train_model.py aktualisiert

train_model.py

@@ -9,13 +9,18 @@ from sklearn.metrics import accuracy_score, classification_report
 from thefuzz import fuzz
 from collections import Counter
 import logging
+import sys
+
+# Importiere deine bestehenden Helfer
+from google_sheet_handler import GoogleSheetHandler
+from helpers import normalize_company_name
 
 # Logging Setup
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # --- Konfiguration ---
 GOLD_STANDARD_FILE = 'erweitertes_matching.csv'
-CRM_ACCOUNTS_FILE = 'CRM_Accounts.csv' # Annahme: Du hast einen Export des CRM Sheets als CSV
+CRM_SHEET_NAME = "CRM_Accounts"
 MODEL_OUTPUT_FILE = 'xgb_model.json'
 TERM_WEIGHTS_OUTPUT_FILE = 'term_weights.joblib'
 
@@ -29,20 +34,11 @@ STOP_TOKENS_BASE = {
 }
 CITY_TOKENS = set()
 
-# --- Hilfsfunktionen (aus dem Original-Skript übernommen) ---
+# --- Hilfsfunktionen ---
 def _tokenize(s: str):
     if not s: return []
     return re.split(r"[^a-z0-9äöüß]+", str(s).lower())
 
-def normalize_company_name(name: str):
-    if not isinstance(name, str): return ''
-    name = name.lower()
-    name = re.sub(r'\(.*?\)', '', name)
-    name = re.sub(r'\[.*?\]', '', name)
-    name = re.sub(r'[^a-z0-9äöüß\s]', ' ', name)
-    name = re.sub(r'\s+', ' ', name).strip()
-    return name
-
 def clean_name_for_scoring(norm_name: str):
     if not norm_name: return "", set()
     tokens = [t for t in _tokenize(norm_name) if len(t) >= 3]
@@ -67,9 +63,13 @@ def create_features(mrec: dict, crec: dict, term_weights: dict):
     features['fuzz_token_set_ratio'] = fuzz.token_set_ratio(clean1, clean2)
     features['fuzz_token_sort_ratio'] = fuzz.token_sort_ratio(clean1, clean2)
     
-    features['domain_match'] = 1 if mrec.get('CRM Website') and str(mrec.get('CRM Website')).strip() != '' and mrec.get('CRM Website') == crec.get('Kandidat Website') else 0
+    # Normalisiere Domains für den Vergleich
-    features['city_match'] = 1 if mrec.get('CRM Ort') and str(mrec.get('CRM Ort')).strip() != '' and mrec.get('CRM Ort') == crec.get('Kandidat Ort') else 0
+    domain1 = str(mrec.get('CRM Website', '')).lower().replace('www.', '').split('/')[0]
-    features['country_match'] = 1 if mrec.get('CRM Land') and str(mrec.get('CRM Land')).strip() != '' and mrec.get('CRM Land') == crec.get('Kandidat Land') else 0
+    domain2 = str(crec.get('Kandidat Website', '')).lower().replace('www.', '').split('/')[0]
+    features['domain_match'] = 1 if domain1 and domain1 == domain2 else 0
+    
+    features['city_match'] = 1 if mrec.get('CRM Ort') and mrec.get('CRM Ort') == crec.get('Kandidat Ort') else 0
+    features['country_match'] = 1 if mrec.get('CRM Land') and mrec.get('CRM Land') == crec.get('Kandidat Land') else 0
     features['country_mismatch'] = 1 if (mrec.get('CRM Land') and crec.get('Kandidat Land') and mrec.get('CRM Land') != crec.get('Kandidat Land')) else 0
     
     overlapping_tokens = toks1 & toks2
@@ -88,63 +88,42 @@ def create_features(mrec: dict, crec: dict, term_weights: dict):
 if __name__ == "__main__":
     logging.info("Starte Trainingsprozess für Duplikats-Checker v5.0")
 
-    # 1. Daten laden
     try:
-        gold_df = pd.read_csv(GOLD_STANDARD_FILE, sep=';')
+        gold_df = pd.read_csv(GOLD_STANDARD_FILE, sep=';', encoding='utf-8')
-        # Lade CRM Daten, um Gewichte zu berechnen. 
+        logging.info(f"{len(gold_df)} Zeilen aus Gold-Standard-Datei '{GOLD_STANDARD_FILE}' geladen.")
-        # Idealerweise wäre dies ein aktueller Export aus dem Google Sheet.
+        
-        # Für die Simulation nehmen wir die Daten aus dem Gold-Standard.
+        logging.info("Verbinde mit Google Sheets, um CRM-Daten zu laden...")
-        # Besser: Lade hier alle 22.000 CRM Accounts.
+        sheet_handler = GoogleSheetHandler()
-        # Annahme: Du hast einen Export als CRM_Accounts.csv im Ordner
+        crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME)
-        try:
+        logging.info(f"{len(crm_df)} CRM Accounts aus Google Sheets geladen.")
-            crm_df = pd.read_csv(CRM_ACCOUNTS_FILE, sep=',') # Passe Trennzeichen ggf. an
-            logging.info(f"{len(crm_df)} CRM Accounts geladen für die Gewichtsberechnung.")
-        except FileNotFoundError:
-            logging.warning(f"'{CRM_ACCOUNTS_FILE}' nicht gefunden. Verwende Daten aus '{GOLD_STANDARD_FILE}' für Gewichte.")
-            crm_df = gold_df.rename(columns={'Kandidat': 'CRM Name'})
 
     except Exception as e:
-        logging.critical(f"Fehler beim Laden der CSV-Dateien: {e}")
+        logging.critical(f"Fehler beim Laden der Daten: {e}")
         sys.exit(1)
 
-    # 2. Daten normalisieren
-    for col in ['CRM Name', 'Kandidat']:
-        gold_df[f'normalized_{col}'] = gold_df[col].astype(str).apply(normalize_company_name)
-    for col in ['CRM Ort', 'Kandidat Ort', 'CRM Land', 'Kandidat Land']:
-         gold_df[col] = gold_df[col].astype(str).str.lower().str.strip()
-
     crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name)
-    
+    gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name)
-    # 3. Term Weights (TF-IDF) auf dem gesamten CRM-Datensatz berechnen
+    gold_df['normalized_Kandidat'] = gold_df['Kandidat'].astype(str).apply(normalize_company_name)
+    for col in ['CRM Ort', 'Kandidat Ort', 'CRM Land', 'Kandidat Land']:
+         gold_df[col] = gold_df[col].astype(str).str.lower().str.strip().replace('nan', '')
+
     term_weights = {token: math.log(len(crm_df) / (count + 1)) for token, count in Counter(t for n in crm_df['normalized_name'] for t in set(clean_name_for_scoring(n)[1])).items()}
     logging.info(f"{len(term_weights)} Wortgewichte berechnet.")
 
-    # 4. Feature-Tabelle und Labels erstellen
     logging.info("Erstelle Features für den Trainingsdatensatz...")
     features_list = []
     labels = []
 
     for _, row in gold_df.iterrows():
-        mrec = {
+        # Nur Zeilen mit einem Kandidaten und einem Best Match verarbeiten
-            'normalized_name': row['normalized_CRM Name'],
+        if pd.notna(row['Kandidat']) and pd.notna(row['Best Match Option']):
-            'CRM Website': row['CRM Website'],
+            mrec = row.to_dict()
-            'CRM Ort': row['CRM Ort'],
+            crec = {'Kandidat Website': row['Kandidat Website'], 'Kandidat Ort': row['Kandidat Ort'], 'Kandidat Land': row['Kandidat Land']}
-            'CRM Land': row['CRM Land']
+            
-        }
-        crec = {
-            'normalized_name': row['normalized_Kandidat'],
-            'Kandidat Website': row['Kandidat Website'],
-            'Kandidat Ort': row['Kandidat Ort'],
-            'Kandidat Land': row['Kandidat Land']
-        }
-        
-        # Nur Zeilen mit einem Kandidaten verarbeiten
-        if pd.notna(row['Kandidat']):
             features = create_features(mrec, crec, term_weights)
             features_list.append(features)
             
-            # Label erstellen: 1 wenn der Kandidat dem Gold-Standard entspricht, sonst 0
+            is_correct_match = 1 if row['Kandidat'] == row['Best Match Option'] else 0
-            is_correct_match = 1 if row['Kandidat'] == row.get('Best Match Option', '') else 0 # Angenommen Spalte G heißt jetzt so
             labels.append(is_correct_match)
 
     X = pd.DataFrame(features_list)
@@ -153,24 +132,21 @@ if __name__ == "__main__":
     logging.info(f"Trainingsdatensatz erstellt mit {X.shape[0]} Beispielen und {X.shape[1]} Features.")
     logging.info(f"Verteilung der Klassen: {Counter(y)}")
 
-    # 5. Modell trainieren
     logging.info("Trainiere das XGBoost-Modell...")
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
     
-    model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss')
+    model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight= (len(y) - sum(y)) / sum(y))
     model.fit(X_train, y_train)
     
     logging.info("Modell erfolgreich trainiert.")
 
-    # 6. Modell validieren
     y_pred = model.predict(X_test)
     accuracy = accuracy_score(y_test, y_pred)
     logging.info(f"\n--- Validierungsergebnis ---")
     logging.info(f"Genauigkeit auf Testdaten: {accuracy:.2%}")
     logging.info("Detaillierter Report:")
-    logging.info("\n" + classification_report(y_test, y_pred))
+    logging.info("\n" + classification_report(y_test, y_pred, zero_division=0))
     
-    # 7. Finales Modell und Gewichte speichern
     model.save_model(MODEL_OUTPUT_FILE)
     joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE)
     logging.info(f"Modell in '{MODEL_OUTPUT_FILE}' und Gewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' erfolgreich gespeichert.")