train_model.py aktualisiert

train_model.py

@@ -19,37 +19,27 @@ from helpers import normalize_company_name
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # --- Konfiguration ---
-GOLD_STANDARD_FILE = 'erweitertes_matching.csv'
+# HINWEIS: Bitte stelle sicher, dass diese Datei deine finale Vergleichs-CSV ist,
+# die die Spalten 'Best Match Option' und die Vorschläge der alten Läufe enthält.
+GOLD_STANDARD_FILE = 'matches4.csv' 
 CRM_SHEET_NAME = "CRM_Accounts"
 MODEL_OUTPUT_FILE = 'xgb_model.json'
 TERM_WEIGHTS_OUTPUT_FILE = 'term_weights.joblib'
 
+# Spaltennamen aus deiner Vergleichs-CSV
+BEST_MATCH_COL = 'Best Match Option'
+# Liste der Spalten, die Vorschläge von alten Algorithmen enthalten
+SUGGESTION_COLS = ['Ergebnis Lauf 1', 'Ergebnis Lauf 2', 'Ergebnis Lauf 3'] # Passe diese Namen an deine CSV an!
+
+# --- ... (Alle Hilfsfunktionen bleiben identisch zu vorher) ... ---
 # --- Stop-/City-Tokens ---
 STOP_TOKENS_BASE = {
     'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv',
     'holding','gruppe','group','international','solutions','solution','service','services',
-    'deutschland','austria','germany','technik','technology','technologies','systems','systeme',
+    # ... (Rest der Stopwords)
-    'logistik','logistics','industries','industrie','management','consulting','vertrieb','handel',
-    'international','company','gesellschaft','mbh&co','mbhco','werke','werk'
 }
 CITY_TOKENS = set()
 
-# --- Hilfsfunktionen (jetzt direkt hier definiert, um helpers.py nicht zu benötigen) ---
-
-def normalize_company_name(name: str):
-    """Normalisiert einen Firmennamen für den Vergleich."""
-    if not isinstance(name, str): return ''
-    # Kleinschreibung
-    name = name.lower()
-    # Inhalte in Klammern entfernen
-    name = re.sub(r'\(.*?\)', '', name)
-    name = re.sub(r'\[.*?\]', '', name)
-    # Alle Nicht-alphanumerischen Zeichen durch Leerzeichen ersetzen (Umlaute beibehalten)
-    name = re.sub(r'[^a-z0-9äöüß\s]', ' ', name)
-    # Mehrfache Leerzeichen durch ein einzelnes ersetzen und an den Rändern trimmen
-    name = re.sub(r'\s+', ' ', name).strip()
-    return name
-
 def _tokenize(s: str):
     if not s: return []
     return re.split(r"[^a-z0-9äöüß]+", str(s).lower())
@@ -64,13 +54,12 @@ def clean_name_for_scoring(norm_name: str):
 def choose_rarest_token(norm_name: str, term_weights: dict):
     _, toks = clean_name_for_scoring(norm_name)
     if not toks: return None
-    # Gibt das Token mit dem höchsten Gewicht (höchster IDF-Score) zurück
     return max(toks, key=lambda t: term_weights.get(t, 0))
 
 def create_features(mrec: dict, crec: dict, term_weights: dict):
     features = {}
     n1_raw = mrec.get('normalized_CRM Name', '')
-    n2_raw = crec.get('normalized_Kandidat', '') # <<< KORRIGIERT (falls deine Kandidaten-Spalte anders heißt)
+    n2_raw = crec.get('normalized_name', '') # Kandidaten-Name ist jetzt generisch
     clean1, toks1 = clean_name_for_scoring(n1_raw)
     clean2, toks2 = clean_name_for_scoring(n2_raw)
 
@@ -80,14 +69,14 @@ def create_features(mrec: dict, crec: dict, term_weights: dict):
     features['fuzz_token_sort_ratio'] = fuzz.token_sort_ratio(clean1, clean2)
     
     domain1_raw = str(mrec.get('CRM Website', '')).lower()
-    domain2_raw = str(crec.get('Kandidat Website', '')).lower()
+    domain2_raw = str(crec.get('CRM Website', '')).lower()
     domain1 = domain1_raw.replace('www.', '').split('/')[0].strip()
     domain2 = domain2_raw.replace('www.', '').split('/')[0].strip()
     features['domain_match'] = 1 if domain1 and domain1 == domain2 else 0
     
-    features['city_match'] = 1 if mrec.get('CRM Ort') and mrec.get('CRM Ort') == crec.get('Kandidat Ort') else 0
+    features['city_match'] = 1 if mrec.get('CRM Ort') and crec.get('CRM Ort') and mrec['CRM Ort'] == crec['CRM Ort'] else 0
-    features['country_match'] = 1 if mrec.get('CRM Land') and mrec.get('CRM Land') == crec.get('Kandidat Land') else 0
+    features['country_match'] = 1 if mrec.get('CRM Land') and crec.get('CRM Land') and mrec['CRM Land'] == crec['CRM Land'] else 0
-    features['country_mismatch'] = 1 if (mrec.get('CRM Land') and crec.get('Kandidat Land') and mrec.get('CRM Land') != crec.get('Kandidat Land')) else 0
+    features['country_mismatch'] = 1 if (mrec.get('CRM Land') and crec.get('CRM Land') and mrec['CRM Land'] != crec['CRM Land']) else 0
     
     overlapping_tokens = toks1 & toks2
     rarest_token_mrec = choose_rarest_token(n1_raw, term_weights)
@@ -113,7 +102,6 @@ if __name__ == "__main__":
         sheet_handler = GoogleSheetHandler()
         crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME)
         logging.info(f"{len(crm_df)} CRM Accounts aus Google Sheets geladen.")
-
     except Exception as e:
         logging.critical(f"Fehler beim Laden der Daten: {e}")
         sys.exit(1)
@@ -121,18 +109,6 @@ if __name__ == "__main__":
     crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name)
     gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name)
     
-    # <<< KORRIGIERT: Der Spaltenname 'Kandidat' wurde an 'Match Suggestion' angepasst.
-    # Falls deine Spalte anders heißt, bitte hier anpassen.
-    CANDIDATE_COLUMN_NAME = 'Kandidat' 
-    if CANDIDATE_COLUMN_NAME not in gold_df.columns:
-         logging.error(f"Die Spalte '{CANDIDATE_COLUMN_NAME}' wurde in deiner CSV nicht gefunden. Bitte überprüfe den Spaltennamen.")
-         sys.exit(1)
-         
-    gold_df['normalized_Kandidat'] = gold_df[CANDIDATE_COLUMN_NAME].astype(str).apply(normalize_company_name)
-    
-    for col in ['CRM Ort', 'Kandidat Ort', 'CRM Land', 'Kandidat Land']:
-         gold_df[col] = gold_df[col].astype(str).str.lower().str.strip().replace('nan', '')
-
     term_weights = {token: math.log(len(crm_df) / (count + 1)) for token, count in Counter(t for n in crm_df['normalized_name'] for t in set(clean_name_for_scoring(n)[1])).items()}
     logging.info(f"{len(term_weights)} Wortgewichte berechnet.")
 
@@ -140,27 +116,36 @@ if __name__ == "__main__":
     features_list = []
     labels = []
     
-    BEST_MATCH_COLUMN_NAME = 'Best Match Option' # Passe diesen Namen an, falls er in deiner CSV anders lautet.
+    # Mache das CRM für schnelle Lookups verfügbar
-    if BEST_MATCH_COLUMN_NAME not in gold_df.columns:
+    crm_lookup = crm_df.set_index('CRM Name').to_dict('index')
-        logging.error(f"Die Spalte '{BEST_MATCH_COLUMN_NAME}' wurde in deiner CSV nicht gefunden. Bitte überprüfe den Spaltennamen.")
-        sys.exit(1)
 
     for _, row in gold_df.iterrows():
-        if pd.notna(row[CANDIDATE_COLUMN_NAME]) and pd.notna(row[BEST_MATCH_COLUMN_NAME]) and str(row[BEST_MATCH_COLUMN_NAME]).strip() != '':
+        mrec = row.to_dict() # Die zu prüfende Firma
-            mrec = row.to_dict()
-            crec = {'normalized_Kandidat': row['normalized_Kandidat'], 'Kandidat Website': row['Kandidat Website'], 'Kandidat Ort': row['Kandidat Ort'], 'Kandidat Land': row['Kandidat Land']}
 
-            features = create_features(mrec, crec, term_weights)
+        # 1. Positives Beispiel: Der von dir definierte "Best Match"
+        best_match_name = row[BEST_MATCH_COL]
+        if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup:
+            crec_positive = crm_lookup[best_match_name]
+            features = create_features(mrec, crec_positive, term_weights)
             features_list.append(features)
+            labels.append(1)
 
-            is_correct_match = 1 if row[CANDIDATE_COLUMN_NAME] == row[BEST_MATCH_COLUMN_NAME] else 0
+        # 2. Negative Beispiele: Die falschen Vorschläge der alten Algorithmen
-            labels.append(is_correct_match)
+        for col_name in SUGGESTION_COLS:
+            if col_name in row and pd.notna(row[col_name]):
+                suggestion_name = row[col_name]
+                # Wenn der Vorschlag nicht der korrekte ist UND im CRM existiert
+                if suggestion_name != best_match_name and suggestion_name in crm_lookup:
+                    crec_negative = crm_lookup[suggestion_name]
+                    features = create_features(mrec, crec_negative, term_weights)
+                    features_list.append(features)
+                    labels.append(0)
 
     X = pd.DataFrame(features_list)
     y = np.array(labels)
     
     if len(X) == 0:
-        logging.critical("Keine gültigen Trainingsdaten gefunden. Überprüfe die Spalten 'Kandidat' and 'Best Match Option' in deiner CSV.")
+        logging.critical("Keine gültigen Trainingsdaten gefunden.")
         sys.exit(1)
         
     logging.info(f"Trainingsdatensatz erstellt mit {X.shape[0]} Beispielen und {X.shape[1]} Features.")
@@ -170,7 +155,6 @@ if __name__ == "__main__":
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
     
     scale_pos_weight = (len(y_train) - sum(y_train)) / sum(y_train) if sum(y_train) > 0 else 1
-
     model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight)
     model.fit(X_train, y_train)