From e052933704d02530c06528483cfbf6f782ab6752 Mon Sep 17 00:00:00 2001 From: Floke Date: Wed, 24 Sep 2025 19:21:45 +0000 Subject: [PATCH] train_model.py aktualisiert --- train_model.py | 91 ++++++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/train_model.py b/train_model.py index c9ecef80..fef357db 100644 --- a/train_model.py +++ b/train_model.py @@ -1,3 +1,4 @@ +# train_model_v3.0.py (final) import pandas as pd import numpy as np import re @@ -10,7 +11,7 @@ from thefuzz import fuzz from collections import Counter import logging import sys - +import os from google_sheet_handler import GoogleSheetHandler from helpers import normalize_company_name @@ -25,91 +26,93 @@ CRM_PREDICTION_FILE = 'crm_for_prediction.pkl' BEST_MATCH_COL = 'Best Match Option' SUGGESTION_COLS = ['V2_Match_Suggestion', 'V3_Match_Suggestion', 'V4_Match_Suggestion'] -STOP_TOKENS_BASE = { - 'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv', - 'holding','gruppe','group','international','solutions','solution','service','services', -} -CITY_TOKENS = set() - +# ... (Alle Hilfsfunktionen bleiben identisch zu Version 2.4/2.5) ... def _tokenize(s: str): if not s: return [] return re.split(r"[^a-z0-9äöüß]+", str(s).lower()) - def clean_name_for_scoring(norm_name: str): + STOP_TOKENS_BASE = {'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv','holding','gruppe','group','international','solutions','solution','service','services'} + CITY_TOKENS = set() if not norm_name: return "", set() tokens = [t for t in _tokenize(norm_name) if len(t) >= 3] stop_union = STOP_TOKENS_BASE | CITY_TOKENS final_tokens = [t for t in tokens if t not in stop_union] return " ".join(final_tokens), set(final_tokens) - def choose_rarest_token(norm_name: str, term_weights: dict): _, toks = clean_name_for_scoring(norm_name) if not toks: return None return max(toks, key=lambda t: term_weights.get(t, 0)) - def create_features(mrec: dict, crec: dict, term_weights: dict): features = {} n1_raw = mrec.get('normalized_CRM Name', '') n2_raw = crec.get('normalized_name', '') clean1, toks1 = clean_name_for_scoring(n1_raw) clean2, toks2 = clean_name_for_scoring(n2_raw) - features['fuzz_ratio'] = fuzz.ratio(n1_raw, n2_raw) features['fuzz_partial_ratio'] = fuzz.partial_ratio(n1_raw, n2_raw) features['fuzz_token_set_ratio'] = fuzz.token_set_ratio(clean1, clean2) features['fuzz_token_sort_ratio'] = fuzz.token_sort_ratio(clean1, clean2) - domain1_raw = str(mrec.get('CRM Website', '')).lower() domain2_raw = str(crec.get('CRM Website', '')).lower() domain1 = domain1_raw.replace('www.', '').split('/')[0].strip() domain2 = domain2_raw.replace('www.', '').split('/')[0].strip() features['domain_match'] = 1 if domain1 and domain1 == domain2 else 0 - features['city_match'] = 1 if mrec.get('CRM Ort') and crec.get('CRM Ort') and mrec['CRM Ort'] == crec['CRM Ort'] else 0 features['country_match'] = 1 if mrec.get('CRM Land') and crec.get('CRM Land') and mrec['CRM Land'] == crec['CRM Land'] else 0 features['country_mismatch'] = 1 if (mrec.get('CRM Land') and crec.get('CRM Land') and mrec['CRM Land'] != crec['CRM Land']) else 0 - overlapping_tokens = toks1 & toks2 rarest_token_mrec = choose_rarest_token(n1_raw, term_weights) - features['rarest_token_overlap'] = 1 if rarest_token_mrec and rarest_token_mrec in toks2 else 0 features['weighted_token_score'] = sum(term_weights.get(t, 0) for t in overlapping_tokens) features['jaccard_similarity'] = len(overlapping_tokens) / len(toks1 | toks2) if len(toks1 | toks2) > 0 else 0 - features['name_len_diff'] = abs(len(n1_raw) - len(n2_raw)) features['candidate_is_shorter'] = 1 if len(n2_raw) < len(n1_raw) else 0 - return features if __name__ == "__main__": - # ... (der gesamte Trainingsprozess bis zum Speichern) ... - log.info("Modell erfolgreich trainiert.") + log.info("Starte Trainingsprozess (v3.0 final)") + try: + gold_df = pd.read_csv(GOLD_STANDARD_FILE, sep=';', encoding='utf-8') + sheet_handler = GoogleSheetHandler() + crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME) + except Exception as e: + log.critical(f"Fehler beim Laden der Daten: {e}") + sys.exit(1) + crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True) + crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name) + gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name) + term_weights = {token: math.log(len(crm_df) / (count + 1)) for token, count in Counter(t for n in crm_df['normalized_name'] for t in set(clean_name_for_scoring(n)[1])).items()} + + features_list, labels = [], [] + crm_lookup = crm_df.set_index('CRM Name').to_dict('index') + suggestion_cols_found = [col for col in gold_df.columns if col in SUGGESTION_COLS] + + for _, row in gold_df.iterrows(): + mrec = row.to_dict() + best_match_name = row.get(BEST_MATCH_COL) + if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup: + features_list.append(create_features(mrec, crm_lookup[best_match_name], term_weights)) + labels.append(1) + for col_name in suggestion_cols_found: + suggestion_name = row.get(col_name) + if pd.notna(suggestion_name) and suggestion_name != best_match_name and suggestion_name in crm_lookup: + features_list.append(create_features(mrec, crm_lookup[suggestion_name], term_weights)) + labels.append(0) + + X, y = pd.DataFrame(features_list), np.array(labels) + log.info(f"Trainingsdatensatz erstellt mit {X.shape[0]} Beispielen. Klassenverteilung: {Counter(y)}") + + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y) + scale_pos_weight = sum(y_train == 0) / sum(y_train) if sum(y_train) > 0 else 1 + model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight) + model.fit(X_train, y_train) + log.info("Modell erfolgreich trainiert.") + y_pred = model.predict(X_test) log.info(f"\n--- Validierungsergebnis ---\nGenauigkeit: {accuracy_score(y_test, y_pred):.2%}\n" + classification_report(y_test, y_pred, zero_division=0)) - try: - model.save_model(MODEL_OUTPUT_FILE) - log.info(f"Modell in '{MODEL_OUTPUT_FILE}' gespeichert.") - - # <<< KORREKTUR: Wir exportieren jetzt in das korrekte Format (.so für Linux) >>> - TREELITE_MODEL_SO_FILE = 'xgb_model.so' - treelite_model = treelite.Model.from_xgboost(model.get_booster()) - - # Dieser Befehl kompiliert das Modell in eine native Bibliothek - treelite_model.export_lib( - toolchain='gcc', - libpath=TREELITE_MODEL_SO_FILE, - params={'parallel_comp': 4}, # Anzahl der CPU-Kerne nutzen - verbose=True - ) - log.info(f"Kompiliertes Modell in '{TREELITE_MODEL_SO_FILE}' gespeichert.") - - joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE) - log.info(f"Wortgewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' gespeichert.") - crm_df.to_pickle(CRM_PREDICTION_FILE) - log.info(f"CRM-Daten in '{CRM_PREDICTION_FILE}' gespeichert.") - log.info("Alle Dateien wurden erfolgreich erstellt.") - - except Exception as e: - log.critical(f"FEHLER BEIM SPEICHERN DER DATEIEN: {e}") \ No newline at end of file + model.save_model(MODEL_OUTPUT_FILE) + joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE) + crm_df.to_pickle(CRM_PREDICTION_FILE) + log.info("Alle 3 Modelldateien erfolgreich erstellt.") \ No newline at end of file