Floke/Brancheneinstufung2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

train_model.py aktualisiert

Browse Source
This commit is contained in:
Floke
2025-09-24 14:23:15 +00:00
parent 894d6f50df
commit 9ff6f97513
1 changed files with 27 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
train_model.py
View File

@@ -4,6 +4,8 @@ import re
import math import math
import joblib import joblib
import xgboost as xgb import xgboost as xgb
import treelite
import treelite_runtime
from sklearn.model_selection import train_test_split from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report from sklearn.metrics import accuracy_score, classification_report
from thefuzz import fuzz from thefuzz import fuzz
@@ -11,22 +13,13 @@ from collections import Counter
import logging import logging
import sys import sys
import os import os
import treelite
import treelite_runtime
# Importiere deine bestehenden Helfer # Importiere deine bestehenden Helfer
from google_sheet_handler import GoogleSheetHandler from google_sheet_handler import GoogleSheetHandler
from helpers import normalize_company_name from helpers import normalize_company_name
# --- Detailliertes Logging Setup --- # Logging Setup
# Wir stellen sicher, dass wir alles sehen logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', handlers=[logging.StreamHandler(sys.stdout)])
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)-8s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout) # Gibt alles in der Konsole aus
]
)
log = logging.getLogger() log = logging.getLogger()
# --- Konfiguration --- # --- Konfiguration ---
@@ -35,6 +28,7 @@ CRM_SHEET_NAME = "CRM_Accounts"
MODEL_OUTPUT_FILE = 'xgb_model.json' MODEL_OUTPUT_FILE = 'xgb_model.json'
TERM_WEIGHTS_OUTPUT_FILE = 'term_weights.joblib' TERM_WEIGHTS_OUTPUT_FILE = 'term_weights.joblib'
CRM_PREDICTION_FILE = 'crm_for_prediction.pkl' CRM_PREDICTION_FILE = 'crm_for_prediction.pkl'
TREELITE_MODEL_FILE = 'xgb_model.treelite'
BEST_MATCH_COL = 'Best Match Option' BEST_MATCH_COL = 'Best Match Option'
SUGGESTION_COLS = ['V2_Match_Suggestion', 'V3_Match_Suggestion', 'V4_Match_Suggestion'] SUGGESTION_COLS = ['V2_Match_Suggestion', 'V3_Match_Suggestion', 'V4_Match_Suggestion']
@@ -43,7 +37,6 @@ SUGGESTION_COLS = ['V2_Match_Suggestion', 'V3_Match_Suggestion', 'V4_Match_Sugge
STOP_TOKENS_BASE = { STOP_TOKENS_BASE = {
'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv', 'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv',
'holding','gruppe','group','international','solutions','solution','service','services', 'holding','gruppe','group','international','solutions','solution','service','services',
# ... (Rest der Stopwords)
} }
CITY_TOKENS = set() CITY_TOKENS = set()
@@ -101,11 +94,10 @@ def create_features(mrec: dict, crec: dict, term_weights: dict):
# --- Haupt-Trainingsskript --- # --- Haupt-Trainingsskript ---
if __name__ == "__main__": if __name__ == "__main__":
log.info("Starte Trainingsprozess für Duplikats-Checker v5.0") log.info("Starte Trainingsprozess für Duplikats-Checker v5.0")
try: try:
log.info(f"Versuche, Gold-Standard-Datei zu laden: '{GOLD_STANDARD_FILE}'") log.info(f"Versuche, Gold-Standard-Datei zu laden: '{GOLD_STANDARD_FILE}'")
if not os.path.exists(GOLD_STANDARD_FILE): if not os.path.exists(GOLD_STANDARD_FILE):
log.critical(f"FEHLER: Die Datei '{GOLD_STANDARD_FILE}' wurde im aktuellen Verzeichnis nicht gefunden.") log.critical(f"FEHLER: Die Datei '{GOLD_STANDARD_FILE}' wurde nicht gefunden.")
sys.exit(1) sys.exit(1)
gold_df = pd.read_csv(GOLD_STANDARD_FILE, sep=';', encoding='utf-8') gold_df = pd.read_csv(GOLD_STANDARD_FILE, sep=';', encoding='utf-8')
log.info(f"{len(gold_df)} Zeilen aus '{GOLD_STANDARD_FILE}' geladen.") log.info(f"{len(gold_df)} Zeilen aus '{GOLD_STANDARD_FILE}' geladen.")
@@ -114,7 +106,6 @@ if __name__ == "__main__":
sheet_handler = GoogleSheetHandler() sheet_handler = GoogleSheetHandler()
crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME) crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME)
log.info(f"{len(crm_df)} CRM Accounts aus Google Sheets geladen.") log.info(f"{len(crm_df)} CRM Accounts aus Google Sheets geladen.")
except Exception as e: except Exception as e:
log.critical(f"Fehler beim Laden der Daten: {e}") log.critical(f"Fehler beim Laden der Daten: {e}")
sys.exit(1) sys.exit(1)
@@ -122,91 +113,56 @@ if __name__ == "__main__":
log.info("Entferne Duplikate aus CRM-Daten für das Training...") log.info("Entferne Duplikate aus CRM-Daten für das Training...")
crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True) crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True)
log.info(f"CRM-Daten auf {len(crm_df)} eindeutige Firmennamen reduziert.") log.info(f"CRM-Daten auf {len(crm_df)} eindeutige Firmennamen reduziert.")
log.info("Normalisiere Firmennamen...")
crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name) crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name)
gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name) gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name)
log.info("Berechne Wortgewichte (TF-IDF)...") log.info("Berechne Wortgewichte (TF-IDF)...")
term_weights = {token: math.log(len(crm_df) / (count + 1)) for token, count in Counter(t for n in crm_df['normalized_name'] for t in set(clean_name_for_scoring(n)[1])).items()} term_weights = {token: math.log(len(crm_df) / (count + 1)) for token, count in Counter(t for n in crm_df['normalized_name'] for t in set(clean_name_for_scoring(n)[1])).items()}
log.info(f"{len(term_weights)} Wortgewichte berechnet.")
log.info("Erstelle Features für den Trainingsdatensatz...")
features_list = []
labels = []
log.info("Erstelle Features für den Trainingsdatensatz...")
features_list, labels = [], []
crm_lookup = crm_df.set_index('CRM Name').to_dict('index') crm_lookup = crm_df.set_index('CRM Name').to_dict('index')
# Finde die Spalten mit den alten Vorschlägen dynamisch
suggestion_cols_found = [col for col in gold_df.columns if col in SUGGESTION_COLS] suggestion_cols_found = [col for col in gold_df.columns if col in SUGGESTION_COLS]
if not suggestion_cols_found:
log.warning(f"Keine Spalten für alte Vorschläge in der CSV gefunden (gesucht: {SUGGESTION_COLS}). Training erfolgt nur mit positiven Beispielen.")
for index, row in gold_df.iterrows(): for _, row in gold_df.iterrows():
mrec = row.to_dict() mrec = row.to_dict()
best_match_name = row.get(BEST_MATCH_COL) best_match_name = row.get(BEST_MATCH_COL)
if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup: if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup:
crec_positive = crm_lookup[best_match_name] features_list.append(create_features(mrec, crm_lookup[best_match_name], term_weights))
features = create_features(mrec, crec_positive, term_weights)
features_list.append(features)
labels.append(1) labels.append(1)
for col_name in suggestion_cols_found: for col_name in suggestion_cols_found:
if col_name in row and pd.notna(row[col_name]): suggestion_name = row.get(col_name)
suggestion_name = row[col_name] if pd.notna(suggestion_name) and suggestion_name != best_match_name and suggestion_name in crm_lookup:
if suggestion_name != best_match_name and suggestion_name in crm_lookup: features_list.append(create_features(mrec, crm_lookup[suggestion_name], term_weights))
crec_negative = crm_lookup[suggestion_name] labels.append(0)
features = create_features(mrec, crec_negative, term_weights)
features_list.append(features)
labels.append(0)
X = pd.DataFrame(features_list) X, y = pd.DataFrame(features_list), np.array(labels)
y = np.array(labels) log.info(f"Trainingsdatensatz erstellt mit {X.shape[0]} Beispielen. Klassenverteilung: {Counter(y)}")
if len(X) == 0:
log.critical("FEHLER: Keine gültigen Trainingsdaten-Paare konnten erstellt werden. Überprüfe die Spaltennamen in der CSV und im Skript.")
sys.exit(1)
log.info(f"Trainingsdatensatz erfolgreich erstellt mit {X.shape[0]} Beispielen und {X.shape[1]} Features.")
log.info(f"Verteilung der Klassen (0=Falsch, 1=Korrekt): {Counter(y)}")
log.info("Trainiere das XGBoost-Modell...") log.info("Trainiere das XGBoost-Modell...")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
scale_pos_weight = sum(y_train == 0) / sum(y_train) if sum(y_train) > 0 else 1
scale_pos_weight = (len(y_train) - sum(y_train)) / sum(y_train) if sum(y_train) > 0 else 1
model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight) model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight)
model.fit(X_train, y_train) model.fit(X_train, y_train)
log.info("Modell erfolgreich trainiert.") log.info("Modell erfolgreich trainiert.")
log.info("Validiere Modell auf Testdaten...") log.info("Validiere Modell auf Testdaten...")
y_pred = model.predict(X_test) y_pred = model.predict(X_test)
accuracy = accuracy_score(y_test, y_pred) log.info(f"\n--- Validierungsergebnis ---\nGenauigkeit: {accuracy_score(y_test, y_pred):.2%}\n" + classification_report(y_test, y_pred, zero_division=0))
log.info(f"\n--- Validierungsergebnis ---")
log.info(f"Genauigkeit auf Testdaten: {accuracy:.2%}")
log.info("Detaillierter Report:")
log.info("\n" + classification_report(y_test, y_pred, zero_division=0))
try: try:
# Speichern des Standard-Modells
model.save_model(MODEL_OUTPUT_FILE) model.save_model(MODEL_OUTPUT_FILE)
logging.info(f"Modell in '{MODEL_OUTPUT_FILE}' erfolgreich gespeichert.") log.info(f"Modell in '{MODEL_OUTPUT_FILE}' gespeichert.")
# KORREKTUR: Wir nutzen den internen Booster für Treelite
treelite_model = treelite.Model.from_xgboost(model.get_booster())
treelite_model.export_lib(toolchain='gcc', libpath=TREELITE_MODEL_FILE, params={'parallel_comp': 4}, verbose=True)
log.info(f"Leichtgewichtiges Modell in '{TREELITE_MODEL_FILE}' gespeichert.")
# NEU: Speichern des Modells im Treelite-Format
TREELITE_MODEL_FILE = 'xgb_model.treelite'
treelite_model = treelite.Model.from_xgboost(model)
treelite_model.export_lib(
toolchain='gcc',
libpath=TREELITE_MODEL_FILE,
params={'parallel_comp': 4}, # Anzahl der CPU-Kerne nutzen
verbose=True
)
logging.info(f"Leichtgewichtiges Modell in '{TREELITE_MODEL_FILE}' erfolgreich gespeichert.")
joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE) joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE)
logging.info(f"Wortgewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' erfolgreich gespeichert.") log.info(f"Wortgewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' gespeichert.")
crm_df.to_pickle(CRM_PREDICTION_FILE) crm_df.to_pickle(CRM_PREDICTION_FILE)
logging.info(f"CRM-Daten in '{CRM_PREDICTION_FILE}' erfolgreich gespeichert.") log.info(f"CRM-Daten in '{CRM_PREDICTION_FILE}' gespeichert.")
except Exception as e: except Exception as e:
logging.critical(f"FEHLER BEIM SPEICHERN DER DATEIEN: {e}") log.critical(f"FEHLER BEIM SPEICHERN DER DATEIEN: {e}")