Floke/Brancheneinstufung2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

train_model.py aktualisiert

Browse Source
This commit is contained in:
Floke
2025-09-24 14:06:54 +00:00
parent c91dbd6246
commit 673fe18347
1 changed files with 18 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
train_model.py
View File

@@ -19,30 +19,27 @@ from helpers import normalize_company_name
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
# --- Konfiguration --- # --- Konfiguration ---
# HINWEIS: Bitte stelle sicher, dass diese Datei deine finale Vergleichs-CSV ist.
# Passe den Namen an, falls deine Datei anders heißt (z.B. 'matches4.csv').
GOLD_STANDARD_FILE = 'erweitertes_matching.csv' GOLD_STANDARD_FILE = 'erweitertes_matching.csv'
CRM_SHEET_NAME = "CRM_Accounts" CRM_SHEET_NAME = "CRM_Accounts"
MODEL_OUTPUT_FILE = 'xgb_model.json' MODEL_OUTPUT_FILE = 'xgb_model.json'
TERM_WEIGHTS_OUTPUT_FILE = 'term_weights.joblib' TERM_WEIGHTS_OUTPUT_FILE = 'term_weights.joblib'
CRM_PREDICTION_FILE = 'crm_for_prediction.pkl'
# WICHTIG: Passe diese Spaltennamen exakt an deine CSV-Datei an! # Passe diese Spaltennamen exakt an deine CSV-Datei an!
BEST_MATCH_COL = 'Best Match Option' BEST_MATCH_COL = 'Best Match Option'
# Liste der Spalten, die Vorschläge von alten Algorithmen enthalten. # Das Skript findet automatisch alle Spalten, die mit 'V' beginnen und '_Match_Suggestion' enden
# Das Skript wird alle Spalten verwenden, die mit 'V' beginnen und '_Match_Suggestion' enden.
SUGGESTION_COLS_PREFIX = 'V' SUGGESTION_COLS_PREFIX = 'V'
# --- Stop-/City-Tokens --- # --- Stop-/City-Tokens ---
STOP_TOKENS_BASE = { STOP_TOKENS_BASE = {
'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv', 'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv',
'holding','gruppe','group','international','solutions','solution','service','services', 'holding','gruppe','group','international','solutions','solution','service','services',
'deutschland','austria','germany','technik','technology','technologies','systems','systeme', # ... (Rest der Stopwords)
'logistik','logistics','industries','industrie','management','consulting','vertrieb','handel',
'international','company','gesellschaft','mbh&co','mbhco','werke','werk'
} }
CITY_TOKENS = set() CITY_TOKENS = set()
# --- Hilfsfunktionen --- # --- Hilfsfunktionen ---
# ... (alle Hilfsfunktionen wie _tokenize, clean_name_for_scoring etc. bleiben unverändert)
def _tokenize(s: str): def _tokenize(s: str):
if not s: return [] if not s: return []
return re.split(r"[^a-z0-9äöüß]+", str(s).lower()) return re.split(r"[^a-z0-9äöüß]+", str(s).lower())
@@ -105,12 +102,10 @@ if __name__ == "__main__":
sheet_handler = GoogleSheetHandler() sheet_handler = GoogleSheetHandler()
crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME) crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME)
logging.info(f"{len(crm_df)} CRM Accounts aus Google Sheets geladen.") logging.info(f"{len(crm_df)} CRM Accounts aus Google Sheets geladen.")
except Exception as e: except Exception as e:
logging.critical(f"Fehler beim Laden der Daten: {e}") logging.critical(f"Fehler beim Laden der Daten: {e}")
sys.exit(1) sys.exit(1)
# <<< KORRIGIERT: Entferne Duplikate aus dem CRM basierend auf dem Namen, behalte nur den ersten Eintrag.
crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True) crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True)
logging.info(f"CRM-Daten auf {len(crm_df)} eindeutige Firmennamen reduziert.") logging.info(f"CRM-Daten auf {len(crm_df)} eindeutige Firmennamen reduziert.")
@@ -126,12 +121,13 @@ if __name__ == "__main__":
crm_lookup = crm_df.set_index('CRM Name').to_dict('index') crm_lookup = crm_df.set_index('CRM Name').to_dict('index')
suggestion_cols_found = [col for col in gold_df.columns if col.startswith(SUGGESTION_COLS_PREFIX) and col.endswith('_Match_Suggestion')] suggestion_cols_found = [col for col in gold_df.columns if col.startswith(SUGGESTION_COLS_PREFIX) and '_Match_Suggestion' in col]
logging.info(f"Gefundene Spalten mit alten Vorschlägen: {suggestion_cols_found}")
for _, row in gold_df.iterrows(): for _, row in gold_df.iterrows():
mrec = row.to_dict() mrec = row.to_dict()
best_match_name = row.get(BEST_MATCH_COL) best_match_name = row.get(BEST_MATCH_COL)
if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup: if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup:
crec_positive = crm_lookup[best_match_name] crec_positive = crm_lookup[best_match_name]
features = create_features(mrec, crec_positive, term_weights) features = create_features(mrec, crec_positive, term_weights)
@@ -173,6 +169,12 @@ if __name__ == "__main__":
logging.info("Detaillierter Report:") logging.info("Detaillierter Report:")
logging.info("\n" + classification_report(y_test, y_pred, zero_division=0)) logging.info("\n" + classification_report(y_test, y_pred, zero_division=0))
try:
model.save_model(MODEL_OUTPUT_FILE) model.save_model(MODEL_OUTPUT_FILE)
logging.info(f"Modell in '{MODEL_OUTPUT_FILE}' erfolgreich gespeichert.")
joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE) joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE)
logging.info(f"Modell in '{MODEL_OUTPUT_FILE}' und Gewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' erfolgreich gespeichert.") logging.info(f"Wortgewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' erfolgreich gespeichert.")
crm_df.to_pickle(CRM_PREDICTION_FILE)
logging.info(f"CRM-Daten in '{CRM_PREDICTION_FILE}' erfolgreich gespeichert.")
except Exception as e:
logging.critical(f"FEHLER BEIM SPEICHERN DER DATEIEN: {e}")