Floke/Brancheneinstufung2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

train_model.py aktualisiert

Browse Source
This commit is contained in:
Floke
2025-09-08 18:27:12 +00:00
parent c1867fa2f1
commit b8d491f0fc
1 changed files with 20 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
train_model.py
View File

@@ -11,8 +11,9 @@ from collections import Counter
import logging import logging
import sys import sys
# Importiere NUR noch den GoogleSheetHandler # Importiere deine bestehenden Helfer
from google_sheet_handler import GoogleSheetHandler from google_sheet_handler import GoogleSheetHandler
from helpers import normalize_company_name
# Logging Setup # Logging Setup
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -68,8 +69,8 @@ def choose_rarest_token(norm_name: str, term_weights: dict):
def create_features(mrec: dict, crec: dict, term_weights: dict): def create_features(mrec: dict, crec: dict, term_weights: dict):
features = {} features = {}
n1_raw = mrec.get('normalized_CRM Name', '') # Angepasst an Spaltennamen n1_raw = mrec.get('normalized_CRM Name', '')
n2_raw = crec.get('normalized_Kandidat', '') # Angepasst an Spaltennamen n2_raw = crec.get('normalized_Kandidat', '') # <<< KORRIGIERT (falls deine Kandidaten-Spalte anders heißt)
clean1, toks1 = clean_name_for_scoring(n1_raw) clean1, toks1 = clean_name_for_scoring(n1_raw)
clean2, toks2 = clean_name_for_scoring(n2_raw) clean2, toks2 = clean_name_for_scoring(n2_raw)
@@ -119,7 +120,16 @@ if __name__ == "__main__":
crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name) crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name)
gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name) gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name)
gold_df['normalized_Kandidat'] = gold_df['Kandidat'].astype(str).apply(normalize_company_name)
# <<< KORRIGIERT: Der Spaltenname 'Kandidat' wurde an 'Match Suggestion' angepasst.
# Falls deine Spalte anders heißt, bitte hier anpassen.
CANDIDATE_COLUMN_NAME = 'Kandidat'
if CANDIDATE_COLUMN_NAME not in gold_df.columns:
logging.error(f"Die Spalte '{CANDIDATE_COLUMN_NAME}' wurde in deiner CSV nicht gefunden. Bitte überprüfe den Spaltennamen.")
sys.exit(1)
gold_df['normalized_Kandidat'] = gold_df[CANDIDATE_COLUMN_NAME].astype(str).apply(normalize_company_name)
for col in ['CRM Ort', 'Kandidat Ort', 'CRM Land', 'Kandidat Land']: for col in ['CRM Ort', 'Kandidat Ort', 'CRM Land', 'Kandidat Land']:
gold_df[col] = gold_df[col].astype(str).str.lower().str.strip().replace('nan', '') gold_df[col] = gold_df[col].astype(str).str.lower().str.strip().replace('nan', '')
@@ -130,20 +140,20 @@ if __name__ == "__main__":
features_list = [] features_list = []
labels = [] labels = []
# Sicherstellen, dass die Spalte 'Best Match Option' existiert, um Fehler zu vermeiden BEST_MATCH_COLUMN_NAME = 'Best Match Option' # Passe diesen Namen an, falls er in deiner CSV anders lautet.
if 'Best Match Option' not in gold_df.columns: if BEST_MATCH_COLUMN_NAME not in gold_df.columns:
logging.error("Die Spalte 'Best Match Option' wurde in deiner CSV nicht gefunden. Bitte überprüfe den Spaltennamen.") logging.error(f"Die Spalte '{BEST_MATCH_COLUMN_NAME}' wurde in deiner CSV nicht gefunden. Bitte überprüfe den Spaltennamen.")
sys.exit(1) sys.exit(1)
for _, row in gold_df.iterrows(): for _, row in gold_df.iterrows():
if pd.notna(row['Kandidat']) and pd.notna(row['Best Match Option']) and str(row['Best Match Option']).strip() != '': if pd.notna(row[CANDIDATE_COLUMN_NAME]) and pd.notna(row[BEST_MATCH_COLUMN_NAME]) and str(row[BEST_MATCH_COLUMN_NAME]).strip() != '':
mrec = row.to_dict() mrec = row.to_dict()
crec = {'normalized_name': row['normalized_Kandidat'], 'Kandidat Website': row['Kandidat Website'], 'Kandidat Ort': row['Kandidat Ort'], 'Kandidat Land': row['Kandidat Land']} crec = {'normalized_Kandidat': row['normalized_Kandidat'], 'Kandidat Website': row['Kandidat Website'], 'Kandidat Ort': row['Kandidat Ort'], 'Kandidat Land': row['Kandidat Land']}
features = create_features(mrec, crec, term_weights) features = create_features(mrec, crec, term_weights)
features_list.append(features) features_list.append(features)
is_correct_match = 1 if row['Kandidat'] == row['Best Match Option'] else 0 is_correct_match = 1 if row[CANDIDATE_COLUMN_NAME] == row[BEST_MATCH_COLUMN_NAME] else 0
labels.append(is_correct_match) labels.append(is_correct_match)
X = pd.DataFrame(features_list) X = pd.DataFrame(features_list)
@@ -159,7 +169,6 @@ if __name__ == "__main__":
logging.info("Trainiere das XGBoost-Modell...") logging.info("Trainiere das XGBoost-Modell...")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
# Balance der Klassen für das Training
scale_pos_weight = (len(y_train) - sum(y_train)) / sum(y_train) if sum(y_train) > 0 else 1 scale_pos_weight = (len(y_train) - sum(y_train)) / sum(y_train) if sum(y_train) > 0 else 1
model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight) model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight)