Floke/Brancheneinstufung2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

train_model.py aktualisiert

Browse Source
This commit is contained in:
Floke
2025-09-10 08:25:50 +00:00
parent db6965927f
commit 601d1ab848
1 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
train_model.py
View File

@@ -28,9 +28,9 @@ TERM_WEIGHTS_OUTPUT_FILE = 'term_weights.joblib'
# WICHTIG: Passe diese Spaltennamen exakt an deine CSV-Datei an!
BEST_MATCH_COL = 'Best Match Option'
# Liste der Spalten, die Vorschläge von alten Algorithmen enthalten
# Füge hier alle Spaltennamen hinzu, die du als "alte Vorschläge" verwenden willst.
SUGGESTION_COLS = ['V2_Match_Suggestion', 'V3_Match_Suggestion', 'V4_Match_Suggestion']
# Liste der Spalten, die Vorschläge von alten Algorithmen enthalten.
# Das Skript wird alle Spalten verwenden, die mit 'V' beginnen und '_Match_Suggestion' enden.
SUGGESTION_COLS_PREFIX = 'V'
# --- Stop-/City-Tokens ---
STOP_TOKENS_BASE = {
@@ -72,7 +72,7 @@ def create_features(mrec: dict, crec: dict, term_weights: dict):
features['fuzz_token_sort_ratio'] = fuzz.token_sort_ratio(clean1, clean2)
domain1_raw = str(mrec.get('CRM Website', '')).lower()
domain2_raw = str(crec.get('CRM Website', '')).lower() # crec ist jetzt ein dict aus dem CRM
domain2_raw = str(crec.get('CRM Website', '')).lower()
domain1 = domain1_raw.replace('www.', '').split('/')[0].strip()
domain2 = domain2_raw.replace('www.', '').split('/')[0].strip()
features['domain_match'] = 1 if domain1 and domain1 == domain2 else 0
@@ -110,6 +110,10 @@ if __name__ == "__main__":
logging.critical(f"Fehler beim Laden der Daten: {e}")
sys.exit(1)
# <<< KORRIGIERT: Entferne Duplikate aus dem CRM basierend auf dem Namen, behalte nur den ersten Eintrag.
crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True)
logging.info(f"CRM-Daten auf {len(crm_df)} eindeutige Firmennamen reduziert.")
crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name)
gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name)
@@ -122,10 +126,11 @@ if __name__ == "__main__":
crm_lookup = crm_df.set_index('CRM Name').to_dict('index')
suggestion_cols_found = [col for col in gold_df.columns if col.startswith(SUGGESTION_COLS_PREFIX) and col.endswith('_Match_Suggestion')]
for _, row in gold_df.iterrows():
mrec = row.to_dict()
# 1. Positives Beispiel: Der von dir definierte "Best Match"
best_match_name = row.get(BEST_MATCH_COL)
if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup:
crec_positive = crm_lookup[best_match_name]
@@ -133,8 +138,7 @@ if __name__ == "__main__":
features_list.append(features)
labels.append(1)
# 2. Negative Beispiele: Die falschen Vorschläge der alten Algorithmen
for col_name in SUGGESTION_COLS:
for col_name in suggestion_cols_found:
if col_name in row and pd.notna(row[col_name]):
suggestion_name = row[col_name]
if suggestion_name != best_match_name and suggestion_name in crm_lookup:
@@ -147,7 +151,7 @@ if __name__ == "__main__":
y = np.array(labels)
if len(X) == 0:
logging.critical("Keine gültigen Trainingsdaten gefunden. Überprüfe die Spaltennamen in der Konfiguration.")
logging.critical("Keine gültigen Trainingsdaten gefunden.")
sys.exit(1)
logging.info(f"Trainingsdatensatz erstellt mit {X.shape[0]} Beispielen und {X.shape[1]} Features.")