Floke/Brancheneinstufung2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

train_model.py aktualisiert

Browse Source
This commit is contained in:
Floke
2025-09-24 15:47:22 +00:00
parent d5d54fa37c
commit 67e63140bf
1 changed files with 27 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
train_model.py
View File

@@ -82,51 +82,34 @@ def create_features(mrec: dict, crec: dict, term_weights: dict):
return features
if __name__ == "__main__":
log.info("Starte Trainingsprozess für Duplikats-Checker v5.0")
try:
gold_df = pd.read_csv(GOLD_STANDARD_FILE, sep=';', encoding='utf-8')
sheet_handler = GoogleSheetHandler()
crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME)
except Exception as e:
log.critical(f"Fehler beim Laden der Daten: {e}")
sys.exit(1)
crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True)
crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name)
gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name)
term_weights = {token: math.log(len(crm_df) / (count + 1)) for token, count in Counter(t for n in crm_df['normalized_name'] for t in set(clean_name_for_scoring(n)[1])).items()}
features_list, labels = [], []
crm_lookup = crm_df.set_index('CRM Name').to_dict('index')
suggestion_cols_found = [col for col in gold_df.columns if col in SUGGESTION_COLS]
for _, row in gold_df.iterrows():
mrec = row.to_dict()
best_match_name = row.get(BEST_MATCH_COL)
if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup:
features_list.append(create_features(mrec, crm_lookup[best_match_name], term_weights))
labels.append(1)
for col_name in suggestion_cols_found:
suggestion_name = row.get(col_name)
if pd.notna(suggestion_name) and suggestion_name != best_match_name and suggestion_name in crm_lookup:
features_list.append(create_features(mrec, crm_lookup[suggestion_name], term_weights))
labels.append(0)
X, y = pd.DataFrame(features_list), np.array(labels)
log.info(f"Trainingsdatensatz erstellt mit {X.shape[0]} Beispielen. Klassenverteilung: {Counter(y)}")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
scale_pos_weight = sum(y_train == 0) / sum(y_train) if sum(y_train) > 0 else 1
model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight)
model.fit(X_train, y_train)
# ... (der gesamte Trainingsprozess bis zum Speichern) ...
log.info("Modell erfolgreich trainiert.")
y_pred = model.predict(X_test)
log.info(f"\n--- Validierungsergebnis ---\nGenauigkeit: {accuracy_score(y_test, y_pred):.2%}\n" + classification_report(y_test, y_pred, zero_division=0))
model.save_model(MODEL_OUTPUT_FILE)
joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE)
crm_df.to_pickle(CRM_PREDICTION_FILE)
log.info("Alle Modelldateien erfolgreich erstellt.")
try:
model.save_model(MODEL_OUTPUT_FILE)
log.info(f"Modell in '{MODEL_OUTPUT_FILE}' gespeichert.")
# <<< KORREKTUR: Wir exportieren jetzt in das korrekte Format (.so für Linux) >>>
TREELITE_MODEL_SO_FILE = 'xgb_model.so'
treelite_model = treelite.Model.from_xgboost(model.get_booster())
# Dieser Befehl kompiliert das Modell in eine native Bibliothek
treelite_model.export_lib(
toolchain='gcc',
libpath=TREELITE_MODEL_SO_FILE,
params={'parallel_comp': 4}, # Anzahl der CPU-Kerne nutzen
verbose=True
)
log.info(f"Kompiliertes Modell in '{TREELITE_MODEL_SO_FILE}' gespeichert.")
joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE)
log.info(f"Wortgewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' gespeichert.")
crm_df.to_pickle(CRM_PREDICTION_FILE)
log.info(f"CRM-Daten in '{CRM_PREDICTION_FILE}' gespeichert.")
log.info("Alle Dateien wurden erfolgreich erstellt.")
except Exception as e:
log.critical(f"FEHLER BEIM SPEICHERN DER DATEIEN: {e}")