train_model.py aktualisiert

train_model.py

@@ -82,51 +82,34 @@ def create_features(mrec: dict, crec: dict, term_weights: dict):
     return features
 
 if __name__ == "__main__":
-    log.info("Starte Trainingsprozess für Duplikats-Checker v5.0")
+    # ... (der gesamte Trainingsprozess bis zum Speichern) ...
-    try:
-        gold_df = pd.read_csv(GOLD_STANDARD_FILE, sep=';', encoding='utf-8')
-        sheet_handler = GoogleSheetHandler()
-        crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME)
-    except Exception as e:
-        log.critical(f"Fehler beim Laden der Daten: {e}")
-        sys.exit(1)
-
-    crm_df.drop_duplicates(subset=['CRM Name'], keep='first', inplace=True)
-    crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name)
-    gold_df['normalized_CRM Name'] = gold_df['CRM Name'].astype(str).apply(normalize_company_name)
-    
-    term_weights = {token: math.log(len(crm_df) / (count + 1)) for token, count in Counter(t for n in crm_df['normalized_name'] for t in set(clean_name_for_scoring(n)[1])).items()}
-    
-    features_list, labels = [], []
-    crm_lookup = crm_df.set_index('CRM Name').to_dict('index')
-    
-    suggestion_cols_found = [col for col in gold_df.columns if col in SUGGESTION_COLS]
-
-    for _, row in gold_df.iterrows():
-        mrec = row.to_dict()
-        best_match_name = row.get(BEST_MATCH_COL)
-        if pd.notna(best_match_name) and str(best_match_name).strip() != '' and best_match_name in crm_lookup:
-            features_list.append(create_features(mrec, crm_lookup[best_match_name], term_weights))
-            labels.append(1)
-        for col_name in suggestion_cols_found:
-            suggestion_name = row.get(col_name)
-            if pd.notna(suggestion_name) and suggestion_name != best_match_name and suggestion_name in crm_lookup:
-                features_list.append(create_features(mrec, crm_lookup[suggestion_name], term_weights))
-                labels.append(0)
-
-    X, y = pd.DataFrame(features_list), np.array(labels)
-    log.info(f"Trainingsdatensatz erstellt mit {X.shape[0]} Beispielen. Klassenverteilung: {Counter(y)}")
-    
-    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
-    scale_pos_weight = sum(y_train == 0) / sum(y_train) if sum(y_train) > 0 else 1
-    model = xgb.XGBClassifier(use_label_encoder=False, eval_metric='logloss', scale_pos_weight=scale_pos_weight)
-    model.fit(X_train, y_train)
     log.info("Modell erfolgreich trainiert.")
 
     y_pred = model.predict(X_test)
     log.info(f"\n--- Validierungsergebnis ---\nGenauigkeit: {accuracy_score(y_test, y_pred):.2%}\n" + classification_report(y_test, y_pred, zero_division=0))
     
+    try:
         model.save_model(MODEL_OUTPUT_FILE)
+        log.info(f"Modell in '{MODEL_OUTPUT_FILE}' gespeichert.")
+        
+        # <<< KORREKTUR: Wir exportieren jetzt in das korrekte Format (.so für Linux) >>>
+        TREELITE_MODEL_SO_FILE = 'xgb_model.so'
+        treelite_model = treelite.Model.from_xgboost(model.get_booster())
+        
+        # Dieser Befehl kompiliert das Modell in eine native Bibliothek
+        treelite_model.export_lib(
+            toolchain='gcc', 
+            libpath=TREELITE_MODEL_SO_FILE, 
+            params={'parallel_comp': 4}, # Anzahl der CPU-Kerne nutzen
+            verbose=True
+        )
+        log.info(f"Kompiliertes Modell in '{TREELITE_MODEL_SO_FILE}' gespeichert.")
+        
         joblib.dump(term_weights, TERM_WEIGHTS_OUTPUT_FILE)
+        log.info(f"Wortgewichte in '{TERM_WEIGHTS_OUTPUT_FILE}' gespeichert.")
         crm_df.to_pickle(CRM_PREDICTION_FILE)
-    log.info("Alle Modelldateien erfolgreich erstellt.")
+        log.info(f"CRM-Daten in '{CRM_PREDICTION_FILE}' gespeichert.")
+        log.info("Alle Dateien wurden erfolgreich erstellt.")
+        
+    except Exception as e:
+        log.critical(f"FEHLER BEIM SPEICHERN DER DATEIEN: {e}")