bugfix
This commit is contained in:
@@ -573,6 +573,161 @@ def normalize_company_name(name):
|
||||
|
||||
return normalized.lower()
|
||||
|
||||
# NEUE Funktion für Wiki-Updates basierend auf ChatGPT Vorschlägen
|
||||
def process_wiki_updates_from_chatgpt(sheet_handler, data_processor):
|
||||
"""
|
||||
Identifiziert Zeilen, bei denen ChatGPT einen alternativen Wiki-Artikel vorgeschlagen hat (S='X', T=URL),
|
||||
kopiert die neue URL nach M, führt ein Reparse der Wiki-Daten (N-R) durch,
|
||||
berechnet die Branche neu (W-Y) und löscht relevante Timestamps (AN, AX, AO, AP).
|
||||
"""
|
||||
debug_print("Starte Modus: Wiki-Updates basierend auf ChatGPT-Vorschlägen...")
|
||||
|
||||
if not sheet_handler.load_data(): return
|
||||
all_data = sheet_handler.get_all_data_with_headers()
|
||||
if not all_data or len(all_data) <= 5: return
|
||||
header_rows = 5
|
||||
data_rows = all_data[header_rows:] # Arbeite mit Daten ohne Header
|
||||
|
||||
# Indizes holen (Beispielhaft, passe Schlüssel an deine COLUMN_MAP an)
|
||||
wiki_konsistenz_idx = COLUMN_MAP.get("Chat Wiki Konsistenzprüfung") # S
|
||||
vorschlag_t_idx = COLUMN_MAP.get("Chat Vorschlag Wiki Artikel") # T
|
||||
wiki_url_m_idx = COLUMN_MAP.get("Wiki URL") # M
|
||||
# Indizes für Wiki-Daten N-R
|
||||
absatz_n_idx = COLUMN_MAP.get("Wiki Absatz")
|
||||
branche_o_idx = COLUMN_MAP.get("Wiki Branche")
|
||||
umsatz_p_idx = COLUMN_MAP.get("Wiki Umsatz")
|
||||
ma_q_idx = COLUMN_MAP.get("Wiki Mitarbeiter")
|
||||
kat_r_idx = COLUMN_MAP.get("Wiki Kategorien")
|
||||
# Indizes für Branch W-Y
|
||||
branch_w_idx = COLUMN_MAP.get("Chat Vorschlag Branche")
|
||||
branch_x_idx = COLUMN_MAP.get("Chat Konsistenz Branche")
|
||||
branch_y_idx = COLUMN_MAP.get("Chat Begründung Abweichung Branche")
|
||||
# Indizes für Timestamps und Version
|
||||
ts_an_idx = COLUMN_MAP.get("Wikipedia Timestamp")
|
||||
ts_ax_idx = COLUMN_MAP.get("Wiki Verif. Timestamp")
|
||||
ts_ao_idx = COLUMN_MAP.get("Timestamp letzte Prüfung")
|
||||
version_ap_idx = COLUMN_MAP.get("Version")
|
||||
# Index für Website Summary AS (wird für Branch-Neuberechnung gebraucht)
|
||||
summary_as_idx = COLUMN_MAP.get("Website Zusammenfassung")
|
||||
# Index für CRM Branche/Beschreibung (für Branch-Neuberechnung)
|
||||
crm_branch_idx = COLUMN_MAP.get("CRM Branche")
|
||||
crm_desc_idx = COLUMN_MAP.get("CRM Beschreibung")
|
||||
|
||||
|
||||
# Prüfe, ob alle Indizes gefunden wurden
|
||||
required_indices = [
|
||||
wiki_konsistenz_idx, vorschlag_t_idx, wiki_url_m_idx, absatz_n_idx, branche_o_idx,
|
||||
umsatz_p_idx, ma_q_idx, kat_r_idx, branch_w_idx, branch_x_idx, branch_y_idx,
|
||||
ts_an_idx, ts_ax_idx, ts_ao_idx, version_ap_idx, summary_as_idx,
|
||||
crm_branch_idx, crm_desc_idx
|
||||
]
|
||||
if None in required_indices:
|
||||
missing = [k for k, v in COLUMN_MAP.items() if v in required_indices and v is None] # Finde fehlende Keys
|
||||
debug_print(f"FEHLER: Mindestens ein benötigter Spaltenindex für Wiki-Updates fehlt in COLUMN_MAP. Fehlende Keys (Beispiel): {missing}")
|
||||
return
|
||||
|
||||
all_sheet_updates = []
|
||||
processed_rows = 0
|
||||
wiki_scraper = data_processor.wiki_scraper # Nutze den Scraper aus dem DataProcessor
|
||||
|
||||
# Gehe durch alle *Daten*-Zeilen
|
||||
for idx, row in enumerate(data_rows):
|
||||
row_num_in_sheet = idx + header_rows + 1 # 1-basierte Zeilennummer
|
||||
|
||||
# Lese Werte sicher aus
|
||||
konsistenz_s = row[wiki_konsistenz_idx] if len(row) > wiki_konsistenz_idx else ""
|
||||
vorschlag_t = row[vorschlag_t_idx] if len(row) > vorschlag_t_idx else ""
|
||||
url_m = row[wiki_url_m_idx] if len(row) > wiki_url_m_idx else ""
|
||||
|
||||
# Bedingung prüfen: S='X' und T ist eine URL und T != M
|
||||
is_update_candidate = False
|
||||
new_url = ""
|
||||
if konsistenz_s.strip().upper() == "X":
|
||||
# Prüfe, ob T eine valide URL ist (einfache Prüfung)
|
||||
if vorschlag_t.strip().lower().startswith(("http://", "https://")):
|
||||
new_url = vorschlag_t.strip()
|
||||
# Prüfe, ob die neue URL anders ist als die alte
|
||||
if new_url != url_m.strip():
|
||||
is_update_candidate = True
|
||||
|
||||
if is_update_candidate:
|
||||
debug_print(f"Zeile {row_num_in_sheet}: Update-Kandidat gefunden. Neue URL: {new_url}")
|
||||
processed_rows += 1
|
||||
|
||||
# --- Schritt 3a: Wiki Reparse ---
|
||||
debug_print(f" -> Reparsing Wiki-Daten für {new_url}...")
|
||||
new_wiki_data = wiki_scraper.extract_company_data(new_url)
|
||||
|
||||
# --- Schritt 3b: Branch Neuberechnung ---
|
||||
# Hole benötigte Daten für Branch-Eval
|
||||
crm_branche = row[crm_branch_idx] if len(row) > crm_branch_idx else ""
|
||||
crm_beschreibung = row[crm_desc_idx] if len(row) > crm_desc_idx else ""
|
||||
website_summary = row[summary_as_idx] if len(row) > summary_as_idx else ""
|
||||
debug_print(f" -> Neuberechnung der Branche...")
|
||||
new_branch_result = evaluate_branche_chatgpt(
|
||||
crm_branche,
|
||||
crm_beschreibung,
|
||||
new_wiki_data.get('branche', 'k.A.'), # Neue Wiki-Branche
|
||||
new_wiki_data.get('categories', 'k.A.'), # Neue Wiki-Kategorien
|
||||
website_summary # Vorhandene Website-Zusammenfassung
|
||||
)
|
||||
|
||||
# --- Schritt 4: Updates sammeln ---
|
||||
# Spaltenbuchstaben ermitteln
|
||||
url_m_letter = sheet_handler._get_col_letter(wiki_url_m_idx + 1)
|
||||
absatz_n_letter = sheet_handler._get_col_letter(absatz_n_idx + 1)
|
||||
branche_o_letter = sheet_handler._get_col_letter(branche_o_idx + 1)
|
||||
umsatz_p_letter = sheet_handler._get_col_letter(umsatz_p_idx + 1)
|
||||
ma_q_letter = sheet_handler._get_col_letter(ma_q_idx + 1)
|
||||
kat_r_letter = sheet_handler._get_col_letter(kat_r_idx + 1)
|
||||
branch_w_letter = sheet_handler._get_col_letter(branch_w_idx + 1)
|
||||
branch_x_letter = sheet_handler._get_col_letter(branch_x_idx + 1)
|
||||
branch_y_letter = sheet_handler._get_col_letter(branch_y_idx + 1)
|
||||
ts_an_letter = sheet_handler._get_col_letter(ts_an_idx + 1)
|
||||
ts_ax_letter = sheet_handler._get_col_letter(ts_ax_idx + 1)
|
||||
ts_ao_letter = sheet_handler._get_col_letter(ts_ao_idx + 1)
|
||||
version_ap_letter = sheet_handler._get_col_letter(version_ap_idx + 1)
|
||||
# Optional: Spalte T leeren/markieren
|
||||
vorschlag_t_letter = sheet_handler._get_col_letter(vorschlag_t_idx + 1)
|
||||
|
||||
|
||||
row_updates = [
|
||||
# Wiki-Daten aktualisieren
|
||||
{'range': f'{url_m_letter}{row_num_in_sheet}', 'values': [[new_url]]}, # Neue URL nach M
|
||||
{'range': f'{absatz_n_letter}{row_num_in_sheet}', 'values': [[new_wiki_data.get('first_paragraph', 'k.A.')]]},
|
||||
{'range': f'{branche_o_letter}{row_num_in_sheet}', 'values': [[new_wiki_data.get('branche', 'k.A.')]]},
|
||||
{'range': f'{umsatz_p_letter}{row_num_in_sheet}', 'values': [[new_wiki_data.get('umsatz', 'k.A.')]]},
|
||||
{'range': f'{ma_q_letter}{row_num_in_sheet}', 'values': [[new_wiki_data.get('mitarbeiter', 'k.A.')]]},
|
||||
{'range': f'{kat_r_letter}{row_num_in_sheet}', 'values': [[new_wiki_data.get('categories', 'k.A.')]]},
|
||||
# Branch-Daten aktualisieren
|
||||
{'range': f'{branch_w_letter}{row_num_in_sheet}', 'values': [[new_branch_result.get("branch", "Fehler")]]},
|
||||
{'range': f'{branch_x_letter}{row_num_in_sheet}', 'values': [[new_branch_result.get("consistency", "Fehler")]]},
|
||||
{'range': f'{branch_y_letter}{row_num_in_sheet}', 'values': [[new_branch_result.get("justification", "Fehler")]]},
|
||||
# Timestamps und Version leeren, um erneute Prüfung zu triggern
|
||||
{'range': f'{ts_an_letter}{row_num_in_sheet}', 'values': [[""]]}, # Wiki Extraktion TS
|
||||
{'range': f'{ts_ax_letter}{row_num_in_sheet}', 'values': [[""]]}, # Wiki Verif. TS
|
||||
{'range': f'{ts_ao_letter}{row_num_in_sheet}', 'values': [[""]]}, # Letzte Prüfung TS
|
||||
{'range': f'{version_ap_letter}{row_num_in_sheet}', 'values': [[""]]}, # Version
|
||||
# Optional: Spalte T leeren oder markieren
|
||||
{'range': f'{vorschlag_t_letter}{row_num_in_sheet}', 'values': [["Korrektur übernommen"]]},
|
||||
]
|
||||
all_sheet_updates.extend(row_updates)
|
||||
|
||||
# Kurze Pause nach jeder Zeile, um APIs nicht zu überlasten
|
||||
time.sleep(Config.RETRY_DELAY) # Pause nach Wiki Reparse + Branch Neuberechnung
|
||||
|
||||
# --- Schritt 5: Batch Update ---
|
||||
if all_sheet_updates:
|
||||
debug_print(f"Sende Batch-Update für {processed_rows} korrigierte Wiki-Einträge ({len(all_sheet_updates)} Zellen)...")
|
||||
success = sheet_handler.batch_update_cells(all_sheet_updates)
|
||||
if success:
|
||||
debug_print(f"Sheet-Update für Wiki-Korrekturen erfolgreich.")
|
||||
else:
|
||||
debug_print(f"FEHLER beim Sheet-Update für Wiki-Korrekturen.")
|
||||
else:
|
||||
debug_print("Keine Zeilen gefunden, die eine Wiki-URL-Korrektur benötigen.")
|
||||
|
||||
debug_print(f"Wiki-Updates basierend auf ChatGPT abgeschlossen. {processed_rows} Zeilen aktualisiert.")
|
||||
|
||||
def extract_numeric_value(raw_value, is_umsatz=False):
|
||||
"""Extrahiert und normalisiert Zahlenwerte (Umsatz in Mio, Mitarbeiter)."""
|
||||
@@ -3579,48 +3734,71 @@ def main():
|
||||
|
||||
# --- Initialisierung ---
|
||||
parser = argparse.ArgumentParser(description="Firmen-Datenanreicherungs-Skript")
|
||||
valid_modes = ["combined", "wiki", "website", "branch", "summarize", "reeval", "website_lookup", "website_details", "contacts", "full_run", "alignment", "train_technician_model"]
|
||||
# NEU: 'update_wiki' hinzugefügt
|
||||
valid_modes = ["combined", "wiki", "website", "branch", "summarize", "reeval",
|
||||
"website_lookup", "website_details", "contacts", "full_run",
|
||||
"alignment", "train_technician_model", "update_wiki"]
|
||||
parser.add_argument("--mode", type=str, help=f"Betriebsmodus ({', '.join(valid_modes)})")
|
||||
parser.add_argument("--limit", type=int, help="Maximale Anzahl zu verarbeitender Zeilen", default=None)
|
||||
# --- NEU: Argumente für Modelltraining ---
|
||||
parser.add_argument("--model_out", type=str, default=MODEL_FILE, help=f"Dateipfad zum Speichern des Modells (Standard: {MODEL_FILE})")
|
||||
parser.add_argument("--imputer_out", type=str, default=IMPUTER_FILE, help=f"Dateipfad zum Speichern des Imputers (Standard: {IMPUTER_FILE})")
|
||||
parser.add_argument("--patterns_out", type=str, default=PATTERNS_FILE_TXT, help=f"Dateipfad zum Speichern der Text-Regeln (Standard: {PATTERNS_FILE_TXT})")
|
||||
|
||||
parser.add_argument("--model_out", type=str, default=MODEL_FILE, help=f"Pfad für Modell (.pkl)")
|
||||
parser.add_argument("--imputer_out", type=str, default=IMPUTER_FILE, help=f"Pfad für Imputer (.pkl)")
|
||||
parser.add_argument("--patterns_out", type=str, default=PATTERNS_FILE_TXT, help=f"Pfad für Regeln (.txt)")
|
||||
args = parser.parse_args()
|
||||
|
||||
Config.load_api_keys()
|
||||
|
||||
# Betriebsmodus ermitteln (wie gehabt)
|
||||
# Betriebsmodus ermitteln
|
||||
mode = None
|
||||
if args.mode and args.mode.lower() in valid_modes: mode = args.mode.lower(); print(f"Betriebsmodus (aus Kommandozeile): {mode}")
|
||||
else: # Interaktive Abfrage (wie gehabt)
|
||||
# ... (print Optionen etc.) ...
|
||||
try: mode_input = input(f"Geben Sie den Modus ein: ").strip().lower();
|
||||
else: # Interaktive Abfrage
|
||||
print("Bitte wählen Sie den Betriebsmodus:")
|
||||
print(" combined: Wiki(AX), Website-Scrape(AR), Summarize(AS), Branch(AO) (Batch, Start bei leerem AO)")
|
||||
print(" wiki: Nur Wikipedia-Verifizierung (AX) (Batch, Start bei leerem AX)")
|
||||
print(" website: Nur Website-Scraping Rohtext (AR) (Batch, Start bei leerem AR)")
|
||||
print(" summarize: Nur Website-Zusammenfassung (AS) (Batch, Start bei leerem AS)")
|
||||
print(" branch: Nur Branchen-Einschätzung (AO) (Batch, Start bei leerem AO)")
|
||||
print(" update_wiki: Wiki-URL aus Spalte T übernehmen & Reparse/Re-Branch") # NEU
|
||||
print(" reeval: Verarbeitet Zeilen mit 'x' (volle Verarbeitung, alle TS prüfen)")
|
||||
print(" website_lookup: Sucht fehlende Websites (D)")
|
||||
print(" website_details:Extrahiert Details für Zeilen mit 'x' (AR)")
|
||||
print(" contacts: Sucht LinkedIn Kontakte (AM)")
|
||||
print(" full_run: Verarbeitet sequentiell ab erster Zeile ohne AO (alle TS prüfen)")
|
||||
print(" alignment: Schreibt Header A1:AX5 (!)")
|
||||
print(" train_technician_model: Trainiert Decision Tree zur Technikerschätzung")
|
||||
try:
|
||||
mode_input = input(f"Geben Sie den Modus ein ({', '.join(valid_modes)}): ").strip().lower()
|
||||
if mode_input in valid_modes: mode = mode_input
|
||||
else: print("Ungültige Eingabe -> combined"); mode = "combined"
|
||||
except Exception as e: print(f"Fehler Modus-Eingabe ({e}) -> combined"); mode = "combined"
|
||||
if mode_input in valid_modes: mode = mode_input
|
||||
else: print("Ungültige Eingabe -> combined"); mode = "combined"
|
||||
|
||||
# Zeilenlimit ermitteln (wie gehabt)
|
||||
# Zeilenlimit ermitteln
|
||||
row_limit = None
|
||||
if args.limit is not None:
|
||||
if args.limit >= 0: row_limit = args.limit; print(f"Zeilenlimit (aus Kommandozeile): {row_limit}")
|
||||
else: print("Warnung: Negatives Limit ignoriert."); row_limit = None
|
||||
elif mode in ["combined", "wiki", "website", "branch", "summarize", "full_run"]:
|
||||
try: limit_input = input("Max Zeilen? (Enter=alle): ");
|
||||
except Exception as e: print(f"Fehler Limit-Eingabe ({e}) -> Kein Limit"); row_limit = None
|
||||
if limit_input.strip():
|
||||
try: limit_val = int(limit_input);
|
||||
except ValueError: print("Ungültige Zahl -> Kein Limit"); row_limit = None
|
||||
if limit_val >= 0: row_limit = limit_val; print(f"Zeilenlimit: {row_limit}")
|
||||
else: print("Negatives Limit -> Kein Limit"); row_limit = None
|
||||
else: row_limit = None; print("Kein Zeilenlimit.")
|
||||
if args.limit >= 0: row_limit = args.limit; print(f"Zeilenlimit (aus Kommandozeile): {row_limit}")
|
||||
else: print("Warnung: Negatives Limit ignoriert."); row_limit = None
|
||||
elif mode in ["combined", "wiki", "website", "branch", "summarize", "full_run"]: # Nur für relevante Modi fragen
|
||||
try:
|
||||
limit_input = input("Max Zeilen? (Enter=alle): ");
|
||||
if limit_input.strip():
|
||||
try:
|
||||
limit_val = int(limit_input)
|
||||
if limit_val >= 0: row_limit = limit_val; print(f"Zeilenlimit: {row_limit}")
|
||||
else: print("Negatives Limit -> Kein Limit"); row_limit = None
|
||||
except ValueError: print("Ungültige Zahl -> Kein Limit"); row_limit = None
|
||||
else: row_limit = None; print("Kein Zeilenlimit.")
|
||||
except Exception as e: print(f"Fehler Limit-Eingabe ({e}) -> Kein Limit"); row_limit = None
|
||||
|
||||
# Logfile initialisieren (wie gehabt)
|
||||
|
||||
# Logfile initialisieren
|
||||
LOG_FILE = create_log_filename(mode)
|
||||
# ... (Logging Startparameter) ...
|
||||
debug_print(f"===== Skript gestartet ====="); debug_print(f"Version: {Config.VERSION}")
|
||||
debug_print(f"Betriebsmodus: {mode}"); # ... (Restliches Logging)
|
||||
debug_print(f"Betriebsmodus: {mode}");
|
||||
limit_log_text = str(row_limit) if row_limit is not None else 'N/A für diesen Modus'
|
||||
if mode in ["combined", "wiki", "website", "branch", "summarize", "full_run"]:
|
||||
limit_log_text = str(row_limit) if row_limit is not None else 'Unbegrenzt'
|
||||
if row_limit == 0: limit_log_text = '0 (Keine Verarbeitung geplant)'
|
||||
debug_print(f"Zeilenlimit: {limit_log_text}")
|
||||
debug_print(f"Logdatei: {LOG_FILE}")
|
||||
|
||||
# --- Vorbereitung ---
|
||||
load_target_schema()
|
||||
@@ -3632,139 +3810,96 @@ def main():
|
||||
start_time = time.time()
|
||||
debug_print(f"Starte Verarbeitung um {datetime.now().strftime('%H:%M:%S')}...")
|
||||
try:
|
||||
# Batch-Modi über Dispatcher
|
||||
if mode in ["wiki", "website", "branch", "summarize", "combined"]:
|
||||
if row_limit == 0: debug_print("Limit 0 -> Skip Dispatcher.")
|
||||
else: run_dispatcher(mode, sheet_handler, row_limit)
|
||||
# Einzelne Zeilen Modi (kein Batch-Dispatcher)
|
||||
elif mode == "reeval": data_processor.process_reevaluation_rows()
|
||||
elif mode == "website_lookup": data_processor.process_serp_website_lookup_for_empty()
|
||||
elif mode == "website_details": data_processor.process_website_details_for_marked_rows()
|
||||
elif mode == "contacts": process_contact_research(sheet_handler)
|
||||
elif mode == "full_run":
|
||||
if row_limit == 0: debug_print("Limit 0 -> Skip full_run.")
|
||||
else: # Logik für full_run (wie gehabt)
|
||||
start_index = sheet_handler.get_start_row_index(check_column_key="Timestamp letzte Prüfung")
|
||||
if start_index != -1 and start_index < len(sheet_handler.get_data()):
|
||||
#... (Berechne num_to_process) ...
|
||||
num_available = len(sheet_handler.get_data()) - start_index
|
||||
num_to_process = min(row_limit, num_available) if row_limit is not None and row_limit >= 0 else num_available
|
||||
if num_to_process > 0: data_processor.process_rows_sequentially(start_index, num_to_process)
|
||||
else: debug_print("Keine Zeilen für full_run.")
|
||||
else: debug_print(f"Startindex {start_index} für full_run ungültig.")
|
||||
elif mode == "alignment": # Logik für Alignment (wie gehabt)
|
||||
# ... (Bestätigungsabfrage und Aufruf) ...
|
||||
print("\nACHTUNG: Überschreibt A1:AX5!");
|
||||
try: confirm = input("Fortfahren? (j/N): ").strip().lower()
|
||||
except Exception as e_input: print(f"Input-Fehler: {e_input}"); confirm = 'n'
|
||||
if confirm == 'j': alignment_demo(sheet_handler.sheet)
|
||||
else: print("Abgebrochen.")
|
||||
else:
|
||||
start_index = sheet_handler.get_start_row_index(check_column_key="Timestamp letzte Prüfung")
|
||||
if start_index != -1 and start_index < len(sheet_handler.get_data()):
|
||||
num_available = len(sheet_handler.get_data()) - start_index
|
||||
num_to_process = min(row_limit, num_available) if row_limit is not None and row_limit >= 0 else num_available
|
||||
if num_to_process > 0:
|
||||
# Übergebe Flags an process_rows_sequentially
|
||||
data_processor.process_rows_sequentially(start_index, num_to_process, process_wiki=True, process_chatgpt=True, process_website=True)
|
||||
else: debug_print("Keine Zeilen für 'full_run' zu verarbeiten.")
|
||||
else: debug_print(f"Startindex {start_index} für 'full_run' ungültig.")
|
||||
elif mode == "alignment":
|
||||
print("\nACHTUNG: Überschreibt A1:AX5!"); # AX statt AS
|
||||
try: confirm = input("Fortfahren? (j/N): ").strip().lower()
|
||||
except Exception as e_input: print(f"Input-Fehler: {e_input}"); confirm = 'n'
|
||||
if confirm == 'j': alignment_demo(sheet_handler.sheet)
|
||||
else: print("Abgebrochen.")
|
||||
|
||||
# --- NEUER BLOCK: Modelltraining ---
|
||||
# --- NEU: Wiki Update Modus ---
|
||||
elif mode == "update_wiki":
|
||||
process_wiki_updates_from_chatgpt(sheet_handler, data_processor)
|
||||
# --- Ende Wiki Update Modus ---
|
||||
|
||||
# Block für Modelltraining (wie von dir bereitgestellt)
|
||||
elif mode == "train_technician_model":
|
||||
debug_print(f"Starte Modus: {mode}")
|
||||
|
||||
# 1. Daten vorbereiten
|
||||
prepared_df = data_processor.prepare_data_for_modeling()
|
||||
|
||||
if prepared_df is None or prepared_df.empty:
|
||||
debug_print("FEHLER: Datenvorbereitung fehlgeschlagen oder keine Daten vorhanden. Modus wird abgebrochen.")
|
||||
else:
|
||||
# 2. Train/Test Split
|
||||
debug_print("Aufteilen der Daten in Trainings- und Testsets (Testgröße 25%, stratifiziert)...")
|
||||
if prepared_df is not None and not prepared_df.empty:
|
||||
debug_print("Aufteilen der Daten...")
|
||||
try:
|
||||
# Features X (ohne Ziel und Hilfsspalten), Target y
|
||||
X = prepared_df.drop(columns=['Techniker_Bucket', 'name', 'Anzahl_Servicetechniker_Numeric'])
|
||||
y = prepared_df['Techniker_Bucket']
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42, stratify=y)
|
||||
debug_print(f"Trainingsdaten: {X_train.shape[0]} Zeilen, {X_train.shape[1]} Features")
|
||||
debug_print(f"Testdaten: {X_test.shape[0]} Zeilen, {X_test.shape[1]} Features")
|
||||
split_successful = True
|
||||
except Exception as e:
|
||||
debug_print(f"FEHLER beim Train/Test Split: {e}"); split_successful = False
|
||||
|
||||
except Exception as e: debug_print(f"FEHLER Split: {e}"); split_successful = False
|
||||
if split_successful:
|
||||
# 3. Imputation fehlender Werte
|
||||
debug_print("Imputation fehlender Werte für 'Finaler_Umsatz', 'Finaler_Mitarbeiter' (Median)...")
|
||||
debug_print("Imputation...")
|
||||
numeric_features = ['Finaler_Umsatz', 'Finaler_Mitarbeiter']
|
||||
try:
|
||||
imputer = SimpleImputer(strategy='median')
|
||||
# WICHTIG: Imputer NUR auf Trainingsdaten fitten!
|
||||
X_train[numeric_features] = imputer.fit_transform(X_train[numeric_features])
|
||||
# Testdaten NUR transformieren
|
||||
X_test[numeric_features] = imputer.transform(X_test[numeric_features])
|
||||
|
||||
# Speichere den Imputer
|
||||
imputer_filename = args.imputer_out # Verwende Argument oder Default
|
||||
with open(imputer_filename, 'wb') as f: pickle.dump(imputer, f)
|
||||
debug_print(f"Median-Imputer trainiert und gespeichert als '{imputer_filename}'.")
|
||||
imputer_filename = args.imputer_out; pickle.dump(imputer, open(imputer_filename, 'wb'))
|
||||
debug_print(f"Imputer gespeichert: '{imputer_filename}'.")
|
||||
imputation_successful = True
|
||||
except Exception as e:
|
||||
debug_print(f"FEHLER bei der Imputation: {e}"); imputation_successful = False
|
||||
|
||||
except Exception as e: debug_print(f"FEHLER Imputation: {e}"); imputation_successful = False
|
||||
if imputation_successful:
|
||||
# 4. Modelltraining & Hyperparameter-Tuning
|
||||
debug_print("Starte Decision Tree Training mit GridSearchCV...")
|
||||
# Verkleinerter Grid zum Testen, kann erweitert werden
|
||||
param_grid = {
|
||||
'criterion': ['gini', 'entropy'],
|
||||
'max_depth': [6, 8, 10, 12],
|
||||
'min_samples_split': [20, 40],
|
||||
'min_samples_leaf': [10, 20],
|
||||
'ccp_alpha': [0.0, 0.001, 0.005]
|
||||
}
|
||||
dtree = DecisionTreeClassifier(random_state=42, class_weight='balanced') # Balanced für ungleiche Klassen?
|
||||
# F1-Score oft besser für unbalancierte Klassen als Accuracy
|
||||
debug_print("Starte Training/GridSearchCV...")
|
||||
param_grid = { 'criterion': ['gini', 'entropy'], 'max_depth': [6, 8, 10, 12, 15], 'min_samples_split': [20, 40, 60], 'min_samples_leaf': [10, 20, 30], 'ccp_alpha': [0.0, 0.001, 0.005]}
|
||||
dtree = DecisionTreeClassifier(random_state=42, class_weight='balanced')
|
||||
grid_search = GridSearchCV(estimator=dtree, param_grid=param_grid, cv=5, scoring='f1_weighted', n_jobs=-1, verbose=1)
|
||||
|
||||
try:
|
||||
grid_search.fit(X_train, y_train)
|
||||
best_estimator = grid_search.best_estimator_
|
||||
debug_print(f"GridSearchCV abgeschlossen.")
|
||||
debug_print(f"Beste Parameter: {grid_search.best_params_}")
|
||||
debug_print(f"Bester Kreuzvalidierungs-Score (F1 Weighted): {grid_search.best_score_:.4f}")
|
||||
|
||||
# Speichere das beste Modell
|
||||
model_filename = args.model_out # Verwende Argument oder Default
|
||||
with open(model_filename, 'wb') as f: pickle.dump(best_estimator, f)
|
||||
debug_print(f"Bestes Modell gespeichert als '{model_filename}'.")
|
||||
debug_print(f"GridSearchCV fertig. Beste Params: {grid_search.best_params_}, Score: {grid_search.best_score_:.4f}")
|
||||
model_filename = args.model_out; pickle.dump(best_estimator, open(model_filename, 'wb'))
|
||||
debug_print(f"Modell gespeichert: '{model_filename}'.")
|
||||
training_successful = True
|
||||
|
||||
except Exception as e_train:
|
||||
debug_print(f"FEHLER während Training/Tuning: {e_train}"); training_successful = False
|
||||
import traceback; debug_print(traceback.format_exc())
|
||||
|
||||
except Exception as e_train: debug_print(f"FEHLER Training: {e_train}"); training_successful = False; import traceback; debug_print(traceback.format_exc())
|
||||
if training_successful:
|
||||
# 5. Evaluation auf dem Test-Set
|
||||
debug_print("Evaluiere bestes Modell auf dem Test-Set...")
|
||||
y_pred = best_estimator.predict(X_test)
|
||||
debug_print("Evaluiere Test-Set..."); y_pred = best_estimator.predict(X_test)
|
||||
test_accuracy = accuracy_score(y_test, y_pred)
|
||||
report = classification_report(y_test, y_pred, zero_division=0, labels=best_estimator.classes_, target_names=best_estimator.classes_)
|
||||
conf_matrix = confusion_matrix(y_test, y_pred, labels=best_estimator.classes_)
|
||||
# Erstelle DataFrame für bessere Lesbarkeit der Matrix
|
||||
conf_matrix_df = pd.DataFrame(conf_matrix, index=best_estimator.classes_, columns=best_estimator.classes_)
|
||||
|
||||
debug_print(f"\n--- Evaluationsergebnisse (Test-Set) ---")
|
||||
debug_print(f"Genauigkeit: {test_accuracy:.4f}")
|
||||
debug_print(f"Klassifikationsbericht:\n{report}")
|
||||
debug_print(f"Konfusionsmatrix:\n{conf_matrix_df}")
|
||||
print(f"\nModell-Evaluation abgeschlossen. Genauigkeit (Test): {test_accuracy:.4f}")
|
||||
print(f"Log für Details: {LOG_FILE}")
|
||||
|
||||
# 6. Muster extrahieren (Text)
|
||||
debug_print("\nExtrahiere Regeln aus dem besten Baum...")
|
||||
debug_print(f"\n--- Evaluation Test-Set ---\nGenauigkeit: {test_accuracy:.4f}\nBericht:\n{report}\nMatrix:\n{conf_matrix_df}"); print(f"\nModell Genauigkeit (Test): {test_accuracy:.4f}")
|
||||
debug_print("\nExtrahiere Regeln...");
|
||||
try:
|
||||
feature_names = list(X_train.columns)
|
||||
class_names = best_estimator.classes_
|
||||
rules_text = export_text(best_estimator, feature_names=feature_names, class_names=class_names, show_weights=True, spacing=3) # Bessere Formatierung
|
||||
# debug_print(f"--- Baumregeln (Text) ---:\n{rules_text}") # Kann sehr lang sein
|
||||
patterns_filename = args.patterns_out # Verwende Argument oder Default
|
||||
feature_names = list(X_train.columns); class_names = best_estimator.classes_
|
||||
rules_text = export_text(best_estimator, feature_names=feature_names, class_names=class_names, show_weights=True, spacing=3)
|
||||
patterns_filename = args.patterns_out;
|
||||
with open(patterns_filename, 'w', encoding='utf-8') as f: f.write(rules_text)
|
||||
debug_print(f"Regeln als Text gespeichert in '{patterns_filename}'.")
|
||||
debug_print(f"Regeln gespeichert: '{patterns_filename}'.")
|
||||
except Exception as e_export: debug_print(f"Fehler Export Regeln: {e_export}")
|
||||
else: debug_print("Datenvorbereitung fehlgeschlagen -> Abbruch ML Training.")
|
||||
|
||||
else:
|
||||
debug_print(f"Unbekannter Modus '{mode}'.")
|
||||
|
||||
except Exception as e: # Fange unerwartete Fehler im Hauptblock ab
|
||||
except Exception as e:
|
||||
debug_print(f"FATAL: Unerwarteter Fehler in main try-Block: {e}")
|
||||
import traceback; debug_print(traceback.format_exc())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user