diff --git a/duplicate_checker.py b/duplicate_checker.py new file mode 100644 index 00000000..57a64a84 --- /dev/null +++ b/duplicate_checker.py @@ -0,0 +1,112 @@ +# duplicate_checker.py + +import logging +import pandas as pd +from thefuzz import fuzz +from config import Config +from helpers import normalize_company_name, simple_normalize_url +from google_sheet_handler import GoogleSheetHandler + +# --- Konfiguration --- +CRM_SHEET_NAME = "CRM_Accounts" +MATCHING_SHEET_NAME = "Matching_Accounts" +SCORE_THRESHOLD = 80 # Mindest-Score, um als "wahrscheinlicher Treffer" zu gelten + +# --- Logging einrichten --- +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +def calculate_similarity(record1, record2): + """Berechnet einen gewichteten Ähnlichkeits-Score zwischen zwei Datensätzen.""" + total_score = 0 + + # 1. Website-Domain (stärkstes Signal) + if record1['normalized_domain'] and record1['normalized_domain'] == record2['normalized_domain']: + total_score += 100 + + # 2. Firmenname (Fuzzy-Signal) + if record1['normalized_name'] and record2['normalized_name']: + name_similarity = fuzz.token_set_ratio(record1['normalized_name'], record2['normalized_name']) + total_score += name_similarity * 0.7 # Gewichtung: 70% + + # 3. Standort (Bestätigungs-Signal) + if record1['CRM Ort'] and record1['CRM Ort'] == record2['CRM Ort']: + if record1['CRM Land'] and record1['CRM Land'] == record2['CRM Land']: + total_score += 20 # Bonus für vollen Standort-Match + + return round(total_score) + +def main(): + """Hauptfunktion zum Laden, Vergleichen und Schreiben der Daten.""" + logging.info("Starte den Duplikats-Check...") + + try: + sheet_handler = GoogleSheetHandler() + except Exception as e: + logging.critical(f"FEHLER bei der Initialisierung des GoogleSheetHandler: {e}") + return + + # 1. Daten laden + logging.info(f"Lade Master-Daten aus '{CRM_SHEET_NAME}'...") + crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME) + if crm_df is None or crm_df.empty: + logging.critical(f"Konnte keine Daten aus '{CRM_SHEET_NAME}' laden. Breche ab.") + return + + logging.info(f"Lade zu prüfende Daten aus '{MATCHING_SHEET_NAME}'...") + matching_df = sheet_handler.get_sheet_as_dataframe(MATCHING_SHEET_NAME) + if matching_df is None or matching_df.empty: + logging.critical(f"Konnte keine Daten aus '{MATCHING_SHEET_NAME}' laden. Breche ab.") + return + + # 2. Daten normalisieren + logging.info("Normalisiere Daten für den Vergleich...") + crm_df['normalized_name'] = crm_df['CRM Name'].astype(str).apply(normalize_company_name) + crm_df['normalized_domain'] = crm_df['CRM Website'].astype(str).apply(simple_normalize_url) + crm_df['CRM Ort'] = crm_df['CRM Ort'].astype(str).str.lower().str.strip() + crm_df['CRM Land'] = crm_df['CRM Land'].astype(str).str.lower().str.strip() + + matching_df['normalized_name'] = matching_df['CRM Name'].astype(str).apply(normalize_company_name) + matching_df['normalized_domain'] = matching_df['CRM Website'].astype(str).apply(simple_normalize_url) + matching_df['CRM Ort'] = matching_df['CRM Ort'].astype(str).str.lower().str.strip() + matching_df['CRM Land'] = matching_df['CRM Land'].astype(str).str.lower().str.strip() + + # 3. Matching-Prozess + logging.info("Starte Matching-Prozess... Dies kann einige Zeit dauern.") + results = [] + + for index, match_row in matching_df.iterrows(): + best_score = 0 + best_match_name = "" + + logging.info(f"Prüfe: {match_row['CRM Name']}...") + + for _, crm_row in crm_df.iterrows(): + score = calculate_similarity(match_row, crm_row) + if score > best_score: + best_score = score + best_match_name = crm_row['CRM Name'] + + if best_score >= SCORE_THRESHOLD: + results.append({'Potenzieller Treffer im CRM': best_match_name, 'Ähnlichkeits-Score': best_score}) + else: + results.append({'Potenzieller Treffer im CRM': '', 'Ähnlichkeits-Score': best_score}) + + # 4. Ergebnisse zusammenführen und schreiben + logging.info("Matching abgeschlossen. Schreibe Ergebnisse zurück ins Sheet...") + result_df = pd.DataFrame(results) + output_df = pd.concat([matching_df.reset_index(drop=True), result_df], axis=1) + + # Entferne die temporären normalisierten Spalten für eine saubere Ausgabe + output_df = output_df.drop(columns=['normalized_name', 'normalized_domain']) + + # Konvertiere DataFrame in Liste von Listen für den Upload + data_to_write = [output_df.columns.values.tolist()] + output_df.values.tolist() + + success = sheet_handler.clear_and_write_data(MATCHING_SHEET_NAME, data_to_write) + if success: + logging.info(f"Ergebnisse erfolgreich in das Tabellenblatt '{MATCHING_SHEET_NAME}' geschrieben.") + else: + logging.error("FEHLER beim Schreiben der Ergebnisse ins Google Sheet.") + +if __name__ == "__main__": + main() \ No newline at end of file