duplicate_checker.py aktualisiert

duplicate_checker.py

@@ -28,16 +28,13 @@ log_path = os.path.join(LOG_DIR, LOG_FILENAME)
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
-
 formatter = logging.Formatter('%(asctime)s - %(levelname)-8s - %(name)s - %(message)s')
-
-# Console Handler
+# Console handler
 console_handler = logging.StreamHandler()
 console_handler.setLevel(logging.INFO)
 console_handler.setFormatter(formatter)
 logger.addHandler(console_handler)
-
-# File Handler
+# File handler
 try:
     file_handler = logging.FileHandler(log_path, mode='a', encoding='utf-8')
     file_handler.setLevel(logging.DEBUG)
@@ -57,7 +54,6 @@ def normalize_company_name(name: str) -> str:
     tokens = [t for t in s.split() if t and t not in stops]
     return ' '.join(tokens)
 
-
 def normalize_domain(url: str) -> str:
     s = str(url).casefold().strip()
     s = re.sub(r'^https?://', '', s)
@@ -68,8 +64,7 @@ def normalize_domain(url: str) -> str:
 
 
 def main():
-    logger.info("Starte den Duplikats-Check (v2.0 mit Logging in /log)...")
-    # GoogleSheetHandler initialisieren
+    logger.info("Starte den Duplikats-Check (v2.0 mit Kandidaten-Logging)...")
     try:
         sheet_handler = GoogleSheetHandler()
         logger.info("GoogleSheetHandler initialisiert")
@@ -77,81 +72,81 @@ def main():
         logger.critical(f"FEHLER bei Initialisierung des GoogleSheetHandler: {e}")
         return
 
-    # CRM-Daten laden
-    logger.info(f"Lade Master-Daten aus '{CRM_SHEET_NAME}'...")
+    # Daten laden
     crm_df = sheet_handler.get_sheet_as_dataframe(CRM_SHEET_NAME)
-    if crm_df is None or crm_df.empty:
-        logger.critical(f"Konnte keine Daten aus '{CRM_SHEET_NAME}' laden. Abbruch.")
-        return
-    logger.info(f"{len(crm_df)} Zeilen aus '{CRM_SHEET_NAME}' geladen")
-
-    # Matching-Daten laden
-    logger.info(f"Lade zu prüfende Daten aus '{MATCHING_SHEET_NAME}'...")
     match_df = sheet_handler.get_sheet_as_dataframe(MATCHING_SHEET_NAME)
-    if match_df is None or match_df.empty:
-        logger.critical(f"Konnte keine Daten aus '{MATCHING_SHEET_NAME}' laden. Abbruch.")
+    if crm_df is None or crm_df.empty or match_df is None or match_df.empty:
+        logger.critical("CRM- oder Matching-Daten leer. Abbruch.")
         return
-    logger.info(f"{len(match_df)} Zeilen aus '{MATCHING_SHEET_NAME}' geladen")
+    logger.info(f"{len(crm_df)} CRM-Zeilen, {len(match_df)} Matching-Zeilen geladen")
 
     # Normalisierung
     for df, label in [(crm_df, 'CRM'), (match_df, 'Matching')]:
         df['norm_name'] = df['CRM Name'].fillna('').apply(normalize_company_name)
         df['norm_domain'] = df['CRM Website'].fillna('').apply(normalize_domain)
         df['city'] = df['CRM Ort'].fillna('').apply(lambda x: str(x).casefold().strip())
-        logger.debug(f"{label}-Daten normalisiert. Erste Zeile: {df.iloc[0].to_dict()}")
+        logger.debug(f"{label}-Daten normalisiert: Beispiel: {df.iloc[0][['norm_name','norm_domain','city']].to_dict()}")
 
-    # Blocking per Domain
+    # Blocking
     indexer = recordlinkage.Index()
     indexer.block('norm_domain')
     candidate_pairs = indexer.index(crm_df, match_df)
-    logger.info(f"Blocking abgeschlossen: {len(candidate_pairs)} Kandidatenpaare gefunden")
+    logger.info(f"Blocking abgeschlossen: {len(candidate_pairs)} Kandidatenpaare")
 
-    # Vergleichsregeln definieren
+    # Compare
     compare = recordlinkage.Compare()
     compare.exact('norm_domain', 'norm_domain', label='domain')
     compare.string('norm_name', 'norm_name', method='jarowinkler', label='name_sim')
     compare.exact('city', 'city', label='city')
     features = compare.compute(candidate_pairs, crm_df, match_df)
-    logger.debug(f"Feature-DataFrame Vorschau:\n{features.head()}")
+    logger.debug(f"Features berechnet: {features.head()}\n...")
 
-    # Score berechnen
-    features['score'] = (
-        WEIGHTS['domain'] * features['domain'] +
-        WEIGHTS['name']   * features['name_sim'] +
-        WEIGHTS['city']   * features['city']
-    )
+    # Score
+    features['score'] = (WEIGHTS['domain']*features['domain'] +
+                         WEIGHTS['name']*features['name_sim'] +
+                         WEIGHTS['city']*features['city'])
     logger.info("Scores berechnet")
 
-    # Best Match pro neuer Zeile mit detailliertem Logging
+    # Per Match Logging
     results = []
+    crm_df_idx = crm_df.reset_index()
     for match_idx, group in features.reset_index().groupby('level_1'):
-        logger.info(f"--- Prüfe: Zeile {match_idx} ---")
-        df_block = group.sort_values('score', ascending=False)
-        logger.debug(f"Kandidaten für Zeile {match_idx}:\n{df_block[['level_0','score','domain','name_sim','city']].to_string(index=False)}")
+        logger.info(f"--- Prüfe Matching-Zeile {match_idx} ---")
+        df_block = group.sort_values('score', ascending=False).copy()
+        # Enrich with CRM fields
+        df_block['CRM Name'] = df_block['level_0'].map(crm_df_idx.set_index('index')['CRM Name'])
+        df_block['CRM Website'] = df_block['level_0'].map(crm_df_idx.set_index('index')['CRM Website'])
+        df_block['CRM Ort'] = df_block['level_0'].map(crm_df_idx.set_index('index')['CRM Ort'])
+        # Log top candidates
+        logger.debug("Kandidaten (CRM_Index, Score, Domain, Name_sim, City, CRM Name):")
+        for _, row in df_block.iterrows():
+            logger.debug(f"  [{int(row['level_0'])}] score={row['score']:.3f} dom={row['domain']} name_sim={row['name_sim']:.3f} city={row['city']} => {row['CRM Name']}")
         top = df_block.iloc[0]
         crm_idx = top['level_0'] if top['score'] >= SCORE_THRESHOLD else None
         if crm_idx is not None:
-            logger.info(f"  --> Match: CRM-Index {crm_idx} mit Score {top['score']:.2f}")
+            logger.info(f"  --> Match: CRM-Index {int(crm_idx)} ({top['CRM Name']}) mit Score {top['score']:.2f}")
         else:
             logger.info(f"  --> Kein Match (höchster Score {top['score']:.2f})")
         results.append((crm_idx, match_idx, top['score']))
 
-    # Ausgabe DataFrame zusammenstellen
-    crm_df = crm_df.reset_index()
-    match_df = match_df.reset_index()
-    output = match_df[['CRM Name','CRM Website','CRM Ort','CRM Land']].copy()
-    output[['Matched CRM Name','Matched CRM Website','Matched CRM Ort','Matched CRM Land','Score']] = ''
-
+    # Prepare output
+    match_df_idx = match_df.reset_index()
+    output = match_df_idx[['CRM Name','CRM Website','CRM Ort','CRM Land']].copy()
+    output['Matched CRM Name'] = ''
+    output['Matched CRM Website'] = ''
+    output['Matched CRM Ort'] = ''
+    output['Matched CRM Land'] = ''
+    output['Score'] = 0.0
     for crm_idx, match_idx, score in results:
         if crm_idx is not None:
-            row_crm = crm_df.loc[crm_df['index']==crm_idx].iloc[0]
-            output.at[match_idx, 'Matched CRM Name'] = row_crm['CRM Name']
-            output.at[match_idx, 'Matched CRM Website'] = row_crm['CRM Website']
-            output.at[match_idx, 'Matched CRM Ort'] = row_crm['CRM Ort']
-            output.at[match_idx, 'Matched CRM Land'] = row_crm['CRM Land']
-        output.at[match_idx, 'Score'] = round(score, 3)
+            crm_row = crm_df_idx[crm_df_idx['index']==crm_idx].iloc[0]
+            output.at[match_idx, 'Matched CRM Name'] = crm_row['CRM Name']
+            output.at[match_idx, 'Matched CRM Website'] = crm_row['CRM Website']
+            output.at[match_idx, 'Matched CRM Ort'] = crm_row['CRM Ort']
+            output.at[match_idx, 'Matched CRM Land'] = crm_row['CRM Land']
+        output.at[match_idx, 'Score'] = round(score,3)
 
-    # Zurückschreiben ins Google Sheet
+    # Write back
     data = [output.columns.tolist()] + output.values.tolist()
     success = sheet_handler.clear_and_write_data(MATCHING_SHEET_NAME, data)
     if success: