From ffad12dae541ef229b70e54ba109ef52a56d196a Mon Sep 17 00:00:00 2001 From: Floke Date: Fri, 5 Sep 2025 08:59:05 +0000 Subject: [PATCH] duplicate_checker.py aktualisiert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NEU: "Shortest Name Tie-Breaker": Bei sehr ähnlichen Scores wird der Kandidat mit dem kürzeren Namen bevorzugt, um das Prinzip der "wirtschaftlichen Einheit" (z.B. Holding) besser abzubilden. - Scoring-Formel und Schwellenwerte erneut feinjustiert für finale Balance. - Golden-Rule und Interaktiver Modus beibehalten. --- duplicate_checker.py | 97 ++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/duplicate_checker.py b/duplicate_checker.py index 46580b5d..dc05c4c8 100644 --- a/duplicate_checker.py +++ b/duplicate_checker.py @@ -1,11 +1,11 @@ -# duplicate_checker.py v3.2 +# duplicate_checker.py v3.3 # Build timestamp is injected into logfile name. -# --- ÄNDERUNGEN v3.2 --- -# - Scoring-Formel und Multiplikatoren neu gewichtet, um einzigartige Namens-Tokens stärker zu bewerten ("Großzügigkeits-Boost"). -# - Schwellenwerte (Thresholds) erneut feinjustiert, um die Balance zwischen korrekten und falschen Treffern zu optimieren. -# - Logik des Domain-Gates beibehalten und sichergestellt, dass es korrekt greift. -# - Golden-Rule und Interaktiver Modus unverändert. +# --- ÄNDERUNGEN v3.3 --- +# - NEU: "Shortest Name Tie-Breaker": Bei sehr ähnlichen Scores wird der Kandidat mit dem kürzeren Namen bevorzugt, +# um das Prinzip der "wirtschaftlichen Einheit" (z.B. Holding) besser abzubilden. +# - Scoring-Formel und Schwellenwerte erneut feinjustiert für finale Balance. +# - Golden-Rule und Interaktiver Modus beibehalten. import os import sys @@ -25,6 +25,7 @@ from google_sheet_handler import GoogleSheetHandler STATUS_DIR = "job_status" def update_status(job_id, status, progress_message): + # ... (Keine Änderungen hier) if not job_id: return status_file = os.path.join(STATUS_DIR, f"{job_id}.json") try: @@ -46,17 +47,20 @@ CRM_SHEET_NAME = "CRM_Accounts" MATCHING_SHEET_NAME = "Matching_Accounts" LOG_DIR = "Log" now = datetime.now().strftime('%Y-%m-%d_%H-%M') -LOG_FILE = f"{now}_duplicate_check_v3.2.txt" +LOG_FILE = f"{now}_duplicate_check_v3.3.txt" -# --- NEU: Angepasste Scoring-Konfiguration v3.2 --- -SCORE_THRESHOLD = 90 # Standard-Schwelle leicht angehoben -SCORE_THRESHOLD_WEAK= 120 # Schwelle für Matches ohne Domain oder Ort angepasst -GOLDEN_MATCH_RATIO = 95 +# --- NEU: Angepasste Scoring-Konfiguration v3.3 --- +SCORE_THRESHOLD = 95 # Standard-Schwelle +SCORE_THRESHOLD_WEAK= 125 # Schwelle für Matches ohne Domain oder Ort +GOLDEN_MATCH_RATIO = 97 # Leicht großzügiger GOLDEN_MATCH_SCORE = 300 -MIN_NAME_SCORE_FOR_DOMAIN = 2.5 # Mindest-Namensscore, damit ein Domain-Match voll zählt +MIN_NAME_SCORE_FOR_DOMAIN = 3.0 + +# Tie-Breaker Konfiguration +TIE_SCORE_DIFF = 15 # Max Score-Unterschied für Tie-Breaking # Interaktiver Modus Konfiguration -INTERACTIVE_SCORE_MIN = 90 +INTERACTIVE_SCORE_MIN = 95 INTERACTIVE_SCORE_DIFF = 20 # Prefilter-Konfiguration @@ -83,8 +87,7 @@ fh.setFormatter(formatter) root.addHandler(fh) logger = logging.getLogger(__name__) logger.info(f"Logging to console and file: {log_path}") -logger.info(f"Starting duplicate_checker.py v3.2 | Build: {now}") - +logger.info(f"Starting duplicate_checker.py v3.3 | Build: {now}") # --- SerpAPI Key laden --- # ... (Keine Änderungen hier) @@ -97,6 +100,7 @@ except Exception as e: logger.warning(f"Fehler beim Laden API-Keys: {e}") serp_key = None + # --- Stop-/City-Tokens --- STOP_TOKENS_BASE = { 'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv', @@ -114,7 +118,7 @@ def _tokenize(s: str): def clean_name_for_scoring(norm_name: str): if not norm_name: return "", set() - tokens = [t for t in _tokenize(norm_name) if len(t) >= 2] # auch 2-Buchstaben-Tokens zulassen + tokens = [t for t in _tokenize(norm_name) if len(t) >= 3] stop_union = STOP_TOKENS_BASE | CITY_TOKENS final_tokens = [t for t in tokens if t not in stop_union] return " ".join(final_tokens), set(final_tokens) @@ -138,13 +142,13 @@ def build_term_weights(crm_df: pd.DataFrame): logger.info(f"Wortgewichte für {len(term_weights)} Tokens berechnet.") return term_weights -# --- Similarity v3.2 --- +# --- Similarity v3.3 --- def calculate_similarity(mrec: dict, crec: dict, term_weights: dict): n1_raw = mrec.get('normalized_name', '') n2_raw = crec.get('normalized_name', '') if fuzz.ratio(n1_raw, n2_raw) >= GOLDEN_MATCH_RATIO: - return GOLDEN_MATCH_SCORE, {'reason': f'Golden Match (Name Ratio >= {GOLDEN_MATCH_RATIO}%)', 'name_score': 100} + return GOLDEN_MATCH_SCORE, {'reason': f'Golden Match (Ratio >= {GOLDEN_MATCH_RATIO}%)', 'name_score': 100} dom1 = mrec.get('normalized_domain','') dom2 = crec.get('normalized_domain','') @@ -156,33 +160,25 @@ def calculate_similarity(mrec: dict, crec: dict, term_weights: dict): clean1, toks1 = clean_name_for_scoring(n1_raw) clean2, toks2 = clean_name_for_scoring(n2_raw) - # --- ÄNDERUNG v3.2: Gewichteter Token Set Score --- - # Belohnt Übereinstimmung, bestraft aber auch fehlende wichtige Wörter name_score = 0 overlapping_tokens = toks1 & toks2 - if overlapping_tokens: - sum_overlap = sum(term_weights.get(token, 0) for token in overlapping_tokens) - sum_toks1 = sum(term_weights.get(token, 0) for token in toks1) - sum_toks2 = sum(term_weights.get(token, 0) for token in toks2) - - if (sum_toks1 + sum_toks2) > 0: - # Dice-Koeffizient auf Basis der Gewichte - name_score = (2 * sum_overlap) / (sum_toks1 + sum_toks2) * 100 - - # Domain-Gate + name_score = sum(term_weights.get(token, 0) for token in overlapping_tokens) + if toks1: + overlap_percentage = len(overlapping_tokens) / len(toks1) + name_score *= (1 + overlap_percentage) + score_domain = 0 - # Name Score für Domain Gate wird jetzt direkt aus der Ratio berechnet, nicht aus dem gewichteten Score if domain_match: - if fuzz.token_set_ratio(clean1, clean2) > 60 or (city_match and country_match): - score_domain = 60 # Starker Bonus + if name_score >= MIN_NAME_SCORE_FOR_DOMAIN: + score_domain = 75 else: - score_domain = 15 # Schwacher Bonus + score_domain = 20 score_location = 25 if (city_match and country_match) else 0 - # --- ÄNDERUNG v3.2: Finale Score-Kalibrierung --- - total = name_score * 1.2 + score_domain + score_location + # --- ÄNDERUNG v3.3: Angepasste Gewichtung --- + total = name_score * 10 + score_domain + score_location penalties = 0 if mrec.get('CRM Land') and crec.get('CRM Land') and not country_match: @@ -203,8 +199,6 @@ def calculate_similarity(mrec: dict, crec: dict, term_weights: dict): return max(0, round(total)), comp # --- Indexe & Hauptfunktion --- -# (Die folgenden Funktionen bleiben strukturell gleich, aber rufen jetzt die angepassten Helper auf) - def build_indexes(crm_df: pd.DataFrame): records = list(crm_df.to_dict('records')) domain_index = {} @@ -227,10 +221,8 @@ def choose_rarest_token(norm_name: str, term_weights: dict): return rarest if term_weights.get(rarest, 0) > 0 else None def main(job_id=None, interactive=False): - logger.info("Starte Duplikats-Check v3.2 (Final Recalibration)") - # ... - # (Code für Initialisierung und Datenladen bleibt identisch zu v3.1) - # ... + logger.info("Starte Duplikats-Check v3.3 (Tie-Breaker Final Calibration)") + # ... (Code für Initialisierung und Datenladen bleibt identisch) ... update_status(job_id, "Läuft", "Initialisiere GoogleSheetHandler...") try: sheet = GoogleSheetHandler() @@ -322,7 +314,7 @@ def main(job_id=None, interactive=False): pf.sort(key=lambda x: x[0], reverse=True) candidate_indices.update([i for _, i in pf[:PREFILTER_LIMIT]]) used_block = f"prefilter:{PREFILTER_MIN_PARTIAL}/{len(pf)}" - + candidates = [crm_records[i] for i in candidate_indices] logger.info(f"Prüfe {processed}/{total}: '{mrow.get('CRM Name','')}' -> {len(candidates)} Kandidaten (Block={used_block})") if not candidates: @@ -339,7 +331,22 @@ def main(job_id=None, interactive=False): logger.debug(f" Kandidat: {cand['name']} | Score={cand['score']} | Comp={cand['comp']}") best_match = scored[0] if scored else None - + + # --- NEU: "Shortest Name Tie-Breaker" Logik --- + if best_match and len(scored) > 1: + best_score = best_match['score'] + second_best_score = scored[1]['score'] + # Wenn Scores sehr nah beieinander liegen UND es kein Golden Match ist + if best_score < GOLDEN_MATCH_SCORE and (best_score - second_best_score) < TIE_SCORE_DIFF: + logger.info(f" Tie-Breaker-Situation erkannt für '{mrow['CRM Name']}'. Scores: {best_score} vs {second_best_score}") + # Finde alle Kandidaten im "Tie-Bereich" + tie_candidates = [c for c in scored if (best_score - c['score']) < TIE_SCORE_DIFF] + # Wähle den Kandidaten mit dem kürzesten Namen + best_match_by_length = min(tie_candidates, key=lambda x: len(x['name'])) + if best_match_by_length['name'] != best_match['name']: + logger.info(f" Tie-Breaker angewendet: '{best_match['name']}' ({best_score}) -> '{best_match_by_length['name']}' ({best_match_by_length['score']}) wegen kürzerem Namen.") + best_match = best_match_by_length + # Interaktiver Modus if interactive and best_match and len(scored) > 1: best_score = best_match['score'] @@ -419,7 +426,7 @@ def main(job_id=None, interactive=False): update_status(job_id, "Fehlgeschlagen", "Fehler beim Schreiben ins Google Sheet.") if __name__=='__main__': - parser = argparse.ArgumentParser(description="Duplicate Checker v3.2") + parser = argparse.ArgumentParser(description="Duplicate Checker v3.3") parser.add_argument("--job-id", type=str, help="Eindeutige ID für den Job-Status.") parser.add_argument("--interactive", action='store_true', help="Aktiviert den interaktiven Modus für unklare Fälle.") args = parser.parse_args()