duplicate_checker.py aktualisiert
- NEU: "Shortest Name Tie-Breaker": Bei sehr ähnlichen Scores wird der Kandidat mit dem kürzeren Namen bevorzugt, um das Prinzip der "wirtschaftlichen Einheit" (z.B. Holding) besser abzubilden. - Scoring-Formel und Schwellenwerte erneut feinjustiert für finale Balance. - Golden-Rule und Interaktiver Modus beibehalten.
This commit is contained in:
@@ -1,11 +1,11 @@
|
||||
# duplicate_checker.py v3.2
|
||||
# duplicate_checker.py v3.3
|
||||
# Build timestamp is injected into logfile name.
|
||||
|
||||
# --- ÄNDERUNGEN v3.2 ---
|
||||
# - Scoring-Formel und Multiplikatoren neu gewichtet, um einzigartige Namens-Tokens stärker zu bewerten ("Großzügigkeits-Boost").
|
||||
# - Schwellenwerte (Thresholds) erneut feinjustiert, um die Balance zwischen korrekten und falschen Treffern zu optimieren.
|
||||
# - Logik des Domain-Gates beibehalten und sichergestellt, dass es korrekt greift.
|
||||
# - Golden-Rule und Interaktiver Modus unverändert.
|
||||
# --- ÄNDERUNGEN v3.3 ---
|
||||
# - NEU: "Shortest Name Tie-Breaker": Bei sehr ähnlichen Scores wird der Kandidat mit dem kürzeren Namen bevorzugt,
|
||||
# um das Prinzip der "wirtschaftlichen Einheit" (z.B. Holding) besser abzubilden.
|
||||
# - Scoring-Formel und Schwellenwerte erneut feinjustiert für finale Balance.
|
||||
# - Golden-Rule und Interaktiver Modus beibehalten.
|
||||
|
||||
import os
|
||||
import sys
|
||||
@@ -25,6 +25,7 @@ from google_sheet_handler import GoogleSheetHandler
|
||||
STATUS_DIR = "job_status"
|
||||
|
||||
def update_status(job_id, status, progress_message):
|
||||
# ... (Keine Änderungen hier)
|
||||
if not job_id: return
|
||||
status_file = os.path.join(STATUS_DIR, f"{job_id}.json")
|
||||
try:
|
||||
@@ -46,17 +47,20 @@ CRM_SHEET_NAME = "CRM_Accounts"
|
||||
MATCHING_SHEET_NAME = "Matching_Accounts"
|
||||
LOG_DIR = "Log"
|
||||
now = datetime.now().strftime('%Y-%m-%d_%H-%M')
|
||||
LOG_FILE = f"{now}_duplicate_check_v3.2.txt"
|
||||
LOG_FILE = f"{now}_duplicate_check_v3.3.txt"
|
||||
|
||||
# --- NEU: Angepasste Scoring-Konfiguration v3.2 ---
|
||||
SCORE_THRESHOLD = 90 # Standard-Schwelle leicht angehoben
|
||||
SCORE_THRESHOLD_WEAK= 120 # Schwelle für Matches ohne Domain oder Ort angepasst
|
||||
GOLDEN_MATCH_RATIO = 95
|
||||
# --- NEU: Angepasste Scoring-Konfiguration v3.3 ---
|
||||
SCORE_THRESHOLD = 95 # Standard-Schwelle
|
||||
SCORE_THRESHOLD_WEAK= 125 # Schwelle für Matches ohne Domain oder Ort
|
||||
GOLDEN_MATCH_RATIO = 97 # Leicht großzügiger
|
||||
GOLDEN_MATCH_SCORE = 300
|
||||
MIN_NAME_SCORE_FOR_DOMAIN = 2.5 # Mindest-Namensscore, damit ein Domain-Match voll zählt
|
||||
MIN_NAME_SCORE_FOR_DOMAIN = 3.0
|
||||
|
||||
# Tie-Breaker Konfiguration
|
||||
TIE_SCORE_DIFF = 15 # Max Score-Unterschied für Tie-Breaking
|
||||
|
||||
# Interaktiver Modus Konfiguration
|
||||
INTERACTIVE_SCORE_MIN = 90
|
||||
INTERACTIVE_SCORE_MIN = 95
|
||||
INTERACTIVE_SCORE_DIFF = 20
|
||||
|
||||
# Prefilter-Konfiguration
|
||||
@@ -83,8 +87,7 @@ fh.setFormatter(formatter)
|
||||
root.addHandler(fh)
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info(f"Logging to console and file: {log_path}")
|
||||
logger.info(f"Starting duplicate_checker.py v3.2 | Build: {now}")
|
||||
|
||||
logger.info(f"Starting duplicate_checker.py v3.3 | Build: {now}")
|
||||
|
||||
# --- SerpAPI Key laden ---
|
||||
# ... (Keine Änderungen hier)
|
||||
@@ -97,6 +100,7 @@ except Exception as e:
|
||||
logger.warning(f"Fehler beim Laden API-Keys: {e}")
|
||||
serp_key = None
|
||||
|
||||
|
||||
# --- Stop-/City-Tokens ---
|
||||
STOP_TOKENS_BASE = {
|
||||
'gmbh','mbh','ag','kg','ug','ohg','se','co','kgaa','inc','llc','ltd','sarl', 'b.v', 'bv',
|
||||
@@ -114,7 +118,7 @@ def _tokenize(s: str):
|
||||
|
||||
def clean_name_for_scoring(norm_name: str):
|
||||
if not norm_name: return "", set()
|
||||
tokens = [t for t in _tokenize(norm_name) if len(t) >= 2] # auch 2-Buchstaben-Tokens zulassen
|
||||
tokens = [t for t in _tokenize(norm_name) if len(t) >= 3]
|
||||
stop_union = STOP_TOKENS_BASE | CITY_TOKENS
|
||||
final_tokens = [t for t in tokens if t not in stop_union]
|
||||
return " ".join(final_tokens), set(final_tokens)
|
||||
@@ -138,13 +142,13 @@ def build_term_weights(crm_df: pd.DataFrame):
|
||||
logger.info(f"Wortgewichte für {len(term_weights)} Tokens berechnet.")
|
||||
return term_weights
|
||||
|
||||
# --- Similarity v3.2 ---
|
||||
# --- Similarity v3.3 ---
|
||||
def calculate_similarity(mrec: dict, crec: dict, term_weights: dict):
|
||||
|
||||
n1_raw = mrec.get('normalized_name', '')
|
||||
n2_raw = crec.get('normalized_name', '')
|
||||
if fuzz.ratio(n1_raw, n2_raw) >= GOLDEN_MATCH_RATIO:
|
||||
return GOLDEN_MATCH_SCORE, {'reason': f'Golden Match (Name Ratio >= {GOLDEN_MATCH_RATIO}%)', 'name_score': 100}
|
||||
return GOLDEN_MATCH_SCORE, {'reason': f'Golden Match (Ratio >= {GOLDEN_MATCH_RATIO}%)', 'name_score': 100}
|
||||
|
||||
dom1 = mrec.get('normalized_domain','')
|
||||
dom2 = crec.get('normalized_domain','')
|
||||
@@ -156,33 +160,25 @@ def calculate_similarity(mrec: dict, crec: dict, term_weights: dict):
|
||||
clean1, toks1 = clean_name_for_scoring(n1_raw)
|
||||
clean2, toks2 = clean_name_for_scoring(n2_raw)
|
||||
|
||||
# --- ÄNDERUNG v3.2: Gewichteter Token Set Score ---
|
||||
# Belohnt Übereinstimmung, bestraft aber auch fehlende wichtige Wörter
|
||||
name_score = 0
|
||||
overlapping_tokens = toks1 & toks2
|
||||
|
||||
if overlapping_tokens:
|
||||
sum_overlap = sum(term_weights.get(token, 0) for token in overlapping_tokens)
|
||||
sum_toks1 = sum(term_weights.get(token, 0) for token in toks1)
|
||||
sum_toks2 = sum(term_weights.get(token, 0) for token in toks2)
|
||||
|
||||
if (sum_toks1 + sum_toks2) > 0:
|
||||
# Dice-Koeffizient auf Basis der Gewichte
|
||||
name_score = (2 * sum_overlap) / (sum_toks1 + sum_toks2) * 100
|
||||
|
||||
# Domain-Gate
|
||||
name_score = sum(term_weights.get(token, 0) for token in overlapping_tokens)
|
||||
if toks1:
|
||||
overlap_percentage = len(overlapping_tokens) / len(toks1)
|
||||
name_score *= (1 + overlap_percentage)
|
||||
|
||||
score_domain = 0
|
||||
# Name Score für Domain Gate wird jetzt direkt aus der Ratio berechnet, nicht aus dem gewichteten Score
|
||||
if domain_match:
|
||||
if fuzz.token_set_ratio(clean1, clean2) > 60 or (city_match and country_match):
|
||||
score_domain = 60 # Starker Bonus
|
||||
if name_score >= MIN_NAME_SCORE_FOR_DOMAIN:
|
||||
score_domain = 75
|
||||
else:
|
||||
score_domain = 15 # Schwacher Bonus
|
||||
score_domain = 20
|
||||
|
||||
score_location = 25 if (city_match and country_match) else 0
|
||||
|
||||
# --- ÄNDERUNG v3.2: Finale Score-Kalibrierung ---
|
||||
total = name_score * 1.2 + score_domain + score_location
|
||||
# --- ÄNDERUNG v3.3: Angepasste Gewichtung ---
|
||||
total = name_score * 10 + score_domain + score_location
|
||||
|
||||
penalties = 0
|
||||
if mrec.get('CRM Land') and crec.get('CRM Land') and not country_match:
|
||||
@@ -203,8 +199,6 @@ def calculate_similarity(mrec: dict, crec: dict, term_weights: dict):
|
||||
return max(0, round(total)), comp
|
||||
|
||||
# --- Indexe & Hauptfunktion ---
|
||||
# (Die folgenden Funktionen bleiben strukturell gleich, aber rufen jetzt die angepassten Helper auf)
|
||||
|
||||
def build_indexes(crm_df: pd.DataFrame):
|
||||
records = list(crm_df.to_dict('records'))
|
||||
domain_index = {}
|
||||
@@ -227,10 +221,8 @@ def choose_rarest_token(norm_name: str, term_weights: dict):
|
||||
return rarest if term_weights.get(rarest, 0) > 0 else None
|
||||
|
||||
def main(job_id=None, interactive=False):
|
||||
logger.info("Starte Duplikats-Check v3.2 (Final Recalibration)")
|
||||
# ...
|
||||
# (Code für Initialisierung und Datenladen bleibt identisch zu v3.1)
|
||||
# ...
|
||||
logger.info("Starte Duplikats-Check v3.3 (Tie-Breaker Final Calibration)")
|
||||
# ... (Code für Initialisierung und Datenladen bleibt identisch) ...
|
||||
update_status(job_id, "Läuft", "Initialisiere GoogleSheetHandler...")
|
||||
try:
|
||||
sheet = GoogleSheetHandler()
|
||||
@@ -322,7 +314,7 @@ def main(job_id=None, interactive=False):
|
||||
pf.sort(key=lambda x: x[0], reverse=True)
|
||||
candidate_indices.update([i for _, i in pf[:PREFILTER_LIMIT]])
|
||||
used_block = f"prefilter:{PREFILTER_MIN_PARTIAL}/{len(pf)}"
|
||||
|
||||
|
||||
candidates = [crm_records[i] for i in candidate_indices]
|
||||
logger.info(f"Prüfe {processed}/{total}: '{mrow.get('CRM Name','')}' -> {len(candidates)} Kandidaten (Block={used_block})")
|
||||
if not candidates:
|
||||
@@ -339,7 +331,22 @@ def main(job_id=None, interactive=False):
|
||||
logger.debug(f" Kandidat: {cand['name']} | Score={cand['score']} | Comp={cand['comp']}")
|
||||
|
||||
best_match = scored[0] if scored else None
|
||||
|
||||
|
||||
# --- NEU: "Shortest Name Tie-Breaker" Logik ---
|
||||
if best_match and len(scored) > 1:
|
||||
best_score = best_match['score']
|
||||
second_best_score = scored[1]['score']
|
||||
# Wenn Scores sehr nah beieinander liegen UND es kein Golden Match ist
|
||||
if best_score < GOLDEN_MATCH_SCORE and (best_score - second_best_score) < TIE_SCORE_DIFF:
|
||||
logger.info(f" Tie-Breaker-Situation erkannt für '{mrow['CRM Name']}'. Scores: {best_score} vs {second_best_score}")
|
||||
# Finde alle Kandidaten im "Tie-Bereich"
|
||||
tie_candidates = [c for c in scored if (best_score - c['score']) < TIE_SCORE_DIFF]
|
||||
# Wähle den Kandidaten mit dem kürzesten Namen
|
||||
best_match_by_length = min(tie_candidates, key=lambda x: len(x['name']))
|
||||
if best_match_by_length['name'] != best_match['name']:
|
||||
logger.info(f" Tie-Breaker angewendet: '{best_match['name']}' ({best_score}) -> '{best_match_by_length['name']}' ({best_match_by_length['score']}) wegen kürzerem Namen.")
|
||||
best_match = best_match_by_length
|
||||
|
||||
# Interaktiver Modus
|
||||
if interactive and best_match and len(scored) > 1:
|
||||
best_score = best_match['score']
|
||||
@@ -419,7 +426,7 @@ def main(job_id=None, interactive=False):
|
||||
update_status(job_id, "Fehlgeschlagen", "Fehler beim Schreiben ins Google Sheet.")
|
||||
|
||||
if __name__=='__main__':
|
||||
parser = argparse.ArgumentParser(description="Duplicate Checker v3.2")
|
||||
parser = argparse.ArgumentParser(description="Duplicate Checker v3.3")
|
||||
parser.add_argument("--job-id", type=str, help="Eindeutige ID für den Job-Status.")
|
||||
parser.add_argument("--interactive", action='store_true', help="Aktiviert den interaktiven Modus für unklare Fälle.")
|
||||
args = parser.parse_args()
|
||||
|
||||
Reference in New Issue
Block a user