Floke/Brancheneinstufung2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

duplicate_checker.py aktualisiert

Browse Source
This commit is contained in:
Floke
2025-08-01 11:51:05 +00:00
parent c3dd21c7cc
commit 220369881a
1 changed files with 21 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
duplicate_checker.py
View File

@@ -1,4 +1,4 @@
# duplicate_checker.py (v2.2 - Multi-Key Blocking & optimiertes Scoring) # duplicate_checker.py (v2.3 - Intelligent Blocking)
import logging import logging
import pandas as pd import pandas as pd
@@ -11,7 +11,13 @@ from collections import defaultdict
# --- Konfiguration --- # --- Konfiguration ---
CRM_SHEET_NAME = "CRM_Accounts" CRM_SHEET_NAME = "CRM_Accounts"
MATCHING_SHEET_NAME = "Matching_Accounts" MATCHING_SHEET_NAME = "Matching_Accounts"
SCORE_THRESHOLD = 85 # Etwas höherer Schwellenwert für bessere Präzision SCORE_THRESHOLD = 85 # Treffer unter diesem Wert werden nicht angezeigt
# NEU: Liste von generischen Wörtern, die für das Blocking ignoriert werden
BLOCKING_STOP_WORDS = {
'gmbh', 'ag', 'co', 'kg', 'se', 'holding', 'gruppe', 'industries', 'systems',
'technik', 'service', 'services', 'solutions', 'management', 'international'
}
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -22,7 +28,6 @@ def calculate_similarity_details(record1, record2):
if record1.get('normalized_domain') and record1['normalized_domain'] != 'k.a.' and record1['normalized_domain'] == record2.get('normalized_domain'): if record1.get('normalized_domain') and record1['normalized_domain'] != 'k.a.' and record1['normalized_domain'] == record2.get('normalized_domain'):
scores['domain'] = 100 scores['domain'] = 100
# Höhere Gewichtung für den Namen, da die Website oft fehlt
if record1.get('normalized_name') and record2.get('normalized_name'): if record1.get('normalized_name') and record2.get('normalized_name'):
scores['name'] = round(fuzz.token_set_ratio(record1['normalized_name'], record2['normalized_name']) * 0.85) scores['name'] = round(fuzz.token_set_ratio(record1['normalized_name'], record2['normalized_name']) * 0.85)
@@ -34,28 +39,27 @@ def calculate_similarity_details(record1, record2):
return {'total': total_score, 'details': scores} return {'total': total_score, 'details': scores}
def create_blocking_keys(name): def create_blocking_keys(name):
"""Erstellt mehrere Blocking Keys für einen Namen, um die Sensitivität zu erhöhen.""" """Erstellt mehrere Blocking Keys aus den signifikanten Wörtern eines Namens."""
if not name: if not name:
return [] return []
words = name.split() # Filtere Stop-Wörter aus der Wortliste
significant_words = [word for word in name.split() if word not in BLOCKING_STOP_WORDS]
keys = set() keys = set()
# 1. Erstes Wort # 1. Erstes signifikantes Wort
if len(words) > 0: if len(significant_words) > 0:
keys.add(words[0]) keys.add(significant_words[0])
# 2. Zweites Wort (falls vorhanden) # 2. Zweites signifikantes Wort (falls vorhanden)
if len(words) > 1: if len(significant_words) > 1:
keys.add(words[1]) keys.add(significant_words[1])
# 3. Erste 4 Buchstaben des ersten Wortes
if len(words) > 0 and len(words[0]) >= 4:
keys.add(words[0][:4])
return list(keys) return list(keys)
def main(): def main():
logging.info("Starte den Duplikats-Check (v2.2 mit Multi-Key Blocking)...") logging.info("Starte den Duplikats-Check (v2.3 mit Intelligent Blocking)...")
# ... (Initialisierung des GoogleSheetHandler bleibt gleich) ...
try: try:
sheet_handler = GoogleSheetHandler() sheet_handler = GoogleSheetHandler()
except Exception as e: except Exception as e:
@@ -69,7 +73,6 @@ def main():
logging.info(f"Lade zu prüfende Daten aus '{MATCHING_SHEET_NAME}'...") logging.info(f"Lade zu prüfende Daten aus '{MATCHING_SHEET_NAME}'...")
matching_df = sheet_handler.get_sheet_as_dataframe(MATCHING_SHEET_NAME) matching_df = sheet_handler.get_sheet_as_dataframe(MATCHING_SHEET_NAME)
if matching_df is None or matching_df.empty: return if matching_df is None or matching_df.empty: return
# Speichere eine saubere Kopie der Originaldaten für die Ausgabe
original_matching_df = matching_df.copy() original_matching_df = matching_df.copy()
logging.info("Normalisiere Daten für den Vergleich...") logging.info("Normalisiere Daten für den Vergleich...")
@@ -95,16 +98,11 @@ def main():
logging.info(f"Prüfe: {match_record['CRM Name']}...") logging.info(f"Prüfe: {match_record['CRM Name']}...")
# Sammle alle einzigartigen Kandidaten aus den Blöcken candidate_pool = {}
candidate_pool = {} # Verwende ein Dict, um Duplikate zu vermeiden
for key in match_record['block_keys']: for key in match_record['block_keys']:
for crm_record in crm_index.get(key, []): for crm_record in crm_index.get(key, []):
# Verwende den CRM Namen als eindeutigen Schlüssel für den Pool
candidate_pool[crm_record['CRM Name']] = crm_record candidate_pool[crm_record['CRM Name']] = crm_record
if not candidate_pool:
logging.debug(" -> Keine Kandidaten im Index gefunden.")
for crm_record in candidate_pool.values(): for crm_record in candidate_pool.values():
score_info = calculate_similarity_details(match_record, crm_record) score_info = calculate_similarity_details(match_record, crm_record)
if score_info['total'] > best_score_info['total']: if score_info['total'] > best_score_info['total']:
@@ -122,6 +120,7 @@ def main():
logging.info("Matching abgeschlossen. Schreibe Ergebnisse zurück ins Sheet...") logging.info("Matching abgeschlossen. Schreibe Ergebnisse zurück ins Sheet...")
result_df = pd.DataFrame(results) result_df = pd.DataFrame(results)
# Originalspalten aus der Kopie nehmen, um saubere Ausgabe zu garantieren
output_df = pd.concat([original_matching_df.reset_index(drop=True), result_df], axis=1) output_df = pd.concat([original_matching_df.reset_index(drop=True), result_df], axis=1)
data_to_write = [output_df.columns.values.tolist()] + output_df.values.tolist() data_to_write = [output_df.columns.values.tolist()] + output_df.values.tolist()