diff --git a/brancheneinstufung.py b/brancheneinstufung.py index 7a27c142..c99f9ee6 100644 --- a/brancheneinstufung.py +++ b/brancheneinstufung.py @@ -18,7 +18,7 @@ except ImportError: # ==================== KONFIGURATION ==================== class Config: - VERSION = "v1.3.16" # v1.3.16: Modus 51 implementiert mit separaten Spalten für Wiki-Confirm, alternative Wiki URL, Branchenvorschlag etc. + VERSION = "v1.3.16" # v1.3.16: Neuer Modus 51 für gezielte Verifizierung (Branche & FSM) LANG = "de" CREDENTIALS_FILE = "service_account.json" SHEET_URL = "https://docs.google.com/spreadsheets/d/1u_gHr9JUfmV1-iviRzbSe3575QEp7KLhK5jFV_gJcgo" @@ -176,7 +176,7 @@ def validate_article_with_chatgpt(crm_data, wiki_data): wiki_headers = "Wikipedia URL;Wikipedia Absatz;Wikipedia Branche;Wikipedia Umsatz;Wikipedia Mitarbeiter;Wikipedia Kategorien" prompt_text = ( "Bitte überprüfe, ob die folgenden beiden Datensätze grundsätzlich zum gleichen Unternehmen gehören. " - "Berücksichtige leichte Abweichungen in Firmennamen und Ort. Wenn sie im Wesentlichen übereinstimmen, antworte mit 'OK'. " + "Berücksichtige dabei leichte Abweichungen in Firmennamen und Ort. Wenn sie im Wesentlichen übereinstimmen, antworte mit 'OK'. " "Andernfalls nenne den wichtigsten Grund und eine kurze Begründung.\n\n" f"CRM-Daten:\n{crm_headers}\n{crm_data}\n\n" f"Wikipedia-Daten:\n{wiki_headers}\n{wiki_data}\n\n" @@ -202,48 +202,18 @@ def validate_article_with_chatgpt(crm_data, wiki_data): debug_print(f"Fehler beim Validierungs-API-Aufruf: {e}") return "k.A." -def load_target_branches(): - try: - with open("ziel_Branchenschema.csv", "r", encoding="utf-8") as csvfile: - reader = csv.reader(csvfile) - branches = [row[0] for row in reader if row and row[0].strip() != ""] - return branches - except Exception as e: - debug_print(f"Fehler beim Laden des Ziel-Branchenschemas: {e}") - return [ - "Gutachter / Versicherungen > Baugutachter", - "Gutachter / Versicherungen > Technische Gutachten", - "Gutachter / Versicherungen > Versicherungsgutachten", - "Gutachter / Versicherungen > Medizinische Gutachten", - "Hersteller / Produzenten > Anlagenbau", - "Hersteller / Produzenten > Automaten (Vending, Slot)", - "Hersteller / Produzenten > Gebäudetechnik Allgemein", - "Hersteller / Produzenten > Gebäudetechnik Heizung, Lüftung, Klima", - "Hersteller / Produzenten > Maschinenbau", - "Hersteller / Produzenten > Medizintechnik", - "Service provider (Dienstleister) > Aufzüge und Rolltreppen", - "Service provider (Dienstleister) > Feuer- und Sicherheitssysteme", - "Service provider (Dienstleister) > Servicedienstleister / Reparatur ohne Produktion", - "Service provider (Dienstleister) > Facility Management", - "Versorger > Telekommunikation" - ] - def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kategorien): - target_branches = load_target_branches() - target_branches_str = "\n".join(target_branches) prompt_text = ( - "Du bist ein Experte im Field Service Management. Hier ist das gültige Ziel-Branchenschema:\n" - f"{target_branches_str}\n\n" - "Ordne anhand der folgenden Informationen das Unternehmen genau einer der oben genannten Branchen zu. " - "Wenn keine der Informationen passt, antworte mit 'k.A.'. Verwende dabei exakt die Schreibweise aus dem Ziel-Branchenschema.\n\n" + "Du bist ein Experte im Field Service Management. Analysiere die folgenden Branchenangaben und ordne das Unternehmen " + "einer der gültigen Branchen zu. Nutze ausschließlich die vorhandenen Informationen.\n\n" f"CRM-Branche: {crm_branche}\n" f"Beschreibung Branche extern: {beschreibung}\n" f"Wikipedia-Branche: {wiki_branche}\n" f"Wikipedia-Kategorien: {wiki_kategorien}\n\n" - "Gib aus im exakten Format (ohne zusätzliche Erklärungen):\n" + "Ordne das Unternehmen exakt einer der gültigen Branchen zu und gib aus:\n" "Branche: \n" - "Konsistenz: \n" - "Begründung: " + "Übereinstimmung: \n" + "Begründung: " ) try: with open("api_key.txt", "r") as f: @@ -266,24 +236,7 @@ def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kateg for line in result.split("\n"): if line.lower().startswith("branche:"): branch = line.split(":", 1)[1].strip() - elif line.lower().startswith("konsistenz:"): - consistency = line.split(":", 1)[1].strip() - elif line.lower().startswith("begründung:"): - justification = line.split(":", 1)[1].strip() - if branch not in target_branches: - debug_print(f"Vorgeschlagene Branche '{branch}' nicht im Ziel-Branchenschema enthalten.") - branch = "k.A." - return {"branch": branch, "consistency": consistency, "justification": justification} - except Exception as e: - debug_print(f"Fehler beim Aufruf der ChatGPT API für Branchenabgleich: {e}") - return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."} - branch = "k.A." - consistency = "k.A." - justification = "" - for line in result.split("\n"): - if line.lower().startswith("branche:"): - branch = line.split(":", 1)[1].strip() - elif line.lower().startswith("konsistenz:"): + elif line.lower().startswith("übereinstimmung:"): consistency = line.split(":", 1)[1].strip() elif line.lower().startswith("begründung:"): justification = line.split(":", 1)[1].strip() @@ -293,16 +246,101 @@ def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kateg return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."} def evaluate_fsm_suitability(company_name, company_data): - # Vorläufig nicht genutzt – Rückgabe "n.v." - return {"suitability": "n.v.", "justification": ""} + try: + with open("api_key.txt", "r") as f: + api_key = f.read().strip() + except Exception as e: + debug_print(f"Fehler beim Lesen des API-Tokens (FSM): {e}") + return {"suitability": "k.A.", "justification": "k.A."} + openai.api_key = api_key + prompt = ( + f"Bitte bewerte, ob das Unternehmen '{company_name}' für den Einsatz einer Field Service Management Lösung geeignet ist. " + "Antworte ausschließlich mit 'Ja' oder 'Nein' und gib eine kurze Begründung." + ) + try: + response = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[{"role": "system", "content": prompt}], + temperature=0.0 + ) + result = response.choices[0].message.content.strip() + debug_print(f"FSM-Eignungsantwort ChatGPT: '{result}'") + suitability = "k.A." + justification = "" + lines = result.split("\n") + if len(lines) == 1: + parts = result.split(" ", 1) + suitability = parts[0].strip() + justification = parts[1].strip() if len(parts) > 1 else "" + else: + for line in lines: + if line.lower().startswith("eignung:"): + suitability = line.split(":", 1)[1].strip() + elif line.lower().startswith("begründung:"): + justification = line.split(":", 1)[1].strip() + if suitability not in ["Ja", "Nein"]: + parts = result.split(" ", 1) + suitability = parts[0].strip() + justification = " ".join(result.split()[1:]).strip() + return {"suitability": suitability, "justification": justification} + except Exception as e: + debug_print(f"Fehler beim Aufruf der ChatGPT API für FSM-Eignungsprüfung: {e}") + return {"suitability": "k.A.", "justification": "k.A."} def evaluate_servicetechnicians_estimate(company_name, company_data): - # Vorläufig nicht genutzt – Rückgabe "n.v." - return "n.v." + try: + with open("serpApiKey.txt", "r") as f: + serp_key = f.read().strip() + except Exception as e: + debug_print(f"Fehler beim Lesen des SerpAPI-Schlüssels (Servicetechniker): {e}") + return "k.A." + try: + with open("api_key.txt", "r") as f: + api_key = f.read().strip() + except Exception as e: + debug_print(f"Fehler beim Lesen des API-Tokens (Servicetechniker): {e}") + return "k.A." + openai.api_key = api_key + prompt = ( + f"Bitte schätze die Anzahl der Servicetechniker des Unternehmens '{company_name}' in einer der folgenden Kategorien: " + "'<50 Techniker', '>100 Techniker', '>200 Techniker', '>500 Techniker'." + ) + try: + response = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[{"role": "system", "content": prompt}], + temperature=0.0 + ) + result = response.choices[0].message.content.strip() + debug_print(f"Schätzung Servicetechniker ChatGPT: '{result}'") + return result + except Exception as e: + debug_print(f"Fehler beim Aufruf der ChatGPT API für Servicetechniker-Schätzung: {e}") + return "k.A." def evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data): - # Vorläufig nicht genutzt – Rückgabe "n.v." - return "n.v." + try: + with open("api_key.txt", "r") as f: + api_key = f.read().strip() + except Exception as e: + debug_print(f"Fehler beim Lesen des API-Tokens (ST-Erklärung): {e}") + return "k.A." + openai.api_key = api_key + prompt = ( + f"Bitte erkläre, warum du für das Unternehmen '{company_name}' die Anzahl der Servicetechniker als '{st_estimate}' geschätzt hast." + ) + try: + response = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[{"role": "system", "content": prompt}], + temperature=0.0 + ) + result = response.choices[0].message.content.strip() + debug_print(f"Servicetechniker-Erklärung ChatGPT: '{result}'") + return result + except Exception as e: + debug_print(f"Fehler beim Aufruf der ChatGPT API für Servicetechniker-Erklärung: {e}") + return "k.A." def map_internal_technicians(value): try: @@ -330,111 +368,60 @@ def wait_for_sheet_update(sheet, cell, expected_value, timeout=5): time.sleep(0.5) return False -# ==================== NEUE FUNKTION: LINKEDIN-KONTAKT-SUCHE (Einzelkontakt) ==================== -def search_linkedin_contact(company_name, website, position_query): - try: - with open("serpApiKey.txt", "r") as f: - serp_key = f.read().strip() - except Exception as e: - debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e)) - return None - # Falls vorhanden, könnte hier auch die Kurzform (Spalte C) verwendet werden - search_name = company_name - query = f'site:linkedin.com/in "{position_query}" "{search_name}"' - debug_print(f"Erstelle LinkedIn-Query: {query}") - params = { - "engine": "google", - "q": query, - "api_key": serp_key, - "hl": "de" - } - try: - response = requests.get("https://serpapi.com/search", params=params) - data = response.json() - debug_print(f"SerpAPI-Response für Query '{query}': {data.get('organic_results', [])[:1]}") - if "organic_results" in data and len(data["organic_results"]) > 0: - result = data["organic_results"][0] - title = result.get("title", "") - debug_print(f"LinkedIn-Suchergebnis-Titel: {title}") - if "–" in title: - parts = title.split("–") - elif "-" in title: - parts = title.split("-") - else: - parts = [title] - if len(parts) >= 2: - name_part = parts[0].strip() - pos = parts[1].split("|")[0].strip() - name_parts = name_part.split(" ", 1) - if len(name_parts) == 2: - firstname, lastname = name_parts - else: - firstname = name_part - lastname = "" - debug_print(f"Kontakt gefunden: {firstname} {lastname}, Position: {pos}") - return {"Firmenname": company_name, "Website": website, "Vorname": firstname, "Nachname": lastname, "Position": pos} - else: - debug_print(f"Kontakt gefunden, aber unvollständige Informationen: {title}") - return {"Firmenname": company_name, "Website": website, "Vorname": "", "Nachname": "", "Position": title} - else: - debug_print(f"Keine LinkedIn-Ergebnisse für Query: {query}") - return None - except Exception as e: - debug_print(f"Fehler bei der SerpAPI-Suche: {e}") - return None - -def count_linkedin_contacts(company_name, website, position_query): - try: - with open("serpApiKey.txt", "r") as f: - serp_key = f.read().strip() - except Exception as e: - debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e)) - return 0 - query = f'site:linkedin.com/in "{position_query}" "{company_name}"' - debug_print(f"Erstelle LinkedIn-Query (Count): {query}") - params = { - "engine": "google", - "q": query, - "api_key": serp_key, - "hl": "de" - } - try: - response = requests.get("https://serpapi.com/search", params=params) - data = response.json() - if "organic_results" in data: - count = len(data["organic_results"]) - debug_print(f"Anzahl Kontakte für Query '{query}': {count}") - return count - else: - debug_print(f"Keine Ergebnisse für Query: {query}") - return 0 - except Exception as e: - debug_print(f"Fehler bei der SerpAPI-Suche (Count): {e}") - return 0 - # ==================== VERIFIZIERUNGS-MODUS (Modus 51) ==================== def _process_verification_row(row_num, row_data): - """ - Aggregiert relevante Informationen für die Verifizierung: - - Firmenname (Spalte B) - - CRM-Beschreibung (Spalte G) - - Wikipedia-URL (Spalte M) - - Wikipedia-Absatz (Spalte N) - - Wikipedia-Kategorien (Spalte R) - """ company_name = row_data[1] if len(row_data) > 1 else "" - crm_description = row_data[6] if len(row_data) > 6 else "" - wiki_url = row_data[12] if len(row_data) > 12 else "k.A." - wiki_absatz = row_data[13] if len(row_data) > 13 else "k.A." - wiki_categories = row_data[17] if len(row_data) > 17 else "k.A." - entry_text = (f"Eintrag {row_num}:\n" - f"Firmenname: {company_name}\n" - f"CRM-Beschreibung: {crm_description}\n" - f"Wikipedia-URL: {wiki_url}\n" - f"Wikipedia-Absatz: {wiki_absatz}\n" - f"Wikipedia-Kategorien: {wiki_categories}\n" - "-----\n") - return entry_text + website = row_data[3] if len(row_data) > 3 else "" + current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."]: + wiki_url = row_data[11].strip() + try: + wiki_data = WikipediaScraper().extract_company_data(wiki_url) + except Exception as e: + debug_print(f"Fehler beim Laden des vorgeschlagenen Wikipedia-Artikels: {e}") + article = WikipediaScraper().search_company_article(company_name, website) + wiki_data = WikipediaScraper().extract_company_data(article.url) if article else { + 'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.', + 'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.', + 'full_infobox': 'k.A.' + } + else: + article = WikipediaScraper().search_company_article(company_name, website) + wiki_data = WikipediaScraper().extract_company_data(article.url) if article else { + 'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.', + 'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.', + 'full_infobox': 'k.A.' + } + wiki_values = [ + row_data[11] if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."] else "k.A.", + wiki_data.get('url', 'k.A.'), + wiki_data.get('first_paragraph', 'k.A.'), + wiki_data.get('branche', 'k.A.'), + wiki_data.get('umsatz', 'k.A.'), + wiki_data.get('mitarbeiter', 'k.A.'), + wiki_data.get('categories', 'k.A.') + ] + gh = GoogleSheetHandler() + gh.sheet.update(values=[wiki_values], range_name=f"L{row_num}:R{row_num}") + # Branchenbewertung: + crm_branch = row_data[6] if len(row_data) > 6 else "k.A." + ext_branch = row_data[7] if len(row_data) > 7 else "k.A." + wiki_branch = wiki_data.get('branche', 'k.A.') + wiki_cats = wiki_data.get('categories', 'k.A.') + branch_result = evaluate_branche_chatgpt(crm_branch, ext_branch, wiki_branch, wiki_cats) + gh.sheet.update(values=[[branch_result["branch"]]], range_name=f"W{row_num}") + gh.sheet.update(values=[[branch_result["consistency"]]], range_name=f"Y{row_num}") + # Validierung mit ChatGPT: + crm_data = ";".join(row_data[1:11]) + wiki_data_str = ";".join(row_data[11:18]) + valid_result = validate_article_with_chatgpt(crm_data, wiki_data_str) + gh.sheet.update(values=[[valid_result]], range_name=f"R{row_num}") + # Schreibe Timestamp, Version und Token Count: + gh.sheet.update(values=[[current_dt]], range_name=f"AO{row_num}") + gh.sheet.update(values=[[Config.VERSION]], range_name=f"AP{row_num}") + # Für Batch-Token-Zählung wird später Spalte AQ aktualisiert. + debug_print(f"Zeile {row_num} verifiziert: Antwort: {valid_result}") + time.sleep(Config.RETRY_DELAY) def process_verification_only(): debug_print("Starte Verifizierungsmodus (Modus 51) im Batch-Prozess...") @@ -443,28 +430,33 @@ def process_verification_only(): sh = gc.open_by_url(Config.SHEET_URL) main_sheet = sh.sheet1 data = main_sheet.get_all_values() - batch_size = Config.BATCH_SIZE batch_entries = [] row_indices = [] - # Prüfe Spalte AO (Index 40) für den Verifizierungstimestamp: nur leere Zeilen verarbeiten for i, row in enumerate(data[1:], start=2): - if len(row) <= 41 or row[40].strip() == "": - entry_text = _process_verification_row(i, row) + if len(row) <= 25 or row[24].strip() == "": + # Hier wird _process_verification_row genutzt + entry_text = (f"Eintrag {i}:\n" + f"Firmenname: {row[1] if len(row)>1 else 'k.A.'}\n" + f"CRM-Beschreibung: {row[7] if len(row)>7 else 'k.A.'}\n" + f"Wikipedia-URL: {row[11] if len(row)>11 else 'k.A.'}\n" + f"Wikipedia-Absatz: {row[12] if len(row)>12 else 'k.A.'}\n" + f"Wikipedia-Kategorien: {row[17] if len(row)>17 else 'k.A.'}\n" + "-----") batch_entries.append(entry_text) row_indices.append(i) - if len(batch_entries) == batch_size: + if len(batch_entries) == Config.BATCH_SIZE: break if not batch_entries: debug_print("Keine Einträge für die Verifizierung gefunden.") return aggregated_prompt = ("Du bist ein Experte in der Verifizierung von Wikipedia-Artikeln für Unternehmen. " - "Für jeden der folgenden Einträge prüfe, ob der vorhandene Wikipedia-Artikel (URL, Absatz, Kategorien) plausibel passt. " + "Für jeden der folgenden Einträge prüfe, ob der vorhandene Wikipedia-Artikel plausibel passt. " "Gib für jeden Eintrag das Ergebnis im Format aus:\n" "Eintrag : \n" - "Dabei gilt:\n" - "- Wenn der Artikel passt, antworte mit 'OK'.\n" - "- Wenn der Artikel unpassend ist, antworte mit 'Alternativer Wikipedia-Artikel vorgeschlagen: | X | '.\n" - "- Wenn kein Artikel gefunden wurde, antworte mit 'Kein Wikipedia-Eintrag vorhanden.'\n\n") + "Antwortoptionen:\n" + "- 'OK' wenn der Artikel passt\n" + "- 'Kein Wikipedia-Eintrag vorhanden.'\n" + "- 'Alternativer Wikipedia-Artikel vorgeschlagen: | X | '\n\n") aggregated_prompt += "\n".join(batch_entries) debug_print("Aggregierter Prompt für Verifizierungs-Batch erstellt.") token_count = "n.v." @@ -520,22 +512,21 @@ def process_verification_only(): main_sheet.update(values=[[wiki_confirm]], range_name=f"S{row_num}") main_sheet.update(values=[[alt_article]], range_name=f"U{row_num}") main_sheet.update(values=[[wiki_explanation]], range_name=f"V{row_num}") - crm_branch = data[row_num-1][6] if len(data[row_num-1]) > 6 else "k.A." - ext_branch = data[row_num-1][7] if len(data[row_num-1]) > 7 else "k.A." + crm_branch = data[row_num-1][7] if len(data[row_num-1]) > 7 else "k.A." + ext_branch = data[row_num-1][8] if len(data[row_num-1]) > 8 else "k.A." wiki_branch = data[row_num-1][14] if len(data[row_num-1]) > 14 else "k.A." wiki_cats = data[row_num-1][17] if len(data[row_num-1]) > 17 else "k.A." branch_result = evaluate_branche_chatgpt(crm_branch, ext_branch, wiki_branch, wiki_cats) main_sheet.update(values=[[branch_result["branch"]]], range_name=f"W{row_num}") main_sheet.update(values=[[branch_result["consistency"]]], range_name=f"Y{row_num}") main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}") - current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S") main_sheet.update(values=[[current_dt]], range_name=f"AO{row_num}") main_sheet.update(values=[[Config.VERSION]], range_name=f"AP{row_num}") debug_print(f"Zeile {row_num} verifiziert: Antwort: {answer}") time.sleep(Config.RETRY_DELAY) debug_print("Verifizierungs-Batch abgeschlossen.") -# ==================== NEUER MODUS: CONTACT RESEARCH (via SerpAPI) ==================== +# ==================== CONTACT RESEARCH (Modus 6) ==================== def process_contact_research(): debug_print("Starte Contact Research (Modus 6)...") gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( @@ -563,7 +554,7 @@ def process_contact_research(): time.sleep(Config.RETRY_DELAY * 1.5) debug_print("Contact Research abgeschlossen.") -# ==================== NEUER MODUS: CONTACTS (LinkedIn) ==================== +# ==================== CONTACTS (Modus 7) ==================== def process_contacts(): debug_print("Starte LinkedIn-Kontaktsuche (Modus 7)...") gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( @@ -582,98 +573,254 @@ def process_contacts(): new_rows = [] for idx, row in enumerate(data[1:], start=2): company_name = row[1] if len(row) > 1 else "" + search_name = row[2].strip() if len(row) > 2 and row[2].strip() not in ["", "k.A."] else company_name website = row[3] if len(row) > 3 else "" debug_print(f"Verarbeite Firma: '{company_name}' (Zeile {idx}), Website: '{website}'") if not company_name or not website: debug_print("Überspringe, da Firmenname oder Website fehlt.") continue for pos in positions: - debug_print(f"Suche nach Position: '{pos}' bei '{company_name}'") - contact = search_linkedin_contact(company_name, website, pos) + debug_print(f"Suche nach Position: '{pos}' bei '{search_name}'") + contact = search_linkedin_contact(search_name, website, pos) if contact: debug_print(f"Kontakt gefunden: {contact}") - new_rows.append([contact["Firmenname"], contact["Website"], "", contact["Vorname"], contact["Nachname"], contact["Position"], "", ""]) + new_rows.append([contact["Firmenname"], website, search_name, contact["Vorname"], contact["Nachname"], contact["Position"], "", ""]) else: - debug_print(f"Kein Kontakt für Position '{pos}' bei '{company_name}' gefunden.") + debug_print(f"Kein Kontakt für Position '{pos}' bei '{search_name}' gefunden.") if new_rows: last_row = len(contacts_sheet.get_all_values()) + 1 range_str = f"A{last_row}:H{last_row + len(new_rows) - 1}" - contacts_sheet.update(range_str, new_rows) + contacts_sheet.update(values=new_rows, range_name=range_str) debug_print(f"{len(new_rows)} Kontakte in 'Contacts' hinzugefügt.") else: debug_print("Keine Kontakte gefunden in der Haupttabelle.") +# ==================== BATCH-TOKEN-ZÄHLUNG (Modus 8) ==================== +def process_batch_token_count(batch_size=10): + import tiktoken + def count_tokens(text, model="gpt-3.5-turbo"): + encoding = tiktoken.encoding_for_model(model) + tokens = encoding.encode(text) + return len(tokens) + debug_print("Starte Batch-Token-Zählung (Modus 8)...") + gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( + Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"])) + sh = gc.open_by_url(Config.SHEET_URL) + main_sheet = sh.sheet1 + data = main_sheet.get_all_values() + for i in range(2, len(data)+1, batch_size): + batch_rows = data[i-1:i-1+batch_size] + aggregated_prompt = "" + for row in batch_rows: + info = [] + if len(row) > 1: + info.append(row[1]) # Firmenname + if len(row) > 2: + info.append(row[2]) # Kurzform + if len(row) > 3: + info.append(row[3]) # Website + if len(row) > 4: + info.append(row[4]) # Ort + if len(row) > 5: + info.append(row[5]) # Beschreibung + if len(row) > 6: + info.append(row[6]) # Aktuelle Branche + aggregated_prompt += "; ".join(info) + "\n" + token_count = count_tokens(aggregated_prompt) + debug_print(f"Batch beginnend in Zeile {i}: {token_count} Tokens") + for j in range(i, min(i+batch_size, len(data)+1)): + main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{j}") + time.sleep(Config.RETRY_DELAY) + debug_print("Batch-Token-Zählung abgeschlossen.") + +# ==================== ALIGNMENT DEMO FÜR HAUPTBLATT & CONTACTS ==================== +def alignment_demo_full(): + alignment_demo(GoogleSheetHandler().sheet) + gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( + Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"])) + sh = gc.open_by_url(Config.SHEET_URL) + try: + contacts_sheet = sh.worksheet("Contacts") + except gspread.exceptions.WorksheetNotFound: + contacts_sheet = sh.add_worksheet(title="Contacts", rows="1000", cols="10") + header = ["Firmenname", "Website", "Kurzform", "Vorname", "Nachname", "Position", "Anrede", "E-Mail"] + contacts_sheet.update(values=[header], range_name="A1:H1") + debug_print("Neues Blatt 'Contacts' erstellt und Header eingetragen.") + alignment_demo(contacts_sheet) + debug_print("Alignment-Demo für Hauptblatt und Contacts abgeschlossen.") + +# ==================== GOOGLE SHEET HANDLER (Hauptdaten) ==================== +class GoogleSheetHandler: + def __init__(self): + self.sheet = None + self.sheet_values = [] + self._connect() + def _connect(self): + scope = ["https://www.googleapis.com/auth/spreadsheets"] + creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope) + self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1 + self.sheet_values = self.sheet.get_all_values() + def get_start_index(self): + filled_n = [row[39] if len(row) > 39 else '' for row in self.sheet_values[1:]] + return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1) + +# ==================== DATA PROCESSOR (Regulärer Modus) ==================== +class DataProcessor: + def __init__(self): + self.sheet_handler = GoogleSheetHandler() + self.wiki_scraper = WikipediaScraper() + def process_rows(self, num_rows=None): + if MODE == "2": + print("Re-Evaluierungsmodus: Verarbeitung aller Zeilen mit 'x' in Spalte A.") + for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2): + if row[0].strip().lower() == "x": + self._process_single_row(i, row, force_all=True) + elif MODE == "3": + print("Alignment-Demo-Modus: Hauptblatt und Contacts aktualisieren.") + alignment_demo_full() + elif MODE == "4": + for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2): + if len(row) <= 39 or row[39].strip() == "": + self._process_single_row(i, row, process_wiki=True, process_chatgpt=False) + elif MODE == "5": + for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2): + if len(row) <= 40 or row[40].strip() == "": + self._process_single_row(i, row, process_wiki=False, process_chatgpt=True) + elif MODE == "51": + for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2): + # Hier: Nur Zeilen ohne Verifizierungstimestamp (Spalte Y, z.B.) werden verarbeitet + if len(row) <= 25 or row[24].strip() == "": + _process_verification_row(i, row) + elif MODE == "8": + process_batch_token_count() + else: + start_index = self.sheet_handler.get_start_index() + print(f"Starte bei Zeile {start_index+1}") + rows_processed = 0 + for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2): + if i < start_index: + continue + if num_rows is not None and rows_processed >= num_rows: + break + self._process_single_row(i, row) + rows_processed += 1 + def _process_single_row(self, row_num, row_data, force_all=False, process_wiki=True, process_chatgpt=True): + company_name = row_data[1] if len(row_data) > 1 else "" + website = row_data[3] if len(row_data) > 3 else "" + current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + # Wiki-Auswertung (Spalten L bis R, Timestamp AO) + if force_all or process_wiki: + if len(row_data) <= 39 or row_data[39].strip() == "": + if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."]: + wiki_url = row_data[11].strip() + try: + wiki_data = self.wiki_scraper.extract_company_data(wiki_url) + except Exception as e: + debug_print(f"Fehler beim Laden des vorgeschlagenen Wikipedia-Artikels: {e}") + article = self.wiki_scraper.search_company_article(company_name, website) + wiki_data = self.wiki_scraper.extract_company_data(article.url) if article else { + 'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.', + 'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.', + 'full_infobox': 'k.A.' + } + else: + article = self.wiki_scraper.search_company_article(company_name, website) + wiki_data = self.wiki_scraper.extract_company_data(article.url) if article else { + 'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.', + 'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.', + 'full_infobox': 'k.A.' + } + wiki_values = [ + row_data[11] if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."] else "k.A.", + wiki_data.get('url', 'k.A.'), + wiki_data.get('first_paragraph', 'k.A.'), + wiki_data.get('branche', 'k.A.'), + wiki_data.get('umsatz', 'k.A.'), + wiki_data.get('mitarbeiter', 'k.A.'), + wiki_data.get('categories', 'k.A.') + ] + self.sheet_handler.sheet.update(values=[wiki_values], range_name=f"L{row_num}:R{row_num}") + self.sheet_handler.sheet.update(values=[[current_dt]], range_name=f"AO{row_num}") + else: + debug_print(f"Zeile {row_num}: Wikipedia-Timestamp bereits gesetzt – überspringe Wiki-Auswertung.") + # ChatGPT-Auswertung (Branche & FSM, etc. – Spalten R, AG, Y, Z, AE, AF; Timestamp in AO, Version in AP) + if force_all or process_chatgpt: + if len(row_data) <= 40 or row_data[40].strip() == "": + crm_umsatz = row_data[9] if len(row_data) > 9 else "k.A." + abgleich_result = compare_umsatz_values(crm_umsatz, wiki_data.get('umsatz', 'k.A.') if 'wiki_data' in locals() else "k.A.") + self.sheet_handler.sheet.update(values=[[abgleich_result]], range_name=f"AG{row_num}") + crm_data = ";".join(row_data[1:11]) + wiki_data_str = ";".join(row_data[11:18]) + valid_result = validate_article_with_chatgpt(crm_data, wiki_data_str) + self.sheet_handler.sheet.update(values=[[valid_result]], range_name=f"R{row_num}") + fsm_result = evaluate_fsm_suitability(company_name, wiki_data if 'wiki_data' in locals() else {}) + self.sheet_handler.sheet.update(values=[[fsm_result["suitability"]]], range_name=f"Y{row_num}") + self.sheet_handler.sheet.update(values=[[fsm_result["justification"]]], range_name=f"Z{row_num}") + st_estimate = evaluate_servicetechnicians_estimate(company_name, wiki_data if 'wiki_data' in locals() else {}) + self.sheet_handler.sheet.update(values=[[st_estimate]], range_name=f"AE{row_num}") + internal_value = row_data[8] if len(row_data) > 8 else "k.A." + internal_category = map_internal_technicians(internal_value) if internal_value != "k.A." else "k.A." + if internal_category != "k.A." and st_estimate != internal_category: + explanation = evaluate_servicetechnicians_explanation(company_name, st_estimate, wiki_data if 'wiki_data' in locals() else {}) + discrepancy = explanation + else: + discrepancy = "ok" + self.sheet_handler.sheet.update(values=[[discrepancy]], range_name=f"AF{row_num}") + self.sheet_handler.sheet.update(values=[[current_dt]], range_name=f"AO{row_num}") + else: + debug_print(f"Zeile {row_num}: ChatGPT-Timestamp bereits gesetzt – überspringe ChatGPT-Auswertung.") + self.sheet_handler.sheet.update(values=[[Config.VERSION]], range_name=f"AP{row_num}") + debug_print(f"✅ Aktualisiert: URL: {(wiki_data.get('url', 'k.A.') if 'wiki_data' in locals() else 'k.A.')}, " + f"Branche: {(wiki_data.get('branche', 'k.A.') if 'wiki_data' in locals() else 'k.A.')}, " + f"Umsatz-Abgleich: {abgleich_result if 'abgleich_result' in locals() else 'k.A.'}, " + f"Validierung: {valid_result if 'valid_result' in locals() else 'k.A.'}, " + f"FSM: {fsm_result['suitability'] if 'fsm_result' in locals() else 'k.A.'}, " + f"Servicetechniker-Schätzung: {st_estimate if 'st_estimate' in locals() else 'k.A.'}") + time.sleep(Config.RETRY_DELAY) + # ==================== MAIN PROGRAMM ==================== if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() - parser.add_argument("--mode", type=str, help="Modus: 1,2,3,4,5,6,7,8 oder 51") + parser.add_argument("--mode", type=str, help="Modus: 1,2,3,4,5,6,7,51 oder 8") parser.add_argument("--num_rows", type=int, default=0, help="Anzahl der zu bearbeitenden Zeilen (nur für Modus 1)") args = parser.parse_args() if not args.mode: print("Modi:") print("1 = Regulärer Modus") print("2 = Re-Evaluierungsmodus (nur Zeilen mit 'x' in Spalte A)") - print("3 = Alignment-Demo (Header in Hauptblatt und Contacts)") + print("3 = Alignment-Demo (Hauptblatt & Contacts)") print("4 = Nur Wikipedia-Suche (Zeilen ohne Wikipedia-Timestamp)") print("5 = Nur ChatGPT-Bewertung (Zeilen ohne ChatGPT-Timestamp)") print("6 = Contact Research (via SerpAPI)") print("7 = Contacts (LinkedIn)") print("8 = Batch-Token-Zählung") - print("51 = Nur Verifizierung (Wikipedia + Brancheneinordnung)") + print("51 = Nur Verifizierung (gezielte Branchen- & FSM-Evaluierung)") args.mode = input("Wählen Sie den Modus: ").strip() MODE = args.mode if MODE == "1": - try: - num_rows = int(input("Wieviele Zeilen sollen überprüft werden? ")) - except Exception as e: - print("Ungültige Eingabe. Bitte eine Zahl eingeben.") - exit(1) + num_rows = args.num_rows if args.num_rows > 0 else int(input("Wieviele Zeilen sollen überprüft werden? ")) processor = DataProcessor() processor.process_rows(num_rows) elif MODE in ["2", "3"]: processor = DataProcessor() processor.process_rows() elif MODE == "4": - gh = GoogleSheetHandler() - start_index = gh.get_start_index(39) # Wiki-Timestamp in Spalte AN - debug_print(f"Wiki-Modus: Starte bei Zeile {start_index+1}") processor = DataProcessor() - processor.process_rows() + for i, row in enumerate(processor.sheet_handler.sheet_values[1:], start=2): + if len(row) <= 39 or row[39].strip() == "": + processor._process_single_row(i, row, process_wiki=True, process_chatgpt=False) elif MODE == "5": - gh = GoogleSheetHandler() - start_index = gh.get_start_index(40) # ChatGPT-Timestamp in Spalte AO - debug_print(f"ChatGPT-Modus: Starte bei Zeile {start_index+1}") processor = DataProcessor() - processor.process_rows() + for i, row in enumerate(processor.sheet_handler.sheet_values[1:], start=2): + if len(row) <= 40 or row[40].strip() == "": + processor._process_single_row(i, row, process_wiki=False, process_chatgpt=True) + elif MODE == "51": + process_verification_only() elif MODE == "6": process_contact_research() elif MODE == "7": process_contacts() elif MODE == "8": - gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( - Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"])) - sh = gc.open_by_url(Config.SHEET_URL) - main_sheet = sh.sheet1 - data = main_sheet.get_all_values() - batch_entries = [] - row_indices = [] - for i, row in enumerate(data[1:], start=2): - batch_entries.append(" ".join(row)) - row_indices.append(i) - if len(batch_entries) == Config.BATCH_SIZE: - break - aggregated_text = "\n".join(batch_entries) - token_count = "n.v." - if tiktoken: - try: - enc = tiktoken.encoding_for_model(Config.TOKEN_MODEL) - token_count = len(enc.encode(aggregated_text)) - except Exception as e: - debug_print(f"Fehler beim Token-Counting: {e}") - for row_num in row_indices: - main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}") - debug_print(f"Batch-Token-Zählung abgeschlossen. Token: {token_count}") - elif MODE == "51": - process_verification_only() + process_batch_token_count() print(f"\n✅ Auswertung abgeschlossen ({Config.VERSION})")