diff --git a/brancheneinstufung.py b/brancheneinstufung.py index a62e254a..a97f81bd 100644 --- a/brancheneinstufung.py +++ b/brancheneinstufung.py @@ -11,6 +11,8 @@ from datetime import datetime from difflib import SequenceMatcher import unicodedata import csv + +# Optional: tiktoken für Token-Zählung (Modus 8) try: import tiktoken except ImportError: @@ -18,7 +20,7 @@ except ImportError: # ==================== KONFIGURATION ==================== class Config: - VERSION = "v1.3.16" # v1.3.16: Neuer Modus 8 (Batch-Token-Zählung in Spalte AQ) & Modus 51 (nur Verifizierung) + VERSION = "v1.3.18" # v1.3.18: Neuer Modus 8 (Batch-Token-Zählung) & Modus 51 (nur Verifizierung) LANG = "de" CREDENTIALS_FILE = "service_account.json" SHEET_URL = "https://docs.google.com/spreadsheets/d/1u_gHr9JUfmV1-iviRzbSe3575QEp7KLhK5jFV_gJcgo" @@ -206,13 +208,17 @@ def validate_article_with_chatgpt(crm_data, wiki_data): return "k.A." def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kategorien): - target_branches = [] - try: - with open("ziel_Branchenschema.csv", "r", encoding="utf-8") as csvfile: - reader = csv.reader(csvfile) - target_branches = [row[0] for row in reader if row] - except Exception as e: - debug_print(f"Fehler beim Laden des Ziel-Branchenschemas: {e}") + # Lade das Ziel-Branchenschema aus der CSV + def load_target_branches(): + try: + with open("ziel_Branchenschema.csv", "r", encoding="utf-8") as csvfile: + reader = csv.reader(csvfile) + branches = [row[0] for row in reader if row] + return branches + except Exception as e: + debug_print(f"Fehler beim Laden des Ziel-Branchenschemas: {e}") + return [] + target_branches = load_target_branches() target_branches_str = "\n".join(target_branches) focus_branches = [ "Gutachter / Versicherungen > Baugutachter", @@ -232,6 +238,13 @@ def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kateg "Versorger > Telekommunikation" ] focus_branches_str = "\n".join(focus_branches) + try: + with open("api_key.txt", "r") as f: + api_key = f.read().strip() + except Exception as e: + debug_print(f"Fehler beim Lesen des API-Tokens (Branche): {e}") + return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."} + openai.api_key = api_key additional_instruction = "" if wiki_branche.strip() == "k.A.": additional_instruction = ( @@ -260,13 +273,6 @@ def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kateg "Übereinstimmung: \n" "Begründung: " ) - try: - with open("api_key.txt", "r") as f: - api_key = f.read().strip() - except Exception as e: - debug_print(f"Fehler beim Lesen des API-Tokens (Branche): {e}") - return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."} - openai.api_key = api_key try: response = openai.ChatCompletion.create( model="gpt-3.5-turbo", @@ -416,7 +422,7 @@ def wait_for_sheet_update(sheet, cell, expected_value, timeout=5): time.sleep(0.5) return False -# ==================== NEUE FUNKTION: LINKEDIN-KONTAKT-SUCHE MIT SERPAPI ==================== +# ==================== NEUE FUNKTION: LINKEDIN-KONTAKT-SUCHE (Einzelkontakt) ==================== def search_linkedin_contact(company_name, website, position_query): try: with open("serpApiKey.txt", "r") as f: @@ -425,7 +431,6 @@ def search_linkedin_contact(company_name, website, position_query): debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e)) return None query = f'site:linkedin.com/in "{position_query}" "{company_name}"' - debug_print(f"Erstelle LinkedIn-Query: {query}") params = { "engine": "google", "q": query, @@ -435,7 +440,6 @@ def search_linkedin_contact(company_name, website, position_query): try: response = requests.get("https://serpapi.com/search", params=params) data = response.json() - debug_print(f"SerpAPI-Response für Query '{query}': {data.get('organic_results', [])[:1]}") if "organic_results" in data and len(data["organic_results"]) > 0: result = data["organic_results"][0] title = result.get("title", "") @@ -471,7 +475,6 @@ def count_linkedin_contacts(company_name, website, position_query): debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e)) return 0 query = f'site:linkedin.com/in "{position_query}" "{company_name}"' - debug_print(f"Erstelle LinkedIn-Query (Count): {query}") params = { "engine": "google", "q": query, @@ -493,10 +496,11 @@ def count_linkedin_contacts(company_name, website, position_query): return 0 # ==================== VERIFIZIERUNGS-MODUS (Modus 51) ==================== -def _process_verification_row(row_num, row_data): +def _process_verification_row(self, row_num, row_data): + # Verarbeitung: Extrahiere relevante Daten für die Verifizierung company_name = row_data[1] if len(row_data) > 1 else "" website = row_data[3] if len(row_data) > 3 else "" - crm_description = row_data[7] if len(row_data) > 7 else "k.A." + crm_description = row_data[7] if len(row_data) > 7 else "" wiki_url = row_data[11] if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."] else "k.A." wiki_absatz = row_data[12] if len(row_data) > 12 else "k.A." wiki_categories = row_data[16] if len(row_data) > 16 else "k.A." @@ -521,7 +525,7 @@ def process_verification_only(): row_indices = [] for i, row in enumerate(data[1:], start=2): if len(row) <= 19 or row[18].strip() == "": - entry_text = _process_verification_row(i, row) + entry_text = _process_verification_row(None, i, row) batch_entries.append(entry_text) row_indices.append(i) if len(batch_entries) == batch_size: @@ -535,7 +539,7 @@ def process_verification_only(): "Eintrag : \n" "Dabei gilt:\n" "- Wenn der Artikel passt, antworte mit 'OK'.\n" - "- Wenn der Artikel nicht passt, antworte mit 'Alternativer Wikipedia-Artikel vorgeschlagen: | X | '.\n" + "- Wenn der Artikel unpassend ist, antworte mit 'Alternativer Wikipedia-Artikel vorgeschlagen: | X | '.\n" "- Wenn kein Artikel gefunden wurde, antworte mit 'Kein Wikipedia-Eintrag vorhanden.'\n\n") aggregated_prompt += "\n".join(batch_entries) debug_print("Aggregierter Prompt für Verifizierungs-Batch erstellt.") @@ -607,59 +611,20 @@ def process_verification_only(): time.sleep(Config.RETRY_DELAY) debug_print("Verifizierungs-Batch abgeschlossen.") -# ==================== NEUER MODUS 8: BATCH-PROZESSING MIT TOKEN-ZÄHLUNG ==================== -def process_batch_token_count(batch_size=10): - import tiktoken - def count_tokens(text, model="gpt-3.5-turbo"): - encoding = tiktoken.encoding_for_model(model) - tokens = encoding.encode(text) - return len(tokens) - debug_print("Starte Batch-Token-Zählung (Modus 8)...") - gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( - Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"])) - sh = gc.open_by_url(Config.SHEET_URL) - main_sheet = sh.sheet1 - data = main_sheet.get_all_values() - for i in range(2, len(data)+1, batch_size): - batch_rows = data[i-1:i-1+batch_size] - aggregated_prompt = "" - for row in batch_rows: - info = [] - if len(row) > 1: - info.append(row[1]) # Firmenname - if len(row) > 2: - info.append(row[2]) # Kurzform - if len(row) > 3: - info.append(row[3]) # Website - if len(row) > 4: - info.append(row[4]) # Ort - if len(row) > 5: - info.append(row[5]) # Beschreibung - if len(row) > 6: - info.append(row[6]) # Aktuelle Branche - aggregated_prompt += "; ".join(info) + "\n" - token_count = count_tokens(aggregated_prompt) - debug_print(f"Batch beginnend in Zeile {i}: {token_count} Tokens") - for j in range(i, min(i+batch_size, len(data)+1)): - main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{j}") - time.sleep(Config.RETRY_DELAY) - debug_print("Batch-Token-Zählung abgeschlossen.") - -# ==================== NEUER MODUS: ALIGNMENT DEMO (Hauptblatt und Contacts) ==================== -def alignment_demo_full(): - alignment_demo(GoogleSheetHandler().sheet) - gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( - Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"])) - sh = gc.open_by_url(Config.SHEET_URL) - try: - contacts_sheet = sh.worksheet("Contacts") - except gspread.exceptions.WorksheetNotFound: - contacts_sheet = sh.add_worksheet(title="Contacts", rows="1000", cols="10") - header = ["Firmenname", "Website", "Kurzform", "Vorname", "Nachname", "Position", "Anrede", "E-Mail"] - contacts_sheet.update("A1:H1", [header]) - debug_print("Neues Blatt 'Contacts' erstellt und Header eingetragen.") - alignment_demo(contacts_sheet) - debug_print("Alignment-Demo für Hauptblatt und Contacts abgeschlossen.") +# ==================== GOOGLE SHEET HANDLER ==================== +class GoogleSheetHandler: + def __init__(self): + self.sheet = None + self.sheet_values = [] + self._connect() + def _connect(self): + scope = ["https://www.googleapis.com/auth/spreadsheets"] + creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope) + self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1 + self.sheet_values = self.sheet.get_all_values() + def get_start_index(self): + filled_n = [row[13] if len(row) > 13 else '' for row in self.sheet_values[1:]] + return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1) # ==================== ALIGNMENT DEMO (Hauptblatt) ==================== def alignment_demo(sheet): @@ -878,24 +843,6 @@ class WikipediaScraper: continue return None -# ==================== GOOGLE SHEET HANDLER ==================== -class GoogleSheetHandler: - def __init__(self): - self.sheet = None - self.sheet_values = [] - self._connect() - def _connect(self): - scope = ["https://www.googleapis.com/auth/spreadsheets"] - creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope) - self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1 - self.sheet_values = self.sheet.get_all_values() - def get_start_index(self): - # Spalte AO entspricht dem Index 40 (wenn A=0) - filled_n = [row[40] if len(row) > 40 else '' for row in self.sheet_values[1:]] - # Da die Datenzeilen in der Tabelle ab Zeile 2 beginnen, - # starten wir die Aufzählung bei 2 - return next((i for i, v in enumerate(filled_n, start=2) if not str(v).strip()), len(filled_n) + 2) - # ==================== DATA PROCESSOR ==================== class DataProcessor: def __init__(self): @@ -908,8 +855,8 @@ class DataProcessor: if row[0].strip().lower() == "x": self._process_single_row(i, row) elif MODE == "3": - print("Alignment-Demo-Modus: Schreibe neue Spaltenüberschriften in Zeile 11200.") - alignment_demo(self.sheet_handler.sheet) + print("Alignment-Demo-Modus: Schreibe neue Spaltenüberschriften in Hauptblatt und Contacts.") + alignment_demo_full() elif MODE == "4": for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2): if len(row) <= 39 or row[39].strip() == "": @@ -935,15 +882,14 @@ class DataProcessor: break self._process_single_row(i, row) rows_processed += 1 + def _process_single_row(self, row_num, row_data, process_wiki=True, process_chatgpt=True): company_name = row_data[1] if len(row_data) > 1 else "" website = row_data[2] if len(row_data) > 2 else "" wiki_update_range = f"K{row_num}:Q{row_num}" - chatgpt_range = f"AF{row_num}" - abgleich_range = f"AG{row_num}" - valid_range = f"R{row_num}" - dt_range = f"AH{row_num}" - ver_range = f"AI{row_num}" + dt_wiki_range = f"AN{row_num}" + dt_chat_range = f"AO{row_num}" + ver_range = f"AP{row_num}" print(f"\n[{datetime.now().strftime('%H:%M:%S')}] Verarbeite Zeile {row_num}: {company_name}") current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S") if process_wiki: @@ -977,46 +923,74 @@ class DataProcessor: company_data.get('categories', 'k.A.') ] self.sheet_handler.sheet.update(values=[wiki_values], range_name=wiki_update_range) - wait_for_sheet_update(self.sheet_handler.sheet, f"K{row_num}", wiki_values[0]) - self.sheet_handler.sheet.update(values=[[current_dt]], range_name=dt_range) + self.sheet_handler.sheet.update(values=[[current_dt]], range_name=dt_wiki_range) else: debug_print(f"Zeile {row_num}: Wikipedia-Timestamp bereits gesetzt – überspringe Wiki-Auswertung.") if process_chatgpt: if len(row_data) <= 40 or row_data[40].strip() == "": crm_umsatz = row_data[8] if len(row_data) > 8 else "k.A." - abgleich_result = compare_umsatz_values(crm_umsatz, company_data.get('umsatz', 'k.A.') if 'company_data' in locals() else "k.A.") - self.sheet_handler.sheet.update(values=[[abgleich_result]], range_name=abgleich_range) + abgleich_result = compare_umsatz_values(crm_umsatz, company_data.get('umsatz', 'k.A.')) + self.sheet_handler.sheet.update(values=[[abgleich_result]], range_name=f"AG{row_num}") crm_data = ";".join(row_data[1:10]) - wiki_data_str = ";".join(row_data[11:17]) + wiki_data_str = ";".join(row_data[11:18]) valid_result = validate_article_with_chatgpt(crm_data, wiki_data_str) - self.sheet_handler.sheet.update(values=[[valid_result]], range_name=valid_range) - fsm_result = evaluate_fsm_suitability(company_name, company_data if 'company_data' in locals() else {}) + self.sheet_handler.sheet.update(values=[[valid_result]], range_name=f"R{row_num}") + fsm_result = evaluate_fsm_suitability(company_name, company_data) self.sheet_handler.sheet.update(values=[[fsm_result["suitability"]]], range_name=f"Y{row_num}") self.sheet_handler.sheet.update(values=[[fsm_result["justification"]]], range_name=f"Z{row_num}") - st_estimate = evaluate_servicetechnicians_estimate(company_name, company_data if 'company_data' in locals() else {}) + st_estimate = evaluate_servicetechnicians_estimate(company_name, company_data) self.sheet_handler.sheet.update(values=[[st_estimate]], range_name=f"AD{row_num}") internal_value = row_data[7] if len(row_data) > 7 else "k.A." internal_category = map_internal_technicians(internal_value) if internal_value != "k.A." else "k.A." if internal_category != "k.A." and st_estimate != internal_category: - explanation = evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data if 'company_data' in locals() else {}) + explanation = evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data) discrepancy = explanation else: discrepancy = "ok" - self.sheet_handler.sheet.update(values=[[discrepancy]], range_name=f"AE{row_num}") - self.sheet_handler.sheet.update(values=[[current_dt]], range_name=chatgpt_range) + self.sheet_handler.sheet.update(values=[[discrepancy]], range_name=f"AF{row_num}") + self.sheet_handler.sheet.update(values=[[current_dt]], range_name=dt_chat_range) else: debug_print(f"Zeile {row_num}: ChatGPT-Timestamp bereits gesetzt – überspringe ChatGPT-Auswertung.") self.sheet_handler.sheet.update(values=[[current_dt]], range_name=ver_range) self.sheet_handler.sheet.update(values=[[Config.VERSION]], range_name=ver_range) - debug_print(f"✅ Aktualisiert: URL: {(company_data.get('url', 'k.A.') if 'company_data' in locals() else 'k.A.')}, " - f"Branche: {(company_data.get('branche', 'k.A.') if 'company_data' in locals() else 'k.A.')}, " - f"Umsatz-Abgleich: {abgleich_result if 'abgleich_result' in locals() else 'k.A.'}, " - f"Validierung: {valid_result if 'valid_result' in locals() else 'k.A.'}, " - f"FSM: {fsm_result['suitability'] if 'fsm_result' in locals() else 'k.A.'}, " - f"Servicetechniker-Schätzung: {st_estimate if 'st_estimate' in locals() else 'k.A.'}") + debug_print(f"✅ Aktualisiert: URL: {company_data.get('url', 'k.A.')}, " + f"Branche: {company_data.get('branche', 'k.A.')}, Umsatz-Abgleich: {abgleich_result}, " + f"Validierung: {valid_result}, " + f"FSM: {fsm_result['suitability']}, Servicetechniker-Schätzung: {st_estimate}") time.sleep(Config.RETRY_DELAY) -# ==================== NEUER MODUS 6: CONTACT RESEARCH (via SerpAPI) ==================== +# ==================== GOOGLE SHEET HANDLER (für Hauptdaten) ==================== +class GoogleSheetHandler: + def __init__(self): + self.sheet = None + self.sheet_values = [] + self._connect() + def _connect(self): + scope = ["https://www.googleapis.com/auth/spreadsheets"] + creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope) + self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1 + self.sheet_values = self.sheet.get_all_values() + def get_start_index(self): + filled_n = [row[13] if len(row) > 13 else '' for row in self.sheet_values[1:]] + return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1) + +# ==================== ALIGNMENT DEMO (Hauptblatt und Contacts) ==================== +def alignment_demo_full(): + alignment_demo(GoogleSheetHandler().sheet) + gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( + Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"])) + sh = gc.open_by_url(Config.SHEET_URL) + try: + contacts_sheet = sh.worksheet("Contacts") + except gspread.exceptions.WorksheetNotFound: + contacts_sheet = sh.add_worksheet(title="Contacts", rows="1000", cols="10") + header = ["Firmenname", "Website", "Kurzform", "Vorname", "Nachname", "Position", "Anrede", "E-Mail"] + contacts_sheet.update(values=[header], range_name="A1:H1") + debug_print("Neues Blatt 'Contacts' erstellt und Header eingetragen.") + alignment_demo(contacts_sheet) + debug_print("Alignment-Demo für Hauptblatt und Contacts abgeschlossen.") + +# ==================== NEUER MODUS: CONTACT RESEARCH (via SerpAPI) ==================== def process_contact_research(): debug_print("Starte Contact Research (Modus 6)...") gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( @@ -1055,7 +1029,7 @@ def process_contacts(): except gspread.exceptions.WorksheetNotFound: contacts_sheet = sh.add_worksheet(title="Contacts", rows="1000", cols="10") header = ["Firmenname", "Website", "Kurzform", "Vorname", "Nachname", "Position", "Anrede", "E-Mail"] - contacts_sheet.update("A1:G1", [header]) + contacts_sheet.update(values=[header], range_name="A1:H1") debug_print("Neues Blatt 'Contacts' erstellt und Header eingetragen.") main_sheet = sh.sheet1 data = main_sheet.get_all_values() @@ -1069,20 +1043,57 @@ def process_contacts(): continue for pos in positions: debug_print(f"Suche nach Position: '{pos}' bei '{search_name}'") - contact = search_linkedin_contact(company_name, website, pos) + contact = search_linkedin_contact(search_name, website, pos) if contact: - debug_print(f"Kontakt gefunden: {contact}") - new_rows.append([contact["Firmenname"], contact["Website"], search_name, contact["Vorname"], contact["Nachname"], contact["Position"], "", ""]) + new_rows.append([contact["Firmenname"], website, search_name, contact["Vorname"], contact["Nachname"], contact["Position"], "", ""]) else: debug_print(f"Kein Kontakt für Position '{pos}' bei '{search_name}' gefunden.") if new_rows: last_row = len(contacts_sheet.get_all_values()) + 1 - range_str = f"A{last_row}:G{last_row + len(new_rows) - 1}" - contacts_sheet.update(range_str, new_rows) + range_str = f"A{last_row}:H{last_row + len(new_rows) - 1}" + contacts_sheet.update(values=new_rows, range_name=range_str) debug_print(f"{len(new_rows)} Kontakte in 'Contacts' hinzugefügt.") else: debug_print("Keine Kontakte gefunden.") +# ==================== NEUER MODUS: BATCH-PROZESSING MIT TOKEN-ZÄHLUNG (Modus 8) ==================== +def process_batch_token_count(batch_size=10): + import tiktoken + def count_tokens(text, model="gpt-3.5-turbo"): + encoding = tiktoken.encoding_for_model(model) + tokens = encoding.encode(text) + return len(tokens) + debug_print("Starte Batch-Token-Zählung (Modus 8)...") + gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name( + Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"])) + sh = gc.open_by_url(Config.SHEET_URL) + main_sheet = sh.sheet1 + data = main_sheet.get_all_values() + for i in range(2, len(data)+1, batch_size): + batch_rows = data[i-1:i-1+batch_size] + aggregated_prompt = "" + for row in batch_rows: + info = [] + if len(row) > 1: + info.append(row[1]) # Firmenname + if len(row) > 2: + info.append(row[2]) # Kurzform + if len(row) > 3: + info.append(row[3]) # Website + if len(row) > 4: + info.append(row[4]) # Ort + if len(row) > 5: + info.append(row[5]) # Beschreibung + if len(row) > 6: + info.append(row[6]) # Aktuelle Branche + aggregated_prompt += "; ".join(info) + "\n" + token_count = count_tokens(aggregated_prompt) + debug_print(f"Batch beginnend in Zeile {i}: {token_count} Tokens") + for j in range(i, min(i+batch_size, len(data)+1)): + main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{j}") + time.sleep(Config.RETRY_DELAY) + debug_print("Batch-Token-Zählung abgeschlossen.") + # ==================== MAIN PROGRAMM ==================== if __name__ == "__main__": import argparse @@ -1090,9 +1101,10 @@ if __name__ == "__main__": parser.add_argument("--mode", type=str, help="Modus: 1,2,3,4,5,6,7,51 oder 8") parser.add_argument("--num_rows", type=int, default=0, help="Anzahl der zu bearbeitenden Zeilen (nur für Modus 1)") args = parser.parse_args() + if not args.mode: print("Modi:") - print("1 = regulärer Modus") + print("1 = Regulärer Modus") print("2 = Re-Evaluierungsmodus (nur Zeilen mit 'x' in Spalte A)") print("3 = Alignment-Demo (Header in Hauptblatt und Contacts)") print("4 = Nur Wikipedia-Suche (Zeilen ohne Wikipedia-Timestamp)") @@ -1102,6 +1114,7 @@ if __name__ == "__main__": print("8 = Batch-Token-Zählung") print("51 = Nur Verifizierung (Wikipedia + Brancheneinordnung)") args.mode = input("Wählen Sie den Modus: ").strip() + MODE = args.mode if MODE == "1": try: