v1.3.7: Wikipedia-only search mode added, ChatGPT calls disabled in mode 4
- Implemented new mode 4 that performs only Wikipedia searches without ChatGPT requests - process_wikipedia_only() updates columns K–Q, AH, and AI in the main sheet - Retained existing functions for modes 1–3 and improved debug output for tracing - Provided complete commit summary and description format with "- " as prefix
This commit is contained in:
@@ -14,7 +14,7 @@ import csv
|
||||
|
||||
# ==================== KONFIGURATION ====================
|
||||
class Config:
|
||||
VERSION = "v1.3.6" # v1.3.6: LinkedIn-Contacts über SerpApi integriert, Bugfixes zu Zeilenbegrenzung und Startindex.
|
||||
VERSION = "v1.3.7" # v1.3.7: Neuer Modus 4: Nur Wikipedia-Suche, keine ChatGPT-Anfragen.
|
||||
LANG = "de"
|
||||
CREDENTIALS_FILE = "service_account.json"
|
||||
SHEET_URL = "https://docs.google.com/spreadsheets/d/1u_gHr9JUfmV1-iviRzbSe3575QEp7KLhK5jFV_gJcgo"
|
||||
@@ -134,441 +134,6 @@ def compare_umsatz_values(crm, wiki):
|
||||
diff_mio = abs(crm_val - wiki_val)
|
||||
return f"Abweichung: {int(round(diff_mio))} Mio €"
|
||||
|
||||
def evaluate_umsatz_chatgpt(company_name, wiki_umsatz):
|
||||
try:
|
||||
with open("api_key.txt", "r") as f:
|
||||
api_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen des API-Tokens: {e}")
|
||||
return "k.A."
|
||||
openai.api_key = api_key
|
||||
prompt = (
|
||||
f"Bitte schätze den Umsatz in Mio. Euro für das Unternehmen '{company_name}'. "
|
||||
f"Die Wikipedia-Daten zeigen: '{wiki_umsatz}'. "
|
||||
"Antworte nur mit der Zahl."
|
||||
)
|
||||
try:
|
||||
response = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.0
|
||||
)
|
||||
result = response.choices[0].message.content.strip()
|
||||
debug_print(f"ChatGPT Umsatzschätzung: '{result}'")
|
||||
try:
|
||||
value = float(result.replace(',', '.'))
|
||||
return str(int(round(value)))
|
||||
except Exception as conv_e:
|
||||
debug_print(f"Fehler bei der Verarbeitung der Umsatzschätzung '{result}': {conv_e}")
|
||||
return result
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für Umsatzschätzung: {e}")
|
||||
return "k.A."
|
||||
|
||||
def validate_article_with_chatgpt(crm_data, wiki_data):
|
||||
crm_headers = "Firmenname;Website;Ort;Beschreibung;Aktuelle Branche;Beschreibung Branche extern;Anzahl Techniker;Umsatz (CRM);Anzahl Mitarbeiter (CRM)"
|
||||
wiki_headers = "Wikipedia URL;Wikipedia Absatz;Wikipedia Branche;Wikipedia Umsatz;Wikipedia Mitarbeiter;Wikipedia Kategorien"
|
||||
prompt_text = (
|
||||
"Bitte überprüfe, ob die folgenden beiden Datensätze grundsätzlich zum gleichen Unternehmen gehören. "
|
||||
"Berücksichtige dabei, dass leichte Abweichungen in Firmennamen (z. B. unterschiedliche Schreibweisen, Mutter-Tochter-Beziehungen) "
|
||||
"oder im Ort (z. B. 'Oberndorf' vs. 'Oberndorf/Neckar') tolerierbar sind. "
|
||||
"Vergleiche insbesondere den Firmennamen, den Ort und die Branche. Unterschiede im Umsatz können bis zu 10% abweichen. "
|
||||
"Wenn die Daten im Wesentlichen übereinstimmen, antworte ausschließlich mit 'OK'. "
|
||||
"Falls nicht, nenne bitte den wichtigsten Grund und eine kurze Begründung, warum die Abweichung plausibel sein könnte.\n\n"
|
||||
f"CRM-Daten:\n{crm_headers}\n{crm_data}\n\n"
|
||||
f"Wikipedia-Daten:\n{wiki_headers}\n{wiki_data}\n\n"
|
||||
"Antwort: "
|
||||
)
|
||||
try:
|
||||
with open("api_key.txt", "r") as f:
|
||||
api_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen des API-Tokens: {e}")
|
||||
return "k.A."
|
||||
openai.api_key = api_key
|
||||
try:
|
||||
response = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "system", "content": prompt_text}],
|
||||
temperature=0.0
|
||||
)
|
||||
result = response.choices[0].message.content.strip()
|
||||
debug_print(f"Validierungsantwort ChatGPT: '{result}'")
|
||||
return result
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Validierungs-API-Aufruf: {e}")
|
||||
return "k.A."
|
||||
|
||||
def evaluate_fsm_suitability(company_name, company_data):
|
||||
try:
|
||||
with open("api_key.txt", "r") as f:
|
||||
api_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen des API-Tokens (FSM): {e}")
|
||||
return {"suitability": "k.A.", "justification": "k.A."}
|
||||
openai.api_key = api_key
|
||||
prompt = (
|
||||
f"Bitte bewerte, ob das Unternehmen '{company_name}' für den Einsatz einer Field Service Management Lösung geeignet ist. "
|
||||
"Berücksichtige, dass ein Unternehmen mit einem technischen Außendienst, idealerweise mit über 50 Technikern und "
|
||||
"Disponenten, die mit der Planung mobiler Ressourcen beschäftigt sind, als geeignet gilt. Nutze dabei verifizierte "
|
||||
"Wikipedia-Daten und deine eigene Einschätzung. Antworte ausschließlich mit 'Ja' oder 'Nein' und gib eine kurze Begründung."
|
||||
)
|
||||
try:
|
||||
response = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "system", "content": prompt}],
|
||||
temperature=0.0
|
||||
)
|
||||
result = response.choices[0].message.content.strip()
|
||||
debug_print(f"FSM-Eignungsantwort ChatGPT: '{result}'")
|
||||
suitability = "k.A."
|
||||
justification = ""
|
||||
lines = result.split("\n")
|
||||
if len(lines) == 1:
|
||||
parts = result.split(" ", 1)
|
||||
suitability = parts[0].strip()
|
||||
justification = parts[1].strip() if len(parts) > 1 else ""
|
||||
else:
|
||||
for line in lines:
|
||||
if line.lower().startswith("eignung:"):
|
||||
suitability = line.split(":", 1)[1].strip()
|
||||
elif line.lower().startswith("begründung:"):
|
||||
justification = line.split(":", 1)[1].strip()
|
||||
if suitability not in ["Ja", "Nein"]:
|
||||
parts = result.split(" ", 1)
|
||||
suitability = parts[0].strip()
|
||||
justification = " ".join(result.split()[1:]).strip()
|
||||
return {"suitability": suitability, "justification": justification}
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für FSM-Eignungsprüfung: {e}")
|
||||
return {"suitability": "k.A.", "justification": "k.A."}
|
||||
|
||||
def evaluate_servicetechnicians_estimate(company_name, company_data):
|
||||
try:
|
||||
with open("serpApiKey.txt", "r") as f:
|
||||
serp_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen des SerpAPI-Schlüssels (Servicetechniker): {e}")
|
||||
return "k.A."
|
||||
# Wir nutzen SerpApi hier nicht, sondern die ChatGPT-Funktion
|
||||
try:
|
||||
with open("api_key.txt", "r") as f:
|
||||
api_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen des API-Tokens (Servicetechniker): {e}")
|
||||
return "k.A."
|
||||
openai.api_key = api_key
|
||||
prompt = (
|
||||
f"Bitte schätze auf Basis öffentlich zugänglicher Informationen (vor allem verifizierte Wikipedia-Daten) "
|
||||
f"die Anzahl der Servicetechniker des Unternehmens '{company_name}' ein. "
|
||||
"Gib die Antwort ausschließlich in einer der folgenden Kategorien aus: "
|
||||
"'<50 Techniker', '>100 Techniker', '>200 Techniker', '>500 Techniker'."
|
||||
)
|
||||
try:
|
||||
response = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "system", "content": prompt}],
|
||||
temperature=0.0
|
||||
)
|
||||
result = response.choices[0].message.content.strip()
|
||||
debug_print(f"Schätzung Servicetechniker ChatGPT: '{result}'")
|
||||
return result
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für Servicetechniker-Schätzung: {e}")
|
||||
return "k.A."
|
||||
|
||||
def evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data):
|
||||
try:
|
||||
with open("api_key.txt", "r") as f:
|
||||
api_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen des API-Tokens (ST-Erklärung): {e}")
|
||||
return "k.A."
|
||||
openai.api_key = api_key
|
||||
prompt = (
|
||||
f"Bitte erkläre, warum du für das Unternehmen '{company_name}' die Anzahl der Servicetechniker als '{st_estimate}' geschätzt hast. "
|
||||
"Berücksichtige dabei öffentlich zugängliche Informationen wie Branche, Umsatz, Mitarbeiterzahl und andere relevante Daten."
|
||||
)
|
||||
try:
|
||||
response = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "system", "content": prompt}],
|
||||
temperature=0.0
|
||||
)
|
||||
result = response.choices[0].message.content.strip()
|
||||
debug_print(f"Servicetechniker-Erklärung ChatGPT: '{result}'")
|
||||
return result
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für Servicetechniker-Erklärung: {e}")
|
||||
return "k.A."
|
||||
|
||||
def map_internal_technicians(value):
|
||||
try:
|
||||
num = int(value)
|
||||
except Exception:
|
||||
return "k.A."
|
||||
if num < 50:
|
||||
return "<50 Techniker"
|
||||
elif num < 100:
|
||||
return ">100 Techniker"
|
||||
elif num < 200:
|
||||
return ">200 Techniker"
|
||||
else:
|
||||
return ">500 Techniker"
|
||||
|
||||
def wait_for_sheet_update(sheet, cell, expected_value, timeout=5):
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
current_value = sheet.acell(cell).value
|
||||
if current_value == expected_value:
|
||||
return True
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen von Zelle {cell}: {e}")
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
# ==================== NEUE FUNKTION: LINKEDIN-KONTAKT-SUCHE MIT SERPAPI ====================
|
||||
def search_linkedin_contact(company_name, website, position_query):
|
||||
try:
|
||||
with open("serpApiKey.txt", "r") as f:
|
||||
serp_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e))
|
||||
return None
|
||||
query = f'site:linkedin.com/in "{position_query}" "{company_name}"'
|
||||
debug_print(f"Erstelle LinkedIn-Query: {query}")
|
||||
params = {
|
||||
"engine": "google",
|
||||
"q": query,
|
||||
"api_key": serp_key,
|
||||
"hl": "de"
|
||||
}
|
||||
try:
|
||||
response = requests.get("https://serpapi.com/search", params=params)
|
||||
data = response.json()
|
||||
debug_print(f"SerpAPI-Response für Query '{query}': {data.get('organic_results', [])[:1]}")
|
||||
if "organic_results" in data and len(data["organic_results"]) > 0:
|
||||
result = data["organic_results"][0]
|
||||
title = result.get("title", "")
|
||||
debug_print(f"LinkedIn-Suchergebnis-Titel: {title}")
|
||||
if "–" in title:
|
||||
parts = title.split("–")
|
||||
elif "-" in title:
|
||||
parts = title.split("-")
|
||||
else:
|
||||
parts = [title]
|
||||
if len(parts) >= 2:
|
||||
name_part = parts[0].strip()
|
||||
pos = parts[1].split("|")[0].strip()
|
||||
name_parts = name_part.split(" ", 1)
|
||||
if len(name_parts) == 2:
|
||||
firstname, lastname = name_parts
|
||||
else:
|
||||
firstname = name_part
|
||||
lastname = ""
|
||||
debug_print(f"Kontakt gefunden: {firstname} {lastname}, Position: {pos}")
|
||||
return {"Firmenname": company_name, "Website": website, "Vorname": firstname, "Nachname": lastname, "Position": pos}
|
||||
else:
|
||||
debug_print(f"Kontakt gefunden, aber unvollständige Informationen: {title}")
|
||||
return {"Firmenname": company_name, "Website": website, "Vorname": "", "Nachname": "", "Position": title}
|
||||
else:
|
||||
debug_print(f"Keine LinkedIn-Ergebnisse für Query: {query}")
|
||||
return None
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler bei der SerpAPI-Suche: {e}")
|
||||
return None
|
||||
|
||||
def process_contacts():
|
||||
debug_print("Starte LinkedIn-Kontaktsuche...")
|
||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
||||
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
||||
sh = gc.open_by_url(Config.SHEET_URL)
|
||||
try:
|
||||
contacts_sheet = sh.worksheet("Contacts")
|
||||
except gspread.exceptions.WorksheetNotFound:
|
||||
contacts_sheet = sh.add_worksheet(title="Contacts", rows="1000", cols="10")
|
||||
header = ["Firmenname", "Website", "Vorname", "Nachname", "Position", "Anrede", "E-Mail"]
|
||||
contacts_sheet.update("A1:G1", [header])
|
||||
debug_print("Neues Blatt 'Contacts' erstellt und Header eingetragen.")
|
||||
main_sheet = sh.sheet1
|
||||
data = main_sheet.get_all_values()
|
||||
positions = ["Serviceleiter", "IT-Leiter", "Leiter After Sales", "Leiter Einsatzplanung"]
|
||||
new_rows = []
|
||||
for idx, row in enumerate(data[1:], start=2):
|
||||
company_name = row[1] if len(row) > 1 else ""
|
||||
website = row[2] if len(row) > 2 else ""
|
||||
debug_print(f"Verarbeite Firma: '{company_name}' (Zeile {idx}), Website: '{website}'")
|
||||
if not company_name or not website:
|
||||
debug_print("Überspringe, da Firmenname oder Website fehlt.")
|
||||
continue
|
||||
for pos in positions:
|
||||
debug_print(f"Suche nach Position: '{pos}' bei '{company_name}'")
|
||||
contact = search_linkedin_contact(company_name, website, pos)
|
||||
if contact:
|
||||
debug_print(f"Kontakt gefunden: {contact}")
|
||||
new_rows.append([contact["Firmenname"], contact["Website"], contact["Vorname"], contact["Nachname"], contact["Position"], "", ""])
|
||||
else:
|
||||
debug_print(f"Kein Kontakt für Position '{pos}' bei '{company_name}' gefunden.")
|
||||
if new_rows:
|
||||
last_row = len(contacts_sheet.get_all_values()) + 1
|
||||
range_str = f"A{last_row}:G{last_row + len(new_rows) - 1}"
|
||||
contacts_sheet.update(range_str, new_rows)
|
||||
debug_print(f"{len(new_rows)} Kontakte in 'Contacts' hinzugefügt.")
|
||||
else:
|
||||
debug_print("Keine Kontakte gefunden in der Haupttabelle.")
|
||||
|
||||
# ==================== BRANCHENABGLEICH PER CHATGPT ====================
|
||||
def load_target_branches():
|
||||
try:
|
||||
with open("ziel_Branchenschema.csv", "r", encoding="utf-8") as csvfile:
|
||||
reader = csv.reader(csvfile)
|
||||
branches = [row[0] for row in reader if row]
|
||||
return branches
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Laden des Ziel-Branchenschemas: {e}")
|
||||
return []
|
||||
|
||||
focus_branches = [
|
||||
"Gutachter / Versicherungen > Baugutachter",
|
||||
"Gutachter / Versicherungen > Technische Gutachten",
|
||||
"Gutachter / Versicherungen > Versicherungsgutachten",
|
||||
"Gutachter / Versicherungen > Medizinische Gutachten",
|
||||
"Hersteller / Produzenten > Anlagenbau",
|
||||
"Hersteller / Produzenten > Automaten (Vending, Slot)",
|
||||
"Hersteller / Produzenten > Gebäudetechnik Allgemein",
|
||||
"Hersteller / Produzenten > Gebäudetechnik Heizung, Lüftung, Klima",
|
||||
"Hersteller / Produzenten > Maschinenbau",
|
||||
"Hersteller / Produzenten > Medizintechnik",
|
||||
"Service provider (Dienstleister) > Aufzüge und Rolltreppen",
|
||||
"Service provider (Dienstleister) > Feuer- und Sicherheitssysteme",
|
||||
"Service provider (Dienstleister) > Servicedienstleister / Reparatur ohne Produktion",
|
||||
"Service provider (Dienstleister) > Facility Management",
|
||||
"Versorger > Telekommunikation"
|
||||
]
|
||||
|
||||
def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kategorien):
|
||||
target_branches = load_target_branches()
|
||||
target_branches_str = "\n".join(target_branches)
|
||||
focus_branches_str = "\n".join(focus_branches)
|
||||
try:
|
||||
with open("api_key.txt", "r") as f:
|
||||
api_key = f.read().strip()
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Lesen des API-Tokens (Branche): {e}")
|
||||
return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."}
|
||||
openai.api_key = api_key
|
||||
|
||||
additional_instruction = ""
|
||||
if wiki_branche.strip() == "k.A.":
|
||||
additional_instruction = (
|
||||
"Da keine Wikipedia-Branche vorliegt, berücksichtige bitte die Wikipedia-Kategorien mit erhöhter Gewichtung, "
|
||||
"insbesondere wenn Hinweise auf Personentransport oder öffentliche Verkehrsdienstleistungen vorliegen. "
|
||||
)
|
||||
|
||||
system_prompt = (
|
||||
"Du bist ein Experte im Field Service Management. Deine Aufgabe ist es, ein Unternehmen basierend auf folgenden Angaben einer Branche zuzuordnen.\n\n"
|
||||
f"CRM-Branche (Spalte F): {crm_branche}\n"
|
||||
f"Branchenbeschreibung (Spalte G): {beschreibung}\n"
|
||||
f"Wikipedia-Branche (Spalte N): {wiki_branche}\n"
|
||||
f"Wikipedia-Kategorien (Spalte Q): {wiki_kategorien}\n\n"
|
||||
+ additional_instruction +
|
||||
"Das Ziel-Branchenschema umfasst ALLE gültigen Branchen, also sowohl Fokusbranchen als auch weitere, z. B. 'Housing > Sozialbau Unternehmen'.\n"
|
||||
"Das vollständige Ziel-Branchenschema lautet:\n"
|
||||
f"{target_branches_str}\n\n"
|
||||
"Falls das Unternehmen mehreren Branchen zugeordnet werden könnte, wähle bitte bevorzugt eine Branche aus der folgenden Fokusliste, sofern zutreffend:\n"
|
||||
f"{focus_branches_str}\n\n"
|
||||
"Gewichtung der Angaben:\n"
|
||||
"1. Wikipedia-Branche (Spalte N) zusammen mit Wikipedia-Kategorien (Spalte Q) (höchste Priorität, wenn verifiziert, ansonsten erhöhte Gewichtung der Kategorien)\n"
|
||||
"2. Branchenbeschreibung (Spalte G)\n"
|
||||
"3. CRM-Branche (Spalte F)\n\n"
|
||||
"Ordne das Unternehmen exakt einer der oben genannten Branchen zu (es dürfen keine zusätzlichen Branchen erfunden werden). "
|
||||
"Bitte antworte in folgendem Format (ohne zusätzliche Informationen):\n"
|
||||
"Branche: <vorgeschlagene Branche>\n"
|
||||
"Übereinstimmung: <ok oder X>\n"
|
||||
"Begründung: <kurze Begründung, falls abweichend, ansonsten leer>"
|
||||
)
|
||||
try:
|
||||
response = openai.ChatCompletion.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "system", "content": system_prompt}],
|
||||
temperature=0.0
|
||||
)
|
||||
result = response.choices[0].message.content.strip()
|
||||
debug_print(f"Branchenabgleich ChatGPT Antwort: '{result}'")
|
||||
branch = "k.A."
|
||||
consistency = "k.A."
|
||||
justification = ""
|
||||
for line in result.split("\n"):
|
||||
if line.lower().startswith("branche:"):
|
||||
branch = line.split(":", 1)[1].strip()
|
||||
elif line.lower().startswith("übereinstimmung:"):
|
||||
consistency = line.split(":", 1)[1].strip()
|
||||
elif line.lower().startswith("begründung:"):
|
||||
justification = line.split(":", 1)[1].strip()
|
||||
return {"branch": branch, "consistency": consistency, "justification": justification}
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für Branchenabgleich: {e}")
|
||||
return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."}
|
||||
|
||||
# ==================== GOOGLE SHEET HANDLER ====================
|
||||
class GoogleSheetHandler:
|
||||
def __init__(self):
|
||||
self.sheet = None
|
||||
self.sheet_values = []
|
||||
self._connect()
|
||||
def _connect(self):
|
||||
scope = ["https://www.googleapis.com/auth/spreadsheets"]
|
||||
creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope)
|
||||
self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1
|
||||
self.sheet_values = self.sheet.get_all_values()
|
||||
def get_start_index(self):
|
||||
filled_n = [row[13] if len(row) > 13 else '' for row in self.sheet_values[1:]]
|
||||
return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1)
|
||||
|
||||
# ==================== ALIGNMENT DEMO (Modus 3) ====================
|
||||
def alignment_demo(sheet):
|
||||
new_headers = [
|
||||
"Spalte A (ReEval Flag)",
|
||||
"Spalte B (Firmenname)",
|
||||
"Spalte C (Website)",
|
||||
"Spalte D (Ort)",
|
||||
"Spalte E (Beschreibung)",
|
||||
"Spalte F (Aktuelle Branche)",
|
||||
"Spalte G (Beschreibung Branche extern)",
|
||||
"Spalte H (Anzahl Techniker CRM)",
|
||||
"Spalte I (Umsatz CRM)",
|
||||
"Spalte J (Anzahl Mitarbeiter CRM)",
|
||||
"Spalte K (Vorschlag Wiki URL)",
|
||||
"Spalte L (Wikipedia URL)",
|
||||
"Spalte M (Wikipedia Absatz)",
|
||||
"Spalte N (Wikipedia Branche)",
|
||||
"Spalte O (Wikipedia Umsatz)",
|
||||
"Spalte P (Wikipedia Mitarbeiter)",
|
||||
"Spalte Q (Wikipedia Kategorien)",
|
||||
"Spalte R (Konsistenzprüfung)",
|
||||
"Spalte S (Begründung bei Inkonsistenz)",
|
||||
"Spalte T (Vorschlag Wiki Artikel ChatGPT)",
|
||||
"Spalte U (Begründung bei Abweichung)",
|
||||
"Spalte V (Vorschlag neue Branche)",
|
||||
"Spalte W (Konsistenzprüfung Branche)",
|
||||
"Spalte X (Begründung Abweichung Branche)",
|
||||
"Spalte Y (FSM Relevanz Ja / Nein)",
|
||||
"Spalte Z (Begründung für FSM Relevanz)",
|
||||
"Spalte AA (Schätzung Anzahl Mitarbeiter)",
|
||||
"Spalte AB (Konsistenzprüfung Mitarbeiterzahl)",
|
||||
"Spalte AC (Begründung für Abweichung Mitarbeiterzahl)",
|
||||
"Spalte AD (Einschätzung Anzahl Servicetechniker)",
|
||||
"Spalte AE (Begründung bei Abweichung Anzahl Servicetechniker)",
|
||||
"Spalte AF (Schätzung Umsatz ChatGPT)",
|
||||
"Spalte AG (Begründung für Abweichung Umsatz)",
|
||||
"Spalte AH (Timestamp letzte Prüfung)",
|
||||
"Spalte AI (Version)"
|
||||
]
|
||||
header_range = "A11200:AI11200"
|
||||
sheet.update(values=[new_headers], range_name=header_range)
|
||||
print("Alignment-Demo abgeschlossen: Neue Spaltenüberschriften in Zeile 11200 geschrieben.")
|
||||
|
||||
# ==================== WIKIPEDIA SCRAPER ====================
|
||||
class WikipediaScraper:
|
||||
def __init__(self):
|
||||
@@ -751,127 +316,7 @@ class WikipediaScraper:
|
||||
continue
|
||||
return None
|
||||
|
||||
# ==================== DATA PROCESSOR ====================
|
||||
class DataProcessor:
|
||||
def __init__(self):
|
||||
self.sheet_handler = GoogleSheetHandler()
|
||||
self.wiki_scraper = WikipediaScraper()
|
||||
def process_rows(self, num_rows=None):
|
||||
if MODE == "2":
|
||||
print("Re-Evaluierungsmodus: Verarbeitung aller Zeilen mit 'x' in Spalte A.")
|
||||
for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2):
|
||||
if row[0].strip().lower() == "x":
|
||||
self._process_single_row(i, row)
|
||||
elif MODE == "3":
|
||||
print("Alignment-Demo-Modus: Schreibe neue Spaltenüberschriften in Zeile 11200.")
|
||||
alignment_demo(self.sheet_handler.sheet)
|
||||
else:
|
||||
start_index = self.sheet_handler.get_start_index()
|
||||
print(f"Starte bei Zeile {start_index+1}")
|
||||
rows_processed = 0
|
||||
for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2):
|
||||
if i < start_index:
|
||||
continue
|
||||
if num_rows is not None and rows_processed >= num_rows:
|
||||
break
|
||||
self._process_single_row(i, row)
|
||||
rows_processed += 1
|
||||
def _process_single_row(self, row_num, row_data):
|
||||
company_name = row_data[1] if len(row_data) > 1 else ""
|
||||
website = row_data[2] if len(row_data) > 2 else ""
|
||||
wiki_update_range = f"K{row_num}:Q{row_num}"
|
||||
chatgpt_range = f"AF{row_num}"
|
||||
abgleich_range = f"AG{row_num}"
|
||||
valid_range = f"R{row_num}"
|
||||
dt_range = f"AH{row_num}"
|
||||
ver_range = f"AI{row_num}"
|
||||
print(f"\n[{datetime.now().strftime('%H:%M:%S')}] Verarbeite Zeile {row_num}: {company_name}")
|
||||
|
||||
if len(row_data) > 10 and row_data[10].strip() not in ["", "k.A."]:
|
||||
wiki_url = row_data[10].strip()
|
||||
try:
|
||||
company_data = self.wiki_scraper.extract_company_data(wiki_url)
|
||||
except Exception as e:
|
||||
debug_print(f"Fehler beim Laden des vorgeschlagenen Wikipedia-Artikels: {e}")
|
||||
article = self.wiki_scraper.search_company_article(company_name, website)
|
||||
company_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
||||
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
||||
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
||||
'full_infobox': 'k.A.'
|
||||
}
|
||||
else:
|
||||
article = self.wiki_scraper.search_company_article(company_name, website)
|
||||
company_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
||||
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
||||
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
||||
'full_infobox': 'k.A.'
|
||||
}
|
||||
|
||||
wiki_values = [
|
||||
row_data[10] if len(row_data) > 10 and row_data[10].strip() not in ["", "k.A."] else "k.A.",
|
||||
company_data.get('url', 'k.A.'),
|
||||
company_data.get('first_paragraph', 'k.A.'),
|
||||
company_data.get('branche', 'k.A.'),
|
||||
company_data.get('umsatz', 'k.A.'),
|
||||
company_data.get('mitarbeiter', 'k.A.'),
|
||||
company_data.get('categories', 'k.A.')
|
||||
]
|
||||
self.sheet_handler.sheet.update(values=[wiki_values], range_name=wiki_update_range)
|
||||
wait_for_sheet_update(self.sheet_handler.sheet, f"K{row_num}", wiki_values[0])
|
||||
time.sleep(3)
|
||||
|
||||
self.sheet_handler.sheet.update(values=[["XX"]], range_name=chatgpt_range)
|
||||
|
||||
crm_umsatz = row_data[8] if len(row_data) > 8 else "k.A."
|
||||
abgleich_result = compare_umsatz_values(crm_umsatz, company_data.get('umsatz', 'k.A.'))
|
||||
self.sheet_handler.sheet.update(values=[[abgleich_result]], range_name=abgleich_range)
|
||||
|
||||
crm_data = ";".join(row_data[1:10])
|
||||
wiki_data = ";".join(row_data[11:17])
|
||||
valid_result = validate_article_with_chatgpt(crm_data, wiki_data)
|
||||
self.sheet_handler.sheet.update(values=[[valid_result]], range_name=valid_range)
|
||||
|
||||
crm_branche = row_data[5] if len(row_data) > 5 else "k.A."
|
||||
beschreibung_branche = row_data[6] if len(row_data) > 6 else "k.A."
|
||||
wiki_branche = company_data.get('branche', 'k.A.')
|
||||
wiki_kategorien = company_data.get('categories', 'k.A.')
|
||||
branche_result = evaluate_branche_chatgpt(crm_branche, beschreibung_branche, wiki_branche, wiki_kategorien)
|
||||
branche_v_range = f"V{row_num}"
|
||||
branche_w_range = f"W{row_num}"
|
||||
branche_x_range = f"X{row_num}"
|
||||
self.sheet_handler.sheet.update(values=[[branche_result["branch"]]], range_name=branche_v_range)
|
||||
self.sheet_handler.sheet.update(values=[[branche_result["consistency"]]], range_name=branche_w_range)
|
||||
self.sheet_handler.sheet.update(values=[[branche_result["justification"]]], range_name=branche_x_range)
|
||||
|
||||
fsm_result = evaluate_fsm_suitability(company_name, company_data)
|
||||
self.sheet_handler.sheet.update(values=[[fsm_result["suitability"]]], range_name=f"Y{row_num}")
|
||||
self.sheet_handler.sheet.update(values=[[fsm_result["justification"]]], range_name=f"Z{row_num}")
|
||||
|
||||
st_estimate = evaluate_servicetechnicians_estimate(company_name, company_data)
|
||||
self.sheet_handler.sheet.update(values=[[st_estimate]], range_name=f"AD{row_num}")
|
||||
internal_value = row_data[7] if len(row_data) > 7 else "k.A."
|
||||
internal_category = map_internal_technicians(internal_value) if internal_value != "k.A." else "k.A."
|
||||
if internal_category != "k.A." and st_estimate != internal_category:
|
||||
explanation = evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data)
|
||||
discrepancy = explanation
|
||||
else:
|
||||
discrepancy = "ok"
|
||||
self.sheet_handler.sheet.update(values=[[discrepancy]], range_name=f"AE{row_num}")
|
||||
|
||||
self.sheet_handler.sheet.update(values=[["XX"]], range_name="AF" + str(row_num))
|
||||
self.sheet_handler.sheet.update(values=[["XX"]], range_name="AG" + str(row_num))
|
||||
|
||||
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
self.sheet_handler.sheet.update(values=[[current_dt]], range_name=dt_range)
|
||||
self.sheet_handler.sheet.update(values=[[Config.VERSION]], range_name=ver_range)
|
||||
|
||||
debug_print(f"✅ Aktualisiert: URL: {company_data.get('url', 'k.A.')}, "
|
||||
f"Branche: {company_data.get('branche', 'k.A.')}, Umsatz-Abgleich: {abgleich_result}, "
|
||||
f"Validierung: {valid_result}, Branchenvorschlag: {branche_result['branch']}, "
|
||||
f"FSM: {fsm_result['suitability']}, Servicetechniker-Schätzung: {st_estimate}")
|
||||
time.sleep(Config.RETRY_DELAY)
|
||||
|
||||
# ==================== GOOGLE SHEET HANDLER (für Hauptdaten) ====================
|
||||
# ==================== GOOGLE SHEET HANDLER (Hauptdaten) ====================
|
||||
class GoogleSheetHandler:
|
||||
def __init__(self):
|
||||
self.sheet = None
|
||||
@@ -886,40 +331,100 @@ class GoogleSheetHandler:
|
||||
filled_n = [row[13] if len(row) > 13 else '' for row in self.sheet_values[1:]]
|
||||
return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1)
|
||||
|
||||
# ==================== NEUER MODUS: CONTACTS ====================
|
||||
def process_contacts():
|
||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
||||
# ==================== ALIGNMENT DEMO (Modus 3) ====================
|
||||
def alignment_demo(sheet):
|
||||
new_headers = [
|
||||
"Spalte A (ReEval Flag)",
|
||||
"Spalte B (Firmenname)",
|
||||
"Spalte C (Website)",
|
||||
"Spalte D (Ort)",
|
||||
"Spalte E (Beschreibung)",
|
||||
"Spalte F (Aktuelle Branche)",
|
||||
"Spalte G (Beschreibung Branche extern)",
|
||||
"Spalte H (Anzahl Techniker CRM)",
|
||||
"Spalte I (Umsatz CRM)",
|
||||
"Spalte J (Anzahl Mitarbeiter CRM)",
|
||||
"Spalte K (Vorschlag Wiki URL)",
|
||||
"Spalte L (Wikipedia URL)",
|
||||
"Spalte M (Wikipedia Absatz)",
|
||||
"Spalte N (Wikipedia Branche)",
|
||||
"Spalte O (Wikipedia Umsatz)",
|
||||
"Spalte P (Wikipedia Mitarbeiter)",
|
||||
"Spalte Q (Wikipedia Kategorien)",
|
||||
"Spalte R (Konsistenzprüfung)",
|
||||
"Spalte S (Begründung bei Inkonsistenz)",
|
||||
"Spalte T (Vorschlag Wiki Artikel ChatGPT)",
|
||||
"Spalte U (Begründung bei Abweichung)",
|
||||
"Spalte V (Vorschlag neue Branche)",
|
||||
"Spalte W (Konsistenzprüfung Branche)",
|
||||
"Spalte X (Begründung Abweichung Branche)",
|
||||
"Spalte Y (FSM Relevanz Ja / Nein)",
|
||||
"Spalte Z (Begründung für FSM Relevanz)",
|
||||
"Spalte AA (Schätzung Anzahl Mitarbeiter)",
|
||||
"Spalte AB (Konsistenzprüfung Mitarbeiterzahl)",
|
||||
"Spalte AC (Begründung für Abweichung Mitarbeiterzahl)",
|
||||
"Spalte AD (Einschätzung Anzahl Servicetechniker)",
|
||||
"Spalte AE (Begründung bei Abweichung Anzahl Servicetechniker)",
|
||||
"Spalte AF (Schätzung Umsatz ChatGPT)",
|
||||
"Spalte AG (Begründung für Abweichung Umsatz)",
|
||||
"Spalte AH (Timestamp letzte Prüfung)",
|
||||
"Spalte AI (Version)"
|
||||
]
|
||||
header_range = "A11200:AI11200"
|
||||
sheet.update(values=[new_headers], range_name=header_range)
|
||||
print("Alignment-Demo abgeschlossen: Neue Spaltenüberschriften in Zeile 11200 geschrieben.")
|
||||
|
||||
# ==================== NEUER MODUS 4: NUR WIKIPEDIA-SUCHE ====================
|
||||
def process_wikipedia_only():
|
||||
debug_print("Starte ausschließlich Wikipedia-Suche (Modus 4)...")
|
||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
||||
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
||||
sh = gc.open_by_url(Config.SHEET_URL)
|
||||
try:
|
||||
contacts_sheet = sh.worksheet("Contacts")
|
||||
except gspread.exceptions.WorksheetNotFound:
|
||||
contacts_sheet = sh.add_worksheet(title="Contacts", rows="1000", cols="10")
|
||||
header = ["Firmenname", "Website", "Vorname", "Nachname", "Position", "Anrede", "E-Mail"]
|
||||
contacts_sheet.update("A1:G1", [header])
|
||||
main_sheet = sh.sheet1
|
||||
data = main_sheet.get_all_values()
|
||||
positions = ["Serviceleiter", "IT-Leiter", "Leiter After Sales", "Leiter Einsatzplanung"]
|
||||
new_rows = []
|
||||
for row in data[1:]:
|
||||
# Wir gehen von der ersten Zeile ohne Timestamp in Spalte AH aus:
|
||||
start_index = GoogleSheetHandler().get_start_index()
|
||||
debug_print(f"Starte bei Zeile {start_index+1}")
|
||||
for i, row in enumerate(data[1:], start=2):
|
||||
if i < start_index:
|
||||
continue
|
||||
company_name = row[1] if len(row) > 1 else ""
|
||||
website = row[2] if len(row) > 2 else ""
|
||||
if not company_name or not website:
|
||||
continue
|
||||
for pos in positions:
|
||||
contact = search_linkedin_contact(company_name, website, pos)
|
||||
if contact:
|
||||
new_rows.append([contact["Firmenname"], contact["Website"], contact["Vorname"], contact["Nachname"], contact["Position"], "", ""])
|
||||
if new_rows:
|
||||
last_row = len(contacts_sheet.get_all_values()) + 1
|
||||
range_str = f"A{last_row}:G{last_row + len(new_rows) - 1}"
|
||||
contacts_sheet.update(range_str, new_rows)
|
||||
debug_print(f"{len(new_rows)} Kontakte in 'Contacts' hinzugefügt.")
|
||||
else:
|
||||
debug_print("Keine Kontakte gefunden.")
|
||||
debug_print(f"Verarbeite Zeile {i}: {company_name}")
|
||||
article = WikipediaScraper().search_company_article(company_name, website)
|
||||
if article:
|
||||
company_data = WikipediaScraper().extract_company_data(article.url)
|
||||
else:
|
||||
company_data = {
|
||||
'url': 'k.A.',
|
||||
'first_paragraph': 'k.A.',
|
||||
'branche': 'k.A.',
|
||||
'umsatz': 'k.A.',
|
||||
'mitarbeiter': 'k.A.',
|
||||
'categories': 'k.A.',
|
||||
'full_infobox': 'k.A.'
|
||||
}
|
||||
wiki_values = [
|
||||
row[10] if len(row) > 10 and row[10].strip() not in ["", "k.A."] else "k.A.",
|
||||
company_data.get('url', 'k.A.'),
|
||||
company_data.get('first_paragraph', 'k.A.'),
|
||||
company_data.get('branche', 'k.A.'),
|
||||
company_data.get('umsatz', 'k.A.'),
|
||||
company_data.get('mitarbeiter', 'k.A.'),
|
||||
company_data.get('categories', 'k.A.')
|
||||
]
|
||||
wiki_range = f"K{i}:Q{i}"
|
||||
main_sheet.update(values=[wiki_values], range_name=wiki_range)
|
||||
debug_print(f"Zeile {i} mit Wikipedia-Daten aktualisiert.")
|
||||
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
main_sheet.update(values=[[current_dt]], range_name=f"AH{i}")
|
||||
main_sheet.update(values=[[Config.VERSION]], range_name=f"AI{i}")
|
||||
time.sleep(Config.RETRY_DELAY)
|
||||
debug_print("Wikipedia-Suche abgeschlossen.")
|
||||
|
||||
# ==================== MAIN PROGRAMM ====================
|
||||
if __name__ == "__main__":
|
||||
print("Modi: 1 = regulärer Modus, 2 = Re-Evaluierungsmodus, 3 = Alignment-Demo, 4 = LinkedIn Contacts")
|
||||
print("Modi: 1 = regulärer Modus, 2 = Re-Evaluierungsmodus, 3 = Alignment-Demo, 4 = Nur Wikipedia-Suche")
|
||||
mode_input = input("Wählen Sie den Modus: ").strip()
|
||||
if mode_input == "2":
|
||||
MODE = "2"
|
||||
@@ -941,5 +446,5 @@ if __name__ == "__main__":
|
||||
processor = DataProcessor()
|
||||
processor.process_rows()
|
||||
elif MODE == "4":
|
||||
process_contacts()
|
||||
print(f"\n✅ Wikipedia-Auswertung abgeschlossen ({Config.VERSION})")
|
||||
process_wikipedia_only()
|
||||
print(f"\n✅ Auswertung abgeschlossen ({Config.VERSION})")
|
||||
|
||||
Reference in New Issue
Block a user