Deepseek Aktualisierung
This commit is contained in:
@@ -2,7 +2,6 @@ import os
|
|||||||
import time
|
import time
|
||||||
import csv
|
import csv
|
||||||
import re
|
import re
|
||||||
import pandas as pd
|
|
||||||
import gspread
|
import gspread
|
||||||
import openai
|
import openai
|
||||||
import wikipedia
|
import wikipedia
|
||||||
@@ -21,202 +20,169 @@ DURCHLÄUFE = int(input("Wieviele Zeilen sollen überprüft werden? "))
|
|||||||
MAX_RETRIES = 3
|
MAX_RETRIES = 3
|
||||||
RETRY_DELAY = 5
|
RETRY_DELAY = 5
|
||||||
|
|
||||||
# === OpenAI API-KEY LADEN ===
|
# === OpenAI INIT ===
|
||||||
with open("api_key.txt", "r") as f:
|
with open("api_key.txt", "r") as f:
|
||||||
openai.api_key = f.read().strip()
|
openai.api_key = f.read().strip()
|
||||||
|
|
||||||
# === GOOGLE SHEET VERBINDUNG ===
|
# === GOOGLE SHEETS ===
|
||||||
scope = ["https://www.googleapis.com/auth/spreadsheets"]
|
scope = ["https://www.googleapis.com/auth/spreadsheets"]
|
||||||
creds = ServiceAccountCredentials.from_json_keyfile_name(CREDENTIALS, scope)
|
creds = ServiceAccountCredentials.from_json_keyfile_name(CREDENTIALS, scope)
|
||||||
sheet = gspread.authorize(creds).open_by_url(SHEET_URL).sheet1
|
client = gspread.authorize(creds)
|
||||||
|
sheet = client.open_by_url(SHEET_URL).sheet1
|
||||||
sheet_values = sheet.get_all_values()
|
sheet_values = sheet.get_all_values()
|
||||||
|
|
||||||
# === STARTINDEX SUCHEN (Spalte N = Index 13) ===
|
# === WIKIPEDIA KONFIG ===
|
||||||
filled_n = [row[13] if len(row) > 13 else '' for row in sheet_values[1:]]
|
|
||||||
start = next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1)
|
|
||||||
print(f"Starte bei Zeile {start+1}")
|
|
||||||
|
|
||||||
wikipedia.set_lang(LANG)
|
wikipedia.set_lang(LANG)
|
||||||
|
WHITELIST_KATEGORIEN = [
|
||||||
|
"unternehmen", "hersteller", "produktion", "industrie",
|
||||||
|
"maschinenbau", "technik", "dienstleistung", "chemie",
|
||||||
|
"pharma", "elektro", "medizin", "bau", "energie",
|
||||||
|
"logistik", "automobil", "handel", "textil", "klima"
|
||||||
|
]
|
||||||
|
|
||||||
# === SYSTEM PROMPT ===
|
# === SYSTEM PROMPT ===
|
||||||
branches = [
|
branches = [...] # Branchenliste hier einfügen
|
||||||
"Hersteller / Produzenten > Maschinenbau",
|
|
||||||
# ... (restliche Branchen wie zuvor)
|
|
||||||
"Gutachter / Versicherungen > Medizinische Gutachten"
|
|
||||||
]
|
|
||||||
|
|
||||||
system_prompt = {
|
system_prompt = {
|
||||||
"role": "system",
|
"role": "system",
|
||||||
"content": (
|
"content": (
|
||||||
"Du bist ein Experte für Brancheneinstufung und FSM-Potenzialbewertung. "
|
"Du bist ein Experte für Brancheneinstufung. Beantworte ausschließlich "
|
||||||
"Bitte beziehe dich ausschließlich auf das konkret genannte Unternehmen. Ähnlich klingende Firmennamen dürfen nicht verwendet werden.\n"
|
"basierend auf den gegebenen Unternehmensdaten. Format: "
|
||||||
"FSM steht für Field Service Management – Software zur Planung und Unterstützung mobiler Techniker.\n"
|
"Wikipedia-Branche;LinkedIn-Branche;Umsatz (Mio €);Empfohlene Neueinstufung;"
|
||||||
"Ziel ist es, Unternehmen mit >50 Technikern im Außeneinsatz zu identifizieren.\n\n"
|
"Begründung;FSM-Relevanz;Techniker-Einschätzung;Techniker-Begründung"
|
||||||
"Struktur: Firmenname; Website; Ort; Aktuelle Einstufung; Beschreibung der Branche Extern\n\n"
|
|
||||||
"Gib deine Antwort im CSV-Format (1 Zeile, 8 Spalten, durch Semikolon getrennt):\n"
|
|
||||||
"Wikipedia-Branche;LinkedIn-Branche;Umsatz (Mio €);Empfohlene Neueinstufung;Begründung;FSM-Relevanz (Ja/Nein/k.A. mit Begründung);Techniker-Einschätzung;Techniker-Begründung\n\n"
|
|
||||||
"Falls ein Wikipedia-Link angegeben ist, vertraue ausschließlich den Angaben aus diesem Artikel für Branche und Umsatz.\n"
|
|
||||||
"Falls kein Wikipedia-Link existiert, gib für 'Wikipedia-Branche' und 'Umsatz (Mio €)' bitte 'k.A.' aus.\n\n"
|
|
||||||
"Ziel-Branchenschema:\n" + "\n".join(branches)
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
WHITELIST_KATEGORIEN = ["unternehmen", "hersteller", "produktion", "industrie",
|
# === HELFERFUNKTIONEN ===
|
||||||
"maschinenbau", "technik", "dienstleistungsunternehmen",
|
def extract_domain(url):
|
||||||
"chemie", "pharma", "elektrotechnik", "medizintechnik"]
|
"""Extrahiert den Domain-Schlüssel aus der URL"""
|
||||||
|
if not url.startswith("http"):
|
||||||
|
url = f"https://{url}"
|
||||||
|
return url.split("//")[-1].split("/")[0].split(".")[0]
|
||||||
|
|
||||||
def extract_domain_key(url):
|
def validate_wikipedia_content(content, name, domain):
|
||||||
"""Extrahiert den Domain-Schlüssel aus der Website-URL"""
|
"""Prüft ob der Artikel zum Unternehmen gehört"""
|
||||||
if not url:
|
name_fragments = name.lower().split()[:2]
|
||||||
return ""
|
return (
|
||||||
clean_url = url.replace("https://", "").replace("http://", "").split("/")[0]
|
any(frag in content.lower() for frag in name_fragments) or
|
||||||
parts = clean_url.split(".")
|
(domain and domain.lower() in content.lower())
|
||||||
return parts[0] if len(parts) > 1 else ""
|
|
||||||
|
|
||||||
def parse_infobox(soup):
|
def parse_infobox(soup):
|
||||||
"""Extrahiert Branche und Umsatz aus der Wikipedia-Infobox"""
|
"""Extrahiert Branche und Umsatz aus der Infobox"""
|
||||||
infobox = soup.find("table", class_=["infobox", "infobox vcard"])
|
|
||||||
branche = umsatz = ""
|
branche = umsatz = ""
|
||||||
|
|
||||||
if infobox:
|
for row in soup.find_all("tr"):
|
||||||
for row in infobox.find_all("tr"):
|
th = row.find("th")
|
||||||
th = row.find("th")
|
td = row.find("td")
|
||||||
td = row.find("td")
|
if not th or not td:
|
||||||
if not th or not td:
|
continue
|
||||||
continue
|
|
||||||
|
|
||||||
# Branchenerkennung
|
header = th.get_text(strip=True).lower()
|
||||||
if "branche" in th.text.lower():
|
value = td.get_text(separator=" ", strip=True)
|
||||||
branche = td.get_text(separator=" ", strip=True)
|
|
||||||
|
|
||||||
# Umsatzerkennung
|
# Branche erkennen
|
||||||
if "umsatz" in th.text.lower():
|
if any(s in header for s in ["branche", "industrie", "tätigkeitsfeld"]):
|
||||||
umsatz_text = td.get_text(strip=True)
|
branche = value
|
||||||
if "Mio" in umsatz_text:
|
|
||||||
match = re.search(r"(\d+[\d.,]*)\s*Mio", umsatz_text)
|
# Umsatz erkennen
|
||||||
if match:
|
if "umsatz" in header:
|
||||||
umsatz = match.group(1).replace(",", ".")
|
if "mio" in value.lower():
|
||||||
|
match = re.search(r"(\d{1,3}(?:[.,]\d{3})*(?:[.,]\d+)?)", value)
|
||||||
|
if match:
|
||||||
|
umsatz = match.group(1).replace(",", ".")
|
||||||
|
|
||||||
return branche, umsatz
|
return branche, umsatz
|
||||||
|
|
||||||
def get_wikipedia_data(name, website_hint=""):
|
def get_wikipedia_data(name, website):
|
||||||
"""Sucht Wikipedia-Daten mit erweiterter Validierung"""
|
"""Holt validierte Wikipedia-Daten"""
|
||||||
domain_key = extract_domain_key(website_hint)
|
domain = extract_domain(website) if website else ""
|
||||||
search_terms = [name, domain_key] if domain_key else [name]
|
|
||||||
|
|
||||||
for term in search_terms:
|
for attempt in range(MAX_RETRIES):
|
||||||
if not term:
|
try:
|
||||||
continue
|
results = wikipedia.search(name, results=3)
|
||||||
|
for title in results:
|
||||||
for attempt in range(MAX_RETRIES):
|
try:
|
||||||
try:
|
page = wikipedia.page(title, auto_suggest=False)
|
||||||
results = wikipedia.search(term, results=3)
|
if not validate_wikipedia_content(page.content, name, domain):
|
||||||
for title in results:
|
|
||||||
try:
|
|
||||||
page = wikipedia.page(title, auto_suggest=False)
|
|
||||||
html = requests.get(page.url, timeout=10).text
|
|
||||||
|
|
||||||
# Validierung der Übereinstimmung
|
|
||||||
content_check = (
|
|
||||||
name.split()[0].lower() in page.content.lower() or
|
|
||||||
(domain_key and domain_key.lower() in html.lower())
|
|
||||||
)
|
|
||||||
|
|
||||||
if content_check:
|
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
|
||||||
branche, umsatz = parse_infobox(soup)
|
|
||||||
|
|
||||||
# Fallback auf Kategorien
|
|
||||||
if not branche:
|
|
||||||
for category in page.categories:
|
|
||||||
if any(kw in category.lower() for kw in WHITELIST_KATEGORIEN):
|
|
||||||
branche = category
|
|
||||||
break
|
|
||||||
|
|
||||||
return page.url, branche or "k.A.", umsatz or "k.A."
|
|
||||||
|
|
||||||
except (wikipedia.exceptions.PageError,
|
|
||||||
wikipedia.exceptions.DisambiguationError,
|
|
||||||
requests.exceptions.RequestException):
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
except Exception as e:
|
soup = BeautifulSoup(requests.get(page.url).text, "html.parser")
|
||||||
print(f"⚠️ Wikipedia-Fehler ({term}, Versuch {attempt+1}): {str(e)[:100]}")
|
branche, umsatz = parse_infobox(soup)
|
||||||
time.sleep(RETRY_DELAY)
|
|
||||||
|
# Fallback auf Kategorien
|
||||||
|
if not branche:
|
||||||
|
for cat in page.categories:
|
||||||
|
if any(kw in cat.lower() for kw in WHITELIST_KATEGORIEN):
|
||||||
|
branche = cat
|
||||||
|
break
|
||||||
|
|
||||||
|
return page.url, branche or "k.A.", umsatz or "k.A."
|
||||||
|
|
||||||
|
except wikipedia.exceptions.PageError:
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
print(f"⚠️ Wikipedia-Fehler ({name}): {str(e)[:100]}")
|
||||||
|
time.sleep(RETRY_DELAY)
|
||||||
|
|
||||||
return "", "k.A.", "k.A."
|
return "", "k.A.", "k.A."
|
||||||
|
|
||||||
def classify_company(row, wikipedia_url=""):
|
def query_gpt(row, wiki_url):
|
||||||
"""Verarbeitet die GPT-Klassifizierung mit Wikipedia-Integration"""
|
"""Verarbeitet die GPT-Abfrage mit verbessertem Error-Handling"""
|
||||||
user_prompt = {
|
user_content = f"{row[0]};{row[1]};{row[2]};{row[4]};{row[5]}\nWikipedia: {wiki_url}"
|
||||||
"role": "user",
|
|
||||||
"content": f"{row[0]};{row[1]};{row[2]};{row[4]};{row[5]}\nWikipedia-Link: {wikipedia_url}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# GPT-Abfrage mit Retry-Logik
|
|
||||||
for attempt in range(MAX_RETRIES):
|
for attempt in range(MAX_RETRIES):
|
||||||
try:
|
try:
|
||||||
response = openai.chat.completions.create(
|
response = openai.chat.completions.create(
|
||||||
model="gpt-3.5-turbo",
|
model="gpt-3.5-turbo",
|
||||||
messages=[system_prompt, user_prompt],
|
messages=[system_prompt, {"role": "user", "content": user_content}],
|
||||||
temperature=0,
|
temperature=0,
|
||||||
request_timeout=15
|
timeout=15
|
||||||
)
|
)
|
||||||
full_text = response.choices[0].message.content.strip()
|
return response.choices[0].message.content.strip()
|
||||||
break
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"⚠️ GPT-Fehler (Versuch {attempt+1}): {str(e)[:100]}")
|
print(f"⚠️ GPT-Fehler (Versuch {attempt+1}): {str(e)[:100]}")
|
||||||
time.sleep(RETRY_DELAY)
|
time.sleep(RETRY_DELAY)
|
||||||
else:
|
|
||||||
print("❌ GPT 3x fehlgeschlagen – setze auf Standardwerte")
|
|
||||||
full_text = "k.A.;k.A.;k.A.;k.A.;k.A.;k.A.;k.A.;k.A."
|
|
||||||
|
|
||||||
# Antwortverarbeitung
|
print("❌ GPT-Abfrage fehlgeschlagen")
|
||||||
csv_line = next((l for l in full_text.splitlines() if ";" in l and not l.startswith("Wikipedia-Branche")), "")
|
return "k.A.;k.A.;k.A.;k.A.;k.A.;k.A.;k.A.;k.A."
|
||||||
parts = [v.strip().strip('"') for v in csv_line.split(";")] if csv_line else ["k.A."] * 8
|
|
||||||
parts += ["k.A."] * (8 - len(parts)) # Padding für fehlende Werte
|
|
||||||
|
|
||||||
# Logging
|
# === HAUPTLOGIK ===
|
||||||
with open(LOG_CSV, "a", newline="", encoding="utf-8") as log:
|
start_index = next((i for i, row in enumerate(sheet_values[1:], start=1) if not row[13].strip()), 1)
|
||||||
writer = csv.writer(log, delimiter=";")
|
|
||||||
writer.writerow([datetime.now().strftime("%Y-%m-%d %H:%M:%S"), row[0], *parts, full_text])
|
|
||||||
|
|
||||||
return parts
|
for i in range(start_index, min(start_index + DURCHLÄUFE, len(sheet_values))):
|
||||||
|
|
||||||
# === HAUPTPROZESS ===
|
|
||||||
for i in range(start, min(start + DURCHLÄUFE, len(sheet_values))):
|
|
||||||
row = sheet_values[i]
|
row = sheet_values[i]
|
||||||
print(f"\n[{datetime.now().strftime('%H:%M:%S')}] Verarbeite Zeile {i+1}: {row[0]}")
|
print(f"\n[{datetime.now().strftime('%H:%M:%S')}] Verarbeite Zeile {i+1}: {row[0]}")
|
||||||
|
|
||||||
# Wikipedia-Datenabfrage
|
# Wikipedia-Daten holen
|
||||||
url, wiki_branche, umsatz = get_wikipedia_data(row[0], row[1])
|
wiki_url, wiki_branche, wiki_umsatz = get_wikipedia_data(row[0], row[1])
|
||||||
|
|
||||||
# GPT-Klassifizierung
|
# GPT-Abfrage
|
||||||
gpt_data = classify_company(row, url)
|
gpt_response = query_gpt(row, wiki_url)
|
||||||
(wiki_gpt, linkedin, umsatz_gpt,
|
gpt_data = [x.strip('"') for x in gpt_response.split(";")][:8]
|
||||||
new_cat, reason, fsm, techniker, techniker_reason) = gpt_data
|
|
||||||
|
|
||||||
# Priorisierung der Wikipedia-Daten
|
# Finale Werte
|
||||||
final_wiki = wiki_branche if url else "k.A."
|
final_branche = wiki_branche if wiki_url else "k.A."
|
||||||
final_umsatz = umsatz if url else "k.A."
|
final_umsatz = wiki_umsatz if wiki_url else "k.A."
|
||||||
|
|
||||||
# Daten für Google Sheet
|
# Google Sheet aktualisieren
|
||||||
values = [
|
update_values = [
|
||||||
final_wiki, # G: Wikipedia-Branche
|
final_branche, # G: Wikipedia-Branche
|
||||||
linkedin, # H: LinkedIn-Branche
|
gpt_data[1], # H: LinkedIn-Branche
|
||||||
final_umsatz, # I: Umsatz
|
final_umsatz, # I: Umsatz
|
||||||
new_cat, # J: Empfohlene Neueinstufung
|
gpt_data[3], # J: Neueinstufung
|
||||||
reason, # K: Begründung
|
gpt_data[4], # K: Begründung
|
||||||
fsm, # L: FSM-Relevanz
|
gpt_data[5], # L: FSM-Relevanz
|
||||||
url, # M: Wikipedia-URL
|
wiki_url, # M: Wikipedia-URL
|
||||||
datetime.now().strftime("%Y-%m-%d %H:%M:%S"), # N: Letzte Prüfung
|
datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
techniker, # O: Techniker-Einschätzung
|
gpt_data[6], # O: Techniker-Einschätzung
|
||||||
techniker_reason # P: Techniker-Begründung
|
gpt_data[7] # P: Techniker-Begründung
|
||||||
]
|
]
|
||||||
|
|
||||||
# Google Sheet Update
|
sheet.update(
|
||||||
sheet.update(f"G{i+1}:P{i+1}", [values])
|
range_name=f"G{i+1}:P{i+1}",
|
||||||
print(f"✅ Aktualisiert: {values[:3]}...")
|
values=[update_values]
|
||||||
|
)
|
||||||
|
print(f"✅ Aktualisiert: {update_values[:3]}...")
|
||||||
time.sleep(RETRY_DELAY)
|
time.sleep(RETRY_DELAY)
|
||||||
|
|
||||||
print("\n✅ Alle Durchläufe erfolgreich abgeschlossen")
|
print("\n✅ Prozess erfolgreich abgeschlossen")
|
||||||
Reference in New Issue
Block a user