v1.3.15 – Modus 51 Batch-Verifizierung, separate Startindizes, Token-Ausgabe in Spalte AQ
In Modus 51 werden nun jeweils 10 Einträge in einem Batch aggregiert und an ChatGPT gesendet. Die Antwort wird so geparst, dass in Spalte W der Branchenvorschlag, in Spalte X der Konsistenzstatus und in Spalte Y die Begründung bei Abweichung eingetragen wird. Zusätzlich wird die Token-Zahl des aggregierten Prompts in Spalte AQ geschrieben. Es wurden separate Startindex-Funktionen implementiert, um Wiki- und ChatGPT-Runs über unterschiedliche Spalten zu steuern.
This commit is contained in:
@@ -11,10 +11,14 @@ from datetime import datetime
|
|||||||
from difflib import SequenceMatcher
|
from difflib import SequenceMatcher
|
||||||
import unicodedata
|
import unicodedata
|
||||||
import csv
|
import csv
|
||||||
|
try:
|
||||||
|
import tiktoken
|
||||||
|
except ImportError:
|
||||||
|
tiktoken = None # Falls tiktoken nicht installiert ist
|
||||||
|
|
||||||
# ==================== KONFIGURATION ====================
|
# ==================== KONFIGURATION ====================
|
||||||
class Config:
|
class Config:
|
||||||
VERSION = "v1.3.13" # v1.3.13: Neuer Modus 8 (Batch-Token-Zählung in Spalte AQ) & Modus 51 (nur Verifizierung)
|
VERSION = "v1.3.15" # v1.3.15: Modus 51 für verifizierte Wikipedia-Artikel in Batches, Ausgabe in Spalten W, X, Y und Token-Zahl in AQ.
|
||||||
LANG = "de"
|
LANG = "de"
|
||||||
CREDENTIALS_FILE = "service_account.json"
|
CREDENTIALS_FILE = "service_account.json"
|
||||||
SHEET_URL = "https://docs.google.com/spreadsheets/d/1u_gHr9JUfmV1-iviRzbSe3575QEp7KLhK5jFV_gJcgo"
|
SHEET_URL = "https://docs.google.com/spreadsheets/d/1u_gHr9JUfmV1-iviRzbSe3575QEp7KLhK5jFV_gJcgo"
|
||||||
@@ -25,6 +29,8 @@ class Config:
|
|||||||
DEBUG = True
|
DEBUG = True
|
||||||
WIKIPEDIA_SEARCH_RESULTS = 5
|
WIKIPEDIA_SEARCH_RESULTS = 5
|
||||||
HTML_PARSER = "html.parser"
|
HTML_PARSER = "html.parser"
|
||||||
|
BATCH_SIZE = 10 # Batch-Größe für Verifizierungsmodus
|
||||||
|
TOKEN_MODEL = "gpt-3.5-turbo" # Für tiktoken
|
||||||
|
|
||||||
# ==================== RETRY-DECORATOR ====================
|
# ==================== RETRY-DECORATOR ====================
|
||||||
def retry_on_failure(func):
|
def retry_on_failure(func):
|
||||||
@@ -170,11 +176,8 @@ def validate_article_with_chatgpt(crm_data, wiki_data):
|
|||||||
wiki_headers = "Wikipedia URL;Wikipedia Absatz;Wikipedia Branche;Wikipedia Umsatz;Wikipedia Mitarbeiter;Wikipedia Kategorien"
|
wiki_headers = "Wikipedia URL;Wikipedia Absatz;Wikipedia Branche;Wikipedia Umsatz;Wikipedia Mitarbeiter;Wikipedia Kategorien"
|
||||||
prompt_text = (
|
prompt_text = (
|
||||||
"Bitte überprüfe, ob die folgenden beiden Datensätze grundsätzlich zum gleichen Unternehmen gehören. "
|
"Bitte überprüfe, ob die folgenden beiden Datensätze grundsätzlich zum gleichen Unternehmen gehören. "
|
||||||
"Berücksichtige dabei, dass leichte Abweichungen in Firmennamen (z. B. unterschiedliche Schreibweisen, Mutter-Tochter-Beziehungen) "
|
"Berücksichtige leichte Abweichungen in Firmennamen und Ort. Wenn sie im Wesentlichen übereinstimmen, antworte mit 'OK'. "
|
||||||
"oder im Ort (z. B. 'Oberndorf' vs. 'Oberndorf/Neckar') tolerierbar sind. "
|
"Andernfalls nenne den wichtigsten Grund und eine kurze Begründung.\n\n"
|
||||||
"Vergleiche insbesondere den Firmennamen, den Ort und die Branche. Unterschiede im Umsatz können bis zu 10% abweichen. "
|
|
||||||
"Wenn die Daten im Wesentlichen übereinstimmen, antworte ausschließlich mit 'OK'. "
|
|
||||||
"Falls nicht, nenne bitte den wichtigsten Grund und eine kurze Begründung, warum die Abweichung plausibel sein könnte.\n\n"
|
|
||||||
f"CRM-Daten:\n{crm_headers}\n{crm_data}\n\n"
|
f"CRM-Daten:\n{crm_headers}\n{crm_data}\n\n"
|
||||||
f"Wikipedia-Daten:\n{wiki_headers}\n{wiki_data}\n\n"
|
f"Wikipedia-Daten:\n{wiki_headers}\n{wiki_data}\n\n"
|
||||||
"Antwort: "
|
"Antwort: "
|
||||||
@@ -201,16 +204,16 @@ def validate_article_with_chatgpt(crm_data, wiki_data):
|
|||||||
|
|
||||||
def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kategorien):
|
def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kategorien):
|
||||||
prompt_text = (
|
prompt_text = (
|
||||||
"Du bist ein Experte im Field Service Management. Analysiere die folgenden Branchenangaben und ordne das Unternehmen "
|
"Du bist ein Experte im Field Service Management. Analysiere die folgenden Branchenangaben und ordne das Unternehmen einer der gültigen Branchen zu. "
|
||||||
"einer der gültigen Branchen zu. Nutze ausschließlich die vorhandenen Informationen.\n\n"
|
"Nutze ausschließlich die vorhandenen Informationen.\n\n"
|
||||||
f"CRM-Branche: {crm_branche}\n"
|
f"CRM-Branche: {crm_branche}\n"
|
||||||
f"Beschreibung Branche extern: {beschreibung}\n"
|
f"Beschreibung Branche extern: {beschreibung}\n"
|
||||||
f"Wikipedia-Branche: {wiki_branche}\n"
|
f"Wikipedia-Branche: {wiki_branche}\n"
|
||||||
f"Wikipedia-Kategorien: {wiki_kategorien}\n\n"
|
f"Wikipedia-Kategorien: {wiki_kategorien}\n\n"
|
||||||
"Ordne das Unternehmen exakt einer der gültigen Branchen zu und gib aus:\n"
|
"Gib aus:\n"
|
||||||
"Branche: <vorgeschlagene Branche>\n"
|
"Branche: <vorgeschlagene Branche>\n"
|
||||||
"Übereinstimmung: <ok oder X>\n"
|
"Übereinstimmung: <OK oder X>\n"
|
||||||
"Begründung: <kurze Begründung, falls abweichend, ansonsten leer>"
|
"Begründung: <Begründung bei Abweichung (leer, wenn OK)>"
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
with open("api_key.txt", "r") as f:
|
with open("api_key.txt", "r") as f:
|
||||||
@@ -243,101 +246,16 @@ def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kateg
|
|||||||
return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."}
|
return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."}
|
||||||
|
|
||||||
def evaluate_fsm_suitability(company_name, company_data):
|
def evaluate_fsm_suitability(company_name, company_data):
|
||||||
try:
|
# In Modus 51 wird diese Funktion nicht aufgerufen.
|
||||||
with open("api_key.txt", "r") as f:
|
return {"suitability": "n.v.", "justification": ""}
|
||||||
api_key = f.read().strip()
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Lesen des API-Tokens (FSM): {e}")
|
|
||||||
return {"suitability": "k.A.", "justification": "k.A."}
|
|
||||||
openai.api_key = api_key
|
|
||||||
prompt = (
|
|
||||||
f"Bitte bewerte, ob das Unternehmen '{company_name}' für den Einsatz einer Field Service Management Lösung geeignet ist. "
|
|
||||||
"Antworte ausschließlich mit 'Ja' oder 'Nein' und gib eine kurze Begründung."
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
response = openai.ChatCompletion.create(
|
|
||||||
model="gpt-3.5-turbo",
|
|
||||||
messages=[{"role": "system", "content": prompt}],
|
|
||||||
temperature=0.0
|
|
||||||
)
|
|
||||||
result = response.choices[0].message.content.strip()
|
|
||||||
debug_print(f"FSM-Eignungsantwort ChatGPT: '{result}'")
|
|
||||||
suitability = "k.A."
|
|
||||||
justification = ""
|
|
||||||
lines = result.split("\n")
|
|
||||||
if len(lines) == 1:
|
|
||||||
parts = result.split(" ", 1)
|
|
||||||
suitability = parts[0].strip()
|
|
||||||
justification = parts[1].strip() if len(parts) > 1 else ""
|
|
||||||
else:
|
|
||||||
for line in lines:
|
|
||||||
if line.lower().startswith("eignung:"):
|
|
||||||
suitability = line.split(":", 1)[1].strip()
|
|
||||||
elif line.lower().startswith("begründung:"):
|
|
||||||
justification = line.split(":", 1)[1].strip()
|
|
||||||
if suitability not in ["Ja", "Nein"]:
|
|
||||||
parts = result.split(" ", 1)
|
|
||||||
suitability = parts[0].strip()
|
|
||||||
justification = " ".join(result.split()[1:]).strip()
|
|
||||||
return {"suitability": suitability, "justification": justification}
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für FSM-Eignungsprüfung: {e}")
|
|
||||||
return {"suitability": "k.A.", "justification": "k.A."}
|
|
||||||
|
|
||||||
def evaluate_servicetechnicians_estimate(company_name, company_data):
|
def evaluate_servicetechnicians_estimate(company_name, company_data):
|
||||||
try:
|
# In Modus 51 wird diese Funktion nicht aufgerufen.
|
||||||
with open("serpApiKey.txt", "r") as f:
|
return "n.v."
|
||||||
serp_key = f.read().strip()
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Lesen des SerpAPI-Schlüssels (Servicetechniker): {e}")
|
|
||||||
return "k.A."
|
|
||||||
try:
|
|
||||||
with open("api_key.txt", "r") as f:
|
|
||||||
api_key = f.read().strip()
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Lesen des API-Tokens (Servicetechniker): {e}")
|
|
||||||
return "k.A."
|
|
||||||
openai.api_key = api_key
|
|
||||||
prompt = (
|
|
||||||
f"Bitte schätze die Anzahl der Servicetechniker des Unternehmens '{company_name}' in einer der folgenden Kategorien: "
|
|
||||||
"'<50 Techniker', '>100 Techniker', '>200 Techniker', '>500 Techniker'."
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
response = openai.ChatCompletion.create(
|
|
||||||
model="gpt-3.5-turbo",
|
|
||||||
messages=[{"role": "system", "content": prompt}],
|
|
||||||
temperature=0.0
|
|
||||||
)
|
|
||||||
result = response.choices[0].message.content.strip()
|
|
||||||
debug_print(f"Schätzung Servicetechniker ChatGPT: '{result}'")
|
|
||||||
return result
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für Servicetechniker-Schätzung: {e}")
|
|
||||||
return "k.A."
|
|
||||||
|
|
||||||
def evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data):
|
def evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data):
|
||||||
try:
|
# In Modus 51 wird diese Funktion nicht aufgerufen.
|
||||||
with open("api_key.txt", "r") as f:
|
return "n.v."
|
||||||
api_key = f.read().strip()
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Lesen des API-Tokens (ST-Erklärung): {e}")
|
|
||||||
return "k.A."
|
|
||||||
openai.api_key = api_key
|
|
||||||
prompt = (
|
|
||||||
f"Bitte erkläre, warum du für das Unternehmen '{company_name}' die Anzahl der Servicetechniker als '{st_estimate}' geschätzt hast."
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
response = openai.ChatCompletion.create(
|
|
||||||
model="gpt-3.5-turbo",
|
|
||||||
messages=[{"role": "system", "content": prompt}],
|
|
||||||
temperature=0.0
|
|
||||||
)
|
|
||||||
result = response.choices[0].message.content.strip()
|
|
||||||
debug_print(f"Servicetechniker-Erklärung ChatGPT: '{result}'")
|
|
||||||
return result
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Aufruf der ChatGPT API für Servicetechniker-Erklärung: {e}")
|
|
||||||
return "k.A."
|
|
||||||
|
|
||||||
def map_internal_technicians(value):
|
def map_internal_technicians(value):
|
||||||
try:
|
try:
|
||||||
@@ -373,7 +291,8 @@ def search_linkedin_contact(company_name, website, position_query):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e))
|
debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e))
|
||||||
return None
|
return None
|
||||||
search_name = company_name # Hier kannst du auch die Kurzform verwenden, falls vorhanden.
|
# Nutze ggf. die Kurzform aus Spalte C, falls vorhanden.
|
||||||
|
search_name = company_name
|
||||||
query = f'site:linkedin.com/in "{position_query}" "{search_name}"'
|
query = f'site:linkedin.com/in "{position_query}" "{search_name}"'
|
||||||
debug_print(f"Erstelle LinkedIn-Query: {query}")
|
debug_print(f"Erstelle LinkedIn-Query: {query}")
|
||||||
params = {
|
params = {
|
||||||
@@ -446,121 +365,155 @@ def count_linkedin_contacts(company_name, website, position_query):
|
|||||||
debug_print(f"Fehler bei der SerpAPI-Suche (Count): {e}")
|
debug_print(f"Fehler bei der SerpAPI-Suche (Count): {e}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
# ==================== NEUE FUNKTION: _process_verification_row ====================
|
# ==================== VERIFIZIERUNGS-MODUS (Modus 51) ====================
|
||||||
def _process_verification_row(self, row_num, row_data):
|
def _process_verification_row(row_num, row_data):
|
||||||
# Verarbeitung nur bis Spalte Y (Begründung Abweichung Branche)
|
"""
|
||||||
|
Aggregiert die relevanten Informationen eines Eintrags für die Verifizierung.
|
||||||
|
Erwartete Spalten (0-basiert):
|
||||||
|
B: Firmenname
|
||||||
|
F: CRM-Beschreibung
|
||||||
|
M: Wiki URL
|
||||||
|
N: Wiki Absatz
|
||||||
|
R: Wiki Kategorien
|
||||||
|
"""
|
||||||
company_name = row_data[1] if len(row_data) > 1 else ""
|
company_name = row_data[1] if len(row_data) > 1 else ""
|
||||||
website = row_data[3] if len(row_data) > 3 else ""
|
crm_description = row_data[5] if len(row_data) > 5 else ""
|
||||||
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
wiki_url = row_data[12] if len(row_data) > 12 else "k.A."
|
||||||
if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."]:
|
wiki_absatz = row_data[13] if len(row_data) > 13 else "k.A."
|
||||||
wiki_url = row_data[11].strip()
|
wiki_categories = row_data[17] if len(row_data) > 17 else "k.A."
|
||||||
try:
|
entry_text = (f"Eintrag {row_num}:\n"
|
||||||
wiki_data = self.wiki_scraper.extract_company_data(wiki_url)
|
f"Firmenname: {company_name}\n"
|
||||||
except Exception as e:
|
f"CRM-Beschreibung: {crm_description}\n"
|
||||||
debug_print(f"Fehler beim Laden des vorgeschlagenen Wikipedia-Artikels: {e}")
|
f"Wikipedia-URL: {wiki_url}\n"
|
||||||
article = self.wiki_scraper.search_company_article(company_name, website)
|
f"Wikipedia-Absatz: {wiki_absatz}\n"
|
||||||
wiki_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
f"Wikipedia-Kategorien: {wiki_categories}\n"
|
||||||
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
"-----\n")
|
||||||
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
return entry_text
|
||||||
'full_infobox': 'k.A.'
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
article = self.wiki_scraper.search_company_article(company_name, website)
|
|
||||||
wiki_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
|
||||||
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
|
||||||
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
|
||||||
'full_infobox': 'k.A.'
|
|
||||||
}
|
|
||||||
wiki_values = [
|
|
||||||
row_data[11] if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."] else "k.A.",
|
|
||||||
wiki_data.get('url', 'k.A.'),
|
|
||||||
wiki_data.get('first_paragraph', 'k.A.'),
|
|
||||||
wiki_data.get('branche', 'k.A.'),
|
|
||||||
wiki_data.get('umsatz', 'k.A.'),
|
|
||||||
wiki_data.get('mitarbeiter', 'k.A.'),
|
|
||||||
wiki_data.get('categories', 'k.A.')
|
|
||||||
]
|
|
||||||
self.sheet_handler.sheet.update(values=[wiki_values], range_name=f"L{row_num}:R{row_num}")
|
|
||||||
crm_branche = row_data[6] if len(row_data) > 6 else "k.A."
|
|
||||||
beschreibung = row_data[7] if len(row_data) > 7 else "k.A."
|
|
||||||
wiki_branche = wiki_data.get('branche', 'k.A.')
|
|
||||||
wiki_kategorien = wiki_data.get('categories', 'k.A.')
|
|
||||||
branche_result = evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kategorien)
|
|
||||||
self.sheet_handler.sheet.update(values=[[branche_result["branch"]]], range_name=f"V{row_num}")
|
|
||||||
self.sheet_handler.sheet.update(values=[[branche_result["consistency"]]], range_name=f"W{row_num}")
|
|
||||||
self.sheet_handler.sheet.update(values=[[branche_result["justification"]]], range_name=f"X{row_num}")
|
|
||||||
crm_data = ";".join(row_data[1:11])
|
|
||||||
wiki_data_str = ";".join(row_data[11:18])
|
|
||||||
valid_result = validate_article_with_chatgpt(crm_data, wiki_data_str)
|
|
||||||
self.sheet_handler.sheet.update(values=[[valid_result]], range_name=f"Y{row_num}")
|
|
||||||
self.sheet_handler.sheet.update(values=[[current_dt]], range_name=f"Z{row_num}")
|
|
||||||
self.sheet_handler.sheet.update(values=[[Config.VERSION]], range_name=f"AA{row_num}")
|
|
||||||
debug_print(f"Zeile {row_num} verifiziert: URL: {wiki_data.get('url', 'k.A.')}, Branche: {wiki_data.get('branche', 'k.A.')}")
|
|
||||||
time.sleep(Config.RETRY_DELAY)
|
|
||||||
|
|
||||||
# Nach Abschluss der DataProcessor-Klasse wird diese Methode zugewiesen:
|
def process_verification_only():
|
||||||
# (Siehe unten nach der Klassendefinition)
|
"""
|
||||||
|
Verifizierungsmodus (Modus 51) im Batch-Prozess.
|
||||||
# ==================== NEUER MODUS 8: BATCH-PROZESSING MIT TOKEN-ZÄHLUNG ====================
|
Es werden jeweils Config.BATCH_SIZE (z.B. 10) Einträge aggregiert.
|
||||||
def process_batch_token_count(batch_size=10):
|
Für jeden Eintrag werden folgende Spalten aktualisiert:
|
||||||
import tiktoken
|
- Spalte W: Branchenvorschlag von ChatGPT
|
||||||
def count_tokens(text, model="gpt-3.5-turbo"):
|
- Spalte X: Konsistenzprüfung (OK oder X)
|
||||||
encoding = tiktoken.encoding_for_model(model)
|
- Spalte Y: Begründung bei Abweichung
|
||||||
tokens = encoding.encode(text)
|
- Spalte AQ: Token-Zahl des aggregierten Prompts (gleich für alle Einträge des Batches)
|
||||||
return len(tokens)
|
- Spalte Z: Verifizierungs-Timestamp
|
||||||
debug_print("Starte Batch-Token-Zählung (Modus 8)...")
|
- Spalte AA: Version
|
||||||
|
"""
|
||||||
|
debug_print("Starte Verifizierungsmodus (Modus 51) im Batch-Prozess...")
|
||||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
||||||
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
||||||
sh = gc.open_by_url(Config.SHEET_URL)
|
sh = gc.open_by_url(Config.SHEET_URL)
|
||||||
main_sheet = sh.sheet1
|
main_sheet = sh.sheet1
|
||||||
data = main_sheet.get_all_values()
|
data = main_sheet.get_all_values()
|
||||||
for i in range(2, len(data)+1, batch_size):
|
batch_size = Config.BATCH_SIZE
|
||||||
batch_rows = data[i-1:i-1+batch_size]
|
batch_entries = []
|
||||||
aggregated_prompt = ""
|
row_indices = []
|
||||||
for row in batch_rows:
|
# Wir prüfen hier Spalte Y (Index 24); wenn leer, dann ist der Eintrag noch nicht verifiziert.
|
||||||
info = []
|
for i, row in enumerate(data[1:], start=2):
|
||||||
if len(row) > 1:
|
if len(row) <= 25 or row[24].strip() == "":
|
||||||
info.append(row[1]) # Firmenname
|
entry_text = _process_verification_row(i, row)
|
||||||
if len(row) > 2:
|
batch_entries.append(entry_text)
|
||||||
info.append(row[2]) # Kurzform
|
row_indices.append(i)
|
||||||
if len(row) > 3:
|
if len(batch_entries) == batch_size:
|
||||||
info.append(row[3]) # Website
|
break
|
||||||
if len(row) > 4:
|
if not batch_entries:
|
||||||
info.append(row[4]) # Ort
|
debug_print("Keine Einträge für die Verifizierung gefunden.")
|
||||||
if len(row) > 5:
|
return
|
||||||
info.append(row[5]) # Beschreibung
|
|
||||||
if len(row) > 6:
|
|
||||||
info.append(row[6]) # Aktuelle Branche
|
|
||||||
aggregated_prompt += "; ".join(info) + "\n"
|
|
||||||
token_count = count_tokens(aggregated_prompt)
|
|
||||||
debug_print(f"Batch beginnend in Zeile {i}: {token_count} Tokens")
|
|
||||||
for j in range(i, min(i+batch_size, len(data)+1)):
|
|
||||||
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{j}")
|
|
||||||
time.sleep(Config.RETRY_DELAY)
|
|
||||||
debug_print("Batch-Token-Zählung abgeschlossen.")
|
|
||||||
|
|
||||||
# ==================== NEUER MODUS: ALIGNMENT DEMO (für Hauptblatt und Contacts) ====================
|
aggregated_prompt = ("Du bist ein Experte im Bereich Unternehmensverifizierung. "
|
||||||
def alignment_demo_full():
|
"Für jeden der folgenden Einträge prüfe, ob der vorhandene Wikipedia-Artikel (URL, Absatz, Kategorien) plausibel zum Unternehmen passt. "
|
||||||
alignment_demo(GoogleSheetHandler().sheet)
|
"Falls ja, antworte für den Eintrag im Format:\n"
|
||||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
"Eintrag X: OK\n"
|
||||||
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
"Falls nein, schlage einen alternativen Wikipedia-Artikel vor (als URL) und gib die Gründe an, "
|
||||||
sh = gc.open_by_url(Config.SHEET_URL)
|
"aber gib nicht denselben Artikel zurück, der bereits vorliegt. "
|
||||||
|
"Wenn kein Artikel gefunden werden kann, antworte mit 'k.A.'\n\n")
|
||||||
|
aggregated_prompt += "\n".join(batch_entries)
|
||||||
|
debug_print("Aggregierter Prompt für Verifizierungs-Batch erstellt.")
|
||||||
|
# Zähle die Token (falls tiktoken verfügbar)
|
||||||
|
token_count = "n.v."
|
||||||
|
if tiktoken:
|
||||||
|
try:
|
||||||
|
enc = tiktoken.encoding_for_model(Config.TOKEN_MODEL)
|
||||||
|
token_count = len(enc.encode(aggregated_prompt))
|
||||||
|
debug_print(f"Token-Zahl für Batch: {token_count}")
|
||||||
|
except Exception as e:
|
||||||
|
debug_print(f"Fehler beim Token-Counting: {e}")
|
||||||
|
# Sende den aggregierten Prompt an ChatGPT
|
||||||
try:
|
try:
|
||||||
contacts_sheet = sh.worksheet("Contacts")
|
with open("api_key.txt", "r") as f:
|
||||||
except gspread.exceptions.WorksheetNotFound:
|
api_key = f.read().strip()
|
||||||
contacts_sheet = sh.add_worksheet(title="Contacts", rows="1000", cols="10")
|
except Exception as e:
|
||||||
header = ["Firmenname", "Website", "Kurzform", "Vorname", "Nachname", "Position", "Anrede", "E-Mail"]
|
debug_print(f"Fehler beim Lesen des API-Tokens (Verifizierung): {e}")
|
||||||
contacts_sheet.update(values=[header], range_name="A1:H1")
|
return
|
||||||
debug_print("Neues Blatt 'Contacts' erstellt und Header eingetragen.")
|
openai.api_key = api_key
|
||||||
alignment_demo(contacts_sheet)
|
try:
|
||||||
debug_print("Alignment-Demo für Hauptblatt und Contacts abgeschlossen.")
|
response = openai.ChatCompletion.create(
|
||||||
|
model=Config.TOKEN_MODEL,
|
||||||
|
messages=[{"role": "system", "content": aggregated_prompt}],
|
||||||
|
temperature=0.0
|
||||||
|
)
|
||||||
|
result = response.choices[0].message.content.strip()
|
||||||
|
debug_print(f"Antwort ChatGPT Verifizierung Batch: {result}")
|
||||||
|
except Exception as e:
|
||||||
|
debug_print(f"Fehler bei der ChatGPT Anfrage für Verifizierung: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
# ==================== ALIGNMENT DEMO (Hauptblatt) ====================
|
# Wir erwarten, dass ChatGPT für jeden Eintrag eine Zeile liefert im Format "Eintrag X: <Antwort>"
|
||||||
|
answers = result.split("\n")
|
||||||
|
for idx, row_num in enumerate(row_indices):
|
||||||
|
answer = "k.A."
|
||||||
|
for line in answers:
|
||||||
|
if line.strip().startswith(f"Eintrag {row_num}:"):
|
||||||
|
answer = line.split(":", 1)[1].strip()
|
||||||
|
break
|
||||||
|
# Falls die Antwort "OK" lautet, setze in Spalte X "OK" und Spalte Y leer;
|
||||||
|
# ansonsten in Spalte X "X" und Spalte Y den Vorschlag.
|
||||||
|
if answer.upper() == "OK":
|
||||||
|
branch_suggestion = "OK"
|
||||||
|
consistency = "OK"
|
||||||
|
justification = ""
|
||||||
|
else:
|
||||||
|
branch_suggestion = answer # hier wird der alternative Artikel (URL) als Vorschlag verwendet
|
||||||
|
consistency = "X"
|
||||||
|
justification = answer # oder ggf. eine ausführlichere Begründung; hier wird derselbe Text genutzt
|
||||||
|
main_sheet.update(values=[[branch_suggestion]], range_name=f"W{row_num}")
|
||||||
|
main_sheet.update(values=[[consistency]], range_name=f"X{row_num}")
|
||||||
|
main_sheet.update(values=[[justification]], range_name=f"Y{row_num}")
|
||||||
|
# Schreibe den Token-Count in Spalte AQ (gleich für alle Einträge dieses Batches)
|
||||||
|
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}")
|
||||||
|
main_sheet.update(values=[[datetime.now().strftime('%Y-%m-%d %H:%M:%S')]], range_name=f"Z{row_num}")
|
||||||
|
main_sheet.update(values=[[Config.VERSION]], range_name=f"AA{row_num}")
|
||||||
|
debug_print(f"Zeile {row_num} verifiziert: Antwort: {answer}")
|
||||||
|
time.sleep(Config.RETRY_DELAY)
|
||||||
|
debug_print("Verifizierungs-Batch abgeschlossen.")
|
||||||
|
|
||||||
|
# ==================== STARTINDEX-FUNKTIONEN ====================
|
||||||
|
class GoogleSheetHandler:
|
||||||
|
def __init__(self):
|
||||||
|
self.sheet = None
|
||||||
|
self.sheet_values = []
|
||||||
|
self._connect()
|
||||||
|
def _connect(self):
|
||||||
|
scope = ["https://www.googleapis.com/auth/spreadsheets"]
|
||||||
|
creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope)
|
||||||
|
self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1
|
||||||
|
self.sheet_values = self.sheet.get_all_values()
|
||||||
|
def get_start_index(self, column_index=39):
|
||||||
|
"""
|
||||||
|
column_index=39 für Wiki (Spalte AN), column_index=40 für ChatGPT (Spalte AO)
|
||||||
|
"""
|
||||||
|
filled_n = [row[column_index] if len(row) > column_index else '' for row in self.sheet_values[1:]]
|
||||||
|
return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1)
|
||||||
|
|
||||||
|
# ==================== ALIGNMENT DEMO (Modus 3) ====================
|
||||||
def alignment_demo(sheet):
|
def alignment_demo(sheet):
|
||||||
new_headers = [
|
new_headers = [
|
||||||
"Spalte A (ReEval Flag)",
|
"Spalte A (ReEval Flag)",
|
||||||
"Spalte B (Firmenname)",
|
"Spalte B (Firmenname)",
|
||||||
"Spalte C (Kurzform des Firmennamens)",
|
"Spalte C (Kurzform Firmenname)",
|
||||||
"Spalte D (Website)",
|
"Spalte D (Website)",
|
||||||
"Spalte E (Ort)",
|
"Spalte E (Ort)",
|
||||||
"Spalte F (Beschreibung)",
|
"Spalte F (Beschreibung)",
|
||||||
@@ -579,14 +532,30 @@ def alignment_demo(sheet):
|
|||||||
"Spalte S (Konsistenzprüfung)",
|
"Spalte S (Konsistenzprüfung)",
|
||||||
"Spalte T (Begründung bei Inkonsistenz)",
|
"Spalte T (Begründung bei Inkonsistenz)",
|
||||||
"Spalte U (Vorschlag Wiki Artikel ChatGPT)",
|
"Spalte U (Vorschlag Wiki Artikel ChatGPT)",
|
||||||
"Spalte V (Begründung bei Abweichung)",
|
"Spalte V (Konsistenzprüfung Branche)",
|
||||||
"Spalte W (Vorschlag neue Branche)",
|
"Spalte W (Vorschlag neue Branche)", # Wird in Modus 51 als Branchenvorschlag genutzt
|
||||||
"Spalte X (Konsistenzprüfung Branche)",
|
"Spalte X (Konsistenzprüfung – OK oder X)",
|
||||||
"Spalte Y (Begründung Abweichung Branche)",
|
"Spalte Y (Begründung Abweichung)",
|
||||||
"Spalte Z (Timestamp Verifizierung)",
|
"Spalte Z (Timestamp Verifizierung)",
|
||||||
"Spalte AA (Version)"
|
"Spalte AA (Version)",
|
||||||
|
"Spalte AB (Schätzung Anzahl Mitarbeiter)",
|
||||||
|
"Spalte AC (Konsistenzprüfung Mitarbeiterzahl)",
|
||||||
|
"Spalte AD (Einschätzung Anzahl Servicetechniker)",
|
||||||
|
"Spalte AE (Begründung bei Abweichung Techniker)",
|
||||||
|
"Spalte AF (Schätzung Umsatz ChatGPT)",
|
||||||
|
"Spalte AG (Begründung Umsatz ChatGPT)",
|
||||||
|
"Spalte AH (Wikipedia-Timestamp)",
|
||||||
|
"Spalte AI (ChatGPT-Timestamp)",
|
||||||
|
"Spalte AJ (Kontakt: Serviceleiter gefunden)",
|
||||||
|
"Spalte AK (Kontakt: IT-Leiter gefunden)",
|
||||||
|
"Spalte AL (Kontakt: Management gefunden)",
|
||||||
|
"Spalte AM (Kontakt: Disponent gefunden)",
|
||||||
|
"Spalte AN (Contact Search Timestamp)",
|
||||||
|
"Spalte AO (Wikipedia Timestamp – für regulären Wiki-Runner)",
|
||||||
|
"Spalte AP (ChatGPT Timestamp – für regulären ChatGPT-Runner)",
|
||||||
|
"Spalte AQ (Token Count Batch)"
|
||||||
]
|
]
|
||||||
header_range = "A11200:AA11200"
|
header_range = "A11200:AQ11200"
|
||||||
sheet.update(values=[new_headers], range_name=header_range)
|
sheet.update(values=[new_headers], range_name=header_range)
|
||||||
print("Alignment-Demo abgeschlossen: Neue Spaltenüberschriften in Zeile 11200 geschrieben.")
|
print("Alignment-Demo abgeschlossen: Neue Spaltenüberschriften in Zeile 11200 geschrieben.")
|
||||||
|
|
||||||
@@ -772,56 +741,22 @@ class WikipediaScraper:
|
|||||||
continue
|
continue
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# ==================== GOOGLE SHEET HANDLER (für Hauptdaten) ====================
|
|
||||||
class GoogleSheetHandler:
|
|
||||||
def __init__(self):
|
|
||||||
self.sheet = None
|
|
||||||
self.sheet_values = []
|
|
||||||
self._connect()
|
|
||||||
def _connect(self):
|
|
||||||
scope = ["https://www.googleapis.com/auth/spreadsheets"]
|
|
||||||
creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope)
|
|
||||||
self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1
|
|
||||||
self.sheet_values = self.sheet.get_all_values()
|
|
||||||
def get_start_index(self):
|
|
||||||
# Verwende Spalte AN (Index 39) als Wikipedia-Timestamp im regulären Modus
|
|
||||||
filled_n = [row[39] if len(row) > 39 else '' for row in self.sheet_values[1:]]
|
|
||||||
return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1)
|
|
||||||
|
|
||||||
# ==================== DATA PROCESSOR ====================
|
# ==================== DATA PROCESSOR ====================
|
||||||
class DataProcessor:
|
class DataProcessor:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.sheet_handler = GoogleSheetHandler()
|
self.sheet_handler = GoogleSheetHandler()
|
||||||
self.wiki_scraper = WikipediaScraper()
|
self.wiki_scraper = WikipediaScraper()
|
||||||
|
|
||||||
def process_rows(self, num_rows=None):
|
def process_rows(self, num_rows=None):
|
||||||
if MODE == "2":
|
if MODE == "2":
|
||||||
print("Re-Evaluierungsmodus: Verarbeitung aller Zeilen mit 'x' in Spalte A.")
|
print("Re-Evaluierungsmodus: Verarbeitung aller Zeilen mit 'x' in Spalte A.")
|
||||||
for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2):
|
for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2):
|
||||||
if row[0].strip().lower() == "x":
|
if row[0].strip().lower() == "x":
|
||||||
self._process_single_row(i, row, force_all=True)
|
self._process_single_row(i, row)
|
||||||
elif MODE == "3":
|
elif MODE == "3":
|
||||||
print("Alignment-Demo-Modus: Schreibe neue Spaltenüberschriften in Hauptblatt und Contacts.")
|
print("Alignment-Demo-Modus: Schreibe neue Spaltenüberschriften in Zeile 11200.")
|
||||||
alignment_demo_full()
|
alignment_demo(self.sheet_handler.sheet)
|
||||||
elif MODE == "4":
|
|
||||||
processor = DataProcessor()
|
|
||||||
for i, row in enumerate(processor.sheet_handler.sheet_values[1:], start=2):
|
|
||||||
if len(row) <= 39 or row[39].strip() == "":
|
|
||||||
processor._process_single_row(i, row, process_wiki=True, process_chatgpt=False)
|
|
||||||
elif MODE == "5":
|
|
||||||
processor = DataProcessor()
|
|
||||||
for i, row in enumerate(processor.sheet_handler.sheet_values[1:], start=2):
|
|
||||||
if len(row) <= 40 or row[40].strip() == "":
|
|
||||||
processor._process_single_row(i, row, process_wiki=False, process_chatgpt=True)
|
|
||||||
elif MODE == "51":
|
|
||||||
processor = DataProcessor()
|
|
||||||
for i, row in enumerate(processor.sheet_handler.sheet_values[1:], start=2):
|
|
||||||
if len(row) <= 25 or row[24].strip() == "":
|
|
||||||
processor._process_verification_row(i, row)
|
|
||||||
elif MODE == "8":
|
|
||||||
process_batch_token_count()
|
|
||||||
else:
|
else:
|
||||||
start_index = self.sheet_handler.get_start_index()
|
start_index = self.sheet_handler.get_start_index(40) # Standardmäßig ChatGPT-Timestamp (Spalte AO)
|
||||||
print(f"Starte bei Zeile {start_index+1}")
|
print(f"Starte bei Zeile {start_index+1}")
|
||||||
rows_processed = 0
|
rows_processed = 0
|
||||||
for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2):
|
for i, row in enumerate(self.sheet_handler.sheet_values[1:], start=2):
|
||||||
@@ -831,89 +766,131 @@ class DataProcessor:
|
|||||||
break
|
break
|
||||||
self._process_single_row(i, row)
|
self._process_single_row(i, row)
|
||||||
rows_processed += 1
|
rows_processed += 1
|
||||||
|
def _process_single_row(self, row_num, row_data):
|
||||||
def _process_single_row(self, row_num, row_data, force_all=False, process_wiki=True, process_chatgpt=True):
|
|
||||||
company_name = row_data[1] if len(row_data) > 1 else ""
|
company_name = row_data[1] if len(row_data) > 1 else ""
|
||||||
website = row_data[3] if len(row_data) > 3 else ""
|
website = row_data[3] if len(row_data) > 3 else ""
|
||||||
wiki_update_range = f"L{row_num}:R{row_num}"
|
wiki_update_range = f"L{row_num}:R{row_num}" # Angenommen, hier kommen Wiki-Daten rein
|
||||||
dt_wiki_range = f"AN{row_num}"
|
# Falls in Spalte L bereits ein Wiki-URL steht, nutze diese
|
||||||
dt_chat_range = f"AO{row_num}"
|
if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."]:
|
||||||
ver_range = f"AP{row_num}"
|
wiki_url = row_data[11].strip()
|
||||||
print(f"\n[{datetime.now().strftime('%H:%M:%S')}] Verarbeite Zeile {row_num}: {company_name}")
|
try:
|
||||||
|
company_data = self.wiki_scraper.extract_company_data(wiki_url)
|
||||||
|
except Exception as e:
|
||||||
|
debug_print(f"Fehler beim Laden des vorgeschlagenen Wikipedia-Artikels: {e}")
|
||||||
|
article = self.wiki_scraper.search_company_article(company_name, website)
|
||||||
|
company_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
||||||
|
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
||||||
|
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
||||||
|
'full_infobox': 'k.A.'
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
article = self.wiki_scraper.search_company_article(company_name, website)
|
||||||
|
company_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
||||||
|
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
||||||
|
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
||||||
|
'full_infobox': 'k.A.'
|
||||||
|
}
|
||||||
|
wiki_values = [
|
||||||
|
company_data.get('url', 'k.A.'),
|
||||||
|
company_data.get('first_paragraph', 'k.A.'),
|
||||||
|
company_data.get('branche', 'k.A.'),
|
||||||
|
company_data.get('umsatz', 'k.A.'),
|
||||||
|
company_data.get('mitarbeiter', 'k.A.'),
|
||||||
|
company_data.get('categories', 'k.A.')
|
||||||
|
]
|
||||||
|
self.sheet_handler.sheet.update(values=[wiki_values], range_name=wiki_update_range)
|
||||||
|
time.sleep(3)
|
||||||
|
# Weitere Verarbeitung (z.B. Umsatz-Abgleich, Brancheneinordnung etc.) würden hier erfolgen,
|
||||||
|
# aber im regulären Modus 1 werden auch FSM und Techniker verarbeitet – das ist hier nicht Teil von Modus 51.
|
||||||
|
# Deshalb bleibt dieser Teil unberührt.
|
||||||
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
if force_all or process_wiki:
|
# Aktualisiere Timestamp und Version in den entsprechenden Spalten (z.B. in Spalte AP für ChatGPT-Timestamp)
|
||||||
if len(row_data) <= 39 or row_data[39].strip() == "":
|
self.sheet_handler.sheet.update(values=[[current_dt]], range_name=f"AP{row_num}")
|
||||||
if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."]:
|
self.sheet_handler.sheet.update(values=[[Config.VERSION]], range_name=f"AQ{row_num}")
|
||||||
wiki_url = row_data[11].strip()
|
debug_print(f"Zeile {row_num} verarbeitet.")
|
||||||
try:
|
|
||||||
wiki_data = self.wiki_scraper.extract_company_data(wiki_url)
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Laden des vorgeschlagenen Wikipedia-Artikels: {e}")
|
|
||||||
article = self.wiki_scraper.search_company_article(company_name, website)
|
|
||||||
wiki_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
|
||||||
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
|
||||||
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
|
||||||
'full_infobox': 'k.A.'
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
article = self.wiki_scraper.search_company_article(company_name, website)
|
|
||||||
wiki_data = self.wiki_scraper.extract_company_data(article.url) if article else {
|
|
||||||
'url': 'k.A.', 'first_paragraph': 'k.A.', 'branche': 'k.A.',
|
|
||||||
'umsatz': 'k.A.', 'mitarbeiter': 'k.A.', 'categories': 'k.A.',
|
|
||||||
'full_infobox': 'k.A.'
|
|
||||||
}
|
|
||||||
wiki_values = [
|
|
||||||
row_data[11] if len(row_data) > 11 and row_data[11].strip() not in ["", "k.A."] else "k.A.",
|
|
||||||
wiki_data.get('url', 'k.A.'),
|
|
||||||
wiki_data.get('first_paragraph', 'k.A.'),
|
|
||||||
wiki_data.get('branche', 'k.A.'),
|
|
||||||
wiki_data.get('umsatz', 'k.A.'),
|
|
||||||
wiki_data.get('mitarbeiter', 'k.A.'),
|
|
||||||
wiki_data.get('categories', 'k.A.')
|
|
||||||
]
|
|
||||||
self.sheet_handler.sheet.update(values=[wiki_values], range_name=wiki_update_range)
|
|
||||||
self.sheet_handler.sheet.update(values=[[current_dt]], range_name=dt_wiki_range)
|
|
||||||
else:
|
|
||||||
debug_print(f"Zeile {row_num}: Wikipedia-Timestamp bereits gesetzt – überspringe Wiki-Auswertung.")
|
|
||||||
if force_all or process_chatgpt:
|
|
||||||
if len(row_data) <= 40 or row_data[40].strip() == "":
|
|
||||||
crm_umsatz = row_data[9] if len(row_data) > 9 else "k.A."
|
|
||||||
abgleich_result = compare_umsatz_values(crm_umsatz, wiki_data.get('umsatz', 'k.A.') if 'wiki_data' in locals() else "k.A.")
|
|
||||||
self.sheet_handler.sheet.update(values=[[abgleich_result]], range_name=f"AG{row_num}")
|
|
||||||
crm_data = ";".join(row_data[1:11])
|
|
||||||
wiki_data_str = ";".join(row_data[11:18])
|
|
||||||
valid_result = validate_article_with_chatgpt(crm_data, wiki_data_str)
|
|
||||||
self.sheet_handler.sheet.update(values=[[valid_result]], range_name=f"R{row_num}")
|
|
||||||
fsm_result = evaluate_fsm_suitability(company_name, wiki_data if 'wiki_data' in locals() else {})
|
|
||||||
self.sheet_handler.sheet.update(values=[[fsm_result["suitability"]]], range_name=f"Y{row_num}")
|
|
||||||
self.sheet_handler.sheet.update(values=[[fsm_result["justification"]]], range_name=f"Z{row_num}")
|
|
||||||
st_estimate = evaluate_servicetechnicians_estimate(company_name, wiki_data if 'wiki_data' in locals() else {})
|
|
||||||
self.sheet_handler.sheet.update(values=[[st_estimate]], range_name=f"AE{row_num}")
|
|
||||||
internal_value = row_data[8] if len(row_data) > 8 else "k.A."
|
|
||||||
internal_category = map_internal_technicians(internal_value) if internal_value != "k.A." else "k.A."
|
|
||||||
if internal_category != "k.A." and st_estimate != internal_category:
|
|
||||||
explanation = evaluate_servicetechnicians_explanation(company_name, st_estimate, wiki_data if 'wiki_data' in locals() else {})
|
|
||||||
discrepancy = explanation
|
|
||||||
else:
|
|
||||||
discrepancy = "ok"
|
|
||||||
self.sheet_handler.sheet.update(values=[[discrepancy]], range_name=f"AF{row_num}")
|
|
||||||
self.sheet_handler.sheet.update(values=[[current_dt]], range_name=dt_chat_range)
|
|
||||||
else:
|
|
||||||
debug_print(f"Zeile {row_num}: ChatGPT-Timestamp bereits gesetzt – überspringe ChatGPT-Auswertung.")
|
|
||||||
self.sheet_handler.sheet.update(values=[[current_dt]], range_name=ver_range)
|
|
||||||
self.sheet_handler.sheet.update(values=[[Config.VERSION]], range_name=ver_range)
|
|
||||||
debug_print(f"✅ Aktualisiert: URL: {(wiki_data.get('url', 'k.A.') if 'wiki_data' in locals() else 'k.A.')}, "
|
|
||||||
f"Branche: {(wiki_data.get('branche', 'k.A.') if 'wiki_data' in locals() else 'k.A.')}, "
|
|
||||||
f"Umsatz-Abgleich: {abgleich_result if 'abgleich_result' in locals() else 'k.A.'}, "
|
|
||||||
f"Validierung: {valid_result if 'valid_result' in locals() else 'k.A.'}, "
|
|
||||||
f"FSM: {fsm_result['suitability'] if 'fsm_result' in locals() else 'k.A.'}, "
|
|
||||||
f"Servicetechniker-Schätzung: {st_estimate if 'st_estimate' in locals() else 'k.A.'}")
|
|
||||||
time.sleep(Config.RETRY_DELAY)
|
time.sleep(Config.RETRY_DELAY)
|
||||||
|
|
||||||
# Hier wird _process_verification_row nach der Definition von DataProcessor zugewiesen.
|
# ==================== MODUS 51: VERIFIZIERUNG (BATCH) ====================
|
||||||
DataProcessor._process_verification_row = _process_verification_row
|
def process_verification_only():
|
||||||
|
debug_print("Starte Verifizierungsmodus (Modus 51) im Batch-Prozess...")
|
||||||
|
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
||||||
|
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
||||||
|
sh = gc.open_by_url(Config.SHEET_URL)
|
||||||
|
main_sheet = sh.sheet1
|
||||||
|
data = main_sheet.get_all_values()
|
||||||
|
batch_size = Config.BATCH_SIZE
|
||||||
|
batch_entries = []
|
||||||
|
row_indices = []
|
||||||
|
# Prüfe hier Spalte Y (Index 24); wenn leer, dann verifizieren.
|
||||||
|
for i, row in enumerate(data[1:], start=2):
|
||||||
|
if len(row) <= 25 or row[24].strip() == "":
|
||||||
|
entry_text = _process_verification_row(i, row)
|
||||||
|
batch_entries.append(entry_text)
|
||||||
|
row_indices.append(i)
|
||||||
|
if len(batch_entries) == batch_size:
|
||||||
|
break
|
||||||
|
if not batch_entries:
|
||||||
|
debug_print("Keine Einträge für die Verifizierung gefunden.")
|
||||||
|
return
|
||||||
|
aggregated_prompt = ("Du bist ein Experte im Bereich Unternehmensverifizierung. "
|
||||||
|
"Für jeden der folgenden Einträge prüfe, ob der vorhandene Wikipedia-Artikel plausibel zum Unternehmen passt. "
|
||||||
|
"Gib für jeden Eintrag das Ergebnis in folgendem Format aus:\n"
|
||||||
|
"Eintrag <Zeilennummer>: <Branchenvorschlag> | <Konsistenz (OK oder X)> | <Begründung>\n"
|
||||||
|
"Wenn der Artikel passt, antworte mit 'OK'. Falls nicht, schlage einen alternativen Artikel (als URL) vor.\n\n")
|
||||||
|
aggregated_prompt += "\n".join(batch_entries)
|
||||||
|
debug_print("Aggregierter Prompt für Verifizierungs-Batch erstellt.")
|
||||||
|
token_count = "n.v."
|
||||||
|
if tiktoken:
|
||||||
|
try:
|
||||||
|
enc = tiktoken.encoding_for_model(Config.TOKEN_MODEL)
|
||||||
|
token_count = len(enc.encode(aggregated_prompt))
|
||||||
|
debug_print(f"Token-Zahl für Batch: {token_count}")
|
||||||
|
except Exception as e:
|
||||||
|
debug_print(f"Fehler beim Token-Counting: {e}")
|
||||||
|
try:
|
||||||
|
with open("api_key.txt", "r") as f:
|
||||||
|
api_key = f.read().strip()
|
||||||
|
except Exception as e:
|
||||||
|
debug_print(f"Fehler beim Lesen des API-Tokens (Verifizierung): {e}")
|
||||||
|
return
|
||||||
|
openai.api_key = api_key
|
||||||
|
try:
|
||||||
|
response = openai.ChatCompletion.create(
|
||||||
|
model=Config.TOKEN_MODEL,
|
||||||
|
messages=[{"role": "system", "content": aggregated_prompt}],
|
||||||
|
temperature=0.0
|
||||||
|
)
|
||||||
|
result = response.choices[0].message.content.strip()
|
||||||
|
debug_print(f"Antwort ChatGPT Verifizierung Batch: {result}")
|
||||||
|
except Exception as e:
|
||||||
|
debug_print(f"Fehler bei der ChatGPT Anfrage für Verifizierung: {e}")
|
||||||
|
return
|
||||||
|
answers = result.split("\n")
|
||||||
|
for idx, row_num in enumerate(row_indices):
|
||||||
|
answer = "k.A."
|
||||||
|
for line in answers:
|
||||||
|
if line.strip().startswith(f"Eintrag {row_num}:"):
|
||||||
|
answer = line.split(":", 1)[1].strip()
|
||||||
|
break
|
||||||
|
if answer.upper() == "OK":
|
||||||
|
branch_suggestion = "OK"
|
||||||
|
consistency = "OK"
|
||||||
|
justification = ""
|
||||||
|
else:
|
||||||
|
branch_suggestion = answer
|
||||||
|
consistency = "X"
|
||||||
|
justification = answer
|
||||||
|
main_sheet.update(values=[[branch_suggestion]], range_name=f"W{row_num}")
|
||||||
|
main_sheet.update(values=[[consistency]], range_name=f"X{row_num}")
|
||||||
|
main_sheet.update(values=[[justification]], range_name=f"Y{row_num}")
|
||||||
|
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}")
|
||||||
|
main_sheet.update(values=[[datetime.now().strftime('%Y-%m-%d %H:%M:%S')]], range_name=f"Z{row_num}")
|
||||||
|
main_sheet.update(values=[[Config.VERSION]], range_name=f"AA{row_num}")
|
||||||
|
debug_print(f"Zeile {row_num} verifiziert: Antwort: {answer}")
|
||||||
|
time.sleep(Config.RETRY_DELAY)
|
||||||
|
debug_print("Verifizierungs-Batch abgeschlossen.")
|
||||||
|
|
||||||
# ==================== NEUER MODUS 6: CONTACT RESEARCH (via SerpAPI) ====================
|
# ==================== NEUER MODUS: CONTACT RESEARCH (via SerpAPI) ====================
|
||||||
def process_contact_research():
|
def process_contact_research():
|
||||||
debug_print("Starte Contact Research (Modus 6)...")
|
debug_print("Starte Contact Research (Modus 6)...")
|
||||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
||||||
@@ -960,74 +937,34 @@ def process_contacts():
|
|||||||
new_rows = []
|
new_rows = []
|
||||||
for idx, row in enumerate(data[1:], start=2):
|
for idx, row in enumerate(data[1:], start=2):
|
||||||
company_name = row[1] if len(row) > 1 else ""
|
company_name = row[1] if len(row) > 1 else ""
|
||||||
search_name = row[2].strip() if len(row) > 2 and row[2].strip() not in ["", "k.A."] else company_name
|
|
||||||
website = row[3] if len(row) > 3 else ""
|
website = row[3] if len(row) > 3 else ""
|
||||||
debug_print(f"Verarbeite Firma: '{company_name}' (Zeile {idx}), Website: '{website}'")
|
debug_print(f"Verarbeite Firma: '{company_name}' (Zeile {idx}), Website: '{website}'")
|
||||||
if not company_name or not website:
|
if not company_name or not website:
|
||||||
debug_print("Überspringe, da Firmenname oder Website fehlt.")
|
debug_print("Überspringe, da Firmenname oder Website fehlt.")
|
||||||
continue
|
continue
|
||||||
for pos in positions:
|
for pos in positions:
|
||||||
debug_print(f"Suche nach Position: '{pos}' bei '{search_name}'")
|
debug_print(f"Suche nach Position: '{pos}' bei '{company_name}'")
|
||||||
contact = search_linkedin_contact(search_name, website, pos)
|
contact = search_linkedin_contact(company_name, website, pos)
|
||||||
if contact:
|
if contact:
|
||||||
debug_print(f"Kontakt gefunden: {contact}")
|
debug_print(f"Kontakt gefunden: {contact}")
|
||||||
new_rows.append([contact["Firmenname"], website, search_name, contact["Vorname"], contact["Nachname"], contact["Position"], "", ""])
|
new_rows.append([contact["Firmenname"], contact["Website"], "", contact["Vorname"], contact["Nachname"], contact["Position"], "", ""])
|
||||||
else:
|
else:
|
||||||
debug_print(f"Kein Kontakt für Position '{pos}' bei '{search_name}' gefunden.")
|
debug_print(f"Kein Kontakt für Position '{pos}' bei '{company_name}' gefunden.")
|
||||||
if new_rows:
|
if new_rows:
|
||||||
last_row = len(contacts_sheet.get_all_values()) + 1
|
last_row = len(contacts_sheet.get_all_values()) + 1
|
||||||
range_str = f"A{last_row}:H{last_row + len(new_rows) - 1}"
|
range_str = f"A{last_row}:H{last_row + len(new_rows) - 1}"
|
||||||
contacts_sheet.update(values=new_rows, range_name=range_str)
|
contacts_sheet.update(range_str, new_rows)
|
||||||
debug_print(f"{len(new_rows)} Kontakte in 'Contacts' hinzugefügt.")
|
debug_print(f"{len(new_rows)} Kontakte in 'Contacts' hinzugefügt.")
|
||||||
else:
|
else:
|
||||||
debug_print("Keine Kontakte gefunden in der Haupttabelle.")
|
debug_print("Keine Kontakte gefunden in der Haupttabelle.")
|
||||||
|
|
||||||
# ==================== NEUER MODUS 8: BATCH-PROZESSING MIT TOKEN-ZÄHLUNG ====================
|
|
||||||
def process_batch_token_count(batch_size=10):
|
|
||||||
import tiktoken
|
|
||||||
def count_tokens(text, model="gpt-3.5-turbo"):
|
|
||||||
encoding = tiktoken.encoding_for_model(model)
|
|
||||||
tokens = encoding.encode(text)
|
|
||||||
return len(tokens)
|
|
||||||
debug_print("Starte Batch-Token-Zählung (Modus 8)...")
|
|
||||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
|
||||||
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
|
||||||
sh = gc.open_by_url(Config.SHEET_URL)
|
|
||||||
main_sheet = sh.sheet1
|
|
||||||
data = main_sheet.get_all_values()
|
|
||||||
for i in range(2, len(data)+1, batch_size):
|
|
||||||
batch_rows = data[i-1:i-1+batch_size]
|
|
||||||
aggregated_prompt = ""
|
|
||||||
for row in batch_rows:
|
|
||||||
info = []
|
|
||||||
if len(row) > 1:
|
|
||||||
info.append(row[1]) # Firmenname
|
|
||||||
if len(row) > 2:
|
|
||||||
info.append(row[2]) # Kurzform
|
|
||||||
if len(row) > 3:
|
|
||||||
info.append(row[3]) # Website
|
|
||||||
if len(row) > 4:
|
|
||||||
info.append(row[4]) # Ort
|
|
||||||
if len(row) > 5:
|
|
||||||
info.append(row[5]) # Beschreibung
|
|
||||||
if len(row) > 6:
|
|
||||||
info.append(row[6]) # Aktuelle Branche
|
|
||||||
aggregated_prompt += "; ".join(info) + "\n"
|
|
||||||
token_count = count_tokens(aggregated_prompt)
|
|
||||||
debug_print(f"Batch beginnend in Zeile {i}: {token_count} Tokens")
|
|
||||||
for j in range(i, min(i+batch_size, len(data)+1)):
|
|
||||||
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{j}")
|
|
||||||
time.sleep(Config.RETRY_DELAY)
|
|
||||||
debug_print("Batch-Token-Zählung abgeschlossen.")
|
|
||||||
|
|
||||||
# ==================== MAIN PROGRAMM ====================
|
# ==================== MAIN PROGRAMM ====================
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import argparse
|
import argparse
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--mode", type=str, help="Modus: 1,2,3,4,5,6,7,51 oder 8")
|
parser.add_argument("--mode", type=str, help="Modus: 1,2,3,4,5,6,7,8 oder 51")
|
||||||
parser.add_argument("--num_rows", type=int, default=0, help="Anzahl der zu bearbeitenden Zeilen (nur für Modus 1)")
|
parser.add_argument("--num_rows", type=int, default=0, help="Anzahl der zu bearbeitenden Zeilen (nur für Modus 1)")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if not args.mode:
|
if not args.mode:
|
||||||
print("Modi:")
|
print("Modi:")
|
||||||
print("1 = Regulärer Modus")
|
print("1 = Regulärer Modus")
|
||||||
@@ -1040,31 +977,61 @@ if __name__ == "__main__":
|
|||||||
print("8 = Batch-Token-Zählung")
|
print("8 = Batch-Token-Zählung")
|
||||||
print("51 = Nur Verifizierung (Wikipedia + Brancheneinordnung)")
|
print("51 = Nur Verifizierung (Wikipedia + Brancheneinordnung)")
|
||||||
args.mode = input("Wählen Sie den Modus: ").strip()
|
args.mode = input("Wählen Sie den Modus: ").strip()
|
||||||
|
|
||||||
MODE = args.mode
|
MODE = args.mode
|
||||||
if MODE == "1":
|
if MODE == "1":
|
||||||
num_rows = args.num_rows if args.num_rows > 0 else int(input("Wieviele Zeilen sollen überprüft werden? "))
|
try:
|
||||||
|
num_rows = int(input("Wieviele Zeilen sollen überprüft werden? "))
|
||||||
|
except Exception as e:
|
||||||
|
print("Ungültige Eingabe. Bitte eine Zahl eingeben.")
|
||||||
|
exit(1)
|
||||||
processor = DataProcessor()
|
processor = DataProcessor()
|
||||||
processor.process_rows(num_rows)
|
processor.process_rows(num_rows)
|
||||||
elif MODE in ["2", "3"]:
|
elif MODE in ["2", "3"]:
|
||||||
processor = DataProcessor()
|
processor = DataProcessor()
|
||||||
processor.process_rows()
|
processor.process_rows()
|
||||||
elif MODE == "4":
|
elif MODE == "4":
|
||||||
|
# Wiki-runner: Startindex anhand Spalte AN (Index 39)
|
||||||
|
gh = GoogleSheetHandler()
|
||||||
|
start_index = gh.get_start_index(39)
|
||||||
|
debug_print(f"Wiki-Modus: Starte bei Zeile {start_index+1}")
|
||||||
processor = DataProcessor()
|
processor = DataProcessor()
|
||||||
for i, row in enumerate(processor.sheet_handler.sheet_values[1:], start=2):
|
processor.process_rows()
|
||||||
if len(row) <= 39 or row[39].strip() == "":
|
|
||||||
processor._process_single_row(i, row, process_wiki=True, process_chatgpt=False)
|
|
||||||
elif MODE == "5":
|
elif MODE == "5":
|
||||||
|
# ChatGPT-runner: Startindex anhand Spalte AO (Index 40)
|
||||||
|
gh = GoogleSheetHandler()
|
||||||
|
start_index = gh.get_start_index(40)
|
||||||
|
debug_print(f"ChatGPT-Modus: Starte bei Zeile {start_index+1}")
|
||||||
processor = DataProcessor()
|
processor = DataProcessor()
|
||||||
for i, row in enumerate(processor.sheet_handler.sheet_values[1:], start=2):
|
processor.process_rows()
|
||||||
if len(row) <= 40 or row[40].strip() == "":
|
|
||||||
processor._process_single_row(i, row, process_wiki=False, process_chatgpt=True)
|
|
||||||
elif MODE == "51":
|
|
||||||
process_verification_only()
|
|
||||||
elif MODE == "6":
|
elif MODE == "6":
|
||||||
process_contact_research()
|
process_contact_research()
|
||||||
elif MODE == "7":
|
elif MODE == "7":
|
||||||
process_contacts()
|
process_contacts()
|
||||||
elif MODE == "8":
|
elif MODE == "8":
|
||||||
process_batch_token_count()
|
# Batch-Token-Zählung: Aggregiere 10 Zeilen und zähle Token
|
||||||
|
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
||||||
|
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
||||||
|
sh = gc.open_by_url(Config.SHEET_URL)
|
||||||
|
main_sheet = sh.sheet1
|
||||||
|
data = main_sheet.get_all_values()
|
||||||
|
batch_entries = []
|
||||||
|
row_indices = []
|
||||||
|
for i, row in enumerate(data[1:], start=2):
|
||||||
|
batch_entries.append(" ".join(row))
|
||||||
|
row_indices.append(i)
|
||||||
|
if len(batch_entries) == Config.BATCH_SIZE:
|
||||||
|
break
|
||||||
|
aggregated_text = "\n".join(batch_entries)
|
||||||
|
token_count = "n.v."
|
||||||
|
if tiktoken:
|
||||||
|
try:
|
||||||
|
enc = tiktoken.encoding_for_model(Config.TOKEN_MODEL)
|
||||||
|
token_count = len(enc.encode(aggregated_text))
|
||||||
|
except Exception as e:
|
||||||
|
debug_print(f"Fehler beim Token-Counting: {e}")
|
||||||
|
for row_num in row_indices:
|
||||||
|
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}")
|
||||||
|
debug_print(f"Batch-Token-Zählung abgeschlossen. Token: {token_count}")
|
||||||
|
elif MODE == "51":
|
||||||
|
process_verification_only()
|
||||||
print(f"\n✅ Auswertung abgeschlossen ({Config.VERSION})")
|
print(f"\n✅ Auswertung abgeschlossen ({Config.VERSION})")
|
||||||
|
|||||||
Reference in New Issue
Block a user