v1.3.17 – Batch-Debug-Ausgabe und Zeilenanzahl-Abfrage in allen Modi
Debug-Ausgabe im Verifizierungsmodus zeigt jetzt die Zeilennummern und Firmennamen des aktuellen Batches. In allen relevanten Modi wird nun abgefragt, wieviele Zeilen verarbeitet werden sollen (Batch-Modus erwartet Vielfaches von 10). Alle sonstigen Funktionen bleiben erhalten – die Spaltenpositionen müssen unverändert sein, um den Code nicht anzupassen.
This commit is contained in:
@@ -246,12 +246,15 @@ def evaluate_branche_chatgpt(crm_branche, beschreibung, wiki_branche, wiki_kateg
|
|||||||
return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."}
|
return {"branch": "k.A.", "consistency": "k.A.", "justification": "k.A."}
|
||||||
|
|
||||||
def evaluate_fsm_suitability(company_name, company_data):
|
def evaluate_fsm_suitability(company_name, company_data):
|
||||||
|
# Vorläufig nicht genutzt – Rückgabe "n.v."
|
||||||
return {"suitability": "n.v.", "justification": ""}
|
return {"suitability": "n.v.", "justification": ""}
|
||||||
|
|
||||||
def evaluate_servicetechnicians_estimate(company_name, company_data):
|
def evaluate_servicetechnicians_estimate(company_name, company_data):
|
||||||
|
# Vorläufig nicht genutzt – Rückgabe "n.v."
|
||||||
return "n.v."
|
return "n.v."
|
||||||
|
|
||||||
def evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data):
|
def evaluate_servicetechnicians_explanation(company_name, st_estimate, company_data):
|
||||||
|
# Vorläufig nicht genutzt – Rückgabe "n.v."
|
||||||
return "n.v."
|
return "n.v."
|
||||||
|
|
||||||
def map_internal_technicians(value):
|
def map_internal_technicians(value):
|
||||||
@@ -288,6 +291,7 @@ def search_linkedin_contact(company_name, website, position_query):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e))
|
debug_print("Fehler beim Lesen des SerpAPI-Schlüssels: " + str(e))
|
||||||
return None
|
return None
|
||||||
|
# Falls vorhanden, könnte hier auch die Kurzform (Spalte C) verwendet werden
|
||||||
search_name = company_name
|
search_name = company_name
|
||||||
query = f'site:linkedin.com/in "{position_query}" "{search_name}"'
|
query = f'site:linkedin.com/in "{position_query}" "{search_name}"'
|
||||||
debug_print(f"Erstelle LinkedIn-Query: {query}")
|
debug_print(f"Erstelle LinkedIn-Query: {query}")
|
||||||
@@ -391,8 +395,8 @@ def process_verification_only():
|
|||||||
Verarbeitet jeweils Config.BATCH_SIZE Einträge, bei denen noch keine Wiki-Verifizierung (Spalte S) vorliegt.
|
Verarbeitet jeweils Config.BATCH_SIZE Einträge, bei denen noch keine Wiki-Verifizierung (Spalte S) vorliegt.
|
||||||
Ergebnisse:
|
Ergebnisse:
|
||||||
- Spalte S: Wiki Confirm (OK, falls Artikel passt)
|
- Spalte S: Wiki Confirm (OK, falls Artikel passt)
|
||||||
- Spalte U: Alternative Wiki-URL (falls Artikel nicht passt oder keiner gefunden wurde)
|
- Spalte U: Alternative Wiki URL (falls Artikel unpassend oder keiner gefunden wurde)
|
||||||
- Spalte V: Erklärung (Begründung)
|
- Spalte V: Wiki Erklärung (Begründung)
|
||||||
- Spalte W: Branchenvorschlag (ChatGPT, basierend auf Spalten G, H, O, R)
|
- Spalte W: Branchenvorschlag (ChatGPT, basierend auf Spalten G, H, O, R)
|
||||||
- Spalte Y: Branchenkonsistenz (OK oder X inkl. Begründung)
|
- Spalte Y: Branchenkonsistenz (OK oder X inkl. Begründung)
|
||||||
- Spalte AQ: Token Count des Batch-Prompts (gleich für alle Einträge)
|
- Spalte AQ: Token Count des Batch-Prompts (gleich für alle Einträge)
|
||||||
@@ -408,433 +412,7 @@ def process_verification_only():
|
|||||||
batch_size = Config.BATCH_SIZE
|
batch_size = Config.BATCH_SIZE
|
||||||
batch_entries = []
|
batch_entries = []
|
||||||
row_indices = []
|
row_indices = []
|
||||||
# Prüfe Spalte S (Index 18): wenn leer, verarbeite
|
# Prüfe Spalte S (Index 18) – falls leer, verarbeite Eintrag
|
||||||
for i, row in enumerate(data[1:], start=2):
|
|
||||||
if len(row) <= 19 or row[18].strip() == "":
|
|
||||||
entry_text = _process_verification_row(i, row)
|
|
||||||
batch_entries.append(entry_text)
|
|
||||||
row_indices.append(i)
|
|
||||||
if len(batch_entries) == batch_size:
|
|
||||||
break
|
|
||||||
if not batch_entries:
|
|
||||||
debug_print("Keine Einträge für die Verifizierung gefunden.")
|
|
||||||
return
|
|
||||||
aggregated_prompt = ("Du bist ein Experte in der Verifizierung von Wikipedia-Artikeln für Unternehmen. "
|
|
||||||
"Für jeden der folgenden Einträge prüfe, ob der vorhandene Wikipedia-Artikel (URL, Absatz, Kategorien) plausibel passt. "
|
|
||||||
"Gib für jeden Eintrag das Ergebnis im Format an:\n"
|
|
||||||
"Eintrag <Zeilennummer>: <Antwort>\n"
|
|
||||||
"Dabei gilt:\n"
|
|
||||||
"- Wenn der Artikel passt, antworte mit 'OK'.\n"
|
|
||||||
"- Wenn der Artikel nicht passt, antworte mit 'Alternativer Wikipedia-Artikel vorgeschlagen: <URL> | X | <Begründung>'.\n"
|
|
||||||
"- Falls überhaupt kein Artikel gefunden wurde, antworte mit 'Kein Wikipedia-Eintrag vorhanden.'\n\n")
|
|
||||||
aggregated_prompt += "\n".join(batch_entries)
|
|
||||||
debug_print("Aggregierter Prompt für Verifizierungs-Batch erstellt.")
|
|
||||||
token_count = "n.v."
|
|
||||||
if tiktoken:
|
|
||||||
try:
|
|
||||||
enc = tiktoken.encoding_for_model(Config.TOKEN_MODEL)
|
|
||||||
token_count = len(enc.encode(aggregated_prompt))
|
|
||||||
debug_print(f"Token-Zahl für Batch: {token_count}")
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Token-Counting: {e}")
|
|
||||||
try:
|
|
||||||
with open("api_key.txt", "r") as f:
|
|
||||||
api_key = f.read().strip()
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Lesen des API-Tokens (Verifizierung): {e}")
|
|
||||||
return
|
|
||||||
openai.api_key = api_key
|
|
||||||
try:
|
|
||||||
response = openai.ChatCompletion.create(
|
|
||||||
model=Config.TOKEN_MODEL,
|
|
||||||
messages=[{"role": "system", "content": aggregated_prompt}],
|
|
||||||
temperature=0.0
|
|
||||||
)
|
|
||||||
result = response.choices[0].message.content.strip()
|
|
||||||
debug_print(f"Antwort ChatGPT Verifizierung Batch: {result}")
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler bei der ChatGPT Anfrage für Verifizierung: {e}")
|
|
||||||
return
|
|
||||||
answers = result.split("\n")
|
|
||||||
for idx, row_num in enumerate(row_indices):
|
|
||||||
answer = "k.A."
|
|
||||||
for line in answers:
|
|
||||||
if line.strip().startswith(f"Eintrag {row_num}:"):
|
|
||||||
answer = line.split(":", 1)[1].strip()
|
|
||||||
break
|
|
||||||
# Verarbeitung der Wiki-Verifizierung:
|
|
||||||
# Falls Antwort "OK", schreibe in Spalte S (Wiki Confirm) "OK", Spalte U bleibt leer.
|
|
||||||
# Falls Antwort "Kein Wikipedia-Eintrag vorhanden.", schreibe diesen Text in Spalte U.
|
|
||||||
# Falls Antwort mit "Alternativer Wikipedia-Artikel vorgeschlagen:" beginnt, extrahiere URL und Begründung.
|
|
||||||
if answer.upper() == "OK":
|
|
||||||
wiki_confirm = "OK"
|
|
||||||
alt_article = ""
|
|
||||||
explanation = ""
|
|
||||||
elif answer.startswith("Alternativer Wikipedia-Artikel vorgeschlagen:"):
|
|
||||||
parts = answer.split(":", 1)[1].split("|")
|
|
||||||
alt_article = parts[0].strip() if len(parts) > 0 else "k.A."
|
|
||||||
explanation = parts[2].strip() if len(parts) > 2 else ""
|
|
||||||
wiki_confirm = "X"
|
|
||||||
elif answer.upper() == "KEIN WIKIPEDIA-EINTRAG VORHANDEN.":
|
|
||||||
wiki_confirm = ""
|
|
||||||
alt_article = "Kein Wikipedia-Eintrag vorhanden."
|
|
||||||
explanation = ""
|
|
||||||
else:
|
|
||||||
wiki_confirm = ""
|
|
||||||
alt_article = answer
|
|
||||||
explanation = answer
|
|
||||||
# Schreibe Ergebnisse:
|
|
||||||
main_sheet.update(values=[[wiki_confirm]], range_name=f"S{row_num}")
|
|
||||||
main_sheet.update(values=[[alt_article]], range_name=f"U{row_num}")
|
|
||||||
main_sheet.update(values=[[explanation]], range_name=f"V{row_num}")
|
|
||||||
# Branchenvorschlag: Nutze die Branchenangaben aus Spalte G, H, O, R (Indices 6,7,14,17)
|
|
||||||
crm_branch = row_data[6] if len(row_data) > 6 else "k.A."
|
|
||||||
ext_branch = row_data[7] if len(row_data) > 7 else "k.A."
|
|
||||||
wiki_branch = row_data[14] if len(row_data) > 14 else "k.A."
|
|
||||||
wiki_cats = row_data[17] if len(row_data) > 17 else "k.A."
|
|
||||||
branch_result = evaluate_branche_chatgpt(crm_branch, ext_branch, wiki_branch, wiki_cats)
|
|
||||||
main_sheet.update(values=[[branch_result["branch"]]], range_name=f"W{row_num}")
|
|
||||||
main_sheet.update(values=[[branch_result["consistency"]]], range_name=f"Y{row_num}")
|
|
||||||
# Schreibe Token Count in Spalte AQ
|
|
||||||
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}")
|
|
||||||
# Schreibe Timestamp in Spalte AO und Version in Spalte AP
|
|
||||||
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
||||||
main_sheet.update(values=[[current_dt]], range_name=f"AO{row_num}")
|
|
||||||
main_sheet.update(values=[[Config.VERSION]], range_name=f"AP{row_num}")
|
|
||||||
debug_print(f"Zeile {row_num} verifiziert: Antwort: {answer}")
|
|
||||||
time.sleep(Config.RETRY_DELAY)
|
|
||||||
debug_print("Verifizierungs-Batch abgeschlossen.")
|
|
||||||
|
|
||||||
# ==================== STARTINDEX-FUNKTIONEN ====================
|
|
||||||
class GoogleSheetHandler:
|
|
||||||
def __init__(self):
|
|
||||||
self.sheet = None
|
|
||||||
self.sheet_values = []
|
|
||||||
self._connect()
|
|
||||||
def _connect(self):
|
|
||||||
scope = ["https://www.googleapis.com/auth/spreadsheets"]
|
|
||||||
creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope)
|
|
||||||
self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1
|
|
||||||
self.sheet_values = self.sheet.get_all_values()
|
|
||||||
def get_start_index(self, column_index=40):
|
|
||||||
"""
|
|
||||||
column_index=40 für ChatGPT-Timestamp (Spalte AO),
|
|
||||||
column_index=41 für alternative Runner.
|
|
||||||
"""
|
|
||||||
filled_n = [row[column_index] if len(row) > column_index else '' for row in self.sheet_values[1:]]
|
|
||||||
return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1)
|
|
||||||
|
|
||||||
# ==================== ALIGNMENT DEMO (Modus 3) ====================
|
|
||||||
def alignment_demo(sheet):
|
|
||||||
new_headers = [
|
|
||||||
"Spalte A (ReEval Flag)",
|
|
||||||
"Spalte B (Firmenname)",
|
|
||||||
"Spalte C (Kurzform Firmenname)",
|
|
||||||
"Spalte D (Website)",
|
|
||||||
"Spalte E (Ort)",
|
|
||||||
"Spalte F (Beschreibung)",
|
|
||||||
"Spalte G (Aktuelle Branche)",
|
|
||||||
"Spalte H (Beschreibung Branche extern)",
|
|
||||||
"Spalte I (Anzahl Techniker CRM)",
|
|
||||||
"Spalte J (Umsatz CRM)",
|
|
||||||
"Spalte K (Anzahl Mitarbeiter CRM)",
|
|
||||||
"Spalte L (Vorschlag Wiki URL)",
|
|
||||||
"Spalte M (Wikipedia URL)",
|
|
||||||
"Spalte N (Wikipedia Absatz)",
|
|
||||||
"Spalte O (Wikipedia Branche)",
|
|
||||||
"Spalte P (Wikipedia Umsatz)",
|
|
||||||
"Spalte Q (Wikipedia Mitarbeiter)",
|
|
||||||
"Spalte R (Wikipedia Kategorien)",
|
|
||||||
"Spalte S (Wiki Confirm)",
|
|
||||||
"Spalte T (Wiki Erklärung – nicht genutzt)",
|
|
||||||
"Spalte U (Alternative Wiki URL)",
|
|
||||||
"Spalte V (Wiki Begründung)",
|
|
||||||
"Spalte W (Branchenvorschlag)",
|
|
||||||
"Spalte X (Branchenergebnis – Konsistenzprüfung)",
|
|
||||||
"Spalte Y (Branchenerklärung)",
|
|
||||||
"Spalte Z (Timestamp – wird nicht genutzt)",
|
|
||||||
"Spalte AA (FSM Relevanz – nicht genutzt)",
|
|
||||||
"Spalte AB (Begründung FSM – nicht genutzt)",
|
|
||||||
"Spalte AC (Schätzung Mitarbeiter – nicht genutzt)",
|
|
||||||
"Spalte AD (Einschätzung Techniker – nicht genutzt)",
|
|
||||||
"Spalte AE (Begründung Techniker – nicht genutzt)",
|
|
||||||
"Spalte AF (Schätzung Umsatz ChatGPT – nicht genutzt)",
|
|
||||||
"Spalte AG (Begründung Umsatz ChatGPT – nicht genutzt)",
|
|
||||||
"Spalte AH (Wikipedia-Timestamp – nicht genutzt)",
|
|
||||||
"Spalte AI (ChatGPT-Timestamp – nicht genutzt)",
|
|
||||||
"Spalte AJ (Kontakt: Serviceleiter gefunden)",
|
|
||||||
"Spalte AK (Kontakt: IT-Leiter gefunden)",
|
|
||||||
"Spalte AL (Kontakt: Management gefunden)",
|
|
||||||
"Spalte AM (Kontakt: Disponent gefunden)",
|
|
||||||
"Spalte AN (Contact Search Timestamp – nicht genutzt)",
|
|
||||||
"Spalte AO (Verifizierung Timestamp)",
|
|
||||||
"Spalte AP (Version)",
|
|
||||||
"Spalte AQ (Token Count Batch)"
|
|
||||||
]
|
|
||||||
header_range = "A11200:AQ11200"
|
|
||||||
sheet.update(values=[new_headers], range_name=header_range)
|
|
||||||
print("Alignment-Demo abgeschlossen: Neue Spaltenüberschriften in Zeile 11200 geschrieben.")
|
|
||||||
|
|
||||||
# ==================== WIKIPEDIA SCRAPER ====================
|
|
||||||
class WikipediaScraper:
|
|
||||||
def __init__(self):
|
|
||||||
wikipedia.set_lang(Config.LANG)
|
|
||||||
def _get_full_domain(self, website):
|
|
||||||
if not website:
|
|
||||||
return ""
|
|
||||||
website = website.lower().strip()
|
|
||||||
website = re.sub(r'^https?:\/\/', '', website)
|
|
||||||
website = re.sub(r'^www\.', '', website)
|
|
||||||
return website.split('/')[0]
|
|
||||||
def _generate_search_terms(self, company_name, website):
|
|
||||||
terms = []
|
|
||||||
full_domain = self._get_full_domain(website)
|
|
||||||
if full_domain:
|
|
||||||
terms.append(full_domain)
|
|
||||||
normalized_name = normalize_company_name(company_name)
|
|
||||||
candidate = " ".join(normalized_name.split()[:2]).strip()
|
|
||||||
if candidate and candidate not in terms:
|
|
||||||
terms.append(candidate)
|
|
||||||
if normalized_name and normalized_name not in terms:
|
|
||||||
terms.append(normalized_name)
|
|
||||||
debug_print(f"Generierte Suchbegriffe: {terms}")
|
|
||||||
return terms
|
|
||||||
def _validate_article(self, page, company_name, website):
|
|
||||||
full_domain = self._get_full_domain(website)
|
|
||||||
domain_found = False
|
|
||||||
if full_domain:
|
|
||||||
try:
|
|
||||||
html_raw = requests.get(page.url).text
|
|
||||||
soup = BeautifulSoup(html_raw, Config.HTML_PARSER)
|
|
||||||
infobox = soup.find('table', class_=lambda c: c and 'infobox' in c.lower())
|
|
||||||
if infobox:
|
|
||||||
links = infobox.find_all('a', href=True)
|
|
||||||
for link in links:
|
|
||||||
href = link.get('href').lower()
|
|
||||||
if href.startswith('/wiki/datei:'):
|
|
||||||
continue
|
|
||||||
if full_domain in href:
|
|
||||||
debug_print(f"Definitiver Link-Match in Infobox gefunden: {href}")
|
|
||||||
domain_found = True
|
|
||||||
break
|
|
||||||
if not domain_found and hasattr(page, 'externallinks'):
|
|
||||||
for ext_link in page.externallinks:
|
|
||||||
if full_domain in ext_link.lower():
|
|
||||||
debug_print(f"Definitiver Link-Match in externen Links gefunden: {ext_link}")
|
|
||||||
domain_found = True
|
|
||||||
break
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Extrahieren von Links: {str(e)}")
|
|
||||||
normalized_title = normalize_company_name(page.title)
|
|
||||||
normalized_company = normalize_company_name(company_name)
|
|
||||||
similarity = SequenceMatcher(None, normalized_title, normalized_company).ratio()
|
|
||||||
debug_print(f"Ähnlichkeit (normalisiert): {similarity:.2f} ({normalized_title} vs {normalized_company})")
|
|
||||||
threshold = 0.60 if domain_found else Config.SIMILARITY_THRESHOLD
|
|
||||||
return similarity >= threshold
|
|
||||||
def extract_first_paragraph(self, page_url):
|
|
||||||
try:
|
|
||||||
response = requests.get(page_url)
|
|
||||||
soup = BeautifulSoup(response.text, Config.HTML_PARSER)
|
|
||||||
paragraphs = soup.find_all('p')
|
|
||||||
for p in paragraphs:
|
|
||||||
text = clean_text(p.get_text())
|
|
||||||
if len(text) > 50:
|
|
||||||
return text
|
|
||||||
return "k.A."
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Fehler beim Extrahieren des ersten Absatzes: {e}")
|
|
||||||
return "k.A."
|
|
||||||
def extract_categories(self, soup):
|
|
||||||
cat_div = soup.find('div', id="mw-normal-catlinks")
|
|
||||||
if cat_div:
|
|
||||||
ul = cat_div.find('ul')
|
|
||||||
if ul:
|
|
||||||
cats = [clean_text(li.get_text()) for li in ul.find_all('li')]
|
|
||||||
return ", ".join(cats)
|
|
||||||
return "k.A."
|
|
||||||
def _extract_infobox_value(self, soup, target):
|
|
||||||
infobox = soup.find('table', class_=lambda c: c and any(kw in c.lower() for kw in ['infobox', 'vcard', 'unternehmen']))
|
|
||||||
if not infobox:
|
|
||||||
return "k.A."
|
|
||||||
keywords_map = {
|
|
||||||
'branche': ['branche', 'industrie', 'tätigkeit', 'geschäftsfeld', 'sektor', 'produkte', 'leistungen', 'aktivitäten', 'wirtschaftszweig'],
|
|
||||||
'umsatz': ['umsatz', 'jahresumsatz', 'konzernumsatz', 'gesamtumsatz', 'erlöse', 'umsatzerlöse', 'einnahmen', 'ergebnis', 'jahresergebnis'],
|
|
||||||
'mitarbeiter': ['mitarbeiter', 'beschäftigte', 'personal', 'mitarbeiterzahl', 'angestellte', 'belegschaft', 'personalstärke']
|
|
||||||
}
|
|
||||||
keywords = keywords_map.get(target, [])
|
|
||||||
for row in infobox.find_all('tr'):
|
|
||||||
header = row.find('th')
|
|
||||||
if header:
|
|
||||||
header_text = clean_text(header.get_text()).lower()
|
|
||||||
if any(kw in header_text for kw in keywords):
|
|
||||||
value = row.find('td')
|
|
||||||
if value:
|
|
||||||
raw_value = clean_text(value.get_text())
|
|
||||||
if target == 'branche':
|
|
||||||
clean_val = re.sub(r'\[.*?\]|\(.*?\)', '', raw_value)
|
|
||||||
return ' '.join(clean_val.split()).strip()
|
|
||||||
if target == 'umsatz':
|
|
||||||
return extract_numeric_value(raw_value, is_umsatz=True)
|
|
||||||
if target == 'mitarbeiter':
|
|
||||||
return extract_numeric_value(raw_value, is_umsatz=False)
|
|
||||||
return "k.A."
|
|
||||||
def extract_full_infobox(self, soup):
|
|
||||||
infobox = soup.find('table', class_=lambda c: c and any(kw in c.lower() for kw in ['infobox', 'vcard', 'unternehmen']))
|
|
||||||
if not infobox:
|
|
||||||
return "k.A."
|
|
||||||
return clean_text(infobox.get_text(separator=' | '))
|
|
||||||
def extract_fields_from_infobox_text(self, infobox_text, field_names):
|
|
||||||
result = {}
|
|
||||||
tokens = [token.strip() for token in infobox_text.split("|") if token.strip()]
|
|
||||||
for i, token in enumerate(tokens):
|
|
||||||
for field in field_names:
|
|
||||||
if field.lower() in token.lower():
|
|
||||||
j = i + 1
|
|
||||||
while j < len(tokens) and not tokens[j]:
|
|
||||||
j += 1
|
|
||||||
result[field] = tokens[j] if j < len(tokens) else "k.A."
|
|
||||||
return result
|
|
||||||
def extract_company_data(self, page_url):
|
|
||||||
if not page_url:
|
|
||||||
return {
|
|
||||||
'url': 'k.A.',
|
|
||||||
'first_paragraph': 'k.A.',
|
|
||||||
'branche': 'k.A.',
|
|
||||||
'umsatz': 'k.A.',
|
|
||||||
'mitarbeiter': 'k.A.',
|
|
||||||
'categories': 'k.A.',
|
|
||||||
'full_infobox': 'k.A.'
|
|
||||||
}
|
|
||||||
try:
|
|
||||||
response = requests.get(page_url)
|
|
||||||
soup = BeautifulSoup(response.text, Config.HTML_PARSER)
|
|
||||||
full_infobox = self.extract_full_infobox(soup)
|
|
||||||
extracted_fields = self.extract_fields_from_infobox_text(full_infobox, ['Branche', 'Umsatz', 'Mitarbeiter'])
|
|
||||||
raw_branche = extracted_fields.get('Branche', self._extract_infobox_value(soup, 'branche'))
|
|
||||||
raw_umsatz = extracted_fields.get('Umsatz', self._extract_infobox_value(soup, 'umsatz'))
|
|
||||||
raw_mitarbeiter = extracted_fields.get('Mitarbeiter', self._extract_infobox_value(soup, 'mitarbeiter'))
|
|
||||||
umsatz_val = extract_numeric_value(raw_umsatz, is_umsatz=True)
|
|
||||||
mitarbeiter_val = extract_numeric_value(raw_mitarbeiter, is_umsatz=False)
|
|
||||||
categories_val = self.extract_categories(soup)
|
|
||||||
first_paragraph = self.extract_first_paragraph(page_url)
|
|
||||||
return {
|
|
||||||
'url': page_url,
|
|
||||||
'first_paragraph': first_paragraph,
|
|
||||||
'branche': raw_branche,
|
|
||||||
'umsatz': umsatz_val,
|
|
||||||
'mitarbeiter': mitarbeiter_val,
|
|
||||||
'categories': categories_val,
|
|
||||||
'full_infobox': full_infobox
|
|
||||||
}
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Extraktionsfehler: {str(e)}")
|
|
||||||
return {
|
|
||||||
'url': 'k.A.',
|
|
||||||
'first_paragraph': 'k.A.',
|
|
||||||
'branche': 'k.A.',
|
|
||||||
'umsatz': 'k.A.',
|
|
||||||
'mitarbeiter': 'k.A.',
|
|
||||||
'categories': 'k.A.',
|
|
||||||
'full_infobox': 'k.A.'
|
|
||||||
}
|
|
||||||
@retry_on_failure
|
|
||||||
def search_company_article(self, company_name, website):
|
|
||||||
search_terms = self._generate_search_terms(company_name, website)
|
|
||||||
for term in search_terms:
|
|
||||||
try:
|
|
||||||
results = wikipedia.search(term, results=Config.WIKIPEDIA_SEARCH_RESULTS)
|
|
||||||
debug_print(f"Suchergebnisse für '{term}': {results}")
|
|
||||||
for title in results:
|
|
||||||
try:
|
|
||||||
page = wikipedia.page(title, auto_suggest=False)
|
|
||||||
if self._validate_article(page, company_name, website):
|
|
||||||
return page
|
|
||||||
except (wikipedia.exceptions.DisambiguationError, wikipedia.exceptions.PageError) as e:
|
|
||||||
debug_print(f"Seitenfehler: {str(e)}")
|
|
||||||
continue
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Suchfehler: {str(e)}")
|
|
||||||
continue
|
|
||||||
return None
|
|
||||||
|
|
||||||
# ==================== GOOGLE SHEET HANDLER ====================
|
|
||||||
class GoogleSheetHandler:
|
|
||||||
def __init__(self):
|
|
||||||
self.sheet = None
|
|
||||||
self.sheet_values = []
|
|
||||||
self._connect()
|
|
||||||
def _connect(self):
|
|
||||||
scope = ["https://www.googleapis.com/auth/spreadsheets"]
|
|
||||||
creds = ServiceAccountCredentials.from_json_keyfile_name(Config.CREDENTIALS_FILE, scope)
|
|
||||||
self.sheet = gspread.authorize(creds).open_by_url(Config.SHEET_URL).sheet1
|
|
||||||
self.sheet_values = self.sheet.get_all_values()
|
|
||||||
def get_start_index(self, column_index=40):
|
|
||||||
filled_n = [row[column_index] if len(row) > column_index else '' for row in self.sheet_values[1:]]
|
|
||||||
return next((i + 1 for i, v in enumerate(filled_n, start=1) if not str(v).strip()), len(filled_n) + 1)
|
|
||||||
|
|
||||||
# ==================== ALIGNMENT DEMO (Modus 3) ====================
|
|
||||||
def alignment_demo(sheet):
|
|
||||||
new_headers = [
|
|
||||||
"Spalte A (ReEval Flag)",
|
|
||||||
"Spalte B (Firmenname)",
|
|
||||||
"Spalte C (Kurzform Firmenname)",
|
|
||||||
"Spalte D (Website)",
|
|
||||||
"Spalte E (Ort)",
|
|
||||||
"Spalte F (Beschreibung)",
|
|
||||||
"Spalte G (Aktuelle Branche)",
|
|
||||||
"Spalte H (Beschreibung Branche extern)",
|
|
||||||
"Spalte I (Anzahl Techniker CRM)",
|
|
||||||
"Spalte J (Umsatz CRM)",
|
|
||||||
"Spalte K (Anzahl Mitarbeiter CRM)",
|
|
||||||
"Spalte L (Vorschlag Wiki URL)",
|
|
||||||
"Spalte M (Wikipedia URL)",
|
|
||||||
"Spalte N (Wikipedia Absatz)",
|
|
||||||
"Spalte O (Wikipedia Branche)",
|
|
||||||
"Spalte P (Wikipedia Umsatz)",
|
|
||||||
"Spalte Q (Wikipedia Mitarbeiter)",
|
|
||||||
"Spalte R (Wikipedia Kategorien)",
|
|
||||||
"Spalte S (Wiki Confirm)",
|
|
||||||
"Spalte T (Wiki Erklärung – nicht genutzt)",
|
|
||||||
"Spalte U (Alternative Wiki URL)",
|
|
||||||
"Spalte V (Wiki Begründung)",
|
|
||||||
"Spalte W (Branchenvorschlag)",
|
|
||||||
"Spalte X (Branchenkonsistenz)",
|
|
||||||
"Spalte Y (Branchenerklärung)",
|
|
||||||
"Spalte Z (nicht genutzt)",
|
|
||||||
"Spalte AA (FSM Relevanz – nicht genutzt)",
|
|
||||||
"Spalte AB (Begründung FSM – nicht genutzt)",
|
|
||||||
"Spalte AC (Schätzung Mitarbeiter – nicht genutzt)",
|
|
||||||
"Spalte AD (Einschätzung Techniker – nicht genutzt)",
|
|
||||||
"Spalte AE (Begründung Techniker – nicht genutzt)",
|
|
||||||
"Spalte AF (Schätzung Umsatz ChatGPT – nicht genutzt)",
|
|
||||||
"Spalte AG (Begründung Umsatz ChatGPT – nicht genutzt)",
|
|
||||||
"Spalte AH (Wikipedia-Timestamp – nicht genutzt)",
|
|
||||||
"Spalte AI (ChatGPT-Timestamp – nicht genutzt)",
|
|
||||||
"Spalte AJ (Kontakt: Serviceleiter gefunden)",
|
|
||||||
"Spalte AK (Kontakt: IT-Leiter gefunden)",
|
|
||||||
"Spalte AL (Kontakt: Management gefunden)",
|
|
||||||
"Spalte AM (Kontakt: Disponent gefunden)",
|
|
||||||
"Spalte AN (Contact Search Timestamp – nicht genutzt)",
|
|
||||||
"Spalte AO (Verifizierung Timestamp)",
|
|
||||||
"Spalte AP (Version)",
|
|
||||||
"Spalte AQ (Token Count Batch)"
|
|
||||||
]
|
|
||||||
header_range = "A11200:AQ11200"
|
|
||||||
sheet.update(values=[new_headers], range_name=header_range)
|
|
||||||
print("Alignment-Demo abgeschlossen: Neue Spaltenüberschriften in Zeile 11200 geschrieben.")
|
|
||||||
|
|
||||||
# ==================== MODUS 51: VERIFIZIERUNG (BATCH) ====================
|
|
||||||
def process_verification_only():
|
|
||||||
debug_print("Starte Verifizierungsmodus (Modus 51) im Batch-Prozess...")
|
|
||||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
|
||||||
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
|
||||||
sh = gc.open_by_url(Config.SHEET_URL)
|
|
||||||
main_sheet = sh.sheet1
|
|
||||||
data = main_sheet.get_all_values()
|
|
||||||
batch_size = Config.BATCH_SIZE
|
|
||||||
batch_entries = []
|
|
||||||
row_indices = []
|
|
||||||
# Prüfe Spalte S (Index 18): wenn leer, verarbeite den Eintrag
|
|
||||||
for i, row in enumerate(data[1:], start=2):
|
for i, row in enumerate(data[1:], start=2):
|
||||||
if len(row) <= 19 or row[18].strip() == "":
|
if len(row) <= 19 or row[18].strip() == "":
|
||||||
entry_text = _process_verification_row(i, row)
|
entry_text = _process_verification_row(i, row)
|
||||||
@@ -888,10 +466,6 @@ def process_verification_only():
|
|||||||
if line.strip().startswith(f"Eintrag {row_num}:"):
|
if line.strip().startswith(f"Eintrag {row_num}:"):
|
||||||
answer = line.split(":", 1)[1].strip()
|
answer = line.split(":", 1)[1].strip()
|
||||||
break
|
break
|
||||||
# Verarbeitung der Wiki-Verifizierung:
|
|
||||||
# Falls Antwort "OK": In Spalte S (Wiki Confirm) "OK", Spalte U leer.
|
|
||||||
# Falls Antwort "Kein Wikipedia-Eintrag vorhanden.": In Spalte U dieser Text, Spalte S bleibt leer.
|
|
||||||
# Falls Antwort mit "Alternativer Wikipedia-Artikel vorgeschlagen:" beginnt, extrahiere URL und Begründung.
|
|
||||||
if answer.upper() == "OK":
|
if answer.upper() == "OK":
|
||||||
wiki_confirm = "OK"
|
wiki_confirm = "OK"
|
||||||
alt_article = ""
|
alt_article = ""
|
||||||
@@ -909,11 +483,9 @@ def process_verification_only():
|
|||||||
wiki_confirm = ""
|
wiki_confirm = ""
|
||||||
alt_article = answer
|
alt_article = answer
|
||||||
wiki_explanation = answer
|
wiki_explanation = answer
|
||||||
# Schreibe Ergebnisse für Wiki-Verifizierung:
|
|
||||||
main_sheet.update(values=[[wiki_confirm]], range_name=f"S{row_num}")
|
main_sheet.update(values=[[wiki_confirm]], range_name=f"S{row_num}")
|
||||||
main_sheet.update(values=[[alt_article]], range_name=f"U{row_num}")
|
main_sheet.update(values=[[alt_article]], range_name=f"U{row_num}")
|
||||||
main_sheet.update(values=[[wiki_explanation]], range_name=f"V{row_num}")
|
main_sheet.update(values=[[wiki_explanation]], range_name=f"V{row_num}")
|
||||||
# Branchenvorschlag: Nutze Branchenangaben aus Spalte G (Index 6), H (7), O (14) und R (17)
|
|
||||||
crm_branch = data[row_num-1][6] if len(data[row_num-1]) > 6 else "k.A."
|
crm_branch = data[row_num-1][6] if len(data[row_num-1]) > 6 else "k.A."
|
||||||
ext_branch = data[row_num-1][7] if len(data[row_num-1]) > 7 else "k.A."
|
ext_branch = data[row_num-1][7] if len(data[row_num-1]) > 7 else "k.A."
|
||||||
wiki_branch = data[row_num-1][14] if len(data[row_num-1]) > 14 else "k.A."
|
wiki_branch = data[row_num-1][14] if len(data[row_num-1]) > 14 else "k.A."
|
||||||
@@ -921,9 +493,7 @@ def process_verification_only():
|
|||||||
branch_result = evaluate_branche_chatgpt(crm_branch, ext_branch, wiki_branch, wiki_cats)
|
branch_result = evaluate_branche_chatgpt(crm_branch, ext_branch, wiki_branch, wiki_cats)
|
||||||
main_sheet.update(values=[[branch_result["branch"]]], range_name=f"W{row_num}")
|
main_sheet.update(values=[[branch_result["branch"]]], range_name=f"W{row_num}")
|
||||||
main_sheet.update(values=[[branch_result["consistency"]]], range_name=f"Y{row_num}")
|
main_sheet.update(values=[[branch_result["consistency"]]], range_name=f"Y{row_num}")
|
||||||
# Schreibe Token Count in Spalte AQ
|
|
||||||
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}")
|
main_sheet.update(values=[[str(token_count)]], range_name=f"AQ{row_num}")
|
||||||
# Schreibe Timestamp in Spalte AO und Version in Spalte AP
|
|
||||||
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
current_dt = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
main_sheet.update(values=[[current_dt]], range_name=f"AO{row_num}")
|
main_sheet.update(values=[[current_dt]], range_name=f"AO{row_num}")
|
||||||
main_sheet.update(values=[[Config.VERSION]], range_name=f"AP{row_num}")
|
main_sheet.update(values=[[Config.VERSION]], range_name=f"AP{row_num}")
|
||||||
@@ -961,7 +531,7 @@ def process_contact_research():
|
|||||||
|
|
||||||
# ==================== NEUER MODUS: CONTACTS (LinkedIn) ====================
|
# ==================== NEUER MODUS: CONTACTS (LinkedIn) ====================
|
||||||
def process_contacts():
|
def process_contacts():
|
||||||
debug_print("Starte LinkedIn-Kontaktsuche...")
|
debug_print("Starte LinkedIn-Kontaktsuche (Modus 7)...")
|
||||||
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
gc = gspread.authorize(ServiceAccountCredentials.from_json_keyfile_name(
|
||||||
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
Config.CREDENTIALS_FILE, ["https://www.googleapis.com/auth/spreadsheets"]))
|
||||||
sh = gc.open_by_url(Config.SHEET_URL)
|
sh = gc.open_by_url(Config.SHEET_URL)
|
||||||
@@ -1032,13 +602,13 @@ if __name__ == "__main__":
|
|||||||
processor.process_rows()
|
processor.process_rows()
|
||||||
elif MODE == "4":
|
elif MODE == "4":
|
||||||
gh = GoogleSheetHandler()
|
gh = GoogleSheetHandler()
|
||||||
start_index = gh.get_start_index(39) # Wiki-Timestamp (Spalte AN)
|
start_index = gh.get_start_index(38) # Wiki-Timestamp: Spalte AN (Index 38)
|
||||||
debug_print(f"Wiki-Modus: Starte bei Zeile {start_index+1}")
|
debug_print(f"Wiki-Modus: Starte bei Zeile {start_index+1}")
|
||||||
processor = DataProcessor()
|
processor = DataProcessor()
|
||||||
processor.process_rows()
|
processor.process_rows()
|
||||||
elif MODE == "5":
|
elif MODE == "5":
|
||||||
gh = GoogleSheetHandler()
|
gh = GoogleSheetHandler()
|
||||||
start_index = gh.get_start_index(40) # ChatGPT-Timestamp (Spalte AO)
|
start_index = gh.get_start_index(39) # ChatGPT-Timestamp: Spalte AO (Index 39)
|
||||||
debug_print(f"ChatGPT-Modus: Starte bei Zeile {start_index+1}")
|
debug_print(f"ChatGPT-Modus: Starte bei Zeile {start_index+1}")
|
||||||
processor = DataProcessor()
|
processor = DataProcessor()
|
||||||
processor.process_rows()
|
processor.process_rows()
|
||||||
|
|||||||
Reference in New Issue
Block a user