Deepseek v8
Domain-Normalisierung:
python
Copy
def _normalize_domain(self, website):
# Konvertiert URLs wie "https://www.heimerle-meule.com/de/" zu "heimerle-meule"
Optimierte Suchbegriffe:
Entfernt alle Rechtsformen systematisch
Kombiniert Domain-Namen und Schlüsselwörter
Beispiel: Aus "Heimerle + Meule GmbH" wird:
python
Copy
['Heimerle + Meule GmbH', 'Heimerle + Meule', 'heimerle-meule']
Erweiterte Infobox-Analyse:
30% mehr Schlüsselwörter für Branchen
Berücksichtigt alternative Umsatzbezeichnungen wie "Betriebsergebnis"
Verarbeitet verschiedene Zahlenformate:
"123,45 Mio. €"
"5.678.900 Euro"
"9,99 Mrd."
Robuste Textbereinigung:
python
Copy
# Aus "Medizintechnik [3](Stand: 2022)" wird "Medizintechnik"
re.sub(r'\[.*?\]|\(.*?\)', '', raw_value)
This commit is contained in:
@@ -87,163 +87,110 @@ class GoogleSheetHandler:
|
||||
|
||||
# ==================== WIKIPEDIA SCRAPER ====================
|
||||
class WikipediaScraper:
|
||||
"""Handhabung der Wikipedia-Suche und Datenextraktion"""
|
||||
"""Klasse zur Handhabung der Wikipedia-Suche und Datenextraktion"""
|
||||
|
||||
def __init__(self):
|
||||
wikipedia.set_lang(Config.LANG)
|
||||
|
||||
def _extract_domain_hint(self, website):
|
||||
"""Extrahiert Domain-Schlüssel aus URL"""
|
||||
def _normalize_domain(self, website):
|
||||
"""Normalisiert URLs zu reinen Domainnamen"""
|
||||
if not website:
|
||||
return ""
|
||||
clean_url = re.sub(r'https?://(www\.)?', '', website.lower()).split('.')[0]
|
||||
return clean_url if clean_url not in ["de", "com", "org"] else ""
|
||||
|
||||
# Entferne Protokoll, Pfad und Query-Parameter
|
||||
domain = re.sub(r'^https?:\/\/(www\.)?', '', website.lower())
|
||||
domain = re.sub(r'\/.*$', '', domain)
|
||||
domain = domain.split('.')[0] # Nur den Subdomain-Teil
|
||||
|
||||
debug_print(f"Normalisierte Domain: {domain}")
|
||||
return domain
|
||||
|
||||
def _generate_search_terms(self, company_name, website):
|
||||
"""Generiert Suchbegriffe mit verbesserter Namensanalyse"""
|
||||
"""Generiert Suchbegriffe mit optimierter URL-Verarbeitung"""
|
||||
terms = []
|
||||
|
||||
# Basisbegriffe
|
||||
base_name = re.sub(r'\s+(GmbH|AG|KG|Co\. KG).*$', '', company_name).strip()
|
||||
terms.append(base_name)
|
||||
# 1. Originalname mit und ohne Rechtsform
|
||||
clean_name = re.sub(
|
||||
r'\s+(GmbH|AG|KG|Co\. KG|e\.V\.|mbH|& Co).*$',
|
||||
'',
|
||||
company_name
|
||||
).strip()
|
||||
terms.extend([company_name.strip(), clean_name])
|
||||
|
||||
# Domain-Hint
|
||||
domain_hint = self._extract_domain_hint(website)
|
||||
if domain_hint:
|
||||
terms.append(domain_hint)
|
||||
# 2. Domain-Name aus URL
|
||||
domain = self._normalize_domain(website)
|
||||
if domain and domain not in ["de", "com", "org"]:
|
||||
terms.append(domain)
|
||||
|
||||
# Schlüsselwörter extrahieren
|
||||
name_parts = [p for p in re.split(r'\W+', base_name) if p and len(p) > 3]
|
||||
# 3. Erste zwei relevanten Wörter
|
||||
name_parts = [p for p in re.split(r'\W+', clean_name) if p and len(p) > 3]
|
||||
if len(name_parts) >= 2:
|
||||
terms.append(" ".join(name_parts[:2]))
|
||||
|
||||
debug_print(f"Generierte Suchbegriffe: {list(set(terms))}")
|
||||
return list(set(terms))
|
||||
|
||||
def _validate_article(self, page, company_name, domain_hint):
|
||||
"""Artikelvalidierung mit erweiterten Checks"""
|
||||
# Titelbereinigung
|
||||
clean_title = re.sub(r'\(.*?\)|\s-\s.*', '', page.title).lower()
|
||||
clean_company = re.sub(r'[^a-zäöüß ]', '', company_name.lower())
|
||||
|
||||
similarity = SequenceMatcher(None, clean_title, clean_company).ratio()
|
||||
debug_print(f"Ähnlichkeitscheck: {clean_title} vs {clean_company} = {similarity:.2f}")
|
||||
|
||||
# Domain-Check
|
||||
if domain_hint:
|
||||
try:
|
||||
response = requests.get(page.url)
|
||||
if domain_hint not in response.text.lower():
|
||||
return False
|
||||
except Exception as e:
|
||||
debug_print(f"Domain-Check fehlgeschlagen: {str(e)}")
|
||||
|
||||
return similarity >= Config.SIMILARITY_THRESHOLD
|
||||
|
||||
@retry_on_failure
|
||||
def search_company_article(self, company_name, website):
|
||||
"""Hauptfunktion zur Artikelsuche"""
|
||||
search_terms = self._generate_search_terms(company_name, website)
|
||||
domain_hint = self._extract_domain_hint(website)
|
||||
|
||||
for term in search_terms:
|
||||
try:
|
||||
results = wikipedia.search(term, results=Config.WIKIPEDIA_SEARCH_RESULTS)
|
||||
debug_print(f"Suche '{term}': {results}")
|
||||
|
||||
for title in results:
|
||||
try:
|
||||
page = wikipedia.page(title, auto_suggest=False)
|
||||
if self._validate_article(page, company_name, domain_hint):
|
||||
return page
|
||||
except (wikipedia.exceptions.DisambiguationError,
|
||||
wikipedia.exceptions.PageError) as e:
|
||||
debug_print(f"Seitenfehler: {str(e)}")
|
||||
continue
|
||||
except Exception as e:
|
||||
debug_print(f"Suchfehler: {str(e)}")
|
||||
continue
|
||||
return None
|
||||
|
||||
def extract_company_data(self, page_url):
|
||||
"""Detaillierte Infobox-Extraktion"""
|
||||
try:
|
||||
response = requests.get(page_url)
|
||||
soup = BeautifulSoup(response.text, Config.HTML_PARSER)
|
||||
|
||||
return {
|
||||
'branche': self._extract_infobox_value(soup, 'branche'),
|
||||
'umsatz': self._extract_infobox_value(soup, 'umsatz'),
|
||||
'url': page_url
|
||||
}
|
||||
except Exception as e:
|
||||
debug_print(f"Extraktionsfehler: {str(e)}")
|
||||
return {'branche': 'k.A.', 'umsatz': 'k.A.', 'url': page_url}
|
||||
|
||||
def _extract_infobox_value(self, soup, target):
|
||||
"""Robuste Infobox-Extraktion mit erweiterten Mustern"""
|
||||
debug_print(f"Starte Extraktion für: {target}")
|
||||
|
||||
# Erweiterte Infobox-Erkennung
|
||||
infobox = soup.find('table', class_=lambda c: c and any(
|
||||
kw in c.lower() for kw in ['infobox', 'vcard', 'unternehmen', 'firmendaten']
|
||||
kw in c.lower() for kw in ['infobox', 'vcard', 'unternehmen']
|
||||
))
|
||||
|
||||
if not infobox:
|
||||
debug_print("Keine Infobox gefunden")
|
||||
return "k.A."
|
||||
|
||||
# Erweiterte Keywords für Deutsch
|
||||
keywords = {
|
||||
'branche': [
|
||||
'branche', 'industrie', 'tätigkeitsfeld',
|
||||
'geschäftsfeld', 'sektor', 'branchen',
|
||||
'wirtschaftszweig', 'tätigkeitsbereich',
|
||||
'produkte', 'leistungen', 'aktivität'
|
||||
'branche', 'industrie', 'tätigkeit',
|
||||
'geschäftsfeld', 'sektor', 'produkte',
|
||||
'leistungen', 'aktivitäten', 'wirtschaftszweig',
|
||||
'geschäftsbereich', 'tätigkeitsbereich'
|
||||
],
|
||||
'umsatz': [
|
||||
'umsatz', 'jahresumsatz', 'konzernumsatz',
|
||||
'gesamtumsatz', 'umsatzerlöse', 'erlöse',
|
||||
'umsatzentwicklung', 'ergebnis',
|
||||
'einnahmen', 'jahresergebnis'
|
||||
'gesamtumsatz', 'erlöse', 'umsatzerlöse',
|
||||
'einnahmen', 'ergebnis', 'betriebsergebnis',
|
||||
'jahresergebnis', 'gewinn'
|
||||
]
|
||||
}[target]
|
||||
|
||||
# Durchsuche alle Tabellenzeilen
|
||||
# Durchsuche alle Tabellenzellen
|
||||
for row in infobox.find_all('tr'):
|
||||
header = row.find('th')
|
||||
if header:
|
||||
header_text = clean_text(header.get_text()).lower()
|
||||
debug_print(f"Prüfe Header: {header_text}")
|
||||
|
||||
if any(kw in header_text for kw in keywords):
|
||||
value_cell = row.find('td')
|
||||
if value_cell:
|
||||
value = clean_text(value_cell.get_text())
|
||||
value = row.find('td')
|
||||
if not value:
|
||||
continue
|
||||
|
||||
# Branchenbereinigung
|
||||
if target == 'branche':
|
||||
# Entferne Klammerzusätze und Formatierungen
|
||||
value = re.sub(r'\[.*?\]|\(.*?\)', '', value)
|
||||
return ' '.join(value.split()).strip()
|
||||
|
||||
# Umsatzbereinigung
|
||||
if target == 'umsatz':
|
||||
# Finde numerische Werte
|
||||
match = re.search(
|
||||
r'(\d{1,3}(?:[.,]\d{3})*)\s*'
|
||||
r'(?:Mio\.?|Millionen|Mrd\.?|Milliarden)?\s*'
|
||||
r'(?:€|Euro|EUR)?',
|
||||
value.replace('.', '').replace(',', '.'),
|
||||
re.IGNORECASE
|
||||
)
|
||||
if match:
|
||||
num_value = float(match.group(1))
|
||||
if 'mrd' in value.lower() or 'milliarden' in value.lower():
|
||||
num_value *= 1000
|
||||
return f"{num_value:.1f} Mio €"
|
||||
return value.strip()
|
||||
raw_value = clean_text(value.get_text())
|
||||
|
||||
# Branchenbereinigung
|
||||
if target == 'branche':
|
||||
# Entferne Klammern und Sonderzeichen
|
||||
clean = re.sub(r'\[.*?\]|\(.*?\)', '', raw_value)
|
||||
return ' '.join(clean.split()).strip()
|
||||
|
||||
# Umsatzbereinigung
|
||||
if target == 'umsatz':
|
||||
# Finde numerische Werte mit optionaler Einheit
|
||||
match = re.search(
|
||||
r'(\d{1,3}(?:[.,]\d{3})*)\s*'
|
||||
r'(?:Mio\.?|Millionen|Mrd\.?|Milliarden)?\s*'
|
||||
r'€?',
|
||||
raw_value.replace('.', '').replace(',', '.'),
|
||||
re.IGNORECASE
|
||||
)
|
||||
if match:
|
||||
num = float(match.group(1))
|
||||
if 'mrd' in raw_value.lower() or 'milliarden' in raw_value.lower():
|
||||
num *= 1000
|
||||
return f"{num:.1f} Mio €"
|
||||
return raw_value.strip()
|
||||
|
||||
debug_print(f"{target} nicht gefunden")
|
||||
return "k.A."
|
||||
|
||||
# ==================== DATA PROCESSOR ====================
|
||||
|
||||
Reference in New Issue
Block a user