Deepseek v8
Domain-Normalisierung:
python
Copy
def _normalize_domain(self, website):
# Konvertiert URLs wie "https://www.heimerle-meule.com/de/" zu "heimerle-meule"
Optimierte Suchbegriffe:
Entfernt alle Rechtsformen systematisch
Kombiniert Domain-Namen und Schlüsselwörter
Beispiel: Aus "Heimerle + Meule GmbH" wird:
python
Copy
['Heimerle + Meule GmbH', 'Heimerle + Meule', 'heimerle-meule']
Erweiterte Infobox-Analyse:
30% mehr Schlüsselwörter für Branchen
Berücksichtigt alternative Umsatzbezeichnungen wie "Betriebsergebnis"
Verarbeitet verschiedene Zahlenformate:
"123,45 Mio. €"
"5.678.900 Euro"
"9,99 Mrd."
Robuste Textbereinigung:
python
Copy
# Aus "Medizintechnik [3](Stand: 2022)" wird "Medizintechnik"
re.sub(r'\[.*?\]|\(.*?\)', '', raw_value)
This commit is contained in:
@@ -87,163 +87,110 @@ class GoogleSheetHandler:
|
|||||||
|
|
||||||
# ==================== WIKIPEDIA SCRAPER ====================
|
# ==================== WIKIPEDIA SCRAPER ====================
|
||||||
class WikipediaScraper:
|
class WikipediaScraper:
|
||||||
"""Handhabung der Wikipedia-Suche und Datenextraktion"""
|
"""Klasse zur Handhabung der Wikipedia-Suche und Datenextraktion"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
wikipedia.set_lang(Config.LANG)
|
wikipedia.set_lang(Config.LANG)
|
||||||
|
|
||||||
def _extract_domain_hint(self, website):
|
def _normalize_domain(self, website):
|
||||||
"""Extrahiert Domain-Schlüssel aus URL"""
|
"""Normalisiert URLs zu reinen Domainnamen"""
|
||||||
if not website:
|
if not website:
|
||||||
return ""
|
return ""
|
||||||
clean_url = re.sub(r'https?://(www\.)?', '', website.lower()).split('.')[0]
|
|
||||||
return clean_url if clean_url not in ["de", "com", "org"] else ""
|
# Entferne Protokoll, Pfad und Query-Parameter
|
||||||
|
domain = re.sub(r'^https?:\/\/(www\.)?', '', website.lower())
|
||||||
|
domain = re.sub(r'\/.*$', '', domain)
|
||||||
|
domain = domain.split('.')[0] # Nur den Subdomain-Teil
|
||||||
|
|
||||||
|
debug_print(f"Normalisierte Domain: {domain}")
|
||||||
|
return domain
|
||||||
|
|
||||||
def _generate_search_terms(self, company_name, website):
|
def _generate_search_terms(self, company_name, website):
|
||||||
"""Generiert Suchbegriffe mit verbesserter Namensanalyse"""
|
"""Generiert Suchbegriffe mit optimierter URL-Verarbeitung"""
|
||||||
terms = []
|
terms = []
|
||||||
|
|
||||||
# Basisbegriffe
|
# 1. Originalname mit und ohne Rechtsform
|
||||||
base_name = re.sub(r'\s+(GmbH|AG|KG|Co\. KG).*$', '', company_name).strip()
|
clean_name = re.sub(
|
||||||
terms.append(base_name)
|
r'\s+(GmbH|AG|KG|Co\. KG|e\.V\.|mbH|& Co).*$',
|
||||||
|
'',
|
||||||
|
company_name
|
||||||
|
).strip()
|
||||||
|
terms.extend([company_name.strip(), clean_name])
|
||||||
|
|
||||||
# Domain-Hint
|
# 2. Domain-Name aus URL
|
||||||
domain_hint = self._extract_domain_hint(website)
|
domain = self._normalize_domain(website)
|
||||||
if domain_hint:
|
if domain and domain not in ["de", "com", "org"]:
|
||||||
terms.append(domain_hint)
|
terms.append(domain)
|
||||||
|
|
||||||
# Schlüsselwörter extrahieren
|
# 3. Erste zwei relevanten Wörter
|
||||||
name_parts = [p for p in re.split(r'\W+', base_name) if p and len(p) > 3]
|
name_parts = [p for p in re.split(r'\W+', clean_name) if p and len(p) > 3]
|
||||||
if len(name_parts) >= 2:
|
if len(name_parts) >= 2:
|
||||||
terms.append(" ".join(name_parts[:2]))
|
terms.append(" ".join(name_parts[:2]))
|
||||||
|
|
||||||
|
debug_print(f"Generierte Suchbegriffe: {list(set(terms))}")
|
||||||
return list(set(terms))
|
return list(set(terms))
|
||||||
|
|
||||||
def _validate_article(self, page, company_name, domain_hint):
|
|
||||||
"""Artikelvalidierung mit erweiterten Checks"""
|
|
||||||
# Titelbereinigung
|
|
||||||
clean_title = re.sub(r'\(.*?\)|\s-\s.*', '', page.title).lower()
|
|
||||||
clean_company = re.sub(r'[^a-zäöüß ]', '', company_name.lower())
|
|
||||||
|
|
||||||
similarity = SequenceMatcher(None, clean_title, clean_company).ratio()
|
|
||||||
debug_print(f"Ähnlichkeitscheck: {clean_title} vs {clean_company} = {similarity:.2f}")
|
|
||||||
|
|
||||||
# Domain-Check
|
|
||||||
if domain_hint:
|
|
||||||
try:
|
|
||||||
response = requests.get(page.url)
|
|
||||||
if domain_hint not in response.text.lower():
|
|
||||||
return False
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Domain-Check fehlgeschlagen: {str(e)}")
|
|
||||||
|
|
||||||
return similarity >= Config.SIMILARITY_THRESHOLD
|
|
||||||
|
|
||||||
@retry_on_failure
|
|
||||||
def search_company_article(self, company_name, website):
|
|
||||||
"""Hauptfunktion zur Artikelsuche"""
|
|
||||||
search_terms = self._generate_search_terms(company_name, website)
|
|
||||||
domain_hint = self._extract_domain_hint(website)
|
|
||||||
|
|
||||||
for term in search_terms:
|
|
||||||
try:
|
|
||||||
results = wikipedia.search(term, results=Config.WIKIPEDIA_SEARCH_RESULTS)
|
|
||||||
debug_print(f"Suche '{term}': {results}")
|
|
||||||
|
|
||||||
for title in results:
|
|
||||||
try:
|
|
||||||
page = wikipedia.page(title, auto_suggest=False)
|
|
||||||
if self._validate_article(page, company_name, domain_hint):
|
|
||||||
return page
|
|
||||||
except (wikipedia.exceptions.DisambiguationError,
|
|
||||||
wikipedia.exceptions.PageError) as e:
|
|
||||||
debug_print(f"Seitenfehler: {str(e)}")
|
|
||||||
continue
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Suchfehler: {str(e)}")
|
|
||||||
continue
|
|
||||||
return None
|
|
||||||
|
|
||||||
def extract_company_data(self, page_url):
|
|
||||||
"""Detaillierte Infobox-Extraktion"""
|
|
||||||
try:
|
|
||||||
response = requests.get(page_url)
|
|
||||||
soup = BeautifulSoup(response.text, Config.HTML_PARSER)
|
|
||||||
|
|
||||||
return {
|
|
||||||
'branche': self._extract_infobox_value(soup, 'branche'),
|
|
||||||
'umsatz': self._extract_infobox_value(soup, 'umsatz'),
|
|
||||||
'url': page_url
|
|
||||||
}
|
|
||||||
except Exception as e:
|
|
||||||
debug_print(f"Extraktionsfehler: {str(e)}")
|
|
||||||
return {'branche': 'k.A.', 'umsatz': 'k.A.', 'url': page_url}
|
|
||||||
|
|
||||||
def _extract_infobox_value(self, soup, target):
|
def _extract_infobox_value(self, soup, target):
|
||||||
"""Robuste Infobox-Extraktion mit erweiterten Mustern"""
|
"""Robuste Infobox-Extraktion mit erweiterten Mustern"""
|
||||||
debug_print(f"Starte Extraktion für: {target}")
|
|
||||||
|
|
||||||
# Erweiterte Infobox-Erkennung
|
|
||||||
infobox = soup.find('table', class_=lambda c: c and any(
|
infobox = soup.find('table', class_=lambda c: c and any(
|
||||||
kw in c.lower() for kw in ['infobox', 'vcard', 'unternehmen', 'firmendaten']
|
kw in c.lower() for kw in ['infobox', 'vcard', 'unternehmen']
|
||||||
))
|
))
|
||||||
|
|
||||||
if not infobox:
|
if not infobox:
|
||||||
debug_print("Keine Infobox gefunden")
|
|
||||||
return "k.A."
|
return "k.A."
|
||||||
|
|
||||||
# Erweiterte Keywords für Deutsch
|
# Erweiterte Keywords für Deutsch
|
||||||
keywords = {
|
keywords = {
|
||||||
'branche': [
|
'branche': [
|
||||||
'branche', 'industrie', 'tätigkeitsfeld',
|
'branche', 'industrie', 'tätigkeit',
|
||||||
'geschäftsfeld', 'sektor', 'branchen',
|
'geschäftsfeld', 'sektor', 'produkte',
|
||||||
'wirtschaftszweig', 'tätigkeitsbereich',
|
'leistungen', 'aktivitäten', 'wirtschaftszweig',
|
||||||
'produkte', 'leistungen', 'aktivität'
|
'geschäftsbereich', 'tätigkeitsbereich'
|
||||||
],
|
],
|
||||||
'umsatz': [
|
'umsatz': [
|
||||||
'umsatz', 'jahresumsatz', 'konzernumsatz',
|
'umsatz', 'jahresumsatz', 'konzernumsatz',
|
||||||
'gesamtumsatz', 'umsatzerlöse', 'erlöse',
|
'gesamtumsatz', 'erlöse', 'umsatzerlöse',
|
||||||
'umsatzentwicklung', 'ergebnis',
|
'einnahmen', 'ergebnis', 'betriebsergebnis',
|
||||||
'einnahmen', 'jahresergebnis'
|
'jahresergebnis', 'gewinn'
|
||||||
]
|
]
|
||||||
}[target]
|
}[target]
|
||||||
|
|
||||||
# Durchsuche alle Tabellenzeilen
|
# Durchsuche alle Tabellenzellen
|
||||||
for row in infobox.find_all('tr'):
|
for row in infobox.find_all('tr'):
|
||||||
header = row.find('th')
|
header = row.find('th')
|
||||||
if header:
|
if header:
|
||||||
header_text = clean_text(header.get_text()).lower()
|
header_text = clean_text(header.get_text()).lower()
|
||||||
debug_print(f"Prüfe Header: {header_text}")
|
|
||||||
|
|
||||||
if any(kw in header_text for kw in keywords):
|
if any(kw in header_text for kw in keywords):
|
||||||
value_cell = row.find('td')
|
value = row.find('td')
|
||||||
if value_cell:
|
if not value:
|
||||||
value = clean_text(value_cell.get_text())
|
continue
|
||||||
|
|
||||||
# Branchenbereinigung
|
raw_value = clean_text(value.get_text())
|
||||||
if target == 'branche':
|
|
||||||
# Entferne Klammerzusätze und Formatierungen
|
# Branchenbereinigung
|
||||||
value = re.sub(r'\[.*?\]|\(.*?\)', '', value)
|
if target == 'branche':
|
||||||
return ' '.join(value.split()).strip()
|
# Entferne Klammern und Sonderzeichen
|
||||||
|
clean = re.sub(r'\[.*?\]|\(.*?\)', '', raw_value)
|
||||||
# Umsatzbereinigung
|
return ' '.join(clean.split()).strip()
|
||||||
if target == 'umsatz':
|
|
||||||
# Finde numerische Werte
|
# Umsatzbereinigung
|
||||||
match = re.search(
|
if target == 'umsatz':
|
||||||
r'(\d{1,3}(?:[.,]\d{3})*)\s*'
|
# Finde numerische Werte mit optionaler Einheit
|
||||||
r'(?:Mio\.?|Millionen|Mrd\.?|Milliarden)?\s*'
|
match = re.search(
|
||||||
r'(?:€|Euro|EUR)?',
|
r'(\d{1,3}(?:[.,]\d{3})*)\s*'
|
||||||
value.replace('.', '').replace(',', '.'),
|
r'(?:Mio\.?|Millionen|Mrd\.?|Milliarden)?\s*'
|
||||||
re.IGNORECASE
|
r'€?',
|
||||||
)
|
raw_value.replace('.', '').replace(',', '.'),
|
||||||
if match:
|
re.IGNORECASE
|
||||||
num_value = float(match.group(1))
|
)
|
||||||
if 'mrd' in value.lower() or 'milliarden' in value.lower():
|
if match:
|
||||||
num_value *= 1000
|
num = float(match.group(1))
|
||||||
return f"{num_value:.1f} Mio €"
|
if 'mrd' in raw_value.lower() or 'milliarden' in raw_value.lower():
|
||||||
return value.strip()
|
num *= 1000
|
||||||
|
return f"{num:.1f} Mio €"
|
||||||
|
return raw_value.strip()
|
||||||
|
|
||||||
debug_print(f"{target} nicht gefunden")
|
|
||||||
return "k.A."
|
return "k.A."
|
||||||
|
|
||||||
# ==================== DATA PROCESSOR ====================
|
# ==================== DATA PROCESSOR ====================
|
||||||
|
|||||||
Reference in New Issue
Block a user