mal schauen, was gemini draus macht
This commit is contained in:
@@ -22,10 +22,12 @@ class TempConfig:
|
||||
# ==============================================================================
|
||||
|
||||
OUTPUT_DIR = "/app/output"
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)-8s - %(name)-25s - %(message)s', force=True)
|
||||
LOG_FORMAT = '%(asctime)s - %(levelname)-8s - %(name)-25s - %(message)s' # <-- DEFINITION HIER
|
||||
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, force=True)
|
||||
logging.getLogger("selenium").setLevel(logging.WARNING)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
log_filename = f"dealfront_run_{time.strftime('%Y%m%d-%H%M%S')}.txt"
|
||||
log_filepath = os.path.join(OUTPUT_DIR, log_filename)
|
||||
try:
|
||||
@@ -82,91 +84,67 @@ class DealfrontScraper:
|
||||
logger.error(f"Konnte Debug-Artefakte nicht speichern: {e}")
|
||||
|
||||
def login_and_find_list(self, search_name):
|
||||
# ... (Diese Methode bleibt unverändert, verwendet aber jetzt TempConfig) ...
|
||||
try:
|
||||
logger.info(f"Navigiere zur Login-Seite: {TempConfig.DEALFRONT_LOGIN_URL}")
|
||||
self.driver.get(TempConfig.DEALFRONT_LOGIN_URL)
|
||||
self.wait.until(EC.visibility_of_element_located((By.NAME, "email"))).send_keys(self.username)
|
||||
self.driver.find_element(By.CSS_SELECTOR, "input[type='password']").send_keys(self.password)
|
||||
self.driver.find_element(By.XPATH, "//button[normalize-space()='Log in']").click()
|
||||
logger.info("Login-Befehl gesendet.")
|
||||
logger.info("Warte auf Dashboard und den 'Prospects finden' Quick-Link...")
|
||||
logger.info("Login-Befehl gesendet. Warte auf Navigation...")
|
||||
|
||||
prospects_link_selector = (By.XPATH, "//a[@data-test-target-product-tile]")
|
||||
prospects_link = self.wait.until(EC.element_to_be_clickable(prospects_link_selector))
|
||||
prospects_link.click()
|
||||
logger.info("'Prospects finden' geklickt.")
|
||||
logger.info(f"Warte auf die Liste der Suchen und klicke auf '{search_name}'...")
|
||||
logger.info("'Prospects finden' geklickt. Lade Suchliste...")
|
||||
|
||||
search_item_selector = (By.XPATH, f"//div[contains(@class, 'truncate') and normalize-space()='{search_name}']")
|
||||
search_item = self.wait.until(EC.element_to_be_clickable(search_item_selector))
|
||||
search_item.click()
|
||||
logger.info(f"Suche '{search_name}' geladen. Warte auf das Rendern der Ergebnistabelle.")
|
||||
first_row_locator = (By.CSS_SELECTOR, ".sticky-column a.t-highlight-text")
|
||||
self.wait.until(
|
||||
EC.visibility_of_element_located(first_row_locator)
|
||||
)
|
||||
time.sleep(5)
|
||||
self.wait.until(EC.element_to_be_clickable(search_item_selector)).click()
|
||||
|
||||
logger.info(f"Suche '{search_name}' geladen. Warte auf Ergebnisse.")
|
||||
first_row_locator = (By.CSS_SELECTOR, "table#t-result-table tbody tr[id]")
|
||||
self.wait.until(EC.visibility_of_element_located(first_row_locator))
|
||||
logger.info("Zielseite mit Ergebnissen erfolgreich erreicht.")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.critical(f"Der Prozess ist fehlgeschlagen: {type(e).__name__}", exc_info=True)
|
||||
logger.critical(f"Prozess bis zum Laden der Liste fehlgeschlagen: {type(e).__name__}", exc_info=True)
|
||||
self._save_debug_artifacts()
|
||||
return False
|
||||
|
||||
def extract_current_page_results(self):
|
||||
# 1) Kurz Implicit-Wait absenken
|
||||
self.driver.implicitly_wait(1)
|
||||
|
||||
# 2) Auf das erste Daten-Element warten
|
||||
first_row_locator = (By.CSS_SELECTOR, ".sticky-column a.t-highlight-text")
|
||||
self.wait.until(EC.visibility_of_element_located(first_row_locator))
|
||||
time.sleep(1)
|
||||
|
||||
"""
|
||||
Extrahiert Firmennamen und Webseiten zeilenbasiert für maximale Zuverlässigkeit.
|
||||
"""
|
||||
try:
|
||||
logger.info("Extrahiere Ergebnisse von der aktuellen Seite...")
|
||||
logger.info("Extrahiere Ergebnisse von der aktuellen Seite (zeilenbasierter Ansatz)...")
|
||||
results = []
|
||||
|
||||
# 3) Warten auf mindestens eine Tabellen-Zeile
|
||||
|
||||
rows_selector = (By.CSS_SELECTOR, "table#t-result-table tbody tr[id]")
|
||||
self.wait.until(EC.presence_of_all_elements_located(rows_selector))
|
||||
# Wir warten nur kurz, da die `login_and_find_list` das Laden schon bestätigt hat.
|
||||
time.sleep(2)
|
||||
rows = self.driver.find_elements(*rows_selector)
|
||||
logger.info(f"{len(rows)} Firmen-Zeilen gefunden.")
|
||||
|
||||
for i, row in enumerate(rows, 1):
|
||||
# Name-Extraktion (bewährter Selector)
|
||||
name_elems = row.find_elements(By.CSS_SELECTOR, ".sticky-column a.t-highlight-text")
|
||||
if not name_elems:
|
||||
logger.warning(f"Zeile {i}: Kein Name-Element gefunden. Überspringe.")
|
||||
try:
|
||||
# Innerhalb jeder Zeile (row) suchen wir jetzt die spezifischen Elemente.
|
||||
company_name_element = row.find_element(By.CSS_SELECTOR, ".sticky-column a.t-highlight-text")
|
||||
company_name = company_name_element.get_attribute("title").strip()
|
||||
|
||||
# Die Website ist in der dritten Zelle (td). Wenn es keinen Link gibt, ist die Zelle trotzdem da.
|
||||
website_cell = row.find_element(By.CSS_SELECTOR, "td:nth-of-type(3)")
|
||||
website = website_cell.text.strip() # .text gibt immer den sichtbaren Text zurück, auch wenn kein <a>-Tag da ist.
|
||||
|
||||
results.append({'name': company_name, 'website': website or "N/A"}) # Fallback für leere Websites
|
||||
except NoSuchElementException:
|
||||
logger.warning(f"Zeile {i}: Wichtige Elemente nicht gefunden. Überspringe.")
|
||||
continue
|
||||
name_elem = name_elems[0]
|
||||
company_name = (name_elem.get_attribute("title") or name_elem.text).strip()
|
||||
|
||||
# Website-Extraktion aus 3. Spalte (externe Links mit target="_blank")
|
||||
web_ext = row.find_elements(By.CSS_SELECTOR, "td:nth-of-type(3) a[target='_blank']")
|
||||
if web_ext:
|
||||
href = web_ext[0].get_attribute("href")
|
||||
# ohne Protokoll und Slash am Ende
|
||||
website = href.split("://")[-1].rstrip("/")
|
||||
else:
|
||||
# Fallback 1: Link ohne target (manchmal vorhanden)
|
||||
web_any = row.find_elements(By.CSS_SELECTOR, "td:nth-of-type(3) a")
|
||||
if web_any:
|
||||
# Domain steht im title-Attribut oder im Text
|
||||
website = (web_any[0].get_attribute("title") or web_any[0].text).strip()
|
||||
else:
|
||||
# Fallback 2: reiner Zellen-Text
|
||||
cell = row.find_elements(By.CSS_SELECTOR, "td:nth-of-type(3)")
|
||||
website = cell[0].text.strip() if cell else ""
|
||||
|
||||
|
||||
|
||||
results.append({'name': company_name, 'website': website})
|
||||
|
||||
logger.info(f"Extraktion abgeschlossen: {len(results)} Firmen.")
|
||||
|
||||
logger.info(f"Extraktion abgeschlossen. {len(results)} Firmen verarbeitet.")
|
||||
return results
|
||||
|
||||
finally:
|
||||
# 4) Implicit-Wait auf Standard zurücksetzen
|
||||
self.driver.implicitly_wait(10)
|
||||
except Exception as e:
|
||||
logger.error(f"Fehler bei der Extraktion: {type(e).__name__}", exc_info=True)
|
||||
self._save_debug_artifacts()
|
||||
return []
|
||||
|
||||
|
||||
def close(self):
|
||||
|
||||
Reference in New Issue
Block a user