dealfront_enrichment.py aktualisiert

dealfront_enrichment.py

@@ -14,8 +14,9 @@ from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 
 # ────────────────────────────────────────────────────────────────
-# Konstanten
+# Einstellung
 LOGIN_URL         = "https://app.dealfront.com/login"
+TARGET_PAGE       = "https://app.dealfront.com/target"
 SEARCH_NAME       = "Facility Management"
 CREDS_FILE        = "dealfront_credentials.json"
 OUTPUT_DIR        = "output"
@@ -32,11 +33,10 @@ def load_creds(path):
         logger.error("Credentials-Datei nicht gefunden: %s", path)
         sys.exit(1)
     with open(path, encoding="utf-8") as f:
-        creds = json.load(f)
+        j = json.load(f)
-    u = creds.get("username")
+    u, p = j.get("username"), j.get("password")
-    p = creds.get("password")
     if not u or not p:
-        logger.error("Username oder Passwort fehlt in %s", path)
+        logger.error("username/password fehlen in %s", path)
         sys.exit(1)
     return u, p
 
@@ -47,34 +47,40 @@ class DealfrontScraper:
         self.user   = user
         self.pwd    = pwd
 
-    def login_and_select_search(self):
+    def login_and_load_target(self):
         # 1) Login
         self.driver.get(LOGIN_URL)
         self.wait.until(EC.visibility_of_element_located(
-            (By.CSS_SELECTOR, "input[type='email'], input[type='text']")
+            (By.CSS_SELECTOR, "input[type='email'],input[type='text']")
         ))
-        self.driver.find_element(By.CSS_SELECTOR, "input[type='email'], input[type='text']").send_keys(self.user)
+        self.driver.find_element(By.CSS_SELECTOR, "input[type='email'],input[type='text']").send_keys(self.user)
         self.driver.find_element(By.CSS_SELECTOR, "input[type='password']").send_keys(self.pwd)
         self.driver.find_element(By.CSS_SELECTOR, "button[type='submit']").click()
 
-        # 2) Auf Sidebar-Liste warten
+        # 2) Warten bis URL sich ändert (Login abgeschlossen)
-        ul_selector = "ul[data-userpilot-id='sidebar-searches-list']"
+        self.wait.until(lambda d: d.current_url != LOGIN_URL)
-        self.wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ul_selector)))
 
-        # 3) Genaue Suche anklicken über das <div title="…">
+        # 3) Direkt zur Target-Seite navigieren
-        xpath = f"//ul[@data-userpilot-id='sidebar-searches-list']//div[@title='{SEARCH_NAME}']"
+        self.driver.get(TARGET_PAGE)
-        elem = self.wait.until(EC.element_to_be_clickable((By.XPATH, xpath)))
+
-        # JS-Click, damit alle Listener feuern
+        # 4) Auf die Sidebar mit deinen vordefinierten Suchen warten
+        sidebar_xpath = "//ul[contains(@class,'sidebar-tree-view')]"
+        self.wait.until(EC.visibility_of_element_located((By.XPATH, sidebar_xpath)))
+
+        # 5) Deine Suche anklicken über das div[@title]
+        search_xpath = f"//ul[contains(@class,'sidebar-tree-view')]//div[@title='{SEARCH_NAME}']"
+        elem = self.wait.until(EC.element_to_be_clickable((By.XPATH, search_xpath)))
         self.driver.execute_script("arguments[0].click();", elem)
 
-        # 4) Auf erste Datenzeile warten
+        # 6) Auf das erste Daten-Element warten, bevor wir extrahieren
         first = (By.CSS_SELECTOR, ".sticky-column a.t-highlight-text")
         self.wait.until(EC.visibility_of_element_located(first))
         time.sleep(1)
 
-    def extract_page(self):
+    def extract_current_page_results(self):
-        # kurz Implicit-Wait 1 s, damit find_elements nicht blocken
+        # kurz Implicit-Wait absenken
         self.driver.implicitly_wait(1)
+
         rows_sel = (By.CSS_SELECTOR, "table#t-result-table tbody tr[id]")
         self.wait.until(EC.presence_of_all_elements_located(rows_sel))
         rows = self.driver.find_elements(*rows_sel)
@@ -95,17 +101,18 @@ class DealfrontScraper:
 
             out.append({"name": name, "website": site})
 
-        # reset Implicit-Wait
+        # Implicit-Wait zurücksetzen
         self.driver.implicitly_wait(10)
         return out
 
-    def click_next(self):
+    def click_next_page(self):
         btns = self.driver.find_elements(By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button")
         if not btns:
             return False
         nxt = btns[-1]
         if not nxt.is_enabled() or "disabled" in nxt.get_attribute("class"):
             return False
+
         current = self.driver.find_element(
             By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button.active"
         ).text
@@ -116,11 +123,11 @@ class DealfrontScraper:
         return True
 
     def run(self):
-        self.login_and_select_search()
+        self.login_and_load_target()
         all_data = []
         while True:
-            all_data.extend(self.extract_page())
+            all_data.extend(self.extract_current_page_results())
-            if not self.click_next():
+            if not self.click_next_page():
                 break
         return all_data
 
@@ -146,7 +153,7 @@ def main():
     with open(out, "w", encoding="utf-8") as f:
         json.dump(data, f, ensure_ascii=False, indent=2)
 
-    print(f"✅ Fertig: {len(data)} Einträge in {out}")
+    logger.info("✅ Fertig: %d Einträge in %s", len(data), out)
 
 if __name__ == "__main__":
     main()