diff --git a/dealfront_enrichment.py b/dealfront_enrichment.py index 6a24b5b3..9b410f37 100644 --- a/dealfront_enrichment.py +++ b/dealfront_enrichment.py @@ -14,13 +14,14 @@ from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC # ──────────────────────────────────────────────────────────────── -# Konstanten -LOGIN_URL = "https://app.dealfront.com/login" -SEARCH_NAME = "Facility Management" -CREDS_FILE = "dealfront_credentials.json" -OUTPUT_DIR = "output" -CHROMEDRIVER_PATH = "/usr/bin/chromedriver" -LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" +# Einstellung +LOGIN_URL = "https://app.dealfront.com/login" +TARGET_PAGE = "https://app.dealfront.com/target" +SEARCH_NAME = "Facility Management" +CREDS_FILE = "dealfront_credentials.json" +OUTPUT_DIR = "output" +CHROMEDRIVER_PATH = "/usr/bin/chromedriver" +LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" # ──────────────────────────────────────────────────────────────── logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, force=True) @@ -32,49 +33,54 @@ def load_creds(path): logger.error("Credentials-Datei nicht gefunden: %s", path) sys.exit(1) with open(path, encoding="utf-8") as f: - creds = json.load(f) - u = creds.get("username") - p = creds.get("password") + j = json.load(f) + u, p = j.get("username"), j.get("password") if not u or not p: - logger.error("Username oder Passwort fehlt in %s", path) + logger.error("username/password fehlen in %s", path) sys.exit(1) return u, p class DealfrontScraper: def __init__(self, driver, wait, user, pwd): - self.driver = driver - self.wait = wait - self.user = user - self.pwd = pwd + self.driver = driver + self.wait = wait + self.user = user + self.pwd = pwd - def login_and_select_search(self): + def login_and_load_target(self): # 1) Login self.driver.get(LOGIN_URL) self.wait.until(EC.visibility_of_element_located( - (By.CSS_SELECTOR, "input[type='email'], input[type='text']") + (By.CSS_SELECTOR, "input[type='email'],input[type='text']") )) - self.driver.find_element(By.CSS_SELECTOR, "input[type='email'], input[type='text']").send_keys(self.user) + self.driver.find_element(By.CSS_SELECTOR, "input[type='email'],input[type='text']").send_keys(self.user) self.driver.find_element(By.CSS_SELECTOR, "input[type='password']").send_keys(self.pwd) self.driver.find_element(By.CSS_SELECTOR, "button[type='submit']").click() - # 2) Auf Sidebar-Liste warten - ul_selector = "ul[data-userpilot-id='sidebar-searches-list']" - self.wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ul_selector))) + # 2) Warten bis URL sich ändert (Login abgeschlossen) + self.wait.until(lambda d: d.current_url != LOGIN_URL) - # 3) Genaue Suche anklicken über das
- xpath = f"//ul[@data-userpilot-id='sidebar-searches-list']//div[@title='{SEARCH_NAME}']" - elem = self.wait.until(EC.element_to_be_clickable((By.XPATH, xpath))) - # JS-Click, damit alle Listener feuern + # 3) Direkt zur Target-Seite navigieren + self.driver.get(TARGET_PAGE) + + # 4) Auf die Sidebar mit deinen vordefinierten Suchen warten + sidebar_xpath = "//ul[contains(@class,'sidebar-tree-view')]" + self.wait.until(EC.visibility_of_element_located((By.XPATH, sidebar_xpath))) + + # 5) Deine Suche anklicken über das div[@title] + search_xpath = f"//ul[contains(@class,'sidebar-tree-view')]//div[@title='{SEARCH_NAME}']" + elem = self.wait.until(EC.element_to_be_clickable((By.XPATH, search_xpath))) self.driver.execute_script("arguments[0].click();", elem) - # 4) Auf erste Datenzeile warten + # 6) Auf das erste Daten-Element warten, bevor wir extrahieren first = (By.CSS_SELECTOR, ".sticky-column a.t-highlight-text") self.wait.until(EC.visibility_of_element_located(first)) time.sleep(1) - def extract_page(self): - # kurz Implicit-Wait 1 s, damit find_elements nicht blocken + def extract_current_page_results(self): + # kurz Implicit-Wait absenken self.driver.implicitly_wait(1) + rows_sel = (By.CSS_SELECTOR, "table#t-result-table tbody tr[id]") self.wait.until(EC.presence_of_all_elements_located(rows_sel)) rows = self.driver.find_elements(*rows_sel) @@ -95,17 +101,18 @@ class DealfrontScraper: out.append({"name": name, "website": site}) - # reset Implicit-Wait + # Implicit-Wait zurücksetzen self.driver.implicitly_wait(10) return out - def click_next(self): + def click_next_page(self): btns = self.driver.find_elements(By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button") if not btns: return False nxt = btns[-1] if not nxt.is_enabled() or "disabled" in nxt.get_attribute("class"): return False + current = self.driver.find_element( By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button.active" ).text @@ -116,18 +123,18 @@ class DealfrontScraper: return True def run(self): - self.login_and_select_search() + self.login_and_load_target() all_data = [] while True: - all_data.extend(self.extract_page()) - if not self.click_next(): + all_data.extend(self.extract_current_page_results()) + if not self.click_next_page(): break return all_data def main(): user, pwd = load_creds(CREDS_FILE) - opts = Options() + opts = Options() opts.add_argument("--headless") opts.add_argument("--no-sandbox") opts.add_argument("--disable-dev-shm-usage") @@ -146,7 +153,7 @@ def main(): with open(out, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) - print(f"✅ Fertig: {len(data)} Einträge in {out}") + logger.info("✅ Fertig: %d Einträge in %s", len(data), out) if __name__ == "__main__": main()