Floke/Brancheneinstufung2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dealfront_enrichment.py aktualisiert

Browse Source
This commit is contained in:
Floke
2025-07-09 12:52:15 +00:00
parent 5fa4e559b7
commit 9c3bd2d5c1
1 changed files with 42 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
dealfront_enrichment.py
View File

@@ -14,13 +14,14 @@ from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
# ──────────────────────────────────────────────────────────────── # ────────────────────────────────────────────────────────────────
# Konstanten # Einstellung
LOGIN_URL = "https://app.dealfront.com/login" LOGIN_URL = "https://app.dealfront.com/login"
SEARCH_NAME = "Facility Management" TARGET_PAGE = "https://app.dealfront.com/target"
CREDS_FILE = "dealfront_credentials.json" SEARCH_NAME = "Facility Management"
OUTPUT_DIR = "output" CREDS_FILE = "dealfront_credentials.json"
CHROMEDRIVER_PATH = "/usr/bin/chromedriver" OUTPUT_DIR = "output"
LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" CHROMEDRIVER_PATH = "/usr/bin/chromedriver"
LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
# ──────────────────────────────────────────────────────────────── # ────────────────────────────────────────────────────────────────
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, force=True) logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, force=True)
@@ -32,49 +33,54 @@ def load_creds(path):
logger.error("Credentials-Datei nicht gefunden: %s", path) logger.error("Credentials-Datei nicht gefunden: %s", path)
sys.exit(1) sys.exit(1)
with open(path, encoding="utf-8") as f: with open(path, encoding="utf-8") as f:
creds = json.load(f) j = json.load(f)
u = creds.get("username") u, p = j.get("username"), j.get("password")
p = creds.get("password")
if not u or not p: if not u or not p:
logger.error("Username oder Passwort fehlt in %s", path) logger.error("username/password fehlen in %s", path)
sys.exit(1) sys.exit(1)
return u, p return u, p
class DealfrontScraper: class DealfrontScraper:
def __init__(self, driver, wait, user, pwd): def __init__(self, driver, wait, user, pwd):
self.driver = driver self.driver = driver
self.wait = wait self.wait = wait
self.user = user self.user = user
self.pwd = pwd self.pwd = pwd
def login_and_select_search(self): def login_and_load_target(self):
# 1) Login # 1) Login
self.driver.get(LOGIN_URL) self.driver.get(LOGIN_URL)
self.wait.until(EC.visibility_of_element_located( self.wait.until(EC.visibility_of_element_located(
(By.CSS_SELECTOR, "input[type='email'], input[type='text']") (By.CSS_SELECTOR, "input[type='email'],input[type='text']")
)) ))
self.driver.find_element(By.CSS_SELECTOR, "input[type='email'], input[type='text']").send_keys(self.user) self.driver.find_element(By.CSS_SELECTOR, "input[type='email'],input[type='text']").send_keys(self.user)
self.driver.find_element(By.CSS_SELECTOR, "input[type='password']").send_keys(self.pwd) self.driver.find_element(By.CSS_SELECTOR, "input[type='password']").send_keys(self.pwd)
self.driver.find_element(By.CSS_SELECTOR, "button[type='submit']").click() self.driver.find_element(By.CSS_SELECTOR, "button[type='submit']").click()
# 2) Auf Sidebar-Liste warten # 2) Warten bis URL sich ändert (Login abgeschlossen)
ul_selector = "ul[data-userpilot-id='sidebar-searches-list']" self.wait.until(lambda d: d.current_url != LOGIN_URL)
self.wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ul_selector)))
# 3) Genaue Suche anklicken über das <div title="…"> # 3) Direkt zur Target-Seite navigieren
xpath = f"//ul[@data-userpilot-id='sidebar-searches-list']//div[@title='{SEARCH_NAME}']" self.driver.get(TARGET_PAGE)
elem = self.wait.until(EC.element_to_be_clickable((By.XPATH, xpath)))
# JS-Click, damit alle Listener feuern # 4) Auf die Sidebar mit deinen vordefinierten Suchen warten
sidebar_xpath = "//ul[contains(@class,'sidebar-tree-view')]"
self.wait.until(EC.visibility_of_element_located((By.XPATH, sidebar_xpath)))
# 5) Deine Suche anklicken über das div[@title]
search_xpath = f"//ul[contains(@class,'sidebar-tree-view')]//div[@title='{SEARCH_NAME}']"
elem = self.wait.until(EC.element_to_be_clickable((By.XPATH, search_xpath)))
self.driver.execute_script("arguments[0].click();", elem) self.driver.execute_script("arguments[0].click();", elem)
# 4) Auf erste Datenzeile warten # 6) Auf das erste Daten-Element warten, bevor wir extrahieren
first = (By.CSS_SELECTOR, ".sticky-column a.t-highlight-text") first = (By.CSS_SELECTOR, ".sticky-column a.t-highlight-text")
self.wait.until(EC.visibility_of_element_located(first)) self.wait.until(EC.visibility_of_element_located(first))
time.sleep(1) time.sleep(1)
def extract_page(self): def extract_current_page_results(self):
# kurz Implicit-Wait 1 s, damit find_elements nicht blocken # kurz Implicit-Wait absenken
self.driver.implicitly_wait(1) self.driver.implicitly_wait(1)
rows_sel = (By.CSS_SELECTOR, "table#t-result-table tbody tr[id]") rows_sel = (By.CSS_SELECTOR, "table#t-result-table tbody tr[id]")
self.wait.until(EC.presence_of_all_elements_located(rows_sel)) self.wait.until(EC.presence_of_all_elements_located(rows_sel))
rows = self.driver.find_elements(*rows_sel) rows = self.driver.find_elements(*rows_sel)
@@ -95,17 +101,18 @@ class DealfrontScraper:
out.append({"name": name, "website": site}) out.append({"name": name, "website": site})
# reset Implicit-Wait # Implicit-Wait zurücksetzen
self.driver.implicitly_wait(10) self.driver.implicitly_wait(10)
return out return out
def click_next(self): def click_next_page(self):
btns = self.driver.find_elements(By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button") btns = self.driver.find_elements(By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button")
if not btns: if not btns:
return False return False
nxt = btns[-1] nxt = btns[-1]
if not nxt.is_enabled() or "disabled" in nxt.get_attribute("class"): if not nxt.is_enabled() or "disabled" in nxt.get_attribute("class"):
return False return False
current = self.driver.find_element( current = self.driver.find_element(
By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button.active" By.CSS_SELECTOR, "nav.eb-pagination a.eb-pagination-button.active"
).text ).text
@@ -116,18 +123,18 @@ class DealfrontScraper:
return True return True
def run(self): def run(self):
self.login_and_select_search() self.login_and_load_target()
all_data = [] all_data = []
while True: while True:
all_data.extend(self.extract_page()) all_data.extend(self.extract_current_page_results())
if not self.click_next(): if not self.click_next_page():
break break
return all_data return all_data
def main(): def main():
user, pwd = load_creds(CREDS_FILE) user, pwd = load_creds(CREDS_FILE)
opts = Options() opts = Options()
opts.add_argument("--headless") opts.add_argument("--headless")
opts.add_argument("--no-sandbox") opts.add_argument("--no-sandbox")
opts.add_argument("--disable-dev-shm-usage") opts.add_argument("--disable-dev-shm-usage")
@@ -146,7 +153,7 @@ def main():
with open(out, "w", encoding="utf-8") as f: with open(out, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2) json.dump(data, f, ensure_ascii=False, indent=2)
print(f"✅ Fertig: {len(data)} Einträge in {out}") logger.info("✅ Fertig: %d Einträge in %s", len(data), out)
if __name__ == "__main__": if __name__ == "__main__":
main() main()