[30388f42] Infrastructure Hardening: Repaired CE/Connector DB schema, fixed frontend styling build, implemented robust echo shield in worker v2.1.1, and integrated Lead Engine into gateway.

lead-engine/ingest.py

@@ -1,4 +1,5 @@
 import re
+from datetime import datetime
 from db import insert_lead
 
 def parse_tradingtwins_email(body):
@@ -28,6 +29,108 @@ def parse_tradingtwins_email(body):
     data['raw_body'] = body
     return data
 
+def is_free_mail(email_addr):
+    """Checks if an email belongs to a known free-mail provider."""
+    if not email_addr: return False
+    free_domains = {
+        'gmail.com', 'googlemail.com', 'outlook.com', 'hotmail.com', 'live.com', 
+        'msn.com', 'icloud.com', 'me.com', 'mac.com', 'yahoo.com', 'ymail.com', 
+        'rocketmail.com', 'gmx.de', 'gmx.net', 'web.de', 't-online.de', 
+        'freenet.de', 'mail.com', 'protonmail.com', 'proton.me', 'online.de'
+    }
+    domain = email_addr.split('@')[-1].lower()
+    return domain in free_domains
+
+def parse_tradingtwins_html(html_body):
+    """
+    Extracts data from the Tradingtwins HTML table structure.
+    Pattern: <p ...>Label:</p>...<p ...>Value</p>
+    """
+    data = {}
+    
+    # Map label names in HTML to our keys
+    field_map = {
+        'Firma': 'company',
+        'Vorname': 'contact_first', 
+        'Nachname': 'contact_last', 
+        'Anrede': 'salutation',
+        'E-Mail': 'email',
+        'Rufnummer': 'phone',
+        'Einsatzzweck': 'purpose', 
+        'Reinigungs-Funktionen': 'cleaning_functions',
+        'Reinigungs-Fläche': 'area', 
+        'PLZ': 'zip',
+        'Stadt': 'city',
+        'Lead-ID': 'source_id' 
+    }
+    
+    for label, key in field_map.items():
+        pattern = fr'>\s*{re.escape(label)}:\s*</p>.*?<p[^>]*>(.*?)</p>'
+        match = re.search(pattern, html_body, re.DOTALL | re.IGNORECASE)
+        if match:
+            raw_val = match.group(1).strip()
+            clean_val = re.sub(r'<[^>]+>', '', raw_val).strip()
+            data[key] = clean_val
+            
+    # Composite fields
+    if data.get('contact_first') and data.get('contact_last'):
+        data['contact'] = f"{data['contact_first']} {data['contact_last']}"
+    
+    # Quality Check: Free mail or missing company
+    email = data.get('email', '')
+    company = data.get('company', '-')
+    
+    data['is_free_mail'] = is_free_mail(email)
+    data['is_low_quality'] = data['is_free_mail'] or company == '-' or not company
+    
+    # Ensure source_id is present and map to 'id' for db.py compatibility
+    if not data.get('source_id'):
+         data['source_id'] = f"tt_unknown_{int(datetime.now().timestamp())}"
+    
+    data['id'] = data['source_id'] # db.py expects 'id' for source_id column
+
+    return data
+    
+def parse_roboplanet_form(html_body):
+    """
+    Parses the Roboplanet website contact form (HTML format).
+    Example: <b>Vorname:</b> Gordana <br><b>Nachname:</b> Dumitrovic <br>...
+    """
+    data = {}
+    
+    # Map label names in HTML to our keys
+    field_map = {
+        'Vorname': 'contact_first', 
+        'Nachname': 'contact_last', 
+        'Email': 'email',
+        'Telefon': 'phone',
+        'Firma': 'company',
+        'PLZ': 'zip',
+        'Nachricht': 'message'
+    }
+    
+    for label, key in field_map.items():
+        # Pattern: <b>Label:</b> Value <br>
+        pattern = fr'<b>{re.escape(label)}:</b>\s*(.*?)\s*<br>'
+        match = re.search(pattern, html_body, re.DOTALL | re.IGNORECASE)
+        if match:
+            raw_val = match.group(1).strip()
+            clean_val = re.sub(r'<[^>]+>', '', raw_val).strip() # Clean any leftover HTML tags
+            data[key] = clean_val
+            
+    # Composite fields
+    if data.get('contact_first') and data.get('contact_last'):
+        data['contact'] = f"{data['contact_first']} {data['contact_last']}"
+
+    # For Roboplanet forms, we use the timestamp as ID or a hash if missing
+    # We need to ensure 'id' is present for db.py compatibility
+    if not data.get('source_id'):
+         data['source_id'] = f"rp_unknown_{int(datetime.now().timestamp())}"
+    data['id'] = data['source_id']
+    
+    data['raw_body'] = html_body
+    return data
+
 def ingest_mock_leads():
     # Mock data from the session context
     leads = [