[31388f42] Feature: Integrate Roboplanet Contact Forms into Lead Engine

This commit integrates the Roboplanet website contact form submissions into the Lead Engine, allowing them to be processed alongside TradingTwins leads.

Key changes:
- **Database Schema Update (db.py):** Added a new source column to the leads table for tracking lead origin (TradingTwins or Website-Formular). Includes a migration check to safely add the column.
- **Improved HTML Parsing (ingest.py):** Refined the `parse_roboplanet_form` function to accurately extract data from the specific HTML structure of Roboplanet contact form emails.
- **Enhanced Ingestion Logic (trading_twins_ingest.py):**
    - Renamed `fetch_tradingtwins_emails` to `fetch_new_leads_emails` and updated it to fetch emails from both lead sources.
    - Modified `process_leads` to dynamically select the correct parser based on email subject.
    - Ensured `source` field is correctly populated and `is_low_quality` checks are applied for both lead types.
- **UI Enhancement (app.py):** Updated the Streamlit UI to visually distinguish lead types with icons and improved the "Low Quality Lead" warning message.

This feature enables a unified processing pipeline for different lead sources and provides better visibility in the Lead Engine dashboard.

lead-engine/app.py

@@ -147,7 +147,11 @@ if not df.empty:
              except:
                  pass
                  
-        with st.expander(f"{date_str} | {row['company_name']}"):
+        # --- DYNAMIC TITLE ---
+        source_icon = "🌐" if row.get('source') == 'Website-Formular' else "🤝"
+        title = f"{source_icon} {row.get('source', 'Lead')} | {date_str} | {row['company_name']}"
+                 
+        with st.expander(title):
             # Metadata Parsing
             meta = {}
             if row.get('lead_metadata'):
@@ -155,8 +159,9 @@ if not df.empty:
                 except: pass
             
             # --- TOP SECTION: QUALITY WARNING ---
+            # Now directly checks the metadata from DB, which is more reliable
             if meta.get('is_low_quality'):
-                st.warning("⚠️ **Low Quality Lead detected** (Free-mail or missing company).")
+                st.warning("⚠️ **Low Quality Lead detected** (Free-mail provider or missing company name). Please verify manually.")
 
             # --- SECTION 1: LEAD INFO & INTELLIGENCE ---
             col_lead, col_intel = st.columns(2)

lead-engine/db.py

@@ -35,13 +35,18 @@ def init_db():
         )
     ''')
     
-    # Simple migration check: check if lead_metadata column exists
+    # Simple migration check: add 'lead_metadata' if not exists
-    try:
+    c.execute("PRAGMA table_info(leads)")
-        c.execute('SELECT lead_metadata FROM leads LIMIT 1')
+    columns = [row[1] for row in c.fetchall()]
-    except sqlite3.OperationalError:
+    
+    if 'lead_metadata' not in columns:
         print("Migrating DB: Adding lead_metadata column...")
         c.execute('ALTER TABLE leads ADD COLUMN lead_metadata TEXT')
         
+    if 'source' not in columns:
+        print("Migrating DB: Adding source column...")
+        c.execute('ALTER TABLE leads ADD COLUMN source TEXT')
+        
     conn.commit()
     conn.close()
 
@@ -71,8 +76,8 @@ def insert_lead(lead_data):
     c = conn.cursor()
     try:
         c.execute('''
-            INSERT INTO leads (source_id, received_at, company_name, contact_name, email, phone, raw_body, lead_metadata, status)
+            INSERT INTO leads (source_id, received_at, company_name, contact_name, email, phone, raw_body, lead_metadata, status, source)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         ''', (
             lead_data.get('id'),
             received_at,
@@ -82,7 +87,8 @@ def insert_lead(lead_data):
             lead_data.get('phone'),
             lead_data.get('raw_body'),
             json.dumps(meta),
-            'new'
+            'new',
+            lead_data.get('source') # Added source
         ))
         conn.commit()
         return True

lead-engine/ingest.py

@@ -28,38 +28,44 @@ def parse_tradingtwins_email(body):
     data['raw_body'] = body
     return data
 
-def parse_roboplanet_form(body):
+def parse_roboplanet_form(html_body):
     """
     Parses the Roboplanet website contact form (HTML format).
-    Example: <b>Vorname:</b> BÄKO <br><b>Nachname:</b> eG <br><b>Email:</b> Alexander.Grau@baeko-hr.de ...
+    Example: <b>Vorname:</b> Gordana <br><b>Nachname:</b> Dumitrovic <br>...
     """
     data = {}
     
-    # Helper to strip HTML tags if needed, but we'll use regex on the content
+    # Map label names in HTML to our keys
-    patterns = {
+    field_map = {
-        'contact_first': r'Vorname:</b>\s*(.*?)\s*<br>',
+        'Vorname': 'contact_first', 
-        'contact_last': r'Nachname:</b>\s*(.*?)\s*<br>',
+        'Nachname': 'contact_last', 
-        'email': r'Email:</b>\s*(.*?)\s*<br>',
+        'Email': 'email',
-        'phone': r'Telefon:</b>\s*(.*?)\s*<br>',
+        'Telefon': 'phone',
-        'company': r'Firma:</b>\s*(.*?)\s*<br>',
+        'Firma': 'company',
-        'zip': r'PLZ:</b>\s*(.*?)\s*<br>',
+        'PLZ': 'zip',
-        'message': r'Nachricht:</b>\s*(.*?)\s*(?:<br>|--|$)'
+        'Nachricht': 'message'
     }
     
-    for key, pattern in patterns.items():
+    for label, key in field_map.items():
-        # Use re.DOTALL for message if it spans lines, but usually it's one block
+        # Pattern: <b>Label:</b> Value <br>
-        match = re.search(pattern, body, re.IGNORECASE | re.DOTALL)
+        pattern = fr'<b>{re.escape(label)}:</b>\s*(.*?)\s*<br>'
+        match = re.search(pattern, html_body, re.DOTALL | re.IGNORECASE)
         if match:
-            # Clean HTML tags from the captured value if any
+            raw_val = match.group(1).strip()
-            val = re.sub(r'<.*?>', '', match.group(1)).strip()
+            clean_val = re.sub(r'<[^>]+>', '', raw_val).strip() # Clean any leftover HTML tags
-            data[key] = val
+            data[key] = clean_val
             
-    # Combine names
+    # Composite fields
-    if 'contact_first' in data and 'contact_last' in data:
+    if data.get('contact_first') and data.get('contact_last'):
         data['contact'] = f"{data['contact_first']} {data['contact_last']}"
-    
+
     # For Roboplanet forms, we use the timestamp as ID or a hash if missing
-    data['raw_body'] = body
+    # We need to ensure 'id' is present for db.py compatibility
+    if not data.get('source_id'):
+         data['source_id'] = f"rp_unknown_{int(datetime.now().timestamp())}"
+    data['id'] = data['source_id']
+    
+    data['raw_body'] = html_body
     return data
 
 def ingest_mock_leads():

lead-engine/trading_twins_ingest.py

@@ -10,13 +10,15 @@ from dotenv import load_dotenv
 # Ensure we can import from root directory
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
-# Import db functions
+# Import db functions and parsers
 try:
     from db import insert_lead, init_db
+    from ingest import parse_roboplanet_form, parse_tradingtwins_html, is_free_mail
 except ImportError:
     # Fallback for direct execution
     sys.path.append(os.path.dirname(__file__))
     from db import insert_lead, init_db
+    from ingest import parse_roboplanet_form, parse_tradingtwins_html, is_free_mail
 
 # Configuration
 env_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '.env'))
@@ -43,15 +45,13 @@ def get_access_token():
     response.raise_for_status()
     return response.json().get("access_token")
 
-def fetch_tradingtwins_emails(token, limit=200):
+def fetch_new_leads_emails(token, limit=200):
     url = f"https://graph.microsoft.com/v1.0/users/{USER_EMAIL}/messages"
     headers = {
         "Authorization": f"Bearer {token}",
         "Content-Type": "application/json"
     }
     
-    # Graph API restriction: 'contains' on subject is often blocked.
-    # Strategy: Fetch metadata + body for last 200 messages and filter client-side.
     params = {
         "$top": limit,
         "$select": "id,subject,receivedDateTime,body",
@@ -65,9 +65,11 @@ def fetch_tradingtwins_emails(token, limit=200):
     
     all_msgs = response.json().get("value", [])
     
-    # Filter strictly for the subject pattern locally
+    # Filter client-side for both TradingTwins and Roboplanet contact forms
-    # Handle case where subject might be None
+    filtered = [m for m in all_msgs if (
-    filtered = [m for m in all_msgs if "Neue Anfrage zum Thema Roboter" in (m.get('subject') or '')]
+        "Neue Anfrage zum Thema Roboter" in (m.get('subject') or '') or
+        "Kontaktformular Roboplanet" in (m.get('subject') or '')
+    )]
     return filtered
 
 def is_free_mail(email_addr):
@@ -137,15 +139,14 @@ def process_leads(auto_sync=False):
     new_count = 0
     try:
         token = get_access_token()
-        emails = fetch_tradingtwins_emails(token)
+        emails = fetch_new_leads_emails(token) # Use the new function
-        logger.info(f"Found {len(emails)} Tradingtwins emails.")
+        logger.info(f"Found {len(emails)} potential lead emails.")
         
         for email in emails:
-            # ... (parsing logic remains same)
+            subject = email.get('subject') or ''
             body = email.get('body', {}).get('content', '')
             received_at_str = email.get('receivedDateTime')
             
-            # Convert ISO string to datetime object
             received_at = None
             if received_at_str:
                 try:
@@ -153,22 +154,53 @@ def process_leads(auto_sync=False):
                 except:
                     pass
 
-            lead_data = parse_tradingtwins_html(body)
+            lead_data = {}
+            source_prefix = "unknown"
+            source_display_name = "Unknown"
+
+            if "Neue Anfrage zum Thema Roboter" in subject:
+                lead_data = parse_tradingtwins_html(body)
+                source_prefix = "tt"
+                source_display_name = "TradingTwins"
+            elif "Kontaktformular Roboplanet" in subject:
+                lead_data = parse_roboplanet_form(body)
+                source_prefix = "rp"
+                source_display_name = "Website-Formular"
+            else:
+                # Should not happen with current filtering, but good for robustness
+                logger.warning(f"Skipping unknown email type: {subject}")
+                continue
+            
+            lead_data['source'] = source_display_name # Add the new source field for the DB
             lead_data['raw_body'] = body
             lead_data['received_at'] = received_at
-            
+
+            # Apply general quality checks (if not already done by parser)
+            if 'is_free_mail' not in lead_data: 
+                lead_data['is_free_mail'] = is_free_mail(lead_data.get('email', ''))
+            if 'is_low_quality' not in lead_data:
+                company_name_check = lead_data.get('company', '')
+                # Consider company name '-' as missing/invalid
+                if company_name_check == '-': company_name_check = '' 
+                lead_data['is_low_quality'] = lead_data['is_free_mail'] or not company_name_check
+
             company_name = lead_data.get('company')
             if not company_name or company_name == '-':
+                # Fallback: if company name is missing, use contact name as company
                 company_name = lead_data.get('contact')
                 lead_data['company'] = company_name
             
             if not company_name:
+                logger.warning(f"Skipping lead due to missing company and contact name: {subject}")
                 continue
                 
-            lead_data['id'] = lead_data.get('source_id') or f"tt_{int(datetime.now().timestamp())}"
+            # Ensure source_id and 'id' for db.py compatibility
+            if not lead_data.get('source_id'):
+                 lead_data['source_id'] = f"{source_prefix}_unknown_{int(datetime.now().timestamp())}"
+            lead_data['id'] = lead_data['source_id'] # db.py expects 'id' for source_id column
             
             if insert_lead(lead_data):
-                logger.info(f" -> Ingested: {company_name}")
+                logger.info(f" -> Ingested ({source_prefix}): {company_name}")
                 new_count += 1
         
         if new_count > 0 and auto_sync: