[31388f42] Add SerpAPI debugging to UI and relax dotenv loading

lead-engine/app.py

@@ -2,9 +2,29 @@ import streamlit as st
 import pandas as pd
 from db import get_leads, init_db
 import json
+import re
+import os
 from enrich import run_sync, refresh_ce_data
 from generate_reply import generate_email_draft
 
+def clean_html_to_text(html_content):
+    """Simple helper to convert HTML email body to readable plain text."""
+    if not html_content:
+        return ""
+    # Remove head and style tags entirely
+    clean = re.sub(r'<head.*?>.*?</head>', '', html_content, flags=re.DOTALL | re.IGNORECASE)
+    clean = re.sub(r'<style.*?>.*?</style>', '', clean, flags=re.DOTALL | re.IGNORECASE)
+    # Replace <br> and </p> with newlines
+    clean = re.sub(r'<br\s*/?>', '\n', clean, flags=re.IGNORECASE)
+    clean = re.sub(r'</p>', '\n', clean, flags=re.IGNORECASE)
+    # Remove all other tags
+    clean = re.sub(r'<.*?>', '', clean)
+    # Decode some common entities
+    clean = clean.replace('&nbsp;', ' ').replace('&amp;', '&').replace('&quot;', '"')
+    # Cleanup multiple newlines
+    clean = re.sub(r'\n\s*\n+', '\n\n', clean).strip()
+    return clean
+
 st.set_page_config(page_title="TradingTwins Lead Engine", layout="wide")
 
 st.title("🚀 Lead Engine: TradingTwins")
@@ -51,6 +71,42 @@ if st.sidebar.button("3. Sync to Company Explorer"):
         except Exception as e:
             st.error(f"Sync Failed: {e}")
 
+if st.sidebar.checkbox("Show System Debug"):
+    st.sidebar.subheader("System Diagnostics")
+    
+    # 1. API Key Check
+    from lookup_role import get_gemini_key
+    key = get_gemini_key()
+    if key:
+        st.sidebar.success(f"Gemini Key found ({key[:5]}...)")
+    else:
+        st.sidebar.error("Gemini Key NOT found!")
+        
+    # 2. SerpAPI Check
+    serp_key = os.getenv("SERP_API")
+    if serp_key:
+        st.sidebar.success(f"SerpAPI Key found ({serp_key[:5]}...)")
+    else:
+        st.sidebar.error("SerpAPI Key NOT found in Env!")
+        
+    # 3. Network Check
+    try:
+        import requests
+        res = requests.get("https://generativelanguage.googleapis.com", timeout=2)
+        st.sidebar.success(f"Gemini API Reachable ({res.status_code})")
+    except Exception as e:
+        st.sidebar.error(f"Network Error: {e}")
+        
+    # 4. Live Lookup Test
+    if st.sidebar.button("Test Role Lookup (Georg Stahl)"):
+        from lookup_role import lookup_person_role
+        with st.sidebar.status("Running Lookup..."):
+            res = lookup_person_role("Georg Stahl", "Klemm Bohrtechnik GmbH")
+            if res:
+                st.sidebar.success(f"Result: {res}")
+            else:
+                st.sidebar.error("Result: None")
+
 # Main View
 leads = get_leads()
 df = pd.DataFrame(leads)
@@ -64,7 +120,16 @@ if not df.empty:
     st.subheader("Lead Pipeline")
     
     for index, row in df.iterrows():
-        with st.expander(f"{row['company_name']} ({row['status']})"):
+        # Format date for title
+        date_str = ""
+        if row.get('received_at'):
+             try:
+                 dt = pd.to_datetime(row['received_at'])
+                 date_str = dt.strftime("%d.%m. %H:%M")
+             except:
+                 pass
+                 
+        with st.expander(f"{date_str} | {row['company_name']} ({row['status']})"):
             c1, c2 = st.columns(2)
             
             # --- Left Column: Lead Data ---
@@ -79,13 +144,29 @@ if not df.empty:
                 except:
                     pass
             
+            role = meta.get('role')
+            if role:
+                c1.info(f"**Role:** {role}")
+            else:
+                if c1.button("🔍 Find Role (SerpAPI)", key=f"role_{row['id']}"):
+                    from enrich import enrich_contact_role
+                    with st.spinner("Searching LinkedIn via Google..."):
+                        found_role = enrich_contact_role(row)
+                        if found_role:
+                            st.success(f"Found: {found_role}")
+                            st.rerun()
+                        else:
+                            st.error("No role found.")
+            
             if meta:
                 c1.write("---")
                 c1.write(f"**Area:** {meta.get('area', '-')}")
                 c1.write(f"**Purpose:** {meta.get('purpose', '-')}")
                 c1.write(f"**Location:** {meta.get('zip', '')} {meta.get('city', '')}")
             
-            with c1.expander("Show Original Email Body"):
+            with c1.expander("Show Original Email Content"):
+                st.text(clean_html_to_text(row['raw_body']))
+                if st.checkbox("Show Raw HTML", key=f"raw_{row['id']}"):
                     st.code(row['raw_body'], language="html")
             
             # --- Right Column: Enrichment & Response ---

lead-engine/lookup_role.py

@@ -3,23 +3,15 @@ import requests
 import re
 from dotenv import load_dotenv
 
-# Load env from root
+# Try loading .env only if file exists (Local Dev), otherwise rely on Docker Env
 env_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '.env'))
-load_dotenv(dotenv_path=env_path, override=True)
+if os.path.exists(env_path):
+    load_dotenv(dotenv_path=env_path, override=True)
 
 SERP_API_KEY = os.getenv("SERP_API")
 
 if not SERP_API_KEY:
-    print(f"DEBUG: Failed to load SERP_API from {env_path}")
+    print(f"DEBUG: SERP_API not found in environment.")
-    # Fallback: try reading directly if file exists
-    try:
-        with open(env_path, 'r') as f:
-            for line in f:
-                if line.startswith('SERP_API='):
-                    SERP_API_KEY = line.split('=')[1].strip().strip('"')
-                    print("DEBUG: Loaded key via manual parsing.")
-    except:
-        pass
 
 import json
 
@@ -75,8 +67,10 @@ def extract_role_with_llm(name, company, search_results):
             # Cleanup: remove punctuation at the end
             role = role.rstrip('.')
             return None if "Unbekannt" in role else role
-    except:
+        else:
-        pass
+            print(f"DEBUG: Gemini API Error {response.status_code}: {response.text}")
+    except Exception as e:
+        print(f"DEBUG: Gemini API Exception: {e}")
     return None
 
 def lookup_person_role(name, company):