Floke
efaa43858d
This commit integrates the Roboplanet website contact form submissions into the Lead Engine, allowing them to be processed alongside TradingTwins leads.
Key changes:
- **Database Schema Update (db.py):** Added a new source column to the leads table for tracking lead origin (TradingTwins or Website-Formular). Includes a migration check to safely add the column.
- **Improved HTML Parsing (ingest.py):** Refined the `parse_roboplanet_form` function to accurately extract data from the specific HTML structure of Roboplanet contact form emails.
- **Enhanced Ingestion Logic (trading_twins_ingest.py):**
- Renamed `fetch_tradingtwins_emails` to `fetch_new_leads_emails` and updated it to fetch emails from both lead sources.
- Modified `process_leads` to dynamically select the correct parser based on email subject.
- Ensured `source` field is correctly populated and `is_low_quality` checks are applied for both lead types.
- **UI Enhancement (app.py):** Updated the Streamlit UI to visually distinguish lead types with icons and improved the "Low Quality Lead" warning message.
This feature enables a unified processing pipeline for different lead sources and provides better visibility in the Lead Engine dashboard.
116 lines
3.6 KiB
Python
116 lines
3.6 KiB
Python
import re
|
|
from db import insert_lead
|
|
|
|
def parse_tradingtwins_email(body):
|
|
data = {}
|
|
|
|
# Simple regex extraction based on the email format
|
|
patterns = {
|
|
'id': r'Lead-ID:\s*(\d+)',
|
|
'company': r'Firma:\s*(.+)',
|
|
'contact_first': r'Vorname:\s*(.+)',
|
|
'contact_last': r'Nachname:\s*(.+)',
|
|
'email': r'E-Mail:\s*([^\s<]+)',
|
|
'phone': r'Rufnummer:\s*([^\n]+)',
|
|
'area': r'Reinigungs-Flche:\s*([^\n]+)',
|
|
'purpose': r'Einsatzzweck:\s*([^\n]+)'
|
|
}
|
|
|
|
for key, pattern in patterns.items():
|
|
match = re.search(pattern, body)
|
|
if match:
|
|
data[key] = match.group(1).strip()
|
|
|
|
# Combine names
|
|
if 'contact_first' in data and 'contact_last' in data:
|
|
data['contact'] = f"{data['contact_first']} {data['contact_last']}"
|
|
|
|
data['raw_body'] = body
|
|
return data
|
|
|
|
def parse_roboplanet_form(html_body):
|
|
"""
|
|
Parses the Roboplanet website contact form (HTML format).
|
|
Example: <b>Vorname:</b> Gordana <br><b>Nachname:</b> Dumitrovic <br>...
|
|
"""
|
|
data = {}
|
|
|
|
# Map label names in HTML to our keys
|
|
field_map = {
|
|
'Vorname': 'contact_first',
|
|
'Nachname': 'contact_last',
|
|
'Email': 'email',
|
|
'Telefon': 'phone',
|
|
'Firma': 'company',
|
|
'PLZ': 'zip',
|
|
'Nachricht': 'message'
|
|
}
|
|
|
|
for label, key in field_map.items():
|
|
# Pattern: <b>Label:</b> Value <br>
|
|
pattern = fr'<b>{re.escape(label)}:</b>\s*(.*?)\s*<br>'
|
|
match = re.search(pattern, html_body, re.DOTALL | re.IGNORECASE)
|
|
if match:
|
|
raw_val = match.group(1).strip()
|
|
clean_val = re.sub(r'<[^>]+>', '', raw_val).strip() # Clean any leftover HTML tags
|
|
data[key] = clean_val
|
|
|
|
# Composite fields
|
|
if data.get('contact_first') and data.get('contact_last'):
|
|
data['contact'] = f"{data['contact_first']} {data['contact_last']}"
|
|
|
|
# For Roboplanet forms, we use the timestamp as ID or a hash if missing
|
|
# We need to ensure 'id' is present for db.py compatibility
|
|
if not data.get('source_id'):
|
|
data['source_id'] = f"rp_unknown_{int(datetime.now().timestamp())}"
|
|
data['id'] = data['source_id']
|
|
|
|
data['raw_body'] = html_body
|
|
return data
|
|
|
|
def ingest_mock_leads():
|
|
# Mock data from the session context
|
|
leads = [
|
|
{
|
|
'id': '2397256',
|
|
'company': 'pronorm Einbauküchen GmbH',
|
|
'contact': 'Jakob Funk',
|
|
'email': 'jakob.funk@pronorm.de',
|
|
'phone': '+49 5733 979175',
|
|
'raw_body': """
|
|
Lead-ID: 2397256
|
|
Firma: pronorm Einbauküchen GmbH
|
|
Vorname: Jakob
|
|
Nachname: Funk
|
|
Reinigungs-Flche: 1.001 - 10.000 qm
|
|
Einsatzzweck: Reinigung von Böden
|
|
"""
|
|
},
|
|
{
|
|
'id': '2414364',
|
|
'company': 'Quad & Rollershop Schwabmünchen GmbH',
|
|
'contact': 'Manfred Bihler',
|
|
'email': 'Rollershop.Schwabmuenchen@web.de',
|
|
'phone': '+49 8232 905246',
|
|
'raw_body': """
|
|
Lead-ID: 2414364
|
|
Firma: Quad & Rollershop Schwabmünchen GmbH
|
|
Vorname: Manfred
|
|
Nachname: Bihler
|
|
Reinigungs-Flche: 301 - 1.000 qm
|
|
Einsatzzweck: Reinigung von Böden
|
|
"""
|
|
}
|
|
]
|
|
|
|
count = 0
|
|
for lead in leads:
|
|
if insert_lead(lead):
|
|
count += 1
|
|
return count
|
|
|
|
if __name__ == "__main__":
|
|
from db import init_db
|
|
init_db()
|
|
print(f"Ingested {ingest_mock_leads()} new leads.")
|