Brancheneinstufung2/lead-engine/ingest.py

import re
from datetime import datetime
from db import insert_lead

def parse_tradingtwins_email(body):
    data = {}
    
    # Simple regex extraction based on the email format
    patterns = {
        'id': r'Lead-ID:\s*(\d+)',
        'company': r'Firma:\s*(.+)',
        'contact_first': r'Vorname:\s*(.+)',
        'contact_last': r'Nachname:\s*(.+)',
        'email': r'E-Mail:\s*([^\s<]+)',
        'phone': r'Rufnummer:\s*([^\n]+)',
        'area': r'Reinigungs-Flche:\s*([^\n]+)',
        'purpose': r'Einsatzzweck:\s*([^\n]+)'
    }
    
    for key, pattern in patterns.items():
        match = re.search(pattern, body)
        if match:
            data[key] = match.group(1).strip()
            
    # Combine names
    if 'contact_first' in data and 'contact_last' in data:
        data['contact'] = f"{data['contact_first']} {data['contact_last']}"
    
    data['raw_body'] = body
    return data

def is_free_mail(email_addr):
    """Checks if an email belongs to a known free-mail provider."""
    if not email_addr: return False
    free_domains = {
        'gmail.com', 'googlemail.com', 'outlook.com', 'hotmail.com', 'live.com', 
        'msn.com', 'icloud.com', 'me.com', 'mac.com', 'yahoo.com', 'ymail.com', 
        'rocketmail.com', 'gmx.de', 'gmx.net', 'web.de', 't-online.de', 
        'freenet.de', 'mail.com', 'protonmail.com', 'proton.me', 'online.de'
    }
    domain = email_addr.split('@')[-1].lower()
    return domain in free_domains

def parse_tradingtwins_html(html_body):
    """
    Extracts data from the Tradingtwins HTML table structure.
    Pattern: <p ...>Label:</p>...<p ...>Value</p>
    """
    data = {}
    
    # Map label names in HTML to our keys
    field_map = {
        'Firma': 'company',
        'Vorname': 'contact_first', 
        'Nachname': 'contact_last', 
        'Anrede': 'salutation',
        'E-Mail': 'email',
        'Rufnummer': 'phone',
        'Einsatzzweck': 'purpose', 
        'Reinigungs-Funktionen': 'cleaning_functions',
        'Reinigungs-Fläche': 'area', 
        'PLZ': 'zip',
        'Stadt': 'city',
        'Lead-ID': 'source_id' 
    }
    
    for label, key in field_map.items():
        pattern = fr'>\s*{re.escape(label)}:\s*</p>.*?<p[^>]*>(.*?)</p>'
        match = re.search(pattern, html_body, re.DOTALL | re.IGNORECASE)
        if match:
            raw_val = match.group(1).strip()
            clean_val = re.sub(r'<[^>]+>', '', raw_val).strip()
            data[key] = clean_val
            
    # Composite fields
    if data.get('contact_first') and data.get('contact_last'):
        data['contact'] = f"{data['contact_first']} {data['contact_last']}"
    
    # Quality Check: Free mail or missing company
    email = data.get('email', '')
    company = data.get('company', '-')
    
    data['is_free_mail'] = is_free_mail(email)
    data['is_low_quality'] = data['is_free_mail'] or company == '-' or not company
    
    # Ensure source_id is present and map to 'id' for db.py compatibility
    if not data.get('source_id'):
         data['source_id'] = f"tt_unknown_{int(datetime.now().timestamp())}"
    
    data['id'] = data['source_id'] # db.py expects 'id' for source_id column

    return data
    
def parse_roboplanet_form(html_body):
    """
    Parses the Roboplanet website contact form (HTML format).
    Example: <b>Vorname:</b> Gordana <br><b>Nachname:</b> Dumitrovic <br>...
    """
    data = {}
    
    # Map label names in HTML to our keys
    field_map = {
        'Vorname': 'contact_first', 
        'Nachname': 'contact_last', 
        'Email': 'email',
        'Telefon': 'phone',
        'Firma': 'company',
        'PLZ': 'zip',
        'Nachricht': 'message'
    }
    
    for label, key in field_map.items():
        # Pattern: <b>Label:</b> Value <br>
        pattern = fr'<b>{re.escape(label)}:</b>\s*(.*?)\s*<br>'
        match = re.search(pattern, html_body, re.DOTALL | re.IGNORECASE)
        if match:
            raw_val = match.group(1).strip()
            clean_val = re.sub(r'<[^>]+>', '', raw_val).strip() # Clean any leftover HTML tags
            data[key] = clean_val
            
    # Composite fields
    if data.get('contact_first') and data.get('contact_last'):
        data['contact'] = f"{data['contact_first']} {data['contact_last']}"

    # For Roboplanet forms, we use the timestamp as ID or a hash if missing
    # We need to ensure 'id' is present for db.py compatibility
    if not data.get('source_id'):
         data['source_id'] = f"rp_unknown_{int(datetime.now().timestamp())}"
    data['id'] = data['source_id']
    
    data['raw_body'] = html_body
    return data

def ingest_mock_leads():
    # Mock data from the session context
    leads = [
        {
            'id': '2397256',
            'company': 'pronorm Einbauküchen GmbH',
            'contact': 'Jakob Funk',
            'email': 'jakob.funk@pronorm.de',
            'phone': '+49 5733 979175',
            'raw_body': """
            Lead-ID: 2397256
            Firma: pronorm Einbauküchen GmbH
            Vorname: Jakob
            Nachname: Funk
            Reinigungs-Flche: 1.001 - 10.000 qm
            Einsatzzweck: Reinigung von Böden
            """
        },
        {
            'id': '2414364',
            'company': 'Quad & Rollershop Schwabmünchen GmbH',
            'contact': 'Manfred Bihler',
            'email': 'Rollershop.Schwabmuenchen@web.de',
            'phone': '+49 8232 905246',
            'raw_body': """
            Lead-ID: 2414364
            Firma: Quad & Rollershop Schwabmünchen GmbH
            Vorname: Manfred
            Nachname: Bihler
            Reinigungs-Flche: 301 - 1.000 qm
            Einsatzzweck: Reinigung von Böden
            """
        }
    ]
    
    count = 0
    for lead in leads:
        if insert_lead(lead):
            count += 1
    return count

if __name__ == "__main__":
    from db import init_db
    init_db()
    print(f"Ingested {ingest_mock_leads()} new leads.")