Docs: Aktualisierung der Dokumentation für Task [2ea88f42]

MIGRATION_PLAN.md

@@ -198,6 +198,11 @@ Um DSGVO-konforme Marketing-Automatisierung zu ermöglichen, wurde eine sichere
 
 ## 7. Historie & Fixes (Jan 2026)
 
+    *   **[MAJOR] v0.9.0: Role Matching Optimization & Portability (March 2026)**
+        *   **Pattern Optimizer:** Asynchrones Hintergrund-System zur automatischen Konsolidierung von Einzel-Matches in mächtige Regex-Regeln via Gemini. Inklusive Konfliktprüfung gegen andere Rollen. Nutzt `ast.literal_eval` für robustes Regex-Parsing.
+        *   **Database Management:** Direkter Up- & Download der SQLite-Datenbank aus dem UI heraus. Automatisches Backup-System bei Upload.
+        *   **Regex Sandbox:** Integriertes Test-Tool für Muster vor der Speicherung in der Datenbank.
+        *   **Smart Suggestions:** Live-Analyse der Kontaktdaten zur Identifikation häufiger Schlüsselwörter pro Rolle als Klick-Vorschläge.
     *   **[CRITICAL] v0.7.4: Service Restoration & Logic Fix (Jan 24, 2026)**
     *   **[STABILITY] v0.7.3: Hardening Metric Parser & Regression Testing (Jan 23, 2026)**
     *   **[STABILITY] v0.7.2: Robust Metric Parsing (Jan 23, 2026)**

company-explorer/backend/app.py

@@ -1,4 +1,4 @@
-from fastapi import FastAPI, Depends, HTTPException, Query, BackgroundTasks
+from fastapi import FastAPI, Depends, HTTPException, Query, BackgroundTasks, UploadFile, File
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import FileResponse
@@ -9,6 +9,9 @@ from datetime import datetime
 import os
 import sys
 import uuid
+import shutil
+import re
+from collections import Counter
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
 import secrets
 
@@ -39,6 +42,7 @@ from .services.discovery import DiscoveryService
 from .services.scraping import ScraperService
 from .services.classification import ClassificationService
 from .services.role_mapping import RoleMappingService
+from .services.optimization import PatternOptimizationService
 
 # Initialize App
 app = FastAPI(
@@ -60,6 +64,14 @@ scraper = ScraperService()
 classifier = ClassificationService() # Now works without args
 discovery = DiscoveryService()
 
+# Global State for Long-Running Optimization Task
+optimization_status = {
+    "state": "idle", # idle, processing, completed, error
+    "progress": 0,
+    "result": None,
+    "error": None
+}
+
 # --- Pydantic Models ---
 class CompanyCreate(BaseModel):
     name: str
@@ -898,6 +910,96 @@ class ClassificationResponse(BaseModel):
     processed: int
     new_patterns: int
 
+class OptimizationProposal(BaseModel):
+    target_role: str
+    regex: str
+    explanation: str
+    priority: int
+    covered_pattern_ids: List[int]
+    covered_titles: List[str]
+    false_positives: List[str]
+
+class ApplyOptimizationRequest(BaseModel):
+    target_role: str
+    regex: str
+    priority: int
+    ids_to_delete: List[int]
+
+def run_optimization_task():
+    global optimization_status
+    optimization_status["state"] = "processing"
+    optimization_status["result"] = None
+    optimization_status["error"] = None
+    
+    from .database import SessionLocal
+    db = SessionLocal()
+    try:
+        optimizer = PatternOptimizationService(db)
+        proposals = optimizer.generate_proposals()
+        optimization_status["result"] = proposals
+        optimization_status["state"] = "completed"
+    except Exception as e:
+        logger.error(f"Optimization task failed: {e}", exc_info=True)
+        optimization_status["state"] = "error"
+        optimization_status["error"] = str(e)
+    finally:
+        db.close()
+
+@app.post("/api/job_roles/optimize-start")
+def start_pattern_optimization(
+    background_tasks: BackgroundTasks,
+    username: str = Depends(authenticate_user)
+):
+    """
+    Starts the optimization analysis in the background.
+    """
+    global optimization_status
+    if optimization_status["state"] == "processing":
+        return {"status": "already_running"}
+    
+    background_tasks.add_task(run_optimization_task)
+    return {"status": "started"}
+
+@app.get("/api/job_roles/optimize-status")
+def get_pattern_optimization_status(
+    username: str = Depends(authenticate_user)
+):
+    """
+    Poll this endpoint to get the result of the optimization.
+    """
+    return optimization_status
+
+@app.post("/api/job_roles/apply-optimization")
+def apply_pattern_optimization(
+    req: ApplyOptimizationRequest,
+    db: Session = Depends(get_db),
+    username: str = Depends(authenticate_user)
+):
+    """
+    Applies a proposal: Creates the new regex and deletes the obsolete exact patterns.
+    """
+    # 1. Create new Regex Pattern
+    # Check duplicate first
+    existing = db.query(JobRolePattern).filter(JobRolePattern.pattern_value == req.regex).first()
+    if not existing:
+        new_pattern = JobRolePattern(
+            pattern_type="regex",
+            pattern_value=req.regex,
+            role=req.target_role,
+            priority=req.priority,
+            created_by="optimizer"
+        )
+        db.add(new_pattern)
+        logger.info(f"Optimization: Created new regex {req.regex} for {req.target_role}")
+    
+    # 2. Delete covered Exact Patterns
+    if req.ids_to_delete:
+        db.query(JobRolePattern).filter(JobRolePattern.id.in_(req.ids_to_delete)).delete(synchronize_session=False)
+        logger.info(f"Optimization: Deleted {len(req.ids_to_delete)} obsolete patterns.")
+        
+    db.commit()
+    return {"status": "success", "message": f"Created regex and removed {len(req.ids_to_delete)} old patterns."}
+
 @app.post("/api/job_roles", response_model=JobRolePatternResponse)
 def create_job_role(
     job_role: JobRolePatternCreate, 
@@ -977,6 +1079,34 @@ def list_raw_job_titles(
     
     return query.order_by(RawJobTitle.count.desc()).limit(limit).all()
 
+@app.get("/api/job_roles/suggestions")
+def get_job_role_suggestions(db: Session = Depends(get_db), username: str = Depends(authenticate_user)):
+    """
+    Analyzes existing contacts to suggest regex patterns based on frequent keywords per role.
+    """
+    contacts = db.query(Contact).filter(Contact.role != None, Contact.job_title != None).all()
+    
+    role_groups = {}
+    for c in contacts:
+        if c.role not in role_groups:
+            role_groups[c.role] = []
+        role_groups[c.role].append(c.job_title)
+        
+    suggestions = {}
+    
+    for role, titles in role_groups.items():
+        all_tokens = []
+        for t in titles:
+            # Simple cleaning: keep alphanum, lower
+            cleaned = re.sub(r'[^\w\s]', ' ', t).lower()
+            tokens = [w for w in cleaned.split() if len(w) > 3] # Ignore short words
+            all_tokens.extend(tokens)
+            
+        common = Counter(all_tokens).most_common(10)
+        suggestions[role] = [{"word": w, "count": c} for w, c in common]
+        
+    return suggestions
+
 @app.get("/api/mistakes")
 def list_reported_mistakes(
     status: Optional[str] = Query(None),
@@ -1024,6 +1154,87 @@ def update_reported_mistake_status(
     logger.info(f"Updated status for mistake {mistake_id} to {mistake.status}")
     return {"status": "success", "mistake": mistake}
 
+# --- Database Management ---
+
+@app.get("/api/admin/database/download")
+def download_database(username: str = Depends(authenticate_user)):
+    """
+    Downloads the current SQLite database file.
+    """
+    db_path = "/app/companies_v3_fixed_2.db"
+    if not os.path.exists(db_path):
+        raise HTTPException(status_code=404, detail="Database file not found")
+    
+    filename = f"companies_backup_{datetime.utcnow().strftime('%Y-%m-%d_%H-%M')}.db"
+    return FileResponse(db_path, media_type="application/octet-stream", filename=filename)
+
+@app.post("/api/admin/database/upload")
+async def upload_database(
+    file: UploadFile = File(...),
+    username: str = Depends(authenticate_user)
+):
+    """
+    Uploads and replaces the SQLite database file. Creating a backup first.
+    """
+    db_path = "/app/companies_v3_fixed_2.db"
+    backup_path = f"{db_path}.bak.{datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')}"
+    
+    try:
+        # Create Backup
+        if os.path.exists(db_path):
+            shutil.copy2(db_path, backup_path)
+            logger.info(f"Created database backup at {backup_path}")
+        
+        # Save new file
+        with open(db_path, "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+            
+        logger.info(f"Database replaced via upload by user {username}")
+        return {"status": "success", "message": "Database uploaded successfully. Please restart the container to apply changes."}
+        
+    except Exception as e:
+        logger.error(f"Database upload failed: {e}", exc_info=True)
+        # Try to restore backup if something went wrong during write
+        if os.path.exists(backup_path):
+            shutil.copy2(backup_path, db_path)
+            logger.warning("Restored database from backup due to upload failure.")
+            
+        raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
+
+# --- Regex Testing ---
+
+class RegexTestRequest(BaseModel):
+    pattern: str
+    pattern_type: str = "regex" # regex, exact, startswith
+    test_string: str
+
+@app.post("/api/job_roles/test-pattern")
+def test_job_role_pattern(req: RegexTestRequest, username: str = Depends(authenticate_user)):
+    """
+    Tests if a given pattern matches a test string.
+    """
+    try:
+        is_match = False
+        normalized_test = req.test_string.lower().strip()
+        pattern = req.pattern.lower().strip()
+        
+        if req.pattern_type == "regex":
+            if re.search(pattern, normalized_test, re.IGNORECASE):
+                is_match = True
+        elif req.pattern_type == "exact":
+            if pattern == normalized_test:
+                is_match = True
+        elif req.pattern_type == "startswith":
+            if normalized_test.startswith(pattern):
+                is_match = True
+                
+        return {"match": is_match}
+    except re.error as e:
+        return {"match": False, "error": f"Invalid Regex: {str(e)}"}
+    except Exception as e:
+        logger.error(f"Pattern test error: {e}")
+        return {"match": False, "error": str(e)}
+
 @app.post("/api/enrich/discover")
 def discover_company(req: AnalysisRequest, background_tasks: BackgroundTasks, db: Session = Depends(get_db), username: str = Depends(authenticate_user)):
     company = db.query(Company).filter(Company.id == req.company_id).first()

company-explorer/backend/scripts/analyze_job_title_patterns.py

@@ -0,0 +1,82 @@
+import sys
+import os
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from collections import Counter
+import re
+
+# Add backend to path to import models
+sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
+
+from backend.config import settings
+from backend.database import Contact, JobRolePattern
+
+def clean_text(text):
+    if not text: return ""
+    # Keep only alphanumeric and spaces
+    text = re.sub(r'[^\w\s]', ' ', text)
+    return text.lower().strip()
+
+def get_ngrams(tokens, n):
+    if len(tokens) < n:
+        return []
+    return [" ".join(tokens[i:i+n]) for i in range(len(tokens)-n+1)]
+
+def analyze_patterns():
+    print(f"Connecting to database: {settings.DATABASE_URL}")
+    engine = create_engine(settings.DATABASE_URL)
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    try:
+        # Fetch all contacts with a role
+        contacts = session.query(Contact).filter(Contact.role != None, Contact.job_title != None).all()
+        print(f"Found {len(contacts)} classified contacts to analyze.")
+
+        role_groups = {}
+        for c in contacts:
+            if c.role not in role_groups:
+                role_groups[c.role] = []
+            role_groups[c.role].append(c.job_title)
+
+        print("\n" + "="*60)
+        print(" JOB TITLE PATTERN ANALYSIS REPORT")
+        print("="*60 + "\n")
+
+        for role, titles in role_groups.items():
+            print(f"--- ROLE: {role} ({len(titles)} samples) ---")
+            
+            # Tokenize all titles
+            all_tokens = []
+            all_bigrams = []
+            
+            for t in titles:
+                cleaned = clean_text(t)
+                tokens = [w for w in cleaned.split() if len(w) > 2] # Ignore short words
+                all_tokens.extend(tokens)
+                all_bigrams.extend(get_ngrams(tokens, 2))
+
+            # Analyze frequencies
+            common_words = Counter(all_tokens).most_common(15)
+            common_bigrams = Counter(all_bigrams).most_common(10)
+
+            print("Top Keywords:")
+            for word, count in common_words:
+                print(f"  - {word}: {count}")
+            
+            print("\nTop Bigrams (Word Pairs):")
+            for bg, count in common_bigrams:
+                print(f"  - \"{bg}\": {count}")
+            
+            print("\nSuggested Regex Components:")
+            top_5_words = [w[0] for w in common_words[:5]]
+            print(f"  ({ '|'.join(top_5_words) })")
+            print("\n" + "-"*30 + "\n")
+
+    except Exception as e:
+        print(f"Error: {e}")
+    finally:
+        session.close()
+
+if __name__ == "__main__":
+    analyze_patterns()

company-explorer/backend/services/optimization.py

@@ -0,0 +1,157 @@
+from sqlalchemy.orm import Session
+from ..database import JobRolePattern, Persona
+from ..lib.core_utils import call_gemini_flash
+import json
+import logging
+import re
+import ast
+
+logger = logging.getLogger(__name__)
+
+class PatternOptimizationService:
+    def __init__(self, db: Session):
+        self.db = db
+
+    def generate_proposals(self):
+        """
+        Analyzes existing EXACT patterns and proposes consolidated REGEX patterns.
+        """
+        # ... (Fetch Data logic remains)
+        # 1. Fetch Data
+        patterns = self.db.query(JobRolePattern).filter(JobRolePattern.pattern_type == "exact").all()
+        
+        # Group by Role
+        roles_data = {}
+        pattern_map = {} 
+        
+        for p in patterns:
+            if p.role not in roles_data:
+                roles_data[p.role] = []
+            roles_data[p.role].append(p.pattern_value)
+            pattern_map[p.pattern_value] = p.id
+
+        if not roles_data:
+            return []
+
+        proposals = []
+
+        # 2. Analyze each role
+        for target_role in roles_data.keys():
+            target_titles = roles_data[target_role]
+            
+            if len(target_titles) < 3:
+                continue
+
+            negative_examples = []
+            for other_role, titles in roles_data.items():
+                if other_role != target_role:
+                    negative_examples.extend(titles[:50]) 
+
+            # 3. Build Prompt
+            prompt = f"""
+            Act as a Regex Optimization Engine for B2B Job Titles.
+            
+            GOAL: Break down the list of 'Positive Examples' into logical CLUSTERS and create a Regex for each cluster.
+            TARGET ROLE: "{target_role}"
+            
+            TITLES TO COVER (Positive Examples):
+            {json.dumps(target_titles)}
+            
+            TITLES TO AVOID (Negative Examples - DO NOT MATCH THESE):
+            {json.dumps(negative_examples[:150])}
+            
+            INSTRUCTIONS:
+            1. Analyze the 'Positive Examples'. Do NOT try to create one single regex for all of them.
+            2. Identify distinct semantic groups.
+            3. Create a Regex for EACH group.
+            4. CRITICAL - CONFLICT HANDLING:
+               - The Regex must NOT match the 'Negative Examples'.
+               - Use Negative Lookahead (e.g. ^(?=.*Manager)(?!.*Facility).*) if needed.
+            5. Aggressiveness: Be bold.
+            
+            OUTPUT FORMAT:
+            Return a valid Python List of Dictionaries. 
+            Example:
+            [
+                {{
+                    "regex": r"(?i).*pattern.*",
+                    "explanation": "Explanation...",
+                    "suggested_priority": 50
+                }}
+            ]
+            Enclose regex patterns in r"..." strings to handle backslashes correctly.
+            """
+
+            try:
+                logger.info(f"Optimizing patterns for role: {target_role} (Positive: {len(target_titles)})")
+                
+                response = call_gemini_flash(prompt) # Removed json_mode=True to allow Python syntax
+                
+                # Cleanup markdown
+                clean_text = response.strip()
+                if clean_text.startswith("```python"):
+                    clean_text = clean_text[9:-3]
+                elif clean_text.startswith("```json"):
+                    clean_text = clean_text[7:-3]
+                elif clean_text.startswith("```"):
+                    clean_text = clean_text[3:-3]
+                clean_text = clean_text.strip()
+                
+                ai_suggestions = []
+                try:
+                    # First try standard JSON
+                    ai_suggestions = json.loads(clean_text)
+                except json.JSONDecodeError:
+                    try:
+                        # Fallback: Python AST Literal Eval (handles r"..." strings)
+                        ai_suggestions = ast.literal_eval(clean_text)
+                    except Exception as e:
+                        logger.error(f"Failed to parse response for {target_role} with JSON and AST. Error: {e}")
+                        continue
+                
+                # Verify and map back IDs
+                for sugg in ai_suggestions:
+                    try:
+                        regex_str = sugg.get('regex')
+                        if not regex_str: continue
+                        
+                        # Python AST already handles r"..." decoding, so regex_str is the raw pattern
+                        regex = re.compile(regex_str)
+                        
+                        # Calculate coverage locally
+                        covered_ids = []
+                        covered_titles_verified = []
+                        
+                        for t in target_titles:
+                            if regex.search(t):
+                                if t in pattern_map:
+                                    covered_ids.append(pattern_map[t])
+                                    covered_titles_verified.append(t)
+                        
+                        # Calculate False Positives
+                        false_positives = []
+                        for t in negative_examples:
+                            if regex.search(t):
+                                false_positives.append(t)
+                        
+                        if len(covered_ids) >= 2 and len(false_positives) == 0:
+                            proposals.append({
+                                "target_role": target_role,
+                                "regex": regex_str,
+                                "explanation": sugg.get('explanation', 'No explanation provided'),
+                                "priority": sugg.get('suggested_priority', 50),
+                                "covered_pattern_ids": covered_ids,
+                                "covered_titles": covered_titles_verified,
+                                "false_positives": false_positives
+                            })
+                            
+                    except re.error:
+                        logger.warning(f"AI generated invalid regex: {sugg.get('regex')}")
+                        continue
+
+            except Exception as e:
+                logger.error(f"Error optimizing patterns for {target_role}: {e}", exc_info=True)
+                continue
+
+        logger.info(f"Optimization complete. Generated {len(proposals)} proposals.")
+        return proposals

company-explorer/frontend/src/components/RoboticsSettings.tsx

@@ -1,6 +1,6 @@
 import { useEffect, useState, useMemo } from 'react'
 import axios from 'axios'
-import { X, Bot, Tag, Target, Users, Plus, Trash2, Save, Flag, Check, Ban, ExternalLink, ChevronDown, Grid } from 'lucide-react'
+import { X, Bot, Tag, Target, Users, Plus, Trash2, Save, Flag, Check, Ban, ExternalLink, ChevronDown, Grid, Database, Download, UploadCloud, Play, AlertTriangle, Lightbulb, Sparkles } from 'lucide-react'
 import clsx from 'clsx'
 import { MarketingMatrixManager } from './MarketingMatrixManager'
 
@@ -47,9 +47,22 @@ type ReportedMistake = {
   updated_at: string;
 }
 
+type SuggestionItem = { word: string, count: number };
+type RoleSuggestions = Record<string, SuggestionItem[]>;
+
+type OptimizationProposal = {
+    target_role: string;
+    regex: string;
+    explanation: string;
+    priority: number;
+    covered_pattern_ids: number[];
+    covered_titles: string[];
+    false_positives: string[];
+}
+
 export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsProps) {
-  const [activeTab, setActiveTab] = useState<'robotics' | 'industries' | 'roles' | 'mistakes' | 'matrix'>(
+  const [activeTab, setActiveTab] = useState<'robotics' | 'industries' | 'roles' | 'matrix' | 'mistakes' | 'database'>(
-    localStorage.getItem('roboticsSettingsActiveTab') as 'robotics' | 'industries' | 'roles' | 'mistakes' | 'matrix' || 'robotics'
+    localStorage.getItem('roboticsSettingsActiveTab') as 'robotics' | 'industries' | 'roles' | 'matrix' | 'mistakes' | 'database' || 'robotics'
   )
   
   const [roboticsCategories, setRoboticsCategories] = useState<any[]>([])
@@ -57,11 +70,25 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
   const [jobRoles, setJobRoles] = useState<JobRolePatternType[]>([])
   const [rawJobTitles, setRawJobTitles] = useState<RawJobTitleType[]>([])
   const [reportedMistakes, setReportedMistakes] = useState<ReportedMistake[]>([])
+  const [suggestions, setSuggestions] = useState<RoleSuggestions>({});
+  const [optimizationProposals, setOptimizationProposals] = useState<OptimizationProposal[]>([]);
+  const [showOptimizationModal, setShowOptimizationModal] = useState(false);
+  
   const [currentMistakeStatusFilter, setCurrentMistakeStatusFilter] = useState<string>("PENDING");
   const [isLoading, setIsLoading] = useState(false);
   const [isClassifying, setIsClassifying] = useState(false);
+  const [isOptimizing, setIsOptimizing] = useState(false);
   const [roleSearch, setRoleSearch] = useState("");
 
+  // Database & Regex State
+  const [fileToUpload, setFileToUpload] = useState<File | null>(null);
+  const [uploadStatus, setUploadStatus] = useState<string>("");
+  const [testPattern, setTestPattern] = useState("");
+  const [testPatternType, setTestPatternType] = useState("regex");
+  const [testString, setTestString] = useState("");
+  const [testResult, setTestResult] = useState<boolean | null>(null);
+  const [testError, setTestError] = useState<string | null>(null);
+
   const groupedAndFilteredRoles = useMemo(() => {
     const grouped = jobRoles.reduce((acc: Record<string, JobRolePatternType[]>, role) => {
         const key = role.role || 'Unassigned';
@@ -91,18 +118,20 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
   const fetchAllData = async () => {
     setIsLoading(true);
     try {
-      const [resRobotics, resIndustries, resJobRoles, resRawTitles, resMistakes] = await Promise.all([
+      const [resRobotics, resIndustries, resJobRoles, resRawTitles, resMistakes, resSuggestions] = await Promise.all([
         axios.get(`${apiBase}/robotics/categories`),
         axios.get(`${apiBase}/industries`),
         axios.get(`${apiBase}/job_roles`),
         axios.get(`${apiBase}/job_roles/raw?unmapped_only=true`), // Ensure we only get unmapped
         axios.get(`${apiBase}/mistakes?status=${currentMistakeStatusFilter}`),
+        axios.get(`${apiBase}/job_roles/suggestions`),
       ]);
       setRoboticsCategories(resRobotics.data);
       setIndustries(resIndustries.data);
       setJobRoles(resJobRoles.data);
       setRawJobTitles(resRawTitles.data);
       setReportedMistakes(resMistakes.data.items);
+      setSuggestions(resSuggestions.data);
     } catch (e) {
       console.error("Failed to fetch settings data:", e);
       alert("Fehler beim Laden der Settings. Siehe Konsole.");
@@ -171,8 +200,6 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
     try {
         await axios.post(`${apiBase}/job_roles/classify-batch`);
         alert("Batch classification started in the background. The list will update automatically as titles are processed. You can close this window.");
-        // Optionally, you can poll for completion or just let the user see the number go down on next refresh.
-        // For now, we just inform the user.
     } catch (e) {
         alert("Failed to start batch classification.");
         console.error(e);
@@ -181,6 +208,74 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
     }
   };
 
+  const handleOptimizePatterns = async () => {
+      setIsOptimizing(true);
+      setOptimizationProposals([]);
+      setShowOptimizationModal(true);
+      
+      try {
+          // 1. Start Task
+          await axios.post(`${apiBase}/job_roles/optimize-start`);
+          
+          // 2. Poll for Status
+          const pollInterval = 2000; // 2 seconds
+          const maxAttempts = 150; // 5 minutes
+          let attempts = 0;
+          
+          const checkStatus = async () => {
+              if (attempts >= maxAttempts) {
+                  alert("Optimization timed out. Please check logs.");
+                  setIsOptimizing(false);
+                  return;
+              }
+              
+              try {
+                  const res = await axios.get(`${apiBase}/job_roles/optimize-status`);
+                  const status = res.data.state;
+                  
+                  if (status === 'completed') {
+                      setOptimizationProposals(res.data.result);
+                      setIsOptimizing(false);
+                  } else if (status === 'error') {
+                      alert(`Optimization failed: ${res.data.error}`);
+                      setIsOptimizing(false);
+                  } else {
+                      attempts++;
+                      setTimeout(checkStatus, pollInterval);
+                  }
+              } catch (e) {
+                  console.error("Polling error", e);
+                  setIsOptimizing(false);
+              }
+          };
+          
+          setTimeout(checkStatus, 1000);
+          
+      } catch (e) {
+          alert("Failed to start optimization.");
+          console.error(e);
+          setShowOptimizationModal(false);
+          setIsOptimizing(false);
+      }
+  };
+
+  const handleApplyOptimization = async (proposal: OptimizationProposal) => {
+      try {
+          await axios.post(`${apiBase}/job_roles/apply-optimization`, {
+              target_role: proposal.target_role,
+              regex: proposal.regex,
+              priority: proposal.priority,
+              ids_to_delete: proposal.covered_pattern_ids
+          });
+          // Remove applied proposal from list
+          setOptimizationProposals(prev => prev.filter(p => p.regex !== proposal.regex));
+          fetchAllData(); // Refresh main list
+      } catch (e) {
+          alert("Failed to apply optimization.");
+          console.error(e);
+      }
+  };
+  
   const handleUpdateJobRole = async (roleId: number, field: string, value: any) => {
     const roleToUpdate = jobRoles.find(r => r.id === roleId);
     if (!roleToUpdate) return;
@@ -199,19 +294,17 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
     } catch (e) {
         alert("Failed to update job role");
         console.error(e);
-        // Revert on failure if needed, but for now just log it
     }
   };
 
-  const handleAddJobRole = async (title?: string) => {
+  const handleAddJobRole = async (value: string, type: 'exact' | 'regex' = 'exact', roleName?: string) => {
-      const patternValue = title || "New Pattern";
       setIsLoading(true);
       try { 
           await axios.post(`${apiBase}/job_roles`, { 
-              pattern_type: "exact",
+              pattern_type: type,
-              pattern_value: patternValue, 
+              pattern_value: value, 
-              role: "Influencer",
+              role: roleName || "Influencer",
-              priority: 100 
+              priority: type === 'regex' ? 80 : 100 
             }); 
           fetchAllData();
       } catch (e) { 
@@ -238,11 +331,66 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
       }
   }
 
+  const handleDownloadDb = () => {
+    window.location.href = `${apiBase}/admin/database/download`;
+  };
+
+  const handleFileSelect = (event: React.ChangeEvent<HTMLInputElement>) => {
+    if (event.target.files && event.target.files[0]) {
+      setFileToUpload(event.target.files[0]);
+      setUploadStatus("");
+    }
+  };
+
+  const handleUploadDb = async () => {
+    if (!fileToUpload) return;
+    if (!window.confirm("WARNING: This will overwrite the current database! A backup will be created, but any recent changes might be lost. You MUST restart the container afterwards. Continue?")) return;
+
+    const formData = new FormData();
+    formData.append("file", fileToUpload);
+
+    setUploadStatus("uploading");
+
+    try {
+      await axios.post(`${apiBase}/admin/database/upload`, formData, {
+        headers: { "Content-Type": "multipart/form-data" },
+      });
+      setUploadStatus("success");
+      alert("Upload successful! Please RESTART the Docker container to apply changes.");
+    } catch (e: any) {
+      console.error(e);
+      setUploadStatus("error");
+      alert(`Upload failed: ${e.response?.data?.detail || e.message}`);
+    }
+  };
+
+  const handleTestPattern = async () => {
+    setTestResult(null);
+    setTestError(null);
+    if (!testPattern || !testString) return;
+
+    try {
+        const res = await axios.post(`${apiBase}/job_roles/test-pattern`, {
+            pattern: testPattern,
+            pattern_type: testPatternType,
+            test_string: testString
+        });
+        if (res.data.error) {
+            setTestError(res.data.error);
+        } else {
+            setTestResult(res.data.match);
+        }
+    } catch (e: any) {
+        setTestError(e.message);
+    }
+  };
+
+
   if (!isOpen) return null
 
   return (
     <div className="fixed inset-0 z-[60] flex items-center justify-center p-4 bg-black/60 backdrop-blur-sm animate-in fade-in duration-200">
-      <div className="bg-white dark:bg-slate-900 w-full max-w-4xl max-h-[85vh] rounded-2xl shadow-2xl border border-slate-200 dark:border-slate-800 flex flex-col overflow-hidden">
+      <div className="bg-white dark:bg-slate-900 w-full max-w-4xl max-h-[85vh] rounded-2xl shadow-2xl border border-slate-200 dark:border-slate-800 flex flex-col overflow-hidden relative">
         {/* Header */}
         <div className="p-6 border-b border-slate-200 dark:border-slate-800 flex justify-between items-center bg-slate-50 dark:bg-slate-950/50">
           <div>
@@ -262,6 +410,7 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
                  { id: 'roles', label: 'Job Role Mapping', icon: Users },
                  { id: 'matrix', label: 'Marketing Matrix', icon: Grid },
                  { id: 'mistakes', label: 'Reported Mistakes', icon: Flag },
+                 { id: 'database', label: 'Database & Regex', icon: Database },
              ].map(t => (
                  <button 
                     key={t.id}
@@ -384,7 +533,10 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
                             className="w-full bg-white dark:bg-slate-950 border border-slate-300 dark:border-slate-700 rounded-md px-3 py-1.5 text-xs text-slate-900 dark:text-white focus:ring-1 focus:ring-blue-500 outline-none"
                         />
                     </div>
-                    <button onClick={() => handleAddJobRole()} className="flex items-center gap-1 px-3 py-1.5 bg-blue-600 hover:bg-blue-500 text-white text-xs font-bold rounded shadow-lg shadow-blue-500/20"><Plus className="h-3 w-3" /> ADD PATTERN</button>
+                    <button onClick={handleOptimizePatterns} className="flex items-center gap-1 px-3 py-1.5 bg-indigo-600 hover:bg-indigo-500 text-white text-xs font-bold rounded shadow-lg shadow-indigo-500/20 mr-2">
+                        <Sparkles className="h-3 w-3" /> OPTIMIZE
+                    </button>
+                    <button onClick={() => handleAddJobRole("New Pattern", "exact", "Influencer")} className="flex items-center gap-1 px-3 py-1.5 bg-blue-600 hover:bg-blue-500 text-white text-xs font-bold rounded shadow-lg shadow-blue-500/20"><Plus className="h-3 w-3" /> ADD PATTERN</button>
                   </div>
 
                   <div className="space-y-2">
@@ -397,7 +549,36 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
                                   </div>
                                   <ChevronDown className="h-4 w-4 text-slate-400 transform group-open:rotate-180 transition-transform" />
                               </summary>
+                              
                               <div className="border-t border-slate-200 dark:border-slate-800">
+                                {/* AI Suggestions Area */}
+                                {suggestions[roleName] && suggestions[roleName].length > 0 && (
+                                    <div className="p-3 bg-blue-50/50 dark:bg-blue-900/10 border-b border-slate-100 dark:border-slate-800/50">
+                                        <div className="flex items-center gap-2 mb-2 text-[10px] uppercase font-bold text-blue-600 dark:text-blue-400">
+                                            <Lightbulb className="h-3 w-3" />
+                                            AI Suggestions (Common Keywords)
+                                        </div>
+                                        <div className="flex flex-wrap gap-2">
+                                            {suggestions[roleName].map(s => (
+                                                <button 
+                                                    key={s.word}
+                                                    onClick={() => {
+                                                        const regex = `(?i).*${s.word}.*`; // Simple starting point
+                                                        if(window.confirm(`Create Regex pattern "${regex}" for role "${roleName}"?`)) {
+                                                            handleAddJobRole(regex, 'regex', roleName);
+                                                        }
+                                                    }}
+                                                    className="flex items-center gap-1.5 px-2 py-1 bg-white dark:bg-slate-900 border border-blue-200 dark:border-blue-900 rounded-full text-[10px] text-slate-700 dark:text-slate-300 hover:border-blue-400 hover:text-blue-600 transition-colors"
+                                                    title={`Found ${s.count} times`}
+                                                >
+                                                    {s.word} <span className="opacity-50 text-[9px]">({s.count})</span>
+                                                    <Plus className="h-2 w-2 text-blue-400" />
+                                                </button>
+                                            ))}
+                                        </div>
+                                    </div>
+                                )}
+
                                 <table className="w-full text-left text-xs">
                                     <thead className="bg-slate-50 dark:bg-slate-900/50 text-slate-500 font-bold uppercase tracking-wider">
                                         <tr>
@@ -455,7 +636,7 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
                                   <tr key={raw.id} className="group hover:bg-white dark:hover:bg-slate-800 transition-colors">
                                       <td className="p-3 font-medium text-slate-600 dark:text-slate-400 italic">{raw.title}</td>
                                       <td className="p-3 text-center"><span className="px-2 py-1 bg-slate-200 dark:bg-slate-800 rounded-full font-bold text-[10px] text-slate-500">{raw.count}x</span></td>
-                                      <td className="p-3 text-center"><button onClick={() => handleAddJobRole(raw.title)} className="p-1 text-blue-500 hover:bg-blue-100 dark:hover:bg-blue-900/30 rounded transition-all"><Plus className="h-4 w-4" /></button></td>
+                                      <td className="p-3 text-center"><button onClick={() => handleAddJobRole(raw.title, "exact", "Influencer")} className="p-1 text-blue-500 hover:bg-blue-100 dark:hover:bg-blue-900/30 rounded transition-all"><Plus className="h-4 w-4" /></button></td>
                                   </tr>
                               ))}
                               {rawJobTitles.length === 0 && (<tr><td colSpan={3} className="p-12 text-center text-slate-400 italic">Discovery inbox is empty. Import raw job titles to see data here.</td></tr>)}
@@ -470,6 +651,7 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
           </div>
 
           <div key="mistakes-content" className={clsx("space-y-4", { 'hidden': isLoading || activeTab !== 'mistakes' })}>
+              {/* ... existing mistakes content ... */}
               <div className="flex justify-between items-center">
                   <h3 className="text-sm font-bold text-slate-700 dark:text-slate-300">Reported Data Mistakes</h3>
                   <div className="flex items-center gap-2">
@@ -548,7 +730,202 @@ export function RoboticsSettings({ isOpen, onClose, apiBase }: RoboticsSettingsP
                   </table>
               </div>
           </div>
+
+          <div key="database-content" className={clsx("space-y-6", { 'hidden': isLoading || activeTab !== 'database' })}>
+              
+              {/* Regex Tester */}
+              <div className="bg-slate-50 dark:bg-slate-950 border border-slate-200 dark:border-slate-800 rounded-xl p-4 space-y-4">
+                  <div>
+                      <h3 className="text-sm font-bold text-slate-700 dark:text-slate-300 flex items-center gap-2"><Bot className="h-4 w-4" /> Regex Tester</h3>
+                      <p className="text-xs text-slate-500">Validate your patterns before adding them to the database.</p>
                   </div>
+                  
+                  <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
+                      <div className="space-y-2">
+                          <label className="text-[10px] uppercase font-bold text-slate-500">Pattern</label>
+                          <div className="flex gap-2">
+                              <select 
+                                value={testPatternType} 
+                                onChange={e => setTestPatternType(e.target.value)}
+                                className="bg-white dark:bg-slate-900 border border-slate-200 dark:border-slate-800 rounded text-xs px-2 focus:ring-1 focus:ring-blue-500 outline-none"
+                              >
+                                  <option value="regex">Regex</option>
+                                  <option value="exact">Exact</option>
+                                  <option value="startswith">Starts With</option>
+                              </select>
+                              <input 
+                                type="text" 
+                                value={testPattern} 
+                                onChange={e => setTestPattern(e.target.value)} 
+                                placeholder="e.g. (leiter|head).{0,15}vertrieb"
+                                className="flex-1 bg-white dark:bg-slate-900 border border-slate-200 dark:border-slate-800 rounded p-2 text-xs font-mono text-slate-800 dark:text-slate-200 focus:ring-1 focus:ring-blue-500 outline-none"
+                              />
+                          </div>
+                      </div>
+                      
+                      <div className="space-y-2">
+                          <label className="text-[10px] uppercase font-bold text-slate-500">Test String (Job Title)</label>
+                          <input 
+                            type="text" 
+                            value={testString} 
+                            onChange={e => setTestString(e.target.value)} 
+                            placeholder="e.g. Leiter Vertrieb und Marketing"
+                            className="w-full bg-white dark:bg-slate-900 border border-slate-200 dark:border-slate-800 rounded p-2 text-xs text-slate-800 dark:text-slate-200 focus:ring-1 focus:ring-blue-500 outline-none"
+                          />
+                      </div>
+                  </div>
+
+                  <div className="flex justify-between items-center pt-2">
+                      <div className="flex items-center gap-2">
+                          <button onClick={handleTestPattern} className="flex items-center gap-1 px-4 py-2 bg-blue-600 hover:bg-blue-500 text-white text-xs font-bold rounded shadow-lg shadow-blue-500/20">
+                              <Play className="h-3 w-3" /> TEST PATTERN
+                          </button>
+                          {testResult !== null && (
+                              <span className={clsx("px-3 py-1.5 rounded text-xs font-bold flex items-center gap-1", testResult ? "bg-green-100 text-green-700" : "bg-red-100 text-red-700")}>
+                                  {testResult ? <Check className="h-3 w-3" /> : <Ban className="h-3 w-3" />}
+                                  {testResult ? "MATCH" : "NO MATCH"}
+                              </span>
+                          )}
+                          {testError && <span className="text-xs text-red-500 font-mono">{testError}</span>}
+                      </div>
+                  </div>
+              </div>
+
+              {/* Database Management */}
+              <div className="bg-slate-50 dark:bg-slate-950 border border-slate-200 dark:border-slate-800 rounded-xl p-4 space-y-4">
+                  <div>
+                      <h3 className="text-sm font-bold text-slate-700 dark:text-slate-300 flex items-center gap-2"><Database className="h-4 w-4" /> Database Management</h3>
+                      <p className="text-xs text-slate-500">Download the full database for offline analysis or restore a backup.</p>
+                  </div>
+
+                  <div className="grid grid-cols-1 md:grid-cols-2 gap-8 items-start">
+                      {/* Download */}
+                      <div className="space-y-3">
+                          <h4 className="text-xs font-bold text-slate-600 dark:text-slate-400 uppercase">Export</h4>
+                          <button onClick={handleDownloadDb} className="w-full flex items-center justify-center gap-2 px-4 py-8 bg-white dark:bg-slate-900 border-2 border-dashed border-slate-300 dark:border-slate-700 hover:border-blue-500 hover:text-blue-500 dark:hover:border-blue-400 dark:hover:text-blue-400 rounded-xl text-slate-500 transition-all group">
+                              <Download className="h-6 w-6 group-hover:scale-110 transition-transform" />
+                              <span className="font-semibold text-sm">Download Database (SQLite)</span>
+                          </button>
+                      </div>
+
+                      {/* Upload */}
+                      <div className="space-y-3">
+                          <h4 className="text-xs font-bold text-slate-600 dark:text-slate-400 uppercase">Restore / Import</h4>
+                          <div className="bg-white dark:bg-slate-900 border border-slate-200 dark:border-slate-800 rounded-xl p-4 space-y-3">
+                              <div className="flex items-center gap-2 text-amber-600 bg-amber-50 dark:bg-amber-900/20 p-2 rounded text-[10px]">
+                                  <AlertTriangle className="h-4 w-4 flex-shrink-0" />
+                                  <span>Warning: Uploading will overwrite the current database. A backup will be created automatically.</span>
+                              </div>
+                              
+                              <input 
+                                type="file" 
+                                onChange={handleFileSelect} 
+                                className="block w-full text-xs text-slate-500
+                                  file:mr-4 file:py-2 file:px-4
+                                  file:rounded-full file:border-0
+                                  file:text-xs file:font-semibold
+                                  file:bg-blue-50 file:text-blue-700
+                                  hover:file:bg-blue-100
+                                "
+                              />
+                              
+                              <button 
+                                onClick={handleUploadDb} 
+                                disabled={!fileToUpload || uploadStatus === 'uploading'}
+                                className={clsx(
+                                    "w-full flex items-center justify-center gap-2 px-4 py-2 text-white text-xs font-bold rounded transition-all",
+                                    !fileToUpload ? "bg-slate-300 dark:bg-slate-800 cursor-not-allowed" : "bg-red-600 hover:bg-red-500 shadow-lg shadow-red-500/20"
+                                )}
+                              >
+                                  {uploadStatus === 'uploading' ? (
+                                      <span className="animate-pulse">Uploading...</span>
+                                  ) : (
+                                      <>
+                                          <UploadCloud className="h-3 w-3" /> UPLOAD & REPLACE DB
+                                      </>
+                                  )}
+                              </button>
+                          </div>
+                      </div>
+                  </div>
+              </div>
+          </div>
+
+        </div>
+
+        {/* Optimization Modal */}
+        {showOptimizationModal && (
+            <div className="absolute inset-0 z-[70] bg-white dark:bg-slate-900 p-6 overflow-y-auto">
+                <div className="max-w-3xl mx-auto space-y-6">
+                    <div className="flex justify-between items-center pb-4 border-b border-slate-200 dark:border-slate-800">
+                        <div>
+                            <h2 className="text-xl font-bold text-slate-900 dark:text-white flex items-center gap-2"><Sparkles className="h-5 w-5 text-indigo-500" /> Pattern Optimization Proposals</h2>
+                            <p className="text-xs text-slate-500 mt-1">AI-generated Regex suggestions to consolidate exact matches.</p>
+                        </div>
+                        <button onClick={() => setShowOptimizationModal(false)} className="p-2 hover:bg-slate-100 dark:hover:bg-slate-800 rounded-full"><X className="h-5 w-5 text-slate-500" /></button>
+                    </div>
+
+                    {isOptimizing ? (
+                        <div className="py-20 text-center space-y-4">
+                            <div className="animate-spin h-8 w-8 border-4 border-indigo-500 border-t-transparent rounded-full mx-auto"></div>
+                            <p className="text-sm text-slate-500 font-medium">Analyzing patterns & checking for conflicts...</p>
+                        </div>
+                    ) : optimizationProposals.length === 0 ? (
+                        <div className="py-20 text-center text-slate-500">
+                            <Check className="h-10 w-10 mx-auto text-green-500 mb-4" />
+                            <p>No optimization opportunities found. Your patterns are already efficient!</p>
+                        </div>
+                    ) : (
+                        <div className="space-y-6">
+                            {optimizationProposals.map((prop, idx) => (
+                                <div key={idx} className="bg-slate-50 dark:bg-slate-950 border border-slate-200 dark:border-slate-800 rounded-xl p-5 shadow-sm">
+                                    <div className="flex justify-between items-start mb-4">
+                                        <div>
+                                            <div className="flex items-center gap-2 mb-1">
+                                                <span className="text-[10px] font-bold uppercase tracking-wider bg-indigo-100 text-indigo-700 px-2 py-0.5 rounded">{prop.target_role}</span>
+                                                <span className="text-[10px] font-bold text-slate-400">Priority: {prop.priority}</span>
+                                            </div>
+                                            <h3 className="font-mono text-sm font-bold text-slate-800 dark:text-slate-200 bg-white dark:bg-slate-900 px-2 py-1 rounded border border-slate-200 dark:border-slate-800 inline-block">{prop.regex}</h3>
+                                        </div>
+                                        <button onClick={() => handleApplyOptimization(prop)} className="bg-indigo-600 hover:bg-indigo-500 text-white text-xs font-bold px-4 py-2 rounded shadow-lg shadow-indigo-500/20 transition-all flex items-center gap-2">
+                                            APPLY & REPLACE {prop.covered_pattern_ids.length} PATTERNS
+                                        </button>
+                                    </div>
+                                    
+                                    <p className="text-xs text-slate-600 dark:text-slate-400 mb-4 italic">{prop.explanation}</p>
+                                    
+                                    <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
+                                        <div className="bg-green-50/50 dark:bg-green-900/10 border border-green-100 dark:border-green-900/30 rounded p-3">
+                                            <div className="text-[10px] font-bold uppercase text-green-600 mb-2 flex items-center gap-1"><Check className="h-3 w-3" /> Covers ({prop.covered_titles.length})</div>
+                                            <div className="flex flex-wrap gap-1.5">
+                                                {prop.covered_titles.map(t => (
+                                                    <span key={t} className="text-[9px] px-1.5 py-0.5 bg-white dark:bg-slate-900 rounded border border-green-200 dark:border-green-800 text-slate-600 dark:text-slate-300">{t}</span>
+                                                ))}
+                                            </div>
+                                        </div>
+                                        
+                                        {prop.false_positives.length > 0 ? (
+                                            <div className="bg-red-50/50 dark:bg-red-900/10 border border-red-100 dark:border-red-900/30 rounded p-3">
+                                                <div className="text-[10px] font-bold uppercase text-red-600 mb-2 flex items-center gap-1"><AlertTriangle className="h-3 w-3" /> Conflicts (Matches other roles!)</div>
+                                                <div className="flex flex-wrap gap-1.5">
+                                                    {prop.false_positives.map(t => (
+                                                        <span key={t} className="text-[9px] px-1.5 py-0.5 bg-white dark:bg-slate-900 rounded border border-red-200 dark:border-red-800 text-slate-600 dark:text-slate-300">{t}</span>
+                                                    ))}
+                                                </div>
+                                            </div>
+                                        ) : (
+                                            <div className="flex items-center justify-center text-[10px] text-slate-400 italic bg-slate-100 dark:bg-slate-900 rounded border border-dashed border-slate-200 dark:border-slate-800">
+                                                No conflicts with other roles detected.
+                                            </div>
+                                        )}
+                                    </div>
+                                </div>
+                            ))}
+                        </div>
+                    )}
+                </div>
+            </div>
+        )}
       </div>
     </div>
   )

tasks.md

@@ -39,6 +39,15 @@
 
 - [x] **Integrität:** Fehlende API-Endpunkte für Firmen-Erstellung, Bulk-Import und Wiki-Overrides wiederhergestellt.
 
+## Persona Segmentierung & Rollen-Matching (v0.9.0 - Abgeschlossen)
+
+- [x] **Database Portability:** Up- & Download der SQLite-Datenbank direkt im UI implementiert (inkl. automatischem Backup).
+- [x] **Pattern Optimizer:** Asynchrones KI-System zur automatischen Generierung von Regex-Mustern aus Einzelregeln.
+- [x] **Konflikt-Management:** KI-gestützte Prüfung von Regex-Regeln gegen andere Rollen (Negative Examples) zur Vermeidung von Fehlzuordnungen.
+- [x] **Regex Sandbox:** Interaktives Test-Tool im Frontend zur Validierung von Mustern gegen echte Jobtitel.
+- [x] **Smart Suggestions:** Live-Analyse der Datenbank zur Anzeige häufiger Schlüsselwörter als Klick-Vorschläge.
+- [x] **Robustheit:** Implementierung eines AST-basierten Parsers für komplexe Regex-Escaping-Szenarien.
+
 ## Lead Engine: Tradingtwins Automation (In Arbeit)
 
 - [x] **E-Mail Ingest:** Automatisierter Import von Leads aus dem Postfach `info@robo-planet.de` via Microsoft Graph API.