Implementierung der Batch-Brancheneinstufung zur Kostenoptimierung
- FEATURE: Brancheneinstufung erfolgt nun in Batches (z.B. 20 Unternehmen pro API-Call), um die Token-Kosten drastisch zu senken. - REFACTOR: Neue Funktion `evaluate_branches_batch` in `helpers.py` erstellt, die den komplexen Batch-Prompt generiert. - REFACTOR: `reclassify_all_branches` in `data_processor.py` überarbeitet, um die Batch-Verarbeitung und das Ergebnis-Mapping zu steuern.
This commit is contained in:
@@ -1462,27 +1462,23 @@ class DataProcessor:
|
|||||||
|
|
||||||
self.logger.info(f"FSM-Pitch-Generierung abgeschlossen. {processed_count} Zeilen bearbeitet.")
|
self.logger.info(f"FSM-Pitch-Generierung abgeschlossen. {processed_count} Zeilen bearbeitet.")
|
||||||
|
|
||||||
def reclassify_all_branches(self, start_sheet_row=None, limit=None):
|
def reclassify_all_branches(self, start_sheet_row=None, limit=None, batch_size=20):
|
||||||
"""
|
"""
|
||||||
Führt für alle relevanten Zeilen eine neue Brancheneinstufung (v2.0) durch.
|
Führt für alle relevanten Zeilen eine neue Brancheneinstufung (v2.0) in Batches durch.
|
||||||
Dieser Modus ist ideal, um nach einer Änderung der Branchen-Definitionen
|
|
||||||
den gesamten Datenbestand zu aktualisieren.
|
|
||||||
"""
|
"""
|
||||||
self.logger.info(f"Starte Modus 'reclassify_branches'. Bereich: {start_sheet_row or 'Start'}, Limit: {limit or 'Unbegrenzt'}")
|
self.logger.info(f"Starte Modus 'reclassify_branches' im Batch-Modus (Größe: {batch_size}). Bereich: {start_sheet_row or 'Start'}, Limit: {limit or 'Unbegrenzt'}")
|
||||||
|
|
||||||
if not self.sheet_handler.load_data():
|
if not self.sheet_handler.load_data():
|
||||||
return
|
return
|
||||||
|
|
||||||
all_data = self.sheet_handler.get_all_data_with_headers()
|
all_data = self.sheet_handler.get_all_data_with_headers()
|
||||||
header_rows = self.sheet_handler._header_rows
|
header_rows = self.sheet_handler._header_rows
|
||||||
|
|
||||||
effective_start = start_sheet_row if start_sheet_row is not None else header_rows + 1
|
effective_start = start_sheet_row if start_sheet_row is not None else header_rows + 1
|
||||||
|
|
||||||
tasks = []
|
tasks = []
|
||||||
for i in range(effective_start - 1, len(all_data)):
|
for i in range(effective_start - 1, len(all_data)):
|
||||||
if limit is not None and len(tasks) >= limit:
|
if limit is not None and len(tasks) >= limit:
|
||||||
break
|
break
|
||||||
|
|
||||||
row_data = all_data[i]
|
row_data = all_data[i]
|
||||||
company_name = self._get_cell_value_safe(row_data, "CRM Name").strip()
|
company_name = self._get_cell_value_safe(row_data, "CRM Name").strip()
|
||||||
if company_name:
|
if company_name:
|
||||||
@@ -1497,43 +1493,56 @@ class DataProcessor:
|
|||||||
all_sheet_updates = []
|
all_sheet_updates = []
|
||||||
now_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
now_timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
for task in tasks:
|
# Verarbeite die Tasks in Batches
|
||||||
row_num = task['row_num']
|
for i in range(0, len(tasks), batch_size):
|
||||||
row_data = task['data']
|
batch_tasks = tasks[i:i + batch_size]
|
||||||
|
self.logger.info(f"Verarbeite Batch {i//batch_size + 1}/{(len(tasks) + batch_size - 1)//batch_size} (Zeilen {batch_tasks[0]['row_num']} bis {batch_tasks[-1]['row_num']})...")
|
||||||
|
|
||||||
self.logger.debug(f"Bewerte Branche für Zeile {row_num}...")
|
# Bereite die Daten für den Batch-Prompt vor
|
||||||
|
companies_data_for_prompt = []
|
||||||
|
for task in batch_tasks:
|
||||||
|
row_data = task['data']
|
||||||
|
companies_data_for_prompt.append({
|
||||||
|
"row_num": task['row_num'],
|
||||||
|
"name": self._get_cell_value_safe(row_data, "CRM Name"),
|
||||||
|
"summary": self._get_cell_value_safe(row_data, "Website Zusammenfassung"),
|
||||||
|
"wiki": self._get_cell_value_safe(row_data, "Wiki Absatz")
|
||||||
|
})
|
||||||
|
|
||||||
try:
|
# Rufe die neue Batch-Funktion auf
|
||||||
result = evaluate_branche_chatgpt(
|
batch_results = evaluate_branches_batch(companies_data_for_prompt)
|
||||||
company_name=self._get_cell_value_safe(row_data, "CRM Name"),
|
|
||||||
website_summary=self._get_cell_value_safe(row_data, "Website Zusammenfassung"),
|
|
||||||
wiki_absatz=self._get_cell_value_safe(row_data, "Wiki Absatz")
|
|
||||||
)
|
|
||||||
|
|
||||||
# Updates für die drei Zielspalten vorbereiten
|
if batch_results:
|
||||||
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Vorschlag Branche"]["index"] + 1)}{row_num}', 'values': [[result.get('branch')]]})
|
# Ordne die Ergebnisse den richtigen Zeilen zu
|
||||||
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Branche Konfidenz"]["index"] + 1)}{row_num}', 'values': [[result.get('confidence')]]})
|
results_by_row = {res['row_num']: res for res in batch_results}
|
||||||
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Begruendung Abweichung Branche"]["index"] + 1)}{row_num}', 'values': [[result.get('justification')]]})
|
|
||||||
# Auch den Timestamp aktualisieren
|
|
||||||
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Timestamp letzte Pruefung"]["index"] + 1)}{row_num}', 'values': [[now_timestamp]]})
|
|
||||||
|
|
||||||
except Exception as e:
|
for task in batch_tasks:
|
||||||
self.logger.error(f"FEHLER bei Branchen-Neubewertung für Zeile {row_num}: {e}")
|
row_num = task['row_num']
|
||||||
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Vorschlag Branche"]["index"] + 1)}{row_num}', 'values': [['FEHLER (Prozess)']]} )
|
result = results_by_row.get(row_num)
|
||||||
|
|
||||||
# Batch-Update Logik (vereinfacht, um es hier zu zeigen)
|
if result:
|
||||||
if len(all_sheet_updates) >= 200: # 50 Zeilen * 4 Updates pro Zeile
|
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Vorschlag Branche"]["index"] + 1)}{row_num}', 'values': [[result.get('Branche')]]})
|
||||||
self.logger.info(f"Sende Batch-Update für {len(all_sheet_updates)//4} Branchen...")
|
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Branche Konfidenz"]["index"] + 1)}{row_num}', 'values': [[result.get('Konfidenz')]]})
|
||||||
self.sheet_handler.batch_update_cells(all_sheet_updates)
|
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Begruendung Abweichung Branche"]["index"] + 1)}{row_num}', 'values': [[result.get('Begruendung')]]})
|
||||||
all_sheet_updates = []
|
else:
|
||||||
time.sleep(1)
|
self.logger.error(f"Kein Ergebnis für Zeile {row_num} im Batch-Resultat gefunden.")
|
||||||
|
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Vorschlag Branche"]["index"] + 1)}{row_num}', 'values': [['FEHLER (Batch-Antwort)']]} )
|
||||||
|
|
||||||
# Letzten Batch senden
|
# Timestamp immer setzen
|
||||||
|
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Timestamp letzte Pruefung"]["index"] + 1)}{row_num}', 'values': [[now_timestamp]]})
|
||||||
|
else:
|
||||||
|
self.logger.error(f"Batch-Verarbeitung für Zeilen {batch_tasks[0]['row_num']} bis {batch_tasks[-1]['row_num']} fehlgeschlagen. Setze Fehlerstatus.")
|
||||||
|
for task in batch_tasks:
|
||||||
|
row_num = task['row_num']
|
||||||
|
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Chat Vorschlag Branche"]["index"] + 1)}{row_num}', 'values': [['FEHLER (Batch-API)']]} )
|
||||||
|
all_sheet_updates.append({'range': f'{self.sheet_handler._get_col_letter(COLUMN_MAP["Timestamp letzte Pruefung"]["index"] + 1)}{row_num}', 'values': [[now_timestamp]]})
|
||||||
|
|
||||||
|
# Finalen Batch-Update senden
|
||||||
if all_sheet_updates:
|
if all_sheet_updates:
|
||||||
self.logger.info(f"Sende finalen Batch-Update für {len(all_sheet_updates)//4} Branchen...")
|
self.logger.info(f"Sende finales Batch-Update für {len(tasks)} bewertete Branchen...")
|
||||||
self.sheet_handler.batch_update_cells(all_sheet_updates)
|
self.sheet_handler.batch_update_cells(all_sheet_updates)
|
||||||
|
|
||||||
self.logger.info("Branchen-Neubewertung abgeschlossen.")
|
self.logger.info("Branchen-Neubewertung im Batch-Modus abgeschlossen.")
|
||||||
|
|
||||||
|
|
||||||
# ==========================================================================
|
# ==========================================================================
|
||||||
|
|||||||
Reference in New Issue
Block a user