bugfix
This commit is contained in:
109
helpers.py
109
helpers.py
@@ -35,10 +35,33 @@ import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import pandas as pd
|
||||
import openai
|
||||
# NEU: Korrigierte Imports für openai v1.x
|
||||
from openai import APIError, RateLimitError, APIConnectionError, BadRequestError, AuthenticationError, Timeout
|
||||
from config import (Config, BRANCH_MAPPING_FILE, URL_CHECK_MARKER, USER_AGENTS, LOG_DIR)
|
||||
|
||||
# ==============================================================================
|
||||
# UNIVERSAL OPENAI v0.x / v1.x IMPORTS
|
||||
# ==============================================================================
|
||||
# This block makes the code compatible with both old (v0.x) and new (v1.x)
|
||||
# versions of the OpenAI library.
|
||||
|
||||
try:
|
||||
# Attempt to import from the new (v1.x) structure
|
||||
from openai import APIError, RateLimitError, APIConnectionError, BadRequestError, AuthenticationError, Timeout
|
||||
IS_OPENAI_V1 = True
|
||||
logging.info("OpenAI library v1.x or higher detected.")
|
||||
except ImportError:
|
||||
# Fallback to the old (v0.x) structure
|
||||
from openai.error import (
|
||||
APIError,
|
||||
RateLimitError,
|
||||
APIConnectionError,
|
||||
InvalidRequestError as BadRequestError, # Alias für Kompatibilität
|
||||
AuthenticationError,
|
||||
Timeout,
|
||||
OpenAIError
|
||||
)
|
||||
IS_OPENAI_V1 = False
|
||||
logging.info("Legacy OpenAI library v0.x detected.")
|
||||
|
||||
# Optionale Bibliotheken
|
||||
try:
|
||||
import tiktoken
|
||||
@@ -94,6 +117,7 @@ def retry_on_failure(func):
|
||||
"""
|
||||
Decorator, der eine Funktion bei bestimmten Fehlern mehrmals wiederholt.
|
||||
Implementiert exponentiellen Backoff mit Jitter.
|
||||
Ist kompatibel mit openai v0.x und v1.x.
|
||||
"""
|
||||
def wrapper(*args, **kwargs):
|
||||
func_name = func.__name__
|
||||
@@ -108,8 +132,8 @@ def retry_on_failure(func):
|
||||
return func(*args, **kwargs)
|
||||
except Exception as e:
|
||||
decorator_logger.error(f"FEHLER bei '{effective_func_name}' (keine Retries konfiguriert). {type(e).__name__} - {str(e)[:150]}...")
|
||||
if not isinstance(e, (requests.exceptions.RequestException, gspread.exceptions.APIError, OpenAIError, wikipedia.exceptions.WikipediaException)):
|
||||
decorator_logger.exception("Details zum Fehler:")
|
||||
# Wir fangen hier jetzt alle Fehler, da die spezifischen unten sind.
|
||||
decorator_logger.exception("Details zum Fehler:")
|
||||
raise e
|
||||
|
||||
for attempt in range(max_retries_config):
|
||||
@@ -118,11 +142,13 @@ def retry_on_failure(func):
|
||||
decorator_logger.warning(f"Wiederhole Versuch {attempt + 1}/{max_retries_config} fuer '{effective_func_name}'...")
|
||||
return func(*args, **kwargs)
|
||||
|
||||
# Fehler, die NICHT wiederholt werden sollen
|
||||
except (gspread.exceptions.SpreadsheetNotFound, AuthenticationError, ValueError, BadRequestError) as e:
|
||||
decorator_logger.critical(f"❌ ENDGUELTIGER FEHLER bei '{effective_func_name}': Permanentes Problem erkannt. {type(e).__name__} - {str(e)[:150]}...")
|
||||
decorator_logger.exception("Details:")
|
||||
raise e
|
||||
|
||||
# HTTP-Fehler, die NICHT wiederholt werden sollen
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if hasattr(e, 'response') and e.response is not None:
|
||||
status_code = e.response.status_code
|
||||
@@ -131,26 +157,30 @@ def retry_on_failure(func):
|
||||
decorator_logger.critical(f"❌ ENDGUELTIGER FEHLER bei '{effective_func_name}': HTTP Fehler {status_code} erhalten ({e.response.reason}). Nicht wiederholbar. {str(e)[:100]}...")
|
||||
decorator_logger.exception("Details:")
|
||||
raise e
|
||||
# Wenn der HTTP-Fehler wiederholbar ist (z.B. 500), wird er unten gefangen
|
||||
pass
|
||||
|
||||
|
||||
# Fehler, die wiederholt werden sollen (inkl. OpenAI-Fehler)
|
||||
except (requests.exceptions.RequestException, gspread.exceptions.APIError, APIError, wikipedia.exceptions.WikipediaException) as e:
|
||||
error_msg = str(e)
|
||||
error_type = type(e).__name__
|
||||
|
||||
if attempt < max_retries_config - 1:
|
||||
wait_time = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||||
|
||||
# Spezifisches Logging für OpenAI-Fehler
|
||||
if isinstance(e, RateLimitError):
|
||||
decorator_logger.warning(f"🚦 RATE LIMIT ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
elif isinstance(e, Timeout) and isinstance(e, OpenAIError):
|
||||
decorator_logger.warning(f"⏰ OPENAI TIMEOUT ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
decorator_logger.warning(f"🚦 OPENAI RATE LIMIT ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
elif isinstance(e, Timeout):
|
||||
decorator_logger.warning(f"⏰ TIMEOUT ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
elif isinstance(e, APIError): # Fängt alle anderen wiederholbaren OpenAI-Fehler
|
||||
decorator_logger.warning(f"🤖 OPENAI API FEHLER ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
|
||||
# Spezifisches Logging für andere Bibliotheken
|
||||
elif isinstance(e, gspread.exceptions.APIError) and hasattr(e, 'response') and e.response is not None and e.response.status_code == 429:
|
||||
decorator_logger.warning(f"🚦 GSPREAD RATE LIMIT ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
elif isinstance(e, requests.exceptions.Timeout):
|
||||
decorator_logger.warning(f"⏰ REQUESTS TIMEOUT ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
elif isinstance(e, requests.exceptions.RequestException):
|
||||
decorator_logger.warning(f"🌐 NETZWERKFEHLER ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
elif isinstance(e, OpenAIError):
|
||||
decorator_logger.warning(f"🤖 OPENAI FEHLER ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
elif isinstance(e, wikipedia.exceptions.WikipediaException):
|
||||
decorator_logger.warning(f"📚 WIKIPEDIA FEHLER ({error_type}) bei '{effective_func_name}' (Versuch {attempt+1}/{max_retries_config}). {error_msg[:150]}... Warte {wait_time:.2f}s...")
|
||||
else:
|
||||
@@ -739,21 +769,12 @@ def initialize_target_schema():
|
||||
def call_openai_chat(prompt, temperature=0.3, model=None):
|
||||
"""
|
||||
Zentrale Funktion fuer OpenAI Chat API Aufrufe.
|
||||
Wird von anderen globalen Helfern oder DataProcessor Methoden aufgerufen.
|
||||
|
||||
Args:
|
||||
prompt (str): Der Prompt-Text an die API.
|
||||
temperature (float, optional): Die Temperatur fuer die Textgenerierung. Defaults to 0.3.
|
||||
model (str, optional): Das zu verwendende OpenAI Modell. Defaults to Config.TOKEN_MODEL.
|
||||
|
||||
Returns:
|
||||
str: Der bereinigte Antwortstring von der API.
|
||||
Wirft Exception bei API-Fehlern nach Retries.
|
||||
Kompatibel mit openai v0.x und v1.x.
|
||||
"""
|
||||
logger = logging.getLogger(__name__)
|
||||
if not Config.API_KEYS.get('openai'):
|
||||
logger.error("Fehler: OpenAI API Key nicht konfiguriert.")
|
||||
raise openai.error.AuthenticationError("OpenAI API Key nicht konfiguriert.")
|
||||
raise AuthenticationError("OpenAI API Key nicht konfiguriert.") # Funktioniert in beiden Versionen
|
||||
|
||||
if not prompt or not isinstance(prompt, str) or not prompt.strip():
|
||||
logger.error("Fehler: Leerer Prompt fuer OpenAI.")
|
||||
@@ -762,35 +783,33 @@ def call_openai_chat(prompt, temperature=0.3, model=None):
|
||||
current_model = model if model else getattr(Config, 'TOKEN_MODEL', 'gpt-3.5-turbo')
|
||||
|
||||
try:
|
||||
# NEU: OpenAI v1.x Client-Instanziierung
|
||||
# Es ist Best Practice, einen Client zu erstellen, anstatt die globalen Methoden zu verwenden.
|
||||
# Der API-Schlüssel wird automatisch aus der Umgebungsvariable oder der Konfiguration gelesen.
|
||||
client = openai.OpenAI(api_key=Config.API_KEYS.get('openai'))
|
||||
|
||||
# logger.debug(f"Sende Prompt an OpenAI ({current_model})...")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=current_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=temperature
|
||||
)
|
||||
|
||||
if not response or not response.choices:
|
||||
logger.error(f"OpenAI Call erfolgreich, aber keine Choices in der Antwort erhalten. Response: {str(response)[:200]}...")
|
||||
raise APIError("Keine Choices in OpenAI Antwort erhalten.", request=None, body=None)
|
||||
|
||||
result = response.choices[0].message.content.strip() if response.choices[0].message and response.choices[0].message.content else ""
|
||||
if IS_OPENAI_V1:
|
||||
# Code für die neue v1.x Bibliothek
|
||||
client = openai.OpenAI(api_key=Config.API_KEYS.get('openai'))
|
||||
response = client.chat.completions.create(
|
||||
model=current_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=temperature
|
||||
)
|
||||
result = response.choices[0].message.content.strip() if response.choices and response.choices[0].message else ""
|
||||
else:
|
||||
# Code für die alte v0.x Bibliothek
|
||||
response = openai.ChatCompletion.create(
|
||||
api_key=Config.API_KEYS.get('openai'), # explizit übergeben
|
||||
model=current_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=temperature
|
||||
)
|
||||
result = response.choices[0].message.content.strip() if response.choices and response.choices[0].message else ""
|
||||
|
||||
if not result:
|
||||
logger.warning(f"OpenAI Call erfolgreich, erhielt aber leeren Inhalt in der Antwort. Prompt Anfang: {prompt[:100]}...")
|
||||
logger.warning(f"OpenAI Call erfolgreich, erhielt aber leeren Inhalt in der Antwort.")
|
||||
return ""
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
# Wird vom @retry_on_failure Decorator gefangen und behandelt.
|
||||
# Wir heben die Exception erneut auf, damit der Decorator sie sehen kann.
|
||||
raise e
|
||||
raise e # Wird vom @retry_on_failure Decorator gefangen
|
||||
|
||||
|
||||
def summarize_website_content(raw_text):
|
||||
|
||||
Reference in New Issue
Block a user