fix: [30388f42] Maximiere DB-Performance und behebe Locking-Endlosschleife
- Aktiviert den SQLite WAL-Modus für echtes Concurrent Reading/Writing. - Optimiert get_next_job, um unnötige EXCLUSIVE-Locks zu vermeiden. - Dies stellt sicher, dass Jobs nach der Verarbeitung korrekt als COMPLETED markiert werden und der Worker nicht in einer Wiederholungsschleife gefangen bleibt.
This commit is contained in:
@@ -11,6 +11,8 @@ class JobQueue:
|
|||||||
|
|
||||||
def _init_db(self):
|
def _init_db(self):
|
||||||
with sqlite3.connect(DB_PATH, timeout=30) as conn:
|
with sqlite3.connect(DB_PATH, timeout=30) as conn:
|
||||||
|
# Enable WAL mode for better concurrency
|
||||||
|
conn.execute("PRAGMA journal_mode=WAL")
|
||||||
conn.execute("""
|
conn.execute("""
|
||||||
CREATE TABLE IF NOT EXISTS jobs (
|
CREATE TABLE IF NOT EXISTS jobs (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
@@ -66,35 +68,31 @@ class JobQueue:
|
|||||||
conn.row_factory = sqlite3.Row
|
conn.row_factory = sqlite3.Row
|
||||||
cursor = conn.cursor()
|
cursor = conn.cursor()
|
||||||
|
|
||||||
# Lock the job
|
cursor.execute("""
|
||||||
cursor.execute("BEGIN EXCLUSIVE")
|
SELECT id, event_type, payload, created_at
|
||||||
try:
|
FROM jobs
|
||||||
cursor.execute("""
|
WHERE status = 'PENDING'
|
||||||
SELECT id, event_type, payload, created_at
|
AND (next_try_at IS NULL OR next_try_at <= datetime('now'))
|
||||||
FROM jobs
|
ORDER BY created_at ASC
|
||||||
WHERE status = 'PENDING'
|
LIMIT 1
|
||||||
AND (next_try_at IS NULL OR next_try_at <= datetime('now'))
|
""")
|
||||||
ORDER BY created_at ASC
|
row = cursor.fetchone()
|
||||||
LIMIT 1
|
|
||||||
""")
|
if row:
|
||||||
row = cursor.fetchone()
|
job = dict(row)
|
||||||
|
# Mark as processing
|
||||||
if row:
|
cursor.execute(
|
||||||
job = dict(row)
|
"UPDATE jobs SET status = 'PROCESSING', updated_at = datetime('now') WHERE id = ?",
|
||||||
# Mark as processing
|
(job['id'],)
|
||||||
cursor.execute(
|
)
|
||||||
"UPDATE jobs SET status = 'PROCESSING', updated_at = datetime('now') WHERE id = ?",
|
conn.commit()
|
||||||
(job['id'],)
|
|
||||||
)
|
|
||||||
conn.commit()
|
|
||||||
else:
|
|
||||||
conn.rollback() # No job found
|
|
||||||
except Exception:
|
|
||||||
conn.rollback()
|
|
||||||
raise
|
|
||||||
|
|
||||||
if job:
|
if job:
|
||||||
job['payload'] = json.loads(job['payload'])
|
try:
|
||||||
|
job['payload'] = json.loads(job['payload'])
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to parse payload for job {job['id']}: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
return job
|
return job
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user