fix(connector): [31e88f42] Implement de-duplication for contact.created
SuperOffice sends two 'contact.created' webhooks for a single new contact. This caused the connector to process the same entity twice, leading to duplicate entries and logs. This commit introduces a de-duplication shield in the worker: - A new method is added to to check for jobs with the same company name that are either 'PROCESSING' or 'COMPLETED' within the last 5 minutes. - The worker now fetches the company name upon receiving a job, updates the job record with the name, and then calls the new de-duplication method. - If a duplicate event is detected, the job is skipped, preventing redundant processing.
This commit is contained in:
@@ -185,6 +185,31 @@ class JobQueue:
|
||||
except Exception as e:
|
||||
logger.critical(f"❌ CRITICAL: Failed to set job {job_id} to FAILED: {e}", exc_info=True)
|
||||
conn.rollback()
|
||||
|
||||
def check_for_recent_duplicate(self, entity_name: str, current_job_id: int) -> bool:
|
||||
"""
|
||||
Checks if another job with the same entity_name has been processed or is processing recently.
|
||||
"""
|
||||
five_minutes_ago = datetime.utcnow() - timedelta(minutes=5)
|
||||
with sqlite3.connect(DB_PATH, timeout=30) as conn:
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT id FROM jobs
|
||||
WHERE entity_name = ?
|
||||
AND id != ?
|
||||
AND status IN ('COMPLETED', 'PROCESSING')
|
||||
AND created_at >= ?
|
||||
""", (entity_name, current_job_id, five_minutes_ago))
|
||||
|
||||
if cursor.fetchone():
|
||||
logger.warning(f"Found recent duplicate job for entity '{entity_name}' (related to job {current_job_id}).")
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Failed to check for duplicate jobs for entity '{entity_name}': {e}", exc_info=True)
|
||||
return False # Fail safe
|
||||
|
||||
def get_stats(self):
|
||||
with sqlite3.connect(DB_PATH, timeout=30) as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
Reference in New Issue
Block a user