feat: field based de-dupe

crm/lead_syncing/doctype/failed_lead_sync_log/failed_lead_sync_log.json

@@ -9,7 +9,9 @@
   "column_break_dhay",
   "source",
   "section_break_fhot",
-  "lead_data"
+  "lead_data",
+  "section_break_knec",
+  "traceback"
  ],
  "fields": [
   {
@@ -43,13 +45,23 @@
   {
    "fieldname": "section_break_fhot",
    "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "section_break_knec",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "traceback",
+   "fieldtype": "Code",
+   "label": "Traceback",
+   "read_only": 1
   }
  ],
  "grid_page_length": 50,
  "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2025-10-19 18:30:15.381205",
+ "modified": "2025-10-19 18:53:32.192551",
  "modified_by": "Administrator",
  "module": "Lead Syncing",
  "name": "Failed Lead Sync Log",

crm/lead_syncing/doctype/failed_lead_sync_log/failed_lead_sync_log.py

@@ -16,6 +16,7 @@ class FailedLeadSyncLog(Document):
 
 		lead_data: DF.Code | None
 		source: DF.Link | None
+		traceback: DF.Code | None
 		type: DF.Literal["Duplicate", "Failure"]
 	# end: auto-generated types
 

crm/lead_syncing/doctype/lead_sync_source/facebook.py

@@ -1,10 +1,15 @@
 import frappe
+from frappe.exceptions import ValidationError
 from frappe.integrations.utils import make_get_request
 
 FB_GRAPH_API_BASE = "https://graph.facebook.com"
 FB_GRAPH_API_VERSION = "v23.0"
 
 
+class DuplicateLeadError(ValidationError):
+	pass
+
+
 def get_fb_graph_api_url(endpoint: str) -> str:
 	if endpoint.startswith("/"):
 		endpoint = endpoint[1:]
@@ -40,18 +45,17 @@ class FacebookSyncSource:
 			crm_lead_data["facebook_form_id"] = self.form_id
 
 			try:
+				self.validate_duplicate_lead(crm_lead_data, question_to_field_map)
 				frappe.get_doc(
 					{
 						"doctype": "CRM Lead",
 						**crm_lead_data,
 					}
 				).insert(ignore_permissions=True)
-			except frappe.UniqueValidationError:
+			except (frappe.UniqueValidationError, DuplicateLeadError):
-				# Skip duplicate leads based on facebook_lead_id
-				# TODO: de-duplication based on field values
 				self.create_failure_log(lead, "Duplicate")
 			except Exception:
-				self.create_failure_log(lead)
+				self.create_failure_log(lead, traceback=frappe.get_traceback(with_context=True))
 
 		self.update_last_synced_at()
 
@@ -89,13 +93,16 @@ class FacebookSyncSource:
 			"Lead Sync Source", self.source_name or {"facebook_lead_form": self.form_id}, "last_synced_at"
 		)
 
-	def create_failure_log(self, lead_data: dict | None = None, type: str = "Failure"):
+	def create_failure_log(
+		self, lead_data: dict | None = None, type: str = "Failure", traceback: str | None = None
+	):
 		return frappe.get_doc(
 			{
 				"doctype": "Failed Lead Sync Log",
 				"type": type,
 				"lead_data": frappe.as_json(lead_data),
-				"source": self.get_source_name()
+				"source": self.get_source_name(),
+				"traceback": traceback,
 			}
 		).insert(ignore_permissions=True)
 
@@ -113,6 +120,12 @@ class FacebookSyncSource:
 
 		return frappe.db.get_value("Lead Sync Source", {"facebook_lead_form": self.form_id}, "name")
 
+	def validate_duplicate_lead(self, lead_data: dict, field_mapping: dict):
+		validation_filters = {crm_field: lead_data[crm_field] for crm_field in field_mapping.values()}
+		validation_filters["facebook_form_id"] = lead_data["facebook_form_id"]  # only for this campaign
+		if frappe.db.exists("CRM Lead", validation_filters):
+			raise DuplicateLeadError
+
 
 @frappe.whitelist()
 def fetch_and_store_pages_from_facebook(access_token: str) -> list[dict]: