feat: de-duplicate leads

crm/fcrm/doctype/crm_lead/crm_lead.json

@@ -55,7 +55,9 @@
   "first_response_time",
   "first_responded_on",
   "log_tab",
-  "status_change_log"
+  "status_change_log",
+  "syncing_tab",
+  "facebook_lead_id"
  ],
  "fields": [
   {
@@ -325,13 +327,24 @@
    "label": "Net Total",
    "options": "currency",
    "read_only": 1
+  },
+  {
+   "fieldname": "syncing_tab",
+   "fieldtype": "Tab Break",
+   "label": "Syncing"
+  },
+  {
+   "fieldname": "facebook_lead_id",
+   "fieldtype": "Data",
+   "label": "Facebook Lead ID",
+   "unique": 1
   }
  ],
  "grid_page_length": 50,
  "image_field": "image",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2025-05-14 19:51:06.184569",
+ "modified": "2025-09-29 19:21:49.483883",
  "modified_by": "Administrator",
  "module": "FCRM",
  "name": "CRM Lead",

crm/fcrm/doctype/crm_lead/crm_lead.py

@@ -14,6 +14,52 @@ from crm.fcrm.doctype.crm_status_change_log.crm_status_change_log import (
 
 
 class CRMLead(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from crm.fcrm.doctype.crm_products.crm_products import CRMProducts
+		from crm.fcrm.doctype.crm_status_change_log.crm_status_change_log import CRMStatusChangeLog
+		from frappe.types import DF
+
+		annual_revenue: DF.Currency
+		communication_status: DF.Link | None
+		converted: DF.Check
+		email: DF.Data | None
+		facebook_lead_id: DF.Data | None
+		first_name: DF.Data
+		first_responded_on: DF.Datetime | None
+		first_response_time: DF.Duration | None
+		gender: DF.Link | None
+		image: DF.AttachImage | None
+		industry: DF.Link | None
+		job_title: DF.Data | None
+		last_name: DF.Data | None
+		lead_name: DF.Data | None
+		lead_owner: DF.Link | None
+		middle_name: DF.Data | None
+		mobile_no: DF.Data | None
+		naming_series: DF.Literal["CRM-LEAD-.YYYY.-"]
+		net_total: DF.Currency
+		no_of_employees: DF.Literal["1-10", "11-50", "51-200", "201-500", "501-1000", "1000+"]
+		organization: DF.Data | None
+		phone: DF.Data | None
+		products: DF.Table[CRMProducts]
+		response_by: DF.Datetime | None
+		salutation: DF.Link | None
+		sla: DF.Link | None
+		sla_creation: DF.Datetime | None
+		sla_status: DF.Literal["", "First Response Due", "Failed", "Fulfilled"]
+		source: DF.Link | None
+		status: DF.Link
+		status_change_log: DF.Table[CRMStatusChangeLog]
+		territory: DF.Link | None
+		total: DF.Currency
+		website: DF.Data | None
+	# end: auto-generated types
+
 	def before_validate(self):
 		self.set_sla()
 

crm/lead_syncing/doctype/lead_sync_source/lead_sync_source.json

@@ -58,14 +58,15 @@
    "fieldname": "facebook_lead_form",
    "fieldtype": "Link",
    "label": "Facebook Lead Form",
-   "options": "Facebook Lead Form"
+   "options": "Facebook Lead Form",
+   "unique": 1
   }
  ],
  "grid_page_length": 50,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2025-09-29 18:45:32.305860",
+ "modified": "2025-09-29 19:03:14.804026",
- "modified_by": "hussain@frappe.io",
+ "modified_by": "Administrator",
  "module": "Lead Syncing",
  "name": "Lead Sync Source",
  "naming_rule": "Autoincrement",

crm/lead_syncing/doctype/lead_sync_source/lead_sync_source.py

@@ -46,6 +46,14 @@ class LeadSyncSource(Document):
 
 def sync_leads_from_facebook(access_token: str, lead_form_id: str) -> None:
 	url = get_fb_graph_api_url(f"/{lead_form_id}/leads")
+	last_synced_at = frappe.db.get_value(
+		"Lead Sync Source", {"facebook_lead_form": lead_form_id}, "last_synced_at"
+	)
+	timestamp = frappe.utils.data.get_timestamp(last_synced_at)
+	filtering = f"filtering=[{{'field':'time_created','operator':'GREATER_THAN','value':{timestamp}}}]"
+	if last_synced_at:
+		url = f"{url}?{filtering}"
+
 	leads = make_get_request(
 		url,
 		params={
@@ -60,24 +68,33 @@ def sync_leads_from_facebook(access_token: str, lead_form_id: str) -> None:
 	)
 
 	# Map form questions to CRM Lead fields
-	question_to_field_map = {q["key"]: q["mapped_to_crm_field"]
+	question_to_field_map = {
-		for q in form_questions
+		q["key"]: q["mapped_to_crm_field"] for q in form_questions if q["mapped_to_crm_field"]
-		if q["mapped_to_crm_field"]
 	}
 
 	for lead in leads:
 		lead_data = {item["name"]: item["values"][0] for item in lead["field_data"]}
-		crm_lead_data = {question_to_field_map.get(k): v for
+		crm_lead_data = {
-			k, v in lead_data.items() if k in question_to_field_map
+			question_to_field_map.get(k): v for k, v in lead_data.items() if k in question_to_field_map
 		}
 		crm_lead_data["source"] = "Facebook"
+		crm_lead_data["facebook_lead_id"] = lead["id"]
 
+		try:
 			frappe.get_doc(
 				{
 					"doctype": "CRM Lead",
 					**crm_lead_data,
 				}
 			).insert(ignore_permissions=True)
+		except frappe.UniqueValidationError:
+			# Skip duplicate leads based on facebook_lead_id
+			frappe.log_error("Duplicate lead skipped")
+			continue
+
+	frappe.db.set_value(
+		"Lead Sync Source", {"facebook_lead_form": lead_form_id}, "last_synced_at", frappe.utils.now()
+	)
 
 
 def fetch_and_store_pages_from_facebook(access_token: str) -> None: