import json
import re
import asyncio
import requests
import uuid
import html
from typing import Dict, Any, Optional, List
from urllib.parse import urljoin, urlparse, parse_qs, urlencode

import jingrow
from jingrow.utils.jinja import render_template
from bs4 import BeautifulSoup
from crawl4ai import AsyncWebCrawler


# ==================== 产品列表页提取函数 ====================

def extract_current_category(soup):
    """提取当前分类名称"""
    # 方法1: 查找 div.sr-txt-title 中的 h1.sr-txt-h2
    title_div = soup.find('div', class_='sr-txt-title')
    if title_div:
        h1_tag = title_div.find('h1', class_='sr-txt-h2')
        if h1_tag:
            category_name = h1_tag.get_text(strip=True)
            if category_name:
                return category_name
    
    # 方法2: 直接查找 h1.sr-txt-h2
    category_title = soup.find('h1', class_='sr-txt-h2')
    if category_title:
        category_name = category_title.get_text(strip=True)
        if category_name:
            return category_name
    
    # 方法3: 查找包含 itemscope 的 div.sr-txt-title 中的 h1
    title_divs = soup.find_all('div', class_='sr-txt-title')
    for div in title_divs:
        if div.get('itemscope'):
            h1_tag = div.find('h1')
            if h1_tag:
                category_name = h1_tag.get_text(strip=True)
                if category_name:
                    return category_name
    
    return None


def extract_products_from_page(soup, base_url=''):
    """从单个页面提取产品信息"""
    products = []
    
    # 找到所有产品项容器
    product_items = soup.find_all('div', class_='prod-result-item')
    
    # 遍历每个产品项，提取信息
    for item in product_items:
        product = {}
        
        # 提取产品ID（从data-prodid属性）
        product['product_id'] = item.get('data-prodid', '')
        
        # 提取产品标题和链接（从 prod-title 中的 a 标签）
        title_div = item.find('div', class_='prod-title')
        if title_div:
            title_link = title_div.find('a')
            if title_link:
                product['title'] = title_link.get_text(strip=True)
                product['url'] = title_link.get('href', '')
                # 确保URL是完整的
                if product['url'] and not product['url'].startswith('http'):
                    if product['url'].startswith('//'):
                        product['url'] = 'https:' + product['url']
                    elif product['url'].startswith('/'):
                        if base_url:
                            parsed = urlparse(base_url)
                            product['url'] = f"{parsed.scheme}://{parsed.netloc}" + product['url']
                        else:
                            product['url'] = base_url + product['url']
        
        # 提取产品价格（从 prod-price 中）
        price_div = item.find('div', class_='prod-price')
        if price_div:
            # 方法1: 从title属性获取完整价格信息
            price_title = price_div.get('title', '')
            if price_title:
                product['price'] = price_title
            else:
                # 方法2: 从内部文本提取
                price_value = price_div.find('span', class_='value')
                price_unit = price_div.find('span', class_='unit')
                if price_value:
                    price_text = price_value.get_text(strip=True)
                    if price_unit:
                        price_text += ' ' + price_unit.get_text(strip=True)
                    product['price'] = price_text
        
        # 提取最小订购量（从 min-order 中）
        min_order_div = item.find('div', class_='min-order')
        if min_order_div:
            # 从title属性获取
            min_order_title = min_order_div.get('title', '')
            if min_order_title:
                product['min_order'] = min_order_title
            else:
                # 从内部文本提取
                min_order_value = min_order_div.find('span', class_='value')
                if min_order_value:
                    product['min_order'] = min_order_value.get_text(strip=True)
        
        # 提取产品图片（从 prod-image 中的 img 标签）
        image_div = item.find('div', class_='prod-image')
        if image_div:
            img_tag = image_div.find('img')
            if img_tag:
                # 优先使用 data-original，其次使用 src
                img_url = img_tag.get('data-original') or img_tag.get('src', '')
                if img_url:
                    # 确保URL是完整的
                    if img_url.startswith('//'):
                        img_url = 'https:' + img_url
                    elif img_url.startswith('/'):
                        if base_url:
                            parsed = urlparse(base_url)
                            img_url = f"{parsed.scheme}://{parsed.netloc}" + img_url
                        else:
                            img_url = base_url + img_url
                    product['image'] = img_url
                    product['image_alt'] = img_tag.get('alt', '')
        
        # 检查是否有视频标记
        video_mark = item.find('div', class_='prod-video-mark')
        product['has_video'] = video_mark is not None
        
        # 只添加有标题的产品（确保数据完整）
        if product.get('title'):
            products.append(product)
    
    return products


def get_next_page_url(soup, current_url):
    """检查是否有下一页，如果有则返回下一页URL"""
    # 查找分页器
    pager = soup.find('div', class_='pager')
    if not pager:
        return None
    
    # 方法1: 查找 class 包含 "next" 的 a 标签
    # 排除包含 "disClick" 的（禁用状态，表示没有下一页）
    next_links = pager.find_all('a', class_=lambda x: x and 'next' in x if x else False)
    
    for next_link in next_links:
        # 检查是否被禁用
        classes = next_link.get('class', [])
        if 'disClick' in classes:
            continue  # 跳过禁用的链接
        
        # 获取href属性
        next_url = next_link.get('href')
        if not next_url:
            continue
        
        # 过滤掉 JavaScript URL
        if next_url.startswith('javascript:'):
            # 尝试从 JavaScript 函数中提取页码，然后构建 URL
            # 例如：javascript:submitSearchByPage(2) -> 提取页码 2
            page_match = re.search(r'submitSearchByPage\((\d+)\)', next_url)
            if page_match:
                page_num = page_match.group(1)
                # 尝试从当前 URL 构建下一页 URL
                # 查找当前 URL 中的页码参数
                parsed = urlparse(current_url)
                query_params = {}
                if parsed.query:
                    query_params = parse_qs(parsed.query)
                
                # 更新页码参数
                # 常见的页码参数名：page, p, pageNum, pageNumber, pn
                page_param_found = False
                for param_name in ['page', 'p', 'pageNum', 'pageNumber', 'pn', 'currentPage']:
                    if param_name in query_params:
                        query_params[param_name] = [page_num]
                        page_param_found = True
                        break
                
                # 如果没有找到页码参数，尝试添加
                if not page_param_found:
                    query_params['page'] = [page_num]
                
                # 重建 URL
                new_query = urlencode(query_params, doseq=True)
                next_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{new_query}"
            else:
                # 无法从 JavaScript URL 提取页码，跳过
                continue
        
        # 确保URL是完整的
        if next_url.startswith('//'):
            next_url = 'https:' + next_url
        elif next_url.startswith('/'):
            # 从当前URL提取基础URL
            parsed = urlparse(current_url)
            base_url = f"{parsed.scheme}://{parsed.netloc}"
            next_url = urljoin(base_url, next_url)
        elif not next_url.startswith(('http://', 'https://', 'file://', 'raw:')):
            # 相对路径
            parsed = urlparse(current_url)
            base_url = f"{parsed.scheme}://{parsed.netloc}"
            next_url = urljoin(base_url, next_url)
        
        # 验证 URL 格式
        if not next_url.startswith(('http://', 'https://', 'file://', 'raw:')):
            continue
        
        # 确保下一页URL与当前URL不同
        if next_url != current_url:
            return next_url
    
    # 方法2: 如果方法1失败，尝试查找所有分页链接，找到当前页的下一个
    all_page_links = pager.find_all('a')
    current_page_num = None
    
    # 尝试从当前 URL 提取页码
    parsed = urlparse(current_url)
    if parsed.query:
        query_params = parse_qs(parsed.query)
        for param_name in ['page', 'p', 'pageNum', 'pageNumber', 'pn', 'currentPage']:
            if param_name in query_params:
                try:
                    current_page_num = int(query_params[param_name][0])
                    break
                except (ValueError, IndexError):
                    pass
    
    # 如果找到了当前页码，尝试找到下一页
    if current_page_num is not None:
        next_page_num = current_page_num + 1
        # 查找包含下一页数字的链接
        for link in all_page_links:
            link_text = link.get_text(strip=True)
            try:
                if int(link_text) == next_page_num:
                    href = link.get('href')
                    if href and not href.startswith('javascript:'):
                        # 构建下一页 URL
                        query_params = parse_qs(parsed.query)
                        for param_name in ['page', 'p', 'pageNum', 'pageNumber', 'pn', 'currentPage']:
                            if param_name in query_params:
                                query_params[param_name] = [str(next_page_num)]
                                break
                        else:
                            query_params['page'] = [str(next_page_num)]
                        
                        new_query = urlencode(query_params, doseq=True)
                        next_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{new_query}"
                        if next_url != current_url:
                            return next_url
            except ValueError:
                continue
    
    return None


# ==================== 产品详情页提取函数 ====================

def extract_product_detail_images(soup, base_url=''):
    """从产品详情页提取所有图片（从 bigImg-wrap 中提取大图）"""
    images = []
    
    # 从 bigImg-wrap 中提取大图
    big_img_wrap = soup.find('div', class_='bigImg-wrap') or soup.find('div', class_='J-bigImg-wrap')
    if big_img_wrap:
        # 查找所有 J-pic-large-item 元素
        pic_large_items = big_img_wrap.find_all('div', class_='J-pic-large-item')
        for item in pic_large_items:
            img_url = None
            
            # 优先从 fsrc 属性获取
            pic_inside = item.find('div', class_='sr-proMainInfo-slide-picInside')
            if pic_inside:
                img_url = pic_inside.get('fsrc')
            
            # 如果没有 fsrc，从 enlargeHref 链接中的 img 获取 src
            if not img_url:
                enlarge_href = item.find('a', class_='enlargeHref')
                if enlarge_href:
                    img_tag = enlarge_href.find('img')
                    if img_tag:
                        img_url = img_tag.get('src')
            
            if img_url:
                # 确保URL是完整的
                if img_url.startswith('//'):
                    img_url = 'https:' + img_url
                elif img_url.startswith('/'):
                    if base_url:
                        parsed = urlparse(base_url)
                        img_url = f"{parsed.scheme}://{parsed.netloc}" + img_url
                    else:
                        img_url = base_url + img_url
                
                if img_url not in images:
                    images.append(img_url)
    
    return images


def extract_product_detail_prices(soup):
    """从产品详情页提取价格信息（统一格式：{"US$718.00":"5-19"}）"""
    prices = []
    
    # 方法1: 提取swiper格式的价格（多个价格项）
    swiper_wrapper = soup.find('div', class_='swiper-wrapper-div')
    if swiper_wrapper:
        # 查找所有价格项
        slide_divs = swiper_wrapper.find_all('div', class_='swiper-slide-div')
        for slide_div in slide_divs:
            # 查找价格容器
            money_container = slide_div.find('div', class_='swiper-money-container')
            # 查找数量容器
            unit_container = slide_div.find('div', class_='swiper-unit-container')
            
            if money_container and unit_container:
                price_text = money_container.get_text(strip=True)
                unit_text = unit_container.get_text(strip=True)
                
                # 提取数量范围（移除"Pieces"等词，只保留数字范围）
                # 使用正则表达式提取数字和符号（如 "5-19", "50+", "20-49"）
                # 匹配数字范围格式：数字-数字 或 数字+
                unit_match = re.search(r'(\d+\s*[-+]\s*\d+|\d+\+)', unit_text)
                if unit_match:
                    unit_text = unit_match.group(1).strip()
                else:
                    # 如果没匹配到，移除"Pieces"等词，提取数字
                    unit_text = unit_text.replace('Pieces', '').replace('Piece', '').strip()
                    # 提取数字，如果没有范围，添加+号表示最小数量
                    num_match = re.search(r'(\d+)', unit_text)
                    if num_match:
                        unit_text = num_match.group(1) + '+'
                    unit_text = ' '.join(unit_text.split())  # 清理多余空格
                
                if price_text and unit_text:
                    # 格式: {"US$718.00":"5-19"}
                    price_obj = {price_text: unit_text}
                    prices.append(price_obj)
    
    # 方法2: 如果swiper格式没有找到，尝试提取only-one-price格式（单个价格范围）
    if not prices:
        only_one_price = soup.find('div', class_='only-one-priceNum')
        if only_one_price:
            # 提取价格范围
            price_td = only_one_price.find('td')
            if price_td:
                price_span = price_td.find('span', class_='only-one-priceNum-td-left')
                if price_span:
                    price_text = price_span.get_text(strip=True)
                    
                    # 提取数量信息
                    price_info_td = only_one_price.find('td', class_='sa-only-property-price')
                    unit_text = ''
                    has_moq = False
                    if price_info_td:
                        # 先检查整个容器的文本是否包含MOQ
                        container_text = price_info_td.get_text(strip=True)
                        if 'MOQ' in container_text or 'moq' in container_text.lower():
                            has_moq = True
                        
                        # 查找所有span标签
                        quantity_spans = price_info_td.find_all('span')
                        for span in quantity_spans:
                            span_text = span.get_text(strip=True)
                            # 提取数字（可能是 "5 Pieces" 或 "5"）
                            num_match = re.search(r'(\d+)', span_text)
                            if num_match:
                                unit_text = num_match.group(1)
                                # 检查当前span是否包含MOQ
                                if 'MOQ' in span_text or 'moq' in span_text.lower():
                                    has_moq = True
                                break
                    
                    # 如果提取到了价格，即使没有数量信息也保存（数量设为空或默认值）
                    if price_text:
                        # 如果没有提取到数量，使用默认值"1+"（表示最小订购量为1）
                        if not unit_text:
                            unit_text = '1+'
                        # only-one-price格式通常表示最小订购量，默认添加+号
                        elif not unit_text.endswith('+'):
                            unit_text += '+'
                        # 统一格式: {"US$1,018.00 - 1,118.00":"5+"}
                        price_obj = {price_text: unit_text}
                        prices.append(price_obj)
    
    return prices


def extract_company_info(soup, base_url=''):
    """从产品详情页提取公司信息"""
    company_info = {
        'company_name': '',
        'company_url': ''
    }
    
    # 查找公司信息容器
    com_div = soup.find('div', class_='sr-com')
    if com_div:
        # 查找公司信息
        com_info = com_div.find('div', class_='sr-com-info')
        if com_info:
            # 查找公司标题
            title_div = com_info.find('div', class_='title-txt')
            if title_div:
                company_link = title_div.find('a')
                if company_link:
                    company_info['company_name'] = company_link.get('title', '') or company_link.get_text(strip=True)
                    company_info['company_url'] = company_link.get('href', '')
                    
                    # 确保URL是完整的
                    if company_info['company_url'] and not company_info['company_url'].startswith('http'):
                        if company_info['company_url'].startswith('//'):
                            company_info['company_url'] = 'https:' + company_info['company_url']
                        elif company_info['company_url'].startswith('/'):
                            if base_url:
                                parsed = urlparse(base_url)
                                company_info['company_url'] = f"{parsed.scheme}://{parsed.netloc}" + company_info['company_url']
                            else:
                                company_info['company_url'] = base_url + company_info['company_url']
    
    return company_info


def extract_product_details(soup):
    """从产品详情页提取Product Details表格信息"""
    details = {}
    
    # 方法1: 查找包含"Product Details"文本的元素
    details_elements = soup.find_all(string=lambda text: text and 'Product Details' in text)
    
    for details_text in details_elements:
        # 向上查找包含表格的容器
        parent = details_text.find_parent()
        
        # 在父元素的后续兄弟元素中查找表格
        current = parent
        for _ in range(5):  # 最多查找5层
            if current:
                # 在当前元素及其后续兄弟中查找表格
                table = current.find('table')
                if not table:
                    # 查找下一个兄弟元素
                    next_sibling = current.find_next_sibling()
                    if next_sibling:
                        table = next_sibling.find('table')
                
                if table:
                    # 提取表格数据
                    rows = table.find_all('tr')
                    for row in rows:
                        th = row.find('th')
                        td = row.find('td')
                        if th and td:
                            key = th.get_text(strip=True).replace(':', '').strip()
                            value = td.get_text(strip=True)
                            if key and value:
                                details[key] = value
                    break  # 找到表格后退出
                
                # 向上查找
                current = current.find_parent()
            else:
                break
    
    # 方法2: 如果没找到，直接查找所有包含th和td的表格
    if not details:
        tables = soup.find_all('table')
        for table in tables:
            rows = table.find_all('tr')
            # 检查表格是否包含Product Details相关的key
            product_detail_keys = ['Customization', 'Type', 'Material', 'Application', 'Max Bear Weight', 'Warranty']
            for row in rows:
                th = row.find('th')
                td = row.find('td')
                if th and td:
                    key = th.get_text(strip=True).replace(':', '').strip()
                    value = td.get_text(strip=True)
                    # 如果key匹配产品详情的关键词，则提取
                    if key and value and any(keyword in key for keyword in product_detail_keys):
                        details[key] = value
    
    return details


def extract_product_title(soup):
    """提取产品标题"""
    # 优先级：h1.sr-proMainInfo-baseInfoH1 > span > og:title > title标签
    h1_tag = soup.find('h1', class_='sr-proMainInfo-baseInfoH1')
    if h1_tag:
        title_span = h1_tag.find('span')
        if title_span:
            return title_span.get_text(strip=True)
    
    # 如果h1没有找到，尝试从og:title提取
    og_title = soup.find('meta', property='og:title')
    if og_title and og_title.get('content'):
        return og_title.get('content').replace('[Hot Item]', '').strip()
    
    # 如果还是没有，从title标签提取
    title_tag = soup.find('title')
    if title_tag:
        title_text = title_tag.get_text(strip=True)
        # 移除 "- Smith Machine and Bench Press price" 等后缀
        if ' - ' in title_text:
            return title_text.split(' - ')[0].strip()
        else:
            return title_text
    
    return ''


def convert_bsc_info_to_table(bsc_info):
    """将 bsc-info 的 div 结构转换为表格格式"""
    if not bsc_info:
        return ''
    
    # 查找所有子区块
    subblocks = bsc_info.find_all('div', class_='sr-layout-subblock')
    if not subblocks:
        return ''
    
    html_parts = []
    
    for subblock in subblocks:
        # 提取标题
        title_elem = subblock.find('div', class_='sr-txt-title')
        title = ''
        if title_elem:
            h2 = title_elem.find('h2')
            if h2:
                title = h2.get_text(strip=True)
        
        # 查找所有 bsc-item
        content_div = subblock.find('div', class_='sr-layout-content')
        if not content_div:
            continue
        
        basic_info_list = content_div.find('div', class_='basic-info-list')
        if not basic_info_list:
            continue
        
        items = basic_info_list.find_all('div', class_='bsc-item')
        if not items:
            continue
        
        # 构建表格HTML
        table_html = []
        if title:
            title_escaped = html.escape(title)
            table_html.append(f'<h2 style="margin: 20px 0 10px 0; font-size: 18px; font-weight: 600;">{title_escaped}</h2>')
        
        table_html.append('<table style="width: 100%; border-collapse: collapse; margin-bottom: 20px; border: 1px solid #e5e7eb;">')
        table_html.append('<tbody>')
        
        for item in items:
            label_elem = item.find('div', class_='bac-item-label')
            value_elem = item.find('div', class_='bac-item-value')
            
            if not label_elem or not value_elem:
                continue
            
            label_text = label_elem.get_text(strip=True)
            value_text = value_elem.get_text(strip=True)
            
            if not label_text and not value_text:
                continue
            
            # HTML转义，确保特殊字符正确显示
            label_text_escaped = html.escape(label_text)
            value_text_escaped = html.escape(value_text)
            
            # 统一使用两列布局：标签列和值列
            table_html.append('<tr>')
            table_html.append(f'<td style="padding: 10px 12px; border: 1px solid #e5e7eb; width: 30%; background-color: #f9fafb; font-weight: 500; vertical-align: top;">{label_text_escaped}</td>')
            table_html.append(f'<td style="padding: 10px 12px; border: 1px solid #e5e7eb; width: 70%; background-color: #ffffff; vertical-align: top;">{value_text_escaped}</td>')
            table_html.append('</tr>')
        
        table_html.append('</tbody>')
        table_html.append('</table>')
        
        html_parts.append(''.join(table_html))
    
    return ''.join(html_parts)


def extract_product_description(soup, base_url=''):
    """从产品详情页提取完整的产品详细描述（只提取 bsc-info 部分，并转换为表格格式）"""
    detail_tab_item = soup.find('div', class_='detail-tab')
    if detail_tab_item:
        # 查找 bsc-info 容器
        bsc_info = detail_tab_item.find('div', class_='bsc-info')
        if bsc_info:
            # 转换为表格格式
            table_html = convert_bsc_info_to_table(bsc_info)
            if table_html:
                return table_html
    
    return ''


# ==================== 爬取函数 ====================

async def crawl_product_detail(crawler, product_url, product_index=None, total_products=None, base_url=''):
    """爬取单个产品详情页并提取详细信息"""
    try:
        result = await crawler.arun(url=product_url, wait_for="networkidle", page_timeout=30000)
        
        if not result.success:
            return {}
        
        soup = BeautifulSoup(result.html, 'html.parser')
        detail_info = {}
        
        # 提取标题
        title = extract_product_title(soup)
        if title:
            detail_info['title'] = title
        
        # 提取图片
        images = extract_product_detail_images(soup, base_url)
        if images:
            detail_info['images'] = images
        
        # 提取价格
        prices = extract_product_detail_prices(soup)
        if prices:
            detail_info['prices'] = prices
        
        # 提取公司信息
        company_info = extract_company_info(soup, base_url)
        if company_info.get('company_name'):
            detail_info['company_name'] = company_info['company_name']
            detail_info['company_url'] = company_info['company_url']
        
        # 提取Product Details
        product_details = extract_product_details(soup)
        if product_details:
            detail_info['Product Details'] = product_details
        
        # 提取完整的产品详细描述
        description = extract_product_description(soup, base_url)
        if description:
            detail_info['description'] = description
        
        return detail_info
        
    except Exception as e:
        return {}


async def crawl_single_list_page(crawler, url, page_num, base_url=''):
    """爬取单个产品列表页"""
    result = await crawler.arun(url=url, wait_for="networkidle")
    
    if not result.success:
        return None, None, None
    
    soup = BeautifulSoup(result.html, 'html.parser')
    
    # 提取产品
    products = extract_products_from_page(soup, base_url)
    
    # 检查是否有下一页
    next_url = get_next_page_url(soup, url)
    
    # 只在第一页提取分类名称
    category_name = None
    if page_num == 1:
        category_name = extract_current_category(soup)
    
    return products, next_url, category_name


def get_branch_upstream_nodes(current_node_id, flow_data):
    """获取同一分支的所有上游节点ID列表"""
    if not current_node_id or not flow_data:
        return []
    
    edges = flow_data.get("edges", [])
    upstream_nodes = set()
    stack = [current_node_id]
    
    while stack:
        node_id = stack.pop()
        for edge in edges:
            if edge.get("target") == node_id:
                source_id = edge.get("source")
                if source_id and source_id not in upstream_nodes:
                    upstream_nodes.add(source_id)
                    stack.append(source_id)
    
    return list(upstream_nodes)


async def download_and_upload_image(image_url: str, record_name: str, record_type: str, field_name: str = None) -> Optional[str]:
    """下载图片并上传到服务器"""
    try:
        # 下载图片
        response = requests.get(image_url, verify=False, timeout=30)
        if response.status_code != 200:
            return None
        
        # 获取文件扩展名
        parsed_url = urlparse(image_url)
        path = parsed_url.path
        file_extension = "jpg"
        if path:
            ext = path.split('.')[-1].lower()
            if ext in ['jpg', 'jpeg', 'png', 'gif', 'webp']:
                file_extension = ext
        
        # 生成文件名
        filename = f"scraped_{uuid.uuid4().hex[:8]}.{file_extension}"
        
        # 上传图片
        upload_result = jingrow.upload_file(
            file_data=response.content,
            filename=filename,
            attached_to_pagetype=record_type,
            attached_to_name=record_name,
            attached_to_field=field_name
        )
        
        if upload_result.get('success'):
            return upload_result.get('file_url')
        return None
    except Exception as e:
        print(f"[ERROR] 图片上传失败 {image_url}: {str(e)}")
        return None


async def upload_images(images: List[str], record_name: str, record_type: str, field_name: str = None) -> List[str]:
    """批量上传图片"""
    uploaded_urls = []
    for image_url in images:
        if not image_url or not image_url.startswith('http'):
            continue
        uploaded_url = await download_and_upload_image(image_url, record_name, record_type, field_name)
        if uploaded_url:
            uploaded_urls.append(uploaded_url)
    return uploaded_urls


def get_or_create_menu(menu_name: str, menu_pagetype: str, category_pagetype: str, category_record_name: str, site: str = None, parent_menu_name: str = None) -> Optional[str]:
    """查找或创建菜单，返回菜单记录的name，失败返回None
    菜单关联到分类，按照分类的层级结构创建
    """
    if not menu_name or not menu_pagetype or not category_record_name:
        return None
    
    if not site:
        return None  # site字段是必填的
    
    try:
        # 通过title和site字段查找菜单
        filters = [["title", "=", menu_name], ["site", "=", site]]
        existing = jingrow.get_list(menu_pagetype, filters=filters, limit=1)
        if existing:
            return existing[0].get("name")
        
        # 根据category_pagetype确定menu_type和对应的category字段
        menu_type = None
        category_field = None
        if "Product Category" in category_pagetype:
            menu_type = "Product Category"
            category_field = "product_category"
        elif "Article Category" in category_pagetype:
            menu_type = "Article Category"
            category_field = "article_category"
        elif "Page Category" in category_pagetype:
            menu_type = "Page Category"
            category_field = "page_category"
        elif "Project Category" in category_pagetype:
            menu_type = "Project Category"
            category_field = "project_category"
        elif "Presentation Category" in category_pagetype:
            menu_type = "Presentation Category"
            category_field = "presentation_category"
        
        if not menu_type or not category_field:
            return None  # 不支持的分类类型
        
        # 创建菜单数据
        menu_data = {
            "title": menu_name,
            "menu_type": menu_type,
            "position": "Header",  # 默认位置
            "status": "Published",  # 默认状态
            "site": site,
            category_field: category_record_name  # 关联到分类
        }
        
        # 查找父菜单并设置
        if parent_menu_name:
            parent_filters = [["title", "=", parent_menu_name], ["site", "=", site]]
            parent_menus = jingrow.get_list(menu_pagetype, filters=parent_filters, limit=1)
            if parent_menus:
                parent_name = parent_menus[0].get("name")
                menu_data["parent_jsite_menu"] = parent_name
        
        created = jingrow.create_pg(menu_pagetype, menu_data)
        if created:
            return created.get("name")
    except Exception:
        pass
    
    return None


def get_or_create_category(category_name: str, category_pagetype: str, site: str = None, parent_category: str = "Products") -> Optional[str]:
    """查找或创建分类，返回分类记录的name，失败返回None
    统一使用title字段查找和创建分类
    如果提供了site，会在查找和创建时使用site字段过滤
    创建分类时会自动设置父分类（默认值为"Products"）
    """
    if not category_name or not category_pagetype:
        return None
    
    try:
        # 通过title字段查找，如果提供了site则加上site过滤
        filters = [["title", "=", category_name]]
        if site:
            filters.append(["site", "=", site])
        existing = jingrow.get_list(category_pagetype, filters=filters, limit=1)
        if existing:
            return existing[0].get("name")
        
        # 不存在则创建
        category_data = {"title": category_name}
        if site:
            category_data["site"] = site
        
        # 查找父分类并设置
        if parent_category:
            parent_filters = [["title", "=", parent_category]]
            if site:
                parent_filters.append(["site", "=", site])
            parent_categories = jingrow.get_list(category_pagetype, filters=parent_filters, limit=1)
            if parent_categories:
                parent_name = parent_categories[0].get("name")
                # 父分类字段名格式：parent_分类pagetype名称（小写下划线格式）
                parent_field = "parent_" + category_pagetype.lower().replace(" ", "_")
                category_data[parent_field] = parent_name
        
        created = jingrow.create_pg(category_pagetype, category_data)
        if created:
            return created.get("name")
    except Exception:
        pass
    
    return None


def map_product_data_to_record(product_data: Dict[str, Any], field_map: List[Dict], label2field: Dict, record_type: str, default_site: str = "", category_name: str = None, category_field: str = None, category_pagetype: str = None, parent_category: str = "Products") -> Dict[str, Any]:
    """将产品数据映射为记录字段"""
    record_data = {}
    mapped_fields = set()
    
    # 识别图片字段（需要特殊处理，不能直接映射）
    image_source_fields = set()
    image_target_fields = set()
    for mapping in field_map:
        from_field = mapping.get("from")
        to_field = mapping.get("to")
        if from_field in ["image", "images"]:
            image_source_fields.add(from_field)
            if to_field:
                image_target_field = label2field.get(str(to_field).strip(), to_field)
                if image_target_field:
                    image_target_fields.add(image_target_field)
    
    # 1. 优先使用 field_map 映射（排除图片字段）
    if field_map:
        # 过滤掉图片字段的映射
        filtered_field_map = [m for m in field_map if m.get("from") not in image_source_fields]
        if filtered_field_map:
            mapped_result = jingrow.map_fields_by_labels(filtered_field_map, product_data, label2field)
            record_data.update(mapped_result)
        mapped_fields.update([m.get("from") for m in field_map if m.get("from")])
    
    # 2. 自动标签映射：处理未映射的字段（排除图片字段）
    for label, value in product_data.items():
        if label in mapped_fields or label in image_source_fields:
            continue
        fieldname = label2field.get(str(label).strip())
        if fieldname is not None and fieldname not in record_data and fieldname not in image_target_fields:
            record_data[fieldname] = value
            mapped_fields.add(label)
    
    # 3. 处理特殊必填字段（如果配置了 default_site，尝试设置 site 字段）
    if default_site:
        # 查找 site 字段
        site_fieldname = None
        for label, fieldname in label2field.items():
            if "网站" in str(label) or "site" in str(label).lower() or fieldname == "site":
                site_fieldname = fieldname
                break
        
        # 如果没找到，尝试使用 "site" 作为字段名
        if not site_fieldname:
            # 检查 "site" 是否在字段名集合中
            if "site" in label2field.values():
                site_fieldname = "site"
            else:
                # 如果 label2field 中有 site 相关的字段，使用第一个
                for fieldname in label2field.values():
                    if "site" in fieldname.lower():
                        site_fieldname = fieldname
                        break
        
        # 如果找到了 site 字段且未设置，则设置默认值
        if site_fieldname and site_fieldname not in record_data:
            site_value = str(default_site).strip()
            # 验证 default_site 不是 URL
            if not site_value.startswith(('http://', 'https://')):
                record_data[site_fieldname] = site_value
    
    # 4. 处理分类字段（Link类型）
    if category_name and category_field and category_pagetype:
        # 获取site值（从record_data中获取，或使用default_site）
        site_value = None
        # 先尝试从record_data中获取site字段值
        for key, value in record_data.items():
            if "site" in key.lower():
                site_value = value
                break
        # 如果record_data中没有，使用default_site
        if not site_value and default_site:
            site_value = str(default_site).strip()
            if site_value.startswith(('http://', 'https://')):
                site_value = None
        
        category_record_name = get_or_create_category(category_name, category_pagetype, site_value, parent_category)
        if category_record_name:
            record_data[category_field] = category_record_name
    
    # 5. 处理字段长度限制
    processed_data = {}
    for key, value in record_data.items():
        if isinstance(value, str) and len(value) > 140 and key in ['title', 'name']:
            processed_data[key] = value[:140]
        elif isinstance(value, (list, dict)):
            try:
                processed_data[key] = json.dumps(value, ensure_ascii=False)
            except:
                processed_data[key] = str(value)
        else:
            processed_data[key] = value
    
    return processed_data


async def create_record_async(product_data: Dict[str, Any], config: Dict[str, Any], label2field: Dict, category_name: str = None) -> Dict[str, Any]:
    """异步创建记录"""
    try:
        record_type = config.get("pagetype")
        field_map = config.get("field_map", [])
        default_site = config.get("default_site", "")
        
        # 获取分类字段配置
        category_field = None
        category_pagetype = None
        if category_name:
            # 从config获取分类字段配置（优先使用新配置方式）
            category_field = config.get("category_field", "category")
            category_pagetype = config.get("category_pagetype", "Jsite Product Category")
            
            # 兼容旧配置方式
            if not category_field or not category_pagetype:
                category_config = config.get("category", {})
                if category_config:
                    category_field = category_field or category_config.get("field")
                    category_pagetype = category_pagetype or category_config.get("pagetype", "Jsite Product Category")
            
            # 如果字段名未配置，尝试从label2field自动查找
            if not category_field:
                for label, fieldname in label2field.items():
                    if "分类" in str(label) or "category" in str(label).lower():
                        category_field = fieldname
                        # 如果pagetype未配置，尝试从字段元数据获取
                        if not category_pagetype or category_pagetype == "Jsite Product Category":
                            try:
                                meta = jingrow.get_meta(record_type)
                                if meta and meta.get("success"):
                                    fields = meta.get("data", {}).get("fields", [])
                                    for field in fields:
                                        if field.get("fieldname") == fieldname and field.get("fieldtype") == "Link":
                                            category_pagetype = field.get("options") or "Jsite Product Category"
                                            break
                            except Exception:
                                pass
                        break
        
        # 获取父分类配置（默认值为"Products"）
        parent_category = config.get("parent_category", "Products")
        
        # 映射字段
        record_data = map_product_data_to_record(product_data, field_map, label2field, record_type, default_site, category_name, category_field, category_pagetype, parent_category)
        
        # 处理图片上传
        image_field = None
        for mapping in field_map:
            if mapping.get("from") in ["image", "images"]:
                to_field = mapping.get("to")
                if to_field:
                    image_field = label2field.get(str(to_field).strip(), to_field)
                    break
        
        # 如果没找到映射，尝试自动查找图片字段
        if not image_field:
            for label, fieldname in label2field.items():
                if "图片" in str(label) or "image" in str(label).lower():
                    image_field = fieldname
                    break
        
        # 上传图片
        images_to_upload = []
        if "images" in product_data and isinstance(product_data["images"], list):
            images_to_upload = product_data["images"]
        elif "image" in product_data and product_data["image"]:
            images_to_upload = [product_data["image"]]
        
        # 先创建记录（需要 record_name 来上传图片）
        # 获取标题作为记录名称
        record_name = record_data.get("title") or record_data.get("name") or f"product_{uuid.uuid4().hex[:8]}"
        if isinstance(record_name, str) and len(record_name) > 140:
            record_name = record_name[:140]
        
        # 创建记录
        created = jingrow.create_pg(record_type, record_data)
        
        if not created:
            return {
                "success": False,
                "error": "创建记录失败",
                "data": product_data
            }
        
        created_name = created.get('name') or record_name
        
        # 上传图片（如果有）
        update_data = {}
        if images_to_upload and image_field:
            uploaded_urls = await upload_images(images_to_upload, created_name, record_type, image_field)
            if uploaded_urls:
                # 只设置第一张图片作为主图（避免多张图片时设置数组）
                # 如果目标字段支持多张图片，可以通过其他方式处理
                update_data[image_field] = uploaded_urls[0]
        
        # 如果有需要更新的数据，更新记录
        if update_data:
            jingrow.update_pg(record_type, created_name, update_data)
        
        return {
            "success": True,
            "record_name": created_name,
            "data": product_data
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e),
            "data": product_data
        }


def prepare_category_and_menu(config: Dict[str, Any], category_name: str = None, site: str = None) -> bool:
    """在采集开始前创建分类和菜单（最佳实践）
    返回True表示成功，False表示失败
    """
    if not category_name:
        return True  # 如果没有分类名称，跳过
    
    try:
        category_pagetype = config.get("category_pagetype", "Jsite Product Category")
        parent_category = config.get("parent_category", "Products")
        menu_pagetype = "Jsite Menu"
        
        # 获取site值
        if not site and config.get("default_site"):
            site = str(config.get("default_site")).strip()
            if site.startswith(('http://', 'https://')):
                site = None
        
        # 1. 确保父分类存在
        parent_category_name = get_or_create_category(parent_category, category_pagetype, site, None)
        if not parent_category_name:
            jingrow.log_error(f"警告：无法创建或找到父分类 {parent_category}")
        
        # 2. 创建分类
        category_record_name = get_or_create_category(category_name, category_pagetype, site, parent_category)
        if not category_record_name:
            jingrow.log_error(f"警告：无法创建或找到分类 {category_name}")
            return False
        
        # 3. 创建对应的菜单（按照同样的层级结构）
        if site:
            # 先确保父菜单存在（如果已存在则直接返回，不会重复创建）
            if parent_category_name:
                get_or_create_menu(parent_category, menu_pagetype, category_pagetype, parent_category_name, site, None)
            
            # 创建分类对应的菜单（如果已存在则直接返回，不会重复创建）
            if category_record_name:
                menu_record_name = get_or_create_menu(category_name, menu_pagetype, category_pagetype, category_record_name, site, parent_category)
                if not menu_record_name:
                    jingrow.log_error(f"警告：无法创建或找到菜单 {category_name}")
                    # 菜单创建失败不影响采集，只记录警告
        
        return True
    except Exception as e:
        jingrow.log_error(f"准备分类和菜单时出错: {str(e)}")
        return False


async def crawl_and_create_list(crawler, start_url: str, config: Dict[str, Any], base_url: str = '', max_pages: int = 100):
    """爬取产品列表并逐条创建记录"""
    record_type = config.get("pagetype")
    if not record_type:
        return {
            "success": False,
            "error": "pagetype 必须配置"
        }
    
    # 获取字段映射
    label2field = jingrow.get_field_mapping_from_jingrow(record_type) or {}
    
    current_url = start_url
    page_num = 1
    # 优先使用配置的分类名称，如果没有配置才从页面提取
    category_name = config.get("category_name")
    created_count = 0
    failed_count = 0
    created_records = []
    failed_records = []
    
    # 在采集开始前，提前创建分类和菜单（最佳实践）
    # 先尝试从第一页提取分类名称（如果需要）
    if not category_name:
        products, _, cat_name = await crawl_single_list_page(crawler, start_url, 1, base_url)
        if cat_name:
            category_name = cat_name
    
    # 获取site值用于创建分类和菜单
    site_value = None
    if config.get("default_site"):
        site_value = str(config.get("default_site")).strip()
        if site_value.startswith(('http://', 'https://')):
            site_value = None
    
    # 提前创建分类和菜单（如果分类名称已确定）
    if category_name:
        prepare_category_and_menu(config, category_name, site_value)
    
    while current_url and page_num <= max_pages:
        # 爬取当前页
        products, next_url, cat_name = await crawl_single_list_page(crawler, current_url, page_num, base_url)
        
        if not products:
            break
        
        # 为每个产品爬取详情并创建记录
        for product in products:
            if not product.get('url'):
                continue
            
            # 爬取产品详情
            detail_info = await crawl_product_detail(crawler, product['url'], None, None, base_url)
            
            # 合并数据（列表页数据 + 详情页数据）
            full_product_data = product.copy()
            if detail_info:
                # 更新标题（使用详情页的标题，更准确）
                if 'title' in detail_info:
                    full_product_data['title'] = detail_info['title']
                
                # 提取副标题（如果有合适的来源字段）
                # 可以从 title 中提取副标题，例如 "主标题 - 副标题"
                if 'title' in full_product_data:
                    title_parts = full_product_data['title'].split(' - ', 1)
                    if len(title_parts) == 2:
                        full_product_data['title'] = title_parts[0].strip()
                        full_product_data['subtitle'] = title_parts[1].strip()
                
                # 更新图片（使用详情页的图片列表，更完整）
                if 'images' in detail_info and detail_info['images']:
                    full_product_data['images'] = detail_info['images']
                    full_product_data['image'] = detail_info['images'][0]
                elif 'image' in full_product_data and not full_product_data.get('images'):
                    # 如果详情页没有图片，使用列表页的图片
                    full_product_data['images'] = [full_product_data['image']]
                
                # 更新价格（使用详情页的价格，更详细）
                if 'prices' in detail_info:
                    full_product_data['prices'] = detail_info['prices']
                
                # 添加公司信息
                if 'company_name' in detail_info:
                    full_product_data['company_name'] = detail_info['company_name']
                if 'company_url' in detail_info:
                    full_product_data['company_url'] = detail_info['company_url']
                
                # 添加Product Details（完整的产品详情）
                if 'Product Details' in detail_info:
                    full_product_data['Product Details'] = detail_info['Product Details']
                
                # 添加完整的产品详细描述
                if 'description' in detail_info:
                    full_product_data['description'] = detail_info['description']
            
            # 异步创建记录（传入分类名称）
            create_result = await create_record_async(full_product_data, config, label2field, category_name)
            
            if create_result.get('success'):
                created_count += 1
                created_records.append({
                    "record_name": create_result.get('record_name'),
                    "title": full_product_data.get('title')
                })
            else:
                failed_count += 1
                failed_records.append({
                    "title": full_product_data.get('title'),
                    "error": create_result.get('error')
                })
            
            # 添加延迟，避免请求过快
            await asyncio.sleep(0.5)
        
        # 检查是否有下一页
        if next_url:
            current_url = next_url
            page_num += 1
            await asyncio.sleep(1)
        else:
            break
    
    return {
        "success": True,
        "category_name": category_name,
        "total_pages": page_num,
        "created_count": created_count,
        "failed_count": failed_count,
        "created_records": created_records,
        "failed_records": failed_records
    }


def execute(context=None, inputs=None, config=None, **kwargs):
    """
    网站采集并创建记录节点（一体化）
    采集一条数据后立即异步创建一条记录
    """
    if context is None:
        context = kwargs.get("context", {})
    if inputs is None:
        inputs = kwargs.get("inputs", {})
    if config is None:
        config = kwargs.get("config", {})
    
    try:
        # 参数处理
        if not isinstance(inputs, dict):
            try:
                inputs = json.loads(inputs) if isinstance(inputs, str) else {}
            except Exception:
                inputs = {}
        
        # 获取配置
        url = config.get("url") or inputs.get("url")
        if not url:
            return {"success": False, "error": "URL 必填"}
        
        # 获取 pagetype
        record_type = config.get("pagetype")
        if not record_type:
            return {"success": False, "error": "pagetype 必须配置"}
        
        # 支持从上游节点获取 URL
        current_node_id = context.get("current_node_id")
        flow_data = context.get("flow_data", {})
        branch_upstream_nodes = get_branch_upstream_nodes(current_node_id, flow_data)
        
        if context and "node_results" in context and branch_upstream_nodes:
            node_results = context["node_results"]
            for upstream_node_id in branch_upstream_nodes:
                if upstream_node_id in node_results:
                    upstream_result = node_results[upstream_node_id]
                    if isinstance(upstream_result, dict):
                        if "ai_raw_response" in upstream_result and isinstance(upstream_result["ai_raw_response"], str):
                            try:
                                raw_response = upstream_result["ai_raw_response"].strip()
                                start_brace = raw_response.find('{')
                                end_brace = raw_response.rfind('}')
                                
                                if start_brace != -1 and end_brace != -1 and end_brace > start_brace:
                                    json_content = raw_response[start_brace:end_brace + 1]
                                    parsed_json = json.loads(json_content)
                                    if isinstance(parsed_json, dict):
                                        for k, v in parsed_json.items():
                                            if k not in inputs or not inputs.get(k):
                                                inputs[k] = v
                            except:
                                pass
                        
                        for k, v in upstream_result.items():
                            if k not in inputs or not inputs.get(k):
                                inputs[k] = v
        
        if inputs.get("url") and not url:
            url = inputs.get("url")
        
        # 构建模板上下文
        template_context = inputs.copy()
        if context and context.get("flow_id"):
            flow_id = context["flow_id"]
            agent_data = jingrow.get_pg("Local Ai Agent", str(flow_id))
            if agent_data:
                for key, value in agent_data.items():
                    if key not in template_context or not template_context.get(key):
                        template_context[key] = value
        
        # 渲染 URL 模板
        try:
            url = render_template(url, template_context)
        except Exception as e:
            return {"success": False, "error": f"URL 模板渲染失败: {str(e)}"}
        
        # URL 验证和规范化
        if not url or not isinstance(url, str):
            return {"success": False, "error": "URL 不能为空"}
        
        url = url.strip()
        if not url:
            return {"success": False, "error": "URL 不能为空"}
        
        if not url.startswith(('http://', 'https://', 'file://', 'raw:')):
            if url.startswith('//'):
                url = 'https:' + url
            elif not url.startswith('/'):
                url = 'https://' + url
            else:
                return {"success": False, "error": f"URL 格式不正确: {url}"}
        
        # 获取配置
        max_pages = config.get("max_pages", 100)
        
        # 提取基础URL
        parsed = urlparse(url)
        base_url = f"{parsed.scheme}://{parsed.netloc}"
        
        # 执行爬取和创建
        async def run_crawl():
            async with AsyncWebCrawler(verbose=True) as crawler:
                return await crawl_and_create_list(crawler, url, config, base_url, max_pages)
        
        result = asyncio.run(run_crawl())
        return result
        
    except Exception as e:
        return {
            "success": False,
            "error": f"执行失败: {str(e)}"
        }

