diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 2de575b..c881623 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -4,6 +4,7 @@ ================================================================================ 【已完成】 + - V10.314 擴大 PChome 候選池與搜尋韌性：PChome 搜尋 API 改為依 limit 掃多頁並對 429/5xx/timeout 做有限重試；feeder 預設每個商品最多 5 組搜尋詞、每詞 20 候選、2 頁，且搜尋清理不再刪掉括號/方括號內的品牌與規格，讓正確候選更有機會進 matcher，而不是長期停在「待對比」。 - V10.312 強化 PChome 商品身份比對防錯配：matcher 開始解析 mg/mcg 劑量、件組套組與多規格集合，60ml+150ml vs 60ml+20ml、10mg vs 20mg、10片 vs 10盒、精華 vs 化妝水都會進硬否決或單位價覆核，不再靠單一規格重疊放行；覆核診斷同步新增「劑量差異」標籤，降低核心比價錯配污染 daily/growth/PPT/AI 分析。 - V10.311 統一競品價差語意：`fetch_competitor_comparison_results()`、competitor PPT 與 OpenClaw competitor prompt 全部改用 `MOMO - PChome`，正值代表 MOMO 較貴 / PChome 低價壓力，負值代表 MOMO 價格優勢；避免 daily/growth 顯示價格壓力但 PPT/AI 反向解讀。 - V10.310 強化 MOMO/PChome 核心比價閉環：PChome feeder 搜尋候選只有強同款 `0.90` 才提前停止，避免第一個 0.76 次佳候選卡掉後續精準搜尋詞；人工否決的候選會被跳過並改挑下一個候選，不再讓已否決商品長期阻塞同 SKU。人工 `reject_identity`、`unit_price_required`、`needs_research` 會立即讓同候選正式 `competitor_prices` 過期，Dashboard 即使尚有舊價也不再顯示正式總價差；手機版比價覆核欄位標籤、覆核按鈕冒泡與候選證據顯示同步修正。 diff --git a/config.py b/config.py index 2c6a644..a89c6d0 100644 --- a/config.py +++ b/config.py @@ -320,7 +320,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.313" +SYSTEM_VERSION = "V10.314" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 6038d8f..d650dcd 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -606,3 +606,4 @@ POSTGRES_HOST=momo-db | 2026-04-17 | 188 .env Telegram token 不正確（split-brain）| 修正為 `8610496165`，188→Telegram message_id=282 確認 | | 2026-04-17 | NIM Tool Calling E2E | 真實 NVIDIA_API_KEY 驗證：dispatched=3, errors=[] | | 2026-05-20 | PChome 商品身份比對仍可能因單一規格重疊誤放行 | V10.312 起 matcher 解析 mg/mcg 劑量、件組套組、多規格集合與同數字不同單位；劑量/容量/重量/件數/品類衝突會硬否決或導向單位價覆核，避免錯配污染 Dashboard、daily/growth、PPT 與 AI 競價分析 | +| 2026-05-20 | 正確 PChome 候選常因只掃第一頁或搜尋詞丟失品牌/規格而未進入 matcher | V10.314 起搜尋 API 依 limit 掃多頁、對暫時性錯誤有限重試；feeder 預設 5 組搜尋詞、20 候選、2 頁，並保留括號/方括號內品牌與規格，提升覆核隊列與正式比價的候選品質 | diff --git a/services/competitor_price_feeder.py b/services/competitor_price_feeder.py index e127729..283da7b 100644 --- a/services/competitor_price_feeder.py +++ b/services/competitor_price_feeder.py @@ -38,10 +38,11 @@ logger = logging.getLogger(__name__) MIN_MATCH_SCORE = 0.76 # 低於此分數不寫入；核心比價寧可待審也不能錯配 REPLACE_DIFFERENT_PRODUCT_SCORE = 0.84 # 已有不同 PChome 商品時，需超高信心才覆蓋 EARLY_STOP_MATCH_SCORE = 0.90 # 搜尋候選池只有強同款才提前停止，避免次佳候選卡住後續精準搜尋詞 -SEARCH_LIMIT = 12 # 每個搜尋詞取 PChome 前 N 筆 -MAX_SEARCH_TERMS = 3 # 每個 MOMO 商品最多嘗試幾組搜尋詞 +SEARCH_LIMIT = int(os.getenv("PCHOME_FEEDER_SEARCH_LIMIT", "20")) # 每個搜尋詞取 PChome 前 N 筆 +MAX_SEARCH_TERMS = int(os.getenv("PCHOME_FEEDER_MAX_SEARCH_TERMS", "5")) # 每個 MOMO 商品最多嘗試幾組搜尋詞 +SEARCH_MAX_PAGES = int(os.getenv("PCHOME_FEEDER_SEARCH_MAX_PAGES", "2")) # 每個搜尋詞最多掃描 PChome 搜尋頁數 BATCH_SIZE = 30 # 每批 DB 寫入筆數 -RATE_DELAY = 0.8 # 每次 PChome 請求間隔（秒） +RATE_DELAY = float(os.getenv("PCHOME_FEEDER_RATE_DELAY", "1.0")) # 每次 PChome 請求間隔（秒） TTL_HOURS = 6 # competitor_prices 快取有效期 REQUEST_TIMEOUT = float(os.getenv("PCHOME_FEEDER_TIMEOUT", "12")) # 避免外部搜尋 API 長時間卡住排程 @@ -103,8 +104,8 @@ def _extract_tags(pchome_product) -> list: def _clean_search_text(value: str) -> str: - value = re.sub(r'[（(][^)）]*[)）]', ' ', value or '') - value = re.sub(r'[【\[].*?[】\]]', ' ', value) + value = re.sub(r'[（）()]', ' ', value or '') + value = re.sub(r'[【】\[\]]', ' ', value) value = re.sub(r'[^\w\u4e00-\u9fff]+', ' ', value) return re.sub(r'\s+', ' ', value).strip() @@ -133,6 +134,7 @@ def _build_search_keywords(momo_name: str) -> list: try: from services.marketplace_product_matcher import build_search_terms terms = build_search_terms(momo_name, max_terms=MAX_SEARCH_TERMS) + terms.append(momo_name) except Exception: logger.debug( "[Feeder] marketplace matcher failed while building search keywords; " @@ -217,7 +219,7 @@ def _search_pchome_candidates(crawler, momo_name: str, keywords: list = None, mo candidates = [] seen_ids = set() for keyword in keywords or _build_search_keywords(momo_name): - ok, _, products = crawler.search_products(keyword, limit=SEARCH_LIMIT) + ok, _, products = crawler.search_products(keyword, limit=SEARCH_LIMIT, max_pages=SEARCH_MAX_PAGES) if not ok or not products: continue for product in products: diff --git a/services/pchome_crawler.py b/services/pchome_crawler.py index 568340c..496145e 100644 --- a/services/pchome_crawler.py +++ b/services/pchome_crawler.py @@ -66,16 +66,26 @@ class PChomeCrawler: # 商品 ID 正則表達式 PRODUCT_ID_PATTERN = re.compile(r'[A-Z]{4}[A-Z0-9]{2}-?[A-Z0-9]{8,10}') - def __init__(self, timeout: int = 30, delay: float = 0.5): + def __init__( + self, + timeout: int = 30, + delay: float = 1.0, + max_retries: int = 2, + retry_backoff: float = 0.8, + ): """ 初始化爬蟲 Args: timeout: 請求超時時間 (秒) delay: 請求間隔延遲 (秒)，避免過度頻繁請求 + max_retries: 暫時性錯誤的重試次數 + retry_backoff: 指數退避基礎秒數 """ self.timeout = timeout self.delay = delay + self.max_retries = max(0, int(max_retries)) + self.retry_backoff = max(0.0, float(retry_backoff)) self.session = requests.Session() self.session.headers.update(self.DEFAULT_HEADERS) self._last_request_time = 0 @@ -87,6 +97,47 @@ class PChomeCrawler: time.sleep(self.delay - elapsed) self._last_request_time = time.time() + def _get_with_retry(self, url: str, **kwargs) -> requests.Response: + """GET with polite rate limiting and bounded retry for transient failures.""" + retryable_statuses = {429, 500, 502, 503, 504} + last_error = None + for attempt in range(self.max_retries + 1): + self._rate_limit() + try: + response = self.session.get(url, **kwargs) + status_code = getattr(response, "status_code", 200) + if ( + status_code in retryable_statuses + and attempt < self.max_retries + ): + last_error = requests.HTTPError( + f"HTTP {status_code} for {url}", + response=response, + ) + else: + response.raise_for_status() + return response + except (requests.Timeout, requests.ConnectionError, requests.HTTPError) as exc: + last_error = exc + response = getattr(exc, "response", None) + status_code = getattr(response, "status_code", None) + if ( + attempt >= self.max_retries + or ( + isinstance(exc, requests.HTTPError) + and status_code not in retryable_statuses + ) + ): + raise + + sleep_sec = self.retry_backoff * (2 ** attempt) + if sleep_sec > 0: + time.sleep(sleep_sec) + + if last_error: + raise last_error + raise requests.RequestException(f"GET failed: {url}") + def _normalize_product_id(self, product_id: str) -> str: """ 正規化商品 ID 格式 @@ -143,9 +194,7 @@ class PChomeCrawler: url = f"{self.BASE_URL}/region/{region_code}" try: - self._rate_limit() - response = self.session.get(url, timeout=self.timeout) - response.raise_for_status() + response = self._get_with_retry(url, timeout=self.timeout) product_ids = self._extract_product_ids_from_html(response.text) logger.info(f"從 {url} 取得 {len(product_ids)} 個商品 ID") @@ -178,16 +227,13 @@ class PChomeCrawler: batch = product_ids[i:i + batch_size] try: - self._rate_limit() - # 呼叫商品 API params = {'id': ','.join(batch)} - response = self.session.get( + response = self._get_with_retry( self.API_URL, params=params, timeout=self.timeout ) - response.raise_for_status() data = response.json() crawled_at = datetime.now() @@ -311,41 +357,66 @@ class PChomeCrawler: return success, message, products - def search_products(self, keyword: str, limit: int = 50) -> Tuple[bool, str, List[PChomeProduct]]: + def search_products( + self, + keyword: str, + limit: int = 50, + max_pages: Optional[int] = None, + ) -> Tuple[bool, str, List[PChomeProduct]]: """ 搜尋商品 (使用搜尋 API) Args: keyword: 搜尋關鍵字 limit: 最多回傳數量 + max_pages: 搜尋結果最多掃描頁數；預設依 limit 最多掃到 3 頁 Returns: (成功與否, 訊息, 商品資料列表) """ search_url = f"https://ecshweb.pchome.com.tw/search/v4.3/all/results" - params = { - 'q': keyword, - 'page': 1, - 'sort': 'rnk/dc', - 'cateid': '24h', - } + limit = max(1, int(limit or 1)) + page_cap = max_pages if max_pages is not None else min(3, max(1, (limit // 20) + 1)) + page_cap = max(1, int(page_cap or 1)) try: - self._rate_limit() - response = self.session.get(search_url, params=params, timeout=self.timeout) - response.raise_for_status() + product_ids = [] + seen_ids = set() + pages_scanned = 0 + for page in range(1, page_cap + 1): + params = { + 'q': keyword, + 'page': page, + 'sort': 'rnk/dc', + 'cateid': '24h', + } + response = self._get_with_retry(search_url, params=params, timeout=self.timeout) + pages_scanned += 1 - data = response.json() - prods = data.get('Prods', []) + data = response.json() + prods = data.get('Prods', []) + if not prods: + break - if not prods: + for item in prods: + product_id = item.get('Id', '') + if not product_id or product_id in seen_ids: + continue + seen_ids.add(product_id) + product_ids.append(product_id) + if len(product_ids) >= limit: + break + if len(product_ids) >= limit: + break + + if not product_ids: return False, "沒有找到符合的商品", [] - # 取得商品 ID - product_ids = [p.get('Id', '') for p in prods[:limit] if p.get('Id')] - # 取得詳細資料 - return self.fetch_product_details(product_ids) + success, message, products = self.fetch_product_details(product_ids[:limit]) + if success: + message = f"{message}；搜尋頁數 {pages_scanned}" + return success, message, products except requests.RequestException as e: logger.error(f"搜尋失敗: {e}") diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index b19ecdc..b023faf 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -418,7 +418,7 @@ def test_ai_product_pick_agent_uses_real_competitor_data_and_dashboard_action(): assert "MAX_SEARCH_TERMS" in feeder_source assert "_build_search_keywords" in feeder_source assert "_search_pchome_candidates" in feeder_source - assert "crawler.search_products(keyword, limit=SEARCH_LIMIT)" in feeder_source + assert "crawler.search_products(keyword, limit=SEARCH_LIMIT, max_pages=SEARCH_MAX_PAGES)" in feeder_source assert "_fetch_unmatched_priority_skus" in feeder_source assert "_fetch_expired_identity_skus" in feeder_source assert "run_expired_identity_refresh" in feeder_source diff --git a/tests/test_pchome_crawler_search.py b/tests/test_pchome_crawler_search.py new file mode 100644 index 0000000..7e6441e --- /dev/null +++ b/tests/test_pchome_crawler_search.py @@ -0,0 +1,149 @@ +from datetime import datetime + +import requests + + +class _FakeResponse: + def __init__(self, payload=None, status_code=200): + self._payload = payload or {} + self.status_code = status_code + self.text = "" + + def json(self): + return self._payload + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError(f"HTTP {self.status_code}", response=self) + + +def test_pchome_search_scans_multiple_pages_until_limit(monkeypatch): + from services.pchome_crawler import PChomeCrawler, PChomeProduct + + crawler = PChomeCrawler(timeout=1, delay=0, max_retries=0) + calls = [] + fetched_ids = [] + + class FakeSession: + headers = {} + + def get(self, url, params=None, timeout=None): + calls.append((url, dict(params or {}), timeout)) + page = int((params or {}).get("page") or 1) + if page == 1: + return _FakeResponse({"Prods": [{"Id": "A001"}, {"Id": "A002"}]}) + if page == 2: + return _FakeResponse({"Prods": [{"Id": "A002"}, {"Id": "A003"}]}) + return _FakeResponse({"Prods": []}) + + def fake_fetch_product_details(product_ids, batch_size=20): + fetched_ids.extend(product_ids) + return True, "details ok", [ + PChomeProduct( + product_id=product_id, + name=f"商品 {product_id}", + price=100, + original_price=120, + discount=17, + image_url="", + product_url=f"https://24h.pchome.com.tw/prod/{product_id}", + stock=10, + store="24h", + rating=None, + review_count=0, + is_on_sale=True, + crawled_at=datetime.now(), + ) + for product_id in product_ids + ] + + crawler.session = FakeSession() + monkeypatch.setattr(crawler, "fetch_product_details", fake_fetch_product_details) + + success, message, products = crawler.search_products("理膚寶水", limit=3, max_pages=3) + + assert success is True + assert "搜尋頁數 2" in message + assert fetched_ids == ["A001", "A002", "A003"] + assert [call[1]["page"] for call in calls] == [1, 2] + assert [product.product_id for product in products] == ["A001", "A002", "A003"] + + +def test_pchome_get_retries_transient_timeout(): + from services.pchome_crawler import PChomeCrawler + + crawler = PChomeCrawler(timeout=1, delay=0, max_retries=1, retry_backoff=0) + calls = [] + + class FakeSession: + headers = {} + + def get(self, url, **kwargs): + calls.append((url, kwargs)) + if len(calls) == 1: + raise requests.Timeout("temporary timeout") + return _FakeResponse({"ok": True}) + + crawler.session = FakeSession() + + response = crawler._get_with_retry("https://example.test/api", timeout=1) + + assert response.json() == {"ok": True} + assert len(calls) == 2 + + +def test_feeder_search_cleanup_preserves_bracket_brand_and_specs(): + from services.competitor_price_feeder import _clean_search_text + + cleaned = _clean_search_text("【蘭蔻】絕對完美玫瑰霜(60ml)+玫瑰精露150ml") + + assert "蘭蔻" in cleaned + assert "60ml" in cleaned + assert "150ml" in cleaned + + +def test_feeder_search_candidate_passes_page_cap(monkeypatch): + from services.competitor_price_feeder import _search_pchome_candidates + from services.pchome_crawler import PChomeProduct + + product = PChomeProduct( + product_id="DDAB01-PAGE2", + name="理膚寶水 B5 修復霜 40ml", + price=679, + original_price=799, + discount=15, + image_url="", + product_url="https://24h.pchome.com.tw/prod/DDAB01-PAGE2", + stock=20, + store="24h", + rating=4.7, + review_count=8, + is_on_sale=True, + crawled_at=datetime.now(), + ) + calls = [] + + class FakeCrawler: + def search_products(self, keyword, **kwargs): + calls.append((keyword, kwargs)) + return True, "ok", [product] + + monkeypatch.setattr( + "services.marketplace_product_matcher.score_marketplace_match", + lambda *_args, **_kwargs: type( + "Diagnostics", + (), + {"score": 0.95}, + )(), + ) + + candidates = _search_pchome_candidates( + FakeCrawler(), + "理膚寶水 B5 修復霜 40ml", + keywords=["理膚寶水 B5 40ml"], + momo_price=699, + ) + + assert candidates == [product] + assert calls[0][1]["limit"] == 20 + assert calls[0][1]["max_pages"] == 2