diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 924ec54..f9eece3 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -4,6 +4,7 @@ ================================================================================ 【已完成】 + - V10.528 將 `recover-stale` 救援 preview 改成輕量雙階段篩選：SQL 從過期 `competitor_prices` 小集合出發，只做 identity_v2、過期、exact/total_price/price_alert_exact 等必要條件並限制候選量，再用 `JOIN LATERAL` 取 ACTIVE 商品最新 MOMO 價；variant / catalog / commercial condition / 高風險名稱訊號改在 Python 對小樣本過濾，避免正式站看板狀態端點因全量 price_records、JSONB + regex 過重查詢拖垮 app worker。 - V10.527 收斂 PChome 過期 identity 搜尋救援隊列：`recover-stale` 不再直接吃全部過期 `identity_v2`，改走 `_fetch_expired_identity_recovery_skus()`，只收既有正式診斷為 `exact_identity / total_price / price_alert_exact` 且無 variant、catalog、commercial condition、count、bundle、unit-price 等阻擋理由的舊配對；名稱含任選、多款、香味、色號、即期、融燭燈、香氛蠟燭等高風險訊號也先排除，避免慢速 fresh search 把人工覆核型 stale pair 全部掃進來。 - V10.526 將 PChome 近門檻重評池與過期 identity 搜尋救援變成可觀測、可操作產線：`preview_retryable_candidate_revalidation()` / `preview_expired_identity_recovery()` 都是 read-only，不啟動 PChome 搜尋、不呼叫 LLM、不寫 DB；`/api/ai/pchome-match/backfill/status` 回傳 `revalidation_preview` / `stale_recovery_preview`，Dashboard 顯示「可重評 / 窄門 / 可救援」數字，並新增「救援過期 40 筆」按鈕呼叫 `/api/ai/pchome-match/recover-stale`，只在舊 PChome ID 缺失或低分時走受控 fresh-search recovery，最後仍經 hard veto、auto price write safety 與 overwrite protection。 - V10.525 補高分 review-gated exact 舊候選重評入口：`run_retryable_candidate_revalidation()` 仍以 `low_score / refresh_low_score / recoverable_low_score` 為主，只額外允許 Beauty Foot / KAMERIA / TS6 / Vaseline 這批已補 focused exact 規則、舊分數 >= 0.95、無商業狀態 / 款式 / 入數 / 組合阻擋理由的 `true_low_confidence` 進窄門重評，讓 V10.523 的安全規則可以實際回收舊資料，不把所有人工審核候選打開。 diff --git a/config.py b/config.py index e325613..1140f65 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.527" +SYSTEM_VERSION = "V10.528" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index ee94530..8cbebe0 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -98,7 +98,7 @@ SQL漏斗(~300筆) - PChome re-score 回收線：`rescore_accepted_current` 只能表示最新版 matcher 判定「可人工採用」，不可直接寫入正式 `competitor_prices`；`fetch_competitor_coverage()` 必須輸出 `rescore_accepted_count`，Dashboard、daily/growth 與 OpenClaw 競品摘要都要把「重算可採用待審」獨立呈現，避免和一般低信心/單位價覆核混在一起。 - PChome 低信心操作分流：Dashboard 與 read-only `/api/pchome-review/queue` 必須把近門檻可救、證據不足、低信心舊候選拆成 `recoverable_low_score`、`true_low_confidence`、`legacy_low_score` 三個可篩選桶；廣義 `low_score` 僅作 repository/export 相容查詢，不可在 UI 中冒充單一操作分流。 - `run_retryable_candidate_revalidation()` 的自動回刷主戰場仍限 `low_score` / `refresh_low_score` / `recoverable_low_score`；`true_low_confidence` 只有在已補 focused exact 規則的窄範圍品線、舊分數 >= 0.95、`comparison_mode='exact_identity'`、含 `strong_exact_spec_match` 且不含 commercial / variant / count / bundle / refill 等阻擋理由時，才可進入重評，不得全面打開人工審核池。 -- `/api/ai/pchome-match/backfill/status` 必須把近門檻重評池與過期 identity 救援池以只讀 `revalidation_preview` / `stale_recovery_preview` 曝光給操作員；預覽只復用正式候選 SQL 並受 limit / 60 秒快取限制，不啟動 PChome 搜尋、不呼叫 LLM、不寫 `competitor_match_attempts` / `competitor_prices`，其中 `review_gated_count` 僅代表窄門 `true_low_confidence` exact 候選，不得被解讀為全量人工池可自動回刷。 +- `/api/ai/pchome-match/backfill/status` 必須把近門檻重評池與過期 identity 救援池以只讀 `revalidation_preview` / `stale_recovery_preview` 曝光給操作員；預覽只復用正式候選 SQL 並受 limit / 60 秒快取限制，不啟動 PChome 搜尋、不呼叫 LLM、不寫 `competitor_match_attempts` / `competitor_prices`。救援 preview 必須從過期 `competitor_prices` 小集合出發並用 `JOIN LATERAL` 取最新 MOMO 價，不得掃全量 `price_records`；其中 `review_gated_count` 僅代表窄門 `true_low_confidence` exact 候選，不得被解讀為全量人工池可自動回刷。 - PChome re-score audit 預設必須先取每個 SKU 的最新 `competitor_match_attempts` 狀態，再套用 status / reason 篩選；舊低信心歷史候選只能透過 `--include-historical-candidates` 明確進入考古掃描，避免已入隊、已否決或已修正 SKU 被舊紀錄重新推回報表。 - production re-score `--apply-accepted` 僅可追加 `rescore_accepted_current` attempt 給人工覆核；執行後需清除 Dashboard / competitor intel cache，且必須抽查 `competitor_prices` / `competitor_price_history` 未新增正式價差。 - production re-score 若曾把 `variant_selection_review` 追加成 `rescore_accepted_current`，必須用 `audit_competitor_match_attempt_rescore.py --retract-variant-accepted` 追加最新 `true_low_confidence` 退回列；此路徑只寫 `competitor_match_attempts`，不得刪歷史紀錄，也不得寫 `competitor_prices` / `competitor_price_history`。 diff --git a/docs/memory/history_logs.md b/docs/memory/history_logs.md index 6f90cc4..9cfd85f 100644 --- a/docs/memory/history_logs.md +++ b/docs/memory/history_logs.md @@ -13,6 +13,7 @@ ## 📅 詳細更新日誌 (考古存檔) ### 2026-06-01：PChome 比價新鮮度操作閉環 +- **V10.528 recover-stale preview 輕量化**: V10.527 的救援隊列在正式站 preview 時曾造成 status API 超時。改為雙階段篩選：SQL 從過期 `competitor_prices` 小集合出發，只做 identity_v2、過期、exact/total_price/price_alert_exact 等必要條件並限制候選量，再用 `JOIN LATERAL` 取 ACTIVE 商品最新 MOMO 價；variant / catalog / commercial condition / 高風險名稱訊號改在 Python 對小樣本過濾，避免 `/api/ai/pchome-match/backfill/status` 因全量 price_records、JSONB + regex preview 查詢拖垮。 - **V10.527 PChome 過期 identity 搜尋救援隊列收斂**: V10.526 production smoke 發現直接對全部過期 `identity_v2` 做 rescue 會把香氛 / 色號 / 目錄款 / 商業狀態差異等人工覆核型 stale pair 送進慢速 fresh search，20 筆耗時 361 秒且 0 筆成功。新增 `_fetch_expired_identity_recovery_skus()` 作為救援專用隊列，只收既有正式診斷為 `exact_identity / total_price / price_alert_exact` 且無 variant、catalog、commercial condition、count、bundle、unit-price 等阻擋理由的舊配對；名稱含任選、多款、香味、色號、即期、融燭燈、香氛蠟燭等高風險訊號先排除。 - **V10.526 PChome 重評預覽與過期 identity 搜尋救援**: `/api/ai/pchome-match/backfill/status` 新增 60 秒快取的 `revalidation_preview` 與 `stale_recovery_preview`，Dashboard 補抓產線顯示「可重評 / 窄門 / 可救援」數字；兩個 preview 都只讀 DB，不啟動 PChome 搜尋、不呼叫 LLM、不寫 `competitor_match_attempts` 或正式價格表。另新增 `/api/ai/pchome-match/recover-stale` 與「救援過期 40 筆」按鈕，對過期 `identity_v2` 先查既有 product_id，只有在舊 ID 缺失或低分時才走受控 fresh-search recovery，最後仍經 hard veto、auto price write safety 與 overwrite protection 才能寫入正式比價。 - **V10.525 高分 review-gated exact 舊候選窄門重評**: `run_retryable_candidate_revalidation()` 保持主戰場為 `low_score / refresh_low_score / recoverable_low_score`，只額外收 Beauty Foot、KAMERIA、TS6、Vaseline 這批已由 V10.523 補 focused exact 規則的 `true_low_confidence` 舊候選。入口要求舊分數 >= 0.95、仍為 `exact_identity`、具備 `strong_exact_spec_match`，且不得含 `commercial_condition_gap`、variant、count、bundle、refill 等阻擋理由；讓已驗證真同款可被回刷，不把整個人工審核池自動打開。 diff --git a/services/competitor_price_feeder.py b/services/competitor_price_feeder.py index aac764a..da24f6d 100644 --- a/services/competitor_price_feeder.py +++ b/services/competitor_price_feeder.py @@ -130,6 +130,28 @@ STALE_IDENTITY_RECOVERY_BLOCK_NAME_PATTERN = ( r"融燭燈|融蠟燈|香氛蠟燭|精油蠟燭|蠟燭|限定|組合任選)" ) + +def _json_list(value) -> list: + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + if isinstance(value, str): + try: + parsed = json.loads(value) + return parsed if isinstance(parsed, list) else [] + except Exception: + return [] + return [] + + +def _has_stale_identity_recovery_block(row: dict) -> bool: + reasons = set(str(reason) for reason in _json_list(row.get("diagnostic_reasons"))) + if reasons & STALE_IDENTITY_RECOVERY_BLOCK_REASONS: + return True + haystack = f"{row.get('name') or ''} {row.get('competitor_product_name') or ''}" + return bool(re.search(STALE_IDENTITY_RECOVERY_BLOCK_NAME_PATTERN, haystack, flags=re.IGNORECASE)) + # ── Feeder 結果 ─────────────────────────────────────── @dataclass class FeederResult: @@ -1325,72 +1347,83 @@ class CompetitorPriceFeeder: raise RuntimeError("需要注入 SQLAlchemy engine") from sqlalchemy import text + candidate_limit = max(1, min(int(limit) * 6, 400)) + target_limit = max(1, min(int(limit), 120)) sql = text(f""" - WITH latest_momo AS ( + WITH expired_competitor AS ( SELECT - p.id AS product_id, - p.i_code AS sku, - p.name, - p.category, - pr.price AS momo_price, - ROW_NUMBER() OVER (PARTITION BY p.id ORDER BY pr.timestamp DESC) AS rn - FROM products p - JOIN price_records pr ON pr.product_id = p.id - WHERE p.status = 'ACTIVE' + cp.sku, + cp.competitor_product_id, + cp.competitor_product_name, + cp.match_score, + cp.expires_at, + COALESCE(cp.match_diagnostic_json->'reasons', '[]'::jsonb) AS diagnostic_reasons + FROM competitor_prices cp + WHERE cp.source = 'pchome' + AND cp.competitor_product_id IS NOT NULL + AND cp.competitor_product_id <> '' + AND cp.expires_at IS NOT NULL + AND cp.expires_at <= CURRENT_TIMESTAMP + AND COALESCE(cp.match_score, 0) >= :match_score_floor + AND COALESCE(cp.tags, '[]'::jsonb) ? 'identity_v2' + AND ( + COALESCE(cp.tags, '[]'::jsonb) ? 'price_basis_total_price' + OR cp.match_diagnostic_json->>'price_basis' = 'total_price' + ) + AND ( + COALESCE(cp.tags, '[]'::jsonb) ? 'alert_tier_price_alert_exact' + OR cp.match_diagnostic_json->>'alert_tier' = 'price_alert_exact' + ) + AND COALESCE(cp.match_diagnostic_json->>'comparison_mode', 'exact_identity') = 'exact_identity' + AND COALESCE(cp.hard_veto, false) = false + ORDER BY cp.expires_at ASC, cp.sku + LIMIT :candidate_limit ) SELECT - lm.product_id, - lm.sku, - lm.name, - lm.category, - lm.momo_price, - cp.competitor_product_id, - cp.competitor_product_name, - cp.match_score, - cp.expires_at - FROM latest_momo lm - JOIN competitor_prices cp - ON cp.sku = lm.sku - AND cp.source = 'pchome' - AND cp.competitor_product_id IS NOT NULL - AND cp.competitor_product_id <> '' - AND cp.expires_at IS NOT NULL - AND cp.expires_at <= CURRENT_TIMESTAMP - AND COALESCE(cp.match_score, 0) >= :match_score_floor - AND COALESCE(cp.tags, '[]'::jsonb) ? 'identity_v2' - WHERE lm.rn = 1 - AND ( - COALESCE(cp.tags, '[]'::jsonb) ? 'price_basis_total_price' - OR cp.match_diagnostic_json->>'price_basis' = 'total_price' - ) - AND ( - COALESCE(cp.tags, '[]'::jsonb) ? 'alert_tier_price_alert_exact' - OR cp.match_diagnostic_json->>'alert_tier' = 'price_alert_exact' - ) - AND COALESCE(cp.match_diagnostic_json->>'comparison_mode', 'exact_identity') = 'exact_identity' - AND COALESCE(cp.hard_veto, false) = false - AND NOT ( - COALESCE(cp.match_diagnostic_json->'reasons', '[]'::jsonb) - ?| array[{STALE_IDENTITY_RECOVERY_BLOCK_SQL_REASON_LIST}] - ) - AND COALESCE(lm.name, '') !~* :blocked_name_pattern - AND COALESCE(cp.competitor_product_name, '') !~* :blocked_name_pattern + p.id AS product_id, + p.i_code AS sku, + p.name, + p.category, + latest_price.price AS momo_price, + ec.competitor_product_id, + ec.competitor_product_name, + ec.match_score, + ec.expires_at, + ec.diagnostic_reasons + FROM expired_competitor ec + JOIN products p + ON p.i_code = ec.sku + AND p.status = 'ACTIVE' + JOIN LATERAL ( + SELECT pr.price + FROM price_records pr + WHERE pr.product_id = p.id + ORDER BY pr.timestamp DESC, pr.id DESC + LIMIT 1 + ) latest_price ON TRUE ORDER BY - cp.expires_at ASC, - lm.momo_price DESC NULLS LAST, - lm.sku - LIMIT :limit + ec.expires_at ASC, + latest_price.price DESC NULLS LAST, + p.i_code """) with self.engine.connect() as conn: rows = conn.execute( sql, { - "limit": max(1, min(int(limit), 120)), + "candidate_limit": candidate_limit, "match_score_floor": MIN_MATCH_SCORE, - "blocked_name_pattern": STALE_IDENTITY_RECOVERY_BLOCK_NAME_PATTERN, }, ).fetchall() - return [dict(r._mapping) for r in rows] + filtered = [] + for row in rows: + payload = dict(row._mapping) + if _has_stale_identity_recovery_block(payload): + continue + payload.pop("diagnostic_reasons", None) + filtered.append(payload) + if len(filtered) >= target_limit: + break + return filtered def _fetch_expired_identity_skus(self, limit: int = 120) -> list: """ diff --git a/tests/test_competitor_match_attempts_persistence.py b/tests/test_competitor_match_attempts_persistence.py index f84ed0c..d78e065 100644 --- a/tests/test_competitor_match_attempts_persistence.py +++ b/tests/test_competitor_match_attempts_persistence.py @@ -113,6 +113,14 @@ def test_competitor_feeder_persists_all_match_attempt_outcomes(): assert "read_only_no_crawl_no_llm_no_db_write" in source assert "run_retryable_candidate_revalidation" in source assert "run_expired_identity_search_recovery" in source + recovery_source = source.split("def _fetch_expired_identity_recovery_skus", 1)[1].split( + "def _fetch_expired_identity_skus", 1 + )[0] + assert "expired_competitor AS" in recovery_source + assert "JOIN LATERAL" in recovery_source + assert "ORDER BY pr.timestamp DESC, pr.id DESC" in recovery_source + assert "ROW_NUMBER() OVER (PARTITION BY p.id" not in recovery_source + assert "candidate_limit" in recovery_source retryable_source = source.split("def _fetch_retryable_candidate_skus", 1)[1].split( "def _fetch_expired_identity_skus", 1 )[0]