From 11896c24dca024cc1543ed1cfe15878f92704037 Mon Sep 17 00:00:00 2001 From: OoO Date: Sun, 31 May 2026 23:16:06 +0800 Subject: [PATCH] =?UTF-8?q?V10.518=20=E6=8B=86=E5=88=86=E6=AF=94=E5=83=B9?= =?UTF-8?q?=E8=A6=86=E8=93=8B=E8=88=87=E5=83=B9=E6=A0=BC=E6=96=B0=E9=AE=AE?= =?UTF-8?q?=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO_NEXT_STEPS.txt | 1 + config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 4 +-- docs/memory/history_logs.md | 1 + routes/dashboard_routes.py | 10 ++++++ scheduler.py | 13 ++++--- services/competitor_intel_repository.py | 39 ++++++++++++++++----- services/competitor_match_review_service.py | 4 ++- services/competitor_price_feeder.py | 2 +- services/hermes_analyst_service.py | 2 +- templates/daily_sales.html | 8 +++-- templates/dashboard_v2.html | 12 +++++-- templates/growth_analysis.html | 8 +++-- tests/test_competitor_intel_cache.py | 10 ++++-- tests/test_frontend_v2_assets.py | 3 +- 15 files changed, 90 insertions(+), 29 deletions(-) diff --git a/TODO_NEXT_STEPS.txt b/TODO_NEXT_STEPS.txt index 19d5e64..0fea965 100644 --- a/TODO_NEXT_STEPS.txt +++ b/TODO_NEXT_STEPS.txt @@ -4,6 +4,7 @@ ================================================================================ 【已完成】 + - V10.518 修正 PChome 比價覆蓋率口徑與新鮮度產線：`fetch_competitor_coverage()` 改拆「身份覆蓋」與「價格新鮮」，覆蓋率不再因 `expires_at` 過期被歸零；首頁 / 業績 / 成長頁同步顯示身份覆蓋、價格新鮮數與新鮮率。PChome 快取 TTL 預設由 6h 改 48h，並把每日 expired identity refresh / retryable / unmatched limits 改為環境變數，預設提高到 1200 / 240 / 240，避免 1800+ 已配對 identity 因刷新量不足長期失效。 - V10.517 補 PChome 近門檻比對安全 exact 與香氛 variant 防線：Lab52 齒妍堂汪汪隊嬰幼兒牙刷 2 入組可由低分區提升為 `exact / total_price / price_alert_exact`；Les nez 香氛融蠟燈不同款式、Time Leisure 香薰蠟燭單側香味款式會被留在覆核 / veto，不再進 recoverable 自動回刷，避免為了壓低 low_score 而錯配款式。 - V10.515 補 Webcrumbs host data 硬性授權：即使正式環境 `DISABLE_LOGIN=true` 讓一般 `@login_required` 放行，`/api/webcrumbs/marketplace-host-data` 仍必須有登入 session 或 `X-Internal-Key` 才能取真實 SKU/價差；`/webcrumbs` 未授權時只注入 `auth_required` 空狀態，避免 inline seed data 公開正式比價資料。 - V10.514 新增 Webcrumbs MOMO/PChome host data read-only API：`/api/webcrumbs/marketplace-host-data` 回傳與 `/webcrumbs` inline seed 相同的登入後 JSON contract，提供 plugin / QA / 其他專案 proxy 驗證；API boundary 明確標示不寫 DB、不呼叫 LLM、不抓外站，只允許 exact / total_price / price_alert_exact 價差摘要。 diff --git a/config.py b/config.py index 018e816..048ce0b 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.517" +SYSTEM_VERSION = "V10.518" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index f8b6b40..b026c13 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -245,7 +245,7 @@ SKU/商品ID = find_col(['商品ID', 'Product ID', 'ID', 'i_code', 'Item Code' | `match_score` | NUMERIC(4,3) | 商品身份比對分數（0~1），< 0.76 不寫入正式快取 | | `tags` | JSONB | 語意標籤，如 `["on_sale","discount_20pct"]` | | `crawled_at` | TIMESTAMP | 爬取時間 | -| `expires_at` | TIMESTAMP | TTL = crawled_at + 6h，過期後 Hermes 忽略 | +| `expires_at` | TIMESTAMP | TTL = crawled_at + 48h（可由 `PCHOME_FEEDER_TTL_HOURS` 調整），過期後 Hermes 忽略；UI 身份覆蓋率不因價格 TTL 過期歸零 | **UNIQUE**: `(sku, source)` — 同一 SKU+來源只有一筆，ON CONFLICT UPDATE @@ -336,7 +336,7 @@ LIMIT 300 ↓ 搜尋 PChome（search_products） ↓ 商品身份比對（marketplace_product_matcher.py） ↓ 提取語意標籤 - ↓ UPSERT competitor_prices（TTL 6h） + ↓ UPSERT competitor_prices（預設 TTL 48h） ↓ [HermesAnalystService.fetch_candidates()] ←← AI Pipeline 消費端 ↓ LEFT JOIN competitor_prices（零網路等待） diff --git a/docs/memory/history_logs.md b/docs/memory/history_logs.md index 85294ba..ae13a0c 100644 --- a/docs/memory/history_logs.md +++ b/docs/memory/history_logs.md @@ -13,6 +13,7 @@ ## 📅 詳細更新日誌 (考古存檔) ### 2026-05-31：Webcrumbs 共用 UI Runtime 與市場情報 writer approval +- **V10.518 PChome 覆蓋率與新鮮度拆分**: 修正比價監控總覽把價格 TTL 過期誤算成「未覆蓋」的產品口徑，`fetch_competitor_coverage()` 現在分開回報 `valid_matches`（identity 覆蓋）、`fresh_matches`（價格新鮮）、`stale_matches` 與 `fresh_match_rate`；首頁、業績與成長頁同步顯示身份覆蓋與價格新鮮。PChome 快取 TTL 預設由 6h 改 48h，並將每日 expired identity refresh / retryable / unmatched limits 改為環境變數，預設提升到 1200 / 240 / 240，避免已建立 identity 的商品因刷新量不足被長期視為無覆蓋。 - **V10.517 PChome near-threshold 比對 hotfix**: 新增 Lab52 齒妍堂汪汪隊嬰幼兒牙刷 2 入組 focused exact identity，讓真同款可進 `exact / total_price / price_alert_exact`；同時補 Les nez 香氛融蠟燈款式選擇 gap 與 Time Leisure 香薰蠟燭香味 gap，將不同款式 / 單側香味候選留在覆核或 veto，不讓它們進 recoverable 自動回刷。測試鎖住 Dashing Diva、Pavaruni、Recipe Box、Lactacyd 與 feeder recoverable 邊界。 - **V10.516 Webcrumbs host data 授權回歸測試**: 新增 Flask runtime 測試，直接重現 `DISABLE_LOGIN=true` 下 `/api/webcrumbs/marketplace-host-data` 必須回 401 且不得組裝真實 host data；同時鎖住 `X-Internal-Key` 與登入 session 可取得敏感 seed、未授權 `/webcrumbs` 只注入 `auth_required` 空狀態，避免後續改動再讓 public 診斷頁洩漏 MOMO/PChome SKU 與價差。 - **V10.515 Webcrumbs host data 硬性授權**: 發現正式環境一般 `@login_required` 可能因 `DISABLE_LOGIN=true` 放行後，為 `/api/webcrumbs/marketplace-host-data` 與 `/webcrumbs` inline seed 加上獨立授權判斷；只有登入 session 或 `X-Internal-Key` 可取得真實 SKU/價差，未授權時只回 `auth_required` 空狀態，避免 public runtime 診斷頁洩漏正式比價資料。 diff --git a/routes/dashboard_routes.py b/routes/dashboard_routes.py index ddaf200..c0d60c7 100644 --- a/routes/dashboard_routes.py +++ b/routes/dashboard_routes.py @@ -681,6 +681,13 @@ def _merge_competitor_review_context(overview, review_context): for status in option['statuses'] ) overview.update({ + 'total_active': int(coverage.get('active_with_price') or overview.get('total_active') or 0), + 'matched_count': int(coverage.get('valid_matches') or overview.get('matched_count') or 0), + 'match_rate': coverage.get('match_rate', overview.get('match_rate') or 0), + 'fresh_match_count': int(coverage.get('fresh_matches') or 0), + 'fresh_match_rate': coverage.get('fresh_match_rate', 0), + 'stale_match_count': int(coverage.get('stale_matches') or 0), + 'pending_match_count': int(coverage.get('pending') or overview.get('pending_match_count') or 0), 'review_queue_count': int(coverage.get('actionable_review_count') or len(review_queue) or 0), 'unit_comparable_count': int(coverage.get('unit_comparable_count') or 0), 'rescore_accepted_count': int( @@ -820,6 +827,9 @@ def _load_competitor_decision_overview(session, latest_items=None): 'total_active': 0, 'matched_count': 0, 'match_rate': 0, + 'fresh_match_count': 0, + 'fresh_match_rate': 0, + 'stale_match_count': 0, 'pchome_advantage_count': 0, 'momo_threat_count': 0, 'near_count': 0, diff --git a/scheduler.py b/scheduler.py index 8c8c3a2..57fe895 100644 --- a/scheduler.py +++ b/scheduler.py @@ -2276,17 +2276,22 @@ def run_pchome_match_backfill_task(): logging.info(f"[Scheduler] [PChomeBackfill] 🚀 啟動待比對補抓任務 | {now_str}") engine = create_engine(DATABASE_PATH) + revalidation_limit = int(os.getenv("PCHOME_BACKFILL_REVALIDATION_LIMIT", "1200")) + expired_refresh_limit = int(os.getenv("PCHOME_BACKFILL_EXPIRED_REFRESH_LIMIT", "1200")) + retryable_limit = int(os.getenv("PCHOME_BACKFILL_RETRYABLE_LIMIT", "240")) + unmatched_limit = int(os.getenv("PCHOME_BACKFILL_UNMATCHED_LIMIT", "240")) + revalidation_result = revalidate_existing_competitor_identities( engine, - limit=500, + limit=revalidation_limit, dry_run=False, include_expired=True, write_attempts=True, ) feeder = CompetitorPriceFeeder(engine=engine) - refresh_result = feeder.run_expired_identity_refresh(limit=240) - retryable_result = feeder.run_retryable_candidate_revalidation(limit=160, min_score=0.70) - feeder_result = feeder.run_unmatched_priority(limit=120) + refresh_result = feeder.run_expired_identity_refresh(limit=expired_refresh_limit) + retryable_result = feeder.run_retryable_candidate_revalidation(limit=retryable_limit, min_score=0.70) + feeder_result = feeder.run_unmatched_priority(limit=unmatched_limit) pick_result = generate_product_pick_list(engine, limit=50) clear_dashboard_cache() clear_competitor_intel_cache() diff --git a/services/competitor_intel_repository.py b/services/competitor_intel_repository.py index 5662fc9..f150f55 100644 --- a/services/competitor_intel_repository.py +++ b/services/competitor_intel_repository.py @@ -705,7 +705,7 @@ def _cached_payload(cache_key: str, producer, ttl_seconds: int = COMPETITOR_INTE def fetch_competitor_coverage(engine) -> dict: return _cached_payload( - f"coverage:v5:floor={PCHOME_MATCH_SCORE_FLOOR}:manual_reviews=1:rescore=1", + f"coverage:v6:floor={PCHOME_MATCH_SCORE_FLOOR}:manual_reviews=1:rescore=1:identity_vs_fresh=1", lambda: _fetch_competitor_coverage_uncached(engine), ) @@ -720,8 +720,11 @@ def _fetch_competitor_coverage_uncached(engine) -> dict: return { "active_with_price": 0, "valid_matches": 0, + "fresh_matches": 0, + "stale_matches": 0, "pending": 0, "match_rate": 0, + "fresh_match_rate": 0, "attempt_status": {}, "unit_comparable_count": 0, "rescore_accepted_count": 0, @@ -771,34 +774,47 @@ def _fetch_competitor_coverage_uncached(engine) -> dict: ) latest_price ON TRUE WHERE p.status = 'ACTIVE' ), - valid_competitor AS ( + identity_competitor AS ( SELECT DISTINCT ON (cp.sku) - cp.sku + cp.sku, + cp.expires_at FROM competitor_prices cp WHERE cp.source = 'pchome' - AND (cp.expires_at IS NULL OR cp.expires_at > CURRENT_TIMESTAMP) AND cp.price IS NOT NULL AND cp.price > 0 AND COALESCE(cp.match_score, 0) >= {PCHOME_MATCH_SCORE_FLOOR} AND COALESCE(cp.tags, '[]'::jsonb) ? 'identity_v2' ORDER BY cp.sku, cp.crawled_at DESC NULLS LAST ), + fresh_competitor AS ( + SELECT sku + FROM identity_competitor + WHERE expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP + ), {attempt_cte} SELECT (SELECT COUNT(*) FROM latest_momo) AS active_with_price, (SELECT COUNT(*) FROM latest_momo lm - JOIN valid_competitor vc ON vc.sku = lm.sku) AS valid_matches, + JOIN identity_competitor ic ON ic.sku = lm.sku) AS valid_matches, (SELECT COUNT(*) FROM latest_momo lm - LEFT JOIN valid_competitor vc ON vc.sku = lm.sku - WHERE vc.sku IS NULL) AS pending, + JOIN fresh_competitor fc ON fc.sku = lm.sku) AS fresh_matches, + (SELECT COUNT(*) + FROM latest_momo lm + JOIN identity_competitor ic ON ic.sku = lm.sku + LEFT JOIN fresh_competitor fc ON fc.sku = lm.sku + WHERE fc.sku IS NULL) AS stale_matches, + (SELECT COUNT(*) + FROM latest_momo lm + LEFT JOIN identity_competitor ic ON ic.sku = lm.sku + WHERE ic.sku IS NULL) AS pending, COALESCE(la.attempt_status, 'never_attempted') AS attempt_status, COUNT(*) AS status_count FROM latest_momo lm - LEFT JOIN valid_competitor vc ON vc.sku = lm.sku + LEFT JOIN identity_competitor ic ON ic.sku = lm.sku LEFT JOIN latest_attempt la ON la.sku = lm.sku - WHERE vc.sku IS NULL + WHERE ic.sku IS NULL GROUP BY COALESCE(la.attempt_status, 'never_attempted') """) with engine.connect() as conn: @@ -806,6 +822,8 @@ def _fetch_competitor_coverage_uncached(engine) -> dict: active = int(rows[0].get("active_with_price") or 0) if rows else 0 valid = int(rows[0].get("valid_matches") or 0) if rows else 0 + fresh = int(rows[0].get("fresh_matches") or 0) if rows else 0 + stale = int(rows[0].get("stale_matches") or 0) if rows else 0 pending = int(rows[0].get("pending") or 0) if rows else 0 statuses = { str(row.get("attempt_status")): int(row.get("status_count") or 0) @@ -817,8 +835,11 @@ def _fetch_competitor_coverage_uncached(engine) -> dict: return { "active_with_price": active, "valid_matches": valid, + "fresh_matches": fresh, + "stale_matches": stale, "pending": pending, "match_rate": round(valid / max(active, 1) * 100, 1), + "fresh_match_rate": round(fresh / max(valid, 1) * 100, 1), "attempt_status": statuses, "unit_comparable_count": unit_count, "rescore_accepted_count": rescore_accepted_count, diff --git a/services/competitor_match_review_service.py b/services/competitor_match_review_service.py index d6ee897..c6b85bd 100644 --- a/services/competitor_match_review_service.py +++ b/services/competitor_match_review_service.py @@ -5,6 +5,7 @@ from __future__ import annotations import json +import os from datetime import datetime, timedelta, timezone from typing import Any @@ -212,7 +213,8 @@ def _promote_manual_match(conn, attempt: dict[str, Any], source: str) -> None: _ensure_competitor_price_history_table(conn) _taipei = timezone(timedelta(hours=8)) - expires_at = (datetime.now(_taipei) + timedelta(hours=6)).strftime("%Y-%m-%d %H:%M:%S") + ttl_hours = int(os.getenv("PCHOME_FEEDER_TTL_HOURS", "48")) + expires_at = (datetime.now(_taipei) + timedelta(hours=ttl_hours)).strftime("%Y-%m-%d %H:%M:%S") match_score = max(_num(attempt.get("best_match_score")) or 0, 0.76) tags = [ "identity_v2", diff --git a/services/competitor_price_feeder.py b/services/competitor_price_feeder.py index 317fc6b..ac687fd 100644 --- a/services/competitor_price_feeder.py +++ b/services/competitor_price_feeder.py @@ -44,7 +44,7 @@ MAX_SEARCH_TERMS = int(os.getenv("PCHOME_FEEDER_MAX_SEARCH_TERMS", "5")) # 每 SEARCH_MAX_PAGES = int(os.getenv("PCHOME_FEEDER_SEARCH_MAX_PAGES", "2")) # 每個搜尋詞最多掃描 PChome 搜尋頁數 BATCH_SIZE = 30 # 每批 DB 寫入筆數 RATE_DELAY = float(os.getenv("PCHOME_FEEDER_RATE_DELAY", "1.0")) # 每次 PChome 請求間隔（秒） -TTL_HOURS = 6 # competitor_prices 快取有效期 +TTL_HOURS = int(os.getenv("PCHOME_FEEDER_TTL_HOURS", "48")) # competitor_prices 價格新鮮度有效期 REQUEST_TIMEOUT = float(os.getenv("PCHOME_FEEDER_TIMEOUT", "12")) # 避免外部搜尋 API 長時間卡住排程 VARIANT_RECALL_SORTS = ("sale/dc", "new/dc") RECOVERABLE_LOW_SCORE_FLOOR = max(MIN_MATCH_SCORE - 0.03, 0.72) diff --git a/services/hermes_analyst_service.py b/services/hermes_analyst_service.py index 0a257f5..4acabc3 100644 --- a/services/hermes_analyst_service.py +++ b/services/hermes_analyst_service.py @@ -396,7 +396,7 @@ class HermesAnalystService: 競價威脅候選漏斗（v2 — 直接 LEFT JOIN competitor_prices） 條件：近7天銷量下滑 > 10% 且 MOMO 自家有歷史價格紀錄 - 擴充：LEFT JOIN competitor_prices（PChome 快取，TTL 6h） + 擴充：LEFT JOIN competitor_prices（PChome 快取，預設 TTL 48h） 帶回 pchome_price + competitor_tags 供 Hermes 情境分析 無競品資料的商品仍回傳，pchome_price=NULL → _batch_analyze 跳過 diff --git a/templates/daily_sales.html b/templates/daily_sales.html index 152a242..c8ddc7a 100644 --- a/templates/daily_sales.html +++ b/templates/daily_sales.html @@ -344,13 +344,17 @@
- 有效配對 + 身份配對 {{ comp_coverage.valid_matches | default(0) | number_format }}
- 覆蓋率 + 身份覆蓋率 {{ comp_coverage.match_rate | default(0) }}%
+
+ 價格新鮮 + {{ comp_coverage.fresh_matches | default(0) | number_format }} +
待審/待補 {{ comp_coverage.pending | default(0) | number_format }} diff --git a/templates/dashboard_v2.html b/templates/dashboard_v2.html index d7868a7..bd82687 100644 --- a/templates/dashboard_v2.html +++ b/templates/dashboard_v2.html @@ -17,9 +17,12 @@
-
比對覆蓋率
+
身份覆蓋率
{{ overview.match_rate | default(0) }}%
-
{{ overview.matched_count | default(0) | number_format }} / {{ overview.total_active | default(total_products) | number_format }} ACTIVE
+
+ {{ overview.matched_count | default(0) | number_format }} / {{ overview.total_active | default(total_products) | number_format }} ACTIVE + · 新鮮 {{ overview.fresh_match_count | default(0) | number_format }} +
PChome 優勢
@@ -50,7 +53,10 @@
資料新鮮度
{{ '已更新' if overview.last_pchome_crawled else '待更新' }}
-
{{ overview.last_pchome_crawled or '尚無 PChome 抓取紀錄' }}
+
+ {{ overview.last_pchome_crawled or '尚無 PChome 抓取紀錄' }} + · 新鮮率 {{ overview.fresh_match_rate | default(0) }}% +
高信心門檻 {{ coverage.match_score_floor | default(0.76) }} - 有效配對 + 身份配對 {{ coverage.valid_matches | default(0) | number_format }} - 覆蓋率 + 身份覆蓋率 {{ coverage.match_rate | default(0) }}% + 價格新鮮 + {{ coverage.fresh_matches | default(0) | number_format }} + 新鮮率 + {{ coverage.fresh_match_rate | default(0) }}% 待審/待補 {{ coverage.pending | default(0) | number_format }} 需單位價覆核 diff --git a/tests/test_competitor_intel_cache.py b/tests/test_competitor_intel_cache.py index 1edd8ae..b8cc9d9 100644 --- a/tests/test_competitor_intel_cache.py +++ b/tests/test_competitor_intel_cache.py @@ -75,10 +75,14 @@ def test_competitor_coverage_counts_only_active_product_intersection(): "def _fetch_manual_review_summary", 1 )[0] - assert "coverage:v5" in source + assert "coverage:v6" in source assert "rescore_accepted_count" in coverage_source assert "(SELECT COUNT(*) FROM valid_competitor) AS valid_matches" not in coverage_source - assert "FROM latest_momo lm\n JOIN valid_competitor vc ON vc.sku = lm.sku" in coverage_source + assert "identity_competitor AS" in coverage_source + assert "fresh_competitor AS" in coverage_source + assert "FROM latest_momo lm\n JOIN identity_competitor ic ON ic.sku = lm.sku" in coverage_source + assert "\"fresh_matches\": fresh" in coverage_source + assert "\"stale_matches\": stale" in coverage_source assert "FROM products p\n JOIN LATERAL" in coverage_source assert "WHERE p.status = 'ACTIVE'" in coverage_source @@ -121,6 +125,8 @@ def test_competitor_review_queue_is_canonical_unit_price_handoff(): assert "人工否決" in daily_template assert "人工單位價" in daily_template assert "competitor_intel.review_queue" in daily_template + assert "coverage.fresh_matches" in growth_template + assert "coverage.fresh_match_rate" in growth_template assert "coverage.unit_comparable_count" in growth_template assert "coverage.rescore_accepted_count" in growth_template assert "coverage.manual_accept_count" in growth_template diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index 58bcb81..0bdf162 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -482,7 +482,8 @@ def test_ai_product_pick_agent_uses_real_competitor_data_and_dashboard_action(): assert "run_expired_identity_refresh" in feeder_source assert "fetch_product_details(requested_ids" in feeder_source assert "run_unmatched_priority" in feeder_source - assert "run_expired_identity_refresh(limit=240)" in scheduler_source + assert "PCHOME_BACKFILL_EXPIRED_REFRESH_LIMIT" in scheduler_source + assert "run_expired_identity_refresh(limit=expired_refresh_limit)" in scheduler_source assert "generate_product_pick_list" in agent_source assert "clear_dashboard_cache()" in route_source