diff --git a/.env.example b/.env.example index 584d418..21f95a5 100644 --- a/.env.example +++ b/.env.example @@ -638,6 +638,15 @@ MOMO_TARGETED_SEARCH_MAX_PRODUCTS=8 MOMO_TARGETED_SEARCH_MAX_TERMS=3 MOMO_TARGETED_SEARCH_MIN_SCORE=0.5 PCHOME_GROWTH_MOMO_BACKFILL_LIMIT=8 +PCHOME_GROWTH_YAHOO_BACKFILL_LIMIT=8 +PCHOME_GROWTH_YAHOO_RECEIPT_ROOT=/app/data/ai_automation/pchome_growth_yahoo_backfill_receipts +PCHOME_GROWTH_YAHOO_HTTP_TIMEOUT_SECONDS=15 +PCHOME_GROWTH_YAHOO_REQUEST_DELAY_SECONDS=0.2 +PCHOME_GROWTH_YAHOO_MAX_TERMS_PER_TARGET=1 +PCHOME_GROWTH_YAHOO_DETAIL_CANDIDATES_PER_TARGET=3 +PCHOME_GROWTH_YAHOO_MIN_SCORE=0.55 +PCHOME_GROWTH_YAHOO_OFFER_TTL_HOURS=30 +PCHOME_GROWTH_YAHOO_ACTIVATION_MIN_VERIFIED=2 PCHOME_GROWTH_BACKFILL_COOLDOWN_HOURS=4 PCHOME_GROWTH_BACKFILL_RETRY_HOURS=1 PCHOME_PUBLIC_API_TIMEOUT_SECONDS=4 diff --git a/config.py b/config.py index a13e31a..7684049 100644 --- a/config.py +++ b/config.py @@ -414,7 +414,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.799" +SYSTEM_VERSION = "V10.800" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 9fa303b..44126f1 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,18 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-07-15 (台北時間) -> **狀態**: 🟠 Partial。V10.799 已在 production 以 exact Gitea object `01329c7048ffb54d7f323e84d8c9de086eb1cd68` 運行。`pchome_comparison_coverage_v3` 把 TOP50 固定為 action-priority scoring 前的營收排名母體,mapping 狀態不再把已完成商品擠出分母;production 最終 readback 為 `21/50 = 42%` 商品覆蓋與 `NT$180,667 / NT$354,062 = 51.0%` 業績覆蓋,母體 fingerprint `7b74504fcc1e1801c2ca2b42` 在三輪 bounded run 前後不變。仍有 `1` 件候選驗證、`28` 件未對應,外部平台 runtime 仍僅 `1/15`。SEC-P0-001 已完成 production closure,SEC-P0-002 維持 production hybrid canary-ready;full asset reconciliation、software supply chain、統一 controlled-apply envelope、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 -> **適用版本**: V10.799 source + production runtime;Gitea Action #1131 仍因 `ewoooc-host` runner 離線而未提供 CD 執行證據 +> **狀態**: 🟠 Partial。V10.799 已在 production 以 exact Gitea object `01329c7048ffb54d7f323e84d8c9de086eb1cd68` 運行。`pchome_comparison_coverage_v3` 把 TOP50 固定為 action-priority scoring 前的營收排名母體,mapping 狀態不再把已完成商品擠出分母;production 最終 readback 為 `21/50 = 42%` 商品覆蓋與 `NT$180,667 / NT$354,062 = 51.0%` 業績覆蓋,母體 fingerprint `7b74504fcc1e1801c2ca2b42` 在三輪 bounded run 前後不變。V10.800 Yahoo Shopping structured adapter 已完成 source candidate,workstation regression 為 `2,105 passed / 9 skipped / 0 failed`,但尚未部署或改變 production KPI;正式外部平台 runtime 仍僅 `1/15`。SEC-P0-001 已完成 production closure,SEC-P0-002 維持 production hybrid canary-ready;full asset reconciliation、software supply chain、統一 controlled-apply envelope、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 +> **適用版本**: V10.799 production runtime + V10.800 source candidate;Gitea Action #1131 仍因 `ewoooc-host` runner 離線而未提供 CD 執行證據 + +--- + +## 零之負四、Yahoo Shopping 公開結構化同品 canary(V10.800 source candidate) + +- Yahoo 搜尋只解析公開頁面的 `script#isoredux-data`,商品頁只解析 schema.org `Product` / `Offer` JSON-LD;PChome 目標商品則由公開 JSONP product API 回讀 `Name`、`Spec`、`Price` 與 `Qty`。所有 URL 都受 HTTPS host/path allowlist、回應大小、timeout、查詢數、detail 數與 request interval 約束,不讀登入 cookie/session,也不繞過平台阻擋。 +- 正式候選必須同時通過 search/detail product ID 一致、TWD 正價、PChome current price、雙方庫存(有證據時)、PChome Spec 出現在來源標題、非「任選/多款/可選」、非括號多規格組、deterministic exact same-item matcher 與獨立 DB post-verifier。只有 exact offer 能寫 `external_offers`;其餘候選只留 receipt,不因 KPI 壓力放寬 matcher。 +- Controlled apply 強制先做零寫入 check-mode,再以同一 `trace_id/run_id/work_item_id` bounded execute;至少兩筆 exact offer 全數獨立回讀成功後,才一次啟用 `active + enabled + write_enabled`。來源尚未啟用但 post-verifier 失敗時也會 quarantine 同 run row;activation 後失敗則同時還原 source state,不刪除歷史資料。compact atomic receipt 由 scheduler 每六小時更新,API/CLI 與 UI 分開顯示正式來源、候選證據與任何 runtime evidence。 +- 最新 workstation 公開 live no-write probe 對 OBgE、Amywo、Lancome、Sebamed 取得三筆 in-stock source-guard/total-price exact 候選,Lancome 因 Yahoo 明確 `ec_has_stock=0` 被阻擋。Critic 另以 Estee 色號、Armani 色號及 Gatsby 香型做反例;唯一曾漏過的括號多規格 listing 已加 guard,Gatsby 聚焦重跑的三筆候選全數阻擋。PChome sales-report fallback 不得冒充 current public price;公開 HTTP 採真正 bounded streaming。這些都是 source validation,不是 production DB、正式 coverage 或平台 activation 證據。 +- Production 仍為 V10.799、`21/50` 商品覆蓋、`51.0%` 業績覆蓋與 `1/15` 正式平台 runtime。V10.800 已通過 `2,105 passed / 9 skipped / 0 failed` 完整 regression;剩餘 exit 必須依序完成 Gitea main/dev、部署 readback、production check-mode、bounded canary、獨立 DB verifier、TOP50 fingerprint/KPI delta、三應用容器健康與 `momo-db` identity unchanged。 --- diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 4f05a18..e38179d 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -1,6 +1,6 @@ # AI Automation Mainline Work Items -> Updated: 2026-07-15 10:48 Asia/Taipei +> Updated: 2026-07-15 12:39 Asia/Taipei > Governance: `global_product_governance_v2` + ADR-038 > Current P0: `GROWTH-P0-001 comparison coverage truth + autonomous refresh` @@ -17,7 +17,7 @@ | Order | ID | Status | Work item | Exit evidence / next machine action | |---:|---|---|---|---| | 1 | `SEC-P0-001` | Completed | Deny-by-default route access control | `governance/evidence/SEC-P0-001-20260711T122758Z.json` plus the current runtime receipt prove anonymous matrix 8/8 denied, public `/metrics` 404, exact internal target up, EwoooC product markers present, Prometheus identity preserved and `momo-db` unchanged. | -| 2 | `GROWTH-P0-001` | In progress (`runtime_partial`) | Comparison coverage truth + autonomous refresh | V10.799 is live. `pchome_comparison_coverage_v3` fixes TOP50 membership before action-priority scoring; fingerprint `7b74504fcc1e1801c2ca2b42` stayed unchanged across all bounded runs. Final production truth is `21 ready + 1 candidate validation + 28 unmatched`, count coverage `42%`, and revenue coverage `NT$180,667 / NT$354,062 = 51.0%`. Two productive runs independently wrote/read back `3 + 2` verified MOMO offers; the next run found 9 candidates but safely closed `no_write_verified` with 8/8 durable target receipts. Sales date `2026-07-13` remains decision-ready inside the `2026-07-15 20:00` arrival SLA; source readiness remains `1/4` and registered platform runtime remains `1/15`. Next: retry the remaining 28 unmatched products only after fresh source evidence, while advancing Lane C structured platform adapters and continuously verifying Lane A report arrival. | +| 2 | `GROWTH-P0-001` | In progress (`runtime_partial`) | Comparison coverage truth + autonomous refresh | V10.799 is live. `pchome_comparison_coverage_v3` fixes TOP50 membership before action-priority scoring; fingerprint `7b74504fcc1e1801c2ca2b42` stayed unchanged across all bounded runs. Final production truth is `21 ready + 1 candidate validation + 28 unmatched`, count coverage `42%`, and revenue coverage `NT$180,667 / NT$354,062 = 51.0%`. V10.800 source candidate adds a bounded Yahoo Shopping public structured adapter, enforced zero-write check-mode, exact same-item guards, independent DB verifier, automatic activation/rollback, six-hour scheduler and formal-versus-candidate UI evidence. Final regression is `2,105 passed / 9 skipped / 0 failed`; latest four-target live no-write probe produced three exact in-stock candidates and correctly blocked one explicit out-of-stock Lancome offer. Next: merge through Gitea, deploy V10.800 and execute the bounded production canary before updating any runtime KPI. | | 3 | `SEC-P0-002` | In progress (`canary_ready`) | Database identity + least-privilege RBAC | V10.789 is live and `governance/auth_identity_runtime_receipt.json` verifies required tables, two active admins, durable lockout, session revocation, trusted proxy policy and no-secret mutation audit readiness. Runtime is intentionally hybrid because zero database-admin success receipts have been captured; `auto` retires shared authority after two durable successes without a manual review gate. Next: capture database-admin login receipts and verify automatic database-only cutover. | | 4 | `SEC-P0-003` | In progress | Webhook trust and replay protection | Telegram secret-token verification code exists; production secret activation remains unproven. Exit: secret provisioned outside source, required mode enabled, invalid-secret 401 and authorized callback canary pass. | | 5 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout, secret-safe `.dockerignore`, commit-bound source receipt and governance gate are active. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | @@ -36,7 +36,7 @@ These lanes are one ordered current P0, not optional side work. They must advanc |---|---|---|---| | A. Sales freshness | In progress (`sla_runtime_closed_source_redundancy_partial`) | Latest sales date `2026-07-13`; before the `2026-07-15 20:00` cutoff, raw lag is `2` but SLA lag is `0`, state is `grace` and decisions remain released. Scheduler receipt `0f24219e0bbb4740b7ad6645e7952296` and explicit canary receipt `6e2df008f65544fe8e557c11ff0dbb93` both persisted `completed_no_write`; durable decision is `no_candidate_fresh_no_write`. Live and persisted readiness now agree at Google Drive `1/4`; HTTPS, IMAP and local remain disabled. | Continue automatic report-arrival reconciliation. At/after 20:00 require `2026-07-14` or automatically block decision use, emit the bounded upstream action and verify the next arrival receipt; keep source redundancy partial until another approved source is live. | | B. Verified same-item evidence | In progress | TOP50 fixed cohort: `21` verified, `1` candidate/source validation, `28` unmatched. Count coverage is `42%`; revenue-weighted coverage is `NT$180,667 / NT$354,062 = 51.0%`. V10.797 outer-pack false exact candidates were quarantined without deletion; V10.798 normalizes nested quantity, and V10.799 prevents mapping status from changing cohort membership. Runs `58bb702057284cd382c82314b418d713` and `cb40a5dcdaa045c69bf9611ebb13bba2` wrote/read back five verified offers; run `14d4aac64d434e34bfbd0f0edb7ef1eb` closed no-write after zero safe candidates. | Wait for fresh source evidence before retrying the same unresolved subset, broaden approved structured platform adapters, preserve deterministic identity/unit/variant gates, and publish both count and revenue coverage deltas with the same scope fingerprint per run. | -| C. Platform runtime coverage | In progress | `1/15` registered external sources currently contributes formal runtime data (`6.7%`). PixelRAG remains evidence-only. | Activate approved structured adapters in source-contract order, use PixelRAG/OCR/VLM only to generate replayable candidates, classify anti-bot pages as blocked evidence, and require provenance/freshness/verifier receipts before formal price promotion. | +| C. Platform runtime coverage | In progress (`source_candidate_ready`) | Production remains `1/15` registered external sources with formal runtime data (`6.7%`); PixelRAG remains evidence-only. V10.800 source adds the first additional structured adapter for Yahoo Shopping and has three current in-stock exact candidates, but no production activation or KPI gain is claimed yet. | Deploy through Gitea, run Yahoo check-mode then bounded execute, require at least two exact offers with independent DB readback before atomically enabling `active + enabled + write_enabled`, and roll back/quarantine same-run rows on any failed post-verifier. | ## P1 @@ -49,8 +49,8 @@ These lanes are one ordered current P0, not optional side work. They must advanc | 16 | `ARCH-P1-001` | In progress | Split oversized policy/executor/verifier modules | Current top debts include 44k-line PChome mapping and 14k-line smoke service. Split by bounded family and independent tests. | | 17 | `UX-P1-001` | In progress | Professional full-site UI/UX | V10.799 production template leads with `業績比價覆蓋`, now backed by 51.0% revenue coverage and 21/50 product transparency instead of the old unqualified 10% card. Mapping and automation evidence remain structured grids with progressive disclosure, and the first viewport distinguishes calendar lag from report-arrival SLA. Existing V10.795 desktop/mobile overflow smoke remains valid for layout; this release still needs a fresh authenticated production screenshot before visible closure is claimed. Continue the same first-viewport, progressive-disclosure, accessibility and state coverage across every remaining primary page. | | 18 | `PIXELRAG-P1-001` | Not started | Ollama-first multimodal embedding benchmark | Verify approved visual embedding on GCP-A -> GCP-B -> 111 and design pgvector-compatible visual metadata; FAISS remains disallowed without ADR. | -| 19 | `MARKET-P1-001` | In progress | Marketplace source contracts | Finish stable, public-boundary, provenance and rate-limit contracts for Shopee, Coupang, Yahoo, ETMall, Friday and Rakuten; blocked pages remain non-product data. | -| 20 | `QA-P1-001` | In progress | Deterministic test and CI governance | V10.799 final source baseline is `2,089 passed / 9 skipped / 0 failed`; focused cohort/reconciliation scope is `37 passed`, Ruff and diff checks are clean. Production exact-object hashes, external health, fixed cohort fingerprint, two positive bounded runs and one no-write run all have readback evidence. Action #1131 remains waiting without runner execution, so CI parity is not claimed. Next: reproduce the same terminal in Gitea CI after runner recovery, split deterministic release tests from nightly integration and production canaries, and prevent workstation disk pressure from invalidating evidence. | +| 19 | `MARKET-P1-001` | In progress | Marketplace source contracts | Yahoo Shopping has a V10.800 source candidate with public-boundary allowlists, bounded timeout/bytes/rate, provenance, current product-detail readback, stock/spec/variant guards and controlled canary. Production activation remains pending; Shopee, Coupang, ETMall, Friday and Rakuten still require equivalent structured contracts, and blocked pages remain non-product data. | +| 20 | `QA-P1-001` | In progress | Deterministic test and CI governance | V10.800 workstation source baseline is `2,105 passed / 9 skipped / 0 failed`; the final focused Yahoo, source-readiness, same-item rollback and modularization scope is `62 passed`. This is source/test evidence only. V10.799 production exact-object hashes, external health and fixed cohort fingerprint remain the runtime baseline; Action #1131 remains waiting without runner execution, so CI parity and V10.800 deployment are not yet claimed. Next: reproduce the terminal through Gitea or bounded exact-object fallback, then run the production Yahoo canary and post-verifier. | ## P2 diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index d593088..8d80e2f 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1217,6 +1217,17 @@ def ai_automation_security_governance_review_api(): )) +@system_public_bp.route('/api/ai-automation/yahoo-shopping-public-offer') +@login_required +def ai_automation_yahoo_shopping_public_offer_api(): + """Read the latest public-safe Yahoo offer automation receipt.""" + from services.yahoo_shopping_public_offer_service import ( + build_yahoo_shopping_runtime_readback, + ) + + return jsonify(build_yahoo_shopping_runtime_readback()) + + @system_public_bp.route('/api/ai-automation/smoke/history/export') @login_required def ai_automation_smoke_history_export(): diff --git a/run_scheduler.py b/run_scheduler.py index c006293..3bbefb9 100644 --- a/run_scheduler.py +++ b/run_scheduler.py @@ -6,7 +6,7 @@ run_scheduler.py — momo-scheduler 容器入口點 每 30 分鐘:revenue_automation_cycle(auto_import → comparison backfill)、whitepage_check 每 1 小時:momo、edm、festival 每 4 小時:competitor_price_feeder、external_offer_sync、icaim_analysis - 每 6 小時:quality_rescore、action_plan_hygiene + 每 6 小時:pchome_growth_yahoo_backfill、quality_rescore、action_plan_hygiene 每 12 小時:dedup_batch 每 10 分鐘:ppt_auto_generation_catchup(補跑被長任務卡過的定期簡報) 每 1 天 :db_backup(03:00)、cleanup_agent_context(03:30)、backup_monitor(04:00)、daily_report(09:00)、roi_monthly_report gate(09:05)、ai_smoke_summary(09:10)、observability_daily_summary(09:30)、pchome_match_backfill(10:30)、pchome_growth_momo_backfill(10:45)、openclaw_meta_analysis(12:00, Phase 4 降頻)、ppt_auto_generation_daily(20:30)、ppt_vision_audit(22:00)、daily_token_report(23:55) @@ -39,6 +39,7 @@ from scheduler import ( run_external_offer_sync_task, run_pchome_match_backfill_task, run_pchome_growth_momo_backfill_task, + run_pchome_growth_yahoo_backfill_task, run_icaim_analysis_task, run_weekly_strategy_task, run_db_backup_task, @@ -191,6 +192,13 @@ def run_pchome_growth_momo_backfill_background_task() -> dict[str, str]: ) +def run_pchome_growth_yahoo_backfill_background_task() -> dict[str, str]: + return _launch_revenue_automation_background( + "pchome-growth-yahoo-backfill", + run_pchome_growth_yahoo_backfill_task, + ) + + def start_revenue_automation_watchdog() -> threading.Thread: """Keep revenue refresh alive even when a legacy schedule job blocks run_pending.""" try: @@ -337,6 +345,9 @@ def _register_schedules(): schedule.every(4).hours.do(run_icaim_analysis_task) logger.info("📅 每 4 小時:icaim_analysis") + schedule.every(6).hours.do(run_pchome_growth_yahoo_backfill_background_task) + logger.info("📅 每 6 小時:pchome_growth_yahoo_backfill(公開來源 + 同款驗證 + canary)") + # Operation Ollama-First v5.0 Phase 4:Meta 自審降頻 6h → 每日 12:00(月省 ~1.875M Gemini tokens) # icaim_analysis 內原本 line 2233/2253 的額外觸發已同步移除(避免重複呼叫) schedule.every().day.at("12:00").do(run_openclaw_meta_analysis_task) diff --git a/scheduler.py b/scheduler.py index b66623c..98afc7e 100644 --- a/scheduler.py +++ b/scheduler.py @@ -2620,6 +2620,14 @@ def run_pchome_growth_momo_backfill_task(): logging.error(f"[Scheduler] [PChomeGrowthMomoBackfill] event_router 失敗: {_router_e}") +def run_pchome_growth_yahoo_backfill_task(): + from services.pchome_growth_yahoo_scheduler_task import ( + run_scheduled_pchome_growth_yahoo_backfill, + ) + + return run_scheduled_pchome_growth_yahoo_backfill(save_stats=_save_stats) + + def run_icaim_analysis_task(): """ ICAIM 競價情報分析排程任務(每 6 小時執行一次) diff --git a/scripts/ops/run_pchome_growth_yahoo_backfill.py b/scripts/ops/run_pchome_growth_yahoo_backfill.py new file mode 100644 index 0000000..89b0439 --- /dev/null +++ b/scripts/ops/run_pchome_growth_yahoo_backfill.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Run or read back the bounded Yahoo public-offer automation lane.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +def main() -> int: + from sqlalchemy import create_engine + + parser = argparse.ArgumentParser() + parser.add_argument("--execute", action="store_true") + parser.add_argument("--limit", type=int, default=8) + parser.add_argument("--receipt-root") + args = parser.parse_args() + + from services.yahoo_shopping_public_offer_service import ( + build_yahoo_shopping_runtime_readback, + run_pchome_growth_yahoo_backfill, + ) + + if not args.execute: + payload = build_yahoo_shopping_runtime_readback( + receipt_root=args.receipt_root, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2, default=str)) + return 0 if payload.get("success") else 1 + + from config import DATABASE_PATH + + engine = create_engine(DATABASE_PATH) + try: + payload = run_pchome_growth_yahoo_backfill( + engine, + limit=max(1, min(args.limit, 20)), + receipt_root=args.receipt_root, + ) + finally: + engine.dispose() + print(json.dumps(payload, ensure_ascii=False, indent=2, default=str)) + return 0 if payload.get("success") else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/agent_actions.py b/services/agent_actions.py index 38113ba..2b2d7d6 100644 --- a/services/agent_actions.py +++ b/services/agent_actions.py @@ -78,6 +78,7 @@ ALLOWED_RETRY_TASKS = { "run_auto_import_task", "run_momo_task", "run_edm_task", "run_competitor_price_feeder_task", "run_backup_monitor_task", "run_pchome_match_backfill_task", "run_pchome_growth_momo_backfill_task", + "run_pchome_growth_yahoo_backfill_task", "run_icaim_analysis_task", "run_festival_task", "run_whitepage_check", "run_icaim_analysis_task", "run_db_backup_task", "run_promo_event_task", } diff --git a/services/external_market_offer_service.py b/services/external_market_offer_service.py index 8e5ccdd..e1559c5 100644 --- a/services/external_market_offer_service.py +++ b/services/external_market_offer_service.py @@ -126,9 +126,9 @@ SOURCE_CONTRACTS = [ "status_code": "paused", "status_label": "待接入", "source_kind": "connector_contract", - "input_methods": ["官方 API", "供應商 API", "手動 CSV"], + "input_methods": ["公開結構化資料", "官方 API", "供應商 API", "手動 CSV"], "data_quality_label": "暫不進告警", - "plain_note": "保留本地平台促銷與價格監控欄位,接入前不影響現有作戰清單。", + "plain_note": "優先使用 Yahoo 公開搜尋資料與商品頁 JSON-LD;通過 PChome 規格、同款 verifier 與來源 canary 後才自動啟用。", }, { "code": "etmall", @@ -1315,7 +1315,6 @@ def sync_targeted_momo_candidates_to_external_offers( ), } - def sync_targeted_momo_review_candidates_to_external_offers( engine, candidates: list[dict[str, Any]], @@ -1633,32 +1632,45 @@ def _normalized_offer_stats(conn) -> dict[str, dict[str, Any]]: if not all(_has_table(conn, table) for table in {"external_market_sources", "external_offers"}): return {} - rows = conn.execute(text(""" + source_columns = { + str(column.get("name") or "") + for column in inspect(conn).get_columns("external_market_sources") + } + write_enabled_sql = ( + "s.write_enabled" if "write_enabled" in source_columns else "s.enabled" + ) + rows = conn.execute(text(f""" SELECT s.code, s.status, s.enabled, + {write_enabled_sql} AS write_enabled, COUNT(o.id) AS offer_count, SUM(CASE WHEN o.data_quality_status IN ('verified', 'usable', 'reviewed') AND COALESCE(o.quality_score, 0) >= 76 + AND (o.expires_at IS NULL OR o.expires_at > CURRENT_TIMESTAMP) THEN 1 ELSE 0 END ) AS usable_offer_count, SUM(CASE - WHEN o.match_status = 'needs_review' - OR o.data_quality_status = 'needs_review' + WHEN ( + o.match_status = 'needs_review' + OR o.data_quality_status = 'needs_review' + ) + AND (o.expires_at IS NULL OR o.expires_at > CURRENT_TIMESTAMP) THEN 1 ELSE 0 END ) AS review_offer_count, MAX(o.observed_at) AS last_seen_at FROM external_market_sources s LEFT JOIN external_offers o ON o.source_code = s.code - GROUP BY s.code, s.status, s.enabled + GROUP BY s.code, s.status, s.enabled, {write_enabled_sql} """)).mappings().all() return { str(row["code"]): { "status": row.get("status"), "enabled": bool(row.get("enabled")), + "write_enabled": bool(row.get("write_enabled")), "offer_count": int(row.get("offer_count") or 0), "usable_offer_count": int(row.get("usable_offer_count") or 0), "review_offer_count": int(row.get("review_offer_count") or 0), @@ -1732,8 +1744,22 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: sources = [dict(source) for source in SOURCE_CONTRACTS] normalized_stats: dict[str, dict[str, Any]] = {} legacy_stats: dict[str, Any] = {"usable_offer_count": 0, "last_seen_at": None} + candidate_runtime_by_source: dict[str, dict[str, Any]] = {} schema_ready = False + try: + from services.yahoo_shopping_public_offer_service import ( + build_yahoo_shopping_runtime_readback, + ) + + yahoo_runtime = build_yahoo_shopping_runtime_readback() + candidate_runtime_by_source["yahoo_shopping"] = yahoo_runtime + except Exception: + logger.warning( + "[ExternalMarket] Yahoo candidate runtime readback failed", + exc_info=True, + ) + if engine is not None: try: with engine.connect() as conn: @@ -1750,8 +1776,10 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: stats = normalized_stats.get(source["code"], {}) runtime_status = str(stats.get("status") or source["status_code"]).strip() or source["status_code"] runtime_enabled = bool(stats.get("enabled")) if stats else source["status_code"] == "active" + runtime_write_enabled = bool(stats.get("write_enabled")) if stats else False source["status_code"] = runtime_status source["enabled"] = runtime_enabled + source["write_enabled"] = runtime_write_enabled source["status_label"] = ( "正在使用" if runtime_status == "active" and runtime_enabled @@ -1773,15 +1801,54 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: source["usable_offer_count"] = usable source["review_offer_count"] = int(stats.get("review_offer_count") or 0) source["last_seen_at"] = last_seen_at - source["has_runtime_data"] = usable > 0 or source["review_offer_count"] > 0 - source["can_alert"] = runtime_status == "active" and runtime_enabled and usable > 0 - if runtime_status == "active" and runtime_enabled: - if usable: - source["plain_state"] = "已接入,可進作戰清單" - elif source["review_offer_count"]: - source["plain_state"] = "已有待確認候選" - else: - source["plain_state"] = "已接入,等待可用資料" + candidate_runtime = candidate_runtime_by_source.get(source["code"], {}) + has_artifact_candidate_data = bool( + candidate_runtime.get("has_candidate_runtime_data") + ) + has_observed_offer_data = usable > 0 or source["review_offer_count"] > 0 + has_formal_runtime_data = bool( + runtime_status == "active" + and runtime_enabled + and usable > 0 + and (source["code"] == "momo_reference" or runtime_write_enabled) + ) + has_candidate_runtime_data = bool( + has_artifact_candidate_data + or (has_observed_offer_data and not has_formal_runtime_data) + ) + candidate_offer_count = int( + candidate_runtime.get("candidate_offer_count") or 0 + ) + if has_observed_offer_data and not has_formal_runtime_data: + candidate_offer_count = max( + candidate_offer_count, + usable + source["review_offer_count"], + ) + source["candidate_runtime"] = candidate_runtime + source["has_observed_offer_data"] = has_observed_offer_data + source["has_runtime_data"] = has_formal_runtime_data + source["has_candidate_runtime_data"] = has_candidate_runtime_data + source["candidate_offer_count"] = candidate_offer_count + source["candidate_last_seen_at"] = candidate_runtime.get("last_seen_at") + source["runtime_stage"] = ( + "formal" + if source["has_runtime_data"] + else "candidate" + if has_candidate_runtime_data + else "registered" + ) + source["can_alert"] = has_formal_runtime_data + if has_formal_runtime_data: + source["status_label"] = "正在使用" + source["plain_state"] = "已接入,可進作戰清單" + elif has_candidate_runtime_data: + source["status_label"] = "候選自動化" + source["plain_state"] = ( + f"已有 {source['candidate_offer_count']} 筆公開來源候選," + "等待正式同款門檻" + ) + elif runtime_status == "active" and runtime_enabled: + source["plain_state"] = "已接入,等待可用資料" else: source["plain_state"] = "先保留接口,不進告警" @@ -1791,6 +1858,17 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: ) paused_count = len(sources) - active_count sources_with_data_count = sum(1 for source in sources if source.get("has_runtime_data")) + candidate_runtime_source_count = sum( + 1 + for source in sources + if source.get("has_candidate_runtime_data") + and not source.get("has_runtime_data") + ) + sources_with_any_runtime_evidence_count = sum( + 1 + for source in sources + if source.get("has_runtime_data") or source.get("has_candidate_runtime_data") + ) usable_count = sum(int(source["usable_offer_count"]) for source in sources) review_offer_count = sum(int(source.get("review_offer_count") or 0) for source in sources) @@ -1801,14 +1879,21 @@ def build_external_source_readiness(engine=None) -> dict[str, Any]: "paused_count": paused_count, "sources_with_data_count": sources_with_data_count, "source_coverage_rate": round(sources_with_data_count / max(len(sources), 1) * 100, 1), + "candidate_runtime_source_count": candidate_runtime_source_count, + "sources_with_any_runtime_evidence_count": sources_with_any_runtime_evidence_count, + "runtime_evidence_coverage_rate": round( + sources_with_any_runtime_evidence_count / max(len(sources), 1) * 100, + 1, + ), "usable_offer_count": usable_count, "review_offer_count": review_offer_count, "sources": sources, "connector_contract": build_connector_contracts(), "offer_evidence_contract": build_offer_evidence_contract(), "plain_summary": ( - f"已列管 {len(sources)} 個來源,{sources_with_data_count} 個已有資料;" - "只有 runtime 已啟用且通過身份與品質驗證的報價會進入決策。" + f"已列管 {len(sources)} 個來源,{sources_with_data_count} 個正式可用、" + f"{candidate_runtime_source_count} 個已有候選 runtime;" + "只有 runtime 已啟用、受控寫入已開放,且通過身份與品質驗證的報價會進入決策。" ), } diff --git a/services/external_marketplace_canary_service.py b/services/external_marketplace_canary_service.py new file mode 100644 index 0000000..96b6f4e --- /dev/null +++ b/services/external_marketplace_canary_service.py @@ -0,0 +1,492 @@ +"""Controlled apply, canary activation, and rollback for external offers.""" + +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from typing import Any + +from sqlalchemy import text + +from services.external_market_offer_service import ( + _ensure_external_market_source_seeds, + _has_table, + _load_json_dict, + _quality_score_from_match, + _targeted_candidate_auto_type, + _targeted_candidate_needs_review, + _targeted_candidate_sync_rank, + _to_float, + _upsert_external_offer, +) +from services.pchome_growth_cache_state import mark_pchome_growth_cache_stale + + +def _targeted_marketplace_candidate_to_external_offer( + candidate: dict[str, Any], + *, + source_code: str, + platform_code: str, + ingestion_method: str, + connector_key: str, + source_label: str, + observed_at: datetime, + ttl_hours: int, +) -> tuple[dict[str, Any] | None, str]: + """Normalize an independently verified marketplace candidate for controlled apply.""" + guard = candidate.get("source_contract_guard") + if not isinstance(guard, dict) or not bool(guard.get("passed")): + return None, "來源契約或商品頁驗證未通過" + reconciliation = candidate.get("same_item_reconciliation") + if not isinstance(reconciliation, dict) or not bool( + reconciliation.get("independent_verifier_passed") + ): + return None, "獨立同款 verifier 未通過" + if _targeted_candidate_auto_type(candidate) != "total_price": + return None, "目前只允許 exact total-price marketplace candidate" + if _targeted_candidate_needs_review(candidate): + return None, "候選仍有規格、款式或促銷條件疑慮" + + source_product_id = str( + candidate.get("product_id") + or candidate.get("goodsCode") + or candidate.get("id") + or "" + ).strip() + pchome_product_id = str(candidate.get("target_pchome_product_id") or "").strip() + source_price = _to_float(candidate.get("price")) + pchome_price = _to_float(candidate.get("target_pchome_price")) + currency = str(candidate.get("currency") or "TWD").upper() + if not source_product_id: + return None, f"缺少{source_label}商品 ID" + if not pchome_product_id: + return None, "缺少 PChome 商品 ID" + if not source_price or source_price <= 0: + return None, f"缺少{source_label}售價" + if not pchome_price or pchome_price <= 0: + return None, "缺少 PChome 公開售價" + if currency != "TWD": + return None, "目前只允許 TWD 公開報價" + + quality_score = _quality_score_from_match(candidate.get("target_match_score")) + if quality_score < 76: + return None, "同款分數低於自動同步門檻" + + title = str( + candidate.get("name") or candidate.get("title") or source_product_id + ).strip() + provenance = candidate.get("source_provenance") + provenance = provenance if isinstance(provenance, dict) else {} + raw_payload = { + "source": connector_key, + "source_code": source_code, + "platform_code": platform_code, + "auto_compare_type": "total_price", + "price_basis": "total_price", + "pchome_public_price": pchome_price, + "pchome_public_name": candidate.get("target_pchome_name"), + "pchome_public_evidence": candidate.get("target_public_evidence") or {}, + "match_score": candidate.get("target_match_score"), + "match_reasons": candidate.get("target_match_reasons") or [], + "comparison_mode": candidate.get("target_comparison_mode"), + "match_type": candidate.get("target_match_type"), + "hard_veto": bool(candidate.get("target_hard_veto")), + "target_gap_pct": candidate.get("target_gap_pct"), + "search_term": candidate.get("target_search_term"), + "source_contract_guard": guard, + "source_provenance": provenance, + "same_item_reconciliation": reconciliation, + "same_item_candidate_fingerprint": candidate.get( + "same_item_candidate_fingerprint" + ), + "tags": [ + "identity_v2", + f"source_{connector_key}", + "public_structured_data", + "product_jsonld_verified", + "price_basis_total_price", + "auto_compare_total_price", + ], + } + return { + "source_code": source_code, + "platform_code": platform_code, + "source_product_id": source_product_id, + "source_offer_key": ( + f"{source_code}:{source_product_id}:{pchome_product_id}:total_price" + ), + "title": title, + "brand": candidate.get("brand"), + "category_text": candidate.get("category") or candidate.get("category_text"), + "product_url": candidate.get("product_url") or candidate.get("url"), + "image_url": candidate.get("image_url"), + "price": source_price, + "original_price": _to_float(candidate.get("original_price")), + "currency": currency, + "stock_status": candidate.get("stock_status") or candidate.get("availability"), + "sold_count": candidate.get("sold_count"), + "rating": _to_float(candidate.get("rating")), + "review_count": candidate.get("review_count"), + "observed_at": observed_at, + "expires_at": observed_at + timedelta(hours=max(1, min(ttl_hours, 72))), + "ingestion_method": ingestion_method, + "connector_key": connector_key, + "pchome_product_id": pchome_product_id, + "momo_sku": None, + "match_status": "verified", + "quality_score": round(quality_score, 2), + "data_quality_status": "verified", + "quality_notes_json": json.dumps( + [ + f"{source_label}公開結構化商品頁已驗證", + "PChome 公開規格與獨立同款 verifier 已通過", + "來源 canary 啟用前已完成 exact DB readback", + ], + ensure_ascii=False, + ), + "raw_payload_json": json.dumps(raw_payload, ensure_ascii=False), + }, "" + + +def sync_targeted_marketplace_candidates_to_external_offers( + engine, + candidates: list[dict[str, Any]], + *, + source_code: str, + platform_code: str, + ingestion_method: str, + connector_key: str, + source_label: str, + dry_run: bool = False, + ttl_hours: int = 30, +) -> dict[str, Any]: + """Controlled apply for exact, independently verified marketplace offers.""" + generated_at = datetime.now().isoformat(timespec="seconds") + candidates = list(candidates or []) + required_tables = {"external_market_sources", "external_offers"} + + with engine.begin() as conn: + missing_tables = sorted( + table for table in required_tables if not _has_table(conn, table) + ) + if missing_tables: + return { + "success": False, + "status": "skipped", + "generated_at": generated_at, + "candidate_count": len(candidates), + "written_count": 0, + "dry_run": dry_run, + "missing_tables": missing_tables, + "message": "外部報價同步暫時無法執行,缺少必要資料表。", + } + + if not dry_run: + _ensure_external_market_source_seeds(conn) + source_exists = conn.execute( + text( + """ + SELECT 1 + FROM external_market_sources + WHERE code = :source_code + """ + ), + {"source_code": source_code}, + ).first() + if not source_exists: + return { + "success": False, + "status": "unregistered_source", + "generated_at": generated_at, + "candidate_count": len(candidates), + "written_count": 0, + "dry_run": False, + "source_code": source_code, + "message": "外部來源尚未完成 registry 登記,本輪沒有寫入。", + } + base_observed_at = datetime.now(timezone.utc) + ranked_offers: list[ + tuple[dict[str, Any], tuple[float, float, float, float]] + ] = [] + skipped_reasons: dict[str, int] = {} + for index, candidate in enumerate(candidates): + offer, reason = _targeted_marketplace_candidate_to_external_offer( + candidate, + source_code=source_code, + platform_code=platform_code, + ingestion_method=ingestion_method, + connector_key=connector_key, + source_label=source_label, + observed_at=base_observed_at + timedelta(microseconds=index), + ttl_hours=ttl_hours, + ) + if offer: + ranked_offers.append((offer, _targeted_candidate_sync_rank(candidate))) + else: + skipped_reasons[reason] = skipped_reasons.get(reason, 0) + 1 + + selected_by_pchome: dict[ + str, + tuple[dict[str, Any], tuple[float, float, float, float]], + ] = {} + for offer, rank in ranked_offers: + key = str(offer.get("pchome_product_id") or "").strip() + existing = selected_by_pchome.get(key) + if existing is None or rank > existing[1]: + selected_by_pchome[key] = (offer, rank) + offers = [offer for offer, _ in selected_by_pchome.values()] + + if not dry_run: + for offer in offers: + _upsert_external_offer(conn, offer) + + if offers and not dry_run: + mark_pchome_growth_cache_stale() + + return { + "success": True, + "status": "dry_run" if dry_run else "synced", + "generated_at": generated_at, + "candidate_count": len(candidates), + "selected_count": len(offers), + "written_count": 0 if dry_run else len(offers), + "dry_run": dry_run, + "source_code": source_code, + "platform_code": platform_code, + "ingestion_method": ingestion_method, + "skipped_reasons": skipped_reasons, + "writes_database_count": 0 if dry_run else len(offers), + "message": ( + f"已把 {source_label} exact 同款寫入外部報價並等待來源 canary。" + if not dry_run + else f"已完成 {source_label} exact 同款寫入預檢,尚未寫入資料。" + ), + } + + +def activate_external_market_source_after_canary( + engine, + *, + source_code: str, + candidates: list[dict[str, Any]], + independent_readback: dict[str, Any], + identity: dict[str, str], + minimum_verified_count: int = 2, +) -> dict[str, Any]: + """Activate a source only after a bounded exact-write canary is read back.""" + candidates = list(candidates or []) + threshold = max(2, min(int(minimum_verified_count or 2), 20)) + readback_count = int(independent_readback.get("readback_pass_count") or 0) + guards_passed = bool(candidates) and all( + isinstance(item.get("source_contract_guard"), dict) + and bool(item["source_contract_guard"].get("passed")) + for item in candidates + ) + independent_verifiers_passed = bool(candidates) and all( + isinstance(item.get("same_item_reconciliation"), dict) + and bool(item["same_item_reconciliation"].get("independent_verifier_passed")) + for item in candidates + ) + can_activate = bool( + independent_readback.get("success") + and len(candidates) >= threshold + and readback_count == len(candidates) + and guards_passed + and independent_verifiers_passed + ) + if not can_activate: + return { + "success": True, + "status": "canary_threshold_not_met", + "source_code": source_code, + "candidate_count": len(candidates), + "readback_pass_count": readback_count, + "minimum_verified_count": threshold, + "guards_passed": guards_passed, + "independent_verifiers_passed": independent_verifiers_passed, + "applied": False, + "writes_database_count": 0, + } + + with engine.begin() as conn: + _ensure_external_market_source_seeds(conn) + previous = ( + conn.execute( + text( + """ + SELECT status, enabled, write_enabled + FROM external_market_sources + WHERE code = :source_code + """ + ), + {"source_code": source_code}, + ) + .mappings() + .first() + ) + if not previous: + return { + "success": False, + "status": "source_registry_missing", + "source_code": source_code, + "applied": False, + "writes_database_count": 0, + } + conn.execute( + text( + """ + UPDATE external_market_sources + SET status = 'active', + enabled = :enabled, + write_enabled = :write_enabled, + updated_at = CURRENT_TIMESTAMP + WHERE code = :source_code + """ + ), + {"source_code": source_code, "enabled": True, "write_enabled": True}, + ) + observed = ( + conn.execute( + text( + """ + SELECT status, enabled, write_enabled + FROM external_market_sources + WHERE code = :source_code + """ + ), + {"source_code": source_code}, + ) + .mappings() + .first() + ) + + applied = bool( + observed + and str(observed.get("status") or "") == "active" + and bool(observed.get("enabled")) + and bool(observed.get("write_enabled")) + ) + if applied: + mark_pchome_growth_cache_stale() + return { + "success": applied, + "status": "activated" if applied else "activation_readback_failed", + "source_code": source_code, + "candidate_count": len(candidates), + "readback_pass_count": readback_count, + "minimum_verified_count": threshold, + "guards_passed": guards_passed, + "independent_verifiers_passed": independent_verifiers_passed, + "applied": applied, + "previous_state": dict(previous or {}), + "observed_state": dict(observed or {}), + "identity": dict(identity or {}), + "writes_database_count": 1 if applied else 0, + } + + +def rollback_external_market_source_canary( + engine, + *, + source_code: str, + ingestion_method: str, + candidates: list[dict[str, Any]], + identity: dict[str, str], + activation_receipt: dict[str, Any], +) -> dict[str, Any]: + """Restore source state and quarantine only rows written by the failed run.""" + previous = activation_receipt.get("previous_state") + previous = previous if isinstance(previous, dict) else {} + run_id = str((identity or {}).get("run_id") or "") + quarantined_ids: list[int] = [] + + with engine.begin() as conn: + for candidate in candidates or []: + rows = ( + conn.execute( + text( + """ + SELECT id, raw_payload_json + FROM external_offers + WHERE source_code = :source_code + AND ingestion_method = :ingestion_method + AND pchome_product_id = :pchome_product_id + AND source_product_id = :source_product_id + ORDER BY observed_at DESC, id DESC + """ + ), + { + "source_code": source_code, + "ingestion_method": ingestion_method, + "pchome_product_id": candidate.get("target_pchome_product_id"), + "source_product_id": ( + candidate.get("product_id") + or candidate.get("goodsCode") + or candidate.get("id") + ), + }, + ) + .mappings() + .all() + ) + for row in rows: + raw = _load_json_dict(row.get("raw_payload_json")) + observed_run_id = str( + (raw.get("same_item_reconciliation") or {}).get("run_id") or "" + ) + if run_id and observed_run_id == run_id: + quarantined_ids.append(int(row["id"])) + + quarantined_ids = sorted(set(quarantined_ids)) + for offer_id in quarantined_ids: + conn.execute( + text( + """ + UPDATE external_offers + SET match_status = 'quarantined', + data_quality_status = 'needs_review', + updated_at = CURRENT_TIMESTAMP + WHERE id = :offer_id + """ + ), + {"offer_id": offer_id}, + ) + + if activation_receipt.get("applied") and previous: + conn.execute( + text( + """ + UPDATE external_market_sources + SET status = :status, + enabled = :enabled, + write_enabled = :write_enabled, + updated_at = CURRENT_TIMESTAMP + WHERE code = :source_code + """ + ), + { + "source_code": source_code, + "status": previous.get("status") or "paused", + "enabled": bool(previous.get("enabled")), + "write_enabled": bool(previous.get("write_enabled")), + }, + ) + + mark_pchome_growth_cache_stale() + return { + "success": True, + "status": "rolled_back_and_quarantined", + "source_code": source_code, + "quarantined_offer_count": len(quarantined_ids), + "source_state_restored": bool(activation_receipt.get("applied") and previous), + "identity": dict(identity or {}), + "writes_database_count": len(quarantined_ids) + + (1 if activation_receipt.get("applied") and previous else 0), + } + + +__all__ = [ + "activate_external_market_source_after_canary", + "rollback_external_market_source_canary", + "sync_targeted_marketplace_candidates_to_external_offers", +] diff --git a/services/pchome_growth_momo_backfill_service.py b/services/pchome_growth_momo_backfill_service.py index 3289041..ba1e1ea 100644 --- a/services/pchome_growth_momo_backfill_service.py +++ b/services/pchome_growth_momo_backfill_service.py @@ -103,6 +103,9 @@ def run_pchome_growth_momo_backfill( sync_func: Callable[[Any, list[dict[str, Any]]], dict[str, Any]] | None = None, sync_review_func: Callable[[Any, list[dict[str, Any]]], dict[str, Any]] | None = None, readback_func: Callable[[Any, list[dict[str, Any]]], dict[str, Any]] | None = None, + source_profile: dict[str, str] | None = None, + source_activation_func: Callable[..., dict[str, Any]] | None = None, + source_rollback_func: Callable[..., dict[str, Any]] | None = None, ) -> dict[str, Any]: """Run a bounded search, independent verifier, apply, and readback loop.""" limit = max(1, min(int(limit or 12), 20)) @@ -111,7 +114,16 @@ def run_pchome_growth_momo_backfill( sync_candidates = sync_func or _default_sync_candidates sync_review_candidates = sync_review_func or _default_sync_review_candidates readback_candidates = readback_func or readback_external_offer_candidates - identity = new_reconciliation_identity() + profile = { + "display_name": "MOMO", + "receipt_source": "momo_targeted_search", + "work_item_id": "GROWTH-P0-001-B", + **(source_profile or {}), + } + source_label = str(profile.get("display_name") or "外部平台") + identity = new_reconciliation_identity( + str(profile.get("work_item_id") or "GROWTH-P0-001-B") + ) generated_at = datetime.now(timezone.utc).isoformat() # The command center contract is top-50. Ranking a smaller pre-sorted slice @@ -165,7 +177,7 @@ def run_pchome_growth_momo_backfill( "success": True, "status": "not_found", "written_count": 0, - "message": "已搜尋 MOMO,但尚未找到可自動寫入的同款或單位價候選。", + "message": f"已搜尋 {source_label},但尚未找到可自動寫入的同款或單位價候選。", } if auto_candidates: external_offer_sync = sync_candidates(engine, auto_candidates) @@ -174,6 +186,19 @@ def run_pchome_growth_momo_backfill( engine, auto_candidates, ) + source_activation = { + "success": True, + "status": "not_requested", + "applied": False, + "writes_database_count": 0, + } + if source_activation_func and auto_candidates: + source_activation = source_activation_func( + engine, + candidates=auto_candidates, + independent_readback=external_offer_sync.get("independent_readback") or {}, + identity=identity, + ) review_candidate_sync = { "success": True, "status": "not_found", @@ -191,6 +216,40 @@ def run_pchome_growth_momo_backfill( sync_result=external_offer_sync, ) written_count = int(external_offer_sync.get("written_count") or 0) + activation_status = str(source_activation.get("status") or "") + activation_failed = bool( + source_activation_func + and auto_candidates + and not source_activation.get("applied") + and activation_status != "canary_threshold_not_met" + ) + source_rollback = { + "success": True, + "status": "not_required", + "writes_database_count": 0, + } + if ( + (not post_verifier.get("all_checks_passed") or activation_failed) + and source_rollback_func + and (written_count > 0 or source_activation.get("applied")) + ): + source_rollback = source_rollback_func( + engine, + candidates=auto_candidates, + identity=identity, + activation_receipt=source_activation, + ) + external_offer_sync["independent_readback"] = readback_candidates( + engine, + auto_candidates, + ) + after_payload = build_payload(engine, 50) + post_verifier = build_coverage_post_verifier( + before_payload=before_payload, + after_payload=after_payload, + verified_candidates=auto_candidates, + sync_result=external_offer_sync, + ) after_snapshot = post_verifier.get("after") or {} unresolved_after = int(after_snapshot.get("candidate_validation_count") or 0) + int( after_snapshot.get("unmatched_count") or 0 @@ -198,6 +257,19 @@ def run_pchome_growth_momo_backfill( if not post_verifier.get("all_checks_passed"): terminal_status = "partial_post_verifier_failed" next_machine_action = "retry_bounded_batch_after_drift_diagnosis" + elif source_activation_func and auto_candidates and not source_activation.get( + "applied" + ): + terminal_status = ( + "partial_source_activation_pending" + if activation_status == "canary_threshold_not_met" + else "partial_source_activation_failed" + ) + next_machine_action = ( + "continue_bounded_source_canary" + if activation_status == "canary_threshold_not_met" + else "retry_source_activation_after_rollback_diagnosis" + ) elif written_count > 0: terminal_status = ( "verified_with_coverage_gain" @@ -224,7 +296,8 @@ def run_pchome_growth_momo_backfill( "generated_at": generated_at, "identity": identity, "source_receipt": { - "source": "momo_targeted_search", + "source": profile.get("receipt_source") or "external_marketplace_search", + "source_label": source_label, "search_success": bool(search_success), "search_message": search_message, "target_count": len(targets), @@ -250,6 +323,7 @@ def run_pchome_growth_momo_backfill( "bounded_target_limit": limit, "formal_price_write_requires_independent_verifier": True, "pixelrag_direct_price_write_allowed": False, + "source_activation_requires_canary_readback": bool(source_activation_func), }, "check_mode": { "receipt_schema_ready": bool(schema_receipt.get("success")), @@ -262,6 +336,8 @@ def run_pchome_growth_momo_backfill( "review_candidate_written_count": int(review_candidate_sync.get("written_count") or 0), "sync_status": external_offer_sync.get("status"), }, + "source_activation": source_activation, + "source_rollback": source_rollback, "post_verifier": post_verifier, "terminal": { "status": terminal_status, @@ -283,16 +359,24 @@ def run_pchome_growth_momo_backfill( "unresolved_after": unresolved_after, } + activation_contract_passed = bool( + not source_activation_func + or not auto_candidates + or source_activation.get("applied") + ) success = bool( post_verifier.get("all_checks_passed") and receipt_persistence.get("success") + and activation_contract_passed ) message = ( - f"已依業績權重掃描 {len(targets)} 個商品,找到 {len(candidates)} 筆候選," + f"已依業績權重掃描 {len(targets)} 個商品並搜尋 {source_label},找到 {len(candidates)} 筆候選," f"獨立驗證 {len(auto_candidates)} 筆,正式寫入並回讀 {post_verifier.get('readback_pass_count', 0)} 筆。" ) if not search_success and not candidates: message = search_message or "已完成 bounded 搜尋,但本輪沒有安全候選。" + elif not activation_contract_passed: + message += " 來源尚未通過 activation canary,報價維持候選狀態。" return { "success": success, @@ -313,6 +397,8 @@ def run_pchome_growth_momo_backfill( "review_count": len(review_candidates), "external_offer_sync": external_offer_sync, "review_candidate_sync": review_candidate_sync, + "source_activation": source_activation, + "source_rollback": source_rollback, "before_stats": before_payload.get("stats") or {}, "after_stats": after_payload.get("stats") or {}, "post_verifier": post_verifier, diff --git a/services/pchome_growth_same_item_reconciliation.py b/services/pchome_growth_same_item_reconciliation.py index 5beed88..4bc4b74 100644 --- a/services/pchome_growth_same_item_reconciliation.py +++ b/services/pchome_growth_same_item_reconciliation.py @@ -43,12 +43,14 @@ def _canonical_hash(payload: Any) -> str: return hashlib.sha256(encoded).hexdigest() -def new_reconciliation_identity() -> dict[str, str]: +def new_reconciliation_identity( + work_item_id: str = WORK_ITEM_ID, +) -> dict[str, str]: run_id = uuid.uuid4().hex return { "trace_id": uuid.uuid4().hex, "run_id": run_id, - "work_item_id": WORK_ITEM_ID, + "work_item_id": str(work_item_id or WORK_ITEM_ID), } @@ -191,6 +193,23 @@ def verify_same_item_candidates( }) continue + source_contract_guard = candidate.get("source_contract_guard") + if isinstance(source_contract_guard, dict) and not bool( + source_contract_guard.get("passed") + ): + blocked.append({ + "target_pchome_product_id": target_id, + "source_product_id": source_id, + "source_name": source_name, + "source_price": _float(candidate.get("price")), + "search_term": str(candidate.get("target_search_term") or ""), + "reason_code": "source_contract_guard_rejected", + "source_contract_guard_reasons": list( + source_contract_guard.get("reasons") or [] + )[:8], + }) + continue + source_price = _float(candidate.get("price")) target_price = _float(target.get("price")) if source_price <= 0 or target_price <= 0: @@ -361,6 +380,9 @@ def verify_same_item_candidates( def readback_external_offer_candidates( engine, candidates: list[dict[str, Any]], + *, + source_code: str = "momo_reference", + ingestion_method: str = "targeted_momo_search", ) -> dict[str, Any]: """Verify committed offers by exact identity on a fresh DB connection.""" expected = list(candidates or []) @@ -390,8 +412,8 @@ def readback_external_offer_candidates( raw_payload_json, observed_at FROM external_offers - WHERE source_code = 'momo_reference' - AND ingestion_method = 'targeted_momo_search' + WHERE source_code = :source_code + AND ingestion_method = :ingestion_method AND pchome_product_id = :pchome_product_id AND source_product_id = :source_product_id ORDER BY observed_at DESC, id DESC @@ -408,6 +430,8 @@ def readback_external_offer_candidates( candidate.get("same_item_candidate_fingerprint") or "" ) row = conn.execute(query, { + "source_code": source_code, + "ingestion_method": ingestion_method, "pchome_product_id": target_id, "source_product_id": source_id, }).mappings().first() diff --git a/services/pchome_growth_yahoo_scheduler_task.py b/services/pchome_growth_yahoo_scheduler_task.py new file mode 100644 index 0000000..6d583ae --- /dev/null +++ b/services/pchome_growth_yahoo_scheduler_task.py @@ -0,0 +1,126 @@ +"""Scheduler adapter for the Yahoo public-offer reconciliation lane.""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime, timedelta, timezone + + +TAIPEI_TZ = timezone(timedelta(hours=8)) + + +def run_scheduled_pchome_growth_yahoo_backfill(*, save_stats): + """Bounded Yahoo public-offer reconciliation with canary and readback.""" + try: + from config import DATABASE_PATH + from sqlalchemy import create_engine + from services.yahoo_shopping_public_offer_service import ( + run_pchome_growth_yahoo_backfill, + ) + + now_str = datetime.now(TAIPEI_TZ).strftime("%Y-%m-%d %H:%M") + limit = max( + 1, + min(int(os.getenv("PCHOME_GROWTH_YAHOO_BACKFILL_LIMIT", "8")), 20), + ) + logging.info( + "[Scheduler] [PChomeGrowthYahooBackfill] starting | %s | limit=%s", + now_str, + limit, + ) + + engine = create_engine(DATABASE_PATH) + try: + result = run_pchome_growth_yahoo_backfill(engine, limit=limit) + finally: + engine.dispose() + + data = result.get("data") or {} + sync_result = data.get("external_offer_sync") or {} + post_verifier = data.get("post_verifier") or {} + activation = data.get("source_activation") or {} + artifact = data.get("artifact_receipt") or {} + identity = data.get("identity") or {} + terminal_status = str(data.get("terminal_status") or "") + stats = { + "status": ( + "Success" + if result.get("success") and terminal_status.startswith("verified") + else "Degraded" + if terminal_status.startswith(("degraded", "partial")) + else "Failed" + if not result.get("success") + else "Skipped" + ), + "trace_id": identity.get("trace_id"), + "run_id": identity.get("run_id"), + "work_item_id": identity.get("work_item_id"), + "terminal_status": terminal_status, + "next_machine_action": data.get("next_machine_action"), + "scanned_products": data.get("scanned_products", 0), + "candidate_count": data.get("candidate_count", 0), + "auto_compare_count": data.get("auto_compare_count", 0), + "review_count": data.get("review_count", 0), + "written_count": sync_result.get("written_count", 0), + "readback_pass_count": post_verifier.get("readback_pass_count", 0), + "post_verifier_status": post_verifier.get("status"), + "source_activation_status": activation.get("status"), + "source_activated": bool(activation.get("applied")), + "artifact_status": artifact.get("status"), + "artifact_sha256": artifact.get("receipt_sha256"), + "message": result.get("message"), + } + logging.info( + "[Scheduler] [PChomeGrowthYahooBackfill] completed | " + "terminal=%s scanned=%s candidates=%s verified=%s written=%s " + "readback=%s activated=%s artifact=%s", + stats["terminal_status"], + stats["scanned_products"], + stats["candidate_count"], + stats["auto_compare_count"], + stats["written_count"], + stats["readback_pass_count"], + stats["source_activated"], + stats["artifact_status"], + ) + save_stats("pchome_growth_yahoo_backfill", stats) + return result + + except Exception as error: + import traceback as _tb + + logging.error( + "[Scheduler] [PChomeGrowthYahooBackfill] task failed | Error: %s", + error, + exc_info=True, + ) + save_stats( + "pchome_growth_yahoo_backfill", + {"status": "Failed", "error": str(error)}, + ) + try: + from services.event_router import notify_failure + + notify_failure( + task_name="run_pchome_growth_yahoo_backfill_task", + error=error, + source="Scheduler.PChomeGrowthYahooBackfill", + event_type="pchome_growth_yahoo_backfill_failure", + priority="P2", + title="PChome 高業績商品 Yahoo 公開來源補齊異常", + trace=_tb.format_exc(), + ) + except Exception as router_error: + logging.error( + "[Scheduler] [PChomeGrowthYahooBackfill] event_router failed: %s", + router_error, + ) + return { + "success": False, + "message": str(error), + "data": {"terminal_status": "failed_with_safe_retry"}, + } + + +__all__ = ["run_scheduled_pchome_growth_yahoo_backfill"] diff --git a/services/pchome_revenue_growth_service.py b/services/pchome_revenue_growth_service.py index 906129f..a8e38a5 100644 --- a/services/pchome_revenue_growth_service.py +++ b/services/pchome_revenue_growth_service.py @@ -418,6 +418,7 @@ def _fetch_normalized_external_offer_map(conn, pchome_product_ids: list[str]) -> AND COALESCE(eo.quality_score, 0) >= 76 AND eo.match_status IN ('verified', 'usable', 'reviewed', 'exact', 'confirmed') AND eo.data_quality_status IN ('verified', 'usable', 'reviewed') + AND (eo.expires_at IS NULL OR eo.expires_at > CURRENT_TIMESTAMP) ) SELECT * FROM latest_offer @@ -1094,12 +1095,18 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] registered_external_source_count = len(source_readiness.get("sources") or []) active_external_source_count = int(source_readiness.get("active_count") or 0) runtime_external_source_count = int(source_readiness.get("sources_with_data_count") or 0) + candidate_runtime_external_source_count = int( + source_readiness.get("candidate_runtime_source_count") or 0 + ) + runtime_evidence_external_source_count = int( + source_readiness.get("sources_with_any_runtime_evidence_count") or 0 + ) active_external_sources = _source_names_by_status(source_readiness, "active") or list(ACTIVE_EXTERNAL_SOURCES) paused_external_sources = _source_names_by_status(source_readiness, "paused") or list(PAUSED_EXTERNAL_SOURCES) observed_external_sources = [ str(source.get("display_name") or source.get("code") or "") for source in source_readiness.get("sources") or [] - if source.get("has_runtime_data") + if source.get("has_runtime_data") or source.get("has_candidate_runtime_data") ] source_scope = { "primary_goal": "提升 PChome 業績", @@ -1107,7 +1114,7 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "active_external_sources": active_external_sources, "paused_external_sources": paused_external_sources, "observed_external_sources": observed_external_sources, - "plain_note": "多平台證據會完整保留;只有 runtime 已啟用且通過同款與品質驗證的報價能驅動正式決策。", + "plain_note": "多平台候選與正式證據分開列示;只有 runtime 已啟用且通過同款與品質驗證的報價能驅動正式決策。", "source_readiness": source_readiness, "offer_evidence_contract": source_readiness.get("offer_evidence_contract") or {}, } @@ -1133,6 +1140,8 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "registered_external_source_count": registered_external_source_count, "active_external_source_count": active_external_source_count, "platforms_with_runtime_data": runtime_external_source_count, + "platforms_with_candidate_runtime_data": candidate_runtime_external_source_count, + "platforms_with_any_runtime_evidence": runtime_evidence_external_source_count, "sales_freshness": _sales_freshness(None), }, "opportunities": [], @@ -1166,6 +1175,8 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "registered_external_source_count": registered_external_source_count, "active_external_source_count": active_external_source_count, "platforms_with_runtime_data": runtime_external_source_count, + "platforms_with_candidate_runtime_data": candidate_runtime_external_source_count, + "platforms_with_any_runtime_evidence": runtime_evidence_external_source_count, "total_sales_7d": 0, "opportunity_sales_7d": 0, "action_counts": {}, @@ -1342,6 +1353,22 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "denominator": registered_external_source_count, "rate": _to_float(source_readiness.get("source_coverage_rate")), "scope": "registered_external_market_sources_with_runtime_data", + "formal": { + "numerator": runtime_external_source_count, + "denominator": registered_external_source_count, + "rate": _to_float(source_readiness.get("source_coverage_rate")), + }, + "candidate_only": { + "numerator": candidate_runtime_external_source_count, + "denominator": registered_external_source_count, + }, + "any_runtime_evidence": { + "numerator": runtime_evidence_external_source_count, + "denominator": registered_external_source_count, + "rate": _to_float( + source_readiness.get("runtime_evidence_coverage_rate") + ), + }, }, } @@ -1380,7 +1407,12 @@ def build_pchome_growth_opportunities(engine, limit: int = 20) -> dict[str, Any] "registered_external_source_count": registered_external_source_count, "active_external_source_count": active_external_source_count, "platforms_with_runtime_data": runtime_external_source_count, + "platforms_with_candidate_runtime_data": candidate_runtime_external_source_count, + "platforms_with_any_runtime_evidence": runtime_evidence_external_source_count, "platform_source_coverage_rate": _to_float(source_readiness.get("source_coverage_rate")), + "platform_runtime_evidence_coverage_rate": _to_float( + source_readiness.get("runtime_evidence_coverage_rate") + ), "comparison_metric_contract": comparison_metric_contract, "blocked_reason_codes": blocked_reason_codes, **catalog_mapping_summary, diff --git a/services/public_source_http_service.py b/services/public_source_http_service.py new file mode 100644 index 0000000..022f61f --- /dev/null +++ b/services/public_source_http_service.py @@ -0,0 +1,44 @@ +"""Bounded HTTP response helpers for public structured-data adapters.""" + +from __future__ import annotations + +from typing import Any + + +def read_bounded_response_text(response: Any, *, max_bytes: int) -> str: + """Read a public response without buffering beyond the configured limit.""" + status_code = int(getattr(response, "status_code", 0) or 0) + if status_code != 200: + raise ValueError(f"public_source_http_{status_code}") + + content_length = str(getattr(response, "headers", {}).get("Content-Length") or "") + if content_length.isdigit() and int(content_length) > max_bytes: + raise ValueError("public_source_response_too_large") + + iter_content = getattr(response, "iter_content", None) + if callable(iter_content): + chunks: list[bytes] = [] + size = 0 + try: + for chunk in iter_content(chunk_size=64 * 1024): + if not chunk: + continue + size += len(chunk) + if size > max_bytes: + raise ValueError("public_source_response_too_large") + chunks.append(chunk) + finally: + close = getattr(response, "close", None) + if callable(close): + close() + content = b"".join(chunks) + else: + content = getattr(response, "content", None) + if content is None: + content = str(getattr(response, "text", "") or "").encode("utf-8") + if len(content) > max_bytes: + raise ValueError("public_source_response_too_large") + return content.decode("utf-8", "ignore") + + +__all__ = ["read_bounded_response_text"] diff --git a/services/yahoo_shopping_public_offer_service.py b/services/yahoo_shopping_public_offer_service.py new file mode 100644 index 0000000..169d4e8 --- /dev/null +++ b/services/yahoo_shopping_public_offer_service.py @@ -0,0 +1,397 @@ +"""Controlled runtime and receipt orchestration for Yahoo public offers.""" + +from __future__ import annotations + +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from services.yahoo_shopping_public_source_adapter import ( + ADAPTER_VERSION, + CONNECTOR_KEY, + DEFAULT_RECEIPT_ROOT, + INGESTION_METHOD, + PLATFORM_CODE, + POLICY, + SOURCE_CODE, + SOURCE_LABEL, + WORK_ITEM_ID, + _bounded_int_env, + _source_contract_guard as _adapter_source_contract_guard, + enrich_targets_with_pchome_public_evidence, + parse_pchome_public_product_jsonp, + parse_yahoo_product_jsonld, + parse_yahoo_search_html, + search_yahoo_candidates_for_pchome_products, +) + + +_source_contract_guard = _adapter_source_contract_guard + + +def _no_write_review_sync( + _engine: Any, + candidates: list[dict[str, Any]], +) -> dict[str, Any]: + return { + "success": True, + "status": "artifact_candidate_only", + "candidate_count": len(candidates or []), + "written_count": 0, + "writes_database_count": 0, + "message": "Yahoo 非 exact 候選只保留在 durable receipt,等待下一輪 AI 證據補強。", + } + + +def _compact_verified_candidates(run: Mapping[str, Any]) -> list[dict[str, Any]]: + verification = run.get("candidate_verification") + verification = verification if isinstance(verification, dict) else {} + return [ + { + "target_pchome_product_id": item.get("target_pchome_product_id"), + "source_product_id": item.get("product_id"), + "source_product_url": item.get("product_url"), + "price": item.get("price"), + "currency": item.get("currency"), + "match_score": item.get("target_match_score"), + "candidate_fingerprint": item.get("same_item_candidate_fingerprint"), + "product_fingerprint_sha256": (item.get("source_provenance") or {}).get( + "product_fingerprint_sha256" + ), + } + for item in verification.get("verified_candidates") or [] + ] + + +def _receipt_summary(data: Mapping[str, Any]) -> dict[str, Any]: + sync = data.get("external_offer_sync") or {} + check_mode = sync.get("check_mode_receipt") or {} + activation = data.get("source_activation") or {} + rollback = data.get("source_rollback") or {} + post_verifier = data.get("post_verifier") or {} + return { + "target_count": int(data.get("target_count") or 0), + "source_candidate_count": int(data.get("candidate_count") or 0), + "verified_candidate_count": int(data.get("auto_compare_count") or 0), + "review_candidate_count": int(data.get("review_count") or 0), + "written_count": int(sync.get("written_count") or 0), + "check_mode_passed": bool( + check_mode.get("success") + and int(check_mode.get("selected_count") or 0) + == int(check_mode.get("candidate_count") or 0) + ), + "readback_pass_count": int(post_verifier.get("readback_pass_count") or 0), + "formal_source_activated": bool(activation.get("applied")), + "writes_database_count": ( + int(sync.get("writes_database_count") or 0) + + int(activation.get("writes_database_count") or 0) + + int(rollback.get("writes_database_count") or 0) + ), + } + + +def _write_compact_receipt( + result: Mapping[str, Any], + *, + receipt_root: str | Path, +) -> dict[str, Any]: + data = result.get("data") if isinstance(result.get("data"), dict) else {} + run = data.get("reconciliation_receipt") + run = run if isinstance(run, dict) else {} + identity = data.get("identity") if isinstance(data.get("identity"), dict) else {} + run_id = str(identity.get("run_id") or "unknown") + summary = _receipt_summary(data) + receipt = { + "policy": POLICY, + "adapter_version": ADAPTER_VERSION, + "work_item_id": WORK_ITEM_ID, + "generated_at": datetime.now(timezone.utc).isoformat(), + "identity": identity, + "success": bool(result.get("success")), + "terminal_status": data.get("terminal_status"), + "next_machine_action": data.get("next_machine_action"), + "source_receipt": run.get("source_receipt") or {}, + "summary": summary, + "verified_candidates": _compact_verified_candidates(run), + "source_activation": data.get("source_activation") or {}, + "source_rollback": data.get("source_rollback") or {}, + "post_verifier": data.get("post_verifier") or {}, + "controlled_apply": { + "public_network_read": True, + "artifact_write": True, + "database_write": bool(summary["writes_database_count"]), + "check_mode_passed": bool(summary["check_mode_passed"]), + "secret_read": False, + "cookie_or_session_read": False, + "login_allowed": False, + "pixelrag_direct_price_write": False, + "independent_same_item_verifier_required": True, + "source_activation_canary_required": True, + }, + "learning_writeback": { + "reconciliation_receipt_persisted": bool( + (data.get("receipt_persistence") or {}).get("success") + ), + "artifact_receipt_persisted": True, + }, + } + root = Path(receipt_root) + root.mkdir(parents=True, exist_ok=True) + target = root / f"{run_id}.json" + temporary = target.with_suffix(".json.tmp") + temporary.write_text( + json.dumps(receipt, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + os.replace(temporary, target) + persisted = json.loads(target.read_text(encoding="utf-8")) + return { + "success": persisted.get("identity") == identity, + "status": "persisted_and_verified", + "receipt_path": str(target), + "receipt_sha256": hashlib.sha256(target.read_bytes()).hexdigest(), + "summary": summary, + } + + +def run_pchome_growth_yahoo_backfill( + engine: Any, + *, + limit: int = 12, + request_get=None, + receipt_root: str | Path | None = None, +) -> dict[str, Any]: + """Run the bounded Yahoo detect-to-verify-to-activate production loop.""" + from services.external_marketplace_canary_service import ( + activate_external_market_source_after_canary, + rollback_external_market_source_canary, + sync_targeted_marketplace_candidates_to_external_offers, + ) + from services.pchome_growth_momo_backfill_service import ( + run_pchome_growth_momo_backfill, + ) + from services.pchome_growth_same_item_reconciliation import ( + readback_external_offer_candidates, + ) + + def search(targets: list[dict[str, Any]], bounded_limit: int): + return search_yahoo_candidates_for_pchome_products( + targets, + bounded_limit, + request_get=request_get, + ) + + def sync(sync_engine: Any, candidates: list[dict[str, Any]]): + sync_kwargs = { + "source_code": SOURCE_CODE, + "platform_code": PLATFORM_CODE, + "ingestion_method": INGESTION_METHOD, + "connector_key": CONNECTOR_KEY, + "source_label": SOURCE_LABEL, + "ttl_hours": _bounded_int_env( + "PCHOME_GROWTH_YAHOO_OFFER_TTL_HOURS", + 30, + 6, + 72, + ), + } + check_mode = sync_targeted_marketplace_candidates_to_external_offers( + sync_engine, + candidates, + dry_run=True, + **sync_kwargs, + ) + if not check_mode.get("success") or int( + check_mode.get("selected_count") or 0 + ) != len(candidates): + return { + "success": False, + "status": "check_mode_failed_no_write", + "candidate_count": len(candidates), + "written_count": 0, + "writes_database_count": 0, + "check_mode_receipt": check_mode, + "message": "Yahoo check-mode 未完整通過,本輪已停止且沒有寫入。", + } + applied = sync_targeted_marketplace_candidates_to_external_offers( + sync_engine, + candidates, + dry_run=False, + **sync_kwargs, + ) + applied["check_mode_receipt"] = check_mode + return applied + + def readback(readback_engine: Any, candidates: list[dict[str, Any]]): + return readback_external_offer_candidates( + readback_engine, + candidates, + source_code=SOURCE_CODE, + ingestion_method=INGESTION_METHOD, + ) + + def activate( + activation_engine: Any, + *, + candidates: list[dict[str, Any]], + independent_readback: dict[str, Any], + identity: dict[str, str], + ): + return activate_external_market_source_after_canary( + activation_engine, + source_code=SOURCE_CODE, + candidates=candidates, + independent_readback=independent_readback, + identity=identity, + minimum_verified_count=_bounded_int_env( + "PCHOME_GROWTH_YAHOO_ACTIVATION_MIN_VERIFIED", + 2, + 2, + 10, + ), + ) + + def rollback( + rollback_engine: Any, + *, + candidates: list[dict[str, Any]], + identity: dict[str, str], + activation_receipt: dict[str, Any], + ): + return rollback_external_market_source_canary( + rollback_engine, + source_code=SOURCE_CODE, + ingestion_method=INGESTION_METHOD, + candidates=candidates, + identity=identity, + activation_receipt=activation_receipt, + ) + + result = run_pchome_growth_momo_backfill( + engine, + limit=limit, + search_func=search, + sync_func=sync, + sync_review_func=_no_write_review_sync, + readback_func=readback, + source_profile={ + "display_name": SOURCE_LABEL, + "receipt_source": CONNECTOR_KEY, + "work_item_id": WORK_ITEM_ID, + }, + source_activation_func=activate, + source_rollback_func=rollback, + ) + try: + artifact = _write_compact_receipt( + result, + receipt_root=receipt_root or DEFAULT_RECEIPT_ROOT, + ) + except Exception as exc: + artifact = { + "success": False, + "status": "artifact_write_failed", + "error_type": type(exc).__name__, + "error": str(exc)[:200], + } + result["success"] = False + result.setdefault("data", {})["artifact_receipt"] = artifact + return result + + +def build_yahoo_shopping_runtime_readback( + *, + receipt_root: str | Path | None = None, + max_age_hours: int = 30, +) -> dict[str, Any]: + """Read the latest public-safe Yahoo runtime receipt without network or DB calls.""" + root = Path(receipt_root or DEFAULT_RECEIPT_ROOT) + paths = sorted( + root.glob("*.json") if root.exists() else [], + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + if not paths: + return { + "success": True, + "status": "not_started", + "source_code": SOURCE_CODE, + "has_candidate_runtime_data": False, + "candidate_offer_count": 0, + "receipt_count": 0, + "writes_database_count": 0, + } + latest_path = paths[0] + try: + receipt = json.loads(latest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return { + "success": False, + "status": "invalid_receipt", + "source_code": SOURCE_CODE, + "has_candidate_runtime_data": False, + "candidate_offer_count": 0, + "receipt_count": len(paths), + "error_type": type(exc).__name__, + "writes_database_count": 0, + } + generated_at = str(receipt.get("generated_at") or "") + try: + generated = datetime.fromisoformat(generated_at.replace("Z", "+00:00")) + if generated.tzinfo is None: + generated = generated.replace(tzinfo=timezone.utc) + age_hours = max( + 0.0, + ( + datetime.now(timezone.utc) - generated.astimezone(timezone.utc) + ).total_seconds() + / 3600, + ) + except ValueError: + age_hours = None + stale = age_hours is None or age_hours > max( + 1, + min(int(max_age_hours or 30), 168), + ) + summary = receipt.get("summary") + summary = summary if isinstance(summary, dict) else {} + candidate_count = int(summary.get("source_candidate_count") or 0) + has_candidate = bool(candidate_count and not stale) + return { + "success": True, + "status": "ready" if has_candidate else "stale" if stale else "no_candidate", + "policy": receipt.get("policy"), + "adapter_version": receipt.get("adapter_version"), + "source_code": SOURCE_CODE, + "platform_code": PLATFORM_CODE, + "has_candidate_runtime_data": has_candidate, + "candidate_offer_count": candidate_count if has_candidate else 0, + "verified_candidate_count": int(summary.get("verified_candidate_count") or 0), + "written_count": int(summary.get("written_count") or 0), + "formal_source_activated": bool(summary.get("formal_source_activated")), + "last_seen_at": generated_at, + "age_hours": round(age_hours, 3) if age_hours is not None else None, + "stale": stale, + "receipt_count": len(paths), + "receipt_path": str(latest_path), + "terminal_status": receipt.get("terminal_status"), + "next_machine_action": receipt.get("next_machine_action"), + "writes_database_count": 0, + } + + +__all__ = [ + "ADAPTER_VERSION", + "DEFAULT_RECEIPT_ROOT", + "POLICY", + "build_yahoo_shopping_runtime_readback", + "enrich_targets_with_pchome_public_evidence", + "parse_pchome_public_product_jsonp", + "parse_yahoo_product_jsonld", + "parse_yahoo_search_html", + "run_pchome_growth_yahoo_backfill", + "search_yahoo_candidates_for_pchome_products", +] diff --git a/services/yahoo_shopping_public_source_adapter.py b/services/yahoo_shopping_public_source_adapter.py new file mode 100644 index 0000000..ef45a2d --- /dev/null +++ b/services/yahoo_shopping_public_source_adapter.py @@ -0,0 +1,798 @@ +"""Public Yahoo/PChome evidence adapter with no session or credential access.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import time +import unicodedata +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Mapping +from urllib.parse import quote, urlparse + +import requests +from bs4 import BeautifulSoup + +from services.public_source_http_service import read_bounded_response_text + + +POLICY = "controlled_yahoo_shopping_public_offer_v1" +ADAPTER_VERSION = "yahoo_shopping_public_jsonld_v1" +WORK_ITEM_ID = "GROWTH-P0-001-C" +SOURCE_CODE = "yahoo_shopping" +PLATFORM_CODE = "yahoo_shopping_tw" +SOURCE_LABEL = "Yahoo 購物" +INGESTION_METHOD = "public_product_jsonld" +CONNECTOR_KEY = "pchome_targeted_yahoo_public_search" +YAHOO_HOST = "tw.buy.yahoo.com" +PCHOME_API_HOST = "ecapi.pchome.com.tw" +YAHOO_SEARCH_URL = "https://tw.buy.yahoo.com/search/product?p={query}" +PCHOME_PUBLIC_PRODUCT_API = "https://ecapi.pchome.com.tw/ecshop/prodapi/v2/prod" +DEFAULT_RECEIPT_ROOT = os.getenv( + "PCHOME_GROWTH_YAHOO_RECEIPT_ROOT", + "/app/data/ai_automation/pchome_growth_yahoo_backfill_receipts" + if Path("/app/data").exists() + else "runtime_artifacts/pchome_growth_yahoo_backfill_receipts", +) +DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; EwoooC-PublicEvidence/1.0)" +MAX_RESPONSE_BYTES = 1_500_000 +SELECTION_MARKERS = ( + "任選", + "多款", + "可選", + "隨機", + "款式可選", + "色號可選", + "香味可選", +) + + +def _bounded_int_env(name: str, default: int, minimum: int, maximum: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except (TypeError, ValueError): + value = default + return max(minimum, min(value, maximum)) + + +def _bounded_float_env( + name: str, + default: float, + minimum: float, + maximum: float, +) -> float: + try: + value = float(os.getenv(name, str(default))) + except (TypeError, ValueError): + value = default + return max(minimum, min(value, maximum)) + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _canonical_hash(payload: Any) -> str: + encoded = json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _compact_text(value: Any) -> str: + text = unicodedata.normalize("NFKC", str(value or "")) + return re.sub(r"\s+", "", text).lower() + + +def _plain_text(value: Any) -> str: + if value in (None, ""): + return "" + return BeautifulSoup(str(value), "html.parser").get_text(" ", strip=True) + + +def _is_public_url(url: str, *, host: str, path_prefix: str = "") -> bool: + parsed = urlparse(str(url or "")) + return bool( + parsed.scheme == "https" + and parsed.hostname == host + and (not path_prefix or parsed.path.startswith(path_prefix)) + and not parsed.username + and not parsed.password + ) + + +def _response_text(response: Any) -> str: + return read_bounded_response_text(response, max_bytes=MAX_RESPONSE_BYTES) + + +def _default_get(url: str, *, timeout: float) -> Any: + return requests.get( + url, + timeout=timeout, + allow_redirects=True, + stream=True, + headers={ + "User-Agent": DEFAULT_USER_AGENT, + "Accept-Language": "zh-TW,zh;q=0.9", + "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8", + }, + ) + + +def _jsonld_nodes(value: Any): + if isinstance(value, dict): + yield value + graph = value.get("@graph") + if graph is not None: + yield from _jsonld_nodes(graph) + elif isinstance(value, list): + for item in value: + yield from _jsonld_nodes(item) + + +def _type_is(node: Mapping[str, Any], expected: str) -> bool: + value = node.get("@type") or node.get("type") + if isinstance(value, str): + return value.lower() == expected.lower() + if isinstance(value, list): + return any(str(item).lower() == expected.lower() for item in value) + return False + + +def _first_mapping(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return dict(value) + if isinstance(value, list): + for item in value: + if isinstance(item, dict): + return dict(item) + return {} + + +def _first_image(value: Any) -> str: + if isinstance(value, str): + return value.strip() + if isinstance(value, dict): + return str(value.get("url") or value.get("contentUrl") or "").strip() + if isinstance(value, list): + for item in value: + image = _first_image(item) + if image: + return image + return "" + + +def _clean_yahoo_product_name(value: Any) -> str: + text = str(value or "").strip() + return re.split(r"\s+\|\s+", text, maxsplit=1)[0].strip() + + +def _has_multi_variant_group_for_spec(title: str, target_spec: str) -> bool: + compact_spec = _compact_text(target_spec) + if not compact_spec: + return False + for group in re.findall(r"[((]([^))]+)[))]", title or ""): + if compact_spec in _compact_text(group) and re.search(r"[//|、]", group): + return True + return False + + +def parse_yahoo_search_html(html: str, *, search_url: str) -> dict[str, Any]: + """Parse Yahoo public search state without executing page scripts.""" + soup = BeautifulSoup(html or "", "html.parser") + script = soup.find("script", id="isoredux-data") + errors: list[str] = [] + payload: dict[str, Any] = {} + if script is None: + errors.append("isoredux_data_missing") + else: + try: + parsed = json.loads(script.string or script.get_text() or "{}") + payload = parsed if isinstance(parsed, dict) else {} + except json.JSONDecodeError: + errors.append("isoredux_data_invalid") + + search = payload.get("search") if isinstance(payload.get("search"), dict) else {} + ecsearch = ( + search.get("ecsearch") if isinstance(search.get("ecsearch"), dict) else {} + ) + hits = ecsearch.get("hits") if isinstance(ecsearch.get("hits"), list) else [] + candidates: list[dict[str, Any]] = [] + for item in hits: + if not isinstance(item, dict): + continue + product_id = str(item.get("ec_productid") or "").strip() + title = str(item.get("ec_title") or "").strip() + product_url = str(item.get("ec_item_url") or "").strip() + price = _to_float(item.get("ec_price")) + if not product_id or not title or price <= 0: + continue + if not _is_public_url(product_url, host=YAHOO_HOST, path_prefix="/gdsale/"): + continue + candidates.append( + { + "product_id": product_id, + "name": title, + "search_price": price, + "original_price": _to_float(item.get("ec_listprice")), + "product_url": product_url, + "image_url": str(item.get("ec_image") or "").strip(), + "brand": str(item.get("ec_brand") or "").strip(), + "seller": str(item.get("ec_storename") or "").strip(), + "stock_status": {"1": "in_stock", "0": "out_of_stock"}.get( + str(item.get("ec_has_stock") or "") + ), + "rating": _to_float(item.get("ec_quality_rating")), + "review_count": int(_to_float(item.get("ec_rating_count"))), + } + ) + + page_title = soup.title.get_text(strip=True) if soup.title else "" + blocked = any( + marker in page_title.lower() + for marker in ("access denied", "captcha", "robot check") + ) + if blocked: + errors.append("blocked_page_detected") + return { + "success": not errors, + "policy": POLICY, + "adapter_version": ADAPTER_VERSION, + "source_method": "public_search_embedded_json", + "search_url": search_url, + "page_title": page_title, + "candidate_count": len(candidates), + "candidates": candidates, + "errors": errors, + "content_sha256": hashlib.sha256((html or "").encode("utf-8")).hexdigest(), + } + + +def parse_yahoo_product_jsonld(html: str, *, product_url: str) -> dict[str, Any]: + """Parse and normalize one Yahoo public Product/Offer JSON-LD payload.""" + soup = BeautifulSoup(html or "", "html.parser") + products: list[dict[str, Any]] = [] + warnings: list[str] = [] + for script in soup.find_all( + "script", + attrs={"type": re.compile(r"ld\+json", re.IGNORECASE)}, + ): + text_value = script.string or script.get_text() or "" + if not text_value.strip(): + continue + try: + data = json.loads(text_value) + except json.JSONDecodeError: + warnings.append("invalid_jsonld_skipped") + continue + for node in _jsonld_nodes(data): + if _type_is(node, "Product"): + products.append(dict(node)) + + if not products: + return { + "success": False, + "policy": POLICY, + "adapter_version": ADAPTER_VERSION, + "product_url": product_url, + "errors": ["product_jsonld_missing"], + "warnings": warnings, + } + + product = products[0] + offer = _first_mapping(product.get("offers")) + brand = _first_mapping(product.get("brand")) + seller = _first_mapping(offer.get("seller")) + rating = _first_mapping(product.get("aggregateRating")) + product_id = str(product.get("sku") or "").strip() + title = _clean_yahoo_product_name(product.get("name")) + price = _to_float(offer.get("price")) + currency = str(offer.get("priceCurrency") or "").strip().upper() + errors = [] + if not product_id: + errors.append("product_id_missing") + if not title: + errors.append("product_title_missing") + if price <= 0: + errors.append("positive_price_missing") + if currency != "TWD": + errors.append("currency_not_twd") + if not _is_public_url(product_url, host=YAHOO_HOST, path_prefix="/gdsale/"): + errors.append("product_url_outside_allowlist") + + normalized = { + "product_id": product_id, + "name": title, + "price": price, + "currency": currency, + "product_url": product_url, + "offer_url": str(offer.get("url") or "").strip(), + "image_url": _first_image(product.get("image")), + "brand": str(brand.get("name") or "").strip(), + "gtin13": str(product.get("gtin13") or "").strip(), + "seller": str(seller.get("name") or "").strip(), + "availability": str(offer.get("availability") or "").strip(), + "rating": _to_float(rating.get("ratingValue")), + "review_count": int(_to_float(rating.get("ratingCount"))), + } + return { + "success": not errors, + "policy": POLICY, + "adapter_version": ADAPTER_VERSION, + "source_method": "public_product_jsonld", + "product_url": product_url, + "product": normalized, + "errors": errors, + "warnings": warnings, + "content_sha256": hashlib.sha256((html or "").encode("utf-8")).hexdigest(), + "product_fingerprint_sha256": _canonical_hash(normalized), + } + + +def parse_pchome_public_product_jsonp(payload: str) -> dict[str, Any]: + """Parse the public PChome JSONP wrapper without evaluating JavaScript.""" + start = str(payload or "").find("jsonp(") + if start < 0: + return {} + start += len("jsonp(") + end = str(payload or "").find(");}catch", start) + if end < 0: + end = str(payload or "").find(");", start) + if end < 0: + return {} + try: + parsed = json.loads(str(payload or "")[start:end]) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _target_current_price(product: Mapping[str, Any], fallback: Any) -> tuple[float, str]: + prices = product.get("Price") if isinstance(product.get("Price"), dict) else {} + for key in ("P", "Low", "M"): + value = _to_float(prices.get(key)) + if value > 0: + return value, key + return _to_float(fallback), "sales_report_fallback" + + +def enrich_targets_with_pchome_public_evidence( + targets: list[dict[str, Any]], + *, + request_get: Callable[..., Any] | None = None, + timeout: float | None = None, +) -> dict[str, Any]: + """Attach current public PChome name, price, and Spec to bounded targets.""" + ids = [ + str(item.get("product_id") or "").strip() + for item in targets or [] + if str(item.get("product_id") or "").strip() + ] + if not ids: + return {"success": True, "evidence_count": 0, "missing_count": 0} + timeout_seconds = timeout or _bounded_float_env( + "PCHOME_GROWTH_YAHOO_HTTP_TIMEOUT_SECONDS", 15, 3, 30 + ) + url = ( + f"{PCHOME_PUBLIC_PRODUCT_API}?id={quote(','.join(ids), safe=',')}" + "&fields=Id,Name,Nick,Price,Spec,Qty&_callback=jsonp" + ) + getter = request_get or _default_get + try: + response = getter(url, timeout=timeout_seconds) + final_url = str(getattr(response, "url", url) or url) + if not _is_public_url(final_url, host=PCHOME_API_HOST, path_prefix="/ecshop/"): + raise ValueError("pchome_public_api_redirect_outside_allowlist") + products = parse_pchome_public_product_jsonp(_response_text(response)) + except Exception as exc: + return { + "success": False, + "evidence_count": 0, + "missing_count": len(ids), + "error_type": type(exc).__name__, + "error": str(exc)[:200], + } + + evidence_count = 0 + for target in targets: + product_id = str(target.get("product_id") or "").strip() + product = products.get(product_id) + if not isinstance(product, dict): + continue + public_name = str(product.get("Name") or target.get("name") or "").strip() + public_spec = _plain_text(product.get("Spec")) + public_price, public_price_field = _target_current_price(product, target.get("price")) + prices = product.get("Price") if isinstance(product.get("Price"), dict) else {} + original_name = str(target.get("name") or "").strip() + original_price = target.get("price") + target["name"] = ( + f"{public_name} {public_spec}" if public_spec else public_name + ).strip() + target["price"] = public_price + target["target_public_evidence"] = { + "source": "pchome_public_product_api", + "product_id": product_id, + "name": public_name, + "spec": public_spec, + "current_price": public_price, + "current_price_field": public_price_field, + "price_fields": { + key: _to_float(prices.get(key)) + for key in ("P", "Low", "M") + if _to_float(prices.get(key)) > 0 + }, + "sales_realized_price": original_price, + "sales_report_name": original_name, + **({"quantity": int(_to_float(product.get("Qty")))} if product.get("Qty") not in (None, "") else {}), + } + evidence_count += 1 + + return { + "success": evidence_count == len(ids), + "evidence_count": evidence_count, + "missing_count": max(0, len(ids) - evidence_count), + "source_url": url, + "source_method": "pchome_public_product_api", + } + + +def _source_contract_guard( + *, + target: Mapping[str, Any], + search_candidate: Mapping[str, Any], + product_readback: Mapping[str, Any], +) -> dict[str, Any]: + detail = product_readback.get("product") + detail = detail if isinstance(detail, dict) else {} + reasons: list[str] = [] + source_title = str(detail.get("name") or search_candidate.get("name") or "") + target_evidence = target.get("target_public_evidence") + target_evidence = target_evidence if isinstance(target_evidence, dict) else {} + target_spec = str(target_evidence.get("spec") or "").strip() + if not product_readback.get("success"): + reasons.append("product_jsonld_invalid") + if str(detail.get("product_id") or "") != str( + search_candidate.get("product_id") or "" + ): + reasons.append("search_detail_product_id_mismatch") + if any(marker in source_title for marker in SELECTION_MARKERS): + reasons.append("source_selection_listing") + if _has_multi_variant_group_for_spec(source_title, target_spec): + reasons.append("source_multi_variant_listing") + if target_spec and _compact_text(target_spec) not in _compact_text(source_title): + reasons.append("target_spec_missing_from_source_title") + if not target_evidence.get("current_price") or target_evidence.get("current_price_field") == "sales_report_fallback": + reasons.append("pchome_current_price_missing") + if "quantity" in target_evidence and int( + _to_float(target_evidence.get("quantity")) + ) <= 0: + reasons.append("pchome_stock_not_available") + source_stock_status = str( + detail.get("availability") or search_candidate.get("stock_status") or "" + ).lower() + if source_stock_status and not source_stock_status.replace("_", "").endswith( + "instock" + ): + reasons.append("source_stock_not_confirmed") + if str(detail.get("currency") or "") != "TWD": + reasons.append("source_currency_not_twd") + if _to_float(detail.get("price")) <= 0: + reasons.append("source_positive_price_missing") + if not _is_public_url( + str(detail.get("product_url") or ""), + host=YAHOO_HOST, + path_prefix="/gdsale/", + ): + reasons.append("source_url_outside_allowlist") + return { + "policy": POLICY, + "version": ADAPTER_VERSION, + "passed": not reasons, + "reasons": reasons, + "public_source_boundary": True, + "rate_limit_contract": "bounded_targets_and_terms", + "provenance_contract": "search_html_hash_plus_product_jsonld_hash", + "target_spec_required_when_present": True, + "stock_gate": "reject_explicit_unavailable_evidence", + "selection_listing_rejected": True, + "direct_price_write_allowed": False, + "requires_independent_same_item_verifier": True, + "requires_activation_canary": True, + } + + +def search_yahoo_candidates_for_pchome_products( + targets: list[dict[str, Any]], + limit: int, + *, + request_get: Callable[..., Any] | None = None, +) -> tuple[bool, str, list[dict[str, Any]]]: + """Discover and validate Yahoo candidates for revenue-weighted PChome targets.""" + from services.marketplace_product_matcher import ( + build_search_terms, + build_unit_price_comparison, + score_marketplace_match, + ) + + target_limit = max(1, min(int(limit or 12), 20)) + term_limit = _bounded_int_env("PCHOME_GROWTH_YAHOO_MAX_TERMS_PER_TARGET", 1, 1, 3) + detail_limit = _bounded_int_env( + "PCHOME_GROWTH_YAHOO_DETAIL_CANDIDATES_PER_TARGET", 3, 1, 5 + ) + minimum_score = _bounded_float_env( + "PCHOME_GROWTH_YAHOO_MIN_SCORE", 0.55, 0.45, 0.85 + ) + timeout = _bounded_float_env("PCHOME_GROWTH_YAHOO_HTTP_TIMEOUT_SECONDS", 15, 3, 30) + raw_getter = request_get or _default_get + request_delay = _bounded_float_env( + "PCHOME_GROWTH_YAHOO_REQUEST_DELAY_SECONDS", + 0.2, + 0.0, + 2.0, + ) + last_request_at = 0.0 + + def getter(url: str, *, timeout: float): + nonlocal last_request_at + if request_get is None and request_delay > 0: + elapsed = time.monotonic() - last_request_at + if elapsed < request_delay: + time.sleep(request_delay - elapsed) + response = raw_getter(url, timeout=timeout) + last_request_at = time.monotonic() + return response + + bounded_targets = list(targets or [])[:target_limit] + pchome_evidence = enrich_targets_with_pchome_public_evidence( + bounded_targets, + request_get=getter, + timeout=timeout, + ) + if not pchome_evidence.get("success"): + return ( + False, + "PChome 公開商品規格或目前售價未完整讀回,本輪已停止正式候選。", + [], + ) + + candidates: list[dict[str, Any]] = [] + searched_target_count = 0 + search_request_count = 0 + product_request_count = 0 + source_errors: list[str] = [] + for target in bounded_targets: + target_id = str(target.get("product_id") or "").strip() + target_name = str(target.get("name") or "").strip() + target_price = _to_float(target.get("price")) + if not target_id or not target_name or target_price <= 0: + continue + searched_target_count += 1 + original_name = str( + (target.get("target_public_evidence") or {}).get("name") or target_name + ).strip() + terms = [original_name] + terms.extend(build_search_terms(original_name, max_terms=term_limit)) + unique_terms: list[str] = [] + for term in terms: + term = re.sub(r"\s+", " ", str(term or "").strip()) + if term and term.lower() not in {item.lower() for item in unique_terms}: + unique_terms.append(term) + if len(unique_terms) >= term_limit: + break + + discovered: dict[str, dict[str, Any]] = {} + for term in unique_terms: + search_url = YAHOO_SEARCH_URL.format(query=quote(term)) + try: + response = getter(search_url, timeout=timeout) + search_request_count += 1 + final_url = str(getattr(response, "url", search_url) or search_url) + if not _is_public_url( + final_url, host=YAHOO_HOST, path_prefix="/search/" + ): + raise ValueError("yahoo_search_redirect_outside_allowlist") + search_readback = parse_yahoo_search_html( + _response_text(response), + search_url=final_url, + ) + except Exception as exc: + source_errors.append(f"{target_id}:{type(exc).__name__}") + continue + for item in search_readback.get("candidates") or []: + product_id = str(item.get("product_id") or "") + if product_id and product_id not in discovered: + discovered[product_id] = { + **item, + "target_search_term": term, + "search_provenance": { + "search_url": final_url, + "content_sha256": search_readback.get("content_sha256"), + "source_method": search_readback.get("source_method"), + }, + } + + preliminaries: list[tuple[float, dict[str, Any]]] = [] + for item in discovered.values(): + diagnostics = score_marketplace_match( + str(item.get("name") or ""), + target_name, + momo_price=_to_float(item.get("search_price")), + competitor_price=target_price, + ) + score = _to_float(getattr(diagnostics, "score", 0.0)) + if score >= minimum_score: + preliminaries.append((score, item)) + preliminaries.sort(key=lambda pair: pair[0], reverse=True) + + for _, search_candidate in preliminaries[:detail_limit]: + product_url = str(search_candidate.get("product_url") or "") + try: + response = getter(product_url, timeout=timeout) + product_request_count += 1 + final_url = str(getattr(response, "url", product_url) or product_url) + if not _is_public_url( + final_url, host=YAHOO_HOST, path_prefix="/gdsale/" + ): + raise ValueError("yahoo_product_redirect_outside_allowlist") + detail_readback = parse_yahoo_product_jsonld( + _response_text(response), + product_url=final_url, + ) + except Exception as exc: + source_errors.append(f"{target_id}:{type(exc).__name__}") + continue + detail = detail_readback.get("product") + detail = detail if isinstance(detail, dict) else {} + source_name = str(detail.get("name") or search_candidate.get("name") or "") + source_price = _to_float(detail.get("price")) + diagnostics = score_marketplace_match( + source_name, + target_name, + momo_price=source_price, + competitor_price=target_price, + ) + score = _to_float(getattr(diagnostics, "score", 0.0)) + if score < minimum_score: + continue + guard = _source_contract_guard( + target=target, + search_candidate=search_candidate, + product_readback=detail_readback, + ) + comparison_mode = str(getattr(diagnostics, "comparison_mode", "") or "") + price_basis = str(getattr(diagnostics, "price_basis", "") or "") + alert_tier = str(getattr(diagnostics, "alert_tier", "") or "") + match_type = str(getattr(diagnostics, "match_type", "") or "") + hard_veto = bool(getattr(diagnostics, "hard_veto", True)) + exact_ready = bool( + guard.get("passed") + and not hard_veto + and comparison_mode == "exact_identity" + and price_basis == "total_price" + and alert_tier == "price_alert_exact" + and match_type == "exact" + ) + unit_comparison: dict[str, Any] = {} + unit_ready = False + if comparison_mode == "unit_comparable" or price_basis == "unit_price": + unit_comparison = build_unit_price_comparison( + source_name, + target_name, + momo_price=source_price, + competitor_price=target_price, + ) + unit_ready = bool( + guard.get("passed") and unit_comparison.get("comparable") + ) + auto_type = ( + "total_price" + if exact_ready + else "unit_price" + if unit_ready + else "manual_review" + ) + gap_pct = ( + round((source_price - target_price) / target_price * 100, 2) + if source_price > 0 and target_price > 0 + else None + ) + source_provenance = { + **(search_candidate.get("search_provenance") or {}), + "product_url": final_url, + "product_content_sha256": detail_readback.get("content_sha256"), + "product_fingerprint_sha256": detail_readback.get( + "product_fingerprint_sha256" + ), + "adapter_version": ADAPTER_VERSION, + "observed_at": datetime.now(timezone.utc).isoformat(), + } + candidates.append( + { + "product_id": detail.get("product_id") + or search_candidate.get("product_id"), + "name": source_name, + "title": source_name, + "price": source_price, + "original_price": search_candidate.get("original_price"), + "currency": detail.get("currency") or "TWD", + "product_url": final_url, + "image_url": detail.get("image_url") + or search_candidate.get("image_url"), + "brand": detail.get("brand") or search_candidate.get("brand"), + "gtin13": detail.get("gtin13"), + "seller": detail.get("seller") or search_candidate.get("seller"), + "stock_status": detail.get("availability") + or search_candidate.get("stock_status"), + "rating": detail.get("rating") or search_candidate.get("rating"), + "review_count": detail.get("review_count") + or search_candidate.get("review_count"), + "target_pchome_product_id": target_id, + "target_pchome_name": target_name, + "target_pchome_price": target_price, + "target_public_evidence": target.get("target_public_evidence") + or {}, + "target_search_term": search_candidate.get("target_search_term"), + "target_match_score": round(score, 3), + "target_match_reasons": list( + getattr(diagnostics, "reasons", ()) or () + ), + "target_comparison_mode": comparison_mode, + "target_match_type": match_type, + "target_alert_tier": alert_tier, + "target_hard_veto": hard_veto, + "target_price_basis": "total_price" if exact_ready else price_basis, + "target_gap_pct": gap_pct, + "target_unit_price_comparison": unit_comparison, + "auto_compare_type": auto_type, + "can_auto_compare": auto_type in {"total_price", "unit_price"}, + "source_strategy": CONNECTOR_KEY, + "source_contract_guard": guard, + "source_provenance": source_provenance, + } + ) + + exact_count = sum( + 1 for item in candidates if item.get("auto_compare_type") == "total_price" + ) + guarded_count = sum( + 1 + for item in candidates + if not bool((item.get("source_contract_guard") or {}).get("passed")) + ) + success = bool(searched_target_count and (candidates or not source_errors)) + message = ( + f"已用 {searched_target_count} 筆高業績商品搜尋 Yahoo," + f"取得 {len(candidates)} 筆商品頁候選,{exact_count} 筆通過來源前置檢查," + f"{guarded_count} 筆因規格或選項風險阻擋。" + ) + if source_errors and not candidates: + message = "Yahoo 公開來源本輪無可用候選,已安全停止且不寫正式價格。" + return success, message, candidates + + +__all__ = [ + "ADAPTER_VERSION", + "CONNECTOR_KEY", + "DEFAULT_RECEIPT_ROOT", + "INGESTION_METHOD", + "PLATFORM_CODE", + "POLICY", + "SOURCE_CODE", + "SOURCE_LABEL", + "WORK_ITEM_ID", + "enrich_targets_with_pchome_public_evidence", + "parse_pchome_public_product_jsonp", + "parse_yahoo_product_jsonld", + "parse_yahoo_search_html", + "search_yahoo_candidates_for_pchome_products", +] diff --git a/templates/ai_intelligence.html b/templates/ai_intelligence.html index d5cb2c1..c392a5a 100644 --- a/templates/ai_intelligence.html +++ b/templates/ai_intelligence.html @@ -1961,6 +1961,11 @@ background: rgba(235, 248, 241, 0.78); } + .growth-source-chip.is-candidate { + border-color: rgba(34, 105, 190, 0.24); + background: rgba(237, 245, 255, 0.8); + } + .growth-source-name { display: flex; align-items: center; @@ -1986,6 +1991,11 @@ color: #1f6d4c; } + .growth-source-chip.is-candidate .growth-source-status { + background: rgba(34, 105, 190, 0.13); + color: #205d9d; + } + .growth-source-detail { display: none; } @@ -2921,11 +2931,11 @@
- 正式資料平台 + 外部平台資料
— / — - 僅計正式 runtime 資料 + 正式與候選分開計算
@@ -4006,7 +4016,10 @@ function renderGrowthCommandCenter(stats = {}, rows = []) { const activeMappedCount = Number(stats.active_mapped_product_count || 0); const platformCount = Number(stats.registered_external_source_count || 0); const runtimePlatformCount = Number(stats.platforms_with_runtime_data || 0); - const platformRate = Number(stats.platform_source_coverage_rate || 0); + const candidateRuntimePlatformCount = Number(stats.platforms_with_candidate_runtime_data || 0); + const runtimeEvidencePlatformCount = Number( + stats.platforms_with_any_runtime_evidence || runtimePlatformCount + ); setText('commandSales7d', sales7d ? formatMoney(sales7d) : '—'); setText('commandSalesDelta', prevSales7d @@ -4026,8 +4039,11 @@ function renderGrowthCommandCenter(stats = {}, rows = []) { setText('commandReviewCount', formatCount(candidateValidationCount)); setText('commandTopCategory', stats.top_category || '—'); setText('commandTopCategorySales', stats.top_category_sales_7d ? formatMoney(stats.top_category_sales_7d) : '等待分類業績'); - setText('commandPlatformCoverage', `${formatCount(runtimePlatformCount)} / ${formatCount(platformCount)}`); - setText('commandPlatformCoverageNote', `${platformRate.toFixed(1)}% · 僅計正式 runtime 資料`); + setText('commandPlatformCoverage', `${formatCount(runtimeEvidencePlatformCount)} / ${formatCount(platformCount)}`); + setText( + 'commandPlatformCoverageNote', + `${formatCount(runtimePlatformCount)} 正式 · ${formatCount(candidateRuntimePlatformCount)} 候選` + ); setText('commandStatusReady', formatCount(mappedCount)); setText('commandStatusReview', formatCount(candidateValidationCount)); setText('commandStatusNeeds', formatCount(unmatchedCount)); @@ -4442,16 +4458,22 @@ function renderGrowthSourceReadiness(sources) { } const visibleSources = sources.filter((source) => - source.status_code === 'active' || source.has_runtime_data + source.status_code === 'active' || source.has_runtime_data || source.has_candidate_runtime_data ); const pendingCount = Math.max(0, sources.length - visibleSources.length); const cards = visibleSources.map((source) => { const usable = Number(source.usable_offer_count || 0); + const candidateCount = Number(source.candidate_offer_count || 0); const detail = usable > 0 ? `${source.data_quality_label || '資料可用'} · ${usable.toLocaleString()} 筆` + : candidateCount > 0 + ? `公開來源候選 · ${candidateCount.toLocaleString()} 筆` : `${source.plain_state || source.plain_note || '等待資料接入'}`; const activeClass = source.status_code === 'active' ? ' is-active' : ''; - return `
+ const candidateClass = source.has_candidate_runtime_data && !source.has_runtime_data + ? ' is-candidate' + : ''; + return `
${escapeHtml(source.display_name || '未命名來源')} ${escapeHtml(source.status_label || '待確認')} diff --git a/tests/test_ai_intelligence_text_density_guardrails.py b/tests/test_ai_intelligence_text_density_guardrails.py index 98f0829..b5df76b 100644 --- a/tests/test_ai_intelligence_text_density_guardrails.py +++ b/tests/test_ai_intelligence_text_density_guardrails.py @@ -92,7 +92,9 @@ def test_ai_intelligence_comparison_coverage_has_explicit_scope_and_partition(): )[0] assert "同款比價證據 · TOP 50" in template - assert "正式資料平台" in template + assert "外部平台資料" in template + assert "正式 ·" in renderer + assert "候選" in renderer assert "比價覆蓋 · TOP 50" in template assert "具正式比價證據" in template assert "候選 / 來源驗證" in template diff --git a/tests/test_external_market_offer_service.py b/tests/test_external_market_offer_service.py index 38ccf38..3274158 100644 --- a/tests/test_external_market_offer_service.py +++ b/tests/test_external_market_offer_service.py @@ -625,6 +625,47 @@ def test_external_source_readiness_uses_legacy_momo_reference_cache(): assert "只有 runtime 已啟用" in payload["plain_summary"] +def test_external_source_readiness_separates_candidate_runtime_from_formal( + monkeypatch, + tmp_path, +): + import json + from datetime import datetime, timezone + from services import yahoo_shopping_public_source_adapter as adapter + from services.external_market_offer_service import build_external_source_readiness + + receipt = { + "policy": "controlled_yahoo_shopping_public_offer_v1", + "adapter_version": "yahoo_shopping_public_jsonld_v1", + "generated_at": datetime.now(timezone.utc).isoformat(), + "summary": { + "source_candidate_count": 4, + "verified_candidate_count": 0, + "written_count": 0, + "formal_source_activated": False, + }, + } + (tmp_path / "candidate-run.json").write_text( + json.dumps(receipt), + encoding="utf-8", + ) + monkeypatch.setattr(adapter, "DEFAULT_RECEIPT_ROOT", str(tmp_path)) + from services import yahoo_shopping_public_offer_service as runtime + + monkeypatch.setattr(runtime, "DEFAULT_RECEIPT_ROOT", str(tmp_path)) + + payload = build_external_source_readiness() + sources = {source["code"]: source for source in payload["sources"]} + + assert payload["sources_with_data_count"] == 0 + assert payload["candidate_runtime_source_count"] == 1 + assert payload["sources_with_any_runtime_evidence_count"] == 1 + assert sources["yahoo_shopping"]["runtime_stage"] == "candidate" + assert sources["yahoo_shopping"]["status_label"] == "候選自動化" + assert sources["yahoo_shopping"]["candidate_offer_count"] == 4 + assert sources["yahoo_shopping"]["can_alert"] is False + + def test_source_seeds_preserve_runtime_activation_and_readiness_uses_it(): from services import external_market_offer_service as service @@ -649,6 +690,142 @@ def test_source_seeds_preserve_runtime_activation_and_readiness_uses_it(): assert bool(source["enabled"]) is True +def test_targeted_marketplace_canary_activates_and_rollback_quarantines(monkeypatch): + from services import external_marketplace_canary_service as canary_service + from services.external_market_offer_service import build_external_source_readiness + from services.pchome_growth_same_item_reconciliation import ( + readback_external_offer_candidates, + ) + + monkeypatch.setattr( + canary_service, + "mark_pchome_growth_cache_stale", + lambda: None, + ) + engine = create_engine("sqlite:///:memory:") + _seed_external_offer_sync_tables(engine) + identity = { + "trace_id": "trace-yahoo", + "run_id": "run-yahoo", + "work_item_id": "GROWTH-P0-001-C", + } + candidates = [] + for index in (1, 2): + candidates.append({ + "product_id": f"Y-{index}", + "name": f"Yahoo 同款商品 {index}", + "price": 800 + index, + "currency": "TWD", + "product_url": f"https://tw.buy.yahoo.com/gdsale/product-{index}.html", + "target_pchome_product_id": f"P-{index}", + "target_pchome_name": f"PChome 同款商品 {index}", + "target_pchome_price": 900 + index, + "target_match_score": 1.0, + "target_match_reasons": [], + "target_comparison_mode": "exact_identity", + "target_match_type": "exact", + "target_alert_tier": "price_alert_exact", + "target_price_basis": "total_price", + "target_hard_veto": False, + "auto_compare_type": "total_price", + "source_contract_guard": {"passed": True, "reasons": []}, + "same_item_candidate_fingerprint": f"fingerprint-{index}", + "same_item_reconciliation": { + **identity, + "independent_verifier_passed": True, + }, + }) + + preview = canary_service.sync_targeted_marketplace_candidates_to_external_offers( + engine, + candidates, + source_code="yahoo_shopping", + platform_code="yahoo_shopping_tw", + ingestion_method="public_product_jsonld", + connector_key="pchome_targeted_yahoo_public_search", + source_label="Yahoo 購物", + dry_run=True, + ) + with engine.connect() as conn: + source_count = conn.execute( + text("SELECT COUNT(*) FROM external_market_sources") + ).scalar_one() + offer_count = conn.execute( + text("SELECT COUNT(*) FROM external_offers") + ).scalar_one() + assert preview["status"] == "dry_run" + assert preview["selected_count"] == 2 + assert preview["writes_database_count"] == 0 + assert source_count == 0 + assert offer_count == 0 + + sync = canary_service.sync_targeted_marketplace_candidates_to_external_offers( + engine, + candidates, + source_code="yahoo_shopping", + platform_code="yahoo_shopping_tw", + ingestion_method="public_product_jsonld", + connector_key="pchome_targeted_yahoo_public_search", + source_label="Yahoo 購物", + ) + readback = readback_external_offer_candidates( + engine, + candidates, + source_code="yahoo_shopping", + ingestion_method="public_product_jsonld", + ) + activation = canary_service.activate_external_market_source_after_canary( + engine, + source_code="yahoo_shopping", + candidates=candidates, + independent_readback=readback, + identity=identity, + minimum_verified_count=2, + ) + + assert sync["written_count"] == 2 + assert readback["success"] is True + assert readback["readback_pass_count"] == 2 + assert activation["status"] == "activated" + assert activation["applied"] is True + assert bool(activation["observed_state"]["write_enabled"]) is True + readiness = build_external_source_readiness(engine) + yahoo = next( + source for source in readiness["sources"] if source["code"] == "yahoo_shopping" + ) + assert yahoo["runtime_stage"] == "formal" + assert yahoo["has_runtime_data"] is True + assert yahoo["write_enabled"] is True + + rollback = canary_service.rollback_external_market_source_canary( + engine, + source_code="yahoo_shopping", + ingestion_method="public_product_jsonld", + candidates=candidates, + identity=identity, + activation_receipt=activation, + ) + + assert rollback["quarantined_offer_count"] == 2 + assert rollback["source_state_restored"] is True + with engine.connect() as conn: + source = conn.execute(text(""" + SELECT status, enabled, write_enabled + FROM external_market_sources + WHERE code = 'yahoo_shopping' + """)).mappings().one() + rows = conn.execute(text(""" + SELECT match_status, data_quality_status + FROM external_offers + WHERE source_code = 'yahoo_shopping' + """)).mappings().all() + assert source["status"] == "paused" + assert bool(source["enabled"]) is False + assert bool(source["write_enabled"]) is False + assert {row["match_status"] for row in rows} == {"quarantined"} + assert {row["data_quality_status"] for row in rows} == {"needs_review"} + + def test_source_aliases_normalize_pixelrag_platform_codes(): from services.external_market_offer_service import normalize_external_offer_payload @@ -708,7 +885,23 @@ def test_external_offer_sync_is_registered_in_scheduler(): assert "sync_legacy_momo_reference_offers" in scheduler_source assert "def run_pchome_growth_momo_backfill_task" in scheduler_source assert "run_pchome_growth_momo_backfill" in scheduler_source + assert "def run_pchome_growth_yahoo_backfill_task" in scheduler_source + assert "run_pchome_growth_yahoo_backfill" in scheduler_source assert "run_external_offer_sync_task" in run_scheduler_source assert "run_pchome_growth_momo_backfill_task" in run_scheduler_source + assert "run_pchome_growth_yahoo_backfill_task" in run_scheduler_source assert "external_offer_sync" in run_scheduler_source assert "pchome_growth_momo_backfill" in run_scheduler_source + assert "pchome_growth_yahoo_backfill" in run_scheduler_source + + +def test_yahoo_public_offer_readback_route_is_login_protected(): + from pathlib import Path + + route_source = Path("routes/system_public_routes.py").read_text(encoding="utf-8") + + marker = "@system_public_bp.route('/api/ai-automation/yahoo-shopping-public-offer')" + assert marker in route_source + route_block = route_source[route_source.index(marker):] + assert "@login_required" in route_block[:200] + assert "build_yahoo_shopping_runtime_readback" in route_block[:600] diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index e6e2424..fae564f 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -1265,15 +1265,18 @@ def test_ai_product_pick_agent_uses_real_competitor_data_and_dashboard_action(): agent_actions_source = (ROOT / "services/agent_actions.py").read_text(encoding="utf-8") assert "def run_pchome_match_backfill_task" in scheduler_source assert "def run_pchome_growth_momo_backfill_task" in scheduler_source + assert "def run_pchome_growth_yahoo_backfill_task" in scheduler_source assert "_save_stats('pchome_match_backfill'" in scheduler_source assert "_save_stats('pchome_growth_momo_backfill'" in scheduler_source assert "retryable_candidate_revalidation_total" in scheduler_source assert "run_pchome_match_backfill_task" in run_scheduler_source assert "run_pchome_growth_momo_backfill_task" in run_scheduler_source + assert "run_pchome_growth_yahoo_backfill_task" in run_scheduler_source assert "每日 10:30:pchome_match_backfill" in run_scheduler_source assert "每日 10:45:pchome_growth_momo_backfill" in run_scheduler_source assert '"run_pchome_match_backfill_task"' in agent_actions_source assert '"run_pchome_growth_momo_backfill_task"' in agent_actions_source + assert '"run_pchome_growth_yahoo_backfill_task"' in agent_actions_source assert "產生今日清單" in template assert "補齊比價資料" in template diff --git a/tests/test_pchome_revenue_growth_service.py b/tests/test_pchome_revenue_growth_service.py index 3ce95fd..947bfce 100644 --- a/tests/test_pchome_revenue_growth_service.py +++ b/tests/test_pchome_revenue_growth_service.py @@ -647,6 +647,105 @@ def test_pchome_growth_momo_backfill_service_targets_unmapped_high_sales_items() assert captured["search_limit"] == 2 +def test_backfill_rolls_back_failed_write_before_source_activation(monkeypatch): + from services import pchome_growth_momo_backfill_service as service + + engine = create_engine("sqlite:///:memory:") + candidate = { + "product_id": "Y-ROLLBACK", + "target_pchome_product_id": "PCH-ROLLBACK", + "auto_compare_type": "total_price", + } + payload = { + "success": True, + "stats": {"candidate_validation_count": 1, "unmatched_count": 0}, + "opportunities": [ + { + "pchome_product_id": "PCH-ROLLBACK", + "product_name": "需回滾商品", + "pchome_price": 900, + "sales_7d": 1000, + "external_price": None, + "recommended_action": {"code": "map_external_product"}, + } + ], + } + rollback_calls = [] + + monkeypatch.setattr( + service, + "ensure_evidence_receipt_table", + lambda _engine: {"success": True}, + ) + monkeypatch.setattr( + service, + "verify_same_item_candidates", + lambda _targets, _candidates, *, identity: { + "verified_candidates": [{**candidate, "same_item_reconciliation": identity}], + "review_candidates": [], + "blocked_candidate_count": 0, + }, + ) + monkeypatch.setattr( + service, + "build_coverage_post_verifier", + lambda **_kwargs: { + "all_checks_passed": False, + "status": "readback_failed", + "readback_pass_count": 0, + "after": {"candidate_validation_count": 1, "unmatched_count": 0}, + }, + ) + monkeypatch.setattr( + service, + "persist_reconciliation_receipts", + lambda *_args, **_kwargs: {"success": True, "readback_count": 1}, + ) + + def rollback(_engine, **kwargs): + rollback_calls.append(kwargs) + return { + "success": True, + "status": "rolled_back_and_quarantined", + "writes_database_count": 1, + "source_state_restored": False, + } + + result = service.run_pchome_growth_momo_backfill( + engine, + limit=1, + build_payload_func=lambda _engine, _limit: payload, + search_func=lambda _targets, _limit: (True, "found", [candidate]), + sync_func=lambda _engine, _candidates: { + "success": True, + "status": "synced", + "written_count": 1, + }, + sync_review_func=lambda _engine, _candidates: { + "success": True, + "written_count": 0, + }, + readback_func=lambda _engine, _candidates: { + "success": False, + "readback_pass_count": 0, + }, + source_profile={"work_item_id": "GROWTH-P0-001-C"}, + source_activation_func=lambda _engine, **_kwargs: { + "success": True, + "status": "canary_threshold_not_met", + "applied": False, + "writes_database_count": 0, + }, + source_rollback_func=rollback, + ) + + assert result["success"] is False + assert result["data"]["terminal_status"] == "partial_post_verifier_failed" + assert result["data"]["source_rollback"]["status"] == "rolled_back_and_quarantined" + assert len(rollback_calls) == 1 + assert rollback_calls[0]["activation_receipt"]["applied"] is False + + def test_pchome_growth_momo_backfill_default_search_uses_deeper_terms(monkeypatch): from services import pchome_growth_momo_backfill_service as service diff --git a/tests/test_pchome_same_item_reconciliation.py b/tests/test_pchome_same_item_reconciliation.py index 6e960ee..eca5397 100644 --- a/tests/test_pchome_same_item_reconciliation.py +++ b/tests/test_pchome_same_item_reconciliation.py @@ -117,6 +117,39 @@ def test_independent_verifier_promotes_exact_and_unit_candidates_but_rejects_con assert result["blocked_candidates"][0]["reasons"] +def test_independent_verifier_rejects_failed_source_contract_before_scoring(): + from services.pchome_growth_same_item_reconciliation import ( + verify_same_item_candidates, + ) + + result = verify_same_item_candidates( + [_target("P-SHADE", "Estee Lauder 粉持久粉底 1W2", 1690, 1)], + [{ + "product_id": "Y-GENERIC", + "name": "Estee Lauder 粉持久粉底", + "price": 1590, + "target_pchome_product_id": "P-SHADE", + "source_contract_guard": { + "passed": False, + "reasons": ["target_spec_missing_from_source_title"], + }, + }], + identity={ + "trace_id": "trace-shade", + "run_id": "run-shade", + "work_item_id": "GROWTH-P0-001-C", + }, + ) + + assert result["selected_candidate_count"] == 0 + assert result["blocked_candidate_count"] == 1 + blocked = result["blocked_candidates"][0] + assert blocked["reason_code"] == "source_contract_guard_rejected" + assert blocked["source_contract_guard_reasons"] == [ + "target_spec_missing_from_source_title" + ] + + def test_independent_verifier_prefers_exact_sheet_pack_over_unit_candidate(): from services.pchome_growth_same_item_reconciliation import ( verify_same_item_candidates, diff --git a/tests/test_run_scheduler_embed_consistency.py b/tests/test_run_scheduler_embed_consistency.py index 4d39f9e..351cc68 100644 --- a/tests/test_run_scheduler_embed_consistency.py +++ b/tests/test_run_scheduler_embed_consistency.py @@ -232,6 +232,7 @@ def test_roi_ai_smoke_and_daily_report_schedules_stay_staggered(): assert 'schedule.every().day.at("10:30").do(run_pchome_match_backfill_task)' in source assert 'schedule.every(30).minutes.do(run_revenue_automation_cycle_background_task)' in source assert 'schedule.every().day.at("10:45").do(run_pchome_growth_momo_backfill_background_task)' in source + assert 'schedule.every(6).hours.do(run_pchome_growth_yahoo_backfill_background_task)' in source assert "start_revenue_automation_watchdog()" in source assert "schedule.every(6).hours.do(run_action_plan_hygiene_task)" in source assert "schedule.every(15).minutes.do(run_ollama_111_usage_guard_check)" in source diff --git a/tests/test_yahoo_shopping_public_offer_service.py b/tests/test_yahoo_shopping_public_offer_service.py new file mode 100644 index 0000000..fb330d2 --- /dev/null +++ b/tests/test_yahoo_shopping_public_offer_service.py @@ -0,0 +1,511 @@ +import json + + +class _FakeResponse: + def __init__(self, url, body, status_code=200): + self.url = url + self.content = body.encode("utf-8") + self.status_code = status_code + + +def _search_html(*, product_id="1001", title="LANCOME 蘭蔻 零粉感水凝粉餅 10g #01"): + payload = { + "search": { + "ecsearch": { + "hits": [ + { + "ec_productid": product_id, + "ec_title": title, + "ec_item_url": ( + f"https://tw.buy.yahoo.com/gdsale/product-{product_id}.html" + ), + "ec_price": "1890", + "ec_listprice": "2100", + "ec_image": "https://img.example.test/1001.jpg", + "ec_brand": "LANCOME", + "ec_storename": "Yahoo購物中心", + "ec_has_stock": "1", + "ec_quality_rating": "4.9", + "ec_rating_count": "18", + } + ] + } + } + } + return ( + "Yahoo 搜尋" + f"" + "" + ) + + +def _product_html(*, product_id="1001", title="LANCOME 蘭蔻 零粉感水凝粉餅 10g #01"): + payload = { + "@context": "https://schema.org", + "@type": "Product", + "sku": product_id, + "name": f"{title} | Yahoo購物中心", + "brand": {"@type": "Brand", "name": "LANCOME"}, + "gtin13": "3614271234567", + "image": ["https://img.example.test/1001.jpg"], + "offers": { + "@type": "Offer", + "price": "1890", + "priceCurrency": "TWD", + "url": f"https://tw.buy.yahoo.com/gdsale/gdsale.asp?gdid={product_id}", + "seller": {"@type": "Organization", "name": "Yahoo購物中心"}, + }, + "aggregateRating": { + "@type": "AggregateRating", + "ratingValue": "4.9", + "ratingCount": "18", + }, + } + return ( + "商品頁" + f"" + "" + ) + + +def test_yahoo_public_parsers_extract_search_and_product_jsonld(): + from services.yahoo_shopping_public_offer_service import ( + parse_yahoo_product_jsonld, + parse_yahoo_search_html, + ) + + search_url = "https://tw.buy.yahoo.com/search/product?p=lancome" + product_url = "https://tw.buy.yahoo.com/gdsale/product-1001.html" + search = parse_yahoo_search_html(_search_html(), search_url=search_url) + product = parse_yahoo_product_jsonld( + _product_html(), + product_url=product_url, + ) + + assert search["success"] is True + assert search["candidate_count"] == 1 + assert search["candidates"][0]["product_id"] == "1001" + assert search["candidates"][0]["search_price"] == 1890 + assert search["content_sha256"] + assert product["success"] is True + assert product["product"]["product_id"] == "1001" + assert product["product"]["price"] == 1890 + assert product["product"]["currency"] == "TWD" + assert product["product"]["gtin13"] == "3614271234567" + assert product["product_fingerprint_sha256"] + + +def test_yahoo_source_contract_rejects_missing_pchome_spec(): + from services.yahoo_shopping_public_offer_service import _source_contract_guard + + product_url = "https://tw.buy.yahoo.com/gdsale/product-1001.html" + guard = _source_contract_guard( + target={ + "target_public_evidence": { + "spec": "1W2", + "current_price": 1690, + } + }, + search_candidate={"product_id": "1001"}, + product_readback={ + "success": True, + "product": { + "product_id": "1001", + "name": "Estee Lauder 粉持久完美持妝粉底", + "price": 1590, + "currency": "TWD", + "product_url": product_url, + }, + }, + ) + + assert guard["passed"] is False + assert "target_spec_missing_from_source_title" in guard["reasons"] + assert guard["direct_price_write_allowed"] is False + + +def test_yahoo_source_contract_rejects_multi_variant_group_even_when_spec_appears(): + from services.yahoo_shopping_public_offer_service import _source_contract_guard + + product_url = "https://tw.buy.yahoo.com/gdsale/product-2002.html" + guard = _source_contract_guard( + target={ + "target_public_evidence": { + "spec": "無香", + "current_price": 179, + } + }, + search_candidate={"product_id": "2002"}, + product_readback={ + "success": True, + "product": { + "product_id": "2002", + "name": "GATSBY 魔法激凍體用噴霧170ml(海洋冰封/無香薄荷)", + "price": 169, + "currency": "TWD", + "product_url": product_url, + }, + }, + ) + + assert guard["passed"] is False + assert "source_multi_variant_listing" in guard["reasons"] + + +def test_yahoo_source_contract_rejects_unavailable_target_or_source_stock(): + from services.yahoo_shopping_public_offer_service import _source_contract_guard + + product_url = "https://tw.buy.yahoo.com/gdsale/product-3003.html" + guard = _source_contract_guard( + target={ + "target_public_evidence": { + "spec": "100ml", + "current_price": 975, + "quantity": 0, + } + }, + search_candidate={"product_id": "3003", "stock_status": "out_of_stock"}, + product_readback={ + "success": True, + "product": { + "product_id": "3003", + "name": "施巴 抗乾敏滋潤浴露100ml", + "price": 299, + "currency": "TWD", + "product_url": product_url, + }, + }, + ) + + assert guard["passed"] is False + assert "pchome_stock_not_available" in guard["reasons"] + assert "source_stock_not_confirmed" in guard["reasons"] + + +def test_yahoo_source_contract_rejects_sales_report_price_fallback(): + from services.yahoo_shopping_public_offer_service import _source_contract_guard + + product_url = "https://tw.buy.yahoo.com/gdsale/product-4004.html" + guard = _source_contract_guard( + target={ + "target_public_evidence": { + "spec": "100ml", + "current_price": 975, + "current_price_field": "sales_report_fallback", + } + }, + search_candidate={"product_id": "4004", "stock_status": "in_stock"}, + product_readback={ + "success": True, + "product": { + "product_id": "4004", + "name": "施巴 抗乾敏滋潤浴露100ml", + "price": 999, + "currency": "TWD", + "product_url": product_url, + }, + }, + ) + + assert guard["passed"] is False + assert "pchome_current_price_missing" in guard["reasons"] + + +def test_yahoo_source_contract_accepts_product_detail_instock_evidence(): + from services.yahoo_shopping_public_offer_service import _source_contract_guard + + product_url = "https://tw.buy.yahoo.com/gdsale/product-5005.html" + guard = _source_contract_guard( + target={ + "target_public_evidence": { + "spec": "100ml", + "current_price": 975, + "current_price_field": "P", + } + }, + search_candidate={"product_id": "5005", "stock_status": "unknown"}, + product_readback={ + "success": True, + "product": { + "product_id": "5005", + "name": "施巴 抗乾敏滋潤浴露100ml", + "price": 999, + "currency": "TWD", + "availability": "https://schema.org/InStock", + "product_url": product_url, + }, + }, + ) + + assert guard["passed"] is True + assert "source_stock_not_confirmed" not in guard["reasons"] + + +def test_public_source_stream_reader_stops_at_byte_limit(): + from services.public_source_http_service import read_bounded_response_text + + class StreamingResponse: + status_code = 200 + headers = {} + closed = False + + def iter_content(self, *, chunk_size): + assert chunk_size > 0 + yield b"1234" + yield b"5678" + + def close(self): + self.closed = True + + response = StreamingResponse() + + try: + read_bounded_response_text(response, max_bytes=6) + except ValueError as exc: + assert str(exc) == "public_source_response_too_large" + else: + raise AssertionError("oversized public response was accepted") + assert response.closed is True + + +def test_yahoo_search_builds_guarded_candidate_from_two_public_sources(monkeypatch): + from services.yahoo_shopping_public_offer_service import ( + search_yahoo_candidates_for_pchome_products, + ) + + monkeypatch.setenv("PCHOME_GROWTH_YAHOO_MAX_TERMS_PER_TARGET", "1") + monkeypatch.setenv("PCHOME_GROWTH_YAHOO_DETAIL_CANDIDATES_PER_TARGET", "1") + pchome_payload = { + "PCH-1": { + "Id": "PCH-1", + "Name": "LANCOME 蘭蔻 零粉感水凝粉餅 10g", + "Spec": "#01", + "Price": {"P": 1690, "M": 1890}, + "Qty": 3, + } + } + + def fake_get(url, *, timeout): + assert timeout > 0 + if "ecapi.pchome.com.tw" in url: + body = f"try{{jsonp({json.dumps(pchome_payload)});}}catch(e){{}}" + return _FakeResponse(url, body) + if "/search/product" in url: + return _FakeResponse(url, _search_html()) + if "/gdsale/" in url: + return _FakeResponse(url, _product_html()) + raise AssertionError(url) + + success, message, candidates = search_yahoo_candidates_for_pchome_products( + [ + { + "product_id": "PCH-1", + "name": "LANCOME 蘭蔻 零粉感水凝粉餅 10g", + "price": 1650, + } + ], + 1, + request_get=fake_get, + ) + + assert success is True + assert "1 筆商品頁候選" in message + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate["target_pchome_price"] == 1690 + assert candidate["target_public_evidence"]["spec"] == "#01" + assert candidate["target_public_evidence"]["current_price_field"] == "P" + assert candidate["target_public_evidence"]["price_fields"] == { + "P": 1690.0, + "M": 1890.0, + } + assert candidate["source_contract_guard"]["passed"] is True + assert candidate["source_provenance"]["content_sha256"] + assert candidate["source_provenance"]["product_content_sha256"] + assert candidate["auto_compare_type"] == "total_price" + + +def test_yahoo_runtime_readback_uses_fresh_compact_receipt(tmp_path): + from services.yahoo_shopping_public_offer_service import ( + _write_compact_receipt, + build_yahoo_shopping_runtime_readback, + ) + + identity = { + "trace_id": "trace-yahoo", + "run_id": "run-yahoo", + "work_item_id": "GROWTH-P0-001-C", + } + result = { + "success": True, + "data": { + "identity": identity, + "terminal_status": "verified_write_readback", + "next_machine_action": "continue_next_revenue_weighted_batch", + "target_count": 2, + "candidate_count": 2, + "auto_compare_count": 2, + "review_count": 0, + "external_offer_sync": { + "written_count": 2, + "writes_database_count": 2, + }, + "source_activation": {"applied": True, "writes_database_count": 1}, + "source_rollback": {"writes_database_count": 0}, + "post_verifier": {"readback_pass_count": 2}, + "receipt_persistence": {"success": True}, + "reconciliation_receipt": { + "source_receipt": {"source": "public_search"}, + "candidate_verification": { + "verified_candidates": [ + { + "target_pchome_product_id": "PCH-1", + "product_id": "1001", + "product_url": "https://tw.buy.yahoo.com/gdsale/product-1001.html", + "price": 1890, + "currency": "TWD", + "target_match_score": 1.0, + "same_item_candidate_fingerprint": "candidate-fingerprint", + "source_provenance": { + "product_fingerprint_sha256": "product-fingerprint" + }, + } + ] + }, + }, + }, + } + + persisted = _write_compact_receipt(result, receipt_root=tmp_path) + readback = build_yahoo_shopping_runtime_readback(receipt_root=tmp_path) + + assert persisted["success"] is True + assert readback["status"] == "ready" + assert readback["has_candidate_runtime_data"] is True + assert readback["candidate_offer_count"] == 2 + assert readback["verified_candidate_count"] == 2 + assert readback["formal_source_activated"] is True + assert readback["writes_database_count"] == 0 + + +def test_yahoo_runtime_forces_check_mode_before_controlled_apply(monkeypatch): + from services import external_marketplace_canary_service as canary_service + from services import pchome_growth_momo_backfill_service as backfill_service + from services import yahoo_shopping_public_offer_service as runtime + + calls = [] + + def fake_sync(_engine, candidates, *, dry_run, **_kwargs): + calls.append(dry_run) + return { + "success": True, + "status": "dry_run" if dry_run else "synced", + "candidate_count": len(candidates), + "selected_count": len(candidates), + "written_count": 0 if dry_run else len(candidates), + "writes_database_count": 0 if dry_run else len(candidates), + } + + def fake_backfill(_engine, **kwargs): + sync = kwargs["sync_func"](_engine, [{"product_id": "Y-1"}]) + return { + "success": True, + "data": { + "external_offer_sync": sync, + "identity": {"run_id": "run-yahoo-check-mode"}, + "reconciliation_receipt": {}, + }, + } + + monkeypatch.setattr( + canary_service, + "sync_targeted_marketplace_candidates_to_external_offers", + fake_sync, + ) + monkeypatch.setattr( + backfill_service, + "run_pchome_growth_momo_backfill", + fake_backfill, + ) + monkeypatch.setattr( + runtime, + "_write_compact_receipt", + lambda _result, receipt_root: { + "success": True, + "receipt_path": str(receipt_root), + }, + ) + + result = runtime.run_pchome_growth_yahoo_backfill( + object(), + receipt_root="/tmp/yahoo-check-mode-test", + ) + + assert calls == [True, False] + sync = result["data"]["external_offer_sync"] + assert sync["written_count"] == 1 + assert sync["check_mode_receipt"]["status"] == "dry_run" + assert sync["check_mode_receipt"]["writes_database_count"] == 0 + + +def test_yahoo_scheduler_adapter_records_terminal_receipt(monkeypatch): + import sqlalchemy + from services import pchome_growth_yahoo_scheduler_task as scheduler_task + from services import yahoo_shopping_public_offer_service as runtime + + captured = {"stats": []} + + class FakeEngine: + def dispose(self): + captured["disposed"] = True + + monkeypatch.setenv("PCHOME_GROWTH_YAHOO_BACKFILL_LIMIT", "5") + monkeypatch.setattr(sqlalchemy, "create_engine", lambda _path: FakeEngine()) + monkeypatch.setattr( + runtime, + "run_pchome_growth_yahoo_backfill", + lambda engine, limit: { + "success": True, + "message": "verified", + "data": { + "identity": { + "trace_id": "trace-scheduler", + "run_id": "run-scheduler", + "work_item_id": "GROWTH-P0-001-C", + }, + "terminal_status": "verified_with_coverage_gain", + "next_machine_action": "continue_next_revenue_weighted_batch", + "scanned_products": limit, + "candidate_count": 4, + "auto_compare_count": 3, + "review_count": 1, + "external_offer_sync": {"written_count": 3}, + "post_verifier": { + "readback_pass_count": 3, + "status": "verified", + }, + "source_activation": { + "status": "activated", + "applied": True, + }, + "artifact_receipt": { + "status": "persisted_and_verified", + "receipt_sha256": "receipt-hash", + }, + }, + }, + ) + + result = scheduler_task.run_scheduled_pchome_growth_yahoo_backfill( + save_stats=lambda key, value: captured["stats"].append((key, value)) + ) + + assert result["success"] is True + assert captured["disposed"] is True + key, stats = captured["stats"][0] + assert key == "pchome_growth_yahoo_backfill" + assert stats["status"] == "Success" + assert stats["work_item_id"] == "GROWTH-P0-001-C" + assert stats["readback_pass_count"] == 3 + assert stats["source_activated"] is True + assert stats["artifact_sha256"] == "receipt-hash"