fix(ai): preserve fail-closed RAG receipt truth

config.py

@@ -414,7 +414,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '')
 # ==========================================
 # 系統版本與路徑
 # ==========================================
-SYSTEM_VERSION = "V10.812"
+SYSTEM_VERSION = "V10.813"
 LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log')
 public_url = PUBLIC_URL  # 用於模板顯示
 

docs/AI_INTELLIGENCE_MODULE_SOT.md

@@ -1,15 +1,15 @@
 # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth
 
 > **最後更新**: 2026-07-17 (台北時間)
-> **狀態**: 🟠 Partial。Production runtime 已是 V10.811；daily scheduled internal RAG candidate canary 與四 AI Agent 產品整合 truth readback 已上線，但不得標示完整整合。Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` 證明 candidate replay `1/1` 成功、2 個 knowledge contracts ready、Telegram acknowledgement 成功；GCP-A connect timeout、GCP-B read timeout 導致 canary failed，且 `MCP_ROUTER_ENABLED=false`、`RAG_ENABLED=false`、MCP/RAG telemetry 均為 0。四 Agent source wiring 為 `4/4`，runtime active `2/4`、healthy `0/4`、closure `3/9`；程式存在或 smoke 顯示 ok 都不能覆蓋這個 runtime truth。V10.812 source 另加入空 embedding 進 DB 前的 fail-closed 修正，尚待部署。
-> **適用版本**: V10.811 production runtime；V10.812 source candidate pending Gitea/deploy/runtime verification
+> **狀態**: 🟠 Partial。Production runtime 已回滾至 V10.811；daily scheduled internal RAG candidate canary 與四 AI Agent 產品整合 truth readback 已上線，但不得標示完整整合。Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` 證明 candidate replay `1/1` 成功、2 個 knowledge contracts ready、Telegram acknowledgement 成功；GCP-A connect timeout、GCP-B read timeout 導致 canary failed，且 `MCP_ROUTER_ENABLED=false`、`RAG_ENABLED=false`、MCP/RAG telemetry 均為 0。V10.812 retry run `47ca3ccd-f7b4-4ec9-97dd-8c14690f33a7` 已在正式環境證明 host preflight 失敗時 `database_call_performed=false`、Telegram acknowledged，隨後因空 log grep verifier 語意錯誤自動回滾。四 Agent source wiring 為 `4/4`，runtime active `2/4`、healthy `0/4`、closure `3/9`；程式存在或 smoke 顯示 ok 都不能覆蓋這個 runtime truth。V10.813 source 進一步讓 latest/top-level receipt 如實保留 no-DB-call terminal，尚待部署。
+> **適用版本**: V10.811 production runtime；V10.813 source candidate pending Gitea/deploy/runtime verification
 
 ---
 
 ## 零之負五、AI Agent 產品整合 truth 與 internal RAG canary（V10.811 source）
 
 - `/api/ai-automation/agent-product-integration` 與 `scripts/ops/report_ai_agent_product_integration.py` 分開輸出四 Agent source/scheduler wiring、七日 `ai_calls` 實際呼叫與錯誤率、MCP/RAG telemetry、action plan/outcome、AutoHeal incident retry，以及九階段 closure。只有四 Agent 全部有健康 runtime、MCP/RAG 已啟用且有 telemetry、internal RAG canary 已通過、受控執行/驗證/重試/學習都有實證時才可回 `fully_integrated`。
-- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次，CLI 只保留給 Windows 99 controlled apply / verifier。每次最多一筆 candidate knowledge receipt，使用 Ollama-first BGE-M3 與 GCP-A/GCP-B consistency probe；V10.812 起任一主要 host 不可達或向量維度錯誤會在 DB 前 fail closed，只有完整 1024-dim embedding 才能進 PostgreSQL `SET TRANSACTION READ ONLY` pgvector probe，最後 rollback。
+- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次，CLI 只保留給 Windows 99 controlled apply / verifier。每次最多一筆 candidate knowledge receipt，使用 Ollama-first BGE-M3 與 GCP-A/GCP-B consistency probe；V10.812 起任一主要 host 不可達或向量維度錯誤會在 DB 前 fail closed，只有完整 1024-dim embedding 才能進 PostgreSQL `SET TRANSACTION READ ONLY` pgvector probe，最後 rollback。V10.813 起 upstream 單節點 fail-safe 不可覆蓋 canary 的雙主機 requirement，latest 與 top-level receipt 必須區分 no-DB-call、read-only transaction 與 rollback terminal。
 - canary 不 INSERT/UPDATE `ai_insights`、`competitor_prices`、`external_offers` 或任何正式價格表；artifact 與 scheduler receipt 共用 `trace_id/run_id/work_item_id`，明確輸出 `transaction_read_only`、similarity、embedding signature、GCP-A/GCP-B reachability、Telegram acknowledgement 與 zero-write/rollback terminal。簽名漂移、任一主要 GCP embedding host 不可達、pgvector probe 失敗或 semantic threshold 未達都必須 fail closed。
 - `bge-m3:latest` 仍是 immutable supply-chain blocker；即使 canary 通過，也只能回 `canary_passed_activation_blocked`，必須先鎖定可重現模型版本，再以 Windows 99 controlled apply 啟用 RAG shadow/canary。MCP runtime 仍受 localhost-only、read-only tool contract與 required secret presence preflight 約束，不可因 registry 已存在就宣稱上線。
 - `AI Agent product integration truth` 與 `Internal RAG candidate canary` 已接入 `/api/ai-automation/smoke`。舊的 NemoTron/ElephantAlpha class/method smoke 保留為 source guard，但不再能單獨代表產品整合完成。

docs/guides/ai_automation_mainline_work_items.md

@@ -23,7 +23,7 @@
 | 5 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout, secret-safe `.dockerignore`, commit-bound source receipt and governance gate are active. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. |
 | 6 | `GOV-P0-001` | In progress | Canonical full asset graph + runtime reconciliation | `governance/ewoooc_asset_inventory.json` seeds hosts, services, data, AI, routes, supply chain, observability and recovery. Exit: same-run probe receipt for every asset; drift auto-creates work items. |
 | 7 | `GOV-P0-002` | Not started | Unified controlled-apply envelope | Introduce one `trace_id/run_id/work_item_id` across sensor, identity, SOT diff, decision, risk, dry-run, execution, verifier, rollback/retry and learning acknowledgement. Start with EventRouter + AutoHeal. |
-| 8 | `RAG-P0-001` | In progress (`production_canary_failed_gcp_unreachable`) | Internal RAG candidate canary | V10.811 production now runs the default-enabled daily 04:45 scheduler lane. Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` produced one fresh Coupang candidate receipt with two RAG-ready knowledge contracts and a successful Telegram acknowledgement, but GCP-A connect timed out and GCP-B read timed out, so the canary correctly stayed partial with zero price/`ai_insights` writes. V10.812 source adds a pre-DB fail-closed guard: missing GCP-A/GCP-B or wrong/empty vectors cannot reach pgvector. Exit still requires both hosts healthy, an executed read-only pgvector receipt, immutable model reference, controlled `RAG_ENABLED` shadow activation, query/hit telemetry and feedback readback. |
+| 8 | `RAG-P0-001` | In progress (`production_canary_failed_gcp_unreachable`) | Internal RAG candidate canary | V10.811 production now runs the default-enabled daily 04:45 scheduler lane. Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` produced one fresh Coupang candidate receipt with two RAG-ready knowledge contracts and a successful Telegram acknowledgement, but GCP-A connect timed out and GCP-B read timed out, so the canary correctly stayed partial with zero price/`ai_insights` writes. V10.812 retry run `47ca3ccd-f7b4-4ec9-97dd-8c14690f33a7` proved the pre-DB fail-closed guard in production (`database_call_performed=false`, Telegram acknowledged) and then auto-rolled back because an empty post-log grep was incorrectly treated as a verifier failure. V10.813 source fixes that evidence semantic and preserves the no-DB-call terminal in latest/top-level readback. Exit still requires both hosts healthy, an executed read-only pgvector receipt, immutable model reference, controlled `RAG_ENABLED` shadow activation, query/hit telemetry and feedback readback. |
 | 9 | `MCP-P0-001` | In progress (`federation_source_ready`) | MCP/RAG production runtime closure | V10.796 source adds a strict public aggregate receipt for canonical `ewoooc` and `momo-pro-system` identities without opening authenticated internal APIs or exposing endpoint/tool payload data. Exit still requires V10.796 production `/health`, two fresh AWOOOI durable receipts with fingerprint recompute, live MCP servers/router/RAG, approved caller/tool boundary and production query canary. Current source readiness must not be reported as runtime closure. |
 | 10 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. |
 | 11 | `REL-P0-001` | In progress (`runtime_verified_cd_degraded`) | Formal deploy and visible proof discipline | Production V10.810 runs exact Gitea object `911393190ded015e384e438c26b68faf50ec260c`; dev merge `e56c613399affb5cf0134eea44b7f78aa8134943` carries the same source. Host 110 still exposes only paused `vibework-dedicated-runner`, with no matching EwoooC runner, so this release has no formal CD execution receipt. The controlled fallback archive SHA-256 is `cd0607d593fd16c8281b5f01839d20d7eba493f49d363cb197c30966c25be5ee`; archive, checksum manifest and rollback (`02e73678125ee0c8908a42be8fff479287180fa2c06f6efe7dce70c2ce7339a4`) are retained at `/home/ollama/momo-deploy-backups/analytics-ui-20260716T080302Z-9113931`. Internal/external `/health` is healthy at V10.810; only `momo-app` was recreated, while scheduler ID `9af8e31fd0eb90f74ce5894af1ae1fbc511362ec90df142fde6442cf737d1181`, Telegram bot ID `1d4bd922f6ad1252177ba40f16ea172dcd5eb1de1b4c999c00f4a3141bd06ca5` and `momo-db` immutable ID `cd092451cb5fd555d0ffff70642e109f3b742882c418beeab631793d1e9dc55d` did not change. Runner recovery remains a release-governance gap and is not replaced by fallback evidence. |
@@ -97,7 +97,7 @@ These are reusable foundations, not proof that the full program is complete.
 | Completed | PromotionGate replay | No production write. |
 | Completed | Embedding-signature guard replay | Signature readiness only. |
 | Completed | Candidate knowledge replay | Internal RAG preview only; no DB/model call. |
-| Production partial | Internal RAG candidate canary | V10.811 executes the scheduled lane and captured a real no-write GCP outage receipt. V10.812 source blocks unavailable/empty embeddings before any DB call; GCP recovery, production redeploy, model pin and RAG shadow activation remain P0. |
+| Production partial | Internal RAG candidate canary | V10.811 executes the scheduled lane and captured a real no-write GCP outage receipt. V10.812 production retry proved unavailable/empty embeddings are blocked before any DB call and auto-rollback works; V10.813 source corrects receipt and empty-log verifier semantics. GCP recovery, production redeploy, model pin and RAG shadow activation remain P0. |
 | Completed | PixelRAG application portfolio | Commerce/RAG/UX/ops/marketing/governance inventory. |
 | Completed | Ollama-first VLM route readiness and replay worker | Evidence-bound artifact output; no direct price write. |
 | Completed | Platform probe worker | Shopee/Coupang barriers become structured fallback/backoff receipts. |

services/internal_rag_candidate_canary_service.py

@@ -124,6 +124,7 @@ def _latest_execution_receipt(root: Path) -> dict[str, Any]:
         payload = json.loads(path.read_text(encoding="utf-8"))
     except (OSError, json.JSONDecodeError):
         return {}
+    transaction_read_only = payload.get("transaction_read_only")
     return {
         "receipt_path": str(path),
         "generated_at": payload.get("generated_at"),
@@ -133,7 +134,21 @@ def _latest_execution_receipt(root: Path) -> dict[str, Any]:
         "manifest_id": payload.get("manifest_id"),
         "embedding_signature": payload.get("embedding_signature"),
         "probe_similarity": payload.get("probe_similarity"),
-        "transaction_read_only": payload.get("transaction_read_only") is True,
+        "database_call_performed": payload.get("database_call_performed") is True,
+        "transaction_read_only": (
+            transaction_read_only
+            if isinstance(transaction_read_only, bool)
+            else None
+        ),
+        "rollback_terminal": payload.get("rollback_terminal"),
+        "error": payload.get("error"),
+        "required_consistency_hosts": list(
+            payload.get("required_consistency_hosts") or []
+        ),
+        "reachable_consistency_hosts": list(
+            payload.get("reachable_consistency_hosts") or []
+        ),
+        "writes_database": payload.get("writes_database") is True,
         "writes_ai_insights": payload.get("writes_ai_insights") is True,
         "writes_price_tables": payload.get("writes_price_tables") is True,
     }
@@ -249,7 +264,27 @@ def _generate_embedding(text: str) -> list[float]:
 def _verify_embedding_consistency() -> dict[str, Any]:
     from services.rag_service import verify_embedding_consistency
 
-    return dict(verify_embedding_consistency())
+    result = dict(verify_embedding_consistency())
+    required_hosts = {"gcp_ollama", "ollama_secondary"}
+    reachable = set(result.get("reachable") or [])
+    upstream_ok = result.get("ok") is True
+    required_hosts_ready = required_hosts.issubset(reachable)
+    errors = list(result.get("errors") or [])
+    if not required_hosts_ready:
+        missing = sorted(required_hosts - reachable)
+        marker = f"required embedding hosts unavailable: {missing}"
+        if marker not in errors:
+            errors.append(marker)
+    result.update(
+        {
+            "ok": upstream_ok and required_hosts_ready,
+            "upstream_ok": upstream_ok,
+            "required_hosts": sorted(required_hosts),
+            "required_hosts_ready": required_hosts_ready,
+            "errors": errors,
+        }
+    )
+    return result
 
 
 def _run_pgvector_probe(
@@ -548,6 +583,22 @@ def run_internal_rag_candidate_canary(
         1 for item in executed_items if item.get("canary_passed") is True
     )
     canary_failed_count = len(executed_items) - canary_passed_count
+    latest_executed_item = executed_items[-1] if executed_items else {}
+    database_items = [
+        item
+        for item in executed_items
+        if item.get("database_call_performed") is True
+    ]
+    database_call_performed = bool(database_items)
+    database_transaction_read_only = (
+        all(item.get("transaction_read_only") is True for item in database_items)
+        if database_items
+        else None
+    )
+    rollback_terminal = str(
+        latest_executed_item.get("rollback_terminal")
+        or ("no_write_terminal" if not executed_items else "missing_execution_terminal")
+    )
     activation_blockers: list[str] = []
     if not is_rag_enabled():
         activation_blockers.append("rag_runtime_disabled")
@@ -567,7 +618,13 @@ def run_internal_rag_candidate_canary(
         next_action = "run_internal_rag_candidate_canary_execute"
     elif canary_failed_count:
         status = "canary_failed"
-        next_action = "repair_embedding_or_pgvector_canary_failure"
+        latest_error = str(latest_executed_item.get("error") or "")
+        if latest_error.startswith("embedding_host_preflight_failed"):
+            next_action = "restore_required_embedding_hosts_then_retry_canary"
+        elif latest_error.startswith("embedding_dimension_preflight_failed"):
+            next_action = "repair_embedding_dimension_then_retry_canary"
+        else:
+            next_action = "repair_embedding_or_pgvector_canary_failure"
     elif activation_blockers:
         status = "canary_passed_activation_blocked"
         next_action = "pin_embedding_model_then_enable_rag_controlled_canary"
@@ -622,13 +679,16 @@ def run_internal_rag_candidate_canary(
             "bounded_candidate_limit": item_limit,
             "network_call": bool(execute and ready_items),
             "model_call": bool(execute and ready_items),
-            "database_transaction_read_only": True,
+            "database_call_performed": database_call_performed,
+            "database_transaction_read_only": database_transaction_read_only,
             "database_write": False,
             "ai_insights_write": False,
             "price_table_write": False,
             "artifact_write": bool(execute and write_receipt),
-            "rollback_terminal": "transaction_rollback_after_read_only_pgvector_probe",
-            "independent_verifier": "pgvector_read_only_similarity_probe",
+            "rollback_terminal": rollback_terminal,
+            "independent_verifier": (
+                "embedding_host_preflight_and_pgvector_read_only_similarity_probe"
+            ),
         },
         "closure_receipt": {
             "sensor_source_receipt": bool(source_items),
@@ -645,11 +705,7 @@ def run_internal_rag_candidate_canary(
                 bool(executed_items)
                 and canary_passed_count == len(executed_items)
             ),
-            "rollback_or_no_write_terminal": (
-                "transaction_rollback_after_read_only_pgvector_probe"
-                if executed_items
-                else "no_write_terminal"
-            ),
+            "rollback_or_no_write_terminal": rollback_terminal,
             "telegram_acknowledgement": "pending_scheduler_dispatch"
             if execute
             else "not_applicable_dry_run",

tests/test_internal_rag_candidate_canary_service.py

@@ -114,6 +114,11 @@ def test_internal_rag_canary_executes_read_only_pgvector_probe(tmp_path, monkeyp
     assert item["writes_ai_insights"] is False
     assert item["writes_price_tables"] is False
     assert payload["latest_execution"]["canary_passed"] is True
+    assert payload["latest_execution"]["database_call_performed"] is True
+    assert payload["latest_execution"]["transaction_read_only"] is True
+    assert payload["latest_execution"]["rollback_terminal"] == (
+        "transaction_rollback_after_read_only_pgvector_probe"
+    )
     assert payload["run_identity"]["work_item_id"] == "RAG-P0-001"
     assert item["run_identity"] == payload["run_identity"]
 
@@ -122,6 +127,7 @@ def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypat
     from services import internal_rag_candidate_canary_service as service
 
     source_root = tmp_path / "candidate"
+    output_root = tmp_path / "canary"
     _write_candidate_receipt(source_root)
     monkeypatch.setattr(
         service, "_generate_embedding", lambda _text: [0.01] * service.RAG_EMBED_DIM
@@ -151,7 +157,9 @@ def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypat
 
     payload = service.run_internal_rag_candidate_canary(
         candidate_knowledge_receipt_root=source_root,
+        output_root=output_root,
         execute=True,
+        write_receipt=True,
     )
 
     assert payload["status"] == "canary_failed"
@@ -161,8 +169,57 @@ def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypat
     assert item["required_consistency_hosts"] == ["gcp_ollama", "ollama_secondary"]
     assert item["error"].startswith("embedding_host_preflight_failed")
     assert item["database_call_performed"] is False
+    assert payload["controlled_apply"]["database_call_performed"] is False
+    assert payload["controlled_apply"]["database_transaction_read_only"] is None
+    assert payload["controlled_apply"]["rollback_terminal"] == (
+        "no_database_call_due_embedding_host_preflight"
+    )
+    assert payload["closure_receipt"]["rollback_or_no_write_terminal"] == (
+        "no_database_call_due_embedding_host_preflight"
+    )
+    assert payload["next_machine_action"] == (
+        "restore_required_embedding_hosts_then_retry_canary"
+    )
+    assert payload["latest_execution"]["database_call_performed"] is False
+    assert payload["latest_execution"]["transaction_read_only"] is None
+    assert payload["latest_execution"]["rollback_terminal"] == (
+        "no_database_call_due_embedding_host_preflight"
+    )
+    assert payload["latest_execution"]["error"].startswith(
+        "embedding_host_preflight_failed"
+    )
     assert database_calls == []
 
+    readback = service.run_internal_rag_candidate_canary(
+        candidate_knowledge_receipt_root=source_root,
+        output_root=output_root,
+    )
+    assert readback["latest_execution"] == payload["latest_execution"]
+
+
+def test_internal_rag_canary_normalises_upstream_single_host_fail_safe(monkeypatch):
+    from services import internal_rag_candidate_canary_service as service
+    from services import rag_service
+
+    monkeypatch.setattr(
+        rag_service,
+        "verify_embedding_consistency",
+        lambda: {
+            "ok": True,
+            "reachable": [],
+            "max_diff": 0.0,
+            "errors": ["only 0 host reachable, cannot cross-verify"],
+        },
+    )
+
+    payload = service._verify_embedding_consistency()
+
+    assert payload["upstream_ok"] is True
+    assert payload["ok"] is False
+    assert payload["required_hosts_ready"] is False
+    assert payload["required_hosts"] == ["gcp_ollama", "ollama_secondary"]
+    assert payload["errors"][-1].startswith("required embedding hosts unavailable")
+
 
 def test_internal_rag_canary_blocks_empty_embedding_before_database(tmp_path, monkeypatch):
     from services import internal_rag_candidate_canary_service as service