diff --git a/config.py b/config.py index ceac9a5..8e1a388 100644 --- a/config.py +++ b/config.py @@ -414,7 +414,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.812" +SYSTEM_VERSION = "V10.813" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index aba038e..918c806 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,15 +1,15 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-07-17 (台北時間) -> **狀態**: 🟠 Partial。Production runtime 已是 V10.811；daily scheduled internal RAG candidate canary 與四 AI Agent 產品整合 truth readback 已上線，但不得標示完整整合。Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` 證明 candidate replay `1/1` 成功、2 個 knowledge contracts ready、Telegram acknowledgement 成功；GCP-A connect timeout、GCP-B read timeout 導致 canary failed，且 `MCP_ROUTER_ENABLED=false`、`RAG_ENABLED=false`、MCP/RAG telemetry 均為 0。四 Agent source wiring 為 `4/4`，runtime active `2/4`、healthy `0/4`、closure `3/9`；程式存在或 smoke 顯示 ok 都不能覆蓋這個 runtime truth。V10.812 source 另加入空 embedding 進 DB 前的 fail-closed 修正，尚待部署。 -> **適用版本**: V10.811 production runtime；V10.812 source candidate pending Gitea/deploy/runtime verification +> **狀態**: 🟠 Partial。Production runtime 已回滾至 V10.811；daily scheduled internal RAG candidate canary 與四 AI Agent 產品整合 truth readback 已上線，但不得標示完整整合。Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` 證明 candidate replay `1/1` 成功、2 個 knowledge contracts ready、Telegram acknowledgement 成功；GCP-A connect timeout、GCP-B read timeout 導致 canary failed，且 `MCP_ROUTER_ENABLED=false`、`RAG_ENABLED=false`、MCP/RAG telemetry 均為 0。V10.812 retry run `47ca3ccd-f7b4-4ec9-97dd-8c14690f33a7` 已在正式環境證明 host preflight 失敗時 `database_call_performed=false`、Telegram acknowledged，隨後因空 log grep verifier 語意錯誤自動回滾。四 Agent source wiring 為 `4/4`，runtime active `2/4`、healthy `0/4`、closure `3/9`；程式存在或 smoke 顯示 ok 都不能覆蓋這個 runtime truth。V10.813 source 進一步讓 latest/top-level receipt 如實保留 no-DB-call terminal，尚待部署。 +> **適用版本**: V10.811 production runtime；V10.813 source candidate pending Gitea/deploy/runtime verification --- ## 零之負五、AI Agent 產品整合 truth 與 internal RAG canary（V10.811 source） - `/api/ai-automation/agent-product-integration` 與 `scripts/ops/report_ai_agent_product_integration.py` 分開輸出四 Agent source/scheduler wiring、七日 `ai_calls` 實際呼叫與錯誤率、MCP/RAG telemetry、action plan/outcome、AutoHeal incident retry，以及九階段 closure。只有四 Agent 全部有健康 runtime、MCP/RAG 已啟用且有 telemetry、internal RAG canary 已通過、受控執行/驗證/重試/學習都有實證時才可回 `fully_integrated`。 -- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次，CLI 只保留給 Windows 99 controlled apply / verifier。每次最多一筆 candidate knowledge receipt，使用 Ollama-first BGE-M3 與 GCP-A/GCP-B consistency probe；V10.812 起任一主要 host 不可達或向量維度錯誤會在 DB 前 fail closed，只有完整 1024-dim embedding 才能進 PostgreSQL `SET TRANSACTION READ ONLY` pgvector probe，最後 rollback。 +- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次，CLI 只保留給 Windows 99 controlled apply / verifier。每次最多一筆 candidate knowledge receipt，使用 Ollama-first BGE-M3 與 GCP-A/GCP-B consistency probe；V10.812 起任一主要 host 不可達或向量維度錯誤會在 DB 前 fail closed，只有完整 1024-dim embedding 才能進 PostgreSQL `SET TRANSACTION READ ONLY` pgvector probe，最後 rollback。V10.813 起 upstream 單節點 fail-safe 不可覆蓋 canary 的雙主機 requirement，latest 與 top-level receipt 必須區分 no-DB-call、read-only transaction 與 rollback terminal。 - canary 不 INSERT/UPDATE `ai_insights`、`competitor_prices`、`external_offers` 或任何正式價格表；artifact 與 scheduler receipt 共用 `trace_id/run_id/work_item_id`，明確輸出 `transaction_read_only`、similarity、embedding signature、GCP-A/GCP-B reachability、Telegram acknowledgement 與 zero-write/rollback terminal。簽名漂移、任一主要 GCP embedding host 不可達、pgvector probe 失敗或 semantic threshold 未達都必須 fail closed。 - `bge-m3:latest` 仍是 immutable supply-chain blocker；即使 canary 通過，也只能回 `canary_passed_activation_blocked`，必須先鎖定可重現模型版本，再以 Windows 99 controlled apply 啟用 RAG shadow/canary。MCP runtime 仍受 localhost-only、read-only tool contract與 required secret presence preflight 約束，不可因 registry 已存在就宣稱上線。 - `AI Agent product integration truth` 與 `Internal RAG candidate canary` 已接入 `/api/ai-automation/smoke`。舊的 NemoTron/ElephantAlpha class/method smoke 保留為 source guard，但不再能單獨代表產品整合完成。 diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 0f1c978..1d2dc8f 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -23,7 +23,7 @@ | 5 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout, secret-safe `.dockerignore`, commit-bound source receipt and governance gate are active. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | | 6 | `GOV-P0-001` | In progress | Canonical full asset graph + runtime reconciliation | `governance/ewoooc_asset_inventory.json` seeds hosts, services, data, AI, routes, supply chain, observability and recovery. Exit: same-run probe receipt for every asset; drift auto-creates work items. | | 7 | `GOV-P0-002` | Not started | Unified controlled-apply envelope | Introduce one `trace_id/run_id/work_item_id` across sensor, identity, SOT diff, decision, risk, dry-run, execution, verifier, rollback/retry and learning acknowledgement. Start with EventRouter + AutoHeal. | -| 8 | `RAG-P0-001` | In progress (`production_canary_failed_gcp_unreachable`) | Internal RAG candidate canary | V10.811 production now runs the default-enabled daily 04:45 scheduler lane. Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` produced one fresh Coupang candidate receipt with two RAG-ready knowledge contracts and a successful Telegram acknowledgement, but GCP-A connect timed out and GCP-B read timed out, so the canary correctly stayed partial with zero price/`ai_insights` writes. V10.812 source adds a pre-DB fail-closed guard: missing GCP-A/GCP-B or wrong/empty vectors cannot reach pgvector. Exit still requires both hosts healthy, an executed read-only pgvector receipt, immutable model reference, controlled `RAG_ENABLED` shadow activation, query/hit telemetry and feedback readback. | +| 8 | `RAG-P0-001` | In progress (`production_canary_failed_gcp_unreachable`) | Internal RAG candidate canary | V10.811 production now runs the default-enabled daily 04:45 scheduler lane. Windows 99 run `4337ac55-f832-4d96-9ab7-99c2d779a49d` produced one fresh Coupang candidate receipt with two RAG-ready knowledge contracts and a successful Telegram acknowledgement, but GCP-A connect timed out and GCP-B read timed out, so the canary correctly stayed partial with zero price/`ai_insights` writes. V10.812 retry run `47ca3ccd-f7b4-4ec9-97dd-8c14690f33a7` proved the pre-DB fail-closed guard in production (`database_call_performed=false`, Telegram acknowledged) and then auto-rolled back because an empty post-log grep was incorrectly treated as a verifier failure. V10.813 source fixes that evidence semantic and preserves the no-DB-call terminal in latest/top-level readback. Exit still requires both hosts healthy, an executed read-only pgvector receipt, immutable model reference, controlled `RAG_ENABLED` shadow activation, query/hit telemetry and feedback readback. | | 9 | `MCP-P0-001` | In progress (`federation_source_ready`) | MCP/RAG production runtime closure | V10.796 source adds a strict public aggregate receipt for canonical `ewoooc` and `momo-pro-system` identities without opening authenticated internal APIs or exposing endpoint/tool payload data. Exit still requires V10.796 production `/health`, two fresh AWOOOI durable receipts with fingerprint recompute, live MCP servers/router/RAG, approved caller/tool boundary and production query canary. Current source readiness must not be reported as runtime closure. | | 10 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. | | 11 | `REL-P0-001` | In progress (`runtime_verified_cd_degraded`) | Formal deploy and visible proof discipline | Production V10.810 runs exact Gitea object `911393190ded015e384e438c26b68faf50ec260c`; dev merge `e56c613399affb5cf0134eea44b7f78aa8134943` carries the same source. Host 110 still exposes only paused `vibework-dedicated-runner`, with no matching EwoooC runner, so this release has no formal CD execution receipt. The controlled fallback archive SHA-256 is `cd0607d593fd16c8281b5f01839d20d7eba493f49d363cb197c30966c25be5ee`; archive, checksum manifest and rollback (`02e73678125ee0c8908a42be8fff479287180fa2c06f6efe7dce70c2ce7339a4`) are retained at `/home/ollama/momo-deploy-backups/analytics-ui-20260716T080302Z-9113931`. Internal/external `/health` is healthy at V10.810; only `momo-app` was recreated, while scheduler ID `9af8e31fd0eb90f74ce5894af1ae1fbc511362ec90df142fde6442cf737d1181`, Telegram bot ID `1d4bd922f6ad1252177ba40f16ea172dcd5eb1de1b4c999c00f4a3141bd06ca5` and `momo-db` immutable ID `cd092451cb5fd555d0ffff70642e109f3b742882c418beeab631793d1e9dc55d` did not change. Runner recovery remains a release-governance gap and is not replaced by fallback evidence. | @@ -97,7 +97,7 @@ These are reusable foundations, not proof that the full program is complete. | Completed | PromotionGate replay | No production write. | | Completed | Embedding-signature guard replay | Signature readiness only. | | Completed | Candidate knowledge replay | Internal RAG preview only; no DB/model call. | -| Production partial | Internal RAG candidate canary | V10.811 executes the scheduled lane and captured a real no-write GCP outage receipt. V10.812 source blocks unavailable/empty embeddings before any DB call; GCP recovery, production redeploy, model pin and RAG shadow activation remain P0. | +| Production partial | Internal RAG candidate canary | V10.811 executes the scheduled lane and captured a real no-write GCP outage receipt. V10.812 production retry proved unavailable/empty embeddings are blocked before any DB call and auto-rollback works; V10.813 source corrects receipt and empty-log verifier semantics. GCP recovery, production redeploy, model pin and RAG shadow activation remain P0. | | Completed | PixelRAG application portfolio | Commerce/RAG/UX/ops/marketing/governance inventory. | | Completed | Ollama-first VLM route readiness and replay worker | Evidence-bound artifact output; no direct price write. | | Completed | Platform probe worker | Shopee/Coupang barriers become structured fallback/backoff receipts. | diff --git a/services/internal_rag_candidate_canary_service.py b/services/internal_rag_candidate_canary_service.py index 5c2b60a..763bf8c 100644 --- a/services/internal_rag_candidate_canary_service.py +++ b/services/internal_rag_candidate_canary_service.py @@ -124,6 +124,7 @@ def _latest_execution_receipt(root: Path) -> dict[str, Any]: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return {} + transaction_read_only = payload.get("transaction_read_only") return { "receipt_path": str(path), "generated_at": payload.get("generated_at"), @@ -133,7 +134,21 @@ def _latest_execution_receipt(root: Path) -> dict[str, Any]: "manifest_id": payload.get("manifest_id"), "embedding_signature": payload.get("embedding_signature"), "probe_similarity": payload.get("probe_similarity"), - "transaction_read_only": payload.get("transaction_read_only") is True, + "database_call_performed": payload.get("database_call_performed") is True, + "transaction_read_only": ( + transaction_read_only + if isinstance(transaction_read_only, bool) + else None + ), + "rollback_terminal": payload.get("rollback_terminal"), + "error": payload.get("error"), + "required_consistency_hosts": list( + payload.get("required_consistency_hosts") or [] + ), + "reachable_consistency_hosts": list( + payload.get("reachable_consistency_hosts") or [] + ), + "writes_database": payload.get("writes_database") is True, "writes_ai_insights": payload.get("writes_ai_insights") is True, "writes_price_tables": payload.get("writes_price_tables") is True, } @@ -249,7 +264,27 @@ def _generate_embedding(text: str) -> list[float]: def _verify_embedding_consistency() -> dict[str, Any]: from services.rag_service import verify_embedding_consistency - return dict(verify_embedding_consistency()) + result = dict(verify_embedding_consistency()) + required_hosts = {"gcp_ollama", "ollama_secondary"} + reachable = set(result.get("reachable") or []) + upstream_ok = result.get("ok") is True + required_hosts_ready = required_hosts.issubset(reachable) + errors = list(result.get("errors") or []) + if not required_hosts_ready: + missing = sorted(required_hosts - reachable) + marker = f"required embedding hosts unavailable: {missing}" + if marker not in errors: + errors.append(marker) + result.update( + { + "ok": upstream_ok and required_hosts_ready, + "upstream_ok": upstream_ok, + "required_hosts": sorted(required_hosts), + "required_hosts_ready": required_hosts_ready, + "errors": errors, + } + ) + return result def _run_pgvector_probe( @@ -548,6 +583,22 @@ def run_internal_rag_candidate_canary( 1 for item in executed_items if item.get("canary_passed") is True ) canary_failed_count = len(executed_items) - canary_passed_count + latest_executed_item = executed_items[-1] if executed_items else {} + database_items = [ + item + for item in executed_items + if item.get("database_call_performed") is True + ] + database_call_performed = bool(database_items) + database_transaction_read_only = ( + all(item.get("transaction_read_only") is True for item in database_items) + if database_items + else None + ) + rollback_terminal = str( + latest_executed_item.get("rollback_terminal") + or ("no_write_terminal" if not executed_items else "missing_execution_terminal") + ) activation_blockers: list[str] = [] if not is_rag_enabled(): activation_blockers.append("rag_runtime_disabled") @@ -567,7 +618,13 @@ def run_internal_rag_candidate_canary( next_action = "run_internal_rag_candidate_canary_execute" elif canary_failed_count: status = "canary_failed" - next_action = "repair_embedding_or_pgvector_canary_failure" + latest_error = str(latest_executed_item.get("error") or "") + if latest_error.startswith("embedding_host_preflight_failed"): + next_action = "restore_required_embedding_hosts_then_retry_canary" + elif latest_error.startswith("embedding_dimension_preflight_failed"): + next_action = "repair_embedding_dimension_then_retry_canary" + else: + next_action = "repair_embedding_or_pgvector_canary_failure" elif activation_blockers: status = "canary_passed_activation_blocked" next_action = "pin_embedding_model_then_enable_rag_controlled_canary" @@ -622,13 +679,16 @@ def run_internal_rag_candidate_canary( "bounded_candidate_limit": item_limit, "network_call": bool(execute and ready_items), "model_call": bool(execute and ready_items), - "database_transaction_read_only": True, + "database_call_performed": database_call_performed, + "database_transaction_read_only": database_transaction_read_only, "database_write": False, "ai_insights_write": False, "price_table_write": False, "artifact_write": bool(execute and write_receipt), - "rollback_terminal": "transaction_rollback_after_read_only_pgvector_probe", - "independent_verifier": "pgvector_read_only_similarity_probe", + "rollback_terminal": rollback_terminal, + "independent_verifier": ( + "embedding_host_preflight_and_pgvector_read_only_similarity_probe" + ), }, "closure_receipt": { "sensor_source_receipt": bool(source_items), @@ -645,11 +705,7 @@ def run_internal_rag_candidate_canary( bool(executed_items) and canary_passed_count == len(executed_items) ), - "rollback_or_no_write_terminal": ( - "transaction_rollback_after_read_only_pgvector_probe" - if executed_items - else "no_write_terminal" - ), + "rollback_or_no_write_terminal": rollback_terminal, "telegram_acknowledgement": "pending_scheduler_dispatch" if execute else "not_applicable_dry_run", diff --git a/tests/test_internal_rag_candidate_canary_service.py b/tests/test_internal_rag_candidate_canary_service.py index 7761edc..a3b1417 100644 --- a/tests/test_internal_rag_candidate_canary_service.py +++ b/tests/test_internal_rag_candidate_canary_service.py @@ -114,6 +114,11 @@ def test_internal_rag_canary_executes_read_only_pgvector_probe(tmp_path, monkeyp assert item["writes_ai_insights"] is False assert item["writes_price_tables"] is False assert payload["latest_execution"]["canary_passed"] is True + assert payload["latest_execution"]["database_call_performed"] is True + assert payload["latest_execution"]["transaction_read_only"] is True + assert payload["latest_execution"]["rollback_terminal"] == ( + "transaction_rollback_after_read_only_pgvector_probe" + ) assert payload["run_identity"]["work_item_id"] == "RAG-P0-001" assert item["run_identity"] == payload["run_identity"] @@ -122,6 +127,7 @@ def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypat from services import internal_rag_candidate_canary_service as service source_root = tmp_path / "candidate" + output_root = tmp_path / "canary" _write_candidate_receipt(source_root) monkeypatch.setattr( service, "_generate_embedding", lambda _text: [0.01] * service.RAG_EMBED_DIM @@ -151,7 +157,9 @@ def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypat payload = service.run_internal_rag_candidate_canary( candidate_knowledge_receipt_root=source_root, + output_root=output_root, execute=True, + write_receipt=True, ) assert payload["status"] == "canary_failed" @@ -161,8 +169,57 @@ def test_internal_rag_canary_requires_both_primary_gcp_hosts(tmp_path, monkeypat assert item["required_consistency_hosts"] == ["gcp_ollama", "ollama_secondary"] assert item["error"].startswith("embedding_host_preflight_failed") assert item["database_call_performed"] is False + assert payload["controlled_apply"]["database_call_performed"] is False + assert payload["controlled_apply"]["database_transaction_read_only"] is None + assert payload["controlled_apply"]["rollback_terminal"] == ( + "no_database_call_due_embedding_host_preflight" + ) + assert payload["closure_receipt"]["rollback_or_no_write_terminal"] == ( + "no_database_call_due_embedding_host_preflight" + ) + assert payload["next_machine_action"] == ( + "restore_required_embedding_hosts_then_retry_canary" + ) + assert payload["latest_execution"]["database_call_performed"] is False + assert payload["latest_execution"]["transaction_read_only"] is None + assert payload["latest_execution"]["rollback_terminal"] == ( + "no_database_call_due_embedding_host_preflight" + ) + assert payload["latest_execution"]["error"].startswith( + "embedding_host_preflight_failed" + ) assert database_calls == [] + readback = service.run_internal_rag_candidate_canary( + candidate_knowledge_receipt_root=source_root, + output_root=output_root, + ) + assert readback["latest_execution"] == payload["latest_execution"] + + +def test_internal_rag_canary_normalises_upstream_single_host_fail_safe(monkeypatch): + from services import internal_rag_candidate_canary_service as service + from services import rag_service + + monkeypatch.setattr( + rag_service, + "verify_embedding_consistency", + lambda: { + "ok": True, + "reachable": [], + "max_diff": 0.0, + "errors": ["only 0 host reachable, cannot cross-verify"], + }, + ) + + payload = service._verify_embedding_consistency() + + assert payload["upstream_ok"] is True + assert payload["ok"] is False + assert payload["required_hosts_ready"] is False + assert payload["required_hosts"] == ["gcp_ollama", "ollama_secondary"] + assert payload["errors"][-1].startswith("required embedding hosts unavailable") + def test_internal_rag_canary_blocks_empty_embedding_before_database(tmp_path, monkeypatch): from services import internal_rag_candidate_canary_service as service