diff --git a/.env.example b/.env.example index 086b9df..63acda1 100644 --- a/.env.example +++ b/.env.example @@ -460,7 +460,8 @@ OLLAMA_HOST_HEALTH_MODEL_PROBE_INCLUDE_111=false OLLAMA_HOST_HEALTH_EMBED_MODEL=bge-m3:latest OLLAMA_HOST_HEALTH_EMBED_TIMEOUT=30 OLLAMA_HOST_HEALTH_EMBED_KEEP_ALIVE=1m -# 111 是 Mac final fallback，不承接 7B+ / vision / long-context / 長輸出任務；落到 111 時自動降級與縮短常駐。 +# 111 的一般 OllamaService fallback 不承接 7B+ / vision / long-context / 長輸出任務。 +# NemoTron 是唯一受控特例：只允許下方 exact-digest qwen3:8b，並強制總 deadline、num_ctx 4096、num_predict 512。 OLLAMA_111_MODEL_FALLBACK=llama3.2:latest OLLAMA_111_MODEL_DOWNGRADE_PATTERNS=qwen3:*,deepseek-r1:*,hermes3:*,llama3.1:*,qwen2.5:*,qwen2.5-coder:*,gemma3:*,minicpm-v:*,llava:*,*:7b*,*:8b*,*:14b*,*:32b*,*:70b* OLLAMA_111_KEEP_ALIVE=5m @@ -480,6 +481,11 @@ NEMOTRON_OLLAMA_FIRST=true NEMOTRON_OLLAMA_MODEL=qwen3:14b NEMOTRON_OLLAMA_TIMEOUT=180 NEMOTRON_OLLAMA_EXPECTED_DIGEST=bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8 +NEMOTRON_OLLAMA_FALLBACK_MODEL=qwen3:8b +NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41 +NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=60 +NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=45 +NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=180 NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=300 NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=10 NEMOTRON_DECISION_CANARY_MAX_AGE_HOURS=26 diff --git a/config.py b/config.py index f75d0b6..73bc03e 100644 --- a/config.py +++ b/config.py @@ -414,7 +414,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.815" +SYSTEM_VERSION = "V10.816" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docker-compose.yml b/docker-compose.yml index 77cffbf..038991f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,6 +107,12 @@ services: - RAG_EMBED_EXPECTED_DIGEST=${RAG_EMBED_EXPECTED_DIGEST:-7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab} - EMBED_CONSISTENCY_TIMEOUT_SEC=${EMBED_CONSISTENCY_TIMEOUT_SEC:-150} - NEMOTRON_OLLAMA_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_EXPECTED_DIGEST:-bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8} + - NEMOTRON_OLLAMA_FALLBACK_MODEL=${NEMOTRON_OLLAMA_FALLBACK_MODEL:-qwen3:8b} + - NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST:-500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41} + - NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC:-60} + - NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_111_ATTEMPT_TIMEOUT_SEC:-45} + - NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=${NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC:-180} + - NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=${NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC:-10} - NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=${NEMOTRON_DECISION_CANARY_TIMEOUT_SEC:-300} # Gemini 只能作緊急備援；即使 .env 有 API key，預設也不得產生付費出站。 - GEMINI_API_HARD_DISABLED=${GEMINI_API_HARD_DISABLED:-true} @@ -250,6 +256,12 @@ services: - RAG_EMBED_EXPECTED_DIGEST=${RAG_EMBED_EXPECTED_DIGEST:-7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab} - EMBED_CONSISTENCY_TIMEOUT_SEC=${EMBED_CONSISTENCY_TIMEOUT_SEC:-150} - NEMOTRON_OLLAMA_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_EXPECTED_DIGEST:-bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8} + - NEMOTRON_OLLAMA_FALLBACK_MODEL=${NEMOTRON_OLLAMA_FALLBACK_MODEL:-qwen3:8b} + - NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST:-500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41} + - NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC:-60} + - NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_111_ATTEMPT_TIMEOUT_SEC:-45} + - NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=${NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC:-180} + - NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=${NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC:-10} - NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=${NEMOTRON_DECISION_CANARY_TIMEOUT_SEC:-300} # Gemini 只能作緊急備援；即使 .env 有 API key，預設也不得產生付費出站。 - GEMINI_API_HARD_DISABLED=${GEMINI_API_HARD_DISABLED:-true} @@ -322,6 +334,12 @@ services: - RAG_EMBED_EXPECTED_DIGEST=${RAG_EMBED_EXPECTED_DIGEST:-7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab} - EMBED_CONSISTENCY_TIMEOUT_SEC=${EMBED_CONSISTENCY_TIMEOUT_SEC:-150} - NEMOTRON_OLLAMA_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_EXPECTED_DIGEST:-bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8} + - NEMOTRON_OLLAMA_FALLBACK_MODEL=${NEMOTRON_OLLAMA_FALLBACK_MODEL:-qwen3:8b} + - NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST=${NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST:-500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41} + - NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC:-60} + - NEMOTRON_111_ATTEMPT_TIMEOUT_SEC=${NEMOTRON_111_ATTEMPT_TIMEOUT_SEC:-45} + - NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC=${NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC:-180} + - NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC=${NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC:-10} - NEMOTRON_DECISION_CANARY_TIMEOUT_SEC=${NEMOTRON_DECISION_CANARY_TIMEOUT_SEC:-300} # Gemini 只能作緊急備援；即使 .env 有 API key，預設也不得產生付費出站。 - GEMINI_API_HARD_DISABLED=${GEMINI_API_HARD_DISABLED:-true} diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 335e45e..2128be3 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,19 +1,21 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth -> **最後更新**: 2026-07-17 (台北時間) -> **狀態**: 🟠 Partial。Production runtime 仍是 V10.813；V10.814 source 已修正 BGE-M3 冷啟動假陰性並加入 NemoTron decision-only canary，但部署與 production canary 尚未在本段 source truth 中宣稱完成。Windows 99 audit run `534c6208-2088-428c-ba6f-f7a63a67f422` 證明舊 project `astral-gateway-484913-d7` 已為 `DELETE_REQUESTED`、GCP-A `22/11434` 不可達；GCP-B Ollama `0.22.1` 可達，`bge-m3:latest` digest `7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab` 的 1024 維 no-write probe 成功但冷啟動需 `116.475s`，`qwen3:14b` digest 為 `bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8`。四 Agent runtime、MCP/RAG flags 與 telemetry 仍沿用 V10.813 production readback；程式、測試、tags 或 digest 可見都不能覆蓋尚未完成的 production closure。 -> **適用版本**: V10.813 production runtime；V10.814 source candidate；AI Agent/MCP/RAG full product integration remains partial +> **最後更新**: 2026-07-22 (台北時間) +> **狀態**: 🟠 Partial。Production runtime 是 V10.815、exact Gitea object `283c8c80c631f5d97315708885413b62ee5a34ea`；Windows 99 controlled run `736e6de1-fbc3-4c9b-bcfe-9b6ef488b932` 已完成 `36/36` source hash、`12/12` runtime env、三應用健康與 `momo-db` identity unchanged 的獨立讀回。PixelRAG receipt -> BGE-M3 -> pgvector no-write canary 已通過，但 `RAG_ENABLED=false` 與 embedding redundancy degraded 仍阻擋 activation。GCP-A 仍不可達；GCP-B exact-digest `qwen3:14b` chat 逾時，較小的 `qwen2.5:7b-instruct` 亦逾時，證明問題包含共享 CPU/queue，而非只靠換小模型即可解決。111 exact-digest `qwen3:8b` decision-only probe 於 `19.053s` 通過。V10.816 source candidate 已把這份實證接成 model-aware、bounded、exact-digest 的 GCP-A -> GCP-B -> 111 fallback；尚未完成本版 production deploy/readback 前，不得宣稱正式 runtime 已採用。 +> **適用版本**: V10.815 production runtime；V10.816 source candidate；AI Agent/MCP/RAG full product integration remains partial --- -## 零之負五、AI Agent 產品整合 truth 與 internal RAG/NemoTron canary（V10.814 source） +## 零之負五、AI Agent 產品整合 truth 與 internal RAG/NemoTron canary（V10.815 production / V10.816 source） - `/api/ai-automation/agent-product-integration` 與 `scripts/ops/report_ai_agent_product_integration.py` 分開輸出四 Agent source/scheduler wiring、七日 `ai_calls` 實際呼叫與錯誤率、MCP/RAG telemetry、action plan/outcome、AutoHeal incident retry，以及九階段 closure。只有四 Agent 全部有健康 runtime、MCP/RAG 已啟用且有 telemetry、internal RAG canary 已通過、受控執行/驗證/重試/學習都有實證時才可回 `fully_integrated`。 -- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次。V10.814 起 consistency probe 會先由 Ollama `/api/tags` 驗證 manifest digest，再以 canary 專用 `150s` timeout 容納 GCP-B 的實測冷啟動；一般 embedding 呼叫仍保留既有短 timeout cap，不把所有 request 放大成 150 秒。至少一台核准 GCP host 必須同時通過 digest 與 1024 維 embedding，111 不可形成 quorum；單一 GCP 可進 degraded read-only pgvector shadow probe，但 `rag_embedding_redundancy_degraded` 仍阻擋正式 activation。 +- `/api/ai-automation/internal-rag-candidate-canary` 只有 GET，永遠是 no-model/no-DB-write readback；production execute 由 `momo-scheduler` 每日 04:45 自動跑一次。V10.815 production 已使用先由 Ollama `/api/tags` 驗證 manifest digest、再以 canary 專用 `150s` timeout 容納 GCP-B 冷啟動的契約；一般 embedding 呼叫仍保留既有短 timeout cap，不把所有 request 放大成 150 秒。至少一台核准 GCP host 必須同時通過 digest 與 1024 維 embedding，111 不可形成 quorum；單一 GCP 可進 degraded read-only pgvector shadow probe，但 `rag_embedding_redundancy_degraded` 仍阻擋正式 activation。 - RAG canary 不 INSERT/UPDATE `ai_insights`、`competitor_prices`、`external_offers` 或任何正式價格表；artifact 與 scheduler receipt 共用 `trace_id/run_id/work_item_id`，明確輸出 `transaction_read_only`、similarity、embedding signature、expected/observed digest、GCP reachability、Telegram acknowledgement 與 zero-write/rollback terminal。digest drift、零核准 GCP host、向量維度錯誤、pgvector probe 失敗或 semantic threshold 未達都必須 fail closed。 - `bge-m3:latest` 只可在 expected digest gate 下使用：source contract 固定 digest，runtime 每次 canary 由 `/api/tags` 比對，漂移時在 embedding/DB 前阻擋。Ollama 官方 `/api/tags` 明確提供模型 `digest`，因此 immutable contract 使用「固定 expected digest + runtime verifier」，不是只信任 floating tag。MCP runtime 仍受 localhost-only、read-only tool contract 與 required secret presence preflight 約束，不可因 registry 已存在就宣稱上線。 -- V10.814 新增 `/api/ai-automation/nemotron-decision-canary`、`scripts/ops/run_nemotron_decision_canary.py` 與每日 05:00 scheduler lane。execute 只用 production 共用 qwen3 tool-calling payload產生 synthetic decision，驗證 expected model digest、allowed tool、synthetic SKU 與 post-call digest，固定 `tool_execution_count=0`、`database_call_performed=false`、`writes_price_tables=false`、`writes_ai_insights=false`、`telegram_sent=false`；scheduler 只在 model canary 後發生命週期 acknowledgement 並原子回寫 receipt。 -- `AI Agent product integration truth` 會讀取 fresh NemoTron decision-only receipt；這份 shadow canary 能證明模型決策路徑真的執行，但不能冒充正式商品 action、agent outcome 或完整 Controlled Apply。舊的 NemoTron/ElephantAlpha class/method smoke 保留為 source guard，也不能單獨代表產品整合完成。 +- V10.815 production 提供 `/api/ai-automation/nemotron-decision-canary`、`scripts/ops/run_nemotron_decision_canary.py` 與每日 05:00 scheduler lane。V10.816 source 將 production dispatcher 與 canary 共用 `services/nemotron_runtime_candidate_service.py`：GCP-A/GCP-B 必須使用 exact-digest `qwen3:14b`，每台 chat 最多 60 秒；最終 111 fallback 必須使用 exact-digest `qwen3:8b`，最多 45 秒並固定 `num_ctx=4096`、`num_predict=512`，所有 payload 固定 `think=false`。Production Ollama + NIM 共用單一 180 秒 monotonic deadline，canary 的 `timeout_sec` 也是整次執行上限，不再對每個候選重新計時。每次呼叫前後都驗證 digest；畸形 `/api/tags` payload/model/details schema 必須 fail closed 並轉下一候選，不能中斷 fallback。實際 model/provider/host/fallback/attempts 必須進 logger、通知 footprint 與 receipt。 +- 模型輸出必須在選定候選前通過 deterministic tool contract：工具在 allowlist、必要欄位與型別正確、SKU 必須來自本次輸入、每個 SKU 恰好一個 call 且全數覆蓋。Production 去重使用 `services/nemotron_dispatch_reservation_service.py` 在三容器共用 `/app/data` 上執行 `flock`、file fsync、atomic replace 與 parent-directory fsync 的 process/container-shared ownership-token store；state 只保存 SHA-256 SKU key、owner token、phase 與 expiry，不保存商品名稱或原始資料。In-flight lease 必須涵蓋最大模型 deadline；每個 handler 前須由原 owner 原子寫入 `side_effect_started` 並切換為四小時 crash quarantine，寫入失敗就不得執行副作用。舊 token 不可 release/commit 新 owner，state/lock 無法讀寫、schema 損壞或平台缺少 process-shared lock 時 fail closed。本地非 production 測試才使用記憶體 backend。告警工具必須有 EventRouter delivered/durable-queue acknowledgement，推薦工具必須有 DB write 或 durable notification，KM 工具必須有持久化 insight，才可提交四小時 TTL；若 durable side effect 已完成但 dedupe commit 未驗證，結果仍按已派發計數、輸出 `dedupe_commit_unverified`，並保留 quarantine，絕不可 release 後重送。模型契約錯誤、reservation ownership lost、boundary 未建立或 handler 未回 durable outcome 時才 release 自己持有的 lease。程序若在 boundary 後、side effect 前崩潰，最多抑制該 SKU 四小時；這是避免重複通知/寫入的 at-most-once 取捨，必須由 receipt/告警揭露而非靜默重試。 +- Decision-only canary 只產生 synthetic decision，並共用 production `_validate_tool_call_contract()` 驗證完整 call set，不可只看第一筆；多餘、重複、未知、缺欄位或未綁定 SKU 的 call 都必須 fail closed。Canary 同時驗證 post-call digest，固定 `tool_execution_count=0`、`database_call_performed=false`、`writes_price_tables=false`、`writes_ai_insights=false`、`telegram_sent=false`。使用 111 成功時狀態必須是 `canary_passed_degraded_fallback`，不能偽裝成 GCP healthy；scheduler 只在 model canary 後發生命週期 acknowledgement 並原子回寫 receipt。111 的 `num_ctx=4096`、`num_predict=512` 是不可被 env 弱化的固定 NemoTron 契約；候選失敗鏈只輸出 label/tier/model/status/digest-match/error-class 的 privacy-safe footprint。 +- `AI Agent product integration truth` 會讀取 fresh NemoTron decision-only receipt；這份 shadow canary能證明模型決策路徑真的執行，但不能冒充正式商品 action、agent outcome 或完整 Controlled Apply。V10.816 的 source/test 綠燈仍需 production deploy、fresh degraded-fallback receipt 與獨立 readback 才能提升 runtime 狀態。 - V10.812 起四 Agent activity 依角色證據判定：Hermes/OpenClaw 仍以實際 `ai_calls` 為主，NemoTron 可加入其派發/insight durable evidence，ElephantAlpha 可由其 `ai_insights`/action plan 證明 active；沒有成功 call 或 verified executed action 時只能是 `runtime_active_unverified`，不得把 deterministic artifact 數量包裝成 healthy integration。 --- @@ -70,6 +72,7 @@ ## 零、LLM 路由紅線（2026-05-12） - 所有 AI Agent、LLM 推理與 embedding 預設必須走 Ollama 三主機級聯：GCP-A `34.87.90.216:11434` → GCP-B `34.21.145.224:11434` → 111 `192.168.0.111:11434`。 +- NemoTron decision runtime 是 model-aware cascade，不得把同一 floating model tag盲送三台：GCP-A/GCP-B 鎖 `qwen3:14b` exact digest，111 最終備援鎖 `qwen3:8b` exact digest與較小輸出上限；GCP capacity degraded 時可維持自架決策可用性，但必須在 receipt/UI 明示 degraded fallback，並持續保留恢復 GCP capacity 的 P0。 - `services/ollama_service.resolve_ollama_host()` 是主機解析契約；`OLLAMA_HOST`、`HERMES_URL`、`EMBEDDING_HOST`、`OLLAMA_API_BASE` 只接受 GCP-A / GCP-B / 111 或 110 的核准轉發端口。 - 188 直連 GCP-A / GCP-B timeout 時，resolver 可先使用同順位 110 proxy rescue：GCP-A direct → `192.168.0.110:11435` → GCP-B direct → `192.168.0.110:11436` → 111。proxy rescue 只是同一順位的可用入口，不代表 GCP direct host 已恢復。 - `OLLAMA_RESOLVE_HOST_HEALTH_SKIP_ENABLED=true` 時，resolver 會讀最近 `host_health_probes`；若 direct GCP-A/GCP-B 在視窗內已被判定不健康，會直接略過該 direct endpoint，先試同順位 proxy rescue，避免每 120 秒 cache refresh 都等待 direct timeout。此 skip 只套用 direct GCP，不套用 110 proxy。 diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index ce16fa8..a7b4aa7 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -1,6 +1,6 @@ # AI Automation Mainline Work Items -> Updated: 2026-07-17 10:28 Asia/Taipei +> Updated: 2026-07-22 Asia/Taipei > Governance: `global_product_governance_v2` + ADR-038 > Current P0: `GROWTH-P0-001 comparison coverage truth + autonomous refresh` @@ -17,16 +17,16 @@ | Order | ID | Status | Work item | Exit evidence / next machine action | |---:|---|---|---|---| | 1 | `SEC-P0-001` | Completed | Deny-by-default route access control | `governance/evidence/SEC-P0-001-20260711T122758Z.json` plus the current runtime receipt prove anonymous matrix 8/8 denied, public `/metrics` 404, exact internal target up, EwoooC product markers present, Prometheus identity preserved and `momo-db` unchanged. | -| 2 | `GROWTH-P0-001` | In progress (`runtime_partial`) | Comparison coverage truth + autonomous refresh | V10.813 is live and retains the growth runtime introduced at exact Gitea object `2647632660673e9f1533e61922c96e4fb4adcb40`. Controlled run `34999f5314054f09909663b46fc42ee2` scanned 20, verified/wrote/read back one exact offer and raised the fixed cohort to `25 ready + 1 candidate validation + 24 unmatched`, `50%` count and `NT$211,667 / NT$354,062 = 59.782%` revenue. Follow-up run `e1508ec3a49241d1965bc99d428b0325` found 19 candidates but safely wrote zero because none passed strict identity/variant/unit verification; its independent terminal is `degraded_no_safe_candidate`. Yahoo remains durably `active + enabled + write_enabled`, and the durable readback proves `formal_source_activated=true` even when the latest run is verified no-write. Formal runtime remains `2/15`. Next: retry the unresolved revenue-weighted batch only with fresh source evidence, then add the next approved structured marketplace adapter without relaxing promotion gates. | +| 2 | `GROWTH-P0-001` | In progress (`runtime_partial`) | Comparison coverage truth + autonomous refresh | V10.815 is live and retains the growth runtime introduced at exact Gitea object `2647632660673e9f1533e61922c96e4fb4adcb40`. Controlled run `34999f5314054f09909663b46fc42ee2` scanned 20, verified/wrote/read back one exact offer and raised the fixed cohort to `25 ready + 1 candidate validation + 24 unmatched`, `50%` count and `NT$211,667 / NT$354,062 = 59.782%` revenue. Follow-up run `e1508ec3a49241d1965bc99d428b0325` found 19 candidates but safely wrote zero because none passed strict identity/variant/unit verification; its independent terminal is `degraded_no_safe_candidate`. Yahoo remains durably `active + enabled + write_enabled`, and the durable readback proves `formal_source_activated=true` even when the latest run is verified no-write. Formal runtime remains `2/15`. Next: retry the unresolved revenue-weighted batch only with fresh source evidence, then add the next approved structured marketplace adapter without relaxing promotion gates. | | 3 | `SEC-P0-002` | In progress (`canary_ready`) | Database identity + least-privilege RBAC | V10.789 is live and `governance/auth_identity_runtime_receipt.json` verifies required tables, two active admins, durable lockout, session revocation, trusted proxy policy and no-secret mutation audit readiness. Runtime is intentionally hybrid because zero database-admin success receipts have been captured; `auto` retires shared authority after two durable successes without a manual review gate. Next: capture database-admin login receipts and verify automatic database-only cutover. | | 4 | `SEC-P0-003` | In progress | Webhook trust and replay protection | Telegram secret-token verification code exists; production secret activation remains unproven. Exit: secret provisioned outside source, required mode enabled, invalid-secret 401 and authorized callback canary pass. | | 5 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout, secret-safe `.dockerignore`, commit-bound source receipt and governance gate are active. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | | 6 | `GOV-P0-001` | In progress | Canonical full asset graph + runtime reconciliation | `governance/ewoooc_asset_inventory.json` seeds hosts, services, data, AI, routes, supply chain, observability and recovery. Exit: same-run probe receipt for every asset; drift auto-creates work items. | | 7 | `GOV-P0-002` | Not started | Unified controlled-apply envelope | Introduce one `trace_id/run_id/work_item_id` across sensor, identity, SOT diff, decision, risk, dry-run, execution, verifier, rollback/retry and learning acknowledgement. Start with EventRouter + AutoHeal. | -| 8 | `RAG-P0-001` | In progress (`source_fix_ready_runtime_canary_pending`) | Internal RAG candidate canary + NemoTron decision-only proof | V10.813 production still preserves the prior fail-closed receipt. Windows 99 audit run `534c6208-2088-428c-ba6f-f7a63a67f422` now proves the old GCP project is `DELETE_REQUESTED`, GCP-A is unreachable, and GCP-B Ollama `0.22.1` has exact `bge-m3` / `qwen3:14b` digests. Its bounded BGE-M3 probe returned 1024 dimensions with zero DB/app writes but required `116.475s`, proving the 10-second consistency timeout was a false-negative source defect. V10.814 source raises only the canary cold-start cap to 150 seconds, verifies `/api/tags` digest before embedding, permits one digest-verified GCP host as degraded shadow quorum while excluding 111, and adds a daily 05:00 NemoTron model decision with zero tool/DB/price/insight writes. Exit: deploy V10.814, pass both production canaries with Telegram/durable receipts, retain redundancy blocker until a canonical replacement GCP-A asset exists, then controlled `RAG_ENABLED` shadow activation plus query/hit/feedback telemetry. | +| 8 | `RAG-P0-001` | In progress (`rag_canary_passed_nemotron_fallback_source_ready`) | Internal RAG candidate canary + NemoTron decision-only proof | V10.815 production run `736e6de1-fbc3-4c9b-bcfe-9b6ef488b932` passed PixelRAG receipt -> exact-digest BGE-M3 -> read-only pgvector canary with zero business writes; activation remains correctly blocked by `RAG_ENABLED=false` and degraded embedding redundancy. GCP-A is unreachable; GCP-B exact-digest `qwen3:14b` chat timed out at 300s and `qwen2.5:7b-instruct` at 240s, while 111 exact-digest `qwen3:8b` produced the required tool call in 19.053s with no tool/DB/Telegram execution. V10.816 source now implements exact-digest GCP-A -> GCP-B -> 111 model-aware fallback with one 180-second production deadline, bounded 111 context/output, deterministic one-call-per-SKU tool validation, truthful failure footprints and a process-shared `side_effect_started` boundary. Durable side effects remain quarantined for four hours even when dedupe commit verification fails, including mixed forced-review/model batches, so app/scheduler/bot cannot reopen the same SKU and duplicate the action；mixed-path dispatched/skipped metrics preserve input-count truth. Focused model/dedupe regression is `93 passed`. Next: deploy V10.816, obtain shared-store and fresh production `canary_passed_degraded_fallback` receipts plus independent zero-write readback; then restore GCP capacity/redundancy before controlled `RAG_ENABLED` shadow activation and query/hit/feedback telemetry. | | 9 | `MCP-P0-001` | In progress (`federation_source_ready`) | MCP/RAG production runtime closure | V10.796 source adds a strict public aggregate receipt for canonical `ewoooc` and `momo-pro-system` identities without opening authenticated internal APIs or exposing endpoint/tool payload data. Exit still requires V10.796 production `/health`, two fresh AWOOOI durable receipts with fingerprint recompute, live MCP servers/router/RAG, approved caller/tool boundary and production query canary. Current source readiness must not be reported as runtime closure. | | 10 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. | -| 11 | `REL-P0-001` | In progress (`runtime_verified_cd_degraded`) | Formal deploy and visible proof discipline | Production V10.813 runs exact Gitea object `a193acc407ef61b9cc6340e8e11aa9d0fb9a39c5`; dev merge `715f90133101e7fdcac243bf999aef05f80e7401` carries the same source. Host 110 still has no matching EwoooC runner, so formal CD remains unavailable and is not replaced by fallback evidence. Windows 99 controlled run `2ef0a084-7aa8-4748-a0db-127fb623d75d` deployed exact archive SHA-256 `65369d5f78c56376bffe8606adc790c046274dc070dfef9b933a6140cd83cc66`; rollback/evidence is retained at `/home/ollama/momo-deploy-backups/ewoooc-20260716T190911Z-a193acc-2ef0a084`. Internal/external `/health` is healthy at V10.813, all 8 source hashes match, app/scheduler/bot identities and immutable `momo-db` ID `cd092451cb5fd555d0ffff70642e109f3b742882c418beeab631793d1e9dc55d` remained unchanged. Public login visibly reports V10.813; authenticated cockpit visual proof remains unavailable in the current browser session. | +| 11 | `REL-P0-001` | In progress (`v10815_runtime_verified_v10816_pending`) | Formal deploy and visible proof discipline | Production V10.815 runs exact Gitea object `283c8c80c631f5d97315708885413b62ee5a34ea`; dev merge `df83c646cdb3fde5e33f045bff19c5a392f84b86` has an identical tree. Host 110 still has no matching EwoooC runner, so formal CD remains unavailable and is not replaced by fallback evidence. Windows 99 controlled run `736e6de1-fbc3-4c9b-bcfe-9b6ef488b932` corrected and deployed the complete 36-file source set, verified `36/36` hashes and `12/12` runtime env checks, recreated only app/scheduler/bot, and left `momo-db` identity unchanged; rollback is retained at `/home/ollama/momo-deploy-backups/ewoooc-20260722T115155Z-283c8c8-full-736e6de1`. Internal/external `/health` is healthy at V10.815 and public JS/CSS match the tested hashes. V10.816 remains source-only until Gitea integration plus a new Windows 99 controlled deployment and independent runtime readback complete. | ### GROWTH-P0-001 Fixed Execution Lanes @@ -36,7 +36,7 @@ These lanes are one ordered current P0, not optional side work. They must advanc |---|---|---|---| | A. Sales freshness | In progress (`sla_runtime_closed_source_redundancy_partial`) | Latest sales date `2026-07-13`; before the `2026-07-15 20:00` cutoff, raw lag is `2` but SLA lag is `0`, state is `grace` and decisions remain released. Scheduler receipt `0f24219e0bbb4740b7ad6645e7952296` and explicit canary receipt `6e2df008f65544fe8e557c11ff0dbb93` both persisted `completed_no_write`; durable decision is `no_candidate_fresh_no_write`. Live and persisted readiness now agree at Google Drive `1/4`; HTTPS, IMAP and local remain disabled. | Continue automatic report-arrival reconciliation. At/after 20:00 require `2026-07-14` or automatically block decision use, emit the bounded upstream action and verify the next arrival receipt; keep source redundancy partial until another approved source is live. | | B. Verified same-item evidence | In progress | TOP50 fixed cohort: `25` verified, `1` candidate/source validation, `24` unmatched. Count coverage is `50%`; revenue-weighted coverage is `NT$211,667 / NT$354,062 = 59.782%`. V10.810 preserves fingerprint `7b74504fcc1e1801c2ca2b42`; run `34999f5314054f09909663b46fc42ee2` added one independently verified offer (`+2.0pp` count, `+1.89pp` revenue), while run `e1508ec3a49241d1965bc99d428b0325` correctly ended no-write. Their durable artifact SHA-256 values are `994beba6dfa70ef8c50031f130916ee155657dd77ac8a2f59b6530b491d45d1e` and `e58be7bb07599871a8f318ad63885bd868c786b0144b81ba0980ae80cf556541`. | Retry unresolved candidates only after fresh evidence arrives, preserve deterministic identity/unit/variant gates, and publish count/revenue deltas against the same fingerprint per run. | -| C. Platform runtime coverage | In progress (`runtime_canary_activated`) | V10.813 is live. Yahoo is durably active and exact-offer canary readback remains valid; an already-active canary now returns `already_active_verified`, `state_changed=false` and `writes_database_count=0`, while latest no-write receipts no longer erase durable activation truth. Formal runtime is `2/15`; PixelRAG remains evidence-only. | Continue bounded refresh on schedule, monitor expiry/recurrence/rollback signals, then implement the next approved structured source contract for Shopee, Coupang, ETMall, Friday or Rakuten without treating blocked pages as product data. | +| C. Platform runtime coverage | In progress (`runtime_canary_activated`) | V10.815 is live. Yahoo is durably active and exact-offer canary readback remains valid; an already-active canary returns `already_active_verified`, `state_changed=false` and `writes_database_count=0`, while latest no-write receipts do not erase durable activation truth. Formal runtime is `2/15`; PixelRAG remains evidence-only. | Continue bounded refresh on schedule, monitor expiry/recurrence/rollback signals, then implement the next approved structured source contract for Shopee, Coupang, ETMall, Friday or Rakuten without treating blocked pages as product data. | ### AI Agent Product Integration Acceptance @@ -45,7 +45,7 @@ This is an acceptance surface inside the current growth P0; it does not reorder | Layer | Current status | Exit evidence | |---|---|---| | Source and scheduler wiring | Source ready (`4/4`) | Hermes, NemoTron, OpenClaw and ElephantAlpha source markers plus scheduler ownership are machine-read and reported separately from runtime. | -| Agent runtime activity | Production partial (`3/4` role-active; `0/4` healthy); V10.814 source canary ready | V10.813 live readback preserves Hermes/OpenClaw call evidence, zero NemoTron runtime activity and ElephantAlpha durable role-owned insights. V10.814 adds a qwen3 decision-only canary that uses the production payload and exact digest but stops before every tool/action/data write; a fresh pass proves shadow model execution only, not formal product action. Exit requires all four role-active and healthy in the bounded window, without treating class presence, configured fallback or shadow canary as an executed business outcome. | +| Agent runtime activity | Production partial; V10.816 bounded fallback source ready | V10.815 runtime proved the RAG no-write lane and separately proved 111 `qwen3:8b` can execute the NemoTron tool contract in 19.053s, but production dispatcher does not use that model-aware fallback until V10.816 is deployed. A fresh decision-only pass proves shadow model execution only, not formal product action. Exit requires all four Agents role-active and healthy in the bounded window, without treating class presence, configured fallback or shadow canary as an executed business outcome. | | MCP/RAG dependency | Runtime disabled / telemetry empty | Production currently reports `MCP_ROUTER_ENABLED=false`, `RAG_ENABLED=false`, zero `mcp_calls` and zero `rag_query_log` activity. Exit requires enabled approved routes, live health, non-zero agent/product telemetry and the internal RAG candidate canary. | | Controlled automation closure | Runtime partial | `/api/ai-automation/agent-product-integration`, CLI and smoke must report Detect -> Normalize -> Correlate -> Decide -> Check -> Controlled Apply -> Verify -> Retry/Rollback -> Learn/Writeback. Completion requires bounded execution, linked outcome/incident verification and durable learning evidence; aggregate source presence is insufficient. | @@ -57,8 +57,8 @@ This bounded interruption is closed and control returns to `GROWTH-P0-001` witho |---|---|---| | Program | Completed for the four primary analysis tabs | Daily sales, sales analysis, growth analysis and monthly summary now share one canonical day/month/range contract; cross-tab links preserve the selected period. | | Asset coverage | `4/4` pages and `2/2` sales async APIs verified | Daily KPI/calendar/charts/Top 10, growth KPI/series, monthly KPI/tables/charts and sales KPI/charts/YoY/detail table all use the active period. Historical current-snapshot mixing and blank monthly charts are replaced by explicit honest states. | -| Runtime closure | Completed in V10.810 at `911393190ded015e384e438c26b68faf50ec260c` | `2026-04` daily stays inside April and bounds extreme DoD/WoW values with traceable markers; growth single-point series remain visible; sales consumes the real `chart_values` payload and aggregates repeated SKU rows before ranking; monthly requests are abortable/deduplicated and publish explicit loading/error/ready states. External HTTPS serves the exact tested JS/CSS checksums and internal/external `/health` report V10.810. | -| Verification | Passed; formal CD still degraded | Final regression: `2,123 passed / 9 skipped / 0 failed`; focused analytics contracts: `49 passed`. The automated visual guard passed `12/12` combinations (`4` pages x desktop/tablet/mobile), covering `126` chart-target observations, period links, actual pixels/elements, honest empty states, date-tick density, console errors and overflow. Formal CD remains degraded because no matching runner executed the release. | +| Runtime closure | Completed in V10.815 at `283c8c80c631f5d97315708885413b62ee5a34ea` | The exact `/sales_analysis` URL with blank date fields now resolves one canonical period and applies the same metric/range/filter contract to KPI, charts, YoY and detail rows. Query, chart and export logic are split into dedicated services; repeated SKU rows are aggregated before ranking, date/range links preserve state, and invalid filters fail safely. Windows 99 independent readback verified all 36 target files, runtime env, public `/health` and exact JS/CSS hashes. | +| Verification | Passed; authenticated visual proof remains pending | V10.815 broad regression: `2,174 passed / 9 skipped / 0 failed`; focused analytics batches: `173 passed` and `70 passed`, plus Python compile, Node syntax, Jinja render and diff checks. Public target route returns the expected login redirect and public assets match SHA-256 `6bb4a985744917ef2243d6d055c85b6803b2baf5c571580f49ef69c536ccdf04` / `d98654ddaac10a49af2351750f350bbbcfc3491d62b14c8a25d02eb4d7803186`. The current browser session cannot provide authenticated chart screenshots, so that evidence is not claimed. | ## P1 @@ -69,10 +69,10 @@ This bounded interruption is closed and control returns to `GROWTH-P0-001` witho | 14 | `APPSEC-P1-001` | In progress | CSP and DOM/XSS hardening | Security headers are present and CSP is report-only. Collect violations, remove high-risk `innerHTML`/inline sinks, then enforce CSP by canary. | | 15 | `APPSEC-P1-002` | Not started | Unsafe shared-cache serialization removal | Replace writable pickle caches in dashboard/daily-sales/EDM/sales with constrained JSON or signed schema. | | 16 | `ARCH-P1-001` | In progress | Split oversized policy/executor/verifier modules | Current top debts include 44k-line PChome mapping and 14k-line smoke service. Split by bounded family and independent tests. | -| 17 | `UX-P1-001` | In progress | Professional full-site UI/UX | V10.810 closes chart rendering defects across all four primary analysis tabs: real payload values render, single points stay visible, extreme percentages no longer flatten normal data, dense date labels auto-skip, Top 50 vendor content uses bounded zoom, and loading/error/empty states are explicit. The `12/12` desktop/tablet/mobile visual matrix has zero horizontal overflow. The broader site-wide first-viewport, progressive-disclosure, accessibility and loading/error/degraded-state audit remains in progress. | +| 17 | `UX-P1-001` | In progress | Professional full-site UI/UX | V10.815 closes the exact sales-analysis period/filter linkage and keeps the prior four-tab rendering guards: real payload values, visible single points, bounded extreme percentages, date-label auto-skip, zoom bounds and explicit loading/error/empty states. Public assets are hash-verified; authenticated V10.815 visual proof and the broader site-wide first-viewport, progressive-disclosure, accessibility and loading/error/degraded-state audit remain in progress. | | 18 | `PIXELRAG-P1-001` | Not started | Ollama-first multimodal embedding benchmark | Verify approved visual embedding on GCP-A -> GCP-B -> 111 and design pgvector-compatible visual metadata; FAISS remains disallowed without ADR. | | 19 | `MARKET-P1-001` | In progress | Marketplace source contracts | Yahoo Shopping remains active in V10.810 production with public-boundary allowlists, bounded streaming/rate, provenance, current product-detail readback, stock/spec/variant guards, source-specific promotion partition, idempotent exact canary activation and durable activation readback across no-write runs. Four fresh verified Yahoo offers now contribute formal evidence across completed batches; non-exact and unit-price candidates do not. Shopee, Coupang, ETMall, Friday and Rakuten still require equivalent structured contracts, and blocked pages remain non-product data. | -| 20 | `QA-P1-001` | In progress | Deterministic test and CI governance | V10.810 final broad regression is `2,123 passed / 9 skipped / 0 failed`; analytics contracts are `49 passed`, and the new runtime chart guard passes `12/12` page/viewport cases with actual pixel/element and console assertions. Production host parity is `18/18` files and public HTTPS parity is `9/9` key JS/CSS assets. No matching EwoooC runner executed this release, so CI/CD parity is not claimed and exact-object fallback receipts remain separate evidence. | +| 20 | `QA-P1-001` | In progress | Deterministic test and CI governance | V10.816 source-candidate broad regression is `2,223 passed / 15 skipped / 0 failed`; focused model/dedupe regression is `93 passed`, and independent ninth-round review found no material issue. V10.815 production parity remains `36/36` source hashes plus `12/12` runtime env checks, and public HTTPS serves its exact tested sales JS/CSS. No matching EwoooC runner has executed V10.816, so formal CI/CD or production parity is not claimed and Windows 99 exact-object deployment receipts remain a separate evidence layer. | ## P2 @@ -97,7 +97,7 @@ These are reusable foundations, not proof that the full program is complete. | Completed | PromotionGate replay | No production write. | | Completed | Embedding-signature guard replay | Signature readiness only. | | Completed | Candidate knowledge replay | Internal RAG preview only; no DB/model call. | -| Source ready; production pending | Internal RAG + NemoTron decision canaries | V10.814 source separates cold-start timeout from normal embedding latency, enforces exact BGE-M3/qwen3 digest readback, permits one approved GCP host only as degraded shadow quorum, and adds zero-tool/zero-data-write NemoTron runtime proof. Production canary receipts, replacement GCP-A control-plane identity, RAG shadow activation and query/outcome telemetry remain P0. | +| Source ready; production pending | Model-aware NemoTron dispatcher fallback | V10.816 source uses one modular exact-digest candidate registry for production and canary: GCP-A/GCP-B `qwen3:14b` with 60-second attempts, then 111 `qwen3:8b` with 45 seconds, `think=false`, fixed `num_ctx=4096` / `num_predict=512` and one total deadline. Production and canary share the full one-call-per-SKU tool contract; malformed model identity schema fails closed and continues the approved candidate chain. Two-phase dedupe uses `/app/data` shared `flock` + fsync + atomic JSON ownership-token leases across Gunicorn workers and app/scheduler/bot containers. Before any handler, the owner persists `side_effect_started` with a four-hour quarantine; only an explicit DB write, durable insight, delivered notification or durable EventRouter queue acknowledgement counts as dispatched. A later commit-verification failure is surfaced but never releases the quarantine, preventing duplicate action after a durable effect. Privacy-safe state hashes SKU values; privacy-safe attempt summaries preserve label/tier/model/status/digest/error-class in structured and recipient-visible footprints. Production deployment, shared-store canary, fresh degraded-fallback receipt and independent readback remain P0; GCP-A replacement, GCP-B capacity and RAG shadow activation remain unresolved rather than hidden by the fallback. | | Completed | PixelRAG application portfolio | Commerce/RAG/UX/ops/marketing/governance inventory. | | Completed | Ollama-first VLM route readiness and replay worker | Evidence-bound artifact output; no direct price write. | | Completed | Platform probe worker | Shopee/Coupang barriers become structured fallback/backoff receipts. | diff --git a/docs/memory/code_modularization_inventory_20260430.md b/docs/memory/code_modularization_inventory_20260430.md index 44aa59e..fd1a2a4 100644 --- a/docs/memory/code_modularization_inventory_20260430.md +++ b/docs/memory/code_modularization_inventory_20260430.md @@ -75,6 +75,7 @@ - 2026-07-15 追記：`services/pchome_growth_same_item_reconciliation.py` 已達 880 行；同商品 identity verifier、exact DB readback、coverage post-verifier 與 durable receipt persistence 應在 `ARCH-P1-001` 拆成獨立 policy/verifier/repository，主模組只保留 bounded orchestration。 - 2026-07-17 追記：Nemotron decision-only canary 的排程執行、Telegram acknowledgement 與 durable receipt 終局已移至 `services/nemotron_decision_canary_scheduler_task.py`（120 行）；`run_scheduler.py` 僅保留薄委派與排程註冊，清冊同步為 1,684 行，下一步仍依序拆 task registration 與 runtime startup。 - 2026-07-22 追記：業績分析的 canonical query、metric aggregate、Other-category contract 與 period-linked Excel export 已抽到 `services/sales_analysis_query_service.py`（391 行）及 `services/sales_analysis_export_service.py`（207 行）；`routes/sales_routes.py` 降為 2,954 行，後續繼續拆 page context 與 legacy pandas API。 +- 2026-07-22 追記：V10.816 將 exact-digest model candidate/identity policy 抽成 `services/nemotron_runtime_candidate_service.py`（234 行）、process/container-shared ownership-token lease 與 side-effect crash quarantine 抽成 `services/nemotron_dispatch_reservation_service.py`（376 行），decision-only runtime proof 維持在 `services/nemotron_decision_canary_service.py`（604 行）；`services/nemoton_dispatcher_service.py` 因補齊共用 tool contract、durable outcome、commit-failure quarantine 與 privacy-safe fallback footprint增至 2,864 行。此輪先完成 P0 自動化正確性；後續 `ARCH-P1-001` 應依序抽出 delivery outcome adapter、tool execution coordinator 與 footprint renderer，dispatcher 只保留 orchestration。 ## 達到或超過 800 行檔案清單 @@ -137,7 +138,7 @@ | 2961 | `services/openclaw_strategist_service.py` | P1 strategist | prompt、query、report、notification 分離 | | 2383 | `services/competitor_intel_repository.py` | P1 repository | query、decision envelope、UI projection、cache 分離 | | 2332 | `services/elephant_alpha_autonomous_engine.py` | P1 engine | trigger、planner、executor、notification 分離 | -| 2166 | `services/nemoton_dispatcher_service.py` | P1 dispatcher | model client、tool parser、decision、delivery 分離 | +| 2864 | `services/nemoton_dispatcher_service.py` | P1 dispatcher | 已抽 exact-digest candidate registry 與 process-shared lease/crash-quarantine store；下一步拆 delivery outcome adapter、tool execution coordinator、footprint renderer | | 2011 | `services/external_market_offer_service.py` | P1 market offers | source registry、normalizer、sync、review、readiness 分離 | | 1917 | `services/market_intel/deployment_readiness.py` | P1 readiness | check registry、runtime probe、projection 分離 | | 1658 | `routes/market_intel_review_report_routes.py` | P1 review route | report query、export、route glue 分離 | diff --git a/scripts/ops/run_nemotron_dispatch_reservation_canary.py b/scripts/ops/run_nemotron_dispatch_reservation_canary.py new file mode 100644 index 0000000..6fdb5ea --- /dev/null +++ b/scripts/ops/run_nemotron_dispatch_reservation_canary.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Run the no-business-side-effect NemoTron shared reservation canary.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.nemotron_dispatch_reservation_service import ( # noqa: E402 + PRODUCTION_STATE_PATH, + run_shared_reservation_canary, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--state-path", default=str(PRODUCTION_STATE_PATH)) + args = parser.parse_args() + payload = run_shared_reservation_canary(state_path=args.state_path) + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + return 0 if payload["success"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/nemoton_dispatcher_service.py b/services/nemoton_dispatcher_service.py index 9376efc..2523040 100644 --- a/services/nemoton_dispatcher_service.py +++ b/services/nemoton_dispatcher_service.py @@ -23,9 +23,17 @@ import os import re import uuid from datetime import datetime +from threading import Lock from typing import Optional import requests from services.mcp_context_service import build_mcp_context +from services.nemotron_runtime_candidate_service import ( + PRIMARY_EXPECTED_DIGEST, + PRIMARY_MODEL, +) +from services.nemotron_dispatch_reservation_service import ( + build_production_shared_reservation_store, +) from config import HERMES_URL # ADR-008 集中化：禁止硬編碼 IP from services.ai_call_logger import log_ai_call # Operation Ollama-First v5.0 P1 @@ -121,12 +129,16 @@ _nim_call_count = {"date": "", "count": 0} # 預設 ON：qwen3:14b 主 → NIM 備援 → Hermes 規則引擎兜底（ADR-004） # 緊急停用（回 NIM-first）：export NEMOTRON_OLLAMA_FIRST=false NEMOTRON_OLLAMA_FIRST = os.getenv("NEMOTRON_OLLAMA_FIRST", "true").lower() == "true" -NEMOTRON_OLLAMA_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b") +NEMOTRON_OLLAMA_MODEL = PRIMARY_MODEL NEMOTRON_OLLAMA_TIMEOUT = int(os.getenv("NEMOTRON_OLLAMA_TIMEOUT", "180")) # 秒 -NEMOTRON_OLLAMA_EXPECTED_DIGEST = os.getenv( - "NEMOTRON_OLLAMA_EXPECTED_DIGEST", - "bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8", -).strip().lower() +NEMOTRON_OLLAMA_EXPECTED_DIGEST = PRIMARY_EXPECTED_DIGEST +try: + NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC = max( + 30, + min(int(os.getenv("NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC", "180")), 600), + ) +except (TypeError, ValueError): + NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC = 180 def _check_nim_quota() -> bool: @@ -143,23 +155,174 @@ def _check_nim_quota() -> bool: # ── 告警去重 (Deduplication) 快取 ──────────────────────── import time _ALERT_CACHE = {} # {sku: timestamp} +_ALERT_INFLIGHT = {} # {sku: {"token": str, "timestamp": float, "phase": str}} _ALERT_TTL_SEC = 4 * 3600 # 預設 4 小時（防止同商品短時間重複告警） +_ALERT_INFLIGHT_TTL_SEC = max( + 15 * 60, + NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC + 120, +) +_ALERT_CACHE_LOCK = Lock() +_SHARED_ALERT_RESERVATION_STORE = build_production_shared_reservation_store( + committed_ttl_sec=_ALERT_TTL_SEC, + lease_ttl_sec=_ALERT_INFLIGHT_TTL_SEC, +) + +def _cleanup_alert_dedupe(now: float) -> None: + expired = [ + key + for key, timestamp in _ALERT_CACHE.items() + if (now - timestamp) >= _ALERT_TTL_SEC + ] + for key in expired: + del _ALERT_CACHE[key] + expired_inflight = [] + for key, lease in _ALERT_INFLIGHT.items(): + timestamp = lease.get("timestamp", 0) if isinstance(lease, dict) else lease + lease_ttl = ( + _ALERT_TTL_SEC + if isinstance(lease, dict) and lease.get("phase") == "side_effect_started" + else _ALERT_INFLIGHT_TTL_SEC + ) + if (now - float(timestamp or 0)) >= lease_ttl: + expired_inflight.append(key) + for key in expired_inflight: + del _ALERT_INFLIGHT[key] + def _is_duplicate_alert(sku: str) -> bool: - """檢查是否在 TTL 內已經告警過。若是則回傳 True，否則記錄當下時間並回傳 False""" + """Return whether a SKU is committed or currently reserved.""" + if _SHARED_ALERT_RESERVATION_STORE is not None: + return _SHARED_ALERT_RESERVATION_STORE.is_duplicate(sku) now = time.time() - last_alert = _ALERT_CACHE.get(sku) - if last_alert and (now - last_alert) < _ALERT_TTL_SEC: + cache_key = str(sku) + with _ALERT_CACHE_LOCK: + _cleanup_alert_dedupe(now) + return cache_key in _ALERT_CACHE or cache_key in _ALERT_INFLIGHT + + +def _reserve_alert(sku: str) -> str | None: + """Atomically reserve a SKU and return an ownership token.""" + if _SHARED_ALERT_RESERVATION_STORE is not None: + return _SHARED_ALERT_RESERVATION_STORE.reserve(sku) + now = time.time() + cache_key = str(sku) + with _ALERT_CACHE_LOCK: + _cleanup_alert_dedupe(now) + if cache_key in _ALERT_CACHE or cache_key in _ALERT_INFLIGHT: + return None + token = uuid.uuid4().hex + _ALERT_INFLIGHT[cache_key] = { + "token": token, + "timestamp": now, + "phase": "reserved", + } + return token + + +def _refresh_alert_reservation(sku: str, token: str) -> bool: + """Renew only the lease owned by this dispatch worker.""" + if _SHARED_ALERT_RESERVATION_STORE is not None: + return _SHARED_ALERT_RESERVATION_STORE.refresh(sku, token) + cache_key = str(sku) + with _ALERT_CACHE_LOCK: + _cleanup_alert_dedupe(time.time()) + lease = _ALERT_INFLIGHT.get(cache_key) + if not isinstance(lease, dict) or lease.get("token") != str(token): + return False + lease["timestamp"] = time.time() return True - _ALERT_CACHE[sku] = now - - # 順便清理過期快取，避免記憶體洩漏 - expired = [k for k, v in _ALERT_CACHE.items() if (now - v) >= _ALERT_TTL_SEC] - for k in expired: - del _ALERT_CACHE[k] - + + +def _begin_alert_side_effect(sku: str, token: str) -> bool: + """Persist the at-most-once boundary before invoking a side effect.""" + if _SHARED_ALERT_RESERVATION_STORE is not None: + return _SHARED_ALERT_RESERVATION_STORE.begin_side_effect(sku, token) + cache_key = str(sku) + with _ALERT_CACHE_LOCK: + _cleanup_alert_dedupe(time.time()) + lease = _ALERT_INFLIGHT.get(cache_key) + if not isinstance(lease, dict) or lease.get("token") != str(token): + return False + lease["phase"] = "side_effect_started" + lease["timestamp"] = time.time() + return True + + +def _mark_alert_dispatched(sku: str, token: str) -> bool: + """Commit only the reservation owned by this dispatch worker.""" + if _SHARED_ALERT_RESERVATION_STORE is not None: + return _SHARED_ALERT_RESERVATION_STORE.commit(sku, token) + with _ALERT_CACHE_LOCK: + cache_key = str(sku) + now = time.time() + _cleanup_alert_dedupe(now) + lease = _ALERT_INFLIGHT.get(cache_key) + if not isinstance(lease, dict) or lease.get("token") != str(token): + return False + del _ALERT_INFLIGHT[cache_key] + _ALERT_CACHE[cache_key] = now + return True + + +def _release_alert_reservation(sku: str, token: str | None) -> bool: + if _SHARED_ALERT_RESERVATION_STORE is not None: + return _SHARED_ALERT_RESERVATION_STORE.release(sku, token) + with _ALERT_CACHE_LOCK: + cache_key = str(sku) + _cleanup_alert_dedupe(time.time()) + lease = _ALERT_INFLIGHT.get(cache_key) + if not isinstance(lease, dict) or lease.get("token") != str(token or ""): + return False + del _ALERT_INFLIGHT[cache_key] + return True + + +def _handler_outcome_is_durable(outcome) -> bool: + """Require an explicit durable side-effect acknowledgement from handlers.""" + return isinstance(outcome, dict) and outcome.get("durable") is True + + +def _require_durable_handler_outcome(outcome, tool_name: str) -> None: + if not _handler_outcome_is_durable(outcome): + raise RuntimeError(f"durable_side_effect_not_confirmed:{tool_name}") + + +def _record_dedupe_commit_after_durable_side_effect( + sku: str, + token: str, + tool_name: str, + errors: list, +) -> bool: + """Commit dedupe state without reopening a completed external side effect.""" + if _mark_alert_dispatched(sku, token): + return True + error = f"dedupe_commit_unverified:{tool_name}({sku})" + errors.append(error) + logger.error( + "[Dispatcher] Durable side effect completed but dedupe commit was not " + "verified; reservation remains quarantined: %s", + error, + ) return False + +def _remaining_timeout(deadline_monotonic: float | None, cap_sec: float) -> float: + cap = max(0.1, float(cap_sec)) + if deadline_monotonic is None: + return cap + remaining = float(deadline_monotonic) - time.monotonic() + if remaining <= 0: + raise requests.Timeout("decision_total_deadline_exhausted") + return min(cap, max(0.001, remaining)) + + +class DecisionModelRuntimeError(RuntimeError): + """Carries privacy-safe candidate attempts into the next fallback layer.""" + + def __init__(self, message: str, *, attempted_candidates: list[dict]): + super().__init__(message) + self.attempted_candidates = list(attempted_candidates) + def _nim_quota_used() -> int: """回傳今日已使用配額數""" today = datetime.now().strftime("%Y-%m-%d") @@ -649,11 +812,95 @@ def _parse_content_fallback(raw_content: str) -> list: return results +class ToolCallContractError(ValueError): + """Model output did not satisfy the deterministic dispatch contract.""" + + +def _validate_tool_call_contract(tool_calls: list, threats: list) -> list: + """Require one schema-valid, input-bound tool call for every threat SKU.""" + expected_skus = {str(getattr(threat, "sku", "")).strip() for threat in threats} + expected_skus.discard("") + if not expected_skus: + raise ToolCallContractError("no_expected_threat_skus") + if not isinstance(tool_calls, list) or len(tool_calls) != len(expected_skus): + raise ToolCallContractError( + f"tool_call_count_mismatch:{len(tool_calls or [])}/{len(expected_skus)}" + ) + + contracts = {} + for tool in TOOLS: + function = tool.get("function") or {} + name = str(function.get("name") or "") + parameters = function.get("parameters") or {} + contracts[name] = { + "required": set(parameters.get("required") or []), + "properties": parameters.get("properties") or {}, + } + + normalized = [] + seen_skus = set() + for index, call in enumerate(tool_calls): + if not isinstance(call, dict): + raise ToolCallContractError(f"tool_call_not_object:{index}") + tool_name = str(call.get("tool") or "").strip() + contract = contracts.get(tool_name) + if contract is None: + raise ToolCallContractError(f"unknown_tool:{tool_name or 'missing'}") + raw_args = call.get("args") + if not isinstance(raw_args, dict): + raise ToolCallContractError(f"tool_args_not_object:{tool_name}") + + missing = [] + for key in contract["required"]: + value = raw_args.get(key) + if value is None or (isinstance(value, str) and not value.strip()): + missing.append(key) + if missing: + raise ToolCallContractError( + f"missing_required_args:{tool_name}:{','.join(sorted(missing))}" + ) + + for key, spec in contract["properties"].items(): + if key not in raw_args: + continue + value = raw_args[key] + expected_type = spec.get("type") + if expected_type == "string" and not isinstance(value, str): + raise ToolCallContractError(f"invalid_arg_type:{tool_name}:{key}:string") + if expected_type == "number" and ( + isinstance(value, bool) or not isinstance(value, (int, float)) + ): + raise ToolCallContractError(f"invalid_arg_type:{tool_name}:{key}:number") + + sku = str(raw_args.get("sku") or "").strip() + if sku not in expected_skus: + raise ToolCallContractError(f"sku_not_in_request:{sku or 'missing'}") + if sku in seen_skus: + raise ToolCallContractError(f"duplicate_sku_tool_call:{sku}") + seen_skus.add(sku) + args = { + key: raw_args[key] + for key in contract["properties"] + if key in raw_args + } + args["sku"] = sku + normalized.append({"tool": tool_name, "args": args}) + + missing_skus = expected_skus - seen_skus + if missing_skus: + raise ToolCallContractError( + f"missing_sku_tool_calls:{','.join(sorted(missing_skus))}" + ) + return normalized + + def build_qwen3_dispatch_payload( threats: list, *, mcp_context: str | None = None, + model: str | None = None, num_predict: int = 2048, + num_ctx: int | None = None, keep_alive: str | int | None = None, ) -> dict: """Build the shared production/canary qwen3 tool-calling request.""" @@ -697,7 +944,7 @@ def build_qwen3_dispatch_payload( "若無法產出合理的繁體中文說明，直接輸出『請人工評估議價空間』。" ) payload = { - "model": NEMOTRON_OLLAMA_MODEL, + "model": str(model or NEMOTRON_OLLAMA_MODEL), "messages": [ {"role": "system", "content": system_prompt}, { @@ -707,16 +954,51 @@ def build_qwen3_dispatch_payload( ], "tools": TOOLS, "stream": False, + "think": False, "options": { "temperature": 0.2, "num_predict": max(32, min(int(num_predict), 2048)), }, } + if num_ctx is not None: + payload["options"]["num_ctx"] = max(512, min(int(num_ctx), 32768)) if keep_alive is not None: payload["keep_alive"] = keep_alive return payload +def _privacy_safe_error_class(value) -> str | None: + if not value: + return None + raw = str(value).strip().split(":", 1)[0] + normalized = re.sub(r"[^A-Za-z0-9_.-]", "_", raw)[:64] + return normalized or "RuntimeError" + + +def _privacy_safe_candidate_attempts(nim_stats: Optional[dict]) -> list[dict]: + if not isinstance(nim_stats, dict): + return [] + upstream = nim_stats.get("upstream_ollama_failure") or {} + attempts = ( + upstream.get("attempted_candidates") + if isinstance(upstream, dict) + else None + ) or nim_stats.get("attempted_candidates") or [] + summaries = [] + for attempt in list(attempts)[:6]: + if not isinstance(attempt, dict): + continue + summaries.append({ + "label": str(attempt.get("label") or "unknown")[:64], + "tier": str(attempt.get("tier") or "unknown")[:32], + "model": str(attempt.get("model") or "unknown")[:80], + "status": str(attempt.get("status") or "unknown")[:48], + "digest_matches": attempt.get("digest_matches") is True, + "error_class": _privacy_safe_error_class(attempt.get("error")), + }) + return summaries + + def _build_footprint_json(hermes_stats: Optional[dict], nim_stats: Optional[dict]) -> dict: """ 建立結構化運算足跡 (用於 DB model_footprint JSONB 欄位) @@ -735,13 +1017,37 @@ def _build_footprint_json(hermes_stats: Optional[dict], nim_stats: Optional[dict "cost_usd": 0, } if nim_stats: + provider = str(nim_stats.get("provider") or "nim") + is_ollama = provider in { + "gcp_ollama", + "ollama_secondary", + "ollama_111", + "ollama_other", + } + platform = "Ollama" if is_ollama else ( + "Deterministic" if provider == "hermes_rule_engine" else "NVIDIA NIM" + ) result["dispatcher"] = { - "model": NIM_MODEL, - "platform": "NVIDIA NIM", + "model": nim_stats.get("model") or NIM_MODEL, + "platform": platform, + "provider": provider, + "host": nim_stats.get("host"), + "host_label": nim_stats.get("host_label"), + "fallback_used": nim_stats.get("fallback_used") is True, "total_tokens": nim_stats.get("total_tokens", 0), "quota_used": nim_stats.get("quota_used", 0), "cost_usd": 0, } + upstream_failure = nim_stats.get("upstream_ollama_failure") or {} + if upstream_failure: + result["dispatcher"]["upstream_ollama_failure"] = { + "attempt_count": int(upstream_failure.get("attempt_count") or 0), + "deadline_exhausted": upstream_failure.get("deadline_exhausted") is True, + "error_class": _privacy_safe_error_class(upstream_failure.get("error")), + } + candidate_attempts = _privacy_safe_candidate_attempts(nim_stats) + if candidate_attempts: + result["dispatcher"]["candidate_attempts"] = candidate_attempts return result @@ -772,10 +1078,34 @@ def _build_footprint_block(hermes_stats: Optional[dict], nim_stats: Optional[dic if nim_stats: tok = nim_stats.get("total_tokens", "?") quota = nim_stats.get("quota_used", "?") - lines.append( - f"• ⚡ 決策: NemoTron NIM | " - f"{tok} Tokens | $0 (配額內 {quota}/{NIM_DAILY_LIMIT})" - ) + provider = str(nim_stats.get("provider") or "nim") + if provider in {"gcp_ollama", "ollama_secondary", "ollama_111", "ollama_other"}: + model = str(nim_stats.get("model") or NEMOTRON_OLLAMA_MODEL) + host_label = str(nim_stats.get("host_label") or provider) + degraded = " | 降級備援" if nim_stats.get("fallback_used") is True else "" + lines.append( + f"• ⚡ 決策: {model} ({host_label}) | " + f"{tok} Tokens | $0 自架{degraded}" + ) + elif provider == "hermes_rule_engine": + lines.append("• ⚡ 決策: Hermes 確定性規則引擎 | $0 自架 | 降級備援") + else: + lines.append( + f"• ⚡ 決策: NemoTron NIM | " + f"{tok} Tokens | $0 (配額內 {quota}/{NIM_DAILY_LIMIT})" + ) + upstream_failure = nim_stats.get("upstream_ollama_failure") or {} + if upstream_failure: + count = int(upstream_failure.get("attempt_count") or 0) + deadline_note = "，總截止時間已用盡" if upstream_failure.get("deadline_exhausted") else "" + lines.append(f"• ↪ Ollama 候選鏈: {count} 次未通過{deadline_note}，已自動降級") + for attempt in _privacy_safe_candidate_attempts(nim_stats): + digest = "ok" if attempt["digest_matches"] else "failed" + error = f" | {attempt['error_class']}" if attempt["error_class"] else "" + lines.append( + f" - {attempt['label']} | {attempt['tier']} | {attempt['model']} | " + f"{attempt['status']} | digest={digest}{error}" + ) else: lines.append(f"• ⚡ 決策: NemoTron NIM | $0 (配額內)") @@ -800,7 +1130,12 @@ class NemotronDispatcher: # ────────────────────────────────────────────── # NIM Tool Calling # ────────────────────────────────────────────── - def _call_nim(self, threats: list) -> tuple: + def _call_nim( + self, + threats: list, + *, + deadline_monotonic: float | None = None, + ) -> tuple: """ 將 Hermes 威脅清單交給 NIM，取得 tool_calls 決策清單 @@ -877,6 +1212,10 @@ class NemotronDispatcher: ) as _ctx: for _attempt in range(3): try: + request_timeout = _remaining_timeout( + deadline_monotonic, + NIM_TIMEOUT, + ) resp = requests.post( f"{NIM_BASE_URL}/chat/completions", headers={ @@ -890,7 +1229,7 @@ class NemotronDispatcher: "tool_choice": "required", "max_tokens": 2048, }, - timeout=NIM_TIMEOUT, + timeout=request_timeout, ) resp.raise_for_status() break @@ -904,7 +1243,14 @@ class NemotronDispatcher: _ctx.fallback_to_caller('hermes_rule_engine') raise if _attempt < 2: - _time.sleep(2 ** _attempt) + backoff_sec = 2 ** _attempt + if deadline_monotonic is not None: + remaining = deadline_monotonic - time.monotonic() + if remaining <= backoff_sec: + raise requests.Timeout( + "decision_total_deadline_exhausted" + ) from e + _time.sleep(backoff_sec) logger.warning(f"[NIM] retry {_attempt + 1}/2 after {e}") else: raise last_err @@ -919,6 +1265,8 @@ class NemotronDispatcher: nim_stats = { "total_tokens": usage.get("total_tokens", 0), "quota_used": _nim_quota_used(), + "provider": "nim", + "model": NIM_MODEL, } choices = body.get("choices", []) @@ -934,21 +1282,22 @@ class NemotronDispatcher: logger.warning(f"[NIM] 0 tool_calls，嘗試從 content 解析：{raw_content[:120]}") results = _parse_content_fallback(raw_content) + results = _validate_tool_call_contract(results, threats) + logger.info(f"[NIM] 收到 {len(results)} 個 tool_calls | tokens={nim_stats['total_tokens']}") return results, nim_stats # ────────────────────────────────────────────── - # GCP Ollama qwen3:14b Tool Calling（Operation Ollama-First v5.0 / Phase 3） + # Digest-locked Ollama tool calling (GCP first, 111 final fallback) # ────────────────────────────────────────────── - def _call_qwen3_dispatch(self, threats: list) -> tuple: + def _call_qwen3_dispatch( + self, + threats: list, + *, + deadline_monotonic: float | None = None, + ) -> tuple: """ - 將 Hermes 威脅清單交給 GCP Ollama qwen3:14b，取得 tool_calls 決策。 - - Why qwen3:14b（A2 web-research 結論，docs/phase0_research_report_20260503.md）： - - Ollama registry 官方頁 + qwenlm.github.io 雙確認 tools capability 可用 - - 預設可關閉 thinking mode（避免 deepseek-r1 的 30s thinking 延遲） - - 14B 體積 9.3GB，與 deepseek-r1:14b 同級 - - 與 NIM 一致採 OpenAI 兼容 chat completion + tools schema + Run the shared tool contract across exact-digest model candidates. Returns: (list of {"tool": str, "args": dict}, dict ollama_stats) @@ -958,11 +1307,14 @@ class NemotronDispatcher: get_host_label, get_provider_tag, mark_unhealthy, - resolve_ollama_host, + ) + from services.nemotron_runtime_candidate_service import ( + IDENTITY_TIMEOUT_SEC, + build_nemotron_runtime_candidates, + inspect_nemotron_model_identity, ) - payload = build_qwen3_dispatch_payload(threats) - + cascade_started = time.monotonic() with log_ai_call( caller='nemotron_dispatch', provider='gcp_ollama', @@ -973,82 +1325,189 @@ class NemotronDispatcher: 'threats_count': len(threats), }, ) as ctx: - attempted_hosts = [] + attempted_candidates = [] body = None host = None + selected_candidate = None + results = [] last_error = None - for _attempt in range(3): - host = resolve_ollama_host().rstrip("/") - if host in attempted_hosts: - break - attempted_hosts.append(host) + for candidate in build_nemotron_runtime_candidates( + primary_model=NEMOTRON_OLLAMA_MODEL, + primary_expected_digest=NEMOTRON_OLLAMA_EXPECTED_DIGEST, + ): + host = candidate.host + attempt = { + "label": candidate.label, + "tier": candidate.tier, + "host": host, + "model": candidate.model, + "identity_ok": False, + "digest_matches": False, + "request_timeout_sec": None, + "num_predict": candidate.num_predict, + "num_ctx": candidate.num_ctx, + "status": "identity_failed", + "error": None, + } + attempted_candidates.append(attempt) try: + identity_timeout = _remaining_timeout( + deadline_monotonic, + IDENTITY_TIMEOUT_SEC, + ) + except requests.Timeout as exc: + last_error = exc + attempt["status"] = "deadline_exhausted" + attempt["error"] = str(exc) + break + + identity = inspect_nemotron_model_identity( + candidate, + request_get=requests.get, + timeout_sec=identity_timeout, + ) + attempt.update({ + "identity_ok": identity.get("ok") is True, + "digest_matches": identity.get("digest_matches") is True, + "identity_elapsed_ms": identity.get("elapsed_ms"), + "error": identity.get("error"), + }) + if identity.get("ok") is not True: + last_error = RuntimeError( + str(identity.get("error") or "model identity failed") + ) + continue + + payload = build_qwen3_dispatch_payload( + threats, + model=candidate.model, + num_predict=candidate.num_predict, + num_ctx=candidate.num_ctx, + ) + try: + request_timeout = _remaining_timeout( + deadline_monotonic, + min( + max(1, NEMOTRON_OLLAMA_TIMEOUT), + candidate.request_timeout_sec, + ), + ) + attempt["request_timeout_sec"] = round(request_timeout, 3) resp = requests.post( f"{host}/api/chat", json=payload, - timeout=NEMOTRON_OLLAMA_TIMEOUT, + timeout=request_timeout, ) resp.raise_for_status() - body = resp.json() + candidate_body = resp.json() + msg = ( + candidate_body.get("message", {}) + if isinstance(candidate_body, dict) + else {} + ) + candidate_results = _parse_tool_calls_struct( + msg.get("tool_calls", []) or [] + ) + if not candidate_results: + candidate_results = _parse_content_fallback( + msg.get("content", "") or "" + ) + candidate_results = _validate_tool_call_contract( + candidate_results, + threats, + ) + post_identity_timeout = _remaining_timeout( + deadline_monotonic, + IDENTITY_TIMEOUT_SEC, + ) + post_identity = inspect_nemotron_model_identity( + candidate, + request_get=requests.get, + timeout_sec=post_identity_timeout, + ) + if post_identity.get("ok") is not True: + raise RuntimeError("model digest changed after decision call") + + body = candidate_body + results = candidate_results + selected_candidate = candidate + attempt["status"] = "selected" ctx.set_provider(get_provider_tag(host)) + ctx.set_model(candidate.model) ctx.add_meta('host', host) ctx.add_meta('host_label', get_host_label(host)) - ctx.add_meta('attempted_hosts', attempted_hosts) + ctx.add_meta('attempted_candidates', attempted_candidates) + ctx.add_meta('fallback_used', candidate.is_fallback) break except Exception as e: last_error = e - # 連線/HTTP 失敗 → 標記主機 unhealthy，下一輪依序嘗試 GCP-B / 111。 + attempt["status"] = "failed" + attempt["error"] = f"{type(e).__name__}: {str(e)[:180]}" mark_unhealthy(host) logger.warning( - "[Dispatcher][qwen3] host=%s 呼叫失敗，嘗試下一台: %s", - host, e, + "[Dispatcher][Ollama] candidate=%s host=%s model=%s failed: %s", + candidate.label, + host, + candidate.model, + e, ) - if body is None: - ctx.set_error( - f"qwen3 call failed after {len(attempted_hosts)} host(s): " + if body is None or selected_candidate is None or not results: + deadline_exhausted = bool( + deadline_monotonic is not None + and time.monotonic() >= deadline_monotonic + ) + message = ( + f"decision model failed after {len(attempted_candidates)} candidate(s): " f"{type(last_error).__name__}: {last_error}" ) + ctx.add_meta('attempted_candidates', attempted_candidates) + ctx.add_meta('fallback_used', False) + ctx.add_meta('deadline_exhausted', deadline_exhausted) + ctx.set_error(message) ctx.fallback_to_caller('nim') - raise RuntimeError(last_error or "qwen3 all hosts failed") + raise DecisionModelRuntimeError( + message, + attempted_candidates=attempted_candidates, + ) ctx.set_tokens( input=body.get('prompt_eval_count', 0), output=body.get('eval_count', 0), ) - msg = body.get('message', {}) if isinstance(body, dict) else {} - tool_calls = msg.get('tool_calls', []) or [] - - # 走共用 tool_calls 結構解析（與 NIM 同一條 helper） - results = _parse_tool_calls_struct(tool_calls) - - if not results: - # qwen3 沒回 tool_calls → 走既有 content fallback 解析 - raw_content = msg.get('content', '') or '' - logger.warning( - f"[Dispatcher][qwen3] 0 tool_calls，嘗試從 content 解析：{raw_content[:120]}" - ) - results = _parse_content_fallback(raw_content) - ollama_stats = { "total_tokens": (body.get('prompt_eval_count', 0) or 0) + (body.get('eval_count', 0) or 0), "host": host, "host_label": get_host_label(host), "provider": get_provider_tag(host), - "model": NEMOTRON_OLLAMA_MODEL, + "model": selected_candidate.model, + "candidate_label": selected_candidate.label, + "fallback_used": selected_candidate.is_fallback, + "attempted_candidates": attempted_candidates, + "cascade_elapsed_ms": round( + (time.monotonic() - cascade_started) * 1000 + ), } logger.info( - f"[Dispatcher][qwen3] 收到 {len(results)} 個 tool_calls | " - f"tokens={ollama_stats['total_tokens']} host={host}" + f"[Dispatcher][Ollama] 收到 {len(results)} 個 tool_calls | " + f"tokens={ollama_stats['total_tokens']} host={host} " + f"model={selected_candidate.model} fallback={selected_candidate.is_fallback}" ) return results, ollama_stats # ────────────────────────────────────────────── # ADR-004：Hermes 規則引擎降級路由 # ────────────────────────────────────────────── - def _hermes_rule_fallback(self, threats: list, hermes_stats: Optional[dict] = None) -> dict: + def _hermes_rule_fallback( + self, + threats: list, + hermes_stats: Optional[dict] = None, + *, + upstream_stats: Optional[dict] = None, + reservation_tokens: Optional[dict] = None, + ) -> dict: """ ADR-004 降級模式：NIM HTTP 429 時，改用確定性規則路由 Hermes 威脅清單。 路由規則與 NIM system prompt 一致，所有 Telegram 告警加 🟡 降級前綴。 @@ -1059,20 +1518,35 @@ class NemotronDispatcher: 3. gap_pct < 0 且 sales_delta > 0 → add_to_recommendation（我方具競爭力） 4. 其餘 → flag_for_human_review（信心不足/複雜情況） """ - degraded_note = "🟡 [降級模式 ADR-004] NIM 配額耗盡，改用 Hermes 規則引擎決策" - footprint = degraded_note + "\n" + _build_footprint_block(hermes_stats, None) + degraded_note = "🟡 [降級模式 ADR-004] AI 模型鏈不可用，改用 Hermes 規則引擎決策" + runtime_stats = { + "degraded": True, + "provider": "hermes_rule_engine", + "model": "deterministic_rules", + **(upstream_stats or {}), + } + footprint = degraded_note + "\n" + _build_footprint_block( + hermes_stats, + runtime_stats, + ) dispatched, errors = 0, [] + owned_tokens = dict(reservation_tokens or {}) for t in threats: + token = owned_tokens.get(str(t.sku)) try: + if not token or not _refresh_alert_reservation(t.sku, token): + raise RuntimeError("alert_reservation_ownership_lost") # B' 軌：每個 threat 預先算金額影響，所有路徑統一注入 impact = _compute_business_impact(t) rl, rp = impact["revenue_loss_7d"], impact["recommended_price"] match_meta = _threat_match_metadata(t) if not _can_direct_price_alert(t): - self._exec_flag_for_human_review( + if not _begin_alert_side_effect(t.sku, token): + raise RuntimeError("alert_side_effect_boundary_rejected") + outcome = self._exec_flag_for_human_review( sku=t.sku, name=t.name, concern=( @@ -1092,12 +1566,24 @@ class NemotronDispatcher: recommended_price=rp, **match_meta, ) + _require_durable_handler_outcome( + outcome, + "flag_for_human_review", + ) + _record_dedupe_commit_after_durable_side_effect( + t.sku, + token, + "flag_for_human_review", + errors, + ) dispatched += 1 continue + if not _begin_alert_side_effect(t.sku, token): + raise RuntimeError("alert_side_effect_boundary_rejected") if t.gap_pct < 5 and t.sales_7d_delta_pct < -30: # Rule 1：價差微小但銷量大跌 → 非定價問題，AI 自動驗證確認 - self._exec_flag_for_human_review( + outcome = self._exec_flag_for_human_review( sku=t.sku, name=t.name, concern=( f"🟡 [規則引擎] 價差僅 {t.gap_pct:+.1f}% 但銷量大跌 " @@ -1113,7 +1599,7 @@ class NemotronDispatcher: ) elif t.gap_pct >= 5 and t.risk == "HIGH": # Rule 2：高價差 HIGH 風險 → 競價告警 - self._exec_trigger_price_alert( + outcome = self._exec_trigger_price_alert( t.sku, t.name, t.gap_pct, t.sales_7d_delta_pct, f"🟡 [規則引擎] {t.recommended_action}", @@ -1125,7 +1611,7 @@ class NemotronDispatcher: ) elif t.gap_pct < 0 and t.sales_7d_delta_pct > 0: # Rule 3：我方具競爭力 + 銷量正成長 → 推薦 - self._exec_add_to_recommendation( + outcome = self._exec_add_to_recommendation( t.sku, t.name, ( f"🟡 [規則引擎] 我方比競品便宜 {abs(t.gap_pct):.1f}%，" @@ -1137,7 +1623,7 @@ class NemotronDispatcher: ) else: # Rule 4：其餘複雜情況 → AI 例外決策 - self._exec_flag_for_human_review( + outcome = self._exec_flag_for_human_review( sku=t.sku, name=t.name, concern=( f"🟡 [規則引擎] 情況複雜或信心不足（信心 {t.confidence:.0%}），" @@ -1150,8 +1636,16 @@ class NemotronDispatcher: revenue_loss_7d=rl, recommended_price=rp, **_threat_match_metadata(t), ) + _require_durable_handler_outcome(outcome, "hermes_rule_fallback") + _record_dedupe_commit_after_durable_side_effect( + t.sku, + token, + "hermes_rule_fallback", + errors, + ) dispatched += 1 except Exception as e: + _release_alert_reservation(t.sku, token) errors.append(f"fallback({t.sku}): {e}") logger.error(f"[Dispatcher][ADR-004] Hermes fallback 失敗 {t.sku}: {e}") @@ -1159,7 +1653,12 @@ class NemotronDispatcher: f"[Dispatcher][ADR-004] Hermes 規則引擎降級完成 " f"dispatched={dispatched} errors={len(errors)}" ) - return {"dispatched": dispatched, "skipped": 0, "errors": errors, "nim_stats": {"degraded": True}} + return { + "dispatched": dispatched, + "skipped": max(0, len(threats) - dispatched), + "errors": errors, + "nim_stats": runtime_stats, + } # ────────────────────────────────────────────── # 語意化訊息格式器 @@ -1381,13 +1880,16 @@ class NemotronDispatcher: competitor_product_id=competitor_product_id, competitor_product_name=competitor_product_name, ) - self._send_telegram(msg, decision_envelope=decision_envelope) + notification_outcome = self._send_telegram( + msg, + decision_envelope=decision_envelope, + ) logger.info( f"[Dispatcher] 競價告警 → {sku} gap={gap_pct:.1f}% sales={sales_delta:.1f}% " f"loss=${revenue_loss_7d:,.0f} rec_price={recommended_price}" ) # ADR-007 雙寫：沉澱到 ai_insights 供日後 RAG - self._sink_insight_to_km( + insight_written = self._sink_insight_to_km( insight_type="price_alert", sku=sku, name=name, content=f"[高危險告警] {name} 價差 {gap_pct:+.1f}% / 銷量 {sales_delta:+.1f}%。行動：{action}", @@ -1402,6 +1904,11 @@ class NemotronDispatcher: "competitor_product_id": competitor_product_id, "decision_envelope": decision_envelope}, ) + return { + "durable": _handler_outcome_is_durable(notification_outcome), + "notification": notification_outcome, + "insight_written": insight_written, + } def _exec_add_to_recommendation( self, @@ -1456,14 +1963,20 @@ class NemotronDispatcher: msg = self._fmt_recommendation( sku, name, reason, confidence, db_written, footprint, ) - self._send_telegram(msg) + notification_outcome = self._send_telegram(msg) # ADR-007 雙寫 - self._sink_insight_to_km( + insight_written = self._sink_insight_to_km( insight_type="recommendation", sku=sku, name=name, content=f"[推薦商品] {name}。原因：{reason}", metadata={"confidence": confidence, "db_written": db_written}, ) + return { + "durable": db_written or _handler_outcome_is_durable(notification_outcome), + "db_written": db_written, + "notification": notification_outcome, + "insight_written": insight_written, + } def _exec_flag_for_human_review( self, @@ -1514,12 +2027,15 @@ class NemotronDispatcher: competitor_product_id=competitor_product_id, competitor_product_name=competitor_product_name, ) - self._send_telegram(msg, decision_envelope=decision_envelope) + notification_outcome = self._send_telegram( + msg, + decision_envelope=decision_envelope, + ) logger.info( f"[Dispatcher] AI 例外決策請求 → {sku} loss=${revenue_loss_7d:,.0f}" ) # ADR-007 雙寫 - self._sink_insight_to_km( + insight_written = self._sink_insight_to_km( insight_type="human_review", sku=sku, name=name, content=f"[AI 例外決策] {name}。疑慮：{concern}", @@ -1534,6 +2050,11 @@ class NemotronDispatcher: "competitor_product_id": competitor_product_id, "decision_envelope": decision_envelope}, ) + return { + "durable": _handler_outcome_is_durable(notification_outcome), + "notification": notification_outcome, + "insight_written": insight_written, + } def _exec_route_to_km( self, @@ -1549,7 +2070,7 @@ class NemotronDispatcher: domain = km_domain if km_domain in _KM_DOMAINS else "price_competition" summary = _sanitize_text(summary, fallback="競價洞察已歸檔") - self._sink_insight_to_km( + insight_written = self._sink_insight_to_km( insight_type=f"km_{domain}", sku=sku, name=name, content=f"[KM 路由 {domain}] {name}：{summary}", @@ -1563,6 +2084,10 @@ class NemotronDispatcher: }, ) logger.info(f"[Dispatcher] KM 路由 → {sku} domain={domain} confidence={confidence:.2f}") + return { + "durable": insight_written is True, + "insight_written": insight_written, + } def _exec_mark_for_relearn( self, @@ -1575,6 +2100,7 @@ class NemotronDispatcher: 不送 Telegram 告警（靜默操作，僅 log）。 """ reason = _sanitize_text(reason, fallback="新數據與歷史洞察矛盾，需重新學習") + db_updated = False try: from database.manager import DatabaseManager db = DatabaseManager() @@ -1590,20 +2116,26 @@ class NemotronDispatcher: """), {"sku": sku}) session.commit() rows = result.rowcount + db_updated = int(rows or 0) > 0 logger.info(f"[Dispatcher] mark_for_relearn → {sku} 共更新 {rows} 筆洞察；原因：{reason}") except Exception as e: logger.warning(f"[Dispatcher] mark_for_relearn DB 更新失敗 ({sku}): {e}") # 同時寫入一筆 relearn 事件到 ai_insights 留存紀錄 - self._sink_insight_to_km( + insight_written = self._sink_insight_to_km( insight_type="relearn_event", sku=sku, name=name, content=f"[重新學習事件] {name}：{reason}", metadata={"sku": sku, "trigger": "nemoton_dispatcher"}, ) + return { + "durable": db_updated or insight_written is True, + "db_updated": db_updated, + "insight_written": insight_written, + } def _sink_insight_to_km(self, insight_type: str, sku: str, name: str, - content: str, metadata: dict = None): + content: str, metadata: dict = None) -> bool: """ ADR-007 雙寫：派發後把決策/洞察沉澱到 ai_insights（供日後 RAG/PPT） period 以當日 YYYY-MM-DD 作為 cache-aside 鍵，同日同 SKU 同 type 會覆蓋。 @@ -1615,7 +2147,7 @@ class NemotronDispatcher: meta = {"sku": sku, "name": name} if metadata: meta.update(metadata) - store_insight( + insight_id = store_insight( insight_type=insight_type, content=content, period=period, @@ -1623,10 +2155,12 @@ class NemotronDispatcher: metadata=meta, ai_model=NIM_MODEL, ) + return insight_id is not None except Exception as e: logger.warning(f"[Dispatcher] sink insight 略過 ({insight_type}/{sku}): {e}") + return False - def _send_telegram(self, message: str, decision_envelope: Optional[dict] = None): + def _send_telegram(self, message: str, decision_envelope: Optional[dict] = None) -> dict: """ ADR-019 Phase 5: 改走 EventRouter 統一入口 舊行為（直接呼叫 Telegram Bot API + MarkdownV2 跳脫）已由 EventRouter @@ -1653,10 +2187,32 @@ class NemotronDispatcher: payload["decision_envelope"] = decision_envelope if not event["id"]: event.pop("id", None) - dispatch_sync(event=event) + result = dispatch_sync(event=event) + durable = bool( + isinstance(result, dict) + and ( + result.get("delivered") is True + or result.get("queued") is True + ) + ) + return { + "durable": durable, + "delivered": bool(isinstance(result, dict) and result.get("delivered") is True), + "queued": bool(isinstance(result, dict) and result.get("queued") is True), + "deduped": bool(isinstance(result, dict) and result.get("deduped") is True), + "silenced": bool(isinstance(result, dict) and result.get("silenced") is True), + } except Exception as e: logger.error(f"[Dispatcher] EventRouter dispatch 失敗: {e}") logger.info(f"[Dispatcher] 告警內容（fallback log）：{message[:200]}") + return { + "durable": False, + "delivered": False, + "queued": False, + "deduped": False, + "silenced": False, + "error_class": type(e).__name__, + } # ────────────────────────────────────────────── # 公開介面 @@ -1764,15 +2320,20 @@ class NemotronDispatcher: """ if not threats: return {"dispatched": 0, "skipped": 0, "errors": [], "nim_stats": {}} - # ── 防線二：Python 絕對獨裁預路由（雙閘門） ── forced_review, nim_candidates = [], [] skipped = 0 + batch_skus = set() + reservation_tokens = {} for t in threats: - if _is_duplicate_alert(t.sku): + sku = str(t.sku) + token = None if sku in batch_skus else _reserve_alert(sku) + if sku in batch_skus or token is None: logger.info(f"[Dispatcher] SKU {t.sku} 在 {int(_ALERT_TTL_SEC/3600)}h 內已告警，觸發去重跳過。") skipped += 1 continue + batch_skus.add(sku) + reservation_tokens[sku] = token gate_a = t.sales_7d_delta_pct <= -95 # 絕對斷崖 gate_b = t.sales_7d_delta_pct <= -80 and abs(t.gap_pct) < 5 # 中度斷崖 + 微價差 @@ -1794,6 +2355,7 @@ class NemotronDispatcher: ) hr_footprint = _build_footprint_block(hermes_stats, None) for t in forced_review: + token = reservation_tokens.get(str(t.sku)) # 依閘門類型組 concern 文字 if t.sales_7d_delta_pct <= -95: concern_text = ( @@ -1808,8 +2370,12 @@ class NemotronDispatcher: "疑似缺貨、下架、或前台異常，請營運人員立即走查。" ) try: + if not token or not _refresh_alert_reservation(t.sku, token): + raise RuntimeError("alert_reservation_ownership_lost") impact = _compute_business_impact(t) - self._exec_flag_for_human_review( + if not _begin_alert_side_effect(t.sku, token): + raise RuntimeError("alert_side_effect_boundary_rejected") + outcome = self._exec_flag_for_human_review( sku=t.sku, name=t.name, concern=concern_text, @@ -1823,12 +2389,36 @@ class NemotronDispatcher: recommended_price=impact["recommended_price"], **_threat_match_metadata(t), ) + _require_durable_handler_outcome( + outcome, + "flag_for_human_review", + ) + _record_dedupe_commit_after_durable_side_effect( + t.sku, + token, + "flag_for_human_review", + errors, + ) dispatched += 1 except Exception as e: + _release_alert_reservation(t.sku, token) + skipped += 1 errors.append(f"forced_review({t.sku}): {e}") logger.error(f"[Dispatcher] 防線二執行失敗 {t.sku}: {e}") - # 若全部被防線二攔截，直接回傳（不浪費 NIM 配額） + # Forced-review 可能先消耗時間；進模型鏈前續租其餘 SKU，避免批次前段 + # 的處理時間侵蝕後段 reservation lease。 + renewed_candidates = [] + for t in nim_candidates: + token = reservation_tokens.get(str(t.sku)) + if token and _refresh_alert_reservation(t.sku, token): + renewed_candidates.append(t) + continue + errors.append(f"reservation({t.sku}): alert_reservation_ownership_lost") + skipped += 1 + nim_candidates = renewed_candidates + + # 若全部被防線二攔截或 reservation 已失效，直接回傳（不浪費 NIM 配額） if not nim_candidates: logger.info(f"[Dispatcher] 全部 {len(forced_review)} 筆由防線二處理（或去重），不呼叫 NIM") return { @@ -1838,6 +2428,10 @@ class NemotronDispatcher: "nim_stats": {}, } + decision_deadline = ( + time.monotonic() + NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC + ) + # ── Operation Ollama-First v5.0 / Phase 3 / A9：qwen3 主路徑（feature flag 灰度）── # NEMOTRON_OLLAMA_FIRST=false 時不進入此分支，僅作緊急退路。 # 若 qwen3 成功取得 tool_calls，沿用既有 TOOL_MAP 執行邏輯（共用 footprint/threat 注入）。 @@ -1845,9 +2439,13 @@ class NemotronDispatcher: qwen3_used = False qwen3_stats: Optional[dict] = None qwen3_tool_calls: Optional[list] = None + qwen3_failure: dict = {} if NEMOTRON_OLLAMA_FIRST: try: - qwen3_tool_calls, qwen3_stats = self._call_qwen3_dispatch(nim_candidates) + qwen3_tool_calls, qwen3_stats = self._call_qwen3_dispatch( + nim_candidates, + deadline_monotonic=decision_deadline, + ) if qwen3_tool_calls: qwen3_used = True logger.info( @@ -1856,38 +2454,59 @@ class NemotronDispatcher: ) else: logger.warning("[Dispatcher][qwen3] 0 tool_calls，fallback 至 NIM") + qwen3_failure = { + "attempt_count": len((qwen3_stats or {}).get("attempted_candidates") or []), + "deadline_exhausted": time.monotonic() >= decision_deadline, + "error": "ollama_chain_returned_no_valid_tool_calls", + "attempted_candidates": (qwen3_stats or {}).get("attempted_candidates") or [], + } except Exception as e: logger.warning(f"[Dispatcher][qwen3] 呼叫失敗 fallback NIM: {e}") # log_ai_call 已在 _call_qwen3_dispatch 內標記 status=error + fallback_to=nim + attempts = list(getattr(e, "attempted_candidates", []) or []) + qwen3_failure = { + "attempt_count": len(attempts), + "deadline_exhausted": time.monotonic() >= decision_deadline, + "error": f"{type(e).__name__}: {str(e)[:180]}", + "attempted_candidates": attempts, + } qwen3_tool_calls = None qwen3_stats = None + upstream_stats = ( + {"upstream_ollama_failure": qwen3_failure} + if qwen3_failure + else {} + ) + # qwen3 主路徑成功 → 直接進入工具執行區塊（跳過 NIM） if qwen3_used: tool_calls = qwen3_tool_calls - # 與既有 NIM 路徑一致的 stats 結構（footprint 顯示用） + # 保留候選、降級與實際模型資訊，供通知、DB 足跡與 runtime readback 使用。 nim_stats = { - "total_tokens": qwen3_stats.get("total_tokens", 0), - "quota_used": _nim_quota_used(), # 配額未動用 - "provider": qwen3_stats.get("provider", "gcp_ollama"), - "model": qwen3_stats.get("model", NEMOTRON_OLLAMA_MODEL), - "host": qwen3_stats.get("host"), - "host_label": qwen3_stats.get("host_label"), + **(qwen3_stats or {}), + "quota_used": _nim_quota_used(), # NIM 配額未動用 } return self._execute_tool_calls( tool_calls=tool_calls, - threats=threats, + threats=nim_candidates, hermes_stats=hermes_stats, nim_stats=nim_stats, pre_dispatched=dispatched, pre_skipped=skipped, pre_errors=errors, + reservation_tokens=reservation_tokens, ) # ── 進入 NIM 路徑（flag=false 緊急主路徑；flag=true 則為 qwen3 失敗備援）── if not NIM_API_KEY: logger.warning("[Dispatcher][ADR-004] NVIDIA_API_KEY 未設定，啟動 Hermes 規則引擎降級") - fb = self._hermes_rule_fallback(nim_candidates, hermes_stats) + fb = self._hermes_rule_fallback( + nim_candidates, + hermes_stats, + upstream_stats=upstream_stats, + reservation_tokens=reservation_tokens, + ) return { "dispatched": dispatched + fb["dispatched"], "skipped": skipped + fb["skipped"], @@ -1897,7 +2516,12 @@ class NemotronDispatcher: if not _check_nim_quota(): logger.warning("[Dispatcher][ADR-004] NIM 配額耗盡，啟動 Hermes 規則引擎降級") - fb = self._hermes_rule_fallback(nim_candidates, hermes_stats) + fb = self._hermes_rule_fallback( + nim_candidates, + hermes_stats, + upstream_stats=upstream_stats, + reservation_tokens=reservation_tokens, + ) return { "dispatched": dispatched + fb["dispatched"], "skipped": skipped + fb["skipped"], @@ -1906,10 +2530,19 @@ class NemotronDispatcher: } try: - tool_calls, nim_stats = self._call_nim(nim_candidates) + tool_calls, nim_stats = self._call_nim( + nim_candidates, + deadline_monotonic=decision_deadline, + ) + nim_stats = {**nim_stats, **upstream_stats} if not tool_calls: logger.warning("[Dispatcher][ADR-004] NIM 0 tool_calls，啟動 Hermes 規則引擎降級") - fb = self._hermes_rule_fallback(nim_candidates, hermes_stats) + fb = self._hermes_rule_fallback( + nim_candidates, + hermes_stats, + upstream_stats=upstream_stats, + reservation_tokens=reservation_tokens, + ) return { "dispatched": dispatched + fb["dispatched"], "skipped": skipped + fb["skipped"], @@ -1919,7 +2552,12 @@ class NemotronDispatcher: except requests.HTTPError as e: if e.response is not None and e.response.status_code == 429: logger.warning("[Dispatcher][ADR-004] NIM HTTP 429，啟動 Hermes 規則引擎降級") - fb = self._hermes_rule_fallback(nim_candidates, hermes_stats) + fb = self._hermes_rule_fallback( + nim_candidates, + hermes_stats, + upstream_stats=upstream_stats, + reservation_tokens=reservation_tokens, + ) return { "dispatched": dispatched + fb["dispatched"], "skipped": skipped + fb["skipped"], @@ -1927,7 +2565,12 @@ class NemotronDispatcher: "nim_stats": fb["nim_stats"], } logger.warning("[Dispatcher][ADR-004] NIM HTTP 錯誤，啟動 Hermes 規則引擎降級: %s", e) - fb = self._hermes_rule_fallback(nim_candidates, hermes_stats) + fb = self._hermes_rule_fallback( + nim_candidates, + hermes_stats, + upstream_stats=upstream_stats, + reservation_tokens=reservation_tokens, + ) return { "dispatched": dispatched + fb["dispatched"], "skipped": skipped + fb["skipped"], @@ -1936,7 +2579,12 @@ class NemotronDispatcher: } except Exception as e: logger.warning("[Dispatcher][ADR-004] NIM 呼叫失敗，啟動 Hermes 規則引擎降級: %s", e) - fb = self._hermes_rule_fallback(nim_candidates, hermes_stats) + fb = self._hermes_rule_fallback( + nim_candidates, + hermes_stats, + upstream_stats=upstream_stats, + reservation_tokens=reservation_tokens, + ) return { "dispatched": dispatched + fb["dispatched"], "skipped": skipped + fb["skipped"], @@ -1946,12 +2594,13 @@ class NemotronDispatcher: return self._execute_tool_calls( tool_calls=tool_calls, - threats=threats, + threats=nim_candidates, hermes_stats=hermes_stats, nim_stats=nim_stats, pre_dispatched=dispatched, pre_skipped=skipped, pre_errors=errors, + reservation_tokens=reservation_tokens, ) # ────────────────────────────────────────────── @@ -1966,17 +2615,21 @@ class NemotronDispatcher: pre_dispatched: int = 0, pre_skipped: int = 0, pre_errors: Optional[list] = None, + reservation_tokens: Optional[dict] = None, ) -> dict: """執行 LLM 回傳的 tool_calls 清單，注入 Python 獨裁的客觀數字 + 金額影響。 被 NIM 路徑與 qwen3 路徑共用，避免雙路雙維護。 """ errors = list(pre_errors or []) dispatched = pre_dispatched + batch_total = pre_dispatched + pre_skipped + len(threats) footprint_text = _build_footprint_block(hermes_stats, nim_stats) footprint_data = _build_footprint_json(hermes_stats, nim_stats) - threat_map = {t.sku: t for t in threats} + threat_map = {str(t.sku): t for t in threats} + owned_tokens = dict(reservation_tokens or {}) + executed_skus = set() TOOL_MAP = { "trigger_price_alert": self._exec_trigger_price_alert, @@ -1990,14 +2643,25 @@ class NemotronDispatcher: tool_name = tc.get("tool") args = dict(tc.get("args", {}) or {}) handler = TOOL_MAP.get(tool_name) + sku = str(args.get("sku") or "").strip() + token = owned_tokens.get(sku) if not handler: + _release_alert_reservation(sku, token) errors.append(f"未知工具: {tool_name}") continue args["footprint"] = footprint_text - t = threat_map.get(args.get("sku")) + t = threat_map.get(sku) + if t is None: + _release_alert_reservation(sku, token) + errors.append(f"工具 SKU 不在輸入範圍: {sku or 'missing'}") + continue + if sku in executed_skus: + _release_alert_reservation(sku, token) + errors.append(f"同一 SKU 重複工具呼叫: {sku}") + continue if tool_name == "trigger_price_alert" and t and not _can_direct_price_alert(t): match_meta = _threat_match_metadata(t) tool_name = "flag_for_human_review" @@ -2040,13 +2704,29 @@ class NemotronDispatcher: args["threat"] = t try: - handler(**args) + if not token or not _refresh_alert_reservation(sku, token): + raise RuntimeError("alert_reservation_ownership_lost") + if not _begin_alert_side_effect(sku, token): + raise RuntimeError("alert_side_effect_boundary_rejected") + outcome = handler(**args) + _require_durable_handler_outcome(outcome, tool_name) + _record_dedupe_commit_after_durable_side_effect( + sku, + token, + tool_name, + errors, + ) + executed_skus.add(sku) dispatched += 1 except Exception as e: + _release_alert_reservation(sku, token) errors.append(f"{tool_name}({args.get('sku', '?')}): {e}") logger.error(f"[Dispatcher] 工具執行失敗 [{tool_name}]: {e}") - skipped = max(0, len(threats) - dispatched) + for sku in threat_map: + if sku not in executed_skus: + _release_alert_reservation(sku, owned_tokens.get(sku)) + skipped = max(0, batch_total - dispatched) # nim_stats 在 qwen3 路徑下會帶 provider='gcp_ollama'，log 出處可區辨 provider = nim_stats.get("provider", "nim") if isinstance(nim_stats, dict) else "nim" logger.info( diff --git a/services/nemotron_decision_canary_service.py b/services/nemotron_decision_canary_service.py index 27b8095..16e79e9 100644 --- a/services/nemotron_decision_canary_service.py +++ b/services/nemotron_decision_canary_service.py @@ -1,8 +1,9 @@ """Decision-only NemoTron runtime canary. The canary exercises the production qwen3 tool-calling payload against an -approved GCP Ollama host, validates the returned decision, and stops before any -tool, Telegram, database, price, or insight write is allowed. +exact-digest GCP-first candidate chain with a bounded 111 fallback, validates +the returned decision, and stops before any tool, Telegram, database, price, or +insight write is allowed. """ from __future__ import annotations @@ -18,24 +19,30 @@ from typing import Any, Mapping import requests +from services.nemotron_runtime_candidate_service import ( + FALLBACK_EXPECTED_DIGEST, + FALLBACK_MODEL, + IDENTITY_TIMEOUT_SEC, + PRIMARY_EXPECTED_DIGEST, + PRIMARY_MODEL, + NemotronRuntimeCandidate, + build_nemotron_runtime_candidates, + inspect_nemotron_model_identity, +) from services.ollama_service import OLLAMA_HOST_PRIMARY, OLLAMA_HOST_SECONDARY POLICY = "controlled_nemotron_decision_only_canary_v1" CANARY_VERSION = "nemotron_decision_only_canary_v1" -NEMOTRON_OLLAMA_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b") -NEMOTRON_OLLAMA_EXPECTED_DIGEST = os.getenv( - "NEMOTRON_OLLAMA_EXPECTED_DIGEST", - "bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8", -).strip().lower() +NEMOTRON_OLLAMA_MODEL = PRIMARY_MODEL +NEMOTRON_OLLAMA_EXPECTED_DIGEST = PRIMARY_EXPECTED_DIGEST +NEMOTRON_OLLAMA_FALLBACK_MODEL = FALLBACK_MODEL +NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST = FALLBACK_EXPECTED_DIGEST DEFAULT_TIMEOUT_SEC = max( 30, min(int(os.getenv("NEMOTRON_DECISION_CANARY_TIMEOUT_SEC", "300")), 600), ) -DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC = max( - 2, - min(int(os.getenv("NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC", "10")), 30), -) +DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC = IDENTITY_TIMEOUT_SEC DEFAULT_MAX_AGE_HOURS = max( 1, min(int(os.getenv("NEMOTRON_DECISION_CANARY_MAX_AGE_HOURS", "26")), 168), @@ -86,52 +93,21 @@ def _parse_datetime(value: Any) -> datetime | None: def _model_identity(host: str) -> dict[str, Any]: - started = time.monotonic() - try: - response = requests.get( - f"{host.rstrip('/')}/api/tags", - timeout=DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC, - ) - response.raise_for_status() - models = response.json().get("models") or [] - except Exception as exc: - return { - "ok": False, - "host": host, - "model": NEMOTRON_OLLAMA_MODEL, - "digest": None, - "digest_matches": False, - "elapsed_ms": round((time.monotonic() - started) * 1000), - "error": f"{type(exc).__name__}: {str(exc)[:180]}", - } - - target = NEMOTRON_OLLAMA_MODEL - target_without_latest = target.removesuffix(":latest") - matched: Mapping[str, Any] = {} - for item in models: - names = { - str(item.get("name") or "").strip(), - str(item.get("model") or "").strip(), - } - if target in names or target_without_latest in names: - matched = item - break - digest = str(matched.get("digest") or "").strip().lower() - digest_matches = bool(digest) and digest == NEMOTRON_OLLAMA_EXPECTED_DIGEST - return { - "ok": bool(matched) and digest_matches, - "host": host, - "model": target, - "digest": digest or None, - "expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST, - "digest_matches": digest_matches, - "parameter_size": str((matched.get("details") or {}).get("parameter_size") or ""), - "quantization_level": str( - (matched.get("details") or {}).get("quantization_level") or "" - ), - "elapsed_ms": round((time.monotonic() - started) * 1000), - "error": None if matched else f"model_not_found:{target}", - } + candidate = NemotronRuntimeCandidate( + label="compatibility_identity", + tier="primary", + host=host.rstrip("/"), + model=NEMOTRON_OLLAMA_MODEL, + expected_digest=NEMOTRON_OLLAMA_EXPECTED_DIGEST, + request_timeout_sec=DEFAULT_TIMEOUT_SEC, + num_predict=160, + num_ctx=None, + ) + return inspect_nemotron_model_identity( + candidate, + request_get=requests.get, + timeout_sec=DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC, + ) def _latest_execution(root: Path, *, now: datetime | None = None) -> dict[str, Any]: @@ -165,7 +141,12 @@ def _latest_execution(root: Path, *, now: datetime | None = None) -> dict[str, A "canary_passed": payload.get("canary_passed") is True, "selected_host": payload.get("selected_host"), "model": payload.get("model"), + "expected_digest": payload.get("expected_digest"), "observed_digest": payload.get("observed_digest"), + "selected_candidate_label": payload.get("selected_candidate_label"), + "fallback_used": payload.get("fallback_used") is True, + "degraded_runtime": payload.get("degraded_runtime") is True, + "attempt_count": int(payload.get("attempt_count") or 0), "decision_tool": payload.get("decision_tool"), "decision_sku": payload.get("decision_sku"), "model_elapsed_ms": payload.get("model_elapsed_ms"), @@ -283,67 +264,204 @@ def run_nemotron_decision_canary( ), } - host_preflights = [ - _model_identity(OLLAMA_HOST_PRIMARY), - _model_identity(OLLAMA_HOST_SECONDARY), - ] - selected = next((item for item in host_preflights if item.get("ok")), None) + from services.nemoton_dispatcher_service import ( + ToolCallContractError, + _parse_content_fallback, + _parse_tool_calls_struct, + _remaining_timeout, + _validate_tool_call_contract, + build_qwen3_dispatch_payload, + ) + + candidates = build_nemotron_runtime_candidates( + primary_model=NEMOTRON_OLLAMA_MODEL, + primary_expected_digest=NEMOTRON_OLLAMA_EXPECTED_DIGEST, + fallback_model=NEMOTRON_OLLAMA_FALLBACK_MODEL, + fallback_expected_digest=NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST, + ) + timeout_cap = max(30, min(int(timeout_sec or DEFAULT_TIMEOUT_SEC), 600)) + canary_started = time.monotonic() + deadline_monotonic = canary_started + timeout_cap + host_preflights: list[dict[str, Any]] = [] + runtime_attempts: list[dict[str, Any]] = [] + selected: dict[str, Any] | None = None + selected_candidate: NemotronRuntimeCandidate | None = None + post_identity: dict[str, Any] = {} model_call_performed = False model_elapsed_ms = 0 decisions: list[dict[str, Any]] = [] decision_format = "none" error: str | None = None - if selected is None: - error = "no_approved_gcp_host_with_expected_nemotron_digest" - else: - from services.nemoton_dispatcher_service import ( - _parse_content_fallback, - _parse_tool_calls_struct, - build_qwen3_dispatch_payload, + expected_tool = "trigger_price_alert" + threat = _SyntheticThreat() + for candidate in candidates: + attempt: dict[str, Any] = { + "candidate": candidate.as_public_dict(), + "identity": {}, + "model_call_performed": False, + "model_elapsed_ms": 0, + "decision_format": "none", + "decision_count": 0, + "status": "identity_failed", + "error": None, + } + try: + identity_timeout = _remaining_timeout( + deadline_monotonic, + DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC, + ) + except requests.Timeout as exc: + attempt["status"] = "deadline_exhausted" + attempt["error"] = str(exc) + runtime_attempts.append(attempt) + error = str(exc) + break + identity = inspect_nemotron_model_identity( + candidate, + request_get=requests.get, + timeout_sec=identity_timeout, ) + host_preflights.append(identity) + attempt["identity"] = identity + attempt["error"] = identity.get("error") + if identity.get("ok") is not True: + runtime_attempts.append(attempt) + error = str(identity.get("error") or "model identity failed") + continue - threat = _SyntheticThreat() payload = build_qwen3_dispatch_payload( [threat], mcp_context="controlled decision-only canary; no live market data", - num_predict=160, + model=candidate.model, + num_predict=candidate.num_predict, + num_ctx=candidate.num_ctx, keep_alive=0, ) - payload["think"] = False started = time.monotonic() model_call_performed = True + attempt["model_call_performed"] = True + candidate_decisions: list[dict[str, Any]] = [] + candidate_decision_format = "none" + candidate_error: str | None = None + candidate_contract_error: str | None = None + candidate_contract_valid = False try: + request_timeout = _remaining_timeout( + deadline_monotonic, + candidate.request_timeout_sec, + ) + attempt["request_timeout_sec"] = round(request_timeout, 3) response = requests.post( - f"{str(selected['host']).rstrip('/')}/api/chat", + f"{candidate.host}/api/chat", json=payload, - timeout=max(30, min(int(timeout_sec or DEFAULT_TIMEOUT_SEC), 600)), + timeout=request_timeout, ) response.raise_for_status() body = response.json() message = body.get("message") or {} - decisions = _parse_tool_calls_struct(message.get("tool_calls") or []) - if decisions: - decision_format = "tool_calls" + candidate_decisions = _parse_tool_calls_struct( + message.get("tool_calls") or [] + ) + if candidate_decisions: + candidate_decision_format = "tool_calls" else: - decisions = _parse_content_fallback(str(message.get("content") or "")) - if decisions: - decision_format = "content_fallback" + candidate_decisions = _parse_content_fallback( + str(message.get("content") or "") + ) + if candidate_decisions: + candidate_decision_format = "content_fallback" + try: + candidate_decisions = _validate_tool_call_contract( + candidate_decisions, + [threat], + ) + candidate_contract_valid = True + except ToolCallContractError as exc: + candidate_contract_error = str(exc)[:240] except Exception as exc: - error = f"{type(exc).__name__}: {str(exc)[:240]}" + candidate_error = f"{type(exc).__name__}: {str(exc)[:240]}" finally: - model_elapsed_ms = round((time.monotonic() - started) * 1000) + elapsed_ms = round((time.monotonic() - started) * 1000) + model_elapsed_ms += elapsed_ms + attempt["model_elapsed_ms"] = elapsed_ms + + candidate_decision = candidate_decisions[0] if candidate_decisions else {} + candidate_args = ( + candidate_decision.get("args") + if isinstance(candidate_decision.get("args"), dict) + else {} + ) + candidate_tool = str(candidate_decision.get("tool") or "") + candidate_sku = str((candidate_args or {}).get("sku") or "") + candidate_post_identity: dict[str, Any] = {} + if candidate_error is None: + try: + post_identity_timeout = _remaining_timeout( + deadline_monotonic, + DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC, + ) + candidate_post_identity = inspect_nemotron_model_identity( + candidate, + request_get=requests.get, + timeout_sec=post_identity_timeout, + ) + except requests.Timeout as exc: + candidate_error = f"{type(exc).__name__}: {str(exc)[:240]}" + candidate_checks = { + "approved_host_selected": True, + "expected_digest_verified_before": identity.get("digest_matches") is True, + "model_call_completed": candidate_error is None, + "decision_present": bool(candidate_decisions), + "decision_contract_valid": candidate_contract_valid, + "decision_tool_allowed": candidate_tool in APPROVED_TOOLS, + "expected_decision_tool_selected": candidate_tool == expected_tool, + "synthetic_sku_preserved": candidate_sku == _SyntheticThreat.sku, + "expected_digest_stable_after": ( + candidate_post_identity.get("digest_matches") is True + ), + "tool_execution_absent": True, + "database_call_absent": True, + "telegram_send_absent": True, + } + attempt.update({ + "decision_format": candidate_decision_format, + "decision_count": len(candidate_decisions), + "decision_tool": candidate_tool or None, + "decision_sku": candidate_sku or None, + "post_model_identity": candidate_post_identity, + "checks": candidate_checks, + "status": "passed" if all(candidate_checks.values()) else "failed", + "error": candidate_error or candidate_contract_error, + "contract_error": candidate_contract_error, + }) + runtime_attempts.append(attempt) + if all(candidate_checks.values()): + selected = identity + selected_candidate = candidate + post_identity = candidate_post_identity + decisions = candidate_decisions + decision_format = candidate_decision_format + error = None + break + error = candidate_error or "candidate decision contract failed" + + if selected_candidate is None: + error = ( + "decision_total_deadline_exhausted" + if time.monotonic() >= deadline_monotonic + else "all_approved_model_candidates_failed" + ) decision = decisions[0] if decisions else {} decision_tool = str(decision.get("tool") or "") decision_args = decision.get("args") if isinstance(decision.get("args"), dict) else {} decision_sku = str((decision_args or {}).get("sku") or "") - expected_tool = "trigger_price_alert" - post_identity = _model_identity(str(selected["host"])) if selected else {} checks = { - "approved_host_selected": selected is not None, + "approved_host_selected": selected_candidate is not None, "expected_digest_verified_before": bool(selected and selected.get("digest_matches")), - "model_call_completed": model_call_performed and error is None, + "model_call_completed": selected_candidate is not None and error is None, "decision_present": bool(decisions), + "decision_contract_valid": selected_candidate is not None, "decision_tool_allowed": decision_tool in APPROVED_TOOLS, "expected_decision_tool_selected": decision_tool == expected_tool, "synthetic_sku_preserved": decision_sku == _SyntheticThreat.sku, @@ -353,7 +471,22 @@ def run_nemotron_decision_canary( "telegram_send_absent": True, } canary_passed = all(checks.values()) - status = "canary_passed" if canary_passed else "canary_failed" + fallback_used = bool(selected_candidate and selected_candidate.is_fallback) + status = ( + "canary_passed_degraded_fallback" + if canary_passed and fallback_used + else "canary_passed" + if canary_passed + else "canary_failed" + ) + selected_model = ( + selected_candidate.model if selected_candidate else NEMOTRON_OLLAMA_MODEL + ) + selected_expected_digest = ( + selected_candidate.expected_digest + if selected_candidate + else NEMOTRON_OLLAMA_EXPECTED_DIGEST + ) execution_receipt: dict[str, Any] = { "policy": POLICY, "canary_version": CANARY_VERSION, @@ -361,14 +494,24 @@ def run_nemotron_decision_canary( "run_identity": run_identity, "status": status, "canary_passed": canary_passed, - "model": NEMOTRON_OLLAMA_MODEL, - "expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST, + "model": selected_model, + "expected_digest": selected_expected_digest, "observed_digest": selected.get("digest") if selected else None, "selected_host": selected.get("host") if selected else None, + "selected_candidate_label": ( + selected_candidate.label if selected_candidate else None + ), + "fallback_used": fallback_used, + "degraded_runtime": fallback_used, "host_preflights": host_preflights, + "runtime_attempts": runtime_attempts, + "attempt_count": len(runtime_attempts), "post_model_identity": post_identity, "model_call_performed": model_call_performed, "model_elapsed_ms": model_elapsed_ms, + "timeout_budget_sec": timeout_cap, + "total_elapsed_ms": round((time.monotonic() - canary_started) * 1000), + "deadline_exhausted": time.monotonic() >= deadline_monotonic, "decision_format": decision_format, "decision_count": len(decisions), "decision_tool": decision_tool or None, @@ -388,8 +531,8 @@ def run_nemotron_decision_canary( "error": error, "rollback_terminal": "decision_only_no_tool_or_data_write", "source_of_truth_diff": { - "expected_model": NEMOTRON_OLLAMA_MODEL, - "expected_digest": NEMOTRON_OLLAMA_EXPECTED_DIGEST, + "expected_model": selected_model, + "expected_digest": selected_expected_digest, "observed_digest": selected.get("digest") if selected else None, "expected_decision_tool": expected_tool, "observed_decision_tool": decision_tool or None, @@ -400,7 +543,7 @@ def run_nemotron_decision_canary( "source_of_truth_diff_recorded": True, "ai_candidate_decision_recorded": bool(decisions), "risk_policy_decision": "medium_decision_only_no_tool_execution", - "check_mode_passed": bool(selected), + "check_mode_passed": canary_passed, "bounded_execution_performed": model_call_performed, "independent_verifier_passed": canary_passed, "rollback_or_no_write_terminal": "decision_only_no_tool_or_data_write", @@ -444,7 +587,9 @@ def run_nemotron_decision_canary( "rollback_terminal": "decision_only_no_tool_or_data_write", }, "next_machine_action": ( - "continue_scheduled_nemotron_decision_canary" + "restore_gcp_decision_runtime_capacity" + if canary_passed and fallback_used + else "continue_scheduled_nemotron_decision_canary" if canary_passed else "repair_nemotron_model_runtime_then_retry_decision_only_canary" ), diff --git a/services/nemotron_dispatch_reservation_service.py b/services/nemotron_dispatch_reservation_service.py new file mode 100644 index 0000000..7c2ef8b --- /dev/null +++ b/services/nemotron_dispatch_reservation_service.py @@ -0,0 +1,376 @@ +"""Process-shared reservation store for NemoTron dispatch side effects.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +import uuid +from pathlib import Path +from typing import Callable + +try: + import fcntl +except ImportError: # pragma: no cover - exercised by compatibility monkeypatch + fcntl = None + + +logger = logging.getLogger(__name__) + +STATE_VERSION = 1 +PRODUCTION_STATE_PATH = Path( + "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json" +) + + +def production_shared_reservation_required() -> bool: + return ( + os.getenv("FLASK_ENV", "").strip().lower() in {"prod", "production"} + or os.getenv("USE_POSTGRESQL", "").strip().lower() == "true" + ) + + +def _sku_key(sku: str) -> str: + value = str(sku or "").strip() + if not value: + return "" + return hashlib.sha256(f"nemotron-dispatch:{value}".encode("utf-8")).hexdigest() + + +class ReservationStateError(RuntimeError): + pass + + +class SharedFileReservationStore: + """Atomic lease/commit state shared by processes using the same bind mount.""" + + def __init__( + self, + state_path: str | Path, + *, + committed_ttl_sec: int, + lease_ttl_sec: int, + clock: Callable[[], float] | None = None, + ): + self.state_path = Path(state_path) + self.lock_path = self.state_path.with_suffix(self.state_path.suffix + ".lock") + self.committed_ttl_sec = max(1, int(committed_ttl_sec)) + self.lease_ttl_sec = max(1, int(lease_ttl_sec)) + self.clock = clock or time.time + + @staticmethod + def _empty_state() -> dict: + return {"version": STATE_VERSION, "leases": {}, "committed": {}} + + def _read_state(self) -> dict: + if not self.state_path.exists(): + return self._empty_state() + try: + payload = json.loads(self.state_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ReservationStateError("reservation_state_unreadable") from exc + if not isinstance(payload, dict) or payload.get("version") != STATE_VERSION: + raise ReservationStateError("reservation_state_schema_invalid") + leases = payload.get("leases") + committed = payload.get("committed") + if not isinstance(leases, dict) or not isinstance(committed, dict): + raise ReservationStateError("reservation_state_schema_invalid") + for lease in leases.values(): + if ( + not isinstance(lease, dict) + or not isinstance(lease.get("token"), str) + or not isinstance(lease.get("expires_at"), (int, float)) + or lease.get("phase", "reserved") not in { + "reserved", + "side_effect_started", + } + ): + raise ReservationStateError("reservation_lease_schema_invalid") + for marker in committed.values(): + if ( + not isinstance(marker, dict) + or not isinstance(marker.get("until"), (int, float)) + ): + raise ReservationStateError("reservation_commit_schema_invalid") + return payload + + def _write_state(self, payload: dict) -> None: + temporary = self.state_path.with_name( + f".{self.state_path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + ) + data = json.dumps(payload, sort_keys=True, separators=(",", ":")) + try: + with temporary.open("w", encoding="utf-8") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary, 0o640) + os.replace(temporary, self.state_path) + directory_fd = os.open(str(self.state_path.parent), os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + + @staticmethod + def _cleanup(payload: dict, now: float) -> bool: + changed = False + for key in [ + key + for key, lease in payload["leases"].items() + if float(lease["expires_at"]) <= now + ]: + del payload["leases"][key] + changed = True + for key in [ + key + for key, marker in payload["committed"].items() + if float(marker["until"]) <= now + ]: + del payload["committed"][key] + changed = True + return changed + + def _locked(self, operation): + if fcntl is None: + raise ReservationStateError("process_shared_lock_unavailable") + self.state_path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) + with self.lock_path.open("a+", encoding="utf-8") as lock_handle: + os.chmod(self.lock_path, 0o640) + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX) + try: + payload = self._read_state() + now = float(self.clock()) + cleanup_changed = self._cleanup(payload, now) + result, operation_changed = operation(payload, now) + if cleanup_changed or operation_changed: + self._write_state(payload) + return result + finally: + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN) + + def reserve(self, sku: str) -> str | None: + key = _sku_key(sku) + if not key: + return None + + def operation(payload, now): + if key in payload["leases"] or key in payload["committed"]: + return None, False + token = uuid.uuid4().hex + payload["leases"][key] = { + "token": token, + "expires_at": now + self.lease_ttl_sec, + "phase": "reserved", + } + return token, True + + try: + return self._locked(operation) + except Exception as exc: + logger.error( + "[NemotronReservation] shared reserve failed closed: %s", + type(exc).__name__, + ) + return None + + def refresh(self, sku: str, token: str) -> bool: + key = _sku_key(sku) + + def operation(payload, now): + lease = payload["leases"].get(key) + if not lease or lease.get("token") != str(token): + return False, False + ttl = ( + self.committed_ttl_sec + if lease.get("phase") == "side_effect_started" + else self.lease_ttl_sec + ) + lease["expires_at"] = now + ttl + return True, True + + try: + return bool(key and self._locked(operation)) + except Exception as exc: + logger.error( + "[NemotronReservation] shared refresh failed closed: %s", + type(exc).__name__, + ) + return False + + def begin_side_effect(self, sku: str, token: str) -> bool: + """Persist the at-most-once boundary before invoking an external effect.""" + key = _sku_key(sku) + + def operation(payload, now): + lease = payload["leases"].get(key) + if not lease or lease.get("token") != str(token): + return False, False + lease["phase"] = "side_effect_started" + lease["expires_at"] = now + self.committed_ttl_sec + return True, True + + try: + return bool(key and self._locked(operation)) + except Exception as exc: + logger.error( + "[NemotronReservation] shared side-effect boundary failed closed: %s", + type(exc).__name__, + ) + return False + + def commit(self, sku: str, token: str) -> bool: + key = _sku_key(sku) + + def operation(payload, now): + lease = payload["leases"].get(key) + if not lease or lease.get("token") != str(token): + return False, False + del payload["leases"][key] + payload["committed"][key] = { + "until": now + self.committed_ttl_sec, + } + return True, True + + try: + return bool(key and self._locked(operation)) + except Exception as exc: + logger.error( + "[NemotronReservation] shared commit failed closed: %s", + type(exc).__name__, + ) + return False + + def release(self, sku: str, token: str | None) -> bool: + key = _sku_key(sku) + + def operation(payload, _now): + lease = payload["leases"].get(key) + if not lease or lease.get("token") != str(token or ""): + return False, False + del payload["leases"][key] + return True, True + + try: + return bool(key and self._locked(operation)) + except Exception as exc: + logger.error( + "[NemotronReservation] shared release failed closed: %s", + type(exc).__name__, + ) + return False + + def is_duplicate(self, sku: str) -> bool: + key = _sku_key(sku) + if not key: + return True + + def operation(payload, _now): + return ( + key in payload["leases"] or key in payload["committed"], + False, + ) + + try: + return bool(self._locked(operation)) + except Exception as exc: + logger.error( + "[NemotronReservation] shared read failed closed: %s", + type(exc).__name__, + ) + return True + + +def build_production_shared_reservation_store( + *, + committed_ttl_sec: int, + lease_ttl_sec: int, +) -> SharedFileReservationStore | None: + if not production_shared_reservation_required(): + return None + return SharedFileReservationStore( + PRODUCTION_STATE_PATH, + committed_ttl_sec=committed_ttl_sec, + lease_ttl_sec=lease_ttl_sec, + ) + + +def run_shared_reservation_canary( + *, + state_path: str | Path = PRODUCTION_STATE_PATH, + committed_ttl_sec: int = 4 * 3600, + lease_ttl_sec: int = 15 * 60, +) -> dict: + """Exercise shared ownership semantics without business side effects.""" + first = SharedFileReservationStore( + state_path, + committed_ttl_sec=committed_ttl_sec, + lease_ttl_sec=lease_ttl_sec, + ) + second = SharedFileReservationStore( + state_path, + committed_ttl_sec=committed_ttl_sec, + lease_ttl_sec=lease_ttl_sec, + ) + synthetic_sku = f"CANARY-NEMOTRON-RESERVATION-{uuid.uuid4().hex}" + token = first.reserve(synthetic_sku) + competing_token = second.reserve(synthetic_sku) if token else None + stale_release_rejected = ( + second.release(synthetic_sku, "stale-canary-token") is False + if token + else False + ) + side_effect_boundary_persisted = ( + first.begin_side_effect(synthetic_sku, token) if token else False + ) + owner_release_succeeded = ( + first.release(synthetic_sku, token) if token else False + ) + terminal_clear = ( + first.is_duplicate(synthetic_sku) is False + if owner_release_succeeded + else False + ) + checks = { + "first_owner_reserved": bool(token), + "competing_owner_rejected": competing_token is None, + "stale_token_release_rejected": stale_release_rejected, + "side_effect_boundary_persisted": side_effect_boundary_persisted, + "owner_release_succeeded": owner_release_succeeded, + "terminal_state_clear": terminal_clear, + } + success = all(checks.values()) + return { + "success": success, + "status": "shared_reservation_canary_passed" if success else "shared_reservation_canary_failed", + "policy": "nemotron_process_shared_reservation_v1", + "state_path": str(Path(state_path)), + "checks": checks, + "check_count": len(checks), + "check_pass_count": sum(checks.values()), + "controlled_apply": { + "writes_shared_reservation_state": True, + "terminal_state_clear": terminal_clear, + "writes_database": False, + "sends_telegram": False, + "executes_business_tool": False, + "reads_secret": False, + }, + } + + +__all__ = [ + "PRODUCTION_STATE_PATH", + "ReservationStateError", + "SharedFileReservationStore", + "build_production_shared_reservation_store", + "production_shared_reservation_required", + "run_shared_reservation_canary", +] diff --git a/services/nemotron_runtime_candidate_service.py b/services/nemotron_runtime_candidate_service.py new file mode 100644 index 0000000..ff1ac56 --- /dev/null +++ b/services/nemotron_runtime_candidate_service.py @@ -0,0 +1,234 @@ +"""Digest-locked Ollama candidates for the decision agent runtime.""" + +from __future__ import annotations + +import os +import time +from dataclasses import asdict, dataclass +from typing import Any, Callable, Mapping + +import requests + +from services.ollama_service import ( + OLLAMA_HOST_FALLBACK, + OLLAMA_HOST_PRIMARY, + OLLAMA_HOST_SECONDARY, +) + + +PRIMARY_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b").strip() +PRIMARY_EXPECTED_DIGEST = os.getenv( + "NEMOTRON_OLLAMA_EXPECTED_DIGEST", + "bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8", +).strip().lower() +FALLBACK_MODEL = os.getenv( + "NEMOTRON_OLLAMA_FALLBACK_MODEL", + "qwen3:8b", +).strip() +FALLBACK_EXPECTED_DIGEST = os.getenv( + "NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST", + "500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41", +).strip().lower() +NEMOTRON_FALLBACK_NUM_CTX = 4096 +NEMOTRON_FALLBACK_NUM_PREDICT = 512 + + +def _bounded_int_env(name: str, default: int, minimum: int, maximum: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except (TypeError, ValueError): + value = default + return max(minimum, min(value, maximum)) + + +IDENTITY_TIMEOUT_SEC = _bounded_int_env( + "NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC", 10, 2, 30 +) +GCP_ATTEMPT_TIMEOUT_SEC = _bounded_int_env( + "NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC", 60, 30, 180 +) +FALLBACK_ATTEMPT_TIMEOUT_SEC = _bounded_int_env( + "NEMOTRON_111_ATTEMPT_TIMEOUT_SEC", 45, 20, 120 +) + + +@dataclass(frozen=True) +class NemotronRuntimeCandidate: + label: str + tier: str + host: str + model: str + expected_digest: str + request_timeout_sec: int + num_predict: int + num_ctx: int | None + + @property + def is_fallback(self) -> bool: + return self.tier == "fallback" + + def as_public_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["is_fallback"] = self.is_fallback + return payload + + +def build_nemotron_runtime_candidates( + *, + primary_model: str | None = None, + primary_expected_digest: str | None = None, + fallback_model: str | None = None, + fallback_expected_digest: str | None = None, +) -> tuple[NemotronRuntimeCandidate, ...]: + """Return the approved GCP-first, 111-final candidate chain.""" + resolved_primary_model = str(primary_model or PRIMARY_MODEL).strip() + resolved_primary_digest = str( + primary_expected_digest or PRIMARY_EXPECTED_DIGEST + ).strip().lower() + resolved_fallback_model = str(fallback_model or FALLBACK_MODEL).strip() + resolved_fallback_digest = str( + fallback_expected_digest or FALLBACK_EXPECTED_DIGEST + ).strip().lower() + return ( + NemotronRuntimeCandidate( + label="gcp_primary", + tier="primary", + host=OLLAMA_HOST_PRIMARY.rstrip("/"), + model=resolved_primary_model, + expected_digest=resolved_primary_digest, + request_timeout_sec=GCP_ATTEMPT_TIMEOUT_SEC, + num_predict=2048, + num_ctx=None, + ), + NemotronRuntimeCandidate( + label="gcp_secondary", + tier="secondary", + host=OLLAMA_HOST_SECONDARY.rstrip("/"), + model=resolved_primary_model, + expected_digest=resolved_primary_digest, + request_timeout_sec=GCP_ATTEMPT_TIMEOUT_SEC, + num_predict=2048, + num_ctx=None, + ), + NemotronRuntimeCandidate( + label="ollama_111_fallback", + tier="fallback", + host=OLLAMA_HOST_FALLBACK.rstrip("/"), + model=resolved_fallback_model, + expected_digest=resolved_fallback_digest, + request_timeout_sec=FALLBACK_ATTEMPT_TIMEOUT_SEC, + num_predict=NEMOTRON_FALLBACK_NUM_PREDICT, + num_ctx=NEMOTRON_FALLBACK_NUM_CTX, + ), + ) + + +def inspect_nemotron_model_identity( + candidate: NemotronRuntimeCandidate, + *, + request_get: Callable[..., Any] | None = None, + timeout_sec: int | None = None, +) -> dict[str, Any]: + """Verify an exact model digest without loading the model.""" + started = time.monotonic() + getter = request_get or requests.get + timeout = max(0.1, min(float(timeout_sec or IDENTITY_TIMEOUT_SEC), 30.0)) + try: + response = getter(f"{candidate.host}/api/tags", timeout=timeout) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, Mapping): + raise ValueError("malformed_tags_payload") + models = payload.get("models", []) + if models is None: + models = [] + if not isinstance(models, list): + raise ValueError("malformed_models_schema") + except Exception as exc: + return { + **candidate.as_public_dict(), + "ok": False, + "digest": None, + "digest_matches": False, + "elapsed_ms": round((time.monotonic() - started) * 1000), + "error": f"{type(exc).__name__}: {str(exc)[:180]}", + } + + target = candidate.model + target_without_latest = target.removesuffix(":latest") + matched: Mapping[str, Any] = {} + if any(not isinstance(item, Mapping) for item in models): + return { + **candidate.as_public_dict(), + "ok": False, + "digest": None, + "digest_matches": False, + "parameter_size": "", + "quantization_level": "", + "elapsed_ms": round((time.monotonic() - started) * 1000), + "error": "malformed_models_schema", + } + if any( + "details" in item and not isinstance(item.get("details"), Mapping) + for item in models + ): + return { + **candidate.as_public_dict(), + "ok": False, + "digest": None, + "digest_matches": False, + "parameter_size": "", + "quantization_level": "", + "elapsed_ms": round((time.monotonic() - started) * 1000), + "error": "malformed_model_details", + } + for item in models: + names = { + str(item.get("name") or "").strip(), + str(item.get("model") or "").strip(), + } + if target in names or target_without_latest in names: + matched = item + break + digest = str(matched.get("digest") or "").strip().lower() + raw_details = matched.get("details", {}) if matched else {} + malformed_details = bool(matched) and not isinstance(raw_details, Mapping) + details = raw_details if isinstance(raw_details, Mapping) else {} + digest_matches = ( + bool(digest) + and digest == candidate.expected_digest + and not malformed_details + ) + error = None + if not matched: + error = f"model_not_found:{target}" + elif malformed_details: + error = "malformed_model_details" + elif not digest_matches: + error = "model_digest_mismatch" + return { + **candidate.as_public_dict(), + "ok": bool(matched) and digest_matches, + "digest": digest or None, + "digest_matches": digest_matches, + "parameter_size": str(details.get("parameter_size") or ""), + "quantization_level": str(details.get("quantization_level") or ""), + "elapsed_ms": round((time.monotonic() - started) * 1000), + "error": error, + } + + +__all__ = [ + "FALLBACK_ATTEMPT_TIMEOUT_SEC", + "FALLBACK_EXPECTED_DIGEST", + "FALLBACK_MODEL", + "GCP_ATTEMPT_TIMEOUT_SEC", + "IDENTITY_TIMEOUT_SEC", + "NemotronRuntimeCandidate", + "NEMOTRON_FALLBACK_NUM_CTX", + "NEMOTRON_FALLBACK_NUM_PREDICT", + "PRIMARY_EXPECTED_DIGEST", + "PRIMARY_MODEL", + "build_nemotron_runtime_candidates", + "inspect_nemotron_model_identity", +] diff --git a/services/ollama_service.py b/services/ollama_service.py index f3fc312..8e97a07 100644 --- a/services/ollama_service.py +++ b/services/ollama_service.py @@ -47,6 +47,14 @@ def approved_ollama_env(name: str, default: str = '') -> str: return default +def _bounded_int_env(name: str, default: int, minimum: int, maximum: int) -> int: + try: + value = int(os.getenv(name, str(default))) + except (TypeError, ValueError): + value = default + return max(minimum, min(value, maximum)) + + # Ollama 設定 - 僅允許 GCP-A → GCP-B → 111 三主機 OLLAMA_HOST_PRIMARY = approved_ollama_env('OLLAMA_HOST_PRIMARY', 'http://34.87.90.216:11434') OLLAMA_HOST_SECONDARY = approved_ollama_env('OLLAMA_HOST_SECONDARY', 'http://34.21.145.224:11434') @@ -66,8 +74,8 @@ EMBED_GCP_FAILURE_COOLDOWN_SEC = int(os.getenv('OLLAMA_EMBED_GCP_FAILURE_COOLDOW EMBED_GCP_FAILURE_NOTICE_SEC = int(os.getenv('OLLAMA_EMBED_GCP_FAILURE_NOTICE_SEC', '30')) FALLBACK_111_KEEP_ALIVE = os.getenv('OLLAMA_111_KEEP_ALIVE', '5m') FALLBACK_111_MAX_TIMEOUT = int(os.getenv('OLLAMA_111_MAX_TIMEOUT', '20')) -FALLBACK_111_NUM_CTX = int(os.getenv('OLLAMA_111_NUM_CTX', '4096')) -FALLBACK_111_NUM_PREDICT = int(os.getenv('OLLAMA_111_NUM_PREDICT', '512')) +FALLBACK_111_NUM_CTX = _bounded_int_env('OLLAMA_111_NUM_CTX', 4096, 512, 4096) +FALLBACK_111_NUM_PREDICT = _bounded_int_env('OLLAMA_111_NUM_PREDICT', 512, 32, 512) FALLBACK_111_MODEL = os.getenv('OLLAMA_111_MODEL_FALLBACK', 'llama3.2:latest') FALLBACK_111_CIRCUIT_CACHE_SEC = int(os.getenv('OLLAMA_111_CIRCUIT_CACHE_SEC', '60')) FALLBACK_111_MODEL_PATTERNS = tuple( diff --git a/tests/test_nemotron_decision_canary_service.py b/tests/test_nemotron_decision_canary_service.py index 33ae54a..dba538c 100644 --- a/tests/test_nemotron_decision_canary_service.py +++ b/tests/test_nemotron_decision_canary_service.py @@ -3,6 +3,8 @@ import json import subprocess import sys +import pytest + class _Response: def __init__(self, payload, status_code=200): @@ -19,12 +21,18 @@ class _Response: return self._payload -def _tags(service, *, digest=None): +def _tags(service, *, model=None, digest=None): + resolved_model = model or service.NEMOTRON_OLLAMA_MODEL + resolved_digest = digest or ( + service.NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST + if resolved_model == service.NEMOTRON_OLLAMA_FALLBACK_MODEL + else service.NEMOTRON_OLLAMA_EXPECTED_DIGEST + ) return { "models": [{ - "name": service.NEMOTRON_OLLAMA_MODEL, - "model": service.NEMOTRON_OLLAMA_MODEL, - "digest": digest or service.NEMOTRON_OLLAMA_EXPECTED_DIGEST, + "name": resolved_model, + "model": resolved_model, + "digest": resolved_digest, "details": { "parameter_size": "14.8B", "quantization_level": "Q4_K_M", @@ -33,6 +41,22 @@ def _tags(service, *, digest=None): } +def _valid_canary_tool_call(): + return { + "function": { + "name": "trigger_price_alert", + "arguments": { + "sku": "CANARY-NEMO-001", + "name": "NemoTron 受控決策驗證品", + "gap_pct": 22.4, + "sales_delta": -35.0, + "action": "建立價格告警候選", + "confidence": 0.91, + }, + } + } + + def test_nemotron_canary_readback_never_calls_network(tmp_path, monkeypatch): from services import nemotron_decision_canary_service as service @@ -113,7 +137,8 @@ def test_nemotron_canary_executes_decision_only_on_verified_secondary( assert posted[0][0].endswith("/api/chat") assert posted[0][1]["keep_alive"] == 0 assert posted[0][1]["think"] is False - assert posted[0][2] == 45 + assert posted[0][1]["options"]["num_predict"] == 2048 + assert posted[0][2] == pytest.approx(45, abs=0.01) assert payload["latest_execution"]["canary_passed"] is True acknowledged = service.acknowledge_nemotron_decision_canary( @@ -157,12 +182,263 @@ def test_nemotron_canary_blocks_model_digest_drift_before_model_call( assert payload["status"] == "canary_failed" assert payload["execution"]["model_call_performed"] is False - assert payload["execution"]["error"] == ( - "no_approved_gcp_host_with_expected_nemotron_digest" + assert payload["execution"]["error"] == "all_approved_model_candidates_failed" + assert payload["execution"]["attempt_count"] == 3 + assert all( + attempt["status"] == "identity_failed" + for attempt in payload["execution"]["runtime_attempts"] ) assert model_calls == [] +def test_nemotron_canary_malformed_primary_tags_falls_through_secondary( + tmp_path, monkeypatch +): + from services import nemotron_decision_canary_service as service + + def fake_get(url, timeout): + if service.OLLAMA_HOST_PRIMARY in url: + return _Response({ + "models": [{ + "name": service.NEMOTRON_OLLAMA_MODEL, + "model": service.NEMOTRON_OLLAMA_MODEL, + "digest": service.NEMOTRON_OLLAMA_EXPECTED_DIGEST, + "details": [], + }] + }) + return _Response(_tags(service)) + + monkeypatch.setattr(service.requests, "get", fake_get) + monkeypatch.setattr( + service.requests, + "post", + lambda *_args, **_kwargs: _Response({ + "message": { + "role": "assistant", + "content": "", + "tool_calls": [_valid_canary_tool_call()], + } + }), + ) + + payload = service.run_nemotron_decision_canary( + output_root=tmp_path, + execute=True, + write_receipt=False, + timeout_sec=45, + ) + + execution = payload["execution"] + assert payload["status"] == "canary_passed" + assert execution["selected_host"] == service.OLLAMA_HOST_SECONDARY + assert execution["runtime_attempts"][0]["status"] == "identity_failed" + assert execution["runtime_attempts"][0]["error"] == "malformed_model_details" + + +def test_nemotron_canary_uses_exact_digest_111_fallback_after_gcp_timeout( + tmp_path, monkeypatch +): + import requests + from services import nemotron_decision_canary_service as service + from services.ollama_service import OLLAMA_HOST_FALLBACK + + def fake_get(url, timeout): + assert timeout == service.DEFAULT_MODEL_IDENTITY_TIMEOUT_SEC + if service.OLLAMA_HOST_PRIMARY in url: + raise requests.ConnectionError("primary unavailable") + if service.OLLAMA_HOST_SECONDARY in url: + return _Response(_tags(service)) + if OLLAMA_HOST_FALLBACK in url: + return _Response( + _tags(service, model=service.NEMOTRON_OLLAMA_FALLBACK_MODEL) + ) + raise AssertionError(f"unexpected identity URL: {url}") + + posted = [] + + def fake_post(url, json, timeout): + posted.append((url, json, timeout)) + if service.OLLAMA_HOST_SECONDARY in url: + raise requests.Timeout("secondary saturated") + return _Response({ + "message": { + "role": "assistant", + "content": "", + "tool_calls": [{ + "function": { + "name": "trigger_price_alert", + "arguments": { + "sku": "CANARY-NEMO-001", + "name": "NemoTron 受控決策驗證品", + "gap_pct": 22.4, + "sales_delta": -35.0, + "action": "建立價格告警候選", + "confidence": 0.91, + }, + } + }], + }, + "prompt_eval_count": 100, + "eval_count": 20, + }) + + monkeypatch.setattr(service.requests, "get", fake_get) + monkeypatch.setattr(service.requests, "post", fake_post) + + payload = service.run_nemotron_decision_canary( + output_root=tmp_path, + execute=True, + write_receipt=True, + timeout_sec=90, + run_id="nemotron-canary-fallback-test-001", + ) + + assert payload["status"] == "canary_passed_degraded_fallback" + execution = payload["execution"] + assert execution["selected_host"] == OLLAMA_HOST_FALLBACK + assert execution["model"] == service.NEMOTRON_OLLAMA_FALLBACK_MODEL + assert execution["fallback_used"] is True + assert execution["degraded_runtime"] is True + assert execution["selected_candidate_label"] == "ollama_111_fallback" + assert execution["attempt_count"] == 3 + assert posted[0][2] == 60 + assert posted[1][2] == 45 + assert posted[1][1]["model"] == service.NEMOTRON_OLLAMA_FALLBACK_MODEL + assert posted[1][1]["options"]["num_ctx"] == 4096 + assert posted[1][1]["options"]["num_predict"] == 512 + assert payload["next_machine_action"] == "restore_gcp_decision_runtime_capacity" + assert payload["latest_execution"]["fallback_used"] is True + + +def test_nemotron_canary_timeout_is_one_total_deadline(tmp_path, monkeypatch): + import requests + from services import nemotron_decision_canary_service as service + + clock = {"now": 50.0} + monkeypatch.setattr(service.time, "monotonic", lambda: clock["now"]) + monkeypatch.setattr( + service, + "inspect_nemotron_model_identity", + lambda candidate, **_kwargs: { + **candidate.as_public_dict(), + "ok": True, + "digest": candidate.expected_digest, + "digest_matches": True, + "elapsed_ms": 0, + "error": None, + }, + ) + posted = [] + + def timeout_post(url, json, timeout): + posted.append((url, timeout)) + clock["now"] += float(timeout) + raise requests.Timeout("simulated timeout") + + monkeypatch.setattr(service.requests, "post", timeout_post) + + payload = service.run_nemotron_decision_canary( + output_root=tmp_path, + execute=True, + write_receipt=False, + timeout_sec=30, + ) + + execution = payload["execution"] + assert payload["status"] == "canary_failed" + assert execution["error"] == "decision_total_deadline_exhausted" + assert execution["deadline_exhausted"] is True + assert execution["total_elapsed_ms"] == 30000 + assert len(posted) == 1 + assert posted[0][1] == 30 + assert execution["attempt_count"] == 2 + + +@pytest.mark.parametrize( + ("tool_calls", "error_fragment"), + [ + ( + [ + _valid_canary_tool_call(), + { + "function": { + "name": "delete_product", + "arguments": {"sku": "CANARY-NEMO-001"}, + } + }, + ], + "tool_call_count_mismatch", + ), + ( + [_valid_canary_tool_call(), _valid_canary_tool_call()], + "tool_call_count_mismatch", + ), + ( + [{ + "function": { + "name": "delete_product", + "arguments": {"sku": "CANARY-NEMO-001"}, + } + }], + "unknown_tool", + ), + ], +) +def test_nemotron_canary_rejects_extra_duplicate_or_unknown_tool_calls( + tmp_path, monkeypatch, tool_calls, error_fragment +): + from services import nemotron_decision_canary_service as service + from services.ollama_service import OLLAMA_HOST_FALLBACK + + def fake_get(url, timeout): + model = ( + service.NEMOTRON_OLLAMA_FALLBACK_MODEL + if OLLAMA_HOST_FALLBACK in url + else service.NEMOTRON_OLLAMA_MODEL + ) + return _Response(_tags(service, model=model)) + + monkeypatch.setattr(service.requests, "get", fake_get) + monkeypatch.setattr( + service.requests, + "post", + lambda *_args, **_kwargs: _Response({ + "message": { + "role": "assistant", + "content": "", + "tool_calls": tool_calls, + } + }), + ) + + payload = service.run_nemotron_decision_canary( + output_root=tmp_path, + execute=True, + write_receipt=False, + timeout_sec=90, + ) + + execution = payload["execution"] + assert payload["status"] == "canary_failed" + assert execution["canary_passed"] is False + assert execution["tool_execution_count"] == 0 + assert execution["database_call_performed"] is False + model_attempts = [ + attempt + for attempt in execution["runtime_attempts"] + if attempt["model_call_performed"] + ] + assert len(model_attempts) == 3 + assert all( + attempt["checks"]["decision_contract_valid"] is False + for attempt in model_attempts + ) + assert all( + error_fragment in str(attempt.get("contract_error") or "") + for attempt in model_attempts + ) + + def test_nemotron_canary_route_is_read_only(): from routes import system_public_routes as routes diff --git a/tests/test_nemotron_decision_envelope.py b/tests/test_nemotron_decision_envelope.py index f4103de..60fd4b4 100644 --- a/tests/test_nemotron_decision_envelope.py +++ b/tests/test_nemotron_decision_envelope.py @@ -62,7 +62,10 @@ def test_send_telegram_attaches_decision_envelope_to_event_router(monkeypatch): "severity": "P2", "guardrails": {"can_auto_execute": False}, } - module.NemotronDispatcher()._send_telegram("hello price alert", decision_envelope=envelope) + outcome = module.NemotronDispatcher()._send_telegram( + "hello price alert", + decision_envelope=envelope, + ) assert len(captured) == 1 event = captured[0] @@ -71,6 +74,25 @@ def test_send_telegram_attaches_decision_envelope_to_event_router(monkeypatch): assert event["decision_envelope"] == envelope assert event["payload"]["decision_envelope"] == envelope assert event["payload"]["raw_message"] == "hello price alert" + assert outcome["durable"] is False + + +def test_send_telegram_requires_delivery_or_durable_queue(monkeypatch): + import services.nemoton_dispatcher_service as module + + fake_event_router = types.ModuleType("services.event_router") + fake_event_router.dispatch_sync = lambda event: { + "delivered": False, + "queued": False, + "errors": ["telegram unavailable"], + } + monkeypatch.setitem(sys.modules, "services.event_router", fake_event_router) + + outcome = module.NemotronDispatcher()._send_telegram("test") + + assert outcome["durable"] is False + assert outcome["delivered"] is False + assert outcome["queued"] is False def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch): @@ -82,15 +104,18 @@ def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch): monkeypatch.setattr( dispatcher, "_send_telegram", - lambda message, decision_envelope=None: sent.append((message, decision_envelope)), + lambda message, decision_envelope=None: ( + sent.append((message, decision_envelope)) + or {"durable": True, "delivered": True, "queued": False} + ), ) monkeypatch.setattr( dispatcher, "_sink_insight_to_km", - lambda *args, **kwargs: stored.append((args, kwargs)), + lambda *args, **kwargs: (stored.append((args, kwargs)) or True), ) - dispatcher._exec_trigger_price_alert( + outcome = dispatcher._exec_trigger_price_alert( sku="SKU-1", name="測試商品", gap_pct=12.5, @@ -115,3 +140,4 @@ def test_exec_trigger_price_alert_persists_decision_envelope(monkeypatch): assert envelope["subject"]["competitor_product_id"] == "PC-1" assert stored assert stored[0][1]["metadata"]["decision_envelope"] == envelope + assert outcome["durable"] is True diff --git a/tests/test_nemotron_dispatch_reservation_service.py b/tests/test_nemotron_dispatch_reservation_service.py new file mode 100644 index 0000000..a96e721 --- /dev/null +++ b/tests/test_nemotron_dispatch_reservation_service.py @@ -0,0 +1,281 @@ +import json +import multiprocessing +from pathlib import Path +import subprocess +import sys + + +def _process_reserve(state_path, start_event, result_queue): + from services.nemotron_dispatch_reservation_service import ( + SharedFileReservationStore, + ) + + store = SharedFileReservationStore( + state_path, + committed_ttl_sec=3600, + lease_ttl_sec=60, + ) + start_event.wait(timeout=5) + result_queue.put(store.reserve("SKU-PROCESS-SHARED")) + + +def test_shared_store_coordinates_independent_instances_and_ttl(tmp_path): + from services.nemotron_dispatch_reservation_service import ( + SharedFileReservationStore, + ) + + clock = {"now": 1000.0} + path = tmp_path / "dedupe" / "state.json" + first = SharedFileReservationStore( + path, + committed_ttl_sec=100, + lease_ttl_sec=20, + clock=lambda: clock["now"], + ) + second = SharedFileReservationStore( + path, + committed_ttl_sec=100, + lease_ttl_sec=20, + clock=lambda: clock["now"], + ) + + token = first.reserve("SKU-SHARED") + assert token + assert second.reserve("SKU-SHARED") is None + assert second.is_duplicate("SKU-SHARED") is True + assert "SKU-SHARED" not in path.read_text(encoding="utf-8") + assert second.release("SKU-SHARED", "stale-token") is False + assert first.refresh("SKU-SHARED", token) is True + assert first.commit("SKU-SHARED", token) is True + assert second.reserve("SKU-SHARED") is None + + clock["now"] += 101 + next_token = second.reserve("SKU-SHARED") + assert next_token and next_token != token + assert second.release("SKU-SHARED", next_token) is True + + +def test_begin_side_effect_persists_extended_quarantine(tmp_path): + from services.nemotron_dispatch_reservation_service import ( + SharedFileReservationStore, + ) + + clock = {"now": 1000.0} + path = tmp_path / "state.json" + first = SharedFileReservationStore( + path, + committed_ttl_sec=100, + lease_ttl_sec=20, + clock=lambda: clock["now"], + ) + second = SharedFileReservationStore( + path, + committed_ttl_sec=100, + lease_ttl_sec=20, + clock=lambda: clock["now"], + ) + + token = first.reserve("SKU-SIDE-EFFECT") + assert token + assert first.begin_side_effect("SKU-SIDE-EFFECT", token) is True + + payload = json.loads(path.read_text(encoding="utf-8")) + lease = next(iter(payload["leases"].values())) + assert lease == { + "token": token, + "expires_at": 1100.0, + "phase": "side_effect_started", + } + + clock["now"] += 21 + assert second.reserve("SKU-SIDE-EFFECT") is None + clock["now"] += 80 + assert second.reserve("SKU-SIDE-EFFECT") + + +def test_shared_store_is_atomic_across_processes(tmp_path): + state_path = str(tmp_path / "process" / "state.json") + context = multiprocessing.get_context("fork") + start_event = context.Event() + result_queue = context.Queue() + processes = [ + context.Process( + target=_process_reserve, + args=(state_path, start_event, result_queue), + ) + for _index in range(6) + ] + for process in processes: + process.start() + start_event.set() + tokens = [result_queue.get(timeout=10) for _process in processes] + for process in processes: + process.join(timeout=10) + assert process.exitcode == 0 + + owners = [token for token in tokens if token] + assert len(owners) == 1 + + +def test_shared_store_corruption_fails_closed(tmp_path): + from services.nemotron_dispatch_reservation_service import ( + SharedFileReservationStore, + ) + + path = tmp_path / "state.json" + path.write_text("{not-json", encoding="utf-8") + store = SharedFileReservationStore( + path, + committed_ttl_sec=3600, + lease_ttl_sec=900, + ) + + assert store.reserve("SKU-CORRUPT") is None + assert store.refresh("SKU-CORRUPT", "token") is False + assert store.commit("SKU-CORRUPT", "token") is False + assert store.release("SKU-CORRUPT", "token") is False + assert store.is_duplicate("SKU-CORRUPT") is True + assert path.read_text(encoding="utf-8") == "{not-json" + + +def test_atomic_write_fsyncs_file_and_parent_directory(tmp_path, monkeypatch): + from services import nemotron_dispatch_reservation_service as service + + fsynced = [] + real_fsync = service.os.fsync + + def record_fsync(fd): + fsynced.append(fd) + return real_fsync(fd) + + monkeypatch.setattr(service.os, "fsync", record_fsync) + store = service.SharedFileReservationStore( + tmp_path / "nested" / "state.json", + committed_ttl_sec=3600, + lease_ttl_sec=900, + ) + + assert store.reserve("SKU-FSYNC") + assert len(fsynced) >= 2 + + +def test_missing_process_shared_lock_fails_closed(monkeypatch, tmp_path): + from services import nemotron_dispatch_reservation_service as service + + monkeypatch.setattr(service, "fcntl", None) + monkeypatch.delenv("FLASK_ENV", raising=False) + monkeypatch.delenv("USE_POSTGRESQL", raising=False) + + assert service.build_production_shared_reservation_store( + committed_ttl_sec=3600, + lease_ttl_sec=900, + ) is None + + store = service.SharedFileReservationStore( + tmp_path / "state.json", + committed_ttl_sec=3600, + lease_ttl_sec=900, + ) + assert store.reserve("SKU-NO-FCNTL") is None + assert store.is_duplicate("SKU-NO-FCNTL") is True + + +def test_production_builder_forces_canonical_shared_data_path(monkeypatch): + from services import nemotron_dispatch_reservation_service as service + + monkeypatch.setenv("FLASK_ENV", "production") + store = service.build_production_shared_reservation_store( + committed_ttl_sec=3600, + lease_ttl_sec=900, + ) + + assert store is not None + assert store.state_path == Path( + "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json" + ) + assert store.lock_path == Path( + "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json.lock" + ) + + +def test_postgres_runtime_cannot_fall_back_to_process_local_memory(monkeypatch): + from services import nemotron_dispatch_reservation_service as service + + monkeypatch.delenv("FLASK_ENV", raising=False) + monkeypatch.setenv("USE_POSTGRESQL", "true") + + assert service.production_shared_reservation_required() is True + assert service.build_production_shared_reservation_store( + committed_ttl_sec=3600, + lease_ttl_sec=900, + ) is not None + + +def test_shared_state_contains_only_hashed_sku_and_timing_metadata(tmp_path): + from services.nemotron_dispatch_reservation_service import ( + SharedFileReservationStore, + ) + + path = tmp_path / "state.json" + store = SharedFileReservationStore( + path, + committed_ttl_sec=3600, + lease_ttl_sec=900, + ) + token = store.reserve("RAW-SKU-MUST-NOT-APPEAR") + payload = json.loads(path.read_text(encoding="utf-8")) + + assert token + assert "RAW-SKU-MUST-NOT-APPEAR" not in path.read_text(encoding="utf-8") + assert set(payload) == {"version", "leases", "committed"} + assert len(next(iter(payload["leases"]))) == 64 + + +def test_production_containers_share_data_mount_and_force_shared_backend(): + compose = ( + Path(__file__).resolve().parents[1] / "docker-compose.yml" + ).read_text(encoding="utf-8") + + assert compose.count("- ./data:/app/data") >= 3 + assert compose.count("- FLASK_ENV=production") >= 3 + + +def test_shared_reservation_canary_has_no_business_side_effects(tmp_path): + from services.nemotron_dispatch_reservation_service import ( + run_shared_reservation_canary, + ) + + payload = run_shared_reservation_canary( + state_path=tmp_path / "canary" / "state.json", + ) + + assert payload["success"] is True + assert payload["check_pass_count"] == payload["check_count"] == 6 + assert payload["controlled_apply"] == { + "writes_shared_reservation_state": True, + "terminal_state_clear": True, + "writes_database": False, + "sends_telegram": False, + "executes_business_tool": False, + "reads_secret": False, + } + + +def test_shared_reservation_canary_cli_is_machine_readable(tmp_path): + completed = subprocess.run( + [ + sys.executable, + "scripts/ops/run_nemotron_dispatch_reservation_canary.py", + "--state-path", + str(tmp_path / "cli" / "state.json"), + ], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + payload = json.loads(completed.stdout) + assert payload["status"] == "shared_reservation_canary_passed" + assert payload["controlled_apply"]["writes_database"] is False diff --git a/tests/test_nemotron_fallback.py b/tests/test_nemotron_fallback.py index 3b7f508..c2f5a44 100644 --- a/tests/test_nemotron_fallback.py +++ b/tests/test_nemotron_fallback.py @@ -9,9 +9,11 @@ def _force_legacy_nim_first(monkeypatch): import services.nemoton_dispatcher_service as module module._ALERT_CACHE.clear() + module._ALERT_INFLIGHT.clear() monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", False) yield module._ALERT_CACHE.clear() + module._ALERT_INFLIGHT.clear() @dataclass @@ -33,6 +35,7 @@ def _patch_execution_methods(monkeypatch, dispatcher): def record(kind): def _inner(*args, **kwargs): calls.append({"kind": kind, "args": args, "kwargs": kwargs}) + return {"durable": True} return _inner monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record("price_alert")) @@ -56,6 +59,207 @@ def test_dispatch_falls_back_to_hermes_rules_without_nim_api_key(monkeypatch): assert calls[0]["kind"] == "price_alert" +def test_forced_review_failure_is_counted_as_skipped(monkeypatch): + import services.nemoton_dispatcher_service as module + + monkeypatch.setattr(module, "NIM_API_KEY", "") + dispatcher = module.NemotronDispatcher() + monkeypatch.setattr( + dispatcher, + "_exec_flag_for_human_review", + lambda *_args, **_kwargs: {"durable": False}, + ) + threat = FakeThreat("SKU-FORCED", "斷崖品") + threat.sales_7d_delta_pct = -100.0 + + result = dispatcher.dispatch([threat], hermes_stats={"duration_sec": 1}) + + assert result["dispatched"] == 0 + assert result["skipped"] == 1 + assert len(result["errors"]) == 1 + assert "durable_side_effect_not_confirmed" in result["errors"][0] + + +def test_hermes_failure_is_counted_as_skipped(monkeypatch): + import services.nemoton_dispatcher_service as module + + monkeypatch.setattr(module, "NIM_API_KEY", "") + dispatcher = module.NemotronDispatcher() + monkeypatch.setattr( + dispatcher, + "_exec_trigger_price_alert", + lambda *_args, **_kwargs: {"durable": False}, + ) + + result = dispatcher.dispatch( + [FakeThreat("SKU-HERMES-FAIL", "告警品")], + hermes_stats={"duration_sec": 1}, + ) + + assert result["dispatched"] == 0 + assert result["skipped"] == 1 + assert len(result["errors"]) == 1 + assert "durable_side_effect_not_confirmed" in result["errors"][0] + + +def test_forced_review_commit_failure_keeps_durable_result_quarantined(monkeypatch): + import services.nemoton_dispatcher_service as module + + monkeypatch.setattr(module, "NIM_API_KEY", "") + dispatcher = module.NemotronDispatcher() + monkeypatch.setattr( + dispatcher, + "_exec_flag_for_human_review", + lambda *_args, **_kwargs: {"durable": True}, + ) + monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False) + released = [] + monkeypatch.setattr( + module, + "_release_alert_reservation", + lambda *args: (released.append(args) or True), + ) + threat = FakeThreat("SKU-FORCED-COMMIT", "斷崖品") + threat.sales_7d_delta_pct = -100.0 + + result = dispatcher.dispatch([threat], hermes_stats={"duration_sec": 1}) + + assert result["dispatched"] == 1 + assert result["skipped"] == 0 + assert result["errors"] == [ + "dedupe_commit_unverified:flag_for_human_review(SKU-FORCED-COMMIT)" + ] + assert released == [] + + +def test_hermes_commit_failure_keeps_durable_result_quarantined(monkeypatch): + import services.nemoton_dispatcher_service as module + + monkeypatch.setattr(module, "NIM_API_KEY", "") + dispatcher = module.NemotronDispatcher() + _patch_execution_methods(monkeypatch, dispatcher) + monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False) + released = [] + monkeypatch.setattr( + module, + "_release_alert_reservation", + lambda *args: (released.append(args) or True), + ) + + result = dispatcher.dispatch( + [FakeThreat("SKU-HERMES-COMMIT", "告警品")], + hermes_stats={"duration_sec": 1}, + ) + + assert result["dispatched"] == 1 + assert result["skipped"] == 0 + assert result["errors"] == [ + "dedupe_commit_unverified:hermes_rule_fallback(SKU-HERMES-COMMIT)" + ] + assert released == [] + + +def test_mixed_batch_cleanup_never_releases_forced_durable_quarantine(monkeypatch): + import services.nemoton_dispatcher_service as module + + monkeypatch.setattr(module, "NIM_API_KEY", "test-key") + monkeypatch.setattr(module, "_check_nim_quota", lambda: True) + dispatcher = module.NemotronDispatcher() + _patch_execution_methods(monkeypatch, dispatcher) + forced = FakeThreat("SKU-FORCED-MIXED", "斷崖品") + forced.sales_7d_delta_pct = -100.0 + candidate = FakeThreat("SKU-NIM-MIXED", "模型候選") + monkeypatch.setattr( + dispatcher, + "_call_nim", + lambda _threats, **_kwargs: ( + [{ + "tool": "trigger_price_alert", + "args": { + "sku": candidate.sku, + "name": candidate.name, + "gap_pct": candidate.gap_pct, + "sales_delta": candidate.sales_7d_delta_pct, + "action": candidate.recommended_action, + "confidence": candidate.confidence, + }, + }], + {"provider": "nim", "total_tokens": 1}, + ), + ) + real_mark = module._mark_alert_dispatched + monkeypatch.setattr( + module, + "_mark_alert_dispatched", + lambda sku, token: ( + False if str(sku) == forced.sku else real_mark(sku, token) + ), + ) + + result = dispatcher.dispatch( + [forced, candidate], + hermes_stats={"duration_sec": 1}, + ) + + assert result["dispatched"] == 2 + assert result["skipped"] == 0 + assert result["errors"] == [ + "dedupe_commit_unverified:flag_for_human_review(SKU-FORCED-MIXED)" + ] + assert module._is_duplicate_alert(forced.sku) is True + assert module._is_duplicate_alert(candidate.sku) is True + + +def test_mixed_batch_counts_failed_model_candidate_as_skipped(monkeypatch): + import services.nemoton_dispatcher_service as module + + monkeypatch.setattr(module, "NIM_API_KEY", "test-key") + monkeypatch.setattr(module, "_check_nim_quota", lambda: True) + dispatcher = module.NemotronDispatcher() + forced = FakeThreat("SKU-FORCED-COUNT", "斷崖品") + forced.sales_7d_delta_pct = -100.0 + candidate = FakeThreat("SKU-NIM-COUNT", "模型候選") + monkeypatch.setattr( + dispatcher, + "_exec_flag_for_human_review", + lambda *_args, **_kwargs: {"durable": True}, + ) + monkeypatch.setattr( + dispatcher, + "_exec_trigger_price_alert", + lambda *_args, **_kwargs: {"durable": False}, + ) + monkeypatch.setattr( + dispatcher, + "_call_nim", + lambda _threats, **_kwargs: ( + [{ + "tool": "trigger_price_alert", + "args": { + "sku": candidate.sku, + "name": candidate.name, + "gap_pct": candidate.gap_pct, + "sales_delta": candidate.sales_7d_delta_pct, + "action": candidate.recommended_action, + "confidence": candidate.confidence, + }, + }], + {"provider": "nim", "total_tokens": 1}, + ), + ) + + result = dispatcher.dispatch( + [forced, candidate], + hermes_stats={"duration_sec": 1}, + ) + + assert result["dispatched"] == 1 + assert result["skipped"] == 1 + assert len(result["errors"]) == 1 + assert "durable_side_effect_not_confirmed" in result["errors"][0] + assert result["dispatched"] + result["skipped"] == 2 + + def test_dispatch_routes_non_exact_match_to_human_review(monkeypatch): import services.nemoton_dispatcher_service as module @@ -83,7 +287,7 @@ def test_dispatch_falls_back_to_hermes_rules_on_nim_timeout(monkeypatch): monkeypatch.setattr(module, "_check_nim_quota", lambda: True) dispatcher = module.NemotronDispatcher() calls = _patch_execution_methods(monkeypatch, dispatcher) - monkeypatch.setattr(dispatcher, "_call_nim", lambda threats: (_ for _ in ()).throw(requests.Timeout("timeout"))) + monkeypatch.setattr(dispatcher, "_call_nim", lambda threats, **_kwargs: (_ for _ in ()).throw(requests.Timeout("timeout"))) result = dispatcher.dispatch([FakeThreat("SKU-2", "測試品")], hermes_stats={"duration_sec": 1}) @@ -101,7 +305,7 @@ def test_dispatch_falls_back_to_hermes_rules_on_zero_tool_calls(monkeypatch): monkeypatch.setattr(module, "_check_nim_quota", lambda: True) dispatcher = module.NemotronDispatcher() calls = _patch_execution_methods(monkeypatch, dispatcher) - monkeypatch.setattr(dispatcher, "_call_nim", lambda threats: ([], {"total_tokens": 10})) + monkeypatch.setattr(dispatcher, "_call_nim", lambda threats, **_kwargs: ([], {"total_tokens": 10})) result = dispatcher.dispatch([FakeThreat("SKU-3", "測試品")], hermes_stats={"duration_sec": 1}) diff --git a/tests/test_nemotron_qwen3_compat.py b/tests/test_nemotron_qwen3_compat.py index 892ff14..b5b4927 100644 --- a/tests/test_nemotron_qwen3_compat.py +++ b/tests/test_nemotron_qwen3_compat.py @@ -39,6 +39,20 @@ class FakeThreat: sales_7d_prev_amount: float = 120000.0 +def _valid_price_alert_call(sku="SKU-Q1"): + return { + "tool": "trigger_price_alert", + "args": { + "sku": sku, + "name": "qwen3 測試品", + "gap_pct": 22.4, + "sales_delta": -35.0, + "action": "跟進降價", + "confidence": 0.85, + }, + } + + class _FakeResp: def __init__(self, payload: dict, status: int = 200): self._payload = payload @@ -59,6 +73,7 @@ def _noop_log_ai_call(*args, **kwargs): class _Ctx: def set_tokens(self, **_kw): pass def set_provider(self, *_a, **_kw): pass + def set_model(self, *_a, **_kw): pass def set_error(self, *_a, **_kw): pass def fallback_to_caller(self, *_a, **_kw): pass def set_cache_hit(self, *_a, **_kw): pass @@ -76,11 +91,13 @@ def _reset_global_state(): import services.nemoton_dispatcher_service as _nem import services.ollama_service as _oss _nem._ALERT_CACHE.clear() + _nem._ALERT_INFLIGHT.clear() _oss._unhealthy_marks.clear() _oss._resolved_host_cache['host'] = None _oss._resolved_host_cache['ts'] = 0 yield _nem._ALERT_CACHE.clear() + _nem._ALERT_INFLIGHT.clear() _oss._unhealthy_marks.clear() _oss._resolved_host_cache['host'] = None _oss._resolved_host_cache['ts'] = 0 @@ -93,6 +110,7 @@ def _patch_execution_methods(monkeypatch, dispatcher): def record(kind): def _inner(*args, **kwargs): calls.append({"kind": kind, "args": args, "kwargs": kwargs}) + return {"durable": True} return _inner monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record("price_alert")) @@ -102,14 +120,27 @@ def _patch_execution_methods(monkeypatch, dispatcher): def _enable_qwen3_path(monkeypatch, module): - """打開 NEMOTRON_OLLAMA_FIRST + 旁路 mcp/log_ai_call/resolve_host 等副作用""" + """打開 Ollama-first，並隔離 MCP、logger 與 model identity 網路副作用。""" monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", True) monkeypatch.setattr(module, "log_ai_call", _noop_log_ai_call) monkeypatch.setattr(module, "build_mcp_context", lambda: "MCP-MOCK") # 確保即使未被呼叫，import 路徑可解析 import services.ollama_service as ollama_module + import services.nemotron_runtime_candidate_service as runtime_module + monkeypatch.setattr(ollama_module, "resolve_ollama_host", lambda: "http://34.87.90.216:11434") monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda *a, **kw: None) + monkeypatch.setattr( + runtime_module, + "inspect_nemotron_model_identity", + lambda candidate, **_kwargs: { + **candidate.as_public_dict(), + "ok": True, + "digest": candidate.expected_digest, + "digest_matches": True, + "error": None, + }, + ) # ───────────────────────────────────────────────────────────── @@ -178,12 +209,7 @@ def test_qwen3_retries_secondary_when_primary_chat_fails(monkeypatch): import services.ollama_service as ollama_module _enable_qwen3_path(monkeypatch, module) - hosts = iter([ - "http://34.87.90.216:11434", - "http://34.21.145.224:11434", - ]) marked = [] - monkeypatch.setattr(ollama_module, "resolve_ollama_host", lambda: next(hosts)) monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda host: marked.append(host)) fake_body = { @@ -227,6 +253,519 @@ def test_qwen3_retries_secondary_when_primary_chat_fails(monkeypatch): assert calls[0]["kind"] == "price_alert" +def test_qwen3_uses_bounded_111_model_after_both_gcp_candidates_fail(monkeypatch): + """兩台 GCP 都逾時時，正式 dispatcher 應改用 exact-digest 111 qwen3:8b。""" + import requests + import services.nemoton_dispatcher_service as module + + _enable_qwen3_path(monkeypatch, module) + posted = [] + fake_body = { + "message": { + "role": "assistant", + "content": "", + "tool_calls": [{ + "function": { + "name": "trigger_price_alert", + "arguments": { + "sku": "SKU-Q1", + "name": "qwen3 測試品", + "gap_pct": 22.4, + "sales_delta": -35.0, + "action": "跟進降價", + "confidence": 0.85, + }, + } + }], + }, + "prompt_eval_count": 180, + "eval_count": 36, + } + + def fake_post(url, json, timeout): + posted.append((url, json, timeout)) + if len(posted) <= 2: + raise requests.Timeout("GCP saturated") + return _FakeResp(fake_body) + + monkeypatch.setattr(module.requests, "post", fake_post) + dispatcher = module.NemotronDispatcher() + calls = _patch_execution_methods(monkeypatch, dispatcher) + + result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0}) + + stats = result["nim_stats"] + assert result["dispatched"] == 1 + assert stats["provider"] == "ollama_111" + assert stats["model"] == "qwen3:8b" + assert stats["candidate_label"] == "ollama_111_fallback" + assert stats["fallback_used"] is True + assert [attempt["status"] for attempt in stats["attempted_candidates"]] == [ + "failed", + "failed", + "selected", + ] + assert posted[2][1]["think"] is False + assert posted[2][1]["options"]["num_predict"] == 512 + assert posted[2][2] == 45 + assert calls[0]["kind"] == "price_alert" + assert "qwen3:8b (111 備援)" in calls[0]["kwargs"]["footprint"] + assert "降級備援" in calls[0]["kwargs"]["footprint"] + + structured = module._build_footprint_json(None, stats) + assert structured["dispatcher"]["platform"] == "Ollama" + assert structured["dispatcher"]["model"] == "qwen3:8b" + assert structured["dispatcher"]["fallback_used"] is True + assert [ + attempt["label"] + for attempt in structured["dispatcher"]["candidate_attempts"] + ] == ["gcp_primary", "gcp_secondary", "ollama_111_fallback"] + assert all( + "host" not in attempt and "error" not in attempt + for attempt in structured["dispatcher"]["candidate_attempts"] + ) + + +def test_invalid_primary_tool_contract_retries_verified_secondary(monkeypatch): + import services.nemoton_dispatcher_service as module + + _enable_qwen3_path(monkeypatch, module) + valid_body = { + "message": { + "tool_calls": [{"function": { + "name": "trigger_price_alert", + "arguments": _valid_price_alert_call()["args"], + }}], + "content": "", + }, + "prompt_eval_count": 100, + "eval_count": 20, + } + invalid_body = { + "message": { + "tool_calls": [{"function": { + "name": "delete_product", + "arguments": {"sku": "SKU-Q1"}, + }}], + "content": "", + }, + } + responses = [_FakeResp(invalid_body), _FakeResp(valid_body)] + monkeypatch.setattr( + module.requests, + "post", + lambda *_args, **_kwargs: responses.pop(0), + ) + dispatcher = module.NemotronDispatcher() + calls = _patch_execution_methods(monkeypatch, dispatcher) + + result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0}) + + attempts = result["nim_stats"]["attempted_candidates"] + assert [attempt["status"] for attempt in attempts] == ["failed", "selected"] + assert "unknown_tool:delete_product" in attempts[0]["error"] + assert result["nim_stats"]["provider"] == "ollama_secondary" + assert result["dispatched"] == 1 + assert calls[0]["kind"] == "price_alert" + + +def test_malformed_primary_tags_schema_falls_through_to_verified_secondary( + monkeypatch, +): + import services.nemoton_dispatcher_service as module + import services.nemotron_runtime_candidate_service as runtime_module + import services.ollama_service as ollama_module + + monkeypatch.setattr(module, "NEMOTRON_OLLAMA_FIRST", True) + monkeypatch.setattr(module, "log_ai_call", _noop_log_ai_call) + monkeypatch.setattr(module, "build_mcp_context", lambda: "MCP-MOCK") + monkeypatch.setattr(ollama_module, "mark_unhealthy", lambda *_args: None) + + def fake_get(url, timeout): + if runtime_module.OLLAMA_HOST_PRIMARY in url: + return _FakeResp({ + "models": [ + None, + { + "name": runtime_module.PRIMARY_MODEL, + "model": runtime_module.PRIMARY_MODEL, + "digest": runtime_module.PRIMARY_EXPECTED_DIGEST, + "details": {}, + }, + ] + }) + return _FakeResp({ + "models": [{ + "name": runtime_module.PRIMARY_MODEL, + "model": runtime_module.PRIMARY_MODEL, + "digest": runtime_module.PRIMARY_EXPECTED_DIGEST, + "details": { + "parameter_size": "14.8B", + "quantization_level": "Q4_K_M", + }, + }] + }) + + valid_body = { + "message": { + "tool_calls": [{"function": { + "name": "trigger_price_alert", + "arguments": _valid_price_alert_call()["args"], + }}], + "content": "", + }, + "prompt_eval_count": 100, + "eval_count": 20, + } + monkeypatch.setattr(module.requests, "get", fake_get) + monkeypatch.setattr( + module.requests, + "post", + lambda *_args, **_kwargs: _FakeResp(valid_body), + ) + dispatcher = module.NemotronDispatcher() + calls = _patch_execution_methods(monkeypatch, dispatcher) + + result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1}) + + attempts = result["nim_stats"]["attempted_candidates"] + assert [attempt["status"] for attempt in attempts] == [ + "identity_failed", + "selected", + ] + assert attempts[0]["error"] == "malformed_models_schema" + assert result["nim_stats"]["provider"] == "ollama_secondary" + assert result["dispatched"] == 1 + assert calls[0]["kind"] == "price_alert" + + +@pytest.mark.parametrize( + ("tool_calls", "threats", "error_fragment"), + [ + ([{"tool": "delete_product", "args": {"sku": "SKU-Q1"}}], [FakeThreat()], "unknown_tool"), + ([_valid_price_alert_call("OTHER")], [FakeThreat()], "sku_not_in_request"), + ([{"tool": "trigger_price_alert", "args": {"sku": "SKU-Q1"}}], [FakeThreat()], "missing_required_args"), + ([_valid_price_alert_call()], [FakeThreat(), FakeThreat(sku="SKU-Q2")], "tool_call_count_mismatch"), + ( + [_valid_price_alert_call(), _valid_price_alert_call()], + [FakeThreat(), FakeThreat(sku="SKU-Q2")], + "duplicate_sku_tool_call", + ), + ], +) +def test_tool_contract_rejects_unbound_incomplete_or_duplicate_calls( + tool_calls, threats, error_fragment +): + import services.nemoton_dispatcher_service as module + + with pytest.raises(module.ToolCallContractError, match=error_fragment): + module._validate_tool_call_contract(tool_calls, threats) + + +def test_failed_handler_does_not_poison_alert_dedupe(monkeypatch): + import services.nemoton_dispatcher_service as module + + dispatcher = module.NemotronDispatcher() + monkeypatch.setattr( + dispatcher, + "_exec_trigger_price_alert", + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("send failed")), + ) + first_token = module._reserve_alert("SKU-Q1") + assert first_token + first = dispatcher._execute_tool_calls( + tool_calls=[_valid_price_alert_call()], + threats=[FakeThreat()], + hermes_stats=None, + nim_stats={"provider": "nim", "total_tokens": 1}, + reservation_tokens={"SKU-Q1": first_token}, + ) + + assert first["dispatched"] == 0 + assert module._is_duplicate_alert("SKU-Q1") is False + + sent = [] + monkeypatch.setattr( + dispatcher, + "_exec_trigger_price_alert", + lambda **kwargs: (sent.append(kwargs) or {"durable": True}), + ) + second_token = module._reserve_alert("SKU-Q1") + assert second_token + second = dispatcher._execute_tool_calls( + tool_calls=[_valid_price_alert_call()], + threats=[FakeThreat()], + hermes_stats=None, + nim_stats={"provider": "nim", "total_tokens": 1}, + reservation_tokens={"SKU-Q1": second_token}, + ) + + assert second["dispatched"] == 1 + assert sent + assert module._is_duplicate_alert("SKU-Q1") is True + + +def test_non_durable_alert_outcome_is_failed_and_releases_reservation(monkeypatch): + import services.nemoton_dispatcher_service as module + + dispatcher = module.NemotronDispatcher() + monkeypatch.setattr( + dispatcher, + "_send_telegram", + lambda *_args, **_kwargs: { + "durable": False, + "delivered": False, + "queued": False, + }, + ) + monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: True) + token = module._reserve_alert("SKU-Q1") + assert token + + result = dispatcher._execute_tool_calls( + tool_calls=[_valid_price_alert_call()], + threats=[FakeThreat()], + hermes_stats=None, + nim_stats={"provider": "nim", "total_tokens": 1}, + reservation_tokens={"SKU-Q1": token}, + ) + + assert result["dispatched"] == 0 + assert result["skipped"] == 1 + assert "durable_side_effect_not_confirmed" in result["errors"][0] + assert module._is_duplicate_alert("SKU-Q1") is False + + +def test_recommendation_requires_db_write_or_durable_notification(monkeypatch): + import services.nemoton_dispatcher_service as module + + dispatcher = module.NemotronDispatcher(engine=None) + monkeypatch.setattr( + dispatcher, + "_send_telegram", + lambda *_args, **_kwargs: {"durable": False}, + ) + monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: True) + token = module._reserve_alert("SKU-Q1") + assert token + + result = dispatcher._execute_tool_calls( + tool_calls=[{ + "tool": "add_to_recommendation", + "args": { + "sku": "SKU-Q1", + "name": "qwen3 測試品", + "reason": "具價格競爭力", + "confidence": 0.85, + }, + }], + threats=[FakeThreat()], + hermes_stats=None, + nim_stats={"provider": "nim", "total_tokens": 1}, + reservation_tokens={"SKU-Q1": token}, + ) + + assert result["dispatched"] == 0 + assert "durable_side_effect_not_confirmed" in result["errors"][0] + assert module._is_duplicate_alert("SKU-Q1") is False + + +def test_km_route_requires_durable_insight_write(monkeypatch): + import services.nemoton_dispatcher_service as module + + dispatcher = module.NemotronDispatcher() + monkeypatch.setattr(dispatcher, "_sink_insight_to_km", lambda **_kwargs: False) + token = module._reserve_alert("SKU-Q1") + assert token + + result = dispatcher._execute_tool_calls( + tool_calls=[{ + "tool": "route_to_km", + "args": { + "sku": "SKU-Q1", + "name": "qwen3 測試品", + "km_domain": "price_competition", + "summary": "競價洞察", + "confidence": 0.85, + }, + }], + threats=[FakeThreat()], + hermes_stats=None, + nim_stats={"provider": "nim", "total_tokens": 1}, + reservation_tokens={"SKU-Q1": token}, + ) + + assert result["dispatched"] == 0 + assert "durable_side_effect_not_confirmed" in result["errors"][0] + assert module._is_duplicate_alert("SKU-Q1") is False + + +def test_alert_reservation_is_atomic_and_releasable(): + import services.nemoton_dispatcher_service as module + + first_token = module._reserve_alert("SKU-Q1") + assert first_token + assert module._reserve_alert("SKU-Q1") is None + assert module._release_alert_reservation("SKU-Q1", first_token) is True + second_token = module._reserve_alert("SKU-Q1") + assert second_token and second_token != first_token + assert module._mark_alert_dispatched("SKU-Q1", second_token) is True + assert module._reserve_alert("SKU-Q1") is None + + +def test_expired_reservation_token_cannot_release_or_commit_new_owner(monkeypatch): + import services.nemoton_dispatcher_service as module + + clock = {"now": 1000.0} + monkeypatch.setattr(module.time, "time", lambda: clock["now"]) + first_token = module._reserve_alert("SKU-Q1") + assert first_token + + clock["now"] += module._ALERT_INFLIGHT_TTL_SEC + 1 + assert module._mark_alert_dispatched("SKU-Q1", first_token) is False + second_token = module._reserve_alert("SKU-Q1") + assert second_token and second_token != first_token + + assert module._release_alert_reservation("SKU-Q1", first_token) is False + assert module._mark_alert_dispatched("SKU-Q1", first_token) is False + assert module._refresh_alert_reservation("SKU-Q1", first_token) is False + assert module._refresh_alert_reservation("SKU-Q1", second_token) is True + assert module._mark_alert_dispatched("SKU-Q1", second_token) is True + assert module._is_duplicate_alert("SKU-Q1") is True + + +def test_concurrent_reservation_has_exactly_one_owner(): + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + + import services.nemoton_dispatcher_service as module + + barrier = Barrier(8) + + def reserve_once(): + barrier.wait() + return module._reserve_alert("SKU-CONCURRENT") + + with ThreadPoolExecutor(max_workers=8) as pool: + tokens = list(pool.map(lambda _index: reserve_once(), range(8))) + + owners = [token for token in tokens if token is not None] + assert len(owners) == 1 + assert module._release_alert_reservation("SKU-CONCURRENT", owners[0]) is True + + +def test_side_effect_boundary_extends_lease_through_slow_handler(monkeypatch): + import services.nemoton_dispatcher_service as module + + clock = {"now": 5000.0} + monkeypatch.setattr(module.time, "time", lambda: clock["now"]) + old_token = module._reserve_alert("SKU-Q1") + assert old_token + new_owner = {} + dispatcher = module.NemotronDispatcher() + + def delayed_handler(**_kwargs): + clock["now"] += module._ALERT_INFLIGHT_TTL_SEC + 1 + new_owner["token"] = module._reserve_alert("SKU-Q1") + return {"durable": True} + + monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", delayed_handler) + result = dispatcher._execute_tool_calls( + tool_calls=[_valid_price_alert_call()], + threats=[FakeThreat()], + hermes_stats=None, + nim_stats={"provider": "nim", "total_tokens": 1}, + reservation_tokens={"SKU-Q1": old_token}, + ) + + assert new_owner["token"] is None + assert result["dispatched"] == 1 + assert result["errors"] == [] + assert module._is_duplicate_alert("SKU-Q1") is True + + +def test_durable_tool_commit_failure_never_releases_quarantine(monkeypatch): + import services.nemoton_dispatcher_service as module + + dispatcher = module.NemotronDispatcher() + monkeypatch.setattr( + dispatcher, + "_exec_trigger_price_alert", + lambda **_kwargs: {"durable": True}, + ) + token = module._reserve_alert("SKU-Q1") + released = [] + monkeypatch.setattr(module, "_mark_alert_dispatched", lambda *_args: False) + monkeypatch.setattr( + module, + "_release_alert_reservation", + lambda *args: (released.append(args) or True), + ) + + result = dispatcher._execute_tool_calls( + tool_calls=[_valid_price_alert_call()], + threats=[FakeThreat()], + hermes_stats=None, + nim_stats={"provider": "nim", "total_tokens": 1}, + reservation_tokens={"SKU-Q1": token}, + ) + + assert result["dispatched"] == 1 + assert result["skipped"] == 0 + assert result["errors"] == [ + "dedupe_commit_unverified:trigger_price_alert(SKU-Q1)" + ] + assert released == [] + assert module._is_duplicate_alert("SKU-Q1") is True + + +def test_dispatch_total_deadline_bounds_ollama_and_nim_chain(monkeypatch): + import requests + import services.nemoton_dispatcher_service as module + + _enable_qwen3_path(monkeypatch, module) + monkeypatch.setattr(module, "NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC", 30) + monkeypatch.setattr(module, "NIM_API_KEY", "fake-key") + monkeypatch.setattr(module, "_check_nim_quota", lambda: True) + clock = {"now": 100.0} + monkeypatch.setattr(module.time, "monotonic", lambda: clock["now"]) + posted = [] + + def timeout_post(url, json=None, timeout=None, **_kwargs): + posted.append((url, timeout)) + clock["now"] += float(timeout or 0) + raise requests.Timeout("simulated timeout") + + monkeypatch.setattr(module.requests, "post", timeout_post) + dispatcher = module.NemotronDispatcher() + calls = _patch_execution_methods(monkeypatch, dispatcher) + + result = dispatcher.dispatch([FakeThreat()], hermes_stats={"duration_sec": 1.0}) + + upstream = result["nim_stats"]["upstream_ollama_failure"] + assert len(posted) == 1 + assert posted[0][1] == 30 + assert clock["now"] == 130.0 + assert upstream["deadline_exhausted"] is True + assert upstream["attempt_count"] == 2 + assert result["nim_stats"]["provider"] == "hermes_rule_engine" + assert "Ollama 候選鏈" in calls[0]["kwargs"]["footprint"] + assert "gcp_primary | primary | qwen3:14b | failed | digest=ok | Timeout" in ( + calls[0]["kwargs"]["footprint"] + ) + structured = module._build_footprint_json(None, result["nim_stats"]) + attempts = structured["dispatcher"]["candidate_attempts"] + assert attempts[0] == { + "label": "gcp_primary", + "tier": "primary", + "model": "qwen3:14b", + "status": "failed", + "digest_matches": True, + "error_class": "Timeout", + } + assert result["dispatched"] == 1 + + # ───────────────────────────────────────────────────────────── # T2. qwen3 沒回 tool_calls 但 content 含 JSON list → fallback 解析 # ───────────────────────────────────────────────────────────── @@ -252,7 +791,7 @@ def test_qwen3_content_only_fallback_parsing(monkeypatch): calls = _patch_execution_methods(monkeypatch, dispatcher) monkeypatch.setattr( dispatcher, "_call_nim", - lambda threats: (_ for _ in ()).throw(AssertionError("NIM 不應被呼叫")), + lambda threats, **_kwargs: (_ for _ in ()).throw(AssertionError("NIM 不應被呼叫")), ) result = dispatcher.dispatch([FakeThreat(confidence=0.45)], hermes_stats={"duration_sec": 1.0}) @@ -325,7 +864,7 @@ def test_qwen3_connection_error_falls_back_to_nim(monkeypatch): calls = _patch_execution_methods(monkeypatch, dispatcher) nim_invoked = {"v": False} - def _fake_nim(threats): + def _fake_nim(threats, **_kwargs): nim_invoked["v"] = True return ( [{ @@ -369,7 +908,7 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch): # 攔 _call_nim 也擲 timeout monkeypatch.setattr( dispatcher, "_call_nim", - lambda threats: (_ for _ in ()).throw(requests.Timeout("NIM timeout")), + lambda threats, **_kwargs: (_ for _ in ()).throw(requests.Timeout("NIM timeout")), ) # 攔住規則引擎內部呼叫的 _exec_*，記錄 concern / reason 文字驗證 🟡 標記 @@ -392,6 +931,7 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch): 'revenue_loss_7d', 'recommended_price'], args, kwargs) captured.append(("human_review", merged)) + return {"durable": True} def record_alert(*args, **kwargs): merged = _merge_positional( @@ -400,9 +940,11 @@ def test_qwen3_and_nim_both_fail_falls_back_to_hermes_rules(monkeypatch): 'revenue_loss_7d', 'recommended_price'], args, kwargs) captured.append(("price_alert", merged)) + return {"durable": True} def record_reco(*args, **kwargs): captured.append(("recommendation", kwargs)) + return {"durable": True} monkeypatch.setattr(dispatcher, "_exec_flag_for_human_review", record_review) monkeypatch.setattr(dispatcher, "_exec_trigger_price_alert", record_alert) @@ -448,7 +990,7 @@ def test_flag_false_preserves_nim_first_emergency_path(monkeypatch): calls = _patch_execution_methods(monkeypatch, dispatcher) monkeypatch.setattr( dispatcher, "_call_nim", - lambda threats: ( + lambda threats, **_kwargs: ( [{ "tool": "trigger_price_alert", "args": { diff --git a/tests/test_nemotron_runtime_candidate_service.py b/tests/test_nemotron_runtime_candidate_service.py new file mode 100644 index 0000000..b659170 --- /dev/null +++ b/tests/test_nemotron_runtime_candidate_service.py @@ -0,0 +1,205 @@ +import json +import os +import subprocess +import sys + +import pytest +import requests + + +class _Response: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +def test_candidate_chain_is_gcp_first_with_exact_digest_111_fallback(): + from services import nemotron_runtime_candidate_service as service + + candidates = service.build_nemotron_runtime_candidates() + + assert [candidate.label for candidate in candidates] == [ + "gcp_primary", + "gcp_secondary", + "ollama_111_fallback", + ] + assert [candidate.model for candidate in candidates] == [ + "qwen3:14b", + "qwen3:14b", + "qwen3:8b", + ] + assert candidates[0].is_fallback is False + assert candidates[2].is_fallback is True + assert candidates[0].request_timeout_sec == 60 + assert candidates[2].request_timeout_sec == 45 + assert candidates[0].num_predict == 2048 + assert candidates[2].num_predict == 512 + assert candidates[0].num_ctx is None + assert candidates[2].num_ctx == 4096 + + +def test_model_identity_fails_closed_on_digest_drift(): + from services import nemotron_runtime_candidate_service as service + + candidate = service.build_nemotron_runtime_candidates()[0] + identity = service.inspect_nemotron_model_identity( + candidate, + request_get=lambda *_args, **_kwargs: _Response({ + "models": [{ + "name": candidate.model, + "model": candidate.model, + "digest": "deadbeef", + "details": {}, + }] + }), + ) + + assert identity["ok"] is False + assert identity["digest_matches"] is False + assert identity["error"] == "model_digest_mismatch" + + +def test_model_identity_reports_network_failure_without_loading_model(): + from services import nemotron_runtime_candidate_service as service + + candidate = service.build_nemotron_runtime_candidates()[2] + + def fail(*_args, **_kwargs): + raise requests.Timeout("offline") + + identity = service.inspect_nemotron_model_identity( + candidate, + request_get=fail, + ) + + assert identity["ok"] is False + assert identity["digest"] is None + assert identity["error"].startswith("Timeout: offline") + + +@pytest.mark.parametrize( + ("payload", "error_fragment"), + [ + (None, "malformed_tags_payload"), + ({"models": {}}, "malformed_models_schema"), + ({"models": [None]}, "malformed_models_schema"), + ({"models": [{"name": "other", "details": None}]}, "malformed_model_details"), + ], +) +def test_model_identity_fails_closed_on_malformed_tags_schema( + payload, error_fragment +): + from services import nemotron_runtime_candidate_service as service + + candidate = service.build_nemotron_runtime_candidates()[0] + identity = service.inspect_nemotron_model_identity( + candidate, + request_get=lambda *_args, **_kwargs: _Response(payload), + ) + + assert identity["ok"] is False + assert identity["digest_matches"] is False + assert error_fragment in str(identity["error"]) + + +def test_model_identity_rejects_malformed_entry_mixed_with_valid_target(): + from services import nemotron_runtime_candidate_service as service + + candidate = service.build_nemotron_runtime_candidates()[0] + identity = service.inspect_nemotron_model_identity( + candidate, + request_get=lambda *_args, **_kwargs: _Response({ + "models": [ + None, + { + "name": candidate.model, + "model": candidate.model, + "digest": candidate.expected_digest, + "details": {}, + }, + ] + }), + ) + + assert identity["ok"] is False + assert identity["digest_matches"] is False + assert identity["error"] == "malformed_models_schema" + + +def test_candidate_attempt_timeout_honors_env_but_111_resources_stay_fixed(): + env = os.environ.copy() + env.update({ + "NEMOTRON_111_ATTEMPT_TIMEOUT_SEC": "33", + "OLLAMA_111_NUM_CTX": "2048", + "OLLAMA_111_NUM_PREDICT": "256", + }) + completed = subprocess.run( + [ + sys.executable, + "-c", + ( + "import json; " + "from services.nemotron_runtime_candidate_service import " + "build_nemotron_runtime_candidates; " + "c=build_nemotron_runtime_candidates()[-1]; " + "print(json.dumps(c.as_public_dict(), sort_keys=True))" + ), + ], + cwd=os.path.dirname(os.path.dirname(__file__)), + env=env, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + fallback = json.loads(completed.stdout.strip()) + assert fallback["request_timeout_sec"] == 33 + assert fallback["num_ctx"] == 4096 + assert fallback["num_predict"] == 512 + + +@pytest.mark.parametrize( + ("num_ctx", "num_predict"), + [ + ("99999", "99999"), + ("-7", "-9"), + ("not-a-number", "invalid"), + ], +) +def test_candidate_resource_contract_cannot_be_weakened_by_environment( + num_ctx, num_predict +): + env = os.environ.copy() + env.update({ + "OLLAMA_111_NUM_CTX": num_ctx, + "OLLAMA_111_NUM_PREDICT": num_predict, + }) + completed = subprocess.run( + [ + sys.executable, + "-c", + ( + "import json; " + "from services.nemotron_runtime_candidate_service import " + "build_nemotron_runtime_candidates; " + "c=build_nemotron_runtime_candidates()[-1]; " + "print(json.dumps(c.as_public_dict(), sort_keys=True))" + ), + ], + cwd=os.path.dirname(os.path.dirname(__file__)), + env=env, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + fallback = json.loads(completed.stdout.strip()) + assert fallback["num_ctx"] == 4096 + assert fallback["num_predict"] == 512 diff --git a/tests/test_phase3f_cleanup_contracts.py b/tests/test_phase3f_cleanup_contracts.py index a0b6061..20a0178 100644 --- a/tests/test_phase3f_cleanup_contracts.py +++ b/tests/test_phase3f_cleanup_contracts.py @@ -132,8 +132,15 @@ def test_env_example_documents_runtime_and_ai_automation_variables(): "N8N_USER", "N8N_WEBHOOK_BASE_URL", "NEMOTRON_OLLAMA_FIRST", + "NEMOTRON_OLLAMA_EXPECTED_DIGEST", + "NEMOTRON_OLLAMA_FALLBACK_EXPECTED_DIGEST", + "NEMOTRON_OLLAMA_FALLBACK_MODEL", "NEMOTRON_OLLAMA_MODEL", "NEMOTRON_OLLAMA_TIMEOUT", + "NEMOTRON_GCP_ATTEMPT_TIMEOUT_SEC", + "NEMOTRON_111_ATTEMPT_TIMEOUT_SEC", + "NEMOTRON_DISPATCH_TOTAL_TIMEOUT_SEC", + "NEMOTRON_MODEL_IDENTITY_TIMEOUT_SEC", "OLLAMA_EMBED_TIMEOUT", "OPENCLAW_ADMIN_USER_IDS", "OPENCLAW_AGENT_DISPATCH", diff --git a/tests/test_qwen3_runtime_usage.py b/tests/test_qwen3_runtime_usage.py index 77c9e5e..92e1b2a 100644 --- a/tests/test_qwen3_runtime_usage.py +++ b/tests/test_qwen3_runtime_usage.py @@ -7,6 +7,7 @@ ROOT = Path(__file__).resolve().parents[1] def test_qwen3_is_active_runtime_model_not_unused_ollama_weight(): openclaw_source = (ROOT / "services" / "openclaw_strategist_service.py").read_text(encoding="utf-8") nemotron_source = (ROOT / "services" / "nemoton_dispatcher_service.py").read_text(encoding="utf-8") + candidate_source = (ROOT / "services" / "nemotron_runtime_candidate_service.py").read_text(encoding="utf-8") router_source = (ROOT / "services" / "llm_model_router.py").read_text(encoding="utf-8") assert "OPENCLAW_QA_OLLAMA_MODEL = os.getenv('OPENCLAW_QA_OLLAMA_MODEL', 'qwen3:14b')" in openclaw_source @@ -16,7 +17,11 @@ def test_qwen3_is_active_runtime_model_not_unused_ollama_weight(): assert "OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE = os.getenv('OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE', '5m')" in openclaw_source assert "keep_alive=OPENCLAW_STRATEGY_OLLAMA_KEEP_ALIVE" in openclaw_source assert 'keep_alive="24h"' not in openclaw_source - assert 'NEMOTRON_OLLAMA_MODEL = os.getenv("NEMOTRON_OLLAMA_MODEL", "qwen3:14b")' in nemotron_source + assert "NEMOTRON_OLLAMA_MODEL = PRIMARY_MODEL" in nemotron_source assert "def _call_qwen3_dispatch(" in nemotron_source - assert "for _attempt in range(3):" in nemotron_source + assert "for candidate in build_nemotron_runtime_candidates(" in nemotron_source + assert "inspect_nemotron_model_identity(" in nemotron_source + assert '"qwen3:14b"' in candidate_source + assert '"qwen3:8b"' in candidate_source + assert 'label="ollama_111_fallback"' in candidate_source assert "'qwen3:14b'" in router_source