From 5dde049a07fd200d88d18caa2e156ce138bb7ece Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 21:56:00 +0800 Subject: [PATCH] fix(ops): report Ollama embed failures accurately --- governance/pchome_growth_program_review.json | 42 ++++++++++++++++---- scripts/ops/diagnose_ollama_gcp_failover.sh | 12 +++++- tests/test_ollama_failover_diagnostic.py | 15 +++++++ 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 tests/test_ollama_failover_diagnostic.py diff --git a/governance/pchome_growth_program_review.json b/governance/pchome_growth_program_review.json index 1d9b319..f4aa317 100644 --- a/governance/pchome_growth_program_review.json +++ b/governance/pchome_growth_program_review.json @@ -1,6 +1,6 @@ { "version": "pchome_growth_program_review_v1", - "generated_at": "2026-07-10T20:43:51+08:00", + "generated_at": "2026-07-10T21:54:36+08:00", "supersedes_status_only_completion": true, "goal": "以新鮮 PChome 業績、可信跨平台同款資料與可歸因動作持續提升業績", "production_baseline": { @@ -22,6 +22,29 @@ "runtime_mcp_router_enabled": false, "runtime_rag_enabled": false }, + "implementation_receipt": { + "feature_source_sha": "b8cc27e4c4ae8cc6a649bc04451b7e94d0611bef", + "dev_integration_sha": "107d8ed", + "runtime_version": "V10.775", + "deployment_mode": "controlled_sync_with_targeted_rollback_archive", + "gitea_cd_run": 1098, + "gitea_cd_status": "waiting_runner_unavailable", + "application_containers_healthy": 3, + "database_container_recreated": false, + "latest_sales_date": "2026-06-24", + "sales_decision_ready": false, + "valid_catalog_mapping_count": 31, + "valid_catalog_mapping_rate_pct": 0.24, + "runtime_platforms_with_data": 1, + "catchup_result": { + "scanned": 8, + "candidates": 3, + "auto_accepted": 0, + "written": 0, + "review_candidates": 3, + "next_due": "cooldown_4.0h" + } + }, "codebase_findings": { "relevant_file_count_approx": 250, "market_intel_module_count": 140, @@ -105,7 +128,7 @@ "order": 4, "id": "PCH-GROWTH-P0-004", "priority": "P0", - "status": "source_complete_runtime_pending", + "status": "runtime_deployed_dependency_blocked", "lane": "cross_platform_revenue_decision", "objective": "Compare every qualified platform offer without letting blocked or observational evidence drive price actions", "completed": [ @@ -113,11 +136,12 @@ "runtime source-policy enforcement", "observation-only evidence separation", "sales freshness decision gate", - "catalog and platform coverage metrics" + "catalog and platform coverage metrics", + "Gitea main and dev integration", + "production V10.775 service-layer readback", + "bounded mapping catch-up receipt" ], "remaining": [ - "Gitea deployment", - "production API readback", "production multi-source offer ingestion" ], "production_acceptance": "fresh sales plus at least two enabled verified platforms produce replayable decisions" @@ -143,7 +167,7 @@ "order": 6, "id": "PCH-GROWTH-P1-001", "priority": "P1", - "status": "source_complete_runtime_pending", + "status": "runtime_deployed_visible_auth_pending", "lane": "professional_ui_ux", "objective": "Provide a compact decision cockpit instead of a status and text wall", "completed": [ @@ -153,7 +177,8 @@ "collapsed pending-source summary", "removed fake sales sparkline", "desktop and 390px mobile visual verification without horizontal overflow", - "repaired the circular accent-token fallback that made primary buttons unreadable" + "repaired the circular accent-token fallback that made primary buttons unreadable", + "production V10.775 static asset readback" ], "remaining": [ "production visible readback", @@ -178,6 +203,9 @@ "enable and verify RAG runtime", "Qwen3-VL Ollama-first visual replay", "PixelRAG receipt to RAG candidate replay", + "restore GCP-A direct runtime and 110 primary proxy", + "recover GCP-B bge-m3 embedding latency below the 30 second verifier SLO", + "restore or formally retire the unreachable 111 fallback route", "agent outcome metrics and durable learning acknowledgement" ], "production_acceptance": "runtime flags enabled and same-run retrieval-to-decision receipts pass independent replay" diff --git a/scripts/ops/diagnose_ollama_gcp_failover.sh b/scripts/ops/diagnose_ollama_gcp_failover.sh index 8afa46a..30880bd 100755 --- a/scripts/ops/diagnose_ollama_gcp_failover.sh +++ b/scripts/ops/diagnose_ollama_gcp_failover.sh @@ -108,8 +108,16 @@ curl_probe "110 proxy secondary" "$PROXY_SECONDARY_URL" "/api/version" 10 && pro if [[ "$RUN_EMBED_PROBE" == "1" ]]; then say "" say "Embedding runtime probe model=${EMBED_MODEL} timeout=${EMBED_TIMEOUT}s" - [[ "$primary_version_ok" == "1" ]] && embed_probe "GCP-A embed" "$PRIMARY_URL" || status_line WARN "GCP-A embed" "skipped because /api/version is not healthy" - [[ "$secondary_version_ok" == "1" ]] && embed_probe "GCP-B embed" "$SECONDARY_URL" || status_line WARN "GCP-B embed" "skipped because /api/version is not healthy" + if [[ "$primary_version_ok" == "1" ]]; then + embed_probe "GCP-A embed" "$PRIMARY_URL" || true + else + status_line WARN "GCP-A embed" "skipped because /api/version is not healthy" + fi + if [[ "$secondary_version_ok" == "1" ]]; then + embed_probe "GCP-B embed" "$SECONDARY_URL" || true + else + status_line WARN "GCP-B embed" "skipped because /api/version is not healthy" + fi if [[ "$INCLUDE_111_EMBED" == "1" ]]; then embed_probe "111 embed" "$FALLBACK_URL" || true else diff --git a/tests/test_ollama_failover_diagnostic.py b/tests/test_ollama_failover_diagnostic.py new file mode 100644 index 0000000..be4e705 --- /dev/null +++ b/tests/test_ollama_failover_diagnostic.py @@ -0,0 +1,15 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_embedding_probe_failure_is_not_reported_as_version_skip(): + script = (ROOT / "scripts" / "ops" / "diagnose_ollama_gcp_failover.sh").read_text( + encoding="utf-8" + ) + + assert 'if [[ "$primary_version_ok" == "1" ]]; then' in script + assert 'if [[ "$secondary_version_ok" == "1" ]]; then' in script + assert 'embed_probe "GCP-B embed" "$SECONDARY_URL" || true' in script + assert '&& embed_probe "GCP-B embed"' not in script