From 541a32a9cbf0af2fd5782f19f52a737e38db370b Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 16 Jul 2026 17:32:01 +0800 Subject: [PATCH] feat(sre): enforce typed controlled automation --- .gitea/workflows/cd.yaml | 1 + agent99-alertmanager-alertchain-poll.ps1 | 316 +++++ agent99-bootstrap.ps1 | 27 +- agent99-contract-check.ps1 | 25 +- agent99-deploy.ps1 | 78 ++ agent99-install-sre-alert-relay.ps1 | 50 +- agent99-sre-alert-relay.ps1 | 76 +- agent99.config.99.example.json | 24 +- agent99.config.example.json | 24 +- apps/api/alert_rules.yaml | 44 +- apps/api/src/api/v1/agents.py | 40 + apps/api/src/api/v1/ai.py | 5 + apps/api/src/api/v1/health.py | 5 + apps/api/src/api/v1/platform/operator_runs.py | 83 +- apps/api/src/api/v1/webhooks.py | 81 +- apps/api/src/core/config.py | 14 + .../awooop_ansible_candidate_backfill_job.py | 45 +- apps/api/src/services/ai_control.py | 449 ++++++- apps/api/src/services/ai_provider_policy.py | 38 +- apps/api/src/services/ai_providers/claude.py | 14 +- apps/api/src/services/ai_providers/gemini.py | 118 +- .../src/services/ai_providers/interfaces.py | 19 + apps/api/src/services/ai_providers/ollama.py | 145 ++- apps/api/src/services/ai_rate_limiter.py | 549 +++++++- apps/api/src/services/ai_router.py | 390 +++++- .../services/alert_chain_emergency_ingress.py | 517 ++++++++ apps/api/src/services/alert_rule_engine.py | 18 +- .../services/awooop_ansible_audit_service.py | 54 +- .../services/awooop_ansible_post_verifier.py | 119 +- .../src/services/cloud_transport_receipt.py | 319 +++++ .../controlled_alert_target_router.py | 282 +++- apps/api/src/services/failover_alerter.py | 120 +- apps/api/src/services/model_version_probe.py | 4 +- .../src/services/ollama_failover_manager.py | 67 +- apps/api/src/services/openclaw.py | 60 +- .../paid_provider_canary_authorization.py | 465 +++++++ .../paid_provider_canary_gate5_run.py | 385 ++++++ .../paid_provider_canary_validation.py | 1137 +++++++++++++++++ .../src/services/platform_operator_service.py | 19 +- ...lio_infrastructure_asset_reconciliation.py | 280 ++++ ...re_k3s_controlled_automation_work_items.py | 67 +- apps/api/src/services/telegram_gateway.py | 20 + .../test_gemini_cost_guard_real_redis.py | 480 +++++-- .../tests/test_ai_control_provider_policy.py | 211 ++- .../tests/test_ai_provider_status_readback.py | 39 + ...est_ai_route_observed_fallback_readback.py | 5 + .../test_ai_router_cache_provider_policy.py | 106 +- ..._ai_router_circuit_identity_fail_closed.py | 327 +++++ .../tests/test_ai_router_diagnose_fallback.py | 40 +- .../test_ai_router_failover_integration.py | 10 +- .../test_alert_chain_emergency_ingress.py | 466 +++++++ .../tests/test_alert_chain_runtime_truth.py | 95 ++ .../tests/test_alertmanager_rule_bypass.py | 80 ++ .../test_awooop_operator_timeline_labels.py | 16 +- apps/api/tests/test_claude_paid_fallback.py | 40 +- .../api/tests/test_cloud_transport_receipt.py | 179 +++ apps/api/tests/test_failover_alerter.py | 85 +- apps/api/tests/test_failover_e2e_dispatch.py | 8 + apps/api/tests/test_gemini_cost_guard.py | 279 +++- .../tests/test_gemini_provider_security.py | 88 +- apps/api/tests/test_gemini_route_manifest.py | 24 +- .../test_generate_monitoring_stabilization.py | 95 +- .../test_host111_ollama_ansible_catalog.py | 203 +++ ...est_monitoring_provider_and_gitea_truth.py | 117 ++ .../api/tests/test_ollama_failover_manager.py | 37 + .../tests/test_ollama_prod_manifest_order.py | 95 +- .../tests/test_ollama_provider_endpoints.py | 123 +- ...test_openclaw_alert_cloud_fallback_gate.py | 24 +- .../test_openclaw_legacy_ollama_failover.py | 48 +- ...st_openclaw_paid_cloud_privacy_boundary.py | 84 +- ...test_paid_provider_canary_authorization.py | 256 ++++ .../test_paid_provider_canary_gate5_run.py | 408 ++++++ .../tests/test_paid_provider_canary_runner.py | 105 ++ .../test_paid_provider_canary_validation.py | 1076 ++++++++++++++++ ...lio_infrastructure_asset_reconciliation.py | 209 +++ ...infrastructure_asset_reconciliation_api.py | 82 ++ ...3s_controlled_automation_work_items_api.py | 137 +- .../api/tests/test_sre_typed_domain_router.py | 272 +++- ...test_telegram_delivery_truth_callers_v2.py | 21 +- ...astructure-asset-reconciliation-handoff.md | 20 + ...ructure-asset-reconciliation.snapshot.json | 614 +++++++++ ...rolled-automation-work-items.snapshot.json | 258 +++- docs/runbooks/ANSIBLE-OPERATING-MODEL.md | 2 +- docs/runbooks/AWOOOP-INFERENCE-GATEWAY.md | 4 +- docs/runbooks/DEPLOY-GCP-OLLAMA-PROXY.md | 232 +--- docs/runbooks/GCP-OLLAMA-WIREGUARD-MESH.md | 18 +- docs/runbooks/OLLAMA-188-RETIREMENT-GATE.md | 14 +- docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md | 291 +---- .../ansible/inventory/group_vars/host_110.yml | 1 - .../110-alertmanager-delivery-recovery.yml | 159 +++ infra/ansible/playbooks/110-devops.yml | 11 - .../ansible/playbooks/111-ollama-fallback.yml | 83 +- .../ansible/playbooks/nginx-sync-readonly.yml | 51 +- infra/ansible/playbooks/nginx-sync.yml | 101 +- .../nginx/templates/110-ollama-proxy.conf.j2 | 104 -- k8s/awoooi-dev/02-configmap.yaml | 8 +- k8s/awoooi-prod/02-network-policy.yaml | 13 +- k8s/awoooi-prod/04-configmap.yaml | 29 +- k8s/awoooi-prod/06-deployment-api.yaml | 6 +- k8s/monitoring/alert-chain-monitor.yaml | 34 +- k8s/monitoring/docker-compose-110.yml | 22 - k8s/monitoring/prometheus.yml | 42 +- ops/alertmanager/alertmanager.yml | 25 +- ops/monitoring/alerts-unified.yml | 95 +- ops/monitoring/alerts.yml | 95 +- ops/monitoring/coverage_report.py | 25 +- ops/monitoring/generate_monitoring.py | 62 +- .../generated/blackbox-targets-generated.yaml | 23 +- .../prometheus-scrape-generated.yaml | 171 ++- .../grafana/dashboards/ollama_failover.json | 19 +- ops/monitoring/ollama_health_rules.yaml | 30 +- ops/monitoring/service-registry.yaml | 117 +- ops/monitoring/validate_coverage.py | 15 +- ops/nginx/deploy-ollama-proxy-110.sh | 127 +- scripts/generate_monitoring.py | 207 ++- scripts/ops/deploy-alertmanager-config.sh | 134 +- .../ops/deploy-alertmanager-runtime-scrape.sh | 413 ++++++ scripts/ops/deploy-alerts.sh | 10 + .../ops/ollama111-fallback-proxy-diagnose.sh | 99 +- scripts/ops/run-paid-provider-canary.py | 64 + ...st_alertmanager_runtime_scrape_contract.py | 38 + .../tests/test_alertmanager_webhook_config.py | 68 + .../agent99-live-preflight.ps1 | 2 +- .../agent99-remote-atomic-deploy-receiver.ps1 | 13 +- .../deploy-agent99-via-windows99-ssh.sh | 9 +- .../test_agent99_live_preflight_decision.py | 4 +- ...remote_atomic_deploy_transport_contract.py | 15 +- .../ai-provider-owner-response-acceptance.py | 15 +- .../security/high-value-config-change-gate.py | 3 +- .../security/nginx-config-drift-detector.py | 9 - 130 files changed, 14849 insertions(+), 1738 deletions(-) create mode 100644 agent99-alertmanager-alertchain-poll.ps1 create mode 100644 apps/api/src/services/alert_chain_emergency_ingress.py create mode 100644 apps/api/src/services/cloud_transport_receipt.py create mode 100644 apps/api/src/services/paid_provider_canary_authorization.py create mode 100644 apps/api/src/services/paid_provider_canary_gate5_run.py create mode 100644 apps/api/src/services/paid_provider_canary_validation.py create mode 100644 apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py create mode 100644 apps/api/tests/test_ai_router_circuit_identity_fail_closed.py create mode 100644 apps/api/tests/test_alert_chain_emergency_ingress.py create mode 100644 apps/api/tests/test_alert_chain_runtime_truth.py create mode 100644 apps/api/tests/test_cloud_transport_receipt.py create mode 100644 apps/api/tests/test_host111_ollama_ansible_catalog.py create mode 100644 apps/api/tests/test_monitoring_provider_and_gitea_truth.py create mode 100644 apps/api/tests/test_paid_provider_canary_authorization.py create mode 100644 apps/api/tests/test_paid_provider_canary_gate5_run.py create mode 100644 apps/api/tests/test_paid_provider_canary_runner.py create mode 100644 apps/api/tests/test_paid_provider_canary_validation.py create mode 100644 apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py create mode 100644 apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py create mode 100644 docs/operations/portfolio-infrastructure-asset-reconciliation-handoff.md create mode 100644 docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json create mode 100644 infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml delete mode 100644 infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2 mode change 100755 => 100644 ops/nginx/deploy-ollama-proxy-110.sh create mode 100755 scripts/ops/deploy-alertmanager-runtime-scrape.sh mode change 100755 => 100644 scripts/ops/ollama111-fallback-proxy-diagnose.sh create mode 100644 scripts/ops/run-paid-provider-canary.py create mode 100644 scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 33091139a..d5e07e93d 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -1087,6 +1087,7 @@ jobs: ../../scripts/backup/gitea-bundle-sample-restore-dry-run.sh \ ../../scripts/ops/backup-from-110.sh \ ../../scripts/ops/deploy-alertmanager-config.sh \ + ../../scripts/ops/deploy-alertmanager-runtime-scrape.sh \ ../../scripts/ops/docker-health-monitor.sh \ ../../scripts/ops/dr-drill.sh \ ../../scripts/ops/notify-awoooi-ops.sh \ diff --git a/agent99-alertmanager-alertchain-poll.ps1 b/agent99-alertmanager-alertchain-poll.ps1 new file mode 100644 index 000000000..b4214b14c --- /dev/null +++ b/agent99-alertmanager-alertchain-poll.ps1 @@ -0,0 +1,316 @@ +param( + [string]$AgentRoot = "C:\Wooo\Agent99", + [string]$ConfigPath = "", + [string]$SourceUrl = "", + [string]$SourceHost = "", + [int]$PollIntervalSeconds = 0, + [string]$IngressUrl = "", + [string]$TokenEnv = "" +) + +$ErrorActionPreference = "Stop" + +$CanonicalSourceHost = "192.168.0.110" +$CanonicalSourceUrl = "http://192.168.0.110:9093/api/v2/alerts" +$CanonicalIngressUrl = "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency" +$CanonicalAlertName = "AlertChainBroken_Alertmanager" +$CanonicalCatalogId = "ansible:110-alertmanager-delivery-recovery" +$DefaultTokenEnv = "AGENT99_SRE_RELAY_TOKEN" + +function Get-AgentField { + param( + [object]$Object, + [string]$Name, + [object]$Default = $null + ) + if ($Object -is [Collections.IDictionary] -and $Object.Contains($Name)) { + return $Object[$Name] + } + if ($Object -and $Object.PSObject.Properties[$Name]) { + return $Object.PSObject.Properties[$Name].Value + } + return $Default +} + +function Get-AgentEnvValue { + param([string]$Name) + foreach ($target in @("Process", "User", "Machine")) { + $value = [Environment]::GetEnvironmentVariable($Name, $target) + if ($value) { return $value } + } + return "" +} + +function Get-AgentSha256Text { + param([string]$Value) + $sha = [Security.Cryptography.SHA256]::Create() + try { + return [BitConverter]::ToString( + $sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($Value)) + ).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Write-AgentJsonAtomically { + param( + [string]$Path, + [object]$Value + ) + $temporaryPath = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $Value | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8 + Move-Item -Force $temporaryPath $Path +} + +if (-not $ConfigPath) { + $ConfigPath = Join-Path $AgentRoot "config\agent99.config.json" +} + +$configuredPoll = $null +if (Test-Path -LiteralPath $ConfigPath -PathType Leaf) { + $config = Get-Content -LiteralPath $ConfigPath -Raw | ConvertFrom-Json + $relay = Get-AgentField $config "sreAlertRelay" $null + $configuredPoll = Get-AgentField $relay "alertChainPoll" $null +} + +if (-not $SourceUrl) { + $SourceUrl = [string](Get-AgentField $configuredPoll "sourceUrl" $CanonicalSourceUrl) +} +if (-not $SourceHost) { + $SourceHost = [string](Get-AgentField $configuredPoll "sourceHost" $CanonicalSourceHost) +} +if ($PollIntervalSeconds -le 0) { + $PollIntervalSeconds = [int](Get-AgentField $configuredPoll "pollIntervalSeconds" 60) +} +if (-not $IngressUrl) { + $IngressUrl = [string](Get-AgentField $configuredPoll "ingressUrl" $CanonicalIngressUrl) +} +if (-not $TokenEnv) { + $TokenEnv = [string](Get-AgentField $configuredPoll "tokenEnv" $DefaultTokenEnv) +} + +$sourceUri = [Uri]$SourceUrl +$ingressUri = [Uri]$IngressUrl +if ( + $SourceHost -ne $CanonicalSourceHost -or + $SourceUrl -ne $CanonicalSourceUrl -or + $sourceUri.Scheme -ne "http" -or + $sourceUri.Host -ne $CanonicalSourceHost -or + $sourceUri.Port -ne 9093 -or + $sourceUri.AbsolutePath -ne "/api/v2/alerts" -or + $sourceUri.Query -or + $sourceUri.UserInfo +) { + throw "alertmanager_poll_source_identity_not_exact" +} +if ( + $IngressUrl -ne $CanonicalIngressUrl -or + $ingressUri.Scheme -ne "https" -or + $ingressUri.Host -ne "awoooi.wooo.work" -or + $ingressUri.AbsolutePath -ne "/api/v1/webhooks/agent99/alertmanager-emergency" -or + $ingressUri.Query -or + $ingressUri.UserInfo +) { + throw "alertmanager_poll_ingress_identity_not_exact" +} +if ( + $PollIntervalSeconds -lt 60 -or + $PollIntervalSeconds -gt 300 -or + ($PollIntervalSeconds % 60) -ne 0 +) { + throw "alertmanager_poll_interval_out_of_bounds" +} +if ($TokenEnv -ne $DefaultTokenEnv) { + throw "alertmanager_poll_token_env_not_allowlisted" +} + +$EvidenceDir = Join-Path $AgentRoot "evidence\alertmanager-alertchain-poll" +$ReceiptDir = Join-Path $EvidenceDir "dedupe-receipts" +New-Item -ItemType Directory -Force -Path $EvidenceDir, $ReceiptDir | Out-Null + +$pollStartedAt = (Get-Date).ToUniversalTime() +$summary = [ordered]@{ + schemaVersion = "agent99_alertmanager_alertchain_poll_summary_v1" + status = "poll_in_progress" + sourceUrl = $CanonicalSourceUrl + sourceHost = $CanonicalSourceHost + pollIntervalSeconds = $PollIntervalSeconds + alertname = $CanonicalAlertName + firstHopAuthenticationUsed = $false + firstHopSecretTransmitted = $false + observedCount = 0 + exactFiringCount = 0 + relayedCount = 0 + deduplicatedCount = 0 + rejectedExactCount = 0 + failedCount = 0 + rawPayloadStored = $false + secretValueLogged = $false + runtimeWritePerformed = $false + runtimeClosureVerified = $false + polledAt = $pollStartedAt.ToString("o") +} + +try { + # First hop is deliberately unauthenticated read-only HTTP on the exact LAN + # host. No credential, bearer header, query parameter or raw payload is saved. + $alerts = @(Invoke-RestMethod ` + -Method Get ` + -Uri $CanonicalSourceUrl ` + -TimeoutSec 15) +} catch { + $summary.status = "alertmanager_poll_source_unavailable" + $summary.failedCount = 1 + $summary.errorType = $_.Exception.GetType().Name + $summary.completedAt = (Get-Date).ToUniversalTime().ToString("o") + $summaryPath = Join-Path $EvidenceDir "poll-$($pollStartedAt.ToString('yyyyMMdd-HHmmss'))-$([guid]::NewGuid().ToString('N')).json" + Write-AgentJsonAtomically -Path $summaryPath -Value $summary + $summary | ConvertTo-Json -Compress -Depth 8 + throw "alertmanager_poll_source_unavailable" +} + +$summary.observedCount = $alerts.Count +foreach ($alert in $alerts) { + $labels = Get-AgentField $alert "labels" $null + if ([string](Get-AgentField $labels "alertname" "") -ne $CanonicalAlertName) { + continue + } + + $status = Get-AgentField $alert "status" $null + $severity = [string](Get-AgentField $labels "severity" "") + $state = [string](Get-AgentField $status "state" "") + $fingerprint = ([string](Get-AgentField $alert "fingerprint" "")).Trim().ToLowerInvariant() + $startsAt = ([string](Get-AgentField $alert "startsAt" "")).Trim() + $parsedStartsAt = [DateTimeOffset]::MinValue + if ( + $severity -ne "critical" -or + $state -ne "active" -or + $fingerprint -notmatch "^[0-9a-f]{8,64}$" -or + -not [DateTimeOffset]::TryParse($startsAt, [ref]$parsedStartsAt) + ) { + $summary.rejectedExactCount++ + continue + } + + $summary.exactFiringCount++ + $sourceStartedAt = $parsedStartsAt.UtcDateTime.ToString("o") + $receiptDigest = Get-AgentSha256Text "$fingerprint|$sourceStartedAt" + $relayReceiptId = "agent99-alertchain-$receiptDigest" + $receiptPath = Join-Path $ReceiptDir "$relayReceiptId.json" + if (Test-Path -LiteralPath $receiptPath -PathType Leaf) { + try { + $existing = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json + if ( + [string](Get-AgentField $existing "relayReceiptId" "") -eq $relayReceiptId -and + [string](Get-AgentField $existing "sourceFingerprint" "") -eq $fingerprint -and + [bool](Get-AgentField $existing "candidateQueued" $false) -and + -not [bool](Get-AgentField $existing "rawPayloadStored" $true) + ) { + $summary.deduplicatedCount++ + continue + } + } catch { + # A malformed public receipt cannot suppress retry. + } + } + + $token = Get-AgentEnvValue $TokenEnv + if (-not $token) { + $summary.failedCount++ + continue + } + + $envelope = [ordered]@{ + schema_version = "agent99_alertmanager_emergency_relay_v1" + relay_receipt_id = $relayReceiptId + source_host = $CanonicalSourceHost + source_url = $CanonicalSourceUrl + source_fingerprint = $fingerprint + source_started_at = $sourceStartedAt + alertname = $CanonicalAlertName + status = "firing" + severity = "critical" + target_resource = "alertmanager" + namespace = "monitoring" + agent99_role = "relay_coordination_only" + executor = "host_ansible_executor" + catalog_id = $CanonicalCatalogId + runtime_write_performed = $false + raw_payload_stored = $false + } + + try { + $response = Invoke-RestMethod ` + -Method Post ` + -Uri $CanonicalIngressUrl ` + -Headers @{ "X-Agent99-Relay-Token" = $token } ` + -ContentType "application/json; charset=utf-8" ` + -Body ($envelope | ConvertTo-Json -Compress -Depth 6) ` + -TimeoutSec 30 + } catch { + $summary.failedCount++ + continue + } + + $identity = Get-AgentField $response "identity" $null + $runId = [string](Get-AgentField $identity "run_id" "") + $traceId = [string](Get-AgentField $identity "trace_id" "") + $workItemId = [string](Get-AgentField $identity "work_item_id" "") + $parsedRunId = [guid]::Empty + $accepted = [bool]( + [string](Get-AgentField $response "schema_version" "") -eq "alert_chain_emergency_ingress_result_v1" -and + [bool](Get-AgentField $response "candidate_persisted" $false) -and + [bool](Get-AgentField $response "candidate_queued" $false) -and + [string](Get-AgentField $response "agent99_role" "") -eq "relay_coordination_only" -and + [string](Get-AgentField $response "executor" "") -eq "host_ansible_executor" -and + [string](Get-AgentField $response "catalog_id" "") -eq $CanonicalCatalogId -and + -not [bool](Get-AgentField $response "runtime_write_performed" $true) -and + -not [bool](Get-AgentField $response "runtime_closure_verified" $true) -and + [guid]::TryParse($runId, [ref]$parsedRunId) -and + $traceId -eq $runId -and + $workItemId.StartsWith("alertchain:awoooi:") + ) + if (-not $accepted) { + $summary.failedCount++ + continue + } + + $publicReceipt = [ordered]@{ + schemaVersion = "agent99_alertmanager_alertchain_poll_receipt_v1" + relayReceiptId = $relayReceiptId + sourceHost = $CanonicalSourceHost + sourceFingerprint = $fingerprint + sourceStartedAt = $sourceStartedAt + candidatePersisted = $true + candidateQueued = $true + runId = $runId + traceId = $traceId + workItemId = $workItemId + agent99Role = "relay_coordination_only" + executor = "host_ansible_executor" + catalogId = $CanonicalCatalogId + firstHopAuthenticationUsed = $false + firstHopSecretTransmitted = $false + runtimeWritePerformed = $false + runtimeClosureVerified = $false + rawPayloadStored = $false + secretValueLogged = $false + receivedAt = (Get-Date).ToUniversalTime().ToString("o") + } + Write-AgentJsonAtomically -Path $receiptPath -Value $publicReceipt + $summary.relayedCount++ +} + +$summary.status = if ($summary.failedCount -eq 0) { + "poll_completed" +} else { + "poll_partial_retry_required" +} +$summary.completedAt = (Get-Date).ToUniversalTime().ToString("o") +$summaryPath = Join-Path $EvidenceDir "poll-$($pollStartedAt.ToString('yyyyMMdd-HHmmss'))-$([guid]::NewGuid().ToString('N')).json" +Write-AgentJsonAtomically -Path $summaryPath -Value $summary +$summary | ConvertTo-Json -Compress -Depth 8 +if ($summary.failedCount -gt 0) { + throw "alertmanager_poll_partial_retry_required" +} diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 40cce581a..68771bfd5 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -224,6 +224,7 @@ $agentFiles = @( "agent99-control-plane.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", + "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-inbox.ps1", "agent99-sre-alert-relay.ps1", "agent99-submit-request.ps1", @@ -322,7 +323,22 @@ if (-not (Test-Path $configPath)) { "enabled": true, "prefix": "http://+:8787/agent99/sre-alert/", "tokenEnv": "AGENT99_SRE_RELAY_TOKEN", - "allowedRemotePrefixes": ["127.", "::1", "192.168.0."] + "allowedRemotePrefixes": ["127.", "::1", "192.168.0."], + "alertChainPoll": { + "enabled": true, + "sourceUrl": "http://192.168.0.110:9093/api/v2/alerts", + "sourceHost": "192.168.0.110", + "pollIntervalSeconds": 60, + "ingressUrl": "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency", + "tokenEnv": "AGENT99_SRE_RELAY_TOKEN", + "alertname": "AlertChainBroken_Alertmanager", + "agent99Role": "relay_coordination_only", + "linuxExecutor": "host_ansible_executor", + "catalogId": "ansible:110-alertmanager-delivery-recovery", + "firstHopAuthenticationUsed": false, + "firstHopSecretTransmitted": false, + "rawPayloadStored": false + } }, "telegram": { "enabled": true, @@ -348,6 +364,15 @@ if (-not (Test-Path $configPath)) { "Wooo-Agent99-SRE-Alert-Relay", "Wooo-Agent99-Self-Health", "Wooo-Agent99-Backup-Health" + ], + "evidenceFreshness": [ + { + "name": "alertmanager_alertchain_poll", + "pattern": "alertmanager-alertchain-poll\\poll-*.json", + "maxAgeMinutes": 5, + "required": true, + "runningTaskName": "Wooo-Agent99-SRE-Alert-Relay" + } ] }, "performance": { diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index da8ccdcc4..97137bd92 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -20,6 +20,7 @@ $requiredFiles = @( "agent99-control-plane.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", + "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-inbox.ps1", "agent99-sre-alert-relay.ps1", "agent99-submit-request.ps1", @@ -90,6 +91,26 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $config.completionCallback.url -match "^https://" -and $config.completionCallback.tokenEnv -eq "AGENT99_SRE_RELAY_TOKEN" ) + $alertChainPoll = if ($config.sreAlertRelay) { + $config.sreAlertRelay.alertChainPoll + } else { + $null + } + $hasAlertChainPoll = [bool]( + $alertChainPoll -and + $alertChainPoll.enabled -eq $true -and + [string]$alertChainPoll.sourceUrl -eq "http://192.168.0.110:9093/api/v2/alerts" -and + [string]$alertChainPoll.sourceHost -eq "192.168.0.110" -and + [int]$alertChainPoll.pollIntervalSeconds -eq 60 -and + [string]$alertChainPoll.ingressUrl -eq "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency" -and + [string]$alertChainPoll.tokenEnv -eq "AGENT99_SRE_RELAY_TOKEN" -and + [string]$alertChainPoll.alertname -eq "AlertChainBroken_Alertmanager" -and + [string]$alertChainPoll.linuxExecutor -eq "host_ansible_executor" -and + [string]$alertChainPoll.catalogId -eq "ansible:110-alertmanager-delivery-recovery" -and + $alertChainPoll.firstHopAuthenticationUsed -eq $false -and + $alertChainPoll.firstHopSecretTransmitted -eq $false -and + $alertChainPoll.rawPayloadStored -eq $false + ) $hasCanonicalHost112Vmx = if ($name -eq "agent99.config.99.example.json") { [bool](@($config.vms | Where-Object { [string]$_.name -eq "host112-kali" -and @@ -97,8 +118,8 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js [string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" }).Count -eq 1) } else { $true } - $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasHost188EdgeRecovery -and $hasCompletionCallback -and $hasCanonicalHost112Vmx - Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator host188EdgeRecovery=$hasHost188EdgeRecovery completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" + $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasHost188EdgeRecovery -and $hasCompletionCallback -and $hasAlertChainPoll -and $hasCanonicalHost112Vmx + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator host188EdgeRecovery=$hasHost188EdgeRecovery completionCallback=$hasCompletionCallback alertChainPoll=$hasAlertChainPoll canonicalHost112Vmx=$hasCanonicalHost112Vmx" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 200fbdbd9..41d7f1ed8 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -35,6 +35,7 @@ $runtimeFiles = @( "agent99-control-plane.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", + "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-inbox.ps1", "agent99-sre-alert-relay.ps1", "agent99-submit-request.ps1", @@ -319,6 +320,83 @@ try { } $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json + Add-DefaultProperty $config "sreAlertRelay" ([pscustomobject]@{}) + foreach ($legacyName in @("alertmanagerPrefix", "emergencyIngressUrlEnv")) { + if ($config.sreAlertRelay.PSObject.Properties[$legacyName]) { + $config.sreAlertRelay.PSObject.Properties.Remove($legacyName) + $script:ConfigMigrations += [pscustomobject]@{ + name = "remove_legacy_alertmanager_inbound_$legacyName" + from = "present" + to = "removed" + } + } + } + Add-DefaultProperty $config.sreAlertRelay "alertChainPoll" ([pscustomobject]@{}) + $alertChainPollContract = [ordered]@{ + enabled = $true + sourceUrl = "http://192.168.0.110:9093/api/v2/alerts" + sourceHost = "192.168.0.110" + pollIntervalSeconds = 60 + ingressUrl = "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency" + tokenEnv = "AGENT99_SRE_RELAY_TOKEN" + alertname = "AlertChainBroken_Alertmanager" + agent99Role = "relay_coordination_only" + linuxExecutor = "host_ansible_executor" + catalogId = "ansible:110-alertmanager-delivery-recovery" + firstHopAuthenticationUsed = $false + firstHopSecretTransmitted = $false + rawPayloadStored = $false + } + foreach ($entry in $alertChainPollContract.GetEnumerator()) { + $name = [string]$entry.Key + $expected = $entry.Value + $current = if ($config.sreAlertRelay.alertChainPoll.PSObject.Properties[$name]) { + $config.sreAlertRelay.alertChainPoll.PSObject.Properties[$name].Value + } else { + $null + } + if ($null -eq $current -or [string]$current -ne [string]$expected) { + Set-AgentProperty $config.sreAlertRelay.alertChainPoll $name $expected + $script:ConfigMigrations += [pscustomobject]@{ + name = "alertchain_poll_$name" + from = if ($null -eq $current) { "missing" } else { [string]$current } + to = [string]$expected + } + } + } + Add-DefaultProperty $config "selfHealth" ([pscustomobject]@{}) + Add-DefaultProperty $config.selfHealth "evidenceFreshness" @() + $freshnessRows = @($config.selfHealth.evidenceFreshness) + $existingPollFreshness = @( + $freshnessRows | + Where-Object { [string]$_.name -eq "alertmanager_alertchain_poll" } + ) + $pollFreshnessValid = [bool]( + $existingPollFreshness.Count -eq 1 -and + [string]$existingPollFreshness[0].pattern -eq "alertmanager-alertchain-poll\poll-*.json" -and + [int]$existingPollFreshness[0].maxAgeMinutes -eq 5 -and + [bool]$existingPollFreshness[0].required -and + [string]$existingPollFreshness[0].runningTaskName -eq "Wooo-Agent99-SRE-Alert-Relay" + ) + if (-not $pollFreshnessValid) { + $freshnessRows = @( + $freshnessRows | + Where-Object { [string]$_.name -ne "alertmanager_alertchain_poll" } + ) + $freshnessRows += [pscustomobject]@{ + name = "alertmanager_alertchain_poll" + pattern = "alertmanager-alertchain-poll\poll-*.json" + maxAgeMinutes = 5 + required = $true + runningTaskName = "Wooo-Agent99-SRE-Alert-Relay" + } + Set-AgentProperty $config.selfHealth "evidenceFreshness" $freshnessRows + $script:ConfigMigrations += [pscustomobject]@{ + name = "alertchain_poll_freshness" + from = if ($existingPollFreshness.Count -eq 0) { "missing" } else { "drifted" } + to = "required_5_minutes_on_outbound_relay_task" + } + } Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{}) $previousHost111User = if ($config.sshUsers.PSObject.Properties["192.168.0.111"]) { [string]$config.sshUsers.PSObject.Properties["192.168.0.111"].Value } else { "" } diff --git a/agent99-install-sre-alert-relay.ps1 b/agent99-install-sre-alert-relay.ps1 index 30b9f80d0..3fc721c88 100644 --- a/agent99-install-sre-alert-relay.ps1 +++ b/agent99-install-sre-alert-relay.ps1 @@ -2,7 +2,11 @@ param( [string]$AgentRoot = "C:\Wooo\Agent99", [string]$StagingDir = "", [string]$RelayPrefix = "http://+:8787/agent99/sre-alert/", - [string]$TokenEnv = "AGENT99_SRE_RELAY_TOKEN" + [string]$TokenEnv = "AGENT99_SRE_RELAY_TOKEN", + [string]$AlertChainPollSourceUrl = "http://192.168.0.110:9093/api/v2/alerts", + [string]$AlertChainPollSourceHost = "192.168.0.110", + [int]$AlertChainPollIntervalSeconds = 60, + [string]$AlertChainIngressUrl = "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency" ) $ErrorActionPreference = "Stop" @@ -18,7 +22,12 @@ $EvidenceDir = Join-Path $AgentRoot "evidence" New-Item -ItemType Directory -Force -Path $BinDir, $PlaybookDir, $EvidenceDir | Out-Null -foreach ($name in @("agent99-sre-alert-relay.ps1", "agent99-register-tasks.ps1", "agent99-bootstrap.ps1")) { +foreach ($name in @( + "agent99-sre-alert-relay.ps1", + "agent99-alertmanager-alertchain-poll.ps1", + "agent99-register-tasks.ps1", + "agent99-bootstrap.ps1" +)) { $src = Join-Path $StagingDir $name if (-not (Test-Path $src)) { throw "missing staged file: $name" @@ -54,6 +63,27 @@ Set-AgentProp $config.sreAlertRelay "enabled" $true Set-AgentProp $config.sreAlertRelay "prefix" $RelayPrefix Set-AgentProp $config.sreAlertRelay "tokenEnv" $TokenEnv Set-AgentProp $config.sreAlertRelay "allowedRemotePrefixes" @("127.", "::1", "192.168.0.") +foreach ($legacyName in @("alertmanagerPrefix", "emergencyIngressUrlEnv")) { + if ($config.sreAlertRelay.PSObject.Properties[$legacyName]) { + $config.sreAlertRelay.PSObject.Properties.Remove($legacyName) + } +} +if (-not $config.sreAlertRelay.PSObject.Properties["alertChainPoll"]) { + $config.sreAlertRelay | Add-Member -NotePropertyName "alertChainPoll" -NotePropertyValue ([pscustomobject]@{}) +} +Set-AgentProp $config.sreAlertRelay.alertChainPoll "enabled" $true +Set-AgentProp $config.sreAlertRelay.alertChainPoll "sourceUrl" $AlertChainPollSourceUrl +Set-AgentProp $config.sreAlertRelay.alertChainPoll "sourceHost" $AlertChainPollSourceHost +Set-AgentProp $config.sreAlertRelay.alertChainPoll "pollIntervalSeconds" $AlertChainPollIntervalSeconds +Set-AgentProp $config.sreAlertRelay.alertChainPoll "ingressUrl" $AlertChainIngressUrl +Set-AgentProp $config.sreAlertRelay.alertChainPoll "tokenEnv" $TokenEnv +Set-AgentProp $config.sreAlertRelay.alertChainPoll "alertname" "AlertChainBroken_Alertmanager" +Set-AgentProp $config.sreAlertRelay.alertChainPoll "agent99Role" "relay_coordination_only" +Set-AgentProp $config.sreAlertRelay.alertChainPoll "linuxExecutor" "host_ansible_executor" +Set-AgentProp $config.sreAlertRelay.alertChainPoll "catalogId" "ansible:110-alertmanager-delivery-recovery" +Set-AgentProp $config.sreAlertRelay.alertChainPoll "firstHopAuthenticationUsed" $false +Set-AgentProp $config.sreAlertRelay.alertChainPoll "firstHopSecretTransmitted" $false +Set-AgentProp $config.sreAlertRelay.alertChainPoll "rawPayloadStored" $false if (-not $config.PSObject.Properties["selfHealth"]) { $config | Add-Member -NotePropertyName "selfHealth" -NotePropertyValue ([pscustomobject]@{}) @@ -92,6 +122,14 @@ if (-not ($freshness | Where-Object { $_.name -eq "sre_alert_relay" } | Select-O required = $false } } +if (-not ($freshness | Where-Object { $_.name -eq "alertmanager_alertchain_poll" } | Select-Object -First 1)) { + $freshness += [pscustomobject]@{ + name = "alertmanager_alertchain_poll" + pattern = "alertmanager-alertchain-poll\poll-*.json" + maxAgeMinutes = 5 + required = $true + } +} Set-AgentProp $config.selfHealth "evidenceFreshness" $freshness $config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $ConfigPath @@ -115,5 +153,13 @@ $latestEvidence = Get-ChildItem -Path $EvidenceDir -Filter "agent99-SreAlertRela latestEvidence = if ($latestEvidence) { $latestEvidence.FullName } else { "" } configRelayEnabled = [bool]$config.sreAlertRelay.enabled configTokenEnv = [string]$config.sreAlertRelay.tokenEnv + alertChainPollMode = "in_process_nonblocking_timer" + configAlertChainPollSourceUrl = [string]$config.sreAlertRelay.alertChainPoll.sourceUrl + configAlertChainPollSourceHost = [string]$config.sreAlertRelay.alertChainPoll.sourceHost + configAlertChainPollIntervalSeconds = [int]$config.sreAlertRelay.alertChainPoll.pollIntervalSeconds + configAlertChainIngressUrl = [string]$config.sreAlertRelay.alertChainPoll.ingressUrl + configAgent99Role = [string]$config.sreAlertRelay.alertChainPoll.agent99Role + configLinuxExecutor = [string]$config.sreAlertRelay.alertChainPoll.linuxExecutor + configAlertChainCatalogId = [string]$config.sreAlertRelay.alertChainPoll.catalogId scheduledTaskListed = @($config.selfHealth.scheduledTasks) -contains "Wooo-Agent99-SRE-Alert-Relay" } | ConvertTo-Json -Depth 6 -Compress diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index 1e3df9010..e0564c87b 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -21,6 +21,7 @@ $SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile" $SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit" $AcceptedIgnoredDir = Join-Path $AlertRoot "ignored" $SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" +$AlertChainPollScript = Join-Path $BinDir "agent99-alertmanager-alertchain-poll.ps1" New-Item -ItemType Directory -Force -Path $EvidenceDir, $RelayReceiptDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir, $SameRunRetryAuditDir | Out-Null @@ -261,6 +262,22 @@ function Send-AgentRelayResponse { $Context.Response.Close() } +function Test-AgentRelayTokenMatch { + param( + [string]$Expected, + [string]$Actual + ) + if (-not $Expected -or -not $Actual) { return $false } + $expectedBytes = [Text.Encoding]::UTF8.GetBytes($Expected) + $actualBytes = [Text.Encoding]::UTF8.GetBytes($Actual) + if ($expectedBytes.Length -ne $actualBytes.Length) { return $false } + $difference = 0 + for ($index = 0; $index -lt $expectedBytes.Length; $index++) { + $difference = $difference -bor ($expectedBytes[$index] -bxor $actualBytes[$index]) + } + return [bool]($difference -eq 0) +} + function Test-AgentRelayRemoteAllowed { param([System.Net.HttpListenerContext]$Context) $remote = [string]$Context.Request.RemoteEndPoint.Address @@ -985,10 +1002,51 @@ function Start-AgentSameRunStatusReconcile { if (-not $Prefix.EndsWith("/")) { $Prefix = "$Prefix/" } - $listener = [System.Net.HttpListener]::new() $listener.Prefixes.Add($Prefix) +if (-not (Test-Path -LiteralPath $AlertChainPollScript -PathType Leaf)) { + throw "alertmanager_alertchain_poll_script_missing" +} +$alertChainPollTimer = [Timers.Timer]::new(1000) +$alertChainPollTimer.AutoReset = $false +$alertChainPollEventId = "Agent99.AlertChain.Poll.$PID" +$alertChainPollEventJob = Register-ObjectEvent ` + -InputObject $alertChainPollTimer ` + -EventName Elapsed ` + -SourceIdentifier $alertChainPollEventId ` + -MessageData @{ + PollScript = $AlertChainPollScript + PollAgentRoot = $AgentRoot + Timer = $alertChainPollTimer + } ` + -Action { + $pollData = $event.MessageData + try { + & $pollData.PollScript -AgentRoot $pollData.PollAgentRoot | Out-Null + } catch { + # The poller writes an allowlisted no-secret failure receipt. Keep this + # in-process timer alive so the next bounded interval retries. + } + $interval = 60 + try { + $path = Join-Path $pollData.PollAgentRoot "config\agent99.config.json" + $config = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json + $candidate = [int]$config.sreAlertRelay.alertChainPoll.pollIntervalSeconds + if ($candidate -ge 60 -and $candidate -le 300 -and ($candidate % 60) -eq 0) { + $interval = $candidate + } + } catch { + $interval = 60 + } + $pollData.Timer.Interval = $interval * 1000 + $pollData.Timer.Start() + } +if (-not $alertChainPollEventJob) { + throw "alertmanager_alertchain_poll_event_not_registered" +} +$alertChainPollTimer.Start() + $startupStamp = Get-Date -Format "yyyyMMdd-HHmmss" $startupEvidence = Join-Path $EvidenceDir "agent99-SreAlertRelay-start-$startupStamp.json" @{ @@ -998,6 +1056,8 @@ $startupEvidence = Join-Path $EvidenceDir "agent99-SreAlertRelay-start-$startupS tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv) allowedRemotePrefixes = $AllowedRemotePrefixes maxBodyBytes = $MaxBodyBytes + alertChainPollMode = "in_process_nonblocking_timer" + alertChainPollEventRegistered = $true runOnce = [bool]$RunOnce startedAt = (Get-Date).ToString("o") } | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $startupEvidence @@ -1009,6 +1069,8 @@ try { ok = $true prefix = $Prefix tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv) + alertChainPollMode = "in_process_nonblocking_timer" + alertChainPollEventRegistered = $true evidence = $startupEvidence } @@ -1029,7 +1091,7 @@ try { continue } $actualToken = [string]$context.Request.Headers["X-Agent99-Relay-Token"] - if ($actualToken -ne $expectedToken) { + if (-not (Test-AgentRelayTokenMatch $expectedToken $actualToken)) { Send-AgentRelayResponse $context 401 @{ ok = $false; error = "invalid_relay_token" } Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "invalid_relay_token" } continue @@ -1181,4 +1243,14 @@ try { } finally { if ($listener.IsListening) { $listener.Stop() } $listener.Close() + if ($alertChainPollTimer) { + $alertChainPollTimer.Stop() + $alertChainPollTimer.Dispose() + } + if ($alertChainPollEventId) { + Unregister-Event -SourceIdentifier $alertChainPollEventId -ErrorAction SilentlyContinue + } + if ($alertChainPollEventJob) { + Remove-Job -Job $alertChainPollEventJob -Force -ErrorAction SilentlyContinue + } } diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 2f9001430..4ff128f1b 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -176,7 +176,22 @@ "127.", "::1", "192.168.0." - ] + ], + "alertChainPoll": { + "enabled": true, + "sourceUrl": "http://192.168.0.110:9093/api/v2/alerts", + "sourceHost": "192.168.0.110", + "pollIntervalSeconds": 60, + "ingressUrl": "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency", + "tokenEnv": "AGENT99_SRE_RELAY_TOKEN", + "alertname": "AlertChainBroken_Alertmanager", + "agent99Role": "relay_coordination_only", + "linuxExecutor": "host_ansible_executor", + "catalogId": "ansible:110-alertmanager-delivery-recovery", + "firstHopAuthenticationUsed": false, + "firstHopSecretTransmitted": false, + "rawPayloadStored": false + } }, "completionCallback": { "enabled": true, @@ -385,6 +400,13 @@ "maxAgeMinutes": 1440, "required": false, "runningTaskName": "Wooo-Agent99-SRE-Alert-Relay" + }, + { + "name": "alertmanager_alertchain_poll", + "pattern": "alertmanager-alertchain-poll\\poll-*.json", + "maxAgeMinutes": 5, + "required": true, + "runningTaskName": "Wooo-Agent99-SRE-Alert-Relay" } ] }, diff --git a/agent99.config.example.json b/agent99.config.example.json index 52eb3558a..c9623e9d5 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -170,7 +170,22 @@ "127.", "::1", "192.168.0." - ] + ], + "alertChainPoll": { + "enabled": true, + "sourceUrl": "http://192.168.0.110:9093/api/v2/alerts", + "sourceHost": "192.168.0.110", + "pollIntervalSeconds": 60, + "ingressUrl": "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency", + "tokenEnv": "AGENT99_SRE_RELAY_TOKEN", + "alertname": "AlertChainBroken_Alertmanager", + "agent99Role": "relay_coordination_only", + "linuxExecutor": "host_ansible_executor", + "catalogId": "ansible:110-alertmanager-delivery-recovery", + "firstHopAuthenticationUsed": false, + "firstHopSecretTransmitted": false, + "rawPayloadStored": false + } }, "completionCallback": { "enabled": true, @@ -318,6 +333,13 @@ "maxAgeMinutes": 1440, "required": false, "runningTaskName": "Wooo-Agent99-SRE-Alert-Relay" + }, + { + "name": "alertmanager_alertchain_poll", + "pattern": "alertmanager-alertchain-poll\\poll-*.json", + "maxAgeMinutes": 5, + "required": true, + "runningTaskName": "Wooo-Agent99-SRE-Alert-Relay" } ] }, diff --git a/apps/api/alert_rules.yaml b/apps/api/alert_rules.yaml index b20402a11..8515d74c1 100644 --- a/apps/api/alert_rules.yaml +++ b/apps/api/alert_rules.yaml @@ -614,12 +614,34 @@ rules: # ── 告警鏈路監控 ──────────────────────────────────────────── - - id: alert_chain_broken - priority: 110 - description: 告警鏈路中斷 + - id: alert_chain_broken_alertmanager + priority: 109 + description: Alertmanager webhook delivery chain degraded match: alertname: - AlertChainBroken_Alertmanager + response: + action_title: "受控診斷並收斂 Alertmanager 告警鏈" + description: "⚙️ 規則匹配: 以 Alertmanager 自身 delivery counter 驗證，再由 host110 exact-container lane 執行 bounded repair 與獨立 E2E verifier。" + suggested_action: INVESTIGATE + kubectl_command: "" + estimated_downtime: "監控盲區持續中" + risk: medium + execution_risk_independent_of_incident_severity: true + responsibility: INFRA + responsibility_reasoning: "Alertmanager delivery chain 屬 host110 容器監控控制面；不得跨到 Sentry、SignOz 或 Kubernetes executor" + secondary_teams: [BE] + optimization: + - type: E2E_TEST + description: "由 alert_chain_independent_delivery_verifier 驗證 Alertmanager counter、AWOOOI receipt 與 Telegram delivery acknowledgement" + command: "" + reasoning: "[權威規則] 固定路由 container:host_110:alertmanager；禁止 LLM 改寫 domain、generic command 或 cross-host fallback。" + + - id: alert_chain_broken + priority: 110 + description: 非 Alertmanager 的告警鏈路中斷 + match: + alertname: - AlertChainBroken_Sentry - AlertChainBroken_SignOz - AlertChainUnhealthy @@ -629,20 +651,20 @@ rules: - alertmanager - no alerts response: - action_title: "診斷告警鏈路中斷" - description: "⚙️ 規則匹配: 告警鏈路異常，可能導致真實告警無法送達 Telegram。" - suggested_action: NO_ACTION - kubectl_command: "kubectl get pods -n monitoring && curl -s http://192.168.0.120:9093/api/v1/status | jq '.data.uptime'" + action_title: "診斷告警鏈並維持原始 domain" + description: "⚙️ 規則匹配: 告警鏈路異常；必須先解析 canonical source，並只使用該 source 的 typed executor/verifier。" + suggested_action: INVESTIGATE + kubectl_command: "" estimated_downtime: "監控盲區持續中" - risk: critical + risk: medium responsibility: INFRA responsibility_reasoning: "告警鏈路屬基礎設施監控體系，需立即修復確保可觀測性" secondary_teams: [BE] optimization: - type: E2E_TEST - description: "發送測試告警驗證整條鏈路" - command: "curl -X POST http://192.168.0.125:32334/api/v1/test-alert -H 'Content-Type: application/json' -d '{\"test\": true}'" - reasoning: "[規則匹配] 告警鏈路中斷等同監控失明，最高優先修復。" + description: "由 alert_chain_independent_delivery_verifier 驗證 Alertmanager counter、AWOOOI receipt 與 Telegram delivery acknowledgement" + command: "" + reasoning: "[規則匹配] 告警鏈路中斷等同監控失明；禁止 generic kubectl/host fallback，也禁止滑落到 Alertmanager、Sentry 或 SignOz 的另一 domain。" # ── GPU / AI 基礎設施 ──────────────────────────────────────── diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index cf81caaa4..8dc4e771b 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -479,6 +479,10 @@ from src.services.p0_cicd_baseline_source_readiness import ( from src.services.package_supply_chain_inventory import ( load_latest_package_supply_chain_inventory, ) +from src.services.portfolio_infrastructure_asset_reconciliation import ( + build_portfolio_infrastructure_public_projection, + load_portfolio_infrastructure_asset_reconciliation, +) from src.services.product_awoooi_manifest_standard import ( load_latest_product_awoooi_manifest_standard, ) @@ -1243,6 +1247,42 @@ async def get_sre_k3s_controlled_automation_work_items() -> dict[str, Any]: ) from exc +@router.get( + "/portfolio-infrastructure-asset-reconciliation", + response_model=dict[str, Any], + summary="取得全產品基礎設施與 AI 自動化資產對帳總帳", + description=( + "回傳外部盤點 131 列、盤點漏項、衝突、143 個 canonical assets、" + "依賴排序工作項與 source/runtime 分離狀態。此端點只讀 committed snapshot，" + "不探測主機、不執行修復、不讀 secret、不呼叫模型或 GitHub；公開投影會移除" + "本機絕對路徑、內網拓撲及 provider 原始端點。" + ), +) +async def get_portfolio_infrastructure_asset_reconciliation() -> dict[str, Any]: + """回傳所有專案、產品、網站與控制面的安全資產對帳投影。""" + + try: + payload = await asyncio.to_thread( + load_portfolio_infrastructure_asset_reconciliation + ) + projection = build_portfolio_infrastructure_public_projection(payload) + return redact_public_lan_topology(projection) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + logger.error( + "portfolio_infrastructure_asset_reconciliation_invalid", + error=str(exc), + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="全產品基礎設施與 AI 自動化資產對帳總帳無效", + ) from exc + + @router.post( "/controlled-alert-target-preview", response_model=dict[str, Any], diff --git a/apps/api/src/api/v1/ai.py b/apps/api/src/api/v1/ai.py index b0118e713..b81988d48 100644 --- a/apps/api/src/api/v1/ai.py +++ b/apps/api/src/api/v1/ai.py @@ -351,6 +351,8 @@ async def get_ai_status() -> dict: gemini_enabled=gemini_enabled, gemini_authentication=authentication, cost_guard=gemini_usage, + gemini_execution_mode=settings.GEMINI_EXECUTION_MODE, + gemini_canary_percent=settings.GEMINI_CANARY_PERCENT, claude_configured=claude_configured, claude_enabled=claude_enabled, claude_authentication=claude_authentication, @@ -371,6 +373,9 @@ async def get_ai_status() -> dict: "gemini_production_executable": route["gemini"][ "production_executable" ], + "gemini_canary_executable": route["gemini"]["canary_executable"], + "gemini_execution_mode": settings.GEMINI_EXECUTION_MODE, + "gemini_canary_percent": settings.GEMINI_CANARY_PERCENT, "gemini_verification_status": authentication.get("verification_status"), "gemini_disable_state_status": disable_state_status, "claude_configured": claude_configured, diff --git a/apps/api/src/api/v1/health.py b/apps/api/src/api/v1/health.py index 68a876df4..e007b3a92 100644 --- a/apps/api/src/api/v1/health.py +++ b/apps/api/src/api/v1/health.py @@ -455,6 +455,8 @@ async def get_ai_usage() -> dict: gemini_enabled=gemini_enabled, gemini_authentication=authentication, cost_guard=gemini_stats, + gemini_execution_mode=settings.GEMINI_EXECUTION_MODE, + gemini_canary_percent=settings.GEMINI_CANARY_PERCENT, claude_configured=claude_configured, claude_enabled=claude_enabled, claude_authentication=claude_authentication, @@ -471,6 +473,8 @@ async def get_ai_usage() -> dict: "gemini_status": { "configured": gemini_configured, "enabled": gemini_enabled, + "execution_mode": settings.GEMINI_EXECUTION_MODE, + "canary_percent": settings.GEMINI_CANARY_PERCENT, "authenticated": authentication.get("authenticated"), "authentication_verified": bool( authentication.get("authentication_verified") is True @@ -478,6 +482,7 @@ async def get_ai_usage() -> dict: "production_executable": route["gemini"][ "production_executable" ], + "canary_executable": route["gemini"]["canary_executable"], "verification_status": authentication.get("verification_status"), "disable_state_status": disable_state_status, }, diff --git a/apps/api/src/api/v1/platform/operator_runs.py b/apps/api/src/api/v1/platform/operator_runs.py index 34ed1acf0..624a0364b 100644 --- a/apps/api/src/api/v1/platform/operator_runs.py +++ b/apps/api/src/api/v1/platform/operator_runs.py @@ -16,13 +16,16 @@ from typing import Any, Literal from uuid import UUID import structlog -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, status from pydantic import BaseModel, Field from src.core.awooop_operator_auth import ( AwoooPOperatorPrincipal, verify_awooop_operator, ) +from src.services.paid_provider_canary_gate5_run import ( + create_paid_provider_canary_gate5_run as create_paid_provider_canary_gate5_run_svc, +) from src.services.platform_operator_service import ( decide_approval as decide_approval_svc, ) @@ -412,6 +415,61 @@ class DecideApprovalResponse(BaseModel): approval_token_jti: str | None +class CreatePaidProviderCanaryAuthorizationRequest(BaseModel): + """Deliberate acknowledgement for the one bounded paid-provider canary.""" + + work_item_id: Literal["AIA-SRE-013"] = Field( + ..., + description="固定 paid provider canary work item", + ) + authorization_scope: Literal[ + "authorize_one_sanitized_five_lane_comparison" + ] = Field( + ..., + description="固定為一次脫敏五路比較；不可擴張 provider、流量或寫入範圍", + ) + + +class CreatePaidProviderCanaryAuthorizationResponse(BaseModel): + schema_version: Literal["paid_provider_canary_gate5_run_v1"] + authorization_ref: UUID + run_id: UUID + project_id: Literal["awoooi"] + work_item_id: Literal["AIA-SRE-013"] + agent_id: Literal["awoooi-paid-provider-canary"] + state: Literal["waiting_approval", "running"] + is_shadow: Literal[False] + is_duplicate: bool + trigger_type: Literal["api"] + trigger_ref: Literal["paid-provider-canary:AIA-SRE-013"] + input_sha256: str = Field(pattern="^[0-9a-f]{64}$") + authorization_scope: Literal[ + "authorize_one_sanitized_five_lane_comparison" + ] + provider_order: tuple[ + Literal["ollama_gcp_a"], + Literal["ollama_gcp_b"], + Literal["ollama_local"], + Literal["claude"], + Literal["gemini"], + ] + paid_provider_canary_percent: Literal[5] + max_output_tokens_per_paid_provider: Literal[512] + max_total_paid_cost_usd: Literal["0.25"] + timeout_at: datetime + approval_ttl_seconds: Literal[900] + provider_call_performed: Literal[False] + route_change_performed: Literal[False] + infrastructure_write_performed: Literal[False] + raw_prompt_persistence_allowed: Literal[False] + raw_response_persistence_allowed: Literal[False] + next_action: Literal[ + "operator_gate5_decision_required", + "verify_authorization_then_run_bounded_canary", + ] + approval_path: str + + @router.get( "/runs/list", response_model=ListRunsResponse, @@ -645,6 +703,29 @@ async def list_approvals( ) +@router.post( + "/approvals/paid-provider-canary/authorize", + response_model=CreatePaidProviderCanaryAuthorizationResponse, + status_code=status.HTTP_202_ACCEPTED, + summary="建立 paid provider canary Gate 5 授權 Run", + description=( + "只允許受信 AwoooP operator 建立一次、15 分鐘、non-shadow 的精準授權列。\n\n" + "此入口不呼叫 Claude/Gemini、不切換 provider route、不執行基礎設施寫入；" + "建立後仍必須經既有 `/approvals/{run_id}/decide` Gate 5 核准。" + ), +) +async def create_paid_provider_canary_authorization( + body: CreatePaidProviderCanaryAuthorizationRequest, + operator: AwoooPOperatorPrincipal = Depends(verify_awooop_operator), # noqa: B008 +) -> dict[str, Any]: + # Pydantic Literal fields deliberately reject any attempt to widen the + # work item or canary scope before the service reaches the database. + _ = body + return await create_paid_provider_canary_gate5_run_svc( + operator_id=operator.operator_id, + ) + + @router.post( "/approvals/{run_id}/decide", response_model=DecideApprovalResponse, diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index a0ef04ae8..f164b095e 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -45,13 +45,17 @@ from src.models.approval import ( ) from src.models.incident import SignalSource from src.models.webhook import AlertPayload, AlertResponse +from src.services.agent99_outcome_ingestion import ingest_agent99_outcome_receipt from src.services.agent99_sre_bridge import ( bridge_alertmanager_to_agent99, resolve_agent99_durable_route, ) -from src.services.agent99_outcome_ingestion import ingest_agent99_outcome_receipt from src.services.alert_analyzer_service import AlertAnalyzer from src.services.alert_approval_guard import guard_alert_approval_action +from src.services.alert_chain_emergency_ingress import ( + AlertChainEmergencyRelayRequest, + process_alert_chain_emergency_relay, +) from src.services.alert_grouping_service import get_alert_grouping_service from src.services.alert_rule_engine import get_incident_type, match_rule from src.services.alertmanager_llm_guard import ( @@ -168,6 +172,64 @@ async def ingest_agent99_outcome_webhook( return receipt +@router.post( + "/agent99/alertmanager-emergency", + response_model=dict[str, Any], + status_code=status.HTTP_202_ACCEPTED, + summary="Receive an authenticated Agent99 Alertmanager poll candidate", + description=( + "Agent99 polls the exact host-110 Alertmanager API without credentials " + "and relays only the exact AlertChainBroken_Alertmanager envelope when " + "the primary webhook is unhealthy. The " + "payload cannot contain labels, annotations, logs, credentials or " + "commands. AWOOOI persists one deterministic candidate and routes the " + "Linux repair to the exact host Ansible executor; Agent99 never applies it." + ), +) +async def ingest_agent99_alertmanager_emergency( + payload: AlertChainEmergencyRelayRequest, + x_agent99_relay_token: str | None = Header( + default=None, + alias="X-Agent99-Relay-Token", + ), +) -> dict[str, Any]: + """Fail closed until the same-run Ansible candidate is durably queued.""" + + configured_token = str(settings.AGENT99_SRE_ALERT_RELAY_TOKEN or "") + if not configured_token: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="agent99_alertmanager_emergency_auth_not_configured", + ) + if not x_agent99_relay_token or not hmac.compare_digest( + configured_token, + x_agent99_relay_token, + ): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="agent99_alertmanager_emergency_authentication_failed", + ) + + receipt = await process_alert_chain_emergency_relay(payload) + if receipt.get("candidate_persisted") is not True: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=str(receipt.get("status") or "candidate_persistence_failed"), + ) + if receipt.get("candidate_queued") is not True: + # Agent99 must retry the same deterministic poll occurrence. A + # durable approval/Incident without the exact same-run executor + # candidate is partial evidence, not a successful machine delivery. + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=str( + receipt.get("status") + or "same_run_ansible_candidate_not_queued" + ), + ) + return receipt + + # ============================================================================= # Incident-Approval 同步 (feedback_incident_approval_sync.md 鐵律) # ============================================================================= @@ -200,15 +262,24 @@ def _should_use_alertmanager_rule_first( rule_response: dict | None, alert_category: str, ) -> bool: - """主機/備份類告警命中權威 YAML 規則時，避免再被 LLM 污染成 K8s 動作。""" + """Exact typed rules precede LLM so the model cannot rewrite domains.""" - if not rule_response or alert_category not in {"host_resource", "backup_failure"}: + if not rule_response: return False - if rule_response.get("rule_id", "") in ("generic_fallback", ""): + rule_id = str(rule_response.get("rule_id") or "") + if rule_id in ("generic_fallback", ""): return False suggested_action = str(rule_response.get("suggested_action", "")).upper() command = str(rule_response.get("kubectl_command", "")).strip().lower() + if alert_category == "alert_chain_health": + return ( + rule_id == "alert_chain_broken_alertmanager" + and suggested_action == "INVESTIGATE" + and not command + ) + if alert_category not in {"host_resource", "backup_failure"}: + return False return ( suggested_action == "NO_ACTION" or suggested_action.startswith("SSH_") @@ -2194,7 +2265,7 @@ async def _process_new_alert_background( reason=( "agent99_durable_route_precedes_llm" if agent99_durable_route - else "host_or_backup_authoritative_rule_precedes_llm" + else "exact_typed_authoritative_rule_precedes_llm" ), ) risk_mapping = { diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index c22fb919a..086b33b75 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -486,6 +486,20 @@ class Settings(BaseSettings): le=3600, description="Gemini failed generation work-item cooldown seconds", ) + GEMINI_EXECUTION_MODE: str = Field( + default="shadow", + pattern="^(shadow|canary|production)$", + description=( + "Gemini route mode: shadow performs no API call; canary uses a " + "deterministic run bucket; production enables the final fallback" + ), + ) + GEMINI_CANARY_PERCENT: int = Field( + default=0, + ge=0, + le=100, + description="Deterministic Gemini canary percentage when mode=canary", + ) CLAUDE_MODEL: str = Field( default="claude-sonnet-5", min_length=1, diff --git a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py index a977071bb..1b41027dc 100644 --- a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py +++ b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py @@ -17,7 +17,7 @@ from collections.abc import Awaitable, Callable, Mapping from datetime import UTC, datetime from types import SimpleNamespace from typing import Any -from uuid import NAMESPACE_URL, uuid4, uuid5 +from uuid import NAMESPACE_URL, UUID, uuid4, uuid5 import structlog from sqlalchemy import text @@ -464,6 +464,7 @@ async def enqueue_ai_decision_ansible_candidate( incident: Any, proposal_data: dict[str, Any], project_id: str = "awoooi", + automation_run_id: str | None = None, recorder: Recorder = record_ansible_decision_audit, evidence_collector: EvidenceCollector = ( collect_ansible_candidate_pre_decision_evidence @@ -546,6 +547,14 @@ async def enqueue_ai_decision_ansible_candidate( "automation_run_id": generation_preflight.get( "existing_candidate_op_id" ), + "trace_id": str( + payload["input"].get("trace_id") + or generation_preflight.get("existing_candidate_op_id") + or "" + ), + "work_item_id": str( + payload["input"].get("work_item_id") or "" + ), "queued": existing_current, "side_effect_performed": False, "single_writer_executor": "awoooi-ansible-executor-broker", @@ -554,12 +563,26 @@ async def enqueue_ai_decision_ansible_candidate( ), } - automation_run_id = str(uuid4()) + if automation_run_id: + try: + candidate_run_id = str(UUID(str(automation_run_id))) + except (TypeError, ValueError): + return { + "schema_version": "ai_decision_controlled_executor_handoff_v1", + "status": "automation_run_id_invalid", + "automation_run_id": None, + "queued": False, + "side_effect_performed": False, + "single_writer_executor": "awoooi-ansible-executor-broker", + "active_blockers": ["automation_run_id_invalid"], + } + else: + candidate_run_id = str(uuid4()) try: snapshot = await evidence_collector( incident=incident, project_id=project_id, - automation_run_id=automation_run_id, + automation_run_id=candidate_run_id, ) evidence_ready = await evidence_verifier( snapshot=snapshot, @@ -569,7 +592,7 @@ async def enqueue_ai_decision_ansible_candidate( logger.warning( "ai_decision_controlled_executor_evidence_failed", incident_id=payload["input"].get("incident_id"), - automation_run_id=automation_run_id, + automation_run_id=candidate_run_id, error_type=type(exc).__name__, ) evidence_ready = False @@ -577,7 +600,11 @@ async def enqueue_ai_decision_ansible_candidate( return { "schema_version": "ai_decision_controlled_executor_handoff_v1", "status": "pre_decision_evidence_not_verified", - "automation_run_id": automation_run_id, + "automation_run_id": candidate_run_id, + "trace_id": str( + payload["input"].get("trace_id") or candidate_run_id + ), + "work_item_id": str(payload["input"].get("work_item_id") or ""), "queued": False, "side_effect_performed": False, "single_writer_executor": "awoooi-ansible-executor-broker", @@ -592,7 +619,7 @@ async def enqueue_ai_decision_ansible_candidate( "AI decision routed to the single-writer Ansible broker; " "check-mode and diff are required before controlled apply" ), - automation_run_id=automation_run_id, + automation_run_id=candidate_run_id, ) return { "schema_version": "ai_decision_controlled_executor_handoff_v1", @@ -601,7 +628,11 @@ async def enqueue_ai_decision_ansible_candidate( if inserted else "existing_candidate_or_write_not_acknowledged" ), - "automation_run_id": automation_run_id, + "automation_run_id": candidate_run_id, + "trace_id": str( + payload["input"].get("trace_id") or candidate_run_id + ), + "work_item_id": str(payload["input"].get("work_item_id") or ""), "idempotency_key": payload["input"]["idempotency_key"], "queued": inserted, "side_effect_performed": False, diff --git a/apps/api/src/services/ai_control.py b/apps/api/src/services/ai_control.py index 6e8a88e0a..b3b3bdad6 100644 --- a/apps/api/src/services/ai_control.py +++ b/apps/api/src/services/ai_control.py @@ -9,7 +9,7 @@ Redis Keys: ai:control:use_router "true"/"false" (TTL: 30天) ai:control:primary_provider legacy compatibility; only "ollama"/"ollama_gcp_a" ai:control:disabled: "1" disabled / paid provider "0" enabled - (paid providers 無 TTL；free providers 30 天) + only under a run-owned expiring lease Usage: /ai status — 顯示所有 Provider 狀態 + 當前路由模式 @@ -21,6 +21,10 @@ Usage: /ai router off — 停用 AIRouter (回滾到舊 fallback chain) """ +import hashlib +import re +from typing import Any + import structlog logger = structlog.get_logger(__name__) @@ -29,9 +33,114 @@ logger = structlog.get_logger(__name__) _AI_ROUTER_KEY = "ai:control:use_router" _PRIMARY_PROVIDER_KEY = "ai:control:primary_provider" _DISABLED_KEY_PREFIX = "ai:control:disabled:" +_PAID_CANARY_LOCK_KEY = "ai:control:paid_canary:activation_lock" +_PAID_CANARY_ACTIVE_RECEIPT_KEY = "ai:control:paid_canary:active_receipt" +_PAID_CANARY_EXECUTION_CLAIM_PREFIX = "ai:control:paid_canary:execution_claim:" +_PAID_CANARY_OWNER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$") +_PAID_CANARY_MIN_LEASE_SECONDS = 60 +_PAID_CANARY_MAX_LEASE_SECONDS = 900 + +_ACQUIRE_PAID_CANARY_LEASE_LUA = r""" +if redis.call('EXISTS', KEYS[1]) == 1 then + return {0, 'activation_lock_held'} +end +if redis.call('EXISTS', KEYS[4]) == 1 then + return {0, 'persistent_activation_already_recorded'} +end +if redis.call('GET', KEYS[2]) == '0' or redis.call('GET', KEYS[3]) == '0' then + return {0, 'paid_provider_already_enabled'} +end +redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) +redis.call('SET', KEYS[2], '0', 'EX', ARGV[2]) +redis.call('SET', KEYS[3], '0', 'EX', ARGV[2]) +return {1, 'activation_lease_acquired'} +""" + +_ROLLBACK_PAID_CANARY_LUA = r""" +local lock_owner = redis.call('GET', KEYS[1]) +local active_owner = redis.call('GET', KEYS[4]) +if lock_owner and lock_owner ~= ARGV[1] then + return {0, 'activation_owner_mismatch'} +end +if active_owner and + string.sub(active_owner, 1, string.len(ARGV[1]) + 1) ~= ARGV[1] .. '|' then + return {0, 'active_receipt_owner_mismatch'} +end +redis.call('SET', KEYS[2], '1') +redis.call('SET', KEYS[3], '1') +redis.call('DEL', KEYS[1]) +redis.call('DEL', KEYS[4]) +return {1, 'paid_canary_disabled'} +""" + +_READ_PAID_CANARY_CONTROL_LUA = r""" +return { + redis.call('GET', KEYS[1]) or '', + redis.call('TTL', KEYS[1]), + redis.call('GET', KEYS[2]) or '', + redis.call('TTL', KEYS[2]), + redis.call('GET', KEYS[3]) or '', + redis.call('TTL', KEYS[3]), + redis.call('GET', KEYS[4]) or '' +} +""" + +_ACQUIRE_PAID_CANARY_EXECUTION_CLAIM_LUA = r""" +if redis.call('EXISTS', KEYS[1]) == 1 then + return {0, 'execution_claim_held'} +end +redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) +return {1, 'execution_claim_acquired'} +""" + +_RELEASE_PAID_CANARY_EXECUTION_CLAIM_LUA = r""" +if redis.call('GET', KEYS[1]) ~= ARGV[1] then + return {0, 'execution_claim_owner_mismatch'} +end +redis.call('DEL', KEYS[1]) +return {1, 'execution_claim_released'} +""" + +_SET_PAID_PROVIDER_STATE_LUA = r""" +redis.call('SET', KEYS[2], '1') +redis.call('SET', KEYS[3], '1') +redis.call('DEL', KEYS[1]) +redis.call('DEL', KEYS[4]) +return {1, 'paired_paid_providers_disabled'} +""" + +_PAID_PROVIDER_ADMISSION_LUA = r""" +local claude_state = redis.call('GET', KEYS[2]) +local gemini_state = redis.call('GET', KEYS[3]) +if claude_state ~= '0' or gemini_state ~= '0' then + return {0, 'paired_paid_state_not_enabled', ''} +end + +local lock_owner = redis.call('GET', KEYS[1]) +local active_owner = redis.call('GET', KEYS[4]) +if lock_owner then + local expected_prefix = ARGV[1] .. '.attempt-' + local run_matches = ARGV[1] ~= '' and + string.sub(lock_owner, 1, string.len(expected_prefix)) == expected_prefix + local lease_live = redis.call('TTL', KEYS[1]) > 0 and + redis.call('TTL', KEYS[2]) > 0 and redis.call('TTL', KEYS[3]) > 0 + local promoted_pending_release = active_owner and + string.sub(active_owner, 1, string.len(lock_owner) + 1) == lock_owner .. '|' and + redis.call('TTL', KEYS[2]) == -1 and redis.call('TTL', KEYS[3]) == -1 + if run_matches and (lease_live or promoted_pending_release) then + return {1, 'run_owned_canary_admitted', lock_owner} + end + return {0, 'run_owned_canary_not_admitted', lock_owner} +end + +if active_owner then + return {0, 'legacy_persistent_activation_blocked', active_owner} +end +return {0, 'paid_activation_receipt_missing', ''} +""" # Free-provider/router compatibility controls retain the historical 30-day -# expiry. Paid-provider explicit control state is intentionally persistent. +# expiry. Paid providers are executable only during a run-owned expiring lease. _CONTROL_KEY_TTL = 30 * 24 * 3600 # 30 days PAID_PROVIDERS = {"claude", "gemini"} @@ -122,19 +231,36 @@ async def clear_primary_provider() -> bool: return False -async def is_provider_disabled(provider: str) -> bool: +async def is_provider_disabled( + provider: str, + *, + run_id: str | None = None, +) -> bool: """檢查 Provider 是否被停用""" try: r = await _get_redis() - val = await r.get(f"{_DISABLED_KEY_PREFIX}{provider}") - # The paid final fallback has no implicit enable state. A missing - # durable control key is disabled until an operator explicitly enables - # it; free Ollama lanes retain their historical missing-key=enabled - # behaviour. if provider in PAID_PROVIDERS: - return val is None or ( - val.decode() if isinstance(val, bytes) else str(val) - ) != "0" + result = await r.eval( + _PAID_PROVIDER_ADMISSION_LUA, + 4, + *_paid_canary_keys(), + str(run_id or ""), + ) + admitted = int(result[0]) == 1 + admission_status = _decode_redis_scalar(result[1]) + admission_receipt = _decode_redis_scalar(result[2]) + if not admitted: + return True + if admission_status == "run_owned_canary_admitted": + return not _valid_paid_canary_owner(admission_receipt) + # Unknown admission states are not forward-compatible for paid + # execution. A server-side script change must be paired with an + # explicit client policy update. + return True + val = await r.get(f"{_DISABLED_KEY_PREFIX}{provider}") + # Free Ollama lanes retain their historical missing-key=enabled + # behaviour. Paid providers are admitted only by the paired, + # receipt-backed transaction above. return val is not None except Exception: # Paid-provider disable truth is fail-closed. Callers can catch this to @@ -148,22 +274,34 @@ async def set_provider_disabled(provider: str, disabled: bool) -> bool: """設定 Provider 啟用/停用""" if provider not in VALID_PROVIDERS: return False + if provider in PAID_PROVIDERS and not disabled: + logger.warning( + "ai_control_paid_provider_generic_enable_blocked", + provider=provider, + reason="run_owned_paid_canary_required", + ) + return False try: r = await _get_redis() key = f"{_DISABLED_KEY_PREFIX}{provider}" - if disabled: - if provider in PAID_PROVIDERS: - # Paid-provider disable truth must not silently expire. - await r.set(key, "1") - else: - await r.set(key, "1", ex=_CONTROL_KEY_TTL) + if provider in PAID_PROVIDERS: + result = await r.eval( + _SET_PAID_PROVIDER_STATE_LUA, + 4, + *_paid_canary_keys(), + ) + if int(result[0]) != 1: + logger.warning( + "ai_control_paid_provider_set_blocked", + provider=provider, + disabled=disabled, + status=_decode_redis_scalar(result[1]), + ) + return False + elif disabled: + await r.set(key, "1", ex=_CONTROL_KEY_TTL) else: - if provider in PAID_PROVIDERS: - # Explicitly persist the enabled state. Deleting the key would - # fail closed to disabled on the next read. - await r.set(key, "0") - else: - await r.delete(key) + await r.delete(key) logger.info("ai_control_provider_set", provider=provider, disabled=disabled) return True except Exception as e: @@ -171,6 +309,264 @@ async def set_provider_disabled(provider: str, disabled: bool) -> bool: return False +def _paid_canary_keys() -> tuple[str, str, str, str]: + return ( + _PAID_CANARY_LOCK_KEY, + f"{_DISABLED_KEY_PREFIX}claude", + f"{_DISABLED_KEY_PREFIX}gemini", + _PAID_CANARY_ACTIVE_RECEIPT_KEY, + ) + + +def _valid_paid_canary_owner(owner: str) -> bool: + return bool(_PAID_CANARY_OWNER_PATTERN.fullmatch(str(owner or ""))) + + +def _paid_canary_execution_claim_key(run_id: str) -> str: + digest = hashlib.sha256(str(run_id).encode("utf-8")).hexdigest() + return f"{_PAID_CANARY_EXECUTION_CLAIM_PREFIX}{digest}" + + +def _decode_redis_scalar(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value or "") + + +async def acquire_paid_provider_canary_lease( + owner: str, + *, + ttl_seconds: int = 300, +) -> bool: + """Acquire one run-owned, crash-safe paid-provider activation lease.""" + + if not _valid_paid_canary_owner(owner): + return False + ttl = int(ttl_seconds) + if not _PAID_CANARY_MIN_LEASE_SECONDS <= ttl <= _PAID_CANARY_MAX_LEASE_SECONDS: + return False + try: + redis = await _get_redis() + result = await redis.eval( + _ACQUIRE_PAID_CANARY_LEASE_LUA, + 4, + *_paid_canary_keys(), + owner, + ttl, + ) + acquired = int(result[0]) == 1 + logger.info( + "ai_control_paid_canary_lease_acquire", + acquired=acquired, + status=_decode_redis_scalar(result[1]), + ttl_seconds=ttl, + ) + return acquired + except Exception as exc: + logger.error( + "ai_control_paid_canary_lease_acquire_failed", + error_type=type(exc).__name__, + ) + return False + + +async def acquire_paid_provider_canary_execution_claim( + run_id: str, + owner: str, + *, + ttl_seconds: int = 120, +) -> bool: + """Serialize one Gate 5 authorization before any provider preflight.""" + + if not _valid_paid_canary_owner(run_id) or not _valid_paid_canary_owner(owner): + return False + ttl = int(ttl_seconds) + if not _PAID_CANARY_MIN_LEASE_SECONDS <= ttl <= _PAID_CANARY_MAX_LEASE_SECONDS: + return False + try: + redis = await _get_redis() + result = await redis.eval( + _ACQUIRE_PAID_CANARY_EXECUTION_CLAIM_LUA, + 1, + _paid_canary_execution_claim_key(run_id), + owner, + ttl, + ) + acquired = int(result[0]) == 1 + logger.info( + "ai_control_paid_canary_execution_claim_acquire", + acquired=acquired, + status=_decode_redis_scalar(result[1]), + ttl_seconds=ttl, + ) + return acquired + except Exception as exc: + logger.error( + "ai_control_paid_canary_execution_claim_acquire_failed", + error_type=type(exc).__name__, + ) + return False + + +async def release_paid_provider_canary_execution_claim( + run_id: str, + owner: str, +) -> bool: + """CAS-release only the caller's pre-execution claim.""" + + if not _valid_paid_canary_owner(run_id) or not _valid_paid_canary_owner(owner): + return False + try: + redis = await _get_redis() + result = await redis.eval( + _RELEASE_PAID_CANARY_EXECUTION_CLAIM_LUA, + 1, + _paid_canary_execution_claim_key(run_id), + owner, + ) + released = int(result[0]) == 1 + logger.info( + "ai_control_paid_canary_execution_claim_release", + released=released, + status=_decode_redis_scalar(result[1]), + ) + return released + except Exception as exc: + logger.error( + "ai_control_paid_canary_execution_claim_release_failed", + error_type=type(exc).__name__, + ) + return False + + +async def promote_paid_provider_canary_activation( + owner: str, + *, + completion_record_id: str, +) -> bool: + """Reject persistent enablement without a separate bounded authorization. + + A Gate 5 receipt authorizes one synthetic comparison only. Keeping this + compatibility function fail closed prevents an internal caller from + turning that receipt into future paid-alert traffic. + """ + + _ = (owner, completion_record_id) + logger.warning( + "ai_control_paid_canary_persistent_promotion_blocked", + reason="separate_time_and_cost_bounded_authorization_required", + ) + return False + + +async def rollback_paid_provider_canary_activation(owner: str) -> bool: + """Fail closed without overwriting another run's activation state.""" + + if not _valid_paid_canary_owner(owner): + return False + try: + redis = await _get_redis() + result = await redis.eval( + _ROLLBACK_PAID_CANARY_LUA, + 4, + *_paid_canary_keys(), + owner, + ) + rolled_back = int(result[0]) == 1 + logger.info( + "ai_control_paid_canary_activation_rollback", + rolled_back=rolled_back, + status=_decode_redis_scalar(result[1]), + ) + return rolled_back + except Exception as exc: + logger.error( + "ai_control_paid_canary_activation_rollback_failed", + error_type=type(exc).__name__, + ) + return False + + +async def get_paid_provider_canary_control_readback( + *, + expected_owner: str | None = None, + expected_completion_record_id: str | None = None, +) -> dict[str, Any]: + """Return a content-free, atomic readback of paid canary control state.""" + + if expected_owner is not None and not _valid_paid_canary_owner(expected_owner): + raise ValueError("paid_canary_owner_invalid") + if expected_completion_record_id is not None and not _valid_paid_canary_owner( + expected_completion_record_id + ): + raise ValueError("paid_canary_completion_record_id_invalid") + redis = await _get_redis() + result = await redis.eval( + _READ_PAID_CANARY_CONTROL_LUA, + 4, + *_paid_canary_keys(), + ) + lock_owner = _decode_redis_scalar(result[0]) + lock_ttl = int(result[1]) + claude_value = _decode_redis_scalar(result[2]) + claude_ttl = int(result[3]) + gemini_value = _decode_redis_scalar(result[4]) + gemini_ttl = int(result[5]) + active_receipt = _decode_redis_scalar(result[6]) + active_owner, separator, completion_record_id = active_receipt.partition("|") + active_completion_receipt_present = bool( + separator and _valid_paid_canary_owner(completion_record_id) + ) + active_completion_receipt_matches = bool( + expected_completion_record_id + and completion_record_id == expected_completion_record_id + ) + claude_disabled = claude_value != "0" + gemini_disabled = gemini_value != "0" + lock_matches = bool(expected_owner and lock_owner == expected_owner) + active_matches = bool(expected_owner and active_owner == expected_owner) + leased_enabled = bool( + lock_matches + and not active_receipt + and not claude_disabled + and not gemini_disabled + and lock_ttl > 0 + and claude_ttl > 0 + and gemini_ttl > 0 + ) + legacy_persistent_state_detected = bool( + active_receipt + or ( + not claude_disabled + and not gemini_disabled + and claude_ttl == -1 + and gemini_ttl == -1 + ) + ) + disabled_verified = bool( + claude_disabled + and gemini_disabled + and not lock_owner + and not active_receipt + ) + return { + "claude_disabled": claude_disabled, + "gemini_disabled": gemini_disabled, + "lock_present": bool(lock_owner), + "lock_owner_matches": lock_matches, + "lock_ttl_seconds": lock_ttl, + "active_receipt_present": bool(active_receipt), + "active_receipt_owner_matches": active_matches, + "active_completion_receipt_present": active_completion_receipt_present, + "active_completion_receipt_matches": active_completion_receipt_matches, + "leased_enabled": leased_enabled, + "persistent_promoted": False, + "persistent_enabled": False, + "legacy_persistent_state_detected": legacy_persistent_state_detected, + "disabled_verified": disabled_verified, + } + + async def get_status_summary() -> str: """ 取得 AI 控制狀態摘要 (Telegram 格式 HTML) @@ -293,6 +689,11 @@ async def handle_ai_command(text: str) -> str: provider = parts[2].lower() if provider not in VALID_PROVIDERS: return f"❌ 未知 Provider: {provider}" + if provider in PAID_PROVIDERS: + return ( + "❌ Claude/Gemini 付費路由只能由 run-owned canary 驗證流程啟用；" + "一般 /ai enable 已 fail closed" + ) ok = await set_provider_disabled(provider, False) return f"✅ {provider} 已啟用" if ok else "❌ 操作失敗" @@ -303,6 +704,8 @@ async def handle_ai_command(text: str) -> str: if provider not in VALID_PROVIDERS: return f"❌ 未知 Provider: {provider}" ok = await set_provider_disabled(provider, True) + if ok and provider in PAID_PROVIDERS: + return "⚠️ Claude/Gemini 已成對停用，activation lease/receipt 已清除" return f"⚠️ {provider} 已停用" if ok else "❌ 操作失敗" elif sub == "cost": diff --git a/apps/api/src/services/ai_provider_policy.py b/apps/api/src/services/ai_provider_policy.py index 0ee77d314..8adb6e1f6 100644 --- a/apps/api/src/services/ai_provider_policy.py +++ b/apps/api/src/services/ai_provider_policy.py @@ -32,6 +32,11 @@ NON_EXECUTABLE_SHADOW_PROVIDERS: frozenset[str] = frozenset( PAID_PROVIDER_ORDER: tuple[str, ...] = ("claude", "gemini") +SANITIZED_CLOUD_PROVIDER_ORDER: tuple[str, ...] = ( + "ollama_gcp_a", + "ollama_gcp_b", + *PAID_PROVIDER_ORDER, +) def production_provider_order( @@ -136,6 +141,8 @@ def production_route_readback( gemini_enabled: bool, gemini_authentication: dict[str, Any] | None = None, cost_guard: dict[str, Any] | None = None, + gemini_execution_mode: str = "production", + gemini_canary_percent: int = 0, claude_configured: bool = False, claude_enabled: bool = False, claude_authentication: dict[str, Any] | None = None, @@ -156,6 +163,18 @@ def production_route_readback( authentication=claude_authentication, cost_guard=claude_cost_guard, ) + gemini_mode = str(gemini_execution_mode or "shadow").strip().lower() + gemini_percent = max(0, min(100, int(gemini_canary_percent or 0))) + gemini["execution_mode"] = gemini_mode + gemini["canary_percent"] = gemini_percent + gemini["production_executable"] = bool( + gemini["route_ready"] and gemini_mode == "production" + ) + gemini["canary_executable"] = bool( + gemini["route_ready"] + and gemini_mode == "canary" + and gemini_percent > 0 + ) execution_mode = str(claude_execution_mode or "shadow").strip().lower() canary_percent = max(0, min(100, int(claude_canary_percent or 0))) claude["execution_mode"] = execution_mode @@ -168,13 +187,14 @@ def production_route_readback( and execution_mode == "canary" and canary_percent > 0 ) - gemini["production_executable"] = bool(gemini["route_ready"]) hops = [ { "position": 1, "provider": "ollama_gcp_a", "identity": "GCP-A", "runtime": "Ollama", + "data_boundary": "cloud", + "transport_security_status": "public_http_stopgap_mesh_pending", "paid": False, "production_executable": True, }, @@ -183,6 +203,8 @@ def production_route_readback( "provider": "ollama_gcp_b", "identity": "GCP-B", "runtime": "Ollama", + "data_boundary": "cloud", + "transport_security_status": "public_http_stopgap_mesh_pending", "paid": False, "production_executable": True, }, @@ -191,6 +213,8 @@ def production_route_readback( "provider": "ollama_local", "identity": "host111", "runtime": "Ollama", + "data_boundary": "local", + "transport_security_status": "private_lan", "paid": False, "production_executable": True, }, @@ -215,6 +239,8 @@ def production_route_readback( "runtime": "gemini-2.5-flash-lite", "paid": True, "production_executable": gemini["production_executable"], + "canary_executable": gemini["canary_executable"], + "execution_mode": gemini_mode, "configured": bool(gemini_configured), "authenticated": gemini["authenticated"], "authentication_verified": gemini["authentication_verified"], @@ -228,6 +254,16 @@ def production_route_readback( "GCP-A -> GCP-B -> host111 Ollama -> " "Anthropic Claude API -> Gemini API" ), + "transport_security": { + "status": "degraded_mesh_pending", + "production_closure": False, + "reason": "gcp_ollama_public_http_stopgap", + "replacement": "wireguard_mesh_10.77.114.x", + "sanitized_generation_gate": ( + "durable_same_run_endpoint_bound_transport_receipt_required" + ), + "receipt_ttl_max_seconds": 120, + }, "hops": hops, "legacy_provider_policy": { provider: "non_executable_shadow_metadata_only" diff --git a/apps/api/src/services/ai_providers/claude.py b/apps/api/src/services/ai_providers/claude.py index 5f4b5a7f4..93277c733 100644 --- a/apps/api/src/services/ai_providers/claude.py +++ b/apps/api/src/services/ai_providers/claude.py @@ -21,6 +21,7 @@ from src.core.config import get_settings from src.plugins.mcp.interfaces import MCPTool from src.services.ai_providers.interfaces import ( AIResult, + bounded_cloud_output_tokens, cloud_context_block_reason, is_provider_enabled_by_env, ) @@ -283,7 +284,10 @@ class ClaudeProvider: try: from src.services.ai_control import is_provider_disabled - if await is_provider_disabled("claude"): + if await is_provider_disabled( + "claude", + run_id=str((context or {}).get("run_id") or ""), + ): return False, "claude_disabled_by_runtime_control", mode, bucket except Exception: return False, "claude_disable_state_unavailable", mode, bucket @@ -332,11 +336,15 @@ class ClaudeProvider: limiter = get_ai_rate_limiter() model_name = str(settings.CLAUDE_MODEL).strip() + max_output_tokens = bounded_cloud_output_tokens( + settings.CLAUDE_MAX_OUTPUT_TOKENS, + context, + ) reservation = await limiter.reserve_generation( "claude", prompt, model=model_name, - max_output_tokens=settings.CLAUDE_MAX_OUTPUT_TOKENS, + max_output_tokens=max_output_tokens, context=context, ) audit = self._audit_metadata( @@ -373,7 +381,7 @@ class ClaudeProvider: }, json={ "model": model_name, - "max_tokens": settings.CLAUDE_MAX_OUTPUT_TOKENS, + "max_tokens": max_output_tokens, "messages": [{"role": "user", "content": prompt}], "tools": [_analysis_tool()], "tool_choice": {"type": "tool", "name": "submit_analysis"}, diff --git a/apps/api/src/services/ai_providers/gemini.py b/apps/api/src/services/ai_providers/gemini.py index ea6c8b027..462dffd26 100644 --- a/apps/api/src/services/ai_providers/gemini.py +++ b/apps/api/src/services/ai_providers/gemini.py @@ -15,6 +15,7 @@ Google Gemini Cloud API (gemini-2.5-flash-lite) from __future__ import annotations +import hashlib import time from typing import Any @@ -24,6 +25,7 @@ import structlog from src.core.config import get_settings from src.services.ai_providers.interfaces import ( AIResult, + bounded_cloud_output_tokens, cloud_context_block_reason, is_provider_enabled_by_env, ) @@ -42,7 +44,9 @@ def classify_gemini_provider_error(error: Exception) -> str: if status_code == 400: try: payload = error.response.json() - error_payload = payload.get("error", {}) if isinstance(payload, dict) else {} + error_payload = ( + payload.get("error", {}) if isinstance(payload, dict) else {} + ) markers: list[str] = [ str(error_payload.get("status", "")), str(error_payload.get("message", "")), @@ -50,8 +54,7 @@ def classify_gemini_provider_error(error: Exception) -> str: for detail in error_payload.get("details", []) or []: if isinstance(detail, dict): markers.extend( - str(detail.get(key, "")) - for key in ("reason", "status") + str(detail.get(key, "")) for key in ("reason", "status") ) joined = " ".join(markers).upper() if "API_KEY_INVALID" in joined or "API KEY NOT VALID" in joined: @@ -75,7 +78,9 @@ class GeminiProvider: async def _get_client(self) -> httpx.AsyncClient: if self._http_client is None or self._http_client.is_closed: - self._http_client = httpx.AsyncClient(timeout=httpx.Timeout(30.0, connect=10.0)) + self._http_client = httpx.AsyncClient( + timeout=httpx.Timeout(30.0, connect=10.0) + ) return self._http_client @property @@ -86,7 +91,11 @@ class GeminiProvider: def is_enabled(self) -> bool: if not settings.GEMINI_API_KEY: return False - return is_provider_enabled_by_env("gemini") + mode = str(getattr(settings, "GEMINI_EXECUTION_MODE", "shadow")).strip().lower() + mode_enabled = mode == "production" or ( + mode == "canary" and int(getattr(settings, "GEMINI_CANARY_PERCENT", 0)) > 0 + ) + return is_provider_enabled_by_env("gemini") and mode_enabled @property def capabilities(self) -> set[str]: @@ -96,20 +105,57 @@ class GeminiProvider: def privacy_level(self) -> str: return "cloud" + @staticmethod + def _execution_mode_gate( + context: dict[str, Any] | None, + ) -> tuple[bool, str, str, int | None]: + mode = str(getattr(settings, "GEMINI_EXECUTION_MODE", "shadow")).strip().lower() + if mode == "production": + return True, "production_route", mode, None + if mode == "shadow": + return False, "gemini_shadow_metadata_only", mode, None + if mode != "canary": + return False, "gemini_execution_mode_invalid", mode, None + + percent = max( + 0, + min(100, int(getattr(settings, "GEMINI_CANARY_PERCENT", 0))), + ) + run_id = str((context or {}).get("run_id") or "") + if percent <= 0 or not run_id: + return False, "gemini_canary_not_selected", mode, None + bucket = int(hashlib.sha256(run_id.encode("utf-8")).hexdigest()[:8], 16) % 100 + return ( + bucket < percent, + ( + "gemini_canary_selected" + if bucket < percent + else "gemini_canary_not_selected" + ), + mode, + bucket, + ) + @staticmethod def _audit_metadata( *, receipt_id: str | None = None, pricing_version: str | None = None, + execution_mode: str | None = None, + canary_bucket: int | None = None, ) -> dict[str, Any]: return { "schema_version": "paid_provider_audit_v1", "paid_provider": True, "provider": "gemini", "fallback_position": 5, - "execution_mode": "production", + "execution_mode": execution_mode + or str(getattr(settings, "GEMINI_EXECUTION_MODE", "shadow")) + .strip() + .lower(), "generation_receipt_id": receipt_id, "pricing_version": pricing_version, + "canary_bucket": canary_bucket, } async def analyze( @@ -117,6 +163,18 @@ class GeminiProvider: prompt: str, context: dict | None = None, ) -> AIResult: + allowed, reason, mode, bucket = self._execution_mode_gate(context) + if not allowed: + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=reason, + audit_metadata=self._audit_metadata( + execution_mode=mode, + canary_bucket=bucket, + ), + ) privacy_block = cloud_context_block_reason(context) if privacy_block: return AIResult( @@ -124,7 +182,10 @@ class GeminiProvider: success=False, provider=self.name, error=privacy_block, - audit_metadata=self._audit_metadata(), + audit_metadata=self._audit_metadata( + execution_mode=mode, + canary_bucket=bucket, + ), ) if not settings.GEMINI_API_KEY: return AIResult( @@ -132,7 +193,10 @@ class GeminiProvider: success=False, provider=self.name, error="GEMINI_API_KEY not configured", - audit_metadata=self._audit_metadata(), + audit_metadata=self._audit_metadata( + execution_mode=mode, + canary_bucket=bucket, + ), ) if not is_provider_enabled_by_env("gemini"): return AIResult( @@ -140,19 +204,28 @@ class GeminiProvider: success=False, provider=self.name, error="gemini_disabled_by_environment", - audit_metadata=self._audit_metadata(), + audit_metadata=self._audit_metadata( + execution_mode=mode, + canary_bucket=bucket, + ), ) try: from src.services.ai_control import is_provider_disabled - if await is_provider_disabled("gemini"): + if await is_provider_disabled( + "gemini", + run_id=str((context or {}).get("run_id") or ""), + ): return AIResult( raw_response="", success=False, provider=self.name, error="gemini_disabled_by_runtime_control", - audit_metadata=self._audit_metadata(), + audit_metadata=self._audit_metadata( + execution_mode=mode, + canary_bucket=bucket, + ), ) except Exception: # Paid fallback is fail-closed when its durable disable-state @@ -162,23 +235,32 @@ class GeminiProvider: success=False, provider=self.name, error="gemini_disable_state_unavailable", - audit_metadata=self._audit_metadata(), + audit_metadata=self._audit_metadata( + execution_mode=mode, + canary_bucket=bucket, + ), ) from src.services.ai_rate_limiter import get_ai_rate_limiter rate_limiter = get_ai_rate_limiter() model_name = settings.GEMINI_MODEL + max_output_tokens = bounded_cloud_output_tokens( + settings.GEMINI_MAX_OUTPUT_TOKENS, + context, + ) reservation = await rate_limiter.reserve_generation( "gemini", prompt, model=model_name, - max_output_tokens=settings.GEMINI_MAX_OUTPUT_TOKENS, + max_output_tokens=max_output_tokens, context=context, ) audit = self._audit_metadata( receipt_id=reservation.receipt_id, pricing_version=reservation.pricing_version, + execution_mode=mode, + canary_bucket=bucket, ) if not reservation.allowed: logger.warning( @@ -205,8 +287,10 @@ class GeminiProvider: json={ "contents": [{"parts": [{"text": prompt}]}], "generationConfig": { - "temperature": registry.get_provider_options("gemini").get("temperature", 0.1), - "maxOutputTokens": settings.GEMINI_MAX_OUTPUT_TOKENS, + "temperature": registry.get_provider_options("gemini").get( + "temperature", 0.1 + ), + "maxOutputTokens": max_output_tokens, "responseMimeType": "application/json", }, }, @@ -313,8 +397,8 @@ class GeminiProvider: try: from src.services.ai_rate_limiter import get_ai_rate_limiter - authentication = await get_ai_rate_limiter().get_provider_authentication_status( - "gemini" + authentication = ( + await get_ai_rate_limiter().get_provider_authentication_status("gemini") ) return bool( authentication.get("authenticated") is True diff --git a/apps/api/src/services/ai_providers/interfaces.py b/apps/api/src/services/ai_providers/interfaces.py index 01619ea16..db3b91a78 100644 --- a/apps/api/src/services/ai_providers/interfaces.py +++ b/apps/api/src/services/ai_providers/interfaces.py @@ -138,6 +138,25 @@ def cloud_context_block_reason( return _walk(context or {}) + +def bounded_cloud_output_tokens( + configured_limit: int, + context: dict[str, Any] | None, +) -> int: + """Narrow a paid synthetic canary output budget without widening defaults.""" + + configured = max(1, int(configured_limit)) + candidate = (context or {}).get("paid_canary_max_output_tokens") + if (context or {}).get("synthetic_validation") is not True: + return configured + if (context or {}).get("infrastructure_remediation_write_allowed") is not False: + return configured + try: + requested = int(candidate) + except (TypeError, ValueError): + return configured + return min(configured, max(1, requested)) + # ============================================================================= # AIResult — 標準化 AI 分析結果 # ============================================================================= diff --git a/apps/api/src/services/ai_providers/ollama.py b/apps/api/src/services/ai_providers/ollama.py index de6a9f7f5..03c3b3603 100644 --- a/apps/api/src/services/ai_providers/ollama.py +++ b/apps/api/src/services/ai_providers/ollama.py @@ -11,8 +11,10 @@ Ollama Provider - Phase 24 ADR-052 from __future__ import annotations +import hashlib import json import time +from collections.abc import Awaitable, Callable import httpx import structlog @@ -21,9 +23,11 @@ from src.core.config import get_settings from src.plugins.mcp.interfaces import MCPTool from src.services.ai_providers.interfaces import ( AIResult, + cloud_context_block_reason, is_provider_enabled_by_env, ) from src.services.ai_providers.tool_schema import openai_tools_for_agent +from src.services.cloud_transport_receipt import verify_cloud_transport_receipt from src.services.model_registry import get_model_registry logger = structlog.get_logger(__name__) @@ -41,12 +45,73 @@ def _normalized_url(value: str | None) -> str: def _is_gcp_alert_lane(endpoint_url: str) -> bool: """Return true for the CPU-only GCP-A/B synchronous alert lane.""" endpoint = _normalized_url(endpoint_url) - return endpoint in { + return bool(endpoint) and endpoint in { _normalized_url(getattr(settings, "OLLAMA_URL", "")), _normalized_url(getattr(settings, "OLLAMA_SECONDARY_URL", "")), } +CloudTransportReceiptVerifier = Callable[..., Awaitable[str | None]] + + +def _gcp_provider_identity(endpoint_url: str) -> str | None: + endpoint = _normalized_url(endpoint_url) + if endpoint == _normalized_url(getattr(settings, "OLLAMA_URL", "")): + return "ollama_gcp_a" + if endpoint == _normalized_url(getattr(settings, "OLLAMA_SECONDARY_URL", "")): + return "ollama_gcp_b" + return None + + +async def _gcp_cloud_input_block_reason( + *, + prompt: str, + context: dict | None, + endpoint_url: str, + receipt_verifier: CloudTransportReceiptVerifier, +) -> str | None: + """Require sanitizer and independent transport receipts on GCP Ollama.""" + + if not _is_gcp_alert_lane(endpoint_url): + return None + block_reason = cloud_context_block_reason(context) + if block_reason: + return block_reason + cloud_context = context or {} + provider = _gcp_provider_identity(endpoint_url) + if provider is None: + return "gcp_transport_provider_identity_unresolved" + transport_reason = await receipt_verifier( + context=cloud_context, + provider=provider, + endpoint_url=endpoint_url, + ) + if transport_reason: + return transport_reason + prompt_sha256 = hashlib.sha256(prompt.encode("utf-8")).hexdigest() + if cloud_context.get("synthetic_validation") is True: + if ( + cloud_context.get("synthetic_prompt_sha256") == prompt_sha256 + and cloud_context.get("infrastructure_remediation_write_allowed") is False + ): + return None + return "gcp_synthetic_prompt_receipt_invalid" + receipt = cloud_context.get("cloud_sanitization_receipt") + if not isinstance(receipt, dict): + return "gcp_cloud_sanitization_receipt_missing" + if ( + receipt.get("schema_version") != "openclaw_paid_cloud_sanitization_v1" + or receipt.get("status") != "verified" + or receipt.get("raw_payload_forwarded") is not False + or receipt.get("raw_log_payload_forwarded") is not False + or receipt.get("secret_value_exposed") is not False + or receipt.get("private_network_value_exposed") is not False + or cloud_context.get("paid_prompt_sha256") != prompt_sha256 + ): + return "gcp_cloud_sanitization_receipt_invalid" + return None + + def _resolve_model_for_endpoint( *, requested_model: str, @@ -88,14 +153,21 @@ def _resolve_model_for_endpoint( class OllamaProvider: """ - Ollama 本地 LLM Provider + GCP-A Ollama Provider - privacy_level: local (可處理機密資料) + privacy_level: cloud（不得接收 require_local/未脫敏資料） capabilities: rca, chat, code_review """ - def __init__(self) -> None: + def __init__( + self, + *, + transport_receipt_verifier: CloudTransportReceiptVerifier | None = None, + ) -> None: self._http_client: httpx.AsyncClient | None = None + self._transport_receipt_verifier = ( + transport_receipt_verifier or verify_cloud_transport_receipt + ) async def _get_client(self) -> httpx.AsyncClient: if self._http_client is None or self._http_client.is_closed: @@ -121,7 +193,7 @@ class OllamaProvider: @property def privacy_level(self) -> str: - return "local" + return "cloud" async def analyze( self, @@ -130,10 +202,26 @@ class OllamaProvider: ) -> AIResult: start = time.perf_counter() try: - client = await self._get_client() - registry = get_model_registry() endpoint_url = self._endpoint_url() + cloud_block_reason = await _gcp_cloud_input_block_reason( + prompt=prompt, + context=context, + endpoint_url=endpoint_url, + receipt_verifier=self._transport_receipt_verifier, + ) + if cloud_block_reason: + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=cloud_block_reason, + audit_metadata={ + "generation_attempted": False, + "transport_boundary": "public_http_degraded", + }, + ) + client = await self._get_client() requested_model = str((context or {}).get("ollama_model") or registry.get_model("ollama", "rca")).strip() model_name = _resolve_model_for_endpoint( requested_model=requested_model, @@ -209,9 +297,45 @@ class OllamaProvider: ) -> AIResult: """Run Ollama chat tool calling loop when the local model supports tools.""" + cloud_block_reason = await _gcp_cloud_input_block_reason( + prompt=prompt, + context=context, + endpoint_url=self._endpoint_url(), + receipt_verifier=self._transport_receipt_verifier, + ) + if cloud_block_reason: + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=cloud_block_reason, + audit_metadata={"generation_attempted": False}, + ) + if not available_tools: return await self.analyze(prompt, context=context) + # GCP-A/B currently cross a public clear-text HTTP boundary. Even when + # the initial prompt has a prompt-bound sanitization receipt, an MCP + # result is new data that has not passed that receipt. A second chat + # iteration would therefore export unverified Kubernetes, database, + # metrics, topology, or log content. Keep cloud tool loops fail closed + # until every tool result has its own structured sanitization receipt or + # the providers move behind an authenticated encrypted mesh. Host111 is + # unaffected because its endpoint is not a GCP alert lane. + if _is_gcp_alert_lane(self._endpoint_url()): + return AIResult( + raw_response="", + success=False, + provider=self.name, + error="gcp_tool_loop_blocked_unverified_tool_results", + audit_metadata={ + "generation_attempted": False, + "tool_execution_attempted": False, + "transport_boundary": "public_http_degraded", + }, + ) + tools_schema = openai_tools_for_agent(available_tools, agent_role) if not tools_schema: return AIResult( @@ -341,13 +465,18 @@ class OllamaLocalProvider(OllamaProvider): Ollama Local fallback Provider 使用 OLLAMA_FALLBACK_URL 作為本地最後防線端點。 - ADR-110 目前設定為 110 nginx proxy → 111 Ollama；188 不得再作為 Ollama provider。 + Host111 is selected directly; host110/188 must never be provider identities. """ @property def name(self) -> str: return "ollama_local" + @property + def privacy_level(self) -> str: + """Host111 is the only Ollama hop permitted for local-only data.""" + return "local" + @property def is_enabled(self) -> bool: import os diff --git a/apps/api/src/services/ai_rate_limiter.py b/apps/api/src/services/ai_rate_limiter.py index 40d8e2c5a..8c8a4ba67 100644 --- a/apps/api/src/services/ai_rate_limiter.py +++ b/apps/api/src/services/ai_rate_limiter.py @@ -23,6 +23,7 @@ import json import math import re from dataclasses import dataclass +from decimal import Decimal, InvalidOperation from typing import Any from uuid import uuid4 @@ -37,7 +38,7 @@ logger = structlog.get_logger(__name__) RATE_LIMITS = { "gemini": { - "rpm": 10, # 每分鐘請求數 + "rpm": 10, # 每分鐘請求數 "daily_requests": 500, # 每日請求數 "daily_tokens": 100_000, # 每日 Token 數 }, @@ -50,7 +51,7 @@ RATE_LIMITS = { # 只保留 RPM 限制 (併發控制) + 極大的 daily 上限 (監控用) # 2026-04-03 ogt: I3 — "nvidia" → "openclaw_nemo" 對齊 AIProviderEnum (Phase 24) "openclaw_nemo": { - "rpm": 10, # 每分鐘請求數 (放寬到 10) + "rpm": 10, # 每分鐘請求數 (放寬到 10) "daily_requests": 99999, # 🔴 免費版無限制！設大數避免誤觸 "daily_tokens": 9999999, # 免費版無限制 }, @@ -78,6 +79,7 @@ COST_LIMITS = { }, } + @dataclass(frozen=True) class PaidProviderPricingPolicy: """Reviewed exact-model pricing used by the paid-fallback guard.""" @@ -104,8 +106,7 @@ GEMINI_PRICING_POLICIES: dict[str, PaidProviderPricingPolicy] = { input_usd_per_million=0.10, output_usd_per_million=0.40, source=( - "https://ai.google.dev/gemini-api/docs/pricing" - "#gemini-2.5-flash-lite" + "https://ai.google.dev/gemini-api/docs/pricing" "#gemini-2.5-flash-lite" ), version="google-ai-standard-text-2026-07-14", checked_at="2026-07-14", @@ -151,13 +152,18 @@ def get_paid_provider_pricing_policy( return get_claude_pricing_policy(model) return None + # Redis Keys REDIS_KEY_PREFIX = "ai_rate:" RPM_KEY = f"{REDIS_KEY_PREFIX}rpm:{{provider}}" DAILY_REQ_KEY = f"{REDIS_KEY_PREFIX}daily_req:{{provider}}:{{date}}" DAILY_TOKEN_KEY = f"{REDIS_KEY_PREFIX}daily_token:{{provider}}:{{date}}" -DAILY_TOKEN_RESERVED_KEY = f"{REDIS_KEY_PREFIX}daily_token_reserved:{{provider}}:{{date}}" -DAILY_COST_MICRO_USD_KEY = f"{REDIS_KEY_PREFIX}daily_cost_micro_usd:{{provider}}:{{date}}" +DAILY_TOKEN_RESERVED_KEY = ( + f"{REDIS_KEY_PREFIX}daily_token_reserved:{{provider}}:{{date}}" +) +DAILY_COST_MICRO_USD_KEY = ( + f"{REDIS_KEY_PREFIX}daily_cost_micro_usd:{{provider}}:{{date}}" +) DAILY_COST_RESERVED_MICRO_USD_KEY = ( f"{REDIS_KEY_PREFIX}daily_cost_reserved_micro_usd:{{provider}}:{{date}}" ) @@ -169,12 +175,15 @@ GENERATION_RUN_IDEMPOTENCY_KEY = f"{REDIS_KEY_PREFIX}run:{{provider}}:{{identity GENERATION_FAILURE_COOLDOWN_KEY = ( f"{REDIS_KEY_PREFIX}failure_cooldown:{{provider}}:{{work_item_hash}}" ) +PAID_RUN_BUDGET_KEY = f"{REDIS_KEY_PREFIX}paid_run_budget:{{budget_id}}" PROVIDER_AUTH_STATUS_KEY = f"{REDIS_KEY_PREFIX}auth_status:{{provider}}" # 2026-03-29 ogt: 累積成本 Key (不過期，手動重置) TOTAL_COST_KEY = f"{REDIS_KEY_PREFIX}total_cost:{{provider}}" COST_ALERT_SENT_KEY = f"{REDIS_KEY_PREFIX}cost_alert_sent:{{provider}}" _MICRO_USD = 1_000_000 +_PAID_CANARY_WORK_ITEM_ID = "AIA-SRE-013" +_PAID_CANARY_MAX_COST_MICRO_USD = 250_000 _CORRELATION_VALUE_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$") @@ -200,6 +209,8 @@ class AIGenerationReservation: run_id: str = "" work_item_id: str = "" identity_hash: str = "" + run_budget_id: str = "" + run_budget_max_micro_usd: int = 0 def actual_cost_usd(self, prompt_tokens: int, completion_tokens: int) -> float: """Calculate provider-reported usage with the reserved pricing policy.""" @@ -220,18 +231,25 @@ local total_cost_limit = tonumber(ARGV[5]) local estimated_tokens = tonumber(ARGV[6]) local estimated_cost = tonumber(ARGV[7]) local daily_ttl = tonumber(ARGV[8]) +local run_budget_required = ARGV[24] == '1' +local run_budget_id = ARGV[25] +local run_budget_max = tonumber(ARGV[26]) or 0 local function receipt(status, reason) local reserved_prompt_tokens = '0' local reserved_completion_tokens = '0' local reserved_total_tokens = '0' local reserved_cost_micro_usd = '0' + local run_budget_reserved_cost_micro_usd = '0' local terminal_state = 'blocked_no_write' if status == 'reserved' then reserved_prompt_tokens = ARGV[16] reserved_completion_tokens = ARGV[17] reserved_total_tokens = ARGV[6] reserved_cost_micro_usd = ARGV[7] + if run_budget_required then + run_budget_reserved_cost_micro_usd = ARGV[7] + end terminal_state = 'pending_finalize' end redis.call('HSET', KEYS[9], @@ -252,6 +270,10 @@ local function receipt(status, reason) 'pricing_checked_at', ARGV[21], 'pricing_input_usd_per_million', ARGV[22], 'pricing_output_usd_per_million', ARGV[23], + 'run_budget_required', ARGV[24], + 'run_budget_id', run_budget_id, + 'run_budget_max_micro_usd', ARGV[26], + 'run_budget_reserved_cost_micro_usd', run_budget_reserved_cost_micro_usd, 'estimated_prompt_tokens', ARGV[16], 'estimated_completion_tokens', ARGV[17], 'estimated_total_tokens', ARGV[6], @@ -293,6 +315,37 @@ local current_total_cost = math.floor( (tonumber(redis.call('GET', KEYS[7]) or '0') * 1000000) + 0.5 ) local reserved_total_cost = tonumber(redis.call('GET', KEYS[8]) or '0') +local run_budget_reserved = 0 +local run_budget_accounted = 0 + +if run_budget_required then + if run_budget_id == '' or run_budget_max <= 0 then + receipt('blocked', 'run_cost_cap_invalid') + return {0, 'run_cost_cap_invalid', ARGV[10]} + end + local existing_budget_id = redis.call('HGET', KEYS[12], 'run_budget_id') + local existing_budget_max = tonumber( + redis.call('HGET', KEYS[12], 'max_cost_micro_usd') or '0' + ) + if existing_budget_id and existing_budget_id ~= run_budget_id then + receipt('blocked', 'run_cost_cap_mismatch') + return {0, 'run_cost_cap_mismatch', ARGV[10]} + end + if existing_budget_max > 0 and existing_budget_max ~= run_budget_max then + receipt('blocked', 'run_cost_cap_mismatch') + return {0, 'run_cost_cap_mismatch', ARGV[10]} + end + run_budget_reserved = tonumber( + redis.call('HGET', KEYS[12], 'reserved_cost_micro_usd') or '0' + ) + run_budget_accounted = tonumber( + redis.call('HGET', KEYS[12], 'accounted_cost_micro_usd') or '0' + ) + if run_budget_accounted + run_budget_reserved + estimated_cost > run_budget_max then + receipt('blocked', 'run_cost_limit') + return {0, 'run_cost_limit', ARGV[10]} + end +end if current_rpm + 1 > rpm_limit then receipt('blocked', 'rpm_limit') @@ -328,6 +381,21 @@ redis.call('EXPIRE', KEYS[6], daily_ttl) -- Cumulative paid-spend reservations do not expire. A missing finalize receipt -- must keep consuming the hard cap until reconciliation/reset, never turn green. redis.call('INCRBY', KEYS[8], estimated_cost) +if run_budget_required then + redis.call('HSET', KEYS[12], + 'schema_version', 'ai_paid_run_budget_receipt_v1', + 'run_budget_id', run_budget_id, + 'run_id', ARGV[14], + 'work_item_id', ARGV[15], + 'max_cost_micro_usd', run_budget_max, + 'status', 'pending_finalize') + redis.call('HSETNX', KEYS[12], 'reserved_cost_micro_usd', '0') + redis.call('HSETNX', KEYS[12], 'accounted_cost_micro_usd', '0') + redis.call('HSETNX', KEYS[12], 'reservation_count', '0') + redis.call('HSETNX', KEYS[12], 'finalized_count', '0') + redis.call('HINCRBY', KEYS[12], 'reserved_cost_micro_usd', estimated_cost) + redis.call('HINCRBY', KEYS[12], 'reservation_count', 1) +end receipt('reserved', 'allowed') return {1, 'allowed', ARGV[10]} """ @@ -391,6 +459,34 @@ elseif actual_tokens <= 0 or actual_cost <= 0 then final_status = 'succeeded_usage_estimated' end +local run_budget_required = redis.call('HGET', KEYS[7], 'run_budget_required') == '1' +local run_budget_id = redis.call('HGET', KEYS[7], 'run_budget_id') or '' +local run_budget_max = tonumber( + redis.call('HGET', KEYS[7], 'run_budget_max_micro_usd') or '0' +) +local run_budget_reserved = 0 +local run_budget_accounted = 0 +if run_budget_required then + if run_budget_id == '' or run_budget_max <= 0 then + return {0, 'run_budget_receipt_invalid'} + end + if redis.call('HGET', KEYS[10], 'run_budget_id') ~= run_budget_id then + return {0, 'run_budget_identity_mismatch'} + end + if tonumber(redis.call('HGET', KEYS[10], 'max_cost_micro_usd') or '0') ~= run_budget_max then + return {0, 'run_budget_cap_mismatch'} + end + run_budget_reserved = tonumber( + redis.call('HGET', KEYS[10], 'reserved_cost_micro_usd') or '-1' + ) + run_budget_accounted = tonumber( + redis.call('HGET', KEYS[10], 'accounted_cost_micro_usd') or '-1' + ) + if run_budget_reserved < estimated_cost or run_budget_accounted < 0 then + return {0, 'run_budget_reservation_mismatch'} + end +end + local token_reserved = tonumber(redis.call('GET', KEYS[2]) or '0') redis.call('SET', KEYS[2], math.max(0, token_reserved - estimated_tokens), 'EX', ARGV[8]) local daily_cost_reserved = tonumber(redis.call('GET', KEYS[4]) or '0') @@ -419,10 +515,30 @@ redis.call('HSET', KEYS[7], 'actual_completion_tokens', ARGV[2], 'actual_total_tokens', actual_tokens, 'actual_cost_micro_usd', ARGV[3], + 'reserved_prompt_tokens', '0', + 'reserved_completion_tokens', '0', + 'reserved_total_tokens', '0', + 'reserved_cost_micro_usd', '0', + 'run_budget_reserved_cost_micro_usd', '0', + 'run_budget_accounted_cost_micro_usd', accounted_cost, 'accounted_tokens', accounted_tokens, 'accounted_cost_micro_usd', accounted_cost, 'usage_source', usage_source) +if run_budget_required then + local remaining_run_reserved = run_budget_reserved - estimated_cost + local new_run_accounted = run_budget_accounted + accounted_cost + local run_budget_status = 'pending_finalize' + if remaining_run_reserved == 0 then run_budget_status = 'finalized' end + if new_run_accounted > run_budget_max then run_budget_status = 'cap_exceeded' end + redis.call('HSET', KEYS[10], + 'reserved_cost_micro_usd', remaining_run_reserved, + 'accounted_cost_micro_usd', new_run_accounted, + 'status', run_budget_status, + 'updated_at', ARGV[5]) + redis.call('HINCRBY', KEYS[10], 'finalized_count', 1) +end + local authentication_state = 'unknown' local verification_status = 'not_verified_last_generation_failed' if success then @@ -476,12 +592,14 @@ class AIRateLimiter: """Lazy load Redis""" if self._redis is None: from src.core.redis_client import get_redis + self._redis = get_redis() return self._redis def _get_today(self) -> str: """取得今日日期 (台北時區)""" from src.utils.timezone import now_taipei + return now_taipei().strftime("%Y-%m-%d") def _seconds_until_tomorrow(self) -> int: @@ -530,6 +648,47 @@ class AIRateLimiter: payload = "\x1f".join(parts).encode("utf-8") return hashlib.sha256(payload).hexdigest() + @classmethod + def _paid_run_budget_contract( + cls, + context: dict[str, Any] | None, + *, + run_id: str, + work_item_id: str, + ) -> tuple[bool, str, int, str | None]: + """Return one bounded aggregate paid-run budget or a fail-closed reason.""" + + payload = context or {} + synthetic_validation = payload.get("synthetic_validation") + canary_work_item = work_item_id == _PAID_CANARY_WORK_ITEM_ID + required = canary_work_item or "paid_canary_max_cost_usd" in payload + if not required: + return False, "", 0, None + + budget_hash = cls._identity_hash(run_id, work_item_id) + budget_id = f"paid-run-{budget_hash[:32]}" + if canary_work_item and synthetic_validation is not True: + return True, budget_id, 0, "run_cost_cap_context_invalid" + + raw_cap = payload.get("paid_canary_max_cost_usd") + if raw_cap is None: + return True, budget_id, 0, "run_cost_cap_missing" + if isinstance(raw_cap, bool): + return True, budget_id, 0, "run_cost_cap_invalid" + try: + cap_usd = Decimal(str(raw_cap)) + except (InvalidOperation, ValueError): + return True, budget_id, 0, "run_cost_cap_invalid" + if not cap_usd.is_finite() or cap_usd <= 0 or cap_usd > Decimal("0.25"): + return True, budget_id, 0, "run_cost_cap_invalid" + cap_micro = cap_usd * _MICRO_USD + if cap_micro != cap_micro.to_integral_value(): + return True, budget_id, 0, "run_cost_cap_invalid" + max_cost_micro_usd = int(cap_micro) + if not 1 <= max_cost_micro_usd <= _PAID_CANARY_MAX_COST_MICRO_USD: + return True, budget_id, 0, "run_cost_cap_invalid" + return True, budget_id, max_cost_micro_usd, None + @staticmethod def _estimate_paid_usage( prompt: str, @@ -561,12 +720,8 @@ class AIRateLimiter: daily_tokens=int(getattr(settings, f"{prefix}_DAILY_TOKEN_LIMIT")), ) costs.update( - daily_cost_usd=float( - getattr(settings, f"{prefix}_DAILY_COST_LIMIT_USD") - ), - total_cost_usd=float( - getattr(settings, f"{prefix}_TOTAL_COST_LIMIT_USD") - ), + daily_cost_usd=float(getattr(settings, f"{prefix}_DAILY_COST_LIMIT_USD")), + total_cost_usd=float(getattr(settings, f"{prefix}_TOTAL_COST_LIMIT_USD")), alert_threshold_usd=float( getattr(settings, f"{prefix}_COST_ALERT_THRESHOLD_USD") ), @@ -582,6 +737,9 @@ class AIRateLimiter: date: str, model: str, context: dict[str, Any] | None, + run_budget_required: bool = False, + run_budget_id: str = "", + run_budget_max_micro_usd: int = 0, ) -> bool: """Best-effort durable no-reservation receipt for pre-pricing blocks.""" from src.utils.timezone import now_taipei_iso @@ -609,6 +767,10 @@ class AIRateLimiter: "run_id": run_id, "work_item_id": work_item_id, "model": safe_model, + "run_budget_required": "1" if run_budget_required else "0", + "run_budget_id": run_budget_id, + "run_budget_max_micro_usd": str(run_budget_max_micro_usd), + "run_budget_reserved_cost_micro_usd": "0", "estimated_prompt_tokens": "0", "estimated_completion_tokens": "0", "estimated_total_tokens": "0", @@ -714,6 +876,52 @@ class AIRateLimiter: run_id, ) receipt_id = f"gen-{identity_hash[:32]}" + ( + run_budget_required, + run_budget_id, + run_budget_max_micro_usd, + run_budget_error, + ) = self._paid_run_budget_contract( + context, + run_id=run_id, + work_item_id=work_item_id, + ) + if run_budget_error: + receipt_written = await self._write_blocked_generation_receipt( + provider=provider, + receipt_id=receipt_id, + reason=run_budget_error, + date=today, + model=model_name, + context=context, + run_budget_required=run_budget_required, + run_budget_id=run_budget_id, + run_budget_max_micro_usd=run_budget_max_micro_usd, + ) + logger.error( + "ai_generation_reservation_failed_closed", + provider=provider, + receipt_id=receipt_id, + reason=run_budget_error, + receipt_written=receipt_written, + ) + return AIGenerationReservation( + provider=provider, + receipt_id=receipt_id, + allowed=False, + reason=run_budget_error, + estimated_prompt_tokens=0, + estimated_completion_tokens=0, + estimated_cost_usd=0.0, + date=today, + model=model_name, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + identity_hash=identity_hash, + run_budget_id=run_budget_id, + run_budget_max_micro_usd=run_budget_max_micro_usd, + ) pricing = get_paid_provider_pricing_policy(provider, model_name) if pricing is None: receipt_written = await self._write_blocked_generation_receipt( @@ -723,6 +931,9 @@ class AIRateLimiter: date=today, model=model_name, context=context, + run_budget_required=run_budget_required, + run_budget_id=run_budget_id, + run_budget_max_micro_usd=run_budget_max_micro_usd, ) logger.error( "ai_generation_reservation_failed_closed", @@ -746,6 +957,8 @@ class AIRateLimiter: run_id=run_id, work_item_id=work_item_id, identity_hash=identity_hash, + run_budget_id=run_budget_id, + run_budget_max_micro_usd=run_budget_max_micro_usd, ) output_limit = int( @@ -777,6 +990,8 @@ class AIRateLimiter: run_id=run_id, work_item_id=work_item_id, identity_hash=identity_hash, + run_budget_id=run_budget_id, + run_budget_max_micro_usd=run_budget_max_micro_usd, ) try: @@ -821,6 +1036,9 @@ class AIRateLimiter: provider=provider, work_item_hash=self._identity_hash(provider, work_item_id), ), + PAID_RUN_BUDGET_KEY.format( + budget_id=(run_budget_id or f"unused-{identity_hash[:32]}") + ), ] result = await redis.eval( _RESERVE_GENERATION_LUA, @@ -849,10 +1067,17 @@ class AIRateLimiter: pricing.checked_at, pricing.input_usd_per_million, pricing.output_usd_per_million, + 1 if run_budget_required else 0, + run_budget_id, + run_budget_max_micro_usd, ) allowed = bool(int(result[0])) raw_reason = result[1] - reason = raw_reason.decode() if isinstance(raw_reason, bytes) else str(raw_reason) + reason = ( + raw_reason.decode() + if isinstance(raw_reason, bytes) + else str(raw_reason) + ) raw_receipt_id = result[2] if len(result) > 2 else receipt_id result_receipt_id = ( raw_receipt_id.decode() @@ -878,6 +1103,8 @@ class AIRateLimiter: run_id=run_id, work_item_id=work_item_id, identity_hash=identity_hash, + run_budget_id=run_budget_id, + run_budget_max_micro_usd=run_budget_max_micro_usd, ) logger.info( "ai_generation_reservation", @@ -980,9 +1207,15 @@ class AIRateLimiter: receipt_id=reservation.receipt_id, ) keys = [ - DAILY_TOKEN_KEY.format(provider=reservation.provider, date=reservation.date), - DAILY_TOKEN_RESERVED_KEY.format(provider=reservation.provider, date=reservation.date), - DAILY_COST_MICRO_USD_KEY.format(provider=reservation.provider, date=reservation.date), + DAILY_TOKEN_KEY.format( + provider=reservation.provider, date=reservation.date + ), + DAILY_TOKEN_RESERVED_KEY.format( + provider=reservation.provider, date=reservation.date + ), + DAILY_COST_MICRO_USD_KEY.format( + provider=reservation.provider, date=reservation.date + ), DAILY_COST_RESERVED_MICRO_USD_KEY.format( provider=reservation.provider, date=reservation.date, @@ -998,6 +1231,12 @@ class AIRateLimiter: reservation.work_item_id or reservation.receipt_id, ), ), + PAID_RUN_BUDGET_KEY.format( + budget_id=( + reservation.run_budget_id + or f"unused-{reservation.identity_hash[:32]}" + ) + ), ] result = await redis.eval( _FINALIZE_GENERATION_LUA, @@ -1021,7 +1260,11 @@ class AIRateLimiter: ) result_code = int(result[0]) raw_status = result[1] - status = raw_status.decode() if isinstance(raw_status, bytes) else str(raw_status) + status = ( + raw_status.decode() + if isinstance(raw_status, bytes) + else str(raw_status) + ) finalized = result_code in (1, 2) logger.info( "ai_generation_receipt_finalized", @@ -1031,7 +1274,9 @@ class AIRateLimiter: finalized=finalized, ) if result_code == 1: - total_raw = await redis.get(TOTAL_COST_KEY.format(provider=reservation.provider)) + total_raw = await redis.get( + TOTAL_COST_KEY.format(provider=reservation.provider) + ) total_cost = float(total_raw or 0.0) _, costs = self._provider_limits(reservation.provider) if total_cost >= costs["alert_threshold_usd"]: @@ -1050,6 +1295,196 @@ class AIRateLimiter: ) return False + async def get_paid_run_budget_readback( + self, + *, + run_id: str, + work_item_id: str, + expected_max_cost_micro_usd: int, + ) -> dict[str, Any]: + """Return only aggregate, non-content paid-run accounting fields.""" + + if any( + not _CORRELATION_VALUE_PATTERN.fullmatch(str(value or "")) + for value in (run_id, work_item_id) + ): + raise ValueError("paid_run_budget_correlation_invalid") + if isinstance(expected_max_cost_micro_usd, bool): + raise ValueError("paid_run_budget_expected_cap_invalid") + expected_cap = int(expected_max_cost_micro_usd) + if not 1 <= expected_cap <= _PAID_CANARY_MAX_COST_MICRO_USD: + raise ValueError("paid_run_budget_expected_cap_invalid") + + budget_hash = self._identity_hash(run_id, work_item_id) + budget_id = f"paid-run-{budget_hash[:32]}" + redis = await self._get_redis() + if redis is None: + raise RuntimeError("redis_unavailable") + raw = await redis.hgetall(PAID_RUN_BUDGET_KEY.format(budget_id=budget_id)) + + def _decode(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value or "") + + receipt = {_decode(key): _decode(value) for key, value in raw.items()} + if not receipt: + return { + "run_budget_id": budget_id, + "found": False, + "correlation_verified": False, + "cap_verified": False, + "terminal_verified": False, + } + + try: + max_cost_micro_usd = int(receipt.get("max_cost_micro_usd") or 0) + reserved_cost_micro_usd = int(receipt.get("reserved_cost_micro_usd") or 0) + accounted_cost_micro_usd = int(receipt.get("accounted_cost_micro_usd") or 0) + reservation_count = int(receipt.get("reservation_count") or 0) + finalized_count = int(receipt.get("finalized_count") or 0) + except ValueError as exc: + raise ValueError("paid_run_budget_receipt_invalid") from exc + + correlation_verified = bool( + receipt.get("schema_version") == "ai_paid_run_budget_receipt_v1" + and receipt.get("run_budget_id") == budget_id + and receipt.get("run_id") == run_id + and receipt.get("work_item_id") == work_item_id + ) + cap_verified = max_cost_micro_usd == expected_cap + terminal_verified = bool( + correlation_verified + and cap_verified + and reserved_cost_micro_usd == 0 + and 0 <= accounted_cost_micro_usd <= max_cost_micro_usd + and reservation_count > 0 + and finalized_count == reservation_count + and receipt.get("status") == "finalized" + ) + return { + "schema_version": receipt.get("schema_version"), + "run_budget_id": budget_id, + "found": True, + "status": receipt.get("status"), + "max_cost_micro_usd": max_cost_micro_usd, + "reserved_cost_micro_usd": reserved_cost_micro_usd, + "accounted_cost_micro_usd": accounted_cost_micro_usd, + "accounted_cost_usd": round( + accounted_cost_micro_usd / _MICRO_USD, + 8, + ), + "reservation_count": reservation_count, + "finalized_count": finalized_count, + "correlation_verified": correlation_verified, + "cap_verified": cap_verified, + "terminal_verified": terminal_verified, + "raw_prompt_persisted": False, + "raw_response_persisted": False, + "secret_value_present": False, + } + + async def get_generation_receipt_readback( + self, + provider: str, + receipt_id: str, + *, + trace_id: str, + run_id: str, + work_item_id: str, + ) -> dict[str, Any]: + """Read one allowlisted paid-generation receipt for verification. + + Generation receipts never contain prompts, responses or credentials. + This method still returns only fields needed to prove correlation, + accounting finalization and reservation release. + """ + + if provider not in {"claude", "gemini"}: + raise ValueError("paid_generation_receipt_provider_invalid") + if not _CORRELATION_VALUE_PATTERN.fullmatch(str(receipt_id or "")): + raise ValueError("paid_generation_receipt_id_invalid") + expected = { + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": work_item_id, + } + if any( + not _CORRELATION_VALUE_PATTERN.fullmatch(str(value or "")) + for value in expected.values() + ): + raise ValueError("paid_generation_receipt_correlation_invalid") + + redis = await self._get_redis() + if redis is None: + raise RuntimeError("redis_unavailable") + raw = await redis.hgetall( + GENERATION_RECEIPT_KEY.format( + provider=provider, + receipt_id=receipt_id, + ) + ) + + def _decode(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value or "") + + receipt = {_decode(key): _decode(value) for key, value in raw.items()} + if not receipt: + return { + "provider": provider, + "receipt_id": receipt_id, + "found": False, + "correlation_verified": False, + "finalized_accounted": False, + "reservation_released": False, + } + + correlation_verified = all( + receipt.get(field) == value for field, value in expected.items() + ) + finalized_accounted = receipt.get("terminal_state") == "finalized_accounted" + reservation_released = all( + int(receipt.get(field) or 0) == 0 + for field in ( + "reserved_prompt_tokens", + "reserved_completion_tokens", + "reserved_total_tokens", + "reserved_cost_micro_usd", + "run_budget_reserved_cost_micro_usd", + ) + ) + accounted_cost_micro_usd = int(receipt.get("accounted_cost_micro_usd") or 0) + return { + "provider": provider, + "receipt_id": receipt_id, + "found": True, + "status": receipt.get("status"), + "terminal_state": receipt.get("terminal_state"), + "success": receipt.get("success") == "1", + "model": receipt.get("model"), + "pricing_version": receipt.get("pricing_version"), + "usage_source": receipt.get("usage_source"), + "accounted_tokens": int(receipt.get("accounted_tokens") or 0), + "accounted_cost_micro_usd": accounted_cost_micro_usd, + "accounted_cost_usd": round( + accounted_cost_micro_usd / _MICRO_USD, + 8, + ), + "run_budget_required": receipt.get("run_budget_required") == "1", + "run_budget_id": receipt.get("run_budget_id") or None, + "run_budget_max_micro_usd": int( + receipt.get("run_budget_max_micro_usd") or 0 + ), + "correlation_verified": correlation_verified, + "finalized_accounted": finalized_accounted, + "reservation_released": reservation_released, + "raw_prompt_persisted": False, + "raw_response_persisted": False, + "secret_value_present": False, + } + async def check_and_increment( self, provider: str, @@ -1096,7 +1531,10 @@ class AIRateLimiter: ) # 發送告警 (只發一次) await self._send_cost_alert(provider, current_cost, cost_limit) - return False, f"🔴 成本超限! ${current_cost:.2f} >= ${cost_limit:.2f}，已自動切換到 Ollama" + return ( + False, + f"🔴 成本超限! ${current_cost:.2f} >= ${cost_limit:.2f}，已自動切換到 Ollama", + ) # 1. 檢查 RPM rpm_key = RPM_KEY.format(provider=provider) @@ -1124,7 +1562,10 @@ class AIRateLimiter: current=current_daily, limit=limits["daily_requests"], ) - return False, f"Daily request limit exceeded ({current_daily}/{limits['daily_requests']})" + return ( + False, + f"Daily request limit exceeded ({current_daily}/{limits['daily_requests']})", + ) # 3. 檢查每日 Token (如果有追蹤) daily_token_key = DAILY_TOKEN_KEY.format(provider=provider, date=today) @@ -1138,7 +1579,10 @@ class AIRateLimiter: current=current_tokens, limit=limits["daily_tokens"], ) - return False, f"Daily token limit exceeded ({current_tokens}/{limits['daily_tokens']})" + return ( + False, + f"Daily token limit exceeded ({current_tokens}/{limits['daily_tokens']})", + ) # 4. 遞增計數器 pipe = r.pipeline() @@ -1197,7 +1641,9 @@ class AIRateLimiter: if new_total >= alert_threshold: await self._send_cost_warning(provider, new_total, alert_threshold) - async def _send_cost_alert(self, provider: str, current_cost: float, limit: float) -> None: + async def _send_cost_alert( + self, provider: str, current_cost: float, limit: float + ) -> None: """ 2026-03-29 ogt: 發送成本超限告警到 Telegram (只發一次) """ @@ -1271,7 +1717,9 @@ class AIRateLimiter: except Exception as e: logger.error("ai_cost_alert_failed", error=str(e)) - async def _send_cost_warning(self, provider: str, current_cost: float, threshold: float) -> None: + async def _send_cost_warning( + self, provider: str, current_cost: float, threshold: float + ) -> None: """ 2026-03-29 ogt: 發送成本接近上限警告 """ @@ -1374,9 +1822,11 @@ class AIRateLimiter: """Read public-safe authentication evidence; configuration is separate.""" from src.core.config import settings - configured = bool( - getattr(settings, f"{provider.upper()}_API_KEY", "") - ) if provider in {"claude", "gemini"} else False + configured = ( + bool(getattr(settings, f"{provider.upper()}_API_KEY", "")) + if provider in {"claude", "gemini"} + else False + ) base = { "provider": provider, "configured": configured, @@ -1392,7 +1842,9 @@ class AIRateLimiter: redis = await self._get_redis() if redis is None: raise RuntimeError("redis_unavailable") - raw = await redis.hgetall(PROVIDER_AUTH_STATUS_KEY.format(provider=provider)) + raw = await redis.hgetall( + PROVIDER_AUTH_STATUS_KEY.format(provider=provider) + ) except Exception as exc: base["verification_status"] = ( "authentication_readback_unavailable" @@ -1419,9 +1871,7 @@ class AIRateLimiter: authenticated = False base.update( authenticated=authenticated, - authentication_verified=( - decoded.get("authentication_verified") == "true" - ), + authentication_verified=(decoded.get("authentication_verified") == "true"), verification_status=decoded.get( "verification_status", "configured_not_verified" ), @@ -1450,14 +1900,18 @@ class AIRateLimiter: rpm_key = RPM_KEY.format(provider=provider) daily_req_key = DAILY_REQ_KEY.format(provider=provider, date=today) daily_token_key = DAILY_TOKEN_KEY.format(provider=provider, date=today) - daily_token_reserved_key = DAILY_TOKEN_RESERVED_KEY.format(provider=provider, date=today) + daily_token_reserved_key = DAILY_TOKEN_RESERVED_KEY.format( + provider=provider, date=today + ) daily_cost_key = DAILY_COST_MICRO_USD_KEY.format(provider=provider, date=today) daily_cost_reserved_key = DAILY_COST_RESERVED_MICRO_USD_KEY.format( provider=provider, date=today, ) total_cost_key = TOTAL_COST_KEY.format(provider=provider) - total_cost_reserved_key = TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider=provider) + total_cost_reserved_key = TOTAL_COST_RESERVED_MICRO_USD_KEY.format( + provider=provider + ) current_rpm = await r.get(rpm_key) current_daily = await r.get(daily_req_key) @@ -1471,13 +1925,15 @@ class AIRateLimiter: request_count = int(current_daily) if current_daily else 0 token_count = int(current_tokens) if current_tokens else 0 token_reserved_count = int(reserved_tokens) if reserved_tokens else 0 - daily_cost_usd = (int(current_daily_cost) if current_daily_cost else 0) / _MICRO_USD + daily_cost_usd = ( + int(current_daily_cost) if current_daily_cost else 0 + ) / _MICRO_USD daily_cost_reserved_usd = ( - (int(reserved_daily_cost) if reserved_daily_cost else 0) / _MICRO_USD - ) + int(reserved_daily_cost) if reserved_daily_cost else 0 + ) / _MICRO_USD total_cost_reserved_usd = ( - (int(reserved_total_cost) if reserved_total_cost else 0) / _MICRO_USD - ) + int(reserved_total_cost) if reserved_total_cost else 0 + ) / _MICRO_USD # 2026-03-29 ogt: 加入成本資訊 cost_info = {} @@ -1497,19 +1953,14 @@ class AIRateLimiter: "reserved": round(total_cost_reserved_usd, 6), "limit": cost_limit["total_cost_usd"], "remaining": round( - cost_limit["total_cost_usd"] - - total_cost_exposure, + cost_limit["total_cost_usd"] - total_cost_exposure, 4, ), "exposure_including_pending": round(total_cost_exposure, 6), "alert_threshold": alert_threshold, - "alert_threshold_reached": ( - total_cost_exposure >= alert_threshold - ), + "alert_threshold_reached": (total_cost_exposure >= alert_threshold), }, - "cost_exceeded": ( - total_cost_exposure >= cost_limit["total_cost_usd"] - ), + "cost_exceeded": (total_cost_exposure >= cost_limit["total_cost_usd"]), "cost_alert": { "basis": "actual_plus_pending_reservations", "exposure_usd": round(total_cost_exposure, 6), @@ -1544,7 +1995,9 @@ class AIRateLimiter: }, } - accounting_gap = request_count > 0 and token_count <= 0 and token_reserved_count <= 0 + accounting_gap = ( + request_count > 0 and token_count <= 0 and token_reserved_count <= 0 + ) authentication = await self.get_provider_authentication_status(provider) @@ -1591,7 +2044,9 @@ class AIRateLimiter: """ r = await self._get_redis() total_cost_key = TOTAL_COST_KEY.format(provider=provider) - total_cost_reserved_key = TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider=provider) + total_cost_reserved_key = TOTAL_COST_RESERVED_MICRO_USD_KEY.format( + provider=provider + ) alert_sent_key = COST_ALERT_SENT_KEY.format(provider=provider) await r.delete(total_cost_key, total_cost_reserved_key, alert_sent_key) diff --git a/apps/api/src/services/ai_router.py b/apps/api/src/services/ai_router.py index 91c881120..e838ebbea 100644 --- a/apps/api/src/services/ai_router.py +++ b/apps/api/src/services/ai_router.py @@ -56,6 +56,7 @@ from src.services.ai_provider_policy import ( PAID_PROVIDER_ORDER, PRODUCTION_OLLAMA_ORDER, PRODUCTION_PROVIDER_ORDER, + SANITIZED_CLOUD_PROVIDER_ORDER, normalize_production_execution_order, ) from src.services.ai_providers.interfaces import AIProvider as AIProviderProtocol @@ -433,7 +434,8 @@ class AIRouter: "alert_fast" if intent == IntentType.ALERT_TRIAGE else intent.value if intent else "general" - ) + ), + emit_side_effects=True, ) observed_fallback = getattr( failover_result, @@ -840,6 +842,14 @@ class _SimpleCircuitBreaker: self._failure_count += 1 self._last_failure_time = time.time() + def cooldown_remaining_seconds(self) -> int: + if self._failure_count < self._failure_threshold: + return 0 + return max( + 0, + int(self._recovery_timeout - (time.time() - self._last_failure_time)), + ) + class AIProviderRegistry: """ @@ -949,6 +959,218 @@ class AIRouterExecutor: self._circuit_breakers[name] = _SimpleCircuitBreaker(name) return self._circuit_breakers[name] + @staticmethod + def _asset_execution_preflight(context: dict | None) -> AIResult | None: + """Reject unresolved identities and open asset circuits before any model call.""" + + context = context or {} + route = context.get("typed_target_route") or context.get( + "typedTargetRoute" + ) + identity = context.get("canonical_asset_identity") + route = route if isinstance(route, dict) else {} + identity = identity if isinstance(identity, dict) else {} + resolution_values = { + str(value or "").strip().lower() + for value in ( + route.get("resolution_status"), + identity.get("resolution_status"), + context.get("asset_identity_resolution_status"), + ) + if str(value or "").strip() + } + canonical_asset_id = str( + route.get("canonical_asset_id") + or identity.get("canonical_id") + or context.get("canonical_asset_id") + or "" + ).strip() + identity_unresolved = bool( + resolution_values + & { + "ambiguous", + "asset_identity_ambiguous", + "asset_identity_unresolved", + "unknown", + "unresolved", + } + ) or bool( + "resolved" in resolution_values + and (route or identity) + and not canonical_asset_id + ) + if identity_unresolved: + identity_seed = str( + context.get("target_resource") or "asset-identity-unresolved" + ) + digest = hashlib.sha256(identity_seed.encode("utf-8")).hexdigest() + return AIResult( + raw_response="", + success=False, + provider="none", + error="asset_identity_unresolved", + audit_metadata={ + "schema_version": "ai_asset_identity_fail_closed_v1", + "status": "asset_identity_unresolved", + "provider_network_call_performed": False, + "cross_provider_fallback_allowed": False, + "cross_domain_fallback_allowed": False, + "drift_work_item_id": f"AIA-ASSET-DRIFT-{digest[:12].upper()}", + }, + ) + + circuit_state = str( + route.get("circuit_state") + or identity.get("circuit_state") + or context.get("asset_circuit_state") + or "" + ).strip().lower() + circuit_open = circuit_state == "open" or any( + value is True + for value in ( + route.get("circuit_open"), + identity.get("circuit_open"), + context.get("asset_circuit_open"), + ) + ) + if not circuit_open: + return None + + asset_seed = canonical_asset_id or str( + context.get("target_resource") or "asset-circuit" + ) + digest = hashlib.sha256(asset_seed.encode("utf-8")).hexdigest() + domain = str( + route.get("target_kind") + or identity.get("asset_domain") + or context.get("asset_domain") + or "unknown" + ).strip().lower() + return AIResult( + raw_response="", + success=False, + provider="none", + error="asset_circuit_open_fail_closed", + audit_metadata={ + "schema_version": "ai_asset_circuit_open_terminal_v1", + "status": "circuit_open", + "circuit_scope": "asset", + "asset_identity_sha256": digest, + "canonical_domain": domain, + "provider_network_call_performed": False, + "cross_provider_fallback_allowed": False, + "cross_domain_fallback_allowed": False, + "allowed_actions": [ + "same_asset_domain_bounded_cooldown_probe", + "create_same_asset_domain_repair_work_item", + ], + "repair_work_item": { + "work_item_id": f"AIA-CIRCUIT-ASSET-{digest[:16].upper()}", + "domain": domain, + "execution_allowed": False, + "required_next_receipt": "bounded_cooldown_or_repair_verifier", + }, + }, + ) + + @staticmethod + def _provider_circuit_open_result( + *, + provider_name: str, + circuit_breaker: _SimpleCircuitBreaker, + context: dict | None, + ) -> AIResult: + work_item_seed = str((context or {}).get("work_item_id") or "") + digest = hashlib.sha256( + f"{provider_name}|{work_item_seed}".encode() + ).hexdigest() + return AIResult( + raw_response="", + success=False, + provider="none", + error=f"provider_circuit_open_fail_closed:{provider_name}", + audit_metadata={ + "schema_version": "ai_provider_circuit_open_terminal_v1", + "status": "circuit_open", + "circuit_scope": "provider", + "canonical_provider_lane": provider_name, + "provider_network_call_performed": False, + "cross_provider_fallback_allowed": False, + "cross_domain_fallback_allowed": False, + "cooldown_remaining_seconds": ( + circuit_breaker.cooldown_remaining_seconds() + ), + "allowed_actions": [ + "same_provider_bounded_cooldown_probe", + "create_same_provider_repair_work_item", + ], + "repair_work_item": { + "work_item_id": f"AIA-CIRCUIT-PROVIDER-{digest[:16].upper()}", + "provider": provider_name, + "execution_allowed": False, + "required_next_receipt": "same_provider_cooldown_verifier", + }, + }, + ) + + @staticmethod + async def _attach_gcp_transport_receipt( + *, + provider_name: str, + provider_context: dict | None, + ) -> tuple[dict | None, str | None]: + """Issue and attach one same-run GCP transport receipt fail-closed.""" + + if provider_name not in {"ollama_gcp_a", "ollama_gcp_b"}: + return provider_context, None + if not isinstance(provider_context, dict): + return None, "gcp_cloud_transport_context_missing" + + from src.services.cloud_transport_receipt import ( + bounded_cloud_transport_receipt_hints, + issue_cloud_transport_receipt, + resolve_gcp_transport_endpoint, + verify_cloud_transport_receipt, + ) + + endpoint_url = resolve_gcp_transport_endpoint(provider_name) + existing_reason = await verify_cloud_transport_receipt( + context=provider_context, + provider=provider_name, + endpoint_url=endpoint_url, + ) + if existing_reason is None: + return provider_context, None + try: + receipt = await issue_cloud_transport_receipt( + trace_id=str(provider_context.get("trace_id") or ""), + run_id=str(provider_context.get("run_id") or ""), + work_item_id=str(provider_context.get("work_item_id") or ""), + provider=provider_name, + endpoint_url=endpoint_url, + ) + except Exception as error: + logger.warning( + "ai_router_gcp_transport_preflight_failed", + provider=provider_name, + existing_reason=existing_reason, + error_type=type(error).__name__, + ) + return None, "gcp_cloud_transport_preflight_failed" + + receipts = bounded_cloud_transport_receipt_hints( + provider_context.get("cloud_transport_receipts") + ) + receipts = [ + item + for item in receipts + if str(item.get("provider") or "") != provider_name + ] + receipts.append(receipt) + updated_context = dict(provider_context) + updated_context["cloud_transport_receipts"] = receipts + return updated_context, None + @staticmethod def _cache_key(prompt: str, context: dict | None) -> str: """生成 Cache Key (與 openclaw.py 相容)""" @@ -980,7 +1202,16 @@ class AIRouterExecutor: for key in ("alert_type", "alertname", "alert_name", "incident_id", "signals") ) if not is_alert: - return prompt, context, None + if ( + (context or {}).get("synthetic_validation") is True + and (context or {}).get("data_classification") == "sanitized" + and (context or {}).get("infrastructure_remediation_write_allowed") + is False + and (context or {}).get("synthetic_prompt_sha256") + == hashlib.sha256(prompt.encode("utf-8")).hexdigest() + ): + return prompt, context, None + return None, None, "cloud_sanitization_receipt_missing" cloud_context = (context or {}).get("paid_cloud_context") paid_prompt = (context or {}).get("paid_cloud_prompt") if not isinstance(cloud_context, dict) or not isinstance(paid_prompt, str): @@ -1172,6 +1403,15 @@ class AIRouterExecutor: Returns: AIResult: 標準化結果 """ + asset_preflight = self._asset_execution_preflight(context) + if asset_preflight is not None: + logger.warning( + "ai_router_asset_identity_or_circuit_blocked", + reason=asset_preflight.error, + audit_schema=asset_preflight.audit_metadata.get("schema_version"), + ) + return asset_preflight + # ① Mock Mode 攔截 (D13) if _settings.MOCK_MODE: logger.info("ai_router_mock_mode") @@ -1231,24 +1471,24 @@ class AIRouterExecutor: require_local=require_local, ) if cloud_privacy_block: - blocked_paid_providers = [ + blocked_cloud_providers = [ provider - for provider in PAID_PROVIDER_ORDER + for provider in SANITIZED_CLOUD_PROVIDER_ORDER if provider in provider_order ] provider_order = [ provider for provider in provider_order - if provider not in PAID_PROVIDER_ORDER + if provider not in SANITIZED_CLOUD_PROVIDER_ORDER ] preflight_errors.extend( f"{provider}: {cloud_privacy_block}" - for provider in blocked_paid_providers + for provider in blocked_cloud_providers ) logger.warning( "ai_router_cloud_context_blocked_pre_cache", reason=cloud_privacy_block, - blocked_providers=blocked_paid_providers, + blocked_providers=blocked_cloud_providers, ) if any(provider in provider_order for provider in PAID_PROVIDER_ORDER): for paid_provider in PAID_PROVIDER_ORDER: @@ -1269,7 +1509,10 @@ class AIRouterExecutor: ) continue try: - if await is_provider_disabled(paid_provider): + if await is_provider_disabled( + paid_provider, + run_id=str((paid_context or context or {}).get("run_id") or ""), + ): provider_order = [ provider for provider in provider_order @@ -1297,6 +1540,24 @@ class AIRouterExecutor: error_type=type(disable_check_error).__name__, ) + # A circuit-open first hop is a terminal route condition, not an + # unavailability receipt authorizing another host or provider. Check it + # before cache lookup so stale generations cannot bypass the circuit. + if provider_order: + first_provider = provider_order[0] + first_circuit = self._get_circuit_breaker(first_provider) + if first_circuit.is_open(): + logger.warning( + "ai_router_provider_circuit_open_fail_closed", + provider=first_provider, + cross_provider_fallback_allowed=False, + ) + return self._provider_circuit_open_result( + provider_name=first_provider, + circuit_breaker=first_circuit, + context=context, + ) + # ② Cache 檢查 (D4) cache_key = self._cache_key(prompt, context) # C3 修復: 移到 try 外避免 UnboundLocalError try: @@ -1393,7 +1654,6 @@ class AIRouterExecutor: bounded_unavailable_providers = ( await self._bounded_ollama_unavailable_receipts(context) ) - circuit_open_ollama_providers: set[str] = set() provider_timeout_seconds: float | None = None try: configured_provider_timeout = float( @@ -1417,6 +1677,27 @@ class AIRouterExecutor: _last_attempted_provider: str | None = None for provider_name in provider_order: + # Opening a provider circuit terminates this canonical lane. It is + # checked before fallback metrics and registry/provider work so the + # skipped hop cannot be reported as an attempted fallback. + cb = self._get_circuit_breaker(provider_name) + if cb.is_open(): + logger.warning( + "ai_router_provider_circuit_open_fail_closed", + provider=provider_name, + cross_provider_fallback_allowed=False, + ) + if _lf_trace_ctx: + try: + _lf_trace_ctx.__exit__(None, None, None) + except Exception: + pass + return self._provider_circuit_open_result( + provider_name=provider_name, + circuit_breaker=cb, + context=context, + ) + # 2026-04-27 Claude Sonnet 4.6: A2 — 若上一輪失敗且本輪開始，表示發生 fallback # 記錄 metric（DIAGNOSE intent 專屬；非 DIAGNOSE 不記，不影響其他路徑） # 2026-04-27 Claude Sonnet 4.6: F6 — fallback metric 只在真實 analyze() 失敗時觸發 @@ -1454,6 +1735,17 @@ class AIRouterExecutor: # legacy logical ``ollama`` alias, while execution receipts # continue to carry the provider's concrete runtime identity. provider = self._registry.get("ollama") + if provider is None and provider_name == "ollama_local" and require_local: + # Older isolated/local registries expose the logical Ollama + # provider only. It may satisfy the concrete host111 lane only + # when the provider itself declares a local privacy boundary; + # production keeps using its explicit ollama_local identity. + legacy_local = self._registry.get("ollama") + if ( + legacy_local is not None + and getattr(legacy_local, "privacy_level", None) == "local" + ): + provider = legacy_local if not provider: # 2026-04-14 Claude Sonnet 4.6: silent skip 改 errors 累積（觀測性） # 2026-04-27 Claude Sonnet 4.6: F6 — 不設 _last_attempted_provider（未真實執行 analyze） @@ -1467,17 +1759,6 @@ class AIRouterExecutor: if hop not in attempted_providers and hop not in bounded_unavailable_providers ] - if circuit_open_ollama_providers: - errors.append( - f"{provider_name}: cloud_blocked_ollama_circuit_open(" - f"{','.join(sorted(circuit_open_ollama_providers))})" - ) - logger.warning( - "ai_router_cloud_blocked_by_ollama_circuit_open", - provider=provider_name, - circuit_open_providers=sorted(circuit_open_ollama_providers), - ) - continue if unresolved_ollama_hops: errors.append( f"{provider_name}: cloud_blocked_ollama_attempt_receipts_missing(" @@ -1500,7 +1781,14 @@ class AIRouterExecutor: errors.append(f"{provider_name}: privacy_skip(non_local)") continue - if alert_requires_ollama_before_cloud and provider.privacy_level == "cloud": + # GCP-A/B are cloud-hosted Ollama and therefore non-local for data + # classification, but they remain the first two free Ollama hops. + # The "Ollama before cloud critic" gate applies only to paid + # external critics, otherwise it would incorrectly block GCP-A/B. + if ( + alert_requires_ollama_before_cloud + and provider_name in PAID_PROVIDER_ORDER + ): if "ollama_local" not in attempted_providers: errors.append(f"{provider_name}: blocked_until_ollama_local_attempted") logger.warning( @@ -1511,23 +1799,6 @@ class AIRouterExecutor: ) continue - # 閘門 1: Circuit Breaker (per-provider, C2 修復) - cb = self._get_circuit_breaker(provider_name) - if cb.is_open(): - if provider_name in PRODUCTION_OLLAMA_ORDER: - circuit_open_ollama_providers.add(provider_name) - if alert_requires_ollama_before_cloud and provider_name.startswith("ollama"): - logger.warning( - "ai_router_alert_ollama_circuit_bypassed", - provider=provider_name, - reason="alert_requires_ollama_before_cloud", - ) - else: - errors.append(f"{provider_name}: circuit_open") - logger.warning("ai_router_circuit_open", provider=provider_name) - # 2026-04-27 Claude Sonnet 4.6: F6 — circuit_open 不設 _last_attempted_provider（未嘗試） - continue - # 閘門 2: Rate Limiter # 2026-04-02 Claude Code: Phase 24 B3 + C1 修復 — Rate Limiter (含 openclaw_nemo) # Paid providers own atomic requests/tokens/cost reservations inside @@ -1549,24 +1820,41 @@ class AIRouterExecutor: sem = self._get_semaphore(provider_name) async with sem: try: - attempted_providers.add(provider_name) provider_prompt = ( paid_prompt - if provider_name in PAID_PROVIDER_ORDER + if provider_name in SANITIZED_CLOUD_PROVIDER_ORDER else prompt ) provider_context = ( paid_context - if provider_name in PAID_PROVIDER_ORDER + if provider_name in SANITIZED_CLOUD_PROVIDER_ORDER else context ) - if provider_name in PAID_PROVIDER_ORDER and ( + if provider_name in SANITIZED_CLOUD_PROVIDER_ORDER and ( provider_prompt is None or provider_context is None ): errors.append( - f"{provider_name}: paid_cloud_execution_inputs_missing" + f"{provider_name}: sanitized_cloud_execution_inputs_missing" ) continue + if ( + provider_name in {"ollama_gcp_a", "ollama_gcp_b"} + and (context or {}).get("cloud_transport_auto_preflight") + is True + ): + provider_context, transport_block = ( + await self._attach_gcp_transport_receipt( + provider_name=provider_name, + provider_context=provider_context, + ) + ) + if transport_block: + errors.append( + f"{provider_name}: {transport_block}" + ) + continue + paid_context = provider_context + attempted_providers.add(provider_name) provider_call = provider.analyze( provider_prompt, provider_context, @@ -1598,7 +1886,7 @@ class AIRouterExecutor: # Paid responses are deliberately non-cacheable: every # paid candidate must re-run current durable policy, # privacy, canary, cost, and ordered-attempt gates. - if provider_name not in PAID_PROVIDER_ORDER: + if provider_name not in SANITIZED_CLOUD_PROVIDER_ORDER: try: redis = get_redis() cache_data = _json.dumps({ @@ -1620,7 +1908,9 @@ class AIRouterExecutor: # D5: 記錄 Langfuse generation if _lf_trace_ctx: try: - paid_provider = provider_name in PAID_PROVIDER_ORDER + cloud_provider = ( + provider_name in SANITIZED_CLOUD_PROVIDER_ORDER + ) prompt_hash = hashlib.sha256( provider_prompt.encode("utf-8") ).hexdigest() @@ -1636,17 +1926,17 @@ class AIRouterExecutor: _lf_trace_ctx.generation( name=f"{provider_name}_call", model=provider_name, - input=None if paid_provider else provider_prompt[:500], + input=None if cloud_provider else provider_prompt[:500], output=( None - if paid_provider + if cloud_provider else result.raw_response[:500] ), usage={"total": result.tokens} if result.tokens else None, metadata={ "cost_usd": result.cost_usd, "latency_ms": round(result.latency_ms, 1), - "content_redacted": paid_provider, + "content_redacted": cloud_provider, "prompt_length": len(provider_prompt), "prompt_sha256": prompt_hash, "response_length": len(result.raw_response), @@ -1808,7 +2098,7 @@ def _init_registry() -> AIProviderRegistry: registry.register(NemotronProvider()) # 2026-05-06 Codex: 188 不再作為 Ollama provider。 - # Local fallback 統一命名為 ollama_local，端點由 OLLAMA_FALLBACK_URL 指向 111/110 proxy。 + # Local fallback 統一命名為 ollama_local，端點由 OLLAMA_FALLBACK_URL 直連 host111。 ollama_local = OllamaLocalProvider() registry.register(ollama_local) @@ -1818,7 +2108,7 @@ def _init_registry() -> AIProviderRegistry: # 修復： # "ollama_gcp_a" alias → 同 OllamaProvider（OLLAMA_URL = GCP-A） # "ollama_gcp_b" → 新 OllamaGcpBProvider（OLLAMA_SECONDARY_URL = GCP-B） - # "ollama_local" → OllamaLocalProvider（OLLAMA_FALLBACK_URL = 111 / 110:11437） + # "ollama_local" → OllamaLocalProvider（OLLAMA_FALLBACK_URL = host111 direct） registry._providers["ollama_gcp_a"] = ollama_gcp_a registry.register(OllamaGcpBProvider()) registry._providers["ollama_local"] = ollama_local diff --git a/apps/api/src/services/alert_chain_emergency_ingress.py b/apps/api/src/services/alert_chain_emergency_ingress.py new file mode 100644 index 000000000..0baf0f53d --- /dev/null +++ b/apps/api/src/services/alert_chain_emergency_ingress.py @@ -0,0 +1,517 @@ +"""Authenticated reduced-envelope ingress for Alertmanager delivery failures. + +The primary Alertmanager webhook cannot be the only producer of its own repair +candidate: when that delivery path is broken, calling it again is a catch-22. +Agent99 therefore polls the exact host-110 Alertmanager API without credentials, +reduces only the exact firing alert, then sends this authenticated no-secret +envelope to AWOOOI. This service binds one deterministic approval/Incident/run +identity and queues the exact host-110 Alertmanager Ansible candidate. Agent99 +is transport and coordination only; it never executes the Linux repair. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime +from typing import Any, Literal +from uuid import NAMESPACE_URL, UUID, uuid5 + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from src.core.logging import get_logger +from src.models.approval import ( + ApprovalRequestCreate, + BlastRadius, + DataImpact, + DryRunCheck, + RiskLevel, +) +from src.services.approval_db import get_approval_service +from src.services.channel_hub import record_alertmanager_event +from src.services.incident_service import ( + create_incident_for_approval, + get_incident_service, +) + +logger = get_logger("awoooi.alert_chain_emergency_ingress") + +ALERT_CHAIN_EMERGENCY_SCHEMA_VERSION = ( + "agent99_alertmanager_emergency_relay_v1" +) +ALERT_CHAIN_EMERGENCY_OWNER_SCHEMA_VERSION = ( + "alert_chain_emergency_ingress_owner_v1" +) +ALERT_CHAIN_ALERTNAME = "AlertChainBroken_Alertmanager" +ALERT_CHAIN_CATALOG_ID = "ansible:110-alertmanager-delivery-recovery" +ALERT_CHAIN_CANONICAL_ASSET_ID = "container:host_110:alertmanager" +ALERT_CHAIN_EXECUTOR = "host_ansible_executor" +ALERT_CHAIN_AGENT99_ROLE = "relay_coordination_only" +ALERT_CHAIN_APPROVAL_FINGERPRINT_DOMAIN = ( + "awoooi.alert_chain_emergency.approval.v1" +) + + +class AlertChainEmergencyRelayRequest(BaseModel): + """Bounded relay envelope; arbitrary labels, annotations and logs are forbidden.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["agent99_alertmanager_emergency_relay_v1"] + relay_receipt_id: str = Field( + pattern=r"^agent99-alertchain-[0-9a-f]{64}$", + max_length=83, + ) + source_host: Literal["192.168.0.110"] + source_url: Literal["http://192.168.0.110:9093/api/v2/alerts"] + source_fingerprint: str = Field( + pattern=r"^[0-9a-f]{8,64}$", + max_length=64, + ) + source_started_at: datetime + alertname: Literal["AlertChainBroken_Alertmanager"] + status: Literal["firing"] + severity: Literal["critical"] + target_resource: Literal["alertmanager"] + namespace: Literal["monitoring"] + agent99_role: Literal["relay_coordination_only"] + executor: Literal["host_ansible_executor"] + catalog_id: Literal["ansible:110-alertmanager-delivery-recovery"] + runtime_write_performed: Literal[False] + raw_payload_stored: Literal[False] + + @field_validator("source_started_at") + @classmethod + def require_timezone(cls, value: datetime) -> datetime: + if value.tzinfo is None: + raise ValueError("source_started_at_timezone_required") + return value.astimezone(UTC) + + +def _canonical_json(value: Any) -> str: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def build_alert_chain_emergency_owner( + payload: AlertChainEmergencyRelayRequest, + *, + project_id: str = "awoooi", +) -> dict[str, Any]: + """Build deterministic IDs for one exact Alertmanager firing episode.""" + + project = str(project_id or "awoooi").strip() or "awoooi" + occurrence = { + "source_host": payload.source_host, + "source_url": payload.source_url, + "source_fingerprint": payload.source_fingerprint, + "source_started_at": payload.source_started_at.isoformat(), + "relay_receipt_id": payload.relay_receipt_id, + } + owner_identity = { + "schema_version": ALERT_CHAIN_EMERGENCY_OWNER_SCHEMA_VERSION, + "project_id": project, + "alertname": ALERT_CHAIN_ALERTNAME, + "canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID, + "source_occurrence": occurrence, + } + owner_key = _canonical_json(owner_identity) + digest = hashlib.sha256(owner_key.encode("utf-8")).hexdigest() + run_id = str(uuid5(NAMESPACE_URL, f"{owner_key}:run")) + return { + **owner_identity, + "owner_identity": owner_identity, + "owner_key_sha256": digest, + "approval_id": str(uuid5(NAMESPACE_URL, f"{owner_key}:approval")), + "incident_id": f"INC-ACR-{digest[:16].upper()}", + "run_id": run_id, + "trace_id": run_id, + "work_item_id": f"alertchain:{project}:{digest[:32]}", + "source_receipt_ref": payload.relay_receipt_id, + "source_fingerprint": payload.source_fingerprint, + "source_started_at": payload.source_started_at.isoformat(), + "agent99_role": ALERT_CHAIN_AGENT99_ROLE, + "executor": ALERT_CHAIN_EXECUTOR, + "catalog_id": ALERT_CHAIN_CATALOG_ID, + "canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID, + "runtime_write_performed": False, + "runtime_closure_verified": False, + } + + +def _approval_storage_fingerprint(owner: dict[str, Any]) -> str: + payload = ( + f"{ALERT_CHAIN_APPROVAL_FINGERPRINT_DOMAIN}\x1f" + f"{_canonical_json(owner['owner_identity'])}" + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _approval_matches_owner(approval: Any, owner: dict[str, Any]) -> bool: + metadata = getattr(approval, "metadata", None) + return bool( + str(getattr(approval, "id", "") or "") == owner["approval_id"] + and isinstance(metadata, dict) + and metadata.get("schema_version") + == ALERT_CHAIN_EMERGENCY_OWNER_SCHEMA_VERSION + and metadata.get("owner_identity") == owner["owner_identity"] + and metadata.get("owner_key_sha256") == owner["owner_key_sha256"] + and metadata.get("run_id") == owner["run_id"] + and metadata.get("trace_id") == owner["trace_id"] + and metadata.get("work_item_id") == owner["work_item_id"] + and metadata.get("agent99_role") == ALERT_CHAIN_AGENT99_ROLE + and metadata.get("executor") == ALERT_CHAIN_EXECUTOR + and metadata.get("catalog_id") == ALERT_CHAIN_CATALOG_ID + and metadata.get("runtime_write_performed") is False + ) + + +def _approval_request(owner: dict[str, Any]) -> ApprovalRequestCreate: + return ApprovalRequestCreate( + action=( + "ANSIBLE_CONTROLLED_CANDIDATE " + "ansible:110-alertmanager-delivery-recovery" + ), + description=( + "Authenticated Agent99 poll relay created the exact " + "Alertmanager delivery recovery candidate. Agent99 remains relay-only; " + "the Linux executor is the bounded host Ansible broker." + ), + risk_level=RiskLevel.MEDIUM, + blast_radius=BlastRadius( + affected_pods=0, + estimated_downtime="0-30s", + related_services=["alertmanager"], + data_impact=DataImpact.WRITE, + ), + dry_run_checks=[ + DryRunCheck( + name="exact-host read-only poll ingress", + passed=True, + message="unauthenticated first hop; authenticated reduced outbound", + ), + DryRunCheck( + name="typed executor boundary", + passed=True, + message=( + "Agent99 relay-only; host_ansible_executor owns Linux apply" + ), + ), + DryRunCheck( + name="no-secret envelope", + passed=True, + message="raw labels, annotations, logs and credentials rejected", + ), + ], + requested_by="Agent99 Alertmanager emergency relay", + incident_id=owner["incident_id"], + metadata={ + **owner, + "preallocated_approval_id": owner["approval_id"], + "owner_policy": "global_product_governance_v2", + "candidate_only": True, + "controlled_apply_allowed": True, + }, + ) + + +def _incident_payload_for_candidate( + incident: Any, + *, + owner: dict[str, Any], +) -> dict[str, Any]: + if hasattr(incident, "model_dump"): + payload = incident.model_dump(mode="json") + elif isinstance(incident, dict): + payload = dict(incident) + else: + payload = {} + payload.update( + { + "incident_id": owner["incident_id"], + "project_id": owner["project_id"], + "alertname": ALERT_CHAIN_ALERTNAME, + "alert_category": "alert_chain_health", + "notification_type": "TYPE-3", + "trace_id": owner["trace_id"], + "run_id": owner["run_id"], + "work_item_id": owner["work_item_id"], + "source_receipt_ref": owner["source_receipt_ref"], + "asset_scope_aliases": [ + ALERT_CHAIN_CANONICAL_ASSET_ID, + "host_110", + "alertmanager", + ], + } + ) + return payload + + +async def process_alert_chain_emergency_relay( + payload: AlertChainEmergencyRelayRequest, + *, + project_id: str = "awoooi", + approval_service: Any | None = None, + incident_service: Any | None = None, + incident_creator: Callable[..., Awaitable[str]] = ( + create_incident_for_approval + ), + candidate_enqueuer: Callable[..., Awaitable[dict[str, Any]]] | None = None, + event_recorder: Callable[..., Awaitable[Any]] = record_alertmanager_event, +) -> dict[str, Any]: + """Persist the deterministic owner and queue the exact Ansible candidate.""" + + if candidate_enqueuer is None: + from src.jobs.awooop_ansible_candidate_backfill_job import ( + enqueue_ai_decision_ansible_candidate, + ) + + candidate_enqueuer = enqueue_ai_decision_ansible_candidate + + owner = build_alert_chain_emergency_owner(payload, project_id=project_id) + approvals = approval_service or get_approval_service() + incidents = incident_service or get_incident_service() + approval_uuid = UUID(owner["approval_id"]) + + approval = await approvals.get_approval(approval_uuid) + created = False + if approval is not None and not _approval_matches_owner(approval, owner): + return { + "schema_version": "alert_chain_emergency_ingress_result_v1", + "status": "durable_owner_identity_mismatch_no_candidate", + "candidate_persisted": False, + "candidate_queued": False, + "runtime_write_performed": False, + "runtime_closure_verified": False, + } + if approval is None: + try: + approval = await approvals.create_approval_with_fingerprint( + request=_approval_request(owner), + fingerprint=_approval_storage_fingerprint(owner), + ) + created = True + except Exception: + approval = await approvals.get_approval(approval_uuid) + if approval is None: + raise + else: + incremented = await approvals.increment_hit_count(approval_uuid) + if incremented is not None: + approval = incremented + + if not _approval_matches_owner(approval, owner): + return { + "schema_version": "alert_chain_emergency_ingress_result_v1", + "status": "durable_owner_readback_mismatch_no_candidate", + "candidate_persisted": False, + "candidate_queued": False, + "runtime_write_performed": False, + "runtime_closure_verified": False, + } + + canonical_incident = await incidents.get_for_readback( + owner["incident_id"], + project_id=project_id, + ) + if canonical_incident is None: + await incident_creator( + approval_id=owner["approval_id"], + risk_level="medium", + target_resource="alertmanager", + namespace="monitoring", + alert_type="custom", + message=( + "Alertmanager main webhook delivery failure received by the " + "authenticated Agent99 secondary machine ingress" + ), + source="alertmanager", + alertname=ALERT_CHAIN_ALERTNAME, + alert_labels={ + "alertname": ALERT_CHAIN_ALERTNAME, + "severity": "critical", + "component": "alertmanager", + "host": "110", + "alert_category": "alert_chain_health", + "fingerprint": payload.source_fingerprint, + "relay_receipt_id": payload.relay_receipt_id, + "canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID, + }, + notification_type="TYPE-3", + alert_category="alert_chain_health", + canonical_incident_id=owner["incident_id"], + ) + await approvals.update_incident_id( + approval_uuid, + owner["incident_id"], + ) + canonical_incident = await incidents.get_for_readback( + owner["incident_id"], + project_id=project_id, + ) + + durable_incident = bool( + canonical_incident is not None + and getattr(canonical_incident, "persisted_to_pg", False) is True + ) + bound_approval = await approvals.get_approval(approval_uuid) + durable_binding = bool( + bound_approval is not None + and _approval_matches_owner(bound_approval, owner) + and str(getattr(bound_approval, "incident_id", "") or "") + == owner["incident_id"] + ) + if not durable_incident or not durable_binding: + return { + "schema_version": "alert_chain_emergency_ingress_result_v1", + "status": "durable_incident_binding_missing_no_candidate", + "candidate_persisted": False, + "candidate_queued": False, + "identity": { + key: owner[key] + for key in ("run_id", "trace_id", "work_item_id") + }, + "runtime_write_performed": False, + "runtime_closure_verified": False, + } + + candidate_incident = _incident_payload_for_candidate( + canonical_incident, + owner=owner, + ) + enqueue_receipt = await candidate_enqueuer( + incident=candidate_incident, + proposal_data={ + "source": "alert_webhook_controlled_router", + "risk_level": "medium", + "action": "enqueue_allowlisted_ansible_check_mode", + "alert_type": "custom", + "alertname": ALERT_CHAIN_ALERTNAME, + "namespace": "monitoring", + "approval_id": owner["approval_id"], + "execution_priority": 20, + "source_fingerprint": payload.source_fingerprint, + "source_occurrence_id": payload.relay_receipt_id, + "source_received_at": payload.source_started_at.isoformat(), + "source_recurrence_verified": True, + "source_recurrence_source": "agent99_secondary_machine_ingress", + "source_recurrence_anchor": payload.relay_receipt_id, + "source_recurrence_durable_readback_verified": True, + "source_canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID, + }, + project_id=project_id, + automation_run_id=owner["run_id"], + ) + queued = bool( + enqueue_receipt.get("queued") is True + and str(enqueue_receipt.get("automation_run_id") or "") + == owner["run_id"] + and str(enqueue_receipt.get("trace_id") or "") + == owner["trace_id"] + and str(enqueue_receipt.get("work_item_id") or "") + == owner["work_item_id"] + ) + + try: + await event_recorder( + project_id=project_id, + alert_id=payload.relay_receipt_id, + alertname=ALERT_CHAIN_ALERTNAME, + severity="critical", + namespace="monitoring", + target_resource="alertmanager", + fingerprint=payload.source_fingerprint, + stage=( + "secondary_ingress_ansible_candidate_queued" + if queued + else "secondary_ingress_candidate_queue_blocked" + ), + notification_type="TYPE-3", + alert_category="alert_chain_health", + incident_id=owner["incident_id"], + approval_id=owner["approval_id"], + repeat_count=int(getattr(bound_approval, "hit_count", 1) or 1), + labels={ + "alertname": ALERT_CHAIN_ALERTNAME, + "component": "alertmanager", + "canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID, + "fingerprint": payload.source_fingerprint, + }, + annotations={ + "relay_role": ALERT_CHAIN_AGENT99_ROLE, + "executor": ALERT_CHAIN_EXECUTOR, + }, + ) + except Exception as exc: + logger.warning( + "alert_chain_secondary_ingress_event_record_failed", + error_type=type(exc).__name__, + incident_id=owner["incident_id"], + ) + + return { + "schema_version": "alert_chain_emergency_ingress_result_v1", + "status": ( + "ansible_candidate_queued_verifier_pending" + if queued + else str( + enqueue_receipt.get("status") + or "ansible_candidate_queue_not_acknowledged" + ) + ), + "candidate_owner_created": created, + "candidate_persisted": True, + "candidate_queued": queued, + "identity": { + key: owner[key] + for key in ( + "approval_id", + "incident_id", + "run_id", + "trace_id", + "work_item_id", + "source_receipt_ref", + ) + }, + "agent99_role": ALERT_CHAIN_AGENT99_ROLE, + "executor": ALERT_CHAIN_EXECUTOR, + "catalog_id": ALERT_CHAIN_CATALOG_ID, + "canonical_asset_id": ALERT_CHAIN_CANONICAL_ASSET_ID, + "enqueue_receipt": { + "status": enqueue_receipt.get("status"), + "automation_run_id": enqueue_receipt.get("automation_run_id"), + "queued": enqueue_receipt.get("queued") is True, + "side_effect_performed": bool( + enqueue_receipt.get("side_effect_performed") + ), + "active_blockers": [ + str(value)[:160] + for value in enqueue_receipt.get("active_blockers") or [] + ][:8], + }, + "runtime_write_performed": False, + "runtime_closure_verified": False, + "active_blockers": ( + [ + "independent_post_verifier_pending", + "incident_closure_and_learning_writeback_pending", + ] + if queued + else [ + str(value)[:160] + for value in enqueue_receipt.get("active_blockers") or [ + "ansible_candidate_queue_not_acknowledged" + ] + ][:8] + ), + "safe_next_action": ( + "ansible_worker_check_apply_verify_writeback_same_run" + if queued + else "retry_same_relay_receipt_after_candidate_queue_blocker" + ), + } diff --git a/apps/api/src/services/alert_rule_engine.py b/apps/api/src/services/alert_rule_engine.py index 08c5b0f3e..7ddb21c96 100644 --- a/apps/api/src/services/alert_rule_engine.py +++ b/apps/api/src/services/alert_rule_engine.py @@ -380,8 +380,22 @@ def match_rule(alert_context: dict) -> dict[str, Any] | None: resp = matched_rule["response"] risk = resp.get("risk", "medium") - # severity=critical 強制升級風險等級 - if severity == "critical" and risk == "medium": + # Incident severity describes operational impact, not necessarily the + # bounded action's execution risk. Exact typed rules may explicitly lock + # execution risk after defining a single executor, rollback and verifier. + # This exception is a code-owned policy decision, not a capability that an + # AI-generated or operator-edited YAML rule may grant itself. Keep the + # allow-list exact so critical incidents cannot be silently downgraded into + # an arbitrary automatic action. + execution_risk_independent = bool( + matched_rule.get("id") == "alert_chain_broken_alertmanager" + and alertname == "AlertChainBroken_Alertmanager" + and resp.get("execution_risk_independent_of_incident_severity") is True + and risk == "medium" + and resp.get("suggested_action") == "INVESTIGATE" + and not resp.get("kubectl_command") + ) + if severity == "critical" and risk == "medium" and not execution_risk_independent: risk = "critical" optimization = [ diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 16f49c11d..64a62556e 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -559,6 +559,54 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "risk_level": "high", "catalog_revision": "2026-07-15-wazuh-alert-ingress-v2", }, + { + "catalog_id": "ansible:110-alertmanager-delivery-recovery", + "playbook_path": ( + "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml" + ), + "check_mode_playbook_path": ( + "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml" + ), + "inventory_hosts": ["host_110"], + "domains": ["alert_chain_health", "alertmanager", "docker_container"], + "keywords": [ + "alertchainbrokenalertmanager", + "alertchainbroken_alertmanager", + "alertmanager delivery", + ], + "activation_keywords": ["alertchainbrokenalertmanager"], + "supports_check_mode": True, + "auto_apply_enabled": True, + "approval_required": False, + "risk_level": "medium", + "canonical_asset_id": "container:host_110:alertmanager", + "catalog_revision": "2026-07-16-alertmanager-exact-container-v1", + }, + { + "catalog_id": "ansible:111-ollama-fallback", + "playbook_path": "infra/ansible/playbooks/111-ollama-fallback.yml", + "check_mode_playbook_path": ( + "infra/ansible/playbooks/111-ollama-fallback.yml" + ), + "inventory_hosts": ["host_111", "host_120", "host_121"], + "domains": ["ai_provider", "ollama", "host_launchagent"], + "keywords": [ + "ollamalocalunavailable", + "ollama111unavailable", + "ollama-local", + "ollama 111", + ], + "activation_keywords": [ + "ollamalocalunavailable", + "ollama111unavailable", + ], + "supports_check_mode": True, + "auto_apply_enabled": True, + "approval_required": False, + "risk_level": "medium", + "canonical_asset_id": "ai-provider:ollama_local", + "catalog_revision": "2026-07-16-host111-launchagent-k3s-path-v1", + }, { "catalog_id": "ansible:110-host-pressure-readonly", "playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml", @@ -812,9 +860,9 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "catalog_id": "ansible:nginx-sync", "playbook_path": "infra/ansible/playbooks/nginx-sync.yml", "check_mode_playbook_path": "infra/ansible/playbooks/nginx-sync-readonly.yml", - "inventory_hosts": ["host_110", "host_188"], - "domains": ["nginx", "proxy", "ollama_proxy", "tls"], - "keywords": ["nginx", "proxy", "ollama", "gcp", "tls", "cert", "502", "upstream"], + "inventory_hosts": ["host_188"], + "domains": ["nginx", "proxy", "tls"], + "keywords": ["nginx", "proxy", "tls", "cert", "502", "upstream"], "supports_check_mode": True, "auto_apply_enabled": True, "approval_required": False, diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index db212a768..547dd6dbb 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -29,6 +29,7 @@ class AssetPostcondition: condition_type: str inventory_host: str probe: str + timeout_seconds: int | None = None @dataclass(frozen=True) @@ -139,6 +140,101 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { "test -s /home/wooo/node_exporter_textfiles/docker_stats.prom", ), ), + "ansible:110-alertmanager-delivery-recovery": ( + AssetPostcondition( + "host_110_alertmanager_container_running", + "service", + "host_110", + "test \"$(docker inspect --format '{{.State.Running}}' alertmanager " + "2>/dev/null)\" = true", + ), + AssetPostcondition( + "host_110_alertmanager_config_valid", + "configuration", + "host_110", + "docker exec alertmanager amtool check-config " + "/etc/alertmanager/alertmanager.yml >/dev/null", + ), + AssetPostcondition( + "host_110_awoooi_webhook_network_boundary", + "network", + "host_110", + "code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 -I " + "https://awoooi.wooo.work/api/v1/webhooks/alertmanager); " + "test \"$code\" = 405 -o \"$code\" = 200", + ), + AssetPostcondition( + "host_110_alertmanager_runtime_target_up", + "metric", + "host_110", + "curl -fsS --get --data-urlencode " + "'query=up{job=\"alertmanager-runtime\"}' " + "http://127.0.0.1:9090/api/v1/query | python3 -c '" + "import json,sys; p=json.load(sys.stdin); r=p.get(\"data\",{}).get(\"result\",[]); " + "raise SystemExit(0 if p.get(\"status\")==\"success\" and len(r)==1 " + "and r[0].get(\"value\",[None,None])[1]==\"1\" else 1)'", + ), + AssetPostcondition( + "host_110_alertmanager_delivery_rule_healthy", + "metric", + "host_110", + "curl -fsS 'http://127.0.0.1:9090/api/v1/rules?type=alert' " + "| python3 -c 'import json,sys; p=json.load(sys.stdin); " + "rules=[r for g in p.get(\"data\",{}).get(\"groups\",[]) " + "for r in g.get(\"rules\",[]) " + "if r.get(\"name\")==\"AlertChainBroken_Alertmanager\"]; " + "raise SystemExit(0 if p.get(\"status\")==\"success\" and len(rules)==1 " + "and rules[0].get(\"health\")==\"ok\" else 1)'", + ), + AssetPostcondition( + "host_110_alertmanager_delivery_alert_resolved_after_window", + "metric", + "host_110", + "for attempt in $(seq 1 72); do " + "curl -fsS --get --data-urlencode " + "'query=ALERTS{alertname=\"AlertChainBroken_Alertmanager\"," + "alertstate=\"firing\"}' http://127.0.0.1:9090/api/v1/query " + "| python3 -c 'import json,sys; p=json.load(sys.stdin); " + "raise SystemExit(0 if p.get(\"status\")==\"success\" " + "and p.get(\"data\",{}).get(\"result\")==[] else 1)' " + "&& exit 0; sleep 10; done; exit 1", + 740, + ), + ), + "ansible:111-ollama-fallback": ( + AssetPostcondition( + "host_111_ollama_launchagent_allowlist_exact", + "configuration", + "host_111", + "test \"$(/usr/libexec/PlistBuddy -c " + "'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' " + "/Users/ooo/Library/LaunchAgents/" + "com.momo.ollama111-allow-proxy.plist)\" = " + "'127.0.0.1/32,192.168.0.111/32,192.168.0.188/32," + "192.168.0.120/32,192.168.0.121/32'", + ), + AssetPostcondition( + "host_111_ollama_local_tags_api", + "service", + "host_111", + "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " + "http://127.0.0.1:11434/api/tags)\" = 200", + ), + AssetPostcondition( + "host_120_to_host111_ollama_tags_api", + "network", + "host_120", + "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " + "http://192.168.0.111:11434/api/tags)\" = 200", + ), + AssetPostcondition( + "host_121_to_host111_ollama_tags_api", + "network", + "host_121", + "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " + "http://192.168.0.111:11434/api/tags)\" = 200", + ), + ), "ansible:110-disk-pressure": ( AssetPostcondition( "host_110_root_disk_below_alert_threshold", @@ -317,24 +413,6 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { "host_188", "test -s /etc/nginx/sites-enabled/all-sites.conf && test -s /etc/nginx/sites-enabled/188-internal-tools-https.conf", ), - AssetPostcondition( - "host_110_nginx_runtime", - "service", - "host_110", - "systemctl is-active --quiet nginx", - ), - AssetPostcondition( - "host_110_ollama_proxy_route", - "route", - "host_110", - "test -s /etc/nginx/sites-enabled/110-ollama-proxy.conf", - ), - AssetPostcondition( - "host_110_ollama_proxy_ports", - "metric", - "host_110", - "ss -ltn | grep -Eq ':11435[[:space:]]' && ss -ltn | grep -Eq ':11436[[:space:]]' && ss -ltn | grep -Eq ':11437[[:space:]]'", - ), ), "ansible:restore-password-auth": tuple( condition @@ -547,7 +625,10 @@ async def run_ansible_asset_post_verifier( ) result = await probe_runner( command, - timeout_seconds=max(1, timeout_seconds), + timeout_seconds=max( + 1, + condition.timeout_seconds or timeout_seconds, + ), ) except Exception as exc: receipts.append({ diff --git a/apps/api/src/services/cloud_transport_receipt.py b/apps/api/src/services/cloud_transport_receipt.py new file mode 100644 index 000000000..6b35de2b1 --- /dev/null +++ b/apps/api/src/services/cloud_transport_receipt.py @@ -0,0 +1,319 @@ +"""Durable, correlation-bound receipts for degraded GCP Ollama transport. + +GCP-A/B currently use a public clear-text HTTP stopgap. A sanitizer receipt +only proves which content was selected; it cannot prove that the exact network +endpoint was independently checked. This module keeps those authorities +separate: + +* ``issue_cloud_transport_receipt`` performs a non-secret ``/api/tags`` + network preflight, writes a short-lived receipt to Redis, then reads it back. +* ``verify_cloud_transport_receipt`` treats caller JSON only as a lookup hint + and independently reads the durable receipt before a generation call. + +No response body, endpoint URL, credential, model name, or raw prompt is +persisted in the receipt. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Awaitable, Callable, Mapping +from datetime import UTC, datetime, timedelta +from typing import Any + +import httpx + +from src.core.redis_client import get_redis + +CLOUD_TRANSPORT_RECEIPT_SCHEMA = "cloud_transport_preflight_receipt_v1" +CLOUD_TRANSPORT_RECEIPT_PREFIX = "ai:cloud_transport_receipt:" +CLOUD_TRANSPORT_MAX_TTL_SECONDS = 120 +CLOUD_TRANSPORT_DEFAULT_TTL_SECONDS = 90 +CLOUD_TRANSPORT_SOURCE = "cloud_transport_preflight.network_probe" +CLOUD_TRANSPORT_VERIFIER = "cloud_transport_preflight.redis_readback" +CLOUD_TRANSPORT_STATUS = "degraded_transport_reachable" +CLOUD_TRANSPORT_BOUNDARY = "public_http_sanitized_candidate_only" + +_ALLOWED_PROVIDERS = frozenset({"ollama_gcp_a", "ollama_gcp_b"}) +_SAFE_CORRELATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9:._/-]{0,191}$") +_SAFE_RECEIPT_ID = re.compile(r"^cloud-transport:[a-f0-9]{32}$") + +TransportProbe = Callable[[str, str], Awaitable[bool]] + + +def resolve_gcp_transport_endpoint(provider: str) -> str: + """Resolve an allowlisted provider without exposing a private provider hook.""" + + from src.services.ollama_endpoint_resolver import resolve_ollama_order + + clean_provider = str(provider or "").strip().lower() + if clean_provider not in _ALLOWED_PROVIDERS: + raise ValueError("cloud_transport_provider_not_allowlisted") + for selection in resolve_ollama_order("canary"): + if selection.provider_name == clean_provider: + return normalize_transport_endpoint(selection.url) + raise ValueError("cloud_transport_provider_endpoint_missing") + + +def normalize_transport_endpoint(endpoint_url: str) -> str: + """Normalize an endpoint for identity hashing without returning it.""" + + return str(endpoint_url or "").strip().rstrip("/") + + +def transport_endpoint_sha256(endpoint_url: str) -> str: + normalized = normalize_transport_endpoint(endpoint_url) + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def _require_correlation(value: Any, field: str) -> str: + clean = str(value or "").strip() + if not _SAFE_CORRELATION.fullmatch(clean): + raise ValueError(f"cloud_transport_{field}_invalid") + return clean + + +def _canonical_json(payload: Mapping[str, Any]) -> str: + return json.dumps(dict(payload), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _parse_timestamp(value: Any) -> datetime | None: + try: + parsed = datetime.fromisoformat(str(value or "").replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return parsed.astimezone(UTC) + + +async def _real_ollama_endpoint_probe(provider: str, endpoint_url: str) -> bool: + """Verify only Ollama's public contract; discard the response body.""" + + if provider not in _ALLOWED_PROVIDERS: + return False + endpoint = normalize_transport_endpoint(endpoint_url) + if not endpoint.startswith(("http://", "https://")): + return False + try: + async with httpx.AsyncClient( + timeout=httpx.Timeout(5.0, connect=3.0), + follow_redirects=False, + ) as client: + response = await client.get(f"{endpoint}/api/tags") + if response.status_code != 200: + return False + payload = response.json() + except (httpx.HTTPError, ValueError, TypeError): + return False + return isinstance(payload, dict) and isinstance(payload.get("models"), list) + + +async def issue_cloud_transport_receipt( + *, + trace_id: str, + run_id: str, + work_item_id: str, + provider: str, + endpoint_url: str, + ttl_seconds: int = CLOUD_TRANSPORT_DEFAULT_TTL_SECONDS, + redis_client: Any | None = None, + network_probe: TransportProbe | None = None, + now: datetime | None = None, +) -> dict[str, Any]: + """Probe, durably write, and read back one non-secret endpoint receipt.""" + + clean_provider = str(provider or "").strip().lower() + if clean_provider not in _ALLOWED_PROVIDERS: + raise ValueError("cloud_transport_provider_not_allowlisted") + clean_trace_id = _require_correlation(trace_id, "trace_id") + clean_run_id = _require_correlation(run_id, "run_id") + clean_work_item_id = _require_correlation(work_item_id, "work_item_id") + endpoint = normalize_transport_endpoint(endpoint_url) + if not endpoint: + raise ValueError("cloud_transport_endpoint_missing") + bounded_ttl = int(ttl_seconds) + if not 1 <= bounded_ttl <= CLOUD_TRANSPORT_MAX_TTL_SECONDS: + raise ValueError("cloud_transport_ttl_out_of_bounds") + + probe = network_probe or _real_ollama_endpoint_probe + if await probe(clean_provider, endpoint) is not True: + raise RuntimeError("cloud_transport_endpoint_preflight_failed") + + observed_at = (now or datetime.now(UTC)).astimezone(UTC) + expires_at = observed_at + timedelta(seconds=bounded_ttl) + endpoint_hash = transport_endpoint_sha256(endpoint) + identity = ":".join( + ( + clean_trace_id, + clean_run_id, + clean_work_item_id, + clean_provider, + endpoint_hash, + observed_at.isoformat(), + ) + ) + receipt_id = f"cloud-transport:{hashlib.sha256(identity.encode()).hexdigest()[:32]}" + receipt: dict[str, Any] = { + "schema_version": CLOUD_TRANSPORT_RECEIPT_SCHEMA, + "receipt_id": receipt_id, + "source": CLOUD_TRANSPORT_SOURCE, + "status": CLOUD_TRANSPORT_STATUS, + "transport_boundary": CLOUD_TRANSPORT_BOUNDARY, + "transport_security_status": "degraded_public_http", + "provider": clean_provider, + "endpoint_sha256": endpoint_hash, + "trace_id": clean_trace_id, + "run_id": clean_run_id, + "work_item_id": clean_work_item_id, + "check_completed": True, + "http_status": 200, + "ollama_contract_verified": True, + "observed_at": observed_at.isoformat(), + "expires_at": expires_at.isoformat(), + "ttl_seconds": bounded_ttl, + "durable_write_ack": True, + "verifier_status": "verified", + "verified_by": CLOUD_TRANSPORT_VERIFIER, + "raw_response_persisted": False, + "endpoint_value_persisted": False, + "secret_value_persisted_or_returned": False, + } + store = redis_client or get_redis() + key = f"{CLOUD_TRANSPORT_RECEIPT_PREFIX}{receipt_id}" + encoded = _canonical_json(receipt) + await store.set(key, encoded, ex=bounded_ttl) + durable_raw = await store.get(key) + if isinstance(durable_raw, bytes): + durable_raw = durable_raw.decode("utf-8") + try: + durable = json.loads(durable_raw) if durable_raw else None + except (TypeError, ValueError, json.JSONDecodeError): + durable = None + if not isinstance(durable, dict) or _canonical_json(durable) != encoded: + raise RuntimeError("cloud_transport_receipt_readback_failed") + return receipt + + +def bounded_cloud_transport_receipt_hints(value: Any) -> list[dict[str, Any]]: + """Copy only receipt-contract fields across the sanitized cloud boundary.""" + + raw_values: list[Any] + if isinstance(value, dict): + raw_values = list(value.values()) + elif isinstance(value, list | tuple): + raw_values = list(value) + else: + return [] + fields = ( + "schema_version", + "receipt_id", + "source", + "status", + "transport_boundary", + "transport_security_status", + "provider", + "endpoint_sha256", + "trace_id", + "run_id", + "work_item_id", + "check_completed", + "http_status", + "ollama_contract_verified", + "observed_at", + "expires_at", + "ttl_seconds", + "durable_write_ack", + "verifier_status", + "verified_by", + "raw_response_persisted", + "endpoint_value_persisted", + "secret_value_persisted_or_returned", + ) + hints: list[dict[str, Any]] = [] + for raw in raw_values[: len(_ALLOWED_PROVIDERS)]: + if not isinstance(raw, dict): + continue + hint = {field: raw.get(field) for field in fields if field in raw} + if str(hint.get("provider") or "") not in _ALLOWED_PROVIDERS: + continue + hints.append(hint) + return hints + + +async def verify_cloud_transport_receipt( + *, + context: dict[str, Any] | None, + provider: str, + endpoint_url: str, + redis_client: Any | None = None, + now: datetime | None = None, +) -> str | None: + """Independently verify the exact current-run receipt from durable truth.""" + + clean_provider = str(provider or "").strip().lower() + if clean_provider not in _ALLOWED_PROVIDERS: + return "gcp_transport_provider_not_allowlisted" + root = context or {} + hints = bounded_cloud_transport_receipt_hints(root.get("cloud_transport_receipts")) + hint = next((item for item in hints if item.get("provider") == clean_provider), None) + if hint is None: + return "gcp_cloud_transport_receipt_missing" + receipt_id = str(hint.get("receipt_id") or "") + if not _SAFE_RECEIPT_ID.fullmatch(receipt_id): + return "gcp_cloud_transport_receipt_invalid" + expected_endpoint_hash = transport_endpoint_sha256(endpoint_url) + expected_fields = { + "schema_version": CLOUD_TRANSPORT_RECEIPT_SCHEMA, + "source": CLOUD_TRANSPORT_SOURCE, + "status": CLOUD_TRANSPORT_STATUS, + "transport_boundary": CLOUD_TRANSPORT_BOUNDARY, + "transport_security_status": "degraded_public_http", + "provider": clean_provider, + "endpoint_sha256": expected_endpoint_hash, + "trace_id": str(root.get("trace_id") or ""), + "run_id": str(root.get("run_id") or ""), + "work_item_id": str(root.get("work_item_id") or ""), + "check_completed": True, + "http_status": 200, + "ollama_contract_verified": True, + "durable_write_ack": True, + "verifier_status": "verified", + "verified_by": CLOUD_TRANSPORT_VERIFIER, + "raw_response_persisted": False, + "endpoint_value_persisted": False, + "secret_value_persisted_or_returned": False, + } + if any(hint.get(field) != expected for field, expected in expected_fields.items()): + return "gcp_cloud_transport_receipt_binding_invalid" + if not all(expected_fields[field] for field in ("trace_id", "run_id", "work_item_id")): + return "gcp_cloud_transport_receipt_correlation_missing" + + try: + store = redis_client or get_redis() + durable_raw = await store.get(f"{CLOUD_TRANSPORT_RECEIPT_PREFIX}{receipt_id}") + if isinstance(durable_raw, bytes): + durable_raw = durable_raw.decode("utf-8") + durable = json.loads(durable_raw) if durable_raw else None + except Exception: + return "gcp_cloud_transport_receipt_readback_unavailable" + if not isinstance(durable, dict): + return "gcp_cloud_transport_receipt_not_durable" + if _canonical_json(durable) != _canonical_json(hint): + return "gcp_cloud_transport_receipt_readback_mismatch" + + observed_at = _parse_timestamp(durable.get("observed_at")) + expires_at = _parse_timestamp(durable.get("expires_at")) + checked_at = (now or datetime.now(UTC)).astimezone(UTC) + if observed_at is None or expires_at is None: + return "gcp_cloud_transport_receipt_ttl_invalid" + lifetime = (expires_at - observed_at).total_seconds() + if not ( + 0 < lifetime <= CLOUD_TRANSPORT_MAX_TTL_SECONDS + and durable.get("ttl_seconds") == int(lifetime) + and observed_at <= checked_at < expires_at + ): + return "gcp_cloud_transport_receipt_expired" + return None diff --git a/apps/api/src/services/controlled_alert_target_router.py b/apps/api/src/services/controlled_alert_target_router.py index ddff584ca..e18e80b6b 100644 --- a/apps/api/src/services/controlled_alert_target_router.py +++ b/apps/api/src/services/controlled_alert_target_router.py @@ -13,6 +13,7 @@ runtime state. from __future__ import annotations +import hashlib import re from collections.abc import Mapping from typing import Any @@ -61,6 +62,11 @@ _NODE_EXPORTER_ALERTS = { "nodeexporterscrapedown", "nodeexporterunhealthy", } +_ALERTMANAGER_DELIVERY_ALERTS = {"alertchainbrokenalertmanager"} +_OLLAMA_LOCAL_RUNTIME_ALERTS = { + "ollamalocalunavailable", + "ollama111unavailable", +} _KUBERNETES_KINDS = { "cronjob", "daemonset", @@ -69,6 +75,10 @@ _KUBERNETES_KINDS = { "pod", "statefulset", } +_KUBERNETES_RUNTIME_UID = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +) +_KUBERNETES_RUNTIME_IDENTITY_SOURCE = "kubernetes_api_uid_readback" _HOSTS = { "99": ("host_99", "192.168.0.99"), "110": ("host_110", "192.168.0.110"), @@ -93,6 +103,133 @@ def _compact_token(value: str | None) -> str: return re.sub(r"[^a-z0-9]+", "", _text_token(value)) +def _kubernetes_drift_work_item_id(*values: str | None) -> str: + key = "|".join(_text_token(value) for value in values) or "missing-target" + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:12].upper() + return f"AIA-ASSET-DRIFT-{digest}" + + +def _kubernetes_identity_evidence( + *, + identity: Mapping[str, Any], + labels: Mapping[str, Any], + namespace: str, + target_resource: str, + runtime_identity_evidence: Mapping[str, Any] | None, +) -> dict[str, Any]: + """Verify a Kubernetes identity without trusting a workload-kind label alone.""" + + kind = _text_token( + str( + labels.get("workload_kind") + or labels.get("kubernetes_kind") + or labels.get("kind") + or "" + ) + ) + name = _text_token( + str( + labels.get("workload_name") + or labels.get(kind) + or labels.get("deployment") + or labels.get("statefulset") + or labels.get("daemonset") + or labels.get("cronjob") + or labels.get("job") + or labels.get("pod") + or "" + ) + ) + label_namespace = _text_token(str(labels.get("namespace") or "")) + requested_namespace = _text_token(namespace) + target_name = _text_token(target_resource) + registry_name = _text_token(str(identity.get("service_name") or "")) + registry_exact = bool( + identity.get("resolution_status") == "resolved" + and identity.get("asset_domain") == "kubernetes_workload" + and requested_namespace + and label_namespace == requested_namespace + and kind in _KUBERNETES_KINDS + and name + and name == target_name + and (not registry_name or registry_name == target_name) + ) + + runtime_receipt = ( + runtime_identity_evidence + if isinstance(runtime_identity_evidence, Mapping) + else {} + ) + runtime_uid = _text_token(str(runtime_receipt.get("runtime_uid") or "")) + runtime_namespace = _text_token( + str(runtime_receipt.get("namespace") or "") + ) + runtime_kind = _text_token(str(runtime_receipt.get("kind") or "")) + runtime_name = _text_token(str(runtime_receipt.get("name") or "")) + runtime_source = _text_token( + str(runtime_receipt.get("source") or "") + ) + runtime_status = _text_token( + str(runtime_receipt.get("status") or "") + ) + runtime_uid_verified = bool( + runtime_receipt.get("schema_version") + == "kubernetes_runtime_identity_receipt_v1" + and bool( + _KUBERNETES_RUNTIME_UID.fullmatch(runtime_uid) + ) + and runtime_source == _KUBERNETES_RUNTIME_IDENTITY_SOURCE + and runtime_status == "verified" + and runtime_receipt.get("durable_readback_ack") is True + and runtime_receipt.get("verified_by") + == "kubernetes_runtime_identity_verifier" + and all( + str(runtime_receipt.get(field) or "").strip() + for field in ( + "receipt_id", + "trace_id", + "run_id", + "work_item_id", + ) + ) + and requested_namespace + and runtime_namespace == requested_namespace + and runtime_kind in _KUBERNETES_KINDS + and runtime_name + and runtime_name == target_name + and (not kind or runtime_kind == kind) + and (not name or runtime_name == name) + ) + + method = ( + "canonical_registry_namespace_kind_name_exact" + if registry_exact + else "kubernetes_runtime_uid_readback" + if runtime_uid_verified + else "unverified" + ) + evidence: dict[str, Any] = { + "schema_version": "kubernetes_asset_identity_evidence_v1", + "status": "verified" if registry_exact or runtime_uid_verified else "unverified", + "method": method, + "namespace": requested_namespace or None, + "kind": kind or runtime_kind or None, + "name": name or runtime_name or target_name or None, + "registry_identity_exact": registry_exact, + "runtime_uid_verified": runtime_uid_verified, + "required": [ + "canonical_registry_identity_plus_exact_namespace_kind_name", + "or_kubernetes_api_runtime_uid_readback", + ], + } + if runtime_uid_verified: + evidence["runtime_uid_sha256"] = hashlib.sha256( + runtime_uid.encode("utf-8") + ).hexdigest() + evidence["runtime_identity_source"] = _KUBERNETES_RUNTIME_IDENTITY_SOURCE + return evidence + + def _host_scope(*values: str | None) -> tuple[str, str] | None: """Resolve only an explicit, known host identity.""" @@ -156,9 +293,15 @@ def _typed_route( allowed_inventory_hosts: list[str] | None = None, drift_work_item_id: str | None = None, stateful_level: str | None = None, + controlled_apply_allowed: bool | None = None, execution_role: str = "policy_observer_and_no_secret_dispatch_relay", ) -> dict[str, Any]: resolved = resolution_status == "resolved" + apply_allowed = ( + resolved and risk_class != "critical" + if controlled_apply_allowed is None + else resolved and risk_class != "critical" and controlled_apply_allowed + ) agent99_dispatch = resolved and target_kind in { "control_plane_recovery", "windows_vmware", @@ -180,7 +323,7 @@ def _typed_route( "risk_class": risk_class, "allowed_catalog_ids": list(allowed_catalog_ids or []), "allowed_inventory_hosts": list(allowed_inventory_hosts or []), - "controlled_apply_allowed": resolved and risk_class != "critical", + "controlled_apply_allowed": apply_allowed, "critical_break_glass_required": risk_class == "critical", "cross_domain_fallback_allowed": False, "circuit_open_behavior": "stay_in_same_domain_and_create_repair_work_item", @@ -216,6 +359,7 @@ def resolve_typed_alert_target( labels: Mapping[str, Any] | None = None, alert_category: str = "", registry: ServiceRegistryClient | None = None, + runtime_identity_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Resolve exactly one domain route; unknown assets never fall back AUTO.""" @@ -269,8 +413,61 @@ def resolve_typed_alert_target( execution_role="single_writer_host112_guest_recovery_orchestrator", ) + if compact_alert in _ALERTMANAGER_DELIVERY_ALERTS: + return _typed_route( + resolution_status="resolved", + target_kind="docker_container", + target_resource="alertmanager", + namespace=namespace, + canonical_asset_id="container:host_110:alertmanager", + host=_HOSTS["110"][1], + executor="host_ansible_executor", + verifier="alert_chain_independent_delivery_verifier", + risk_class="medium", + allowed_catalog_ids=["ansible:110-alertmanager-delivery-recovery"], + allowed_inventory_hosts=["host_110"], + execution_role="single_writer_alert_chain_container_executor", + ) + + if compact_alert in _OLLAMA_LOCAL_RUNTIME_ALERTS: + return _typed_route( + resolution_status="resolved", + target_kind="host_launchagent", + target_resource="ollama-local", + namespace=namespace, + canonical_asset_id="ai-provider:ollama_local", + host=_HOSTS["111"][1], + executor="host_ansible_executor", + verifier="ollama111_k3s_path_independent_verifier", + risk_class="medium", + allowed_catalog_ids=["ansible:111-ollama-fallback"], + allowed_inventory_hosts=["host_111", "host_120", "host_121"], + execution_role="single_writer_host111_launchagent_executor", + ) + if any(marker in text for marker in _BACKUP_RESTORE_MARKERS): identity = registry.resolve_identity(target_resource) + generic_backup_signal = _compact_token(target_resource) in { + "", + "backuprestore", + } + if ( + identity.get("resolution_status") != "resolved" + and not generic_backup_signal + ): + return _typed_route( + resolution_status="asset_identity_unresolved", + target_kind="unknown", + target_resource=target_resource, + namespace=namespace, + canonical_asset_id=None, + host=None, + executor=None, + verifier=None, + risk_class="critical", + drift_work_item_id=identity["drift_work_item_id"], + stateful_level=StatefulLevel.UNRESOLVED.value, + ) return _typed_route( resolution_status="resolved", target_kind="backup_restore", @@ -409,6 +606,54 @@ def resolve_typed_alert_target( identity = registry.resolve_identity(target_resource) if identity["resolution_status"] == "resolved": domain = str(identity["asset_domain"]) + if domain == "kubernetes_workload": + identity_evidence = _kubernetes_identity_evidence( + identity=identity, + labels=labels, + namespace=namespace, + target_resource=target_resource, + runtime_identity_evidence=runtime_identity_evidence, + ) + if identity_evidence["status"] != "verified": + route = _typed_route( + resolution_status="asset_identity_unresolved", + target_kind="unknown", + target_resource=target_resource, + namespace=namespace, + canonical_asset_id=None, + host=None, + executor=None, + verifier=None, + risk_class="high", + drift_work_item_id=_kubernetes_drift_work_item_id( + namespace, + target_resource, + str(labels.get("workload_kind") or ""), + ), + stateful_level=StatefulLevel.UNRESOLVED.value, + ) + route["resolution_reason"] = ( + "kubernetes_identity_evidence_missing_or_mismatch" + ) + route["identity_evidence"] = identity_evidence + return route + route = _typed_route( + resolution_status="resolved", + target_kind="kubernetes_workload", + target_resource=target_resource, + namespace=namespace, + canonical_asset_id=str(identity["canonical_id"]), + host="k3s", + executor=str(identity.get("executor") or "") or None, + verifier=str(identity.get("verifier") or "") or None, + risk_class="medium", + stateful_level=str(identity.get("stateful_level") or ""), + controlled_apply_allowed=bool( + identity.get("controlled_apply_allowed") + ), + ) + route["identity_evidence"] = identity_evidence + return route inventory_host = _host_scope(str(identity.get("host") or "")) catalog_ids = list(identity.get("allowed_catalog_ids") or []) exact_sentry_consumer = ( @@ -445,13 +690,21 @@ def resolve_typed_alert_target( allowed_catalog_ids=catalog_ids, allowed_inventory_hosts=[inventory_host[0]] if inventory_host else [], stateful_level=stateful_level, + controlled_apply_allowed=bool( + identity.get("controlled_apply_allowed") + ), ) - kubernetes_kind = _text_token( - str(labels.get("workload_kind") or labels.get("kubernetes_kind") or "") + identity_evidence = _kubernetes_identity_evidence( + identity=identity, + labels=labels, + namespace=namespace, + target_resource=target_resource, + runtime_identity_evidence=runtime_identity_evidence, ) - if kubernetes_kind in _KUBERNETES_KINDS: - return _typed_route( + if identity_evidence["runtime_uid_verified"] is True: + kubernetes_kind = str(identity_evidence["kind"]) + route = _typed_route( resolution_status="resolved", target_kind="kubernetes_workload", target_resource=target_resource, @@ -465,8 +718,10 @@ def resolve_typed_alert_target( verifier="kubernetes_rollout_verifier", risk_class="medium", ) + route["identity_evidence"] = identity_evidence + return route - return _typed_route( + route = _typed_route( resolution_status="asset_identity_unresolved", target_kind="unknown", target_resource=target_resource, @@ -479,6 +734,15 @@ def resolve_typed_alert_target( drift_work_item_id=identity["drift_work_item_id"], stateful_level=StatefulLevel.UNRESOLVED.value, ) + if any( + _text_token(str(labels.get(key) or "")) + for key in ("workload_kind", "kubernetes_kind", "runtime_uid") + ): + route["resolution_reason"] = ( + "kubernetes_identity_evidence_missing_or_mismatch" + ) + route["identity_evidence"] = identity_evidence + return route def resolve_typed_incident_target( @@ -500,6 +764,11 @@ def resolve_typed_incident_target( if isinstance(first_signal.get("labels"), Mapping) else {} ) + runtime_identity_evidence = ( + first_signal.get("runtime_identity_evidence") + if isinstance(first_signal.get("runtime_identity_evidence"), Mapping) + else None + ) alertname = str( incident.get("alertname") or first_signal.get("alert_name") @@ -539,6 +808,7 @@ def resolve_typed_incident_target( labels=labels, alert_category=str(incident.get("alert_category") or ""), registry=registry, + runtime_identity_evidence=runtime_identity_evidence, ) diff --git a/apps/api/src/services/failover_alerter.py b/apps/api/src/services/failover_alerter.py index ad4f825d5..c10ba0938 100644 --- a/apps/api/src/services/failover_alerter.py +++ b/apps/api/src/services/failover_alerter.py @@ -1,9 +1,9 @@ """Ollama 容災 / 自動恢復 / Gemini 帳單 Telegram 告警 設計原則： -- 每次 failover 觸發都通知（用戶明確指示） -- 但用 10min Redis dedup TTL 防同樣狀態重複告警 -- 三種告警類型：failover_triggered / recovery_succeeded / gemini_quota_exceeded +- 健康探測只回報 route degraded observation，不宣稱已切換或已修復 +- 以 canonical provider + route state 做至少 1h Redis dedup +- 三種告警類型：route_degraded_observation / recovery_succeeded / gemini_quota_exceeded 2026-04-25 P1.5 by Claude Engineer-D — Telegram Alerter for Ollama Failover """ @@ -19,13 +19,49 @@ from typing import Any import structlog TAIPEI_TZ = timezone(timedelta(hours=8)) -DEDUP_TTL_SEC = 600 # 10 min（故障切換用） +DEDUP_TTL_SEC = 3600 # 1h（同一 canonical route degraded state 不重複通知） RECOVERY_DEDUP_TTL_SEC = 3600 # 1h — GCP 健康閃爍時 1 小時內不重複告警 QUOTA_DEDUP_TTL_SEC = 86400 # 24h（每日 quota 告警只發一次） logger = structlog.get_logger(__name__) _TELEGRAM_BOT_URL_RE = re.compile(r"(api\.telegram\.org/bot)[^/\s]+") +_PROVIDER_ALIASES = { + "ollama": "ollama_gcp_a", + "ollama_111": "ollama_local", +} +_CANONICAL_PROVIDER_IDS = frozenset( + { + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + "paid_fallback_blocked", + "unknown_provider", + } +) +_CANONICAL_HEALTH_STATES = frozenset( + {"healthy", "slow", "offline", "unknown", "not_checked", "not_probed"} +) + + +def _canonical_provider_id(value: Any) -> str: + """Return a display-safe canonical provider identity, never a transport URL.""" + + candidate = str(value or "unknown_provider").strip().lower() + candidate = _PROVIDER_ALIASES.get(candidate, candidate) + if candidate in _CANONICAL_PROVIDER_IDS: + return candidate + return "unknown_provider" + + +def _canonical_health_state(value: Any) -> str: + candidate = str(value or "unknown").strip().lower() + if candidate in _CANONICAL_HEALTH_STATES: + return candidate + return "unknown" + class FailoverAlerter: """Ollama 容災 Telegram 告警 @@ -43,30 +79,58 @@ class FailoverAlerter: self._memory_dedup_max_size = 1000 async def alert_failover(self, event: dict[str, Any]) -> None: - """Ollama 故障切換告警 — 10min dedup - # 2026-05-03 ogt: ADR-110 三層容災，故障主機從 event["failed_host"] 動態讀取 + """Report a health-observed degraded route without claiming execution. + + Compatibility keeps the historic method name, but this path has no + generation/executor receipt and therefore cannot report failover as + activated. All display fields are canonical identities rather than + transport URLs or probe-model metadata. """ - to_provider = event.get("to_provider", "unknown") - dedup_key = f"alert:failover:{to_provider}" + from_provider = _canonical_provider_id( + event.get("from_provider", event.get("selected_primary")) + ) + to_provider = _canonical_provider_id(event.get("to_provider")) + route_state = "degraded_health_observation" + primary_health = _canonical_health_state(event.get("primary_health")) + candidate_health = _canonical_health_state(event.get("candidate_health")) + candidate_state = ( + "policy_candidate" + if to_provider in {"claude", "gemini", "paid_fallback_blocked"} + else "health_candidate" + ) + dedup_key = ( + "alert:ollama_route_degraded:" + f"{from_provider}:{to_provider}:{route_state}:" + f"{primary_health}:{candidate_health}" + ) if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC): - logger.debug("failover_alert_dedup_skipped", to_provider=to_provider) + logger.debug( + "ollama_route_degraded_alert_dedup_skipped", + from_provider=from_provider, + to_provider=to_provider, + route_state=route_state, + ) return - reason = event.get("reason", "unknown") - model = event.get("model", "?") timestamp = event.get("timestamp", datetime.now(TAIPEI_TZ).isoformat()) - fallback_chain_str = event.get("fallback_chain_str", "?") - # 2026-05-03 ogt: ADR-110 — 故障主機動態，不再硬編碼 111 - failed_host = event.get("failed_host", "Ollama") msg = ( - f"*Ollama 容災激活*\n\n" - f"故障主機：{_escape_md(failed_host)}\n" - f"故障狀態：{_escape_md(reason)}\n" - f"切換目標：{_escape_md(to_provider)} \\(model: {_escape_md(model)}\\)\n" - f"切換時間：{_escape_md(timestamp)}\n\n" - f"Fallback 鏈：{_escape_md(fallback_chain_str)}\n\n" - f"自動恢復服務持續監控，3 次 HEALTHY 後自動切回" + "⚠️ *Ollama 路由健康降級｜觀察中*\n\n" + f"狀態：{_escape_md(route_state)}\n" + f"資產：{_escape_md(from_provider)}\n" + f"候選路徑：{_escape_md(from_provider)} → {_escape_md(to_provider)}\n" + f"健康：{_escape_md(primary_health)} → {_escape_md(candidate_health)}\n" + f"判定：{_escape_md(candidate_state)}｜尚未執行\n\n" + "🤖 *AI／Agent 實際工作*\n" + f"├ Agent：{_escape_md('ollama_health_monitor')}\n" + f"├ 動作：{_escape_md('provider_health_probe')}\n" + f"├ Executor：{_escape_md('none')}\n" + f"├ Verifier：{_escape_md('health_probe')}\n" + f"├ LLM：{_escape_md('none')}｜0 tokens｜$0\n" + f"├ Receipt：{_escape_md('not_created_for_observation')}\n" + f"└ Evidence：{_escape_md('health_observation_only')}\n\n" + f"時間：{_escape_md(timestamp)}\n" + "後續：受控 Executor 成功且獨立 Verifier 有 receipt 後，才宣告路由啟用或修復" ) delivered = await self._deliver_reserved( msg, @@ -74,9 +138,19 @@ class FailoverAlerter: ttl=DEDUP_TTL_SEC, ) if delivered: - logger.info("failover_alert_sent", to_provider=to_provider) + logger.info( + "ollama_route_degraded_alert_sent", + from_provider=from_provider, + to_provider=to_provider, + route_state=route_state, + ) else: - logger.warning("failover_alert_no_send", to_provider=to_provider) + logger.warning( + "ollama_route_degraded_alert_no_send", + from_provider=from_provider, + to_provider=to_provider, + route_state=route_state, + ) async def alert_recovery(self, event: dict[str, Any]) -> None: """Ollama 自動恢復告警 — 1h dedup per host diff --git a/apps/api/src/services/model_version_probe.py b/apps/api/src/services/model_version_probe.py index c013fe1e6..f34746a10 100644 --- a/apps/api/src/services/model_version_probe.py +++ b/apps/api/src/services/model_version_probe.py @@ -5,7 +5,7 @@ AI Provider 版本探測 — 為每個 Provider 提供 get_version() Provider: - ollama : 34.143.170.20 GCP-A Ollama (primary) — 2026-05-03 ogt: ADR-110 GCP-A Primary - - ollama_local : 192.168.0.111 / 110 proxy Ollama (local fallback) + - ollama_local : 192.168.0.111 direct Ollama (local fallback) - claude : Anthropic Claude (版本 = model name) - gemini : Google Gemini API (版本 = model name) - openclaw_nemo : OpenClaw NemoTron (版本 = OPENCLAW_DEFAULT_MODEL) @@ -62,7 +62,7 @@ async def probe_ollama_version(url: str, model: str) -> ProviderVersionInfo: _GCP_OLLAMA_IPS = {"34.143.170.20", "34.21.145.224"} if any(ip in url for ip in _GCP_OLLAMA_IPS): provider_name = "ollama" - elif "192.168.0.111" in url or "192.168.0.110:11437" in url: + elif "192.168.0.111" in url: provider_name = "ollama_local" else: provider_name = "ollama_remote" diff --git a/apps/api/src/services/ollama_failover_manager.py b/apps/api/src/services/ollama_failover_manager.py index 955fee474..ada2d6d2e 100644 --- a/apps/api/src/services/ollama_failover_manager.py +++ b/apps/api/src/services/ollama_failover_manager.py @@ -166,12 +166,12 @@ class OllamaFailoverManager: """ Ollama 自動容災管理器 - 並行檢查 111 + 188，依健康狀態選擇最佳路由。 + 檢查 canonical GCP-A / GCP-B / Local provider 健康狀態，保留固定路由。 使用方式： manager = OllamaFailoverManager() result = await manager.select_provider() - # result.primary.url → 使用的 Ollama URL + # result.primary.url → 固定第一個 Ollama execution endpoint # result.fallback_chain → 依序 fallback 2026-04-25 Claude Engineer-C (P1.1b) @@ -198,6 +198,8 @@ class OllamaFailoverManager: self, task_type: str = "", context: dict | None = None, + *, + emit_side_effects: bool = False, ) -> OllamaRoutingResult: """ 三層 Ollama 容災路由（ADR-110 修正版 2026-05-04）。所有工作固定 @@ -210,15 +212,17 @@ class OllamaFailoverManager: Args: task_type: 任務類型（僅供 audit，不改變 provider 順序） context: 額外上下文（預留） + emit_side_effects: 預設 False；只有 production action path 可明確 + 傳 True 啟用 audit、metric、Telegram 與 recovery callback Returns: OllamaRoutingResult """ # 2026-05-04 ogt: 改用語意中性名稱 primary/secondary/tertiary， # 避免 gcp_a/gcp_b/local 與實際 URL 脫鉤造成 log 誤導 - url_primary = self._settings.OLLAMA_URL # 110:11435 → GCP-A (nginx proxy) - url_secondary = self._settings.OLLAMA_SECONDARY_URL # 110:11436 → GCP-B (nginx proxy) - url_tertiary = self._settings.OLLAMA_FALLBACK_URL # 110:11437 → Local 111 (nginx proxy) + url_primary = self._settings.OLLAMA_URL # canonical GCP-A endpoint + url_secondary = self._settings.OLLAMA_SECONDARY_URL # canonical GCP-B endpoint + url_tertiary = self._settings.OLLAMA_FALLBACK_URL # canonical Local endpoint def _to_health(r, label: str) -> HealthReport: if isinstance(r, Exception): @@ -280,8 +284,11 @@ class OllamaFailoverManager: # Provider selection never consumes paid quota. Claude/Gemini providers # each reserve requests/tokens/cost atomically immediately before calls. - # 寫入 audit_log（best-effort） - await self._write_failover_audit(result) + # Status/readback probes must not emit failover metrics, Telegram, or + # recovery-state writes. Production action callers retain the existing + # behavior and must own the explicit side-effect boundary. + if emit_side_effects: + await self._write_failover_audit(result) def _status(report: HealthReport | None) -> str: return report.status.value if report else "not_checked" @@ -300,12 +307,13 @@ class OllamaFailoverManager: health_gcp_a=health_gcp_a.status.value, health_gcp_b=_status(health_gcp_b), health_local=_status(health_local), + side_effects_emitted=emit_side_effects, ) # Recovery tracks the first health-observed fallback candidate, while # the selected execution primary remains fixed at GCP-A. # 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2 - if self._recovery_callback is not None: + if emit_side_effects and self._recovery_callback is not None: try: recovery_observation = ( result.observed_fallback.provider_name @@ -586,17 +594,17 @@ class OllamaFailoverManager: async def _write_failover_audit(self, result: OllamaRoutingResult) -> None: """ - 切換觸發時寫 structlog audit（best-effort）+ Telegram 告警 + Production route action 發現健康降級時寫 structlog + observation 告警 # 2026-04-25 critic-fix Part2 B1 by Claude Engineer-C2 # 原 AuditLog DB 寫入使用不存在的欄位（service/action/target/status/metadata） # → SQLAlchemy crash → except 吃掉 → 零稽核 # 修法：刪除 DB 寫入路徑，改用 structlog only（audit 不依賴 DB schema） - # 2026-04-25 P1.5 by Claude Engineer-D — 新增 Telegram 告警（dedup 10min） + # 2026-04-25 P1.5 by Claude Engineer-D — 新增 Telegram 告警 service="ollama_failover"（per 任務規格） - 僅在 primary 非 111 時記錄（真正發生切換） + 僅記錄 health observation；未取得 executor receipt 前不宣稱切換 """ observed = result.observed_fallback if observed is None: @@ -619,9 +627,9 @@ class OllamaFailoverManager: logger.debug("ollama_failover_metric_error", error=str(_metric_err)) logger.info( - "ollama_failover_triggered", + "ollama_route_degraded_observed", service="ollama_failover", - action="failover_triggered", + action="route_health_observation", selected_primary=result.primary.provider_name, observed_fallback=observed.provider_name, reason=result.routing_reason, @@ -632,34 +640,25 @@ class OllamaFailoverManager: health_local=result.health_local.status.value if result.health_local else "not_configured", ) - # Telegram 告警（首次切換才通知，dedup 10min 內建） - # 2026-04-25 P1.5 by Claude Engineer-D — 告警失敗不阻斷主路由邏輯 - # 2026-05-03 ogt: ADR-110 — failed_host 動態計算，顯示哪台 GCP/Local 故障 + # Telegram 僅回報 canonical route-degraded observation。不得顯示 + # transport URL、health-check model，或宣稱 executor 已完成切換。 try: from src.services.failover_alerter import get_failover_alerter - from src.services.ollama_health_monitor import HealthStatus - fallback_chain_str = " → ".join( - p.provider_name for p in result.fallback_chain - ) - # 計算故障主機描述（哪層 Ollama 不健康，用實際 URL 不用硬編碼標籤） - _failed = [] - if result.health_gcp_a.status != HealthStatus.HEALTHY: - _failed.append(self._settings.OLLAMA_URL) - if result.health_gcp_b and result.health_gcp_b.status != HealthStatus.HEALTHY: - _failed.append(self._settings.OLLAMA_SECONDARY_URL or "secondary") - if result.health_local and result.health_local.status != HealthStatus.HEALTHY: - _failed.append(self._settings.OLLAMA_FALLBACK_URL or "tertiary") - failed_host = " + ".join(_failed) if _failed else "Ollama" + + observed_health = "not_probed" + if observed.provider_name == "ollama_gcp_b" and result.health_gcp_b: + observed_health = result.health_gcp_b.status.value + elif observed.provider_name == "ollama_local" and result.health_local: + observed_health = result.health_local.status.value alerter = get_failover_alerter() await alerter.alert_failover({ "to_provider": observed.provider_name, - "selected_primary": result.primary.provider_name, + "from_provider": result.primary.provider_name, "observation_only": True, - "model": observed.model, - "reason": result.routing_reason, + "route_state": "degraded_health_observation", + "primary_health": result.health_gcp_a.status.value, + "candidate_health": observed_health, "timestamp": datetime.datetime.now(TAIPEI_TZ).isoformat(), - "fallback_chain_str": fallback_chain_str, - "failed_host": failed_host, }) except Exception as e: logger.warning("failover_alert_failed", error=str(e)) diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 649e8ee2d..31d6c54c7 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -38,6 +38,9 @@ from src.models.ai import ( OpenClawDecision, SuggestedAction, ) +from src.services.cloud_transport_receipt import ( + bounded_cloud_transport_receipt_hints, +) from src.services.langfuse_client import langfuse_trace from src.services.model_registry import get_model_registry from src.services.ollama_failover_manager import get_ollama_failover_manager @@ -345,10 +348,19 @@ def build_paid_cloud_alert_context(alert_context: dict | None) -> tuple[dict | N "data_classification": "sanitized", "sanitized_payload": payload, "cloud_sanitization_receipt": receipt, + "cloud_transport_boundary": "public_http_sanitized_candidate_only", "contains_secret": False, "secret_value_exposed": False, "raw_log_payload_forwarded": False, } + transport_receipts = bounded_cloud_transport_receipt_hints( + source.get("cloud_transport_receipts") + ) + if transport_receipts: + # A sanitizer never creates or verifies transport evidence. It copies + # only bounded lookup hints; the GCP provider independently reads the + # exact durable receipt immediately before generation. + cloud_context["cloud_transport_receipts"] = transport_receipts return cloud_context, "verified_allowlisted_sanitization_receipt" @@ -579,7 +591,7 @@ class OpenClawService: key in alert_context for key in ("alert_type", "alertname", "alert_name", "incident_id", "signals") ): - return prompt, alert_context, None + return None, None, "cloud_sanitization_receipt_missing" cloud_context = alert_context.get("paid_cloud_context") reason = _verified_paid_cloud_context_reason( cloud_context if isinstance(cloud_context, dict) else None @@ -624,7 +636,10 @@ class OpenClawService: """Resolve the only production route: GCP-A, GCP-B, 111, Claude, Gemini.""" provider_order: list[str] = [] try: - route = await get_ollama_failover_manager().select_provider(task_type=task_type) + route = await get_ollama_failover_manager().select_provider( + task_type=task_type, + emit_side_effects=True, + ) provider_order = [ endpoint.provider_name for endpoint in route.all_endpoints_in_order() @@ -1004,7 +1019,13 @@ class OpenClawService: # 完整移除時機: Phase 24 完整驗收後 (ADR-052 D11) # ========================================================================= - async def _call_ollama(self, prompt: str, *, ollama_only: bool = False) -> tuple[str, bool]: + async def _call_ollama( + self, + prompt: str, + *, + alert_context: dict | None = None, + ollama_only: bool = False, + ) -> tuple[str, bool]: """ 呼叫 Ollama (支援 JSON Mode)。 @@ -1028,7 +1049,9 @@ class OpenClawService: endpoints: list[tuple[str, str]] = [] try: - route = await get_ollama_failover_manager().select_provider() + route = await get_ollama_failover_manager().select_provider( + emit_side_effects=True, + ) endpoints = [ (endpoint.provider_name, endpoint.url) for endpoint in route.all_endpoints_in_order() @@ -1074,20 +1097,39 @@ class OpenClawService: ] last_error = "" + cloud_prompt, _cloud_context, cloud_block_reason = ( + self._paid_cloud_execution_inputs(prompt, alert_context) + ) for provider_name, endpoint_url in endpoints: + provider_prompt = prompt + if provider_name in {"ollama_gcp_a", "ollama_gcp_b"}: + if cloud_block_reason or cloud_prompt is None: + last_error = ( + f"{provider_name} blocked: " + f"{cloud_block_reason or 'sanitized_prompt_missing'}" + ) + logger.info( + "legacy_gcp_ollama_context_blocked", + provider=provider_name, + reason=( + cloud_block_reason or "sanitized_prompt_missing" + ), + ) + continue + provider_prompt = cloud_prompt try: logger.info( "ollama_request_start", provider=provider_name, url=f"{endpoint_url}/api/generate", - prompt_length=len(prompt), + prompt_length=len(provider_prompt), ) response = await client.post( f"{endpoint_url}/api/generate", json={ "model": model_name, - "prompt": prompt, + "prompt": provider_prompt, "stream": False, "format": "json", # 強制 JSON 輸出 "options": { @@ -1579,6 +1621,11 @@ class OpenClawService: # webhooks alert_context 從未注入 task_type → Ollama fallback 永遠 30s timeout # 對齊 decision.intent 後 Ollama fallback 真正能跑完 exec_context = dict(routing_context) + if is_incident_alert: + # GCP-A/B use a degraded public-HTTP stopgap. The router + # must issue and durably read back an exact same-run + # transport receipt before either sanitized generation. + exec_context["cloud_transport_auto_preflight"] = True if decision.intent == IntentType.DIAGNOSE: exec_context["task_type"] = "diagnose" if self._alert_enforces_ollama_first(routing_context): @@ -1747,6 +1794,7 @@ class OpenClawService: if provider == "ollama": response, success = await self._call_ollama( prompt, + alert_context=alert_context, ollama_only=self._alert_enforces_ollama_first(alert_context), ) elif provider == "gemini": diff --git a/apps/api/src/services/paid_provider_canary_authorization.py b/apps/api/src/services/paid_provider_canary_authorization.py new file mode 100644 index 000000000..da7ab465c --- /dev/null +++ b/apps/api/src/services/paid_provider_canary_authorization.py @@ -0,0 +1,465 @@ +"""Fail-closed durable authorization readback for the paid-provider canary. + +The verifier reuses the AwoooP Operator Console Gate 5 production evidence +chain. It is intentionally read-only: it validates an already-approved run +and returns a non-sensitive receipt, but it never creates, approves, consumes, +or mutates an authorization. + +An authorization is valid only when the run is bound to the exact committed +paid-canary input contract, is non-shadow and executable, has one durable +``operator_console.approve`` journal step plus its matching audit row, is still +inside the Gate 5 lifetime, and has not appeared in a prior paid-canary start +receipt. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Protocol +from uuid import UUID + +import structlog +from sqlalchemy import func, select, text + +from src.db.awooop_models import AwoooPRunState, AwoooPRunStepJournal +from src.db.base import get_db_context + +logger = structlog.get_logger(__name__) + +PAID_CANARY_PROJECT_ID = "awoooi" +PAID_CANARY_WORK_ITEM_ID = "AIA-SRE-013" +PAID_CANARY_AGENT_ID = "awoooi-paid-provider-canary" +PAID_CANARY_TRIGGER_TYPE = "api" +PAID_CANARY_TRIGGER_REF = "paid-provider-canary:AIA-SRE-013" +PAID_CANARY_APPROVAL_STEP = "operator_console.approve" +PAID_CANARY_APPROVAL_AUDIT_ACTION = "run.approval.approve" +PAID_CANARY_START_ACTION = "claude_gemini_sanitized_sre_canary" +PAID_CANARY_APPROVAL_TTL_SECONDS = 900 +PAID_CANARY_PERCENT = 5 +_SAFE_TRACE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,191}$") + +_PAID_CANARY_AUTHORIZATION_INPUT: dict[str, Any] = { + "schema_version": "paid_provider_canary_authorization_input_v1", + "project_id": PAID_CANARY_PROJECT_ID, + "work_item_id": PAID_CANARY_WORK_ITEM_ID, + "action": "authorize_bounded_paid_provider_canary", + "scope": "one_sanitized_five_lane_comparison", + "provider_order": [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ], + "paid_provider_canary_percent": 5, + "authorization_run_is_execution_run": True, + "authorization_run_must_be_in_canary_bucket": True, + "max_output_tokens_per_paid_provider": 512, + "max_total_paid_cost_usd": "0.25", + "infrastructure_remediation_write_allowed": False, + "raw_prompt_persistence_allowed": False, + "raw_response_persistence_allowed": False, +} + + +def paid_provider_canary_authorization_input() -> dict[str, Any]: + """Return a detached copy of the exact Gate 5 input contract.""" + + return json.loads(json.dumps(_PAID_CANARY_AUTHORIZATION_INPUT)) + + +def paid_provider_canary_authorization_input_sha256() -> str: + """Return the hash that must be present on ``awooop_run_state``.""" + + canonical = json.dumps( + _PAID_CANARY_AUTHORIZATION_INPUT, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def paid_provider_canary_bucket(run_id: UUID | str) -> int: + """Return the deterministic rollout bucket for the exact Gate 5 run.""" + + canonical = str(run_id).strip() + return int(hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:8], 16) % 100 + + +def paid_provider_canary_run_selected(run_id: UUID | str) -> bool: + """Keep authorization and paid execution on one selected run identity.""" + + return paid_provider_canary_bucket(run_id) < PAID_CANARY_PERCENT + + +@dataclass(frozen=True, slots=True) +class PaidProviderCanaryAuthorizationEvidence: + """Minimal durable evidence returned by the DB reader.""" + + run_id: UUID + trace_id: str | None + project_id: str + agent_id: str + state: str + trigger_type: str | None + trigger_ref: str | None + is_shadow: bool + input_sha256: str | None + timeout_at: datetime | None + approval_step_count: int + approval_step_status: str | None + approval_step_was_blocked: bool | None + approval_completed_at: datetime | None + approval_audit_count: int + approval_audit_created_at: datetime | None + prior_canary_start_count: int + + +class PaidProviderCanaryAuthorizationEvidenceReader(Protocol): + """Read-only seam used by production DB readback and unit tests.""" + + async def read( + self, + *, + run_id: UUID, + project_id: str, + work_item_id: str, + ) -> PaidProviderCanaryAuthorizationEvidence | None: ... + + +class PaidProviderCanaryAuthorizationError(RuntimeError): + """Safe, machine-readable fail-closed authorization error.""" + + def __init__(self, error_code: str, receipt: dict[str, Any]) -> None: + super().__init__(error_code) + self.error_code = error_code + self.receipt = receipt + + +class _SqlPaidProviderCanaryAuthorizationEvidenceReader: + """Production readback from the AwoooP run, journal and audit ledgers.""" + + async def read( + self, + *, + run_id: UUID, + project_id: str, + work_item_id: str, + ) -> PaidProviderCanaryAuthorizationEvidence | None: + async with get_db_context(project_id) as db: + run_result = await db.execute( + select(AwoooPRunState).where( + AwoooPRunState.run_id == run_id, + AwoooPRunState.project_id == project_id, + ) + ) + run = run_result.scalar_one_or_none() + if run is None: + return None + + step_result = await db.execute( + select( + func.count(AwoooPRunStepJournal.step_id), + func.max(AwoooPRunStepJournal.result_status), + func.bool_or(AwoooPRunStepJournal.was_blocked), + func.max(AwoooPRunStepJournal.completed_at), + ).where( + AwoooPRunStepJournal.run_id == run_id, + AwoooPRunStepJournal.project_id == project_id, + AwoooPRunStepJournal.tool_name == PAID_CANARY_APPROVAL_STEP, + ) + ) + step_row = step_result.one() + + # ``audit_logs`` is the durable sink used by decide_approval(). + # Query it directly because the legacy ORM model does not expose + # the AwoooP action/resource/details columns present in production. + audit_result = await db.execute( + text( + """ + SELECT COUNT(*) AS approval_audit_count, + MAX(created_at) AS approval_audit_created_at + FROM audit_logs + WHERE project_id = :project_id + AND action = :action + AND resource_type = 'run' + AND resource_id = :run_id + AND details ->> '_run_id' = :run_id + AND details ->> 'decision' = 'approve' + AND details ->> 'new_state' = 'running' + AND COALESCE(details ->> 'approver_id', '') <> '' + """ + ), + { + "project_id": project_id, + "action": PAID_CANARY_APPROVAL_AUDIT_ACTION, + "run_id": str(run_id), + }, + ) + audit_row = audit_result.mappings().one() + + # A prior start receipt consumes this authorization for replay + # protection. The query intentionally matches the exact UUID and + # work item recorded in the immutable operation context; project + # ownership was already proven by the tenant-scoped run query. + used_result = await db.execute( + text( + """ + SELECT COUNT(*) AS prior_canary_start_count + FROM alert_operation_log + WHERE event_type = 'EXECUTION_STARTED' + AND action_detail = :action_detail + AND context ->> 'authorization_ref' = :run_id + AND context ->> 'work_item_id' = :work_item_id + """ + ), + { + "action_detail": PAID_CANARY_START_ACTION, + "run_id": str(run_id), + "work_item_id": work_item_id, + }, + ) + used_row = used_result.mappings().one() + + return PaidProviderCanaryAuthorizationEvidence( + run_id=run.run_id, + trace_id=run.trace_id, + project_id=run.project_id, + agent_id=run.agent_id, + state=run.state, + trigger_type=run.trigger_type, + trigger_ref=run.trigger_ref, + is_shadow=bool(run.is_shadow), + input_sha256=run.input_sha256, + timeout_at=run.timeout_at, + approval_step_count=int(step_row[0] or 0), + approval_step_status=step_row[1], + approval_step_was_blocked=( + bool(step_row[2]) if step_row[2] is not None else None + ), + approval_completed_at=step_row[3], + approval_audit_count=int(audit_row["approval_audit_count"] or 0), + approval_audit_created_at=audit_row["approval_audit_created_at"], + prior_canary_start_count=int(used_row["prior_canary_start_count"] or 0), + ) + + +def _utc(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def _blocked_receipt( + *, + authorization_ref: str, + checked_at: datetime, + error_code: str, +) -> dict[str, Any]: + return { + "schema_version": "paid_provider_canary_authorization_receipt_v1", + "status": "blocked_with_safe_next_action", + "authorized": False, + "error_code": error_code, + "authorization_ref": authorization_ref, + "project_id": PAID_CANARY_PROJECT_ID, + "work_item_id": PAID_CANARY_WORK_ITEM_ID, + "checked_at": checked_at.isoformat(), + "read_only_verification": True, + "authorization_consumed_by_verifier": False, + "raw_input_persisted_or_returned": False, + "operator_identity_returned": False, + "secret_value_read_or_returned": False, + } + + +async def verify_paid_provider_canary_authorization( + authorization_ref: str, + *, + evidence_reader: PaidProviderCanaryAuthorizationEvidenceReader | None = None, + now: datetime | None = None, +) -> dict[str, Any]: + """Read and verify one exact, unexpired and unused Gate 5 authorization. + + The function never raises for malformed or unavailable evidence. It + returns a fail-closed receipt suitable for safe operation-log projection. + Use :func:`require_paid_provider_canary_authorization` when an exception is + more convenient for a command boundary. + """ + + checked_at = _utc(now) or datetime.now(UTC) + clean_ref = str(authorization_ref or "").strip() + try: + run_id = UUID(clean_ref) + except (TypeError, ValueError, AttributeError): + return _blocked_receipt( + authorization_ref=clean_ref, + checked_at=checked_at, + error_code="paid_canary_authorization_ref_must_be_uuid_run_id", + ) + if clean_ref != str(run_id): + return _blocked_receipt( + authorization_ref=clean_ref, + checked_at=checked_at, + error_code="paid_canary_authorization_ref_not_canonical_uuid", + ) + + reader = evidence_reader or _SqlPaidProviderCanaryAuthorizationEvidenceReader() + try: + evidence = await reader.read( + run_id=run_id, + project_id=PAID_CANARY_PROJECT_ID, + work_item_id=PAID_CANARY_WORK_ITEM_ID, + ) + except Exception as exc: + logger.warning( + "paid_canary_authorization_readback_failed", + run_id=str(run_id), + error_type=type(exc).__name__, + ) + return _blocked_receipt( + authorization_ref=clean_ref, + checked_at=checked_at, + error_code="paid_canary_authorization_evidence_unavailable", + ) + if evidence is None: + return _blocked_receipt( + authorization_ref=clean_ref, + checked_at=checked_at, + error_code="paid_canary_authorization_run_not_found", + ) + + expected_input_sha256 = paid_provider_canary_authorization_input_sha256() + checks = { + "run_id_exact": evidence.run_id == run_id, + "trace_id_present": bool( + _SAFE_TRACE_ID.fullmatch(str(evidence.trace_id or "").strip()) + ), + "authorization_run_selected": paid_provider_canary_run_selected( + evidence.run_id + ), + "project_exact": evidence.project_id == PAID_CANARY_PROJECT_ID, + "agent_exact": evidence.agent_id == PAID_CANARY_AGENT_ID, + "state_executable": evidence.state == "running", + "non_shadow": evidence.is_shadow is False, + "trigger_type_exact": evidence.trigger_type == PAID_CANARY_TRIGGER_TYPE, + "trigger_ref_exact": evidence.trigger_ref == PAID_CANARY_TRIGGER_REF, + "input_sha256_exact": evidence.input_sha256 == expected_input_sha256, + "approval_step_exactly_once": evidence.approval_step_count == 1, + "approval_step_success": evidence.approval_step_status == "success", + "approval_step_not_blocked": evidence.approval_step_was_blocked is False, + "approval_step_completed": evidence.approval_completed_at is not None, + "approval_audit_exactly_once": evidence.approval_audit_count == 1, + "approval_audit_durable": evidence.approval_audit_created_at is not None, + "authorization_unused": evidence.prior_canary_start_count == 0, + } + + approval_completed_at = _utc(evidence.approval_completed_at) + approval_audit_created_at = _utc(evidence.approval_audit_created_at) + run_timeout_at = _utc(evidence.timeout_at) + approval_expires_at = ( + approval_completed_at + + timedelta(seconds=PAID_CANARY_APPROVAL_TTL_SECONDS) + if approval_completed_at is not None + else None + ) + expiry_candidates = [ + value + for value in (approval_expires_at, run_timeout_at) + if value is not None + ] + effective_expires_at = min(expiry_candidates) if len(expiry_candidates) == 2 else None + checks["run_timeout_present"] = run_timeout_at is not None + checks["bounded_expiry_computable"] = effective_expires_at is not None + checks["approval_audit_time_correlated"] = bool( + approval_completed_at is not None + and approval_audit_created_at is not None + and approval_completed_at <= approval_audit_created_at + and approval_audit_created_at <= checked_at + timedelta(seconds=5) + ) + checks["authorization_not_expired"] = bool( + effective_expires_at is not None + and approval_completed_at is not None + and approval_completed_at <= checked_at + and checked_at < effective_expires_at + ) + + failed_checks = [name for name, passed in checks.items() if passed is not True] + if failed_checks: + receipt = _blocked_receipt( + authorization_ref=clean_ref, + checked_at=checked_at, + error_code=f"paid_canary_authorization_{failed_checks[0]}_failed", + ) + receipt["checks"] = checks + receipt["failed_checks"] = failed_checks + receipt["prior_canary_start_count"] = max( + 0, + evidence.prior_canary_start_count, + ) + return receipt + + assert effective_expires_at is not None + return { + "schema_version": "paid_provider_canary_authorization_receipt_v1", + "status": "authorized", + "authorized": True, + "error_code": None, + "authorization_ref": clean_ref, + "trace_id": str(evidence.trace_id), + "run_id": clean_ref, + "project_id": PAID_CANARY_PROJECT_ID, + "work_item_id": PAID_CANARY_WORK_ITEM_ID, + "checked_at": checked_at.isoformat(), + "expires_at": effective_expires_at.isoformat(), + "valid_for_seconds": max( + 0, + int((effective_expires_at - checked_at).total_seconds()), + ), + "contract": { + "agent_id": PAID_CANARY_AGENT_ID, + "state": "running", + "trigger_type": PAID_CANARY_TRIGGER_TYPE, + "trigger_ref": PAID_CANARY_TRIGGER_REF, + "input_sha256": expected_input_sha256, + "operator_approval_step": PAID_CANARY_APPROVAL_STEP, + "operator_approval_audit": PAID_CANARY_APPROVAL_AUDIT_ACTION, + "same_run_identity": True, + "canary_bucket": paid_provider_canary_bucket(evidence.run_id), + }, + "checks": checks, + "prior_canary_start_count": 0, + "reuse_check": "clear_at_read_time", + "read_only_verification": True, + "authorization_consumed_by_verifier": False, + "raw_input_persisted_or_returned": False, + "operator_identity_returned": False, + "secret_value_read_or_returned": False, + } + + +async def require_paid_provider_canary_authorization( + authorization_ref: str, + *, + evidence_reader: PaidProviderCanaryAuthorizationEvidenceReader | None = None, + now: datetime | None = None, +) -> dict[str, Any]: + """Return an authorized receipt or raise a safe fail-closed error.""" + + receipt = await verify_paid_provider_canary_authorization( + authorization_ref, + evidence_reader=evidence_reader, + now=now, + ) + if receipt.get("authorized") is not True: + raise PaidProviderCanaryAuthorizationError( + str(receipt.get("error_code") or "paid_canary_authorization_blocked"), + receipt, + ) + return receipt diff --git a/apps/api/src/services/paid_provider_canary_gate5_run.py b/apps/api/src/services/paid_provider_canary_gate5_run.py new file mode 100644 index 000000000..f94794963 --- /dev/null +++ b/apps/api/src/services/paid_provider_canary_gate5_run.py @@ -0,0 +1,385 @@ +"""Operator-only Gate 5 run creation for the paid-provider canary. + +The generic Platform Runs API intentionally remains shadow-only. This module +provides the one narrow exception required to create an executable Gate 5 +authorization row for the bounded Claude/Gemini canary. It does not call a +provider, enable a route, or persist prompts/responses; it only creates (or +returns) one exact, short-lived ``waiting_approval`` run. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Any, Protocol +from uuid import UUID + +import structlog +from sqlalchemy import or_, select, text, update + +from src.db.awooop_models import AwoooPRunState +from src.db.base import get_db_context +from src.services.audit_sink import write_audit +from src.services.paid_provider_canary_authorization import ( + PAID_CANARY_AGENT_ID, + PAID_CANARY_APPROVAL_TTL_SECONDS, + PAID_CANARY_PROJECT_ID, + PAID_CANARY_TRIGGER_REF, + PAID_CANARY_TRIGGER_TYPE, + PAID_CANARY_WORK_ITEM_ID, + paid_provider_canary_authorization_input, + paid_provider_canary_authorization_input_sha256, + paid_provider_canary_bucket, + paid_provider_canary_run_selected, +) +from src.services.platform_runtime import _new_trace_id, _uuid7 + +logger = structlog.get_logger(__name__) + +PAID_CANARY_AUTHORIZATION_SCOPE = "authorize_one_sanitized_five_lane_comparison" +PAID_CANARY_CREATE_AUDIT_ACTION = "run.created.paid_provider_canary_authorization" +_PAID_CANARY_CREATE_LOCK_KEY = "awoooi:AIA-SRE-013:paid-provider-canary:gate5" +_PAID_CANARY_ACTIVE_STATES = ("waiting_approval", "running") + + +def _new_selected_paid_canary_run_id() -> UUID: + """Generate one Gate 5 identity that is itself inside the 5% canary.""" + + for _ in range(1000): + candidate = _uuid7() + if paid_provider_canary_run_selected(candidate): + return candidate + raise RuntimeError("paid_provider_canary_selected_run_id_unavailable") + + +@dataclass(frozen=True, slots=True) +class PaidProviderCanaryGate5RunRecord: + """Non-sensitive result of the bounded Gate 5 run write.""" + + run_id: UUID + project_id: str + agent_id: str + state: str + trigger_type: str + trigger_ref: str + is_shadow: bool + input_sha256: str + timeout_at: datetime + trace_id: str + + +class PaidProviderCanaryGate5RunRepository(Protocol): + """Transactional seam for focused unit tests and PostgreSQL production.""" + + async def create_or_get( + self, + *, + run_id: UUID, + trace_id: str, + now: datetime, + timeout_at: datetime, + input_sha256: str, + ) -> tuple[PaidProviderCanaryGate5RunRecord, bool]: ... + + +AuditWriter = Callable[..., Awaitable[None]] + + +def _utc_naive(value: datetime) -> datetime: + if value.tzinfo is None: + return value + return value.astimezone(UTC).replace(tzinfo=None) + + +def _record_from_run(run: AwoooPRunState) -> PaidProviderCanaryGate5RunRecord: + if run.timeout_at is None or run.input_sha256 is None or run.trace_id is None: + raise RuntimeError("paid_provider_canary_gate5_run_incomplete") + return PaidProviderCanaryGate5RunRecord( + run_id=run.run_id, + project_id=run.project_id, + agent_id=run.agent_id, + state=run.state, + trigger_type=str(run.trigger_type or ""), + trigger_ref=str(run.trigger_ref or ""), + is_shadow=bool(run.is_shadow), + input_sha256=run.input_sha256, + timeout_at=run.timeout_at, + trace_id=run.trace_id, + ) + + +class _SqlPaidProviderCanaryGate5RunRepository: + """Serialize exact Gate 5 creation and prevent concurrent duplicates.""" + + async def create_or_get( + self, + *, + run_id: UUID, + trace_id: str, + now: datetime, + timeout_at: datetime, + input_sha256: str, + ) -> tuple[PaidProviderCanaryGate5RunRecord, bool]: + db_now = _utc_naive(now) + db_timeout_at = _utc_naive(timeout_at) + + async with get_db_context(PAID_CANARY_PROJECT_ID) as db: + # A transaction-scoped advisory lock makes the read-then-insert + # idempotent even though the legacy table has no partial unique + # index for this one authorization lane. + await db.execute( + text("SELECT pg_advisory_xact_lock(hashtext(:lock_key))"), + {"lock_key": _PAID_CANARY_CREATE_LOCK_KEY}, + ) + + # Expired, never-approved requests are terminalized so they do not + # remain forever in the operator approval queue. + await db.execute( + update(AwoooPRunState) + .where( + AwoooPRunState.project_id == PAID_CANARY_PROJECT_ID, + or_( + AwoooPRunState.agent_id == PAID_CANARY_AGENT_ID, + AwoooPRunState.trigger_ref == PAID_CANARY_TRIGGER_REF, + ), + AwoooPRunState.state == "waiting_approval", + AwoooPRunState.timeout_at <= db_now, + ) + .values( + state="timeout", + completed_at=db_now, + error_code="E-PAID-CANARY-AUTH-EXPIRED", + error_detail="paid provider canary Gate 5 approval expired", + ) + ) + + existing_result = await db.execute( + select(AwoooPRunState) + .where( + AwoooPRunState.project_id == PAID_CANARY_PROJECT_ID, + or_( + AwoooPRunState.agent_id == PAID_CANARY_AGENT_ID, + AwoooPRunState.trigger_ref == PAID_CANARY_TRIGGER_REF, + ), + AwoooPRunState.state.in_(_PAID_CANARY_ACTIVE_STATES), + AwoooPRunState.timeout_at > db_now, + ) + .order_by(AwoooPRunState.created_at.desc()) + .with_for_update() + ) + active_runs = list(existing_result.scalars().all()) + if len(active_runs) > 1: + raise RuntimeError( + "paid_provider_canary_multiple_active_gate5_runs" + ) + existing = active_runs[0] if active_runs else None + if existing is not None: + active_contract_checks = { + "agent_exact": existing.agent_id == PAID_CANARY_AGENT_ID, + "trigger_type_exact": ( + existing.trigger_type == PAID_CANARY_TRIGGER_TYPE + ), + "trigger_ref_exact": ( + existing.trigger_ref == PAID_CANARY_TRIGGER_REF + ), + "non_shadow": existing.is_shadow is False, + "input_sha256_exact": ( + existing.input_sha256 == input_sha256 + ), + "authorization_run_selected": ( + paid_provider_canary_run_selected(existing.run_id) + ), + } + active_contract_errors = [ + name + for name, passed in active_contract_checks.items() + if passed is not True + ] + if active_contract_errors: + raise RuntimeError( + "paid_provider_canary_active_gate5_contract_drift:" + + ",".join(active_contract_errors) + ) + return _record_from_run(existing), True + + run = AwoooPRunState( + run_id=run_id, + project_id=PAID_CANARY_PROJECT_ID, + agent_id=PAID_CANARY_AGENT_ID, + state="waiting_approval", + trace_id=trace_id, + trigger_type=PAID_CANARY_TRIGGER_TYPE, + trigger_ref=PAID_CANARY_TRIGGER_REF, + is_shadow=False, + input_sha256=input_sha256, + timeout_at=db_timeout_at, + max_attempts=1, + ) + db.add(run) + await db.flush() + return _record_from_run(run), False + + +def paid_provider_canary_approval_contract_errors( + run: AwoooPRunState, + *, + now: datetime | None = None, +) -> list[str] | None: + """Validate paid-canary rows before the generic approve transition. + + ``None`` means this is not a paid-canary row. A list means it is related + to the lane; the list is empty only for the exact current contract. + Matching either canonical identity field is enough to fail closed against + a partially forged or drifted row. + """ + + is_related = ( + run.agent_id == PAID_CANARY_AGENT_ID + or run.trigger_ref == PAID_CANARY_TRIGGER_REF + ) + if not is_related: + return None + + checked_at = _utc_naive(now or datetime.now(UTC)) + expected_input_sha256 = paid_provider_canary_authorization_input_sha256() + checks = { + "project_exact": run.project_id == PAID_CANARY_PROJECT_ID, + "agent_exact": run.agent_id == PAID_CANARY_AGENT_ID, + "state_waiting_approval": run.state == "waiting_approval", + "non_shadow": run.is_shadow is False, + "trigger_type_exact": run.trigger_type == PAID_CANARY_TRIGGER_TYPE, + "trigger_ref_exact": run.trigger_ref == PAID_CANARY_TRIGGER_REF, + "input_sha256_exact": run.input_sha256 == expected_input_sha256, + "authorization_run_selected": paid_provider_canary_run_selected( + run.run_id + ), + "timeout_present": run.timeout_at is not None, + "timeout_not_expired": bool( + run.timeout_at is not None + and _utc_naive(run.timeout_at) > checked_at + ), + } + return [name for name, passed in checks.items() if passed is not True] + + +async def create_paid_provider_canary_gate5_run( + *, + operator_id: str, + repository: PaidProviderCanaryGate5RunRepository | None = None, + audit_writer: AuditWriter | None = None, + now: datetime | None = None, +) -> dict[str, Any]: + """Create/get the exact short-lived authorization row; execute nothing.""" + + if not str(operator_id or "").strip(): + raise ValueError("paid_provider_canary_operator_identity_required") + created_at = now or datetime.now(UTC) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=UTC) + else: + created_at = created_at.astimezone(UTC) + timeout_at = created_at + timedelta( + seconds=PAID_CANARY_APPROVAL_TTL_SECONDS + ) + authorization_input = paid_provider_canary_authorization_input() + input_sha256 = paid_provider_canary_authorization_input_sha256() + run_id = _new_selected_paid_canary_run_id() + trace_id = _new_trace_id() + + writer = repository or _SqlPaidProviderCanaryGate5RunRepository() + record, is_duplicate = await writer.create_or_get( + run_id=run_id, + trace_id=trace_id, + now=created_at, + timeout_at=timeout_at, + input_sha256=input_sha256, + ) + + write_creation_audit = audit_writer or write_audit + await write_creation_audit( + project_id=PAID_CANARY_PROJECT_ID, + action=PAID_CANARY_CREATE_AUDIT_ACTION, + resource_type="run", + resource_id=str(record.run_id), + details={ + "requested_by": operator_id, + "agent_id": PAID_CANARY_AGENT_ID, + "trigger_type": PAID_CANARY_TRIGGER_TYPE, + "trigger_ref": PAID_CANARY_TRIGGER_REF, + "work_item_id": PAID_CANARY_WORK_ITEM_ID, + "is_duplicate": is_duplicate, + "is_shadow": False, + "state": record.state, + "input_sha256": input_sha256, + "approval_ttl_seconds": PAID_CANARY_APPROVAL_TTL_SECONDS, + "provider_call_performed": False, + "route_change_performed": False, + "infrastructure_write_performed": False, + }, + run_id=str(record.run_id), + trace_id=record.trace_id, + ) + + logger.info( + "paid_provider_canary_gate5_run_ready", + run_id=str(record.run_id), + state=record.state, + is_duplicate=is_duplicate, + timeout_at=record.timeout_at.isoformat(), + ) + return { + "schema_version": "paid_provider_canary_gate5_run_v1", + "authorization_ref": str(record.run_id), + "run_id": str(record.run_id), + "project_id": record.project_id, + "work_item_id": PAID_CANARY_WORK_ITEM_ID, + "agent_id": record.agent_id, + "state": record.state, + "is_shadow": record.is_shadow, + "is_duplicate": is_duplicate, + "trigger_type": record.trigger_type, + "trigger_ref": record.trigger_ref, + "input_sha256": record.input_sha256, + "authorization_scope": PAID_CANARY_AUTHORIZATION_SCOPE, + "provider_order": authorization_input["provider_order"], + "paid_provider_canary_percent": authorization_input[ + "paid_provider_canary_percent" + ], + "canary_bucket": paid_provider_canary_bucket(record.run_id), + "authorization_run_is_execution_run": True, + "max_output_tokens_per_paid_provider": authorization_input[ + "max_output_tokens_per_paid_provider" + ], + "max_total_paid_cost_usd": authorization_input[ + "max_total_paid_cost_usd" + ], + "timeout_at": record.timeout_at, + "approval_ttl_seconds": PAID_CANARY_APPROVAL_TTL_SECONDS, + "provider_call_performed": False, + "route_change_performed": False, + "infrastructure_write_performed": authorization_input[ + "infrastructure_remediation_write_allowed" + ], + "raw_prompt_persistence_allowed": authorization_input[ + "raw_prompt_persistence_allowed" + ], + "raw_response_persistence_allowed": authorization_input[ + "raw_response_persistence_allowed" + ], + "next_action": ( + "operator_gate5_decision_required" + if record.state == "waiting_approval" + else "verify_authorization_then_run_bounded_canary" + ), + "approval_path": f"/api/v1/platform/approvals/{record.run_id}/decide", + } + + +__all__ = [ + "PAID_CANARY_AUTHORIZATION_SCOPE", + "PaidProviderCanaryGate5RunRecord", + "PaidProviderCanaryGate5RunRepository", + "create_paid_provider_canary_gate5_run", + "paid_provider_canary_approval_contract_errors", +] diff --git a/apps/api/src/services/paid_provider_canary_validation.py b/apps/api/src/services/paid_provider_canary_validation.py new file mode 100644 index 000000000..b87607a60 --- /dev/null +++ b/apps/api/src/services/paid_provider_canary_validation.py @@ -0,0 +1,1137 @@ +"""Bounded live canary for Claude and Gemini paid SRE providers. + +The canary uses one committed, non-sensitive synthetic incident. Callers cannot +submit prompts or runtime evidence. Provider response text is parsed in memory +and discarded; only contract checks, cost, latency and durable receipt IDs are +returned or written to the operation ledger. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import re +from collections.abc import Awaitable, Callable +from typing import Any +from uuid import uuid4 + +from src.core.config import settings +from src.models.ai import OpenClawDecision, SuggestedAction +from src.repositories.alert_operation_log_repository import ( + AlertOperationLogRepository, + get_alert_operation_log_repository, +) +from src.services.ai_providers.claude import ClaudeProvider +from src.services.ai_providers.gemini import GeminiProvider +from src.services.ai_providers.interfaces import AIResult +from src.services.ai_providers.ollama import ( + OllamaGcpBProvider, + OllamaLocalProvider, + OllamaProvider, +) +from src.services.cloud_transport_receipt import ( + issue_cloud_transport_receipt, + resolve_gcp_transport_endpoint, +) +from src.services.paid_provider_canary_authorization import ( + paid_provider_canary_bucket, + paid_provider_canary_run_selected, +) + +_SAFE_RUN_REF = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") +_SAFE_TRACE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,191}$") +_WORK_ITEM_ID = "AIA-SRE-013" +_ACTIVATION_LEASE_SECONDS = 300 +_PAID_CANARY_MAX_OUTPUT_TOKENS = 512 +_PAID_CANARY_MAX_COST_USD = 0.25 +_PAID_CANARY_MAX_COST_MICRO_USD = 250_000 +_SUCCESSFUL_RECEIPT_STATUSES = {"succeeded", "succeeded_usage_estimated"} +_SAFE_PROVIDER_ERROR_CODES = { + "authentication_rejected", + "claude_accounting_finalize_failed", + "claude_canary_not_selected", + "claude_disable_state_unavailable", + "claude_disabled_by_environment", + "claude_disabled_by_runtime_control", + "claude_execution_mode_invalid", + "claude_not_configured", + "claude_provider_refusal", + "claude_response_missing_content", + "claude_shadow_metadata_only", + "fixture_provider_unavailable", + "gemini_accounting_finalize_failed", + "gemini_canary_not_selected", + "gemini_disable_state_unavailable", + "gemini_disabled_by_environment", + "gemini_disabled_by_runtime_control", + "gemini_execution_mode_invalid", + "gemini_shadow_metadata_only", + "provider_canary_timeout", + "provider_rate_limited", + "provider_refusal", + "provider_timeout", + "provider_transport_error", +} +_SAFE_COST_GUARD_REASONS = { + "cost_guard_unavailable", + "daily_cost_limit", + "daily_request_limit", + "daily_token_limit", + "failure_cooldown", + "idempotent_replay", + "pricing_policy_missing", + "rpm_limit", + "run_cost_cap_context_invalid", + "run_cost_cap_invalid", + "run_cost_cap_mismatch", + "run_cost_cap_missing", + "run_cost_limit", + "total_cost_limit", +} +_PROMPT = """You are an SRE decision critic. Evaluate this synthetic, sanitized +K3s observation only. The deployment awoooi-api in namespace awoooi-prod has +desired_replicas=3, ready_replicas=3, available_replicas=3, alert_firing=false, +recent_change=false, error_rate_percent=0.1 and p99_ms=120. No secret, customer +data, log body, credential, host address or production payload is present. +Return one JSON object matching the supplied decision contract. The safe and +expected decision is NO_ACTION for target_resource awoooi-api, namespace +awoooi-prod, with an empty kubectl_command, risk_level low, data_impact NONE, +and confidence at least 0.80. Do not invent an incident or remediation. +Return exactly these JSON keys: suggested_action, target_resource, namespace, +kubectl_command, risk_level, blast_radius, reasoning, confidence. blast_radius +must contain affected_pods, estimated_downtime, related_services and +data_impact.""" + + +def select_claude_canary_run_id(run_ref: str, percent: int) -> tuple[str, int]: + """Select one deterministic run bucket without widening the canary percent.""" + + clean_ref = str(run_ref or "").strip() + if not _SAFE_RUN_REF.fullmatch(clean_ref): + raise ValueError("invalid_paid_provider_canary_run_ref") + bounded_percent = max(0, min(100, int(percent))) + if bounded_percent <= 0: + raise ValueError("claude_canary_percent_must_be_positive") + for suffix in range(1000): + run_id = f"{_WORK_ITEM_ID}-{clean_ref}-{suffix}" + bucket = int(hashlib.sha256(run_id.encode("utf-8")).hexdigest()[:8], 16) % 100 + if bucket < bounded_percent: + return run_id, bucket + raise ValueError("claude_canary_bucket_selection_failed") + + +def _score_result( + result: Any, + *, + provider_label: str | None = None, +) -> dict[str, Any]: + checks = { + "provider_call_succeeded": bool(result.success), + "decision_json_valid": False, + "expected_no_action": False, + "canonical_target_exact": False, + "namespace_exact": False, + "no_command_generated": False, + "no_write_data_impact": False, + "confidence_at_least_0_80": False, + } + if result.success: + try: + payload = json.loads(result.raw_response) + decision = OpenClawDecision.model_validate(payload) + checks.update( + decision_json_valid=True, + expected_no_action=( + decision.suggested_action == SuggestedAction.NO_ACTION + ), + canonical_target_exact=(decision.target_resource == "awoooi-api"), + namespace_exact=(decision.namespace == "awoooi-prod"), + no_command_generated=not str(decision.kubectl_command or "").strip(), + no_write_data_impact=( + str(decision.blast_radius.data_impact.value) + in {"NONE", "READ_ONLY"} + ), + confidence_at_least_0_80=(decision.confidence >= 0.80), + ) + except (TypeError, ValueError, json.JSONDecodeError): + pass + + passed = sum(value is True for value in checks.values()) + receipt_id = (result.audit_metadata or {}).get("generation_receipt_id") + raw_error = str(result.error or "").strip() + if not raw_error: + safe_error_code = None + elif raw_error in _SAFE_PROVIDER_ERROR_CODES: + safe_error_code = raw_error + elif raw_error.startswith("cost_guard_blocked:") and ( + raw_error.removeprefix("cost_guard_blocked:") in _SAFE_COST_GUARD_REASONS + ): + safe_error_code = raw_error + elif re.fullmatch(r"http_[1-5][0-9]{2}", raw_error): + safe_error_code = raw_error + elif re.fullmatch(r"unhandled_[A-Za-z][A-Za-z0-9_]{0,79}", raw_error): + safe_error_code = raw_error + else: + safe_error_code = "provider_error_redacted" + return { + "provider": provider_label or result.provider, + "success": bool(result.success), + "error_code": safe_error_code, + "generation_receipt_id": receipt_id, + "tokens": max(0, int(result.tokens or 0)), + "cost_usd": round(max(0.0, float(result.cost_usd or 0.0)), 8), + "latency_ms": round(max(0.0, float(result.latency_ms or 0.0)), 2), + "contract_checks": checks, + "contract_score_percent": round((passed / len(checks)) * 100), + "raw_prompt_persisted": False, + "raw_response_persisted": False, + "secret_value_read_or_returned": False, + } + + +def _paid_receipt_verified( + result: dict[str, Any], + receipt: dict[str, Any], + run_budget: dict[str, Any], +) -> bool: + """Require independent, successful, exactly correlated accounting truth.""" + + return bool( + receipt.get("found") is True + and receipt.get("provider") == result.get("provider") + and receipt.get("receipt_id") == result.get("generation_receipt_id") + and receipt.get("correlation_verified") is True + and receipt.get("finalized_accounted") is True + and receipt.get("reservation_released") is True + and receipt.get("success") is True + and receipt.get("status") in _SUCCESSFUL_RECEIPT_STATUSES + and int(receipt.get("accounted_tokens") or 0) > 0 + and int(receipt.get("accounted_cost_micro_usd") or 0) > 0 + and receipt.get("run_budget_required") is True + and receipt.get("run_budget_id") == run_budget.get("run_budget_id") + and int(receipt.get("run_budget_max_micro_usd") or 0) + == int(run_budget.get("max_cost_micro_usd") or 0) + ) + + +async def _call_provider_safely( + provider: Any, + *, + context: dict[str, Any], + timeout_seconds: float, +) -> AIResult: + try: + return await asyncio.wait_for( + provider.analyze(_PROMPT, context=context), + timeout=timeout_seconds, + ) + except TimeoutError: + return AIResult( + raw_response="", + success=False, + provider=str(getattr(provider, "name", "unknown"))[:40], + error="provider_canary_timeout", + ) + except Exception as exc: + return AIResult( + raw_response="", + success=False, + provider=str(getattr(provider, "name", "unknown"))[:40], + error=f"unhandled_{type(exc).__name__}"[:120], + ) + + +async def _close_provider_safely(provider: Any) -> bool: + try: + await asyncio.wait_for(provider.close(), timeout=5.0) + return True + except BaseException: + # Provider close failure cannot expose response content or widen the + # canary. The verifier still relies on finalized generation receipts. + return False + + +async def _acquire_paid_canary_lease(owner: str) -> bool: + from src.services.ai_control import acquire_paid_provider_canary_lease + + return await acquire_paid_provider_canary_lease( + owner, + ttl_seconds=_ACTIVATION_LEASE_SECONDS, + ) + + +async def _acquire_paid_canary_execution_claim(run_id: str, owner: str) -> bool: + from src.services.ai_control import ( + acquire_paid_provider_canary_execution_claim, + ) + + return await acquire_paid_provider_canary_execution_claim( + run_id, + owner, + ttl_seconds=120, + ) + + +async def _release_paid_canary_execution_claim(run_id: str, owner: str) -> bool: + from src.services.ai_control import ( + release_paid_provider_canary_execution_claim, + ) + + return await release_paid_provider_canary_execution_claim(run_id, owner) + + +async def _require_paid_canary_authorization( + authorization_ref: str, +) -> dict[str, Any]: + from src.services.paid_provider_canary_authorization import ( + require_paid_provider_canary_authorization, + ) + + return await require_paid_provider_canary_authorization(authorization_ref) + + +async def _rollback_paid_canary(owner: str) -> bool: + from src.services.ai_control import rollback_paid_provider_canary_activation + + return await rollback_paid_provider_canary_activation(owner) + + +async def _paid_canary_control_readback( + owner: str | None = None, + completion_record_id: str | None = None, +) -> dict[str, Any]: + from src.services.ai_control import get_paid_provider_canary_control_readback + + return await get_paid_provider_canary_control_readback( + expected_owner=owner, + expected_completion_record_id=completion_record_id, + ) + + +async def _rollback_paid_canary_verified( + owner: str, +) -> tuple[bool, bool, dict[str, Any]]: + """Retry bounded fail-closed compensation and require atomic readback.""" + + applied = False + readback: dict[str, Any] = { + "disabled_verified": False, + "readback_status": "not_started", + } + for _attempt in range(3): + try: + applied = bool(await _rollback_paid_canary(owner)) or applied + readback = await asyncio.wait_for( + _paid_canary_control_readback(owner), + timeout=5.0, + ) + readback["readback_status"] = "verified" + if readback.get("disabled_verified") is True: + return applied, True, readback + except BaseException as exc: + readback = { + "disabled_verified": False, + "readback_status": "unavailable", + "error_code": f"rollback_{type(exc).__name__}"[:120], + } + return applied, False, readback + + +async def _shielded_rollback_paid_canary_verified( + owner: str, +) -> tuple[bool, bool, dict[str, Any]]: + """Let compensation finish even while the caller is being cancelled.""" + + task = asyncio.create_task(_rollback_paid_canary_verified(owner)) + try: + return await asyncio.shield(task) + except BaseException: + return await task + + +async def _generation_receipt_readback( + *, + provider: str, + receipt_id: str, + trace_id: str, + run_id: str, +) -> dict[str, Any]: + try: + from src.services.ai_rate_limiter import get_ai_rate_limiter + + return await asyncio.wait_for( + get_ai_rate_limiter().get_generation_receipt_readback( + provider, + receipt_id, + trace_id=trace_id, + run_id=run_id, + work_item_id=_WORK_ITEM_ID, + ), + timeout=5.0, + ) + except Exception as exc: + return { + "provider": provider, + "receipt_id": receipt_id, + "found": False, + "correlation_verified": False, + "finalized_accounted": False, + "reservation_released": False, + "error_code": f"receipt_readback_{type(exc).__name__}"[:120], + } + + +async def _run_budget_readback(*, run_id: str) -> dict[str, Any]: + try: + from src.services.ai_rate_limiter import get_ai_rate_limiter + + return await asyncio.wait_for( + get_ai_rate_limiter().get_paid_run_budget_readback( + run_id=run_id, + work_item_id=_WORK_ITEM_ID, + expected_max_cost_micro_usd=_PAID_CANARY_MAX_COST_MICRO_USD, + ), + timeout=5.0, + ) + except Exception as exc: + return { + "found": False, + "correlation_verified": False, + "cap_verified": False, + "terminal_verified": False, + "error_code": f"run_budget_readback_{type(exc).__name__}"[:120], + } + + +async def run_paid_provider_canary_validation( + *, + run_ref: str, + operator_id: str, + authorization_ref: str, + ollama_gcp_a_provider: Any | None = None, + ollama_gcp_b_provider: Any | None = None, + ollama_local_provider: Any | None = None, + claude_provider: Any | None = None, + gemini_provider: Any | None = None, + operation_repository: AlertOperationLogRepository | None = None, + cloud_transport_receipt_issuer: Callable[..., Awaitable[dict[str, Any]]] + | None = None, +) -> dict[str, Any]: + """Run one five-lane comparison and a bounded paid-provider canary.""" + + if str(settings.CLAUDE_EXECUTION_MODE).strip().lower() != "canary": + raise ValueError("claude_execution_mode_must_be_canary") + if str(settings.GEMINI_EXECUTION_MODE).strip().lower() != "canary": + raise ValueError("gemini_execution_mode_must_be_canary") + clean_authorization_ref = str(authorization_ref or "").strip() + if not _SAFE_RUN_REF.fullmatch(clean_authorization_ref): + raise ValueError("invalid_paid_provider_canary_authorization_ref") + ollama_model = str(settings.ALERT_OLLAMA_MODEL or "").strip() + if not ollama_model: + raise ValueError("paid_provider_canary_alert_ollama_model_missing") + # Paid-provider routing is a critical control change. Verify the exact, + # unexpired and unused durable Gate 5 receipt before recording a start row, + # constructing provider clients, touching runtime control or making any + # network call. The verifier deliberately returns no operator identity or + # raw authorization payload. + authorization_receipt = await _require_paid_canary_authorization( + clean_authorization_ref + ) + clean_run_ref = str(run_ref or "").strip() + if not _SAFE_RUN_REF.fullmatch(clean_run_ref): + raise ValueError("invalid_paid_provider_canary_run_ref") + # The executable Gate 5 row is the controlled-apply run. A caller-supplied + # source label must never mint a second run identity after approval. + run_id = clean_authorization_ref + canary_percent = min( + settings.CLAUDE_CANARY_PERCENT, + settings.GEMINI_CANARY_PERCENT, + ) + canary_bucket = paid_provider_canary_bucket(run_id) + trace_id = str(authorization_receipt.get("trace_id") or "").strip() + if ( + authorization_receipt.get("authorization_ref") != run_id + or authorization_receipt.get("run_id") != run_id + or authorization_receipt.get("work_item_id") != _WORK_ITEM_ID + or not paid_provider_canary_run_selected(run_id) + or canary_bucket >= canary_percent + or not _SAFE_TRACE_ID.fullmatch(trace_id) + ): + raise RuntimeError("paid_canary_authorization_same_run_binding_failed") + activation_owner = f"{run_id}.attempt-{uuid4().hex[:12]}" + execution_claim_owner = f"{run_id}.claim-{uuid4().hex[:12]}" + execution_claim_acquired = await _acquire_paid_canary_execution_claim( + run_id, + execution_claim_owner, + ) + if not execution_claim_acquired: + raise RuntimeError("paid_canary_authorization_execution_claim_held") + execution_claim_released = False + # The sanitizer has no authority to acknowledge a public HTTP transport. + # A separate source/network preflight writes and reads back one short-lived + # receipt per exact GCP endpoint before either endpoint can receive the + # synthetic prompt. Tests inject this seam; production uses the real + # /api/tags probe plus Redis durability check. + receipt_issuer = cloud_transport_receipt_issuer or issue_cloud_transport_receipt + gcp_transport_providers = ("ollama_gcp_a", "ollama_gcp_b") + repository = operation_repository or get_alert_operation_log_repository() + try: + raw_gcp_transport_results = await asyncio.gather( + *( + receipt_issuer( + trace_id=trace_id, + run_id=run_id, + work_item_id=_WORK_ITEM_ID, + provider=provider, + endpoint_url=resolve_gcp_transport_endpoint(provider), + ) + for provider in gcp_transport_providers + ), + return_exceptions=True, + ) + gcp_transport_receipts: list[dict[str, Any]] = [] + gcp_transport_preflight: list[dict[str, Any]] = [] + for provider, result in zip( + gcp_transport_providers, + raw_gcp_transport_results, + strict=True, + ): + if isinstance(result, dict) and result.get("provider") == provider: + gcp_transport_receipts.append(result) + gcp_transport_preflight.append( + { + "provider": provider, + "status": "verified", + "receipt_id": result.get("receipt_id"), + } + ) + else: + gcp_transport_preflight.append( + { + "provider": provider, + "status": "blocked_no_transport_receipt", + "error_code": ( + f"preflight_{type(result).__name__}" + if isinstance(result, BaseException) + else "preflight_receipt_invalid" + ), + } + ) + context = { + "data_classification": "sanitized", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "synthetic_validation": True, + "synthetic_prompt_sha256": hashlib.sha256( + _PROMPT.encode("utf-8") + ).hexdigest(), + "cloud_transport_receipts": gcp_transport_receipts, + "cloud_transport_boundary": "public_http_sanitized_candidate_only", + "task_type": "diagnose", + "allow_gcp_heavy_model": True, + "ollama_model": ollama_model, + "infrastructure_remediation_write_allowed": False, + "paid_canary_max_output_tokens": _PAID_CANARY_MAX_OUTPUT_TOKENS, + "paid_canary_max_cost_usd": _PAID_CANARY_MAX_COST_USD, + } + started = await repository.append( + "EXECUTION_STARTED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="claude_gemini_sanitized_sre_canary"[:200], + success=None, + context={ + "schema_version": "paid_provider_canary_start_v1", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "providers": [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ], + "authorization_ref": clean_authorization_ref, + "legacy_run_ref": clean_run_ref, + "authorization_run_is_execution_run": True, + "authorization_verified_at": authorization_receipt["checked_at"], + "authorization_expires_at": authorization_receipt["expires_at"], + "data_classification": "sanitized", + "cloud_transport_receipt_ids": [ + receipt["receipt_id"] for receipt in gcp_transport_receipts + ], + "cloud_transport_receipt_readback_verified": ( + len(gcp_transport_receipts) == len(gcp_transport_providers) + ), + "cloud_transport_preflight": gcp_transport_preflight, + "transport_security_status": "degraded_public_http", + "raw_prompt_persisted": False, + "raw_response_persisted": False, + }, + project_id="awoooi", + ) + if started is None: + raise RuntimeError("paid_provider_canary_start_receipt_not_recorded") + finally: + execution_claim_released = await _release_paid_canary_execution_claim( + run_id, + execution_claim_owner, + ) + + providers = [ + ("ollama_gcp_a", ollama_gcp_a_provider or OllamaProvider(), 120.0), + ("ollama_gcp_b", ollama_gcp_b_provider or OllamaGcpBProvider(), 120.0), + ("ollama_local", ollama_local_provider or OllamaLocalProvider(), 120.0), + ("claude", claude_provider or ClaudeProvider(), 45.0), + ("gemini", gemini_provider or GeminiProvider(), 45.0), + ] + scored_results: list[dict[str, Any]] = [] + provider_close_results: list[bool] = [] + providers_closed = False + activation_attempted = False + terminal_success = False + rollback_applied = False + rollback_verified = False + rollback_readback: dict[str, Any] = { + "disabled_verified": False, + "readback_status": "not_started", + } + safe_receipt: dict[str, Any] | None = None + try: + # Baselines run while paid routes remain fail closed. This minimizes + # the duration of the critical paid-provider control-state change. + for route_lane, provider, timeout_seconds in providers[:3]: + result = await _call_provider_safely( + provider, + context=context, + timeout_seconds=timeout_seconds, + ) + scored_results.append(_score_result(result, provider_label=route_lane)) + try: + previous_paid_state = await asyncio.wait_for( + _paid_canary_control_readback(), + timeout=5.0, + ) + except Exception as exc: + raise RuntimeError("paid_provider_canary_control_readback_failed") from exc + if previous_paid_state.get("disabled_verified") is not True: + raise RuntimeError("paid_provider_canary_initial_state_not_disabled") + + # Set this before the Redis call because a connection loss can make a + # successfully applied lease look like a failed response. Compensation + # must therefore run even when acquire returns false or raises. + activation_attempted = True + if not await _acquire_paid_canary_lease(activation_owner): + raise RuntimeError("paid_provider_canary_activation_lease_failed") + try: + leased_readback = await asyncio.wait_for( + _paid_canary_control_readback(activation_owner), + timeout=5.0, + ) + except Exception as exc: + raise RuntimeError("paid_provider_canary_enable_readback_failed") from exc + if leased_readback.get("leased_enabled") is not True: + raise RuntimeError("paid_provider_canary_enable_not_verified") + control_change = await repository.append( + "CHANGE_APPLIED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="paid_provider_canary_atomic_enable"[:200], + success=True, + context={ + "schema_version": "paid_provider_canary_control_change_v1", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "authorization_ref": clean_authorization_ref, + "authorization_run_is_execution_run": True, + "providers": ["claude", "gemini"], + "execution_mode": "canary", + "canary_percent": min( + settings.CLAUDE_CANARY_PERCENT, + settings.GEMINI_CANARY_PERCENT, + ), + "activation_kind": "run_owned_ttl_lease", + "activation_ttl_seconds": _ACTIVATION_LEASE_SECONDS, + "persistent_gate_enabled": False, + "lease_readback_verified": True, + "production_execution_mode_enabled": False, + }, + project_id="awoooi", + ) + if control_change is None: + raise RuntimeError("paid_provider_canary_control_receipt_not_recorded") + + for route_lane, provider, timeout_seconds in providers[3:]: + result = await _call_provider_safely( + provider, + context=context, + timeout_seconds=timeout_seconds, + ) + scored_results.append(_score_result(result, provider_label=route_lane)) + provider_close_results = await asyncio.gather( + *(_close_provider_safely(provider) for _, provider, _ in providers) + ) + providers_closed = True + + paid_results = [ + row for row in scored_results if row["provider"] in {"claude", "gemini"} + ] + paid_receipt_readbacks = [] + for row in paid_results: + receipt_id = str(row.get("generation_receipt_id") or "") + if receipt_id: + paid_receipt_readbacks.append( + await _generation_receipt_readback( + provider=row["provider"], + receipt_id=receipt_id, + trace_id=trace_id, + run_id=run_id, + ) + ) + else: + paid_receipt_readbacks.append( + { + "provider": row["provider"], + "receipt_id": None, + "found": False, + "correlation_verified": False, + "finalized_accounted": False, + "reservation_released": False, + "success": False, + "error_code": "generation_receipt_id_missing", + } + ) + run_budget_readback = await _run_budget_readback(run_id=run_id) + generation_accounted_cost_micro_usd = sum( + max(0, int(receipt.get("accounted_cost_micro_usd") or 0)) + for receipt in paid_receipt_readbacks + ) + run_budget_readback_passed = bool( + run_budget_readback.get("found") is True + and run_budget_readback.get("correlation_verified") is True + and run_budget_readback.get("cap_verified") is True + and run_budget_readback.get("terminal_verified") is True + and int(run_budget_readback.get("max_cost_micro_usd") or 0) + == _PAID_CANARY_MAX_COST_MICRO_USD + and int(run_budget_readback.get("reserved_cost_micro_usd") or 0) == 0 + and 0 + <= int(run_budget_readback.get("accounted_cost_micro_usd") or -1) + <= _PAID_CANARY_MAX_COST_MICRO_USD + and int(run_budget_readback.get("accounted_cost_micro_usd") or -1) + == generation_accounted_cost_micro_usd + and int(run_budget_readback.get("reservation_count") or 0) == 2 + and int(run_budget_readback.get("finalized_count") or 0) == 2 + ) + receipt_readback_passed = bool( + len(paid_receipt_readbacks) == 2 + and run_budget_readback_passed + and all( + _paid_receipt_verified(result, receipt, run_budget_readback) + for result, receipt in zip( + paid_results, + paid_receipt_readbacks, + strict=True, + ) + ) + ) + paid_activation_verifier_passed = bool( + len(paid_results) == 2 + and all( + row["success"] + and row["contract_score_percent"] == 100 + and bool(row["generation_receipt_id"]) + for row in paid_results + ) + and all(provider_close_results) + and receipt_readback_passed + ) + if ( + paid_results[0]["contract_score_percent"] + == paid_results[1]["contract_score_percent"] + ): + comparison = "initial_canary_tie_no_overall_winner" + else: + comparison = ( + max( + paid_results, + key=lambda row: row["contract_score_percent"], + )["provider"] + + "_higher_contract_score_in_initial_canary" + ) + + baseline_results = scored_results[:3] + five_lane_comparison_ready = bool( + len(scored_results) == 5 + and all( + row["success"] and row["contract_score_percent"] == 100 + for row in scored_results + ) + ) + paid_provider_contract_verifier_passed = paid_activation_verifier_passed + promotion_eligible = bool( + paid_provider_contract_verifier_passed + and five_lane_comparison_ready + and run_budget_readback_passed + ) + ranking = [ + row["provider"] + for row in sorted( + scored_results, + key=lambda row: ( + -row["contract_score_percent"], + row["latency_ms"], + row["provider"], + ), + ) + ] + + comparison_ready = None + rollback_change = None + activation_failure = None + final_control_readback: dict[str, Any] = {} + if promotion_eligible: + comparison_ready = await repository.append( + "PRE_FLIGHT_PASSED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="paid_provider_canary_comparison_verified"[:200], + success=True, + context={ + "schema_version": "paid_provider_canary_comparison_verified_v1", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "authorization_ref": clean_authorization_ref, + "generation_receipt_ids": [ + row["generation_receipt_id"] for row in paid_results + ], + "receipt_readback_passed": True, + "run_budget_id": run_budget_readback.get("run_budget_id"), + "run_budget_accounted_cost_micro_usd": ( + run_budget_readback.get("accounted_cost_micro_usd") + ), + "run_budget_readback_passed": True, + "contract_verifier_passed": True, + "authorization_scope": "one_sanitized_five_lane_comparison", + "persistent_promotion_applied": False, + }, + project_id="awoooi", + ) + paid_activation_verifier_passed = comparison_ready is not None + + if not promotion_eligible: + # A partial five-lane comparison is not successful. Both paid routes + # still return to the exact disabled state. + paid_activation_verifier_passed = False + + # Gate 5 authorizes exactly one synthetic comparison. Success never + # promotes future real-alert traffic; the same run must atomically + # disable both paid routes and independently read that state back. + ( + rollback_applied, + rollback_verified, + rollback_readback, + ) = await _shielded_rollback_paid_canary_verified(activation_owner) + final_control_readback = rollback_readback + bounded_terminal_verified = bool( + paid_activation_verifier_passed and rollback_verified + ) + if bounded_terminal_verified: + rollback_change = await repository.append( + "CHANGE_APPLIED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="paid_provider_canary_bounded_rollback"[:200], + success=True, + context={ + "schema_version": "paid_provider_canary_bounded_rollback_v1", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "authorization_ref": clean_authorization_ref, + "paired_paid_routes_disabled": True, + "persistent_gate_enabled": False, + "rollback_readback": rollback_readback, + }, + project_id="awoooi", + ) + bounded_terminal_verified = rollback_change is not None + if not bounded_terminal_verified: + activation_failure = await repository.append( + "GUARDRAIL_BLOCKED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="paid_provider_canary_bounded_terminal_failed"[:200], + success=False, + error_message=( + "paid_provider_canary_rollback_not_verified" + if paid_activation_verifier_passed + else "paid_provider_canary_comparison_not_verified" + ), + context={ + "schema_version": "paid_provider_canary_terminal_failed_v1", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "rollback_applied": rollback_applied, + "rollback_verified": rollback_verified, + }, + project_id="awoooi", + ) + + safe_receipt = { + "schema_version": "paid_provider_sre_canary_receipt_v4", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "source_receipt": { + "kind": "committed_sanitized_synthetic_k3s_observation", + "data_classification": "sanitized", + "ollama_model": ollama_model, + "gcp_transport": { + "status": "degraded_public_http", + "sanitized_only": True, + "receipt_ids": [ + receipt["receipt_id"] for receipt in gcp_transport_receipts + ], + "preflight": gcp_transport_preflight, + "same_run_bound": bool( + len(gcp_transport_receipts) == len(gcp_transport_providers) + and all( + receipt.get("run_id") == run_id + and receipt.get("trace_id") == trace_id + and receipt.get("work_item_id") == _WORK_ITEM_ID + for receipt in gcp_transport_receipts + ) + ), + "durable_readback_verified": bool( + len(gcp_transport_receipts) == len(gcp_transport_providers) + and all( + receipt.get("durable_write_ack") is True + and receipt.get("verifier_status") == "verified" + for receipt in gcp_transport_receipts + ) + ), + "encrypted_transport_claimed": False, + }, + }, + "normalized_asset_identity": { + "canonical_asset_id": "service:awoooi-api", + "typed_domain": "kubernetes_workload", + }, + "policy_decision": { + "risk": "critical_paid_provider_canary", + "authorization_ref": clean_authorization_ref, + "durable_authorization_receipt": authorization_receipt, + "authorization_execution_claim": { + "acquired": execution_claim_acquired, + "released_after_durable_start": execution_claim_released, + "same_run_bound": True, + }, + "infrastructure_remediation_write_allowed": False, + "provider_order_changed": False, + "claude_canary_bucket": canary_bucket, + "gemini_canary_bucket": canary_bucket, + "production_execution_mode_enabled": False, + "maximum_paid_cost_usd": _PAID_CANARY_MAX_COST_USD, + "maximum_output_tokens_per_paid_provider": ( + _PAID_CANARY_MAX_OUTPUT_TOKENS + ), + "authorization_scope": "one_sanitized_five_lane_comparison", + "persistent_promotion_authorized": False, + }, + "source_of_truth_diff": { + "previous_paid_control_state": previous_paid_state, + "leased_enable_readback": leased_readback, + "final_control_readback": final_control_readback, + "target": "one_comparison_then_paired_disabled_readback", + "control_change_record_id": getattr(control_change, "id", None), + }, + "bounded_execution": { + "provider_order": [row[0] for row in providers], + "provider_calls_attempted": len(providers), + "provider_control_writes_attempted": 2, + "workload_or_remediation_writes_performed": 0, + "same_fixture_forced_per_lane_no_route_change": True, + "all_provider_clients_closed": all(provider_close_results), + "results": scored_results, + }, + "independent_verifier": { + "passed": bool( + bounded_terminal_verified and five_lane_comparison_ready + ), + "paid_activation_verifier_passed": (bounded_terminal_verified), + "bounded_activation_and_rollback_verified": (bounded_terminal_verified), + "paid_provider_contract_verifier_passed": ( + paid_provider_contract_verifier_passed + ), + "five_lane_verifier_passed": bool( + bounded_terminal_verified and five_lane_comparison_ready + ), + "paid_provider_comparison": comparison, + "generation_receipt_readback_passed": receipt_readback_passed, + "generation_receipt_readbacks": paid_receipt_readbacks, + "run_budget_readback_passed": run_budget_readback_passed, + "run_budget_readback": run_budget_readback, + "generation_accounted_cost_micro_usd": ( + generation_accounted_cost_micro_usd + ), + "five_lane_comparison_ready": five_lane_comparison_ready, + "ollama_baseline_available_count": sum( + row["success"] for row in baseline_results + ), + "contract_ranking": ranking, + "single_scenario_can_prove_overall_model_superiority": False, + }, + "rollback_or_no_write_terminal": { + "rollback": "run-owned CAS disables both paid gates and removes the active receipt; deployment rollback sets both modes to shadow", + "rollback_applied": rollback_applied, + "rollback_verified": rollback_verified, + "rollback_readback": rollback_readback, + "paid_canary_state": ( + "validated_comparison_rollback_verified" + if bounded_terminal_verified + else ( + "failed_rollback_verified" + if rollback_verified + else "rollback_unverified" + ) + ), + "persistent_paid_routes_enabled": False, + "infrastructure_remediation_terminal": "no_write", + }, + "learning_writeback": { + "status": "pending_provider_scorecard_km_rag_mcp_playbook_ack", + }, + } + completed = await repository.append( + "EXECUTION_COMPLETED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="claude_gemini_sanitized_sre_canary"[:200], + success=bool(bounded_terminal_verified and five_lane_comparison_ready), + error_message=( + None + if bounded_terminal_verified and five_lane_comparison_ready + else ( + "five_lane_comparison_partial" + if paid_activation_verifier_passed and rollback_verified + else "provider_canary_or_rollback_verifier_failed" + ) + ), + context=safe_receipt, + project_id="awoooi", + ) + if completed is None: + bounded_terminal_verified = False + safe_receipt["independent_verifier"]["passed"] = False + safe_receipt["rollback_or_no_write_terminal"].update( + rollback_applied=rollback_applied, + rollback_verified=rollback_verified, + rollback_readback=rollback_readback, + paid_canary_state=( + "completion_writeback_missing_fail_closed" + if rollback_verified + else "rollback_unverified" + ), + ) + + terminal_success = bool( + bounded_terminal_verified + and completed is not None + and rollback_change is not None + ) + safe_receipt["source_of_truth_diff"][ + "final_control_readback" + ] = final_control_readback + safe_receipt["rollback_or_no_write_terminal"].update( + rollback_applied=rollback_applied, + rollback_verified=rollback_verified, + rollback_readback=rollback_readback, + paid_canary_state=( + "validated_comparison_rollback_verified" + if terminal_success + else ( + "failed_rollback_verified" + if rollback_verified + else "rollback_unverified" + ) + ), + ) + safe_receipt["independent_verifier"][ + "paid_activation_verifier_passed" + ] = terminal_success + safe_receipt["independent_verifier"]["five_lane_verifier_passed"] = bool( + terminal_success and five_lane_comparison_ready + ) + safe_receipt["independent_verifier"]["passed"] = bool( + terminal_success and five_lane_comparison_ready + ) + safe_receipt["durable_operation_writeback"] = { + "start_recorded": started is not None, + "lease_change_recorded": control_change is not None, + "lease_change_record_id": getattr(control_change, "id", None), + "comparison_ready_recorded": comparison_ready is not None, + "comparison_ready_record_id": getattr(comparison_ready, "id", None), + "rollback_change_recorded": rollback_change is not None, + "rollback_change_record_id": getattr(rollback_change, "id", None), + "activation_failure_recorded": activation_failure is not None, + "activation_failure_record_id": getattr(activation_failure, "id", None), + "completion_recorded": completed is not None, + "completion_record_id": getattr(completed, "id", None), + } + return safe_receipt + except BaseException as exc: + if activation_attempted and not terminal_success: + ( + rollback_applied, + rollback_verified, + rollback_readback, + ) = await _shielded_rollback_paid_canary_verified(activation_owner) + failure_write = asyncio.create_task( + repository.append( + "EXECUTION_COMPLETED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="claude_gemini_sanitized_sre_canary"[:200], + success=False, + error_message="paid_provider_canary_exception_fail_closed", + context={ + "schema_version": "paid_provider_canary_exception_v1", + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": _WORK_ITEM_ID, + "error_code": type(exc).__name__[:80], + "rollback_applied": rollback_applied, + "rollback_verified": rollback_verified, + "rollback_readback": rollback_readback, + "raw_prompt_persisted": False, + "raw_response_persisted": False, + }, + project_id="awoooi", + ) + ) + try: + await asyncio.shield(failure_write) + except BaseException: + pass + raise + finally: + if not providers_closed: + await asyncio.gather( + *(_close_provider_safely(provider) for _, provider, _ in providers) + ) + if activation_attempted and not terminal_success and not rollback_verified: + ( + rollback_applied, + rollback_verified, + rollback_readback, + ) = await _shielded_rollback_paid_canary_verified(activation_owner) + if safe_receipt is not None: + safe_receipt["rollback_or_no_write_terminal"].update( + rollback_applied=rollback_applied, + rollback_verified=rollback_verified, + rollback_readback=rollback_readback, + paid_canary_state=( + "failed_rollback_verified" + if rollback_verified + else "rollback_unverified" + ), + ) diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index 35c01bd79..edd3b8957 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -81,6 +81,9 @@ from src.services.operator_summary_cache import ( get_cached_operator_summary_async, store_operator_summary_async, ) +from src.services.paid_provider_canary_gate5_run import ( + paid_provider_canary_approval_contract_errors, +) from src.services.run_state_machine import transition from src.services.snapshot_paths import resolve_repo_root @@ -3572,7 +3575,10 @@ async def get_ai_route_status( try: route = await asyncio.wait_for( - get_ollama_failover_manager().select_provider(task_type=workload), + get_ollama_failover_manager().select_provider( + task_type=workload, + emit_side_effects=False, + ), timeout=_AI_ROUTE_STATUS_SELECT_TIMEOUT_SECONDS, ) except TimeoutError: @@ -8864,6 +8870,17 @@ async def decide_approval( status_code=status.HTTP_409_CONFLICT, detail=f"run {run_id!r} 目前狀態為 {run.state!r}，無法審核（需為 waiting_approval）", ) + paid_canary_contract_errors = ( + paid_provider_canary_approval_contract_errors(run) + ) + if paid_canary_contract_errors: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "paid_provider_canary_gate5_contract_invalid:" + + ",".join(paid_canary_contract_errors) + ), + ) is_projection_only_gate5 = run.trigger_type == _ADR100_GATE5_PROJECTION_TRIGGER approval_token_jti: str | None = None diff --git a/apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py b/apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py new file mode 100644 index 000000000..5a705703c --- /dev/null +++ b/apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py @@ -0,0 +1,280 @@ +"""Portfolio-wide infrastructure inventory reconciliation. + +The committed snapshot deliberately separates an imported inventory claim from +current source truth and production runtime truth. Group defaults keep the +snapshot reviewable; this loader expands every member into a complete asset +record and validates the fixed typed-executor contract. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from src.services.snapshot_paths import default_operations_dir + +_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) +_SNAPSHOT_FILE = "portfolio-infrastructure-asset-reconciliation.snapshot.json" +_SCHEMA_VERSION = "portfolio_infrastructure_asset_reconciliation_v1" +_PROGRAM_ID = "AIA-SRE-P0-20260715" +_ALLOWED_DOMAINS = { + "kubernetes_workload", + "host_systemd", + "docker_container", + "windows_vmware", + "database", + "backup_restore", + "control_plane_recovery", + "ai_provider", + "control_plane_service", + "unknown", +} +_EXPECTED_EXECUTORS = { + "kubernetes_workload": "kubernetes_controlled_executor", + "host_systemd": "host_ansible_executor", + "docker_container": "host_ansible_executor_with_exact_container_playbook", + "windows_vmware": "Agent99", + "database": "db_bounded_executor", + "backup_restore": "backup_restore_break_glass", + "control_plane_recovery": "Agent99", + "ai_provider": "deterministic_provider_router", + "control_plane_service": "single_control_plane_executor", + "unknown": None, +} +_REQUIRED_ASSET_FIELDS = { + "canonical_id", + "inventory_labels", + "source_rows", + "category", + "product_id", + "project_id", + "site_id", + "owner_lane", + "runtime_identity", + "source_truth_state", + "live_truth_state", + "reconciliation_state", + "findings", + "source_refs", + "domain_router", + "executor", + "verifier", + "monitoring", + "alerting", + "telegram_destination", + "backup_restore", + "learning_targets", + "priority", + "dependencies", + "next_action", +} +_LEARNING_TARGETS = ["KM", "RAG", "MCP", "PlayBook"] +_PUBLIC_RUNTIME_IDENTITY_ALIASES = { + "ai-provider:ollama_gcp_a": "provider:GCP-A/Ollama", + "ai-provider:ollama_gcp_b": "provider:GCP-B/Ollama", +} + + +def load_portfolio_infrastructure_asset_reconciliation( + operations_dir: Path | None = None, +) -> dict[str, Any]: + """Load, expand, and validate the portfolio inventory reconciliation.""" + + directory = operations_dir or _DEFAULT_OPERATIONS_DIR + path = directory / _SNAPSHOT_FILE + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError(f"{path}: expected JSON object") + if payload.get("schema_version") != _SCHEMA_VERSION: + raise ValueError(f"{path}: unexpected schema_version") + if payload.get("program_id") != _PROGRAM_ID: + raise ValueError(f"{path}: unexpected program_id") + + source = _dict(payload.get("source_inventory"), "source_inventory", path) + if source.get("live_truth_accepted") is not False: + raise ValueError(f"{path}: imported health claims cannot be live truth") + if source.get("runtime_probe_performed") is not False: + raise ValueError(f"{path}: snapshot must not claim an unrecorded live probe") + category_counts = _dict( + source.get("source_category_counts"), "source_category_counts", path + ) + if sum(category_counts.values()) != source.get("inventory_item_row_count"): + raise ValueError(f"{path}: source category counts do not cover imported rows") + + governance = _dict(payload.get("governance"), "governance", path) + if governance.get("github_frozen") is not True: + raise ValueError(f"{path}: GitHub freeze must remain active") + if governance.get("cross_domain_fallback_allowed") is not False: + raise ValueError(f"{path}: cross-domain fallback must remain forbidden") + if governance.get("unknown_asset_terminal") != "asset_identity_unresolved": + raise ValueError(f"{path}: unknown assets must fail closed") + + assets = _expand_assets(payload, path) + canonical_ids = [str(asset["canonical_id"]) for asset in assets] + if len(canonical_ids) != len(set(canonical_ids)): + raise ValueError(f"{path}: duplicate canonical asset id") + + mapped_rows: list[str] = [] + omission_count = 0 + for asset in assets: + _validate_asset(asset, path) + rows = asset["source_rows"] + mapped_rows.extend(rows) + if asset["reconciliation_state"] == "missing_from_imported_inventory": + omission_count += 1 + if rows: + raise ValueError(f"{path}: omitted asset cannot claim source rows") + if len(mapped_rows) != len(set(mapped_rows)): + raise ValueError(f"{path}: imported source row mapped more than once") + if len(mapped_rows) != source.get("inventory_item_row_count"): + raise ValueError(f"{path}: imported inventory row coverage mismatch") + + work_items = _list(payload.get("reconciliation_work_items"), "work_items", path) + known_work_ids = {str(item.get("id") or "") for item in work_items} + if "" in known_work_ids or len(known_work_ids) != len(work_items): + raise ValueError(f"{path}: work item ids must be unique and non-empty") + expected_order = list(range(1, len(work_items) + 1)) + if [item.get("order") for item in work_items] != expected_order: + raise ValueError(f"{path}: work item order must be contiguous") + priorities = [item.get("priority") for item in work_items] + if any(priority not in {"P0", "P1"} for priority in priorities): + raise ValueError(f"{path}: work item priority must be P0 or P1") + first_p1 = priorities.index("P1") if "P1" in priorities else len(priorities) + if "P0" in priorities[first_p1:]: + raise ValueError(f"{path}: P0 work must precede P1 work") + for item in work_items: + unknown = set(item.get("dependencies") or []) - known_work_ids + if unknown: + raise ValueError(f"{path}: {item['id']} has unknown dependencies") + + rollups = _dict(payload.get("rollups"), "rollups", path) + if rollups.get("imported_item_rows") != len(mapped_rows): + raise ValueError(f"{path}: rollup imported row mismatch") + if rollups.get("canonical_assets") != len(assets): + raise ValueError(f"{path}: rollup canonical asset mismatch") + if rollups.get("inventory_omissions") != omission_count: + raise ValueError(f"{path}: rollup omission mismatch") + conflicts = _list(payload.get("conflict_register"), "conflict_register", path) + if rollups.get("conflicts") != len(conflicts): + raise ValueError(f"{path}: rollup conflict mismatch") + conflict_assets = { + str(canonical_id) + for conflict in conflicts + for canonical_id in (conflict.get("assets") or []) + if canonical_id != "all-imported-assets" + } + unknown_conflict_assets = conflict_assets - set(canonical_ids) + if unknown_conflict_assets: + raise ValueError(f"{path}: conflict register references unknown assets") + if rollups.get("conflict_assets") != len(conflict_assets): + raise ValueError(f"{path}: rollup conflict asset mismatch") + source_reconciled_assets = sum( + asset.get("source_truth_state") == "source_reconciled_runtime_pending" + for asset in assets + ) + if rollups.get("source_reconciled_assets") != source_reconciled_assets: + raise ValueError(f"{path}: rollup source-reconciled asset mismatch") + if rollups.get("reconciliation_work_items") != len(work_items): + raise ValueError(f"{path}: rollup work item mismatch") + if rollups.get("runtime_closed_assets") != 0: + raise ValueError(f"{path}: reconciliation cannot manufacture runtime closure") + + result = dict(payload) + result["assets"] = assets + return result + + +def build_portfolio_infrastructure_public_projection( + payload: dict[str, Any], +) -> dict[str, Any]: + """Return the complete management projection without local paths/raw endpoints. + + The expanded ``assets`` list is the canonical public shape. ``asset_groups`` + is removed because it duplicates the same records and still contains the + imported topology strings before endpoint redaction. + """ + + projection = dict(payload) + projection.pop("asset_groups", None) + source_inventory = dict(projection["source_inventory"]) + source_inventory["source_path"] = ( + "external_inventory:infrastructure_inventory.md" + ) + projection["source_inventory"] = source_inventory + assets: list[dict[str, Any]] = [] + for raw_asset in projection["assets"]: + asset = dict(raw_asset) + public_identity = _PUBLIC_RUNTIME_IDENTITY_ALIASES.get( + str(asset["canonical_id"]) + ) + if public_identity: + asset["runtime_identity"] = public_identity + assets.append(asset) + projection["assets"] = assets + projection["public_projection"] = { + "topology_redacted": True, + "source_path_redacted": True, + "expanded_assets_authoritative": True, + } + return projection + + +def _expand_assets(payload: dict[str, Any], path: Path) -> list[dict[str, Any]]: + groups = _list(payload.get("asset_groups"), "asset_groups", path) + expanded: list[dict[str, Any]] = [] + for group_index, group in enumerate(groups): + if not isinstance(group, dict): + raise ValueError(f"{path}: asset_groups[{group_index}] must be an object") + common = _dict(group.get("common"), "asset_group.common", path) + category = str(group.get("category") or "") + group_id = str(group.get("group_id") or "") + if not category or not group_id: + raise ValueError(f"{path}: every asset group requires id/category") + for member in _list(group.get("members"), "asset_group.members", path): + if not isinstance(member, dict): + raise ValueError(f"{path}: asset member must be an object") + asset = dict(common) + asset.update(member) + asset["category"] = category + asset["group_id"] = group_id + expanded.append(asset) + return expanded + + +def _validate_asset(asset: dict[str, Any], path: Path) -> None: + missing = sorted(_REQUIRED_ASSET_FIELDS - asset.keys()) + if missing: + raise ValueError(f"{path}: {asset.get('canonical_id')} missing {missing}") + domain = str(asset.get("domain_router") or "") + if domain not in _ALLOWED_DOMAINS: + raise ValueError(f"{path}: {asset['canonical_id']} has invalid domain") + if asset.get("executor") != _EXPECTED_EXECUTORS[domain]: + raise ValueError(f"{path}: {asset['canonical_id']} executor/domain mismatch") + if asset.get("learning_targets") != _LEARNING_TARGETS: + raise ValueError(f"{path}: {asset['canonical_id']} learning targets incomplete") + if asset.get("live_truth_state") == "verified_healthy": + raise ValueError(f"{path}: snapshot health cannot become live truth") + if asset.get("priority") not in {"P0", "P1", "P2"}: + raise ValueError(f"{path}: {asset['canonical_id']} priority invalid") + if not isinstance(asset.get("monitoring"), dict): + raise ValueError(f"{path}: {asset['canonical_id']} monitoring required") + if not isinstance(asset.get("alerting"), dict): + raise ValueError(f"{path}: {asset['canonical_id']} alerting required") + if not isinstance(asset.get("backup_restore"), dict): + raise ValueError(f"{path}: {asset['canonical_id']} backup contract required") + if domain == "unknown" and asset.get("executor") is not None: + raise ValueError(f"{path}: unknown asset must not have executor") + + +def _dict(value: Any, label: str, path: Path) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError(f"{path}: {label} must be an object") + return value + + +def _list(value: Any, label: str, path: Path) -> list[Any]: + if not isinstance(value, list): + raise ValueError(f"{path}: {label} must be an array") + return value diff --git a/apps/api/src/services/sre_k3s_controlled_automation_work_items.py b/apps/api/src/services/sre_k3s_controlled_automation_work_items.py index e59ef2dbc..c58ec14c7 100644 --- a/apps/api/src/services/sre_k3s_controlled_automation_work_items.py +++ b/apps/api/src/services/sre_k3s_controlled_automation_work_items.py @@ -18,7 +18,7 @@ _PIPELINE = [ "Canonical Asset Normalize", "Typed Domain Router", "HolmesGPT Investigator", - "Ollama RCA / Claude Fallback / Gemini Critic", + "Ollama RCA / Gemini Critic", "Deterministic Policy", "Single Controlled Executor", "Independent Verifier", @@ -33,6 +33,7 @@ _REQUIRED_DOMAINS = { "database", "backup_restore", "unknown", + "control_plane_recovery", } _VALID_PRIORITIES = {"P0", "P1", "P2"} _VALID_RISKS = {"low", "medium", "high", "critical"} @@ -43,6 +44,14 @@ _VALID_STATUSES = { "policy_active", "runtime_closed", } +_REQUIRED_IMMEDIATE_EXECUTION_ORDER = [ + "AIA-SRE-017", + "AIA-SRE-002", + "AIA-SRE-004", + "AIA-SRE-013", + "AIA-SRE-014", + "AIA-SRE-015", +] _REQUIRED_ITEM_FIELDS = { "id", "order", @@ -99,10 +108,32 @@ def load_sre_k3s_controlled_automation_work_items( provider_hops[2].get("identity") != "host111" ): raise ValueError(f"{path}: third provider hop must be host111 Ollama") - if provider.get("gemini_paid_call_allowed") is not False: - raise ValueError(f"{path}: Gemini paid calls require an explicit cost cap") - if provider.get("claude_paid_call_allowed") is not False: - raise ValueError(f"{path}: Claude paid calls require an explicit cost cap") + paid_call_states = { + provider.get("claude_paid_call_allowed"), + provider.get("gemini_paid_call_allowed"), + } + if paid_call_states not in ({False}, {True}): + raise ValueError(f"{path}: paid provider gates must change together") + if paid_call_states == {True}: + if provider.get("paid_execution_mode") != "canary": + raise ValueError(f"{path}: paid calls require canary execution mode") + percent = provider.get("paid_canary_percent") + if not isinstance(percent, int) or not 1 <= percent <= 5: + raise ValueError(f"{path}: paid canary percent must be between 1 and 5") + authorization_ref = str(provider.get("paid_canary_authorization_ref") or "") + if not authorization_ref.startswith("AIA-SRE-013-user-approved-"): + raise ValueError(f"{path}: paid canary authorization reference missing") + required_gates = set( + provider.get("required_before_paid_provider_enablement") or [] + ) + if not { + "daily_and_monthly_cost_cap", + "per_incident_token_cap", + "rate_limit_and_circuit_breaker", + "rollback_to_ollama_chain", + "production_cost_readback", + }.issubset(required_gates): + raise ValueError(f"{path}: paid calls require explicit cost controls") if provider.get("production_provider_route_switch_allowed") is not False: raise ValueError(f"{path}: production provider switch must remain gated") @@ -168,6 +199,31 @@ def load_sre_k3s_controlled_automation_work_items( f"{sorted(unknown_dependencies)}" ) + immediate_queue = _list(payload, "immediate_execution_queue", path) + queue_ids: list[str] = [] + for position, row in enumerate(immediate_queue, start=1): + if not isinstance(row, dict): + raise ValueError( + f"{path}: immediate_execution_queue[{position - 1}] must be an object" + ) + if row.get("position") != position: + raise ValueError( + f"{path}: immediate_execution_queue position must be contiguous" + ) + work_item_id = str(row.get("work_item_id") or "") + if work_item_id not in known_ids: + raise ValueError( + f"{path}: immediate queue references unknown work item {work_item_id!r}" + ) + for required_field in ("scope", "blocking_reason", "terminal_condition"): + if not str(row.get(required_field) or "").strip(): + raise ValueError( + f"{path}: immediate queue {work_item_id}.{required_field} required" + ) + queue_ids.append(work_item_id) + if queue_ids != _REQUIRED_IMMEDIATE_EXECUTION_ORDER: + raise ValueError(f"{path}: immediate execution safety order changed") + current_p0 = _dict(payload, "current_p0", path) if current_p0.get("id") not in known_ids: raise ValueError(f"{path}: current_p0.id must be a work item") @@ -210,6 +266,7 @@ def build_sre_k3s_program_projection(payload: dict[str, Any]) -> dict[str, Any]: "status": payload["status"], "scope_complete": payload["scope_complete"], "current_p0": payload["current_p0"], + "immediate_execution_queue": payload["immediate_execution_queue"], "rollups": payload["rollups"], "provider_order": payload["provider_policy"]["ordered_route"], "claude_paid_call_allowed": payload["provider_policy"][ diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 59fc1ecf0..92d399ffd 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -8226,6 +8226,26 @@ class TelegramGateway: if probable_cause: text += f"└─ 可能根因：{html.escape(_smart_truncate(probable_cause, 320))}\n" + if alert_category == "alertchain_health": + agent_label = "AwoooP typed alert-chain lifecycle router" + verifier_label = "source-specific delivery metric + AWOOOI E2E receipt pending" + elif alert_category == "flywheel_health": + agent_label = "AwoooP flywheel lifecycle controller" + verifier_label = "flywheel metric + durable lifecycle receipt pending" + else: + agent_label = "AwoooP meta-system lifecycle controller" + verifier_label = "domain-specific independent verifier pending" + + text += ( + "\n🤖 AI / Agent 工作證據\n" + f"├ Agent：{html.escape(agent_label)}\n" + "├ LLM：此通知未附 generation receipt，不宣稱已使用或未使用\n" + "├ 動作：已記錄診斷並建立 canonical lifecycle；未宣稱修復\n" + "├ Executor：none（尚無 bounded execution receipt）\n" + f"├ Verifier：{html.escape(verifier_label)}\n" + f"└ Receipt：{html.escape(approval_id)}\n" + ) + # 2026-04-16 ogt: 移除 flywheel_diag / flywheel_dashboard (3-part ghost button，無 handler) # 鐵律: 寧可沒按鈕，不可有死按鈕 (feedback_no_ghost_buttons.md) silence_nonce = self._security.generate_callback_nonce(approval_id, "silence") diff --git a/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py b/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py index 67e43f2f4..4a28f65f3 100644 --- a/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py +++ b/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py @@ -21,6 +21,7 @@ import redis.asyncio as redis_async from src.core import config as config_module from src.services.ai_rate_limiter import ( + _FINALIZE_GENERATION_LUA, DAILY_COST_MICRO_USD_KEY, DAILY_COST_RESERVED_MICRO_USD_KEY, DAILY_REQ_KEY, @@ -28,6 +29,7 @@ from src.services.ai_rate_limiter import ( DAILY_TOKEN_RESERVED_KEY, GENERATION_RECEIPT_KEY, GENERATION_RUN_IDEMPOTENCY_KEY, + PAID_RUN_BUDGET_KEY, REDIS_KEY_PREFIX, TOTAL_COST_KEY, TOTAL_COST_RESERVED_MICRO_USD_KEY, @@ -295,30 +297,39 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( ) ) assert finalized == [True] * 5 - assert int( - await client.get( - DAILY_TOKEN_RESERVED_KEY.format( - provider=provider, - date="2026-07-14", + assert ( + int( + await client.get( + DAILY_TOKEN_RESERVED_KEY.format( + provider=provider, + date="2026-07-14", + ) ) + or 0 ) - or 0 - ) == 0 - assert int( - await client.get( - DAILY_COST_RESERVED_MICRO_USD_KEY.format( - provider=provider, - date="2026-07-14", + == 0 + ) + assert ( + int( + await client.get( + DAILY_COST_RESERVED_MICRO_USD_KEY.format( + provider=provider, + date="2026-07-14", + ) ) + or 0 ) - or 0 - ) == 0 - assert int( - await client.get( - TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider=provider) + == 0 + ) + assert ( + int( + await client.get( + TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider=provider) + ) + or 0 ) - or 0 - ) == 0 + == 0 + ) await client.flushdb() await client.aclose() @@ -358,6 +369,7 @@ async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( "trace_id": "trace-same-run", "run_id": "run-same-run", "work_item_id": "work-cooldown", + "paid_canary_max_cost_usd": 0.25, } reservations = await asyncio.gather( @@ -375,20 +387,38 @@ async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( allowed = [reservation for reservation in reservations if reservation.allowed] assert len(allowed) == 1 - assert {reservation.reason for reservation in reservations if not reservation.allowed} == { - "idempotent_replay" - } + assert { + reservation.reason for reservation in reservations if not reservation.allowed + } == {"idempotent_replay"} assert len({reservation.receipt_id for reservation in reservations}) == 1 - assert int( - await client.get(DAILY_REQ_KEY.format(provider=provider, date="2026-07-14")) - or 0 - ) == 1 + assert ( + int( + await client.get(DAILY_REQ_KEY.format(provider=provider, date="2026-07-14")) + or 0 + ) + == 1 + ) + pending_budget = await limiter.get_paid_run_budget_readback( + run_id="run-same-run", + work_item_id="work-cooldown", + expected_max_cost_micro_usd=250_000, + ) + assert pending_budget["reservation_count"] == 1 + assert pending_budget["finalized_count"] == 0 assert await limiter.finalize_generation( allowed[0], success=False, error_code="isolated_failure", ) + finalized_budget = await limiter.get_paid_run_budget_readback( + run_id="run-same-run", + work_item_id="work-cooldown", + expected_max_cost_micro_usd=250_000, + ) + assert finalized_budget["reservation_count"] == 1 + assert finalized_budget["finalized_count"] == 1 + assert finalized_budget["reserved_cost_micro_usd"] == 0 cooldown_block = await limiter.reserve_generation( provider, "retry after failure", @@ -398,6 +428,7 @@ async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( "trace_id": "trace-retry", "run_id": "run-retry", "work_item_id": "work-cooldown", + "paid_canary_max_cost_usd": 0.25, }, ) assert cooldown_block.allowed is False @@ -459,15 +490,18 @@ async def test_claude_daily_and_total_cost_caps_include_pending_reservations( assert first.estimated_cost_usd == boundary_usd assert daily_block.allowed is False assert daily_block.reason == "daily_cost_limit" - assert int( - await client.get( - DAILY_COST_RESERVED_MICRO_USD_KEY.format( - provider="claude", - date="2026-07-14", + assert ( + int( + await client.get( + DAILY_COST_RESERVED_MICRO_USD_KEY.format( + provider="claude", + date="2026-07-14", + ) ) + or 0 ) - or 0 - ) == 18 + == 18 + ) assert await limiter.finalize_generation( first, @@ -475,15 +509,18 @@ async def test_claude_daily_and_total_cost_caps_include_pending_reservations( prompt_tokens=1, completion_tokens=1, ) - assert int( - await client.get( - DAILY_COST_MICRO_USD_KEY.format( - provider="claude", - date="2026-07-14", + assert ( + int( + await client.get( + DAILY_COST_MICRO_USD_KEY.format( + provider="claude", + date="2026-07-14", + ) ) + or 0 ) - or 0 - ) == 18 + == 18 + ) await client.flushdb() _configure_paid_provider_limits( @@ -523,12 +560,15 @@ async def test_claude_daily_and_total_cost_caps_include_pending_reservations( assert total_first.allowed is True assert total_block.allowed is False assert total_block.reason == "total_cost_limit" - assert int( - await client.get( - TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider="claude") + assert ( + int( + await client.get( + TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider="claude") + ) + or 0 ) - or 0 - ) == 18 + == 18 + ) assert await limiter.finalize_generation( total_first, @@ -592,27 +632,36 @@ async def test_claude_finalize_is_atomic_and_idempotent_under_real_redis_race( assert finalized_receipt[b"status"] == b"succeeded" assert finalized_receipt[b"actual_prompt_tokens"] == b"7" assert finalized_receipt[b"actual_completion_tokens"] == b"3" - assert int( - await client.get( - DAILY_TOKEN_KEY.format(provider="claude", date="2026-07-14") - ) - or 0 - ) == 10 - assert int( - await client.get( - DAILY_COST_MICRO_USD_KEY.format( - provider="claude", - date="2026-07-14", + assert ( + int( + await client.get( + DAILY_TOKEN_KEY.format(provider="claude", date="2026-07-14") ) + or 0 ) - or 0 - ) == 66 - assert int( - await client.get( - TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider="claude") + == 10 + ) + assert ( + int( + await client.get( + DAILY_COST_MICRO_USD_KEY.format( + provider="claude", + date="2026-07-14", + ) + ) + or 0 ) - or 0 - ) == 0 + == 66 + ) + assert ( + int( + await client.get( + TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider="claude") + ) + or 0 + ) + == 0 + ) adversarial = await asyncio.gather( *( @@ -628,12 +677,15 @@ async def test_claude_finalize_is_atomic_and_idempotent_under_real_redis_race( ) assert adversarial == [True] * 20 assert await client.hgetall(receipt_key) == finalized_receipt - assert int( - await client.get( - DAILY_TOKEN_KEY.format(provider="claude", date="2026-07-14") + assert ( + int( + await client.get( + DAILY_TOKEN_KEY.format(provider="claude", date="2026-07-14") + ) + or 0 ) - or 0 - ) == 10 + == 10 + ) await client.flushdb() await client.aclose() @@ -724,13 +776,297 @@ async def test_finalized_receipt_and_run_claim_survive_configured_ttl_and_replay assert replay.allowed is False assert replay.reason == "idempotent_replay" assert replay.receipt_id == first.receipt_id - assert int( - await client.get(DAILY_REQ_KEY.format(provider="gemini", date="2026-07-14")) - or 0 - ) == 1 + assert ( + int( + await client.get(DAILY_REQ_KEY.format(provider="gemini", date="2026-07-14")) + or 0 + ) + == 1 + ) assert await client.hgetall(receipt_key) == finalized_receipt assert await client.ttl(receipt_key) == -1 assert await client.ttl(idempotency_key) == -1 await client.flushdb() await client.aclose() + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_cross_provider_run_budget_cap_is_atomic_under_real_redis_race( + ephemeral_redis_url: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) + await client.ping() + await client.flushdb() + _configure_paid_provider_limits(monkeypatch, "claude") + _configure_paid_provider_limits(monkeypatch, "gemini") + + limiter = AIRateLimiter() + limiter._redis = client + limiter._get_today = lambda: "2026-07-14" # type: ignore[method-assign] + limiter._seconds_until_tomorrow = lambda: 3600 # type: ignore[method-assign] + run_id = "run-cross-provider-cap-race" + common_context = { + "run_id": run_id, + "work_item_id": "AIA-SRE-013", + "synthetic_validation": True, + # Claude reserves 18 micro-USD and Gemini reserves 1 micro-USD for this + # one-token fixture. The aggregate cap permits only one atomic winner. + "paid_canary_max_cost_usd": 0.000018, + } + + reservations = await asyncio.gather( + limiter.reserve_generation( + "claude", + "x", + model="claude-sonnet-5", + max_output_tokens=1, + context={**common_context, "trace_id": "trace-cap-race-claude"}, + ), + limiter.reserve_generation( + "gemini", + "x", + model="gemini-2.5-flash-lite", + max_output_tokens=1, + context={**common_context, "trace_id": "trace-cap-race-gemini"}, + ), + ) + + allowed = [row for row in reservations if row.allowed] + blocked = [row for row in reservations if not row.allowed] + assert len(allowed) == 1 + assert len(blocked) == 1 + assert blocked[0].reason == "run_cost_limit" + assert allowed[0].run_budget_id == blocked[0].run_budget_id + + pending = await limiter.get_paid_run_budget_readback( + run_id=run_id, + work_item_id="AIA-SRE-013", + expected_max_cost_micro_usd=18, + ) + assert pending["reservation_count"] == 1 + assert pending["finalized_count"] == 0 + assert pending["reserved_cost_micro_usd"] in {1, 18} + assert pending["terminal_verified"] is False + + assert await limiter.finalize_generation( + allowed[0], + success=False, + error_code="isolated_run_budget_cleanup", + ) + terminal = await limiter.get_paid_run_budget_readback( + run_id=run_id, + work_item_id="AIA-SRE-013", + expected_max_cost_micro_usd=18, + ) + assert terminal["reserved_cost_micro_usd"] == 0 + assert terminal["reservation_count"] == terminal["finalized_count"] == 1 + assert terminal["accounted_cost_micro_usd"] in {1, 18} + assert terminal["terminal_verified"] is True + + await client.flushdb() + await client.aclose() + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_cross_provider_run_budget_finalize_is_idempotent( + ephemeral_redis_url: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) + await client.ping() + await client.flushdb() + _configure_paid_provider_limits(monkeypatch, "claude") + _configure_paid_provider_limits(monkeypatch, "gemini") + + limiter = AIRateLimiter() + limiter._redis = client + limiter._get_today = lambda: "2026-07-14" # type: ignore[method-assign] + limiter._seconds_until_tomorrow = lambda: 3600 # type: ignore[method-assign] + run_id = "run-cross-provider-finalize-race" + context = { + "run_id": run_id, + "work_item_id": "AIA-SRE-013", + "synthetic_validation": True, + "paid_canary_max_cost_usd": 0.25, + } + claude, gemini = await asyncio.gather( + limiter.reserve_generation( + "claude", + "x", + model="claude-sonnet-5", + max_output_tokens=1, + context={**context, "trace_id": "trace-finalize-claude"}, + ), + limiter.reserve_generation( + "gemini", + "x", + model="gemini-2.5-flash-lite", + max_output_tokens=1, + context={**context, "trace_id": "trace-finalize-gemini"}, + ), + ) + assert claude.allowed is True + assert gemini.allowed is True + + finalized = await asyncio.gather( + *( + limiter.finalize_generation( + reservation, + success=True, + prompt_tokens=1, + completion_tokens=1, + ) + for reservation in (claude, gemini) + for _ in range(10) + ) + ) + assert finalized == [True] * 20 + receipt = await limiter.get_paid_run_budget_readback( + run_id=run_id, + work_item_id="AIA-SRE-013", + expected_max_cost_micro_usd=250_000, + ) + assert receipt["reservation_count"] == 2 + assert receipt["finalized_count"] == 2 + assert receipt["reserved_cost_micro_usd"] == 0 + assert receipt["accounted_cost_micro_usd"] == 19 + assert receipt["terminal_verified"] is True + + await client.flushdb() + await client.aclose() + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_paid_run_budget_finalize_failure_retains_exposure( + ephemeral_redis_url: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) + await client.ping() + await client.flushdb() + _configure_paid_provider_limits(monkeypatch, "gemini") + + class _FinalizeFailRedis: + def __init__(self, delegate) -> None: + self._delegate = delegate + + async def eval(self, script: str, *args): + if script == _FINALIZE_GENERATION_LUA: + raise ConnectionError("isolated finalize failure") + return await self._delegate.eval(script, *args) + + def __getattr__(self, name: str): + return getattr(self._delegate, name) + + limiter = AIRateLimiter() + limiter._redis = _FinalizeFailRedis(client) + limiter._get_today = lambda: "2026-07-14" # type: ignore[method-assign] + limiter._seconds_until_tomorrow = lambda: 3600 # type: ignore[method-assign] + run_id = "run-finalize-failure-retains-reservation" + reservation = await limiter.reserve_generation( + "gemini", + "x", + model="gemini-2.5-flash-lite", + max_output_tokens=1, + context={ + "trace_id": "trace-finalize-failure", + "run_id": run_id, + "work_item_id": "AIA-SRE-013", + "synthetic_validation": True, + "paid_canary_max_cost_usd": 0.25, + }, + ) + assert reservation.allowed is True + assert ( + await limiter.finalize_generation( + reservation, + success=True, + prompt_tokens=1, + completion_tokens=1, + ) + is False + ) + + readback = await limiter.get_paid_run_budget_readback( + run_id=run_id, + work_item_id="AIA-SRE-013", + expected_max_cost_micro_usd=250_000, + ) + assert readback["reserved_cost_micro_usd"] == 1 + assert readback["accounted_cost_micro_usd"] == 0 + assert readback["reservation_count"] == 1 + assert readback["finalized_count"] == 0 + assert readback["terminal_verified"] is False + + limiter._redis = client + assert await limiter.finalize_generation( + reservation, + success=False, + error_code="isolated_cleanup", + ) + await client.flushdb() + await client.aclose() + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_paid_run_budgets_are_isolated_by_run_identity( + ephemeral_redis_url: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) + await client.ping() + await client.flushdb() + _configure_paid_provider_limits(monkeypatch, "claude") + + limiter = AIRateLimiter() + limiter._redis = client + limiter._get_today = lambda: "2026-07-14" # type: ignore[method-assign] + limiter._seconds_until_tomorrow = lambda: 3600 # type: ignore[method-assign] + + reservations = await asyncio.gather( + *( + limiter.reserve_generation( + "claude", + "x", + model="claude-sonnet-5", + max_output_tokens=1, + context={ + "trace_id": f"trace-cross-run-{index}", + "run_id": f"run-cross-run-{index}", + "work_item_id": "AIA-SRE-013", + "synthetic_validation": True, + "paid_canary_max_cost_usd": 0.000018, + }, + ) + for index in range(2) + ) + ) + assert [row.allowed for row in reservations] == [True, True] + assert reservations[0].run_budget_id != reservations[1].run_budget_id + assert all( + await asyncio.gather( + *( + client.exists(PAID_RUN_BUDGET_KEY.format(budget_id=row.run_budget_id)) + for row in reservations + ) + ) + ) + + assert await asyncio.gather( + *( + limiter.finalize_generation( + row, + success=False, + error_code="isolated_cross_run_cleanup", + ) + for row in reservations + ) + ) == [True, True] + await client.flushdb() + await client.aclose() diff --git a/apps/api/tests/test_ai_control_provider_policy.py b/apps/api/tests/test_ai_control_provider_policy.py index 1fb8c6d64..efe618625 100644 --- a/apps/api/tests/test_ai_control_provider_policy.py +++ b/apps/api/tests/test_ai_control_provider_policy.py @@ -9,14 +9,27 @@ from src.services.ai_providers.interfaces import is_provider_enabled_by_env class _ControlRedis: - def __init__(self, value: str | None = None) -> None: + def __init__( + self, + value: str | None = None, + *, + eval_results: list[list[object]] | None = None, + ) -> None: self.value = value self.set = AsyncMock(return_value=True) self.delete = AsyncMock(return_value=1) + self.eval_results = list(eval_results or []) + self.eval_calls: list[tuple[object, ...]] = [] async def get(self, _key: str) -> str | None: return self.value + async def eval(self, *args): + self.eval_calls.append(args) + if not self.eval_results: + raise AssertionError("unexpected Redis eval") + return self.eval_results.pop(0) + def test_gemini_environment_gate_defaults_disabled( monkeypatch: pytest.MonkeyPatch, @@ -45,7 +58,13 @@ async def test_paid_disable_state_is_fail_closed_when_redis_is_unavailable( async def test_gemini_missing_disable_key_is_disabled_and_control_is_persistent( monkeypatch: pytest.MonkeyPatch, ) -> None: - redis = _ControlRedis(value=None) + redis = _ControlRedis( + value=None, + eval_results=[ + [0, "paired_paid_state_not_enabled", ""], + [1, "paid_provider_state_set"], + ], + ) async def _redis() -> _ControlRedis: return redis @@ -54,10 +73,190 @@ async def test_gemini_missing_disable_key_is_disabled_and_control_is_persistent( assert await ai_control.is_provider_disabled("gemini") is True assert await ai_control.set_provider_disabled("gemini", True) is True - redis.set.assert_awaited_with("ai:control:disabled:gemini", "1") - assert await ai_control.set_provider_disabled("gemini", False) is True - redis.set.assert_awaited_with("ai:control:disabled:gemini", "0") - redis.delete.assert_not_awaited() + assert await ai_control.set_provider_disabled("gemini", False) is False + assert len(redis.eval_calls) == 2 + disable_call = redis.eval_calls[1] + assert "ai:control:disabled:claude" in disable_call + assert "ai:control:disabled:gemini" in disable_call + assert "ai:control:paid_canary:activation_lock" in disable_call + assert "ai:control:paid_canary:active_receipt" in disable_call + + +@pytest.mark.asyncio +async def test_paid_canary_lease_is_run_owned_and_persistent_promotion_is_blocked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _ControlRedis( + eval_results=[ + [1, "activation_lease_acquired"], + [1, "paid_canary_disabled"], + ] + ) + + async def _redis() -> _ControlRedis: + return redis + + monkeypatch.setattr(ai_control, "_get_redis", _redis) + + owner = "AIA-SRE-013-run.attempt-123456789abc" + assert await ai_control.acquire_paid_provider_canary_lease(owner) is True + assert await ai_control.promote_paid_provider_canary_activation( + owner, + completion_record_id="row-4", + ) is False + assert await ai_control.rollback_paid_provider_canary_activation(owner) is True + assert len(redis.eval_calls) == 2 + + +@pytest.mark.asyncio +async def test_paid_canary_execution_claim_is_atomic_and_owner_released( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _ControlRedis( + eval_results=[ + [1, "execution_claim_acquired"], + [0, "execution_claim_held"], + [0, "execution_claim_owner_mismatch"], + [1, "execution_claim_released"], + ] + ) + + async def _redis() -> _ControlRedis: + return redis + + monkeypatch.setattr(ai_control, "_get_redis", _redis) + run_id = "267a3d26-3c29-470f-8fe3-388a2f408f39" + owner = f"{run_id}.claim-123456789abc" + + assert ( + await ai_control.acquire_paid_provider_canary_execution_claim( + run_id, + owner, + ) + is True + ) + assert ( + await ai_control.acquire_paid_provider_canary_execution_claim( + run_id, + f"{run_id}.claim-deadbeef0000", + ) + is False + ) + assert ( + await ai_control.release_paid_provider_canary_execution_claim( + run_id, + f"{run_id}.claim-deadbeef0000", + ) + is False + ) + assert ( + await ai_control.release_paid_provider_canary_execution_claim(run_id, owner) + is True + ) + claim_keys = [call[2] for call in redis.eval_calls] + assert len(set(claim_keys)) == 1 + assert str(claim_keys[0]).startswith( + "ai:control:paid_canary:execution_claim:" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "admission_result", + [ + [0, "paired_paid_state_not_enabled", ""], + [0, "paid_activation_receipt_missing", ""], + ], +) +async def test_stale_single_or_paired_legacy_zero_keys_fail_closed( + monkeypatch: pytest.MonkeyPatch, + admission_result: list[object], +) -> None: + redis = _ControlRedis(eval_results=[admission_result]) + + async def _redis() -> _ControlRedis: + return redis + + monkeypatch.setattr(ai_control, "_get_redis", _redis) + + assert await ai_control.is_provider_disabled( + "claude", + run_id="AIA-SRE-013-run", + ) is True + + +@pytest.mark.asyncio +async def test_legacy_persistent_pair_is_never_admitted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + owner = "AIA-SRE-013-run.attempt-123456789abc" + redis = _ControlRedis( + eval_results=[ + [0, "legacy_persistent_activation_blocked", f"{owner}|row-4"] + ] + ) + + async def _redis() -> _ControlRedis: + return redis + + monkeypatch.setattr(ai_control, "_get_redis", _redis) + + assert await ai_control.is_provider_disabled("gemini") is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "receipt", + [ + "AIA-SRE-013-run.attempt-123456789abc", + "AIA-SRE-013-run.attempt-123456789abc|", + "|row-4", + "AIA-SRE-013-run.attempt-123456789abc|row-4|extra", + ], +) +async def test_malformed_persistent_activation_receipt_fails_closed( + monkeypatch: pytest.MonkeyPatch, + receipt: str, +) -> None: + redis = _ControlRedis( + eval_results=[[1, "persistent_paid_activation_admitted", receipt]] + ) + + async def _redis() -> _ControlRedis: + return redis + + monkeypatch.setattr(ai_control, "_get_redis", _redis) + + assert await ai_control.is_provider_disabled("claude") is True + + +@pytest.mark.asyncio +async def test_persistent_readback_requires_exact_completion_record( + monkeypatch: pytest.MonkeyPatch, +) -> None: + owner = "AIA-SRE-013-run.attempt-123456789abc" + atomic_state = ["", -2, "0", -1, "0", -1, f"{owner}|row-4"] + redis = _ControlRedis(eval_results=[atomic_state, atomic_state]) + + async def _redis() -> _ControlRedis: + return redis + + monkeypatch.setattr(ai_control, "_get_redis", _redis) + + matching = await ai_control.get_paid_provider_canary_control_readback( + expected_owner=owner, + expected_completion_record_id="row-4", + ) + mismatching = await ai_control.get_paid_provider_canary_control_readback( + expected_owner=owner, + expected_completion_record_id="row-5", + ) + + assert matching["persistent_enabled"] is False + assert matching["legacy_persistent_state_detected"] is True + assert matching["active_completion_receipt_matches"] is True + assert mismatching["persistent_enabled"] is False + assert mismatching["active_completion_receipt_matches"] is False @pytest.mark.asyncio diff --git a/apps/api/tests/test_ai_provider_status_readback.py b/apps/api/tests/test_ai_provider_status_readback.py index 009063cb9..d34c4e186 100644 --- a/apps/api/tests/test_ai_provider_status_readback.py +++ b/apps/api/tests/test_ai_provider_status_readback.py @@ -95,6 +95,8 @@ def _configure( "CLAUDE_API_KEY", "configured-test-value", ) + monkeypatch.setattr(config_module.settings, "GEMINI_EXECUTION_MODE", "production") + monkeypatch.setattr(config_module.settings, "GEMINI_CANARY_PERCENT", 0) monkeypatch.setattr(config_module.settings, "CLAUDE_EXECUTION_MODE", "shadow") monkeypatch.setattr(config_module.settings, "CLAUDE_CANARY_PERCENT", 0) monkeypatch.setenv("ENABLE_GEMINI", "true") @@ -164,6 +166,22 @@ async def test_verified_auth_and_complete_accounting_enable_cost_guard_readback( assert result["gemini_production_executable"] is True +@pytest.mark.asyncio +async def test_verified_gemini_canary_is_not_reported_as_production( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _configure(monkeypatch, limiter=_Limiter(_usage(authenticated=True))) + monkeypatch.setattr(config_module.settings, "GEMINI_EXECUTION_MODE", "canary") + monkeypatch.setattr(config_module.settings, "GEMINI_CANARY_PERCENT", 5) + + result = await get_ai_status() + + assert result["gemini_production_executable"] is False + assert result["gemini_canary_executable"] is True + assert result["gemini_execution_mode"] == "canary" + assert result["gemini_canary_percent"] == 5 + + @pytest.mark.parametrize( "accounting,readback_status", [ @@ -198,6 +216,27 @@ def test_cost_guard_static_readback_fails_closed_without_complete_nondegraded_ac assert route["hops"][4]["production_executable"] is False +def test_route_readback_exposes_cloud_boundary_and_mesh_gap() -> None: + route = production_route_readback( + gemini_configured=False, + gemini_enabled=False, + ) + + assert route["hops"][0]["data_boundary"] == "cloud" + assert route["hops"][1]["data_boundary"] == "cloud" + assert route["hops"][2]["data_boundary"] == "local" + assert route["transport_security"] == { + "status": "degraded_mesh_pending", + "production_closure": False, + "reason": "gcp_ollama_public_http_stopgap", + "replacement": "wireguard_mesh_10.77.114.x", + "sanitized_generation_gate": ( + "durable_same_run_endpoint_bound_transport_receipt_required" + ), + "receipt_ttl_max_seconds": 120, + } + + @pytest.mark.parametrize("missing_field", ["pricing_policy", "cost_exceeded"]) def test_cost_guard_static_readback_requires_affirmative_pricing_and_cost_proof( missing_field: str, diff --git a/apps/api/tests/test_ai_route_observed_fallback_readback.py b/apps/api/tests/test_ai_route_observed_fallback_readback.py index 113294d9d..219d19a16 100644 --- a/apps/api/tests/test_ai_route_observed_fallback_readback.py +++ b/apps/api/tests/test_ai_route_observed_fallback_readback.py @@ -71,6 +71,11 @@ async def test_all_ollama_offline_never_labels_paid_fallback_selected_primary( response = await platform_operator_service.get_ai_route_status("deep_rca") + manager.select_provider.assert_awaited_once_with( + task_type="deep_rca", + emit_side_effects=False, + ) + assert response["schema_version"] == "awooop_ai_route_status_v2" assert response["selection_mode"] == "fixed_production_order" assert response["selected_provider"] == "ollama_gcp_a" diff --git a/apps/api/tests/test_ai_router_cache_provider_policy.py b/apps/api/tests/test_ai_router_cache_provider_policy.py index bf9ee8e75..bd2066378 100644 --- a/apps/api/tests/test_ai_router_cache_provider_policy.py +++ b/apps/api/tests/test_ai_router_cache_provider_policy.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import hashlib import json from datetime import UTC, datetime, timedelta from typing import Any @@ -48,6 +49,18 @@ class _FakeRedis: async def set(self, key: str, value: str, ex: int | None = None) -> None: self.set_calls.append((key, value, ex)) + async def eval(self, *_args: object) -> list[object]: + if self.gemini_disabled: + return [0, "paired_paid_state_not_enabled", ""] + run_id = str(_args[-1] or "") + if not run_id: + return [0, "run_owned_canary_not_admitted", ""] + return [ + 1, + "run_owned_canary_admitted", + f"{run_id}.attempt-fixture", + ] + class _FakeProvider: name = "ollama_gcp_a" @@ -95,6 +108,26 @@ class _CloudProvider: return AIResult(raw_response='{"provider":"gemini"}', success=True, provider=self.name) +class _CloudOllamaProvider(_CloudProvider): + def __init__(self, name: str) -> None: + super().__init__() + self.name = name + + +class _SuccessfulLocalProvider(_FailingLocalProvider): + async def analyze( + self, + prompt: str, + context: dict[str, Any] | None = None, + ) -> AIResult: + self.calls += 1 + return AIResult( + raw_response='{"provider":"ollama_local"}', + success=True, + provider=self.name, + ) + + class _SlowLocalProvider(_FailingLocalProvider): async def analyze( self, @@ -125,6 +158,18 @@ def _paid_alert_context(**overrides: Any) -> dict[str, Any]: return context +def _synthetic_cloud_context(prompt: str, **overrides: Any) -> dict[str, Any]: + return { + "intent_hint": "diagnose", + "synthetic_validation": True, + "data_classification": "sanitized", + "synthetic_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "infrastructure_remediation_write_allowed": False, + "run_id": "AIA-SRE-013-test", + **overrides, + } + + def _durable_unavailable_receipt(provider: str, run_id: str) -> dict[str, Any]: observed_at = datetime.now(UTC) return { @@ -159,17 +204,15 @@ async def test_executor_skips_cached_cloud_provider_when_ollama_lane_is_required result = await AIRouterExecutor(registry).execute( prompt="diagnose alert", provider_order=["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], - context={ - "intent_hint": "diagnose", - "alert_type": "HostHighCpuLoad", - "data_classification": "sanitized", - }, + context=_paid_alert_context(intent_hint="diagnose"), ) assert result.provider == "ollama_gcp_a" assert result.raw_response == '{"provider":"fresh_ollama"}' assert fake_provider.calls == 1 - assert fake_redis.set_calls + # GCP Ollama is a remote sanitized-cloud hop; its response is deliberately + # non-cacheable even though it is the first free provider. + assert fake_redis.set_calls == [] @pytest.mark.asyncio @@ -187,7 +230,10 @@ async def test_executor_applies_gemini_runtime_disable_before_cache_lookup( result = await AIRouterExecutor(registry).execute( prompt="non-alert summary", provider_order=["ollama_gcp_a", "gemini"], - context={"intent_hint": "summary", "data_classification": "sanitized"}, + context=_synthetic_cloud_context( + "non-alert summary", + intent_hint="summary", + ), ) assert result.provider == "ollama_gcp_a" @@ -211,7 +257,10 @@ async def test_executor_applies_gemini_environment_disable_before_cache_lookup( result = await AIRouterExecutor(registry).execute( prompt="non-alert summary", provider_order=["ollama_gcp_a", "gemini"], - context={"intent_hint": "summary", "data_classification": "sanitized"}, + context=_synthetic_cloud_context( + "non-alert summary", + intent_hint="summary", + ), ) assert result.provider == "ollama_gcp_a" @@ -259,12 +308,10 @@ async def test_alert_cloud_backup_waits_for_real_ollama_local_attempt( result = await AIRouterExecutor(registry).execute( prompt="diagnose alert", provider_order=["ollama_gcp_a", "gemini"], - context={ - "intent_hint": "diagnose", - "alert_type": "HostHighCpuLoad", - "alert_requires_ollama_before_cloud": True, - "data_classification": "sanitized", - }, + context=_paid_alert_context( + intent_hint="diagnose", + alert_requires_ollama_before_cloud=True, + ), ) assert result.success is False @@ -272,6 +319,36 @@ async def test_alert_cloud_backup_waits_for_real_ollama_local_attempt( assert gemini.calls == 0 +@pytest.mark.asyncio +async def test_require_local_skips_both_gcp_ollama_hops_and_uses_host111( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_redis = _FakeRedis(cached_provider="none") + gcp_a = _CloudOllamaProvider("ollama_gcp_a") + gcp_b = _CloudOllamaProvider("ollama_gcp_b") + local = _SuccessfulLocalProvider("ollama_local") + registry = AIProviderRegistry() + registry.register(gcp_a) + registry.register(gcp_b) + registry.register(local) + + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: fake_redis) + + result = await AIRouterExecutor(registry).execute( + prompt="local-only diagnosis", + provider_order=["ollama_gcp_a", "ollama_gcp_b", "ollama_local"], + context={"intent_hint": "diagnose"}, + require_local=True, + ) + + assert result.success is True + assert result.provider == "ollama_local" + assert gcp_a.calls == 0 + assert gcp_b.calls == 0 + assert local.calls == 1 + + @pytest.mark.asyncio async def test_alert_cloud_backup_allowed_after_ollama_local_attempt( monkeypatch: pytest.MonkeyPatch, @@ -371,6 +448,7 @@ async def test_provider_unsuccessful_results_count_toward_circuit_breaker( result = await executor.execute( prompt=f"failure {index}", provider_order=["ollama_gcp_a"], + context=_synthetic_cloud_context(f"failure {index}"), ) assert result.success is False diff --git a/apps/api/tests/test_ai_router_circuit_identity_fail_closed.py b/apps/api/tests/test_ai_router_circuit_identity_fail_closed.py new file mode 100644 index 000000000..48ef4b91d --- /dev/null +++ b/apps/api/tests/test_ai_router_circuit_identity_fail_closed.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import hashlib +from typing import Any + +import pytest + +from src.services import ai_router as ai_router_module +from src.services.ai_providers.interfaces import AIResult +from src.services.ai_router import AIProviderRegistry, AIRouterExecutor + + +class _Redis: + async def get(self, _key: str) -> None: + return None + + async def set( + self, + _key: str, + _value: str, + ex: int | None = None, + ) -> None: + _ = ex + + +class _Provider: + is_enabled = True + privacy_level = "cloud" + capabilities = {"rca", "chat"} + + def __init__(self, name: str, *, success: bool) -> None: + self.name = name + self.success = success + self.calls = 0 + self.contexts: list[dict[str, Any] | None] = [] + + async def analyze( + self, + _prompt: str, + _context: dict[str, Any] | None = None, + ) -> AIResult: + self.calls += 1 + self.contexts.append(_context) + return AIResult( + raw_response="ok" if self.success else "", + success=self.success, + provider=self.name, + error=None if self.success else "forced_failure", + ) + + +def _context(**overrides: Any) -> dict[str, Any]: + prompt = "sanitized circuit test" + return { + "synthetic_validation": True, + "data_classification": "sanitized", + "synthetic_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "infrastructure_remediation_write_allowed": False, + "trace_id": "trace-circuit-test", + "run_id": "run-circuit-test", + "work_item_id": "AIA-CIRCUIT-TEST", + **overrides, + } + + +def _executor(*providers: _Provider) -> AIRouterExecutor: + registry = AIProviderRegistry() + for provider in providers: + registry.register(provider) + return AIRouterExecutor(registry) + + +@pytest.fixture(autouse=True) +def _runtime(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: _Redis()) + + +@pytest.mark.asyncio +async def test_first_provider_circuit_open_is_terminal_without_any_network_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def _unexpected_issue(**_kwargs: Any) -> dict[str, Any]: + raise AssertionError("circuit-open route must not probe another endpoint") + + monkeypatch.setattr( + "src.services.cloud_transport_receipt.issue_cloud_transport_receipt", + _unexpected_issue, + ) + providers = [ + _Provider("ollama_gcp_a", success=False), + _Provider("ollama_gcp_b", success=True), + _Provider("ollama_local", success=True), + ] + executor = _executor(*providers) + circuit = executor._get_circuit_breaker("ollama_gcp_a") + circuit._failure_count = circuit._failure_threshold + circuit._last_failure_time = 10**12 + + result = await executor.execute( + prompt="sanitized circuit test", + provider_order=[provider.name for provider in providers], + context=_context( + alert_requires_ollama_before_cloud=True, + cloud_transport_auto_preflight=True, + ), + ) + + assert result.success is False + assert result.error == "provider_circuit_open_fail_closed:ollama_gcp_a" + assert [provider.calls for provider in providers] == [0, 0, 0] + assert result.audit_metadata["canonical_provider_lane"] == "ollama_gcp_a" + assert result.audit_metadata["provider_network_call_performed"] is False + assert result.audit_metadata["cross_provider_fallback_allowed"] is False + assert result.audit_metadata["cross_domain_fallback_allowed"] is False + assert result.audit_metadata["repair_work_item"]["provider"] == ( + "ollama_gcp_a" + ) + + +@pytest.mark.asyncio +async def test_downstream_circuit_open_stops_after_prior_provider_failure() -> None: + providers = [ + _Provider("ollama_gcp_a", success=False), + _Provider("ollama_gcp_b", success=True), + _Provider("ollama_local", success=True), + ] + executor = _executor(*providers) + circuit = executor._get_circuit_breaker("ollama_gcp_b") + circuit._failure_count = circuit._failure_threshold + circuit._last_failure_time = 10**12 + + result = await executor.execute( + prompt="sanitized circuit test", + provider_order=[provider.name for provider in providers], + context=_context(), + ) + + assert result.error == "provider_circuit_open_fail_closed:ollama_gcp_b" + assert [provider.calls for provider in providers] == [1, 0, 0] + assert result.audit_metadata["canonical_provider_lane"] == "ollama_gcp_b" + + +@pytest.mark.asyncio +async def test_closed_circuits_preserve_normal_gcp_a_gcp_b_host111_order() -> None: + providers = [ + _Provider("ollama_gcp_a", success=False), + _Provider("ollama_gcp_b", success=True), + _Provider("ollama_local", success=True), + ] + + result = await _executor(*providers).execute( + prompt="sanitized circuit test", + provider_order=[provider.name for provider in providers], + context=_context(), + ) + + assert result.success is True + assert result.provider == "ollama_gcp_b" + assert [provider.calls for provider in providers] == [1, 1, 0] + + +@pytest.mark.asyncio +async def test_unresolved_or_ambiguous_asset_identity_never_calls_provider() -> None: + providers = [ + _Provider("ollama_gcp_a", success=True), + _Provider("ollama_gcp_b", success=True), + _Provider("ollama_local", success=True), + ] + result = await _executor(*providers).execute( + prompt="sanitized circuit test", + provider_order=[provider.name for provider in providers], + context=_context( + target_resource="ambiguous-api", + typed_target_route={ + "resolution_status": "asset_identity_unresolved", + "target_kind": "unknown", + "canonical_asset_id": None, + }, + ), + ) + + assert result.error == "asset_identity_unresolved" + assert [provider.calls for provider in providers] == [0, 0, 0] + assert result.audit_metadata["provider_network_call_performed"] is False + assert result.audit_metadata["drift_work_item_id"].startswith( + "AIA-ASSET-DRIFT-" + ) + + +@pytest.mark.asyncio +async def test_asset_circuit_open_stays_in_same_domain_without_model_call() -> None: + providers = [ + _Provider("ollama_gcp_a", success=True), + _Provider("ollama_gcp_b", success=True), + _Provider("ollama_local", success=True), + ] + result = await _executor(*providers).execute( + prompt="sanitized circuit test", + provider_order=[provider.name for provider in providers], + context=_context( + target_resource="awoooi-api", + typed_target_route={ + "resolution_status": "resolved", + "target_kind": "kubernetes_workload", + "canonical_asset_id": "service:awoooi-api", + "circuit_state": "open", + }, + ), + ) + + assert result.error == "asset_circuit_open_fail_closed" + assert [provider.calls for provider in providers] == [0, 0, 0] + assert result.audit_metadata["canonical_domain"] == "kubernetes_workload" + assert result.audit_metadata["cross_domain_fallback_allowed"] is False + assert result.audit_metadata["repair_work_item"]["domain"] == ( + "kubernetes_workload" + ) + + +@pytest.mark.asyncio +async def test_normal_sanitized_alert_issues_and_attaches_same_run_gcp_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + issued: list[dict[str, Any]] = [] + + async def _verify(**_kwargs: Any) -> str: + return "gcp_cloud_transport_receipt_missing" + + async def _issue(**kwargs: Any) -> dict[str, Any]: + issued.append(kwargs) + return { + "schema_version": "cloud_transport_preflight_receipt_v1", + "receipt_id": "cloud-transport:" + "a" * 32, + "provider": kwargs["provider"], + "trace_id": kwargs["trace_id"], + "run_id": kwargs["run_id"], + "work_item_id": kwargs["work_item_id"], + } + + monkeypatch.setattr( + "src.services.cloud_transport_receipt.verify_cloud_transport_receipt", + _verify, + ) + monkeypatch.setattr( + "src.services.cloud_transport_receipt.issue_cloud_transport_receipt", + _issue, + ) + monkeypatch.setattr( + "src.services.cloud_transport_receipt.resolve_gcp_transport_endpoint", + lambda provider: f"http://{provider}.invalid:11434", + ) + providers = [ + _Provider("ollama_gcp_a", success=True), + _Provider("ollama_gcp_b", success=True), + _Provider("ollama_local", success=True), + ] + + result = await _executor(*providers).execute( + prompt="sanitized circuit test", + provider_order=[provider.name for provider in providers], + context=_context(cloud_transport_auto_preflight=True), + ) + + assert result.provider == "ollama_gcp_a" + assert [provider.calls for provider in providers] == [1, 0, 0] + assert issued == [ + { + "trace_id": "trace-circuit-test", + "run_id": "run-circuit-test", + "work_item_id": "AIA-CIRCUIT-TEST", + "provider": "ollama_gcp_a", + "endpoint_url": "http://ollama_gcp_a.invalid:11434", + } + ] + assert providers[0].contexts[0] is not None + assert providers[0].contexts[0]["cloud_transport_receipts"][0][ + "receipt_id" + ] == ("cloud-transport:" + "a" * 32) + + +@pytest.mark.asyncio +async def test_failed_gcp_a_transport_preflight_skips_to_gcp_b_not_circuit_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def _verify(**_kwargs: Any) -> str: + return "gcp_cloud_transport_receipt_missing" + + async def _issue(**kwargs: Any) -> dict[str, Any]: + if kwargs["provider"] == "ollama_gcp_a": + raise RuntimeError("bounded preflight failure") + return { + "schema_version": "cloud_transport_preflight_receipt_v1", + "receipt_id": "cloud-transport:" + "b" * 32, + "provider": kwargs["provider"], + "trace_id": kwargs["trace_id"], + "run_id": kwargs["run_id"], + "work_item_id": kwargs["work_item_id"], + } + + monkeypatch.setattr( + "src.services.cloud_transport_receipt.verify_cloud_transport_receipt", + _verify, + ) + monkeypatch.setattr( + "src.services.cloud_transport_receipt.issue_cloud_transport_receipt", + _issue, + ) + monkeypatch.setattr( + "src.services.cloud_transport_receipt.resolve_gcp_transport_endpoint", + lambda provider: f"http://{provider}.invalid:11434", + ) + providers = [ + _Provider("ollama_gcp_a", success=True), + _Provider("ollama_gcp_b", success=True), + _Provider("ollama_local", success=True), + ] + + result = await _executor(*providers).execute( + prompt="sanitized circuit test", + provider_order=[provider.name for provider in providers], + context=_context(cloud_transport_auto_preflight=True), + ) + + assert result.success is True + assert result.provider == "ollama_gcp_b" + assert [provider.calls for provider in providers] == [0, 1, 0] diff --git a/apps/api/tests/test_ai_router_diagnose_fallback.py b/apps/api/tests/test_ai_router_diagnose_fallback.py index dca8d38df..240fdd11a 100644 --- a/apps/api/tests/test_ai_router_diagnose_fallback.py +++ b/apps/api/tests/test_ai_router_diagnose_fallback.py @@ -7,6 +7,7 @@ executable chain or authorize a direct Gemini jump. from __future__ import annotations +import hashlib from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -35,6 +36,18 @@ PRODUCTION_ENUM_ORDER = [ PRODUCTION_NAME_ORDER = [provider.value for provider in PRODUCTION_ENUM_ORDER] +def _synthetic_cloud_context(prompt: str, **overrides: Any) -> dict[str, Any]: + return { + "intent_hint": "diagnose", + "synthetic_validation": True, + "data_classification": "sanitized", + "synthetic_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "infrastructure_remediation_write_allowed": False, + "run_id": "AIA-SRE-013-test", + **overrides, + } + + class _NoCacheRedis: def __init__(self, *, gemini_disabled: bool = False) -> None: self.gemini_disabled = gemini_disabled @@ -50,6 +63,18 @@ class _NoCacheRedis: async def set(self, key: str, value: str, ex: int | None = None) -> None: self.set_calls.append((key, value, ex)) + async def eval(self, *_args: object) -> list[object]: + if self.gemini_disabled: + return [0, "paired_paid_state_not_enabled", ""] + run_id = str(_args[-1] or "") + if not run_id: + return [0, "run_owned_canary_not_admitted", ""] + return [ + 1, + "run_owned_canary_admitted", + f"{run_id}.attempt-fixture", + ] + class _Provider: is_enabled = True @@ -139,7 +164,7 @@ async def test_health_selected_gemini_cannot_become_router_primary() -> None: decision = await router.route( "diagnose cascading failure", - context={"intent_hint": "diagnose", "data_classification": "sanitized"}, + context=_synthetic_cloud_context("diagnose alert"), ) assert decision.selected_provider == AIProviderEnum.OLLAMA_GCP_A @@ -180,7 +205,7 @@ async def test_executor_rejects_direct_cloud_or_legacy_only_route( result = await executor.execute( prompt="bypass attempt", provider_order=["openclaw_nemo", "gemini", "claude"], - context={"intent_hint": "diagnose", "data_classification": "sanitized"}, + context=_synthetic_cloud_context("diagnose alert"), ) assert result.success is False @@ -207,7 +232,7 @@ async def test_executor_runs_exact_five_hops_in_order( result = await AIRouterExecutor(_registry(*providers)).execute( prompt="diagnose alert", provider_order=["ollama", "openclaw_nemo", "gemini", "claude"], - context={"intent_hint": "diagnose", "data_classification": "sanitized"}, + context=_synthetic_cloud_context("diagnose alert"), ) assert result.success is True @@ -233,12 +258,13 @@ async def test_disabled_gemini_is_never_readded_or_called( result = await AIRouterExecutor(_registry(*providers)).execute( prompt="diagnose alert", provider_order=PRODUCTION_NAME_ORDER, - context={"intent_hint": "diagnose", "data_classification": "sanitized"}, + context=_synthetic_cloud_context("diagnose alert"), ) assert result.success is False + assert "claude: disabled_by_runtime_control" in str(result.error) assert "gemini: disabled_by_runtime_control" in str(result.error) - assert [provider.calls for provider in providers] == [1, 1, 1, 1, 0] + assert [provider.calls for provider in providers] == [1, 1, 1, 0, 0] @pytest.mark.asyncio @@ -301,5 +327,5 @@ async def test_router_blocks_paid_cloud_before_cache_for_unclassified_or_sensiti ) assert result.success is False - assert [provider.calls for provider in providers] == [1, 1, 1, 0, 0] - assert "cloud_context_" in str(result.error) + assert [provider.calls for provider in providers] == [0, 0, 1, 0, 0] + assert "cloud_sanitization_receipt_missing" in str(result.error) diff --git a/apps/api/tests/test_ai_router_failover_integration.py b/apps/api/tests/test_ai_router_failover_integration.py index 24b8adfbf..c4a3f8d5e 100644 --- a/apps/api/tests/test_ai_router_failover_integration.py +++ b/apps/api/tests/test_ai_router_failover_integration.py @@ -70,7 +70,10 @@ async def test_health_observation_never_reorders_execution_contract( ) assert decision.observed_fallback_provider == expected_observation assert "selected primary=ollama_gcp_a" in decision.routing_reason - manager.select_provider.assert_awaited_once() + manager.select_provider.assert_awaited_once_with( + task_type="diagnose", + emit_side_effects=True, + ) @pytest.mark.asyncio @@ -89,4 +92,7 @@ async def test_health_probe_failure_cannot_authorize_cloud_jump() -> None: assert [provider for provider, _model in decision.fallback_chain] == ( EXPECTED_FALLBACK ) - manager.select_provider.assert_awaited_once() + manager.select_provider.assert_awaited_once_with( + task_type="delete", + emit_side_effects=True, + ) diff --git a/apps/api/tests/test_alert_chain_emergency_ingress.py b/apps/api/tests/test_alert_chain_emergency_ingress.py new file mode 100644 index 000000000..d1795b26e --- /dev/null +++ b/apps/api/tests/test_alert_chain_emergency_ingress.py @@ -0,0 +1,466 @@ +from __future__ import annotations + +# ruff: noqa: E402 +import json +import os +from pathlib import Path +from types import SimpleNamespace + +os.environ.setdefault( + "DATABASE_URL", + "postgresql+asyncpg://test:test@localhost/test", +) + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import ValidationError + +from src.api.v1 import webhooks as webhooks_api +from src.core.config import settings +from src.services.alert_chain_emergency_ingress import ( + ALERT_CHAIN_AGENT99_ROLE, + ALERT_CHAIN_CANONICAL_ASSET_ID, + ALERT_CHAIN_CATALOG_ID, + ALERT_CHAIN_EXECUTOR, + AlertChainEmergencyRelayRequest, + build_alert_chain_emergency_owner, + process_alert_chain_emergency_relay, +) + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def payload() -> dict: + return { + "schema_version": "agent99_alertmanager_emergency_relay_v1", + "relay_receipt_id": f"agent99-alertchain-{'a' * 64}", + "source_host": "192.168.0.110", + "source_url": "http://192.168.0.110:9093/api/v2/alerts", + "source_fingerprint": "0123456789abcdef", + "source_started_at": "2026-07-16T13:40:20+08:00", + "alertname": "AlertChainBroken_Alertmanager", + "status": "firing", + "severity": "critical", + "target_resource": "alertmanager", + "namespace": "monitoring", + "agent99_role": "relay_coordination_only", + "executor": "host_ansible_executor", + "catalog_id": "ansible:110-alertmanager-delivery-recovery", + "runtime_write_performed": False, + "raw_payload_stored": False, + } + + +class FakeApprovalService: + def __init__(self) -> None: + self.approval = None + self.created_requests = [] + self.incremented = 0 + + async def get_approval(self, approval_id): + if self.approval and str(self.approval.id) == str(approval_id): + return self.approval + return None + + async def create_approval_with_fingerprint(self, *, request, fingerprint): + self.created_requests.append((request, fingerprint)) + metadata = dict(request.metadata or {}) + approval_id = metadata.pop("preallocated_approval_id") + self.approval = SimpleNamespace( + id=approval_id, + metadata=metadata, + incident_id=request.incident_id, + hit_count=1, + ) + return self.approval + + async def increment_hit_count(self, _approval_id): + self.incremented += 1 + self.approval.hit_count += 1 + return self.approval + + async def update_incident_id(self, _approval_id, incident_id): + self.approval.incident_id = incident_id + + +class FakeIncident: + def __init__(self, incident_id: str) -> None: + self.incident_id = incident_id + self.persisted_to_pg = True + + def model_dump(self, *, mode: str): + assert mode == "json" + return { + "incident_id": self.incident_id, + "status": "investigating", + "severity": "P2", + "affected_services": ["alertmanager"], + "signals": [ + { + "alert_name": "AlertChainBroken_Alertmanager", + "labels": { + "alertname": "AlertChainBroken_Alertmanager", + "component": "alertmanager", + }, + "annotations": {}, + } + ], + } + + +class FakeIncidentService: + def __init__(self) -> None: + self.incident = None + + async def get_for_readback(self, incident_id, *, project_id): + assert project_id == "awoooi" + if self.incident and self.incident.incident_id == incident_id: + return self.incident + return None + + +def app_client() -> TestClient: + app = FastAPI() + app.include_router(webhooks_api.router, prefix="/api/v1") + return TestClient(app) + + +def test_relay_model_rejects_raw_content_and_executor_drift() -> None: + with pytest.raises(ValidationError): + AlertChainEmergencyRelayRequest.model_validate( + {**payload(), "raw_log": "must not enter machine ingress"} + ) + with pytest.raises(ValidationError): + AlertChainEmergencyRelayRequest.model_validate( + {**payload(), "executor": "Agent99"} + ) + with pytest.raises(ValidationError): + AlertChainEmergencyRelayRequest.model_validate( + {**payload(), "source_started_at": "2026-07-16T13:40:20"} + ) + with pytest.raises(ValidationError): + AlertChainEmergencyRelayRequest.model_validate( + {**payload(), "source_host": "192.168.0.111"} + ) + + +def test_agent99_poll_is_read_only_first_hop_and_reduced_authenticated_outbound() -> None: + source = ( + REPO_ROOT / "agent99-alertmanager-alertchain-poll.ps1" + ).read_text( + encoding="utf-8" + ) + first_hop = source.split("$alerts = @(Invoke-RestMethod", 1)[1].split( + "} catch", 1 + )[0] + + assert "http://192.168.0.110:9093/api/v2/alerts" in source + assert '$sourceUri.Host -ne $CanonicalSourceHost' in source + assert '$sourceUri.Port -ne 9093' in source + assert '$sourceUri.AbsolutePath -ne "/api/v2/alerts"' in source + assert "-Method Get" in first_hop + assert "-Headers" not in first_hop + assert "token" not in first_hop.lower() + assert '$state -ne "active"' in source + assert "AlertChainBroken_Alertmanager" in source + assert 'Headers @{ "X-Agent99-Relay-Token" = $token }' in source + assert "agent99_alertmanager_emergency_relay_v1" in source + assert "source_host = $CanonicalSourceHost" in source + assert "source_url = $CanonicalSourceUrl" in source + assert 'agent99_role = "relay_coordination_only"' in source + assert 'executor = "host_ansible_executor"' in source + assert 'catalog_id = $CanonicalCatalogId' in source + assert "candidate_queued" in source + assert "runtime_write_performed = $false" in source + assert "raw_payload_stored = $false" in source + assert "dedupe-receipts" in source + assert "rawPayloadStored = $false" in source + assert "Start-AgentSreAlertInbox" not in source + assert "agent99-run.ps1" not in source + + listener = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text( + encoding="utf-8" + ) + assert "alertmanager-emergency" not in listener + assert "AlertmanagerPrefix" not in listener + + +def test_agent99_install_and_bootstrap_publish_poll_contract() -> None: + installer = (REPO_ROOT / "agent99-install-sre-alert-relay.ps1").read_text( + encoding="utf-8" + ) + bootstrap = (REPO_ROOT / "agent99-bootstrap.ps1").read_text( + encoding="utf-8" + ) + + for source in (installer, bootstrap): + assert "http://192.168.0.110:9093/api/v2/alerts" in source + assert "pollIntervalSeconds" in source + assert "relay_coordination_only" in source + assert "host_ansible_executor" in source + assert "ansible:110-alertmanager-delivery-recovery" in source + assert '"alertmanagerPrefix":' not in source + assert 'Set-AgentProp $config.sreAlertRelay "alertmanagerPrefix"' not in source + + relay_script = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text( + encoding="utf-8" + ) + task_script = (REPO_ROOT / "agent99-register-tasks.ps1").read_text( + encoding="utf-8" + ) + assert "Register-ObjectEvent" in relay_script + assert "in_process_nonblocking_timer" in relay_script + assert "$alertChainPollTimer.AutoReset = $false" in relay_script + assert "$pollData.Timer.Start()" in relay_script + assert "Unregister-Event -SourceIdentifier $alertChainPollEventId" in relay_script + assert "Start-Job" not in relay_script + assert "Wooo-Agent99-AlertChain-Poll" not in task_script + + for config_name in ( + "agent99.config.example.json", + "agent99.config.99.example.json", + ): + config = json.loads((REPO_ROOT / config_name).read_text(encoding="utf-8")) + relay = config["sreAlertRelay"] + assert "alertmanagerPrefix" not in relay + poll = relay["alertChainPoll"] + assert poll["sourceUrl"] == ( + "http://192.168.0.110:9093/api/v2/alerts" + ) + assert poll["sourceHost"] == "192.168.0.110" + assert poll["pollIntervalSeconds"] == 60 + assert poll["agent99Role"] == "relay_coordination_only" + assert poll["linuxExecutor"] == "host_ansible_executor" + assert poll["catalogId"] == ( + "ansible:110-alertmanager-delivery-recovery" + ) + assert poll["firstHopSecretTransmitted"] is False + assert poll["rawPayloadStored"] is False + + +def test_owner_identity_is_stable_and_binds_same_run_trace_work_item() -> None: + request = AlertChainEmergencyRelayRequest.model_validate(payload()) + + first = build_alert_chain_emergency_owner(request) + second = build_alert_chain_emergency_owner(request) + + assert first == second + assert first["run_id"] == first["trace_id"] + assert first["work_item_id"].startswith("alertchain:awoooi:") + assert first["agent99_role"] == ALERT_CHAIN_AGENT99_ROLE + assert first["executor"] == ALERT_CHAIN_EXECUTOR + assert first["catalog_id"] == ALERT_CHAIN_CATALOG_ID + assert first["canonical_asset_id"] == ALERT_CHAIN_CANONICAL_ASSET_ID + + +@pytest.mark.asyncio +async def test_secondary_ingress_persists_and_queues_exact_ansible_candidate() -> None: + request = AlertChainEmergencyRelayRequest.model_validate(payload()) + owner = build_alert_chain_emergency_owner(request) + approvals = FakeApprovalService() + incidents = FakeIncidentService() + enqueue_calls = [] + events = [] + + async def create_incident(**kwargs): + assert kwargs["canonical_incident_id"] == owner["incident_id"] + assert kwargs["alertname"] == "AlertChainBroken_Alertmanager" + assert kwargs["namespace"] == "monitoring" + incidents.incident = FakeIncident(owner["incident_id"]) + return owner["incident_id"] + + async def enqueue(**kwargs): + enqueue_calls.append(kwargs) + assert kwargs["automation_run_id"] == owner["run_id"] + assert kwargs["incident"]["run_id"] == owner["run_id"] + assert kwargs["incident"]["trace_id"] == owner["trace_id"] + assert kwargs["incident"]["work_item_id"] == owner["work_item_id"] + assert kwargs["proposal_data"]["source_recurrence_verified"] is True + return { + "status": "controlled_check_mode_queued", + "automation_run_id": owner["run_id"], + "trace_id": owner["trace_id"], + "work_item_id": owner["work_item_id"], + "queued": True, + "side_effect_performed": False, + "active_blockers": [], + } + + async def record_event(**kwargs): + events.append(kwargs) + + result = await process_alert_chain_emergency_relay( + request, + approval_service=approvals, + incident_service=incidents, + incident_creator=create_incident, + candidate_enqueuer=enqueue, + event_recorder=record_event, + ) + + assert result["status"] == "ansible_candidate_queued_verifier_pending" + assert result["candidate_persisted"] is True + assert result["candidate_queued"] is True + assert result["identity"]["run_id"] == result["identity"]["trace_id"] + assert result["identity"]["work_item_id"] == owner["work_item_id"] + assert result["agent99_role"] == "relay_coordination_only" + assert result["executor"] == "host_ansible_executor" + assert result["runtime_write_performed"] is False + assert result["runtime_closure_verified"] is False + assert len(enqueue_calls) == 1 + assert events[0]["stage"] == "secondary_ingress_ansible_candidate_queued" + + repeated = await process_alert_chain_emergency_relay( + request, + approval_service=approvals, + incident_service=incidents, + incident_creator=create_incident, + candidate_enqueuer=enqueue, + event_recorder=record_event, + ) + assert repeated["identity"] == result["identity"] + assert approvals.incremented == 1 + + +@pytest.mark.asyncio +async def test_secondary_ingress_keeps_persisted_candidate_partial_when_queue_blocks() -> None: + request = AlertChainEmergencyRelayRequest.model_validate(payload()) + owner = build_alert_chain_emergency_owner(request) + approvals = FakeApprovalService() + incidents = FakeIncidentService() + + async def create_incident(**_kwargs): + incidents.incident = FakeIncident(owner["incident_id"]) + return owner["incident_id"] + + async def blocked_enqueue(**_kwargs): + return { + "status": "pre_decision_evidence_not_verified", + "automation_run_id": owner["run_id"], + "trace_id": owner["trace_id"], + "work_item_id": owner["work_item_id"], + "queued": False, + "side_effect_performed": False, + "active_blockers": ["pre_decision_evidence_not_verified"], + } + + async def no_event(**_kwargs): + return None + + result = await process_alert_chain_emergency_relay( + request, + approval_service=approvals, + incident_service=incidents, + incident_creator=create_incident, + candidate_enqueuer=blocked_enqueue, + event_recorder=no_event, + ) + + assert result["candidate_persisted"] is True + assert result["candidate_queued"] is False + assert result["runtime_closure_verified"] is False + assert result["safe_next_action"].startswith("retry_same_relay_receipt") + + +@pytest.mark.asyncio +async def test_secondary_ingress_rejects_deterministic_owner_collision() -> None: + request = AlertChainEmergencyRelayRequest.model_validate(payload()) + owner = build_alert_chain_emergency_owner(request) + approvals = FakeApprovalService() + approvals.approval = SimpleNamespace( + id=owner["approval_id"], + metadata={"schema_version": "untrusted_owner"}, + incident_id=owner["incident_id"], + hit_count=1, + ) + enqueue_calls = 0 + + async def must_not_enqueue(**_kwargs): + nonlocal enqueue_calls + enqueue_calls += 1 + return {} + + result = await process_alert_chain_emergency_relay( + request, + approval_service=approvals, + incident_service=FakeIncidentService(), + candidate_enqueuer=must_not_enqueue, + ) + + assert result["status"] == "durable_owner_identity_mismatch_no_candidate" + assert result["candidate_persisted"] is False + assert result["candidate_queued"] is False + assert enqueue_calls == 0 + + +def test_endpoint_requires_auth_and_fails_closed_until_queue_ack(monkeypatch) -> None: + monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected") + + missing = app_client().post( + "/api/v1/webhooks/agent99/alertmanager-emergency", + json=payload(), + ) + assert missing.status_code == 401 + + async def blocked(_payload): + return { + "status": "pre_decision_evidence_not_verified", + "candidate_persisted": True, + "candidate_queued": False, + } + + monkeypatch.setattr( + webhooks_api, + "process_alert_chain_emergency_relay", + blocked, + ) + response = app_client().post( + "/api/v1/webhooks/agent99/alertmanager-emergency", + headers={"X-Agent99-Relay-Token": "expected"}, + json=payload(), + ) + assert response.status_code == 503 + assert response.json()["detail"] == "pre_decision_evidence_not_verified" + + +def test_endpoint_returns_public_same_run_receipt(monkeypatch) -> None: + monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected") + request = AlertChainEmergencyRelayRequest.model_validate(payload()) + owner = build_alert_chain_emergency_owner(request) + + async def accepted(_payload): + return { + "schema_version": "alert_chain_emergency_ingress_result_v1", + "status": "ansible_candidate_queued_verifier_pending", + "candidate_persisted": True, + "candidate_queued": True, + "identity": { + "run_id": owner["run_id"], + "trace_id": owner["trace_id"], + "work_item_id": owner["work_item_id"], + }, + "agent99_role": "relay_coordination_only", + "executor": "host_ansible_executor", + "catalog_id": ALERT_CHAIN_CATALOG_ID, + "runtime_write_performed": False, + "runtime_closure_verified": False, + } + + monkeypatch.setattr( + webhooks_api, + "process_alert_chain_emergency_relay", + accepted, + ) + response = app_client().post( + "/api/v1/webhooks/agent99/alertmanager-emergency", + headers={"X-Agent99-Relay-Token": "expected"}, + json=payload(), + ) + + assert response.status_code == 202 + body = response.json() + assert body["identity"]["run_id"] == body["identity"]["trace_id"] + assert body["agent99_role"] == "relay_coordination_only" + assert body["executor"] == "host_ansible_executor" + assert body["runtime_write_performed"] is False diff --git a/apps/api/tests/test_alert_chain_runtime_truth.py b/apps/api/tests/test_alert_chain_runtime_truth.py new file mode 100644 index 000000000..c82f44d11 --- /dev/null +++ b/apps/api/tests/test_alert_chain_runtime_truth.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import re +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +CANONICAL_RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml" +MIRROR_RULES = ROOT / "ops" / "monitoring" / "alerts.yml" +K8S_RULES = ROOT / "k8s" / "monitoring" / "alert-chain-monitor.yaml" +PROMETHEUS_CONFIG = ROOT / "k8s" / "monitoring" / "prometheus.yml" +RECOVERY_PLAYBOOK = ( + ROOT / "infra" / "ansible" / "playbooks" / "110-alertmanager-delivery-recovery.yml" +) +RUNTIME_SCRAPE_DEPLOY = ROOT / "scripts" / "ops" / "deploy-alertmanager-runtime-scrape.sh" + + +def _alert_rule(path: Path) -> dict: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + groups = payload["spec"]["groups"] if "spec" in payload else payload["groups"] + return next( + rule + for group in groups + for rule in group.get("rules", []) + if rule.get("alert") == "AlertChainBroken_Alertmanager" + ) + + +def _normalized_expr(path: Path) -> str: + return re.sub(r"\s+", "", _alert_rule(path)["expr"]) + + +def test_alertmanager_delivery_truth_is_scraped_from_single_runtime() -> None: + payload = yaml.safe_load(PROMETHEUS_CONFIG.read_text(encoding="utf-8")) + job = next( + row + for row in payload["scrape_configs"] + if row.get("job_name") == "alertmanager-runtime" + ) + + assert job["metrics_path"] == "/metrics" + assert job["static_configs"] == [ + { + "targets": ["alertmanager:9093"], + "labels": { + "host": "110", + "service": "alertmanager", + "env": "prod", + }, + } + ] + blackbox = next( + row + for row in payload["scrape_configs"] + if row.get("job_name") == "blackbox-tcp" + ) + assert "192.168.0.110:9093" in blackbox["static_configs"][0]["targets"] + + +def test_alert_chain_rule_uses_monotonic_alertmanager_counters_in_all_sources() -> None: + expressions = { + _normalized_expr(path) + for path in (CANONICAL_RULES, MIRROR_RULES, K8S_RULES) + } + + assert len(expressions) == 1 + expression = expressions.pop() + assert "alertmanager_notification_requests_failed_total" in expression + assert "alertmanager_notification_requests_total" in expression + assert 'job="alertmanager-runtime"' in expression + assert 'integration="webhook"' in expression + assert 'receiver="awoooi-webhook"' in expression + assert ">=5" in expression + assert "awoooi_webhook_requests_total" not in expression + + +def test_alert_chain_rule_is_exact_asset_controlled_and_not_no_action() -> None: + for path in (CANONICAL_RULES, MIRROR_RULES, K8S_RULES): + rule = _alert_rule(path) + labels = rule["labels"] + assert labels["component"] == "alertmanager" + assert labels["auto_repair"] == "true" + assert labels["alert_category"] == "alert_chain_health" + assert rule["for"] == "5m" + if path in {CANONICAL_RULES, MIRROR_RULES}: + assert labels["layer"] == "host" + + +def test_runtime_scrape_and_recovery_preflight_require_primary_receiver_label() -> None: + for path in (RECOVERY_PLAYBOOK, RUNTIME_SCRAPE_DEPLOY): + source = path.read_text(encoding="utf-8") + assert 'receiver="awoooi-webhook"' in source + assert "alertmanager_notification_requests_total" in source + assert "alertmanager_notification_requests_failed_total" in source diff --git a/apps/api/tests/test_alertmanager_rule_bypass.py b/apps/api/tests/test_alertmanager_rule_bypass.py index 64f09135b..520d574c3 100644 --- a/apps/api/tests/test_alertmanager_rule_bypass.py +++ b/apps/api/tests/test_alertmanager_rule_bypass.py @@ -15,7 +15,9 @@ from src.api.v1.webhooks import ( _should_use_alertmanager_rule_first, ) from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES +from src.services import alert_rule_engine as alert_rule_engine_module from src.services.alert_approval_guard import ApprovalActionGuardResult +from src.services.alert_rule_engine import match_rule from src.services.alertmanager_llm_guard import ( ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS, alertmanager_llm_inflight_key, @@ -117,6 +119,84 @@ def test_generic_fallback_does_not_use_rule_first(): assert _should_use_alertmanager_rule_first(rule_response, "host_resource") is False +def test_exact_alertmanager_chain_rule_precedes_llm() -> None: + rule_response = { + "rule_id": "alert_chain_broken_alertmanager", + "suggested_action": "INVESTIGATE", + "kubectl_command": "", + } + + assert _should_use_alertmanager_rule_first( + rule_response, + "alert_chain_health", + ) is True + + +def test_critical_alertmanager_incident_keeps_bounded_execution_risk_medium() -> None: + rule_response = match_rule( + { + "labels": {"alertname": "AlertChainBroken_Alertmanager"}, + "alert_type": "alert_chain_health", + "message": "Alertmanager webhook delivery error rate above threshold", + "severity": "critical", + "target_resource": "alertmanager", + } + ) + + assert rule_response is not None + assert rule_response["rule_id"] == "alert_chain_broken_alertmanager" + assert rule_response["risk_level"] == "medium" + assert rule_response["suggested_action"] == "INVESTIGATE" + assert rule_response["kubectl_command"] == "" + + +def test_mutable_rule_cannot_self_authorize_medium_risk_for_critical_incident( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + alert_rule_engine_module, + "_load_rules", + lambda: [ + { + "id": "ai_generated_critical_override", + "priority": 1, + "match": {"alertname": ["CriticalGeneratedRule"]}, + "response": { + "action_title": "generated", + "description": "generated", + "risk": "medium", + "execution_risk_independent_of_incident_severity": True, + "suggested_action": "INVESTIGATE", + "kubectl_command": "", + }, + } + ], + ) + + rule_response = match_rule( + { + "labels": {"alertname": "CriticalGeneratedRule"}, + "severity": "critical", + } + ) + + assert rule_response is not None + assert rule_response["risk_level"] == "critical" + + +def test_other_alert_chain_rule_cannot_claim_alertmanager_exact_lane() -> None: + rule_response = { + "rule_id": "alert_chain_broken", + "suggested_action": "INVESTIGATE", + "kubectl_command": "", + } + + assert _should_use_alertmanager_rule_first( + rule_response, + "alert_chain_health", + ) is False + + def test_manual_gate_reasons_escalate_to_emergency_intervention(): assert _should_escalate_auto_approve_rejection("no_executable_action") is True assert _should_escalate_auto_approve_rejection("no_playbook") is True diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index d3b2559bc..04e55327f 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -4478,7 +4478,13 @@ def test_ai_route_lane_state_rejects_partial_gemini_execution_proof( @pytest.mark.asyncio async def test_ai_route_status_times_out_before_slow_provider_checks(monkeypatch) -> None: class SlowFailoverManager: - async def select_provider(self, task_type: str = "general") -> None: + async def select_provider( + self, + task_type: str = "general", + *, + emit_side_effects: bool = False, + ) -> None: + assert emit_side_effects is False await asyncio.sleep(0.05) async def fake_connectivity(endpoint): @@ -4554,7 +4560,13 @@ async def test_ai_route_status_lightweight_fallback_observes_claude_before_gemin monkeypatch, ) -> None: class SlowFailoverManager: - async def select_provider(self, task_type: str = "general") -> None: + async def select_provider( + self, + task_type: str = "general", + *, + emit_side_effects: bool = False, + ) -> None: + assert emit_side_effects is False await asyncio.sleep(0.05) async def fake_offline_connectivity(endpoint): diff --git a/apps/api/tests/test_claude_paid_fallback.py b/apps/api/tests/test_claude_paid_fallback.py index a39b8d0e4..46363e1f1 100644 --- a/apps/api/tests/test_claude_paid_fallback.py +++ b/apps/api/tests/test_claude_paid_fallback.py @@ -118,6 +118,40 @@ class _ClaudeTerminalResponse: } +@pytest.mark.asyncio +async def test_claude_stale_control_admission_blocks_before_reservation_or_http() -> None: + client = MagicMock() + client.is_closed = False + client.post = AsyncMock() + limiter = MagicMock() + limiter.reserve_generation = AsyncMock() + disabled_readback = AsyncMock(return_value=True) + provider = ClaudeProvider() + provider._http_client = client + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + disabled_readback, + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ): + result = await provider.analyze("sanitized metadata", context=_context()) + + assert result.success is False + assert result.error == "claude_disabled_by_runtime_control" + disabled_readback.assert_awaited_once_with( + "claude", + run_id="run-claude-1", + ) + limiter.reserve_generation.assert_not_awaited() + client.post.assert_not_awaited() + + class _ClaudeRefusalResponse: def raise_for_status(self) -> None: return None @@ -271,7 +305,11 @@ async def test_paid_provider_sensitive_context_blocks_before_control_cost_or_htt provider_settings = ( _settings(mode="production") if provider.name == "claude" - else SimpleNamespace(GEMINI_API_KEY="configured-test-value") + else SimpleNamespace( + GEMINI_API_KEY="configured-test-value", + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, + ) ) with patch(f"{provider_module}.settings", provider_settings), patch( diff --git a/apps/api/tests/test_cloud_transport_receipt.py b/apps/api/tests/test_cloud_transport_receipt.py new file mode 100644 index 000000000..2e4c1c87d --- /dev/null +++ b/apps/api/tests/test_cloud_transport_receipt.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest + +from src.services.cloud_transport_receipt import ( + CLOUD_TRANSPORT_RECEIPT_PREFIX, + issue_cloud_transport_receipt, + verify_cloud_transport_receipt, +) + + +class _Redis: + def __init__(self) -> None: + self.values: dict[str, str] = {} + self.set_calls: list[tuple[str, int | None]] = [] + + async def set(self, key: str, value: str, ex: int | None = None) -> bool: + self.values[key] = value + self.set_calls.append((key, ex)) + return True + + async def get(self, key: str) -> str | None: + return self.values.get(key) + + +async def _reachable_probe(provider: str, endpoint_url: str) -> bool: + assert provider in {"ollama_gcp_a", "ollama_gcp_b"} + assert endpoint_url.startswith("http://") + return True + + +def _context(receipt: dict[str, Any]) -> dict[str, Any]: + return { + "data_classification": "sanitized", + "trace_id": receipt["trace_id"], + "run_id": receipt["run_id"], + "work_item_id": receipt["work_item_id"], + "cloud_transport_receipts": [receipt], + } + + +@pytest.mark.asyncio +async def test_issue_and_verify_receipt_uses_network_probe_and_durable_readback() -> None: + store = _Redis() + now = datetime(2026, 7, 16, 5, 0, tzinfo=UTC) + endpoint = "http://gcp-a.internal.invalid:11434" + + receipt = await issue_cloud_transport_receipt( + trace_id="trace-1", + run_id="run-1", + work_item_id="AIA-SRE-013", + provider="ollama_gcp_a", + endpoint_url=endpoint, + redis_client=store, + network_probe=_reachable_probe, + now=now, + ) + + assert receipt["transport_security_status"] == "degraded_public_http" + assert receipt["transport_boundary"] == "public_http_sanitized_candidate_only" + assert receipt["durable_write_ack"] is True + assert receipt["verifier_status"] == "verified" + assert receipt["endpoint_value_persisted"] is False + assert endpoint not in json.dumps(receipt) + assert store.set_calls == [ + (f"{CLOUD_TRANSPORT_RECEIPT_PREFIX}{receipt['receipt_id']}", 90) + ] + assert ( + await verify_cloud_transport_receipt( + context=_context(receipt), + provider="ollama_gcp_a", + endpoint_url=endpoint, + redis_client=store, + now=now + timedelta(seconds=5), + ) + is None + ) + + +@pytest.mark.asyncio +async def test_caller_only_self_signed_receipt_is_not_executable() -> None: + store = _Redis() + now = datetime(2026, 7, 16, 5, 0, tzinfo=UTC) + receipt = await issue_cloud_transport_receipt( + trace_id="trace-1", + run_id="run-1", + work_item_id="AIA-SRE-013", + provider="ollama_gcp_a", + endpoint_url="http://gcp-a.invalid:11434", + redis_client=store, + network_probe=_reachable_probe, + now=now, + ) + store.values.clear() + + reason = await verify_cloud_transport_receipt( + context=_context(receipt), + provider="ollama_gcp_a", + endpoint_url="http://gcp-a.invalid:11434", + redis_client=store, + now=now + timedelta(seconds=5), + ) + + assert reason == "gcp_cloud_transport_receipt_not_durable" + + +@pytest.mark.asyncio +async def test_receipt_fails_closed_on_same_run_endpoint_or_ttl_mismatch() -> None: + store = _Redis() + now = datetime(2026, 7, 16, 5, 0, tzinfo=UTC) + receipt = await issue_cloud_transport_receipt( + trace_id="trace-1", + run_id="run-1", + work_item_id="AIA-SRE-013", + provider="ollama_gcp_b", + endpoint_url="http://gcp-b.invalid:11434", + redis_client=store, + network_probe=_reachable_probe, + now=now, + ) + + wrong_run = _context(receipt) + wrong_run["run_id"] = "run-2" + assert ( + await verify_cloud_transport_receipt( + context=wrong_run, + provider="ollama_gcp_b", + endpoint_url="http://gcp-b.invalid:11434", + redis_client=store, + now=now + timedelta(seconds=5), + ) + == "gcp_cloud_transport_receipt_binding_invalid" + ) + assert ( + await verify_cloud_transport_receipt( + context=_context(receipt), + provider="ollama_gcp_b", + endpoint_url="http://different.invalid:11434", + redis_client=store, + now=now + timedelta(seconds=5), + ) + == "gcp_cloud_transport_receipt_binding_invalid" + ) + assert ( + await verify_cloud_transport_receipt( + context=_context(receipt), + provider="ollama_gcp_b", + endpoint_url="http://gcp-b.invalid:11434", + redis_client=store, + now=now + timedelta(seconds=91), + ) + == "gcp_cloud_transport_receipt_expired" + ) + + +@pytest.mark.asyncio +async def test_failed_network_preflight_writes_no_false_green_receipt() -> None: + store = _Redis() + + async def _unreachable(_provider: str, _endpoint: str) -> bool: + return False + + with pytest.raises(RuntimeError, match="cloud_transport_endpoint_preflight_failed"): + await issue_cloud_transport_receipt( + trace_id="trace-1", + run_id="run-1", + work_item_id="AIA-SRE-013", + provider="ollama_gcp_a", + endpoint_url="http://gcp-a.invalid:11434", + redis_client=store, + network_probe=_unreachable, + ) + + assert store.values == {} + assert store.set_calls == [] diff --git a/apps/api/tests/test_failover_alerter.py b/apps/api/tests/test_failover_alerter.py index f7f69a3cb..00b5706e9 100644 --- a/apps/api/tests/test_failover_alerter.py +++ b/apps/api/tests/test_failover_alerter.py @@ -1,7 +1,7 @@ """FailoverAlerter 單元測試 — P1.5 Telegram 容災告警 四大 testcase（覆蓋 status 文件 line 99 指定範圍）： -1. test_alert_failover_dedup — 同 to_provider 第二次被 10min dedup +1. test_alert_failover_dedup — 同 canonical route state 第二次被 1h dedup 2. test_alert_recovery_send — 正常發送 + Markdown 訊息結構 3. test_no_telegram_chat_id_noop — chat_id 缺失時不發送（fail-soft） 4. test_quota_alert_dedup_24h — quota 告警 86400s TTL（每日一次） @@ -38,9 +38,24 @@ def _reset_singleton(): @pytest.fixture def mock_redis(): - """Mock Redis：set 第一次回 True（NX 成功），第二次回 None（已存在）""" + """Minimal stateful Redis mock for reservation/finalization truth.""" redis = MagicMock() - redis.set = AsyncMock(side_effect=[True, None, True, None]) + values: dict[str, str] = {} + + async def _set(key, value, *, ex=None, nx=False): + _ = ex + if nx and key in values: + return None + values[key] = value + return True + + async def _delete(key): + values.pop(key, None) + return 1 + + redis.set = AsyncMock(side_effect=_set) + redis.delete = AsyncMock(side_effect=_delete) + redis.values = values return redis @@ -51,15 +66,41 @@ def mock_telegram_send(): `_send()` 在函式內 inline import，必須 mock 來源 module 而非 alerter module。 """ with patch("src.services.telegram_gateway.get_telegram_gateway") as mock_gw: + from src.services import telegram_gateway as gateway_module + + destination_binding = gateway_module._telegram_destination_binding(-1001) gateway = MagicMock() - gateway.send_alert_notification = AsyncMock() + gateway.send_alert_notification = AsyncMock(return_value={ + "ok": True, + "_awooop_delivery_status": "sent", + "_awooop_provider_send_performed": True, + "result": {"message_id": 42, "chat": {"id": -1001}}, + "_awoooi_canonical_route_receipt": { + "schema_version": "telegram_canonical_egress_receipt_v1", + "decision": "allowed", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": destination_binding, + "sender_bot_alias": "tsenyang_bot", + "provider_send_performed": True, + }, + gateway_module._DELIVERY_CONTEXT_KEY: { + "schema_version": "telegram_delivery_context_v1", + "method": "sendMessage", + "sender_bot_alias": "tsenyang_bot", + "destination_alias": "awoooi_sre_war_room", + "destination_binding": destination_binding, + "payload_destination_binding": destination_binding, + "provider_destination_binding": destination_binding, + "destination_binding_verified": True, + }, + }) gateway.canonical_destination_chat_id.return_value = "sre-room" mock_gw.return_value = gateway yield gateway # ============================================================================= -# Case 1: failover dedup（同 to_provider 第二次被攔） +# Case 1: route-degraded observation dedup（同 canonical state 第二次被攔） # ============================================================================= @pytest.mark.asyncio @@ -67,10 +108,13 @@ async def test_alert_failover_dedup(mock_redis, mock_telegram_send): alerter = FailoverAlerter(redis_client=mock_redis) event = { - "to_provider": "gemini", - "reason": "111 unhealthy", - "model": "qwen3:8b", - "fallback_chain_str": "gemini → ollama_local", + "from_provider": "ollama_gcp_a", + "to_provider": "ollama_gcp_b", + "primary_health": "offline", + "candidate_health": "healthy", + # Legacy fields must never leak into the observation card. + "failed_host": "http://192.168.0.110:11435", + "model": "gemma3:4b", } # 第 1 次：dedup pass，發送 @@ -81,9 +125,30 @@ async def test_alert_failover_dedup(mock_redis, mock_telegram_send): await alerter.alert_failover(event) assert mock_telegram_send.send_alert_notification.await_count == 1 # 仍是 1 - # 驗證 dedup TTL = 10min + # 驗證 canonical from/to/state fingerprint + 至少 1h TTL。 + reservation_key = mock_redis.set.await_args_list[0].args[0] + assert reservation_key.startswith( + "alert:ollama_route_degraded:ollama_gcp_a:ollama_gcp_b:" + "degraded_health_observation:offline:healthy" + ) assert mock_redis.set.await_args_list[0].kwargs["ex"] == DEDUP_TTL_SEC assert mock_redis.set.await_args_list[0].kwargs["nx"] is True + assert DEDUP_TTL_SEC >= 3600 + + sent = mock_telegram_send.send_alert_notification.await_args.kwargs["text"] + assert "Ollama 路由健康降級" in sent + assert "AI／Agent 實際工作" in sent + assert "ollama\\_health\\_monitor" in sent + assert "Executor：none" in sent + assert "Verifier：health\\_probe" in sent + assert "LLM：none" in sent + assert "Receipt：not\\_created\\_for\\_observation" in sent + assert "Evidence：health\\_observation\\_only" in sent + assert "容災激活" not in sent + assert "切換目標" not in sent + assert "自動切回" not in sent + assert "192\\.168\\.0\\.110" not in sent + assert "gemma3" not in sent # ============================================================================= diff --git a/apps/api/tests/test_failover_e2e_dispatch.py b/apps/api/tests/test_failover_e2e_dispatch.py index 902b464fd..e257f8153 100644 --- a/apps/api/tests/test_failover_e2e_dispatch.py +++ b/apps/api/tests/test_failover_e2e_dispatch.py @@ -53,6 +53,14 @@ def test_ollama_local_provider_privacy_level(): assert p.privacy_level == "local" +def test_gcp_ollama_providers_are_not_local_data_boundaries(): + """GCP-A/B must never satisfy require_local even though Ollama is self-hosted.""" + from src.services.ai_providers.ollama import OllamaGcpBProvider, OllamaProvider + + assert OllamaProvider().privacy_level == "cloud" + assert OllamaGcpBProvider().privacy_level == "cloud" + + # ============================================================================= # B1：is_enabled 邏輯 # ============================================================================= diff --git a/apps/api/tests/test_gemini_cost_guard.py b/apps/api/tests/test_gemini_cost_guard.py index 346370e1b..b07532312 100644 --- a/apps/api/tests/test_gemini_cost_guard.py +++ b/apps/api/tests/test_gemini_cost_guard.py @@ -12,6 +12,7 @@ from src.services.ai_rate_limiter import ( _FINALIZE_GENERATION_LUA, _RESERVE_GENERATION_LUA, _WRITE_BLOCKED_GENERATION_RECEIPT_LUA, + PAID_RUN_BUDGET_KEY, AIGenerationReservation, AIRateLimiter, get_gemini_pricing_policy, @@ -28,7 +29,9 @@ def _gemini_runtime_control_enabled(monkeypatch: pytest.MonkeyPatch) -> None: ) -def _reservation(*, allowed: bool = True, reason: str = "allowed") -> AIGenerationReservation: +def _reservation( + *, allowed: bool = True, reason: str = "allowed" +) -> AIGenerationReservation: pricing = get_gemini_pricing_policy("gemini-2.5-flash-lite") assert pricing is not None return AIGenerationReservation( @@ -50,7 +53,9 @@ def _reservation(*, allowed: bool = True, reason: str = "allowed") -> AIGenerati class _ReserveRedis: - def __init__(self, result: list[object] | None = None, error: Exception | None = None) -> None: + def __init__( + self, result: list[object] | None = None, error: Exception | None = None + ) -> None: self.result = result or [1, b"allowed"] self.error = error self.eval_calls: list[tuple[object, ...]] = [] @@ -120,11 +125,122 @@ class _ReceiptPipeline: return [1, 1] +class _ReceiptReadRedis: + async def hgetall(self, _key: str) -> dict[bytes, bytes]: + return { + b"provider": b"gemini", + b"receipt_id": b"receipt-readback-1", + b"status": b"succeeded", + b"terminal_state": b"finalized_accounted", + b"success": b"1", + b"trace_id": b"trace-readback", + b"run_id": b"run-readback", + b"work_item_id": b"AIA-SRE-013", + b"model": b"gemini-2.5-flash-lite", + b"pricing_version": b"pricing-v1", + b"usage_source": b"provider_metadata", + b"accounted_tokens": b"120", + b"accounted_cost_micro_usd": b"42", + b"run_budget_required": b"1", + b"run_budget_id": b"paid-run-readback", + b"run_budget_max_micro_usd": b"250000", + b"run_budget_reserved_cost_micro_usd": b"0", + b"reserved_prompt_tokens": b"0", + b"reserved_completion_tokens": b"0", + b"reserved_total_tokens": b"0", + b"reserved_cost_micro_usd": b"0", + } + + +@pytest.mark.asyncio +async def test_generation_receipt_readback_is_allowlisted_and_correlated() -> None: + limiter = AIRateLimiter() + limiter._redis = _ReceiptReadRedis() + + receipt = await limiter.get_generation_receipt_readback( + "gemini", + "receipt-readback-1", + trace_id="trace-readback", + run_id="run-readback", + work_item_id="AIA-SRE-013", + ) + + assert receipt["found"] is True + assert receipt["correlation_verified"] is True + assert receipt["finalized_accounted"] is True + assert receipt["reservation_released"] is True + assert receipt["accounted_tokens"] == 120 + assert receipt["accounted_cost_usd"] == 0.000042 + assert receipt["accounted_cost_micro_usd"] == 42 + assert receipt["run_budget_required"] is True + assert receipt["raw_prompt_persisted"] is False + assert "trace_id" not in receipt + + class _UsageRedis: async def get(self, _key: str) -> None: return None +class _RunBudgetReadRedis: + async def hgetall(self, _key: str) -> dict[bytes, bytes]: + return { + b"schema_version": b"ai_paid_run_budget_receipt_v1", + b"run_budget_id": b"paid-run-6b098d09eeef680a5b119ca0b4d31713", + b"run_id": b"run-readback", + b"work_item_id": b"AIA-SRE-013", + b"status": b"finalized", + b"max_cost_micro_usd": b"250000", + b"reserved_cost_micro_usd": b"0", + b"accounted_cost_micro_usd": b"2000", + b"reservation_count": b"2", + b"finalized_count": b"2", + } + + +@pytest.mark.asyncio +async def test_paid_run_budget_readback_is_allowlisted_and_exactly_correlated() -> None: + limiter = AIRateLimiter() + run_id = "run-readback" + work_item_id = "AIA-SRE-013" + budget_hash = limiter._identity_hash(run_id, work_item_id) + budget_id = f"paid-run-{budget_hash[:32]}" + redis = _RunBudgetReadRedis() + + async def _hgetall(_key: str) -> dict[bytes, bytes]: + payload = await _RunBudgetReadRedis().hgetall(_key) + payload[b"run_budget_id"] = budget_id.encode() + return payload + + redis.hgetall = _hgetall # type: ignore[method-assign] + limiter._redis = redis + + receipt = await limiter.get_paid_run_budget_readback( + run_id=run_id, + work_item_id=work_item_id, + expected_max_cost_micro_usd=250_000, + ) + + assert receipt == { + "schema_version": "ai_paid_run_budget_receipt_v1", + "run_budget_id": budget_id, + "found": True, + "status": "finalized", + "max_cost_micro_usd": 250_000, + "reserved_cost_micro_usd": 0, + "accounted_cost_micro_usd": 2000, + "accounted_cost_usd": 0.002, + "reservation_count": 2, + "finalized_count": 2, + "correlation_verified": True, + "cap_verified": True, + "terminal_verified": True, + "raw_prompt_persisted": False, + "raw_response_persisted": False, + "secret_value_present": False, + } + + @pytest.mark.asyncio async def test_gemini_reservation_is_one_atomic_eval_with_all_hard_limits( monkeypatch: pytest.MonkeyPatch, @@ -150,7 +266,7 @@ async def test_gemini_reservation_is_one_atomic_eval_with_all_hard_limits( assert len(redis.eval_calls) == 1 call = redis.eval_calls[0] assert call[0] == _RESERVE_GENERATION_LUA - assert call[1] == 11 + assert call[1] == 12 serialized = " ".join(str(value) for value in call) assert "must-not-be-stored" not in serialized assert "daily_request_limit" in _RESERVE_GENERATION_LUA @@ -162,16 +278,143 @@ async def test_gemini_reservation_is_one_atomic_eval_with_all_hard_limits( assert "pricing_source" in _RESERVE_GENERATION_LUA assert "pricing_version" in _RESERVE_GENERATION_LUA assert "pricing_checked_at" in _RESERVE_GENERATION_LUA + assert "run_cost_limit" in _RESERVE_GENERATION_LUA assert "'EXPIRE', KEYS[8]" not in _RESERVE_GENERATION_LUA -def test_paid_reservation_identity_and_receipts_have_no_ttl_reopen_path() -> None: - assert "redis.call('SET', KEYS[10], ARGV[10], 'NX')" in ( - _RESERVE_GENERATION_LUA +@pytest.mark.asyncio +async def test_paid_canary_cap_is_part_of_the_same_atomic_reservation_eval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _ReserveRedis() + limiter = AIRateLimiter() + limiter._redis = redis + monkeypatch.setattr(limiter, "_get_today", lambda: "2026-07-14") + monkeypatch.setattr(limiter, "_seconds_until_tomorrow", lambda: 3600) + + reservation = await limiter.reserve_generation( + "gemini", + "sanitized canary", + context={ + "trace_id": "trace-paid-canary", + "run_id": "run-paid-canary", + "work_item_id": "AIA-SRE-013", + "synthetic_validation": True, + "paid_canary_max_cost_usd": 0.25, + }, ) + + assert reservation.allowed is True + assert reservation.run_budget_id.startswith("paid-run-") + assert reservation.run_budget_max_micro_usd == 250_000 + assert len(redis.eval_calls) == 1 + call = redis.eval_calls[0] + assert call[0] == _RESERVE_GENERATION_LUA + assert call[1] == 12 + assert call[13] == PAID_RUN_BUDGET_KEY.format(budget_id=reservation.run_budget_id) + assert call[-3:] == (1, reservation.run_budget_id, 250_000) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("cap_marker", "expected_reason"), + [ + (None, "run_cost_cap_missing"), + (True, "run_cost_cap_invalid"), + (0, "run_cost_cap_invalid"), + (-1, "run_cost_cap_invalid"), + (0.0000001, "run_cost_cap_invalid"), + (0.250001, "run_cost_cap_invalid"), + ("NaN", "run_cost_cap_invalid"), + ("Infinity", "run_cost_cap_invalid"), + ("not-a-number", "run_cost_cap_invalid"), + ], +) +async def test_paid_canary_missing_or_invalid_cap_fails_closed_before_reservation( + cap_marker: object, + expected_reason: str, +) -> None: + redis = _ReserveRedis() + limiter = AIRateLimiter() + limiter._redis = redis + context = { + "trace_id": "trace-invalid-cap", + "run_id": f"run-invalid-cap-{expected_reason}", + "work_item_id": "AIA-SRE-013", + "synthetic_validation": True, + } + if cap_marker is not None: + context["paid_canary_max_cost_usd"] = cap_marker + + reservation = await limiter.reserve_generation( + "gemini", + "sanitized canary", + context=context, + ) + + assert reservation.allowed is False + assert reservation.reason == expected_reason + assert len(redis.eval_calls) == 1 + assert redis.eval_calls[0][0] == _WRITE_BLOCKED_GENERATION_RECEIPT_LUA + assert redis.hsets[0][1]["terminal_state"] == "blocked_no_write" + assert redis.hsets[0][1]["run_budget_required"] == "1" + + +@pytest.mark.asyncio +async def test_unrelated_synthetic_call_does_not_inherit_aia_sre_013_run_cap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _ReserveRedis() + limiter = AIRateLimiter() + limiter._redis = redis + monkeypatch.setattr(limiter, "_get_today", lambda: "2026-07-14") + monkeypatch.setattr(limiter, "_seconds_until_tomorrow", lambda: 3600) + + reservation = await limiter.reserve_generation( + "gemini", + "unrelated synthetic fixture", + context={ + "trace_id": "trace-unrelated-synthetic", + "run_id": "run-unrelated-synthetic", + "work_item_id": "OTHER-SYNTHETIC-WORK", + "synthetic_validation": True, + }, + ) + + assert reservation.allowed is True + assert reservation.run_budget_id == "" + assert reservation.run_budget_max_micro_usd == 0 + + +@pytest.mark.asyncio +async def test_aia_sre_013_requires_exact_synthetic_marker_before_reservation() -> None: + redis = _ReserveRedis() + limiter = AIRateLimiter() + limiter._redis = redis + + reservation = await limiter.reserve_generation( + "gemini", + "canary missing exact marker", + context={ + "trace_id": "trace-canary-marker-missing", + "run_id": "run-canary-marker-missing", + "work_item_id": "AIA-SRE-013", + "paid_canary_max_cost_usd": 0.25, + }, + ) + + assert reservation.allowed is False + assert reservation.reason == "run_cost_cap_context_invalid" + assert redis.eval_calls[0][0] == _WRITE_BLOCKED_GENERATION_RECEIPT_LUA + + +def test_paid_reservation_identity_and_receipts_have_no_ttl_reopen_path() -> None: + assert "redis.call('SET', KEYS[10], ARGV[10], 'NX')" in (_RESERVE_GENERATION_LUA) assert "redis.call('EXPIRE', KEYS[9]" not in _RESERVE_GENERATION_LUA assert "redis.call('EXPIRE', KEYS[10]" not in _RESERVE_GENERATION_LUA + assert "redis.call('EXPIRE', KEYS[12]" not in _RESERVE_GENERATION_LUA assert "redis.call('EXPIRE', KEYS[7]" not in _FINALIZE_GENERATION_LUA + assert "redis.call('EXPIRE', KEYS[10]" not in _FINALIZE_GENERATION_LUA assert "EXPIRE" not in _WRITE_BLOCKED_GENERATION_RECEIPT_LUA @@ -283,8 +526,7 @@ async def test_gemini_usage_readback_publishes_limits_and_pricing_policy() -> No "model": "gemini-2.5-flash-lite", "supported": True, "source": ( - "https://ai.google.dev/gemini-api/docs/pricing" - "#gemini-2.5-flash-lite" + "https://ai.google.dev/gemini-api/docs/pricing" "#gemini-2.5-flash-lite" ), "version": "google-ai-standard-text-2026-07-14", "checked_at": "2026-07-14", @@ -338,7 +580,7 @@ async def test_gemini_finalize_uses_atomic_receipt_transition( assert finalized is True assert redis.eval_calls[0][0] == _FINALIZE_GENERATION_LUA - assert redis.eval_calls[0][1] == 9 + assert redis.eval_calls[0][1] == 10 @pytest.mark.asyncio @@ -378,6 +620,8 @@ async def test_gemini_provider_does_not_call_api_when_cost_guard_blocks() -> Non GEMINI_API_KEY="configured", GEMINI_MODEL="gemini-2.5-flash-lite", GEMINI_MAX_OUTPUT_TOKENS=2048, + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, ), ), patch( "src.services.ai_rate_limiter.get_ai_rate_limiter", @@ -407,17 +651,22 @@ async def test_gemini_provider_does_not_reserve_or_call_when_runtime_disabled( client.post = AsyncMock() limiter = MagicMock() limiter.reserve_generation = AsyncMock() + disabled_readback = AsyncMock(return_value=True) monkeypatch.setattr( ai_control_module, "is_provider_disabled", - AsyncMock(return_value=True), + disabled_readback, ) provider = GeminiProvider() provider._http_client = client with patch( "src.services.ai_providers.gemini.settings", - SimpleNamespace(GEMINI_API_KEY="configured-test-value"), + SimpleNamespace( + GEMINI_API_KEY="configured-test-value", + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, + ), ), patch( "src.services.ai_rate_limiter.get_ai_rate_limiter", return_value=limiter, @@ -434,6 +683,10 @@ async def test_gemini_provider_does_not_reserve_or_call_when_runtime_disabled( assert result.success is False assert result.error == "gemini_disabled_by_runtime_control" + disabled_readback.assert_awaited_once_with( + "gemini", + run_id="run-disabled", + ) limiter.reserve_generation.assert_not_awaited() client.post.assert_not_awaited() @@ -457,6 +710,8 @@ async def test_gemini_provider_failure_finalizes_estimate_charged_receipt() -> N GEMINI_API_KEY="configured", GEMINI_MODEL="gemini-2.5-flash-lite", GEMINI_MAX_OUTPUT_TOKENS=2048, + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, ), ), patch( "src.services.ai_providers.gemini.get_model_registry", @@ -507,6 +762,8 @@ async def test_gemini_success_without_usage_metadata_is_not_zero_false_green() - GEMINI_API_KEY="configured", GEMINI_MODEL="gemini-2.5-flash-lite", GEMINI_MAX_OUTPUT_TOKENS=2048, + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, ), ), patch( "src.services.ai_providers.gemini.get_model_registry", diff --git a/apps/api/tests/test_gemini_provider_security.py b/apps/api/tests/test_gemini_provider_security.py index 68d34ea2f..1c282fbe7 100644 --- a/apps/api/tests/test_gemini_provider_security.py +++ b/apps/api/tests/test_gemini_provider_security.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -14,6 +15,15 @@ from src.services.ai_rate_limiter import ( ) +def _gemini_canary_run_id(percent: int) -> tuple[str, int]: + for index in range(1000): + run_id = f"gemini-canary-{index}" + bucket = int(hashlib.sha256(run_id.encode()).hexdigest()[:8], 16) % 100 + if bucket < percent: + return run_id, bucket + raise AssertionError("missing deterministic canary bucket") + + @pytest.fixture(autouse=True) def _gemini_runtime_control_enabled(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("ENABLE_GEMINI", "true") @@ -30,7 +40,7 @@ class _FakeGeminiResponse: def json(self) -> dict: return { - "candidates": [{"content": {"parts": [{"text": "{\"ok\": true}"}]}}], + "candidates": [{"content": {"parts": [{"text": '{"ok": true}'}]}}], "usageMetadata": { "promptTokenCount": 3, "candidatesTokenCount": 2, @@ -40,6 +50,68 @@ class _FakeGeminiResponse: } +@pytest.mark.asyncio +async def test_gemini_shadow_and_unselected_canary_perform_no_external_call() -> None: + provider = GeminiProvider() + client = MagicMock() + client.is_closed = False + client.post = AsyncMock() + provider._http_client = client + + with patch( + "src.services.ai_providers.gemini.settings", + SimpleNamespace( + GEMINI_API_KEY="configured-test-value", + GEMINI_EXECUTION_MODE="shadow", + GEMINI_CANARY_PERCENT=0, + ), + ): + shadow = await provider.analyze( + "synthetic", + context={"data_classification": "sanitized", "run_id": "run-shadow"}, + ) + + selected_run, selected_bucket = _gemini_canary_run_id(5) + unselected_run = next( + f"gemini-unselected-{index}" + for index in range(1000) + if int( + hashlib.sha256(f"gemini-unselected-{index}".encode()).hexdigest()[:8], + 16, + ) + % 100 + >= 5 + ) + with patch( + "src.services.ai_providers.gemini.settings", + SimpleNamespace( + GEMINI_API_KEY="configured-test-value", + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ), + ): + unselected = await provider.analyze( + "synthetic", + context={ + "data_classification": "sanitized", + "run_id": unselected_run, + }, + ) + allowed, reason, mode, bucket = provider._execution_mode_gate( + {"run_id": selected_run} + ) + + assert shadow.error == "gemini_shadow_metadata_only" + assert unselected.error == "gemini_canary_not_selected" + assert (allowed, reason, mode, bucket) == ( + True, + "gemini_canary_selected", + "canary", + selected_bucket, + ) + client.post.assert_not_called() + + @pytest.mark.asyncio async def test_gemini_provider_uses_header_auth_not_query_string(): client = MagicMock() @@ -79,6 +151,8 @@ async def test_gemini_provider_uses_header_auth_not_query_string(): GEMINI_API_KEY="fake-key", GEMINI_MODEL="gemini-2.5-flash-lite", GEMINI_MAX_OUTPUT_TOKENS=2048, + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, ), ), patch( "src.services.ai_providers.gemini.get_model_registry", @@ -170,6 +244,8 @@ async def test_authentication_rejection_writes_bounded_receipt_code() -> None: GEMINI_API_KEY="configured-test-value", GEMINI_MODEL="gemini-2.5-flash-lite", GEMINI_MAX_OUTPUT_TOKENS=1, + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, ), ), patch( "src.services.ai_providers.gemini.get_model_registry", @@ -195,9 +271,7 @@ async def test_authentication_rejection_writes_bounded_receipt_code() -> None: success=False, error_code="authentication_rejected", ) - assert "SECRET_PROVIDER_PAYLOAD_MUST_NOT_BE_LOGGED" not in str( - warning.call_args - ) + assert "SECRET_PROVIDER_PAYLOAD_MUST_NOT_BE_LOGGED" not in str(warning.call_args) assert warning.call_args.kwargs["error_code"] == "authentication_rejected" assert warning.call_args.kwargs["error_type"] == "HTTPStatusError" assert "error" not in warning.call_args.kwargs @@ -217,7 +291,11 @@ async def test_health_requires_durable_authenticated_verified_receipt() -> None: with patch( "src.services.ai_providers.gemini.settings", - SimpleNamespace(GEMINI_API_KEY="configured-test-value"), + SimpleNamespace( + GEMINI_API_KEY="configured-test-value", + GEMINI_EXECUTION_MODE="production", + GEMINI_CANARY_PERCENT=0, + ), ), patch( "src.services.ai_providers.gemini.is_provider_enabled_by_env", return_value=True, diff --git a/apps/api/tests/test_gemini_route_manifest.py b/apps/api/tests/test_gemini_route_manifest.py index ebfb5bdae..cc497f988 100644 --- a/apps/api/tests/test_gemini_route_manifest.py +++ b/apps/api/tests/test_gemini_route_manifest.py @@ -31,13 +31,18 @@ def test_prod_and_dev_publish_exact_five_hop_fallback() -> None: "k8s/awoooi-dev/02-configmap.yaml", ): data = _load(path)["data"] + production = "awoooi-prod" in path assert json.loads(data["AI_FALLBACK_ORDER"]) == PRODUCTION_PROVIDER_ORDER assert data["ALERT_AI_ENFORCE_OLLAMA_FIRST"] == "true" assert data["ALERT_AI_STRICT_PROVIDER_CHAIN"] == "true" assert data["CLAUDE_MODEL"] == "claude-sonnet-5" - assert data["ENABLE_CLAUDE"] == "false" - assert data["CLAUDE_EXECUTION_MODE"] == "shadow" - assert data["CLAUDE_CANARY_PERCENT"] == "0" + assert data["ENABLE_CLAUDE"] == ("true" if production else "false") + assert data["CLAUDE_EXECUTION_MODE"] == ( + "canary" if production else "shadow" + ) + assert data["CLAUDE_CANARY_PERCENT"] == ( + "5" if production else "0" + ) assert data["CLAUDE_DAILY_QUOTA"] == "50" assert data["CLAUDE_RPM_LIMIT"] == "2" assert data["CLAUDE_DAILY_TOKEN_LIMIT"] == "100000" @@ -51,6 +56,13 @@ def test_prod_and_dev_publish_exact_five_hop_fallback() -> None: assert data["GEMINI_DAILY_COST_LIMIT_USD"] == "5.0" assert data["GEMINI_TOTAL_COST_LIMIT_USD"] == "5.0" assert data["GEMINI_COST_ALERT_THRESHOLD_USD"] == "4.0" + assert data["ENABLE_GEMINI"] == ("true" if production else "false") + assert data["GEMINI_EXECUTION_MODE"] == ( + "canary" if production else "shadow" + ) + assert data["GEMINI_CANARY_PERCENT"] == ( + "5" if production else "0" + ) def test_model_registry_publishes_only_supported_paid_fallback_model() -> None: @@ -125,9 +137,9 @@ def test_prod_runtime_endpoints_and_nemotron_override_match_strict_chain() -> No for item in deployment["spec"]["template"]["spec"]["containers"][0]["env"] } - assert env["OLLAMA_URL"] == "http://192.168.0.110:11435" - assert env["OLLAMA_SECONDARY_URL"] == "http://192.168.0.110:11436" - assert env["OLLAMA_FALLBACK_URL"] == "http://192.168.0.110:11437" + assert env["OLLAMA_URL"] == "http://34.143.170.20:11434" + assert env["OLLAMA_SECONDARY_URL"] == "http://34.21.145.224:11434" + assert env["OLLAMA_FALLBACK_URL"] == "http://192.168.0.111:11434" assert env["ENABLE_NEMOTRON_COLLABORATION"] == "false" diff --git a/apps/api/tests/test_generate_monitoring_stabilization.py b/apps/api/tests/test_generate_monitoring_stabilization.py index d9a178fbc..87e8b05b3 100644 --- a/apps/api/tests/test_generate_monitoring_stabilization.py +++ b/apps/api/tests/test_generate_monitoring_stabilization.py @@ -5,7 +5,6 @@ import sys import unittest from pathlib import Path - SCRIPT_PATH = Path(__file__).resolve().parents[3] / "scripts" / "generate_monitoring.py" SPEC = importlib.util.spec_from_file_location("generate_monitoring", SCRIPT_PATH) generate_monitoring = importlib.util.module_from_spec(SPEC) @@ -14,9 +13,18 @@ sys.dont_write_bytecode = True SPEC.loader.exec_module(generate_monitoring) -def targets_payload(down_jobs: set[str] | None = None, missing_jobs: set[str] | None = None): +def targets_payload( + down_jobs: set[str] | None = None, + missing_jobs: set[str] | None = None, + down_targets: set[str] | None = None, + missing_targets: set[str] | None = None, + duplicate_targets: set[str] | None = None, +): down_jobs = down_jobs or set() missing_jobs = missing_jobs or set() + down_targets = down_targets or set() + missing_targets = missing_targets or set() + duplicate_targets = duplicate_targets or set() active_targets = [] for job in generate_monitoring.EXPECTED_JOBS: @@ -29,6 +37,17 @@ def targets_payload(down_jobs: set[str] | None = None, missing_jobs: set[str] | } ) + for target_id, expected in generate_monitoring.EXPECTED_TARGETS.items(): + if target_id in missing_targets: + continue + target = { + "labels": dict(expected["labels"]), + "health": "down" if target_id in down_targets else "up", + } + active_targets.append(target) + if target_id in duplicate_targets: + active_targets.append(dict(target)) + return {"activeTargets": active_targets} @@ -73,6 +92,78 @@ class GenerateMonitoringStabilizationTest(unittest.TestCase): self.assertEqual(report["stabilization"]["attempt"], 2) self.assertEqual(report["stabilization"]["status"], "cleared") + def test_healthy_blackbox_job_cannot_hide_missing_gcp_provider_targets(self): + report = generate_monitoring.build_stabilized_report( + lambda: targets_payload( + missing_targets=set(generate_monitoring.EXPECTED_TARGETS) + ), + attempts=1, + sleep_seconds=0, + emit_status=False, + ) + + self.assertEqual(report["summary"]["expected_coverage_pct"], 100.0) + self.assertEqual( + report["missing_expected_targets"], + ["ollama_gcp_a_cloud_edge", "ollama_gcp_b_cloud_edge"], + ) + self.assertEqual(report["stabilization"]["status"], "failed") + + def test_stabilized_report_retries_down_gcp_target(self): + snapshots = [ + targets_payload(down_targets={"ollama_gcp_a_cloud_edge"}), + targets_payload(), + ] + + report = generate_monitoring.build_stabilized_report( + lambda: snapshots.pop(0), + attempts=3, + sleep_seconds=0, + emit_status=False, + ) + + self.assertEqual(report["down_expected_targets"], []) + self.assertEqual(report["stabilization"]["attempt"], 2) + self.assertEqual(report["stabilization"]["status"], "cleared") + + def test_duplicate_exact_provider_target_fails_closed(self): + report = generate_monitoring.build_stabilized_report( + lambda: targets_payload( + duplicate_targets={"ollama_gcp_b_cloud_edge"} + ), + attempts=1, + sleep_seconds=0, + emit_status=False, + ) + + self.assertEqual( + report["ambiguous_expected_targets"], ["ollama_gcp_b_cloud_edge"] + ) + self.assertEqual(report["stabilization"]["status"], "failed") + + def test_retired_github_exporter_is_policy_drift_not_extra_coverage(self): + payload = targets_payload() + payload["activeTargets"].append( + { + "labels": { + "job": "github-actions", + "instance": "retired-exporter:9504", + }, + "health": "up", + } + ) + + report = generate_monitoring.build_stabilized_report( + lambda: payload, + attempts=1, + sleep_seconds=0, + emit_status=False, + ) + + self.assertEqual(report["forbidden_jobs_present"], ["github-actions"]) + self.assertEqual(report["summary"]["forbidden_jobs_present"], 1) + self.assertEqual(report["stabilization"]["status"], "failed") + def test_stabilized_report_keeps_real_down_after_attempts_exhausted(self): def fetch_targets(): return targets_payload(down_jobs={"awoooi-api"}) diff --git a/apps/api/tests/test_host111_ollama_ansible_catalog.py b/apps/api/tests/test_host111_ollama_ansible_catalog.py new file mode 100644 index 000000000..bb6964fb7 --- /dev/null +++ b/apps/api/tests/test_host111_ollama_ansible_catalog.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + +from src.services.awooop_ansible_audit_service import ( + _catalog_hints, + get_ansible_catalog_item, +) +from src.services.awooop_ansible_post_verifier import postconditions_for_catalog +from src.services.controlled_alert_target_router import resolve_typed_alert_target + +ROOT = Path(__file__).resolve().parents[3] +PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "111-ollama-fallback.yml" +MONITORING_REGISTRY = ROOT / "ops" / "monitoring" / "service-registry.yaml" +CANONICAL_ALERTS = ROOT / "ops" / "monitoring" / "alerts-unified.yml" +MIRROR_ALERTS = ROOT / "ops" / "monitoring" / "alerts.yml" +CATALOG_ID = "ansible:111-ollama-fallback" +EXPECTED_ALLOWLIST = ( + "127.0.0.1/32,192.168.0.111/32,192.168.0.188/32," + "192.168.0.120/32,192.168.0.121/32" +) + + +def _incident(alertname: str = "OllamaLocalUnavailable") -> dict: + return { + "incident_id": "INC-OLLAMA-111-UNAVAILABLE", + "project_id": "awoooi", + "alertname": alertname, + "alert_category": "ai_provider", + "affected_services": ["ollama-local"], + "signals": [ + { + "alert_name": alertname, + "labels": { + "component": "ollama-local", + "host": "111", + }, + } + ], + } + + +def test_host111_alerts_route_to_exact_launchagent_lane() -> None: + for alertname in ("OllamaLocalUnavailable", "Ollama111Unavailable"): + route = resolve_typed_alert_target( + alertname=alertname, + target_resource="ollama-local", + namespace="", + labels={"host": "111"}, + alert_category="ai_provider", + ) + + assert route["resolution_status"] == "resolved" + assert route["target_kind"] == "host_launchagent" + assert route["canonical_asset_id"] == "ai-provider:ollama_local" + assert route["host"] == "192.168.0.111" + assert route["executor"] == "host_ansible_executor" + assert route["verifier"] == "ollama111_k3s_path_independent_verifier" + assert route["risk_class"] == "medium" + assert route["allowed_catalog_ids"] == [CATALOG_ID] + assert route["allowed_inventory_hosts"] == [ + "host_111", + "host_120", + "host_121", + ] + assert route["cross_domain_fallback_allowed"] is False + + +def test_host111_alert_selects_one_exact_bounded_catalog() -> None: + hints = _catalog_hints(_incident(), None) + + assert hints["match_mode"] == "typed_domain_router_v2" + assert hints["decision_effect"] == "bounded_domain_candidate" + assert [row["catalog_id"] for row in hints["candidates"]] == [CATALOG_ID] + candidate = hints["candidates"][0] + assert candidate["inventory_hosts"] == ["host_111", "host_120", "host_121"] + assert candidate["supports_check_mode"] is True + assert candidate["auto_apply_enabled"] is True + assert candidate["approval_required"] is False + assert candidate["risk_level"] == "medium" + assert hints["cross_domain_fallback_allowed"] is False + + +def test_host111_catalog_and_postconditions_cover_all_runtime_origins() -> None: + catalog = get_ansible_catalog_item(CATALOG_ID) + conditions = postconditions_for_catalog(CATALOG_ID) + + assert catalog is not None + assert catalog["playbook_path"].endswith("111-ollama-fallback.yml") + assert catalog["check_mode_playbook_path"] == catalog["playbook_path"] + assert catalog["canonical_asset_id"] == "ai-provider:ollama_local" + assert catalog["inventory_hosts"] == ["host_111", "host_120", "host_121"] + assert catalog["domains"] == ["ai_provider", "ollama", "host_launchagent"] + assert catalog["catalog_revision"] == ( + "2026-07-16-host111-launchagent-k3s-path-v1" + ) + + assert {condition.condition_id for condition in conditions} == { + "host_111_ollama_launchagent_allowlist_exact", + "host_111_ollama_local_tags_api", + "host_120_to_host111_ollama_tags_api", + "host_121_to_host111_ollama_tags_api", + } + assert {condition.inventory_host for condition in conditions} == { + "host_111", + "host_120", + "host_121", + } + allowlist = next( + condition + for condition in conditions + if condition.condition_id == "host_111_ollama_launchagent_allowlist_exact" + ) + assert EXPECTED_ALLOWLIST in allowlist.probe + assert "192.168.0.110/32" not in allowlist.probe + assert all("systemctl" not in condition.probe for condition in conditions) + assert sum("http://192.168.0.111:11434/api/tags" in row.probe for row in conditions) == 2 + assert any("http://127.0.0.1:11434/api/tags" in row.probe for row in conditions) + + +def test_host111_playbook_is_launchagent_bounded_and_check_mode_safe() -> None: + payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) + source = PLAYBOOK.read_text(encoding="utf-8") + play = payload[0] + + assert play["hosts"] == "host_111" + assert play["gather_facts"] is False + assert play["vars"]["proxy_label"] == "com.momo.ollama111-allow-proxy" + assert play["vars"]["allowed_cidrs"] == EXPECTED_ALLOWLIST + assert "192.168.0.110/32" not in play["vars"]["allowed_cidrs"].split(",") + assert "launchctl bootstrap" in source + assert "systemctl" not in source + assert "docker restart" not in source + + tasks = {task["name"]: task for task in play["tasks"]} + assert tasks["Ollama111 | 讀取現行 allowlist"]["check_mode"] is False + assert tasks["Ollama111 | 讀回生效 allowlist"]["check_mode"] is False + check_receipt = tasks["Ollama111 | check-mode 發布精確 source diff"] + assert check_receipt["when"] == "ansible_check_mode" + assert check_receipt["ansible.builtin.debug"]["msg"]["persistent_writes_performed"] == 0 + + path_readback = tasks["Ollama111 | 從每個 K3s node 驗證 direct provider path"] + assert path_readback["delegate_to"] == "{{ item }}" + assert path_readback["loop"] == ["host_120", "host_121"] + assert path_readback["changed_when"] is False + assert path_readback["ansible.builtin.uri"]["status_code"] == 200 + + +def test_monitoring_registry_uses_launchagent_and_exact_catalog_action() -> None: + payload = yaml.safe_load(MONITORING_REGISTRY.read_text(encoding="utf-8")) + service = next( + row + for row in payload["services"] + if row.get("name") == "ollama-local" and row.get("type") == "ai-provider" + ) + + assert service["host"] == "192.168.0.111" + assert service["runtime_manager"] == "launchagent" + assert service["runtime_unit"] == "com.momo.ollama111-allow-proxy" + assert service["auto_repair"] == { + "enabled": True, + "actions": [CATALOG_ID], + } + assert service["monitoring"]["probe_origins"] == ["host120", "host121"] + assert service["monitoring"]["runtime_verified"] is False + + +def test_host111_sensor_separates_failure_from_missing_series_and_host110() -> None: + expressions: dict[str, set[str]] = { + "OllamaLocalUnavailable": set(), + "OllamaLocalHealthReceiptMissing": set(), + } + for path in (CANONICAL_ALERTS, MIRROR_ALERTS): + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + rules = { + row["alert"]: row + for group in payload["groups"] + for row in group.get("rules", []) + if row.get("alert") in expressions + } + failed = rules["OllamaLocalUnavailable"] + missing = rules["OllamaLocalHealthReceiptMissing"] + expressions["OllamaLocalUnavailable"].add( + "".join(str(failed["expr"]).split()) + ) + expressions["OllamaLocalHealthReceiptMissing"].add( + "".join(str(missing["expr"]).split()) + ) + assert failed["labels"]["canonical_asset_id"] == "ai-provider:ollama_local" + assert failed["labels"]["auto_repair"] == "true" + assert missing["labels"]["canonical_asset_id"] == ( + "signal:ollama_local_health_receipt" + ) + assert missing["labels"]["auto_repair"] == "false" + + assert all(len(values) == 1 for values in expressions.values()) + failure_expression = expressions["OllamaLocalUnavailable"].pop() + missing_expression = expressions["OllamaLocalHealthReceiptMissing"].pop() + assert failure_expression == 'ollama_health_status{host="111"}==0' + assert missing_expression == 'absent(ollama_health_status{host="111"})' + assert "110" not in failure_expression + missing_expression diff --git a/apps/api/tests/test_monitoring_provider_and_gitea_truth.py b/apps/api/tests/test_monitoring_provider_and_gitea_truth.py new file mode 100644 index 000000000..eb2a25c12 --- /dev/null +++ b/apps/api/tests/test_monitoring_provider_and_gitea_truth.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[3] +REGISTRY = ROOT / "ops/monitoring/service-registry.yaml" +SCRAPE = ROOT / "ops/monitoring/generated/prometheus-scrape-generated.yaml" +BLACKBOX = ROOT / "ops/monitoring/generated/blackbox-targets-generated.yaml" +PROMETHEUS = ROOT / "k8s/monitoring/prometheus.yml" +COMPOSE = ROOT / "k8s/monitoring/docker-compose-110.yml" +DISCOVERY = ROOT / "scripts/generate_monitoring.py" +OLLAMA_RULES = ROOT / "ops/monitoring/ollama_health_rules.yaml" +OLLAMA_DASHBOARD = ( + ROOT / "ops/monitoring/grafana/dashboards/ollama_failover.json" +) + + +def test_ollama_provider_health_uses_api_tags_blackbox_not_native_metrics() -> None: + registry = yaml.safe_load(REGISTRY.read_text(encoding="utf-8")) + providers = { + item["name"]: item + for item in registry["services"] + if item.get("type") == "ai-provider" + } + assert set(providers) == {"ollama-gcp-a", "ollama-gcp-b", "ollama-local"} + assert all( + item["monitoring"]["prometheus_scrape"] is False + for item in providers.values() + ) + assert providers["ollama-gcp-a"]["monitoring"]["blackbox_target_enabled"] + assert providers["ollama-gcp-b"]["monitoring"]["blackbox_target_enabled"] + assert not providers["ollama-local"]["monitoring"]["blackbox_target_enabled"] + assert providers["ollama-local"]["monitoring"]["agent_verifier"] + + scrape_jobs = { + item["job_name"] + for item in yaml.safe_load(SCRAPE.read_text(encoding="utf-8"))[ + "scrape_configs" + ] + } + assert scrape_jobs.isdisjoint(providers) + + blackbox_targets = { + item["labels"].get("provider"): item + for item in yaml.safe_load(BLACKBOX.read_text(encoding="utf-8")) + if item["labels"].get("provider") + } + assert set(blackbox_targets) == {"ollama_gcp_a", "ollama_gcp_b"} + assert blackbox_targets["ollama_gcp_a"] == { + "url": "http://34.143.170.20:11434/api/tags", + "labels": { + "boundary": "cloud_edge_availability_only", + "coverage_state": "source_candidate_runtime_pending", + "criticality": "P0", + "owner": "ai-team", + "probe_origin": "host110", + "provider": "ollama_gcp_a", + "service": "ollama-gcp-a", + }, + } + assert blackbox_targets["ollama_gcp_b"] == { + "url": "http://34.21.145.224:11434/api/tags", + "labels": { + "boundary": "cloud_edge_availability_only", + "coverage_state": "source_candidate_runtime_pending", + "criticality": "P0", + "owner": "ai-team", + "probe_origin": "host110", + "provider": "ollama_gcp_b", + "service": "ollama-gcp-b", + }, + } + + prometheus = PROMETHEUS.read_text(encoding="utf-8") + assert prometheus.count("provider: ollama_gcp_a") == 1 + assert prometheus.count("provider: ollama_gcp_b") == 1 + assert "192.168.0.111:11434" not in prometheus + + +def test_ollama_dashboard_and_orphan_rules_cannot_false_green() -> None: + legacy_rules = OLLAMA_RULES.read_text(encoding="utf-8") + dashboard = json.loads(OLLAMA_DASHBOARD.read_text(encoding="utf-8")) + availability = next(panel for panel in dashboard["panels"] if panel["id"] == 1) + + assert "superseded_by=global_product_governance_v2" in legacy_rules + assert "- alert: OllamaInstanceDown" not in legacy_rules + assert "up{job=~\"ollama_gcp" not in legacy_rules + + assert availability["title"] == "GCP-A/B Cloud Edge 可用性" + assert "independent K3s verifier pending" in availability["description"] + expressions = [target["expr"] for target in availability["targets"]] + assert len(expressions) == 2 + assert all("probe_success" in expression for expression in expressions) + assert all("absent(" in expression for expression in expressions) + assert all('job="blackbox-http"' in expression for expression in expressions) + assert all("ollama_local" not in expression for expression in expressions) + assert all("ollama_111" not in expression for expression in expressions) + + +def test_active_monitoring_uses_gitea_native_metrics_under_github_freeze() -> None: + prometheus = PROMETHEUS.read_text(encoding="utf-8") + compose = COMPOSE.read_text(encoding="utf-8") + discovery = DISCOVERY.read_text(encoding="utf-8") + + assert "job_name: 'gitea-native'" in prometheus + assert "192.168.0.110:3001" in prometheus + assert "github-actions" not in prometheus + assert "github-exporter" not in prometheus + assert "github-exporter" not in compose + assert "GITHUB_TOKEN" not in compose + assert '"gitea-native": "Gitea native metrics"' in discovery + assert '"github-actions": "GitHub Actions exporter"' not in discovery + assert "FORBIDDEN_JOBS" in discovery + assert "retired GitHub exporter must be removed from production" in discovery diff --git a/apps/api/tests/test_ollama_failover_manager.py b/apps/api/tests/test_ollama_failover_manager.py index f4409c980..f82f38024 100644 --- a/apps/api/tests/test_ollama_failover_manager.py +++ b/apps/api/tests/test_ollama_failover_manager.py @@ -374,6 +374,43 @@ class TestSelectProvider: assert URL_GCP_B in called_urls assert URL_LOCAL in called_urls + @pytest.mark.asyncio + async def test_default_selection_emits_no_audit_or_recovery_side_effects(self): + """Probe defaults closed; metric/TG/recovery require explicit production opt-in.""" + manager, _ = self._make_three_layer_mock( + gcp_a_status=HealthStatus.OFFLINE, + gcp_b_status=HealthStatus.HEALTHY, + local_status=HealthStatus.OFFLINE, + ) + recovery_callback = AsyncMock() + manager._recovery_callback = recovery_callback + audit = AsyncMock() + + with patch.object(manager, "_write_failover_audit", audit): + result = await manager.select_provider() + + assert result.observed_fallback.provider_name == "ollama_gcp_b" + audit.assert_not_awaited() + recovery_callback.assert_not_awaited() + + @pytest.mark.asyncio + async def test_production_selection_can_explicitly_emit_side_effects(self): + """Production action caller can retain the explicit observation boundary.""" + manager, _ = self._make_three_layer_mock( + gcp_a_status=HealthStatus.OFFLINE, + gcp_b_status=HealthStatus.HEALTHY, + local_status=HealthStatus.OFFLINE, + ) + recovery_callback = AsyncMock() + manager._recovery_callback = recovery_callback + audit = AsyncMock() + + with patch.object(manager, "_write_failover_audit", audit): + result = await manager.select_provider(emit_side_effects=True) + + audit.assert_awaited_once_with(result) + recovery_callback.assert_awaited_once_with("ollama_gcp_b") + @pytest.mark.asyncio async def test_select_provider_gcp_a_healthy_primary_ollama(self): """GCP-A HEALTHY → primary=ollama_gcp_a（或向下相容 ollama）""" diff --git a/apps/api/tests/test_ollama_prod_manifest_order.py b/apps/api/tests/test_ollama_prod_manifest_order.py index 4fea91600..9719f7fce 100644 --- a/apps/api/tests/test_ollama_prod_manifest_order.py +++ b/apps/api/tests/test_ollama_prod_manifest_order.py @@ -1,15 +1,15 @@ from __future__ import annotations from pathlib import Path +from urllib.parse import urlparse import yaml - ROOT = Path(__file__).resolve().parents[3] EXPECTED_ORDER = { - "OLLAMA_URL": "http://192.168.0.110:11435", - "OLLAMA_SECONDARY_URL": "http://192.168.0.110:11436", - "OLLAMA_FALLBACK_URL": "http://192.168.0.110:11437", + "OLLAMA_URL": "http://34.143.170.20:11434", + "OLLAMA_SECONDARY_URL": "http://34.21.145.224:11434", + "OLLAMA_FALLBACK_URL": "http://192.168.0.111:11434", } @@ -19,6 +19,11 @@ def _load_single_doc(path: Path) -> dict: return docs[0] +def _load_named_doc(path: Path, *, name: str) -> dict: + docs = [doc for doc in yaml.safe_load_all(path.read_text()) if doc] + return next(doc for doc in docs if doc["metadata"]["name"] == name) + + def test_prod_configmap_preserves_ollama_policy_order() -> None: configmap = _load_single_doc(ROOT / "k8s/awoooi-prod/04-configmap.yaml") @@ -35,3 +40,85 @@ def test_prod_deployment_preserves_ollama_policy_order() -> None: assert {key: env[key] for key in EXPECTED_ORDER} == EXPECTED_ORDER assert {env[key] for key in EXPECTED_ORDER} == set(EXPECTED_ORDER.values()) + + +def test_retired_host110_ollama_proxy_is_absent_from_runtime_manifests() -> None: + for relative in ( + "k8s/awoooi-prod/04-configmap.yaml", + "k8s/awoooi-prod/06-deployment-api.yaml", + "k8s/awoooi-dev/02-configmap.yaml", + ): + text = (ROOT / relative).read_text(encoding="utf-8") + assert "192.168.0.110:11435" not in text + assert "192.168.0.110:11436" not in text + assert "192.168.0.110:11437" not in text + + +def test_prod_network_policy_allows_only_canonical_ollama_egress() -> None: + policy = _load_named_doc( + ROOT / "k8s/awoooi-prod/02-network-policy.yaml", + name="allow-required-egress", + ) + ollama_rules = [ + rule + for rule in policy["spec"]["egress"] + if any(int(port["port"]) == 11434 for port in rule.get("ports", [])) + ] + + assert len(ollama_rules) == 1 + assert ollama_rules[0]["ports"] == [{"protocol": "TCP", "port": 11434}] + assert { + target["ipBlock"]["cidr"] for target in ollama_rules[0]["to"] + } == { + f"{urlparse(url).hostname}/32" for url in EXPECTED_ORDER.values() + } + + host110_rules = [ + rule + for rule in policy["spec"]["egress"] + if any( + target.get("ipBlock", {}).get("cidr") == "192.168.0.110/32" + for target in rule.get("to", []) + ) + ] + assert host110_rules + assert {11434, 11435, 11436, 11437}.isdisjoint( + { + int(port["port"]) + for rule in host110_rules + for port in rule.get("ports", []) + } + ) + + +def test_host111_allowlist_admits_k3s_direct_and_retires_host110_transport() -> None: + playbook = ( + ROOT / "infra/ansible/playbooks/111-ollama-fallback.yml" + ).read_text(encoding="utf-8") + assert "192.168.0.120/32" in playbook + assert "192.168.0.121/32" in playbook + assert "192.168.0.110/32" not in playbook + + retired_deployer = ( + ROOT / "ops/nginx/deploy-ollama-proxy-110.sh" + ).read_text(encoding="utf-8") + assert "exit 78" in retired_deployer + assert "NGINX_CONF=" not in retired_deployer + assert "proxy_pass" not in retired_deployer + + +def test_nginx_catalog_cannot_recreate_retired_host110_ollama_transport() -> None: + for relative in ( + "infra/ansible/playbooks/nginx-sync.yml", + "infra/ansible/playbooks/nginx-sync-readonly.yml", + ): + text = (ROOT / relative).read_text(encoding="utf-8") + assert "hosts: host_110" not in text + assert "110-ollama-proxy.conf" not in text + assert "11435" not in text + assert "11436" not in text + assert "11437" not in text + + assert not ( + ROOT / "infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2" + ).exists() diff --git a/apps/api/tests/test_ollama_provider_endpoints.py b/apps/api/tests/test_ollama_provider_endpoints.py index 8527500be..c08e54ef9 100644 --- a/apps/api/tests/test_ollama_provider_endpoints.py +++ b/apps/api/tests/test_ollama_provider_endpoints.py @@ -1,6 +1,8 @@ from __future__ import annotations +import hashlib from typing import Any +from unittest.mock import AsyncMock import pytest @@ -46,6 +48,29 @@ class _FakeClient: return _FakeResponse() +def _synthetic_context(prompt: str, **overrides: Any) -> dict[str, Any]: + return { + "data_classification": "sanitized", + "trace_id": "trace-test", + "run_id": "run-test", + "work_item_id": "AIA-SRE-013", + "synthetic_validation": True, + "synthetic_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "cloud_transport_receipts": [ + {"provider": "ollama_gcp_a", "receipt_id": "verified-a"}, + {"provider": "ollama_gcp_b", "receipt_id": "verified-b"}, + ], + "infrastructure_remediation_write_allowed": False, + **overrides, + } + + +async def _injected_transport_verifier(**kwargs: Any) -> str | None: + receipts = (kwargs.get("context") or {}).get("cloud_transport_receipts") or [] + if not any(item.get("provider") == kwargs["provider"] for item in receipts): + return "gcp_cloud_transport_receipt_missing" + return None + @pytest.mark.asyncio async def test_ollama_gcp_b_analyze_uses_secondary_url(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(ollama_module, "get_model_registry", lambda: _FakeRegistry()) @@ -58,14 +83,19 @@ async def test_ollama_gcp_b_analyze_uses_secondary_url(monkeypatch: pytest.Monke monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") client = _FakeClient() - provider = OllamaGcpBProvider() + provider = OllamaGcpBProvider( + transport_receipt_verifier=_injected_transport_verifier + ) async def _get_client() -> _FakeClient: return client monkeypatch.setattr(provider, "_get_client", _get_client) - result = await provider.analyze("diagnose", context={"task_type": "diagnose"}) + result = await provider.analyze( + "diagnose", + context=_synthetic_context("diagnose", task_type="diagnose"), + ) assert result.success is True assert result.provider == "ollama_gcp_b" @@ -83,14 +113,17 @@ async def test_ollama_gcp_a_allows_heavy_diagnose_model( monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") client = _FakeClient() - provider = OllamaProvider() + provider = OllamaProvider(transport_receipt_verifier=_injected_transport_verifier) async def _get_client() -> _FakeClient: return client monkeypatch.setattr(provider, "_get_client", _get_client) - result = await provider.analyze("diagnose", context={"task_type": "diagnose"}) + result = await provider.analyze( + "diagnose", + context=_synthetic_context("diagnose", task_type="diagnose"), + ) assert result.success is True assert client.posted_urls == ["http://primary:11435/api/generate"] @@ -107,14 +140,17 @@ async def test_ollama_gcp_a_coerces_non_diagnosis_heavy_model_to_health_model( monkeypatch.setattr(ollama_module.settings, "OLLAMA_HEALTH_CHECK_MODEL", "gemma3:4b") client = _FakeClient() - provider = OllamaProvider() + provider = OllamaProvider(transport_receipt_verifier=_injected_transport_verifier) async def _get_client() -> _FakeClient: return client monkeypatch.setattr(provider, "_get_client", _get_client) - result = await provider.analyze("background summary", context={"task_type": "summary"}) + result = await provider.analyze( + "background summary", + context=_synthetic_context("background summary", task_type="summary"), + ) assert result.success is True assert client.posted_urls == ["http://primary:11435/api/generate"] @@ -131,7 +167,7 @@ async def test_ollama_gcp_a_can_explicitly_allow_heavy_model( monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") client = _FakeClient() - provider = OllamaProvider() + provider = OllamaProvider(transport_receipt_verifier=_injected_transport_verifier) async def _get_client() -> _FakeClient: return client @@ -140,13 +176,84 @@ async def test_ollama_gcp_a_can_explicitly_allow_heavy_model( result = await provider.analyze( "deep diagnose", - context={"task_type": "diagnose", "allow_gcp_heavy_model": True}, + context=_synthetic_context( + "deep diagnose", + task_type="diagnose", + allow_gcp_heavy_model=True, + ), ) assert result.success is True assert client.posted_payloads[0]["model"] == "qwen3:14b" +@pytest.mark.asyncio +async def test_gcp_ollama_rejects_unreceipted_prompt_before_network_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ollama_module.settings, "OLLAMA_URL", "http://primary:11435") + monkeypatch.setattr( + ollama_module.settings, + "OLLAMA_SECONDARY_URL", + "http://secondary:11436", + ) + client = _FakeClient() + provider = OllamaProvider(transport_receipt_verifier=_injected_transport_verifier) + + async def _get_client() -> _FakeClient: + return client + + monkeypatch.setattr(provider, "_get_client", _get_client) + + result = await provider.analyze( + "raw operational log", + context={"data_classification": "sanitized"}, + ) + + assert result.success is False + assert result.error == "gcp_cloud_transport_receipt_missing" + assert result.audit_metadata["generation_attempted"] is False + assert client.posted_urls == [] + + +@pytest.mark.asyncio +async def test_gcp_ollama_blocks_tool_loop_before_tool_or_network_execution( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ollama_module.settings, "OLLAMA_URL", "http://primary:11435") + monkeypatch.setattr( + ollama_module.settings, + "OLLAMA_SECONDARY_URL", + "http://secondary:11436", + ) + client = _FakeClient() + provider = OllamaProvider(transport_receipt_verifier=_injected_transport_verifier) + tool_executor = AsyncMock() + + async def _get_client() -> _FakeClient: + return client + + monkeypatch.setattr(provider, "_get_client", _get_client) + prompt = "sanitized investigation" + + result = await provider.analyze_with_tools( + prompt, + available_tools=[object()], # type: ignore[list-item] + tool_executor=tool_executor, + context=_synthetic_context(prompt, task_type="diagnose"), + ) + + assert result.success is False + assert result.error == "gcp_tool_loop_blocked_unverified_tool_results" + assert result.audit_metadata == { + "generation_attempted": False, + "tool_execution_attempted": False, + "transport_boundary": "public_http_degraded", + } + tool_executor.assert_not_awaited() + assert client.posted_urls == [] + + @pytest.mark.asyncio async def test_ollama_base_provider_health_uses_endpoint_hook( monkeypatch: pytest.MonkeyPatch, diff --git a/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py b/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py index fa6fe2c56..ea9f3c067 100644 --- a/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py +++ b/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py @@ -45,15 +45,25 @@ class _FakeRoute: class _FakeFailoverManager: def __init__(self) -> None: - self.task_types: list[str] = [] + self.calls: list[tuple[str, bool]] = [] - async def select_provider(self, task_type: str = "general") -> _FakeRoute: - self.task_types.append(task_type) + async def select_provider( + self, + task_type: str = "general", + *, + emit_side_effects: bool = False, + ) -> _FakeRoute: + self.calls.append((task_type, emit_side_effects)) return _FakeRoute() class _UnorderedFailoverManager: - async def select_provider(self, task_type: str = "general") -> SimpleNamespace: + async def select_provider( + self, + task_type: str = "general", + *, + emit_side_effects: bool = False, + ) -> SimpleNamespace: return SimpleNamespace( all_endpoints_in_order=lambda: [ _FakeEndpoint("ollama_local"), @@ -170,7 +180,7 @@ async def test_alert_context_uses_exact_ollama_lane_then_gemini( "ollama_local", "gemini", ] - assert fake_failover.task_types == ["alert_fast"] + assert fake_failover.calls == [("alert_fast", True)] assert fake_router.contexts[0]["intent_hint"] == "alert_triage" assert fake_executor.context is not None assert fake_executor.context["task_type"] == "alert_fast" @@ -277,7 +287,7 @@ async def test_explicit_ai_governance_context_uses_ollama_first( "ollama_local", "gemini", ] - assert fake_failover.task_types == ["diagnose"] + assert fake_failover.calls == [("diagnose", True)] @pytest.mark.asyncio @@ -297,7 +307,7 @@ async def test_alert_context_uses_gcp_a_gcp_b_then_111_order( ) assert provider_order == ["ollama_gcp_a", "ollama_gcp_b", "ollama_local"] - assert fake_failover.task_types == ["alert_fast"] + assert fake_failover.calls == [("alert_fast", True)] @pytest.mark.asyncio diff --git a/apps/api/tests/test_openclaw_legacy_ollama_failover.py b/apps/api/tests/test_openclaw_legacy_ollama_failover.py index f72995d56..492bb8320 100644 --- a/apps/api/tests/test_openclaw_legacy_ollama_failover.py +++ b/apps/api/tests/test_openclaw_legacy_ollama_failover.py @@ -35,8 +35,10 @@ class _FakeRoute: class _FakeManager: def __init__(self, endpoints: list[_FakeEndpoint]) -> None: self._endpoints = endpoints + self.emit_side_effects: list[bool] = [] - async def select_provider(self) -> _FakeRoute: + async def select_provider(self, *, emit_side_effects: bool = False) -> _FakeRoute: + self.emit_side_effects.append(emit_side_effects) return _FakeRoute(self._endpoints) @@ -62,6 +64,18 @@ class _FakeClient: return _FakeResponse() +def _prepared_alert_context(service: OpenClawService) -> dict[str, Any]: + context = service._prepare_paid_cloud_route_context( + { + "alert_type": "HostHighCpuLoad", + "source": "test-alert-producer", + "target_resource": "node-110", + } + ) + assert context is not None + return context + + @pytest.mark.asyncio async def test_legacy_ollama_uses_failover_order_before_gemini( monkeypatch: pytest.MonkeyPatch, @@ -69,17 +83,18 @@ async def test_legacy_ollama_uses_failover_order_before_gemini( monkeypatch.setattr(openclaw_module, "get_model_registry", lambda: _FakeRegistry()) monkeypatch.setattr(openclaw_module.settings, "OPENCLAW_TIMEOUT", 30) monkeypatch.setattr(openclaw_module.settings, "OLLAMA_DIAGNOSE_TIMEOUT_SECONDS", 200) + manager = _FakeManager( + [ + _FakeEndpoint("ollama_gcp_a", "http://gcp-a:11435"), + _FakeEndpoint("ollama_gcp_b", "http://gcp-b:11436"), + _FakeEndpoint("ollama_local", "http://local-111:11434"), + _FakeEndpoint("gemini", ""), + ], + ) monkeypatch.setattr( openclaw_module, "get_ollama_failover_manager", - lambda: _FakeManager( - [ - _FakeEndpoint("ollama_gcp_a", "http://gcp-a:11435"), - _FakeEndpoint("ollama_gcp_b", "http://gcp-b:11436"), - _FakeEndpoint("ollama_local", "http://local-111:11434"), - _FakeEndpoint("gemini", ""), - ], - ), + lambda: manager, ) client = _FakeClient(fail_urls={"http://gcp-a:11435/api/generate"}) @@ -90,7 +105,10 @@ async def test_legacy_ollama_uses_failover_order_before_gemini( monkeypatch.setattr(service, "_get_client", _get_client) - result, ok = await service._call_ollama("diagnose") + result, ok = await service._call_ollama( + "diagnose", + alert_context=_prepared_alert_context(service), + ) assert ok is True assert result == '{"action_title":"ok"}' @@ -98,6 +116,7 @@ async def test_legacy_ollama_uses_failover_order_before_gemini( "http://gcp-a:11435/api/generate", "http://gcp-b:11436/api/generate", ] + assert manager.emit_side_effects == [True] @pytest.mark.asyncio @@ -110,10 +129,11 @@ async def test_legacy_ollama_falls_back_to_configured_three_layer_urls( monkeypatch.setattr(openclaw_module.settings, "OLLAMA_URL", "http://gcp-a:11435") monkeypatch.setattr(openclaw_module.settings, "OLLAMA_SECONDARY_URL", "http://gcp-b:11436") monkeypatch.setattr(openclaw_module.settings, "OLLAMA_FALLBACK_URL", "http://local-111:11434") + manager = _FakeManager([_FakeEndpoint("gemini", "")]) monkeypatch.setattr( openclaw_module, "get_ollama_failover_manager", - lambda: _FakeManager([_FakeEndpoint("gemini", "")]), + lambda: manager, ) client = _FakeClient( @@ -129,7 +149,10 @@ async def test_legacy_ollama_falls_back_to_configured_three_layer_urls( monkeypatch.setattr(service, "_get_client", _get_client) - result, ok = await service._call_ollama("diagnose") + result, ok = await service._call_ollama( + "diagnose", + alert_context=_prepared_alert_context(service), + ) assert ok is True assert result == '{"action_title":"ok"}' @@ -138,3 +161,4 @@ async def test_legacy_ollama_falls_back_to_configured_three_layer_urls( "http://gcp-b:11436/api/generate", "http://local-111:11434/api/generate", ] + assert manager.emit_side_effects == [True] diff --git a/apps/api/tests/test_openclaw_paid_cloud_privacy_boundary.py b/apps/api/tests/test_openclaw_paid_cloud_privacy_boundary.py index 7dfd7dd8c..4cd8a1396 100644 --- a/apps/api/tests/test_openclaw_paid_cloud_privacy_boundary.py +++ b/apps/api/tests/test_openclaw_paid_cloud_privacy_boundary.py @@ -47,6 +47,18 @@ class _Redis: async def set(self, key: str, value: str, ex: int | None = None) -> None: self.set_calls.append((key, value, ex)) + async def eval(self, *_args: object) -> list[object]: + if self.paid_disabled: + return [0, "paired_paid_state_not_enabled", ""] + run_id = str(_args[-1] or "") + if not run_id: + return [0, "run_owned_canary_not_admitted", ""] + return [ + 1, + "run_owned_canary_admitted", + f"{run_id}.attempt-fixture", + ] + class _Provider: is_enabled = True @@ -172,6 +184,50 @@ def test_cloud_sanitizer_attests_only_allowlisted_structured_fields() -> None: assert receipt["raw_log_payload_forwarded"] is False assert receipt["secret_value_exposed"] is False assert receipt["excluded_field_count"] == 5 + assert "cloud_transport_degraded_acknowledged" not in cloud_context + assert "cloud_transport_receipts" not in cloud_context + + +def test_cloud_sanitizer_only_copies_allowlisted_transport_receipt_hints() -> None: + alert = _raw_alert() + alert["cloud_transport_receipts"] = [ + { + "schema_version": "cloud_transport_preflight_receipt_v1", + "receipt_id": "cloud-transport:" + "a" * 32, + "source": "cloud_transport_preflight.network_probe", + "status": "degraded_transport_reachable", + "transport_boundary": "public_http_sanitized_candidate_only", + "transport_security_status": "degraded_public_http", + "provider": "ollama_gcp_a", + "endpoint_sha256": "b" * 64, + "trace_id": "trace-1", + "run_id": "run-1", + "work_item_id": "AIA-SRE-013", + "check_completed": True, + "http_status": 200, + "ollama_contract_verified": True, + "observed_at": "2026-07-16T00:00:00+00:00", + "expires_at": "2026-07-16T00:01:30+00:00", + "ttl_seconds": 90, + "durable_write_ack": True, + "verifier_status": "verified", + "verified_by": "cloud_transport_preflight.redis_readback", + "raw_response_persisted": False, + "endpoint_value_persisted": False, + "secret_value_persisted_or_returned": False, + "untrusted_extra": "must-not-cross", + } + ] + + cloud_context, reason = build_paid_cloud_alert_context(alert) + + assert reason == "verified_allowlisted_sanitization_receipt" + assert cloud_context is not None + hints = cloud_context["cloud_transport_receipts"] + assert len(hints) == 1 + assert hints[0]["provider"] == "ollama_gcp_a" + assert "untrusted_extra" not in hints[0] + assert "must-not-cross" not in json.dumps(cloud_context) def test_cloud_sanitizer_redacts_bare_keys_and_all_private_network_ranges() -> None: @@ -292,7 +348,7 @@ async def test_openclaw_outer_cache_never_writes_paid_result( @pytest.mark.asyncio -async def test_actual_executor_uses_original_prompt_for_ollama_and_sanitized_prompt_for_paid( +async def test_actual_executor_uses_sanitized_prompt_for_gcp_and_paid_cloud( monkeypatch: pytest.MonkeyPatch, ) -> None: redis = _Redis() @@ -301,13 +357,13 @@ async def test_actual_executor_uses_original_prompt_for_ollama_and_sanitized_pro gcp_a = _Provider( "ollama_gcp_a", success=False, - privacy_level="local", + privacy_level="cloud", attempt_order=attempt_order, ) gcp_b = _Provider( "ollama_gcp_b", success=False, - privacy_level="local", + privacy_level="cloud", attempt_order=attempt_order, ) local = _Provider( @@ -366,12 +422,13 @@ async def test_actual_executor_uses_original_prompt_for_ollama_and_sanitized_pro "claude", "gemini", ] - assert [provider.calls[0][0] for provider in (gcp_a, gcp_b, local)] == [ - original_prompt, - original_prompt, - original_prompt, - ] + cloud_prompt = gcp_a.calls[0][0] + assert gcp_b.calls[0][0] == cloud_prompt + assert local.calls[0][0] == original_prompt + assert cloud_prompt != original_prompt + assert "raw-secret-must-never-reach-cloud" not in cloud_prompt paid_prompt, paid_context = gemini.calls[0] + assert paid_prompt == cloud_prompt assert paid_prompt != original_prompt assert "raw-secret-must-never-reach-cloud" not in paid_prompt assert paid_context is not None @@ -510,7 +567,7 @@ async def test_actual_executor_cache_parse_error_is_content_bounded( return None return f'{{"response":"{secret}", invalid' - provider = _Provider("ollama_gcp_a", success=True, privacy_level="local") + provider = _Provider("ollama_local", success=True, privacy_level="local") registry = AIProviderRegistry() registry.register(provider) monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) @@ -522,7 +579,7 @@ async def test_actual_executor_cache_parse_error_is_content_bounded( with capture_logs() as captured_logs: result = await AIRouterExecutor(registry).execute( prompt="local prompt", - provider_order=["ollama_gcp_a"], + provider_order=["ollama_local"], context={"intent_hint": "diagnose"}, ) @@ -582,7 +639,7 @@ async def test_actual_executor_rejects_self_signed_cloud_receipt( @pytest.mark.asyncio -async def test_ollama_circuit_open_cannot_slide_to_paid_cloud( +async def test_ollama_circuit_open_cannot_slide_to_another_host_or_paid_cloud( monkeypatch: pytest.MonkeyPatch, ) -> None: redis = _Redis() @@ -610,8 +667,11 @@ async def test_ollama_circuit_open_cannot_slide_to_paid_cloud( ) assert result.success is False + assert [provider.calls for provider in providers] == [[], [], [], []] assert providers[-1].calls == [] - assert "cloud_blocked_ollama_circuit_open" in str(result.error) + assert result.error == "provider_circuit_open_fail_closed:ollama_gcp_a" + assert result.audit_metadata["cross_provider_fallback_allowed"] is False + assert result.audit_metadata["cross_domain_fallback_allowed"] is False @pytest.mark.asyncio diff --git a/apps/api/tests/test_paid_provider_canary_authorization.py b/apps/api/tests/test_paid_provider_canary_authorization.py new file mode 100644 index 000000000..8964167da --- /dev/null +++ b/apps/api/tests/test_paid_provider_canary_authorization.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import os +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from uuid import UUID, uuid4 + +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") + +from src.services.paid_provider_canary_authorization import ( # noqa: E402 + PAID_CANARY_AGENT_ID, + PAID_CANARY_PROJECT_ID, + PAID_CANARY_TRIGGER_REF, + PAID_CANARY_TRIGGER_TYPE, + PAID_CANARY_WORK_ITEM_ID, + PaidProviderCanaryAuthorizationError, + PaidProviderCanaryAuthorizationEvidence, + paid_provider_canary_authorization_input, + paid_provider_canary_authorization_input_sha256, + require_paid_provider_canary_authorization, + verify_paid_provider_canary_authorization, +) + +_SELECTED_RUN_ID = UUID("267a3d26-3c29-470f-8fe3-388a2f408f39") +_TRACE_ID = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" + + +class _Reader: + def __init__( + self, + evidence: PaidProviderCanaryAuthorizationEvidence | None, + *, + error: Exception | None = None, + ) -> None: + self.evidence = evidence + self.error = error + self.calls: list[tuple[UUID, str, str]] = [] + + async def read( + self, + *, + run_id: UUID, + project_id: str, + work_item_id: str, + ) -> PaidProviderCanaryAuthorizationEvidence | None: + self.calls.append((run_id, project_id, work_item_id)) + if self.error is not None: + raise self.error + return self.evidence + + +def _evidence( + *, + now: datetime, + run_id: UUID | None = None, +) -> PaidProviderCanaryAuthorizationEvidence: + return PaidProviderCanaryAuthorizationEvidence( + run_id=run_id or _SELECTED_RUN_ID, + trace_id=_TRACE_ID, + project_id=PAID_CANARY_PROJECT_ID, + agent_id=PAID_CANARY_AGENT_ID, + state="running", + trigger_type=PAID_CANARY_TRIGGER_TYPE, + trigger_ref=PAID_CANARY_TRIGGER_REF, + is_shadow=False, + input_sha256=paid_provider_canary_authorization_input_sha256(), + timeout_at=now + timedelta(minutes=5), + approval_step_count=1, + approval_step_status="success", + approval_step_was_blocked=False, + approval_completed_at=now - timedelta(minutes=1), + approval_audit_count=1, + approval_audit_created_at=now - timedelta(seconds=50), + prior_canary_start_count=0, + ) + + +def test_authorization_input_is_detached_and_hash_is_stable() -> None: + first = paid_provider_canary_authorization_input() + second = paid_provider_canary_authorization_input() + + first["provider_order"].append("unexpected") + + assert second["project_id"] == "awoooi" + assert second["work_item_id"] == "AIA-SRE-013" + assert "unexpected" not in second["provider_order"] + assert len(paid_provider_canary_authorization_input_sha256()) == 64 + assert ( + paid_provider_canary_authorization_input_sha256() + == paid_provider_canary_authorization_input_sha256() + ) + + +@pytest.mark.asyncio +async def test_exact_gate5_evidence_returns_non_sensitive_authorized_receipt() -> None: + now = datetime(2026, 7, 16, 8, 0, tzinfo=UTC) + evidence = _evidence(now=now) + reader = _Reader(evidence) + + receipt = await verify_paid_provider_canary_authorization( + str(evidence.run_id), + evidence_reader=reader, + now=now, + ) + + assert receipt["authorized"] is True + assert receipt["status"] == "authorized" + assert receipt["authorization_ref"] == str(evidence.run_id) + assert receipt["run_id"] == str(evidence.run_id) + assert receipt["trace_id"] == _TRACE_ID + assert receipt["project_id"] == "awoooi" + assert receipt["work_item_id"] == "AIA-SRE-013" + assert receipt["valid_for_seconds"] == 300 + assert receipt["checks"]["authorization_unused"] is True + assert receipt["checks"]["authorization_run_selected"] is True + assert receipt["contract"]["same_run_identity"] is True + assert receipt["reuse_check"] == "clear_at_read_time" + assert receipt["read_only_verification"] is True + assert receipt["authorization_consumed_by_verifier"] is False + assert receipt["operator_identity_returned"] is False + assert receipt["secret_value_read_or_returned"] is False + assert "approver_id" not in str(receipt) + assert reader.calls == [ + (evidence.run_id, PAID_CANARY_PROJECT_ID, PAID_CANARY_WORK_ITEM_ID) + ] + + +@pytest.mark.asyncio +async def test_authorization_ref_must_be_canonical_uuid_without_db_read() -> None: + reader = _Reader(None) + + malformed = await verify_paid_provider_canary_authorization( + "AIA-SRE-013-user-approved-20260716", + evidence_reader=reader, + ) + uppercase = await verify_paid_provider_canary_authorization( + str(uuid4()).upper(), + evidence_reader=reader, + ) + + assert malformed["authorized"] is False + assert malformed["error_code"] == ( + "paid_canary_authorization_ref_must_be_uuid_run_id" + ) + assert uppercase["authorized"] is False + assert uppercase["error_code"] == ( + "paid_canary_authorization_ref_not_canonical_uuid" + ) + assert reader.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("field", "value", "failed_check"), + [ + ("project_id", "other", "project_exact"), + ( + "run_id", + UUID("00000000-0000-4000-8000-000000000001"), + "authorization_run_selected", + ), + ("agent_id", "generic-agent", "agent_exact"), + ("state", "waiting_approval", "state_executable"), + ("is_shadow", True, "non_shadow"), + ("trigger_type", "schedule", "trigger_type_exact"), + ("trigger_ref", "paid-provider-canary:OTHER", "trigger_ref_exact"), + ("input_sha256", "0" * 64, "input_sha256_exact"), + ("trace_id", None, "trace_id_present"), + ("approval_step_count", 0, "approval_step_exactly_once"), + ("approval_step_status", "failed", "approval_step_success"), + ("approval_step_was_blocked", True, "approval_step_not_blocked"), + ("approval_audit_count", 0, "approval_audit_exactly_once"), + ("prior_canary_start_count", 1, "authorization_unused"), + ], +) +async def test_contract_drift_fails_closed( + field: str, + value: object, + failed_check: str, +) -> None: + now = datetime(2026, 7, 16, 8, 0, tzinfo=UTC) + evidence = replace(_evidence(now=now), **{field: value}) + + receipt = await verify_paid_provider_canary_authorization( + str(evidence.run_id), + evidence_reader=_Reader(evidence), + now=now, + ) + + assert receipt["authorized"] is False + assert failed_check in receipt["failed_checks"] + assert receipt["checks"][failed_check] is False + + +@pytest.mark.asyncio +async def test_missing_timeout_or_expired_approval_fails_closed() -> None: + now = datetime(2026, 7, 16, 8, 0, tzinfo=UTC) + missing_timeout = replace(_evidence(now=now), timeout_at=None) + expired = replace( + _evidence(now=now), + approval_completed_at=now - timedelta(minutes=16), + timeout_at=now + timedelta(minutes=5), + ) + + missing_receipt = await verify_paid_provider_canary_authorization( + str(missing_timeout.run_id), + evidence_reader=_Reader(missing_timeout), + now=now, + ) + expired_receipt = await verify_paid_provider_canary_authorization( + str(expired.run_id), + evidence_reader=_Reader(expired), + now=now, + ) + + assert missing_receipt["authorized"] is False + assert missing_receipt["checks"]["run_timeout_present"] is False + assert expired_receipt["authorized"] is False + assert expired_receipt["checks"]["authorization_not_expired"] is False + + +@pytest.mark.asyncio +async def test_db_error_and_missing_run_are_blocked_without_error_detail() -> None: + run_id = uuid4() + + unavailable = await verify_paid_provider_canary_authorization( + str(run_id), + evidence_reader=_Reader(None, error=RuntimeError("database dsn secret")), + ) + missing = await verify_paid_provider_canary_authorization( + str(run_id), + evidence_reader=_Reader(None), + ) + + assert unavailable["error_code"] == ( + "paid_canary_authorization_evidence_unavailable" + ) + assert "database dsn secret" not in str(unavailable) + assert missing["error_code"] == "paid_canary_authorization_run_not_found" + + +@pytest.mark.asyncio +async def test_require_helper_raises_only_safe_error_code() -> None: + run_id = uuid4() + + with pytest.raises(PaidProviderCanaryAuthorizationError) as raised: + await require_paid_provider_canary_authorization( + str(run_id), + evidence_reader=_Reader(None), + ) + + assert raised.value.error_code == "paid_canary_authorization_run_not_found" + assert str(raised.value) == raised.value.error_code + assert raised.value.receipt["authorized"] is False diff --git a/apps/api/tests/test_paid_provider_canary_gate5_run.py b/apps/api/tests/test_paid_provider_canary_gate5_run.py new file mode 100644 index 000000000..3cf271a66 --- /dev/null +++ b/apps/api/tests/test_paid_provider_canary_gate5_run.py @@ -0,0 +1,408 @@ +from __future__ import annotations + +import os +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from uuid import UUID + +import pytest +from fastapi import HTTPException +from pydantic import ValidationError + +os.environ.setdefault( + "DATABASE_URL", + "postgresql+asyncpg://test:test@localhost/test", +) + +from src.api.v1.platform import operator_runs # noqa: E402 +from src.api.v1.platform import runs as public_runs # noqa: E402 +from src.core.awooop_operator_auth import AwoooPOperatorPrincipal # noqa: E402 +from src.db.awooop_models import AwoooPRunState # noqa: E402 +from src.services import platform_operator_service, platform_runtime # noqa: E402 +from src.services.paid_provider_canary_authorization import ( # noqa: E402 + PAID_CANARY_AGENT_ID, + PAID_CANARY_PROJECT_ID, + PAID_CANARY_TRIGGER_REF, + PAID_CANARY_TRIGGER_TYPE, + paid_provider_canary_authorization_input_sha256, +) +from src.services.paid_provider_canary_gate5_run import ( # noqa: E402 + PAID_CANARY_AUTHORIZATION_SCOPE, + PaidProviderCanaryGate5RunRecord, + create_paid_provider_canary_gate5_run, + paid_provider_canary_approval_contract_errors, +) + +_SELECTED_RUN_ID = UUID("267a3d26-3c29-470f-8fe3-388a2f408f39") + + +class _Repository: + def __init__(self, *, duplicate: bool = False, state: str = "waiting_approval") -> None: + self.duplicate = duplicate + self.state = state + self.calls: list[dict[str, object]] = [] + + async def create_or_get(self, **kwargs: object) -> tuple[ + PaidProviderCanaryGate5RunRecord, + bool, + ]: + self.calls.append(kwargs) + run_id = kwargs["run_id"] + timeout_at = kwargs["timeout_at"] + input_sha256 = kwargs["input_sha256"] + trace_id = kwargs["trace_id"] + assert isinstance(run_id, UUID) + assert isinstance(timeout_at, datetime) + assert isinstance(input_sha256, str) + assert isinstance(trace_id, str) + if self.duplicate: + run_id = _SELECTED_RUN_ID + return ( + PaidProviderCanaryGate5RunRecord( + run_id=run_id, + project_id=PAID_CANARY_PROJECT_ID, + agent_id=PAID_CANARY_AGENT_ID, + state=self.state, + trigger_type=PAID_CANARY_TRIGGER_TYPE, + trigger_ref=PAID_CANARY_TRIGGER_REF, + is_shadow=False, + input_sha256=input_sha256, + timeout_at=timeout_at, + trace_id=trace_id, + ), + self.duplicate, + ) + + +@pytest.mark.asyncio +async def test_operator_service_creates_exact_bounded_non_shadow_gate5_run() -> None: + now = datetime(2026, 7, 16, 9, 0, tzinfo=UTC) + repository = _Repository() + audit_calls: list[dict[str, object]] = [] + + async def audit_writer(**kwargs: object) -> None: + audit_calls.append(kwargs) + + response = await create_paid_provider_canary_gate5_run( + operator_id="telegram:123456", + repository=repository, + audit_writer=audit_writer, + now=now, + ) + + assert response["project_id"] == "awoooi" + assert response["work_item_id"] == "AIA-SRE-013" + assert response["agent_id"] == "awoooi-paid-provider-canary" + assert response["state"] == "waiting_approval" + assert response["is_shadow"] is False + assert response["trigger_type"] == "api" + assert response["trigger_ref"] == "paid-provider-canary:AIA-SRE-013" + assert response["input_sha256"] == paid_provider_canary_authorization_input_sha256() + assert response["authorization_scope"] == PAID_CANARY_AUTHORIZATION_SCOPE + assert response["provider_order"] == [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ] + assert response["paid_provider_canary_percent"] == 5 + assert response["canary_bucket"] < 5 + assert response["authorization_run_is_execution_run"] is True + assert response["max_output_tokens_per_paid_provider"] == 512 + assert response["max_total_paid_cost_usd"] == "0.25" + assert response["timeout_at"] == now + timedelta(minutes=15) + assert response["approval_ttl_seconds"] == 900 + assert response["provider_call_performed"] is False + assert response["route_change_performed"] is False + assert response["infrastructure_write_performed"] is False + assert response["raw_prompt_persistence_allowed"] is False + assert response["raw_response_persistence_allowed"] is False + assert response["next_action"] == "operator_gate5_decision_required" + assert UUID(response["authorization_ref"]) == UUID(response["run_id"]) + assert "telegram:123456" not in str(response) + + assert len(repository.calls) == 1 + call = repository.calls[0] + assert call["now"] == now + assert call["timeout_at"] == now + timedelta(minutes=15) + assert call["input_sha256"] == paid_provider_canary_authorization_input_sha256() + assert len(audit_calls) == 1 + assert audit_calls[0]["resource_id"] == response["run_id"] + assert audit_calls[0]["run_id"] == response["run_id"] + details = audit_calls[0]["details"] + assert isinstance(details, dict) + assert details["provider_call_performed"] is False + assert details["route_change_performed"] is False + + +@pytest.mark.asyncio +async def test_duplicate_active_authorization_is_reused_without_scope_expansion() -> None: + repository = _Repository(duplicate=True, state="running") + + async def audit_writer(**_: object) -> None: + return None + + response = await create_paid_provider_canary_gate5_run( + operator_id="ops@example.com", + repository=repository, + audit_writer=audit_writer, + now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC), + ) + + assert response["is_duplicate"] is True + assert response["state"] == "running" + assert response["run_id"] == str(_SELECTED_RUN_ID) + assert response["max_total_paid_cost_usd"] == "0.25" + assert response["infrastructure_write_performed"] is False + assert response["next_action"] == ( + "verify_authorization_then_run_bounded_canary" + ) + + +def _run( + *, + now: datetime, + agent_id: str = PAID_CANARY_AGENT_ID, + trigger_ref: str = PAID_CANARY_TRIGGER_REF, +) -> AwoooPRunState: + return AwoooPRunState( + run_id=_SELECTED_RUN_ID, + project_id=PAID_CANARY_PROJECT_ID, + agent_id=agent_id, + state="waiting_approval", + trace_id="00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + trigger_type=PAID_CANARY_TRIGGER_TYPE, + trigger_ref=trigger_ref, + is_shadow=False, + input_sha256=paid_provider_canary_authorization_input_sha256(), + timeout_at=now + timedelta(minutes=15), + ) + + +def test_paid_canary_approval_contract_is_exact_and_expiry_bound() -> None: + now = datetime(2026, 7, 16, 9, 0, tzinfo=UTC) + exact = _run(now=now) + expired = _run(now=now) + expired.timeout_at = now - timedelta(seconds=1) + shadow = _run(now=now) + shadow.is_shadow = True + drifted_hash = _run(now=now) + drifted_hash.input_sha256 = "0" * 64 + + assert paid_provider_canary_approval_contract_errors(exact, now=now) == [] + assert paid_provider_canary_approval_contract_errors(expired, now=now) == [ + "timeout_not_expired" + ] + assert paid_provider_canary_approval_contract_errors(shadow, now=now) == [ + "non_shadow" + ] + assert paid_provider_canary_approval_contract_errors( + drifted_hash, + now=now, + ) == ["input_sha256_exact"] + + +def test_unrelated_generic_run_does_not_enter_paid_canary_contract() -> None: + now = datetime(2026, 7, 16, 9, 0, tzinfo=UTC) + generic = _run( + now=now, + agent_id="generic-agent", + trigger_ref="generic:api:request", + ) + + assert paid_provider_canary_approval_contract_errors(generic, now=now) is None + + +@pytest.mark.asyncio +async def test_generic_approval_service_rejects_expired_paid_canary_before_transition( + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = datetime.now(UTC) + expired = _run(now=now) + expired.timeout_at = now - timedelta(seconds=1) + + class _Result: + def scalar_one_or_none(self) -> AwoooPRunState: + return expired + + class _Db: + async def execute(self, *_: object, **__: object) -> _Result: + return _Result() + + @asynccontextmanager + async def fake_db_context(project_id: str): + assert project_id == "awoooi" + yield _Db() + + monkeypatch.setattr( + platform_operator_service, + "get_db_context", + fake_db_context, + ) + + with pytest.raises(HTTPException) as exc: + await platform_operator_service.decide_approval( + run_id=str(expired.run_id), + project_id="awoooi", + decision="approve", + approver_id="telegram:123456", + reason="bounded canary", + ) + + assert exc.value.status_code == 409 + assert "timeout_not_expired" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_operator_route_uses_authenticated_principal_and_fixed_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + async def fake_create(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return { + "schema_version": "paid_provider_canary_gate5_run_v1", + "authorization_ref": "018f2d04-4c37-7a18-b764-df0df0cbe111", + "run_id": "018f2d04-4c37-7a18-b764-df0df0cbe111", + "project_id": "awoooi", + "work_item_id": "AIA-SRE-013", + "agent_id": "awoooi-paid-provider-canary", + "state": "waiting_approval", + "is_shadow": False, + "is_duplicate": False, + "trigger_type": "api", + "trigger_ref": "paid-provider-canary:AIA-SRE-013", + "input_sha256": "1" * 64, + "authorization_scope": PAID_CANARY_AUTHORIZATION_SCOPE, + "provider_order": [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ], + "paid_provider_canary_percent": 5, + "max_output_tokens_per_paid_provider": 512, + "max_total_paid_cost_usd": "0.25", + "timeout_at": datetime(2026, 7, 16, 9, 15, tzinfo=UTC), + "approval_ttl_seconds": 900, + "provider_call_performed": False, + "route_change_performed": False, + "infrastructure_write_performed": False, + "raw_prompt_persistence_allowed": False, + "raw_response_persistence_allowed": False, + "next_action": "operator_gate5_decision_required", + "approval_path": ( + "/api/v1/platform/approvals/" + "018f2d04-4c37-7a18-b764-df0df0cbe111/decide" + ), + } + + monkeypatch.setattr( + operator_runs, + "create_paid_provider_canary_gate5_run_svc", + fake_create, + ) + body = operator_runs.CreatePaidProviderCanaryAuthorizationRequest( + work_item_id="AIA-SRE-013", + authorization_scope=PAID_CANARY_AUTHORIZATION_SCOPE, + ) + principal = AwoooPOperatorPrincipal( + operator_id="telegram:123456", + auth_method="operator_api_key", + ) + + response = await operator_runs.create_paid_provider_canary_authorization( + body, + principal, + ) + validated = ( + operator_runs.CreatePaidProviderCanaryAuthorizationResponse.model_validate( + response + ) + ) + + assert response["state"] == "waiting_approval" + assert validated.provider_order[-2:] == ("claude", "gemini") + assert captured == {"operator_id": "telegram:123456"} + + with pytest.raises(ValidationError): + operator_runs.CreatePaidProviderCanaryAuthorizationRequest.model_validate( + { + "work_item_id": "OTHER", + "authorization_scope": PAID_CANARY_AUTHORIZATION_SCOPE, + } + ) + + +def test_paid_canary_route_declares_operator_auth_dependency() -> None: + route = next( + route + for route in operator_runs.router.routes + if getattr(route, "path", "") + == "/approvals/paid-provider-canary/authorize" + ) + + assert any( + dependency.call is operator_runs.verify_awooop_operator + for dependency in route.dependant.dependencies + ) + + +@pytest.mark.asyncio +async def test_generic_runtime_service_persists_shadow_true( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[AwoooPRunState] = [] + + class _Db: + def add(self, run: AwoooPRunState) -> None: + captured.append(run) + + @asynccontextmanager + async def fake_db_context(project_id: str): + assert project_id == "awoooi" + yield _Db() + + monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context) + + _, is_duplicate = await platform_runtime.create_run( + project_id="awoooi", + agent_id="generic-agent", + trigger_type="api", + input_payload={"safe": True}, + ) + + assert is_duplicate is False + assert len(captured) == 1 + assert captured[0].is_shadow is True + assert captured[0].state == "pending" + + +@pytest.mark.asyncio +async def test_generic_public_run_creation_remains_shadow_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fake_create_run(**_: object) -> tuple[UUID, bool]: + return UUID("018f2d04-4c37-7a18-b764-df0df0cbe222"), False + + async def fake_write_audit(**kwargs: object) -> None: + details = kwargs["details"] + assert isinstance(details, dict) + assert details["is_shadow"] is True + + monkeypatch.setattr(public_runs, "create_run", fake_create_run) + monkeypatch.setattr(public_runs, "write_audit", fake_write_audit) + + response = await public_runs.create_platform_run( + public_runs.CreateRunRequest( + project_id="awoooi", + agent_id="generic-agent", + trigger_type="api", + ) + ) + + assert response.is_shadow is True + assert response.message == "Run 已接受（shadow mode）" diff --git a/apps/api/tests/test_paid_provider_canary_runner.py b/apps/api/tests/test_paid_provider_canary_runner.py new file mode 100644 index 000000000..dae215a26 --- /dev/null +++ b/apps/api/tests/test_paid_provider_canary_runner.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +RUNNER = REPO_ROOT / "scripts" / "ops" / "run-paid-provider-canary.py" + + +def test_runner_help_bootstraps_api_import_from_repo_root() -> None: + env = dict(os.environ) + env["PYTHONDONTWRITEBYTECODE"] = "1" + + result = subprocess.run( + [sys.executable, str(RUNNER), "--help"], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + + assert result.returncode == 0 + assert "--run-ref" in result.stdout + assert "ModuleNotFoundError" not in result.stderr + + +def test_runner_redacts_unexpected_exception_text( + monkeypatch, + capsys, +) -> None: + spec = importlib.util.spec_from_file_location("paid_canary_runner", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + leak = "http://192.168.0.111:11434/?key=secret-like-value" + + async def _failed(**_kwargs): + raise LookupError(leak) + + monkeypatch.setattr(module, "_run_validation", _failed) + monkeypatch.setattr( + sys, + "argv", + [ + str(RUNNER), + "--run-ref", + "source-sha", + "--authorization-ref", + "user-approved-20260716", + ], + ) + + assert module.main() == 2 + output = capsys.readouterr() + payload = json.loads(output.out) + assert payload["error_code"] == "canary_failed_LookupError" + assert leak not in output.out + assert leak not in output.err + + +def test_runner_returns_nonzero_for_partial_five_lane_validation( + monkeypatch, + capsys, +) -> None: + spec = importlib.util.spec_from_file_location("paid_canary_runner_partial", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + async def _partial(**_kwargs): + return { + "schema_version": "paid_provider_sre_canary_receipt_v3", + "independent_verifier": { + "passed": False, + "paid_activation_verifier_passed": True, + "five_lane_verifier_passed": False, + }, + } + + monkeypatch.setattr(module, "_run_validation", _partial) + monkeypatch.setattr( + sys, + "argv", + [ + str(RUNNER), + "--run-ref", + "source-sha", + "--authorization-ref", + "user-approved-20260716", + ], + ) + + assert module.main() == 1 + payload = json.loads(capsys.readouterr().out) + assert payload["independent_verifier"][ + "paid_activation_verifier_passed" + ] is True + assert payload["independent_verifier"]["five_lane_verifier_passed"] is False diff --git a/apps/api/tests/test_paid_provider_canary_validation.py b/apps/api/tests/test_paid_provider_canary_validation.py new file mode 100644 index 000000000..d88cd39a6 --- /dev/null +++ b/apps/api/tests/test_paid_provider_canary_validation.py @@ -0,0 +1,1076 @@ +from __future__ import annotations + +# ruff: noqa: E402 +import asyncio +import json +import os +from types import SimpleNamespace + +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") + +from src.services.ai_providers.interfaces import AIResult +from src.services.paid_provider_canary_validation import ( + run_paid_provider_canary_validation, + select_claude_canary_run_id, +) + +_AUTHORIZATION_REF = "267a3d26-3c29-470f-8fe3-388a2f408f39" +_AUTHORIZATION_TRACE_ID = ( + "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" +) + + +class _Provider: + def __init__( + self, + name: str, + *, + action: str = "NO_ACTION", + success: bool = True, + include_receipt: bool = True, + close_fails: bool = False, + error: str = "fixture_provider_unavailable", + cancel: bool = False, + ) -> None: + self.name = name + self.closed = False + self.action = action + self.success = success + self.include_receipt = include_receipt + self.close_fails = close_fails + self.error = error + self.cancel = cancel + self.calls = 0 + + async def analyze(self, prompt: str, context: dict) -> AIResult: + self.calls += 1 + assert "sanitized" in prompt + assert context["data_classification"] == "sanitized" + assert context["work_item_id"] == "AIA-SRE-013" + assert context["task_type"] == "diagnose" + assert context["ollama_model"] == "qwen3:14b" + if self.cancel: + raise asyncio.CancelledError + if not self.success: + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=self.error, + ) + return AIResult( + raw_response=json.dumps( + { + "suggested_action": self.action, + "target_resource": "awoooi-api", + "namespace": "awoooi-prod", + "kubectl_command": "", + "risk_level": "low", + "blast_radius": { + "affected_pods": 0, + "estimated_downtime": "0s", + "related_services": [], + "data_impact": "NONE", + }, + "reasoning": "Synthetic workload is healthy.", + "confidence": 0.95, + } + ), + success=True, + provider=self.name, + tokens=120, + cost_usd=0.001, + latency_ms=25, + audit_metadata=( + {"generation_receipt_id": f"gen-{self.name}"} + if self.include_receipt + else {} + ), + ) + + async def close(self) -> None: + self.closed = True + if self.close_fails: + raise RuntimeError("synthetic close failure") + + +class _Repository: + def __init__(self, *, omit_start: bool = False) -> None: + self.rows: list[dict] = [] + self.omit_start = omit_start + + async def append(self, event_type: str, **kwargs): + self.rows.append({"event_type": event_type, **kwargs}) + if event_type == "EXECUTION_STARTED" and self.omit_start: + return None + return SimpleNamespace(id=f"row-{len(self.rows)}") + + +def _configure_runtime_control(monkeypatch) -> dict[str, object]: + from src.services import paid_provider_canary_validation as module + + state: dict[str, object] = { + "claude": True, + "gemini": True, + "lock_owner": None, + "active_owner": None, + "active_completion_record_id": None, + "lease_ttl": -2, + "execution_claim_owner": None, + } + + def _readback( + owner: str | None = None, + completion_record_id: str | None = None, + ) -> dict: + lock_matches = bool(owner and state["lock_owner"] == owner) + active_matches = bool(owner and state["active_owner"] == owner) + completion_present = bool(state["active_completion_record_id"]) + claude_disabled = bool(state["claude"]) + gemini_disabled = bool(state["gemini"]) + return { + "claude_disabled": claude_disabled, + "gemini_disabled": gemini_disabled, + "lock_present": state["lock_owner"] is not None, + "lock_owner_matches": lock_matches, + "lock_ttl_seconds": state["lease_ttl"], + "active_receipt_present": state["active_owner"] is not None, + "active_receipt_owner_matches": active_matches, + "active_completion_receipt_present": completion_present, + "active_completion_receipt_matches": bool( + completion_record_id + and state["active_completion_record_id"] == completion_record_id + ), + "leased_enabled": bool( + lock_matches + and state["active_owner"] is None + and not claude_disabled + and not gemini_disabled + and int(state["lease_ttl"]) > 0 + ), + "persistent_promoted": bool( + lock_matches + and active_matches + and completion_present + and not claude_disabled + and not gemini_disabled + and state["lease_ttl"] == -1 + ), + "persistent_enabled": bool( + state["lock_owner"] is None + and active_matches + and completion_present + and ( + completion_record_id is None + or state["active_completion_record_id"] == completion_record_id + ) + and not claude_disabled + and not gemini_disabled + and state["lease_ttl"] == -1 + ), + "disabled_verified": bool( + claude_disabled + and gemini_disabled + and state["lock_owner"] is None + and state["active_owner"] is None + and state["active_completion_record_id"] is None + ), + } + + async def _control_readback( + owner: str | None = None, + completion_record_id: str | None = None, + ) -> dict: + return _readback(owner, completion_record_id) + + async def _acquire(owner: str) -> bool: + if not _readback()["disabled_verified"]: + return False + state.update( + claude=False, + gemini=False, + lock_owner=owner, + active_owner=None, + active_completion_record_id=None, + lease_ttl=300, + ) + return True + + async def _rollback(owner: str) -> bool: + if state["lock_owner"] not in {None, owner}: + return False + if state["active_owner"] not in {None, owner}: + return False + state.update( + claude=True, + gemini=True, + lock_owner=None, + active_owner=None, + active_completion_record_id=None, + lease_ttl=-2, + ) + return True + + async def _acquire_execution_claim(run_id: str, owner: str) -> bool: + assert run_id == _AUTHORIZATION_REF + if state["execution_claim_owner"] is not None: + return False + state["execution_claim_owner"] = owner + return True + + async def _release_execution_claim(run_id: str, owner: str) -> bool: + assert run_id == _AUTHORIZATION_REF + if state["execution_claim_owner"] != owner: + return False + state["execution_claim_owner"] = None + return True + + async def _receipt_readback( + *, + provider: str, + receipt_id: str, + trace_id: str, + run_id: str, + ) -> dict: + assert trace_id == _AUTHORIZATION_TRACE_ID + assert run_id == _AUTHORIZATION_REF + from src.services.ai_rate_limiter import AIRateLimiter + + budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") + budget_id = f"paid-run-{budget_hash[:32]}" + return { + "provider": provider, + "receipt_id": receipt_id, + "found": True, + "correlation_verified": True, + "finalized_accounted": True, + "reservation_released": True, + "success": True, + "status": "succeeded", + "accounted_tokens": 120, + "accounted_cost_micro_usd": 1000, + "accounted_cost_usd": 0.001, + "run_budget_required": True, + "run_budget_id": budget_id, + "run_budget_max_micro_usd": 250_000, + } + + async def _run_budget_readback(*, run_id: str) -> dict: + from src.services.ai_rate_limiter import AIRateLimiter + + budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") + return { + "schema_version": "ai_paid_run_budget_receipt_v1", + "run_budget_id": f"paid-run-{budget_hash[:32]}", + "found": True, + "status": "finalized", + "max_cost_micro_usd": 250_000, + "reserved_cost_micro_usd": 0, + "accounted_cost_micro_usd": 2000, + "accounted_cost_usd": 0.002, + "reservation_count": 2, + "finalized_count": 2, + "correlation_verified": True, + "cap_verified": True, + "terminal_verified": True, + } + + async def _authorization_readback(authorization_ref: str) -> dict: + assert authorization_ref == _AUTHORIZATION_REF + return { + "schema_version": "paid_provider_canary_authorization_receipt_v1", + "status": "authorized", + "authorized": True, + "error_code": None, + "authorization_ref": authorization_ref, + "run_id": authorization_ref, + "trace_id": _AUTHORIZATION_TRACE_ID, + "project_id": "awoooi", + "work_item_id": "AIA-SRE-013", + "checked_at": "2026-07-16T00:00:00+00:00", + "expires_at": "2026-07-16T00:15:00+00:00", + "valid_for_seconds": 900, + "contract": { + "agent_id": "awoooi-paid-provider-canary", + "state": "running", + "trigger_type": "api", + "trigger_ref": "paid-provider-canary:AIA-SRE-013", + "input_sha256": "fixture-contract-sha256", + "operator_approval_step": "operator_console.approve", + "operator_approval_audit": "run.approval.approve", + }, + "prior_canary_start_count": 0, + "reuse_check": "clear_at_read_time", + "read_only_verification": True, + "authorization_consumed_by_verifier": False, + "raw_input_persisted_or_returned": False, + "operator_identity_returned": False, + "secret_value_read_or_returned": False, + } + + async def _transport_receipt_issuer(**kwargs) -> dict: + provider = kwargs["provider"] + return { + "schema_version": "cloud_transport_preflight_receipt_v1", + "receipt_id": ( + "cloud-transport:" + ("a" if provider.endswith("_a") else "b") * 32 + ), + "source": "cloud_transport_preflight.network_probe", + "status": "degraded_transport_reachable", + "transport_boundary": "public_http_sanitized_candidate_only", + "transport_security_status": "degraded_public_http", + "provider": provider, + "endpoint_sha256": "a" * 64, + "trace_id": kwargs["trace_id"], + "run_id": kwargs["run_id"], + "work_item_id": kwargs["work_item_id"], + "check_completed": True, + "http_status": 200, + "ollama_contract_verified": True, + "observed_at": "2026-07-16T00:00:00+00:00", + "expires_at": "2026-07-16T00:01:30+00:00", + "ttl_seconds": 90, + "durable_write_ack": True, + "verifier_status": "verified", + "verified_by": "cloud_transport_preflight.redis_readback", + "raw_response_persisted": False, + "endpoint_value_persisted": False, + "secret_value_persisted_or_returned": False, + } + + monkeypatch.setattr( + module, + "_require_paid_canary_authorization", + _authorization_readback, + ) + monkeypatch.setattr( + module, + "issue_cloud_transport_receipt", + _transport_receipt_issuer, + ) + monkeypatch.setattr(module, "_paid_canary_control_readback", _control_readback) + monkeypatch.setattr(module, "_acquire_paid_canary_lease", _acquire) + monkeypatch.setattr( + module, + "_acquire_paid_canary_execution_claim", + _acquire_execution_claim, + ) + monkeypatch.setattr( + module, + "_release_paid_canary_execution_claim", + _release_execution_claim, + ) + monkeypatch.setattr(module, "_rollback_paid_canary", _rollback) + monkeypatch.setattr(module, "_generation_receipt_readback", _receipt_readback) + monkeypatch.setattr(module, "_run_budget_readback", _run_budget_readback) + return state + + +def test_canary_run_selection_is_deterministic_and_inside_percent() -> None: + first = select_claude_canary_run_id("source-sha", 5) + second = select_claude_canary_run_id("source-sha", 5) + + assert first == second + assert first[1] < 5 + + +@pytest.mark.asyncio +async def test_durable_authorization_blocks_before_receipt_or_provider_call( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + + async def _blocked(_authorization_ref: str) -> dict: + raise RuntimeError("paid_canary_authorization_expired") + + monkeypatch.setattr(module, "_require_paid_canary_authorization", _blocked) + providers = [ + _Provider("ollama", include_receipt=False), + _Provider("ollama_gcp_b", include_receipt=False), + _Provider("ollama_local", include_receipt=False), + _Provider("claude"), + _Provider("gemini"), + ] + repository = _Repository() + + with pytest.raises(RuntimeError, match="paid_canary_authorization_expired"): + await run_paid_provider_canary_validation( + run_ref="source-sha-auth-blocked", + operator_id="test-operator", + authorization_ref="00000000-0000-4000-8000-000000000001", + ollama_gcp_a_provider=providers[0], + ollama_gcp_b_provider=providers[1], + ollama_local_provider=providers[2], + claude_provider=providers[3], + gemini_provider=providers[4], + operation_repository=repository, + ) + + assert repository.rows == [] + assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0] + + +@pytest.mark.asyncio +async def test_atomic_execution_claim_blocks_concurrent_authorization_reuse( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + _configure_runtime_control(monkeypatch) + + async def _claim_held(_run_id: str, _owner: str) -> bool: + return False + + monkeypatch.setattr( + module, + "_acquire_paid_canary_execution_claim", + _claim_held, + ) + providers = [_Provider(name) for name in ("a", "b", "c", "d", "e")] + repository = _Repository() + + with pytest.raises( + RuntimeError, + match="paid_canary_authorization_execution_claim_held", + ): + await run_paid_provider_canary_validation( + run_ref="source-sha-concurrent", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=providers[0], + ollama_gcp_b_provider=providers[1], + ollama_local_provider=providers[2], + claude_provider=providers[3], + gemini_provider=providers[4], + operation_repository=repository, + ) + + assert repository.rows == [] + assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0] + + +@pytest.mark.asyncio +async def test_canary_discards_content_and_writes_bounded_receipt(monkeypatch) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + gcp_a = _Provider("ollama", include_receipt=False) + gcp_b = _Provider("ollama_gcp_b", include_receipt=False) + local = _Provider("ollama_local", include_receipt=False) + claude = _Provider("claude") + gemini = _Provider("gemini") + repository = _Repository() + + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=gcp_a, + ollama_gcp_b_provider=gcp_b, + ollama_local_provider=local, + claude_provider=claude, + gemini_provider=gemini, + operation_repository=repository, + ) + + assert receipt["independent_verifier"]["passed"] is True + assert receipt["independent_verifier"]["paid_activation_verifier_passed"] is True + assert receipt["independent_verifier"]["five_lane_verifier_passed"] is True + assert receipt["independent_verifier"]["paid_provider_comparison"] == ( + "initial_canary_tie_no_overall_winner" + ) + assert ( + receipt["policy_decision"]["durable_authorization_receipt"]["authorized"] + is True + ) + assert receipt["bounded_execution"]["provider_order"] == [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ] + assert receipt["source_receipt"]["gcp_transport"] == { + "status": "degraded_public_http", + "sanitized_only": True, + "receipt_ids": [ + "cloud-transport:" + "a" * 32, + "cloud-transport:" + "b" * 32, + ], + "preflight": [ + { + "provider": "ollama_gcp_a", + "status": "verified", + "receipt_id": "cloud-transport:" + "a" * 32, + }, + { + "provider": "ollama_gcp_b", + "status": "verified", + "receipt_id": "cloud-transport:" + "b" * 32, + }, + ], + "same_run_bound": True, + "durable_readback_verified": True, + "encrypted_transport_claimed": False, + } + assert [ + row["contract_score_percent"] for row in receipt["bounded_execution"]["results"] + ] == [100, 100, 100, 100, 100] + assert all( + "raw_response" not in row for row in receipt["bounded_execution"]["results"] + ) + assert receipt["durable_operation_writeback"] == { + "start_recorded": True, + "lease_change_recorded": True, + "lease_change_record_id": "row-2", + "comparison_ready_recorded": True, + "comparison_ready_record_id": "row-3", + "rollback_change_recorded": True, + "rollback_change_record_id": "row-4", + "activation_failure_recorded": False, + "activation_failure_record_id": None, + "completion_recorded": True, + "completion_record_id": "row-5", + } + assert [row["event_type"] for row in repository.rows] == [ + "EXECUTION_STARTED", + "CHANGE_APPLIED", + "PRE_FLIGHT_PASSED", + "CHANGE_APPLIED", + "EXECUTION_COMPLETED", + ] + assert claude.closed is True + assert gemini.closed is True + assert state["claude"] is True + assert state["gemini"] is True + assert state["lock_owner"] is None + assert state["active_owner"] is None + assert state["execution_claim_owner"] is None + assert receipt["policy_decision"]["authorization_execution_claim"] == { + "acquired": True, + "released_after_durable_start": True, + "same_run_bound": True, + } + assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == ( + "validated_comparison_rollback_verified" + ) + assert receipt["policy_decision"]["persistent_promotion_authorized"] is False + + +@pytest.mark.asyncio +async def test_canary_verifier_rejects_a_mutating_false_positive(monkeypatch) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha-failed", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude", action="RESTART_DEPLOYMENT"), + gemini_provider=_Provider("gemini"), + operation_repository=_Repository(), + ) + + assert receipt["independent_verifier"]["passed"] is False + assert ( + receipt["bounded_execution"]["results"][3]["contract_checks"][ + "expected_no_action" + ] + is False + ) + assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True + assert state["claude"] is True + assert state["gemini"] is True + assert state["lock_owner"] is None + assert state["active_owner"] is None + assert state["execution_claim_owner"] is None + + +@pytest.mark.asyncio +async def test_ollama_baseline_failure_forces_paired_paid_canary_rollback( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha-partial-baseline", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", success=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude"), + gemini_provider=_Provider("gemini"), + operation_repository=_Repository(), + ) + + assert receipt["independent_verifier"]["passed"] is False + assert receipt["independent_verifier"]["paid_activation_verifier_passed"] is False + assert ( + receipt["independent_verifier"]["paid_provider_contract_verifier_passed"] + is True + ) + assert receipt["independent_verifier"]["five_lane_verifier_passed"] is False + assert receipt["independent_verifier"]["five_lane_comparison_ready"] is False + assert receipt["independent_verifier"]["ollama_baseline_available_count"] == 2 + assert receipt["bounded_execution"]["results"][0]["provider"] == ("ollama_gcp_a") + assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True + assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == ( + "failed_rollback_verified" + ) + assert state["claude"] is True + assert state["gemini"] is True + assert state["active_owner"] is None + + +@pytest.mark.asyncio +async def test_missing_paid_generation_receipt_forces_atomic_rollback( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha-missing-receipt", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude", include_receipt=False), + gemini_provider=_Provider("gemini"), + operation_repository=_Repository(), + ) + + assert receipt["independent_verifier"]["passed"] is False + assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == ( + "failed_rollback_verified" + ) + assert state["claude"] is True + assert state["gemini"] is True + assert state["lock_owner"] is None + assert state["active_owner"] is None + + +@pytest.mark.asyncio +async def test_no_paid_or_baseline_call_without_durable_start_receipt( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + providers = [_Provider(name) for name in ("a", "b", "c", "d", "e")] + + with pytest.raises( + RuntimeError, + match="paid_provider_canary_start_receipt_not_recorded", + ): + await run_paid_provider_canary_validation( + run_ref="source-sha-no-start", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=providers[0], + ollama_gcp_b_provider=providers[1], + ollama_local_provider=providers[2], + claude_provider=providers[3], + gemini_provider=providers[4], + operation_repository=_Repository(omit_start=True), + ) + + assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0] + assert state["claude"] is True + assert state["gemini"] is True + assert state["lock_owner"] is None + assert state["active_owner"] is None + + +@pytest.mark.asyncio +async def test_failed_exact_receipt_cannot_promote_and_rolls_back(monkeypatch) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + + async def _failed_receipt(**kwargs) -> dict: + return { + "provider": kwargs["provider"], + "receipt_id": kwargs["receipt_id"], + "found": True, + "correlation_verified": True, + "finalized_accounted": True, + "reservation_released": True, + "success": False, + "status": "failed_estimate_charged", + "accounted_tokens": 120, + "accounted_cost_usd": 0.001, + } + + monkeypatch.setattr(module, "_generation_receipt_readback", _failed_receipt) + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha-failed-exact-receipt", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude"), + gemini_provider=_Provider("gemini"), + operation_repository=_Repository(), + ) + + assert receipt["independent_verifier"]["passed"] is False + assert ( + receipt["independent_verifier"]["generation_receipt_readback_passed"] is False + ) + assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True + assert state["claude"] is True + assert state["gemini"] is True + + +@pytest.mark.asyncio +async def test_aggregate_run_budget_must_be_finalized_under_exact_cap( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + + async def _pending_budget(*, run_id: str) -> dict: + from src.services.ai_rate_limiter import AIRateLimiter + + budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") + return { + "schema_version": "ai_paid_run_budget_receipt_v1", + "run_budget_id": f"paid-run-{budget_hash[:32]}", + "found": True, + "status": "pending_finalize", + "max_cost_micro_usd": 250_000, + "reserved_cost_micro_usd": 1000, + "accounted_cost_micro_usd": 1000, + "reservation_count": 2, + "finalized_count": 1, + "correlation_verified": True, + "cap_verified": True, + "terminal_verified": False, + } + + monkeypatch.setattr(module, "_run_budget_readback", _pending_budget) + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha-pending-run-budget", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude"), + gemini_provider=_Provider("gemini"), + operation_repository=_Repository(), + ) + + assert receipt["independent_verifier"]["passed"] is False + assert receipt["independent_verifier"]["run_budget_readback_passed"] is False + assert ( + receipt["independent_verifier"]["generation_receipt_readback_passed"] is False + ) + assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True + assert state["claude"] is True + assert state["gemini"] is True + + +@pytest.mark.asyncio +async def test_canary_rejects_generation_and_aggregate_accounting_mismatch( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + + async def _mismatched_budget(*, run_id: str) -> dict: + from src.services.ai_rate_limiter import AIRateLimiter + + budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") + return { + "schema_version": "ai_paid_run_budget_receipt_v1", + "run_budget_id": f"paid-run-{budget_hash[:32]}", + "found": True, + "status": "finalized", + "max_cost_micro_usd": 250_000, + "reserved_cost_micro_usd": 0, + "accounted_cost_micro_usd": 1999, + "reservation_count": 2, + "finalized_count": 2, + "correlation_verified": True, + "cap_verified": True, + "terminal_verified": True, + } + + monkeypatch.setattr(module, "_run_budget_readback", _mismatched_budget) + repository = _Repository() + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha-accounting-mismatch", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude"), + gemini_provider=_Provider("gemini"), + operation_repository=repository, + ) + + assert receipt["independent_verifier"]["run_budget_readback_passed"] is False + assert receipt["independent_verifier"]["passed"] is False + assert "PRE_FLIGHT_PASSED" not in {row["event_type"] for row in repository.rows} + assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True + assert state["claude"] is True + assert state["gemini"] is True + + +@pytest.mark.asyncio +async def test_cancelled_paid_call_runs_verified_compensation(monkeypatch) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + repository = _Repository() + + with pytest.raises(asyncio.CancelledError): + await run_paid_provider_canary_validation( + run_ref="source-sha-cancelled", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude", cancel=True), + gemini_provider=_Provider("gemini"), + operation_repository=repository, + ) + + assert state["claude"] is True + assert state["gemini"] is True + assert state["lock_owner"] is None + assert state["active_owner"] is None + assert repository.rows[-1]["event_type"] == "EXECUTION_COMPLETED" + assert repository.rows[-1]["success"] is False + assert repository.rows[-1]["context"]["rollback_verified"] is True + + +@pytest.mark.asyncio +async def test_ambiguous_lease_failure_is_compensated_and_verified(monkeypatch) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + state = _configure_runtime_control(monkeypatch) + + async def _ambiguous_acquire(owner: str) -> bool: + state.update( + claude=False, + gemini=False, + lock_owner=owner, + active_owner=None, + lease_ttl=300, + ) + return False + + monkeypatch.setattr(module, "_acquire_paid_canary_lease", _ambiguous_acquire) + with pytest.raises( + RuntimeError, + match="paid_provider_canary_activation_lease_failed", + ): + await run_paid_provider_canary_validation( + run_ref="source-sha-ambiguous-lease", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude"), + gemini_provider=_Provider("gemini"), + operation_repository=_Repository(), + ) + + assert state["claude"] is True + assert state["gemini"] is True + assert state["lock_owner"] is None + + +@pytest.mark.asyncio +async def test_provider_error_text_is_never_written_to_safe_receipt( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + monkeypatch.setattr( + module, + "settings", + SimpleNamespace( + CLAUDE_EXECUTION_MODE="canary", + CLAUDE_CANARY_PERCENT=5, + GEMINI_EXECUTION_MODE="canary", + GEMINI_CANARY_PERCENT=5, + ALERT_OLLAMA_MODEL="qwen3:14b", + ), + ) + _configure_runtime_control(monkeypatch) + unsafe_error = "http://192.168.0.111:11434/?key=secret-like-value" + receipt = await run_paid_provider_canary_validation( + run_ref="source-sha-sanitized-error", + operator_id="test-operator", + authorization_ref=_AUTHORIZATION_REF, + ollama_gcp_a_provider=_Provider( + "ollama", + success=False, + include_receipt=False, + error=unsafe_error, + ), + ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), + ollama_local_provider=_Provider("ollama_local", include_receipt=False), + claude_provider=_Provider("claude"), + gemini_provider=_Provider("gemini"), + operation_repository=_Repository(), + ) + + serialized = json.dumps(receipt, sort_keys=True) + assert unsafe_error not in serialized + assert "192.168.0.111" not in serialized + assert "secret-like-value" not in serialized + assert receipt["bounded_execution"]["results"][0]["error_code"] == ( + "provider_error_redacted" + ) diff --git a/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py new file mode 100644 index 000000000..4b8c448cc --- /dev/null +++ b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from src.services.portfolio_infrastructure_asset_reconciliation import ( + load_portfolio_infrastructure_asset_reconciliation, +) + +ROOT = Path(__file__).resolve().parents[3] +OPERATIONS = ROOT / "docs" / "operations" +SNAPSHOT = OPERATIONS / "portfolio-infrastructure-asset-reconciliation.snapshot.json" + + +def test_loader_maps_every_imported_inventory_row_without_claiming_live_truth() -> None: + payload = load_portfolio_infrastructure_asset_reconciliation() + + mapped_rows = [row for asset in payload["assets"] for row in asset["source_rows"]] + assert len(mapped_rows) == 131 + assert len(mapped_rows) == len(set(mapped_rows)) + assert {"L012", "L168", "L206", "L259"}.issubset(mapped_rows) + assert payload["rollups"]["runtime_closed_assets"] == 0 + assert payload["rollups"]["source_reconciled_assets"] == 6 + assert payload["rollups"]["conflicts"] == 11 + assert payload["rollups"]["conflict_assets"] == 36 + assert payload["rollups"]["canonical_assets"] == 144 + assert all( + asset["live_truth_state"] != "verified_healthy" for asset in payload["assets"] + ) + assert payload["source_inventory"]["version_history"][0]["source_line_count"] == 241 + assert payload["source_inventory"]["source_line_count"] == 284 + + +def test_reconciliation_includes_missing_control_plane_and_provider_assets() -> None: + payload = load_portfolio_infrastructure_asset_reconciliation() + assets = {asset["canonical_id"]: asset for asset in payload["assets"]} + + required = { + "windows-vmware:host_99", + "service:ollama:host110", + "host:111", + "ai-provider:ollama_gcp_a", + "ai-provider:ollama_gcp_b", + "ai-provider:claude", + "ai-provider:gemini", + "control-plane:holmesgpt-investigator", + "signal:alertmanager-webhook-chain", + "control-plane:km-rag", + "control-plane:mcp-gateway", + "control-plane:playbook-registry", + "control-plane:independent-verifier-registry", + } + assert required.issubset(assets) + imported_omissions = required - {"service:ollama:host110"} + assert all( + assets[canonical_id]["reconciliation_state"] + == "missing_from_imported_inventory" + for canonical_id in imported_omissions + ) + assert assets["service:ollama:host110"]["reconciliation_state"] == ( + "retired_tombstone_source_ready_runtime_absence_pending" + ) + assert assets["windows-vmware:host_99"]["executor"] == "Agent99" + assert assets["host:111"]["executor"] == "host_ansible_executor" + assert assets["ai-provider:ollama_gcp_a"]["source_truth_state"] == ( + "source_reconciled_runtime_pending" + ) + assert "WireGuard" in " ".join( + assets["ai-provider:ollama_gcp_a"]["findings"] + ) or "public HTTP" in " ".join( + assets["ai-provider:ollama_gcp_a"]["findings"] + ) + + +def test_provider_alert_chain_and_gitea_runtime_gaps_are_explicit() -> None: + payload = load_portfolio_infrastructure_asset_reconciliation() + assets = {asset["canonical_id"]: asset for asset in payload["assets"]} + + tombstone = assets["service:ollama:host110"] + assert tombstone["runtime_identity"].startswith("retired-tombstone:") + assert tombstone["monitoring"]["coverage"] == ( + "source_tombstone_runtime_absence_pending" + ) + + host111 = assets["host:111"] + assert "macos-launchd" in host111["runtime_identity"] + assert {"host120_origin_generation", "host121_origin_generation"}.issubset( + host111["monitoring"]["signals"] + ) + assert host111["monitoring"]["coverage"] == ( + "source_candidate_runtime_sensor_missing" + ) + + for provider_id in ("ai-provider:ollama_gcp_a", "ai-provider:ollama_gcp_b"): + provider = assets[provider_id] + assert provider["transport_policy"] == { + "observed": "public_http", + "execution_scope": "sanitized_candidate_only", + "tool_loop": "fail_closed", + "promotion_requires": "secure_mesh_or_tls_plus_runtime_readback", + } + assert provider["monitoring"]["coverage"] == ( + "source_candidate_runtime_target_missing" + ) + + for provider_id in ("ai-provider:claude", "ai-provider:gemini"): + provider = assets[provider_id] + assert provider["credential_metadata"]["raw_value_recorded"] is False + assert provider["credential_metadata"]["raw_value_readback_allowed"] is False + assert provider["authorization"] == { + "gate": "Gate5", + "durable_run_binding_required": True, + "status": "pending", + } + assert provider["canary"]["paired_rollback_required"] is True + assert provider["canary"]["cost_caps_verified"] is False + + alert_chain = assets["signal:alertmanager-webhook-chain"] + assert "Host99 Agent99" in " ".join(alert_chain["findings"]) + assert "first-hop credential" in " ".join(alert_chain["findings"]) + assert alert_chain["monitoring"]["coverage"] == ( + "source_primary_and_independent_pull_ready_runtime_pending" + ) + gitea = assets["service:gitea"] + exporter = assets["service:gitea-exporter:host110"] + assert gitea["monitoring"]["coverage"] == "runtime_target_missing" + assert exporter["monitoring"]["coverage"] == ( + "source_candidate_runtime_target_missing" + ) + + +def test_source_reconciled_alert_and_gitea_assets_remain_runtime_pending() -> None: + payload = load_portfolio_infrastructure_asset_reconciliation() + assets = {asset["canonical_id"]: asset for asset in payload["assets"]} + + assert assets["service:prometheus"]["runtime_identity"] == ( + "host110/docker/prometheus" + ) + assert assets["service:alertmanager"]["runtime_identity"] == ( + "host110/docker/alertmanager" + ) + assert assets["service:alertmanager"]["live_truth_state"] == ( + "not_probed_this_reconciliation" + ) + assert assets["service:gitea-exporter:host110"]["reconciliation_state"] == ( + "source_retirement_ready_runtime_pending" + ) + assert assets["signal:alertmanager-webhook-chain"]["source_truth_state"] == ( + "source_reconciled_runtime_pending" + ) + + +def test_host112_expansion_and_removed_asset_drift() -> None: + payload = load_portfolio_infrastructure_asset_reconciliation() + assets = {asset["canonical_id"]: asset for asset in payload["assets"]} + + required = { + "service:wazuh-indexer:host112", + "service:wazuh-dashboard:host112", + "service:awoooi-guardian:host112", + "service:filebeat:host112", + "service:wg-easy:host112", + "service:node-exporter:host112", + "package:metasploit:host112", + "package:openvas:host112", + "schedule:host112:wazuh-guardian-hourly", + "schedule:host112:daily-0200", + } + assert required.issubset(assets) + assert assets["service:wg-easy:host112"]["reconciliation_state"] == ( + "github_freeze_and_network_boundary_conflict" + ) + assert assets["service:fail2ban:host188"]["reconciliation_state"] == ( + "missing_from_imported_inventory" + ) + + +def test_every_expanded_asset_has_learning_and_typed_execution_contract() -> None: + payload = load_portfolio_infrastructure_asset_reconciliation() + expected = ["KM", "RAG", "MCP", "PlayBook"] + + assert all(asset["learning_targets"] == expected for asset in payload["assets"]) + assert all( + asset["alerting"]["cross_domain_fallback_allowed"] is False + for asset in payload["assets"] + ) + assert all(asset["verifier"] for asset in payload["assets"]) + priorities = [item["priority"] for item in payload["reconciliation_work_items"]] + assert priorities == ["P0"] * 10 + ["P1"] * 2 + + +def test_loader_rejects_runtime_false_green(tmp_path: Path) -> None: + payload = json.loads(SNAPSHOT.read_text(encoding="utf-8")) + payload["source_inventory"]["live_truth_accepted"] = True + (tmp_path / SNAPSHOT.name).write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="cannot be live truth"): + load_portfolio_infrastructure_asset_reconciliation(tmp_path) + + +def test_loader_rejects_cross_domain_executor_drift(tmp_path: Path) -> None: + payload = json.loads(SNAPSHOT.read_text(encoding="utf-8")) + payload["asset_groups"][0]["common"]["executor"] = "generic_fallback" + (tmp_path / SNAPSHOT.name).write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="executor/domain mismatch"): + load_portfolio_infrastructure_asset_reconciliation(tmp_path) diff --git a/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py new file mode 100644 index 000000000..a0886220d --- /dev/null +++ b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +# ruff: noqa: E402, I001 + +import os + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") + +import src.api.v1.agents as agents_module +from src.api.v1.agents import router + + +def _client() -> TestClient: + app = FastAPI() + app.include_router(router, prefix="/api/v1") + return TestClient(app) + + +def test_endpoint_returns_complete_redacted_portfolio_reconciliation() -> None: + response = _client().get( + "/api/v1/agents/portfolio-infrastructure-asset-reconciliation" + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["schema_version"] == ( + "portfolio_infrastructure_asset_reconciliation_v1" + ) + assert payload["program_id"] == "AIA-SRE-P0-20260715" + assert payload["rollups"]["imported_item_rows"] == 131 + assert payload["rollups"]["canonical_assets"] == 144 + assert payload["rollups"]["inventory_omissions"] == 18 + assert payload["rollups"]["conflicts"] == 11 + assert payload["rollups"]["conflict_assets"] == 36 + assert payload["rollups"]["reconciliation_work_items"] == 12 + assert payload["rollups"]["runtime_closed_assets"] == 0 + assert len(payload["assets"]) == 144 + assert "asset_groups" not in payload + + serialized = response.text + assert "192.168.0." not in serialized + assert "/Users/ogt/" not in serialized + assert "34.143.170.20" not in serialized + assert "34.21.145.224" not in serialized + + +def test_endpoint_maps_missing_snapshot_to_404(monkeypatch) -> None: + def _missing(): + raise FileNotFoundError("snapshot missing") + + monkeypatch.setattr( + agents_module, + "load_portfolio_infrastructure_asset_reconciliation", + _missing, + ) + response = _client().get( + "/api/v1/agents/portfolio-infrastructure-asset-reconciliation" + ) + + assert response.status_code == 404 + + +def test_endpoint_maps_invalid_snapshot_to_500(monkeypatch) -> None: + def _invalid(): + raise ValueError("invalid snapshot") + + monkeypatch.setattr( + agents_module, + "load_portfolio_infrastructure_asset_reconciliation", + _invalid, + ) + response = _client().get( + "/api/v1/agents/portfolio-infrastructure-asset-reconciliation" + ) + + assert response.status_code == 500 + assert response.json()["detail"] == ( + "全產品基礎設施與 AI 自動化資產對帳總帳無效" + ) diff --git a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py index 2255e1dc8..ba35e266c 100644 --- a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py +++ b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py @@ -27,13 +27,23 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - payload = load_sre_k3s_controlled_automation_work_items() assert payload["program_id"] == "AIA-SRE-P0-20260715" - assert payload["current_p0"]["id"] == "AIA-SRE-004" + assert payload["current_p0"]["id"] == "AIA-SRE-013" + assert [ + row["work_item_id"] for row in payload["immediate_execution_queue"] + ] == [ + "AIA-SRE-017", + "AIA-SRE-002", + "AIA-SRE-004", + "AIA-SRE-013", + "AIA-SRE-014", + "AIA-SRE-015", + ] assert payload["architecture"]["pipeline"] == [ "Alert", "Canonical Asset Normalize", "Typed Domain Router", "HolmesGPT Investigator", - "Ollama RCA / Claude Fallback / Gemini Critic", + "Ollama RCA / Gemini Critic", "Deterministic Policy", "Single Controlled Executor", "Independent Verifier", @@ -46,8 +56,10 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "claude", "gemini", ] - assert payload["provider_policy"]["claude_paid_call_allowed"] is False - assert payload["provider_policy"]["gemini_paid_call_allowed"] is False + assert payload["provider_policy"]["claude_paid_call_allowed"] is True + assert payload["provider_policy"]["gemini_paid_call_allowed"] is True + assert payload["provider_policy"]["paid_execution_mode"] == "canary" + assert payload["provider_policy"]["paid_canary_percent"] == 5 assert payload["provider_policy"]["route_label"] == ( "GCP-A -> GCP-B -> host111 Ollama -> Anthropic Claude -> Gemini API" ) @@ -56,13 +68,46 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "provider": "ollama_local", "identity": "host111", } + assert payload["provider_policy"]["durable_gate5_authorization"]["status"] == ( + "pending" + ) + assert ( + payload["provider_policy"]["durable_gate5_authorization"] + ["free_form_authorization_ref_is_sufficient"] + is False + ) + assert payload["provider_policy"]["paid_pair_contract"] == { + "enable_and_rollback_as_pair": True, + "partial_five_lane_terminal": ( + "rollback_both_paid_providers_and_fail_canary" + ), + "daily_monthly_and_per_incident_cost_caps_required": True, + "source_status": ( + "same_run_gate5_atomic_single_use_claim_and_aggregate_cap_ready" + ), + "runtime_status": "paid_canary_pending", + } + for provider_id in ("ollama_gcp_a", "ollama_gcp_b"): + boundary = payload["provider_policy"]["cloud_transport_contract"][ + provider_id + ] + assert boundary["observed_transport"] == "public_http" + assert boundary["execution_scope"] == "sanitized_candidate_only" + assert boundary["tool_loop_terminal"] == "fail_closed" + assert boundary["exact_prometheus_target_runtime_status"] == "missing" + for provider_id in ("claude", "gemini"): + metadata = payload["provider_policy"]["credential_metadata"][provider_id] + assert metadata["user_supplied_credential_noted"] is True + assert metadata["raw_value_recorded_in_repository"] is False + assert metadata["raw_value_readback_allowed"] is False assert payload["agent99_host_operations_bridge"]["single_executor_domains"] == [ "windows_vmware", "control_plane_recovery", ] - assert payload["agent99_host_operations_bridge"][ - "unknown_asset_dispatch_allowed" - ] is False + assert ( + payload["agent99_host_operations_bridge"]["unknown_asset_dispatch_allowed"] + is False + ) assert payload["rollups"] == { "total_items": 18, "by_priority": {"P0": 16, "P1": 2}, @@ -80,6 +125,27 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - } +def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None: + payload = load_sre_k3s_controlled_automation_work_items() + items = {item["id"]: item for item in payload["work_items"]} + + assert "retired tombstone" in " ".join(items["AIA-SRE-002"]["confirmed_truth"]) + assert "host120/host121" in " ".join(items["AIA-SRE-002"]["runtime_gaps"]) + assert "public-HTTP" in items["AIA-SRE-013"]["exit_condition"] + assert "durable Gate5" in " ".join(items["AIA-SRE-013"]["runtime_gaps"]) + assert "host99 Agent99" in " ".join(items["AIA-SRE-017"]["runtime_gaps"]) + assert "atomic 15-file bundle" in items["AIA-SRE-017"]["next_action"] + assert "gitea-native" in " ".join(items["AIA-SRE-018"]["runtime_gaps"]) + assert payload["completion_contract"]["evidence_layers"] == { + "source_test": "partial_source_evidence_only", + "gitea_cd_deploy_marker": "pending_for_this_program_sha", + "production_runtime": "zero_work_items_closed", + "visible_ui_telegram": "pending_same_run_receipts", + "learning_writeback": "pending_same_run_durable_acknowledgements", + } + assert all(item["status"] != "runtime_closed" for item in items.values()) + + def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None: payload = load_sre_k3s_controlled_automation_work_items() projection = build_sre_k3s_program_projection(payload) @@ -87,21 +153,43 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None: assert projection["rollups"]["source_implemented_items"] == 8 assert projection["rollups"]["runtime_closed_items"] == 0 assert projection["rollups"]["program_completion_percent"] == 0 - assert projection["domain_count"] == 7 - assert projection["claude_paid_call_allowed"] is False - assert projection["gemini_paid_call_allowed"] is False + assert projection["domain_count"] == 8 + assert projection["claude_paid_call_allowed"] is True + assert projection["gemini_paid_call_allowed"] is True + assert projection["immediate_execution_queue"][0]["work_item_id"] == ( + "AIA-SRE-017" + ) assert "work_items" not in projection -def test_loader_rejects_gemini_paid_enablement_without_cost_gate( +def test_loader_rejects_immediate_execution_queue_reordering( tmp_path: Path, ) -> None: payload = json.loads(SNAPSHOT.read_text(encoding="utf-8")) - payload["provider_policy"]["gemini_paid_call_allowed"] = True + payload["immediate_execution_queue"][0], payload["immediate_execution_queue"][1] = ( + payload["immediate_execution_queue"][1], + payload["immediate_execution_queue"][0], + ) + for position, row in enumerate(payload["immediate_execution_queue"], start=1): + row["position"] = position target = tmp_path / SNAPSHOT.name target.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(ValueError, match="explicit cost cap"): + with pytest.raises(ValueError, match="safety order changed"): + load_sre_k3s_controlled_automation_work_items(tmp_path) + + +def test_loader_rejects_paid_enablement_without_cost_gate( + tmp_path: Path, +) -> None: + payload = json.loads(SNAPSHOT.read_text(encoding="utf-8")) + payload["provider_policy"]["required_before_paid_provider_enablement"].remove( + "production_cost_readback" + ) + target = tmp_path / SNAPSHOT.name + target.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="explicit cost controls"): load_sre_k3s_controlled_automation_work_items(tmp_path) @@ -130,9 +218,10 @@ def test_endpoint_returns_full_work_ledger() -> None: assert payload["program_id"] == "AIA-SRE-P0-20260715" assert len(payload["work_items"]) == 18 assert payload["rollups"]["runtime_closed_items"] == 0 - assert payload["agent99_host_operations_bridge"][ - "completion_callback" - ] == "/api/v1/agents/agent99/completion-callback" + assert ( + payload["agent99_host_operations_bridge"]["completion_callback"] + == "/api/v1/agents/agent99/completion-callback" + ) def test_typed_target_preview_is_no_write_and_fail_closed() -> None: @@ -170,20 +259,14 @@ def test_typed_target_preview_is_no_write_and_fail_closed() -> None: "asset_identity_unresolved" ) assert unresolved_payload["route"]["executor"] is None - assert unresolved_payload["route"]["agent99_bridge"][ - "dispatch_allowed" - ] is False + assert unresolved_payload["route"]["agent99_bridge"]["dispatch_allowed"] is False assert agent99.status_code == 200 agent99_payload = agent99.json() assert agent99_payload["dispatch_performed"] is False - assert agent99_payload["route"]["canonical_asset_id"] == ( - "windows-vmware:host_99" - ) + assert agent99_payload["route"]["canonical_asset_id"] == ("windows-vmware:host_99") assert agent99_payload["route"]["executor"] == "Agent99" - assert agent99_payload["route"]["agent99_bridge"][ - "dispatch_allowed" - ] is True + assert agent99_payload["route"]["agent99_bridge"]["dispatch_allowed"] is True def test_wazuh_typed_target_preview_is_bounded_and_public_safe() -> None: @@ -211,8 +294,6 @@ def test_wazuh_typed_target_preview_is_bounded_and_public_safe() -> None: assert route["resolution_status"] == "resolved" assert route["canonical_asset_id"] == "service:wazuh-manager" assert route["host"] == "host:kali-readonly" - assert route["allowed_catalog_ids"] == [ - "ansible:wazuh-manager-posture-readback" - ] + assert route["allowed_catalog_ids"] == ["ansible:wazuh-manager-posture-readback"] assert route["allowed_inventory_hosts"] == ["host_112"] assert route["cross_domain_fallback_allowed"] is False diff --git a/apps/api/tests/test_sre_typed_domain_router.py b/apps/api/tests/test_sre_typed_domain_router.py index cfe1d9cb7..c1a51f7bc 100644 --- a/apps/api/tests/test_sre_typed_domain_router.py +++ b/apps/api/tests/test_sre_typed_domain_router.py @@ -79,6 +79,51 @@ def _sentry_incident() -> dict: } +def test_alert_chain_broken_routes_to_exact_host110_container_lane() -> None: + route = resolve_typed_alert_target( + alertname="AlertChainBroken_Alertmanager", + target_resource="alert-chain", + namespace="monitoring", + labels={"component": "alertmanager"}, + alert_category="alert_chain_health", + ) + + assert route["resolution_status"] == "resolved" + assert route["target_kind"] == "docker_container" + assert route["canonical_asset_id"] == "container:host_110:alertmanager" + assert route["host"] == "192.168.0.110" + assert route["executor"] == "host_ansible_executor" + assert route["verifier"] == "alert_chain_independent_delivery_verifier" + assert route["risk_class"] == "medium" + assert route["controlled_apply_allowed"] is True + assert route["cross_domain_fallback_allowed"] is False + assert route["allowed_catalog_ids"] == [ + "ansible:110-alertmanager-delivery-recovery" + ] + assert route["allowed_inventory_hosts"] == ["host_110"] + + catalog = get_ansible_catalog_item( + "ansible:110-alertmanager-delivery-recovery" + ) + assert catalog is not None + assert catalog["auto_apply_enabled"] is True + assert catalog["inventory_hosts"] == ["host_110"] + assert postconditions_for_catalog(catalog["catalog_id"]) + + +def test_other_alert_chain_domains_do_not_fall_into_alertmanager_container() -> None: + route = resolve_typed_alert_target( + alertname="AlertChainBroken_Sentry", + target_resource="unregistered-sentry-webhook", + namespace="monitoring", + labels={"component": "sentry"}, + alert_category="alert_chain_health", + ) + + assert route["canonical_asset_id"] != "container:host_110:alertmanager" + assert route["cross_domain_fallback_allowed"] is False + + def _sentry_runtime_incident() -> SimpleNamespace: return SimpleNamespace( incident_id="INC-20260714-471307", @@ -325,11 +370,152 @@ def test_unknown_and_kubernetes_assets_never_enter_ansible_fallback() -> None: assert unknown["decision_effect"] == "create_asset_drift_work_item" assert unknown["candidates"] == [] assert unknown["drift_work_item_id"].startswith("AIA-ASSET-DRIFT-") - assert kubernetes["decision_effect"] == "delegate_to_domain_executor" + assert kubernetes["decision_effect"] == "create_asset_drift_work_item" assert kubernetes["candidates"] == [] - assert kubernetes["typed_target_route"]["executor"] == ( - "kubernetes_controlled_executor" + assert kubernetes["typed_target_route"]["resolution_status"] == ( + "asset_identity_unresolved" ) + assert kubernetes["typed_target_route"]["executor"] is None + assert kubernetes["typed_target_route"]["controlled_apply_allowed"] is False + + +def test_kubernetes_registry_identity_requires_exact_namespace_kind_and_name() -> None: + route = resolve_typed_alert_target( + alertname="DeploymentUnavailable", + target_resource="awoooi-api", + namespace="awoooi-prod", + labels={ + "namespace": "awoooi-prod", + "workload_kind": "deployment", + "workload_name": "awoooi-api", + }, + alert_category="kubernetes", + registry=ServiceRegistryClient(REGISTRY), + ) + + assert route["resolution_status"] == "resolved" + assert route["target_kind"] == "kubernetes_workload" + assert route["canonical_asset_id"] == "service:awoooi-api" + assert route["executor"] == "kubernetes_controlled_executor" + assert route["verifier"] == "kubernetes_rollout_verifier" + assert route["controlled_apply_allowed"] is True + assert route["identity_evidence"] == { + "schema_version": "kubernetes_asset_identity_evidence_v1", + "status": "verified", + "method": "canonical_registry_namespace_kind_name_exact", + "namespace": "awoooi-prod", + "kind": "deployment", + "name": "awoooi-api", + "registry_identity_exact": True, + "runtime_uid_verified": False, + "required": [ + "canonical_registry_identity_plus_exact_namespace_kind_name", + "or_kubernetes_api_runtime_uid_readback", + ], + } + + +def test_unregistered_kubernetes_target_requires_verified_runtime_uid_evidence() -> None: + labels = { + "namespace": "tenant-a", + "workload_kind": "deployment", + "workload_name": "runtime-only-api", + } + route = resolve_typed_alert_target( + alertname="DeploymentUnavailable", + target_resource="runtime-only-api", + namespace="tenant-a", + labels=labels, + alert_category="kubernetes", + registry=ServiceRegistryClient(REGISTRY), + runtime_identity_evidence={ + "schema_version": "kubernetes_runtime_identity_receipt_v1", + "receipt_id": "k8s-runtime:tenant-a:deployment:runtime-only-api", + "trace_id": "trace-k8s-runtime", + "run_id": "run-k8s-runtime", + "work_item_id": "AIA-K8S-RUNTIME-IDENTITY", + "source": "kubernetes_api_uid_readback", + "status": "verified", + "verified_by": "kubernetes_runtime_identity_verifier", + "durable_readback_ack": True, + "runtime_uid": "12345678-1234-4abc-8def-1234567890ab", + "namespace": "tenant-a", + "kind": "deployment", + "name": "runtime-only-api", + }, + ) + + assert route["resolution_status"] == "resolved" + assert route["target_kind"] == "kubernetes_workload" + assert route["canonical_asset_id"] == ( + "k8s:tenant-a:deployment:runtime-only-api" + ) + assert route["identity_evidence"]["runtime_uid_verified"] is True + assert route["identity_evidence"]["runtime_identity_source"] == ( + "kubernetes_api_uid_readback" + ) + assert len(route["identity_evidence"]["runtime_uid_sha256"]) == 64 + assert "runtime_uid" not in route["identity_evidence"] + + +def test_kubernetes_runtime_uid_claim_with_mismatched_name_fails_closed() -> None: + route = resolve_typed_alert_target( + alertname="DeploymentUnavailable", + target_resource="runtime-only-api", + namespace="tenant-a", + labels={ + "namespace": "tenant-a", + "workload_kind": "deployment", + "workload_name": "runtime-only-api", + }, + alert_category="kubernetes", + registry=ServiceRegistryClient(REGISTRY), + runtime_identity_evidence={ + "schema_version": "kubernetes_runtime_identity_receipt_v1", + "receipt_id": "k8s-runtime:tenant-a:deployment:other-api", + "trace_id": "trace-k8s-runtime", + "run_id": "run-k8s-runtime", + "work_item_id": "AIA-K8S-RUNTIME-IDENTITY", + "source": "kubernetes_api_uid_readback", + "status": "verified", + "verified_by": "kubernetes_runtime_identity_verifier", + "durable_readback_ack": True, + "runtime_uid": "12345678-1234-4abc-8def-1234567890ab", + "namespace": "tenant-a", + "kind": "deployment", + "name": "other-api", + }, + ) + + assert route["resolution_status"] == "asset_identity_unresolved" + assert route["target_kind"] == "unknown" + assert route["executor"] is None + assert route["controlled_apply_allowed"] is False + assert route["identity_evidence"]["runtime_uid_verified"] is False + assert route["drift_work_item_id"].startswith("AIA-ASSET-DRIFT-") + + +def test_self_asserted_runtime_uid_labels_are_not_identity_evidence() -> None: + route = resolve_typed_alert_target( + alertname="DeploymentUnavailable", + target_resource="runtime-only-api", + namespace="tenant-a", + labels={ + "namespace": "tenant-a", + "workload_kind": "deployment", + "workload_name": "runtime-only-api", + "runtime_uid": "12345678-1234-4abc-8def-1234567890ab", + "runtime_identity_source": "kubernetes_api_uid_readback", + "runtime_identity_status": "verified", + }, + alert_category="kubernetes", + registry=ServiceRegistryClient(REGISTRY), + ) + + assert route["resolution_status"] == "asset_identity_unresolved" + assert route["executor"] is None + assert route["controlled_apply_allowed"] is False + assert route["identity_evidence"]["runtime_uid_verified"] is False def test_backup_restore_is_critical_readback_only() -> None: @@ -349,6 +535,81 @@ def test_backup_restore_is_critical_readback_only() -> None: assert route["agent99_bridge"]["dispatch_allowed"] is False +def test_registry_auto_flag_cannot_override_critical_backup_break_glass() -> None: + registry = SimpleNamespace( + resolve_identity=lambda _target: { + "resolution_status": "resolved", + "canonical_id": "data-protection:vault-service", + "service_name": "vault-service", + "asset_domain": "backup_restore", + "host": "192.168.0.110", + "stateful_level": "AUTO", + "executor": "backup_restore_break_glass", + "verifier": "backup_restore_readback_verifier", + "allowed_catalog_ids": [], + "controlled_apply_allowed": True, + } + ) + + route = resolve_typed_alert_target( + alertname="DataProtectionSignal", + target_resource="vault-service", + namespace="", + labels={}, + alert_category="data-protection", + registry=registry, + ) + + assert route["resolution_status"] == "resolved" + assert route["target_kind"] == "backup_restore" + assert route["risk_class"] == "critical" + assert route["critical_break_glass_required"] is True + assert route["controlled_apply_allowed"] is False + assert route["cross_domain_fallback_allowed"] is False + + +def test_unknown_backup_asset_fails_closed_and_creates_drift_work_item() -> None: + route = resolve_typed_alert_target( + alertname="BackupRestoreEscrowMissing", + target_resource="unregistered-offsite-vault", + namespace="", + labels={}, + registry=ServiceRegistryClient(REGISTRY), + ) + + assert route["resolution_status"] == "asset_identity_unresolved" + assert route["target_kind"] == "unknown" + assert route["executor"] is None + assert route["verifier"] is None + assert route["controlled_apply_allowed"] is False + assert route["critical_break_glass_required"] is True + assert route["cross_domain_fallback_allowed"] is False + assert route["drift_work_item_id"].startswith("AIA-ASSET-DRIFT-") + + +def test_blocked_database_keeps_bounded_executor_but_denies_target_level_apply() -> None: + registry = ServiceRegistryClient(REGISTRY) + identity = registry.resolve_identity("postgres") + route = resolve_typed_alert_target( + alertname="PostgreSQLConnectionsHigh", + target_resource="postgres", + namespace="awoooi-prod", + labels={"service": "postgres"}, + alert_category="database", + registry=registry, + ) + + assert identity["stateful_level"] == "BLOCK" + assert identity["controlled_apply_allowed"] is False + assert route["resolution_status"] == "resolved" + assert route["target_kind"] == "database" + assert route["executor"] == "db_bounded_executor" + assert route["verifier"] == "db_independent_verifier" + assert route["stateful_level"] == "BLOCK" + assert route["controlled_apply_allowed"] is False + assert route["cross_domain_fallback_allowed"] is False + + def test_agent99_payload_carries_same_typed_host_operations_contract() -> None: payload = build_agent99_sre_alert( alert_id="INC-COLD-START-99", @@ -517,7 +778,10 @@ async def test_agent99_rejects_cross_domain_mutating_dispatch_before_transport( assert result["status"] == "failed" assert result["reason"] == "agent99_typed_dispatch_not_allowed" assert result["dispatchPerformed"] is False - assert result["typedTargetRoute"]["target_kind"] == "kubernetes_workload" + assert result["typedTargetRoute"]["target_kind"] == "unknown" + assert result["typedTargetRoute"]["resolution_status"] == ( + "asset_identity_unresolved" + ) assert dispatched == [] diff --git a/apps/api/tests/test_telegram_delivery_truth_callers_v2.py b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py index 18904e837..7c42583e4 100644 --- a/apps/api/tests/test_telegram_delivery_truth_callers_v2.py +++ b/apps/api/tests/test_telegram_delivery_truth_callers_v2.py @@ -156,18 +156,27 @@ async def test_failover_commits_dedup_and_sent_log_only_after_provider_ack( ) alerter = failover_module.FailoverAlerter(redis_client=redis) - await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) + event = { + "from_provider": "ollama_gcp_a", + "to_provider": "ollama_gcp_b", + "primary_health": "offline", + "candidate_health": "healthy", + } + await alerter.alert_failover(event) - dedup_key = "alert:failover:ollama_gcp_b:dedup" + dedup_key = ( + "alert:ollama_route_degraded:ollama_gcp_a:ollama_gcp_b:" + "degraded_health_observation:offline:healthy:dedup" + ) assert dedup_key not in redis.values - assert ("info", "failover_alert_sent") not in logger.events - assert ("warning", "failover_alert_no_send") in logger.events + assert ("info", "ollama_route_degraded_alert_sent") not in logger.events + assert ("warning", "ollama_route_degraded_alert_no_send") in logger.events gateway.receipt = SENT_RECEIPT - await alerter.alert_failover({"to_provider": "ollama_gcp_b"}) + await alerter.alert_failover(event) assert redis.values[dedup_key] == "sent" - assert ("info", "failover_alert_sent") in logger.events + assert ("info", "ollama_route_degraded_alert_sent") in logger.events @pytest.mark.asyncio diff --git a/docs/operations/portfolio-infrastructure-asset-reconciliation-handoff.md b/docs/operations/portfolio-infrastructure-asset-reconciliation-handoff.md new file mode 100644 index 000000000..86788cf6a --- /dev/null +++ b/docs/operations/portfolio-infrastructure-asset-reconciliation-handoff.md @@ -0,0 +1,20 @@ +# Portfolio Infrastructure Reconciliation Handoff +1. Current P0 remains `AIA-SRE-013`; the safety queue remains `017 -> 002 -> 004 -> 013 -> 014 -> 015` and this inventory creates no competing P0. +2. Active inventory is 284 lines, SHA `c21416023775cc7fa96952ea3f06be389246269d11130902ae039aa0bd463df4`; all 131 non-summary rows map once. +3. Canonical assets are 144, imported omissions remain 18, conflicts are 11 across 36 assets; runtime closure remains `0`. +4. Host110 Ollama is removed and retained only as a canonical tombstone; runtime/monitoring absence is not yet verified. +5. Host111 is the third provider hop and runs as a macOS LaunchAgent, with local plus host120/host121 independent verification still pending. +6. Host111 also lacks a fresh canonical production sensor/rule series; LaunchAgent source readiness is not runtime closure. +7. GCP-A/B are public-HTTP sanitized candidates only; unsanitized payloads and every cloud tool loop must fail closed before network access. +8. Exact GCP-A/B Prometheus targets are missing in production; source/config targets do not count as runtime coverage. +9. Claude/Gemini credential material was user-supplied, but raw values are not recorded/read back; protected-secret metadata verification is pending. +10. Claude/Gemini paid canary needs durable Gate5 run authorization, daily/monthly/per-incident caps and one paired enable/rollback receipt. +11. Any partial five-lane result rolls back both paid providers; the sanitized five-lane production canary has not run. +12. AlertChainBroken source now uses host99 Agent99 exact-host read-only polling plus a reduced HTTPS relay, so it is independent of the primary webhook and sends no first-hop credential; atomic deployment, freshness/dedupe, current alert resolution and same-run repair/learning receipts remain P0 runtime work. +13. The gitea-native source candidate has no exact production target while the legacy GitHub exporter remains visible runtime drift. +14. GitHub stays frozen; legacy exporter/ghcr/runner claims cannot be normalized into healthy Gitea coverage. +15. Product alerts remain blocked until exact product-only Telegram routes exist; every card names AI/provider/action/executor/verifier/receipt. +16. Backup/restore remains read-only critical break-glass; every stateful asset needs freshness/offsite/escrow/drill evidence. +17. Source/test, Gitea CD/deploy marker, production runtime, visible UI/Telegram and learning writeback are separate evidence layers. +18. No lower evidence layer may mark a higher layer complete; no work item is runtime-closed for this program SHA. +19. Next: complete focused contracts, deploy one SHA, then verify tombstones/targets/second ingress/paid gates with same-run receipts. diff --git a/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json b/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json new file mode 100644 index 000000000..55a6a0354 --- /dev/null +++ b/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json @@ -0,0 +1,614 @@ +{ + "schema_version": "portfolio_infrastructure_asset_reconciliation_v1", + "governance_version": "global_product_governance_v2", + "program_id": "AIA-SRE-P0-20260715", + "generated_at": "2026-07-16T15:18:00+08:00", + "status": "all_imported_rows_mapped_source_conflicts_and_omissions_recorded_runtime_unverified", + "scope_complete": false, + "source_inventory": { + "active_version_id": "inventory-c214160-20260716", + "label": "WOOO 基礎設施服務工具盤點表（完整版）", + "source_path": "/Users/ogt/.gemini/antigravity-ide/brain/4e1bfd7f-cc1d-4f4f-94b0-df62ecb40386/infrastructure_inventory.md", + "sha256": "c21416023775cc7fa96952ea3f06be389246269d11130902ae039aa0bd463df4", + "declared_updated_date": "2026-07-16", + "source_line_count": 284, + "inventory_item_row_count": 131, + "source_category_counts": { + "hosts": 5, + "databases": 15, + "cache_queue": 8, + "monitoring_observability": 18, + "error_tracking": 3, + "ai_llm": 5, + "workflow_automation": 1, + "container_registry_cicd": 9, + "storage_backup": 2, + "gateway_proxy": 7, + "host112_security": 23, + "applications": 24, + "k3s_components": 11 + }, + "live_truth_accepted": false, + "runtime_probe_performed": false, + "interpretation": "The imported docker-ps and health labels are historical evidence only. Every asset remains runtime-unverified until an exact production receipt is correlated to this source revision.", + "version_history": [ + { + "version_id": "inventory-2e8580c-20260716", + "sha256": "2e8580c6465fe072990bc3736625c1a81614201d6cb7acadcc27338f96acaf22", + "source_line_count": 241, + "inventory_item_row_count": 112, + "disposition": "superseded_by_inventory-c214160-20260716_preserved_as_reconciliation_history" + } + ] + }, + "governance": { + "pipeline": [ + "Alert", + "Canonical Asset Normalize", + "Typed Domain Router", + "HolmesGPT Investigator", + "Ollama RCA / Gemini Critic", + "Deterministic Policy", + "Single Controlled Executor", + "Independent Verifier", + "Incident Closure + KM/RAG/MCP/PlayBook" + ], + "provider_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local_host111", "claude", "gemini"], + "github_frozen": true, + "cross_domain_fallback_allowed": false, + "unknown_asset_terminal": "asset_identity_unresolved", + "snapshot_health_is_live_truth": false, + "critical_backup_restore_default": "read_only_investigation_break_glass_for_write", + "alert_attribution_required": [ + "decision_engine", + "ai_models_used", + "agent_actions", + "executor", + "independent_verifier", + "same_run_receipt" + ] + }, + "asset_groups": [ + { + "group_id": "imported-hosts", + "category": "host", + "common": { + "product_id": "shared-infrastructure", + "project_id": "wooo-platform", + "site_id": "on-prem-lan", + "owner_lane": "infrastructure_ops", + "source_truth_state": "source_registry_partial", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "host_systemd", + "executor": "host_ansible_executor", + "verifier": "host_runtime_independent_verifier", + "monitoring": {"signals": ["node_reachability", "cpu", "memory", "disk", "service_freshness"], "coverage": "partial"}, + "alerting": {"lifecycle": "canonical_incident_only", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "readback_first", "target": "host_config_and_service_specific_contracts", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P0", + "dependencies": ["PORT-001"], + "next_action": "Run exact-host read-only identity and service probes, then bind each result to a durable asset receipt." + }, + "members": [ + {"canonical_id": "host:110", "inventory_labels": ["DevOps", "192.168.0.110"], "source_rows": ["L012"], "runtime_identity": "192.168.0.110", "reconciliation_state": "matched_with_source_drift", "findings": ["Inventory claims healthy containers but no same-run receipt exists", "Host110 Ollama has been removed; every former endpoint/proxy identity must resolve only to a retired tombstone", "Source tombstone and monitoring cleanup are not production proof of endpoint absence", "Legacy GitHub exporter runtime remains visible while the exact gitea-native target is missing"], "source_refs": ["infra/ansible/inventory/hosts.yml", "infra/ansible/inventory/group_vars/host_110.yml", "ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml"], "next_action": "Verify the Host110 Ollama tombstone against runtime and monitoring, verify gitea-native target freshness, and retire only the exact legacy GitHub exporter under bounded apply."}, + {"canonical_id": "host:188", "inventory_labels": ["AI/Data", "192.168.0.188"], "source_rows": ["L013"], "runtime_identity": "192.168.0.188", "reconciliation_state": "matched_with_role_conflict", "findings": ["Inventory says AI/Data without GPU", "Imported local Ollama identity conflicts with the approved host111 provider route", "Disk 64 percent is an unverified point-in-time claim"], "source_refs": ["infra/ansible/inventory/hosts.yml", "infra/ansible/inventory/group_vars/host_188.yml", "ops/config/service-registry.yaml"]}, + {"canonical_id": "host:120", "inventory_labels": ["K3s Master", "K3s Master (192.168.0.120)"], "source_rows": ["L014", "L249"], "runtime_identity": "192.168.0.120", "reconciliation_state": "matched_runtime_unverified", "findings": ["Control-plane role is source-visible; health is not live-verified"], "source_refs": ["infra/ansible/inventory/hosts.yml", "ops/monitoring/service-registry.yaml"], "domain_router": "control_plane_recovery", "executor": "Agent99", "verifier": "cold_start_independent_scorecard_verifier"}, + {"canonical_id": "host:121", "inventory_labels": ["K3s Backup", "K3s Backup (192.168.0.121)"], "source_rows": ["L015", "L250"], "runtime_identity": "192.168.0.121", "reconciliation_state": "role_conflict", "findings": ["Imported inventory calls this a backup/control-plane node while monitoring registry calls it k3s-worker"], "source_refs": ["infra/ansible/inventory/hosts.yml", "ops/monitoring/service-registry.yaml"], "domain_router": "control_plane_recovery", "executor": "Agent99", "verifier": "cold_start_independent_scorecard_verifier", "next_action": "Resolve exact K3s node roles from live kubectl node identity before any recovery action."}, + {"canonical_id": "host:112", "inventory_labels": ["Security", "192.168.0.112"], "source_rows": ["L016"], "runtime_identity": "192.168.0.112", "reconciliation_state": "matched_runtime_unverified", "findings": ["Wazuh/Kali source identity exists; imported healthy state is not a current receipt"], "source_refs": ["infra/ansible/inventory/hosts.yml", "ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml"], "owner_lane": "security_ops"} + ] + }, + { + "group_id": "imported-databases", + "category": "database", + "common": { + "product_id": "shared-infrastructure", + "project_id": "database-estate", + "site_id": "on-prem-lan", + "owner_lane": "database_ops", + "source_truth_state": "inventory_claim_and_partial_registry", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "database", + "executor": "db_bounded_executor", + "verifier": "db_independent_verifier", + "monitoring": {"signals": ["readiness", "connections", "replication", "capacity", "backup_freshness"], "coverage": "partial"}, + "alerting": {"lifecycle": "database_typed_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "critical_read_only_default", "target": "gap:no_exact_product_restore_contract", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P0", + "dependencies": ["PORT-001", "PORT-007"], + "next_action": "Resolve exact database/container/port identity, collect read-only health and backup freshness, then create bounded DB verifier work." + }, + "members": [ + {"canonical_id": "database:awoooi:prod", "inventory_labels": ["awoooi_prod"], "source_rows": ["L025"], "runtime_identity": "host188/postgresql/awoooi_prod", "reconciliation_state": "identity_scope_conflict", "findings": ["Inventory combines AWOOOI business database and K3s datastore into one purpose", "Canonical service registry names the container postgres, not awoooi_prod"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-awoooi.sh"], "product_id": "awoooi", "project_id": "awoooi", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-awoooi.sh", "freshness": "unverified"}}, + {"canonical_id": "database:momo-pro:postgres", "inventory_labels": ["momo-db"], "source_rows": ["L026"], "runtime_identity": "host188/docker/momo-db", "reconciliation_state": "matched_source_runtime_unverified", "findings": ["Exact exposed port and database name are not proven by the inventory row"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-momo.sh"], "product_id": "momo-pro", "project_id": "momo-pro-system", "owner_lane": "momo_product_ops", "telegram_destination": "blocked:momo_configured_chat_unresolved", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-momo.sh", "freshness": "unverified"}}, + {"canonical_id": "database:vibework:postgres", "inventory_labels": ["vibework-production-postgres-1"], "source_rows": ["L027"], "runtime_identity": "host188/docker/vibework-production-postgres-1", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["No exact AWOOOI service registry entry or committed restore contract found"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "vibework", "project_id": "vibework", "owner_lane": "vibework_product_ops", "telegram_destination": "blocked:vibework_route_not_implemented"}, + {"canonical_id": "database:2026fifa:timescaledb", "inventory_labels": ["current-fifa2026-postgres-1", "TimescaleDB current-fifa2026-postgres-1"], "source_rows": ["L028", "L043"], "runtime_identity": "host188/docker/current-fifa2026-postgres-1", "reconciliation_state": "duplicate_inventory_row_and_registry_gap", "findings": ["Same instance is counted once as PostgreSQL and again as TimescaleDB", "No exact restore drill is registered"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "database:k3s:postgres-recovery", "inventory_labels": ["k3s-postgres-recovery"], "source_rows": ["L029"], "runtime_identity": "host188/docker/k3s-postgres-recovery", "reconciliation_state": "recovery_asset_without_restore_receipt", "findings": ["Running recovery container is not proof of current K3s datastore recovery readiness"], "source_refs": ["docs/security/backup-restore-escrow-inventory.snapshot.json"], "product_id": "shared-infrastructure", "project_id": "k3s-control-plane"}, + {"canonical_id": "database:test:pgvector-b5-8801", "inventory_labels": ["pg-test-b5-8801-1"], "source_rows": ["L030"], "runtime_identity": "host188/docker/pg-test-b5-8801-1", "reconciliation_state": "stale_exited_candidate", "findings": ["Imported state says Exited", "No deletion is authorized; ownership and retention must be resolved first"], "source_refs": ["ops/config/service-registry.yaml"], "product_id": "unknown", "project_id": "unknown", "owner_lane": "asset_identity", "domain_router": "unknown", "executor": null, "verifier": "asset_identity_drift_verifier", "next_action": "Create deterministic drift work item; do not prune or restart until owner and data classification are proven."}, + {"canonical_id": "database:harbor:postgres", "inventory_labels": ["harbor-db"], "source_rows": ["L031"], "runtime_identity": "host110/docker/harbor-db", "reconciliation_state": "matched_runtime_unverified", "findings": ["Stateful BLOCK policy exists; health and backup freshness remain unverified"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-harbor.sh"], "project_id": "harbor", "owner_lane": "devops_team", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-harbor.sh", "freshness": "unverified"}}, + {"canonical_id": "database:langfuse:postgres", "inventory_labels": ["langfuse-db"], "source_rows": ["L032"], "runtime_identity": "host110/docker/langfuse-db", "reconciliation_state": "matched_runtime_unverified", "findings": ["Stateful BLOCK policy exists; LLM trace privacy and restore evidence remain unverified"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-langfuse.sh"], "project_id": "langfuse", "owner_lane": "ai_team", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-langfuse.sh", "freshness": "unverified"}}, + {"canonical_id": "database:agent-bounty:postgres", "inventory_labels": ["agent_bounty_db"], "source_rows": ["L033"], "runtime_identity": "host110/docker/agent_bounty_db", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["No exact backup/restore verifier is registered"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "agent-bounty-protocol", "project_id": "agent-bounty-protocol", "owner_lane": "agent_bounty_product_ops", "telegram_destination": "blocked:vibe_ai_agent_receipt_gap"}, + {"canonical_id": "database:stockplatform-v2:postgres", "inventory_labels": ["stockplatform-v2-postgres-1"], "source_rows": ["L034"], "runtime_identity": "host110/docker/stockplatform-v2-postgres-1", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Product destination ownership and restore contract are unresolved"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "stockplatform-v2", "project_id": "stockPlatform", "owner_lane": "stockplatform_product_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"}, + {"canonical_id": "database:awooogo:postgres", "inventory_labels": ["awooo-go-db-1"], "source_rows": ["L035"], "runtime_identity": "host110/docker/awooo-go-db-1", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Product Telegram egress is disabled and restore evidence is absent"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "awooogo", "project_id": "AwoooGo", "owner_lane": "awooogo_product_ops", "telegram_destination": "disabled:awooogo"}, + {"canonical_id": "database:bitan-pharmacy:restore", "inventory_labels": ["bitan-pg-restore"], "source_rows": ["L036"], "runtime_identity": "host110/docker/bitan-pg-restore", "reconciliation_state": "restore_marker_without_drill_receipt", "findings": ["Container name implies restore but no isolated drill receipt is linked"], "source_refs": ["docs/security/backup-restore-escrow-inventory.snapshot.json"], "product_id": "bitan-pharmacy", "project_id": "bitan-pharmacy", "owner_lane": "bitan_product_ops", "telegram_destination": "blocked:bitan_destination_unassigned"}, + {"canonical_id": "database:sentry:postgres", "inventory_labels": ["Sentry PG (內建)"], "source_rows": ["L037"], "runtime_identity": "host110/sentry-self-hosted/postgres", "reconciliation_state": "matched_runtime_unverified", "findings": ["Exact container identity is not present in the imported row"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-sentry.sh"], "project_id": "sentry", "owner_lane": "devops_team", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-sentry.sh", "freshness": "unverified"}}, + {"canonical_id": "database:signoz:clickhouse", "inventory_labels": ["signoz-clickhouse"], "source_rows": ["L042"], "runtime_identity": "unresolved:host110_or_host188/signoz-clickhouse", "reconciliation_state": "host_identity_conflict", "findings": ["Imported inventory places ClickHouse on host110 while monitoring registry places clickhouse on host188"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "scripts/backup/clickhouse-native-backup.sh"], "project_id": "signoz", "owner_lane": "observability_ops", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/clickhouse-native-restore-drill.sh", "freshness": "unverified"}, "next_action": "Resolve exact host/container identity before any DB executor or restore drill."} + ] + }, + { + "group_id": "imported-cache-queue", + "category": "cache_queue", + "common": { + "product_id": "shared-infrastructure", + "project_id": "cache-queue-estate", + "site_id": "on-prem-lan", + "owner_lane": "database_ops", + "source_truth_state": "inventory_claim_and_partial_registry", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "database", + "executor": "db_bounded_executor", + "verifier": "db_independent_verifier", + "monitoring": {"signals": ["readiness", "memory", "connections", "queue_depth", "persistence_freshness"], "coverage": "partial"}, + "alerting": {"lifecycle": "stateful_cache_queue_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "critical_read_only_default", "target": "gap:no_exact_persistence_contract", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P1", + "dependencies": ["PORT-001", "PORT-007"], + "next_action": "Verify exact instance identity and persistence mode before generating any bounded action." + }, + "members": [ + {"canonical_id": "cache:awoooi:redis", "inventory_labels": ["redis-server (systemd)"], "source_rows": ["L052"], "runtime_identity": "host188/systemd/redis-server", "reconciliation_state": "service_type_conflict", "findings": ["Imported row says systemd while monitoring registry models redis as Docker on port 6380"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml"], "product_id": "awoooi", "project_id": "awoooi", "priority": "P0"}, + {"canonical_id": "cache:clawbot:redis", "inventory_labels": ["clawbot-redis"], "source_rows": ["L053"], "runtime_identity": "host188/docker/clawbot-redis", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Persistence and backup contract are not registered"], "source_refs": ["scripts/backup/backup-clawbot.sh"], "product_id": "clawbot-openclaw", "project_id": "clawbot", "owner_lane": "ai_agent_runtime", "telegram_destination": "blocked:clawbot_cross_product_route"}, + {"canonical_id": "cache:2026fifa:redis", "inventory_labels": ["current-fifa2026-redis-1"], "source_rows": ["L054"], "runtime_identity": "host188/docker/current-fifa2026-redis-1", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Product cache has no canonical persistence/restore contract"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "cache:harbor:redis", "inventory_labels": ["harbor-redis"], "source_rows": ["L055"], "runtime_identity": "host110/docker/harbor-redis", "reconciliation_state": "matched_runtime_unverified", "findings": ["Critical HITL stateful policy exists; runtime health remains unverified"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-harbor.sh"], "project_id": "harbor", "owner_lane": "devops_team"}, + {"canonical_id": "cache:agent-bounty:redis", "inventory_labels": ["agent_bounty_redis"], "source_rows": ["L056"], "runtime_identity": "host110/docker/agent_bounty_redis", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Product route receipt and persistence contract are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "agent-bounty-protocol", "project_id": "agent-bounty-protocol", "owner_lane": "agent_bounty_product_ops", "telegram_destination": "blocked:vibe_ai_agent_receipt_gap"}, + {"canonical_id": "cache:stockplatform-v2:redis", "inventory_labels": ["stockplatform-v2-redis-1"], "source_rows": ["L057"], "runtime_identity": "host110/docker/stockplatform-v2-redis-1", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Product ownership and persistence contract are unresolved"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "stockplatform-v2", "project_id": "stockPlatform", "owner_lane": "stockplatform_product_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"}, + {"canonical_id": "cache:sentry:redis", "inventory_labels": ["Sentry Redis (內建)"], "source_rows": ["L058"], "runtime_identity": "host110/sentry-self-hosted/redis", "reconciliation_state": "matched_runtime_unverified", "findings": ["Exact container and queue health receipt are absent"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-sentry.sh"], "project_id": "sentry", "owner_lane": "devops_team"}, + {"canonical_id": "queue:signoz:zookeeper", "inventory_labels": ["signoz-zookeeper-1"], "source_rows": ["L063"], "runtime_identity": "host110/docker/signoz-zookeeper-1", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Host placement and dependent ClickHouse/Kafka topology are not reconciled"], "source_refs": ["ops/monitoring/service-registry.yaml", "scripts/backup/backup-signoz.sh"], "project_id": "signoz", "owner_lane": "observability_ops"} + ] + }, + { + "group_id": "imported-observability", + "category": "monitoring_observability", + "common": { + "product_id": "shared-infrastructure", + "project_id": "observability-platform", + "site_id": "on-prem-and-k3s", + "owner_lane": "observability_ops", + "source_truth_state": "monitoring_registry_and_inventory_conflict", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "docker_container", + "executor": "host_ansible_executor_with_exact_container_playbook", + "verifier": "container_runtime_independent_verifier", + "monitoring": {"signals": ["self_health", "pipeline_freshness", "target_coverage", "delivery_receipts"], "coverage": "partial"}, + "alerting": {"lifecycle": "observability_platform_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-monitoring.sh", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P1", + "dependencies": ["PORT-001", "PORT-002", "PORT-010"], + "next_action": "Resolve exact deployment identity, verify generated target parity, and attach lifecycle receipts before claiming observability coverage." + }, + "members": [ + {"canonical_id": "service:prometheus", "inventory_labels": ["Prometheus"], "source_rows": ["L071"], "runtime_identity": "host110/docker/prometheus", "source_truth_state": "source_reconciled_runtime_pending", "reconciliation_state": "source_identity_reconciled_runtime_pending", "findings": ["Monitoring registry now declares the exact host110 Docker identity", "Production container, generated target parity and scrape freshness remain unverified"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "k8s/monitoring/prometheus.yml", "scripts/ops/deploy-alertmanager-runtime-scrape.sh"], "priority": "P0", "verifier": "prometheus_runtime_target_and_rule_verifier"}, + {"canonical_id": "service:grafana", "inventory_labels": ["Grafana"], "source_rows": ["L072"], "runtime_identity": "host110/docker/grafana", "reconciliation_state": "source_partial_runtime_unverified", "findings": ["Source policy exists but live dashboard/runtime parity is not proven"], "source_refs": ["ops/config/service-registry.yaml", "docs/security/monitoring-alerting-observability-inventory.snapshot.json"]}, + {"canonical_id": "service:alertmanager", "inventory_labels": ["Alertmanager"], "source_rows": ["L073"], "runtime_identity": "host110/docker/alertmanager", "source_truth_state": "source_reconciled_runtime_pending", "reconciliation_state": "source_control_path_reconciled_runtime_pending", "findings": ["Monitoring registry and typed router now declare the exact host110 Docker identity", "Alert rule now reads Alertmanager monotonic delivery counters instead of process-local API counters", "Exact bounded Ansible recovery and independent verifier are source-ready; production same-run receipt is pending"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "k8s/monitoring/alert-chain-monitor.yaml", "ops/alertmanager/alertmanager.yml", "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml", "scripts/ops/deploy-alertmanager-runtime-scrape.sh"], "priority": "P0", "verifier": "alertmanager_delivery_chain_independent_verifier", "next_action": "Deploy the canonical scrape/rule and exact playbook, then correlate receiver, API ingest, Telegram receipt, retry/rollback and KM writeback under one run."}, + {"canonical_id": "service:signoz", "inventory_labels": ["SignOz"], "source_rows": ["L074"], "runtime_identity": "unresolved:host110-ui-host188-collector", "reconciliation_state": "split_plane_identity_conflict", "findings": ["Inventory collapses UI, collector, and storage into one host110 service while source registries split them across hosts"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "scripts/backup/backup-signoz.sh"]}, + {"canonical_id": "service:otel-collector:signoz", "inventory_labels": ["OTel Collector"], "source_rows": ["L075"], "runtime_identity": "unresolved:host110-or-host188/otel-collector", "reconciliation_state": "host_identity_conflict", "findings": ["Inventory says host110 while monitoring registry says host188 port 24317"], "source_refs": ["ops/monitoring/service-registry.yaml", "docs/security/monitoring-alerting-observability-inventory.snapshot.json"], "domain_router": "unknown", "executor": null, "verifier": "asset_identity_drift_verifier"}, + {"canonical_id": "service:blackbox-exporter", "inventory_labels": ["Blackbox Exporter"], "source_rows": ["L076"], "runtime_identity": "host110/docker/blackbox-exporter", "reconciliation_state": "matched_runtime_unverified", "findings": ["Generated target freshness and live probe results are not current truth"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/generated/blackbox-targets-generated.yaml"]}, + {"canonical_id": "service:node-exporter:host110", "inventory_labels": ["Node Exporter host110"], "source_rows": ["L077"], "runtime_identity": "host110/docker/node-exporter", "reconciliation_state": "inventory_only_runtime_unverified", "findings": ["No exact canonical service entry links this exporter to host110"], "source_refs": ["ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "service:node-exporter:host188", "inventory_labels": ["Node Exporter host188"], "source_rows": ["L078"], "runtime_identity": "host188/docker/node-exporter:9100", "reconciliation_state": "inventory_only_runtime_unverified", "findings": ["No exact canonical service entry links this exporter to host188"], "source_refs": ["ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "service:cadvisor:host110", "inventory_labels": ["cAdvisor"], "source_rows": ["L079"], "runtime_identity": "host110/docker/cadvisor", "reconciliation_state": "inventory_only_runtime_unverified", "findings": ["Image/version claim and scrape freshness need source and runtime receipts"], "source_refs": ["docs/security/monitoring-alerting-observability-inventory.snapshot.json"]}, + {"canonical_id": "service:redis-exporter:host188", "inventory_labels": ["Redis Exporter"], "source_rows": ["L080"], "runtime_identity": "host188/docker/redis-exporter:9121", "reconciliation_state": "inventory_only_runtime_unverified", "findings": ["Exporter target must be reconciled with the conflicting Redis systemd/Docker identity"], "source_refs": ["ops/monitoring/generated/prometheus-scrape-generated.yaml"]}, + {"canonical_id": "service:postgres-exporter:host188", "inventory_labels": ["Postgres Exporter"], "source_rows": ["L081"], "runtime_identity": "host188/docker/postgres-exporter:9187", "reconciliation_state": "inventory_only_runtime_unverified", "findings": ["One exporter does not prove coverage for every product database"], "source_refs": ["docs/security/monitoring-alerting-observability-inventory.snapshot.json"]}, + {"canonical_id": "service:nginx-exporter:host188", "inventory_labels": ["Nginx Exporter"], "source_rows": ["L082"], "runtime_identity": "host188/docker/nginx-exporter:9113", "reconciliation_state": "inventory_only_runtime_unverified", "findings": ["Target freshness and exact Nginx instance coverage are unverified"], "source_refs": ["ops/monitoring/generated/prometheus-scrape-generated.yaml"]}, + {"canonical_id": "service:gitea-exporter:host110", "inventory_labels": ["GitHub/Gitea Exporter"], "source_rows": ["L083"], "runtime_identity": "host110/docker/promhippie-github-exporter", "source_truth_state": "source_reconciled_runtime_pending", "reconciliation_state": "source_retirement_ready_runtime_pending", "findings": ["GitHub exporter and token references are removed from the source compose contract", "Prometheus contains a Gitea-native source candidate, but the exact production target is missing", "The legacy GitHub exporter remains visible in production and requires bounded removal plus independent readback", "Source retirement cannot be promoted to runtime closure until gitea-native freshness and legacy absence are both verified"], "source_refs": ["k8s/monitoring/docker-compose-110.yml", "k8s/monitoring/prometheus.yml", "scripts/generate_monitoring.py"], "priority": "P0", "monitoring": {"signals": ["gitea_native_target_up", "gitea_native_scrape_freshness", "legacy_github_exporter_absence"], "coverage": "source_candidate_runtime_target_missing"}, "next_action": "First make the exact gitea-native target observable; then use a bounded host110 compose apply to retire only the legacy exporter and independently verify target freshness plus absence of GitHub runtime traffic."}, + {"canonical_id": "k8s:kube-state-metrics", "inventory_labels": ["kube-state-metrics"], "source_rows": ["L084"], "runtime_identity": "k3s/kube-state-metrics/deployment", "reconciliation_state": "source_visible_runtime_unverified", "findings": ["K3s deployment is registered; live rollout and scrape freshness are unverified"], "source_refs": ["ops/monitoring/service-registry.yaml"], "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "k8s:node-problem-detector", "inventory_labels": ["Node Problem Detector"], "source_rows": ["L085"], "runtime_identity": "k3s/node-problem-detector", "reconciliation_state": "missing_from_monitoring_service_registry", "findings": ["Inventory claims running but no exact registry member is present"], "source_refs": ["k8s/monitoring"], "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "k8s:otel-daemonset", "inventory_labels": ["OTel DaemonSet"], "source_rows": ["L086"], "runtime_identity": "k3s/observability/otel-daemonset", "reconciliation_state": "source_partial_runtime_unverified", "findings": ["Node coverage and telemetry delivery freshness need independent verification"], "source_refs": ["k8s/monitoring"], "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "k8s:event-exporter", "inventory_labels": ["Event Exporter"], "source_rows": ["L087"], "runtime_identity": "k3s/observability/event-exporter", "reconciliation_state": "source_visible_runtime_unverified", "findings": ["Registry entry exists; event-to-metric freshness is unverified"], "source_refs": ["ops/monitoring/service-registry.yaml"], "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "service:langfuse", "inventory_labels": ["Langfuse"], "source_rows": ["L088"], "runtime_identity": "host110/docker/langfuse", "reconciliation_state": "matched_runtime_unverified", "findings": ["Trace ingest, privacy boundary, and backup freshness are unverified"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "scripts/backup/backup-langfuse.sh"], "owner_lane": "ai_team", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-langfuse.sh", "freshness": "unverified"}} + ] + }, + { + "group_id": "imported-error-tracking", + "category": "error_tracking", + "common": { + "product_id": "shared-infrastructure", + "project_id": "sentry-self-hosted", + "site_id": "host110", + "owner_lane": "observability_ops", + "source_truth_state": "partial_exact_container_registry", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "docker_container", + "executor": "host_ansible_executor_with_exact_container_playbook", + "verifier": "container_runtime_independent_verifier", + "monitoring": {"signals": ["container_health", "ingestion", "queue_lag", "error_event_freshness"], "coverage": "partial"}, + "alerting": {"lifecycle": "sentry_exact_container_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-sentry.sh", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P1", + "dependencies": ["PORT-001", "PORT-004"], + "next_action": "Bind exact host110 container aliases to bounded PlayBooks and require independent queue/ingest verification." + }, + "members": [ + {"canonical_id": "service:sentry:web", "inventory_labels": ["Sentry Self-Hosted"], "source_rows": ["L096"], "runtime_identity": "host110/sentry-self-hosted/web", "reconciliation_state": "matched_runtime_unverified", "findings": ["Exact-container recovery catalog exists but production runtime receipt is pending"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "service:sentry:relay", "inventory_labels": ["Sentry Relay"], "source_rows": ["L097"], "runtime_identity": "host110/sentry-self-hosted/relay", "reconciliation_state": "image_supply_chain_conflict", "findings": ["Imported image points to ghcr.io, which is forbidden by the GitHub freeze unless replaced by an internal immutable mirror"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json", "ops/config/service-registry.yaml"], "priority": "P0"}, + {"canonical_id": "service:sentry:snuba", "inventory_labels": ["Snuba"], "source_rows": ["L098"], "runtime_identity": "host110/sentry-self-hosted/snuba", "reconciliation_state": "image_supply_chain_and_exact_worker_gap", "findings": ["Imported image points to ghcr.io", "Only the profiling consumer has an exact recovery catalog; other workers need identity-only monitoring until registered"], "source_refs": ["ops/config/service-registry.yaml"], "priority": "P0"} + ] + }, + { + "group_id": "imported-ai-runtime", + "category": "ai_llm", + "common": { + "product_id": "awoooi", + "project_id": "ai-control-plane", + "site_id": "on-prem-lan", + "owner_lane": "ai_team", + "source_truth_state": "conflicts_with_approved_provider_route", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "docker_container", + "executor": "host_ansible_executor_with_exact_container_playbook", + "verifier": "container_runtime_independent_verifier", + "monitoring": {"signals": ["health", "latency", "tokens", "cost", "route_state", "receipt_freshness"], "coverage": "partial"}, + "alerting": {"lifecycle": "ai_provider_or_agent_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "readback_first", "target": "scripts/backup/backup-ai-artifacts.sh", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P0", + "dependencies": ["PORT-001", "PORT-003", "PORT-011"], + "next_action": "Reconcile provider and agent identity before route evaluation; never treat a proxy host as the AI provider." + }, + "members": [ + {"canonical_id": "service:ollama:host188", "inventory_labels": ["Ollama systemd host188"], "source_rows": ["L106"], "runtime_identity": "host188/systemd/ollama:11434", "reconciliation_state": "stale_provider_identity_conflict", "findings": ["Imported inventory identifies host188 Ollama, while the approved route uses host111 as the local hop", "No fallback is allowed to select another host/domain by generic target"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json"], "domain_router": "host_systemd", "executor": "host_ansible_executor", "verifier": "host111_ollama_route_identity_verifier", "next_action": "Create identity drift work; determine whether host188 Ollama is retired, non-routing, or a separately owned asset without changing the five-hop order."}, + {"canonical_id": "model:host188:nemotron-mini", "inventory_labels": ["nemotron-mini:latest"], "source_rows": ["L107"], "runtime_identity": "host188/ollama/model/nemotron-mini:latest", "reconciliation_state": "model_catalog_conflict", "findings": ["Imported model conflicts with monitoring registry local model list and approved provider identity", "Latest tag is mutable and not an immutable deployment identity"], "source_refs": ["ops/monitoring/service-registry.yaml"], "domain_router": "control_plane_service", "executor": "single_control_plane_executor", "verifier": "model_catalog_and_digest_verifier"}, + {"canonical_id": "service:open-webui", "inventory_labels": ["Open WebUI"], "source_rows": ["L108"], "runtime_identity": "host188/docker/open-webui:3010", "reconciliation_state": "source_partial_runtime_unverified", "findings": ["UI availability does not prove Ollama route correctness"], "source_refs": ["infra/ansible/inventory/group_vars/host_188.yml", "scripts/backup/backup-open-webui.sh"]}, + {"canonical_id": "service:openclaw", "inventory_labels": ["OpenClaw"], "source_rows": ["L109"], "runtime_identity": "host188/docker/openclaw:8089", "reconciliation_state": "port_and_service_identity_conflict", "findings": ["Ansible source maps openclaw to port 8088 while imported inventory says 8089", "No current durable agent decision receipt is implied by container health"], "source_refs": ["infra/ansible/inventory/group_vars/host_188.yml", "ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "service:clawbot:v5", "inventory_labels": ["ClawBot v5", "clawbot"], "source_rows": ["L110", "L232"], "runtime_identity": "host188/docker/clawbot:8088", "reconciliation_state": "duplicate_inventory_row_and_route_drift", "findings": ["Same service appears in AI and application tables", "Telegram route is blocked by cross-product bot ownership drift"], "source_refs": ["scripts/backup/backup-clawbot.sh", "docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "clawbot-openclaw", "project_id": "clawbot", "owner_lane": "ai_agent_runtime", "telegram_destination": "blocked:clawbot_cross_product_route"} + ] + }, + { + "group_id": "imported-workflow-automation", + "category": "workflow_automation", + "common": { + "product_id": "shared-infrastructure", + "project_id": "workflow-automation", + "site_id": "host188", + "owner_lane": "automation_ops", + "source_truth_state": "ansible_source_partial", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "docker_container", + "executor": "host_ansible_executor_with_exact_container_playbook", + "verifier": "container_runtime_independent_verifier", + "monitoring": {"signals": ["health", "workflow_failures", "queue_depth", "delivery_receipts"], "coverage": "gap"}, + "alerting": {"lifecycle": "workflow_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "critical_read_only_default", "target": "gap:no_n8n_restore_contract", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P1", + "dependencies": ["PORT-001", "PORT-011"], + "next_action": "Register n8n workflow inventory, owners, failure receipts, and exact-host restore verifier." + }, + "members": [ + {"canonical_id": "service:n8n:host188", "inventory_labels": ["n8n"], "source_rows": ["L118"], "runtime_identity": "host188/docker/n8n:5678", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Ansible directory exists but workflow inventory, backup, and failure routing are absent"], "source_refs": ["infra/ansible/inventory/group_vars/host_188.yml"]} + ] + }, + { + "group_id": "imported-cicd-registry", + "category": "container_registry_cicd", + "common": { + "product_id": "shared-infrastructure", + "project_id": "gitea-harbor-cd", + "site_id": "on-prem-and-k3s", + "owner_lane": "devops_team", + "source_truth_state": "source_partial_runtime_unverified", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "docker_container", + "executor": "host_ansible_executor_with_exact_container_playbook", + "verifier": "container_runtime_independent_verifier", + "monitoring": {"signals": ["health", "queue", "build", "deploy_marker", "artifact_digest", "runner_freshness"], "coverage": "partial"}, + "alerting": {"lifecycle": "gitea_cd_or_registry_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "critical_read_only_default", "target": "service_specific_backup_contract", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P0", + "dependencies": ["PORT-001", "PORT-004", "PORT-007"], + "next_action": "Verify Gitea-only source control, internal artifact provenance, exact runner ownership, and backup/restore freshness." + }, + "members": [ + {"canonical_id": "service:harbor", "inventory_labels": ["Harbor"], "source_rows": ["L126"], "runtime_identity": "host110/docker/harbor-core:5000", "reconciliation_state": "matched_runtime_unverified", "findings": ["Critical HITL policy exists; current health, immutable image, and backup receipts are unverified"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "scripts/backup/backup-harbor.sh"], "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-harbor.sh", "freshness": "unverified"}}, + {"canonical_id": "service:harbor:trivy", "inventory_labels": ["Trivy via Harbor"], "source_rows": ["L127"], "runtime_identity": "host110/docker/trivy-adapter", "reconciliation_state": "inventory_only_runtime_unverified", "findings": ["Scanner freshness and vulnerability database provenance are not linked"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"]}, + {"canonical_id": "service:docker-registry:backup", "inventory_labels": ["Docker Registry 備用"], "source_rows": ["L128"], "runtime_identity": "unresolved:host110-or-host188/docker-registry", "reconciliation_state": "host_identity_conflict", "findings": ["Imported row says host110 while host188 Ansible variables define docker_registry", "Backup registry promotion policy is not registered"], "source_refs": ["infra/ansible/inventory/group_vars/host_188.yml"], "domain_router": "unknown", "executor": null, "verifier": "asset_identity_drift_verifier"}, + {"canonical_id": "service:gitea", "inventory_labels": ["Gitea"], "source_rows": ["L129"], "runtime_identity": "host110/docker/gitea:3001", "reconciliation_state": "matched_runtime_unverified", "findings": ["Gitea is the sole source-control authority; runtime and backup freshness need exact receipts", "The exact gitea-native Prometheus target is missing in production, so source configuration cannot prove metrics coverage", "The legacy GitHub exporter must remain a separate drift until bounded removal and absence verification"], "source_refs": ["ops/config/service-registry.yaml", "product.awoooi.yaml", "scripts/backup/backup-gitea.sh", "k8s/monitoring/prometheus.yml"], "monitoring": {"signals": ["gitea_native_target_up", "gitea_native_scrape_freshness", "legacy_exporter_absence"], "coverage": "runtime_target_missing"}, "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/gitea-full-backup-restore-drill.sh", "freshness": "unverified"}, "next_action": "Restore and verify the exact gitea-native metrics target before retiring the legacy exporter; keep both checks separate from Gitea application health."}, + {"canonical_id": "runner:gitea:vibework-host110", "inventory_labels": ["Act Runner VibeWork 110"], "source_rows": ["L130"], "runtime_identity": "host110/gitea-act-runner/vibework", "reconciliation_state": "paused_capacity_and_legacy_registry_gap", "findings": ["Imported state says paused", "Monitoring registry still describes a GitHub Actions runner instead of this Gitea runner"], "source_refs": ["ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "runner:gitea:vibework-deployer-host188", "inventory_labels": ["Act Runner VibeWork deployer"], "source_rows": ["L131"], "runtime_identity": "host188/gitea-act-runner/vibework-deployer", "reconciliation_state": "missing_from_monitoring_registry", "findings": ["Current health and single-writer/CD ownership are unverified"], "source_refs": ["ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "runner:gitea:vibework-dedicated-host188", "inventory_labels": ["Act Runner VibeWork dedicated"], "source_rows": ["L132"], "runtime_identity": "host188/gitea-act-runner/vibework-dedicated", "reconciliation_state": "missing_from_monitoring_registry", "findings": ["Exact labels, capacity, and production deploy scope are unverified"], "source_refs": ["ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "runner:gitea:stockplatform-host188", "inventory_labels": ["Act Runner StockPlatform"], "source_rows": ["L133"], "runtime_identity": "host188/gitea-act-runner/stockplatform", "reconciliation_state": "missing_from_monitoring_registry", "findings": ["Exact labels, capacity, and product ownership are unverified"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "stockplatform-v2", "project_id": "stockPlatform", "owner_lane": "stockplatform_product_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"}, + {"canonical_id": "k8s:argocd", "inventory_labels": ["ArgoCD", "K3s ArgoCD"], "source_rows": ["L134", "L251"], "runtime_identity": "k3s/argocd", "reconciliation_state": "source_visible_runtime_unverified", "findings": ["Multiple ArgoCD components are registered; live sync and rollout truth remain unverified"], "source_refs": ["ops/monitoring/service-registry.yaml", "docs/security/k8s-argocd-manifest-inventory.snapshot.json"], "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"} + ] + }, + { + "group_id": "imported-storage-backup", + "category": "storage_backup", + "common": { + "product_id": "shared-infrastructure", + "project_id": "backup-dr", + "site_id": "on-prem-and-k3s", + "owner_lane": "dr_ops", + "source_truth_state": "source_policy_visible_runtime_unverified", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "backup_restore", + "executor": "backup_restore_break_glass", + "verifier": "backup_restore_readback_verifier", + "monitoring": {"signals": ["freshness", "integrity", "offsite", "escrow", "restore_drill"], "coverage": "partial"}, + "alerting": {"lifecycle": "dr_readback_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "critical_read_only_default", "target": "backup_status_and_isolated_restore_drill", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P0", + "dependencies": ["PORT-001", "PORT-007"], + "next_action": "Read backup status, offsite, escrow, and isolated restore receipts; do not run restore or write markers without break-glass." + }, + "members": [ + {"canonical_id": "storage:minio:host188", "inventory_labels": ["MinIO"], "source_rows": ["L142"], "runtime_identity": "host188/docker/minio:9000", "reconciliation_state": "matched_runtime_unverified", "findings": ["MinIO service identity exists; backup object freshness, offsite, and escrow remain unverified"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "docs/security/backup-restore-escrow-inventory.snapshot.json"]}, + {"canonical_id": "k8s:velero", "inventory_labels": ["Velero", "K3s Velero"], "source_rows": ["L143", "L252"], "runtime_identity": "k3s/velero", "reconciliation_state": "source_visible_runtime_unverified", "findings": ["Deployment presence is not proof of backup freshness or restore success"], "source_refs": ["ops/monitoring/service-registry.yaml", "docs/security/backup-restore-escrow-inventory.snapshot.json"], "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "velero_readback_and_rollout_verifier"} + ] + }, + { + "group_id": "imported-gateways", + "category": "gateway_proxy", + "common": { + "product_id": "shared-infrastructure", + "project_id": "network-gateway", + "site_id": "on-prem-lan", + "owner_lane": "network_ops", + "source_truth_state": "inventory_and_ansible_partial", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "docker_container", + "executor": "host_ansible_executor_with_exact_container_playbook", + "verifier": "container_runtime_independent_verifier", + "monitoring": {"signals": ["health", "tls", "route", "upstream", "latency"], "coverage": "partial"}, + "alerting": {"lifecycle": "gateway_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "readback_first", "target": "scripts/backup/backup-configs.sh", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P1", + "dependencies": ["PORT-001", "PORT-009"], + "next_action": "Resolve exact gateway identity and run route/TLS/upstream readback before bounded reload or restart." + }, + "members": [ + {"canonical_id": "service:nginx:host188", "inventory_labels": ["Nginx systemd"], "source_rows": ["L151"], "runtime_identity": "host188/systemd/nginx:80,443", "reconciliation_state": "source_visible_runtime_unverified", "findings": ["Imported purpose mentions multiple products but exact route ownership is not reconciled"], "source_refs": ["infra/ansible/inventory/group_vars/host_188.yml", "scripts/backup/backup-public-routes.sh"], "domain_router": "host_systemd", "executor": "host_ansible_executor", "verifier": "host_route_tls_independent_verifier"}, + {"canonical_id": "service:harbor:nginx", "inventory_labels": ["Nginx Harbor"], "source_rows": ["L152"], "runtime_identity": "host110/docker/harbor-nginx:5000", "reconciliation_state": "source_partial_runtime_unverified", "findings": ["Exact container health and upstream route receipt are absent"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-harbor.sh"], "project_id": "harbor"}, + {"canonical_id": "service:sentry:nginx", "inventory_labels": ["Nginx Sentry"], "source_rows": ["L153"], "runtime_identity": "host110/sentry-self-hosted/nginx:9000", "reconciliation_state": "source_partial_runtime_unverified", "findings": ["Exact container health and Sentry upstream receipt are absent"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-sentry.sh"], "project_id": "sentry"}, + {"canonical_id": "service:stockplatform-v2:nginx", "inventory_labels": ["Nginx stockplatform-v2"], "source_rows": ["L154"], "runtime_identity": "host110/docker/stockplatform-v2-nginx:31235", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Product route ownership and alert destination are unresolved"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "stockplatform-v2", "project_id": "stockPlatform", "owner_lane": "stockplatform_product_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"}, + {"canonical_id": "service:bitan:nginx", "inventory_labels": ["Nginx Bitan"], "source_rows": ["L155"], "runtime_identity": "host110/docker/bitan-nginx", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Product gateway has no canonical Telegram route or backup verifier"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "bitan-pharmacy", "project_id": "bitan-pharmacy", "owner_lane": "bitan_product_ops", "telegram_destination": "blocked:bitan_destination_unassigned"}, + {"canonical_id": "service:bitan:postgrest", "inventory_labels": ["PostgREST"], "source_rows": ["L156"], "runtime_identity": "host110/docker/bitan-postgrest:3000", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Database API must use Bitan-specific policy and DB verifier; generic container restart is insufficient"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "bitan-pharmacy", "project_id": "bitan-pharmacy", "owner_lane": "bitan_product_ops", "telegram_destination": "blocked:bitan_destination_unassigned"}, + {"canonical_id": "network:keepalived:vip-unresolved", "inventory_labels": ["Keepalived"], "source_rows": ["L157"], "runtime_identity": "unresolved:vip-192.168.0.125-or-192.168.0.200", "reconciliation_state": "critical_vip_identity_conflict", "findings": ["Imported inventory claims VIP 192.168.0.125 on host188 plus K3s", "Ansible host110/host188 vars claim VIP 192.168.0.200", "No network cutover is authorized"], "source_refs": ["infra/ansible/inventory/group_vars/host_110.yml", "infra/ansible/inventory/group_vars/host_188.yml"], "priority": "P0", "domain_router": "unknown", "executor": null, "verifier": "vip_identity_drift_verifier", "next_action": "Create fail-closed drift item and reconcile live VIP ownership read-only before any keepalived action."} + ] + }, + { + "group_id": "imported-security", + "category": "security", + "common": { + "product_id": "shared-infrastructure", + "project_id": "security-platform", + "site_id": "on-prem-lan", + "owner_lane": "security_ops", + "source_truth_state": "source_registry_partial", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "host_systemd", + "executor": "host_ansible_executor", + "verifier": "host_runtime_independent_verifier", + "monitoring": {"signals": ["service_health", "agent_freshness", "security_events", "scan_receipts"], "coverage": "partial"}, + "alerting": {"lifecycle": "security_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_security_incident_lifecycle", + "backup_restore": {"mode": "critical_read_only_default", "target": "security_config_and_evidence_contract", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P0", + "dependencies": ["PORT-001", "PORT-005"], + "next_action": "Run bounded no-secret posture readback and attach exact-host verifier receipts; active scans remain break-glass." + }, + "members": [ + {"canonical_id": "service:wazuh-manager", "inventory_labels": ["Wazuh Manager"], "source_rows": ["L168"], "runtime_identity": "host112/systemd/wazuh-manager", "reconciliation_state": "bounded_source_ready_runtime_unverified", "findings": ["Imported Active 22h and memory labels remain unverified point-in-time claims", "Exact no-write posture catalog exists; active response and arbitrary mutation remain unavailable"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml"], "verifier": "iwooos_wazuh_controlled_executor_runtime"}, + {"canonical_id": "service:wazuh-indexer:host112", "inventory_labels": ["Wazuh Indexer"], "source_rows": ["L169"], "runtime_identity": "host112/systemd/wazuh-indexer:9200,9300", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Indexer state, disk, shard health, and backup/restore verifier are not registered", "OpenSearch-compatible data mutation remains database-bounded"], "source_refs": ["ops/monitoring/alerts-unified.yml"], "domain_router": "database", "executor": "db_bounded_executor", "verifier": "wazuh_indexer_independent_verifier"}, + {"canonical_id": "service:wazuh-dashboard:host112", "inventory_labels": ["Wazuh Dashboard"], "source_rows": ["L170"], "runtime_identity": "host112/systemd/wazuh-dashboard:443", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["TLS, auth-safe UI health, and upstream Indexer/Manager receipts are not registered"], "source_refs": ["ops/monitoring/alerts-unified.yml"], "verifier": "wazuh_dashboard_independent_verifier"}, + {"canonical_id": "service:wazuh-agent:host188", "inventory_labels": ["Wazuh Agent host188"], "source_rows": ["L171"], "runtime_identity": "host188/systemd/wazuh-agent", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Agent-manager connectivity and event freshness are not registered"], "source_refs": ["infra/ansible/inventory/hosts.yml"]}, + {"canonical_id": "service:kali-scanner:host112", "inventory_labels": ["kali-scanner.service"], "source_rows": ["L176"], "runtime_identity": "host112/systemd/kali-scanner:8080", "reconciliation_state": "source_partial_runtime_unverified", "findings": ["Only TCP blackbox is registered", "Active or credentialed scans remain critical break-glass"], "source_refs": ["ops/monitoring/service-registry.yaml"]}, + {"canonical_id": "service:awoooi-guardian:host112", "inventory_labels": ["awoooi-guardian.service"], "source_rows": ["L177"], "runtime_identity": "host112/systemd/awoooi-guardian", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Multi-host monitoring scope, action boundary, and independent verifier are not registered"], "source_refs": ["ops/monitoring/alerts-unified.yml"]}, + {"canonical_id": "service:filebeat:host112", "inventory_labels": ["filebeat.service"], "source_rows": ["L178"], "runtime_identity": "host112/systemd/filebeat", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Log delivery freshness and Wazuh Indexer backpressure receipts are missing"], "source_refs": ["ops/monitoring/alerts-unified.yml"], "verifier": "filebeat_delivery_independent_verifier"}, + {"canonical_id": "service:wg-easy:host112", "inventory_labels": ["wg-easy"], "source_rows": ["L183"], "runtime_identity": "host112/docker/wg-easy:51820,51821", "reconciliation_state": "github_freeze_and_network_boundary_conflict", "findings": ["Imported image points to ghcr.io and conflicts with the GitHub freeze", "VPN/firewall cutover is critical break-glass", "Web UI exposure and route ownership are unverified"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "domain_router": "docker_container", "executor": "host_ansible_executor_with_exact_container_playbook", "verifier": "wireguard_route_and_container_independent_verifier"}, + {"canonical_id": "service:node-exporter:host112", "inventory_labels": ["node-exporter host112"], "source_rows": ["L184"], "runtime_identity": "host112/docker/node-exporter:9100", "reconciliation_state": "missing_from_monitoring_service_registry", "findings": ["Current scrape freshness and exact image provenance are not registered"], "source_refs": ["ops/monitoring/generated/prometheus-scrape-generated.yaml"], "domain_router": "docker_container", "executor": "host_ansible_executor_with_exact_container_playbook", "verifier": "node_exporter_scrape_independent_verifier"}, + {"canonical_id": "package:wazuh-suite:host112:4.9.2", "inventory_labels": ["Wazuh Manager/Indexer/Dashboard 4.9.2"], "source_rows": ["L189"], "runtime_identity": "host112/packages/wazuh-suite:4.9.2", "reconciliation_state": "package_version_claim_runtime_unverified", "findings": ["Composite package row duplicates three runtime services", "Upgrade and rollback require component-specific replay and canary"], "source_refs": ["ops/config/service-registry.yaml"], "verifier": "host_package_version_independent_verifier"}, + {"canonical_id": "package:nmap-zenmap:host112", "inventory_labels": ["nmap / zenmap"], "source_rows": ["L190"], "runtime_identity": "host112/packages/nmap-zenmap", "reconciliation_state": "security_tool_inventory_only", "findings": ["Installed tool is not authorization for active scanning", "Version pair requires immutable package provenance"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "verifier": "security_tool_inventory_verifier"}, + {"canonical_id": "package:metasploit:host112", "inventory_labels": ["Metasploit Framework"], "source_rows": ["L191"], "runtime_identity": "host112/packages/metasploit", "reconciliation_state": "critical_security_tool_inventory_only", "findings": ["Credentialed exploit activity is critical break-glass", "Installed version and package provenance are unverified"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "verifier": "security_tool_inventory_verifier"}, + {"canonical_id": "package:burp-suite:host112", "inventory_labels": ["Burp Suite"], "source_rows": ["L192"], "runtime_identity": "host112/packages/burp-suite", "reconciliation_state": "critical_security_tool_inventory_only", "findings": ["Active web scanning is critical break-glass", "Edition, license, and package provenance are unverified"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "verifier": "security_tool_inventory_verifier"}, + {"canonical_id": "package:sqlmap:host112", "inventory_labels": ["sqlmap"], "source_rows": ["L193"], "runtime_identity": "host112/packages/sqlmap", "reconciliation_state": "critical_security_tool_inventory_only", "findings": ["SQL injection scanning is critical break-glass", "Installed version and package provenance are unverified"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "verifier": "security_tool_inventory_verifier"}, + {"canonical_id": "package:nikto:host112", "inventory_labels": ["nikto"], "source_rows": ["L194"], "runtime_identity": "host112/packages/nikto", "reconciliation_state": "critical_security_tool_inventory_only", "findings": ["Active web scanning is critical break-glass", "Installed version and package provenance are unverified"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "verifier": "security_tool_inventory_verifier"}, + {"canonical_id": "package:openvas:host112", "inventory_labels": ["OpenVAS / ospd-openvas"], "source_rows": ["L195"], "runtime_identity": "host112/packages/openvas-ospd", "reconciliation_state": "critical_security_tool_inventory_only", "findings": ["Active vulnerability scanning is critical break-glass", "Feed freshness and daemon state are not registered"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "verifier": "security_tool_and_feed_independent_verifier"}, + {"canonical_id": "package:python-elasticsearch:host112", "inventory_labels": ["python3-elasticsearch"], "source_rows": ["L196"], "runtime_identity": "host112/packages/python3-elasticsearch", "reconciliation_state": "package_inventory_only", "findings": ["Client compatibility with Wazuh Indexer 4.9.2 is not verified"], "source_refs": ["docs/evaluations/package_supply_chain_inventory_2026-06-04.json"], "verifier": "host_package_version_independent_verifier"}, + {"canonical_id": "service:haveged:host112", "inventory_labels": ["haveged"], "source_rows": ["L197"], "runtime_identity": "host112/systemd/haveged", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["Entropy service health and dependency need are unverified"], "source_refs": ["infra/ansible/inventory/hosts.yml"]}, + {"canonical_id": "service:lightdm:host112", "inventory_labels": ["LightDM"], "source_rows": ["L198"], "runtime_identity": "host112/systemd/lightdm", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["GUI display-manager health is not a Wazuh completion receipt", "Recovery must remain exact-host and independently verified"], "source_refs": ["docs/operations/host112-guest-recovery-20260711.snapshot.json"]}, + {"canonical_id": "schedule:host112:wazuh-guardian-hourly", "inventory_labels": ["每小時整點 host112 security schedule"], "source_rows": ["L203"], "runtime_identity": "host112/crontab/hourly-security-monitor", "reconciliation_state": "schedule_command_and_receipt_unregistered", "findings": ["Schedule label alone does not identify exact command, owner, idempotency, or output receipt"], "source_refs": ["ops/monitoring/alerts-unified.yml"], "verifier": "host_schedule_freshness_independent_verifier"}, + {"canonical_id": "schedule:host112:daily-0800", "inventory_labels": ["每日 08:00 host112 security schedule"], "source_rows": ["L204"], "runtime_identity": "host112/crontab/daily-0800", "reconciliation_state": "schedule_command_and_receipt_unregistered", "findings": ["Exact command, owner, risk, and delivery receipt are absent"], "source_refs": ["ops/monitoring/alerts-unified.yml"], "verifier": "host_schedule_freshness_independent_verifier"}, + {"canonical_id": "schedule:host112:weekly-sunday-0900", "inventory_labels": ["每週日 09:00 host112 security schedule"], "source_rows": ["L205"], "runtime_identity": "host112/crontab/weekly-sunday-0900", "reconciliation_state": "schedule_command_and_receipt_unregistered", "findings": ["Exact command, active-scan boundary, and result receipt are absent"], "source_refs": ["ops/monitoring/alerts-unified.yml"], "verifier": "host_schedule_freshness_independent_verifier"}, + {"canonical_id": "schedule:host112:daily-0200", "inventory_labels": ["每日 02:00 host112 security schedule"], "source_rows": ["L206"], "runtime_identity": "host112/crontab/daily-0200", "reconciliation_state": "schedule_command_and_receipt_unregistered", "findings": ["Exact command, backup/log retention boundary, and result receipt are absent"], "source_refs": ["ops/monitoring/alerts-unified.yml"], "verifier": "host_schedule_freshness_independent_verifier"}, + {"canonical_id": "service:fail2ban:host188", "inventory_labels": ["Fail2Ban historical inventory row"], "source_rows": [], "historical_source_rows": ["inventory-2e8580c-20260716:L167"], "runtime_identity": "host188/systemd/fail2ban", "reconciliation_state": "missing_from_imported_inventory", "findings": ["Asset was present in the 241-line inventory but disappeared from active c214160 without a retirement receipt", "Ban policy, rollback, and independent verifier remain unregistered"], "source_refs": ["infra/ansible/inventory/hosts.yml"], "next_action": "Create removal drift work; verify whether Fail2Ban is retired or omitted without restarting or changing bans."} + ] + }, + { + "group_id": "imported-product-applications", + "category": "application_service", + "common": { + "product_id": "unknown", + "project_id": "unknown", + "site_id": "on-prem-lan", + "owner_lane": "asset_identity", + "source_truth_state": "imported_runtime_claim_without_product_manifest", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "docker_container", + "executor": "host_ansible_executor_with_exact_container_playbook", + "verifier": "container_runtime_independent_verifier", + "monitoring": {"signals": ["health", "latency", "errors", "data_freshness", "worker_lag"], "coverage": "gap"}, + "alerting": {"lifecycle": "product_specific_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "blocked:canonical_product_route_required", + "backup_restore": {"mode": "critical_read_only_default", "target": "gap:product_backup_restore_contract", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P1", + "dependencies": ["PORT-001", "PORT-006", "PORT-007", "PORT-008"], + "next_action": "Adopt a product manifest, exact container registry, product-only Telegram route, backup contract, and same-run verifier before controlled apply." + }, + "members": [ + {"canonical_id": "service:momo-pro:web", "inventory_labels": ["momo-pro-system"], "source_rows": ["L218"], "runtime_identity": "host188/docker/momo-pro-system:5003", "reconciliation_state": "product_manifest_and_registry_gap", "findings": ["Imported health is unverified; product destination returns chat_not_found"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json", "scripts/backup/backup-momo.sh"], "product_id": "momo-pro", "project_id": "momo-pro-system", "owner_lane": "momo_product_ops", "telegram_destination": "blocked:momo_configured_chat_unresolved", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-momo.sh", "freshness": "unverified"}}, + {"canonical_id": "service:momo-pro:scheduler", "inventory_labels": ["momo-scheduler"], "source_rows": ["L219"], "runtime_identity": "host188/docker/momo-scheduler", "reconciliation_state": "product_manifest_and_registry_gap", "findings": ["Scheduler freshness and idempotent replay contract are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "momo-pro", "project_id": "momo-pro-system", "owner_lane": "momo_product_ops", "telegram_destination": "blocked:momo_configured_chat_unresolved"}, + {"canonical_id": "service:momo-pro:telegram-bot", "inventory_labels": ["momo-telegram-bot"], "source_rows": ["L220"], "runtime_identity": "host188/docker/momo-telegram-bot", "reconciliation_state": "destination_unreachable", "findings": ["Configured destination returns chat_not_found; sending remains blocked"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "momo-pro", "project_id": "momo-pro-system", "owner_lane": "momo_product_ops", "telegram_destination": "blocked:momo_configured_chat_unresolved"}, + {"canonical_id": "service:vibework:web", "inventory_labels": ["vibework-production-web-1"], "source_rows": ["L221"], "runtime_identity": "host188/docker/vibework-production-web-1:32336", "reconciliation_state": "product_manifest_and_route_gap", "findings": ["Standalone VibeWork Telegram route is not implemented"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "vibework", "project_id": "vibework", "owner_lane": "vibework_product_ops", "telegram_destination": "blocked:vibework_route_not_implemented"}, + {"canonical_id": "service:vibework:notification-worker", "inventory_labels": ["vibework-production-notification-worker-1"], "source_rows": ["L222"], "runtime_identity": "host188/docker/vibework-production-notification-worker-1", "reconciliation_state": "product_manifest_and_receipt_gap", "findings": ["Notification durability and delivery receipts are unverified"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "vibework", "project_id": "vibework", "owner_lane": "vibework_product_ops", "telegram_destination": "blocked:vibework_route_not_implemented"}, + {"canonical_id": "site:tsenyang-website", "inventory_labels": ["tsenyang-website"], "source_rows": ["L223"], "runtime_identity": "host188/docker/tsenyang-website:3000", "reconciliation_state": "route_ownership_conflict", "findings": ["Telegram private route is shared with StockPlatform and ownership is unresolved"], "source_refs": ["infra/ansible/inventory/group_vars/host_188.yml", "docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "tsenyang-website", "project_id": "tsenyang-website", "owner_lane": "tsenyang_site_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"}, + {"canonical_id": "service:2026fifa:web", "inventory_labels": ["current-fifa2026-web-1"], "source_rows": ["L224"], "runtime_identity": "host188/docker/current-fifa2026-web-1:3108", "reconciliation_state": "product_manifest_and_route_gap", "findings": ["Canonical product destination and durable receipt backend are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:2026fifa:backend", "inventory_labels": ["current-fifa2026-backend-1"], "source_rows": ["L225"], "runtime_identity": "host188/docker/current-fifa2026-backend-1:8000", "reconciliation_state": "product_manifest_and_route_gap", "findings": ["API SLO and backup contract are not registered in this repo"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:2026fifa:odds-worker", "inventory_labels": ["current-fifa2026-odds-worker-1"], "source_rows": ["L226"], "runtime_identity": "host188/docker/current-fifa2026-odds-worker-1", "reconciliation_state": "freshness_and_route_gap", "findings": ["Odds source freshness, provider ownership, and replay verifier are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:2026fifa:fixtures-worker", "inventory_labels": ["current-fifa2026-fixtures-worker-1"], "source_rows": ["L227"], "runtime_identity": "host188/docker/current-fifa2026-fixtures-worker-1", "reconciliation_state": "freshness_and_route_gap", "findings": ["Fixture source freshness and replay verifier are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:2026fifa:calendar-cache-worker", "inventory_labels": ["current-fifa2026-calendar-cache-worker-1"], "source_rows": ["L228"], "runtime_identity": "host188/docker/current-fifa2026-calendar-cache-worker-1", "reconciliation_state": "freshness_and_route_gap", "findings": ["Cache freshness and rebuild verifier are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:2026fifa:agent-review-worker", "inventory_labels": ["current-fifa2026-agent-review-worker-1"], "source_rows": ["L229"], "runtime_identity": "host188/docker/current-fifa2026-agent-review-worker-1", "reconciliation_state": "ai_review_receipt_gap", "findings": ["AI reviewer model, decision receipt, and cost attribution are not registered"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:2026fifa:news-worker", "inventory_labels": ["current-fifa2026-news-worker-1"], "source_rows": ["L230"], "runtime_identity": "host188/docker/current-fifa2026-news-worker-1", "reconciliation_state": "source_freshness_and_route_gap", "findings": ["News source provenance/freshness and replay verifier are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:2026fifa:alerts", "inventory_labels": ["current-fifa2026-alerts-1"], "source_rows": ["L231"], "runtime_identity": "host188/docker/current-fifa2026-alerts-1", "reconciliation_state": "alert_destination_and_receipt_gap", "findings": ["Product alert service has no canonical destination or durable delivery receipt"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, + {"canonical_id": "service:agent-bounty:web", "inventory_labels": ["agent_bounty_web"], "source_rows": ["L233"], "runtime_identity": "host110/docker/agent_bounty_web:3005", "reconciliation_state": "durable_route_receipt_gap", "findings": ["Bot-to-group route is source-proven but durable delivery receipt is missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "agent-bounty-protocol", "project_id": "agent-bounty-protocol", "owner_lane": "agent_bounty_product_ops", "telegram_destination": "blocked:vibe_ai_agent_receipt_gap"}, + {"canonical_id": "service:awooogo:web", "inventory_labels": ["awooo-go-web-1"], "source_rows": ["L234"], "runtime_identity": "host110/docker/awooo-go-web-1:32190", "reconciliation_state": "telegram_disabled_and_manifest_gap", "findings": ["Product Telegram egress is intentionally disabled"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "awooogo", "project_id": "AwoooGo", "owner_lane": "awooogo_product_ops", "telegram_destination": "disabled:awooogo"}, + {"canonical_id": "service:awooogo:api", "inventory_labels": ["awooo-go-api-1"], "source_rows": ["L235"], "runtime_identity": "host110/docker/awooo-go-api-1", "reconciliation_state": "telegram_disabled_and_manifest_gap", "findings": ["Product API monitoring and backup contract are not registered here"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "awooogo", "project_id": "AwoooGo", "owner_lane": "awooogo_product_ops", "telegram_destination": "disabled:awooogo"}, + {"canonical_id": "service:awooogo:security-agent", "inventory_labels": ["awooo-go-security-agent-1"], "source_rows": ["L236"], "runtime_identity": "host110/docker/awooo-go-security-agent-1", "reconciliation_state": "security_agent_contract_gap", "findings": ["Exact authority, active-response boundary, and verifier are not registered"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "awooogo", "project_id": "AwoooGo", "owner_lane": "awooogo_product_ops", "telegram_destination": "disabled:awooogo", "priority": "P0"}, + {"canonical_id": "service:bitan-pharmacy:postgrest", "inventory_labels": ["bitan-pharmacy-postgrest"], "source_rows": ["L237"], "runtime_identity": "host110/docker/bitan-pharmacy-postgrest", "reconciliation_state": "product_route_and_db_verifier_gap", "findings": ["Canonical product destination and bounded DB/API verifier are missing"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "bitan-pharmacy", "project_id": "bitan-pharmacy", "owner_lane": "bitan_product_ops", "telegram_destination": "blocked:bitan_destination_unassigned"}, + {"canonical_id": "service:open-design", "inventory_labels": ["open-design"], "source_rows": ["L238"], "runtime_identity": "host110/docker/open-design", "reconciliation_state": "asset_identity_unresolved", "findings": ["Product owner, source repository, monitoring, alert route, and backup contract are absent"], "source_refs": ["ops/config/service-registry.yaml"], "domain_router": "unknown", "executor": null, "verifier": "asset_identity_drift_verifier", "next_action": "Create deterministic drift work item; prohibit fallback AUTO until product ownership and source truth are registered."}, + {"canonical_id": "service:stockplatform-v2:web", "inventory_labels": ["stockplatform-v2-web-1"], "source_rows": ["L239"], "runtime_identity": "host110/docker/stockplatform-v2-web-1:3000", "reconciliation_state": "route_ownership_and_manifest_gap", "findings": ["Telegram private route ownership conflicts with TSENYANG Website"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "stockplatform-v2", "project_id": "stockPlatform", "owner_lane": "stockplatform_product_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"}, + {"canonical_id": "service:stockplatform-v2:admin", "inventory_labels": ["stockplatform-v2-admin-1"], "source_rows": ["L240"], "runtime_identity": "host110/docker/stockplatform-v2-admin-1:3001", "reconciliation_state": "admin_surface_and_route_gap", "findings": ["Admin auth, smoke, and alert ownership are not linked to a product manifest"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "stockplatform-v2", "project_id": "stockPlatform", "owner_lane": "stockplatform_product_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"}, + {"canonical_id": "service:stockplatform-v2:api", "inventory_labels": ["stockplatform-v2-api-1"], "source_rows": ["L241"], "runtime_identity": "host110/docker/stockplatform-v2-api-1:8000", "reconciliation_state": "api_data_and_route_gap", "findings": ["Market-data freshness, API SLO, and product alert ownership are not registered here"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "stockplatform-v2", "project_id": "stockPlatform", "owner_lane": "stockplatform_product_ops", "telegram_destination": "blocked:stockplatform_tsenyang_ownership_drift"} + ] + }, + { + "group_id": "imported-k3s-controls", + "category": "k3s_component", + "common": { + "product_id": "shared-infrastructure", + "project_id": "k3s-control-plane", + "site_id": "k3s", + "owner_lane": "k3s_ops", + "source_truth_state": "inventory_claim_manifest_lookup_required", + "live_truth_state": "not_probed_this_reconciliation", + "domain_router": "kubernetes_workload", + "executor": "kubernetes_controlled_executor", + "verifier": "kubernetes_rollout_verifier", + "monitoring": {"signals": ["rollout", "readiness", "events", "policy_drift"], "coverage": "partial"}, + "alerting": {"lifecycle": "k3s_component_incident", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "readback_first", "target": "k3s_manifest_and_velero_contract", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P1", + "dependencies": ["PORT-001", "PORT-005", "PORT-010"], + "next_action": "Resolve exact namespace/kind/name and verify rollout/policy state before Kubernetes controlled apply." + }, + "members": [ + {"canonical_id": "k8s:kured", "inventory_labels": ["Kured"], "source_rows": ["L253"], "runtime_identity": "k3s/kured", "reconciliation_state": "critical_reboot_boundary_gap", "findings": ["Host reboot is critical break-glass; inventory presence cannot authorize Kured actions"], "source_refs": ["k8s"], "priority": "P0"}, + {"canonical_id": "k8s:descheduler", "inventory_labels": ["Descheduler"], "source_rows": ["L254"], "runtime_identity": "k3s/descheduler", "reconciliation_state": "manifest_and_runtime_verifier_gap", "findings": ["Rescheduling blast radius and post-verifier are not linked"], "source_refs": ["k8s"]}, + {"canonical_id": "k8s:vpa", "inventory_labels": ["VPA"], "source_rows": ["L255"], "runtime_identity": "k3s/vpa", "reconciliation_state": "recommendation_mode_unverified", "findings": ["Imported row says recommendation mode but live mode and recommendation freshness are unverified"], "source_refs": ["k8s"]}, + {"canonical_id": "k8s:hpa", "inventory_labels": ["HPA"], "source_rows": ["L256"], "runtime_identity": "k3s/hpa", "reconciliation_state": "policy_coverage_unverified", "findings": ["Target workload coverage and scaling verifier are not enumerated"], "source_refs": ["k8s"]}, + {"canonical_id": "k8s:pdb", "inventory_labels": ["PDB"], "source_rows": ["L257"], "runtime_identity": "k3s/pdb", "reconciliation_state": "policy_coverage_unverified", "findings": ["Protected workload coverage and disruption allowance are not enumerated"], "source_refs": ["k8s"]}, + {"canonical_id": "k8s:network-policy", "inventory_labels": ["NetworkPolicy"], "source_rows": ["L258"], "runtime_identity": "k3s/networkpolicy", "reconciliation_state": "policy_coverage_unverified", "findings": ["Presence does not prove every workload/provider path is allowed or isolated"], "source_refs": ["k8s/awoooi-prod/02-network-policy.yaml"]}, + {"canonical_id": "k8s:rbac", "inventory_labels": ["RBAC"], "source_rows": ["L259"], "runtime_identity": "k3s/rbac", "reconciliation_state": "policy_coverage_unverified", "findings": ["Presence does not prove least privilege or executor identity"], "source_refs": ["k8s"]} + ] + }, + { + "group_id": "missing-control-plane-and-runtime-assets", + "category": "inventory_omission", + "common": { + "product_id": "awoooi", + "project_id": "ai-control-plane", + "site_id": "production", + "owner_lane": "AIControlPlane", + "source_truth_state": "source_visible_missing_from_imported_inventory", + "live_truth_state": "not_probed_this_reconciliation", + "reconciliation_state": "missing_from_imported_inventory", + "domain_router": "control_plane_service", + "executor": "single_control_plane_executor", + "verifier": "control_plane_independent_verifier", + "monitoring": {"signals": ["health", "freshness", "same_run_receipt", "coverage"], "coverage": "inventory_gap"}, + "alerting": {"lifecycle": "canonical_incident_only", "cross_domain_fallback_allowed": false, "required_ai_attribution": true}, + "telegram_destination": "awoooi_sre_war_room_for_shared_P0_P1_lifecycle_only", + "backup_restore": {"mode": "readback_first", "target": "component_specific_contract_required", "freshness": "unverified"}, + "learning_targets": ["KM", "RAG", "MCP", "PlayBook"], + "priority": "P0", + "dependencies": ["PORT-001", "PORT-005", "PORT-011"], + "next_action": "Add the canonical asset to the reconciled source registry and require same-run production readback." + }, + "members": [ + {"canonical_id": "windows-vmware:host_99", "inventory_labels": ["Agent99 Windows/VMware control plane"], "source_rows": [], "runtime_identity": "192.168.0.99/Agent99", "findings": ["Agent99 is required for Windows/VMware and control-plane recovery but absent from the imported inventory", "Agent99 also owns the read-only host110 Alertmanager poll/reduced relay coordination role; Linux remediation still routes only to the host Ansible executor", "The atomic 15-file runtime bundle includes the poller but production deployment/readback is pending"], "source_refs": ["docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json", "k8s/awoooi-prod/04-configmap.yaml", "agent99-alertmanager-alertchain-poll.ps1", "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"], "owner_lane": "Agent99", "domain_router": "windows_vmware", "executor": "Agent99", "verifier": "agent99_independent_runtime_verifier"}, + {"canonical_id": "service:ollama:host110", "inventory_labels": ["retired Host110 Ollama tombstone"], "source_rows": [], "runtime_identity": "retired-tombstone:host110/ollama/no-endpoint", "source_truth_state": "retired_tombstone_source_ready_runtime_absence_pending", "reconciliation_state": "retired_tombstone_source_ready_runtime_absence_pending", "findings": ["Host110 Ollama was removed and is forbidden as a provider, proxy, primary, secondary or failover target", "A tombstone is retained so stale sensors and routes create drift instead of resurrecting the endpoint", "Production runtime and Prometheus absence receipts are still missing"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json"], "owner_lane": "ai_team", "domain_router": "host_systemd", "executor": "host_ansible_executor", "verifier": "host110_ollama_runtime_and_monitoring_absence_verifier", "monitoring": {"signals": ["runtime_endpoint_absence", "prometheus_target_absence", "route_reference_absence"], "coverage": "source_tombstone_runtime_absence_pending"}, "next_action": "Verify no Host110 Ollama process, proxy, target, rule or provider route exists; preserve the tombstone and create drift on any recurrence."}, + {"canonical_id": "host:111", "inventory_labels": ["host111 local Ollama"], "source_rows": [], "runtime_identity": "192.168.0.111/macos-launchd/ollama", "findings": ["Approved third provider hop and Ansible host are absent from the imported inventory", "The runtime manager is a macOS LaunchAgent, not systemd", "The exact LaunchAgent playbook requires local verification plus independent origin probes from host120 and host121", "The canonical host111 Ollama sensor/rule has no fresh production series"], "source_refs": ["infra/ansible/inventory/hosts.yml", "infra/ansible/playbooks/111-ollama-fallback.yml", "ops/monitoring/service-registry.yaml", "ops/monitoring/alerts-unified.yml", "apps/api/src/services/awooop_ansible_post_verifier.py"], "owner_lane": "ai_team", "domain_router": "host_systemd", "executor": "host_ansible_executor", "verifier": "host111_launchagent_local_and_120_121_origin_verifier", "monitoring": {"signals": ["launchagent_state", "local_generation", "host120_origin_generation", "host121_origin_generation", "canonical_alert_series"], "coverage": "source_candidate_runtime_sensor_missing"}, "next_action": "Check/apply the exact LaunchAgent catalog, verify local plus host120/host121 origins, and require a fresh canonical sensor series before provider readiness."}, + {"canonical_id": "ai-provider:ollama_gcp_a", "inventory_labels": ["GCP-A Ollama"], "source_rows": [], "runtime_identity": "34.143.170.20:11434", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Approved first provider hop is absent from the imported inventory", "Observed transport is public HTTP and is candidate-only until secure mesh/TLS promotion", "Only sanitized prompts may cross this boundary; unsanitized input and every tool loop fail closed before network access", "The exact Prometheus provider target is missing in production"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/monitoring/prometheus.yml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "apps/api/src/services/ai_provider_policy.py"], "owner_lane": "ai_team", "domain_router": "ai_provider", "executor": "deterministic_provider_router", "verifier": "provider_health_generation_and_transport_receipt_verifier", "monitoring": {"signals": ["exact_target_up", "scrape_freshness", "sanitization_receipt", "tool_loop_denial", "transport_boundary"], "coverage": "source_candidate_runtime_target_missing"}, "transport_policy": {"observed": "public_http", "execution_scope": "sanitized_candidate_only", "tool_loop": "fail_closed", "promotion_requires": "secure_mesh_or_tls_plus_runtime_readback"}, "next_action": "Deploy the exact blackbox target, verify fresh series and sanitization/tool-loop denials, then retain candidate-only status until secure transport promotion."}, + {"canonical_id": "ai-provider:ollama_gcp_b", "inventory_labels": ["GCP-B Ollama"], "source_rows": [], "runtime_identity": "34.21.145.224:11434", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Approved second provider hop is absent from the imported inventory", "Observed transport is public HTTP and is candidate-only until secure mesh/TLS promotion", "Only sanitized prompts may cross this boundary; unsanitized input and every tool loop fail closed before network access", "The exact Prometheus provider target is missing in production"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/monitoring/prometheus.yml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "apps/api/src/services/ai_provider_policy.py"], "owner_lane": "ai_team", "domain_router": "ai_provider", "executor": "deterministic_provider_router", "verifier": "provider_health_generation_and_transport_receipt_verifier", "monitoring": {"signals": ["exact_target_up", "scrape_freshness", "sanitization_receipt", "tool_loop_denial", "transport_boundary"], "coverage": "source_candidate_runtime_target_missing"}, "transport_policy": {"observed": "public_http", "execution_scope": "sanitized_candidate_only", "tool_loop": "fail_closed", "promotion_requires": "secure_mesh_or_tls_plus_runtime_readback"}, "next_action": "Deploy the exact blackbox target, verify fresh series and sanitization/tool-loop denials, then retain candidate-only status until secure transport promotion."}, + {"canonical_id": "ai-provider:claude", "inventory_labels": ["Anthropic Claude API"], "source_rows": [], "runtime_identity": "protected-secret-reference/anthropic", "findings": ["Approved fourth provider hop and paid canary are absent from the imported inventory", "User-supplied credential material was noted but no raw value is recorded or readable from this snapshot", "A protected-secret metadata verifier and durable Gate5 authorization bound to the run are pending", "Paid enablement requires paired Claude/Gemini rollback, daily/monthly/per-incident cost caps and production cost readback", "The sanitized five-lane paid canary has not run"], "source_refs": ["docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "apps/api/src/services/paid_provider_canary_validation.py"], "owner_lane": "ai_team", "domain_router": "ai_provider", "executor": "deterministic_provider_router", "verifier": "paid_provider_generation_and_cost_receipt_verifier", "credential_metadata": {"user_supplied_credential_noted": true, "raw_value_recorded": false, "raw_value_readback_allowed": false, "protected_secret_reference_status": "pending"}, "authorization": {"gate": "Gate5", "durable_run_binding_required": true, "status": "pending"}, "canary": {"percent": 5, "status": "pending", "paired_rollback_required": true, "cost_caps_verified": false}}, + {"canonical_id": "ai-provider:gemini", "inventory_labels": ["Gemini API"], "source_rows": [], "runtime_identity": "protected-secret-reference/gemini", "findings": ["Approved final provider hop and paid canary are absent from the imported inventory", "User-supplied credential material was noted but no raw value is recorded or readable from this snapshot", "A protected-secret metadata verifier and durable Gate5 authorization bound to the run are pending", "Paid enablement requires paired Claude/Gemini rollback, daily/monthly/per-incident cost caps and production cost readback", "The sanitized five-lane paid canary has not run"], "source_refs": ["docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "apps/api/src/services/paid_provider_canary_validation.py"], "owner_lane": "ai_team", "domain_router": "ai_provider", "executor": "deterministic_provider_router", "verifier": "paid_provider_generation_and_cost_receipt_verifier", "credential_metadata": {"user_supplied_credential_noted": true, "raw_value_recorded": false, "raw_value_readback_allowed": false, "protected_secret_reference_status": "pending"}, "authorization": {"gate": "Gate5", "durable_run_binding_required": true, "status": "pending"}, "canary": {"percent": 5, "status": "pending", "paired_rollback_required": true, "cost_caps_verified": false}}, + {"canonical_id": "control-plane:holmesgpt-investigator", "inventory_labels": ["HolmesGPT Investigator"], "source_rows": [], "runtime_identity": "source_candidate/runtime_not_promoted", "findings": ["Required pipeline investigator is absent from the infrastructure inventory and runtime promotion remains pending"], "source_refs": ["docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json"], "owner_lane": "InvestigationPlane"}, + {"canonical_id": "k8s:awoooi-prod:api", "inventory_labels": ["AWOOOI production API"], "source_rows": [], "runtime_identity": "k3s/awoooi-prod/deployment/awoooi-api", "findings": ["Production API is in the monitoring registry but absent from the imported infrastructure inventory"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/awoooi-prod/06-deployment-api.yaml"], "owner_lane": "backend_team", "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "k8s:awoooi-prod:web", "inventory_labels": ["AWOOOI production Web"], "source_rows": [], "runtime_identity": "k3s/awoooi-prod/deployment/awoooi-web", "findings": ["Production Web is in the monitoring registry but absent from the imported infrastructure inventory"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/awoooi-prod"], "owner_lane": "frontend_team", "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "k8s:awoooi-prod:worker", "inventory_labels": ["AWOOOI production Worker"], "source_rows": [], "runtime_identity": "k3s/awoooi-prod/deployment/awoooi-worker", "findings": ["Worker is in the monitoring registry but absent from the imported infrastructure inventory"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/awoooi-prod"], "owner_lane": "backend_team", "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "k8s:awoooi-prod:broker", "inventory_labels": ["AWOOOI production Broker"], "source_rows": [], "runtime_identity": "k3s/awoooi-prod/deployment/awoooi-broker", "findings": ["Broker is part of production deployment/readback but absent from the imported inventory and monitoring service registry"], "source_refs": ["k8s/awoooi-prod", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json"], "owner_lane": "backend_team", "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, + {"canonical_id": "signal:alertmanager-webhook-chain", "inventory_labels": ["AlertChainBroken_Alertmanager"], "source_rows": [], "runtime_identity": "alertmanager->awoooi-api-webhook->incident->telegram plus host99 Agent99 exact-host pull fallback", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Canonical signal is locked to container:host_110:alertmanager", "Rule, concise attribution card, exact bounded recovery and independent verifier are source-ready", "Host99 Agent99 independently polls only the exact active/critical AlertChain signal from host110 without a first-hop credential or raw-payload persistence", "The reduced event is relayed over HTTPS with stable dedupe while Linux execution remains in the host Ansible lane", "Production deployment, poll freshness/dedupe, current alert resolution and same-run learning acknowledgements remain pending"], "source_refs": ["k8s/monitoring/alert-chain-monitor.yaml", "ops/alertmanager/alertmanager.yml", "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-relay.ps1", "apps/api/src/api/v1/webhooks.py", "apps/api/src/services/controlled_alert_target_router.py", "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml", "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"], "owner_lane": "observability_ops", "verifier": "alert_delivery_chain_plus_agent99_pull_independent_verifier", "monitoring": {"signals": ["alertmanager_delivery_counters", "primary_webhook_health", "agent99_pull_freshness", "dedupe_receipt", "telegram_delivery_receipt"], "coverage": "source_primary_and_independent_pull_ready_runtime_pending"}, "next_action": "Deploy the receiver-scoped scrape/rule/playbook and the atomic Agent99 15-file bundle; run check/apply/verify under one trace/run/work item; then require Alertmanager resolution, Telegram and KM/RAG/MCP/PlayBook acknowledgements before closure."}, + {"canonical_id": "control-plane:telegram-routing-registry", "inventory_labels": ["Canonical Telegram routing registry"], "source_rows": [], "runtime_identity": "docs/security/telegram-canonical-routing-registry.snapshot.json", "findings": ["Nine product routes are blocked/disabled/not implemented; scattering alerts is forbidden"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "owner_lane": "NotificationGovernance", "verifier": "telegram_route_and_delivery_receipt_verifier"}, + {"canonical_id": "control-plane:km-rag", "inventory_labels": ["Knowledge Base and RAG"], "source_rows": [], "runtime_identity": "awoooi-api/knowledge-rag", "findings": ["Required learning target is implemented in source but absent from the imported inventory and runtime closure is unverified"], "source_refs": ["apps/api/src/api/v1/knowledge.py", "apps/api/src/api/v1/rag.py"], "owner_lane": "LearningPlane", "verifier": "km_rag_durable_write_ack_verifier"}, + {"canonical_id": "control-plane:mcp-gateway", "inventory_labels": ["Internal MCP Gateway"], "source_rows": [], "runtime_identity": "awoooi-api/mcp-control-plane", "findings": ["MCP catalog has broad product coverage but is absent from the imported inventory; write adapters must remain policy controlled"], "source_refs": ["docs/operations/mcp-control-plane-catalog.snapshot.json"], "owner_lane": "MCPControlPlane", "verifier": "mcp_gateway_audit_and_runtime_verifier"}, + {"canonical_id": "control-plane:playbook-registry", "inventory_labels": ["PlayBook registry"], "source_rows": [], "runtime_identity": "awoooi-api/playbook-service", "findings": ["Required candidate/rollback/trust asset is absent from the imported inventory"], "source_refs": ["apps/api/src/services/playbook_service.py"], "owner_lane": "AutomationControlPlane", "verifier": "playbook_trust_and_execution_receipt_verifier"}, + {"canonical_id": "control-plane:independent-verifier-registry", "inventory_labels": ["Independent verifier registry"], "source_rows": [], "runtime_identity": "awoooi-api/verifier-registry", "findings": ["All assets require an independent verifier, but the inventory does not enumerate verifier ownership or coverage"], "source_refs": ["docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json"], "owner_lane": "VerifierPlane", "verifier": "verifier_registry_self_check"} + ] + } + ], + "conflict_register": [ + {"id": "PORT-CONFLICT-001", "priority": "P0", "assets": ["service:alertmanager", "signal:alertmanager-webhook-chain"], "conflict": "Alertmanager source identity, receiver-scoped metrics, exact repair path and host99 Agent99 independent pull/reduced relay are reconciled in source; production deployment, poll freshness/dedupe, alert resolution and same-run closure receipts remain pending.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-003", "AIA-SRE-004", "AIA-SRE-014", "AIA-SRE-015", "AIA-SRE-017"]}, + {"id": "PORT-CONFLICT-002", "priority": "P0", "assets": ["service:ollama:host110", "service:ollama:host188", "host:111", "ai-provider:ollama_gcp_a", "ai-provider:ollama_gcp_b", "ai-provider:claude", "ai-provider:gemini"], "conflict": "Host110 is source-tombstoned but runtime absence is unverified; host111 is a LaunchAgent without local plus host120/121 same-run verification or a fresh sensor; GCP-A/GCP-B remain public-HTTP sanitized candidates with exact Prometheus targets missing; Claude/Gemini still lack durable Gate5 authorization, protected-secret metadata readback, verified cost caps, paired rollback and a completed five-lane canary.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-004", "AIA-SRE-008", "AIA-SRE-013", "AIA-SRE-014"]}, + {"id": "PORT-CONFLICT-003", "priority": "P0", "assets": ["service:gitea-exporter:host110", "service:sentry:relay", "service:sentry:snuba", "runner:gitea:vibework-host110"], "conflict": "GitHub exporter source retirement is ready, but the exact gitea-native production target is missing and the legacy exporter remains visible; remaining ghcr/runner claims also conflict with the global GitHub freeze.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-004", "AIA-SRE-014", "AIA-SRE-018"]}, + {"id": "PORT-CONFLICT-004", "priority": "P0", "assets": ["network:keepalived:vip-unresolved"], "conflict": "Inventory VIP 192.168.0.125 conflicts with Ansible VIP 192.168.0.200 and host membership differs.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-003"]}, + {"id": "PORT-CONFLICT-005", "priority": "P0", "assets": ["service:signoz", "service:otel-collector:signoz", "database:signoz:clickhouse"], "conflict": "SignOz UI, collector, and ClickHouse host placement is inconsistent across imported inventory and registries.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-014"]}, + {"id": "PORT-CONFLICT-006", "priority": "P0", "assets": ["database:awoooi:prod", "database:k3s:postgres-recovery"], "conflict": "AWOOOI business database and K3s datastore/recovery identities are conflated; multiple same-host port 5432 claims are not externally disambiguated.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-010", "AIA-SRE-011"]}, + {"id": "PORT-CONFLICT-007", "priority": "P0", "assets": ["control-plane:telegram-routing-registry"], "conflict": "Most product Telegram routes are blocked, disabled, or not implemented; shared-war-room fallback is forbidden for raw product monitoring.", "linked_work_items": ["AIA-SRE-017"]}, + {"id": "PORT-CONFLICT-008", "priority": "P0", "assets": ["control-plane:km-rag", "control-plane:mcp-gateway", "control-plane:playbook-registry", "control-plane:independent-verifier-registry"], "conflict": "Imported inventory omits the learning and verifier control planes required to make AI automation complete.", "linked_work_items": ["AIA-SRE-014", "AIA-SRE-015"]}, + {"id": "PORT-CONFLICT-009", "priority": "P0", "assets": ["windows-vmware:host_99", "host:111", "k8s:awoooi-prod:api", "k8s:awoooi-prod:web", "k8s:awoooi-prod:worker", "k8s:awoooi-prod:broker"], "conflict": "Core execution and production workload assets are absent from the imported inventory.", "linked_work_items": ["AIA-SRE-006", "AIA-SRE-007", "AIA-SRE-009", "AIA-SRE-013"]}, + {"id": "PORT-CONFLICT-010", "priority": "P1", "assets": ["all-imported-assets"], "conflict": "All imported healthy/up labels are uncorrelated point-in-time claims and cannot close runtime work items.", "linked_work_items": ["AIA-SRE-014", "AIA-SRE-016", "AIA-SRE-018"]}, + {"id": "PORT-CONFLICT-011", "priority": "P0", "assets": ["service:wg-easy:host112", "package:metasploit:host112", "package:burp-suite:host112", "package:sqlmap:host112", "package:nikto:host112", "package:openvas:host112", "service:fail2ban:host188"], "conflict": "The expanded host112 inventory adds a forbidden ghcr image claim and multiple active-scan tools without authorization/receipt contracts, while Fail2Ban disappeared without a retirement receipt.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-003", "AIA-SRE-008", "AIA-SRE-014", "AIA-SRE-016"]} + ], + "priority_policy": { + "current_p0_unchanged": "AIA-SRE-013", + "P0": "identity, execution safety, active critical alert chain, provider route, backup integrity, monitoring and learning closure", + "P1": "portfolio manifest adoption and 24-hour effectiveness scorecard after P0 control paths are safe", + "ordering_rule": "all P0 reconciliation work precedes P1; declared dependencies are mandatory and an external blocker may not stop unrelated ready P0 work" + }, + "reconciliation_work_items": [ + {"id": "PORT-001", "order": 1, "priority": "P0", "title": "Canonical portfolio asset normalization and exact identity graph", "dependencies": [], "linked_program_items": ["AIA-SRE-001", "AIA-SRE-002", "AIA-SRE-003"], "terminal_condition": "Every imported and omitted asset has exact owner, runtime identity, source truth, and fail-closed drift receipt."}, + {"id": "PORT-002", "order": 2, "priority": "P0", "title": "Alertmanager webhook chain same-run diagnosis, bounded repair, verifier, and closure", "dependencies": ["PORT-001"], "linked_program_items": ["AIA-SRE-004", "AIA-SRE-014", "AIA-SRE-015", "AIA-SRE-017"], "terminal_condition": "Alertmanager receiver through API ingest and Telegram durable receipt closes under one run; host99 Agent99 exact-host read-only polling creates the reduced repair event when the primary webhook is fully broken, without a first-hop credential or raw-payload persistence; repeated unchanged alerts are suppressed without hiding recurrence."}, + {"id": "PORT-003", "order": 3, "priority": "P0", "title": "Ollama provider identity and retired transport cleanup", "dependencies": ["PORT-001"], "linked_program_items": ["AIA-SRE-002", "AIA-SRE-004", "AIA-SRE-008", "AIA-SRE-013", "AIA-SRE-014"], "terminal_condition": "Provider readback shows GCP-A, GCP-B, host111, Claude, Gemini in order; Host110 is only a verified retired tombstone; host111 LaunchAgent closes local and host120/121 verification with a fresh sensor; GCP public-HTTP hops remain sanitized candidate-only and tool loops fail closed."}, + {"id": "PORT-004", "order": 4, "priority": "P0", "title": "Sunset generic auto-repair and GitHub legacy supply/runtime paths", "dependencies": ["PORT-001"], "linked_program_items": ["AIA-SRE-002", "AIA-SRE-004", "AIA-SRE-014", "AIA-SRE-018"], "terminal_condition": "No generic SSH/docker/delete/cross-provider fallback or GitHub source/image/API path can execute; gitea-native exact metrics are fresh; the legacy GitHub exporter is absent; exact typed executors and internal immutable artifacts are verified."}, + {"id": "PORT-005", "order": 5, "priority": "P0", "title": "Register host99, host111, K3s nodes, host112 security tools/schedules and control-plane assets", "dependencies": ["PORT-001", "PORT-004"], "linked_program_items": ["AIA-SRE-006", "AIA-SRE-007", "AIA-SRE-008", "AIA-SRE-009", "AIA-SRE-014", "AIA-SRE-016"], "terminal_condition": "Host and control-plane inventories reconcile with Agent99/Ansible/Kubernetes single-executor routes; host112 services, containers, packages and schedules have no-secret inventory, active-scan break-glass boundaries and independent readbacks."}, + {"id": "PORT-006", "order": 6, "priority": "P0", "title": "Canonical per-product Telegram destinations and concise AI-work attribution", "dependencies": ["PORT-001", "PORT-002"], "linked_program_items": ["AIA-SRE-017"], "terminal_condition": "Every emitted alert has exact product route, decision engine/models/actions/executor/verifier/receipt and no raw product alert leaks into the shared war room."}, + {"id": "PORT-007", "order": 7, "priority": "P0", "title": "Portfolio backup/restore, escrow, freshness and isolated drill coverage", "dependencies": ["PORT-001", "PORT-004"], "linked_program_items": ["AIA-SRE-010", "AIA-SRE-011"], "terminal_condition": "Every stateful asset has readback freshness, escrow/offsite state and an isolated restore verifier; writes remain critical break-glass."}, + {"id": "PORT-009", "order": 8, "priority": "P0", "title": "Resolve VIP, SignOz, database, port and service placement conflicts", "dependencies": ["PORT-001", "PORT-005"], "linked_program_items": ["AIA-SRE-002", "AIA-SRE-003", "AIA-SRE-010", "AIA-SRE-014"], "terminal_condition": "All identity conflicts have read-only live evidence, canonical resolution, source update, and drift verifier without blind mutation."}, + {"id": "PORT-010", "order": 9, "priority": "P0", "title": "Generated monitoring/alert coverage and source-to-runtime parity", "dependencies": ["PORT-001", "PORT-004", "PORT-009"], "linked_program_items": ["AIA-SRE-002", "AIA-SRE-013", "AIA-SRE-014", "AIA-SRE-016", "AIA-SRE-017", "AIA-SRE-018"], "terminal_condition": "Every canonical asset has fresh metrics/logs/traces/alerts, exact owner, generated target parity, and a tested lifecycle route; source targets alone never count as runtime coverage; GCP-A/GCP-B, host111 and gitea-native exact series are production-visible."}, + {"id": "PORT-011", "order": 10, "priority": "P0", "title": "KM/RAG/MCP/PlayBook and verifier target attachment for every asset", "dependencies": ["PORT-001", "PORT-004", "PORT-007", "PORT-010"], "linked_program_items": ["AIA-SRE-012", "AIA-SRE-014", "AIA-SRE-015"], "terminal_condition": "Every verified run writes durable acknowledgements to all learning targets and never marks learning complete from metadata-only refs."}, + {"id": "PORT-008", "order": 11, "priority": "P1", "title": "Adopt product manifests for every imported project/site", "dependencies": ["PORT-001", "PORT-004"], "linked_program_items": ["AIA-SRE-002", "AIA-SRE-015"], "terminal_condition": "Every product/site has Gitea authority, runtime, contracts, operation boundaries and owner without creating repos or using GitHub."}, + {"id": "PORT-012", "order": 12, "priority": "P1", "title": "24-hour replay, cost, recurrence and production closure scorecard", "dependencies": ["PORT-002", "PORT-003", "PORT-005", "PORT-006", "PORT-007", "PORT-008", "PORT-009", "PORT-010", "PORT-011"], "linked_program_items": ["AIA-SRE-013", "AIA-SRE-016", "AIA-SRE-018"], "terminal_condition": "Production same-run closure publishes MTTA, MTTR, false positive, recurrence, human intervention, verifier, rollback, freshness, coverage and paid-provider cost without false green; paid evidence includes durable Gate5 authorization, protected-secret metadata only, daily/monthly/per-incident caps and paired Claude/Gemini rollback on any partial result."} + ], + "rollups": { + "imported_item_rows": 131, + "canonical_assets": 144, + "inventory_omissions": 18, + "conflicts": 11, + "conflict_assets": 36, + "reconciliation_work_items": 12, + "source_reconciled_assets": 6, + "runtime_closed_assets": 0, + "program_completion_percent": 0, + "asset_coverage_status": "all_imported_rows_mapped_but_identity_and_runtime_coverage_partial", + "runtime_closure_status": "not_started_for_this_reconciliation" + } +} diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index c2124732a..a3578b7a7 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -2,14 +2,14 @@ "schema_version": "sre_k3s_controlled_automation_work_items_v1", "governance_version": "global_product_governance_v2", "program_id": "AIA-SRE-P0-20260715", - "generated_at": "2026-07-15T16:15:01+08:00", - "status": "phase_1_canonical_identity_and_typed_routing_in_progress", + "generated_at": "2026-07-16T15:18:00+08:00", + "status": "phase_3_paid_provider_bounded_canary_in_progress", "scope_complete": false, "current_p0": { - "id": "AIA-SRE-004", - "title": "Canonical Asset Normalize 與 Typed Domain Router production closure", - "phase": "phase_1", - "terminal_condition": "同一 source SHA 通過 focused/full tests、Gitea CD、production registry parity、typed route replay 與 Sentry exact-container verifier readback" + "id": "AIA-SRE-013", + "title": "Claude/Gemini protected-secret 受控上線與五路 SRE 效果驗證", + "phase": "phase_3", + "terminal_condition": "同一 source SHA 通過 focused/full tests 與 Gitea CD；durable Gate5 authorization receipt 綁定同一 run；protected-secret metadata、GCP-A/GCP-B/host111/Claude/Gemini 同 fixture、Claude/Gemini finalized receipt、latency/token/cost、paired atomic rollback 與 production status readback 全部可驗證；不代表尚未完成的 HolmesGPT 主 pipeline promotion" }, "architecture": { "pipeline": [ @@ -17,7 +17,7 @@ "Canonical Asset Normalize", "Typed Domain Router", "HolmesGPT Investigator", - "Ollama RCA / Claude Fallback / Gemini Critic", + "Ollama RCA / Gemini Critic", "Deterministic Policy", "Single Controlled Executor", "Independent Verifier", @@ -30,7 +30,8 @@ "trace_id", "run_id", "work_item_id" - ] + ], + "provider_failover_is_not_a_pipeline_stage": true }, "provider_policy": { "ordered_route": [ @@ -71,12 +72,70 @@ "ollama_role": "primary_rca_and_action_planning", "claude_role": "paid_architecture_rca_and_debug_fallback", "gemini_role": "final_fallback_and_optional_critic_only", - "claude_paid_call_allowed": false, - "gemini_paid_call_allowed": false, - "claude_cost_cap_status": "source_configured_shadow_runtime_readback_pending", - "gemini_cost_cap_status": "source_configured_runtime_readback_pending", + "claude_paid_call_allowed": true, + "gemini_paid_call_allowed": true, + "paid_execution_mode": "canary", + "paid_canary_percent": 5, + "paid_canary_authorization_ref": "AIA-SRE-013-user-approved-20260716", + "paid_canary_authorization_status": "user_direction_received_durable_gate5_receipt_pending", + "durable_gate5_authorization": { + "required": true, + "binding": [ + "trace_id", + "run_id", + "work_item_id", + "provider_pair", + "cost_caps", + "expiry" + ], + "accepted_source": "durable_operation_log_or_authorization_receipt", + "free_form_authorization_ref_is_sufficient": false, + "status": "pending" + }, + "paid_canary_scope": "one committed sanitized fixture per provider; no infrastructure remediation", + "claude_cost_cap_status": "source_configured_canary_runtime_readback_pending", + "gemini_cost_cap_status": "source_configured_canary_runtime_readback_pending", "claude_credential_status": "runtime_secret_reference_verifier_pending_no_secret_value_read", "gemini_credential_status": "runtime_secret_reference_verifier_pending_no_secret_value_read", + "credential_metadata": { + "claude": { + "user_supplied_credential_noted": true, + "raw_value_recorded_in_repository": false, + "raw_value_readback_allowed": false, + "protected_secret_reference_status": "pending_secure_ingress_and_metadata_verifier" + }, + "gemini": { + "user_supplied_credential_noted": true, + "raw_value_recorded_in_repository": false, + "raw_value_readback_allowed": false, + "protected_secret_reference_status": "pending_secure_ingress_and_metadata_verifier" + } + }, + "cloud_transport_contract": { + "ollama_gcp_a": { + "observed_transport": "public_http", + "execution_scope": "sanitized_candidate_only", + "tool_loop_terminal": "fail_closed", + "pre_generation_receipt": "same_run_api_tags_probe_plus_redis_exact_readback_source_ready_runtime_pending", + "exact_prometheus_target_runtime_status": "missing" + }, + "ollama_gcp_b": { + "observed_transport": "public_http", + "execution_scope": "sanitized_candidate_only", + "tool_loop_terminal": "fail_closed", + "pre_generation_receipt": "same_run_api_tags_probe_plus_redis_exact_readback_source_ready_runtime_pending", + "exact_prometheus_target_runtime_status": "missing" + }, + "production_sensitive_payload_allowed": false, + "secure_mesh_or_tls_required_for_promotion": true + }, + "paid_pair_contract": { + "enable_and_rollback_as_pair": true, + "partial_five_lane_terminal": "rollback_both_paid_providers_and_fail_canary", + "daily_monthly_and_per_incident_cost_caps_required": true, + "source_status": "same_run_gate5_atomic_single_use_claim_and_aggregate_cap_ready", + "runtime_status": "paid_canary_pending" + }, "production_provider_route_switch_allowed": false, "required_before_paid_provider_enablement": [ "offline_replay_scorecard", @@ -153,6 +212,57 @@ "verifier": null, "terminal": "asset_identity_unresolved", "fallback": "forbidden" + }, + { + "domain": "control_plane_recovery", + "executor": "Agent99", + "verifier": "cold_start_independent_scorecard_verifier", + "fallback": "forbidden", + "exact_target_required": true + } + ], + "immediate_execution_queue": [ + { + "position": 1, + "work_item_id": "AIA-SRE-017", + "scope": "alert_chain_delivery_truth_and_emergency_card_attribution", + "blocking_reason": "firing AlertChainBroken invalidates downstream automation and verifier trust", + "terminal_condition": "Alertmanager monotonic delivery metrics are scraped; Agent99 on host99 independently polls the exact host110 active/critical AlertChain signal without a first-hop credential or raw-payload persistence and relays only the reduced envelope over HTTPS; the false NodePort-counter alert resolves; emergency cards state AI/Agent/executor/verifier truth" + }, + { + "position": 2, + "work_item_id": "AIA-SRE-002", + "scope": "retire_host110_ollama_identity_and_reconcile_portfolio_inventory", + "blocking_reason": "stale provider identity produces false failover alerts and corrupts provider comparison", + "terminal_condition": "Host110 Ollama is a canonical retired tombstone; GCP-A, GCP-B and host111 are direct identities; host111 LaunchAgent is independently verified from host120/121; exact provider sensors are fresh; no host110 Ollama endpoint remains in runtime or monitoring" + }, + { + "position": 3, + "work_item_id": "AIA-SRE-004", + "scope": "critical_fail_closed_and_exact_alert_chain_domain_route", + "blocking_reason": "critical or cross-domain candidates must not reach an executor through legacy fallback", + "terminal_condition": "critical override stays no-write and AlertChainBroken resolves only to host110 Alertmanager container lane" + }, + { + "position": 4, + "work_item_id": "AIA-SRE-013", + "scope": "five_lane_paid_provider_canary", + "blocking_reason": "paid calls are allowed only after positions 1-3 have independent readback", + "terminal_condition": "all five providers return comparable safe receipts; public-cloud prompts remain sanitized and tool loops fail closed; the Gate5 authorization UUID is the exact execution run/trace, has an atomic one-use claim and canary bucket proof; Claude/Gemini use a run-owned lease, paired rollback and verified aggregate cost caps" + }, + { + "position": 5, + "work_item_id": "AIA-SRE-014", + "scope": "independent_verifier_coverage", + "blocking_reason": "no apply may close without a domain-specific verifier", + "terminal_condition": "every mutating catalog and provider lane has an independent terminal readback" + }, + { + "position": 6, + "work_item_id": "AIA-SRE-015", + "scope": "incident_and_learning_writeback", + "blocking_reason": "source, CD or Telegram visibility does not prove learning closure", + "terminal_condition": "same-run incident, KM, RAG, MCP and PlayBook acknowledgements are durable" } ], "work_items": [ @@ -190,14 +300,34 @@ ], "source_refs": [ "ops/config/service-registry.yaml", + "ops/monitoring/service-registry.yaml", + "ops/monitoring/generated/prometheus-scrape-generated.yaml", + "ops/monitoring/generated/blackbox-targets-generated.yaml", + "docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json", + "docs/operations/portfolio-infrastructure-asset-reconciliation-handoff.md", + "apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py", + "apps/api/src/api/v1/agents.py", + "apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py", "scripts/ops/render-service-registry-configmap.py", "k8s/awoooi-prod/15-service-registry-configmap.yaml" ], "executor": "service registry renderer plus Gitea CD", "verifier": "source hash and exact embedded YAML parity", "rollback": "reapply previous ConfigMap from prior Gitea SHA", - "exit_condition": "production ConfigMap source hash and API identity lookup match Gitea main", - "next_action": "deploy and read back ConfigMap annotation" + "exit_condition": "production ConfigMap, monitoring inventory and portfolio asset graph match live canonical identities; Host110 Ollama is present only as a retired tombstone; GCP-A/GCP-B exact targets and the host111 LaunchAgent sensor are fresh; inventory snapshots are never treated as runtime health", + "confirmed_truth": [ + "Host110 Ollama has been removed and must remain a retired tombstone, never a fallback endpoint", + "Host111 is the third provider hop and runs as a macOS LaunchAgent, not systemd", + "GCP-A and GCP-B source targets are public HTTP sanitized candidates only", + "gitea-native source target is not production runtime evidence while the legacy GitHub exporter remains visible" + ], + "runtime_gaps": [ + "Host110 Ollama absence has no same-run production tombstone verifier receipt", + "Host111 LaunchAgent lacks completed local plus host120/host121 origin readback and a fresh exact sensor series", + "GCP-A/GCP-B exact Prometheus target series are missing in production", + "gitea-native exact target is missing and legacy GitHub exporter runtime drift is unresolved" + ], + "next_action": "deploy the reconciled tombstone/targets, then verify Host110 absence, Host111 LaunchAgent from local and host120/121, GCP-A/GCP-B exact Prometheus series, gitea-native target freshness and legacy exporter absence under one source SHA" }, { "id": "AIA-SRE-003", @@ -237,13 +367,18 @@ "apps/api/src/services/controlled_alert_target_router.py", "apps/api/src/services/awooop_ansible_audit_service.py", "apps/api/src/services/awooop_ansible_check_mode_service.py", - "apps/api/src/api/v1/agents.py" + "apps/api/src/api/v1/agents.py", + "ops/monitoring/alerts-unified.yml" ], "executor": "typed policy router", "verifier": "domain/catalog/host/canonical ID scope verifier", "rollback": "revert router source while keeping unknown fail-closed", - "exit_condition": "every candidate is exact-domain scoped and circuit-open never selects another host/domain", - "next_action": "run full Ansible replay regression" + "exit_condition": "every candidate is exact-domain scoped; circuit-open never selects another host/domain; GCP-A/GCP-B unsanitized prompts and every cloud tool loop fail closed before a network call", + "runtime_gaps": [ + "source tests do not prove production router enforcement", + "Agent99 exact-host Alertmanager pull and reduced HTTPS relay are source-implemented but have no host99 production receipt or primary-path outage verifier" + ], + "next_action": "run critical override, AlertChainBroken exact-host, cloud sanitized-input/tool-loop fail-closed and full Ansible replay regressions; then verify the same policy in production" }, { "id": "AIA-SRE-005", @@ -332,8 +467,12 @@ "executor": "host_ansible_executor", "verifier": "host_runtime_independent_verifier", "rollback": "catalog-specific compensating action", - "exit_condition": "every host action has one exact inventory host, check mode, bounded apply and postcondition", - "next_action": "close catalogs still backed by broad convergence playbooks" + "exit_condition": "every host action has one exact inventory host, runtime manager identity, check mode, bounded apply and postcondition; host111 uses its LaunchAgent catalog and is verified locally plus from host120/host121", + "runtime_gaps": [ + "host111 LaunchAgent catalog and sensor are source candidates without production apply receipt", + "host120 and host121 independent origin probes have no same-run production verifier receipt" + ], + "next_action": "close broad catalogs, then check/apply the exact host111 LaunchAgent playbook and verify local, host120 and host121 readbacks plus the canonical sensor rule" }, { "id": "AIA-SRE-009", @@ -429,7 +568,7 @@ "order": 13, "priority": "P0", "phase": "phase_3", - "title": "Ollama RCA、Claude fallback 與 Gemini Critic/cost gate", + "title": "GCP-A/GCP-B/host111 Ollama、Claude、Gemini RCA/critic 與付費 canary/cost gate", "owner_lane": "ModelRouter", "risk": "critical", "status": "source_implemented_runtime_pending", @@ -440,13 +579,35 @@ "apps/api/src/services/ai_router.py", "apps/api/src/services/ollama_failover_manager.py", "apps/api/src/services/ai_providers/claude.py", - "apps/api/src/services/ai_provider_policy.py" + "apps/api/src/services/ai_providers/gemini.py", + "apps/api/src/services/ai_provider_policy.py", + "apps/api/src/services/failover_alerter.py", + "apps/api/src/services/cloud_transport_receipt.py", + "apps/api/src/services/paid_provider_canary_authorization.py", + "apps/api/src/services/paid_provider_canary_gate5_run.py", + "apps/api/src/services/paid_provider_canary_validation.py", + "apps/api/src/services/ai_rate_limiter.py", + "scripts/ops/run-paid-provider-canary.py", + "k8s/awoooi-prod/04-configmap.yaml", + "k8s/awoooi-prod/06-deployment-api.yaml" ], "executor": "ordered model router", "verifier": "provider-attributed replay accuracy, latency and cost readback", - "rollback": "Claude/Gemini disabled; remain on ordered Ollama chain", - "exit_condition": "provider order is enforced and paid providers cannot run without explicit cost cap", - "next_action": "deploy the disabled-by-default source route, verify secret metadata without reading values, then run replay, shadow comparison and bounded canary" + "rollback": "Claude/Gemini are disabled together on any partial five-lane result; remain on the ordered Ollama chain", + "exit_condition": "provider order is enforced; GCP public-HTTP hops remain sanitized candidate-only with cloud tool loops fail-closed until secure transport promotion; paid providers remain 5 percent canary only; the durable Gate5 UUID is the same execution run/trace and is atomically single-use; protected-secret metadata, paired rollback, aggregate cost cap, finalized receipts, latency/token/cost and production readback are verified without secret-value exposure", + "confirmed_truth": [ + "The user supplied Claude and Gemini credential material, but raw values are not repository data and must not be copied or read back", + "GCP-A/GCP-B are currently public HTTP boundaries and cannot receive unsanitized or tool-loop payloads", + "Claude and Gemini enablement, canary outcome and rollback form one atomic pair" + ], + "runtime_gaps": [ + "durable Gate5 authorization receipt bound to the canary run is missing", + "Claude/Gemini protected-secret reference metadata verification is pending", + "daily/monthly/per-incident cost-cap and production cost readback are pending", + "GCP-A/GCP-B exact Prometheus targets are absent from production", + "sanitized five-lane paid canary has not run and no paired rollback receipt exists" + ], + "next_action": "first close the ordered alert-chain, Host110 tombstone/Host111 verifier and critical fail-closed safety queue; then create the durable Gate5 receipt, verify protected-secret metadata without reading values, deploy cost caps, execute one sanitized five-lane fixture, and roll back both paid providers unless all five lanes are comparable" }, { "id": "AIA-SRE-014", @@ -471,8 +632,12 @@ "executor": "none", "verifier": "domain-specific external readback", "rollback": "failed verifier triggers same-domain retry or compensating action", - "exit_condition": "100 percent of mutating catalogs have independent postconditions", - "next_action": "reconcile catalog IDs against postcondition registry" + "exit_condition": "100 percent of mutating catalogs have independent production postconditions; host111 includes local plus host120/host121 origin verification and provider monitoring requires exact fresh series", + "runtime_gaps": [ + "source-level verifier definitions are not production verifier receipts", + "Host111 LaunchAgent, GCP provider targets, gitea-native metrics and the host99 Alertmanager pull/relay remain without independent runtime closure" + ], + "next_action": "reconcile catalog IDs against postcondition registry, then attach same-run runtime receipts for Host111 origins, GCP exact targets, gitea-native metrics and the host99 Alertmanager pull/relay" }, { "id": "AIA-SRE-015", @@ -528,13 +693,26 @@ "AIA-SRE-015" ], "source_refs": [ - "ops/config/telegram-routing-registry.yaml" + "ops/config/telegram-routing-registry.yaml", + "ops/alertmanager/alertmanager.yml", + "k8s/monitoring/prometheus.yml", + "ops/monitoring/alerts-unified.yml", + "scripts/ops/deploy-alertmanager-runtime-scrape.sh", + "agent99-alertmanager-alertchain-poll.ps1", + "agent99-sre-alert-relay.ps1", + "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh", + "apps/api/src/services/failover_alerter.py", + "apps/api/src/services/telegram_gateway.py" ], "executor": "canonical Telegram gateway", "verifier": "delivery receipt plus desktop/mobile visible smoke", "rollback": "suppress duplicate projection; retain incident ledger", - "exit_condition": "no alert is sprayed across groups/bots and every card links one canonical run", - "next_action": "reconcile project/product/alert class routing against the typed domain" + "exit_condition": "no alert is sprayed across groups/bots; every card states AI/provider/Agent action, executor, verifier and receipt truth; AlertChainBroken uses receiver-scoped delivery counters plus host99 Agent99 exact-host read-only polling independent of the broken webhook; UI status polling never emits failover alerts", + "runtime_gaps": [ + "the host99 Agent99 independent pull/reduced relay has no production deployment, freshness, dedupe or controlled-repair receipt", + "the proposed rule/scrape/card path has no production alert-resolution or same-run learning receipt" + ], + "next_action": "deploy Alertmanager runtime scrape/canonical rules and the host99 Agent99 exact-host poller through the atomic 15-file bundle; verify receiver-scoped freshness, dedupe and the current firing alert resolution; then reconcile every project/product/alert class against its typed domain and canonical Telegram destination" }, { "id": "AIA-SRE-018", @@ -556,13 +734,19 @@ "AIA-SRE-015" ], "source_refs": [ - ".gitea/workflows/cd.yaml" + ".gitea/workflows/cd.yaml", + "docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json" ], "executor": "Gitea controlled CD", "verifier": "deploy marker, image SHA, API readback and desktop/mobile smoke", "rollback": "previous immutable image revision and ConfigMap source SHA", - "exit_condition": "source/test/CD/runtime/UI evidence all match one deployed SHA", - "next_action": "commit phase 1, normal push after shared writer release, then terminal CD/readback" + "exit_condition": "source/test/CD/runtime/UI evidence all match one deployed SHA; gitea-native exact metrics are fresh and the retired legacy GitHub exporter is absent from production", + "runtime_gaps": [ + "no program item is runtime closed for this source SHA", + "gitea-native exact target is missing while legacy GitHub exporter runtime drift remains", + "source/test readiness, CD terminal, production runtime and visible UI evidence are separate incomplete layers" + ], + "next_action": "commit phase 1, normal push after shared writer release, then terminal CD plus production API/runtime/UI readback and exact gitea-native/legacy-exporter verification" } ], "rollups": { @@ -598,6 +782,14 @@ "km_rag_mcp_playbook_writeback_ack" ], "terminal_without_all_stages": "partial_or_degraded_with_safe_next_action", - "source_test_cd_green_is_runtime_closure": false + "source_test_cd_green_is_runtime_closure": false, + "evidence_layers": { + "source_test": "partial_source_evidence_only", + "gitea_cd_deploy_marker": "pending_for_this_program_sha", + "production_runtime": "zero_work_items_closed", + "visible_ui_telegram": "pending_same_run_receipts", + "learning_writeback": "pending_same_run_durable_acknowledgements" + }, + "promotion_rule": "a lower evidence layer cannot promote an incomplete higher evidence layer" } } diff --git a/docs/runbooks/ANSIBLE-OPERATING-MODEL.md b/docs/runbooks/ANSIBLE-OPERATING-MODEL.md index 7c9365c59..b8ca16774 100644 --- a/docs/runbooks/ANSIBLE-OPERATING-MODEL.md +++ b/docs/runbooks/ANSIBLE-OPERATING-MODEL.md @@ -32,7 +32,7 @@ Git repo |---|---|---| | 主機 inventory | `infra/ansible/inventory/hosts.yml` | 記錄 110 / 120 / 121 / 188 / 112 | | 188 public nginx routes | `infra/ansible/roles/nginx/templates/*` + `playbooks/nginx-sync.yml` | `/etc/nginx/sites-enabled/*` | -| 110 Ollama proxy | `110-ollama-proxy.conf.j2` | `/etc/nginx/sites-enabled/110-ollama-proxy.conf` | +| 111 Ollama local boundary | `playbooks/111-ollama-fallback.yml` | K3s 120/121 allowlist + direct `/api/tags` readback；host110 不在路徑 | | 110 cold-start monitor | `roles/cold-start-monitor` | `/home/wooo/scripts`、cron、node-exporter textfile | | 110 runner guardrails | `roles/runner-guardrails` | `actions.runner.*` systemd drop-ins | | 110/188 Docker/systemd/storage/backup/runaway-process textfile exporters | `roles/host-textfile-exporters` | `/home/*/node_exporter_textfiles/docker_stats.prom`、`storage_health.prom`、`backup_health.prom`、110 `systemd_units.prom`、110 `host_runaway_process.prom` | diff --git a/docs/runbooks/AWOOOP-INFERENCE-GATEWAY.md b/docs/runbooks/AWOOOP-INFERENCE-GATEWAY.md index 3a92711de..d96eed90c 100644 --- a/docs/runbooks/AWOOOP-INFERENCE-GATEWAY.md +++ b/docs/runbooks/AWOOOP-INFERENCE-GATEWAY.md @@ -144,8 +144,8 @@ Every request must emit: Set provider env back to raw endpoints: ```yaml -OLLAMA_URL: "http://192.168.0.110:11435" -OLLAMA_SECONDARY_URL: "http://192.168.0.110:11436" +OLLAMA_URL: "http://34.143.170.20:11434" +OLLAMA_SECONDARY_URL: "http://34.21.145.224:11434" OLLAMA_FALLBACK_URL: "http://192.168.0.111:11434" ``` diff --git a/docs/runbooks/DEPLOY-GCP-OLLAMA-PROXY.md b/docs/runbooks/DEPLOY-GCP-OLLAMA-PROXY.md index d9e8f5310..a29a98adb 100644 --- a/docs/runbooks/DEPLOY-GCP-OLLAMA-PROXY.md +++ b/docs/runbooks/DEPLOY-GCP-OLLAMA-PROXY.md @@ -1,225 +1,13 @@ -# GCP Ollama Nginx Proxy 部署指南 +# Host110 GCP Ollama Proxy（已退役） -> ADR-110 三層容災 — 啟用 GCP Ollama 的關鍵步驟 -> -> 2026-05-05 修正：此 runbook 只保留為過渡 / rollback bridge。正式方案改用 -> ADR-125 的 WireGuard private mesh 與 AwoooP Inference Gateway。新部署不得把 -> GCP `11434/tcp` 對 `0.0.0.0/0` 長期開放。 +`superseded_by=global_product_governance_v2` ---- +- owner：AssetIdentity +- expiry：2026-07-16 +- replacement：GCP-A → GCP-B → host111 → Claude API → Gemini API +- verifier：`apps/api/tests/test_ollama_prod_manifest_order.py` -## 背景 - -GCP Ollama (34.143.170.20 / 34.21.145.224) 已部署完成。 -最初透過 192.168.0.110 (DevOps 金庫) 架設 nginx 反向代理，讓 K3s Pod 走內網連線 GCP Ollama。 - -2026-05-06 現況：K3s Pod 已可直連 GCP-A/GCP-B `11434/tcp`，且 production -暫時改用 direct endpoint，避開 110 上舊 `conf.d/ollama-gcp-proxy.conf` -的 120s `proxy_read_timeout`。正式長期方案仍是 ADR-125 WireGuard mesh。 - ---- - -## 部署檔案 - -| 檔案 | 用途 | -|-----|------| -| `infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2` | nginx 配置模板 | -| `infra/ansible/playbooks/nginx-sync.yml` | Ansible Playbook | - ---- - -## 執行部署 - -```bash -# 1. 進入 Ansible 目錄 -cd /Users/ogt/awoooi/infra/ansible - -# 2. 部署到 110 (Dry-run 先驗證) -ansible-playbook -i inventory/hosts.yml playbooks/nginx-sync.yml --tags 110 --check - -# 3. 正式部署 -ansible-playbook -i inventory/hosts.yml playbooks/nginx-sync.yml --tags 110 -``` - ---- - -## 驗證部署 - -### 從 110 本機驗證 - -```bash -# 測試 GCP-A proxy -curl http://127.0.0.1:11435/api/tags - -# 測試 GCP-B proxy -curl http://127.0.0.1:11436/api/tags -``` - -### 從 K3s Node 驗證 - -```bash -# 進入 K3s node (120 或 121) -ssh wooo@192.168.0.120 - -# 測試連線 110 proxy -curl http://192.168.0.110:11435/api/tags -curl http://192.168.0.110:11436/api/tags -``` - -### 從 K8s Pod 驗證 - -```bash -# 進入 API Pod -kubectl exec -it -n awoooi-prod deployment/awoooi-api -- bash - -# 測試連線 -apt-get update && apt-get install -y curl -curl http://192.168.0.110:11435/api/tags -``` - ---- - -## 啟用 GCP Ollama - -代理部署完成後，修改 ConfigMap 啟用 GCP 端點： - -```bash -# 編輯 ConfigMap -kubectl edit configmap -n awoooi-prod awoooi-config -``` - -若使用 nginx bridge，修改以下欄位： - -```yaml -# 修改前 -OLLAMA_URL: "http://192.168.0.111:11434" -OLLAMA_SECONDARY_URL: "http://192.168.0.110:11435" -OLLAMA_FALLBACK_URL: "http://192.168.0.110:11436" - -# 修改後 (啟用 GCP-A 作為 Primary) -OLLAMA_URL: "http://192.168.0.110:11435" # GCP-A via proxy -OLLAMA_SECONDARY_URL: "http://192.168.0.110:11436" # GCP-B via proxy -OLLAMA_FALLBACK_URL: "http://192.168.0.111:11434" # Local GPU 最後防線 -``` - -重啟 Deployment： - -```bash -kubectl rollout restart deployment/awoooi-api -n awoooi-prod -``` - -若需要繞過 110 bridge timeout，使用 direct GCP endpoint： - -```yaml -OLLAMA_URL: "http://34.143.170.20:11434" -OLLAMA_SECONDARY_URL: "http://34.21.145.224:11434" -OLLAMA_FALLBACK_URL: "http://192.168.0.111:11434" -OLLAMA_DIAGNOSE_TIMEOUT_SECONDS: "300" -INCIDENT_LLM_TIMEOUT_SECONDS: "360" -``` - ---- - -## 確認模型已載入 - -GCP Ollama 必須已載入以下模型： - -```bash -# GCP-A 檢查 -curl http://34.143.170.20:11434/api/tags | jq '.models[].name' - -# 必須包含: -# - bge-m3:latest (embedding) -# - qwen2.5:7b-instruct (health check) -# - qwen3:14b (RCA analysis) -# - hermes3:latest (tool calling) -# - deepseek-r1:14b (reasoning) -``` - -若模型未載入，SSH 到 GCP 主機執行： - -```bash -ollama pull bge-m3:latest -ollama pull qwen2.5:7b-instruct -ollama pull qwen3:14b -ollama pull hermes3:latest -ollama pull deepseek-r1:14b -``` - ---- - -## 部署檢查清單 - -- [ ] Ansible playbook 執行成功 (110) -- [ ] 110:11435 監聽確認 (`ss -tlnp | grep 11435`) -- [ ] 110:11436 監聽確認 (`ss -tlnp | grep 11436`) -- [ ] K3s node 可連線 110:11435/11436 -- [ ] K8s Pod 可連線 110:11435/11436 -- [ ] GCP-A/B 模型已載入 -- [ ] ConfigMap 已修改 -- [ ] Deployment 已重啟 -- [ ] API Pod 啟動無錯誤 -- [ ] 推理測試成功 (檢查 latency < 10s) - ---- - -## 常見問題 - -### 1. K3s Pod 連線被拒絕 - -檢查 NetworkPolicy： -```bash -kubectl describe networkpolicy -n awoooi-prod allow-required-egress -``` - -確認包含： -```yaml -- to: - - ipBlock: - cidr: 192.168.0.110/32 - ports: - - protocol: TCP - port: 11435 - - protocol: TCP - port: 11436 -``` - -### 2. nginx 無法連線 GCP - -檢查 110 外網連線： -```bash -curl -v http://34.143.170.20:11434/api/tags -``` - -若失敗，只允許短時間確認 GCP 防火牆是否對 110 的固定出口 IP 開放 -`11434/tcp`。不得把 `0.0.0.0/0:11434` 當成正式設定。 - -正式切換請改走 [GCP-OLLAMA-WIREGUARD-MESH.md](GCP-OLLAMA-WIREGUARD-MESH.md)。 - -### 3. 模型載入但推理失敗 - -檢查 GCP VM 記憶體/CPU 使用率： -```bash -# GCP Console → Compute Engine → VM 執行個體 → 監控 -``` - -若記憶體不足，升級機型或減少同時載入模型數量。 - ---- - -## 相關文件 - -- ADR-110: GCP 三層容災架構 -- ADR-125: GCP Ollama Private Mesh and AwoooP Inference Gateway -- `k8s/awoooi-prod/04-configmap.yaml` -- `k8s/awoooi-prod/02-network-policy.yaml` -- `docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md` -- `docs/runbooks/GCP-OLLAMA-WIREGUARD-MESH.md` -- `docs/runbooks/AWOOOP-INFERENCE-GATEWAY.md` - ---- - -## 負責人 - -- 建立: Claude Sonnet 4.6 — 2026-05-04 -- 審查: 首席架構師 ogt +本文件保留原路徑只為阻止舊連結失效；沒有可執行部署手順。Host110 不再是 +Ollama provider、proxy 或 rollback target，任何工具都不得重建舊 Nginx 配置或 +listener。現行操作請使用 `docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md`；transport +目標請使用 `docs/runbooks/GCP-OLLAMA-WIREGUARD-MESH.md`。 diff --git a/docs/runbooks/GCP-OLLAMA-WIREGUARD-MESH.md b/docs/runbooks/GCP-OLLAMA-WIREGUARD-MESH.md index a2bce3323..29503a840 100644 --- a/docs/runbooks/GCP-OLLAMA-WIREGUARD-MESH.md +++ b/docs/runbooks/GCP-OLLAMA-WIREGUARD-MESH.md @@ -19,8 +19,8 @@ Current production endpoints: | Variable | Endpoint | Meaning | |----------|----------|---------| -| `OLLAMA_URL` | `http://192.168.0.110:11435` | GCP-A through 110 nginx | -| `OLLAMA_SECONDARY_URL` | `http://192.168.0.110:11436` | GCP-B through 110 nginx | +| `OLLAMA_URL` | GCP-A direct public HTTP | Temporary degraded stopgap | +| `OLLAMA_SECONDARY_URL` | GCP-B direct public HTTP | Temporary degraded stopgap | | `OLLAMA_FALLBACK_URL` | `http://192.168.0.111:11434` | Local 111 | This is a bridge. Do not treat the public proxy as the final architecture. @@ -29,7 +29,6 @@ This is a bridge. Do not treat the public proxy as the final architecture. | Host | WireGuard IP | Notes | |------|--------------|-------| -| 110 | `10.77.114.10` | DevOps host and rollback bridge | | 120 | `10.77.114.120` | K3s node | | 121 | `10.77.114.121` | K3s node | | 111 | `10.77.114.111` | Local Ollama fallback | @@ -213,8 +212,8 @@ After mesh cutover, allow only mesh endpoints for Ollama: port: 11434 ``` -Do not remove the `192.168.0.110:11435/11436` rules until rollback is no longer -needed. +Host110 proxy rules are retired and must never be restored. Keep the exact +direct GCP public egress only until mesh canary and rollback verification pass. ## Shadow Validation @@ -261,13 +260,13 @@ bash scripts/ops/ollama-topology-check.sh ```bash kubectl -n awoooi-prod set env deploy/awoooi-api \ - OLLAMA_URL=http://192.168.0.110:11435 \ - OLLAMA_SECONDARY_URL=http://192.168.0.110:11436 \ + OLLAMA_URL=http://34.143.170.20:11434 \ + OLLAMA_SECONDARY_URL=http://34.21.145.224:11434 \ OLLAMA_FALLBACK_URL=http://192.168.0.111:11434 kubectl -n awoooi-prod set env deploy/awoooi-worker \ - OLLAMA_URL=http://192.168.0.110:11435 \ - OLLAMA_SECONDARY_URL=http://192.168.0.110:11436 \ + OLLAMA_URL=http://34.143.170.20:11434 \ + OLLAMA_SECONDARY_URL=http://34.21.145.224:11434 \ OLLAMA_FALLBACK_URL=http://192.168.0.111:11434 ``` @@ -277,4 +276,3 @@ kubectl -n awoooi-prod set env deploy/awoooi-worker \ - Alert lane Gemini usage is zero except documented all-Ollama outages. - Public GCP `11434/tcp` is closed. - Operator runbook records peer public keys and rollback owner. - diff --git a/docs/runbooks/OLLAMA-188-RETIREMENT-GATE.md b/docs/runbooks/OLLAMA-188-RETIREMENT-GATE.md index 324e5153d..f34f7094c 100644 --- a/docs/runbooks/OLLAMA-188-RETIREMENT-GATE.md +++ b/docs/runbooks/OLLAMA-188-RETIREMENT-GATE.md @@ -4,10 +4,14 @@ `192.168.0.188:11434` 已不再是 AWOOOI / AwoooP 的 Ollama provider。正式順序是： -1. GCP-A：`192.168.0.110:11435` -2. GCP-B：`192.168.0.110:11436` -3. Local 111：`192.168.0.110:11437` -4. Gemini / Claude：雲端備援，不是主路徑 +1. GCP-A：canonical direct GCP Ollama（cloud boundary） +2. GCP-B：canonical direct GCP Ollama（cloud boundary） +3. Local 111：`192.168.0.111:11434`（local boundary） +4. Claude API：付費 canary／fallback +5. Gemini API：最後付費備援 + +Host110 provider/proxy 已由 `global_product_governance_v2` 退役；不得作為 +rollback target。GCP direct public HTTP 仍是 mesh 上線前的 degraded stopgap。 188 主機仍承載 PostgreSQL、Redis、SigNoz、OpenClaw 等服務，所以不能把「解除 188 Ollama」誤解成「動 188 主機」。退場只針對 `ollama.service` 與 `:11434`。 @@ -105,7 +109,7 @@ curl -sS --max-time 3 http://192.168.0.188:11434/api/tags || echo LAN_CLOSED ## 今日發現 - `awoooi-prod` provider registry 已沒有 `ollama_188`。 -- `awoooi-dev` 原本仍使用 `OLLAMA_URL=http://192.168.0.188:11434`，已 live patch 到 GCP-A/GCP-B/111 proxy。 +- `awoooi-dev` 原本仍使用 `OLLAMA_URL=http://192.168.0.188:11434`，source 已改為 GCP-A/GCP-B/111 direct route；production readback 另行驗證。 - Prometheus 舊 blackbox target 已移除並 reload，避免退役後誤報。 - 188 journal 中近期少量推理 POST 來源為 `192.168.0.88`；後續確認 `.88` 是 188 的 default gateway，不是下游主機。 - 2026-05-06 14:53 短窗口 Gate 綠燈：repo、K8s env、Prometheus target、dev health check 均已避開 188。 diff --git a/docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md b/docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md index bd31cf6ab..65b0745c4 100644 --- a/docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md +++ b/docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md @@ -1,263 +1,74 @@ -# RUNBOOK-OLLAMA-FAILOVER.md -# Ollama 容災監控 Runbook -# 2026-04-26 P2.3 by Claude Sonnet 4.6 (tool-expert) -# 2026-06-04 Codex 更新：ADR-110 現行順序為 GCP-A → GCP-B → 111 local → Gemini；111 經 110:11437 Nginx proxy 進入，不直接由 K8s Pod 打 111。 -# 對應告警規則: ops/monitoring/ollama_health_rules.yaml -# 對應 Dashboard: ops/monitoring/grafana/dashboards/ollama_failover.json +# Ollama／付費 Critic 容災 Runbook ---- +狀態：`global_product_governance_v2`，2026-07-16 Asia/Taipei。 -## 現行拓樸與一鍵診斷 +## 唯一 production 順序 -現行 AI 推理順序： +1. `ollama_gcp_a` — GCP-A Ollama，cloud data boundary。 +2. `ollama_gcp_b` — GCP-B Ollama，cloud data boundary。 +3. `ollama_local` — host111 Ollama，唯一 local-only hop。 +4. `claude` — Anthropic Claude API，付費 canary／fallback。 +5. `gemini` — Gemini API，最後付費備援。 -| 順位 | Provider | 入口 | -|------|----------|------| -| 1 | GCP-A | `http://192.168.0.110:11435` | -| 2 | GCP-B | `http://192.168.0.110:11436` | -| 3 | 111 local fallback | `http://192.168.0.110:11437` → `192.168.0.111:11434` | -| 4 | Gemini | 只在三層 Ollama 都不可用後作最後備援 | +Host110 不再是 Ollama provider、proxy 或 fallback。任何告警、runtime +manifest、Ansible PlayBook、verifier 或 UI 若將 host110 顯示為 Ollama +identity，必須標成 `asset_identity_drift`，不得切換或自動重建舊 transport。 -111 是 Mac / LaunchAgent 管理的 local fallback；不要用舊的 `systemctl` / `nvidia-smi` 手順判斷它。若 `11437` 回 `502`，優先分辨是 110 proxy 本身壞掉，還是 110 到 111 的 LAN/ARP 不通。 +GCP-A/B 目前的 direct public HTTP 是暫時 transport，不能算完整 production +closure。P0 replacement 是 WireGuard `10.77.114.x` mesh；完成 replay、canary、 +runtime readback 後才能關閉 public listener。 -只讀診斷： +## 只讀診斷 ```bash bash scripts/ops/ollama111-fallback-proxy-diagnose.sh ``` -常見判讀： +輸出只包含 canonical provider、data boundary、HTTP 健康狀態與 production +route readback 狀態；不輸出 key、prompt、response 或 host110 proxy 資訊。 -| 證據 | 判讀 | 行動 | -|------|------|------| -| `ollama_gcp_a=up` 或 `ollama_gcp_b=up`，但 `ollama_local=down` | 第三層 fallback 單點故障，核心 AI lane 仍可用 | 不要改 provider order；先修 111 主機或 LAN | -| 110 `ip neigh 192.168.0.111 INCOMPLETE` | 110 找不到 111 MAC，通常是 111 關機、睡眠、Wi-Fi/LAN 斷線或 IP 漂移 | 恢復 111 電源/網路，再重跑診斷 | -| 110 direct `192.168.0.111:11434` OK，但 `127.0.0.1:11437` 502 | 110 Nginx proxy 配置或程序異常 | 檢查 `110-ollama-proxy.conf` 與 Nginx reload | -| SSH 到 `ollama-111-gpu` OK，但 `127.0.0.1:11434` 失敗 | 111 本機 Ollama 或 allowlist LaunchAgent 異常 | 檢查 `com.momo.ollama111-allow-proxy` 與 Ollama | +判讀： ---- +| 證據 | 判定 | 安全下一步 | +|---|---|---| +| GCP-A healthy | 保持第一順位 | 不切換 | +| GCP-A unavailable、GCP-B healthy | 第二順位可用 | 建立 GCP-A transport work item；不改固定順序 | +| GCP-A/B unavailable、host111 healthy | local hop 可用 | 保持 local-only privacy boundary | +| host111 unavailable | local-only 任務 fail closed | 由 host111 Ansible executor check/apply/verify | +| Claude/Gemini 被使用 | 前三跳皆有 bounded unavailable receipts | 驗證 sanitized receipt、token/cost cap、rollback | +| 任一 circuit open | 該 candidate 不得滑到別台主機或別 domain | 保留 exact failure receipt，等待 cooldown／同 domain repair | -## Grafana Dashboard 使用說明 +## Alert 與自動化契約 -Dashboard 路徑：`Ollama 容災監控`（uid: `ollama-failover-p23`） -匯入方式：Grafana UI → Dashboards → Import → Upload JSON file → 選 `ops/monitoring/grafana/dashboards/ollama_failover.json` +每個 failover／recovery lifecycle 必須顯示： -### Panel 1 — Ollama 可用性 (Stat) +- canonical source 與 provider identity； +- Ollama RCA／Claude 或 Gemini Critic 是否實際呼叫； +- deterministic policy decision； +- executor（若沒有必須明示 `none`）； +- independent verifier 與 receipt ID； +- token、cost、latency及 paid budget state； +- rollback／no-write terminal； +- KM、RAG、MCP、PlayBook durable acknowledgement。 -**看什麼**：`up{job=~"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111"}` × 100，顯示每個 Ollama provider endpoint 的 scrape 存活狀態。 +只讀 route/status 查詢不得觸發 failover Telegram、切換 provider 或修改 Redis。 +同一 canonical transition 依 TTL 去重；selected primary 與 observed fallback 必須 +分開呈現，不能把 fallback 說成 primary 已切換。 -| 顏色 | 意義 | -|------|------| -| 綠色 100% | Prometheus 探測正常，主機在線 | -| 黃色 | 部分 endpoint 離線，系統應進入容災 | -| 紅色 0% | Ollama provider pool 全離線，高風險 | +## Privacy 與成本 -**注意**：此面板反映 Prometheus scrape 狀態，需要 scrape job 命名對齊 `ollama_gcp_a` / `ollama_gcp_b` / `ollama_local`。 -設定檔位於 `ops/monitoring/generated/prometheus-scrape-generated.yaml`。 +- `require_local=true` 只能使用 host111；GCP-A/B 不得因自架 Ollama 被誤認為 local。 +- Claude/Gemini 只接收已提交 synthetic fixture 或具可信 sanitization receipt 的資料。 +- provider key 只由 protected runtime secret reference 注入；禁止讀回或記錄明文。 +- 付費 lane 必須先有 run-owned TTL lease、request/token/daily/monthly cost cap、 + generation accounting receipt 和 paired rollback。 +- 單一 canary scenario 只能比較該 fixture 的正確率、延遲與成本，不能宣稱模型全面勝出。 ---- +## Runtime closure -### Panel 2 — 推理延遲 P50 / P99 (Time Series) +不能用「health 200」、「CD success」或 Telegram 卡片單獨結案。完整 terminal： -**看什麼**：推理延遲分位數。 +`Detect → Normalize → Correlate → Decide → Check → Controlled Apply → Verify → Retry/Rollback → Learn/Writeback` -| 門檻 | 含義 | -|------|------| -| < 10s (P50) | HEALTHY — 正常使用 111 | -| 10–30s (P50) | SLOW — 系統已切至 Gemini | -| > 30s (P99) | DEGRADED — 應觸發 failover | - -**⚠️ BACKLOG 警告**：`ollama_inference_duration_seconds_bucket` 尚未在 API 暴露（需在 `_check_inference()` 加 Histogram.observe()）。 -面板顯示 "No Data" 是正常的，等 backlog 補完後啟用。 - ---- - -### Panel 3 — AI Provider 路由分布 (Pie Chart) - -**看什麼**：過去 5 分鐘各 provider 被選中的請求比例。 - -| 分布 | 意義 | -|------|------| -| ollama / ollama_gcp_a 佔 >90% | 正常，GCP-A 健康 | -| ollama_gcp_b 佔多數 | GCP-A SLOW/DEGRADED/OFFLINE，容災到 GCP-B | -| ollama_local 出現 | GCP-A/B 均不可用，容災到 111 local | -| gemini 佔多數 | Ollama provider pool 全部不可用，使用付費備援 | -| 全部 nemotron/claude | 極端情況，所有主力 provider 失敗 | - ---- - -### Panel 4 — Failover / Recovery 觸發次數 (Bar Chart) - -**看什麼**：每小時 failover（橘）和 recovery（綠）的觸發次數。 - -| 模式 | 意義 | -|------|------| -| 兩條都接近 0 | 正常，111 穩定運行中 | -| 橘色上升後綠色跟上 | Auto recovery 正常：切出後又切回 | -| 橘色上升，綠色不動 | **`OllamaRecoveryStuck` alert**，見下方 runbook | -| 橘色持續高頻（>5/h） | **`OllamaFailoverFrequent` alert**，111 不穩定 | - ---- - -## Alert Runbook - -### `OllamaInstanceDown` — Ollama 主機離線 - -**觸發條件**：`up{job=~"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111"} == 0` 持續 2 分鐘。 - -**影響評估**： -- 系統應依序切至 GCP-B / 111 local / Gemini（查 Panel 3 與 `/api/v1/health` 確認） -- 查 Panel 4 是否有 Failover 計數上升 - -**排查步驟**： - -```bash -# 步驟 1：跑完整只讀診斷，先分辨是 110 proxy、111 LAN、或 111 本機服務 -bash scripts/ops/ollama111-fallback-proxy-diagnose.sh - -# 步驟 2：若 110 到 111 是 No route to host / ip neigh INCOMPLETE -# 代表 111 主機或網路不可達；先恢復 111 電源/網路，不要重啟 API。 - -# 步驟 3：若 SSH 到 111 可行，再查本機 Ollama 與 LaunchAgent -ssh ollama-111-gpu 'curl -sS -m 5 http://127.0.0.1:11434/api/tags' -ssh ollama-111-gpu 'launchctl print gui/501/com.momo.ollama111-allow-proxy | head -80' -``` - -**恢復確認**： -`/api/v1/health` 的 `ollama_local.status=up`，且 `scripts/ops/ollama111-fallback-proxy-diagnose.sh` 中 110 `proxy_11437` 回 HTTP 200。 - ---- - -### `OllamaFailoverFrequent` — Failover 頻率過高 - -**觸發條件**：`rate(ollama_failover_triggered_total[1h]) > 5` 持續 10 分鐘（每小時超過 5 次切換）。 - -**影響評估**： -- 服務本身仍可用（Gemini 在接手） -- 但 Gemini 配額消耗加速，有觸發 `GeminiQuotaApproaching` 的風險 - -**排查步驟**： - -```bash -# 步驟 1：確認 111 近況（反覆 OFFLINE/HEALTHY 之間跳動？） -bash scripts/ops/ollama111-fallback-proxy-diagnose.sh - -# 步驟 2：查 API log 找 failover 原因 -kubectl logs -n awoooi-prod deploy/api --since=30m | grep "ollama_failover_triggered" - -# 步驟 3：查推理延遲（是否長期在 SLOW 邊界？） -kubectl logs -n awoooi-prod deploy/api --since=30m | grep "ollama_health_checked" - -# 步驟 4：如果是 111 本機 Ollama 問題，需依 LaunchAgent runbook 執行維護；不可直接用 Linux systemctl 手順。 -``` - ---- - -### `OllamaRecoveryStuck` — Auto Recovery 停滯 - -**觸發條件**：`ollama_health_status{host="111"} == 1 AND ollama_current_primary_is_ollama == 0` 持續 5 分鐘。 -（111 已 HEALTHY 但路由仍走 Gemini） - -**影響評估**： -- API 功能正常（Gemini 在服務） -- 但 Gemini 配額持續消耗，111 GPU 資源浪費 - -**排查步驟**： - -```bash -# 步驟 1：確認 OllamaAutoRecoveryService 是否在運行 -kubectl logs -n awoooi-prod deploy/api --since=10m | grep "ollama_auto_recovery" - -# 步驟 2：查 recovery service 狀態 -kubectl logs -n awoooi-prod deploy/api --since=10m | grep -E "ollama_auto_recovery_started|ollama_auto_recovery_stopped|ollama_auto_recovery_loop_error" - -# 步驟 3：查 current_primary Redis key -kubectl exec -n awoooi-prod deploy/api -- python -c " -import asyncio -from src.core.redis_client import get_redis -async def check(): - r = get_redis() - val = await r.get('ollama:current_primary') - print('current_primary:', val) -asyncio.run(check()) -" - -# 步驟 4：如果 recovery service 掛了，重啟 API pod（會重新啟動 lifespan） -kubectl rollout restart deployment/api -n awoooi-prod -kubectl rollout status deployment/api -n awoooi-prod -``` - ---- - -### `GeminiQuotaApproaching` — Gemini 配額 >80% - -**觸發條件**：`gemini_daily_call_count / gemini_daily_quota > 0.8` 持續 5 分鐘。 - -**注意**：`gemini_daily_quota` 來自 `settings.GEMINI_DAILY_QUOTA`（預設 1000）。 -`gemini_daily_call_count` 從 Redis key `ollama:gemini_daily_count:{YYYY-MM-DD}` 讀取並刷新 Gauge。 - -**影響評估**： -- 當日 Gemini 配額即將耗盡 -- 耗盡後系統會自動切至 188 CPU-only 備援（qwen2.5:7b-instruct），速度較慢 - -**行動步驟**： - -```bash -# 步驟 1：確認當日 Gemini 使用量 -kubectl exec -n awoooi-prod deploy/api -- python -c " -import asyncio, datetime -from src.core.redis_client import get_redis -async def check(): - r = get_redis() - today = datetime.date.today().isoformat() - val = await r.get(f'ollama:gemini_daily_count:{today}') - print(f'gemini_daily_count[{today}]:', val) -asyncio.run(check()) -" - -# 步驟 2：確認 111 是否能快速恢復（讓流量切回 Ollama） -bash scripts/ops/ollama111-fallback-proxy-diagnose.sh - -# 步驟 3：如需增加配額，修改 settings -# k8s/awoooi-prod/04-configmap.yaml.patch-* 找 GEMINI_DAILY_QUOTA -# 改完後 kubectl apply + rollout restart - -# 步驟 4：緊急手動重置計數（謹慎使用，只在確認誤計時才用） -# kubectl exec -n awoooi-prod deploy/api -- redis-cli DEL "ollama:gemini_daily_count:$(date +%Y-%m-%d)" -``` - ---- - -## Metric 清單 - -| Metric | 類型 | 狀態 | 說明 | -|--------|------|------|------| -| `up{job="ollama_gcp_a"}` | Gauge | ✅ 現有 | Prometheus scrape 存活 | -| `up{job="ollama_gcp_b"}` | Gauge | ✅ 現有 | Prometheus scrape 存活 | -| `up{job="ollama_local"}` | Gauge | ✅ 現有 | Prometheus scrape 存活 | -| `ollama_failover_triggered_total` | Counter | ✅ P2.3 補入 | failover 切換次數，labels: from_provider, to_provider | -| `ollama_recovery_triggered_total` | Counter | ✅ P2.3 補入 | recovery 切回次數，labels: from_provider | -| `ollama_health_status{host}` | Gauge | ✅ P2.3 補入 | 健康狀態 1=healthy, 0=not_healthy | -| `ollama_current_primary_is_ollama` | Gauge | ✅ P2.3 補入 | 1=primary 是 ollama, 0=failover 中 | -| `ai_router_selected_provider_total` | Counter | ✅ P2.3 補入 | AI router 選擇次數，labels: provider | -| `gemini_daily_call_count` | Gauge | ✅ P2.3 補入 | 今日 Gemini 呼叫次數 | -| `gemini_daily_quota` | Gauge | ✅ P2.3 補入 | Gemini 每日配額 | -| `ollama_inference_duration_seconds` | Histogram | ⏳ BACKLOG | 推理延遲分布，需在 `_check_inference()` 加 observe | -| `post_execution_verification_total` | Counter | ⏳ BACKLOG | Verifier 執行次數，需 auto_repair_service.py 補入 | -| `post_execution_verification_failed_total` | Counter | ⏳ BACKLOG | Verifier 失敗次數，需 auto_repair_service.py 補入 | - -## Backlog 補完指引 - -### `ollama_inference_duration_seconds` - -在 `apps/api/src/services/ollama_health_monitor.py` 的 `_check_inference()` 方法結尾，加： - -```python -from src.core.metrics import OLLAMA_INFERENCE_DURATION # 需先在 metrics.py 加 Histogram -OLLAMA_INFERENCE_DURATION.labels(host=host_label).observe(latency_ms / 1000) -``` - -### `post_execution_verification_*` - -在 `apps/api/src/services/auto_repair_service.py` 的 verifier 路徑，加 Counter inc()。 -需先確認 verifier 執行點（grep `post_execution` 或 `verif` 找入口）。 +若 GCP mesh、host111 reachability、paid accounting、independent verifier 或 learning +ack 任一缺失，狀態維持 `partial/degraded/blocked_with_safe_next_action`。 diff --git a/infra/ansible/inventory/group_vars/host_110.yml b/infra/ansible/inventory/group_vars/host_110.yml index 9ef921f2c..fd810813e 100644 --- a/infra/ansible/inventory/group_vars/host_110.yml +++ b/infra/ansible/inventory/group_vars/host_110.yml @@ -36,7 +36,6 @@ wooo_aiops_dir: /home/wooo/apps/wooo-aiops wooo_aiops_pm2_name: wooo-aiops # GitHub Runners -github_runner_count: 5 # keepalived keepalived_role: BACKUP diff --git a/infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml b/infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml new file mode 100644 index 000000000..b7c8e7b3c --- /dev/null +++ b/infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml @@ -0,0 +1,159 @@ +--- +- name: Bounded host 110 Alertmanager delivery recovery + hosts: host_110 + gather_facts: false + become: false + vars: + alertmanager_container: alertmanager + alertmanager_config: /etc/alertmanager/alertmanager.yml + awoooi_webhook_url: https://awoooi.wooo.work/api/v1/webhooks/alertmanager + + tasks: + - name: Inspect exact Alertmanager container running state + ansible.builtin.command: + argv: + - docker + - inspect + - --format + - "{{ '{{' }}.State.Running{{ '}}' }}" + - "{{ alertmanager_container }}" + register: alertmanager_running + check_mode: false + changed_when: false + failed_when: false + + - name: Pin the pre-apply container state for exact rollback ownership + ansible.builtin.set_fact: + alertmanager_was_running: >- + {{ alertmanager_running.rc == 0 + and (alertmanager_running.stdout | trim) == 'true' }} + alertmanager_started_by_run: false + changed_when: false + + - name: Validate the AWOOOI webhook network boundary without creating an incident + ansible.builtin.command: + argv: + - curl + - -sS + - -o + - /dev/null + - -w + - "%{http_code}" + - --max-time + - "10" + - -I + - "{{ awoooi_webhook_url }}" + register: alertmanager_webhook_probe + check_mode: false + changed_when: false + failed_when: >- + alertmanager_webhook_probe.rc != 0 + or (alertmanager_webhook_probe.stdout | trim) not in ['200', '405'] + + - name: Execute the bounded exact-container recovery + block: + - name: Start only the exact stopped Alertmanager container during apply + ansible.builtin.command: + argv: + - docker + - start + - "{{ alertmanager_container }}" + when: + - not ansible_check_mode + - not (alertmanager_was_running | bool) + changed_when: true + + - name: Record ownership only when this run started the container + ansible.builtin.set_fact: + alertmanager_started_by_run: true + when: + - not ansible_check_mode + - not (alertmanager_was_running | bool) + changed_when: false + + - name: Wait for exact Alertmanager readiness after controlled start + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - http://127.0.0.1:9093/-/ready + register: alertmanager_ready + retries: 6 + delay: 5 + until: alertmanager_ready.rc == 0 + check_mode: false + changed_when: false + when: not ansible_check_mode + + - name: Validate the active Alertmanager configuration + ansible.builtin.command: + argv: + - docker + - exec + - "{{ alertmanager_container }}" + - amtool + - check-config + - "{{ alertmanager_config }}" + check_mode: false + changed_when: false + when: (alertmanager_was_running | bool) or not ansible_check_mode + + - name: Require exact delivery request metric families before mutation + ansible.builtin.shell: | + set -euo pipefail + metrics="$(curl -fsS --max-time 5 http://127.0.0.1:9093/metrics)" + grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${metrics}" + grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${metrics}" + args: + executable: /bin/bash + check_mode: false + changed_when: false + when: (alertmanager_was_running | bool) or not ansible_check_mode + + - name: Reload only the already-running exact Alertmanager container + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - -X + - POST + - http://127.0.0.1:9093/-/reload + when: + - not ansible_check_mode + - alertmanager_was_running | bool + changed_when: true + + - name: Verify readiness immediately after bounded reload or start + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - http://127.0.0.1:9093/-/ready + check_mode: false + changed_when: false + when: not ansible_check_mode + + rescue: + - name: Restore stopped state only when this run started Alertmanager + ansible.builtin.command: + argv: + - docker + - stop + - --time + - "10" + - "{{ alertmanager_container }}" + when: alertmanager_started_by_run | default(false) | bool + changed_when: true + failed_when: false + + - name: Fail closed after bounded recovery or rollback failure + ansible.builtin.fail: + msg: >- + alertmanager_delivery_recovery_failed; + rollback_started_container={{ alertmanager_started_by_run | default(false) | bool }} diff --git a/infra/ansible/playbooks/110-devops.yml b/infra/ansible/playbooks/110-devops.yml index 9130aefc1..54ef9d725 100644 --- a/infra/ansible/playbooks/110-devops.yml +++ b/infra/ansible/playbooks/110-devops.yml @@ -143,17 +143,6 @@ when: bitan_status.stdout == "" tags: bitan - # ======================================================================== - # GitHub Runners - # ======================================================================== - - name: "Runners | 確認 runner 服務狀態" - ansible.builtin.systemd: - name: "github-runner-{{ item }}" - register: runner_status - loop: "{{ range(1, github_runner_count + 1) | list }}" - ignore_errors: true - tags: runners - # ======================================================================== # keepalived # ======================================================================== diff --git a/infra/ansible/playbooks/111-ollama-fallback.yml b/infra/ansible/playbooks/111-ollama-fallback.yml index dc7acb867..faa29c0dd 100644 --- a/infra/ansible/playbooks/111-ollama-fallback.yml +++ b/infra/ansible/playbooks/111-ollama-fallback.yml @@ -1,6 +1,6 @@ --- -# AWOOOI Ansible — 192.168.0.111 Ollama fallback proxy -# 原則: 111 的 Ollama 仍只綁本機，由 allowlist proxy 控制哪些主機能進入。 +# AWOOOI Ansible — 192.168.0.111 canonical local Ollama lane +# 原則: 111 的 Ollama 由 allowlist proxy 控制；K3s nodes 直接連入，不再經 host110。 - name: "111 Ollama fallback allowlist 收斂" hosts: host_111 @@ -9,17 +9,18 @@ proxy_label: com.momo.ollama111-allow-proxy proxy_user_uid: 501 proxy_plist: /Users/ooo/Library/LaunchAgents/com.momo.ollama111-allow-proxy.plist - allowed_cidrs: "127.0.0.1/32,192.168.0.111/32,192.168.0.188/32,192.168.0.110/32" + allowed_cidrs: "127.0.0.1/32,192.168.0.111/32,192.168.0.188/32,192.168.0.120/32,192.168.0.121/32" tasks: - name: "Ollama111 | 讀取現行 allowlist" ansible.builtin.command: cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}" register: current_allowlist + check_mode: false changed_when: false tags: ["111", "ollama-fallback"] - - name: "Ollama111 | 放行 110 proxy 進入 111 fallback" + - name: "Ollama111 | 收斂 K3s direct provider allowlist" ansible.builtin.command: cmd: "/usr/libexec/PlistBuddy -c 'Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ allowed_cidrs }}' {{ proxy_plist }}" when: current_allowlist.stdout != allowed_cidrs @@ -27,15 +28,89 @@ notify: Restart ollama111 allow proxy tags: ["111", "ollama-fallback"] + - name: "Ollama111 | 套用 allowlist handler 後再驗證" + ansible.builtin.meta: flush_handlers + + - name: "Ollama111 | 讀回生效 allowlist" + ansible.builtin.command: + cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}" + register: effective_allowlist + check_mode: false + changed_when: false + tags: ["111", "ollama-fallback"] + + - name: "Ollama111 | 驗證 K3s origins 並排除 retired host110" + ansible.builtin.assert: + that: + - effective_allowlist.stdout == allowed_cidrs + - "'192.168.0.120/32' in effective_allowlist.stdout.split(',')" + - "'192.168.0.121/32' in effective_allowlist.stdout.split(',')" + fail_msg: ollama111_allowlist_runtime_identity_mismatch + changed_when: false + when: not ansible_check_mode + tags: ["111", "ollama-fallback"] + + - name: "Ollama111 | check-mode 發布精確 source diff" + ansible.builtin.debug: + msg: + schema_version: ollama111_allowlist_check_v1 + current_allowlist_sha_only: true + would_change: "{{ current_allowlist.stdout != allowed_cidrs }}" + expected_k3s_origins: + - host120 + - host121 + retired_host110_allowed_after_apply: false + persistent_writes_performed: 0 + when: ansible_check_mode + changed_when: current_allowlist.stdout != allowed_cidrs + tags: ["111", "ollama-fallback"] + - name: "Ollama111 | 驗證本機 Ollama tag API" ansible.builtin.uri: url: http://127.0.0.1:11434/api/tags method: GET return_content: false status_code: 200 + check_mode: false changed_when: false tags: ["111", "ollama-fallback"] + - name: "Ollama111 | 從每個 K3s node 驗證 direct provider path" + ansible.builtin.uri: + url: http://192.168.0.111:11434/api/tags + method: GET + return_content: false + status_code: 200 + timeout: 10 + delegate_to: "{{ item }}" + loop: + - host_120 + - host_121 + register: k3s_direct_provider_readback + check_mode: false + changed_when: false + when: >- + not ansible_check_mode + or current_allowlist.stdout == allowed_cidrs + tags: ["111", "ollama-fallback", "independent-verifier"] + + - name: "Ollama111 | 發布 K3s path verifier receipt" + ansible.builtin.debug: + msg: + schema_version: ollama111_k3s_path_verifier_v1 + canonical_asset_id: ai-provider:ollama_local + probe_origins: + - host120 + - host121 + http_statuses: >- + {{ k3s_direct_provider_readback.results + | map(attribute='status') | list }} + retired_host110_allowed: false + runtime_verified: true + changed_when: false + when: not ansible_check_mode + tags: ["111", "ollama-fallback", "independent-verifier"] + handlers: - name: Restart ollama111 allow proxy ansible.builtin.shell: diff --git a/infra/ansible/playbooks/nginx-sync-readonly.yml b/infra/ansible/playbooks/nginx-sync-readonly.yml index bd288c4c8..86ede9ecd 100644 --- a/infra/ansible/playbooks/nginx-sync-readonly.yml +++ b/infra/ansible/playbooks/nginx-sync-readonly.yml @@ -2,6 +2,8 @@ # Read-only preflight for the nginx-sync catalog entry. # This path never uses sudo and remains safe when the privileged apply executor # is unavailable. The mutating nginx-sync.yml remains the controlled apply path. +# Host110 Ollama proxy checks were retired on 2026-07-16; this catalog entry is +# now scoped only to the host188 Nginx assets it actually owns. - name: "188 Nginx read-only evidence" hosts: host_188 @@ -43,52 +45,3 @@ public_gateway_config_present: "{{ public_gateway_config.stat.exists | default(false) }}" internal_tools_config_present: "{{ internal_tools_config.stat.exists | default(false) }}" https_listener_present: "{{ ':443 ' in (listening_sockets.stdout | default('')) }}" - -- name: "110 Ollama proxy read-only evidence" - hosts: host_110 - gather_facts: false - become: false - - tasks: - - name: "Preflight | collect remote user" - ansible.builtin.command: - cmd: "id -un" - register: remote_identity - changed_when: false - check_mode: false - - - name: "Nginx | inspect managed Ollama proxy configuration" - ansible.builtin.stat: - path: /etc/nginx/sites-enabled/110-ollama-proxy.conf - register: managed_proxy_config - failed_when: false - - - name: "Nginx | inspect stale Ollama proxy configuration" - ansible.builtin.stat: - path: /etc/nginx/conf.d/ollama-gcp-proxy.conf - register: stale_proxy_config - failed_when: false - - - name: "Ollama | probe configured proxy listeners" - ansible.builtin.uri: - url: "http://127.0.0.1:{{ item }}/api/tags" - method: GET - return_content: false - status_code: [200, 502, 503, 504] - timeout: 6 - loop: - - 11435 - - 11436 - - 11437 - register: proxy_health - failed_when: false - changed_when: false - check_mode: false - - - name: "Evidence | emit 110 read-only summary" - ansible.builtin.debug: - msg: - remote_user: "{{ remote_identity.stdout | default('unknown') }}" - managed_proxy_config_present: "{{ managed_proxy_config.stat.exists | default(false) }}" - stale_proxy_config_present: "{{ stale_proxy_config.stat.exists | default(false) }}" - proxy_http_statuses: "{{ proxy_health.results | default([]) | map(attribute='status', default=-1) | list }}" diff --git a/infra/ansible/playbooks/nginx-sync.yml b/infra/ansible/playbooks/nginx-sync.yml index 0c62fe055..7063f0798 100644 --- a/infra/ansible/playbooks/nginx-sync.yml +++ b/infra/ansible/playbooks/nginx-sync.yml @@ -3,6 +3,9 @@ # 原則: Nginx conf 不再直接手改，所有修改透過此 Playbook 部署 # 執行: ansible-playbook -i inventory/hosts.yml playbooks/nginx-sync.yml --tags 188 # 乾跑: ansible-playbook -i inventory/hosts.yml playbooks/nginx-sync.yml --check +# superseded host110 Ollama proxy lane: global_product_governance_v2 +# owner=AssetIdentity; expiry=2026-07-16 +# replacement=direct GCP-A/GCP-B/host111 provider identities - name: "188 Nginx conf 同步" hosts: host_188 @@ -48,101 +51,3 @@ ansible.builtin.systemd: name: nginx state: reloaded - -- name: "110 Ollama GCP Proxy 部署" - hosts: host_110 - become: true - vars: - ansible_become_pass: "{{ vault_sudo_password | default(omit) }}" - ollama_proxy_src: "{{ playbook_dir }}/../roles/nginx/templates/110-ollama-proxy.conf.j2" - ollama_proxy_dest: /etc/nginx/sites-enabled/110-ollama-proxy.conf - - tasks: - - name: "Nginx | 確認 110 nginx 無 all-sites-from-188.conf 在 sites-enabled" - ansible.builtin.stat: - path: /etc/nginx/sites-enabled/all-sites-from-188.conf - register: stale_conf - tags: ["110", "nginx"] - - - name: "Nginx | 警告：110 仍有 all-sites-from-188.conf (已非 188 管控)" - ansible.builtin.debug: - msg: "⚠️ 110 sites-enabled 仍有 all-sites-from-188.conf，應已封存" - when: stale_conf.stat.exists - tags: ["110", "nginx"] - - # ADR-110: Ollama GCP 三層容災 — 110 作為 nginx proxy 轉發 K3s 流量到 GCP - - name: "Nginx | 部署 Ollama GCP Proxy 配置" - ansible.builtin.template: - src: "{{ ollama_proxy_src }}" - dest: "{{ ollama_proxy_dest }}" - owner: root - group: root - mode: "0644" - backup: true - notify: Reload nginx 110 - tags: ["110", "nginx", "ollama-proxy"] - - - name: "Nginx | 檢查舊 conf.d Ollama proxy 是否仍存在" - ansible.builtin.stat: - path: /etc/nginx/conf.d/ollama-gcp-proxy.conf - register: stale_ollama_conf - tags: ["110", "nginx", "ollama-proxy"] - - - name: "Nginx | 建立舊 conf.d Ollama proxy 備份目錄" - ansible.builtin.file: - path: /var/backups/awoooi/nginx - state: directory - owner: root - group: root - mode: "0755" - when: stale_ollama_conf.stat.exists - tags: ["110", "nginx", "ollama-proxy"] - - - name: "Nginx | 備份舊 conf.d Ollama proxy" - ansible.builtin.copy: - remote_src: true - src: /etc/nginx/conf.d/ollama-gcp-proxy.conf - dest: "/var/backups/awoooi/nginx/ollama-gcp-proxy.conf.{{ ansible_date_time.iso8601_basic }}" - owner: root - group: root - mode: "0644" - when: stale_ollama_conf.stat.exists - tags: ["110", "nginx", "ollama-proxy"] - - - name: "Nginx | 移除舊 conf.d Ollama proxy，避免 11435/11436 重複 server block" - ansible.builtin.file: - path: /etc/nginx/conf.d/ollama-gcp-proxy.conf - state: absent - when: stale_ollama_conf.stat.exists - notify: Reload nginx 110 - tags: ["110", "nginx", "ollama-proxy"] - - - name: "Nginx | 測試 110 設定" - ansible.builtin.command: - cmd: "nginx -t" - changed_when: false - tags: ["110", "nginx", "ollama-proxy"] - - - name: "Nginx | 確認 nginx 已啟動" - ansible.builtin.systemd: - name: nginx - state: started - enabled: true - tags: ["110", "nginx", "ollama-proxy"] - - - name: "Nginx | 驗證 Ollama proxy 端口監聽" - ansible.builtin.wait_for: - port: "{{ item }}" - host: 127.0.0.1 - timeout: 10 - loop: - - 11435 # GCP-A - - 11436 # GCP-B - - 11437 # Local Fallback - tags: ["110", "nginx", "ollama-proxy"] - - handlers: - - name: Reload nginx 110 - ansible.builtin.systemd: - name: nginx - state: reloaded diff --git a/infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2 b/infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2 deleted file mode 100644 index 026284784..000000000 --- a/infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2 +++ /dev/null @@ -1,104 +0,0 @@ -# 110 Ollama GCP Proxy — ADR-110 三層容災 -# 讓 K3s 叢集內可透過內網 110 存取 GCP 外網 Ollama -# 建立時間: 2026-05-04 -# 部署: ansible-playbook -i inventory/hosts.yml playbooks/nginx-sync.yml --tags 110 - -# ============================================================ -# Ollama GCP-A Primary (port 11435 → 34.143.170.20:11434) -# ============================================================ -server { - listen 11435; - listen [::]:11435; - server_name _; - - access_log /var/log/nginx/ollama-gcp-a-access.log; - error_log /var/log/nginx/ollama-gcp-a-error.log warn; - - location / { - proxy_pass http://34.143.170.20:11434; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # Ollama 推理可能較慢，給較長超時 - proxy_connect_timeout 10s; - proxy_send_timeout 300s; - proxy_read_timeout 300s; - - # 支援 streaming response - proxy_buffering off; - proxy_cache off; - } - - # 健康檢查端點 - location /nginx-health { - access_log off; - return 200 "Ollama GCP-A Proxy OK\n"; - add_header Content-Type text/plain; - } -} - -# ============================================================ -# Ollama GCP-B Secondary (port 11436 → 34.21.145.224:11434) -# ============================================================ -server { - listen 11436; - listen [::]:11436; - server_name _; - - access_log /var/log/nginx/ollama-gcp-b-access.log; - error_log /var/log/nginx/ollama-gcp-b-error.log warn; - - location / { - proxy_pass http://34.21.145.224:11434; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_connect_timeout 10s; - proxy_send_timeout 300s; - proxy_read_timeout 300s; - - proxy_buffering off; - proxy_cache off; - } - - location /nginx-health { - access_log off; - return 200 "Ollama GCP-B Proxy OK\n"; - add_header Content-Type text/plain; - } -} - -# ============================================================ -# Ollama Local Fallback (port 11437 → 192.168.0.111:11434) -# ADR-110 第三層：GCP-A 與 GCP-B 均不可用時的本機備援 -# ============================================================ -server { - listen 11437; - listen [::]:11437; - server_name _; - - access_log /var/log/nginx/ollama-local-access.log; - error_log /var/log/nginx/ollama-local-error.log warn; - - location / { - proxy_pass http://192.168.0.111:11434; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_connect_timeout 5s; - proxy_send_timeout 300s; - proxy_read_timeout 300s; - - proxy_buffering off; - proxy_cache off; - } - - location /nginx-health { - access_log off; - return 200 "Ollama Local Fallback Proxy OK\n"; - add_header Content-Type text/plain; - } -} diff --git a/k8s/awoooi-dev/02-configmap.yaml b/k8s/awoooi-dev/02-configmap.yaml index cf28afacd..e4d51da03 100644 --- a/k8s/awoooi-dev/02-configmap.yaml +++ b/k8s/awoooi-dev/02-configmap.yaml @@ -14,9 +14,9 @@ data: SYSTEM_NAME: "awoooi" # 2026-05-06 Codex: dev 也必須走正式 Ollama pool，禁止回到 188 legacy Ollama。 - OLLAMA_URL: "http://192.168.0.110:11435" - OLLAMA_SECONDARY_URL: "http://192.168.0.110:11436" - OLLAMA_FALLBACK_URL: "http://192.168.0.110:11437" + OLLAMA_URL: "http://34.143.170.20:11434" + OLLAMA_SECONDARY_URL: "http://34.21.145.224:11434" + OLLAMA_FALLBACK_URL: "http://192.168.0.111:11434" ALERT_AI_ENFORCE_OLLAMA_FIRST: "true" ALERT_AI_ALLOW_CLOUD_FALLBACK: "true" ALERT_AI_STRICT_PROVIDER_CHAIN: "true" @@ -45,6 +45,8 @@ data: # Historical key: auth-status receipt TTL only. Generation/run receipts are durable. GEMINI_USAGE_RECEIPT_TTL_SECONDS: "604800" GEMINI_FAILURE_COOLDOWN_SECONDS: "60" + GEMINI_EXECUTION_MODE: "shadow" + GEMINI_CANARY_PERCENT: "0" ENABLE_GEMINI: "false" CLAUDE_MODEL: "claude-sonnet-5" CLAUDE_DAILY_QUOTA: "50" diff --git a/k8s/awoooi-prod/02-network-policy.yaml b/k8s/awoooi-prod/02-network-policy.yaml index e96ddac98..0ce8d08ff 100644 --- a/k8s/awoooi-prod/02-network-policy.yaml +++ b/k8s/awoooi-prod/02-network-policy.yaml @@ -1,8 +1,9 @@ # AWOOOI 正式環境零信任網路策略 # 負責人: CIO -# 版本: v1.8 -# 日期: 2026-07-11 +# 版本: v1.9 +# 日期: 2026-07-16 # 變更: +# - v1.9: Ollama egress 固定為 GCP-A/GCP-B/Host111 的 /32:11434；移除已退役 Host110 proxy ports # - v1.8: SSH egress 僅允許 dedicated Ansible executor broker；修正 legacy allow 規則 # - v1.7: ADR-110 — 新增 GCP-A/GCP-B Ollama egress（GCP 三層容災） # - v1.6: 新增 K3s node 120/121 SSH egress，供 SSH MCP 主機診斷/修復 @@ -165,14 +166,6 @@ spec: port: 4317 - protocol: TCP port: 4318 - # 2026-05-25 Codex: ADR-110 production Ollama proxy pool. - # 11435=GCP-A, 11436=GCP-B, 11437=Local 111 fallback. - - protocol: TCP - port: 11435 - - protocol: TCP - port: 11436 - - protocol: TCP - port: 11437 # 允許訪問 192.168.0.112 安全掃描服務 - to: diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index 56c4cba1b..776417fa5 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -17,12 +17,12 @@ data: # 服務端點 (非機密) # 2026-04-16 ogt + Claude Sonnet 4.6: 改指向 111（GPU 機，RTX） # 188 = CPU-only Ollama，推理極慢（>60s）；111 有 GPU，avg 10s - # 2026-05-25 Codex: keep ADR-110 production policy visible and ordered. - # High-volume callers use short endpoint cooldowns when GCP-A/B are down, - # but health/status must still show GCP-A -> GCP-B -> 111 -> Claude -> Gemini. - OLLAMA_URL: "http://192.168.0.110:11435" - OLLAMA_SECONDARY_URL: "http://192.168.0.110:11436" - OLLAMA_FALLBACK_URL: "http://192.168.0.110:11437" + # 2026-07-16 Codex: host110 no longer owns an Ollama transport/proxy. + # Runtime identity and transport now point at the canonical providers directly; + # provider order remains GCP-A -> GCP-B -> host111 -> Claude -> Gemini. + OLLAMA_URL: "http://34.143.170.20:11434" + OLLAMA_SECONDARY_URL: "http://34.21.145.224:11434" + OLLAMA_FALLBACK_URL: "http://192.168.0.111:11434" OPENCLAW_URL: "http://192.168.0.188:8088" KALI_SCANNER_URL: "http://192.168.0.112:8080" SIGNOZ_INTERNAL_URL: "http://192.168.0.110:8080" @@ -82,9 +82,14 @@ data: # Historical key: auth-status receipt TTL only. Generation/run receipts are durable. GEMINI_USAGE_RECEIPT_TTL_SECONDS: "604800" GEMINI_FAILURE_COOLDOWN_SECONDS: "60" - ENABLE_GEMINI: "false" - # Claude Sonnet 5 is source-integrated but starts shadow/no-call. Promotion is - # shadow -> deterministic canary -> production, with an independent budget. + # AIA-SRE-013: paid routes are online only in the deterministic 5% canary. + # Persistent Redis control remains fail closed until the bounded verifier + # atomically enables both providers after its durable start receipt. + GEMINI_EXECUTION_MODE: "canary" + GEMINI_CANARY_PERCENT: "5" + ENABLE_GEMINI: "true" + # Claude/Gemini promotion remains shadow -> deterministic canary -> + # production. This change activates only the bounded canary, not production. CLAUDE_MODEL: "claude-sonnet-5" CLAUDE_DAILY_QUOTA: "50" CLAUDE_RPM_LIMIT: "2" @@ -95,9 +100,9 @@ data: CLAUDE_MAX_OUTPUT_TOKENS: "4096" CLAUDE_USAGE_RECEIPT_TTL_SECONDS: "604800" CLAUDE_FAILURE_COOLDOWN_SECONDS: "120" - CLAUDE_EXECUTION_MODE: "shadow" - CLAUDE_CANARY_PERCENT: "0" - ENABLE_CLAUDE: "false" + CLAUDE_EXECUTION_MODE: "canary" + CLAUDE_CANARY_PERCENT: "5" + ENABLE_CLAUDE: "true" OPENCLAW_DEFAULT_MODEL: "qwen2.5:7b-instruct" OPENCLAW_TIMEOUT: "120" OLLAMA_DIAGNOSE_TIMEOUT_SECONDS: "300" diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 05a7478ef..dd1ec6406 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -214,11 +214,11 @@ spec: # 188/OpenClaw is the single getUpdates owner. value: "false" - name: OLLAMA_URL - value: "http://192.168.0.110:11435" # 2026-05-25 Codex: GCP-A via 110 proxy; health cooldown protects noisy offline probes + value: "http://34.143.170.20:11434" # Canonical GCP-A; host110 proxy retired 2026-07-16 - name: OLLAMA_SECONDARY_URL - value: "http://192.168.0.110:11436" # 2026-05-25 Codex: GCP-B via 110 proxy; fallback only after GCP-A is unavailable + value: "http://34.21.145.224:11434" # Canonical GCP-B - name: OLLAMA_FALLBACK_URL - value: "http://192.168.0.110:11437" # 2026-05-25 Codex: 111 via 110 proxy before Gemini + value: "http://192.168.0.111:11434" # Canonical host111 local Ollama - name: ALERT_AI_ALLOW_CLOUD_FALLBACK value: "true" # Gemini 只作 GCP-A → GCP-B → 111 全失敗後的備援 - name: ALERT_AI_ENFORCE_OLLAMA_FIRST diff --git a/k8s/monitoring/alert-chain-monitor.yaml b/k8s/monitoring/alert-chain-monitor.yaml index 70728fe54..a96013179 100644 --- a/k8s/monitoring/alert-chain-monitor.yaml +++ b/k8s/monitoring/alert-chain-monitor.yaml @@ -26,20 +26,36 @@ spec: # ----------------------------------------------------------------- - alert: AlertChainBroken_Alertmanager expr: | - sum(rate(awoooi_webhook_requests_total{ - source="alertmanager", - status!="success" - }[5m])) / sum(rate(awoooi_webhook_requests_total{ - source="alertmanager" - }[5m])) > 0.1 - for: 10m + ( + sum(increase(alertmanager_notification_requests_failed_total{ + job="alertmanager-runtime", + integration="webhook", + receiver="awoooi-webhook" + }[10m])) + / + clamp_min(sum(increase(alertmanager_notification_requests_total{ + job="alertmanager-runtime", + integration="webhook", + receiver="awoooi-webhook" + }[10m])), 1) + ) > 0.1 + and + sum(increase(alertmanager_notification_requests_total{ + job="alertmanager-runtime", + integration="webhook", + receiver="awoooi-webhook" + }[10m])) >= 5 + for: 5m labels: severity: critical service: alert-chain + component: alertmanager team: platform + auto_repair: "true" + alert_category: alert_chain_health annotations: - summary: "Alertmanager Webhook 錯誤率 > 10%" - description: "告警可能無法正確送達 AWOOOI API" + summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" + description: "Alertmanager 自身且 receiver=awoooi-webhook 的單調 delivery counter 在 10 分鐘至少 5 次嘗試下，失敗率持續超過 10%；Telegram 與 Agent99 poll 不納入分母。" runbook_url: "https://awoooi.internal/runbooks/alert-chain" # ----------------------------------------------------------------- diff --git a/k8s/monitoring/docker-compose-110.yml b/k8s/monitoring/docker-compose-110.yml index 07c75c89f..666452366 100644 --- a/k8s/monitoring/docker-compose-110.yml +++ b/k8s/monitoring/docker-compose-110.yml @@ -116,28 +116,6 @@ services: networks: - monitoring - # === Phase 5: GitHub Exporter (OPS.176) === - github-exporter: - image: promhippie/github-exporter:latest - container_name: github-exporter - restart: unless-stopped - ports: - - '9504:9504' - environment: - - GITHUB_EXPORTER_TOKEN=${GITHUB_TOKEN} - - GITHUB_EXPORTER_REPOS=owenhytsai/wooo-aiops,owenhytsai/clawbot-v5 - - GITHUB_EXPORTER_LOG_LEVEL=info - networks: - - monitoring - labels: - - 'com.wooo.service=github-exporter' - - 'com.wooo.phase=phase-5' - logging: - driver: json-file - options: - max-size: '10m' - max-file: '3' - networks: monitoring: driver: bridge diff --git a/k8s/monitoring/prometheus.yml b/k8s/monitoring/prometheus.yml index 067b868dd..7d1a5ce5e 100644 --- a/k8s/monitoring/prometheus.yml +++ b/k8s/monitoring/prometheus.yml @@ -62,6 +62,22 @@ scrape_configs: - https://www.tsenyang.com - http://stock.wooo.work - https://bitan.wooo.work + # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES + - targets: + - http://34.143.170.20:11434/api/tags + labels: + service: ollama-cloud-edge + provider: ollama_gcp_a + probe_origin: host110 + boundary: cloud_edge_availability_only + - targets: + - http://34.21.145.224:11434/api/tags + labels: + service: ollama-cloud-edge + provider: ollama_gcp_b + probe_origin: host110 + boundary: cloud_edge_availability_only + # END AWOOOI OLLAMA CLOUD EDGE PROBES relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -83,6 +99,7 @@ scrape_configs: - 192.168.0.125:32335 # 110 服務 - 192.168.0.110:9090 + - 192.168.0.110:9093 - 192.168.0.110:3001 - 192.168.0.110:9000 - 192.168.0.110:5000 @@ -136,13 +153,14 @@ scrape_configs: host: "188" service: "clawbot" - # === GitHub Actions Exporter (Phase 5 OPS.176) === - - job_name: 'github-actions' + # === Gitea native metrics (GitHub freeze replacement, 2026-07-16) === + - job_name: 'gitea-native' scrape_interval: 60s + metrics_path: /metrics static_configs: - - targets: ['github-exporter:9504'] + - targets: ['192.168.0.110:3001'] labels: - source: 'github-actions' + source: 'gitea' # === AWOOOI API Metrics (Phase O Wave A.2, 2026-04-02) === - job_name: 'awoooi-api' @@ -155,6 +173,22 @@ scrape_configs: service: 'awoooi-api' env: 'prod' + # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH + # === Alertmanager delivery truth (2026-07-16) === + # Scrape Alertmanager's own monotonic delivery counters. The API NodePort + # load-balances across process-local counters and must not be used to compute + # webhook error rates. + - job_name: 'alertmanager-runtime' + scrape_interval: 30s + metrics_path: /metrics + static_configs: + - targets: ['alertmanager:9093'] + labels: + host: '110' + service: 'alertmanager' + env: 'prod' + # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH + # === ADR-074 M1: 飛輪健康度指標 (2026-04-12 ogt) === - job_name: 'awoooi-flywheel' scrape_interval: 5m diff --git a/ops/alertmanager/alertmanager.yml b/ops/alertmanager/alertmanager.yml index 44fcbe0e1..d92066aba 100644 --- a/ops/alertmanager/alertmanager.yml +++ b/ops/alertmanager/alertmanager.yml @@ -4,7 +4,8 @@ # 修正後: https://awoooi.wooo.work/api/v1/webhooks/alertmanager (AWOOOI public API，複數，正確) # 根據 feedback_alertmanager_awoooi_flow.md 鐵律 # 2026-04-09 Claude Sonnet 4.6 Asia/Taipei: 新增 Telegram Fallback (ADR-035) -# 架構: awoooi-webhook (主路徑) + telegram-direct (告警鏈路緊急旁路) +# 架構: awoooi-webhook (主路徑) + telegram-direct +# Agent99 對 AlertChainBroken_Alertmanager 採主動唯讀 poll，不是 receiver。 # telegram-direct 只允許處理 AWOOOI API / AlertChain 自身異常；一般 critical 必須走 AWOOOI API 治理鏈。 # 旁路目的地必須是 AwoooI SRE 戰情室；OPENCLAW_TG_CHAT_ID 只允許作缺值時的 fail-soft fallback。 # ⚠️ bot_token/chat_id 部署時由 secrets 替換，此檔為模板 @@ -37,7 +38,7 @@ route: - alertname=~"AWOOOIApiDown|AlertmanagerDown|AlertChainBroken_.*|AlertChainUnhealthy|NoAlertsReceived2Hours" receiver: 'telegram-direct' group_wait: 10s - repeat_interval: 30m + repeat_interval: 2h continue: true - matchers: - severity="critical" @@ -72,14 +73,22 @@ receivers: chat_id: SRE_GROUP_CHAT_ID_PLACEHOLDER parse_mode: 'HTML' message: | - 🚨 [Alertmanager Emergency] + 🚨 SRE BYPASS｜告警主鏈異常 {{ range .Alerts }} - ├ {{ .Labels.alertname }} - ├ 嚴重度: {{ .Labels.severity }} - ├ 主機: {{ .Labels.host }}{{ .Labels.instance }} - └ {{ .Annotations.summary }} + {{ .Labels.severity }}｜{{ .Labels.alertname }} + ├ 資產：{{ if .Labels.component }}{{ .Labels.component }}{{ else if .Labels.service }}{{ .Labels.service }}{{ else }}alert-chain{{ end }} + ├ 狀態：{{ .Annotations.summary }} + └ 影響：AWOOOI 主鏈 delivery / automation receipt 可信度下降 {{ end }} - ⚠️ AWOOOI API / 告警鏈路可能異常，此為 SRE 戰情室緊急旁路 + + 🤖 自動化證據 + ├ 判斷：Prometheus deterministic rule（未呼叫 LLM） + ├ Agent：Alertmanager emergency router + ├ 動作：僅送緊急旁路；未宣稱已修復 + ├ Executor：none + └ Verifier：source-specific delivery metric + AWOOOI E2E receipt pending + + 下一步：由 canonical typed route 建立受控修復與獨立驗證；同一 firing state 2 小時內不重送。 send_resolved: false inhibit_rules: diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 42a2dd517..5efcfde4a 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -15,6 +15,81 @@ groups: + # ========================================================================= + # AI provider cloud-edge availability (source + missing-series truth) + # ========================================================================= + - name: ollama_cloud_edge_availability + interval: 30s + rules: + - alert: OllamaGcpACloudEdgeUnavailable + expr: | + (probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"} == 0) + or absent(probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"}) + for: 2m + labels: + severity: critical + layer: ai-provider + component: ollama-gcp-a + team: ai + alert_category: ollama_failover + auto_repair: "false" + annotations: + summary: "GCP-A Ollama cloud-edge probe missing or unavailable" + description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts GCP-B next; no global provider mutation is authorized." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + + - alert: OllamaGcpBCloudEdgeUnavailable + expr: | + (probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"} == 0) + or absent(probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"}) + for: 2m + labels: + severity: critical + layer: ai-provider + component: ollama-gcp-b + team: ai + alert_category: ollama_failover + auto_repair: "false" + annotations: + summary: "GCP-B Ollama cloud-edge probe missing or unavailable" + description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts host111 next; no global provider mutation is authorized." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + + - alert: OllamaLocalUnavailable + expr: | + ollama_health_status{host="111"} == 0 + for: 2m + labels: + severity: warning + layer: ai-provider + component: ollama-local + host: "111" + canonical_asset_id: "ai-provider:ollama_local" + team: ai + alert_category: ai_provider + auto_repair: "true" + annotations: + summary: "Host111 Ollama local provider unavailable or missing" + description: "K3s-side provider health receipt proves Host111 unhealthy. Route only to the Host111 LaunchAgent Ansible PlayBook; verify independently from host120 and host121, and never probe or repair retired Host110 Ollama." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + + - alert: OllamaLocalHealthReceiptMissing + expr: absent(ollama_health_status{host="111"}) + for: 2m + labels: + severity: warning + layer: observability + component: ollama-health-receipt + host: "111" + canonical_asset_id: "signal:ollama_local_health_receipt" + team: ai + alert_category: monitoring_drift + auto_repair: "false" + annotations: + summary: "Host111 Ollama health receipt is missing" + description: "The K3s-side health signal is absent, so Host111 health is unknown rather than failed. Create a monitoring drift work item; do not run the Host111 repair PlayBook from missing-series evidence." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + # ========================================================================= # Full-stack recovery scorecard recording rules # ========================================================================= @@ -810,17 +885,23 @@ groups: rules: - alert: AlertChainBroken_Alertmanager expr: | - sum(rate(awoooi_webhook_requests_total{source="alertmanager",status!="success"}[5m])) - / sum(rate(awoooi_webhook_requests_total{source="alertmanager"}[5m])) > 0.1 - for: 10m + ( + sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) + / clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])), 1) + ) > 0.1 + and + sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) >= 5 + for: 5m labels: severity: critical - layer: k8s + layer: host + component: alertmanager team: platform - auto_repair: "false" + auto_repair: "true" + alert_category: "alert_chain_health" annotations: - summary: "Alertmanager Webhook 錯誤率 > 10%" - description: "告警鏈路可能斷裂，請執行 E2E 驗證" + summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" + description: "只使用 receiver=awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" - alert: AlertChainBroken_Sentry expr: | diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index aedfd6577..d1a39d21d 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -15,6 +15,81 @@ groups: + # ========================================================================= + # AI provider cloud-edge availability (source + missing-series truth) + # ========================================================================= + - name: ollama_cloud_edge_availability + interval: 30s + rules: + - alert: OllamaGcpACloudEdgeUnavailable + expr: | + (probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"} == 0) + or absent(probe_success{job="blackbox-http",provider="ollama_gcp_a",probe_origin="host110"}) + for: 2m + labels: + severity: critical + layer: ai-provider + component: ollama-gcp-a + team: ai + alert_category: ollama_failover + auto_repair: "false" + annotations: + summary: "GCP-A Ollama cloud-edge probe missing or unavailable" + description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts GCP-B next; no global provider mutation is authorized." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + + - alert: OllamaGcpBCloudEdgeUnavailable + expr: | + (probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"} == 0) + or absent(probe_success{job="blackbox-http",provider="ollama_gcp_b",probe_origin="host110"}) + for: 2m + labels: + severity: critical + layer: ai-provider + component: ollama-gcp-b + team: ai + alert_category: ollama_failover + auto_repair: "false" + annotations: + summary: "GCP-B Ollama cloud-edge probe missing or unavailable" + description: "Host110 cloud-edge probe failed or is absent. Per-request deterministic routing remains in the same AI-provider domain and attempts host111 next; no global provider mutation is authorized." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + + - alert: OllamaLocalUnavailable + expr: | + ollama_health_status{host="111"} == 0 + for: 2m + labels: + severity: warning + layer: ai-provider + component: ollama-local + host: "111" + canonical_asset_id: "ai-provider:ollama_local" + team: ai + alert_category: ai_provider + auto_repair: "true" + annotations: + summary: "Host111 Ollama local provider unavailable or missing" + description: "K3s-side provider health receipt proves Host111 unhealthy. Route only to the Host111 LaunchAgent Ansible PlayBook; verify independently from host120 and host121, and never probe or repair retired Host110 Ollama." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + + - alert: OllamaLocalHealthReceiptMissing + expr: absent(ollama_health_status{host="111"}) + for: 2m + labels: + severity: warning + layer: observability + component: ollama-health-receipt + host: "111" + canonical_asset_id: "signal:ollama_local_health_receipt" + team: ai + alert_category: monitoring_drift + auto_repair: "false" + annotations: + summary: "Host111 Ollama health receipt is missing" + description: "The K3s-side health signal is absent, so Host111 health is unknown rather than failed. Create a monitoring drift work item; do not run the Host111 repair PlayBook from missing-series evidence." + runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md" + # ========================================================================= # 主機層告警 (host_alerts) # ========================================================================= @@ -491,17 +566,23 @@ groups: rules: - alert: AlertChainBroken_Alertmanager expr: | - sum(rate(awoooi_webhook_requests_total{source="alertmanager",status!="success"}[5m])) - / sum(rate(awoooi_webhook_requests_total{source="alertmanager"}[5m])) > 0.1 - for: 10m + ( + sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) + / clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])), 1) + ) > 0.1 + and + sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) >= 5 + for: 5m labels: severity: critical - layer: k8s + layer: host + component: alertmanager team: platform - auto_repair: "false" + auto_repair: "true" + alert_category: "alert_chain_health" annotations: - summary: "Alertmanager Webhook 錯誤率 > 10%" - description: "告警鏈路可能斷裂，請執行 E2E 驗證" + summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" + description: "只使用 receiver=awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" - alert: AlertChainBroken_Sentry expr: | diff --git a/ops/monitoring/coverage_report.py b/ops/monitoring/coverage_report.py index a794939cb..8a032ac4c 100755 --- a/ops/monitoring/coverage_report.py +++ b/ops/monitoring/coverage_report.py @@ -83,7 +83,12 @@ def analyze_services(registry: dict) -> dict: "by_type": {}, "by_criticality": {}, "monitoring": { - "prometheus": {"covered": 0, "missing": []}, + "prometheus": { + "covered": 0, + "missing": [], + "source_declared": 0, + "generated_unactivated": [], + }, "sentry": {"covered": 0, "missing": []}, "otel": {"covered": 0, "missing": []}, "langfuse": {"covered": 0, "missing": []}, @@ -109,7 +114,23 @@ def analyze_services(registry: dict) -> dict: # 監控覆蓋 monitoring = svc.get("monitoring", {}) - for key in ["prometheus", "sentry", "otel", "langfuse"]: + prometheus_source_declared = bool( + monitoring.get("prometheus") + or monitoring.get("blackbox_target_enabled") + or monitoring.get("agent_verifier") + ) + if prometheus_source_declared: + stats["monitoring"]["prometheus"]["source_declared"] += 1 + if monitoring.get("runtime_verified") is True: + stats["monitoring"]["prometheus"]["covered"] += 1 + elif prometheus_source_declared: + stats["monitoring"]["prometheus"]["generated_unactivated"].append( + name + ) + else: + stats["monitoring"]["prometheus"]["missing"].append(name) + + for key in ["sentry", "otel", "langfuse"]: if monitoring.get(key): stats["monitoring"][key]["covered"] += 1 else: diff --git a/ops/monitoring/generate_monitoring.py b/ops/monitoring/generate_monitoring.py index 6c8c3014a..757cbfd77 100755 --- a/ops/monitoring/generate_monitoring.py +++ b/ops/monitoring/generate_monitoring.py @@ -87,7 +87,7 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]: scrape_configs.append(config) elif svc.get("type") == "docker": - # Docker 直接 scrape + # Host service exposing native Prometheus metrics. host = svc.get("host", "localhost") port = svc.get("port", 8080) config = { @@ -96,7 +96,7 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]: "targets": [f"{host}:{port}"], "labels": { "service": svc["name"], - "type": "docker", + "type": svc.get("type"), "owner": svc.get("owner", "unknown"), "criticality": svc.get("criticality", "P2") } @@ -144,25 +144,43 @@ def generate_blackbox_targets(registry: dict) -> list[dict]: for svc in registry.get("services", []): health_endpoint = svc.get("health_endpoint") health_type = svc.get("health_type", "http") + monitoring = svc.get("monitoring", {}) + + if ( + svc.get("type") == "ai-provider" + and monitoring.get("blackbox_target_enabled") is not True + ): + continue if health_endpoint and health_type == "http": if svc.get("type") == "k8s-deployment": # K8s 內部 DNS url = f"http://{svc['name']}.{svc.get('namespace', 'default')}.svc.cluster.local:{svc.get('port', 8080)}{health_endpoint}" - elif svc.get("type") == "docker": + elif svc.get("type") in {"docker", "ai-provider"}: host = svc.get("host", "localhost") port = svc.get("port", 8080) url = f"http://{host}:{port}{health_endpoint}" else: continue + labels = { + "service": svc["name"], + "criticality": svc.get("criticality", "P2"), + "owner": svc.get("owner", "unknown"), + } + if svc.get("type") == "ai-provider": + labels.update({ + "provider": svc["name"].replace("-", "_"), + "probe_origin": monitoring.get("probe_origin", "unresolved"), + "boundary": monitoring.get("data_boundary", "unresolved"), + "coverage_state": monitoring.get( + "coverage_state", + "generated_unactivated", + ), + }) targets.append({ "url": url, - "labels": { - "service": svc["name"], - "criticality": svc.get("criticality", "P2"), - "owner": svc.get("owner", "unknown") - } + "labels": labels, }) # API 端點 @@ -191,6 +209,9 @@ def validate_coverage(registry: dict) -> dict: report = { "total_services": 0, "monitored_services": 0, + "source_declared_services": 0, + "generated_unactivated": [], + "runtime_verified_services": 0, "coverage_percent": 0.0, "missing_prometheus": [], "missing_health_endpoint": [], @@ -202,8 +223,18 @@ def validate_coverage(registry: dict) -> dict: report["total_services"] += 1 monitoring = svc.get("monitoring", {}) - if monitoring.get("prometheus"): + source_declared = bool( + monitoring.get("prometheus") + or monitoring.get("blackbox_target_enabled") + or monitoring.get("agent_verifier") + ) + if source_declared: + report["source_declared_services"] += 1 + if monitoring.get("runtime_verified") is True: report["monitored_services"] += 1 + report["runtime_verified_services"] += 1 + elif source_declared: + report["generated_unactivated"].append(svc["name"]) else: report["missing_prometheus"].append(svc["name"]) @@ -258,8 +289,17 @@ def print_coverage_report(report: dict): print(" AWOOOI Monitoring Coverage Report") print("=" * 60) print(f"\n Total Services: {report['total_services']}") - print(f" Monitored: {report['monitored_services']}") - print(f" Coverage: {report['coverage_percent']}%") + print(f" Source Declared: {report['source_declared_services']}") + print(f" Runtime Verified: {report['runtime_verified_services']}") + print(f" Runtime Coverage: {report['coverage_percent']}%") + + if report["generated_unactivated"]: + print( + f"\n Generated/Declared But Runtime Pending " + f"({len(report['generated_unactivated'])}):" + ) + for svc in report["generated_unactivated"]: + print(f" - {svc}") if report["missing_prometheus"]: print(f"\n Missing Prometheus Monitoring ({len(report['missing_prometheus'])}):") diff --git a/ops/monitoring/generated/blackbox-targets-generated.yaml b/ops/monitoring/generated/blackbox-targets-generated.yaml index c4dac3b55..62b167f96 100644 --- a/ops/monitoring/generated/blackbox-targets-generated.yaml +++ b/ops/monitoring/generated/blackbox-targets-generated.yaml @@ -17,27 +17,30 @@ criticality: P0 owner: devops-team service: prometheus - url: http://prometheus.monitoring.svc.cluster.local:9090/-/ready + url: http://192.168.0.110:9090/-/ready - labels: criticality: P0 owner: devops-team service: alertmanager - url: http://alertmanager.monitoring.svc.cluster.local:9093/-/ready + url: http://192.168.0.110:9093/-/ready - labels: + boundary: cloud_edge_availability_only + coverage_state: source_candidate_runtime_pending criticality: P0 owner: ai-team + probe_origin: host110 + provider: ollama_gcp_a service: ollama-gcp-a - url: http://192.168.0.110:11435/api/tags + url: http://34.143.170.20:11434/api/tags - labels: + boundary: cloud_edge_availability_only + coverage_state: source_candidate_runtime_pending criticality: P0 owner: ai-team + probe_origin: host110 + provider: ollama_gcp_b service: ollama-gcp-b - url: http://192.168.0.110:11436/api/tags -- labels: - criticality: P0 - owner: ai-team - service: ollama-local - url: http://192.168.0.110:11437/api/tags + url: http://34.21.145.224:11434/api/tags - labels: criticality: P0 owner: ai-team @@ -47,7 +50,7 @@ criticality: P2 owner: devops-team service: signoz-ui - url: http://192.168.0.188:3301/ + url: http://192.168.0.110:8080/api/v1/health - labels: criticality: P1 owner: devops-team diff --git a/ops/monitoring/generated/prometheus-scrape-generated.yaml b/ops/monitoring/generated/prometheus-scrape-generated.yaml index 5c33abfa3..8c09e3d78 100644 --- a/ops/monitoring/generated/prometheus-scrape-generated.yaml +++ b/ops/monitoring/generated/prometheus-scrape-generated.yaml @@ -67,15 +67,151 @@ scrape_configs: - source_labels: - __meta_kubernetes_pod_name target_label: pod -- job_name: alertmanager +- job_name: argocd-applicationset-controller kubernetes_sd_configs: - namespaces: names: - - monitoring + - argocd role: pod relabel_configs: - action: keep - regex: alertmanager + regex: argocd-applicationset-controller + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: argocd-dex-server + kubernetes_sd_configs: + - namespaces: + names: + - argocd + role: pod + relabel_configs: + - action: keep + regex: argocd-dex-server + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: argocd-notifications-controller + kubernetes_sd_configs: + - namespaces: + names: + - argocd + role: pod + relabel_configs: + - action: keep + regex: argocd-notifications-controller + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: argocd-redis + kubernetes_sd_configs: + - namespaces: + names: + - argocd + role: pod + relabel_configs: + - action: keep + regex: argocd-redis + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: argocd-repo-server + kubernetes_sd_configs: + - namespaces: + names: + - argocd + role: pod + relabel_configs: + - action: keep + regex: argocd-repo-server + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: awoooi-api + kubernetes_sd_configs: + - namespaces: + names: + - awoooi-dev + role: pod + relabel_configs: + - action: keep + regex: awoooi-api + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: kube-state-metrics + kubernetes_sd_configs: + - namespaces: + names: + - kube-state-metrics + role: pod + relabel_configs: + - action: keep + regex: kube-state-metrics + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: event-exporter + kubernetes_sd_configs: + - namespaces: + names: + - observability + role: pod + relabel_configs: + - action: keep + regex: event-exporter + source_labels: + - __meta_kubernetes_pod_label_app + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod +- job_name: velero + kubernetes_sd_configs: + - namespaces: + names: + - velero + role: pod + relabel_configs: + - action: keep + regex: velero source_labels: - __meta_kubernetes_pod_label_app - source_labels: @@ -84,17 +220,6 @@ scrape_configs: - source_labels: - __meta_kubernetes_pod_name target_label: pod -- job_name: ollama - static_configs: - - labels: - criticality: P0 - owner: ai-team - service: ollama - type: docker - targets: - - 192.168.0.110:11435 - - 192.168.0.110:11436 - - 192.168.0.110:11437 - job_name: openclaw static_configs: - labels: @@ -122,24 +247,6 @@ scrape_configs: type: docker targets: - 192.168.0.188:5432 -- job_name: signoz-collector - static_configs: - - labels: - criticality: P1 - owner: devops-team - service: signoz-collector - type: docker - targets: - - 192.168.0.188:24317 -- job_name: signoz-ui - static_configs: - - labels: - criticality: P2 - owner: devops-team - service: signoz-ui - type: docker - targets: - - 192.168.0.188:3301 - job_name: clickhouse static_configs: - labels: diff --git a/ops/monitoring/grafana/dashboards/ollama_failover.json b/ops/monitoring/grafana/dashboards/ollama_failover.json index b185e5650..54184fee8 100644 --- a/ops/monitoring/grafana/dashboards/ollama_failover.json +++ b/ops/monitoring/grafana/dashboards/ollama_failover.json @@ -35,7 +35,7 @@ "annotations": { "list": [] }, - "description": "Ollama 容災監控 — 可用性、推理延遲、AI 路由分布、Failover/Recovery 觸發 | P2.3 2026-04-26 台北時區", + "description": "Ollama 容災監控 — GCP-A/B cloud-edge probe truth、推理延遲、AI 路由分布、Failover/Recovery 觸發。Host111 由 K3s host120/121 independent verifier 補證，尚未取得 runtime receipt 時不得假綠。", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -81,14 +81,23 @@ }, "textMode": "auto" }, - "title": "Ollama 可用性", - "description": "up{job=~\"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111\"} × 100\n- 綠色 100% = 主機在線\n- 紅色 0% = 主機離線（容災應已觸發）\n\n資料來源: Prometheus scrape job ollama_gcp_a / ollama_gcp_b / ollama_local", + "title": "GCP-A/B Cloud Edge 可用性", + "description": "GCP-A/B 僅顯示 host110 blackbox /api/tags 的 probe_success；absent series 明確轉為 0，避免 blackbox-http 其他網站健康時假綠。\n\nHost111 不從已退役的 host110 proxy/probe；狀態為 independent K3s verifier pending，需 host120/121 verifier runtime receipt 才能另行標綠。", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "up{job=~\"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111\"} * 100", - "legendFormat": "{{ job }}", + "expr": "100 * (max by (provider) (probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_a\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}) or label_replace(0 * absent(probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_a\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}), \"provider\", \"ollama_gcp_a\", \"\", \"\"))", + "legendFormat": "GCP-A", "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "100 * (max by (provider) (probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_b\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}) or label_replace(0 * absent(probe_success{job=\"blackbox-http\",provider=\"ollama_gcp_b\",probe_origin=\"host110\",boundary=\"cloud_edge_availability_only\"}), \"provider\", \"ollama_gcp_b\", \"\", \"\"))", + "legendFormat": "GCP-B", + "refId": "B" } ], "type": "stat" diff --git a/ops/monitoring/ollama_health_rules.yaml b/ops/monitoring/ollama_health_rules.yaml index b6e653cac..af3bd4cda 100644 --- a/ops/monitoring/ollama_health_rules.yaml +++ b/ops/monitoring/ollama_health_rules.yaml @@ -5,6 +5,15 @@ # 部署目標: 與 alerts-unified.yml 一起部署到 192.168.0.110:/home/wooo/monitoring/alerts.yml # 部署方式: 手動合併至 alerts-unified.yml，或 scripts/ops/deploy-alerts.sh 支援多檔時直接引用 # +# Legacy policy sunset (2026-07-16): +# superseded_by=global_product_governance_v2 +# OllamaInstanceDown 已移除。它依賴不存在的 ollama_gcp_a / ollama_gcp_b / +# ollama_local / ollama_111 native scrape jobs，會在 series 缺失時假綠。 +# Active source truth is ops/monitoring/alerts-unified.yml: +# - OllamaGcpACloudEdgeUnavailable +# - OllamaGcpBCloudEdgeUnavailable +# Host111 不從退役 host110 probe；由 K3s host120/121 independent verifier 補證。 +# # 標籤規範 (對齊 alerts-unified.yml): # layer: ai-provider # team: ai @@ -26,27 +35,6 @@ groups: interval: 30s rules: - # ----------------------------------------------------------------------- - # 🔴 [ACTIVE] Ollama 主機離線 - # metric: up{job=~"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111"} - # 前置條件: Prometheus scrape job 命名對齊 ADR-110 provider pool - # (設定位於 ops/monitoring/generated/prometheus-scrape-generated.yaml) - # ----------------------------------------------------------------------- - - alert: OllamaInstanceDown - expr: up{job=~"ollama_gcp_a|ollama_gcp_b|ollama_local|ollama_111"} == 0 - for: 2m - labels: - severity: critical - layer: ai-provider - team: ai - auto_repair: "false" - alert_category: "ollama_failover" - annotations: - summary: "Ollama {{ $labels.job }} 離線 ({{ $labels.instance }})" - description: "Prometheus 探測 Ollama {{ $labels.job }} 失敗超過 2 分鐘。預期容災應已觸發，路由已切 Gemini。" - runbook: "docs/runbooks/RUNBOOK-OLLAMA-FAILOVER.md#ollama-instance-down" - action: "curl http://34.143.170.20:11434/api/tags（GCP-A）或 curl http://34.21.145.224:11434/api/tags（GCP-B）或 ssh wooo@192.168.0.111 'systemctl status ollama'（Local 後備）" - # ----------------------------------------------------------------------- # 🟡 [ACTIVE] Failover 觸發頻率過高 # metric: ollama_failover_triggered_total{from_provider,to_provider} diff --git a/ops/monitoring/service-registry.yaml b/ops/monitoring/service-registry.yaml index ffc41e6fe..3cb416b9d 100644 --- a/ops/monitoring/service-registry.yaml +++ b/ops/monitoring/service-registry.yaml @@ -100,12 +100,13 @@ services: # --- Prometheus --- - name: prometheus - type: k8s-deployment - namespace: monitoring + type: docker + host: 192.168.0.110 port: 9090 health_endpoint: /-/ready monitoring: - prometheus: false # 自己監控自己會循環 + prometheus: true + prometheus_scrape: false # canonical self-scrape job is managed explicitly sentry: false alerts: - service_down @@ -114,15 +115,21 @@ services: # --- Alertmanager --- - name: alertmanager - type: k8s-deployment - namespace: monitoring + type: docker + host: 192.168.0.110 port: 9093 health_endpoint: /-/ready monitoring: prometheus: true + prometheus_scrape: false # alertmanager-runtime job owns exact delivery truth sentry: false alerts: - service_down + - delivery_error_rate + auto_repair: + enabled: true + actions: + - ansible:110-alertmanager-delivery-recovery owner: devops-team criticality: P0 @@ -241,17 +248,23 @@ services: criticality: P1 # ============================================================================= -# Docker 容器 (192.168.0.188 - AI/Web 中心) +# AI Provider 與 Docker 服務 # ============================================================================= - # --- Ollama LLM --- - - name: ollama - type: docker - host: 192.168.0.188 + # --- Ollama provider pool（host110 proxy 已於 2026-07-16 退役）--- + - name: ollama-gcp-a + type: ai-provider + host: 34.143.170.20 port: 11434 health_endpoint: /api/tags monitoring: - prometheus: true + prometheus: false + prometheus_scrape: false + blackbox_target_enabled: true + coverage_state: source_candidate_runtime_pending + probe_origin: host110 + data_boundary: cloud_edge_availability_only + runtime_verified: false sentry: false otel: false alerts: @@ -261,7 +274,63 @@ services: auto_repair: enabled: true actions: - - restart_container + - route_same_domain_next_hop + owner: ai-team + criticality: P0 + + - name: ollama-gcp-b + type: ai-provider + host: 34.21.145.224 + port: 11434 + health_endpoint: /api/tags + monitoring: + prometheus: false + prometheus_scrape: false + blackbox_target_enabled: true + coverage_state: source_candidate_runtime_pending + probe_origin: host110 + data_boundary: cloud_edge_availability_only + runtime_verified: false + sentry: false + otel: false + alerts: + - service_down + - inference_timeout + - model_load_failed + auto_repair: + enabled: true + actions: + - route_same_domain_next_hop + owner: ai-team + criticality: P0 + + - name: ollama-local + type: ai-provider + host: 192.168.0.111 + port: 11434 + health_endpoint: /api/tags + runtime_manager: launchagent + runtime_unit: com.momo.ollama111-allow-proxy + monitoring: + prometheus: false + prometheus_scrape: false + blackbox_target_enabled: false + agent_verifier: true + coverage_state: k3s_path_verifier_runtime_pending + probe_origins: + - host120 + - host121 + runtime_verified: false + sentry: false + otel: false + alerts: + - service_down + - inference_timeout + - model_load_failed + auto_repair: + enabled: true + actions: + - ansible:111-ollama-fallback owner: ai-team criticality: P0 @@ -425,24 +494,6 @@ services: owner: ai-team criticality: P2 - # --- GitHub Actions Runner --- - - name: github-runner - type: systemd - host: 192.168.0.110 - service_name: actions.runner.owenhytsai-awoooi.awoooi-110.service - monitoring: - prometheus: true - sentry: false - alerts: - - runner_offline - - job_stuck - auto_repair: - enabled: true - actions: - - restart_service - owner: devops-team - criticality: P0 - # ============================================================================= # 主機節點 # ============================================================================= @@ -759,12 +810,6 @@ alert_templates: severity: warning auto_repair: switch_model - runner_offline: - expr: 'github_runner_status == 0' - for: 5m - severity: critical - auto_repair: restart_service - # --- NVIDIA Nemotron 告警 (Phase 20) --- # 2026-03-29 ogt: ADR-036 新增 circuit_breaker_open: diff --git a/ops/monitoring/validate_coverage.py b/ops/monitoring/validate_coverage.py index a76199cb8..3bf20bce8 100755 --- a/ops/monitoring/validate_coverage.py +++ b/ops/monitoring/validate_coverage.py @@ -181,7 +181,20 @@ def validate_registry(registry: dict) -> ValidationResult: total = len(services) if services else 1 coverage = { - 'prometheus': sum(1 for s in services if s.get('monitoring', {}).get('prometheus')) / total, + 'prometheus_source_declared': sum( + 1 + for s in services + if ( + s.get('monitoring', {}).get('prometheus') + or s.get('monitoring', {}).get('blackbox_target_enabled') + or s.get('monitoring', {}).get('agent_verifier') + ) + ) / total, + 'prometheus_runtime_verified': sum( + 1 + for s in services + if s.get('monitoring', {}).get('runtime_verified') is True + ) / total, 'sentry': sum(1 for s in services if s.get('monitoring', {}).get('sentry')) / total, 'otel': sum(1 for s in services if s.get('monitoring', {}).get('otel')) / total, 'alerts': sum(1 for s in services if s.get('alerts')) / total, diff --git a/ops/nginx/deploy-ollama-proxy-110.sh b/ops/nginx/deploy-ollama-proxy-110.sh old mode 100755 new mode 100644 index 8a183e7dc..d1a466e7c --- a/ops/nginx/deploy-ollama-proxy-110.sh +++ b/ops/nginx/deploy-ollama-proxy-110.sh @@ -1,121 +1,10 @@ -#!/bin/bash -# GCP Ollama Nginx Proxy 部署腳本 (110 手動執行) -# ADR-110 三層容災 — 讓 K3s 透過內網存取 GCP Ollama -# 執行: ssh wooo@192.168.0.110 'sudo bash -s' < deploy-ollama-proxy-110.sh - +#!/usr/bin/env bash +# superseded_by=global_product_governance_v2 +# owner=AssetIdentity; expiry=2026-07-16 +# replacement=direct GCP-A/GCP-B/host111 provider identities +# verifier=apps/api/tests/test_ollama_prod_manifest_order.py set -euo pipefail -echo "🚀 部署 GCP Ollama Nginx Proxy (110)..." - -# 配置內容 -NGINX_CONF="/etc/nginx/sites-enabled/110-ollama-proxy.conf" - -# 備份現有配置 -if [ -f "$NGINX_CONF" ]; then - echo "📦 備份現有配置..." - cp "$NGINX_CONF" "${NGINX_CONF}.backup.$(date +%Y%m%d%H%M%S)" -fi - -# 寫入 nginx 配置 -echo "📝 寫入 nginx 配置..." -cat > "$NGINX_CONF" << 'EOF' -# 110 Ollama GCP Proxy — ADR-110 三層容災 -# 讓 K3s 叢集內可透過內網 110 存取 GCP 外網 Ollama - -# ============================================================ -# Ollama GCP-A Primary (port 11435 → 34.143.170.20:11434) -# ============================================================ -server { - listen 11435; - listen [::]:11435; - server_name _; - - access_log /var/log/nginx/ollama-gcp-a-access.log; - error_log /var/log/nginx/ollama-gcp-a-error.log warn; - - location / { - proxy_pass http://34.143.170.20:11434; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # Ollama 推理可能較慢，給較長超時 - proxy_connect_timeout 10s; - proxy_send_timeout 300s; - proxy_read_timeout 300s; - - # 支援 streaming response - proxy_buffering off; - proxy_cache off; - } - - # 健康檢查端點 - location /nginx-health { - access_log off; - return 200 "Ollama GCP-A Proxy OK\n"; - add_header Content-Type text/plain; - } -} - -# ============================================================ -# Ollama GCP-B Secondary (port 11436 → 34.21.145.224:11434) -# ============================================================ -server { - listen 11436; - listen [::]:11436; - server_name _; - - access_log /var/log/nginx/ollama-gcp-b-access.log; - error_log /var/log/nginx/ollama-gcp-b-error.log warn; - - location / { - proxy_pass http://34.21.145.224:11434; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_connect_timeout 10s; - proxy_send_timeout 300s; - proxy_read_timeout 300s; - - proxy_buffering off; - proxy_cache off; - } - - location /nginx-health { - access_log off; - return 200 "Ollama GCP-B Proxy OK\n"; - add_header Content-Type text/plain; - } -} -EOF - -# 測試 nginx 配置 -echo "🧪 測試 nginx 配置..." -nginx -t - -# 重載 nginx -echo "🔄 重載 nginx..." -systemctl reload nginx - -# 驗證端口監聽 -echo "🔍 驗證端口監聽..." -sleep 2 -ss -tlnp | grep -E '11435|11436' || true - -# 本地測試 -echo "🌐 本地測試 proxy..." -echo "測試 GCP-A proxy (11435)..." -curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:11435/api/tags || echo "連線失敗" -echo "" - -echo "測試 GCP-B proxy (11436)..." -curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:11436/api/tags || echo "連線失敗" -echo "" - -echo "✅ 部署完成！" -echo "" -echo "下一步：" -echo "1. 從 K3s node 測試: curl http://192.168.0.110:11435/api/tags" -echo "2. 修改 K8s ConfigMap 指向 110:11435/11436" -echo "3. 重啟 awoooi-api deployment" +printf '%s\n' \ + 'BLOCKED: host110 Ollama transport is retired; this script cannot deploy it.' >&2 +exit 78 diff --git a/scripts/generate_monitoring.py b/scripts/generate_monitoring.py index f2fd7e63e..e1a162f20 100644 --- a/scripts/generate_monitoring.py +++ b/scripts/generate_monitoring.py @@ -45,7 +45,41 @@ EXPECTED_JOBS = { "prometheus": "Prometheus self-scrape", "blackbox-http": "Blackbox HTTP probe", "blackbox-tcp": "Blackbox TCP probe", - "github-actions": "GitHub Actions exporter", + "gitea-native": "Gitea native metrics", +} + +# Jobs alone are not sufficient evidence for multi-target scrape jobs. A +# healthy website target can keep ``blackbox-http`` green while both AI +# provider probes are absent. Keep the exact cloud-edge identities here so +# source/runtime reconciliation fails closed on missing, down, or duplicated +# provider targets. +EXPECTED_TARGETS = { + "ollama_gcp_a_cloud_edge": { + "description": "GCP-A Ollama cloud-edge /api/tags from host110", + "labels": { + "job": "blackbox-http", + "provider": "ollama_gcp_a", + "probe_origin": "host110", + "boundary": "cloud_edge_availability_only", + "instance": "http://34.143.170.20:11434/api/tags", + }, + }, + "ollama_gcp_b_cloud_edge": { + "description": "GCP-B Ollama cloud-edge /api/tags from host110", + "labels": { + "job": "blackbox-http", + "provider": "ollama_gcp_b", + "probe_origin": "host110", + "boundary": "cloud_edge_availability_only", + "instance": "http://34.21.145.224:11434/api/tags", + }, + }, +} + +# GitHub is globally frozen. A legacy exporter still appearing UP must be +# reported as policy drift, not counted as harmless surplus coverage. +FORBIDDEN_JOBS = { + "github-actions": "retired GitHub exporter must be removed from production", } # 允許 down 的 target (已知問題，不影響覆蓋率計算) @@ -116,6 +150,42 @@ def analyze_targets(targets_data: dict) -> dict: return jobs +def analyze_expected_targets(targets_data: dict) -> dict: + """Resolve exact multi-target identities without allowing job-level false green.""" + active = targets_data.get("activeTargets", []) + target_status: dict[str, dict] = {} + + for target_id, expected in EXPECTED_TARGETS.items(): + expected_labels = expected["labels"] + matches = [ + target + for target in active + if all( + target.get("labels", {}).get(label) == value + for label, value in expected_labels.items() + ) + ] + health_values = [str(target.get("health", "unknown")) for target in matches] + if not matches: + state = "missing" + elif len(matches) != 1: + state = "ambiguous" + elif health_values == ["up"]: + state = "up" + else: + state = "down" + + target_status[target_id] = { + "description": expected["description"], + "expected_labels": expected_labels, + "match_count": len(matches), + "health": health_values, + "state": state, + } + + return target_status + + def build_report(jobs: dict) -> dict: """建立覆蓋率報告""" total_jobs = len(jobs) @@ -132,6 +202,7 @@ def build_report(jobs: dict) -> dict: expected_covered = sum(1 for j in EXPECTED_JOBS if j in jobs and jobs[j]["up"]) coverage_pct = round(expected_covered / len(EXPECTED_JOBS) * 100, 1) + forbidden_jobs_present = [job for job in FORBIDDEN_JOBS if job in jobs] return { "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), @@ -143,23 +214,65 @@ def build_report(jobs: dict) -> dict: "down_jobs": down_jobs, "real_down_jobs": len(real_down_jobs), "expected_coverage_pct": coverage_pct, + "forbidden_jobs_present": len(forbidden_jobs_present), }, "jobs": jobs, "expected_jobs": EXPECTED_JOBS, "known_down": KNOWN_DOWN_TARGETS, "real_down_jobs": list(real_down_jobs.keys()), "missing_expected": [j for j in EXPECTED_JOBS if j not in jobs], + "forbidden_jobs": FORBIDDEN_JOBS, + "forbidden_jobs_present": forbidden_jobs_present, } def build_report_from_targets(targets_data: dict) -> dict: """從 Prometheus targets API payload 建立覆蓋率報告""" - return build_report(analyze_targets(targets_data)) + report = build_report(analyze_targets(targets_data)) + expected_targets = analyze_expected_targets(targets_data) + missing_expected_targets = [ + target_id + for target_id, target in expected_targets.items() + if target["state"] == "missing" + ] + down_expected_targets = [ + target_id + for target_id, target in expected_targets.items() + if target["state"] == "down" + ] + ambiguous_expected_targets = [ + target_id + for target_id, target in expected_targets.items() + if target["state"] == "ambiguous" + ] + report["expected_targets"] = expected_targets + report["missing_expected_targets"] = missing_expected_targets + report["down_expected_targets"] = down_expected_targets + report["ambiguous_expected_targets"] = ambiguous_expected_targets + report["summary"].update( + { + "expected_target_count": len(expected_targets), + "expected_targets_up": sum( + target["state"] == "up" for target in expected_targets.values() + ), + "expected_targets_missing": len(missing_expected_targets), + "expected_targets_down": len(down_expected_targets), + "expected_targets_ambiguous": len(ambiguous_expected_targets), + } + ) + return report def report_needs_stabilization(report: dict) -> bool: """是否需要重查，避免 post-deploy 瞬間 scrape 狀態造成 false red.""" - return bool(report["real_down_jobs"] or report["missing_expected"]) + return bool( + report["real_down_jobs"] + or report["missing_expected"] + or report["missing_expected_targets"] + or report["down_expected_targets"] + or report["ambiguous_expected_targets"] + or report["forbidden_jobs_present"] + ) def stabilization_reason(report: dict) -> str: @@ -168,6 +281,24 @@ def stabilization_reason(report: dict) -> str: parts.append(f"real_down={','.join(report['real_down_jobs'])}") if report["missing_expected"]: parts.append(f"missing_expected={','.join(report['missing_expected'])}") + if report["missing_expected_targets"]: + parts.append( + "missing_expected_targets=" + f"{','.join(report['missing_expected_targets'])}" + ) + if report["down_expected_targets"]: + parts.append( + f"down_expected_targets={','.join(report['down_expected_targets'])}" + ) + if report["ambiguous_expected_targets"]: + parts.append( + "ambiguous_expected_targets=" + f"{','.join(report['ambiguous_expected_targets'])}" + ) + if report["forbidden_jobs_present"]: + parts.append( + f"forbidden_jobs_present={','.join(report['forbidden_jobs_present'])}" + ) return "; ".join(parts) if parts else "stable" @@ -226,18 +357,26 @@ def print_human_report(report: dict) -> None: """輸出人可讀格式報告""" s = report["summary"] print(f"\n{'='*60}") - print(f" AWOOOI 監控覆蓋率報告") + print(" AWOOOI 監控覆蓋率報告") print(f" 生成時間: {report['generated_at']}") print(f"{'='*60}") - print(f"\n📊 總覽") + print("\n📊 總覽") print(f" Jobs 總數: {s['total_jobs']}") print(f" 全部 UP: {s['up_jobs']}") print(f" 部分 UP: {s['partial_jobs']}") print(f" 全部 DOWN: {s['down_jobs']}") print(f" 真實問題 (非已知): {s['real_down_jobs']}") print(f" 預期覆蓋率: {s['expected_coverage_pct']}% ({COVERAGE_THRESHOLD}% 門檻)") + print( + " 精確 Target: " + f"{s['expected_targets_up']}/{s['expected_target_count']} UP " + f"(missing={s['expected_targets_missing']}, " + f"down={s['expected_targets_down']}, " + f"ambiguous={s['expected_targets_ambiguous']})" + ) + print(f" 禁止 Job 漂移: {s['forbidden_jobs_present']}") - print(f"\n✅ 預期服務狀態") + print("\n✅ 預期服務狀態") for job, desc in report["expected_jobs"].items(): jobs = report["jobs"] if job not in jobs: @@ -256,24 +395,42 @@ def print_human_report(report: dict) -> None: if job in report["jobs"] and report["jobs"][job]["down"] ] if known_down_present: - print(f"\n⚠️ 已知 DOWN (不影響覆蓋率)") + print("\n⚠️ 已知 DOWN (不影響覆蓋率)") for job, reason in known_down_present: print(f" {job:<30} {reason}") if report["real_down_jobs"]: - print(f"\n🔴 需處理的 DOWN targets") + print("\n🔴 需處理的 DOWN targets") for job in report["real_down_jobs"]: instances = report["jobs"][job].get("down", []) print(f" {job}: {', '.join(instances)}") if report["missing_expected"]: - print(f"\n🔴 缺少預期服務監控") + print("\n🔴 缺少預期服務監控") for job in report["missing_expected"]: print(f" {job}: {report['expected_jobs'][job]}") + if report["forbidden_jobs_present"]: + print("\n🔴 禁止的 legacy monitoring job 仍存在") + for job in report["forbidden_jobs_present"]: + print(f" {job}: {report['forbidden_jobs'][job]}") + + if ( + report["missing_expected_targets"] + or report["down_expected_targets"] + or report["ambiguous_expected_targets"] + ): + print("\n🔴 精確 Target identity/readback 未通過") + for target_id, target in report["expected_targets"].items(): + if target["state"] != "up": + print( + f" {target_id}: {target['state']} " + f"matches={target['match_count']} {target['description']}" + ) + stabilization = report.get("stabilization") if stabilization and stabilization["attempt"] > 1: - print(f"\n⏱️ Prometheus target 穩定化") + print("\n⏱️ Prometheus target 穩定化") print( " " f"{stabilization['status']} after " @@ -282,7 +439,19 @@ def print_human_report(report: dict) -> None: pct = s["expected_coverage_pct"] threshold = COVERAGE_THRESHOLD - if pct >= threshold and not report["real_down_jobs"]: + exact_targets_healthy = not ( + report["missing_expected_targets"] + or report["down_expected_targets"] + or report["ambiguous_expected_targets"] + ) + monitoring_contract_healthy = ( + exact_targets_healthy and not report["forbidden_jobs_present"] + ) + if ( + pct >= threshold + and not report["real_down_jobs"] + and monitoring_contract_healthy + ): print(f"\n✅ 監控健康: 覆蓋率 {pct}% >= {threshold}%，無真實問題\n") elif pct >= threshold: print(f"\n⚠️ 覆蓋率達標 ({pct}%)，但有 {s['real_down_jobs']} 個真實 DOWN 需處理\n") @@ -335,7 +504,21 @@ def main() -> None: if args.check: pct = report["summary"]["expected_coverage_pct"] real_down = report["summary"]["real_down_jobs"] - if pct < COVERAGE_THRESHOLD or real_down > 0: + exact_target_failures = sum( + report["summary"][key] + for key in ( + "expected_targets_missing", + "expected_targets_down", + "expected_targets_ambiguous", + ) + ) + forbidden_jobs = report["summary"]["forbidden_jobs_present"] + if ( + pct < COVERAGE_THRESHOLD + or real_down > 0 + or exact_target_failures > 0 + or forbidden_jobs > 0 + ): sys.exit(1) diff --git a/scripts/ops/deploy-alertmanager-config.sh b/scripts/ops/deploy-alertmanager-config.sh index 2f71e2a25..012db8d77 100755 --- a/scripts/ops/deploy-alertmanager-config.sh +++ b/scripts/ops/deploy-alertmanager-config.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash # Render and deploy ops/alertmanager/alertmanager.yml to the 110 Docker Alertmanager. # -# This script keeps the live direct-Telegram emergency route aligned with Git: +# This script keeps the direct-Telegram route aligned: # - inject Telegram bot token and SRE group chat id from K8s secret or env # - validate with amtool before touching the live config -# - back up the live file -# - keep the bind-mounted live file inode and readable permissions intact +# - create a 0600 rollback copy without a world-readable intermediate +# - keep the bind-mounted live file inode and protect it as runtime-UID 0400 +# - verify container readability before reload; fail closed with secure rollback # - reload Alertmanager with SIGHUP # # Usage: @@ -105,14 +106,20 @@ target = Path(sys.argv[2]) text = template.read_text() text = text.replace("TELEGRAM_BOT_TOKEN_PLACEHOLDER", os.environ["TELEGRAM_BOT_TOKEN"]) text = text.replace("SRE_GROUP_CHAT_ID_PLACEHOLDER", os.environ["SRE_GROUP_CHAT_ID"]) -if "TELEGRAM_BOT_TOKEN_PLACEHOLDER" in text or "SRE_GROUP_CHAT_ID_PLACEHOLDER" in text: +if any( + placeholder in text + for placeholder in ( + "TELEGRAM_BOT_TOKEN_PLACEHOLDER", + "SRE_GROUP_CHAT_ID_PLACEHOLDER", + ) +): raise SystemExit("unreplaced secret placeholder remains in rendered config") target.write_text(text) PY log "Validating rendered config with live Alertmanager amtool on ${TARGET_HOST}" ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \ - "docker exec -i alertmanager sh -c 'cat >/tmp/alertmanager-rendered.yml && amtool check-config /tmp/alertmanager-rendered.yml'" \ + "docker exec -i alertmanager amtool check-config /dev/stdin" \ < "$tmp_rendered" if [[ "$DRY_RUN" == "--dry-run" ]]; then @@ -127,18 +134,115 @@ ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \ ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" "bash -s" <&2 + exit 1 +} +[[ -f "\$staged" && ! -L "\$staged" ]] || { + echo 'ERROR: staged Alertmanager config must be a regular non-symlink file' >&2 + exit 1 +} +sudo -n true + +# Read the effective host UID/GID from the container process itself. This also +# works when Docker user namespaces remap container identities. +container_pid="\$(docker inspect --format '{{.State.Pid}}' "\$container")" +[[ "\$container_pid" =~ ^[1-9][0-9]*$ ]] || { + echo 'ERROR: Alertmanager container host PID is unavailable' >&2 + exit 1 +} +status_path="/proc/\${container_pid}/status" +[[ -r "\$status_path" ]] || { + echo 'ERROR: Alertmanager runtime identity cannot be verified' >&2 + exit 1 +} +runtime_uid="\$(awk '/^Uid:/{print \$2; exit}' "\$status_path")" +runtime_gid="\$(awk '/^Gid:/{print \$2; exit}' "\$status_path")" +[[ "\$runtime_uid" =~ ^[0-9]+$ && "\$runtime_gid" =~ ^[0-9]+$ ]] || { + echo 'ERROR: Alertmanager runtime UID/GID is invalid' >&2 + exit 1 +} + backup="\${target}.bak.\$(date +%Y%m%d%H%M%S)" -cp "\$target" "\$backup" -# Alertmanager bind-mounts a single file. Keep the existing inode instead of mv'ing -# a replacement over it, then restore readable permissions for the container user. -cat /tmp/alertmanager.yml.new > "\$target" -chmod 0644 "\$target" -rm -f /tmp/alertmanager.yml.new -docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml -docker kill -s HUP alertmanager >/dev/null +operator_uid="\$(id -u)" +operator_gid="\$(id -g)" +umask 077 +# install applies 0600 while creating the copy; cp followed by chmod would leave +# a secret-bearing backup readable during the gap. +sudo -n install -m 0600 -o "\$operator_uid" -g "\$operator_gid" "\$target" "\$backup" +[[ "\$(stat -c '%a' "\$backup")" == '600' ]] || { + echo 'ERROR: secure Alertmanager rollback copy was not created' >&2 + exit 1 +} + +# Harden legacy copies produced by older versions of this script without +# reading or printing their contents. +target_dir="\$(dirname "\$target")" +target_base="\$(basename "\$target")" +while IFS= read -r -d '' legacy_backup; do + sudo -n chmod 0600 "\$legacy_backup" +done < <(find "\$target_dir" -maxdepth 1 -type f -name "\${target_base}.bak.*" -print0) + +reload_attempted=false +rollback_secure() { + local rc="\${1:-1}" + trap - ERR + set +e + if [[ -f "\$backup" ]]; then + # Remove access before restoring bytes so rollback cannot briefly recreate + # the legacy world-readable state. + sudo -n chmod 0000 "\$target" + sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target" + sudo -n chmod 0400 "\$target" + sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$backup" "\$target" + rollback_verified=false + if docker exec "\$container" sh -ec \ + 'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' \ + >/dev/null 2>&1; then + rollback_verified=true + fi + if [[ "\$reload_attempted" == 'true' && "\$rollback_verified" == 'true' ]]; then + docker kill -s HUP "\$container" >/dev/null 2>&1 + fi + fi + rm -f "\$staged" + echo 'ERROR: Alertmanager deploy failed; secure rollback attempted' >&2 + exit "\$rc" +} +trap 'rollback_secure \$?' ERR + +# Alertmanager bind-mounts this single file. Preserve its inode and remove all +# access before changing ownership, so no secret bytes are ever written while +# legacy group/other read bits remain. Validate the permission model against the +# running container before replacing the old config bytes. +sudo -n chmod 0000 "\$target" +sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target" +sudo -n chmod 0400 "\$target" +[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]] +[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]] +[[ "\$(stat -c '%a' "\$target")" == '400' ]] +docker exec "\$container" sh -ec \ + 'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' + +sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$staged" "\$target" +rm -f "\$staged" +[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]] +[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]] +[[ "\$(stat -c '%a' "\$target")" == '400' ]] +docker exec "\$container" sh -ec \ + 'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' + +reload_attempted=true +docker kill -s HUP "\$container" >/dev/null sleep 2 -docker inspect alertmanager --format 'status={{.State.Status}} started={{.State.StartedAt}}' -echo "backup=\$backup" +docker inspect "\$container" --format 'status={{.State.Status}} started={{.State.StartedAt}}' +[[ "\$(docker inspect "\$container" --format '{{.State.Status}}')" == 'running' ]] +trap - ERR +echo "backup=\$backup config_mode=0400 runtime_owner=\${runtime_uid}:\${runtime_gid}" REMOTE log "Alertmanager config deployed and reloaded" diff --git a/scripts/ops/deploy-alertmanager-runtime-scrape.sh b/scripts/ops/deploy-alertmanager-runtime-scrape.sh new file mode 100755 index 000000000..53c34aa93 --- /dev/null +++ b/scripts/ops/deploy-alertmanager-runtime-scrape.sh @@ -0,0 +1,413 @@ +#!/usr/bin/env bash +# Bounded Prometheus scrape-source convergence for Alertmanager delivery truth. + +set -euo pipefail + +TRACE_ID="${TRACE_ID:?TRACE_ID is required}" +RUN_ID="${RUN_ID:?RUN_ID is required}" +WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}" +TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" +PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}" +ALERTMANAGER_METRICS_URL="${ALERTMANAGER_METRICS_URL:-http://127.0.0.1:9093/metrics}" +PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}" +REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}" +MODE="${1:---dry-run}" + +case "${MODE}" in + apply|--dry-run) ;; + *) printf 'usage: %s [--dry-run|apply]\n' "$0" >&2; exit 64 ;; +esac + +for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do + case "${value}" in + *[!A-Za-z0-9._-]*|'') + printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2 + exit 64 + ;; + esac +done + +ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \ + bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \ + "${PROMETHEUS_URL}" "${ALERTMANAGER_METRICS_URL}" \ + "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE' +set -euo pipefail + +TRACE_ID="$1" +RUN_ID="$2" +WORK_ITEM_ID="$3" +MODE="$4" +PROMETHEUS_URL="$5" +ALERTMANAGER_METRICS_URL="$6" +PROMETHEUS_CONTAINER="$7" +REMOTE_CONFIG="$8" +REMOTE_CANDIDATE="/tmp/awoooi-prometheus-alertmanager-runtime.${RUN_ID}.yml" +CONTAINER_CANDIDATE="/etc/prometheus/.awoooi-alertmanager-runtime.${RUN_ID}.yml" +BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}" +APPLY_STARTED=0 +DEPLOY_VERIFIED=0 +SOURCE_SHA="" +CANDIDATE_SHA="" + +exec 9>/tmp/awoooi-prometheus-config.lock +if ! flock -n 9; then + printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=prometheus_config_apply_in_progress\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2 + exit 75 +fi + +cleanup() { + rm -f "${REMOTE_CANDIDATE}" + docker exec -u 0 "${PROMETHEUS_CONTAINER}" \ + rm -f "${CONTAINER_CANDIDATE}" >/dev/null 2>&1 || true +} + +rollback() { + local rollback_rc=0 + local current_sha backup_sha restored_sha + current_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1 + backup_sha="$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" || rollback_rc=1 + if [ "${rollback_rc}" -eq 0 ] && [ "${current_sha}" = "${SOURCE_SHA}" ]; then + printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=no_write_source_intact source_sha=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}" + return 0 + fi + if [ "${rollback_rc}" -ne 0 ] \ + || [ "${current_sha}" != "${CANDIDATE_SHA}" ] \ + || [ "${backup_sha}" != "${SOURCE_SHA}" ]; then + rollback_rc=1 + else + cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" || rollback_rc=1 + restored_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1 + [ "${restored_sha}" = "${SOURCE_SHA}" ] || rollback_rc=1 + fi + if [ "${rollback_rc}" -eq 0 ]; then + curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null \ + || rollback_rc=1 + fi + if [ "${rollback_rc}" -eq 0 ]; then + curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null || rollback_rc=1 + fi + if [ "${rollback_rc}" -eq 0 ]; then + printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" + return 0 + fi + printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed ready=false backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" >&2 + return 1 +} + +on_exit() { + rc=$? + trap - EXIT + set +e + cleanup + rollback_rc=0 + if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \ + && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then + rollback || rollback_rc=$? + fi + if [ "${rollback_rc}" -ne 0 ]; then + exit 70 + fi + exit "${rc}" +} +trap on_exit EXIT + +test -f "${REMOTE_CONFIG}" +test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true +curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null +METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")" +grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}" +grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}" +SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" + +python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY' +from pathlib import Path +import re +import sys + +source = Path(sys.argv[1]) +candidate = Path(sys.argv[2]) +text = source.read_text(encoding="utf-8") + +begin = " # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n" +end = " # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n" +block = """ # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH + - job_name: 'alertmanager-runtime' + scrape_interval: 30s + metrics_path: /metrics + static_configs: + - targets: ['alertmanager:9093'] + labels: + host: '110' + service: 'alertmanager' + env: 'prod' + # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH + +""" + +if text.count(begin) != text.count(end) or text.count(begin) > 1: + raise SystemExit("alertmanager runtime managed block is ambiguous") +if begin in text: + pattern = re.escape(begin) + r".*?" + re.escape(end) + r"\n?" + text, count = re.subn(pattern, block, text, count=1, flags=re.S) + if count != 1: + raise SystemExit("managed block replacement failed") +else: + legacy_pattern = ( + r"(?ms)^ - job_name: ['\"]alertmanager-runtime['\"]\n" + r".*?(?=^ - job_name:|\Z)" + ) + legacy_matches = list(re.finditer(legacy_pattern, text)) + if len(legacy_matches) == 1: + text = re.sub(legacy_pattern, block, text, count=1) + elif legacy_matches: + raise SystemExit("legacy alertmanager runtime job is ambiguous") + else: + anchors = ( + " # === AWOOOI API Metrics", + " - job_name: 'awoooi-api'\n", + ) + anchor = next((value for value in anchors if value in text), None) + if anchor is None or text.count(anchor) != 1: + raise SystemExit("awoooi-api insertion anchor missing or ambiguous") + text = text.replace(anchor, block + anchor, 1) + +provider_begin = " # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES\n" +provider_end = " # END AWOOOI OLLAMA CLOUD EDGE PROBES\n" +provider_block = """ # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES + - targets: + - http://34.143.170.20:11434/api/tags + labels: + service: ollama-cloud-edge + provider: ollama_gcp_a + probe_origin: host110 + boundary: cloud_edge_availability_only + - targets: + - http://34.21.145.224:11434/api/tags + labels: + service: ollama-cloud-edge + provider: ollama_gcp_b + probe_origin: host110 + boundary: cloud_edge_availability_only + # END AWOOOI OLLAMA CLOUD EDGE PROBES +""" +if ( + text.count(provider_begin) != text.count(provider_end) + or text.count(provider_begin) > 1 +): + raise SystemExit("Ollama cloud edge managed block is ambiguous") +if provider_begin in text: + provider_pattern = ( + re.escape(provider_begin) + + r".*?" + + re.escape(provider_end) + ) + text, count = re.subn( + provider_pattern, + provider_block, + text, + count=1, + flags=re.S, + ) + if count != 1: + raise SystemExit("Ollama cloud edge managed block replacement failed") +else: + blackbox_match = re.search( + r"(?ms)^ - job_name: ['\"]blackbox-http['\"]\n" + r".*?(?=^ - job_name:|\Z)", + text, + ) + if blackbox_match is None: + raise SystemExit("blackbox-http job missing") + blackbox_job = blackbox_match.group(0) + if "provider: ollama_gcp_" in blackbox_job: + raise SystemExit("unmanaged Ollama cloud edge targets are ambiguous") + relabel_anchor = " relabel_configs:\n" + if blackbox_job.count(relabel_anchor) != 1: + raise SystemExit("blackbox-http relabel anchor missing or ambiguous") + patched_job = blackbox_job.replace( + relabel_anchor, + provider_block + relabel_anchor, + 1, + ) + text = text[: blackbox_match.start()] + patched_job + text[blackbox_match.end() :] + +if text.count("job_name: 'alertmanager-runtime'") != 1: + raise SystemExit("alertmanager runtime job must exist exactly once") +if "targets: ['alertmanager:9093']" not in text: + raise SystemExit("canonical Alertmanager target missing") +if text.count("provider: ollama_gcp_a") != 1: + raise SystemExit("canonical GCP-A cloud edge target missing or ambiguous") +if text.count("provider: ollama_gcp_b") != 1: + raise SystemExit("canonical GCP-B cloud edge target missing or ambiguous") +if "192.168.0.111:11434" in text: + raise SystemExit("host111 must not be probed from retired host110 boundary") +candidate.write_text(text, encoding="utf-8") +PY + +docker cp "${REMOTE_CANDIDATE}" \ + "${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE}" >/dev/null +docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${CONTAINER_CANDIDATE}" + +CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')" +test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}" +printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass persistent_writes_performed=0 temporary_validation_artifact=true\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \ + "${SOURCE_SHA}" "${CANDIDATE_SHA}" + +if [ "${MODE}" = "--dry-run" ]; then + printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_persistent_write\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" + exit 0 +fi + +cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}" +test "$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}" +test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}" +APPLY_STARTED=1 +cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}" +test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}" +curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null +curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null +printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=alertmanager_runtime_scrape_converged active_sha=%s backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${CANDIDATE_SHA}" \ + "${BACKUP_CONFIG}" + +target_readback() { + curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c ' +import json +import sys + +targets = json.load(sys.stdin)["data"]["activeTargets"] +matches = [ + target for target in targets + if target.get("labels", {}).get("job") == "alertmanager-runtime" +] +if len(matches) != 1: + print(f"{len(matches)}|missing|never") +else: + target = matches[0] + print("{}|{}|{}".format( + len(matches), + target.get("health", "unknown"), + target.get("lastScrape", "never"), + )) +' +} + +provider_target_readback() { + curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c ' +import json +import sys + +targets = json.load(sys.stdin)["data"]["activeTargets"] +providers = ("ollama_gcp_a", "ollama_gcp_b") +matches = { + provider: [ + target + for target in targets + if target.get("labels", {}).get("job") == "blackbox-http" + and target.get("labels", {}).get("provider") == provider + and target.get("labels", {}).get("probe_origin") == "host110" + ] + for provider in providers +} +ready = all( + len(matches[provider]) == 1 + and matches[provider][0].get("lastScrape", "never") != "never" + for provider in providers +) +values = [] +for provider in providers: + target = matches[provider][0] if len(matches[provider]) == 1 else {} + values.extend([ + target.get("lastScrape", "never"), + target.get("health", "missing"), + ]) +print("|".join(["1" if ready else "0", *values])) +' +} + +FIRST_SCRAPE="" +for attempt in $(seq 1 9); do + IFS='|' read -r count health last_scrape <<<"$(target_readback)" + printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \ + "${last_scrape}" + if [ "${count}" = 1 ] && [ "${health}" = up ] \ + && [ "${last_scrape}" != never ]; then + FIRST_SCRAPE="${last_scrape}" + break + fi + sleep 10 +done +test -n "${FIRST_SCRAPE}" + +FIRST_GCP_A_SCRAPE="" +FIRST_GCP_B_SCRAPE="" +for attempt in $(seq 1 9); do + IFS='|' read -r providers_ready gcp_a_scrape gcp_a_health \ + gcp_b_scrape gcp_b_health <<<"$(provider_target_readback)" + printf 'provider_target_probe trace_id=%s run_id=%s attempt=%s ready=%s gcp_a_health=%s gcp_b_health=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${providers_ready}" \ + "${gcp_a_health}" "${gcp_b_health}" + if [ "${providers_ready}" = 1 ]; then + FIRST_GCP_A_SCRAPE="${gcp_a_scrape}" + FIRST_GCP_B_SCRAPE="${gcp_b_scrape}" + break + fi + sleep 10 +done +test -n "${FIRST_GCP_A_SCRAPE}" +test -n "${FIRST_GCP_B_SCRAPE}" + +sleep 31 +IFS='|' read -r count health SECOND_SCRAPE <<<"$(target_readback)" +test "${count}" = 1 +test "${health}" = up +test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}" +IFS='|' read -r providers_ready SECOND_GCP_A_SCRAPE gcp_a_health \ + SECOND_GCP_B_SCRAPE gcp_b_health <<<"$(provider_target_readback)" +test "${providers_ready}" = 1 +test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}" +test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}" + +SERIES_COUNT="$(curl -fsS --get \ + --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \ + "${PROMETHEUS_URL}/api/v1/query" | python3 -c ' +import json +import sys +print(len(json.load(sys.stdin)["data"]["result"])) +')" +test "${SERIES_COUNT}" = 1 + +FAILED_SERIES_COUNT="$(curl -fsS --get \ + --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \ + "${PROMETHEUS_URL}/api/v1/query" | python3 -c ' +import json +import sys +print(len(json.load(sys.stdin)["data"]["result"])) +')" +test "${FAILED_SERIES_COUNT}" = 1 + +PROVIDER_SERIES_COUNT="$(curl -fsS --get \ + --data-urlencode 'query=probe_success{job="blackbox-http",provider=~"ollama_gcp_a|ollama_gcp_b",probe_origin="host110"}' \ + "${PROMETHEUS_URL}/api/v1/query" | python3 -c ' +import json +import sys +print(len(json.load(sys.stdin)["data"]["result"])) +')" +test "${PROVIDER_SERIES_COUNT}" = 2 +test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}" + +DEPLOY_VERIFIED=1 +printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=two_scrapes_up first=%s second=%s delivery_series_count=%s failed_series_count=%s provider_series_count=%s gcp_a_health=%s gcp_b_health=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \ + "${SECOND_SCRAPE}" "${SERIES_COUNT}" "${FAILED_SERIES_COUNT}" \ + "${PROVIDER_SERIES_COUNT}" "${gcp_a_health}" "${gcp_b_health}" +printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" +REMOTE diff --git a/scripts/ops/deploy-alerts.sh b/scripts/ops/deploy-alerts.sh index 728edc91c..bf881c797 100755 --- a/scripts/ops/deploy-alerts.sh +++ b/scripts/ops/deploy-alerts.sh @@ -185,6 +185,16 @@ if [[ "$NO_ALERTS_QUERY" != *'source="alertmanager"'* ]]; then fi log "✅ NoAlertsReceived2Hours query 已限制 alertmanager 主鏈路" +ALERT_CHAIN_BROKEN_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'AlertChainBroken_Alertmanager'), ''))") +if [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_failed_total'* ]] \ + || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_total'* ]] \ + || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver="awoooi-webhook"'* ]] \ + || [[ "$ALERT_CHAIN_BROKEN_QUERY" == *'awoooi_webhook_requests_total'* ]]; then + echo "ERROR: AlertChainBroken_Alertmanager 未使用 primary awoooi-webhook monotonic delivery truth: ${ALERT_CHAIN_BROKEN_QUERY}" + exit 1 +fi +log "✅ AlertChainBroken_Alertmanager 只使用 primary awoooi-webhook monotonic delivery truth" + SOURCE_PROVIDER_STALE_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'SourceProviderIngestionStale'), ''))") if [[ "$SOURCE_PROVIDER_STALE_QUERY" != *'source=~"sentry|signoz"'* ]]; then echo "ERROR: SourceProviderIngestionStale query 未限制 Sentry/SignOz provider freshness: ${SOURCE_PROVIDER_STALE_QUERY}" diff --git a/scripts/ops/ollama111-fallback-proxy-diagnose.sh b/scripts/ops/ollama111-fallback-proxy-diagnose.sh old mode 100755 new mode 100644 index 6a31fc8cb..f71439e10 --- a/scripts/ops/ollama111-fallback-proxy-diagnose.sh +++ b/scripts/ops/ollama111-fallback-proxy-diagnose.sh @@ -1,78 +1,35 @@ #!/usr/bin/env bash +# Compatibility entrypoint retained for operators; the host110 proxy lane is +# retired. All probes use the canonical direct provider identities. set -euo pipefail -# Read-only diagnosis for ADR-110 local fallback: -# API Pod -> 110:11437 Nginx proxy -> 111:11434 Ollama allowlist proxy. +GCP_A_URL="${GCP_A_URL:-http://34.143.170.20:11434}" +GCP_B_URL="${GCP_B_URL:-http://34.21.145.224:11434}" +HOST111_URL="${HOST111_URL:-http://192.168.0.111:11434}" +PUBLIC_STATUS_URL="${PUBLIC_STATUS_URL:-https://awoooi.wooo.work/api/v1/ai/status}" +CURL_TIMEOUT="${CURL_TIMEOUT:-8}" -NAMESPACE="${NAMESPACE:-awoooi-prod}" -DEPLOYMENT="${DEPLOYMENT:-awoooi-api}" -PROXY_HOST="${PROXY_HOST:-wooo@192.168.0.110}" -LOCAL_HOST="${LOCAL_HOST:-ollama-111-gpu}" -LOCAL_IP="${LOCAL_IP:-192.168.0.111}" -PROXY_PORT="${PROXY_PORT:-11437}" -LOCAL_PORT="${LOCAL_PORT:-11434}" -SSH_TIMEOUT="${SSH_TIMEOUT:-8}" -CURL_TIMEOUT="${CURL_TIMEOUT:-5}" - -section() { - printf '\n== %s ==\n' "$1" +probe() { + local provider="$1" + local url="$2" + local code + code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \ + "${url}/api/tags" 2>/dev/null || true)" + case "${code}" in + 200) printf 'provider=%s boundary=%s status=healthy http_code=200\n' \ + "${provider}" "$3" ;; + *) printf 'provider=%s boundary=%s status=unavailable http_code=%s\n' \ + "${provider}" "$3" "${code:-000}" ;; + esac } -run_local() { - set +e - "$@" - local rc=$? - set -e - printf 'exit_code=%s\n' "$rc" -} +printf 'schema_version=ollama_direct_route_diagnosis_v1 writes_performed=0\n' +probe ollama_gcp_a "${GCP_A_URL}" cloud +probe ollama_gcp_b "${GCP_B_URL}" cloud +probe ollama_local "${HOST111_URL}" local -section "Production health provider chain" -run_local curl -sS -m 12 https://awoooi.wooo.work/api/v1/health - -section "API pod view" -set +e -kubectl -n "${NAMESPACE}" exec -i "deploy/${DEPLOYMENT}" -- sh -lc " - set +e - echo OLLAMA_URL=\$OLLAMA_URL - echo OLLAMA_SECONDARY_URL=\$OLLAMA_SECONDARY_URL - echo OLLAMA_FALLBACK_URL=\$OLLAMA_FALLBACK_URL - curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://192.168.0.110:${PROXY_PORT}/api/tags | head -60 -" -printf 'exit_code=%s\n' "$?" -set -e - -section "110 proxy upstream reachability" -run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${PROXY_HOST}" " - set +e - echo route: - ip route get ${LOCAL_IP} - echo neigh: - ip neigh show ${LOCAL_IP} - echo ping: - ping -c 3 -W 2 ${LOCAL_IP} - echo direct_111: - curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://${LOCAL_IP}:${LOCAL_PORT}/api/tags | head -60 - echo proxy_11437: - curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${PROXY_PORT}/api/tags | head -60 - echo nginx_recent_errors: - tail -20 /var/log/nginx/ollama-local-error.log 2>/dev/null || true -" - -section "111 host direct SSH view" -run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${LOCAL_HOST}" " - set +e - hostname - date - curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${LOCAL_PORT}/api/tags | head -60 - launchctl print gui/501/com.momo.ollama111-allow-proxy 2>/dev/null | head -60 || true - pmset -g custom 2>/dev/null | sed -n '1,80p' || true -" - -cat <<'EOF' - -Interpretation: -- If 110 shows "ip neigh INCOMPLETE", "Destination Host Unreachable", or "No route to host", - the 111 host or LAN path is down. Do not restart API or change provider order. -- If 110 can reach 111 directly but 11437 returns 502, inspect/reload the 110 Nginx proxy. -- If 111 direct SSH works but 127.0.0.1:11434 fails, recover the 111 Ollama/allowlist LaunchAgent. -EOF +status_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \ + "${PUBLIC_STATUS_URL}" 2>/dev/null || true)" +printf 'production_route_readback_http=%s\n' "${status_code:-000}" +printf '%s\n' \ + 'terminal=read_only provider_order=ollama_gcp_a,ollama_gcp_b,ollama_local,claude,gemini host110_transport_selected=false' diff --git a/scripts/ops/run-paid-provider-canary.py b/scripts/ops/run-paid-provider-canary.py new file mode 100644 index 000000000..dbd55f3e1 --- /dev/null +++ b/scripts/ops/run-paid-provider-canary.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Execute the bounded five-lane SRE comparison and paid-provider canary.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_API_ROOT = _REPO_ROOT / "apps" / "api" +if str(_API_ROOT) not in sys.path: + sys.path.insert(0, str(_API_ROOT)) + + +async def _run_validation(**kwargs): + from src.services.paid_provider_canary_validation import ( + run_paid_provider_canary_validation, + ) + + return await run_paid_provider_canary_validation(**kwargs) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--run-ref", required=True) + parser.add_argument("--operator-id", default="codex-controlled-canary") + parser.add_argument("--authorization-ref", required=True) + args = parser.parse_args() + try: + receipt = asyncio.run( + _run_validation( + run_ref=args.run_ref, + operator_id=args.operator_id, + authorization_ref=args.authorization_ref, + ) + ) + except BaseException as exc: + error_code = str(exc) + if not re.fullmatch(r"[a-z][a-z0-9_]{0,119}", error_code): + error_code = f"canary_failed_{type(exc).__name__}" + print( + json.dumps( + { + "schema_version": "paid_provider_sre_canary_error_v1", + "status": "blocked_with_safe_next_action", + "error_code": error_code[:120], + "raw_prompt_persisted": False, + "raw_response_persisted": False, + "secret_value_returned": False, + }, + sort_keys=True, + ) + ) + return 2 + print(json.dumps(receipt, ensure_ascii=False, sort_keys=True)) + return 0 if receipt["independent_verifier"]["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py b/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py new file mode 100644 index 000000000..ca58a54b6 --- /dev/null +++ b/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-runtime-scrape.sh" + + +def test_controlled_scrape_apply_has_check_rollback_and_independent_verifier() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert 'TRACE_ID="${TRACE_ID:?TRACE_ID is required}"' in text + assert 'RUN_ID="${RUN_ID:?RUN_ID is required}"' in text + assert 'WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"' in text + assert "promtool check config" in text + assert "writes_performed=0" in text + assert "rollback_receipt" in text + assert "terminal=rollback_failed ready=false" in text + assert "flock -n 9" in text + assert '!= "${CANDIDATE_SHA}"' in text + assert '!= "${SOURCE_SHA}"' in text + assert "alertmanager_notification_requests_total" in text + assert "alertmanager_notification_requests_failed_total" in text + assert 'receiver="awoooi-webhook"' in text + assert text.count('receiver="awoooi-webhook"') >= 4 + assert "two_scrapes_up" in text + assert "delivery_series_count" in text + + +def test_controlled_scrape_apply_never_replaces_the_full_live_config() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert "BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH" in text + assert "job_name: 'alertmanager-runtime'" in text + assert "targets: ['alertmanager:9093']" in text + assert "legacy_pattern" in text + assert "k8s/monitoring/prometheus.yml" not in text diff --git a/scripts/ops/tests/test_alertmanager_webhook_config.py b/scripts/ops/tests/test_alertmanager_webhook_config.py index 2c17eab16..9e79025a4 100644 --- a/scripts/ops/tests/test_alertmanager_webhook_config.py +++ b/scripts/ops/tests/test_alertmanager_webhook_config.py @@ -7,6 +7,7 @@ import yaml ROOT = Path(__file__).resolve().parents[3] ALERTMANAGER_CONFIG = ROOT / "ops" / "alertmanager" / "alertmanager.yml" +DEPLOY_SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-config.sh" def _config() -> dict: @@ -37,3 +38,70 @@ def test_emergency_direct_telegram_stays_limited_to_alert_chain() -> None: route_text = str(_config()["route"]) assert "AWOOOIApiDown" in route_text assert 'severity="critical"' in route_text + + +def test_emergency_card_is_bounded_and_has_honest_automation_attribution() -> None: + receiver = _receiver("telegram-direct") + telegram = receiver["telegram_configs"][0] + message = telegram["message"] + + assert "SRE BYPASS｜告警主鏈異常" in message + assert "Prometheus deterministic rule（未呼叫 LLM）" in message + assert "Executor：none" in message + assert "未宣稱已修復" in message + assert "Verifier：source-specific delivery metric" in message + assert "故障主機" not in message + telegram_route = next( + route + for route in _config()["route"]["routes"] + if route["receiver"] == "telegram-direct" + ) + assert telegram_route["repeat_interval"] == "2h" + + +def test_alertmanager_has_no_agent99_inbound_receiver_or_secret_first_hop() -> None: + config = _config() + assert all( + item.get("receiver") != "agent99-alert-chain-relay" + for item in config["route"]["routes"] + ) + assert all( + receiver.get("name") != "agent99-alert-chain-relay" + for receiver in config["receivers"] + ) + source = ALERTMANAGER_CONFIG.read_text(encoding="utf-8") + assert "192.168.0.99:8787" not in source + assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source + assert "authorization:" not in source + + +def test_alertmanager_deploy_does_not_inject_agent99_relay_secret() -> None: + source = DEPLOY_SCRIPT.read_text(encoding="utf-8") + assert "AGENT99_SRE_ALERT_RELAY_TOKEN" not in source + assert 'os.environ["AGENT99_SRE_RELAY_TOKEN"]' not in source + assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source + assert "unreplaced secret placeholder remains" in source + assert "set -x" not in source + + +def test_secret_bearing_alertmanager_config_and_backups_are_not_world_readable() -> None: + source = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + assert "install -m 0600" in source + assert 'chmod 0600 "\\$legacy_backup"' in source + assert 'chmod 0400 "\\$target"' in source + assert 'chmod 0000 "\\$target"' in source + assert "chmod 0644" not in source + assert "/proc/\\${container_pid}/status" in source + assert "runtime_uid" in source + assert "runtime_gid" in source + assert "test -r /etc/alertmanager/alertmanager.yml" in source + assert "rollback_secure" in source + assert "secure rollback attempted" in source + assert "amtool check-config /etc/alertmanager/alertmanager.yml" in source + assert "amtool check-config /dev/stdin" in source + assert "/tmp/alertmanager-rendered.yml" not in source + assert "trap 'rm -f \"\\$staged\"' EXIT" in source + assert source.index('chmod 0000 "\\$target"') < source.index( + "sudo -n sh -c 'cat \"\\$1\" > \"\\$2\"'" + ) diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 7821c2427..0efa45133 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -456,7 +456,7 @@ $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select- $decision = Get-AgentLivePreflightDecision ` -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` -Manifest $manifest ` - -ExpectedRuntimeFileCount 14 ` + -ExpectedRuntimeFileCount 15 ` -TaskFailureCount $taskFailures.Count ` -RelayListenerCount $relayListeners.Count ` -RequiredEvidenceStaleCount $requiredEvidenceStale.Count ` diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index d67aa952a..b9790a3ff 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -24,6 +24,7 @@ $FixedRuntimeFiles = @( "agent99-control-plane.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", + "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-inbox.ps1", "agent99-sre-alert-relay.ps1", "agent99-submit-request.ps1", @@ -716,7 +717,7 @@ try { } $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant() if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" } - if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" } + if ([int]$envelope.expectedRuntimeFileCount -ne 15) { throw "invalid_expected_runtime_file_count" } $traceId = [string]$envelope.traceId $runId = [string]$envelope.runId @@ -751,7 +752,7 @@ try { $sourceManifest = [pscustomobject]@{ schemaVersion = "agent99_remote_source_manifest_v1" sourceRevision = $sourceRevision - fileCount = 14 + fileCount = 15 manifestSha256 = $manifestDigest files = @($FixedRuntimeFiles | ForEach-Object { [pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() } @@ -788,7 +789,7 @@ try { status = "check_ready" mode = "check" sourceRevision = $sourceRevision - expectedRuntimeFileCount = 14 + expectedRuntimeFileCount = 15 manifestSha256 = $manifestDigest plannedStagingPath = $stagePath runtimeManifest = $currentManifest @@ -875,7 +876,7 @@ try { if ( [string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or [string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or - [int]$existingSourceManifest.fileCount -ne 14 -or + [int]$existingSourceManifest.fileCount -ne 15 -or [string]$existingSourceManifest.manifestSha256 -ne $manifestDigest ) { throw "existing_staging_manifest_mismatch" } } catch { @@ -899,7 +900,7 @@ try { [string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and $live.runtimeMatched -and $live.sourceRevision -eq $sourceRevision -and - $live.fileCount -eq 14 -and + $live.fileCount -eq 15 -and $live.mismatchCount -eq 0 -and $liveLauncherContract.ok -and [string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256 @@ -1135,7 +1136,7 @@ try { $runtimeManifest.exists -and $runtimeManifest.runtimeMatched -and $runtimeManifest.sourceRevision -eq $sourceRevision -and - $runtimeManifest.fileCount -eq 14 -and + $runtimeManifest.fileCount -eq 15 -and $runtimeManifest.mismatchCount -eq 0 -and $launcherContract.ok -and [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index 72f51d02d..278c1560d 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -19,6 +19,7 @@ RUNTIME_FILES=( "agent99-control-plane.ps1" "agent99-deploy.ps1" "agent99-register-tasks.ps1" + "agent99-alertmanager-alertchain-poll.ps1" "agent99-sre-alert-inbox.ps1" "agent99-sre-alert-relay.ps1" "agent99-submit-request.ps1" @@ -234,7 +235,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7] output_path = Path(sys.argv[7]) runtime_names = sys.argv[8:] -if len(runtime_names) != 14 or len(set(runtime_names)) != 14: +if len(runtime_names) != 15 or len(set(runtime_names)) != 15: raise SystemExit("fixed_runtime_file_contract_failed") files: list[dict[str, str]] = [] @@ -273,7 +274,7 @@ envelope = { "runId": run_id, "workItemId": work_item_id, "sourceRevision": source_revision, - "expectedRuntimeFileCount": 14, + "expectedRuntimeFileCount": 15, "manifestSha256": manifest_sha256, "files": files, "livePreflight": { @@ -425,14 +426,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20] script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue' $a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json' $r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''} -if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 14-and$m.mismatchCount-eq 0-and$rows.Count-eq 14-and$seen.Count-eq 14-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} +if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 15-and$m.mismatchCount-eq 0-and$rows.Count-eq 15-and$seen.Count-eq 15-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} $c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''} try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name} $z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false} $ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0) $status=if($ready){'check_ready'}else{'check_failed_no_apply'} $next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'} -[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=14;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 +[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=15;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 if(-not$ready){exit 65}''' script = ( script.replace("__SOURCE_REVISION__", source_revision) diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index a8d1cefe6..eed38bdc6 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]: "exists": True, "sourceRevision": "a" * 40, "runtimeMatched": True, - "fileCount": 14, + "fileCount": 15, "mismatchCount": 0, "parseError": "", }, @@ -81,7 +81,7 @@ $case = @' $parameters = @{{ AgentRootPresent = [bool]$case.agentRootPresent Manifest = $case.manifest - ExpectedRuntimeFileCount = 14 + ExpectedRuntimeFileCount = 15 TaskFailureCount = [int]$case.taskFailureCount RelayListenerCount = [int]$case.relayListenerCount RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index a228d554f..7c9e23707 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -25,6 +25,7 @@ EXPECTED_RUNTIME_FILES = ( "agent99-control-plane.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", + "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-inbox.ps1", "agent99-sre-alert-relay.ps1", "agent99-submit-request.ps1", @@ -77,7 +78,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None assert sender_files == EXPECTED_RUNTIME_FILES assert receiver_files == EXPECTED_RUNTIME_FILES assert deployer_files == EXPECTED_RUNTIME_FILES - assert "expectedRuntimeFileCount\": 14" in sender + assert "expectedRuntimeFileCount\": 15" in sender assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver assert "required_runtime_file_missing" in receiver assert "duplicate_runtime_file" in receiver @@ -230,7 +231,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source assert "$livePreflight.sourceRevision -eq $sourceRevision" in source assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source - assert '$runtimeManifest.fileCount -eq 14' in source + assert '$runtimeManifest.fileCount -eq 15' in source assert '$runtimeManifest.mismatchCount -eq 0' in source assert 'status = "deployed_verified"' in source assert '$failurePhase = "deploy"' in source @@ -699,8 +700,9 @@ def test_check_mode_uses_bounded_control_command_without_ssh_stdin( " *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n" " *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) " "printf '%040d\\n' 0; exit 0 ;;\n" - " *\" cat-file -e \"*) exit 0 ;;\n" - " *\" diff --quiet \"*) exit 0 ;;\n" + " *\" cat-file -e \"*) exit 0 ;;\n" + " *\" ls-files --error-unmatch \"*) exit 0 ;;\n" + " *\" diff --quiet \"*) exit 0 ;;\n" "esac\n" f"exec {shlex.quote(real_git)} \"$@\"\n", encoding="utf-8", @@ -854,8 +856,9 @@ def test_apply_uses_ephemeral_file_transport_and_cleans_up_without_ssh_stdin( " *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n" " *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) " "printf '%040d\\n' 0; exit 0 ;;\n" - " *\" cat-file -e \"*) exit 0 ;;\n" - " *\" diff --quiet \"*) exit 0 ;;\n" + " *\" cat-file -e \"*) exit 0 ;;\n" + " *\" ls-files --error-unmatch \"*) exit 0 ;;\n" + " *\" diff --quiet \"*) exit 0 ;;\n" "esac\n" f"exec {shlex.quote(real_git)} \"$@\"\n", encoding="utf-8", diff --git a/scripts/security/ai-provider-owner-response-acceptance.py b/scripts/security/ai-provider-owner-response-acceptance.py index ac4eb6738..5c0a54e26 100644 --- a/scripts/security/ai-provider-owner-response-acceptance.py +++ b/scripts/security/ai-provider-owner-response-acceptance.py @@ -189,19 +189,20 @@ SURFACES = [ "requires_live_evidence": True, }, { - "surface_id": "ollama_proxy_gateway", - "label": "Ollama proxy gateway / local fallback boundary", - "expected_scope": "proxy route、local fallback、gateway health", - "config_kind": "proxy_gateway", + "surface_id": "ollama_provider_transport_boundary", + "label": "Ollama direct route / mesh transport / local fallback boundary", + "expected_scope": "GCP direct/mesh transport、host111 local boundary、provider health", + "config_kind": "provider_transport", "source_refs": [ - "infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2", + "ops/monitoring/service-registry.yaml", + "k8s/awoooi-prod/04-configmap.yaml", + "infra/ansible/playbooks/111-ollama-fallback.yml", "apps/api/src/services/ollama_endpoint_resolver.py", "apps/api/src/services/ollama_failover_manager.py", - "apps/api/src/services/provider_proxy.py", ], "write_capable_surface": True, "paid_provider_related": False, - "data_egress_related": False, + "data_egress_related": True, "requires_live_evidence": True, }, { diff --git a/scripts/security/high-value-config-change-gate.py b/scripts/security/high-value-config-change-gate.py index 688f2c0ef..5060b3e4a 100644 --- a/scripts/security/high-value-config-change-gate.py +++ b/scripts/security/high-value-config-change-gate.py @@ -337,7 +337,8 @@ CATEGORIES = [ "apps/api/src/services/ai_providers/**", "apps/api/src/services/**/*model*", "apps/api/src/services/**/*provider*", - "infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2", + "ops/monitoring/service-registry.yaml", + "k8s/awoooi-prod/04-configmap.yaml", "docs/ai/**", "docs/**/*Ollama*", ), diff --git a/scripts/security/nginx-config-drift-detector.py b/scripts/security/nginx-config-drift-detector.py index e5e7f0448..57be0c4ce 100644 --- a/scripts/security/nginx-config-drift-detector.py +++ b/scripts/security/nginx-config-drift-detector.py @@ -53,15 +53,6 @@ SOURCES = [ control_tier="C0", owner_gate="public_tools_owner_response_required", ), - NginxSource( - config_id="host110_ollama_proxy", - host="192.168.0.110", - role="ollama_proxy_gateway", - source_path="infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2", - live_path="/etc/nginx/sites-enabled/110-ollama-proxy.conf", - control_tier="C1", - owner_gate="ai_provider_proxy_owner_response_required", - ), ]