From 4af6487ba63bc86d9c38730e25dabb1b9c2789fc Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 16 Jul 2026 17:38:45 +0800 Subject: [PATCH] fix(alerting): align Alertmanager metric schema --- ...lio_infrastructure_asset_reconciliation.py | 38 ++++- .../tests/test_alert_chain_runtime_truth.py | 19 ++- ...lio_infrastructure_asset_reconciliation.py | 29 +++- ...infrastructure_asset_reconciliation_api.py | 2 +- ...ructure-asset-reconciliation.snapshot.json | 14 +- ...rolled-automation-work-items.snapshot.json | 11 +- .../110-alertmanager-delivery-recovery.yml | 4 +- k8s/monitoring/alert-chain-monitor.yaml | 8 +- k8s/monitoring/prometheus.yml | 10 ++ ops/monitoring/alerts-unified.yml | 8 +- ops/monitoring/alerts.yml | 8 +- .../ops/deploy-alertmanager-runtime-scrape.sh | 14 +- scripts/ops/deploy-alerts.sh | 4 +- scripts/ops/retire-host110-ollama-proxy.sh | 154 ++++++++++++++++++ ...st_alertmanager_runtime_scrape_contract.py | 6 +- ...ost110_ollama_proxy_retirement_contract.py | 38 +++++ 16 files changed, 324 insertions(+), 43 deletions(-) create mode 100755 scripts/ops/retire-host110-ollama-proxy.sh create mode 100644 scripts/ops/tests/test_host110_ollama_proxy_retirement_contract.py diff --git a/apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py b/apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py index 5a705703c..1fd1985a4 100644 --- a/apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py +++ b/apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py @@ -178,8 +178,42 @@ def load_portfolio_infrastructure_asset_reconciliation( raise ValueError(f"{path}: rollup source-reconciled asset mismatch") if rollups.get("reconciliation_work_items") != len(work_items): raise ValueError(f"{path}: rollup work item mismatch") - if rollups.get("runtime_closed_assets") != 0: - raise ValueError(f"{path}: reconciliation cannot manufacture runtime closure") + runtime_closed_assets: list[dict[str, Any]] = [] + for asset in assets: + runtime_closure = asset.get("runtime_closure") + if runtime_closure is None: + continue + if not isinstance(runtime_closure, dict): + raise ValueError(f"{path}: {asset['canonical_id']} invalid runtime closure") + if runtime_closure.get("status") not in { + "verified_absent", + "verified_healthy", + }: + continue + required_receipt_fields = { + "trace_id", + "run_id", + "work_item_id", + "post_verifier", + } + missing_receipts = sorted( + key + for key in required_receipt_fields + if not str(runtime_closure.get(key) or "").strip() + ) + if missing_receipts: + raise ValueError( + f"{path}: {asset['canonical_id']} runtime closure missing " + f"{missing_receipts}" + ) + runtime_closed_assets.append(asset) + if rollups.get("runtime_closed_assets") != len(runtime_closed_assets): + raise ValueError(f"{path}: rollup runtime-closed asset mismatch") + expected_completion_percent = round( + len(runtime_closed_assets) * 100 / max(len(assets), 1) + ) + if rollups.get("program_completion_percent") != expected_completion_percent: + raise ValueError(f"{path}: rollup program completion mismatch") result = dict(payload) result["assets"] = assets diff --git a/apps/api/tests/test_alert_chain_runtime_truth.py b/apps/api/tests/test_alert_chain_runtime_truth.py index c82f44d11..068415857 100644 --- a/apps/api/tests/test_alert_chain_runtime_truth.py +++ b/apps/api/tests/test_alert_chain_runtime_truth.py @@ -50,6 +50,15 @@ def test_alertmanager_delivery_truth_is_scraped_from_single_runtime() -> None: }, } ] + assert job["metric_relabel_configs"] == [ + { + "source_labels": ["__name__", "integration"], + "separator": ";", + "regex": "alertmanager_notification_requests(_failed)?_total;webhook", + "target_label": "receiver_contract", + "replacement": "awoooi-webhook", + } + ] blackbox = next( row for row in payload["scrape_configs"] @@ -70,7 +79,7 @@ def test_alert_chain_rule_uses_monotonic_alertmanager_counters_in_all_sources() assert "alertmanager_notification_requests_total" in expression assert 'job="alertmanager-runtime"' in expression assert 'integration="webhook"' in expression - assert 'receiver="awoooi-webhook"' in expression + assert 'receiver_contract="awoooi-webhook"' in expression assert ">=5" in expression assert "awoooi_webhook_requests_total" not in expression @@ -87,9 +96,13 @@ def test_alert_chain_rule_is_exact_asset_controlled_and_not_no_action() -> None: assert labels["layer"] == "host" -def test_runtime_scrape_and_recovery_preflight_require_primary_receiver_label() -> None: +def test_runtime_scrape_and_recovery_preflight_match_production_metric_schema() -> None: for path in (RECOVERY_PLAYBOOK, RUNTIME_SCRAPE_DEPLOY): source = path.read_text(encoding="utf-8") - assert 'receiver="awoooi-webhook"' in source + assert 'integration="webhook"' in source assert "alertmanager_notification_requests_total" in source assert "alertmanager_notification_requests_failed_total" in source + + runtime_source = RUNTIME_SCRAPE_DEPLOY.read_text(encoding="utf-8") + assert "target_label: receiver_contract" in runtime_source + assert "replacement: 'awoooi-webhook'" in runtime_source diff --git a/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py index 4b8c448cc..cb5332506 100644 --- a/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py +++ b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation.py @@ -21,7 +21,10 @@ def test_loader_maps_every_imported_inventory_row_without_claiming_live_truth() assert len(mapped_rows) == 131 assert len(mapped_rows) == len(set(mapped_rows)) assert {"L012", "L168", "L206", "L259"}.issubset(mapped_rows) - assert payload["rollups"]["runtime_closed_assets"] == 0 + assert payload["rollups"]["runtime_closed_assets"] == 1 + assert payload["rollups"]["runtime_closure_status"] == ( + "started_partial_1_of_144" + ) assert payload["rollups"]["source_reconciled_assets"] == 6 assert payload["rollups"]["conflicts"] == 11 assert payload["rollups"]["conflict_assets"] == 36 @@ -60,7 +63,7 @@ def test_reconciliation_includes_missing_control_plane_and_provider_assets() -> for canonical_id in imported_omissions ) assert assets["service:ollama:host110"]["reconciliation_state"] == ( - "retired_tombstone_source_ready_runtime_absence_pending" + "retired_tombstone_runtime_absence_verified" ) assert assets["windows-vmware:host_99"]["executor"] == "Agent99" assert assets["host:111"]["executor"] == "host_ansible_executor" @@ -81,7 +84,10 @@ def test_provider_alert_chain_and_gitea_runtime_gaps_are_explicit() -> None: tombstone = assets["service:ollama:host110"] assert tombstone["runtime_identity"].startswith("retired-tombstone:") assert tombstone["monitoring"]["coverage"] == ( - "source_tombstone_runtime_absence_pending" + "runtime_absence_verified_20260716" + ) + assert tombstone["bounded_executor_receipt"] == ( + "run-aia-sre-002-20260716-1752" ) host111 = assets["host:111"] @@ -207,3 +213,20 @@ def test_loader_rejects_cross_domain_executor_drift(tmp_path: Path) -> None: with pytest.raises(ValueError, match="executor/domain mismatch"): load_portfolio_infrastructure_asset_reconciliation(tmp_path) + + +def test_loader_rejects_runtime_closure_without_same_run_receipt( + tmp_path: Path, +) -> None: + payload = json.loads(SNAPSHOT.read_text(encoding="utf-8")) + host110 = next( + asset + for group in payload["asset_groups"] + for asset in group["members"] + if asset["canonical_id"] == "service:ollama:host110" + ) + host110["runtime_closure"].pop("post_verifier") + (tmp_path / SNAPSHOT.name).write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="runtime closure missing"): + load_portfolio_infrastructure_asset_reconciliation(tmp_path) diff --git a/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py index a0886220d..c0e996baa 100644 --- a/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py +++ b/apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py @@ -36,7 +36,7 @@ def test_endpoint_returns_complete_redacted_portfolio_reconciliation() -> None: assert payload["rollups"]["conflicts"] == 11 assert payload["rollups"]["conflict_assets"] == 36 assert payload["rollups"]["reconciliation_work_items"] == 12 - assert payload["rollups"]["runtime_closed_assets"] == 0 + assert payload["rollups"]["runtime_closed_assets"] == 1 assert len(payload["assets"]) == 144 assert "asset_groups" not in payload diff --git a/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json b/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json index 55a6a0354..f121002b7 100644 --- a/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json +++ b/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json @@ -196,7 +196,7 @@ "members": [ {"canonical_id": "service:prometheus", "inventory_labels": ["Prometheus"], "source_rows": ["L071"], "runtime_identity": "host110/docker/prometheus", "source_truth_state": "source_reconciled_runtime_pending", "reconciliation_state": "source_identity_reconciled_runtime_pending", "findings": ["Monitoring registry now declares the exact host110 Docker identity", "Production container, generated target parity and scrape freshness remain unverified"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "k8s/monitoring/prometheus.yml", "scripts/ops/deploy-alertmanager-runtime-scrape.sh"], "priority": "P0", "verifier": "prometheus_runtime_target_and_rule_verifier"}, {"canonical_id": "service:grafana", "inventory_labels": ["Grafana"], "source_rows": ["L072"], "runtime_identity": "host110/docker/grafana", "reconciliation_state": "source_partial_runtime_unverified", "findings": ["Source policy exists but live dashboard/runtime parity is not proven"], "source_refs": ["ops/config/service-registry.yaml", "docs/security/monitoring-alerting-observability-inventory.snapshot.json"]}, - {"canonical_id": "service:alertmanager", "inventory_labels": ["Alertmanager"], "source_rows": ["L073"], "runtime_identity": "host110/docker/alertmanager", "source_truth_state": "source_reconciled_runtime_pending", "reconciliation_state": "source_control_path_reconciled_runtime_pending", "findings": ["Monitoring registry and typed router now declare the exact host110 Docker identity", "Alert rule now reads Alertmanager monotonic delivery counters instead of process-local API counters", "Exact bounded Ansible recovery and independent verifier are source-ready; production same-run receipt is pending"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "k8s/monitoring/alert-chain-monitor.yaml", "ops/alertmanager/alertmanager.yml", "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml", "scripts/ops/deploy-alertmanager-runtime-scrape.sh"], "priority": "P0", "verifier": "alertmanager_delivery_chain_independent_verifier", "next_action": "Deploy the canonical scrape/rule and exact playbook, then correlate receiver, API ingest, Telegram receipt, retry/rollback and KM writeback under one run."}, + {"canonical_id": "service:alertmanager", "inventory_labels": ["Alertmanager"], "source_rows": ["L073"], "runtime_identity": "host110/docker/alertmanager", "source_truth_state": "source_reconciled_runtime_pending", "reconciliation_state": "source_control_path_reconciled_runtime_pending", "findings": ["Monitoring registry and typed router now declare the exact host110 Docker identity", "Production readback proves this Alertmanager exports notification counters with integration labels but no native receiver label", "The source contract has exactly one webhook receiver and Prometheus stamps receiver_contract=awoooi-webhook only on the two webhook counter families", "The corrected bounded dry-run passed promtool with no persistent write; apply and production same-run receipt remain pending"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "k8s/monitoring/alert-chain-monitor.yaml", "ops/alertmanager/alertmanager.yml", "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml", "scripts/ops/deploy-alertmanager-runtime-scrape.sh"], "priority": "P0", "verifier": "alertmanager_delivery_chain_independent_verifier", "next_action": "Deploy the production-schema-compatible scrape/rule and exact playbook, then correlate receiver contract, API ingest, Telegram receipt, retry/rollback and KM writeback under one run."}, {"canonical_id": "service:signoz", "inventory_labels": ["SignOz"], "source_rows": ["L074"], "runtime_identity": "unresolved:host110-ui-host188-collector", "reconciliation_state": "split_plane_identity_conflict", "findings": ["Inventory collapses UI, collector, and storage into one host110 service while source registries split them across hosts"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "scripts/backup/backup-signoz.sh"]}, {"canonical_id": "service:otel-collector:signoz", "inventory_labels": ["OTel Collector"], "source_rows": ["L075"], "runtime_identity": "unresolved:host110-or-host188/otel-collector", "reconciliation_state": "host_identity_conflict", "findings": ["Inventory says host110 while monitoring registry says host188 port 24317"], "source_refs": ["ops/monitoring/service-registry.yaml", "docs/security/monitoring-alerting-observability-inventory.snapshot.json"], "domain_router": "unknown", "executor": null, "verifier": "asset_identity_drift_verifier"}, {"canonical_id": "service:blackbox-exporter", "inventory_labels": ["Blackbox Exporter"], "source_rows": ["L076"], "runtime_identity": "host110/docker/blackbox-exporter", "reconciliation_state": "matched_runtime_unverified", "findings": ["Generated target freshness and live probe results are not current truth"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/generated/blackbox-targets-generated.yaml"]}, @@ -545,7 +545,7 @@ }, "members": [ {"canonical_id": "windows-vmware:host_99", "inventory_labels": ["Agent99 Windows/VMware control plane"], "source_rows": [], "runtime_identity": "192.168.0.99/Agent99", "findings": ["Agent99 is required for Windows/VMware and control-plane recovery but absent from the imported inventory", "Agent99 also owns the read-only host110 Alertmanager poll/reduced relay coordination role; Linux remediation still routes only to the host Ansible executor", "The atomic 15-file runtime bundle includes the poller but production deployment/readback is pending"], "source_refs": ["docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json", "k8s/awoooi-prod/04-configmap.yaml", "agent99-alertmanager-alertchain-poll.ps1", "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"], "owner_lane": "Agent99", "domain_router": "windows_vmware", "executor": "Agent99", "verifier": "agent99_independent_runtime_verifier"}, - {"canonical_id": "service:ollama:host110", "inventory_labels": ["retired Host110 Ollama tombstone"], "source_rows": [], "runtime_identity": "retired-tombstone:host110/ollama/no-endpoint", "source_truth_state": "retired_tombstone_source_ready_runtime_absence_pending", "reconciliation_state": "retired_tombstone_source_ready_runtime_absence_pending", "findings": ["Host110 Ollama was removed and is forbidden as a provider, proxy, primary, secondary or failover target", "A tombstone is retained so stale sensors and routes create drift instead of resurrecting the endpoint", "Production runtime and Prometheus absence receipts are still missing"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json"], "owner_lane": "ai_team", "domain_router": "host_systemd", "executor": "host_ansible_executor", "verifier": "host110_ollama_runtime_and_monitoring_absence_verifier", "monitoring": {"signals": ["runtime_endpoint_absence", "prometheus_target_absence", "route_reference_absence"], "coverage": "source_tombstone_runtime_absence_pending"}, "next_action": "Verify no Host110 Ollama process, proxy, target, rule or provider route exists; preserve the tombstone and create drift on any recurrence."}, + {"canonical_id": "service:ollama:host110", "inventory_labels": ["retired Host110 Ollama tombstone"], "source_rows": [], "runtime_identity": "retired-tombstone:host110/ollama/no-endpoint", "source_truth_state": "retired_tombstone_runtime_absence_verified", "reconciliation_state": "retired_tombstone_runtime_absence_verified", "findings": ["Host110 Ollama is forbidden as a provider, proxy, primary, secondary or failover target", "A tombstone is retained so stale sensors and routes create drift instead of resurrecting the endpoint", "AIA-SRE-002 run-aia-sre-002-20260716-1752 removed the exact stale Nginx 11435 proxy with a root-only rollback backup", "Independent post-verifier proved listeners 11435/11434=0, Ollama containers and systemd units=0, stale config paths=0, Nginx active/config valid and Prometheus host110 Ollama targets=0"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "scripts/ops/retire-host110-ollama-proxy.sh"], "owner_lane": "ai_team", "domain_router": "host_systemd", "executor": "host_ansible_executor", "bounded_executor_receipt": "run-aia-sre-002-20260716-1752", "runtime_closure": {"status": "verified_absent", "trace_id": "trace-aia-sre-002-20260716-1752", "run_id": "run-aia-sre-002-20260716-1752", "work_item_id": "AIA-SRE-002", "post_verifier": "host110_ollama_runtime_and_monitoring_absence_verifier"}, "verifier": "host110_ollama_runtime_and_monitoring_absence_verifier", "monitoring": {"signals": ["runtime_endpoint_absence", "prometheus_target_absence", "route_reference_absence"], "coverage": "runtime_absence_verified_20260716"}, "next_action": "Preserve the tombstone and create a drift work item on any Host110 Ollama process, proxy, listener, target, rule or provider-route recurrence."}, {"canonical_id": "host:111", "inventory_labels": ["host111 local Ollama"], "source_rows": [], "runtime_identity": "192.168.0.111/macos-launchd/ollama", "findings": ["Approved third provider hop and Ansible host are absent from the imported inventory", "The runtime manager is a macOS LaunchAgent, not systemd", "The exact LaunchAgent playbook requires local verification plus independent origin probes from host120 and host121", "The canonical host111 Ollama sensor/rule has no fresh production series"], "source_refs": ["infra/ansible/inventory/hosts.yml", "infra/ansible/playbooks/111-ollama-fallback.yml", "ops/monitoring/service-registry.yaml", "ops/monitoring/alerts-unified.yml", "apps/api/src/services/awooop_ansible_post_verifier.py"], "owner_lane": "ai_team", "domain_router": "host_systemd", "executor": "host_ansible_executor", "verifier": "host111_launchagent_local_and_120_121_origin_verifier", "monitoring": {"signals": ["launchagent_state", "local_generation", "host120_origin_generation", "host121_origin_generation", "canonical_alert_series"], "coverage": "source_candidate_runtime_sensor_missing"}, "next_action": "Check/apply the exact LaunchAgent catalog, verify local plus host120/host121 origins, and require a fresh canonical sensor series before provider readiness."}, {"canonical_id": "ai-provider:ollama_gcp_a", "inventory_labels": ["GCP-A Ollama"], "source_rows": [], "runtime_identity": "34.143.170.20:11434", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Approved first provider hop is absent from the imported inventory", "Observed transport is public HTTP and is candidate-only until secure mesh/TLS promotion", "Only sanitized prompts may cross this boundary; unsanitized input and every tool loop fail closed before network access", "The exact Prometheus provider target is missing in production"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/monitoring/prometheus.yml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "apps/api/src/services/ai_provider_policy.py"], "owner_lane": "ai_team", "domain_router": "ai_provider", "executor": "deterministic_provider_router", "verifier": "provider_health_generation_and_transport_receipt_verifier", "monitoring": {"signals": ["exact_target_up", "scrape_freshness", "sanitization_receipt", "tool_loop_denial", "transport_boundary"], "coverage": "source_candidate_runtime_target_missing"}, "transport_policy": {"observed": "public_http", "execution_scope": "sanitized_candidate_only", "tool_loop": "fail_closed", "promotion_requires": "secure_mesh_or_tls_plus_runtime_readback"}, "next_action": "Deploy the exact blackbox target, verify fresh series and sanitization/tool-loop denials, then retain candidate-only status until secure transport promotion."}, {"canonical_id": "ai-provider:ollama_gcp_b", "inventory_labels": ["GCP-B Ollama"], "source_rows": [], "runtime_identity": "34.21.145.224:11434", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Approved second provider hop is absent from the imported inventory", "Observed transport is public HTTP and is candidate-only until secure mesh/TLS promotion", "Only sanitized prompts may cross this boundary; unsanitized input and every tool loop fail closed before network access", "The exact Prometheus provider target is missing in production"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/monitoring/prometheus.yml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "apps/api/src/services/ai_provider_policy.py"], "owner_lane": "ai_team", "domain_router": "ai_provider", "executor": "deterministic_provider_router", "verifier": "provider_health_generation_and_transport_receipt_verifier", "monitoring": {"signals": ["exact_target_up", "scrape_freshness", "sanitization_receipt", "tool_loop_denial", "transport_boundary"], "coverage": "source_candidate_runtime_target_missing"}, "transport_policy": {"observed": "public_http", "execution_scope": "sanitized_candidate_only", "tool_loop": "fail_closed", "promotion_requires": "secure_mesh_or_tls_plus_runtime_readback"}, "next_action": "Deploy the exact blackbox target, verify fresh series and sanitization/tool-loop denials, then retain candidate-only status until secure transport promotion."}, @@ -566,8 +566,8 @@ } ], "conflict_register": [ - {"id": "PORT-CONFLICT-001", "priority": "P0", "assets": ["service:alertmanager", "signal:alertmanager-webhook-chain"], "conflict": "Alertmanager source identity, receiver-scoped metrics, exact repair path and host99 Agent99 independent pull/reduced relay are reconciled in source; production deployment, poll freshness/dedupe, alert resolution and same-run closure receipts remain pending.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-003", "AIA-SRE-004", "AIA-SRE-014", "AIA-SRE-015", "AIA-SRE-017"]}, - {"id": "PORT-CONFLICT-002", "priority": "P0", "assets": ["service:ollama:host110", "service:ollama:host188", "host:111", "ai-provider:ollama_gcp_a", "ai-provider:ollama_gcp_b", "ai-provider:claude", "ai-provider:gemini"], "conflict": "Host110 is source-tombstoned but runtime absence is unverified; host111 is a LaunchAgent without local plus host120/121 same-run verification or a fresh sensor; GCP-A/GCP-B remain public-HTTP sanitized candidates with exact Prometheus targets missing; Claude/Gemini still lack durable Gate5 authorization, protected-secret metadata readback, verified cost caps, paired rollback and a completed five-lane canary.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-004", "AIA-SRE-008", "AIA-SRE-013", "AIA-SRE-014"]}, + {"id": "PORT-CONFLICT-001", "priority": "P0", "assets": ["service:alertmanager", "signal:alertmanager-webhook-chain"], "conflict": "Alertmanager source identity, production-compatible integration counters plus sole-webhook receiver contract, exact repair path and host99 Agent99 independent pull/reduced relay are reconciled in source; production apply, poll freshness/dedupe, alert resolution and same-run closure receipts remain pending.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-003", "AIA-SRE-004", "AIA-SRE-014", "AIA-SRE-015", "AIA-SRE-017"]}, + {"id": "PORT-CONFLICT-002", "priority": "P0", "assets": ["service:ollama:host110", "service:ollama:host188", "host:111", "ai-provider:ollama_gcp_a", "ai-provider:ollama_gcp_b", "ai-provider:claude", "ai-provider:gemini"], "conflict": "Host110 tombstone runtime absence is verified under AIA-SRE-002 and must remain absent; host111 is a LaunchAgent without local plus host120/121 same-run verification or a fresh sensor; GCP-A/GCP-B remain public-HTTP sanitized candidates with exact Prometheus targets missing; Claude/Gemini still lack a completed five-lane canary and durable cost receipt.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-004", "AIA-SRE-008", "AIA-SRE-013", "AIA-SRE-014"]}, {"id": "PORT-CONFLICT-003", "priority": "P0", "assets": ["service:gitea-exporter:host110", "service:sentry:relay", "service:sentry:snuba", "runner:gitea:vibework-host110"], "conflict": "GitHub exporter source retirement is ready, but the exact gitea-native production target is missing and the legacy exporter remains visible; remaining ghcr/runner claims also conflict with the global GitHub freeze.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-004", "AIA-SRE-014", "AIA-SRE-018"]}, {"id": "PORT-CONFLICT-004", "priority": "P0", "assets": ["network:keepalived:vip-unresolved"], "conflict": "Inventory VIP 192.168.0.125 conflicts with Ansible VIP 192.168.0.200 and host membership differs.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-003"]}, {"id": "PORT-CONFLICT-005", "priority": "P0", "assets": ["service:signoz", "service:otel-collector:signoz", "database:signoz:clickhouse"], "conflict": "SignOz UI, collector, and ClickHouse host placement is inconsistent across imported inventory and registries.", "linked_work_items": ["AIA-SRE-002", "AIA-SRE-014"]}, @@ -606,9 +606,9 @@ "conflict_assets": 36, "reconciliation_work_items": 12, "source_reconciled_assets": 6, - "runtime_closed_assets": 0, - "program_completion_percent": 0, + "runtime_closed_assets": 1, + "program_completion_percent": 1, "asset_coverage_status": "all_imported_rows_mapped_but_identity_and_runtime_coverage_partial", - "runtime_closure_status": "not_started_for_this_reconciliation" + "runtime_closure_status": "started_partial_1_of_144" } } diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index a3578b7a7..a6649b673 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -308,6 +308,7 @@ "apps/api/src/services/portfolio_infrastructure_asset_reconciliation.py", "apps/api/src/api/v1/agents.py", "apps/api/tests/test_portfolio_infrastructure_asset_reconciliation_api.py", + "scripts/ops/retire-host110-ollama-proxy.sh", "scripts/ops/render-service-registry-configmap.py", "k8s/awoooi-prod/15-service-registry-configmap.yaml" ], @@ -317,17 +318,17 @@ "exit_condition": "production ConfigMap, monitoring inventory and portfolio asset graph match live canonical identities; Host110 Ollama is present only as a retired tombstone; GCP-A/GCP-B exact targets and the host111 LaunchAgent sensor are fresh; inventory snapshots are never treated as runtime health", "confirmed_truth": [ "Host110 Ollama has been removed and must remain a retired tombstone, never a fallback endpoint", + "AIA-SRE-002 run-aia-sre-002-20260716-1752 removed the exact stale Nginx 11435 proxy with rollback backup and independently verified listeners 11435/11434=0, Ollama containers=0, stale config paths=0, nginx active/test pass and Prometheus host110 Ollama targets=0", "Host111 is the third provider hop and runs as a macOS LaunchAgent, not systemd", "GCP-A and GCP-B source targets are public HTTP sanitized candidates only", "gitea-native source target is not production runtime evidence while the legacy GitHub exporter remains visible" ], "runtime_gaps": [ - "Host110 Ollama absence has no same-run production tombstone verifier receipt", "Host111 LaunchAgent lacks completed local plus host120/host121 origin readback and a fresh exact sensor series", "GCP-A/GCP-B exact Prometheus target series are missing in production", "gitea-native exact target is missing and legacy GitHub exporter runtime drift is unresolved" ], - "next_action": "deploy the reconciled tombstone/targets, then verify Host110 absence, Host111 LaunchAgent from local and host120/121, GCP-A/GCP-B exact Prometheus series, gitea-native target freshness and legacy exporter absence under one source SHA" + "next_action": "deploy the reconciled tombstone/targets, preserve the verified Host110 absence receipt, then verify Host111 LaunchAgent from local and host120/121, GCP-A/GCP-B exact Prometheus series, gitea-native target freshness and legacy exporter absence under one source SHA" }, { "id": "AIA-SRE-003", @@ -707,12 +708,12 @@ "executor": "canonical Telegram gateway", "verifier": "delivery receipt plus desktop/mobile visible smoke", "rollback": "suppress duplicate projection; retain incident ledger", - "exit_condition": "no alert is sprayed across groups/bots; every card states AI/provider/Agent action, executor, verifier and receipt truth; AlertChainBroken uses receiver-scoped delivery counters plus host99 Agent99 exact-host read-only polling independent of the broken webhook; UI status polling never emits failover alerts", + "exit_condition": "no alert is sprayed across groups/bots; every card states AI/provider/Agent action, executor, verifier and receipt truth; AlertChainBroken uses Alertmanager integration-scoped delivery counters stamped with the independently tested sole-webhook receiver contract plus host99 Agent99 exact-host read-only polling independent of the broken webhook; UI status polling never emits failover alerts", "runtime_gaps": [ "the host99 Agent99 independent pull/reduced relay has no production deployment, freshness, dedupe or controlled-repair receipt", - "the proposed rule/scrape/card path has no production alert-resolution or same-run learning receipt" + "production Alertmanager exposes integration-only notification counters; the bounded check now passes after adding a scrape-time receiver_contract label, but apply, alert resolution and same-run learning receipts remain pending" ], - "next_action": "deploy Alertmanager runtime scrape/canonical rules and the host99 Agent99 exact-host poller through the atomic 15-file bundle; verify receiver-scoped freshness, dedupe and the current firing alert resolution; then reconcile every project/product/alert class against its typed domain and canonical Telegram destination" + "next_action": "deploy the production-schema-compatible Alertmanager runtime scrape/canonical rules and the host99 Agent99 exact-host poller through the atomic 15-file bundle; verify receiver-contract freshness, dedupe and the current firing alert resolution; then reconcile every project/product/alert class against its typed domain and canonical Telegram destination" }, { "id": "AIA-SRE-018", diff --git a/infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml b/infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml index b7c8e7b3c..815276e70 100644 --- a/infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml +++ b/infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml @@ -104,8 +104,8 @@ ansible.builtin.shell: | set -euo pipefail metrics="$(curl -fsS --max-time 5 http://127.0.0.1:9093/metrics)" - grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${metrics}" - grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${metrics}" + grep -Eq '^alertmanager_notification_requests_total\{[^}]*integration="webhook"' <<<"${metrics}" + grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*integration="webhook"' <<<"${metrics}" args: executable: /bin/bash check_mode: false diff --git a/k8s/monitoring/alert-chain-monitor.yaml b/k8s/monitoring/alert-chain-monitor.yaml index a96013179..a2e391117 100644 --- a/k8s/monitoring/alert-chain-monitor.yaml +++ b/k8s/monitoring/alert-chain-monitor.yaml @@ -30,20 +30,20 @@ spec: sum(increase(alertmanager_notification_requests_failed_total{ job="alertmanager-runtime", integration="webhook", - receiver="awoooi-webhook" + receiver_contract="awoooi-webhook" }[10m])) / clamp_min(sum(increase(alertmanager_notification_requests_total{ job="alertmanager-runtime", integration="webhook", - receiver="awoooi-webhook" + receiver_contract="awoooi-webhook" }[10m])), 1) ) > 0.1 and sum(increase(alertmanager_notification_requests_total{ job="alertmanager-runtime", integration="webhook", - receiver="awoooi-webhook" + receiver_contract="awoooi-webhook" }[10m])) >= 5 for: 5m labels: @@ -55,7 +55,7 @@ spec: alert_category: alert_chain_health annotations: summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" - description: "Alertmanager 自身且 receiver=awoooi-webhook 的單調 delivery counter 在 10 分鐘至少 5 次嘗試下，失敗率持續超過 10%；Telegram 與 Agent99 poll 不納入分母。" + description: "Alertmanager 自身的 webhook 單調 delivery counter，經唯一 webhook receiver source contract 標記為 awoooi-webhook；10 分鐘至少 5 次嘗試且失敗率持續超過 10%。Telegram 與 Agent99 poll 不納入分母。" runbook_url: "https://awoooi.internal/runbooks/alert-chain" # ----------------------------------------------------------------- diff --git a/k8s/monitoring/prometheus.yml b/k8s/monitoring/prometheus.yml index 7d1a5ce5e..6b1c9e2d2 100644 --- a/k8s/monitoring/prometheus.yml +++ b/k8s/monitoring/prometheus.yml @@ -187,6 +187,16 @@ scrape_configs: host: '110' service: 'alertmanager' env: 'prod' + # Production Alertmanager exposes notification counters by integration, + # not by receiver. The committed config has exactly one webhook receiver + # (awoooi-webhook), so stamp that independently tested source contract onto + # only the two webhook counter families at scrape time. + metric_relabel_configs: + - source_labels: [__name__, integration] + separator: ';' + regex: 'alertmanager_notification_requests(_failed)?_total;webhook' + target_label: receiver_contract + replacement: 'awoooi-webhook' # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH # === ADR-074 M1: 飛輪健康度指標 (2026-04-12 ogt) === diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 5efcfde4a..593257bd2 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -886,11 +886,11 @@ groups: - alert: AlertChainBroken_Alertmanager expr: | ( - sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) - / clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])), 1) + sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}[10m])) + / clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}[10m])), 1) ) > 0.1 and - sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) >= 5 + sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}[10m])) >= 5 for: 5m labels: severity: critical @@ -901,7 +901,7 @@ groups: alert_category: "alert_chain_health" annotations: summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" - description: "只使用 receiver=awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" + description: "只使用經唯一 webhook receiver source contract 標記為 awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" - alert: AlertChainBroken_Sentry expr: | diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index d1a39d21d..18cee8149 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -567,11 +567,11 @@ groups: - alert: AlertChainBroken_Alertmanager expr: | ( - sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) - / clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])), 1) + sum(increase(alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}[10m])) + / clamp_min(sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}[10m])), 1) ) > 0.1 and - sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}[10m])) >= 5 + sum(increase(alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}[10m])) >= 5 for: 5m labels: severity: critical @@ -582,7 +582,7 @@ groups: alert_category: "alert_chain_health" annotations: summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" - description: "只使用 receiver=awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" + description: "只使用經唯一 webhook receiver source contract 標記為 awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" - alert: AlertChainBroken_Sentry expr: | diff --git a/scripts/ops/deploy-alertmanager-runtime-scrape.sh b/scripts/ops/deploy-alertmanager-runtime-scrape.sh index 53c34aa93..9683a2e7d 100755 --- a/scripts/ops/deploy-alertmanager-runtime-scrape.sh +++ b/scripts/ops/deploy-alertmanager-runtime-scrape.sh @@ -119,8 +119,8 @@ test -f "${REMOTE_CONFIG}" test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")" -grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}" -grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}" +grep -Eq '^alertmanager_notification_requests_total\{[^}]*integration="webhook"' <<<"${METRICS_SNAPSHOT}" +grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*integration="webhook"' <<<"${METRICS_SNAPSHOT}" SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY' @@ -144,6 +144,12 @@ block = """ # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH host: '110' service: 'alertmanager' env: 'prod' + metric_relabel_configs: + - source_labels: [__name__, integration] + separator: ';' + regex: 'alertmanager_notification_requests(_failed)?_total;webhook' + target_label: receiver_contract + replacement: 'awoooi-webhook' # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH """ @@ -376,7 +382,7 @@ test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}" test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}" SERIES_COUNT="$(curl -fsS --get \ - --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \ + --data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}' \ "${PROMETHEUS_URL}/api/v1/query" | python3 -c ' import json import sys @@ -385,7 +391,7 @@ print(len(json.load(sys.stdin)["data"]["result"])) test "${SERIES_COUNT}" = 1 FAILED_SERIES_COUNT="$(curl -fsS --get \ - --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \ + --data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver_contract="awoooi-webhook"}' \ "${PROMETHEUS_URL}/api/v1/query" | python3 -c ' import json import sys diff --git a/scripts/ops/deploy-alerts.sh b/scripts/ops/deploy-alerts.sh index bf881c797..66614fed7 100755 --- a/scripts/ops/deploy-alerts.sh +++ b/scripts/ops/deploy-alerts.sh @@ -188,12 +188,12 @@ log "✅ NoAlertsReceived2Hours query 已限制 alertmanager 主鏈路" ALERT_CHAIN_BROKEN_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'AlertChainBroken_Alertmanager'), ''))") if [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_failed_total'* ]] \ || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_total'* ]] \ - || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver="awoooi-webhook"'* ]] \ + || [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver_contract="awoooi-webhook"'* ]] \ || [[ "$ALERT_CHAIN_BROKEN_QUERY" == *'awoooi_webhook_requests_total'* ]]; then echo "ERROR: AlertChainBroken_Alertmanager 未使用 primary awoooi-webhook monotonic delivery truth: ${ALERT_CHAIN_BROKEN_QUERY}" exit 1 fi -log "✅ AlertChainBroken_Alertmanager 只使用 primary awoooi-webhook monotonic delivery truth" +log "✅ AlertChainBroken_Alertmanager 使用唯一 webhook source contract 的 monotonic delivery truth" SOURCE_PROVIDER_STALE_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'SourceProviderIngestionStale'), ''))") if [[ "$SOURCE_PROVIDER_STALE_QUERY" != *'source=~"sentry|signoz"'* ]]; then diff --git a/scripts/ops/retire-host110-ollama-proxy.sh b/scripts/ops/retire-host110-ollama-proxy.sh new file mode 100755 index 000000000..419a7d7ef --- /dev/null +++ b/scripts/ops/retire-host110-ollama-proxy.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Retire only the stale host110 Nginx listener that formerly proxied Ollama. + +set -euo pipefail + +TRACE_ID="${TRACE_ID:?TRACE_ID is required}" +RUN_ID="${RUN_ID:?RUN_ID is required}" +WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}" +TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" +MODE="${1:---check}" + +case "${MODE}" in + apply|--check) ;; + *) printf 'usage: %s [--check|apply]\n' "$0" >&2; exit 64 ;; +esac + +for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do + case "${value}" in + *[!A-Za-z0-9._-]*|'') + printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2 + exit 64 + ;; + esac +done + +ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \ + bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" <<'REMOTE' +set -euo pipefail + +TRACE_ID="$1" +RUN_ID="$2" +WORK_ITEM_ID="$3" +MODE="$4" +CONFIG_PATH="/etc/nginx/sites-enabled/110-ollama-proxy.conf" +BACKUP_PATH="/var/backups/awoooi/110-ollama-proxy.conf.${RUN_ID}" +LOCK_PATH="/tmp/awoooi-host110-ollama-proxy-retirement.lock" +APPLY_STARTED=0 +VERIFIED=0 +SOURCE_SHA="" + +exec 9>"${LOCK_PATH}" +if ! flock -n 9; then + printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=retirement_apply_in_progress\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2 + exit 75 +fi + +rollback() { + local rollback_rc=0 + if [ "${APPLY_STARTED}" -eq 1 ] && [ "${VERIFIED}" -eq 0 ]; then + sudo -n test -f "${BACKUP_PATH}" || rollback_rc=1 + if [ "${rollback_rc}" -eq 0 ]; then + sudo -n install -o root -g root -m 0644 \ + "${BACKUP_PATH}" "${CONFIG_PATH}" || rollback_rc=1 + fi + if [ "${rollback_rc}" -eq 0 ]; then + sudo -n nginx -t >/dev/null 2>&1 || rollback_rc=1 + sudo -n systemctl reload nginx || rollback_rc=1 + fi + if [ "${rollback_rc}" -eq 0 ]; then + printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=restored source_sha=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}" + return 0 + fi + printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_PATH}" >&2 + return 1 + fi +} + +on_exit() { + local rc=$? + local rollback_rc=0 + trap - EXIT + set +e + if [ "${rc}" -ne 0 ]; then + rollback || rollback_rc=$? + fi + if [ "${rollback_rc}" -ne 0 ]; then + exit 70 + fi + exit "${rc}" +} +trap on_exit EXIT + +sudo -n true +if ! sudo -n test -e "${CONFIG_PATH}"; then + test "$(sudo -n grep -RIl --include='*.conf' --include='nginx.conf' '11435' /etc/nginx 2>/dev/null | wc -l | tr -d ' ')" = "0" + test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0" + test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0" + test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0" + sudo -n systemctl is-active --quiet nginx + sudo -n nginx -t >/dev/null 2>&1 + printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s state=already_retired listener_11435=0 listener_11434=0 ollama_containers=0 stale_config_paths=0 nginx_active=true nginx_test=pass persistent_writes_performed=0\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" + printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed_already_converged_no_write\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" + exit 0 +fi +sudo -n test -f "${CONFIG_PATH}" +test "$(sudo -n stat -c '%F' "${CONFIG_PATH}")" = "regular file" +test "$(sudo -n stat -c '%U:%G:%a' "${CONFIG_PATH}")" = "root:root:644" +test "$(sudo -n grep -Ec '^[[:space:]]*listen[[:space:]]+.*11435' "${CONFIG_PATH}")" = "2" +PROXY_PASS_COUNT="$(sudo -n grep -Ec '^[[:space:]]*proxy_pass[[:space:]]+http://[^;]+:11434' "${CONFIG_PATH}" || true)" +ALL_PROXY_PASS_COUNT="$(sudo -n grep -Ec '^[[:space:]]*proxy_pass[[:space:]]+' "${CONFIG_PATH}" || true)" +test "${PROXY_PASS_COUNT}" -ge 1 +test "${PROXY_PASS_COUNT}" = "${ALL_PROXY_PASS_COUNT}" +test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "2" +test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0" +test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0" +SOURCE_SHA="$(sudo -n sha256sum "${CONFIG_PATH}" | awk '{print $1}')" +sudo -n nginx -t >/dev/null 2>&1 + +printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s exact_path=%s source_sha=%s listener_11435=2 listener_11434=0 ollama_containers=0 nginx_test=pass persistent_writes_performed=0\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \ + "${CONFIG_PATH}" "${SOURCE_SHA}" + +if [ "${MODE}" = "--check" ]; then + printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_write\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" + exit 0 +fi + +sudo -n install -d -o root -g root -m 0700 /var/backups/awoooi +sudo -n install -o root -g root -m 0600 "${CONFIG_PATH}" "${BACKUP_PATH}" +test "$(sudo -n sha256sum "${BACKUP_PATH}" | awk '{print $1}')" = "${SOURCE_SHA}" +APPLY_STARTED=1 +sudo -n rm -- "${CONFIG_PATH}" +sudo -n nginx -t >/dev/null 2>&1 +sudo -n systemctl reload nginx + +for _ in $(seq 1 10); do + if [ "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0" ]; then + break + fi + sleep 1 +done + +sudo -n test ! -e "${CONFIG_PATH}" +test "$(sudo -n grep -RIl --include='*.conf' --include='nginx.conf' '11435' /etc/nginx 2>/dev/null | wc -l | tr -d ' ')" = "0" +test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0" +test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0" +test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0" +sudo -n systemctl is-active --quiet nginx +sudo -n nginx -t >/dev/null 2>&1 +VERIFIED=1 + +printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=retire_host110_ollama_proxy source_sha=%s backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}" "${BACKUP_PATH}" +printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s listener_11435=0 listener_11434=0 ollama_containers=0 stale_config_paths=0 nginx_active=true nginx_test=pass\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" +printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" +REMOTE diff --git a/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py b/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py index ca58a54b6..816d1ed20 100644 --- a/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py +++ b/scripts/ops/tests/test_alertmanager_runtime_scrape_contract.py @@ -22,8 +22,10 @@ def test_controlled_scrape_apply_has_check_rollback_and_independent_verifier() - assert '!= "${SOURCE_SHA}"' in text assert "alertmanager_notification_requests_total" in text assert "alertmanager_notification_requests_failed_total" in text - assert 'receiver="awoooi-webhook"' in text - assert text.count('receiver="awoooi-webhook"') >= 4 + assert 'integration="webhook"' in text + assert "target_label: receiver_contract" in text + assert "replacement: 'awoooi-webhook'" in text + assert text.count('receiver_contract="awoooi-webhook"') >= 2 assert "two_scrapes_up" in text assert "delivery_series_count" in text diff --git a/scripts/ops/tests/test_host110_ollama_proxy_retirement_contract.py b/scripts/ops/tests/test_host110_ollama_proxy_retirement_contract.py new file mode 100644 index 000000000..70914d71e --- /dev/null +++ b/scripts/ops/tests/test_host110_ollama_proxy_retirement_contract.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "ops" / "retire-host110-ollama-proxy.sh" + + +def test_retirement_is_exact_bounded_and_rollback_capable() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert 'TRACE_ID="${TRACE_ID:?TRACE_ID is required}"' in text + assert 'RUN_ID="${RUN_ID:?RUN_ID is required}"' in text + assert 'WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"' in text + assert 'CONFIG_PATH="/etc/nginx/sites-enabled/110-ollama-proxy.conf"' in text + assert "flock -n 9" in text + assert "source_sha=" in text + assert "persistent_writes_performed=0" in text + assert "rollback_receipt" in text + assert "terminal=rollback_failed" in text + assert "sudo -n nginx -t" in text + assert "sudo -n systemctl reload nginx" in text + assert "completed_already_converged_no_write" in text + assert "state=already_retired" in text + + +def test_retirement_proves_the_retired_identity_cannot_reappear() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert "listener_11435=0" in text + assert "listener_11434=0" in text + assert "ollama_containers=0" in text + assert "stale_config_paths=0" in text + assert "grep -RIl" in text + assert "rm -rf" not in text + assert "pkill" not in text + assert "killall" not in text