From fa51ab9022edbdce884e6886757b58b147cc947f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 01:37:44 +0800 Subject: [PATCH] feat(sre): register independent verifier coverage --- .../services/independent_verifier_registry.py | 231 ++++++++++++++++++ ...re_k3s_controlled_automation_work_items.py | 17 ++ .../test_independent_verifier_registry.py | 81 ++++++ ...3s_controlled_automation_work_items_api.py | 29 ++- ...rolled-automation-work-items.snapshot.json | 20 +- 5 files changed, 369 insertions(+), 9 deletions(-) create mode 100644 apps/api/src/services/independent_verifier_registry.py create mode 100644 apps/api/tests/test_independent_verifier_registry.py diff --git a/apps/api/src/services/independent_verifier_registry.py b/apps/api/src/services/independent_verifier_registry.py new file mode 100644 index 000000000..9fd3ba166 --- /dev/null +++ b/apps/api/src/services/independent_verifier_registry.py @@ -0,0 +1,231 @@ +"""Machine-readable source coverage for every controlled SRE verifier lane. + +This registry is source truth only. It proves that a typed executor is bound +to a separate verifier contract; it never turns source coverage into a +production receipt or runtime closure. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from typing import Any + +SCHEMA_VERSION = "independent_verifier_registry_v1" + +_DOMAIN_CONTRACTS: tuple[dict[str, Any], ...] = ( + { + "domain": "kubernetes_workload", + "mutating": True, + "executor": "kubernetes_controlled_executor", + "verifier": "kubernetes_rollout_verifier", + "runtime_write_allowed": True, + "receipt_required": True, + "source_ref": "apps/api/src/services/kubernetes_rollout_verifier.py", + }, + { + "domain": "host_systemd", + "mutating": True, + "executor": "host_ansible_executor", + "verifier": "asset_specific_read_only_host_postconditions", + "runtime_write_allowed": True, + "receipt_required": True, + "source_ref": "apps/api/src/services/awooop_ansible_post_verifier.py", + }, + { + "domain": "docker_container", + "mutating": True, + "executor": "host_ansible_executor", + "verifier": "asset_specific_read_only_host_postconditions", + "runtime_write_allowed": True, + "receipt_required": True, + "source_ref": "apps/api/src/services/awooop_ansible_post_verifier.py", + }, + { + "domain": "windows_vmware", + "mutating": True, + "executor": "Agent99", + "verifier": "agent99_independent_runtime_verifier", + "runtime_write_allowed": True, + "receipt_required": True, + "source_ref": "apps/api/src/services/agent99_controlled_dispatch_ledger.py", + }, + { + "domain": "database", + "mutating": True, + "executor": "db_bounded_executor", + "verifier": "db_independent_verifier", + "runtime_write_allowed": True, + "receipt_required": True, + "source_ref": "apps/api/src/services/db_bounded_executor.py", + }, + { + "domain": "control_plane_recovery", + "mutating": True, + "executor": "Agent99", + "verifier": "agent99_independent_runtime_verifier", + "runtime_write_allowed": True, + "receipt_required": True, + "source_ref": "apps/api/src/services/agent99_controlled_dispatch_ledger.py", + }, + { + "domain": "backup_restore", + "mutating": False, + "executor": None, + "break_glass_executor": "backup_restore_break_glass", + "verifier": "backup_restore_readback_verifier", + "runtime_write_allowed": False, + "receipt_required": True, + "source_ref": "apps/api/src/services/backup_restore_signal_automation.py", + }, + { + "domain": "unknown", + "mutating": False, + "executor": None, + "verifier": "asset_identity_drift_verifier", + "runtime_write_allowed": False, + "receipt_required": True, + "source_ref": "apps/api/src/services/controlled_alert_target_router.py", + }, +) + +_EXTERNAL_READBACKS: tuple[dict[str, Any], ...] = ( + { + "readback_id": "host111_provider_origins", + "verifier": "asset_specific_read_only_host_postconditions", + "required_origins": ["host_111", "host_120", "host_121"], + "runtime_status": "receipt_pending", + }, + { + "readback_id": "gcp_provider_exact_fresh_series", + "verifier": "provider_health_generation_and_transport_receipt_verifier", + "required_targets": ["ollama_gcp_a", "ollama_gcp_b"], + "runtime_status": "receipt_pending", + }, + { + "readback_id": "gitea_native_metrics", + "verifier": "gitea_native_metrics_freshness_verifier", + "runtime_status": "receipt_pending", + }, + { + "readback_id": "host99_alertmanager_pull_relay", + "verifier": "agent99_alertmanager_pull_relay_verifier", + "runtime_status": "receipt_pending", + }, +) + + +def build_independent_verifier_registry( + *, + ansible_catalog: Iterable[Mapping[str, Any]] | None = None, + ansible_registered_catalog_ids: Iterable[str] | None = None, +) -> dict[str, Any]: + """Return deterministic source coverage without performing a runtime read.""" + + if ansible_catalog is None: + from src.services.awooop_ansible_audit_service import list_ansible_catalog + + ansible_catalog = list_ansible_catalog() + if ansible_registered_catalog_ids is None: + from src.services.awooop_ansible_post_verifier import registered_catalog_ids + + ansible_registered_catalog_ids = registered_catalog_ids() + + catalog_rows = [dict(row) for row in ansible_catalog] + catalog_ids = { + str(row.get("catalog_id") or "").strip() + for row in catalog_rows + if str(row.get("catalog_id") or "").strip() + } + registered_ids = { + str(catalog_id).strip() + for catalog_id in ansible_registered_catalog_ids + if str(catalog_id).strip() + } + missing_postconditions = sorted(catalog_ids - registered_ids) + orphan_postconditions = sorted(registered_ids - catalog_ids) + unsafe_auto_apply = sorted( + str(row.get("catalog_id") or "") + for row in catalog_rows + if row.get("auto_apply_enabled") is True + and row.get("supports_check_mode") is not True + ) + invalid_break_glass = sorted( + str(row.get("catalog_id") or "") + for row in catalog_rows + if row.get("supports_check_mode") is not True + and not ( + row.get("auto_apply_enabled") is False + and row.get("break_glass_required") is True + ) + ) + + domain_issues: list[str] = [] + for contract in _DOMAIN_CONTRACTS: + domain = str(contract["domain"]) + executor = contract.get("executor") + verifier = str(contract.get("verifier") or "") + if not verifier: + domain_issues.append(f"{domain}:verifier_missing") + if contract.get("receipt_required") is not True: + domain_issues.append(f"{domain}:receipt_not_required") + if contract.get("mutating") is True: + if not executor: + domain_issues.append(f"{domain}:executor_missing") + if str(executor) == verifier: + domain_issues.append(f"{domain}:executor_self_verifies") + elif contract.get("runtime_write_allowed") is not False: + domain_issues.append(f"{domain}:read_only_domain_write_allowed") + + source_blockers = [ + *(f"ansible_postcondition_missing:{item}" for item in missing_postconditions), + *(f"ansible_postcondition_orphan:{item}" for item in orphan_postconditions), + *(f"ansible_auto_apply_without_check_mode:{item}" for item in unsafe_auto_apply), + *(f"ansible_non_break_glass_without_check_mode:{item}" for item in invalid_break_glass), + *domain_issues, + ] + source_coverage_complete = not source_blockers + domains = [ + {**contract, "cross_domain_fallback_allowed": False} + for contract in _DOMAIN_CONTRACTS + ] + + return { + "schema_version": SCHEMA_VERSION, + "status": ( + "source_coverage_complete_runtime_receipts_pending" + if source_coverage_complete + else "source_coverage_incomplete_fail_closed" + ), + "source_coverage_complete": source_coverage_complete, + "runtime_closed": False, + "domain_contracts": domains, + "ansible_catalog_coverage": { + "catalog_count": len(catalog_ids), + "check_mode_catalog_count": sum( + row.get("supports_check_mode") is True for row in catalog_rows + ), + "auto_apply_catalog_count": sum( + row.get("auto_apply_enabled") is True for row in catalog_rows + ), + "registered_postcondition_count": len(registered_ids), + "missing_postcondition_catalog_ids": missing_postconditions, + "orphan_postcondition_catalog_ids": orphan_postconditions, + }, + "external_readbacks": [dict(row) for row in _EXTERNAL_READBACKS], + "source_blockers": source_blockers, + "runtime_blockers": [ + "production_same_run_verifier_receipts_pending", + "source_coverage_is_not_runtime_closure", + ], + "rollups": { + "domain_count": len(domains), + "mutating_domain_count": sum( + contract["mutating"] is True for contract in domains + ), + "read_only_domain_count": sum( + contract["mutating"] is False for contract in domains + ), + "source_blocker_count": len(source_blockers), + "runtime_closed_domain_count": 0, + }, + } diff --git a/apps/api/src/services/sre_k3s_controlled_automation_work_items.py b/apps/api/src/services/sre_k3s_controlled_automation_work_items.py index 859696a1a..acfc63208 100644 --- a/apps/api/src/services/sre_k3s_controlled_automation_work_items.py +++ b/apps/api/src/services/sre_k3s_controlled_automation_work_items.py @@ -279,12 +279,21 @@ def load_sre_k3s_controlled_automation_work_items( completion = _dict(payload, "completion_contract", path) if completion.get("source_test_cd_green_is_runtime_closure") is not False: raise ValueError(f"{path}: source/CD cannot be runtime closure") + from src.services.independent_verifier_registry import ( + build_independent_verifier_registry, + ) + + verifier_registry = build_independent_verifier_registry() + if verifier_registry.get("source_coverage_complete") is not True: + raise ValueError(f"{path}: independent verifier source coverage incomplete") + payload["independent_verifier_registry"] = verifier_registry return payload def build_sre_k3s_program_projection(payload: dict[str, Any]) -> dict[str, Any]: """Return a bounded cockpit projection without dropping truth labels.""" + verifier_registry = payload["independent_verifier_registry"] return { "schema_version": payload["schema_version"], "program_id": payload["program_id"], @@ -302,6 +311,14 @@ def build_sre_k3s_program_projection(payload: dict[str, Any]) -> dict[str, Any]: "gemini_paid_call_allowed" ], "domain_count": len(payload["domain_routes"]), + "independent_verifier_registry": { + "status": verifier_registry["status"], + "source_coverage_complete": verifier_registry[ + "source_coverage_complete" + ], + "runtime_closed": verifier_registry["runtime_closed"], + "rollups": verifier_registry["rollups"], + }, "agent99_bridge": payload["agent99_host_operations_bridge"], "full_readback_api": ( "/api/v1/agents/sre-k3s-controlled-automation-work-items" diff --git a/apps/api/tests/test_independent_verifier_registry.py b/apps/api/tests/test_independent_verifier_registry.py new file mode 100644 index 000000000..712b84344 --- /dev/null +++ b/apps/api/tests/test_independent_verifier_registry.py @@ -0,0 +1,81 @@ +from src.services.independent_verifier_registry import ( + build_independent_verifier_registry, +) + + +def test_registry_covers_every_typed_domain_without_false_runtime_closure() -> None: + registry = build_independent_verifier_registry() + + assert registry["status"] == ( + "source_coverage_complete_runtime_receipts_pending" + ) + assert registry["source_coverage_complete"] is True + assert registry["runtime_closed"] is False + assert registry["rollups"] == { + "domain_count": 8, + "mutating_domain_count": 6, + "read_only_domain_count": 2, + "source_blocker_count": 0, + "runtime_closed_domain_count": 0, + } + assert registry["ansible_catalog_coverage"] == { + "catalog_count": 17, + "check_mode_catalog_count": 16, + "auto_apply_catalog_count": 13, + "registered_postcondition_count": 17, + "missing_postcondition_catalog_ids": [], + "orphan_postcondition_catalog_ids": [], + } + by_domain = { + row["domain"]: row for row in registry["domain_contracts"] + } + assert set(by_domain) == { + "kubernetes_workload", + "host_systemd", + "docker_container", + "windows_vmware", + "database", + "backup_restore", + "unknown", + "control_plane_recovery", + } + assert all( + row["cross_domain_fallback_allowed"] is False + for row in by_domain.values() + ) + assert all( + row["executor"] != row["verifier"] + for row in by_domain.values() + if row["mutating"] is True + ) + assert by_domain["backup_restore"]["runtime_write_allowed"] is False + assert by_domain["backup_restore"]["executor"] is None + assert by_domain["unknown"]["runtime_write_allowed"] is False + assert by_domain["unknown"]["executor"] is None + assert all( + row["runtime_status"] == "receipt_pending" + for row in registry["external_readbacks"] + ) + + +def test_registry_fails_closed_when_one_catalog_loses_its_postcondition() -> None: + catalog = [ + { + "catalog_id": "ansible:one", + "supports_check_mode": True, + "auto_apply_enabled": True, + } + ] + registry = build_independent_verifier_registry( + ansible_catalog=catalog, + ansible_registered_catalog_ids=(), + ) + + assert registry["source_coverage_complete"] is False + assert registry["status"] == "source_coverage_incomplete_fail_closed" + assert registry["ansible_catalog_coverage"][ + "missing_postcondition_catalog_ids" + ] == ["ansible:one"] + assert registry["source_blockers"] == [ + "ansible_postcondition_missing:ansible:one" + ] diff --git a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py index 5ef6266f3..36f046809 100644 --- a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py +++ b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py @@ -116,11 +116,10 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "total_items": 18, "by_priority": {"P0": 16, "P1": 2}, "by_status": { - "source_implemented_runtime_pending": 14, - "in_progress": 1, + "source_implemented_runtime_pending": 15, "planned": 3, }, - "source_implemented_items": 14, + "source_implemented_items": 15, "runtime_closed_items": 0, "program_completion_percent": 0, "asset_coverage_status": "partial", @@ -152,6 +151,12 @@ def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None: assert "KM/RAG/MCP/PlayBook" in " ".join( items["AIA-SRE-013"]["runtime_gaps"] ) + assert items["AIA-SRE-014"]["status"] == ( + "source_implemented_runtime_pending" + ) + assert "all eight typed domains" in " ".join( + items["AIA-SRE-014"]["confirmed_truth"] + ) assert "host99 Agent99" in " ".join(items["AIA-SRE-017"]["runtime_gaps"]) assert items["AIA-SRE-009"]["status"] == ( "source_implemented_runtime_pending" @@ -195,10 +200,22 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None: payload = load_sre_k3s_controlled_automation_work_items() projection = build_sre_k3s_program_projection(payload) - assert projection["rollups"]["source_implemented_items"] == 14 + assert projection["rollups"]["source_implemented_items"] == 15 assert projection["rollups"]["runtime_closed_items"] == 0 assert projection["rollups"]["program_completion_percent"] == 0 assert projection["domain_count"] == 8 + assert projection["independent_verifier_registry"] == { + "status": "source_coverage_complete_runtime_receipts_pending", + "source_coverage_complete": True, + "runtime_closed": False, + "rollups": { + "domain_count": 8, + "mutating_domain_count": 6, + "read_only_domain_count": 2, + "source_blocker_count": 0, + "runtime_closed_domain_count": 0, + }, + } assert projection["claude_paid_call_allowed"] is True assert projection["gemini_paid_call_allowed"] is True assert projection["immediate_execution_queue"][0]["work_item_id"] == ( @@ -263,6 +280,10 @@ def test_endpoint_returns_full_work_ledger() -> None: assert payload["program_id"] == "AIA-SRE-P0-20260715" assert len(payload["work_items"]) == 18 assert payload["rollups"]["runtime_closed_items"] == 0 + assert payload["independent_verifier_registry"][ + "source_coverage_complete" + ] is True + assert payload["independent_verifier_registry"]["runtime_closed"] is False assert ( payload["agent99_host_operations_bridge"]["completion_callback"] == "/api/v1/agents/agent99/completion-callback" diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index 0abd146f5..c7cbbcdb1 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -724,7 +724,7 @@ "title": "Independent verifier registry 完整覆蓋", "owner_lane": "PostExecutionVerifier", "risk": "high", - "status": "in_progress", + "status": "source_implemented_runtime_pending", "dependencies": [ "AIA-SRE-005", "AIA-SRE-007", @@ -733,18 +733,29 @@ "AIA-SRE-010" ], "source_refs": [ + "apps/api/src/services/independent_verifier_registry.py", "apps/api/src/services/awooop_ansible_post_verifier.py", + "apps/api/src/services/kubernetes_rollout_verifier.py", + "apps/api/src/services/agent99_controlled_dispatch_ledger.py", + "apps/api/src/services/db_bounded_executor.py", + "apps/api/src/services/backup_restore_signal_automation.py", "apps/api/src/services/post_execution_verifier.py" ], "executor": "none", "verifier": "domain-specific external readback", "rollback": "failed verifier triggers same-domain retry or compensating action", "exit_condition": "100 percent of mutating catalogs have independent production postconditions; host111 includes local plus host120/host121 origin verification and provider monitoring requires exact fresh series", + "confirmed_truth": [ + "machine-readable registry covers all eight typed domains and keeps executor identity separate from verifier identity for all six mutating domains", + "all 17 Ansible catalogs have registered independent postconditions; every auto-apply catalog is check-mode capable and the sole exception is explicit break-glass", + "K3s, Host/Container, Agent99, DB, Backup/restore and unknown-asset lanes expose domain-specific verifier contracts with cross-domain fallback disabled", + "Host111 three-origin, GCP exact-series, gitea-native metrics and host99 Alertmanager relay readbacks are registered as runtime receipt requirements rather than source completion evidence" + ], "runtime_gaps": [ "source-level verifier definitions are not production verifier receipts", "Host111 LaunchAgent, GCP provider targets, gitea-native metrics and the host99 Alertmanager pull/relay remain without independent runtime closure" ], - "next_action": "reconcile catalog IDs against postcondition registry, then attach same-run runtime receipts for Host111 origins, GCP exact targets, gitea-native metrics and the host99 Alertmanager pull/relay" + "next_action": "deploy one exact source revision through the authorized release lane, then attach same-run production receipts for K3s, Ansible, Agent99, DB and Backup readback plus Host111 origins, GCP exact targets, gitea-native metrics and host99 Alertmanager pull/relay; any missing receipt remains fail-closed" }, { "id": "AIA-SRE-015", @@ -886,11 +897,10 @@ "P1": 2 }, "by_status": { - "source_implemented_runtime_pending": 14, - "in_progress": 1, + "source_implemented_runtime_pending": 15, "planned": 3 }, - "source_implemented_items": 14, + "source_implemented_items": 15, "runtime_closed_items": 0, "program_completion_percent": 0, "asset_coverage_status": "partial",