wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(sre): register independent verifier coverage

Browse Source
This commit is contained in:
Your Name
2026-07-19 01:37:44 +08:00
parent e48155c2d6
commit fa51ab9022
5 changed files with 369 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
apps/api/src/services/independent_verifier_registry.py Normal file
View File

@@ -0,0 +1,231 @@
"""Machine-readable source coverage for every controlled SRE verifier lane.
This registry is source truth only. It proves that a typed executor is bound
to a separate verifier contract; it never turns source coverage into a
production receipt or runtime closure.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any
SCHEMA_VERSION = "independent_verifier_registry_v1"
_DOMAIN_CONTRACTS: tuple[dict[str, Any], ...] = (
{
"domain": "kubernetes_workload",
"mutating": True,
"executor": "kubernetes_controlled_executor",
"verifier": "kubernetes_rollout_verifier",
"runtime_write_allowed": True,
"receipt_required": True,
"source_ref": "apps/api/src/services/kubernetes_rollout_verifier.py",
},
{
"domain": "host_systemd",
"mutating": True,
"executor": "host_ansible_executor",
"verifier": "asset_specific_read_only_host_postconditions",
"runtime_write_allowed": True,
"receipt_required": True,
"source_ref": "apps/api/src/services/awooop_ansible_post_verifier.py",
},
{
"domain": "docker_container",
"mutating": True,
"executor": "host_ansible_executor",
"verifier": "asset_specific_read_only_host_postconditions",
"runtime_write_allowed": True,
"receipt_required": True,
"source_ref": "apps/api/src/services/awooop_ansible_post_verifier.py",
},
{
"domain": "windows_vmware",
"mutating": True,
"executor": "Agent99",
"verifier": "agent99_independent_runtime_verifier",
"runtime_write_allowed": True,
"receipt_required": True,
"source_ref": "apps/api/src/services/agent99_controlled_dispatch_ledger.py",
},
{
"domain": "database",
"mutating": True,
"executor": "db_bounded_executor",
"verifier": "db_independent_verifier",
"runtime_write_allowed": True,
"receipt_required": True,
"source_ref": "apps/api/src/services/db_bounded_executor.py",
},
{
"domain": "control_plane_recovery",
"mutating": True,
"executor": "Agent99",
"verifier": "agent99_independent_runtime_verifier",
"runtime_write_allowed": True,
"receipt_required": True,
"source_ref": "apps/api/src/services/agent99_controlled_dispatch_ledger.py",
},
{
"domain": "backup_restore",
"mutating": False,
"executor": None,
"break_glass_executor": "backup_restore_break_glass",
"verifier": "backup_restore_readback_verifier",
"runtime_write_allowed": False,
"receipt_required": True,
"source_ref": "apps/api/src/services/backup_restore_signal_automation.py",
},
{
"domain": "unknown",
"mutating": False,
"executor": None,
"verifier": "asset_identity_drift_verifier",
"runtime_write_allowed": False,
"receipt_required": True,
"source_ref": "apps/api/src/services/controlled_alert_target_router.py",
},
)
_EXTERNAL_READBACKS: tuple[dict[str, Any], ...] = (
{
"readback_id": "host111_provider_origins",
"verifier": "asset_specific_read_only_host_postconditions",
"required_origins": ["host_111", "host_120", "host_121"],
"runtime_status": "receipt_pending",
},
{
"readback_id": "gcp_provider_exact_fresh_series",
"verifier": "provider_health_generation_and_transport_receipt_verifier",
"required_targets": ["ollama_gcp_a", "ollama_gcp_b"],
"runtime_status": "receipt_pending",
},
{
"readback_id": "gitea_native_metrics",
"verifier": "gitea_native_metrics_freshness_verifier",
"runtime_status": "receipt_pending",
},
{
"readback_id": "host99_alertmanager_pull_relay",
"verifier": "agent99_alertmanager_pull_relay_verifier",
"runtime_status": "receipt_pending",
},
)
def build_independent_verifier_registry(
*,
ansible_catalog: Iterable[Mapping[str, Any]] | None = None,
ansible_registered_catalog_ids: Iterable[str] | None = None,
) -> dict[str, Any]:
"""Return deterministic source coverage without performing a runtime read."""
if ansible_catalog is None:
from src.services.awooop_ansible_audit_service import list_ansible_catalog
ansible_catalog = list_ansible_catalog()
if ansible_registered_catalog_ids is None:
from src.services.awooop_ansible_post_verifier import registered_catalog_ids
ansible_registered_catalog_ids = registered_catalog_ids()
catalog_rows = [dict(row) for row in ansible_catalog]
catalog_ids = {
str(row.get("catalog_id") or "").strip()
for row in catalog_rows
if str(row.get("catalog_id") or "").strip()
}
registered_ids = {
str(catalog_id).strip()
for catalog_id in ansible_registered_catalog_ids
if str(catalog_id).strip()
}
missing_postconditions = sorted(catalog_ids - registered_ids)
orphan_postconditions = sorted(registered_ids - catalog_ids)
unsafe_auto_apply = sorted(
str(row.get("catalog_id") or "")
for row in catalog_rows
if row.get("auto_apply_enabled") is True
and row.get("supports_check_mode") is not True
)
invalid_break_glass = sorted(
str(row.get("catalog_id") or "")
for row in catalog_rows
if row.get("supports_check_mode") is not True
and not (
row.get("auto_apply_enabled") is False
and row.get("break_glass_required") is True
)
)
domain_issues: list[str] = []
for contract in _DOMAIN_CONTRACTS:
domain = str(contract["domain"])
executor = contract.get("executor")
verifier = str(contract.get("verifier") or "")
if not verifier:
domain_issues.append(f"{domain}:verifier_missing")
if contract.get("receipt_required") is not True:
domain_issues.append(f"{domain}:receipt_not_required")
if contract.get("mutating") is True:
if not executor:
domain_issues.append(f"{domain}:executor_missing")
if str(executor) == verifier:
domain_issues.append(f"{domain}:executor_self_verifies")
elif contract.get("runtime_write_allowed") is not False:
domain_issues.append(f"{domain}:read_only_domain_write_allowed")
source_blockers = [
*(f"ansible_postcondition_missing:{item}" for item in missing_postconditions),
*(f"ansible_postcondition_orphan:{item}" for item in orphan_postconditions),
*(f"ansible_auto_apply_without_check_mode:{item}" for item in unsafe_auto_apply),
*(f"ansible_non_break_glass_without_check_mode:{item}" for item in invalid_break_glass),
*domain_issues,
]
source_coverage_complete = not source_blockers
domains = [
{**contract, "cross_domain_fallback_allowed": False}
for contract in _DOMAIN_CONTRACTS
]
return {
"schema_version": SCHEMA_VERSION,
"status": (
"source_coverage_complete_runtime_receipts_pending"
if source_coverage_complete
else "source_coverage_incomplete_fail_closed"
),
"source_coverage_complete": source_coverage_complete,
"runtime_closed": False,
"domain_contracts": domains,
"ansible_catalog_coverage": {
"catalog_count": len(catalog_ids),
"check_mode_catalog_count": sum(
row.get("supports_check_mode") is True for row in catalog_rows
),
"auto_apply_catalog_count": sum(
row.get("auto_apply_enabled") is True for row in catalog_rows
),
"registered_postcondition_count": len(registered_ids),
"missing_postcondition_catalog_ids": missing_postconditions,
"orphan_postcondition_catalog_ids": orphan_postconditions,
},
"external_readbacks": [dict(row) for row in _EXTERNAL_READBACKS],
"source_blockers": source_blockers,
"runtime_blockers": [
"production_same_run_verifier_receipts_pending",
"source_coverage_is_not_runtime_closure",
],
"rollups": {
"domain_count": len(domains),
"mutating_domain_count": sum(
contract["mutating"] is True for contract in domains
),
"read_only_domain_count": sum(
contract["mutating"] is False for contract in domains
),
"source_blocker_count": len(source_blockers),
"runtime_closed_domain_count": 0,
},
}

17
apps/api/src/services/sre_k3s_controlled_automation_work_items.py
View File

@@ -279,12 +279,21 @@ def load_sre_k3s_controlled_automation_work_items(
completion = _dict(payload, "completion_contract", path)
if completion.get("source_test_cd_green_is_runtime_closure") is not False:
raise ValueError(f"{path}: source/CD cannot be runtime closure")
from src.services.independent_verifier_registry import (
build_independent_verifier_registry,
)
verifier_registry = build_independent_verifier_registry()
if verifier_registry.get("source_coverage_complete") is not True:
raise ValueError(f"{path}: independent verifier source coverage incomplete")
payload["independent_verifier_registry"] = verifier_registry
return payload
def build_sre_k3s_program_projection(payload: dict[str, Any]) -> dict[str, Any]:
"""Return a bounded cockpit projection without dropping truth labels."""
verifier_registry = payload["independent_verifier_registry"]
return {
"schema_version": payload["schema_version"],
"program_id": payload["program_id"],
@@ -302,6 +311,14 @@ def build_sre_k3s_program_projection(payload: dict[str, Any]) -> dict[str, Any]:
"gemini_paid_call_allowed"
],
"domain_count": len(payload["domain_routes"]),
"independent_verifier_registry": {
"status": verifier_registry["status"],
"source_coverage_complete": verifier_registry[
"source_coverage_complete"
],
"runtime_closed": verifier_registry["runtime_closed"],
"rollups": verifier_registry["rollups"],
},
"agent99_bridge": payload["agent99_host_operations_bridge"],
"full_readback_api": (
"/api/v1/agents/sre-k3s-controlled-automation-work-items"

81
apps/api/tests/test_independent_verifier_registry.py Normal file
View File

@@ -0,0 +1,81 @@
from src.services.independent_verifier_registry import (
build_independent_verifier_registry,
)
def test_registry_covers_every_typed_domain_without_false_runtime_closure() -> None:
registry = build_independent_verifier_registry()
assert registry["status"] == (
"source_coverage_complete_runtime_receipts_pending"
)
assert registry["source_coverage_complete"] is True
assert registry["runtime_closed"] is False
assert registry["rollups"] == {
"domain_count": 8,
"mutating_domain_count": 6,
"read_only_domain_count": 2,
"source_blocker_count": 0,
"runtime_closed_domain_count": 0,
}
assert registry["ansible_catalog_coverage"] == {
"catalog_count": 17,
"check_mode_catalog_count": 16,
"auto_apply_catalog_count": 13,
"registered_postcondition_count": 17,
"missing_postcondition_catalog_ids": [],
"orphan_postcondition_catalog_ids": [],
}
by_domain = {
row["domain"]: row for row in registry["domain_contracts"]
}
assert set(by_domain) == {
"kubernetes_workload",
"host_systemd",
"docker_container",
"windows_vmware",
"database",
"backup_restore",
"unknown",
"control_plane_recovery",
}
assert all(
row["cross_domain_fallback_allowed"] is False
for row in by_domain.values()
)
assert all(
row["executor"] != row["verifier"]
for row in by_domain.values()
if row["mutating"] is True
)
assert by_domain["backup_restore"]["runtime_write_allowed"] is False
assert by_domain["backup_restore"]["executor"] is None
assert by_domain["unknown"]["runtime_write_allowed"] is False
assert by_domain["unknown"]["executor"] is None
assert all(
row["runtime_status"] == "receipt_pending"
for row in registry["external_readbacks"]
)
def test_registry_fails_closed_when_one_catalog_loses_its_postcondition() -> None:
catalog = [
{
"catalog_id": "ansible:one",
"supports_check_mode": True,
"auto_apply_enabled": True,
}
]
registry = build_independent_verifier_registry(
ansible_catalog=catalog,
ansible_registered_catalog_ids=(),
)
assert registry["source_coverage_complete"] is False
assert registry["status"] == "source_coverage_incomplete_fail_closed"
assert registry["ansible_catalog_coverage"][
"missing_postcondition_catalog_ids"
] == ["ansible:one"]
assert registry["source_blockers"] == [
"ansible_postcondition_missing:ansible:one"
]

29
apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py
View File

@@ -116,11 +116,10 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
"total_items": 18,
"by_priority": {"P0": 16, "P1": 2},
"by_status": {
"source_implemented_runtime_pending": 14,
"in_progress": 1,
"source_implemented_runtime_pending": 15,
"planned": 3,
},
"source_implemented_items": 14,
"source_implemented_items": 15,
"runtime_closed_items": 0,
"program_completion_percent": 0,
"asset_coverage_status": "partial",
@@ -152,6 +151,12 @@ def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None:
assert "KM/RAG/MCP/PlayBook" in " ".join(
items["AIA-SRE-013"]["runtime_gaps"]
)
assert items["AIA-SRE-014"]["status"] == (
"source_implemented_runtime_pending"
)
assert "all eight typed domains" in " ".join(
items["AIA-SRE-014"]["confirmed_truth"]
)
assert "host99 Agent99" in " ".join(items["AIA-SRE-017"]["runtime_gaps"])
assert items["AIA-SRE-009"]["status"] == (
"source_implemented_runtime_pending"
@@ -195,10 +200,22 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None:
payload = load_sre_k3s_controlled_automation_work_items()
projection = build_sre_k3s_program_projection(payload)
assert projection["rollups"]["source_implemented_items"] == 14
assert projection["rollups"]["source_implemented_items"] == 15
assert projection["rollups"]["runtime_closed_items"] == 0
assert projection["rollups"]["program_completion_percent"] == 0
assert projection["domain_count"] == 8
assert projection["independent_verifier_registry"] == {
"status": "source_coverage_complete_runtime_receipts_pending",
"source_coverage_complete": True,
"runtime_closed": False,
"rollups": {
"domain_count": 8,
"mutating_domain_count": 6,
"read_only_domain_count": 2,
"source_blocker_count": 0,
"runtime_closed_domain_count": 0,
},
}
assert projection["claude_paid_call_allowed"] is True
assert projection["gemini_paid_call_allowed"] is True
assert projection["immediate_execution_queue"][0]["work_item_id"] == (
@@ -263,6 +280,10 @@ def test_endpoint_returns_full_work_ledger() -> None:
assert payload["program_id"] == "AIA-SRE-P0-20260715"
assert len(payload["work_items"]) == 18
assert payload["rollups"]["runtime_closed_items"] == 0
assert payload["independent_verifier_registry"][
"source_coverage_complete"
] is True
assert payload["independent_verifier_registry"]["runtime_closed"] is False
assert (
payload["agent99_host_operations_bridge"]["completion_callback"]
== "/api/v1/agents/agent99/completion-callback"

20
docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json
View File

@@ -724,7 +724,7 @@
"title": "Independent verifier registry 完整覆蓋",
"owner_lane": "PostExecutionVerifier",
"risk": "high",
"status": "in_progress",
"status": "source_implemented_runtime_pending",
"dependencies": [
"AIA-SRE-005",
"AIA-SRE-007",
@@ -733,18 +733,29 @@
"AIA-SRE-010"
],
"source_refs": [
"apps/api/src/services/independent_verifier_registry.py",
"apps/api/src/services/awooop_ansible_post_verifier.py",
"apps/api/src/services/kubernetes_rollout_verifier.py",
"apps/api/src/services/agent99_controlled_dispatch_ledger.py",
"apps/api/src/services/db_bounded_executor.py",
"apps/api/src/services/backup_restore_signal_automation.py",
"apps/api/src/services/post_execution_verifier.py"
],
"executor": "none",
"verifier": "domain-specific external readback",
"rollback": "failed verifier triggers same-domain retry or compensating action",
"exit_condition": "100 percent of mutating catalogs have independent production postconditions; host111 includes local plus host120/host121 origin verification and provider monitoring requires exact fresh series",
"confirmed_truth": [
"machine-readable registry covers all eight typed domains and keeps executor identity separate from verifier identity for all six mutating domains",
"all 17 Ansible catalogs have registered independent postconditions; every auto-apply catalog is check-mode capable and the sole exception is explicit break-glass",
"K3s, Host/Container, Agent99, DB, Backup/restore and unknown-asset lanes expose domain-specific verifier contracts with cross-domain fallback disabled",
"Host111 three-origin, GCP exact-series, gitea-native metrics and host99 Alertmanager relay readbacks are registered as runtime receipt requirements rather than source completion evidence"
],
"runtime_gaps": [
"source-level verifier definitions are not production verifier receipts",
"Host111 LaunchAgent, GCP provider targets, gitea-native metrics and the host99 Alertmanager pull/relay remain without independent runtime closure"
],
"next_action": "reconcile catalog IDs against postcondition registry, then attach same-run runtime receipts for Host111 origins, GCP exact targets, gitea-native metrics and the host99 Alertmanager pull/relay"
"next_action": "deploy one exact source revision through the authorized release lane, then attach same-run production receipts for K3s, Ansible, Agent99, DB and Backup readback plus Host111 origins, GCP exact targets, gitea-native metrics and host99 Alertmanager pull/relay; any missing receipt remains fail-closed"
},
{
"id": "AIA-SRE-015",
@@ -886,11 +897,10 @@
"P1": 2
},
"by_status": {
"source_implemented_runtime_pending": 14,
"in_progress": 1,
"source_implemented_runtime_pending": 15,
"planned": 3
},
"source_implemented_items": 14,
"source_implemented_items": 15,
"runtime_closed_items": 0,
"program_completion_percent": 0,
"asset_coverage_status": "partial",