from __future__ import annotations # ruff: noqa: E402, I001 import os from pathlib import Path from uuid import UUID import pytest import yaml os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") from src.services.cpu_p99_controlled_recovery import ( # noqa: E402 build_cpu_resource_post_verifier_receipt, build_cpu_p99_closure, build_cpu_p99_controlled_candidate, build_signoz_api_p99_post_verifier_receipt, persist_cpu_p99_closure, record_cpu_p99_closure, ) from src.services.independent_verifier_registry import ( # noqa: E402 build_independent_verifier_registry, ) from src.services.service_registry import ServiceRegistryClient # noqa: E402 ROOT = Path(__file__).resolve().parents[3] RUN_ID = "6dc24cae-46ae-51a5-8278-7a9fbdafaa34" TRACE_ID = "cpu-p99-trace-034" WORK_ITEM_ID = "AIA-CPU-P99-034" OBSERVED_AT = "2026-07-22T04:00:00+00:00" class _MemoryCandidateStore: def __init__(self) -> None: self.records: dict[str, tuple[str, dict]] = {} async def __call__(self, record: dict, project_id: str) -> dict: assert project_id == "awoooi" fingerprint = record["fingerprint"] existing = self.records.get(fingerprint) if existing is not None: return { "event_id": existing[0], "created": False, "record": existing[1], } event_id = str(UUID(int=len(self.records) + 1)) self.records[fingerprint] = (event_id, record) return {"event_id": event_id, "created": True, "record": record} def _common(receipt_id: str) -> dict[str, object]: return { "receipt_id": receipt_id, "trace_id": TRACE_ID, "run_id": RUN_ID, "work_item_id": WORK_ITEM_ID, "observed_at": OBSERVED_AT, "freshness_verified": True, "max_age_seconds": 900, } def _resource( *, canonical_asset_id: str = "service:awoooi-api", asset_domain: str = "kubernetes_workload", alertname: str = "HostHighCpuLoad", namespace: str = "awoooi-prod", labels: dict[str, str] | None = None, ) -> dict[str, object]: return { **_common("resource-receipt-034"), "schema_version": "cpu_resource_evidence_v1", "source": "prometheus_readonly_query", "durable_readback_ack": True, "alertname": alertname, "alert_category": "host_resource", "canonical_asset_id": canonical_asset_id, "asset_domain": asset_domain, "metric_name": "cpu_usage_percent", "observed_value": 93.2, "threshold": 80.0, "namespace": namespace, "labels": labels or { "namespace": "awoooi-prod", "workload_kind": "deployment", "workload_name": "awoooi-api", }, } def _latency() -> dict[str, object]: return { **_common("latency-receipt-034"), "schema_version": "signoz_api_p99_evidence_v1", "source": "signoz_readonly_query", "durable_readback_ack": True, "canonical_asset_id": "service:awoooi-api", "service_name": "awoooi-api", "namespace": "awoooi-prod", "p99_seconds": 3.4, "threshold_seconds": 2.0, } def _logs(root_asset: str = "service:awoooi-api") -> dict[str, object]: return { **_common("logs-receipt-034"), "schema_version": "sanitized_log_correlation_receipt_v1", "durable_readback_ack": True, "sanitized": True, "untrusted_evidence": True, "raw_log_recorded": False, "evidence_refs": ["incident-evidence:cpu-p99:034"], "canonical_asset_ids": list(dict.fromkeys([root_asset, "service:awoooi-api"])), "root_cause_asset_id": root_asset, "rca_status": "correlated", } def _change(root_asset: str = "service:awoooi-api") -> dict[str, object]: return { **_common("change-receipt-034"), "schema_version": "recent_change_correlation_receipt_v1", "durable_readback_ack": True, "correlation_status": "no_change_in_window", "canonical_asset_ids": [root_asset], "root_cause_asset_id": root_asset, "change_refs": [], } @pytest.mark.asyncio async def test_exact_kubernetes_cpu_p99_candidate_is_durable_and_deduplicated() -> None: store = _MemoryCandidateStore() first = await build_cpu_p99_controlled_candidate( root_cause_target="awoooi-api", resource_evidence=_resource(), latency_evidence=_latency(), log_evidence=_logs(), recent_change_evidence=_change(), persist_candidate=store, ) duplicate = await build_cpu_p99_controlled_candidate( root_cause_target="awoooi-api", resource_evidence=_resource(), latency_evidence=_latency(), log_evidence=_logs(), recent_change_evidence=_change(), persist_candidate=store, ) assert first["accepted"] is True assert first["created"] is True assert first["status"] == "bounded_repair_candidate_recorded" assert first["typed_domain"] == "kubernetes_workload" assert first["executor"] == "kubernetes_controlled_executor" assert first["verifier"] == "kubernetes_rollout_verifier" assert first["repair_dispatch_allowed"] is True assert first["runtime_mutation_performed"] is False assert first["agent99_dispatch_performed"] is False assert duplicate["created"] is False assert duplicate["deduplicated"] is True assert duplicate["candidate_event_id"] == first["candidate_event_id"] assert len(store.records) == 1 record = next(iter(store.records.values()))[1] assert record["repair_candidate"]["apply_authorized_by_this_service"] is False assert record["policy"]["provider_call_allowed"] is False assert record["policy"]["runtime_mutation_allowed"] is False assert record["evidence"]["raw_logs_recorded"] is False record["repair_candidate"]["executor"] = "Agent99" mismatched_duplicate = await build_cpu_p99_controlled_candidate( root_cause_target="awoooi-api", resource_evidence=_resource(), latency_evidence=_latency(), log_evidence=_logs(), recent_change_evidence=_change(), persist_candidate=store, ) assert mismatched_duplicate["accepted"] is False assert mismatched_duplicate["status"] == "durable_candidate_receipt_mismatch" assert mismatched_duplicate["runtime_mutation_performed"] is False @pytest.mark.asyncio async def test_host_pressure_uses_no_write_ansible_lane_not_kubernetes_or_agent99() -> ( None ): root_asset = "host:192.168.0.110" store = _MemoryCandidateStore() result = await build_cpu_p99_controlled_candidate( root_cause_target="host-pressure-controller", resource_evidence=_resource( canonical_asset_id=root_asset, asset_domain="host_systemd", alertname="HostLoadAverageSustainedHigh", namespace="", labels={"host": "110", "host_type": "bare_metal"}, ), latency_evidence=_latency(), log_evidence=_logs(root_asset), recent_change_evidence=_change(root_asset), persist_candidate=store, ) assert result["status"] == "no_write_host_rca_candidate_recorded" assert result["typed_domain"] == "host_systemd" assert result["executor"] == "host_ansible_executor" assert result["action_id"] == "ansible:110-host-pressure-readonly" assert result["repair_dispatch_allowed"] is False assert result["agent99_dispatch_performed"] is False assert "kubectl" not in str(result).lower() @pytest.mark.asyncio async def test_exact_database_root_cause_beats_generic_host_route_and_stays_no_write() -> ( None ): root_asset = "service:signoz-clickhouse" store = _MemoryCandidateStore() result = await build_cpu_p99_controlled_candidate( root_cause_target="signoz-clickhouse", resource_evidence=_resource( canonical_asset_id=root_asset, asset_domain="database", namespace="", labels={"host": "110", "host_type": "bare_metal"}, ), latency_evidence=_latency(), log_evidence=_logs(root_asset), recent_change_evidence=_change(root_asset), persist_candidate=store, ) assert result["status"] == "no_write_database_rca_candidate_recorded" assert result["typed_domain"] == "database" assert result["executor"] == "db_bounded_executor" assert result["verifier"] == "db_independent_verifier" assert result["repair_dispatch_allowed"] is False assert result["agent99_dispatch_performed"] is False assert "restart" not in result["action_id"] assert "kubectl" not in result["action_id"] @pytest.mark.asyncio async def test_unknown_asset_creates_only_durable_drift_receipt() -> None: store = _MemoryCandidateStore() result = await build_cpu_p99_controlled_candidate( root_cause_target="mystery-cpu-source", resource_evidence=_resource( canonical_asset_id="unresolved", asset_domain="unknown", namespace="", labels={"host_type": "bare_metal"}, ), latency_evidence=_latency(), log_evidence=_logs("unresolved"), recent_change_evidence=_change("unresolved"), persist_candidate=store, ) assert result["accepted"] is True assert result["status"] == "asset_identity_unresolved" assert result["candidate_created"] is False assert result["repair_dispatch_allowed"] is False assert result["executor"] is None assert result["verifier"] is None assert result["runtime_mutation_performed"] is False assert len(store.records) == 1 record = next(iter(store.records.values()))[1] assert record["kind"] == "asset_identity_drift_work_item" assert record["cross_domain_fallback_allowed"] is False @pytest.mark.asyncio async def test_untrusted_or_misaligned_logs_create_no_candidate() -> None: store = _MemoryCandidateStore() logs = _logs() logs["raw_log_recorded"] = True result = await build_cpu_p99_controlled_candidate( root_cause_target="awoooi-api", resource_evidence=_resource(), latency_evidence=_latency(), log_evidence=logs, recent_change_evidence=_change(), persist_candidate=store, ) assert result["accepted"] is False assert result["status"] == "canonical_evidence_alignment_blocked" assert "sanitized_log_correlation_unverified" in result["active_blockers"] assert result["runtime_mutation_performed"] is False assert store.records == {} def _execution_receipt(candidate: dict) -> dict: common = { "trace_id": TRACE_ID, "run_id": RUN_ID, "work_item_id": WORK_ITEM_ID, } repair = candidate["repair_candidate"] return { **common, "schema_version": "typed_bounded_execution_receipt_v1", "receipt_id": "execution-receipt-034", "status": "applied", "durable_writeback_ack": True, "applied_at": "2026-07-22T03:59:00+00:00", "candidate_fingerprint": candidate["fingerprint"], "canonical_asset_id": candidate["canonical_asset_id"], "action_id": repair["action_id"], "executor": repair["executor"], "runtime_mutation_performed": True, "cross_domain_fallback_performed": False, "domain_verifier": repair["verifier"], "domain_verifier_status": "verified", "domain_verifier_independent": True, "domain_verifier_receipt_id": "kubernetes-verifier-receipt-034", "durable_domain_verifier_ack": True, } def _post_readbacks(candidate: dict, execution: dict) -> tuple[dict, dict]: resource = { **_common("prometheus-post-readback-034"), "schema_version": "prometheus_cpu_resource_post_readback_v1", "source": "prometheus_independent_query", "trusted_server_readback": True, "independent": True, "durable_readback_ack": True, "candidate_fingerprint": candidate["fingerprint"], "execution_receipt_id": execution["receipt_id"], "canonical_asset_id": candidate["canonical_asset_id"], "metric_name": candidate["evidence"]["resource_metric"], "observed_value": 55.0, "runtime_mutation_performed": False, "cross_domain_fallback_performed": False, } latency = { **_common("signoz-post-readback-034"), "schema_version": "signoz_api_p99_post_readback_v1", "source": "signoz_independent_query", "trusted_server_readback": True, "independent": True, "durable_readback_ack": True, "candidate_fingerprint": candidate["fingerprint"], "execution_receipt_id": execution["receipt_id"], "canonical_asset_id": "service:awoooi-api", "service_name": "awoooi-api", "namespace": "awoooi-prod", "p99_seconds": 1.2, "runtime_mutation_performed": False, "cross_domain_fallback_performed": False, } return resource, latency def _closure_receipts(candidate: dict) -> tuple[dict, dict, dict]: execution = _execution_receipt(candidate) resource_readback, latency_readback = _post_readbacks(candidate, execution) resource = build_cpu_resource_post_verifier_receipt( candidate=candidate, execution_receipt=execution, metric_readback=resource_readback, ) latency = build_signoz_api_p99_post_verifier_receipt( candidate=candidate, execution_receipt=execution, signoz_readback=latency_readback, ) return execution, resource, latency @pytest.mark.asyncio async def test_closure_requires_same_run_resource_and_latency_verifiers() -> None: store = _MemoryCandidateStore() result = await build_cpu_p99_controlled_candidate( root_cause_target="awoooi-api", resource_evidence=_resource(), latency_evidence=_latency(), log_evidence=_logs(), recent_change_evidence=_change(), persist_candidate=store, ) candidate = next(iter(store.records.values()))[1] execution, resource, latency = _closure_receipts(candidate) closed = build_cpu_p99_closure( candidate=candidate, execution_receipt=execution, resource_verifier_receipt=resource, latency_verifier_receipt=latency, ) assert closed["closed"] is True assert closed["status"] == "verified_closed" assert closed["resource_verifier_closed"] is True assert closed["latency_verifier_closed"] is True assert closed["same_run_receipts"] is True assert resource["verifier"] == "cpu_resource_independent_verifier" assert latency["verifier"] == "signoz_api_latency_independent_verifier" _, stale_latency_readback = _post_readbacks(candidate, execution) stale_latency_readback["freshness_verified"] = False stale_latency = build_signoz_api_p99_post_verifier_receipt( candidate=candidate, execution_receipt=execution, signoz_readback=stale_latency_readback, ) assert stale_latency["status"] == "blocked" assert "post_readback_freshness_unverified" in stale_latency["active_blockers"] execution["runtime_mutation_performed"] = False no_apply = build_cpu_p99_closure( candidate=candidate, execution_receipt=execution, resource_verifier_receipt=resource, latency_verifier_receipt=latency, ) assert no_apply["closed"] is False assert "typed_execution_receipt_unverified" in no_apply["active_blockers"] execution["runtime_mutation_performed"] = True latency["run_id"] = "91abf6c8-5001-4b17-a538-3205d6f85757" blocked = build_cpu_p99_closure( candidate=candidate, execution_receipt=execution, resource_verifier_receipt=resource, latency_verifier_receipt=latency, ) assert blocked["closed"] is False assert blocked["same_run_receipts"] is False assert "same_run_run_id_mismatch" in blocked["active_blockers"] assert result["runtime_mutation_performed"] is False @pytest.mark.asyncio async def test_verified_closure_requires_durable_append_ack() -> None: store = _MemoryCandidateStore() await build_cpu_p99_controlled_candidate( root_cause_target="awoooi-api", resource_evidence=_resource(), latency_evidence=_latency(), log_evidence=_logs(), recent_change_evidence=_change(), persist_candidate=store, ) candidate = next(iter(store.records.values()))[1] execution, resource, latency = _closure_receipts(candidate) persisted: list[dict] = [] unacknowledged = build_cpu_p99_closure( candidate=candidate, execution_receipt=execution, resource_verifier_receipt=resource, latency_verifier_receipt=latency, ) assert await persist_cpu_p99_closure(candidate, unacknowledged, "awoooi") is False async def persist_closure(candidate_record, closure, project_id): assert candidate_record is candidate assert project_id == "awoooi" persisted.append(dict(closure)) return True pending = await record_cpu_p99_closure( candidate=candidate, execution_receipt={**execution, "runtime_mutation_performed": False}, resource_verifier_receipt=resource, latency_verifier_receipt=latency, persist_closure=persist_closure, ) assert pending["closed"] is False assert pending["durable_closure_ack"] is False assert persisted == [] closure = await record_cpu_p99_closure( candidate=candidate, execution_receipt=execution, resource_verifier_receipt=resource, latency_verifier_receipt=latency, persist_closure=persist_closure, ) assert closure["closed"] is True assert closure["durable_closure_ack"] is True assert persisted[0]["durable_closure_ack"] is True def test_alert_and_registry_contract_require_correlation_not_single_signal_apply() -> ( None ): signoz = yaml.safe_load( (ROOT / "ops/signoz/alerting/rules.yaml").read_text(encoding="utf-8") ) p99 = next( rule for group in signoz["groups"] for rule in group.get("rules", []) if rule.get("alert") == "APIHighLatencyP99" ) labels = p99["labels"] assert labels["auto_repair"] == "false" assert labels["canonical_asset_id"] == "service:awoooi-api" assert labels["namespace"] == "awoooi-prod" assert labels["workload_kind"] == "deployment" assert labels["workload_name"] == "awoooi-api" assert "不得因 P99 單一訊號直接 restart" in p99["annotations"]["runbook"] for path in ( ROOT / "ops/monitoring/alerts-unified.yml", ROOT / "ops/monitoring/alerts.yml", ): payload = yaml.safe_load(path.read_text(encoding="utf-8")) host_cpu = next( rule for group in payload["groups"] for rule in group.get("rules", []) if rule.get("alert") == "HostHighCpuLoad" ) assert host_cpu["labels"]["auto_repair"] == "false" identity = ServiceRegistryClient( ROOT / "ops/config/service-registry.yaml" ).resolve_identity("awoooi-api") assert identity["resolution_status"] == "resolved" assert identity["canonical_id"] == "service:awoooi-api" assert identity["kubernetes_namespace"] == "awoooi-prod" assert identity["kubernetes_kind"] == "deployment" assert identity["kubernetes_name"] == "awoooi-api" readbacks = { row["readback_id"]: row for row in build_independent_verifier_registry()["external_readbacks"] } assert readbacks["cpu_resource_postcondition"]["required_source"] == ( "prometheus_independent_query" ) assert readbacks["api_p99_postcondition"]["required_source"] == ( "signoz_independent_query" ) assert readbacks["cpu_resource_postcondition"]["runtime_status"] == ( "receipt_pending" ) assert readbacks["api_p99_postcondition"]["runtime_status"] == ("receipt_pending")