553 lines
20 KiB
Python
553 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
# ruff: noqa: E402, I001
|
|
|
|
import os
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test")
|
|
|
|
from src.services.cpu_p99_controlled_recovery import ( # noqa: E402
|
|
build_cpu_resource_post_verifier_receipt,
|
|
build_cpu_p99_closure,
|
|
build_cpu_p99_controlled_candidate,
|
|
build_signoz_api_p99_post_verifier_receipt,
|
|
persist_cpu_p99_closure,
|
|
record_cpu_p99_closure,
|
|
)
|
|
from src.services.independent_verifier_registry import ( # noqa: E402
|
|
build_independent_verifier_registry,
|
|
)
|
|
from src.services.service_registry import ServiceRegistryClient # noqa: E402
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
RUN_ID = "6dc24cae-46ae-51a5-8278-7a9fbdafaa34"
|
|
TRACE_ID = "cpu-p99-trace-034"
|
|
WORK_ITEM_ID = "AIA-CPU-P99-034"
|
|
OBSERVED_AT = "2026-07-22T04:00:00+00:00"
|
|
|
|
|
|
class _MemoryCandidateStore:
|
|
def __init__(self) -> None:
|
|
self.records: dict[str, tuple[str, dict]] = {}
|
|
|
|
async def __call__(self, record: dict, project_id: str) -> dict:
|
|
assert project_id == "awoooi"
|
|
fingerprint = record["fingerprint"]
|
|
existing = self.records.get(fingerprint)
|
|
if existing is not None:
|
|
return {
|
|
"event_id": existing[0],
|
|
"created": False,
|
|
"record": existing[1],
|
|
}
|
|
event_id = str(UUID(int=len(self.records) + 1))
|
|
self.records[fingerprint] = (event_id, record)
|
|
return {"event_id": event_id, "created": True, "record": record}
|
|
|
|
|
|
def _common(receipt_id: str) -> dict[str, object]:
|
|
return {
|
|
"receipt_id": receipt_id,
|
|
"trace_id": TRACE_ID,
|
|
"run_id": RUN_ID,
|
|
"work_item_id": WORK_ITEM_ID,
|
|
"observed_at": OBSERVED_AT,
|
|
"freshness_verified": True,
|
|
"max_age_seconds": 900,
|
|
}
|
|
|
|
|
|
def _resource(
|
|
*,
|
|
canonical_asset_id: str = "service:awoooi-api",
|
|
asset_domain: str = "kubernetes_workload",
|
|
alertname: str = "HostHighCpuLoad",
|
|
namespace: str = "awoooi-prod",
|
|
labels: dict[str, str] | None = None,
|
|
) -> dict[str, object]:
|
|
return {
|
|
**_common("resource-receipt-034"),
|
|
"schema_version": "cpu_resource_evidence_v1",
|
|
"source": "prometheus_readonly_query",
|
|
"durable_readback_ack": True,
|
|
"alertname": alertname,
|
|
"alert_category": "host_resource",
|
|
"canonical_asset_id": canonical_asset_id,
|
|
"asset_domain": asset_domain,
|
|
"metric_name": "cpu_usage_percent",
|
|
"observed_value": 93.2,
|
|
"threshold": 80.0,
|
|
"namespace": namespace,
|
|
"labels": labels
|
|
or {
|
|
"namespace": "awoooi-prod",
|
|
"workload_kind": "deployment",
|
|
"workload_name": "awoooi-api",
|
|
},
|
|
}
|
|
|
|
|
|
def _latency() -> dict[str, object]:
|
|
return {
|
|
**_common("latency-receipt-034"),
|
|
"schema_version": "signoz_api_p99_evidence_v1",
|
|
"source": "signoz_readonly_query",
|
|
"durable_readback_ack": True,
|
|
"canonical_asset_id": "service:awoooi-api",
|
|
"service_name": "awoooi-api",
|
|
"namespace": "awoooi-prod",
|
|
"p99_seconds": 3.4,
|
|
"threshold_seconds": 2.0,
|
|
}
|
|
|
|
|
|
def _logs(root_asset: str = "service:awoooi-api") -> dict[str, object]:
|
|
return {
|
|
**_common("logs-receipt-034"),
|
|
"schema_version": "sanitized_log_correlation_receipt_v1",
|
|
"durable_readback_ack": True,
|
|
"sanitized": True,
|
|
"untrusted_evidence": True,
|
|
"raw_log_recorded": False,
|
|
"evidence_refs": ["incident-evidence:cpu-p99:034"],
|
|
"canonical_asset_ids": list(dict.fromkeys([root_asset, "service:awoooi-api"])),
|
|
"root_cause_asset_id": root_asset,
|
|
"rca_status": "correlated",
|
|
}
|
|
|
|
|
|
def _change(root_asset: str = "service:awoooi-api") -> dict[str, object]:
|
|
return {
|
|
**_common("change-receipt-034"),
|
|
"schema_version": "recent_change_correlation_receipt_v1",
|
|
"durable_readback_ack": True,
|
|
"correlation_status": "no_change_in_window",
|
|
"canonical_asset_ids": [root_asset],
|
|
"root_cause_asset_id": root_asset,
|
|
"change_refs": [],
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_exact_kubernetes_cpu_p99_candidate_is_durable_and_deduplicated() -> None:
|
|
store = _MemoryCandidateStore()
|
|
first = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="awoooi-api",
|
|
resource_evidence=_resource(),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs(),
|
|
recent_change_evidence=_change(),
|
|
persist_candidate=store,
|
|
)
|
|
duplicate = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="awoooi-api",
|
|
resource_evidence=_resource(),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs(),
|
|
recent_change_evidence=_change(),
|
|
persist_candidate=store,
|
|
)
|
|
|
|
assert first["accepted"] is True
|
|
assert first["created"] is True
|
|
assert first["status"] == "bounded_repair_candidate_recorded"
|
|
assert first["typed_domain"] == "kubernetes_workload"
|
|
assert first["executor"] == "kubernetes_controlled_executor"
|
|
assert first["verifier"] == "kubernetes_rollout_verifier"
|
|
assert first["repair_dispatch_allowed"] is True
|
|
assert first["runtime_mutation_performed"] is False
|
|
assert first["agent99_dispatch_performed"] is False
|
|
assert duplicate["created"] is False
|
|
assert duplicate["deduplicated"] is True
|
|
assert duplicate["candidate_event_id"] == first["candidate_event_id"]
|
|
assert len(store.records) == 1
|
|
|
|
record = next(iter(store.records.values()))[1]
|
|
assert record["repair_candidate"]["apply_authorized_by_this_service"] is False
|
|
assert record["policy"]["provider_call_allowed"] is False
|
|
assert record["policy"]["runtime_mutation_allowed"] is False
|
|
assert record["evidence"]["raw_logs_recorded"] is False
|
|
|
|
record["repair_candidate"]["executor"] = "Agent99"
|
|
mismatched_duplicate = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="awoooi-api",
|
|
resource_evidence=_resource(),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs(),
|
|
recent_change_evidence=_change(),
|
|
persist_candidate=store,
|
|
)
|
|
assert mismatched_duplicate["accepted"] is False
|
|
assert mismatched_duplicate["status"] == "durable_candidate_receipt_mismatch"
|
|
assert mismatched_duplicate["runtime_mutation_performed"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_host_pressure_uses_no_write_ansible_lane_not_kubernetes_or_agent99() -> (
|
|
None
|
|
):
|
|
root_asset = "host:192.168.0.110"
|
|
store = _MemoryCandidateStore()
|
|
result = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="host-pressure-controller",
|
|
resource_evidence=_resource(
|
|
canonical_asset_id=root_asset,
|
|
asset_domain="host_systemd",
|
|
alertname="HostLoadAverageSustainedHigh",
|
|
namespace="",
|
|
labels={"host": "110", "host_type": "bare_metal"},
|
|
),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs(root_asset),
|
|
recent_change_evidence=_change(root_asset),
|
|
persist_candidate=store,
|
|
)
|
|
|
|
assert result["status"] == "no_write_host_rca_candidate_recorded"
|
|
assert result["typed_domain"] == "host_systemd"
|
|
assert result["executor"] == "host_ansible_executor"
|
|
assert result["action_id"] == "ansible:110-host-pressure-readonly"
|
|
assert result["repair_dispatch_allowed"] is False
|
|
assert result["agent99_dispatch_performed"] is False
|
|
assert "kubectl" not in str(result).lower()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_exact_database_root_cause_beats_generic_host_route_and_stays_no_write() -> (
|
|
None
|
|
):
|
|
root_asset = "service:signoz-clickhouse"
|
|
store = _MemoryCandidateStore()
|
|
result = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="signoz-clickhouse",
|
|
resource_evidence=_resource(
|
|
canonical_asset_id=root_asset,
|
|
asset_domain="database",
|
|
namespace="",
|
|
labels={"host": "110", "host_type": "bare_metal"},
|
|
),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs(root_asset),
|
|
recent_change_evidence=_change(root_asset),
|
|
persist_candidate=store,
|
|
)
|
|
|
|
assert result["status"] == "no_write_database_rca_candidate_recorded"
|
|
assert result["typed_domain"] == "database"
|
|
assert result["executor"] == "db_bounded_executor"
|
|
assert result["verifier"] == "db_independent_verifier"
|
|
assert result["repair_dispatch_allowed"] is False
|
|
assert result["agent99_dispatch_performed"] is False
|
|
assert "restart" not in result["action_id"]
|
|
assert "kubectl" not in result["action_id"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unknown_asset_creates_only_durable_drift_receipt() -> None:
|
|
store = _MemoryCandidateStore()
|
|
result = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="mystery-cpu-source",
|
|
resource_evidence=_resource(
|
|
canonical_asset_id="unresolved",
|
|
asset_domain="unknown",
|
|
namespace="",
|
|
labels={"host_type": "bare_metal"},
|
|
),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs("unresolved"),
|
|
recent_change_evidence=_change("unresolved"),
|
|
persist_candidate=store,
|
|
)
|
|
|
|
assert result["accepted"] is True
|
|
assert result["status"] == "asset_identity_unresolved"
|
|
assert result["candidate_created"] is False
|
|
assert result["repair_dispatch_allowed"] is False
|
|
assert result["executor"] is None
|
|
assert result["verifier"] is None
|
|
assert result["runtime_mutation_performed"] is False
|
|
assert len(store.records) == 1
|
|
record = next(iter(store.records.values()))[1]
|
|
assert record["kind"] == "asset_identity_drift_work_item"
|
|
assert record["cross_domain_fallback_allowed"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_untrusted_or_misaligned_logs_create_no_candidate() -> None:
|
|
store = _MemoryCandidateStore()
|
|
logs = _logs()
|
|
logs["raw_log_recorded"] = True
|
|
|
|
result = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="awoooi-api",
|
|
resource_evidence=_resource(),
|
|
latency_evidence=_latency(),
|
|
log_evidence=logs,
|
|
recent_change_evidence=_change(),
|
|
persist_candidate=store,
|
|
)
|
|
|
|
assert result["accepted"] is False
|
|
assert result["status"] == "canonical_evidence_alignment_blocked"
|
|
assert "sanitized_log_correlation_unverified" in result["active_blockers"]
|
|
assert result["runtime_mutation_performed"] is False
|
|
assert store.records == {}
|
|
|
|
|
|
def _execution_receipt(candidate: dict) -> dict:
|
|
common = {
|
|
"trace_id": TRACE_ID,
|
|
"run_id": RUN_ID,
|
|
"work_item_id": WORK_ITEM_ID,
|
|
}
|
|
repair = candidate["repair_candidate"]
|
|
return {
|
|
**common,
|
|
"schema_version": "typed_bounded_execution_receipt_v1",
|
|
"receipt_id": "execution-receipt-034",
|
|
"status": "applied",
|
|
"durable_writeback_ack": True,
|
|
"applied_at": "2026-07-22T03:59:00+00:00",
|
|
"candidate_fingerprint": candidate["fingerprint"],
|
|
"canonical_asset_id": candidate["canonical_asset_id"],
|
|
"action_id": repair["action_id"],
|
|
"executor": repair["executor"],
|
|
"runtime_mutation_performed": True,
|
|
"cross_domain_fallback_performed": False,
|
|
"domain_verifier": repair["verifier"],
|
|
"domain_verifier_status": "verified",
|
|
"domain_verifier_independent": True,
|
|
"domain_verifier_receipt_id": "kubernetes-verifier-receipt-034",
|
|
"durable_domain_verifier_ack": True,
|
|
}
|
|
|
|
|
|
def _post_readbacks(candidate: dict, execution: dict) -> tuple[dict, dict]:
|
|
resource = {
|
|
**_common("prometheus-post-readback-034"),
|
|
"schema_version": "prometheus_cpu_resource_post_readback_v1",
|
|
"source": "prometheus_independent_query",
|
|
"trusted_server_readback": True,
|
|
"independent": True,
|
|
"durable_readback_ack": True,
|
|
"candidate_fingerprint": candidate["fingerprint"],
|
|
"execution_receipt_id": execution["receipt_id"],
|
|
"canonical_asset_id": candidate["canonical_asset_id"],
|
|
"metric_name": candidate["evidence"]["resource_metric"],
|
|
"observed_value": 55.0,
|
|
"runtime_mutation_performed": False,
|
|
"cross_domain_fallback_performed": False,
|
|
}
|
|
latency = {
|
|
**_common("signoz-post-readback-034"),
|
|
"schema_version": "signoz_api_p99_post_readback_v1",
|
|
"source": "signoz_independent_query",
|
|
"trusted_server_readback": True,
|
|
"independent": True,
|
|
"durable_readback_ack": True,
|
|
"candidate_fingerprint": candidate["fingerprint"],
|
|
"execution_receipt_id": execution["receipt_id"],
|
|
"canonical_asset_id": "service:awoooi-api",
|
|
"service_name": "awoooi-api",
|
|
"namespace": "awoooi-prod",
|
|
"p99_seconds": 1.2,
|
|
"runtime_mutation_performed": False,
|
|
"cross_domain_fallback_performed": False,
|
|
}
|
|
return resource, latency
|
|
|
|
|
|
def _closure_receipts(candidate: dict) -> tuple[dict, dict, dict]:
|
|
execution = _execution_receipt(candidate)
|
|
resource_readback, latency_readback = _post_readbacks(candidate, execution)
|
|
resource = build_cpu_resource_post_verifier_receipt(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
metric_readback=resource_readback,
|
|
)
|
|
latency = build_signoz_api_p99_post_verifier_receipt(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
signoz_readback=latency_readback,
|
|
)
|
|
return execution, resource, latency
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_closure_requires_same_run_resource_and_latency_verifiers() -> None:
|
|
store = _MemoryCandidateStore()
|
|
result = await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="awoooi-api",
|
|
resource_evidence=_resource(),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs(),
|
|
recent_change_evidence=_change(),
|
|
persist_candidate=store,
|
|
)
|
|
candidate = next(iter(store.records.values()))[1]
|
|
execution, resource, latency = _closure_receipts(candidate)
|
|
|
|
closed = build_cpu_p99_closure(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
resource_verifier_receipt=resource,
|
|
latency_verifier_receipt=latency,
|
|
)
|
|
assert closed["closed"] is True
|
|
assert closed["status"] == "verified_closed"
|
|
assert closed["resource_verifier_closed"] is True
|
|
assert closed["latency_verifier_closed"] is True
|
|
assert closed["same_run_receipts"] is True
|
|
assert resource["verifier"] == "cpu_resource_independent_verifier"
|
|
assert latency["verifier"] == "signoz_api_latency_independent_verifier"
|
|
|
|
_, stale_latency_readback = _post_readbacks(candidate, execution)
|
|
stale_latency_readback["freshness_verified"] = False
|
|
stale_latency = build_signoz_api_p99_post_verifier_receipt(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
signoz_readback=stale_latency_readback,
|
|
)
|
|
assert stale_latency["status"] == "blocked"
|
|
assert "post_readback_freshness_unverified" in stale_latency["active_blockers"]
|
|
|
|
execution["runtime_mutation_performed"] = False
|
|
no_apply = build_cpu_p99_closure(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
resource_verifier_receipt=resource,
|
|
latency_verifier_receipt=latency,
|
|
)
|
|
assert no_apply["closed"] is False
|
|
assert "typed_execution_receipt_unverified" in no_apply["active_blockers"]
|
|
execution["runtime_mutation_performed"] = True
|
|
|
|
latency["run_id"] = "91abf6c8-5001-4b17-a538-3205d6f85757"
|
|
blocked = build_cpu_p99_closure(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
resource_verifier_receipt=resource,
|
|
latency_verifier_receipt=latency,
|
|
)
|
|
assert blocked["closed"] is False
|
|
assert blocked["same_run_receipts"] is False
|
|
assert "same_run_run_id_mismatch" in blocked["active_blockers"]
|
|
assert result["runtime_mutation_performed"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_verified_closure_requires_durable_append_ack() -> None:
|
|
store = _MemoryCandidateStore()
|
|
await build_cpu_p99_controlled_candidate(
|
|
root_cause_target="awoooi-api",
|
|
resource_evidence=_resource(),
|
|
latency_evidence=_latency(),
|
|
log_evidence=_logs(),
|
|
recent_change_evidence=_change(),
|
|
persist_candidate=store,
|
|
)
|
|
candidate = next(iter(store.records.values()))[1]
|
|
execution, resource, latency = _closure_receipts(candidate)
|
|
persisted: list[dict] = []
|
|
|
|
unacknowledged = build_cpu_p99_closure(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
resource_verifier_receipt=resource,
|
|
latency_verifier_receipt=latency,
|
|
)
|
|
assert await persist_cpu_p99_closure(candidate, unacknowledged, "awoooi") is False
|
|
|
|
async def persist_closure(candidate_record, closure, project_id):
|
|
assert candidate_record is candidate
|
|
assert project_id == "awoooi"
|
|
persisted.append(dict(closure))
|
|
return True
|
|
|
|
pending = await record_cpu_p99_closure(
|
|
candidate=candidate,
|
|
execution_receipt={**execution, "runtime_mutation_performed": False},
|
|
resource_verifier_receipt=resource,
|
|
latency_verifier_receipt=latency,
|
|
persist_closure=persist_closure,
|
|
)
|
|
assert pending["closed"] is False
|
|
assert pending["durable_closure_ack"] is False
|
|
assert persisted == []
|
|
|
|
closure = await record_cpu_p99_closure(
|
|
candidate=candidate,
|
|
execution_receipt=execution,
|
|
resource_verifier_receipt=resource,
|
|
latency_verifier_receipt=latency,
|
|
persist_closure=persist_closure,
|
|
)
|
|
|
|
assert closure["closed"] is True
|
|
assert closure["durable_closure_ack"] is True
|
|
assert persisted[0]["durable_closure_ack"] is True
|
|
|
|
|
|
def test_alert_and_registry_contract_require_correlation_not_single_signal_apply() -> (
|
|
None
|
|
):
|
|
signoz = yaml.safe_load(
|
|
(ROOT / "ops/signoz/alerting/rules.yaml").read_text(encoding="utf-8")
|
|
)
|
|
p99 = next(
|
|
rule
|
|
for group in signoz["groups"]
|
|
for rule in group.get("rules", [])
|
|
if rule.get("alert") == "APIHighLatencyP99"
|
|
)
|
|
labels = p99["labels"]
|
|
assert labels["auto_repair"] == "false"
|
|
assert labels["canonical_asset_id"] == "service:awoooi-api"
|
|
assert labels["namespace"] == "awoooi-prod"
|
|
assert labels["workload_kind"] == "deployment"
|
|
assert labels["workload_name"] == "awoooi-api"
|
|
assert "不得因 P99 單一訊號直接 restart" in p99["annotations"]["runbook"]
|
|
|
|
for path in (
|
|
ROOT / "ops/monitoring/alerts-unified.yml",
|
|
ROOT / "ops/monitoring/alerts.yml",
|
|
):
|
|
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
host_cpu = next(
|
|
rule
|
|
for group in payload["groups"]
|
|
for rule in group.get("rules", [])
|
|
if rule.get("alert") == "HostHighCpuLoad"
|
|
)
|
|
assert host_cpu["labels"]["auto_repair"] == "false"
|
|
|
|
identity = ServiceRegistryClient(
|
|
ROOT / "ops/config/service-registry.yaml"
|
|
).resolve_identity("awoooi-api")
|
|
assert identity["resolution_status"] == "resolved"
|
|
assert identity["canonical_id"] == "service:awoooi-api"
|
|
assert identity["kubernetes_namespace"] == "awoooi-prod"
|
|
assert identity["kubernetes_kind"] == "deployment"
|
|
assert identity["kubernetes_name"] == "awoooi-api"
|
|
|
|
readbacks = {
|
|
row["readback_id"]: row
|
|
for row in build_independent_verifier_registry()["external_readbacks"]
|
|
}
|
|
assert readbacks["cpu_resource_postcondition"]["required_source"] == (
|
|
"prometheus_independent_query"
|
|
)
|
|
assert readbacks["api_p99_postcondition"]["required_source"] == (
|
|
"signoz_independent_query"
|
|
)
|
|
assert readbacks["cpu_resource_postcondition"]["runtime_status"] == (
|
|
"receipt_pending"
|
|
)
|
|
assert readbacks["api_p99_postcondition"]["runtime_status"] == ("receipt_pending")
|