feat(aiops): add typed CPU P99 recovery closure
This commit is contained in:
Your Name
@@ -67,6 +67,10 @@ _NODE_EXPORTER_ALERTS = {
|
||||
"nodeexporterscrapedown",
|
||||
"nodeexporterunhealthy",
|
||||
}
|
||||
_HOST_CPU_PRESSURE_ALERTS = {
|
||||
"hosthighcpuload",
|
||||
"hostloadaveragesustainedhigh",
|
||||
}
|
||||
_ALERTMANAGER_DELIVERY_ALERTS = {"alertchainbrokenalertmanager"}
|
||||
_PROVIDER_FRESHNESS_ALERTS = {
|
||||
"providerfreshnesssignal",
|
||||
@@ -704,6 +708,18 @@ def resolve_typed_alert_target(
|
||||
host110_pressure_alert = compact_alert.startswith(
|
||||
"host110sustainedmoderatepressure"
|
||||
)
|
||||
host_cpu_pressure_alert = (
|
||||
compact_alert in _HOST_CPU_PRESSURE_ALERTS or host110_pressure_alert
|
||||
)
|
||||
pressure_target_identity = (
|
||||
registry.resolve_identity(target_resource)
|
||||
if host_cpu_pressure_alert and target_resource
|
||||
else {}
|
||||
)
|
||||
pressure_target_is_exact_service = bool(
|
||||
pressure_target_identity.get("resolution_status") == "resolved"
|
||||
and pressure_target_identity.get("asset_domain") != "unknown"
|
||||
)
|
||||
allowed_catalog_ids: list[str] = []
|
||||
if compact_alert in _DISK_ALERTS and host_scope is not None:
|
||||
if host_scope[0] in {"host_110", "host_188"}:
|
||||
@@ -714,9 +730,10 @@ def resolve_typed_alert_target(
|
||||
if host_scope[0] == "host_110":
|
||||
allowed_catalog_ids = ["ansible:110-devops"]
|
||||
elif (
|
||||
host110_pressure_alert
|
||||
host_cpu_pressure_alert
|
||||
and host_scope is not None
|
||||
and host_scope[0] == "host_110"
|
||||
and not pressure_target_is_exact_service
|
||||
):
|
||||
allowed_catalog_ids = ["ansible:110-host-pressure-readonly"]
|
||||
elif (
|
||||
@@ -729,7 +746,7 @@ def resolve_typed_alert_target(
|
||||
if host_scope is not None and (
|
||||
compact_alert in _DISK_ALERTS
|
||||
or compact_alert in _NODE_EXPORTER_ALERTS
|
||||
or host110_pressure_alert
|
||||
or (host_cpu_pressure_alert and not pressure_target_is_exact_service)
|
||||
or "wazuh" in text
|
||||
):
|
||||
return _typed_route(
|
||||
@@ -744,6 +761,7 @@ def resolve_typed_alert_target(
|
||||
risk_class="medium",
|
||||
allowed_catalog_ids=allowed_catalog_ids,
|
||||
allowed_inventory_hosts=[host_scope[0]],
|
||||
controlled_apply_allowed=(False if host_cpu_pressure_alert else None),
|
||||
)
|
||||
|
||||
identity = registry.resolve_identity(target_resource)
|
||||
|
||||
1423
apps/api/src/services/cpu_p99_controlled_recovery.py
Normal file
1423
apps/api/src/services/cpu_p99_controlled_recovery.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -111,6 +111,20 @@ _EXTERNAL_READBACKS: tuple[dict[str, Any], ...] = (
|
||||
"verifier": "agent99_alertmanager_pull_relay_verifier",
|
||||
"runtime_status": "receipt_pending",
|
||||
},
|
||||
{
|
||||
"readback_id": "cpu_resource_postcondition",
|
||||
"verifier": "cpu_resource_independent_verifier",
|
||||
"required_source": "prometheus_independent_query",
|
||||
"source_ref": "apps/api/src/services/cpu_p99_controlled_recovery.py",
|
||||
"runtime_status": "receipt_pending",
|
||||
},
|
||||
{
|
||||
"readback_id": "api_p99_postcondition",
|
||||
"verifier": "signoz_api_latency_independent_verifier",
|
||||
"required_source": "signoz_independent_query",
|
||||
"source_ref": "apps/api/src/services/cpu_p99_controlled_recovery.py",
|
||||
"runtime_status": "receipt_pending",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
||||
552
apps/api/tests/test_cpu_p99_controlled_recovery.py
Normal file
552
apps/api/tests/test_cpu_p99_controlled_recovery.py
Normal file
@@ -0,0 +1,552 @@
|
||||
from __future__ import annotations
|
||||
|
||||
# ruff: noqa: E402, I001
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test")
|
||||
|
||||
from src.services.cpu_p99_controlled_recovery import ( # noqa: E402
|
||||
build_cpu_resource_post_verifier_receipt,
|
||||
build_cpu_p99_closure,
|
||||
build_cpu_p99_controlled_candidate,
|
||||
build_signoz_api_p99_post_verifier_receipt,
|
||||
persist_cpu_p99_closure,
|
||||
record_cpu_p99_closure,
|
||||
)
|
||||
from src.services.independent_verifier_registry import ( # noqa: E402
|
||||
build_independent_verifier_registry,
|
||||
)
|
||||
from src.services.service_registry import ServiceRegistryClient # noqa: E402
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
RUN_ID = "6dc24cae-46ae-51a5-8278-7a9fbdafaa34"
|
||||
TRACE_ID = "cpu-p99-trace-034"
|
||||
WORK_ITEM_ID = "AIA-CPU-P99-034"
|
||||
OBSERVED_AT = "2026-07-22T04:00:00+00:00"
|
||||
|
||||
|
||||
class _MemoryCandidateStore:
|
||||
def __init__(self) -> None:
|
||||
self.records: dict[str, tuple[str, dict]] = {}
|
||||
|
||||
async def __call__(self, record: dict, project_id: str) -> dict:
|
||||
assert project_id == "awoooi"
|
||||
fingerprint = record["fingerprint"]
|
||||
existing = self.records.get(fingerprint)
|
||||
if existing is not None:
|
||||
return {
|
||||
"event_id": existing[0],
|
||||
"created": False,
|
||||
"record": existing[1],
|
||||
}
|
||||
event_id = str(UUID(int=len(self.records) + 1))
|
||||
self.records[fingerprint] = (event_id, record)
|
||||
return {"event_id": event_id, "created": True, "record": record}
|
||||
|
||||
|
||||
def _common(receipt_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"receipt_id": receipt_id,
|
||||
"trace_id": TRACE_ID,
|
||||
"run_id": RUN_ID,
|
||||
"work_item_id": WORK_ITEM_ID,
|
||||
"observed_at": OBSERVED_AT,
|
||||
"freshness_verified": True,
|
||||
"max_age_seconds": 900,
|
||||
}
|
||||
|
||||
|
||||
def _resource(
|
||||
*,
|
||||
canonical_asset_id: str = "service:awoooi-api",
|
||||
asset_domain: str = "kubernetes_workload",
|
||||
alertname: str = "HostHighCpuLoad",
|
||||
namespace: str = "awoooi-prod",
|
||||
labels: dict[str, str] | None = None,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
**_common("resource-receipt-034"),
|
||||
"schema_version": "cpu_resource_evidence_v1",
|
||||
"source": "prometheus_readonly_query",
|
||||
"durable_readback_ack": True,
|
||||
"alertname": alertname,
|
||||
"alert_category": "host_resource",
|
||||
"canonical_asset_id": canonical_asset_id,
|
||||
"asset_domain": asset_domain,
|
||||
"metric_name": "cpu_usage_percent",
|
||||
"observed_value": 93.2,
|
||||
"threshold": 80.0,
|
||||
"namespace": namespace,
|
||||
"labels": labels
|
||||
or {
|
||||
"namespace": "awoooi-prod",
|
||||
"workload_kind": "deployment",
|
||||
"workload_name": "awoooi-api",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _latency() -> dict[str, object]:
|
||||
return {
|
||||
**_common("latency-receipt-034"),
|
||||
"schema_version": "signoz_api_p99_evidence_v1",
|
||||
"source": "signoz_readonly_query",
|
||||
"durable_readback_ack": True,
|
||||
"canonical_asset_id": "service:awoooi-api",
|
||||
"service_name": "awoooi-api",
|
||||
"namespace": "awoooi-prod",
|
||||
"p99_seconds": 3.4,
|
||||
"threshold_seconds": 2.0,
|
||||
}
|
||||
|
||||
|
||||
def _logs(root_asset: str = "service:awoooi-api") -> dict[str, object]:
|
||||
return {
|
||||
**_common("logs-receipt-034"),
|
||||
"schema_version": "sanitized_log_correlation_receipt_v1",
|
||||
"durable_readback_ack": True,
|
||||
"sanitized": True,
|
||||
"untrusted_evidence": True,
|
||||
"raw_log_recorded": False,
|
||||
"evidence_refs": ["incident-evidence:cpu-p99:034"],
|
||||
"canonical_asset_ids": list(dict.fromkeys([root_asset, "service:awoooi-api"])),
|
||||
"root_cause_asset_id": root_asset,
|
||||
"rca_status": "correlated",
|
||||
}
|
||||
|
||||
|
||||
def _change(root_asset: str = "service:awoooi-api") -> dict[str, object]:
|
||||
return {
|
||||
**_common("change-receipt-034"),
|
||||
"schema_version": "recent_change_correlation_receipt_v1",
|
||||
"durable_readback_ack": True,
|
||||
"correlation_status": "no_change_in_window",
|
||||
"canonical_asset_ids": [root_asset],
|
||||
"root_cause_asset_id": root_asset,
|
||||
"change_refs": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exact_kubernetes_cpu_p99_candidate_is_durable_and_deduplicated() -> None:
|
||||
store = _MemoryCandidateStore()
|
||||
first = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="awoooi-api",
|
||||
resource_evidence=_resource(),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs(),
|
||||
recent_change_evidence=_change(),
|
||||
persist_candidate=store,
|
||||
)
|
||||
duplicate = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="awoooi-api",
|
||||
resource_evidence=_resource(),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs(),
|
||||
recent_change_evidence=_change(),
|
||||
persist_candidate=store,
|
||||
)
|
||||
|
||||
assert first["accepted"] is True
|
||||
assert first["created"] is True
|
||||
assert first["status"] == "bounded_repair_candidate_recorded"
|
||||
assert first["typed_domain"] == "kubernetes_workload"
|
||||
assert first["executor"] == "kubernetes_controlled_executor"
|
||||
assert first["verifier"] == "kubernetes_rollout_verifier"
|
||||
assert first["repair_dispatch_allowed"] is True
|
||||
assert first["runtime_mutation_performed"] is False
|
||||
assert first["agent99_dispatch_performed"] is False
|
||||
assert duplicate["created"] is False
|
||||
assert duplicate["deduplicated"] is True
|
||||
assert duplicate["candidate_event_id"] == first["candidate_event_id"]
|
||||
assert len(store.records) == 1
|
||||
|
||||
record = next(iter(store.records.values()))[1]
|
||||
assert record["repair_candidate"]["apply_authorized_by_this_service"] is False
|
||||
assert record["policy"]["provider_call_allowed"] is False
|
||||
assert record["policy"]["runtime_mutation_allowed"] is False
|
||||
assert record["evidence"]["raw_logs_recorded"] is False
|
||||
|
||||
record["repair_candidate"]["executor"] = "Agent99"
|
||||
mismatched_duplicate = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="awoooi-api",
|
||||
resource_evidence=_resource(),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs(),
|
||||
recent_change_evidence=_change(),
|
||||
persist_candidate=store,
|
||||
)
|
||||
assert mismatched_duplicate["accepted"] is False
|
||||
assert mismatched_duplicate["status"] == "durable_candidate_receipt_mismatch"
|
||||
assert mismatched_duplicate["runtime_mutation_performed"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_host_pressure_uses_no_write_ansible_lane_not_kubernetes_or_agent99() -> (
|
||||
None
|
||||
):
|
||||
root_asset = "host:192.168.0.110"
|
||||
store = _MemoryCandidateStore()
|
||||
result = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="host-pressure-controller",
|
||||
resource_evidence=_resource(
|
||||
canonical_asset_id=root_asset,
|
||||
asset_domain="host_systemd",
|
||||
alertname="HostLoadAverageSustainedHigh",
|
||||
namespace="",
|
||||
labels={"host": "110", "host_type": "bare_metal"},
|
||||
),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs(root_asset),
|
||||
recent_change_evidence=_change(root_asset),
|
||||
persist_candidate=store,
|
||||
)
|
||||
|
||||
assert result["status"] == "no_write_host_rca_candidate_recorded"
|
||||
assert result["typed_domain"] == "host_systemd"
|
||||
assert result["executor"] == "host_ansible_executor"
|
||||
assert result["action_id"] == "ansible:110-host-pressure-readonly"
|
||||
assert result["repair_dispatch_allowed"] is False
|
||||
assert result["agent99_dispatch_performed"] is False
|
||||
assert "kubectl" not in str(result).lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exact_database_root_cause_beats_generic_host_route_and_stays_no_write() -> (
|
||||
None
|
||||
):
|
||||
root_asset = "service:signoz-clickhouse"
|
||||
store = _MemoryCandidateStore()
|
||||
result = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="signoz-clickhouse",
|
||||
resource_evidence=_resource(
|
||||
canonical_asset_id=root_asset,
|
||||
asset_domain="database",
|
||||
namespace="",
|
||||
labels={"host": "110", "host_type": "bare_metal"},
|
||||
),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs(root_asset),
|
||||
recent_change_evidence=_change(root_asset),
|
||||
persist_candidate=store,
|
||||
)
|
||||
|
||||
assert result["status"] == "no_write_database_rca_candidate_recorded"
|
||||
assert result["typed_domain"] == "database"
|
||||
assert result["executor"] == "db_bounded_executor"
|
||||
assert result["verifier"] == "db_independent_verifier"
|
||||
assert result["repair_dispatch_allowed"] is False
|
||||
assert result["agent99_dispatch_performed"] is False
|
||||
assert "restart" not in result["action_id"]
|
||||
assert "kubectl" not in result["action_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_asset_creates_only_durable_drift_receipt() -> None:
|
||||
store = _MemoryCandidateStore()
|
||||
result = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="mystery-cpu-source",
|
||||
resource_evidence=_resource(
|
||||
canonical_asset_id="unresolved",
|
||||
asset_domain="unknown",
|
||||
namespace="",
|
||||
labels={"host_type": "bare_metal"},
|
||||
),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs("unresolved"),
|
||||
recent_change_evidence=_change("unresolved"),
|
||||
persist_candidate=store,
|
||||
)
|
||||
|
||||
assert result["accepted"] is True
|
||||
assert result["status"] == "asset_identity_unresolved"
|
||||
assert result["candidate_created"] is False
|
||||
assert result["repair_dispatch_allowed"] is False
|
||||
assert result["executor"] is None
|
||||
assert result["verifier"] is None
|
||||
assert result["runtime_mutation_performed"] is False
|
||||
assert len(store.records) == 1
|
||||
record = next(iter(store.records.values()))[1]
|
||||
assert record["kind"] == "asset_identity_drift_work_item"
|
||||
assert record["cross_domain_fallback_allowed"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untrusted_or_misaligned_logs_create_no_candidate() -> None:
|
||||
store = _MemoryCandidateStore()
|
||||
logs = _logs()
|
||||
logs["raw_log_recorded"] = True
|
||||
|
||||
result = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="awoooi-api",
|
||||
resource_evidence=_resource(),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=logs,
|
||||
recent_change_evidence=_change(),
|
||||
persist_candidate=store,
|
||||
)
|
||||
|
||||
assert result["accepted"] is False
|
||||
assert result["status"] == "canonical_evidence_alignment_blocked"
|
||||
assert "sanitized_log_correlation_unverified" in result["active_blockers"]
|
||||
assert result["runtime_mutation_performed"] is False
|
||||
assert store.records == {}
|
||||
|
||||
|
||||
def _execution_receipt(candidate: dict) -> dict:
|
||||
common = {
|
||||
"trace_id": TRACE_ID,
|
||||
"run_id": RUN_ID,
|
||||
"work_item_id": WORK_ITEM_ID,
|
||||
}
|
||||
repair = candidate["repair_candidate"]
|
||||
return {
|
||||
**common,
|
||||
"schema_version": "typed_bounded_execution_receipt_v1",
|
||||
"receipt_id": "execution-receipt-034",
|
||||
"status": "applied",
|
||||
"durable_writeback_ack": True,
|
||||
"applied_at": "2026-07-22T03:59:00+00:00",
|
||||
"candidate_fingerprint": candidate["fingerprint"],
|
||||
"canonical_asset_id": candidate["canonical_asset_id"],
|
||||
"action_id": repair["action_id"],
|
||||
"executor": repair["executor"],
|
||||
"runtime_mutation_performed": True,
|
||||
"cross_domain_fallback_performed": False,
|
||||
"domain_verifier": repair["verifier"],
|
||||
"domain_verifier_status": "verified",
|
||||
"domain_verifier_independent": True,
|
||||
"domain_verifier_receipt_id": "kubernetes-verifier-receipt-034",
|
||||
"durable_domain_verifier_ack": True,
|
||||
}
|
||||
|
||||
|
||||
def _post_readbacks(candidate: dict, execution: dict) -> tuple[dict, dict]:
|
||||
resource = {
|
||||
**_common("prometheus-post-readback-034"),
|
||||
"schema_version": "prometheus_cpu_resource_post_readback_v1",
|
||||
"source": "prometheus_independent_query",
|
||||
"trusted_server_readback": True,
|
||||
"independent": True,
|
||||
"durable_readback_ack": True,
|
||||
"candidate_fingerprint": candidate["fingerprint"],
|
||||
"execution_receipt_id": execution["receipt_id"],
|
||||
"canonical_asset_id": candidate["canonical_asset_id"],
|
||||
"metric_name": candidate["evidence"]["resource_metric"],
|
||||
"observed_value": 55.0,
|
||||
"runtime_mutation_performed": False,
|
||||
"cross_domain_fallback_performed": False,
|
||||
}
|
||||
latency = {
|
||||
**_common("signoz-post-readback-034"),
|
||||
"schema_version": "signoz_api_p99_post_readback_v1",
|
||||
"source": "signoz_independent_query",
|
||||
"trusted_server_readback": True,
|
||||
"independent": True,
|
||||
"durable_readback_ack": True,
|
||||
"candidate_fingerprint": candidate["fingerprint"],
|
||||
"execution_receipt_id": execution["receipt_id"],
|
||||
"canonical_asset_id": "service:awoooi-api",
|
||||
"service_name": "awoooi-api",
|
||||
"namespace": "awoooi-prod",
|
||||
"p99_seconds": 1.2,
|
||||
"runtime_mutation_performed": False,
|
||||
"cross_domain_fallback_performed": False,
|
||||
}
|
||||
return resource, latency
|
||||
|
||||
|
||||
def _closure_receipts(candidate: dict) -> tuple[dict, dict, dict]:
|
||||
execution = _execution_receipt(candidate)
|
||||
resource_readback, latency_readback = _post_readbacks(candidate, execution)
|
||||
resource = build_cpu_resource_post_verifier_receipt(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
metric_readback=resource_readback,
|
||||
)
|
||||
latency = build_signoz_api_p99_post_verifier_receipt(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
signoz_readback=latency_readback,
|
||||
)
|
||||
return execution, resource, latency
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_closure_requires_same_run_resource_and_latency_verifiers() -> None:
|
||||
store = _MemoryCandidateStore()
|
||||
result = await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="awoooi-api",
|
||||
resource_evidence=_resource(),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs(),
|
||||
recent_change_evidence=_change(),
|
||||
persist_candidate=store,
|
||||
)
|
||||
candidate = next(iter(store.records.values()))[1]
|
||||
execution, resource, latency = _closure_receipts(candidate)
|
||||
|
||||
closed = build_cpu_p99_closure(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
resource_verifier_receipt=resource,
|
||||
latency_verifier_receipt=latency,
|
||||
)
|
||||
assert closed["closed"] is True
|
||||
assert closed["status"] == "verified_closed"
|
||||
assert closed["resource_verifier_closed"] is True
|
||||
assert closed["latency_verifier_closed"] is True
|
||||
assert closed["same_run_receipts"] is True
|
||||
assert resource["verifier"] == "cpu_resource_independent_verifier"
|
||||
assert latency["verifier"] == "signoz_api_latency_independent_verifier"
|
||||
|
||||
_, stale_latency_readback = _post_readbacks(candidate, execution)
|
||||
stale_latency_readback["freshness_verified"] = False
|
||||
stale_latency = build_signoz_api_p99_post_verifier_receipt(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
signoz_readback=stale_latency_readback,
|
||||
)
|
||||
assert stale_latency["status"] == "blocked"
|
||||
assert "post_readback_freshness_unverified" in stale_latency["active_blockers"]
|
||||
|
||||
execution["runtime_mutation_performed"] = False
|
||||
no_apply = build_cpu_p99_closure(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
resource_verifier_receipt=resource,
|
||||
latency_verifier_receipt=latency,
|
||||
)
|
||||
assert no_apply["closed"] is False
|
||||
assert "typed_execution_receipt_unverified" in no_apply["active_blockers"]
|
||||
execution["runtime_mutation_performed"] = True
|
||||
|
||||
latency["run_id"] = "91abf6c8-5001-4b17-a538-3205d6f85757"
|
||||
blocked = build_cpu_p99_closure(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
resource_verifier_receipt=resource,
|
||||
latency_verifier_receipt=latency,
|
||||
)
|
||||
assert blocked["closed"] is False
|
||||
assert blocked["same_run_receipts"] is False
|
||||
assert "same_run_run_id_mismatch" in blocked["active_blockers"]
|
||||
assert result["runtime_mutation_performed"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verified_closure_requires_durable_append_ack() -> None:
|
||||
store = _MemoryCandidateStore()
|
||||
await build_cpu_p99_controlled_candidate(
|
||||
root_cause_target="awoooi-api",
|
||||
resource_evidence=_resource(),
|
||||
latency_evidence=_latency(),
|
||||
log_evidence=_logs(),
|
||||
recent_change_evidence=_change(),
|
||||
persist_candidate=store,
|
||||
)
|
||||
candidate = next(iter(store.records.values()))[1]
|
||||
execution, resource, latency = _closure_receipts(candidate)
|
||||
persisted: list[dict] = []
|
||||
|
||||
unacknowledged = build_cpu_p99_closure(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
resource_verifier_receipt=resource,
|
||||
latency_verifier_receipt=latency,
|
||||
)
|
||||
assert await persist_cpu_p99_closure(candidate, unacknowledged, "awoooi") is False
|
||||
|
||||
async def persist_closure(candidate_record, closure, project_id):
|
||||
assert candidate_record is candidate
|
||||
assert project_id == "awoooi"
|
||||
persisted.append(dict(closure))
|
||||
return True
|
||||
|
||||
pending = await record_cpu_p99_closure(
|
||||
candidate=candidate,
|
||||
execution_receipt={**execution, "runtime_mutation_performed": False},
|
||||
resource_verifier_receipt=resource,
|
||||
latency_verifier_receipt=latency,
|
||||
persist_closure=persist_closure,
|
||||
)
|
||||
assert pending["closed"] is False
|
||||
assert pending["durable_closure_ack"] is False
|
||||
assert persisted == []
|
||||
|
||||
closure = await record_cpu_p99_closure(
|
||||
candidate=candidate,
|
||||
execution_receipt=execution,
|
||||
resource_verifier_receipt=resource,
|
||||
latency_verifier_receipt=latency,
|
||||
persist_closure=persist_closure,
|
||||
)
|
||||
|
||||
assert closure["closed"] is True
|
||||
assert closure["durable_closure_ack"] is True
|
||||
assert persisted[0]["durable_closure_ack"] is True
|
||||
|
||||
|
||||
def test_alert_and_registry_contract_require_correlation_not_single_signal_apply() -> (
|
||||
None
|
||||
):
|
||||
signoz = yaml.safe_load(
|
||||
(ROOT / "ops/signoz/alerting/rules.yaml").read_text(encoding="utf-8")
|
||||
)
|
||||
p99 = next(
|
||||
rule
|
||||
for group in signoz["groups"]
|
||||
for rule in group.get("rules", [])
|
||||
if rule.get("alert") == "APIHighLatencyP99"
|
||||
)
|
||||
labels = p99["labels"]
|
||||
assert labels["auto_repair"] == "false"
|
||||
assert labels["canonical_asset_id"] == "service:awoooi-api"
|
||||
assert labels["namespace"] == "awoooi-prod"
|
||||
assert labels["workload_kind"] == "deployment"
|
||||
assert labels["workload_name"] == "awoooi-api"
|
||||
assert "不得因 P99 單一訊號直接 restart" in p99["annotations"]["runbook"]
|
||||
|
||||
for path in (
|
||||
ROOT / "ops/monitoring/alerts-unified.yml",
|
||||
ROOT / "ops/monitoring/alerts.yml",
|
||||
):
|
||||
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
host_cpu = next(
|
||||
rule
|
||||
for group in payload["groups"]
|
||||
for rule in group.get("rules", [])
|
||||
if rule.get("alert") == "HostHighCpuLoad"
|
||||
)
|
||||
assert host_cpu["labels"]["auto_repair"] == "false"
|
||||
|
||||
identity = ServiceRegistryClient(
|
||||
ROOT / "ops/config/service-registry.yaml"
|
||||
).resolve_identity("awoooi-api")
|
||||
assert identity["resolution_status"] == "resolved"
|
||||
assert identity["canonical_id"] == "service:awoooi-api"
|
||||
assert identity["kubernetes_namespace"] == "awoooi-prod"
|
||||
assert identity["kubernetes_kind"] == "deployment"
|
||||
assert identity["kubernetes_name"] == "awoooi-api"
|
||||
|
||||
readbacks = {
|
||||
row["readback_id"]: row
|
||||
for row in build_independent_verifier_registry()["external_readbacks"]
|
||||
}
|
||||
assert readbacks["cpu_resource_postcondition"]["required_source"] == (
|
||||
"prometheus_independent_query"
|
||||
)
|
||||
assert readbacks["api_p99_postcondition"]["required_source"] == (
|
||||
"signoz_independent_query"
|
||||
)
|
||||
assert readbacks["cpu_resource_postcondition"]["runtime_status"] == (
|
||||
"receipt_pending"
|
||||
)
|
||||
assert readbacks["api_p99_postcondition"]["runtime_status"] == ("receipt_pending")
|
||||
@@ -155,8 +155,8 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
|
||||
"active_or_completed_commitments": 68,
|
||||
"by_status": {
|
||||
"analysis_or_governance_complete": 6,
|
||||
"source_implemented_runtime_pending": 31,
|
||||
"in_progress": 29,
|
||||
"source_implemented_runtime_pending": 32,
|
||||
"in_progress": 28,
|
||||
"not_started_or_no_current_evidence": 2,
|
||||
"superseded": 2,
|
||||
},
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "sre_ai_agent_conversation_commitments_v1",
|
||||
"program_id": "AIA-SRE-P0-20260715",
|
||||
"generated_at": "2026-07-22T12:33:42+08:00",
|
||||
"generated_at": "2026-07-22T15:36:56+08:00",
|
||||
"source_scope": "This Codex AI Agent task only; cross-task delegation envelopes are excluded.",
|
||||
"authoritative_for_execution_order": false,
|
||||
"raw_conversation_embedded": false,
|
||||
@@ -51,7 +51,7 @@
|
||||
|
||||
{"id":"AIA-CONV-032","category":"named_incident","status":"source_implemented_runtime_pending","title":"INC-20260711-11C751 cold-start-gate 補 PlayBook、transport、rollback 與 verifier","linked_work_items":["AIA-SRE-004","AIA-SRE-006","AIA-SRE-015"],"terminal_condition":"namespace identity 修正後完成一次 bounded same-run repair 或明確 no-write terminal。","source_evidence":["source namespace separated from normalized host_recovery execution identity","exact inventory scope absence blocks Recover before single-flight claim or Agent99 transport","durable alert operation receipt records the explicit no-write terminal","consumer-validated source no-write verifier receipt with runtime closure false"]},
|
||||
{"id":"AIA-CONV-033","category":"named_incident","status":"source_implemented_runtime_pending","title":"INC-20260711-D037E5 修復 stale candidate 阻擋、自動排隊與 durable closure","linked_work_items":["AIA-SRE-007","AIA-SRE-015"],"terminal_condition":"新候選可 claim，apply 不重複，verifier、Telegram 與 learning receipts 同 run 完成。","source_evidence":["D037E5 canonical workload identity resolves to the Kubernetes executor/verifier without consulting legacy Ansible candidates","fingerprint-stable durable candidate event supports pending and pre-dispatch lease-expired atomic claim","dedicated feature-gated Kubernetes executor job owns a bounded one-candidate claim tick and is not started by API, signal worker or Ansible broker","single-dispatch barrier suppresses duplicate apply after dispatch starts","durable closure requires lifecycle, independent verifier, Telegram provider acknowledgement and learning receipts on one execution run"]},
|
||||
{"id":"AIA-CONV-034","category":"named_incident","status":"in_progress","title":"AWOOOI CPU 高負載與 P99 上升完成 RCA、bounded repair 與 verifier","linked_work_items":["AIA-SRE-004","AIA-SRE-006","AIA-SRE-010"],"terminal_condition":"相關 metrics/logs/changes 對齊 canonical asset，修復後 latency 與資源 verifier 關閉。"},
|
||||
{"id":"AIA-CONV-034","category":"named_incident","status":"source_implemented_runtime_pending","title":"AWOOOI CPU 高負載與 P99 上升完成 RCA、bounded repair 與 verifier","linked_work_items":["AIA-SRE-004","AIA-SRE-006","AIA-SRE-010"],"terminal_condition":"相關 metrics/logs/changes 對齊 canonical asset，修復後 latency 與資源 verifier 關閉。","source_evidence":["same-run canonical CPU/P99 metrics, sanitized log and recent-change receipt correlation","exact typed Kubernetes, Host Ansible and database RCA lanes with unknown-asset fail-closed behavior","fingerprint-deduplicated durable candidate receipt without provider, Agent99 or runtime mutation","durable closure requires typed execution plus deterministic independent resource and SigNoz P99 verifier receipts on one run"]},
|
||||
{"id":"AIA-CONV-035","category":"named_incident","status":"in_progress","title":"Backup/restore escrow 補 freshness、escrow metadata、restore drill 與 DR scorecard","linked_work_items":["AIA-SRE-011","AIA-SRE-016"],"terminal_condition":"read-only evidence 齊全；缺 escrow/restore receipt 時保持 blocked 而非假綠。"},
|
||||
{"id":"AIA-CONV-036","category":"named_incident","status":"source_implemented_runtime_pending","title":"Sentry Snuba DockerContainerUnhealthy 反覆告警自動修復","linked_work_items":["AIA-SRE-005"],"terminal_condition":"exact-container replay 完成 bounded recovery、健康 verifier 與 recurrence fence。"},
|
||||
{"id":"AIA-CONV-037","category":"named_incident","status":"source_implemented_runtime_pending","title":"AlertChainBroken_Alertmanager webhook error 大於 10% 自動旁路與修復","linked_work_items":["AIA-SRE-014","AIA-SRE-017"],"terminal_condition":"integration-scoped counters、旁路 delivery、receiver contract 與恢復 receipt 全部成立。"},
|
||||
@@ -98,8 +98,8 @@
|
||||
"active_or_completed_commitments": 68,
|
||||
"by_status": {
|
||||
"analysis_or_governance_complete": 6,
|
||||
"source_implemented_runtime_pending": 31,
|
||||
"in_progress": 29,
|
||||
"source_implemented_runtime_pending": 32,
|
||||
"in_progress": 28,
|
||||
"not_started_or_no_current_evidence": 2,
|
||||
"superseded": 2
|
||||
},
|
||||
|
||||
@@ -9,7 +9,7 @@ metadata:
|
||||
app: awoooi
|
||||
component: service-registry
|
||||
annotations:
|
||||
awoooi.wooo.work/source-sha256: "1a2777467c6d7b7912617f2fa6d22f590fa95a469aab0a8ff477d9725b425854"
|
||||
awoooi.wooo.work/source-sha256: "b1ad3cdd9ad3e041afd57f867738027b7668f72df63d0f238d7fa4befc584f9f"
|
||||
data:
|
||||
service-registry.yaml: |
|
||||
# ops/config/service-registry.yaml
|
||||
@@ -190,6 +190,11 @@ data:
|
||||
host: "k3s"
|
||||
stateful_level: AUTO
|
||||
asset_domain: kubernetes_workload
|
||||
executor: kubernetes_controlled_executor
|
||||
verifier: kubernetes_rollout_verifier
|
||||
kubernetes_namespace: awoooi-prod
|
||||
kubernetes_kind: deployment
|
||||
kubernetes_name: awoooi-api
|
||||
containers: []
|
||||
|
||||
- name: awoooi-web
|
||||
|
||||
@@ -176,6 +176,11 @@ services:
|
||||
host: "k3s"
|
||||
stateful_level: AUTO
|
||||
asset_domain: kubernetes_workload
|
||||
executor: kubernetes_controlled_executor
|
||||
verifier: kubernetes_rollout_verifier
|
||||
kubernetes_namespace: awoooi-prod
|
||||
kubernetes_kind: deployment
|
||||
kubernetes_name: awoooi-api
|
||||
containers: []
|
||||
|
||||
- name: awoooi-web
|
||||
|
||||
@@ -116,7 +116,7 @@ groups:
|
||||
severity: warning
|
||||
layer: systemd-188
|
||||
team: ops
|
||||
auto_repair: "true"
|
||||
auto_repair: "false"
|
||||
# MCP Phase 2a (ADR-071, 2026-04-11 Claude Sonnet 4.6): SSH MCP 路由標籤
|
||||
mcp_provider: "ssh_host"
|
||||
host_type: "bare_metal"
|
||||
|
||||
@@ -51,9 +51,18 @@ groups:
|
||||
severity: warning
|
||||
source: signoz
|
||||
team: backend
|
||||
alert_category: service_latency
|
||||
auto_repair: "false"
|
||||
canonical_asset_id: "service:awoooi-api"
|
||||
asset_domain: kubernetes_workload
|
||||
target_resource: awoooi-api
|
||||
namespace: awoooi-prod
|
||||
workload_kind: deployment
|
||||
workload_name: awoooi-api
|
||||
annotations:
|
||||
summary: "API P99 延遲 > 2s"
|
||||
description: "服務 {{ $labels.service_name }} P99: {{ $value }}s"
|
||||
runbook: "先建立 CPU/P99 correlation receipt，要求同一 run 的 canonical resource metrics、sanitized logs 與 recent-change evidence；不得因 P99 單一訊號直接 restart。只有 typed executor 完成 bounded repair，且 resource 與 SigNoz latency independent verifier 都通過，才可 closure。"
|
||||
|
||||
- alert: APIHighLatencyP95
|
||||
expr: |
|
||||
|
||||
Reference in New Issue
Block a user