wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(sre): require Sentry recurrence closure

Browse Source
This commit is contained in:
Your Name
2026-07-19 12:08:49 +08:00
parent 4d1dfe7def
commit da59c89e1c
8 changed files with 725 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
apps/api/tests/test_ansible_verified_closure.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import hashlib
import inspect
import json
from contextlib import asynccontextmanager
@@ -126,6 +127,283 @@ def _claim() -> service.AnsibleCheckModeClaim:
)
def _sentry_claim() -> service.AnsibleCheckModeClaim:
claim = _claim()
return replace(
claim,
catalog_id="ansible:110-sentry-profiling-consumer-recovery",
playbook_path=(
"infra/ansible/playbooks/"
"110-sentry-profiling-consumer-recovery.yml"
),
apply_playbook_path=(
"infra/ansible/playbooks/"
"110-sentry-profiling-consumer-recovery.yml"
),
inventory_hosts=("host_110",),
input_payload={
**claim.input_payload,
"project_id": "awoooi",
"typed_target_route": {
"canonical_asset_id": "service:sentry",
},
"source_recurrence": {
"schema_version": "alert_source_recurrence_receipt_v1",
"verified": True,
"fingerprint": "a" * 32,
"occurrence_id": "alert-20260719010101",
"observed_at": datetime.now(UTC).isoformat(),
"identity_anchor": "webhook_payload",
"canonical_asset_id": "service:sentry",
"source": "current_alert_webhook",
"durable_readback_verified": False,
},
},
)
@pytest.mark.asyncio
async def test_sentry_source_recurrence_fence_passes_only_with_both_anchors(
monkeypatch: pytest.MonkeyPatch,
) -> None:
db = _SequenceDB(
_MappingResult(),
_MappingResult({
"apply_anchor_found": True,
"source_anchor_found": True,
"newer_recurrence_found": False,
}),
)
@asynccontextmanager
async def db_context(project_id: str):
assert project_id == "awoooi"
yield db
monkeypatch.setattr(service, "get_db_context", db_context)
receipt = await service._verify_sentry_source_recurrence_fence(
_sentry_claim(),
apply_op_id="00000000-0000-0000-0000-000000000901",
host_postconditions_passed=True,
)
assert receipt["verified"] is True
assert receipt["durable_readback_verified"] is True
assert receipt["direct_webhook_recurrence_covered"] is True
assert receipt["active_blockers"] == []
sql = db.statements[1]
assert "WITH apply_boundary AS MATERIALIZED" in sql
assert "source_anchor AS MATERIALIZED" in sql
assert "newer_recurrence AS MATERIALIZED" in sql
assert "event_row.received_at > apply_boundary.created_at" in sql
assert "'{source_refs,event_ids}'" in sql
assert "'{source_refs,fingerprints}'" in sql
assert db.parameters[1]["occurrence_id"] == "alert-20260719010101"
assert db.parameters[1]["fingerprint"] == "a" * 32
@pytest.mark.asyncio
@pytest.mark.parametrize(
("row", "expected_blocker"),
[
(
{
"apply_anchor_found": False,
"source_anchor_found": True,
"newer_recurrence_found": False,
},
"source_recurrence_apply_anchor_missing",
),
(
{
"apply_anchor_found": True,
"source_anchor_found": False,
"newer_recurrence_found": False,
},
"source_recurrence_event_anchor_missing",
),
(
{
"apply_anchor_found": True,
"source_anchor_found": True,
"newer_recurrence_found": True,
},
"source_recurrence_observed_after_apply",
),
(
{
"apply_anchor_found": True,
"source_anchor_found": True,
"newer_recurrence_found": None,
},
"source_recurrence_readback_invalid",
),
],
)
async def test_sentry_source_recurrence_fence_fails_closed(
monkeypatch: pytest.MonkeyPatch,
row: dict,
expected_blocker: str,
) -> None:
db = _SequenceDB(_MappingResult(), _MappingResult(row))
@asynccontextmanager
async def db_context(_project_id: str):
yield db
monkeypatch.setattr(service, "get_db_context", db_context)
receipt = await service._verify_sentry_source_recurrence_fence(
_sentry_claim(),
apply_op_id="00000000-0000-0000-0000-000000000902",
host_postconditions_passed=True,
)
assert receipt["verified"] is False
assert expected_blocker in receipt["active_blockers"]
@pytest.mark.asyncio
async def test_sentry_post_verifier_wires_durable_recurrence_fence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
host_receipt = {
"schema_version": service.ANSIBLE_POST_VERIFIER_SCHEMA_VERSION,
"verifier": "asset_specific_read_only_host_postconditions",
"independent_source": "broker_ssh_host_runtime_readback",
"automation_run_id": _sentry_claim().input_payload["automation_run_id"],
"apply_op_id": "00000000-0000-0000-0000-000000000903",
"catalog_id": "ansible:110-sentry-profiling-consumer-recovery",
"verification_result": "success",
"all_postconditions_passed": True,
"technical_postconditions_passed": True,
"required_postcondition_count": 3,
"passed_postcondition_count": 3,
"postconditions": [
{"condition_id": f"host-{index}", "passed": True}
for index in range(3)
],
"active_blockers": [],
"writes_on_verify": False,
"executor_returncode": 0,
}
host_verifier = AsyncMock(return_value=host_receipt)
monkeypatch.setattr(
service,
"run_ansible_asset_post_verifier",
host_verifier,
)
fence = {
"schema_version": "sentry_source_recurrence_fence_receipt_v1",
"condition_id": "awoooi_sentry_source_recurrence_after_apply",
"condition_type": "source_recurrence",
"verified": True,
"passed": True,
"durable_readback_verified": True,
"writes_on_verify": False,
"raw_output_stored": False,
"source_fingerprint_sha256": hashlib.sha256(("a" * 32).encode()).hexdigest(),
"source_occurrence_id_sha256": hashlib.sha256(
b"alert-20260719010101"
).hexdigest(),
"active_blockers": [],
}
monkeypatch.setattr(
service,
"_verify_sentry_source_recurrence_fence",
AsyncMock(return_value=fence),
)
receipt, reused = await service._resolve_ansible_post_verifier_receipt(
_sentry_claim(),
service.AnsibleRunResult(0, "", "", 1),
apply_op_id="00000000-0000-0000-0000-000000000903",
)
assert reused is False
assert receipt["verification_result"] == "success"
assert receipt["source_recurrence_fence_verified"] is True
assert receipt["required_postcondition_count"] == 4
assert receipt["passed_postcondition_count"] == 4
assert receipt["postconditions"][-1]["condition_type"] == (
"source_recurrence"
)
host_verifier.reset_mock()
replayed_receipt, replayed_reused = (
await service._resolve_ansible_post_verifier_receipt(
_sentry_claim(),
service.AnsibleRunResult(0, "", "", 1),
apply_op_id="00000000-0000-0000-0000-000000000903",
durable_verifier_receipt=host_receipt,
)
)
assert replayed_reused is False
assert replayed_receipt["source_recurrence_fence_verified"] is True
host_verifier.assert_awaited_once()
def test_sentry_durable_verifier_cannot_bypass_source_recurrence_fence() -> None:
claim = _sentry_claim()
apply_op_id = "00000000-0000-0000-0000-000000000904"
host_receipt = {
"schema_version": service.ANSIBLE_POST_VERIFIER_SCHEMA_VERSION,
"automation_run_id": claim.input_payload["automation_run_id"],
"apply_op_id": apply_op_id,
"catalog_id": claim.catalog_id,
"verification_result": "success",
"all_postconditions_passed": True,
"required_postcondition_count": 3,
"passed_postcondition_count": 3,
"postconditions": [
{"condition_id": f"host-{index}", "passed": True}
for index in range(3)
],
"active_blockers": [],
"writes_on_verify": False,
"executor_returncode": 0,
}
row = {"durable_verifier_receipt": host_receipt}
assert service._durable_post_verifier_receipt_for_backfill(
row,
claim,
apply_op_id=apply_op_id,
) is None
fence = {
"schema_version": "sentry_source_recurrence_fence_receipt_v1",
"condition_id": "awoooi_sentry_source_recurrence_after_apply",
"condition_type": "source_recurrence",
"verified": True,
"passed": True,
"durable_readback_verified": True,
"writes_on_verify": False,
"source_fingerprint_sha256": hashlib.sha256(("a" * 32).encode()).hexdigest(),
"source_occurrence_id_sha256": hashlib.sha256(
b"alert-20260719010101"
).hexdigest(),
"active_blockers": [],
}
fenced = service._attach_sentry_source_recurrence_fence(
host_receipt,
fence,
)
assert service._durable_post_verifier_receipt_for_backfill(
{"durable_verifier_receipt": fenced},
claim,
apply_op_id=apply_op_id,
) == fenced
fenced["source_recurrence_fence"]["source_fingerprint_sha256"] = "b" * 64
assert service._durable_post_verifier_receipt_for_backfill(
{"durable_verifier_receipt": fenced},
claim,
apply_op_id=apply_op_id,
) is None
def _verified_result() -> service.AnsibleRunResult:
return service.AnsibleRunResult(
returncode=0,

7
apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py
View File

@@ -155,9 +155,9 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
"active_or_completed_commitments": 68,
"by_status": {
"analysis_or_governance_complete": 6,
"source_implemented_runtime_pending": 24,
"source_implemented_runtime_pending": 25,
"in_progress": 32,
"not_started_or_no_current_evidence": 6,
"not_started_or_no_current_evidence": 5,
"superseded": 2,
},
"product_runtime_closed_commitments": 0,
@@ -168,6 +168,9 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
assert commitments["AIA-CONV-012"]["title"].startswith("Claude API")
assert commitments["AIA-CONV-013"]["title"].startswith("Gemini API")
assert "Host112" in commitments["AIA-CONV-049"]["title"]
assert commitments["AIA-CONV-036"]["status"] == (
"source_implemented_runtime_pending"
)
assert commitments["AIA-CONV-050"]["status"] == (
"source_implemented_runtime_pending"
)

84
apps/api/tests/test_sre_typed_domain_router.py
View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import hashlib
import json
import os
import subprocess
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
@@ -60,6 +61,7 @@ PLAYBOOK = (
)
SENTRY_CONTAINER = "sentry-self-hosted-snuba-profiling-functions-consumer-1"
SENTRY_CATALOG = "ansible:110-sentry-profiling-consumer-recovery"
SENTRY_FINGERPRINT = "a" * 32
WAZUH_CATALOG = "ansible:wazuh-manager-posture-readback"
@@ -169,7 +171,7 @@ def _sentry_runtime_incident() -> SimpleNamespace:
"container_name": SENTRY_CONTAINER,
"namespace": "default",
"host": "110",
"fingerprint": "sentry-profiling-fingerprint",
"fingerprint": SENTRY_FINGERPRINT,
},
annotations={},
)
@@ -182,11 +184,13 @@ def _sentry_runtime_proposal(occurrence_id: str) -> dict:
"source": "alert_webhook_controlled_router",
"risk_level": "medium",
"action": SENTRY_CONTAINER,
"source_fingerprint": "sentry-profiling-fingerprint",
"source_fingerprint": SENTRY_FINGERPRINT,
"source_occurrence_id": occurrence_id,
"source_recurrence_verified": True,
"source_recurrence_source": "current_alert_webhook",
"source_recurrence_anchor": "webhook_payload",
"source_received_at": "2026-07-19T00:00:00+00:00",
"source_canonical_asset_id": "service:sentry",
}
@@ -1388,9 +1392,12 @@ def test_webhook_recurrence_receipt_requires_explicit_occurrence_identity() -> N
assert payload is not None
recurrence = payload["input"]["source_recurrence"]
assert recurrence["fingerprint"] == "sentry-profiling-fingerprint"
assert recurrence["fingerprint"] == SENTRY_FINGERPRINT
assert recurrence["occurrence_id"] is None
assert recurrence["verified"] is False
assert recurrence["observed_at"] == "2026-07-19T00:00:00+00:00"
assert recurrence["canonical_asset_id"] == "service:sentry"
assert recurrence["source"] == "current_alert_webhook"
def test_sentry_playbook_and_post_verifier_are_exact_container_bounded() -> None:
@@ -1405,8 +1412,26 @@ def test_sentry_playbook_and_post_verifier_are_exact_container_bounded() -> None
assert catalog["inventory_hosts"] == ["host_110"]
assert catalog["auto_apply_enabled"] is True
assert catalog["risk_level"] == "medium"
assert len(postconditions) == 2
assert catalog["catalog_revision"] == (
"2026-07-19-sentry-exact-container-v2"
)
assert len(postconditions) == 3
assert all(row.inventory_host == "host_110" for row in postconditions)
recurrence = next(
row
for row in postconditions
if row.condition_id
== "host_110_sentry_profiling_alert_resolved_after_window"
)
assert recurrence.condition_type == "metric"
assert recurrence.timeout_seconds == 660
assert 'alertname=\"DockerContainerUnhealthy\"' in recurrence.probe
assert 'job=\"docker-health-monitor\"' in recurrence.probe
assert 'host=\"110\"' in recurrence.probe
assert SENTRY_CONTAINER in recurrence.probe
assert "result\")==[]" in recurrence.probe
assert "clean_samples=$((clean_samples + 1))" in recurrence.probe
assert 'test "$clean_samples" -ge 37' in recurrence.probe
assert source.count(SENTRY_CONTAINER) == 1
for forbidden in (
"docker compose",
@@ -1418,6 +1443,55 @@ def test_sentry_playbook_and_post_verifier_are_exact_container_bounded() -> None
assert forbidden not in source.lower()
def test_sentry_alert_probe_requires_consecutive_clean_samples(
tmp_path: Path,
) -> None:
recurrence = next(
row
for row in postconditions_for_catalog(SENTRY_CATALOG)
if row.condition_id
== "host_110_sentry_profiling_alert_resolved_after_window"
)
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
count_file = tmp_path / "curl-count"
fake_curl = fake_bin / "curl"
fake_curl.write_text(
"""#!/usr/bin/env bash
count=0
test ! -f "$SENTRY_PROBE_COUNT_FILE" || count=$(cat "$SENTRY_PROBE_COUNT_FILE")
count=$((count + 1))
printf '%s' "$count" > "$SENTRY_PROBE_COUNT_FILE"
if test "$count" -eq 2; then
printf '%s\n' '{"status":"success","data":{"result":[{"metric":{"alertname":"DockerContainerUnhealthy"}}]}}'
else
printf '%s\n' '{"status":"success","data":{"result":[]}}'
fi
""",
encoding="utf-8",
)
fake_sleep = fake_bin / "sleep"
fake_sleep.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8")
fake_curl.chmod(0o755)
fake_sleep.chmod(0o755)
result = subprocess.run(
["bash", "-c", recurrence.probe],
check=False,
capture_output=True,
text=True,
timeout=10,
env={
**os.environ,
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"SENTRY_PROBE_COUNT_FILE": str(count_file),
},
)
assert result.returncode == 0, result.stderr
assert count_file.read_text(encoding="utf-8") == "39"
def test_converged_webhook_recurrence_reenters_generation_preflight() -> None:
source = (ROOT / "apps/api/src/api/v1/webhooks.py").read_text(
encoding="utf-8"
@@ -1426,6 +1500,8 @@ def test_converged_webhook_recurrence_reenters_generation_preflight() -> None:
assert '"source_fingerprint": resolved_source_fingerprint' in source
assert '"source_occurrence_id": source_alert_id' in source
assert '"source_recurrence_verified": bool(' in source
assert '"source_received_at": datetime.now(UTC).isoformat()' in source
assert '"source_canonical_asset_id": typed_target_route.get(' in source
assert source.count(
'getattr(updated_approval, "incident_id", None)\n'
" and _controlled_ai_policy_allows"