wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(automation): bind backup approvals to full owner

Browse Source
This commit is contained in:
ogt
2026-07-14 11:24:50 +08:00
parent bbd479a36c
commit 8bee637466
3 changed files with 875 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
apps/api/tests/integration/test_backup_restore_prod_db_regressions.py
View File

@@ -23,9 +23,7 @@ def _explicit_local_test_database_url() -> str:
if not database_url:
pytest.skip("BACKUP_RESTORE_TEST_DATABASE_URL is required for real PG tests")
parsed = urlsplit(
database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
)
parsed = urlsplit(database_url.replace("postgresql+asyncpg://", "postgresql://", 1))
if parsed.hostname not in {None, "", "localhost", "127.0.0.1", "::1"}:
pytest.fail("backup/restore PG regression must target a local database")
database_name = parsed.path.rsplit("/", 1)[-1].strip().lower()
@@ -45,7 +43,8 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
async with engine.connect() as connection:
transaction = await connection.begin()
try:
await connection.execute(text("""
await connection.execute(
text("""
CREATE TEMP TABLE awooop_run_state (
run_id UUID PRIMARY KEY,
project_id VARCHAR(64) NOT NULL,
@@ -69,7 +68,8 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
timeout_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) ON COMMIT DROP
"""))
""")
)
await connection.execute(
backfill._CURSOR_INSERT_SQL,
{
@@ -77,9 +77,7 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
"project_id": "awoooi",
"agent_id": backfill.BACKFILL_CURSOR_AGENT_ID,
"input_sha256": "a" * 64,
"error_detail": json.dumps(
backfill._initial_cursor("awoooi")
),
"error_detail": json.dumps(backfill._initial_cursor("awoooi")),
},
)
await connection.execute(
@@ -127,19 +125,23 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
assert finished.scalar_one() == item_run_id
rows = (
await connection.execute(text("""
(
await connection.execute(
text("""
SELECT run_id, cost_usd
FROM awooop_run_state
ORDER BY run_id
"""))
).mappings().all()
""")
)
)
.mappings()
.all()
)
assert {row["run_id"] for row in rows} == {
cursor_run_id,
item_run_id,
}
assert {row["cost_usd"] for row in rows} == {
Decimal("0.0000")
}
assert {row["cost_usd"] for row in rows} == {Decimal("0.0000")}
finally:
await transaction.rollback()
finally:
@@ -166,12 +168,14 @@ async def test_domain_separated_fingerprints_avoid_cross_domain_pg_collision() -
async with engine.connect() as connection:
transaction = await connection.begin()
try:
await connection.execute(text("""
await connection.execute(
text("""
CREATE TEMP TABLE approval_records (
source_identity TEXT NOT NULL,
fingerprint VARCHAR(64) NOT NULL UNIQUE
) ON COMMIT DROP
"""))
""")
)
await connection.execute(
text("""
INSERT INTO approval_records (
@@ -186,12 +190,18 @@ async def test_domain_separated_fingerprints_avoid_cross_domain_pg_collision() -
},
)
persisted = (
await connection.execute(text("""
(
await connection.execute(
text("""
SELECT source_identity, fingerprint
FROM approval_records
ORDER BY source_identity
"""))
).mappings().all()
""")
)
)
.mappings()
.all()
)
assert len(persisted) == 2
assert {row["fingerprint"] for row in persisted} == {
stored_long_fingerprint,
@@ -204,6 +214,159 @@ async def test_domain_separated_fingerprints_avoid_cross_domain_pg_collision() -
await engine.dispose()
@pytest.mark.asyncio
async def test_full_owner_pg_replay_does_not_reuse_resolved_prior_occurrence() -> None:
"""A source-only v2 row cannot bind a later startsAt to its Incident."""
engine = create_async_engine(_explicit_local_test_database_url(), echo=False)
source_fingerprint = "legacy-backup-occurrence:" + ("a" * 64)
common = {
"project_id": "awoooi",
"source_fingerprint": source_fingerprint,
"source_event_fingerprint": "legacy-backup:message:hash",
}
previous_owner = (
backup_restore_alertmanager_ingress.build_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-11T10:00:00Z",
)
)
previous_legacy = (
backup_restore_alertmanager_ingress._legacy_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-11T10:00:00Z",
)
)
current_owner = (
backup_restore_alertmanager_ingress.build_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-12T10:00:00Z",
)
)
current_legacy = (
backup_restore_alertmanager_ingress._legacy_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-12T10:00:00Z",
)
)
source_only_v2 = backup_restore_alertmanager_ingress._approval_storage_fingerprint(
source_fingerprint
)
current_v3 = (
backup_restore_alertmanager_ingress._approval_owner_storage_fingerprint(
current_owner
)
)
assert previous_owner["approval_id"] == previous_legacy["approval_id"]
try:
async with engine.connect() as connection:
transaction = await connection.begin()
try:
await connection.execute(
text("""
CREATE TEMP TABLE approval_owner_replay (
id UUID PRIMARY KEY,
fingerprint VARCHAR(64) NOT NULL,
incident_id TEXT NOT NULL,
status TEXT NOT NULL,
extra_metadata JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) ON COMMIT DROP
""")
)
await connection.execute(
text("""
INSERT INTO approval_owner_replay (
id, fingerprint, incident_id, status, extra_metadata
) VALUES (
:id, :fingerprint, :incident_id, 'resolved',
CAST(:extra_metadata AS JSONB)
)
"""),
{
"id": UUID(previous_owner["approval_id"]),
"fingerprint": source_only_v2,
"incident_id": previous_owner["incident_id"],
"extra_metadata": json.dumps(previous_legacy),
},
)
async def find_by_fingerprint(
*,
fingerprint: str,
debounce_minutes: int,
):
assert debounce_minutes == 30
row = (
(
await connection.execute(
text("""
SELECT id, incident_id, status, extra_metadata
FROM approval_owner_replay
WHERE fingerprint = :fingerprint
ORDER BY created_at DESC
LIMIT 1
"""),
{"fingerprint": fingerprint},
)
)
.mappings()
.one_or_none()
)
if row is None:
return None
return SimpleNamespace(
id=row["id"],
incident_id=row["incident_id"],
status=row["status"],
metadata=dict(row["extra_metadata"]),
)
service = SimpleNamespace(find_by_fingerprint=find_by_fingerprint)
incompatible = await (
backup_restore_alertmanager_ingress._find_compatible_owner_approval(
service,
owner=current_owner,
legacy_owner=current_legacy,
)
)
assert incompatible is None
await connection.execute(
text("""
INSERT INTO approval_owner_replay (
id, fingerprint, incident_id, status, extra_metadata
) VALUES (
:id, :fingerprint, :incident_id, 'approved',
CAST(:extra_metadata AS JSONB)
)
"""),
{
"id": UUID(current_owner["approval_id"]),
"fingerprint": current_v3,
"incident_id": current_owner["incident_id"],
"extra_metadata": json.dumps(current_owner),
},
)
replay = await (
backup_restore_alertmanager_ingress._find_compatible_owner_approval(
service,
owner=current_owner,
legacy_owner=current_legacy,
)
)
assert replay is not None
assert str(replay.id) == current_owner["approval_id"]
assert replay.incident_id == current_owner["incident_id"]
assert replay.incident_id != previous_owner["incident_id"]
assert len(current_v3) == 64
finally:
await transaction.rollback()
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction(
monkeypatch: pytest.MonkeyPatch,
@@ -217,9 +380,7 @@ async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction
):
separator = "&" if "?" in sqlalchemy_url else "?"
sqlalchemy_url = f"{sqlalchemy_url}{separator}ssl=disable"
direct_url = platform_operator_service._asyncpg_direct_database_url(
sqlalchemy_url
)
direct_url = platform_operator_service._asyncpg_direct_database_url(sqlalchemy_url)
direct_query = dict(parse_qsl(urlsplit(direct_url).query))
assert "ssl" not in direct_query
assert direct_query["sslmode"] == "disable"
@@ -271,16 +432,18 @@ async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction
)
""",
UUID("66cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92"),
json.dumps({
"ai_automation_alert_card": {
"event_type": "backup_restore_escrow_signal",
"lane": "backup_restore_escrow_triage",
"target": "backup_restore",
"delivery_receipt_readback_required": True,
"runtime_write_gate_count": 0,
},
"source_refs": {"fingerprints": ["backup-fixture"]},
}),
json.dumps(
{
"ai_automation_alert_card": {
"event_type": "backup_restore_escrow_signal",
"lane": "backup_restore_escrow_triage",
"target": "backup_restore",
"delivery_receipt_readback_required": True,
"runtime_write_gate_count": 0,
},
"source_refs": {"fingerprints": ["backup-fixture"]},
}
),
)
original_connect = asyncpg.connect
@@ -316,6 +479,4 @@ async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction
assert summary["backup_restore_total"] == 1
assert len(rows) == 1
assert rows[0]["project_id"] == "awoooi"
assert rows[0]["alert_card"]["event_type"] == (
"backup_restore_escrow_signal"
)
assert rows[0]["alert_card"]["event_type"] == ("backup_restore_escrow_signal")

421
apps/api/tests/test_backup_restore_alertmanager_ingress.py
View File

@@ -78,6 +78,14 @@ def test_backup_ingress_owner_is_stable_per_source_occurrence() -> None:
assert first == replay
assert new_firing["approval_id"] != first["approval_id"]
assert first["schema_version"] == "backup_restore_ingress_owner_v2"
assert first["owner_key_encoding"] == "canonical_json_v1"
assert first["identity_id_scheme"] == "canonical_json_v2"
assert first["source_occurrence"] == {
"source_event_fingerprint": "native-fingerprint",
"source_started_at": "2026-07-11T10:00:00Z",
"fallback_source_fingerprint": "",
}
assert first["route_id"] == "agent99:backup_health:BackupCheck"
assert first["single_dispatch_owner"] == "alertmanager_raw_signal"
assert first["telegram_role"] == "receipt_projection_only"
@@ -85,6 +93,87 @@ def test_backup_ingress_owner_is_stable_per_source_occurrence() -> None:
assert first["read_only"] is True
def test_backup_owner_canonical_framing_breaks_colon_and_pipe_collision() -> None:
left = ingress.build_backup_restore_ingress_owner(
project_id="alpha:beta",
source_fingerprint="gamma",
source_event_fingerprint="delta|epsilon",
source_started_at="zeta",
)
right = ingress.build_backup_restore_ingress_owner(
project_id="alpha",
source_fingerprint="beta:gamma",
source_event_fingerprint="delta",
source_started_at="epsilon|zeta",
)
left_legacy = ingress._legacy_backup_restore_ingress_owner(
project_id="alpha:beta",
source_fingerprint="gamma",
source_event_fingerprint="delta|epsilon",
source_started_at="zeta",
)
right_legacy = ingress._legacy_backup_restore_ingress_owner(
project_id="alpha",
source_fingerprint="beta:gamma",
source_event_fingerprint="delta",
source_started_at="epsilon|zeta",
)
assert left_legacy["approval_id"] == right_legacy["approval_id"]
assert left["approval_id"] != right["approval_id"]
assert left["owner_key_sha256"] != right["owner_key_sha256"]
assert ingress._legacy_owner_reuse_is_unambiguous(left) is False
assert ingress._legacy_owner_reuse_is_unambiguous(right) is False
def test_backup_owner_canonical_framing_breaks_empty_component_collision() -> None:
event_only = ingress.build_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint="source-fingerprint",
source_event_fingerprint="same-value",
source_started_at="",
)
started_only = ingress.build_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint="source-fingerprint",
source_event_fingerprint="",
source_started_at="same-value",
)
assert event_only["legacy_approval_id"] == started_only["legacy_approval_id"]
assert event_only["approval_id"] != started_only["approval_id"]
assert ingress._legacy_owner_reuse_is_unambiguous(event_only) is False
assert ingress._legacy_owner_reuse_is_unambiguous(started_only) is False
def test_backup_legacy_89_char_claim_keeps_existing_deterministic_ids() -> None:
source = "legacy-backup-occurrence:" + ("a" * 64)
kwargs = {
"project_id": "awoooi",
"source_fingerprint": source,
"source_event_fingerprint": "legacy-backup:message:hash",
"source_started_at": "2026-07-11T10:00:00Z",
}
owner = ingress.build_backup_restore_ingress_owner(**kwargs)
legacy = ingress._legacy_backup_restore_ingress_owner(**kwargs)
assert len(source) == 89
assert owner["schema_version"] == "backup_restore_ingress_owner_v2"
assert owner["identity_id_scheme"] == "legacy_colon_v1"
assert owner["legacy_identity_compatibility"] is True
assert owner["approval_id"] == legacy["approval_id"]
assert owner["incident_id"] == legacy["incident_id"]
assert owner["work_item_id"] == legacy["work_item_id"]
assert owner["approval_id"] == "0896722a-a4ea-574b-9c1a-c389b30c14ae"
assert owner["incident_id"] == "INC-BRR-544862858131AD18"
assert owner["work_item_id"] == (
"backupcheck:awoooi:544862858131ad1803baa9fea9f27903"
)
assert ingress._approval_owner_storage_fingerprint(owner) != (
ingress._approval_storage_fingerprint(source)
)
def test_backup_approval_fingerprint_is_domain_separated_and_bounded() -> None:
short = "source-fingerprint"
long = "legacy-backup-occurrence:" + ("a" * 64)
@@ -92,9 +181,7 @@ def test_backup_approval_fingerprint_is_domain_separated_and_bounded() -> None:
stored_short = ingress._approval_storage_fingerprint(short)
stored_long = ingress._approval_storage_fingerprint(long)
stored_deliberate_raw = ingress._approval_storage_fingerprint(
legacy_long_digest
)
stored_deliberate_raw = ingress._approval_storage_fingerprint(legacy_long_digest)
assert stored_short != short
assert stored_long != legacy_long_digest
@@ -102,21 +189,51 @@ def test_backup_approval_fingerprint_is_domain_separated_and_bounded() -> None:
assert len(stored_short) == ingress.APPROVAL_FINGERPRINT_MAX_LENGTH
assert len(stored_long) == ingress.APPROVAL_FINGERPRINT_MAX_LENGTH
assert ingress._approval_storage_fingerprint(long) == stored_long
assert ingress._approval_fingerprint_lookup_candidates(short) == (
short_owner = ingress.build_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint=short,
source_event_fingerprint="short-occurrence",
)
long_owner = ingress.build_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint=long,
source_event_fingerprint="long-occurrence",
)
short_candidates = ingress._approval_fingerprint_lookup_candidates(short_owner)
long_candidates = ingress._approval_fingerprint_lookup_candidates(long_owner)
assert short_candidates == (
ingress._approval_owner_storage_fingerprint(short_owner),
stored_short,
short,
)
assert ingress._approval_fingerprint_lookup_candidates(long) == (
assert long_candidates == (
ingress._approval_owner_storage_fingerprint(long_owner),
stored_long,
legacy_long_digest,
)
assert short_candidates[0] != long_candidates[0]
assert len(short_candidates[0]) == ingress.APPROVAL_FINGERPRINT_MAX_LENGTH
@pytest.mark.asyncio
async def test_backup_approval_legacy_lookup_rejects_deliberate_collision() -> None:
long = "legacy-backup-occurrence:" + ("a" * 64)
deliberate_raw = hashlib.sha256(long.encode("utf-8")).hexdigest()
canonical, legacy = ingress._approval_fingerprint_lookup_candidates(long)
owner = ingress.build_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint=long,
source_event_fingerprint="legacy-backup:message:hash",
source_started_at="2026-07-11T10:00:00Z",
)
legacy_owner = ingress._legacy_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint=long,
source_event_fingerprint="legacy-backup:message:hash",
source_started_at="2026-07-11T10:00:00Z",
)
canonical, source_only_v2, legacy = ingress._approval_fingerprint_lookup_candidates(
owner
)
assert legacy == deliberate_raw
wrong_source = SimpleNamespace(
@@ -131,23 +248,38 @@ async def test_backup_approval_legacy_lookup_rejects_deliberate_collision() -> N
assert debounce_minutes == 30
return wrong_source if fingerprint == legacy else None
result = await ingress._find_compatible_source_approval(
result = await ingress._find_compatible_owner_approval(
SimpleNamespace(find_by_fingerprint=find_by_fingerprint),
source_fingerprint=long,
owner=owner,
legacy_owner=legacy_owner,
)
assert result is None
assert calls == [canonical, legacy]
assert calls == [canonical, source_only_v2, legacy]
@pytest.mark.asyncio
async def test_backup_approval_legacy_89_char_lookup_remains_compatible() -> None:
source = "legacy-backup-occurrence:" + ("a" * 64)
canonical, legacy = ingress._approval_fingerprint_lookup_candidates(source)
owner = ingress.build_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint=source,
source_event_fingerprint="legacy-backup:message:hash",
source_started_at="2026-07-11T10:00:00Z",
)
legacy_owner = ingress._legacy_backup_restore_ingress_owner(
project_id="awoooi",
source_fingerprint=source,
source_event_fingerprint="legacy-backup:message:hash",
source_started_at="2026-07-11T10:00:00Z",
)
canonical, source_only_v2, legacy = ingress._approval_fingerprint_lookup_candidates(
owner
)
existing = SimpleNamespace(
id=UUID("22222222-2222-4222-8222-222222222222"),
incident_id="INC-LEGACY-89-CHAR",
metadata={"source_fingerprint": source},
id=UUID(legacy_owner["approval_id"]),
incident_id=legacy_owner["incident_id"],
metadata=dict(legacy_owner),
)
calls: list[str] = []
@@ -156,14 +288,51 @@ async def test_backup_approval_legacy_89_char_lookup_remains_compatible() -> Non
assert debounce_minutes == 30
return existing if fingerprint == legacy else None
result = await ingress._find_compatible_source_approval(
result = await ingress._find_compatible_owner_approval(
SimpleNamespace(find_by_fingerprint=find_by_fingerprint),
source_fingerprint=source,
owner=owner,
legacy_owner=legacy_owner,
)
assert result is existing
assert result.incident_id == "INC-LEGACY-89-CHAR"
assert calls == [canonical, legacy]
assert result.incident_id == legacy_owner["incident_id"]
assert calls == [canonical, source_only_v2, legacy]
@pytest.mark.asyncio
async def test_backup_source_only_v2_lookup_reuses_only_full_legacy_owner() -> None:
source = "legacy-backup-occurrence:" + ("b" * 64)
kwargs = {
"project_id": "awoooi",
"source_fingerprint": source,
"source_event_fingerprint": "legacy-backup:message-two:hash",
"source_started_at": "2026-07-11T11:00:00Z",
}
owner = ingress.build_backup_restore_ingress_owner(**kwargs)
legacy_owner = ingress._legacy_backup_restore_ingress_owner(**kwargs)
canonical, source_only_v2, _legacy = (
ingress._approval_fingerprint_lookup_candidates(owner)
)
existing = SimpleNamespace(
id=UUID(legacy_owner["approval_id"]),
incident_id=legacy_owner["incident_id"],
metadata=dict(legacy_owner),
)
calls: list[str] = []
async def find_by_fingerprint(*, fingerprint: str, debounce_minutes: int):
calls.append(fingerprint)
assert debounce_minutes == 30
return existing if fingerprint == source_only_v2 else None
result = await ingress._find_compatible_owner_approval(
SimpleNamespace(find_by_fingerprint=find_by_fingerprint),
owner=owner,
legacy_owner=legacy_owner,
)
assert result is existing
assert calls == [canonical, source_only_v2]
@pytest.mark.asyncio
@@ -207,9 +376,7 @@ async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortc
"component": "backup_restore",
"namespace": "awoooi-prod",
},
annotations={
"summary": "backup freshness missing; readback required"
},
annotations={"summary": "backup freshness missing; readback required"},
startsAt=datetime.now(UTC).isoformat(),
fingerprint="native-backup-fingerprint",
generatorURL="http://alertmanager/graph",
@@ -249,23 +416,34 @@ class _FakeApprovalService:
async def find_by_fingerprint(self, **_kwargs):
self.calls.append("legacy_owner_read")
expected = ingress._approval_fingerprint_lookup_candidates(
str(_signal_kwargs()["source_fingerprint"])
expected_owner = ingress.build_backup_restore_ingress_owner(
project_id=str(_signal_kwargs()["project_id"]),
source_fingerprint=str(_signal_kwargs()["source_fingerprint"]),
source_event_fingerprint=str(_signal_kwargs()["source_event_fingerprint"]),
source_started_at=str(_signal_kwargs()["source_started_at"]),
)
expected = ingress._approval_fingerprint_lookup_candidates(expected_owner)
assert _kwargs["fingerprint"] in expected
return None
async def create_approval_with_fingerprint(self, *, request, fingerprint):
self.calls.append("owner_create")
self.created_request = request
assert fingerprint == ingress._approval_storage_fingerprint(
str(_signal_kwargs()["source_fingerprint"])
expected_owner = ingress.build_backup_restore_ingress_owner(
project_id=str(_signal_kwargs()["project_id"]),
source_fingerprint=str(_signal_kwargs()["source_fingerprint"]),
source_event_fingerprint=str(_signal_kwargs()["source_event_fingerprint"]),
source_started_at=str(_signal_kwargs()["source_started_at"]),
)
assert fingerprint == ingress._approval_owner_storage_fingerprint(
expected_owner
)
self.approval = SimpleNamespace(
id=UUID(request.metadata["preallocated_approval_id"]),
incident_id=request.incident_id,
hit_count=1,
risk_level=request.risk_level,
metadata=dict(request.metadata),
)
return self.approval
@@ -307,12 +485,14 @@ async def test_backup_ingress_binds_canonical_incident_then_dispatches_and_reads
assert kwargs["incident_id"] == incident_id
assert kwargs["route_id"] == "agent99:backup_health:BackupCheck"
assert kwargs["fingerprint"] == _signal_kwargs()["source_fingerprint"]
identity.update({
"incident_id": incident_id,
"run_id": "12345678-1234-5678-9234-567812345678",
"trace_id": "00-11111111111111111111111111111111-2222222222222222-01",
"work_item_id": kwargs["work_item_id"],
})
identity.update(
{
"incident_id": incident_id,
"run_id": "12345678-1234-5678-9234-567812345678",
"trace_id": "00-11111111111111111111111111111111-2222222222222222-01",
"work_item_id": kwargs["work_item_id"],
}
)
return {
"status": "dispatched",
"dispatchPerformed": True,
@@ -361,12 +541,8 @@ async def test_backup_ingress_binds_canonical_incident_then_dispatches_and_reads
assert calls.index("owner_create") < calls.index("incident_create")
assert calls.index("incident_create") < calls.index("incident_bind")
assert calls.index("incident_bind") < calls.index("incident_canonical_readback")
assert calls.index("incident_canonical_readback") < calls.index(
"agent99_dispatch"
)
assert calls.index("agent99_dispatch") < calls.index(
"dispatch_receipt_readback"
)
assert calls.index("incident_canonical_readback") < calls.index("agent99_dispatch")
assert calls.index("agent99_dispatch") < calls.index("dispatch_receipt_readback")
assert result["status"] == "backupcheck_dispatched_verifier_pending"
assert result["receipt_persisted"] is True
assert result["accepted"] is True
@@ -379,9 +555,12 @@ async def test_backup_ingress_binds_canonical_incident_then_dispatches_and_reads
assert calls.count("owner_create") == 1
assert calls.count("incident_create") == 1
assert calls.count("owner_recurrence") == 1
assert approval_service.approval.fingerprint == hashlib.sha256(
str(_signal_kwargs()["source_fingerprint"]).encode("utf-8")
).hexdigest()
assert (
approval_service.approval.fingerprint
== hashlib.sha256(
str(_signal_kwargs()["source_fingerprint"]).encode("utf-8")
).hexdigest()
)
assert approval_service.created_request.risk_level.value == "low"
assert approval_service.created_request.metadata["telegram_role"] == (
"receipt_projection_only"
@@ -389,6 +568,146 @@ async def test_backup_ingress_binds_canonical_incident_then_dispatches_and_reads
assert "run_restore" in result["prohibited_actions"]
@pytest.mark.asyncio
async def test_same_source_new_occurrence_does_not_inherit_resolved_incident(
monkeypatch: pytest.MonkeyPatch,
) -> None:
previous_kwargs = _signal_kwargs()
current_kwargs = {
**previous_kwargs,
"alert_id": "backup-alert-new-occurrence",
"source_started_at": "2026-07-12T10:00:00Z",
}
previous_owner = ingress.build_backup_restore_ingress_owner(
project_id=str(previous_kwargs["project_id"]),
source_fingerprint=str(previous_kwargs["source_fingerprint"]),
source_event_fingerprint=str(previous_kwargs["source_event_fingerprint"]),
source_started_at=str(previous_kwargs["source_started_at"]),
)
current_owner = ingress.build_backup_restore_ingress_owner(
project_id=str(current_kwargs["project_id"]),
source_fingerprint=str(current_kwargs["source_fingerprint"]),
source_event_fingerprint=str(current_kwargs["source_event_fingerprint"]),
source_started_at=str(current_kwargs["source_started_at"]),
)
previous = SimpleNamespace(
id=UUID(previous_owner["approval_id"]),
incident_id=previous_owner["incident_id"],
hit_count=1,
metadata=dict(previous_owner),
status="approved",
)
class OccurrenceApprovalService:
def __init__(self) -> None:
self.created = None
self.created_request = None
async def get_approval(self, approval_id: UUID):
if self.created is not None and self.created.id == approval_id:
return self.created
if previous.id == approval_id:
return previous
return None
async def find_by_fingerprint(
self,
*,
fingerprint: str,
debounce_minutes: int,
):
assert debounce_minutes == 30
if fingerprint in {
ingress._approval_storage_fingerprint(
str(current_kwargs["source_fingerprint"])
),
hashlib.sha256(
str(current_kwargs["source_fingerprint"]).encode("utf-8")
).hexdigest(),
}:
return previous
return None
async def create_approval_with_fingerprint(self, *, request, fingerprint):
assert request.incident_id is None
assert fingerprint == ingress._approval_owner_storage_fingerprint(
current_owner
)
self.created_request = request
self.created = SimpleNamespace(
id=UUID(request.metadata["preallocated_approval_id"]),
incident_id=None,
hit_count=1,
metadata=dict(request.metadata),
status="approved",
)
return self.created
async def increment_hit_count(self, _approval_id: UUID):
raise AssertionError("new occurrence must not increment prior owner")
async def update_incident_id(self, approval_id: UUID, incident_id: str):
assert self.created is not None
assert approval_id == self.created.id
self.created.incident_id = incident_id
approval_service = OccurrenceApprovalService()
async def create_incident(**kwargs):
assert kwargs["approval_id"] == current_owner["approval_id"]
assert kwargs["canonical_incident_id"] == current_owner["incident_id"]
return str(kwargs["canonical_incident_id"])
async def canonical_readback(incident_id: str, *, project_id: str):
assert incident_id == current_owner["incident_id"]
assert incident_id != previous_owner["incident_id"]
assert project_id == "awoooi"
return SimpleNamespace(status="investigating", persisted_to_pg=True)
identity = {
"incident_id": current_owner["incident_id"],
"run_id": "42345678-1234-5678-9234-567812345678",
"trace_id": "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01",
"work_item_id": current_owner["work_item_id"],
}
async def dispatch(**kwargs):
assert kwargs["incident_id"] == current_owner["incident_id"]
assert kwargs["approval_id"] == current_owner["approval_id"]
return {
"status": "dispatched",
"dispatchPerformed": True,
"identity": identity,
}
async def readback(**kwargs):
assert kwargs["incident_id"] == current_owner["incident_id"]
return {
"identity": identity,
"dispatch_receipt": {"accepted": True, "inbox_triggered": True},
"verifier": {"status": "pending"},
}
monkeypatch.setattr(ingress, "get_approval_service", lambda: approval_service)
monkeypatch.setattr(ingress, "create_incident_for_approval", create_incident)
monkeypatch.setattr(
ingress,
"get_incident_service",
lambda: SimpleNamespace(get_for_readback=canonical_readback),
)
monkeypatch.setattr(ingress, "bridge_alertmanager_to_agent99", dispatch)
monkeypatch.setattr(ingress, "read_agent99_dispatch_receipt", readback)
monkeypatch.setattr(ingress, "record_alertmanager_event", AsyncMock())
result = await ingress.process_backup_restore_alertmanager_signal(**current_kwargs)
assert result["status"] == "backupcheck_dispatched_verifier_pending"
assert result["owner_created"] is True
assert result["incident_id"] == current_owner["incident_id"]
assert result["incident_id"] != previous_owner["incident_id"]
assert approval_service.created_request.incident_id is None
class _FakeDispatchLedger:
def __init__(self) -> None:
self.receipt = None
@@ -452,17 +771,19 @@ async def test_backup_bridge_postgres_identity_deduplicates_transport(
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: transported.append(payload)
or {
"schema_version": "agent99_sre_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"transport": "relay",
"alert_id": str(payload["id"]),
"kind": str(payload["kind"]),
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
},
lambda payload: (
transported.append(payload)
or {
"schema_version": "agent99_sre_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"transport": "relay",
"alert_id": str(payload["id"]),
"kind": str(payload["kind"]),
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
}
),
)
common = {
"alertname": "BackupCredentialEscrowEvidenceMissing",