wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 5 Packages Projects Releases Wiki Activity

fix(automation): bind backup approvals to full owner

Browse Source
This commit is contained in:
ogt
2026-07-14 11:24:50 +08:00
parent bbd479a36c
commit 8bee637466
3 changed files with 875 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

231
apps/api/tests/integration/test_backup_restore_prod_db_regressions.py
View File

@@ -23,9 +23,7 @@ def _explicit_local_test_database_url() -> str:
if not database_url:
pytest.skip("BACKUP_RESTORE_TEST_DATABASE_URL is required for real PG tests")
parsed = urlsplit(
database_url.replace("postgresql+asyncpg://", "postgresql://", 1)
)
parsed = urlsplit(database_url.replace("postgresql+asyncpg://", "postgresql://", 1))
if parsed.hostname not in {None, "", "localhost", "127.0.0.1", "::1"}:
pytest.fail("backup/restore PG regression must target a local database")
database_name = parsed.path.rsplit("/", 1)[-1].strip().lower()
@@ -45,7 +43,8 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
async with engine.connect() as connection:
transaction = await connection.begin()
try:
await connection.execute(text("""
await connection.execute(
text("""
CREATE TEMP TABLE awooop_run_state (
run_id UUID PRIMARY KEY,
project_id VARCHAR(64) NOT NULL,
@@ -69,7 +68,8 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
timeout_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) ON COMMIT DROP
"""))
""")
)
await connection.execute(
backfill._CURSOR_INSERT_SQL,
{
@@ -77,9 +77,7 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
"project_id": "awoooi",
"agent_id": backfill.BACKFILL_CURSOR_AGENT_ID,
"input_sha256": "a" * 64,
"error_detail": json.dumps(
backfill._initial_cursor("awoooi")
),
"error_detail": json.dumps(backfill._initial_cursor("awoooi")),
},
)
await connection.execute(
@@ -127,19 +125,23 @@ async def test_backfill_run_inserts_satisfy_no_default_cost_column() -> None:
assert finished.scalar_one() == item_run_id
rows = (
await connection.execute(text("""
(
await connection.execute(
text("""
SELECT run_id, cost_usd
FROM awooop_run_state
ORDER BY run_id
"""))
).mappings().all()
""")
)
)
.mappings()
.all()
)
assert {row["run_id"] for row in rows} == {
cursor_run_id,
item_run_id,
}
assert {row["cost_usd"] for row in rows} == {
Decimal("0.0000")
}
assert {row["cost_usd"] for row in rows} == {Decimal("0.0000")}
finally:
await transaction.rollback()
finally:
@@ -166,12 +168,14 @@ async def test_domain_separated_fingerprints_avoid_cross_domain_pg_collision() -
async with engine.connect() as connection:
transaction = await connection.begin()
try:
await connection.execute(text("""
await connection.execute(
text("""
CREATE TEMP TABLE approval_records (
source_identity TEXT NOT NULL,
fingerprint VARCHAR(64) NOT NULL UNIQUE
) ON COMMIT DROP
"""))
""")
)
await connection.execute(
text("""
INSERT INTO approval_records (
@@ -186,12 +190,18 @@ async def test_domain_separated_fingerprints_avoid_cross_domain_pg_collision() -
},
)
persisted = (
await connection.execute(text("""
(
await connection.execute(
text("""
SELECT source_identity, fingerprint
FROM approval_records
ORDER BY source_identity
"""))
).mappings().all()
""")
)
)
.mappings()
.all()
)
assert len(persisted) == 2
assert {row["fingerprint"] for row in persisted} == {
stored_long_fingerprint,
@@ -204,6 +214,159 @@ async def test_domain_separated_fingerprints_avoid_cross_domain_pg_collision() -
await engine.dispose()
@pytest.mark.asyncio
async def test_full_owner_pg_replay_does_not_reuse_resolved_prior_occurrence() -> None:
"""A source-only v2 row cannot bind a later startsAt to its Incident."""
engine = create_async_engine(_explicit_local_test_database_url(), echo=False)
source_fingerprint = "legacy-backup-occurrence:" + ("a" * 64)
common = {
"project_id": "awoooi",
"source_fingerprint": source_fingerprint,
"source_event_fingerprint": "legacy-backup:message:hash",
}
previous_owner = (
backup_restore_alertmanager_ingress.build_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-11T10:00:00Z",
)
)
previous_legacy = (
backup_restore_alertmanager_ingress._legacy_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-11T10:00:00Z",
)
)
current_owner = (
backup_restore_alertmanager_ingress.build_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-12T10:00:00Z",
)
)
current_legacy = (
backup_restore_alertmanager_ingress._legacy_backup_restore_ingress_owner(
**common,
source_started_at="2026-07-12T10:00:00Z",
)
)
source_only_v2 = backup_restore_alertmanager_ingress._approval_storage_fingerprint(
source_fingerprint
)
current_v3 = (
backup_restore_alertmanager_ingress._approval_owner_storage_fingerprint(
current_owner
)
)
assert previous_owner["approval_id"] == previous_legacy["approval_id"]
try:
async with engine.connect() as connection:
transaction = await connection.begin()
try:
await connection.execute(
text("""
CREATE TEMP TABLE approval_owner_replay (
id UUID PRIMARY KEY,
fingerprint VARCHAR(64) NOT NULL,
incident_id TEXT NOT NULL,
status TEXT NOT NULL,
extra_metadata JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) ON COMMIT DROP
""")
)
await connection.execute(
text("""
INSERT INTO approval_owner_replay (
id, fingerprint, incident_id, status, extra_metadata
) VALUES (
:id, :fingerprint, :incident_id, 'resolved',
CAST(:extra_metadata AS JSONB)
)
"""),
{
"id": UUID(previous_owner["approval_id"]),
"fingerprint": source_only_v2,
"incident_id": previous_owner["incident_id"],
"extra_metadata": json.dumps(previous_legacy),
},
)
async def find_by_fingerprint(
*,
fingerprint: str,
debounce_minutes: int,
):
assert debounce_minutes == 30
row = (
(
await connection.execute(
text("""
SELECT id, incident_id, status, extra_metadata
FROM approval_owner_replay
WHERE fingerprint = :fingerprint
ORDER BY created_at DESC
LIMIT 1
"""),
{"fingerprint": fingerprint},
)
)
.mappings()
.one_or_none()
)
if row is None:
return None
return SimpleNamespace(
id=row["id"],
incident_id=row["incident_id"],
status=row["status"],
metadata=dict(row["extra_metadata"]),
)
service = SimpleNamespace(find_by_fingerprint=find_by_fingerprint)
incompatible = await (
backup_restore_alertmanager_ingress._find_compatible_owner_approval(
service,
owner=current_owner,
legacy_owner=current_legacy,
)
)
assert incompatible is None
await connection.execute(
text("""
INSERT INTO approval_owner_replay (
id, fingerprint, incident_id, status, extra_metadata
) VALUES (
:id, :fingerprint, :incident_id, 'approved',
CAST(:extra_metadata AS JSONB)
)
"""),
{
"id": UUID(current_owner["approval_id"]),
"fingerprint": current_v3,
"incident_id": current_owner["incident_id"],
"extra_metadata": json.dumps(current_owner),
},
)
replay = await (
backup_restore_alertmanager_ingress._find_compatible_owner_approval(
service,
owner=current_owner,
legacy_owner=current_legacy,
)
)
assert replay is not None
assert str(replay.id) == current_owner["approval_id"]
assert replay.incident_id == current_owner["incident_id"]
assert replay.incident_id != previous_owner["incident_id"]
assert len(current_v3) == 64
finally:
await transaction.rollback()
finally:
await engine.dispose()
@pytest.mark.asyncio
async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction(
monkeypatch: pytest.MonkeyPatch,
@@ -217,9 +380,7 @@ async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction
):
separator = "&" if "?" in sqlalchemy_url else "?"
sqlalchemy_url = f"{sqlalchemy_url}{separator}ssl=disable"
direct_url = platform_operator_service._asyncpg_direct_database_url(
sqlalchemy_url
)
direct_url = platform_operator_service._asyncpg_direct_database_url(sqlalchemy_url)
direct_query = dict(parse_qsl(urlsplit(direct_url).query))
assert "ssl" not in direct_query
assert direct_query["sslmode"] == "disable"
@@ -271,16 +432,18 @@ async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction
)
""",
UUID("66cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92"),
json.dumps({
"ai_automation_alert_card": {
"event_type": "backup_restore_escrow_signal",
"lane": "backup_restore_escrow_triage",
"target": "backup_restore",
"delivery_receipt_readback_required": True,
"runtime_write_gate_count": 0,
},
"source_refs": {"fingerprints": ["backup-fixture"]},
}),
json.dumps(
{
"ai_automation_alert_card": {
"event_type": "backup_restore_escrow_signal",
"lane": "backup_restore_escrow_triage",
"target": "backup_restore",
"delivery_receipt_readback_required": True,
"runtime_write_gate_count": 0,
},
"source_refs": {"fingerprints": ["backup-fixture"]},
}
),
)
original_connect = asyncpg.connect
@@ -316,6 +479,4 @@ async def test_direct_readback_rewrites_ssl_and_keeps_rls_context_in_transaction
assert summary["backup_restore_total"] == 1
assert len(rows) == 1
assert rows[0]["project_id"] == "awoooi"
assert rows[0]["alert_card"]["event_type"] == (
"backup_restore_escrow_signal"
)
assert rows[0]["alert_card"]["event_type"] == ("backup_restore_escrow_signal")