6cf8429d17
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s
Restore D037 durable incident readback without weakening database failure semantics. Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints. Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls. Keep SSH service refusal visible while separating it from broker network-policy reachability.
370 lines
15 KiB
Python
370 lines
15 KiB
Python
from src.services.backup_restore_signal_automation import (
|
|
build_backup_restore_signal_automation_contract,
|
|
extract_backup_restore_evidence,
|
|
)
|
|
|
|
|
|
def test_backup_restore_contract_is_durable_and_fail_closed() -> None:
|
|
contract = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
evidence_text=(
|
|
'<b>Top evidence</b>\n'
|
|
'├ <code>alertname="BackupAggregateRunFailed" '
|
|
'escrow_missing=5 restic failed</code>\n'
|
|
'<b>建議下一步</b>'
|
|
),
|
|
source_refs={
|
|
"fingerprints": [
|
|
"ai_automation_alert_card:backup_restore_escrow_signal:"
|
|
"backup_restore_escrow_triage"
|
|
]
|
|
},
|
|
receipt_id="97ff9f41-7cdc-4893-8aab-634286fb128c",
|
|
run_id="0b25cafa-00a4-520b-8946-a3ea467b13fc",
|
|
source_row_exists=True,
|
|
contract_origin="synthesized_at_read",
|
|
declared_gates=[
|
|
"controlled_playbook_queue",
|
|
"runtime_write_gate=controlled",
|
|
],
|
|
declared_runtime_write_gate_count=1,
|
|
declared_runtime_write_gate_state="controlled",
|
|
declared_controlled_playbook_queue=True,
|
|
declared_learning_writeback_status="ready",
|
|
)
|
|
|
|
assert contract["work_item"]["work_item_id"].startswith("BRR-WI-")
|
|
assert contract["work_item"]["owner"] == "backup_dr"
|
|
assert contract["contract_origin"] == "synthesized_at_read"
|
|
assert contract["contract_persisted_at_send"] is False
|
|
assert contract["policy"]["policy_version"] == (
|
|
"backup_restore_readback_policy_v2"
|
|
)
|
|
assert contract["policy"]["effective_at"] == "2026-07-11T19:30:58+08:00"
|
|
assert contract["policy"]["source_hash"].startswith("sha256:")
|
|
provenance = contract["contract_provenance"]
|
|
assert provenance["source_contract_present_at_send"] is False
|
|
assert provenance["recorded_policy_present"] is False
|
|
assert provenance["recorded_immutable_declared_present"] is False
|
|
assert provenance["synthesized_at_read"] is True
|
|
assert contract["immutable_declared"]["runtime_write_gate_count"] == 1
|
|
assert contract["immutable_declared"]["controlled_playbook_queue"] is True
|
|
assert contract["effective_policy_projection"]["runtime_write_gate_count"] == 0
|
|
assert contract["work_item"]["status"] == "projected_open_evidence_failed"
|
|
assert contract["candidate"]["status"] == (
|
|
"projected_source_row_observed_dispatch_receipt_missing"
|
|
)
|
|
assert contract["candidate"]["execution_state"] == "not_started"
|
|
assert contract["candidate"]["agent99_dispatch"]["dispatched"] is False
|
|
assert contract["receipt"]["status"] == (
|
|
"projected_from_historical_source_row"
|
|
)
|
|
assert contract["receipt"]["source_row_exists"] is True
|
|
assert contract["receipt"]["contract_persisted_at_send"] is False
|
|
assert contract["receipt"]["is_agent99_execution_receipt"] is False
|
|
assert contract["verifier"]["status"] == (
|
|
"projected_dispatch_receipt_missing_verifier_not_started"
|
|
)
|
|
assert contract["verifier"]["terminal"] is False
|
|
assert contract["verifier"]["no_false_green"] is True
|
|
assert set(contract["verifier"]["failed_fields"]) == {
|
|
"backup_status",
|
|
"escrow",
|
|
}
|
|
assert "source_resolution_receipt" in contract["verifier"]["missing_fields"]
|
|
assert contract["runtime_write_allowed"] is False
|
|
assert contract["runtime_execution_authorized"] is False
|
|
assert contract["production_write_performed"] is False
|
|
assert "run_restore" in contract["prohibited_actions"]
|
|
assert "remote_delete" in contract["prohibited_actions"]
|
|
assert "retention_change" in contract["prohibited_actions"]
|
|
assert "escrow_marker_write" in contract["prohibited_actions"]
|
|
assert all(
|
|
{"asset_id", "owner", "status", "next_action"} <= set(asset)
|
|
for asset in contract["asset_ledger"]
|
|
)
|
|
|
|
|
|
def test_backup_restore_contract_never_closes_without_source_resolution() -> None:
|
|
observations = [
|
|
{
|
|
"field": field,
|
|
"status": "healthy",
|
|
"observed": True,
|
|
"safe_value": "explicit_success",
|
|
}
|
|
for field in (
|
|
"backup_status",
|
|
"freshness",
|
|
"offsite_verify",
|
|
"escrow",
|
|
"restore_drill",
|
|
)
|
|
]
|
|
|
|
contract = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
evidence_observations=observations,
|
|
receipt_id="durable-message",
|
|
run_id="durable-run",
|
|
source_row_exists=True,
|
|
contract_origin="synthesized_at_read",
|
|
)
|
|
|
|
assert contract["work_item"]["status"] == (
|
|
"projected_open_evidence_incomplete"
|
|
)
|
|
assert contract["verifier"]["evidence_result"] == "degraded"
|
|
assert contract["verifier"]["terminal"] is False
|
|
assert contract["verifier"]["missing_fields"] == [
|
|
"source_resolution_receipt"
|
|
]
|
|
assert contract["closure_state"] == "partial_degraded_safe_next_action"
|
|
|
|
|
|
def test_backup_restore_work_item_clusters_recurrence_but_receipts_stay_unique() -> None:
|
|
shared = {
|
|
"project_id": "awoooi",
|
|
"source_refs": {
|
|
"fingerprints": [
|
|
"ai_automation_alert_card:backup_restore_escrow_signal:"
|
|
"backup_restore_escrow_triage"
|
|
]
|
|
},
|
|
"source_row_exists": True,
|
|
"contract_origin": "synthesized_at_read",
|
|
}
|
|
first = build_backup_restore_signal_automation_contract(
|
|
**shared,
|
|
receipt_id="message-1",
|
|
run_id="run-1",
|
|
)
|
|
second = build_backup_restore_signal_automation_contract(
|
|
**shared,
|
|
receipt_id="message-2",
|
|
run_id="run-2",
|
|
)
|
|
|
|
assert first["work_item"]["work_item_id"] == second["work_item"]["work_item_id"]
|
|
assert first["candidate"]["candidate_id"] != second["candidate"]["candidate_id"]
|
|
assert first["receipt"]["receipt_id"] != second["receipt"]["receipt_id"]
|
|
assert first["verifier"]["verifier_id"] != second["verifier"]["verifier_id"]
|
|
|
|
|
|
def test_backup_dispatch_requires_accepted_triggered_backupcheck_receipt() -> None:
|
|
receipt = {
|
|
"schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
|
|
"status": "accepted_inbox_triggered",
|
|
"transport": "relay",
|
|
"alert_id": "awoooi-backup-alert",
|
|
"kind": "backup_health",
|
|
"suggested_mode": "BackupCheck",
|
|
"accepted": True,
|
|
"inbox_triggered": True,
|
|
"receipt_persisted": True,
|
|
"runtime_closure_verified": False,
|
|
"run_id": "agent99-run-123",
|
|
"trace_id": "00-12345678901234567890123456789012-1234567890123456-01",
|
|
"work_item_id": "agent99-dispatch:awoooi:INC-BACKUP:backupcheck",
|
|
}
|
|
contract = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
receipt_id="source-message",
|
|
run_id="source-run",
|
|
source_row_exists=True,
|
|
contract_origin="persisted_at_send",
|
|
contract_persisted_at_send=True,
|
|
agent99_dispatch_receipt=receipt,
|
|
)
|
|
|
|
dispatch = contract["candidate"]["agent99_dispatch"]
|
|
assert dispatch["status"] == "accepted_inbox_triggered"
|
|
assert dispatch["dispatched"] is True
|
|
assert dispatch["receipt_persisted"] is True
|
|
assert dispatch["identity_complete"] is True
|
|
assert dispatch["agent99_run_id"] == "agent99-run-123"
|
|
provenance = contract["contract_provenance"]
|
|
assert provenance["source_contract_present_at_send"] is True
|
|
assert provenance["recorded_policy_present"] is True
|
|
assert provenance["recorded_policy_source_hash"].startswith("sha256:")
|
|
assert provenance["recorded_immutable_declared_present"] is True
|
|
binding = contract["candidate"]["identity_binding"]
|
|
assert binding["status"] == "bound_dispatch_verifier_pending"
|
|
assert binding["complete"] is True
|
|
assert binding["brr_work_item_id"] == contract["work_item"]["work_item_id"]
|
|
assert binding["brr_verifier_id"] == contract["verifier"]["verifier_id"]
|
|
assert binding["agent99_run_id"] == "agent99-run-123"
|
|
assert binding["agent99_work_item_id"].startswith("agent99-dispatch:")
|
|
assert contract["candidate"]["execution_state"] == "dispatched"
|
|
assert contract["candidate"]["status"] == "dispatched_verifier_pending"
|
|
assert contract["verifier"]["status"] == "dispatched_verifier_pending"
|
|
assert contract["verifier"]["terminal"] is False
|
|
assert contract["verifier"]["agent99_outcome"]["status"] == (
|
|
"dispatch_accepted_outcome_pending"
|
|
)
|
|
assert contract["verifier"]["agent99_outcome"]["required_source_schema"] == (
|
|
"agent99_outcome_contract_v1"
|
|
)
|
|
|
|
wrong_route = dict(receipt, suggested_mode="ProviderFreshness")
|
|
blocked = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
receipt_id="source-message",
|
|
run_id="source-run",
|
|
source_row_exists=True,
|
|
contract_origin="persisted_at_send",
|
|
contract_persisted_at_send=True,
|
|
agent99_dispatch_receipt=wrong_route,
|
|
)
|
|
assert blocked["candidate"]["execution_state"] == "not_started"
|
|
assert blocked["candidate"]["agent99_dispatch"]["status"] == (
|
|
"dispatch_route_mismatch_fail_closed"
|
|
)
|
|
assert blocked["verifier"]["terminal"] is False
|
|
|
|
not_durable = dict(receipt, receipt_persisted=False)
|
|
blocked = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
receipt_id="source-message",
|
|
run_id="source-run",
|
|
source_row_exists=True,
|
|
contract_origin="persisted_at_send",
|
|
contract_persisted_at_send=True,
|
|
agent99_dispatch_receipt=not_durable,
|
|
)
|
|
assert blocked["candidate"]["execution_state"] == "not_started"
|
|
assert blocked["candidate"]["agent99_dispatch"]["status"] == (
|
|
"dispatch_receipt_not_persisted_fail_closed"
|
|
)
|
|
|
|
legacy = dict(
|
|
receipt,
|
|
schema_version="telegram_agent99_dispatch_receipt_v1",
|
|
receipt_persisted=False,
|
|
)
|
|
legacy_contract = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
receipt_id="legacy-source-message",
|
|
run_id="legacy-source-run",
|
|
source_row_exists=True,
|
|
contract_origin="synthesized_at_read",
|
|
agent99_dispatch_receipt=legacy,
|
|
)
|
|
assert legacy_contract["candidate"]["execution_state"] == "not_started"
|
|
assert legacy_contract["candidate"]["agent99_dispatch"]["status"] == (
|
|
"legacy_dispatch_receipt_unbound"
|
|
)
|
|
assert legacy_contract["candidate"]["identity_binding"]["complete"] is False
|
|
assert legacy_contract["verifier"]["terminal"] is False
|
|
|
|
failed_verifier_receipt = dict(
|
|
receipt,
|
|
post_verifier_passed=False,
|
|
current_run_state="failed",
|
|
current_verifier={
|
|
"schema_version": "agent99_independent_verifier_receipt_v1",
|
|
"status": "failed",
|
|
"run_id": receipt["run_id"],
|
|
"trace_id": receipt["trace_id"],
|
|
"work_item_id": receipt["work_item_id"],
|
|
"verifier_passed": False,
|
|
"source_event_resolved": False,
|
|
},
|
|
)
|
|
failed_contract = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
receipt_id="failed-source-message",
|
|
run_id="failed-source-run",
|
|
source_row_exists=True,
|
|
contract_origin="persisted_at_send",
|
|
contract_persisted_at_send=True,
|
|
agent99_dispatch_receipt=failed_verifier_receipt,
|
|
)
|
|
assert failed_contract["candidate"]["status"] == (
|
|
"verifier_failed_retry_required"
|
|
)
|
|
assert failed_contract["verifier"]["status"] == "failed_terminal"
|
|
assert failed_contract["verifier"]["result"] == "failed"
|
|
assert failed_contract["verifier"]["terminal"] is True
|
|
assert failed_contract["closure_state"] == (
|
|
"partial_degraded_safe_next_action"
|
|
)
|
|
|
|
false_green_receipt = dict(
|
|
receipt,
|
|
post_verifier_passed=True,
|
|
runtime_closure_verified=True,
|
|
closure_complete=True,
|
|
current_run_state="completed",
|
|
current_verifier={
|
|
"schema_version": "agent99_independent_verifier_receipt_v1",
|
|
"status": "success",
|
|
"run_id": receipt["run_id"],
|
|
"trace_id": receipt["trace_id"],
|
|
"work_item_id": receipt["work_item_id"],
|
|
"outcome_state": "resolved",
|
|
"transport_ok": True,
|
|
"verifier_passed": True,
|
|
"source_event_resolved": True,
|
|
"verified_at": "2026-07-11T20:20:00+08:00",
|
|
"evidence_refs": {},
|
|
},
|
|
current_learning_writeback={
|
|
"schema_version": "agent99_learning_writeback_receipt_v1",
|
|
"status": "success",
|
|
"run_id": receipt["run_id"],
|
|
"trace_id": receipt["trace_id"],
|
|
"work_item_id": receipt["work_item_id"],
|
|
"receipt_refs": {
|
|
"incident_closure_receipt_id": "inc-close-1",
|
|
"telegram_lifecycle_receipt_id": "tg-life-1",
|
|
"km_writeback_ack_id": "km-ack-1",
|
|
"playbook_trust_writeback_ack_id": "pb-ack-1",
|
|
"dr_scorecard_writeback_ack_id": "dr-ack-1",
|
|
},
|
|
},
|
|
)
|
|
false_green_contract = build_backup_restore_signal_automation_contract(
|
|
project_id="awoooi",
|
|
receipt_id="false-green-source-message",
|
|
run_id="false-green-source-run",
|
|
source_row_exists=True,
|
|
contract_origin="persisted_at_send",
|
|
contract_persisted_at_send=True,
|
|
agent99_dispatch_receipt=false_green_receipt,
|
|
)
|
|
assert false_green_contract["verifier"]["terminal"] is False
|
|
assert false_green_contract["candidate"]["agent99_dispatch"][
|
|
"runtime_closure_verified"
|
|
] is False
|
|
assert set(false_green_contract["verifier"][
|
|
"missing_backup_evidence_refs"
|
|
]) == {
|
|
"agent99_outcome_receipt_id",
|
|
"post_verifier_evidence_ref",
|
|
"source_event_evidence_ref",
|
|
"backup_status_evidence_ref",
|
|
"freshness_evidence_ref",
|
|
"offsite_verify_evidence_ref",
|
|
"escrow_evidence_ref",
|
|
"restore_drill_evidence_ref",
|
|
"source_resolution_receipt_ref",
|
|
}
|
|
|
|
|
|
def test_backup_evidence_negative_phrases_override_good_tokens() -> None:
|
|
observations = extract_backup_restore_evidence(
|
|
"<b>Top evidence</b>\n"
|
|
"├ <code>backup_status: 1 but not ok</code>\n"
|
|
"└ <code>restore_drill: 1 but not verified</code>\n"
|
|
"<b>建議下一步</b>"
|
|
)
|
|
by_field = {item["field"]: item for item in observations}
|
|
|
|
assert by_field["backup_status"]["status"] == "failed"
|
|
assert by_field["backup_status"]["safe_value"] == "explicit_negative_result"
|
|
assert by_field["restore_drill"]["status"] == "failed"
|
|
assert by_field["restore_drill"]["safe_value"] == (
|
|
"explicit_negative_result"
|
|
)
|