1036 lines
39 KiB
Python
1036 lines
39 KiB
Python
"""Fail-closed automation contract for backup / restore / escrow signals.
|
|
|
|
The Telegram card is only a sensor receipt. This module turns that receipt
|
|
into deterministic work-item, candidate, durable-receipt and verifier
|
|
identities without running backup, restore, prune, remote delete, retention
|
|
changes, or escrow writes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import html
|
|
import re
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any
|
|
|
|
BACKUP_RESTORE_EVENT_TYPE = "backup_restore_escrow_signal"
|
|
BACKUP_RESTORE_LANE = "backup_restore_escrow_triage"
|
|
BACKUP_RESTORE_POLICY_VERSION = "backup_restore_readback_policy_v2"
|
|
BACKUP_RESTORE_POLICY_EFFECTIVE_AT = "2026-07-11T19:30:58+08:00"
|
|
BACKUP_RESTORE_POLICY_SOURCE_REF = (
|
|
"apps/api/src/services/backup_restore_signal_automation.py:"
|
|
"backup_restore_readback_policy_v2"
|
|
)
|
|
_BACKUP_RESTORE_POLICY_CANONICAL = "|".join(
|
|
[
|
|
BACKUP_RESTORE_POLICY_VERSION,
|
|
BACKUP_RESTORE_POLICY_EFFECTIVE_AT,
|
|
BACKUP_RESTORE_EVENT_TYPE,
|
|
BACKUP_RESTORE_LANE,
|
|
"controlled_playbook_queue=readback_only",
|
|
"runtime_write_gate=0_until_verifier",
|
|
"backup_status,freshness,offsite_verify,escrow,restore_drill,source_resolution_receipt",
|
|
"no_backup_restore_delete_retention_escrow_write_or_secret_read",
|
|
]
|
|
)
|
|
BACKUP_RESTORE_POLICY_SOURCE_HASH = (
|
|
"sha256:"
|
|
+ hashlib.sha256(_BACKUP_RESTORE_POLICY_CANONICAL.encode("utf-8")).hexdigest()
|
|
)
|
|
|
|
_TOP_EVIDENCE_RE = re.compile(
|
|
r"<b>\s*Top evidence\s*</b>(?P<body>.*?)(?:<b>\s*(?:建議下一步|AI 自動處置佇列|禁止事項)\s*</b>|$)",
|
|
re.IGNORECASE | re.DOTALL,
|
|
)
|
|
_TAG_RE = re.compile(r"<[^>]+>")
|
|
_SPACE_RE = re.compile(r"\s+")
|
|
_SAFE_ID_RE = re.compile(r"[^a-zA-Z0-9_.:-]+")
|
|
|
|
_REQUIRED_EVIDENCE = (
|
|
"backup_status",
|
|
"freshness",
|
|
"offsite_verify",
|
|
"escrow",
|
|
"restore_drill",
|
|
"source_resolution_receipt",
|
|
)
|
|
_BACKUP_VERIFIER_EVIDENCE_REF_KEYS = {
|
|
"agent99_outcome_receipt_id",
|
|
"post_verifier_evidence_ref",
|
|
"source_event_evidence_ref",
|
|
"backup_status_evidence_ref",
|
|
"freshness_evidence_ref",
|
|
"offsite_verify_evidence_ref",
|
|
"escrow_evidence_ref",
|
|
"restore_drill_evidence_ref",
|
|
"source_resolution_receipt_ref",
|
|
}
|
|
|
|
_FIELD_ALIASES = {
|
|
"backup_status": (
|
|
"backup_status",
|
|
"backup-status",
|
|
"backup status",
|
|
"backupaggregaterun",
|
|
"backup aggregate",
|
|
"last_backup_success",
|
|
"last backup success",
|
|
),
|
|
"freshness": (
|
|
"freshness",
|
|
"backup_age",
|
|
"backup age",
|
|
"last_success_age",
|
|
"last success age",
|
|
),
|
|
"offsite_verify": (
|
|
"offsite_verify",
|
|
"offsite verify",
|
|
"offsite_status",
|
|
"offsite status",
|
|
"rclone_check",
|
|
),
|
|
"escrow": (
|
|
"escrow_missing",
|
|
"escrow missing",
|
|
"escrow_status",
|
|
"escrow status",
|
|
"credential_escrow",
|
|
),
|
|
"restore_drill": (
|
|
"restore_drill",
|
|
"restore drill",
|
|
"restore_test",
|
|
"restore test",
|
|
"restore_status",
|
|
"restore status",
|
|
),
|
|
"source_resolution_receipt": (
|
|
"source_resolution_receipt",
|
|
"source resolution receipt",
|
|
"alert_resolved_receipt",
|
|
"alert resolved receipt",
|
|
),
|
|
}
|
|
|
|
_BAD_TOKENS = (
|
|
"failed",
|
|
"failure",
|
|
"missing",
|
|
"blocked",
|
|
"stale",
|
|
"expired",
|
|
"timeout",
|
|
"degraded",
|
|
"unhealthy",
|
|
"false",
|
|
"error",
|
|
)
|
|
_NEGATIVE_PHRASES = (
|
|
"not ok",
|
|
"not verified",
|
|
"not healthy",
|
|
"not fresh",
|
|
"not current",
|
|
"not complete",
|
|
"not resolved",
|
|
"not passed",
|
|
"never verified",
|
|
"unverified",
|
|
)
|
|
_GOOD_TOKENS = (
|
|
"success",
|
|
"succeeded",
|
|
"healthy",
|
|
"verified",
|
|
"passed",
|
|
"fresh",
|
|
"current",
|
|
"complete",
|
|
"resolved",
|
|
"true",
|
|
"ok",
|
|
)
|
|
|
|
_PROHIBITED_ACTIONS = (
|
|
"run_backup",
|
|
"run_restore",
|
|
"restore_over_production",
|
|
"remote_delete",
|
|
"retention_change",
|
|
"restic_prune",
|
|
"escrow_marker_write",
|
|
"read_backup_contents",
|
|
"read_secret_values",
|
|
)
|
|
|
|
|
|
def backup_restore_readback_policy() -> dict[str, Any]:
|
|
"""Return the immutable source policy used for effective projections."""
|
|
|
|
return {
|
|
"schema_version": "backup_restore_readback_policy_v2",
|
|
"policy_version": BACKUP_RESTORE_POLICY_VERSION,
|
|
"effective_at": BACKUP_RESTORE_POLICY_EFFECTIVE_AT,
|
|
"source_ref": BACKUP_RESTORE_POLICY_SOURCE_REF,
|
|
"source_hash": BACKUP_RESTORE_POLICY_SOURCE_HASH,
|
|
"historical_projection_supported": True,
|
|
"send_time_contract_supported": True,
|
|
"runtime_write_requires_terminal_verifier": True,
|
|
"metadata_refs_are_learning_writeback": False,
|
|
"agent99_dispatch_is_runtime_closure": False,
|
|
}
|
|
|
|
|
|
def _normalized_agent99_dispatch_receipt(
|
|
receipt: Mapping[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
"""Validate the persisted dispatch receipt without treating it as outcome."""
|
|
|
|
if not isinstance(receipt, Mapping) or not receipt:
|
|
return {
|
|
"schema_version": "backup_restore_agent99_dispatch_readback_v1",
|
|
"status": "dispatch_receipt_missing",
|
|
"receipt_present": False,
|
|
"dispatched": False,
|
|
"accepted": False,
|
|
"inbox_triggered": False,
|
|
"queue_accepted": False,
|
|
"dispatch_identity_matched": False,
|
|
"backupcheck_route": False,
|
|
"receipt_persisted": False,
|
|
"identity_complete": False,
|
|
"identity_binding_valid": False,
|
|
"agent99_run_id": None,
|
|
"trace_id": None,
|
|
"agent99_work_item_id": None,
|
|
"alert_id": None,
|
|
"runtime_closure_verified": False,
|
|
"post_verifier_passed": False,
|
|
"post_verifier_failed": False,
|
|
"backup_evidence_refs_complete": False,
|
|
"missing_backup_evidence_refs": sorted(
|
|
_BACKUP_VERIFIER_EVIDENCE_REF_KEYS
|
|
),
|
|
"learning_writeback_complete": False,
|
|
"current_run_state": None,
|
|
"current_verifier_status": None,
|
|
"current_learning_writeback_status": None,
|
|
"next_action": "persist_agent99_dispatch_receipt_before_verifier",
|
|
}
|
|
|
|
source_schema = str(receipt.get("schema_version") or "")
|
|
accepted = receipt.get("accepted") is True
|
|
inbox_triggered = (
|
|
receipt.get("inbox_triggered") is True
|
|
or receipt.get("inboxTriggered") is True
|
|
)
|
|
queue_accepted = receipt.get("queue_accepted") is True
|
|
dispatch_identity_matched = (
|
|
receipt.get("dispatch_identity_matched") is True
|
|
)
|
|
kind = str(receipt.get("kind") or "")
|
|
suggested_mode = str(
|
|
receipt.get("suggested_mode") or receipt.get("suggestedMode") or ""
|
|
)
|
|
identity = receipt.get("identity")
|
|
if not isinstance(identity, Mapping):
|
|
identity = receipt.get("agent99_dispatch_identity")
|
|
if not isinstance(identity, Mapping):
|
|
identity = {}
|
|
agent99_run_id = str(
|
|
receipt.get("run_id") or identity.get("run_id") or ""
|
|
).strip()
|
|
trace_id = str(receipt.get("trace_id") or identity.get("trace_id") or "").strip()
|
|
agent99_work_item_id = str(
|
|
receipt.get("work_item_id") or identity.get("work_item_id") or ""
|
|
).strip()
|
|
legacy_schema = source_schema == "telegram_agent99_dispatch_receipt_v1"
|
|
projection_schema = (
|
|
source_schema == "telegram_agent99_dispatch_receipt_projection_v1"
|
|
)
|
|
valid_schema = legacy_schema or projection_schema
|
|
receipt_persisted = receipt.get("receipt_persisted") is True
|
|
identity_binding_valid = receipt.get("identity_binding_valid") is not False
|
|
identity_complete = bool(
|
|
agent99_run_id and trace_id and agent99_work_item_id
|
|
)
|
|
backupcheck_route = kind == "backup_health" and suggested_mode == "BackupCheck"
|
|
current_verifier = receipt.get("current_verifier")
|
|
if not isinstance(current_verifier, Mapping):
|
|
current_verifier = {}
|
|
current_learning = receipt.get("current_learning_writeback")
|
|
if not isinstance(current_learning, Mapping):
|
|
current_learning = {}
|
|
|
|
def same_stage_identity(stage: Mapping[str, Any]) -> bool:
|
|
return bool(
|
|
identity_complete
|
|
and str(stage.get("run_id") or "").strip() == agent99_run_id
|
|
and str(stage.get("trace_id") or "").strip() == trace_id
|
|
and str(stage.get("work_item_id") or "").strip()
|
|
== agent99_work_item_id
|
|
)
|
|
|
|
verifier_evidence_refs = current_verifier.get("evidence_refs")
|
|
if not isinstance(verifier_evidence_refs, Mapping):
|
|
verifier_evidence_refs = {}
|
|
missing_backup_evidence_refs = sorted(
|
|
key
|
|
for key in _BACKUP_VERIFIER_EVIDENCE_REF_KEYS
|
|
if not str(verifier_evidence_refs.get(key) or "").strip()
|
|
)
|
|
backup_evidence_refs_complete = not missing_backup_evidence_refs
|
|
post_verifier_passed = bool(
|
|
receipt.get("post_verifier_passed") is True
|
|
and current_verifier.get("schema_version")
|
|
== "agent99_independent_verifier_receipt_v1"
|
|
and current_verifier.get("status") == "success"
|
|
and current_verifier.get("verifier_passed") is True
|
|
and current_verifier.get("source_event_resolved") is True
|
|
and current_verifier.get("transport_ok") is True
|
|
and current_verifier.get("outcome_state") == "resolved"
|
|
and bool(str(current_verifier.get("verified_at") or "").strip())
|
|
and backup_evidence_refs_complete
|
|
and same_stage_identity(current_verifier)
|
|
)
|
|
post_verifier_failed = bool(
|
|
current_verifier.get("schema_version")
|
|
== "agent99_independent_verifier_receipt_v1"
|
|
and current_verifier.get("status") == "failed"
|
|
and same_stage_identity(current_verifier)
|
|
)
|
|
learning_refs = current_learning.get("receipt_refs")
|
|
if not isinstance(learning_refs, Mapping):
|
|
learning_refs = {}
|
|
learning_writeback_complete = bool(
|
|
current_learning.get("schema_version")
|
|
== "agent99_learning_writeback_receipt_v1"
|
|
and current_learning.get("status") == "success"
|
|
and same_stage_identity(current_learning)
|
|
and {
|
|
"incident_closure_receipt_id",
|
|
"telegram_lifecycle_receipt_id",
|
|
"km_writeback_ack_id",
|
|
"playbook_trust_writeback_ack_id",
|
|
"dr_scorecard_writeback_ack_id",
|
|
}.issubset(learning_refs)
|
|
)
|
|
runtime_closure_verified = bool(
|
|
receipt.get("runtime_closure_verified") is True
|
|
and receipt.get("closure_complete") is True
|
|
and str(receipt.get("current_run_state") or "") == "completed"
|
|
and post_verifier_passed
|
|
and learning_writeback_complete
|
|
)
|
|
dispatched = bool(
|
|
projection_schema
|
|
and receipt_persisted
|
|
and identity_complete
|
|
and identity_binding_valid
|
|
and accepted
|
|
and inbox_triggered
|
|
and queue_accepted
|
|
and dispatch_identity_matched
|
|
and backupcheck_route
|
|
)
|
|
if not valid_schema:
|
|
status = "dispatch_receipt_schema_invalid"
|
|
next_action = "repair_agent99_dispatch_receipt_schema"
|
|
elif legacy_schema:
|
|
# The old Telegram mirror dispatched independently and lacked the
|
|
# durable run/trace/work-item binding. Keep it as declared history,
|
|
# never as evidence that this candidate was dispatched.
|
|
status = "legacy_dispatch_receipt_unbound"
|
|
next_action = "project_durable_agent99_dispatch_receipt_with_identity"
|
|
elif not receipt_persisted:
|
|
status = "dispatch_receipt_not_persisted_fail_closed"
|
|
next_action = "persist_agent99_dispatch_receipt_before_verifier"
|
|
elif not identity_complete:
|
|
status = "dispatch_identity_incomplete_fail_closed"
|
|
next_action = "repair_agent99_run_trace_work_item_binding"
|
|
elif not identity_binding_valid:
|
|
status = "current_ledger_identity_mismatch_fail_closed"
|
|
next_action = "repair_agent99_run_trace_work_item_binding"
|
|
elif (
|
|
not accepted
|
|
or not inbox_triggered
|
|
or not queue_accepted
|
|
or not dispatch_identity_matched
|
|
):
|
|
status = "dispatch_queue_receipt_not_verified"
|
|
next_action = "retry_idempotent_agent99_backupcheck_dispatch"
|
|
elif not backupcheck_route:
|
|
status = "dispatch_route_mismatch_fail_closed"
|
|
next_action = "repair_backupcheck_route_before_dispatch_replay"
|
|
elif runtime_closure_verified:
|
|
status = "closed_verified_learning_written"
|
|
next_action = "monitor_backup_restore_recurrence"
|
|
elif post_verifier_passed:
|
|
status = "verifier_passed_learning_writeback_pending"
|
|
next_action = "complete_km_playbook_and_lifecycle_writeback"
|
|
elif post_verifier_failed:
|
|
status = "verifier_failed_no_runtime_closure"
|
|
next_action = "collect_failed_verifier_evidence_then_retry_bounded_readback"
|
|
else:
|
|
status = "accepted_queue_persisted"
|
|
next_action = "ingest_agent99_backupcheck_outcome_and_independent_dr_verifier"
|
|
|
|
return {
|
|
"schema_version": "backup_restore_agent99_dispatch_readback_v1",
|
|
"source_schema_version": source_schema,
|
|
"status": status,
|
|
"receipt_present": True,
|
|
"dispatched": dispatched,
|
|
"accepted": accepted,
|
|
"inbox_triggered": inbox_triggered,
|
|
"queue_accepted": queue_accepted,
|
|
"dispatch_identity_matched": dispatch_identity_matched,
|
|
"backupcheck_route": backupcheck_route,
|
|
"receipt_persisted": receipt_persisted,
|
|
"identity_complete": identity_complete,
|
|
"identity_binding_valid": identity_binding_valid,
|
|
"kind": kind,
|
|
"suggested_mode": suggested_mode,
|
|
"transport": str(receipt.get("transport") or ""),
|
|
"alert_id": str(receipt.get("alert_id") or "") or None,
|
|
"agent99_run_id": agent99_run_id or None,
|
|
"trace_id": trace_id or None,
|
|
"agent99_work_item_id": agent99_work_item_id or None,
|
|
"stores_raw_response": False,
|
|
"runtime_closure_verified": bool(
|
|
runtime_closure_verified
|
|
),
|
|
"post_verifier_passed": post_verifier_passed,
|
|
"post_verifier_failed": post_verifier_failed,
|
|
"backup_evidence_refs_complete": backup_evidence_refs_complete,
|
|
"missing_backup_evidence_refs": missing_backup_evidence_refs,
|
|
"learning_writeback_complete": learning_writeback_complete,
|
|
"current_run_state": str(receipt.get("current_run_state") or "") or None,
|
|
"current_verifier_status": str(current_verifier.get("status") or "") or None,
|
|
"current_learning_writeback_status": (
|
|
str(current_learning.get("status") or "") or None
|
|
),
|
|
"next_action": next_action,
|
|
}
|
|
|
|
|
|
def _safe_id(value: object, *, fallback: str) -> str:
|
|
cleaned = _SAFE_ID_RE.sub("-", str(value or "")).strip("-._:")
|
|
return (cleaned or fallback)[:96]
|
|
|
|
|
|
def _stable_digest(*values: object) -> str:
|
|
material = "|".join(str(value or "").strip().lower() for value in values)
|
|
return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
|
|
|
|
|
|
def _evidence_lines(text: str) -> list[str]:
|
|
"""Return only the normalized card's public-safe Top evidence section."""
|
|
|
|
match = _TOP_EVIDENCE_RE.search(text or "")
|
|
body = match.group("body") if match else text or ""
|
|
lines: list[str] = []
|
|
for raw_line in body.splitlines():
|
|
plain = html.unescape(_TAG_RE.sub(" ", raw_line))
|
|
plain = _SPACE_RE.sub(" ", plain).strip(" ├└\t")
|
|
if plain:
|
|
lines.append(plain[:240])
|
|
if len(lines) >= 8:
|
|
break
|
|
return lines
|
|
|
|
|
|
def _numeric_assignment(text: str, aliases: Sequence[str]) -> float | None:
|
|
for alias in aliases:
|
|
match = re.search(
|
|
rf"{re.escape(alias)}\s*[:=]\s*(?P<value>-?\d+(?:\.\d+)?)",
|
|
text,
|
|
re.IGNORECASE,
|
|
)
|
|
if match:
|
|
try:
|
|
return float(match.group("value"))
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _field_observation(field: str, lines: Sequence[str]) -> dict[str, Any]:
|
|
aliases = _FIELD_ALIASES[field]
|
|
matching = [line for line in lines if any(alias in line.lower() for alias in aliases)]
|
|
if not matching:
|
|
return {
|
|
"field": field,
|
|
"status": "missing",
|
|
"observed": False,
|
|
"safe_value": None,
|
|
}
|
|
|
|
joined = " ".join(matching).lower()
|
|
number = _numeric_assignment(joined, aliases)
|
|
|
|
# Explicit negation must win over both a numeric token and positive
|
|
# substrings (for example, "not verified" contains "verified").
|
|
if any(phrase in joined for phrase in _NEGATIVE_PHRASES):
|
|
status = "failed"
|
|
safe_value: object = "explicit_negative_result"
|
|
elif field == "escrow" and number is not None:
|
|
status = "healthy" if number == 0 else "failed"
|
|
safe_value = int(number) if number.is_integer() else number
|
|
elif field in {"backup_status", "offsite_verify", "restore_drill"} and number is not None:
|
|
status = "healthy" if number > 0 else "failed"
|
|
safe_value = int(number) if number.is_integer() else number
|
|
elif any(token in joined for token in _BAD_TOKENS):
|
|
status = "failed"
|
|
safe_value = "explicit_failure_or_missing"
|
|
elif any(token in joined for token in _GOOD_TOKENS):
|
|
status = "healthy"
|
|
safe_value = "explicit_success"
|
|
else:
|
|
status = "unverified"
|
|
safe_value = "observed_without_verifiable_result"
|
|
|
|
return {
|
|
"field": field,
|
|
"status": status,
|
|
"observed": True,
|
|
"safe_value": safe_value,
|
|
}
|
|
|
|
|
|
def extract_backup_restore_evidence(text: str) -> list[dict[str, Any]]:
|
|
"""Extract only allowlisted, non-secret backup readback facts."""
|
|
|
|
lines = _evidence_lines(text)
|
|
return [_field_observation(field, lines) for field in _REQUIRED_EVIDENCE]
|
|
|
|
|
|
def _normalize_observations(
|
|
observations: Sequence[Mapping[str, Any]] | None,
|
|
*,
|
|
evidence_text: str,
|
|
) -> list[dict[str, Any]]:
|
|
if not observations:
|
|
return extract_backup_restore_evidence(evidence_text)
|
|
|
|
by_field = {
|
|
str(item.get("field") or ""): dict(item)
|
|
for item in observations
|
|
if isinstance(item, Mapping)
|
|
}
|
|
normalized: list[dict[str, Any]] = []
|
|
for field in _REQUIRED_EVIDENCE:
|
|
item = by_field.get(field) or {}
|
|
status = str(item.get("status") or "missing")
|
|
if status not in {"healthy", "failed", "unverified", "missing"}:
|
|
status = "unverified"
|
|
normalized.append({
|
|
"field": field,
|
|
"status": status,
|
|
"observed": bool(item.get("observed")) if item else False,
|
|
"safe_value": item.get("safe_value") if item else None,
|
|
})
|
|
return normalized
|
|
|
|
|
|
def build_backup_restore_signal_automation_contract(
|
|
*,
|
|
project_id: str,
|
|
event_type: str = BACKUP_RESTORE_EVENT_TYPE,
|
|
lane: str = BACKUP_RESTORE_LANE,
|
|
target: str = "backup_restore",
|
|
evidence_text: str = "",
|
|
evidence_observations: Sequence[Mapping[str, Any]] | None = None,
|
|
source_refs: Mapping[str, Any] | None = None,
|
|
receipt_id: object = "",
|
|
run_id: object = "",
|
|
source_row_exists: bool = False,
|
|
contract_origin: str = "generated_for_send",
|
|
contract_persisted_at_send: bool = False,
|
|
recorded_contract_origin: str = "",
|
|
recorded_policy: Mapping[str, Any] | None = None,
|
|
recorded_immutable_declared: Mapping[str, Any] | None = None,
|
|
agent99_dispatch_receipt: Mapping[str, Any] | None = None,
|
|
declared_gates: Sequence[str] | None = None,
|
|
declared_runtime_write_gate_count: int = 0,
|
|
declared_runtime_write_gate_state: str = "unknown",
|
|
declared_controlled_playbook_queue: bool = False,
|
|
declared_candidate_only: bool = False,
|
|
declared_learning_writeback_status: str = "not_evaluated",
|
|
) -> dict[str, Any]:
|
|
"""Build a deterministic, no-write backup readback work package."""
|
|
|
|
if contract_origin not in {
|
|
"generated_for_send",
|
|
"persisted_at_send",
|
|
"synthesized_at_read",
|
|
}:
|
|
contract_origin = "synthesized_at_read"
|
|
synthesized_at_read = contract_origin == "synthesized_at_read"
|
|
policy = backup_restore_readback_policy()
|
|
refs = source_refs or {}
|
|
fingerprints = refs.get("fingerprints")
|
|
source_fingerprint = (
|
|
str(fingerprints[0])
|
|
if isinstance(fingerprints, list) and fingerprints
|
|
else f"{event_type}:{lane}:{target}"
|
|
)
|
|
cluster_digest = _stable_digest(project_id, event_type, lane, target, source_fingerprint)
|
|
receipt_token = _safe_id(receipt_id, fallback=f"pending-{cluster_digest}")
|
|
run_token = _safe_id(run_id, fallback=f"unbound-{cluster_digest}")
|
|
work_item_id = f"BRR-WI-{cluster_digest.upper()}"
|
|
candidate_id = f"BRR-CAND-{_stable_digest(work_item_id, receipt_token).upper()}"
|
|
durable_receipt_id = f"BRR-RCPT-{_stable_digest(project_id, receipt_token).upper()}"
|
|
verifier_id = f"BRR-VER-{_stable_digest(work_item_id, run_token).upper()}"
|
|
dispatch = _normalized_agent99_dispatch_receipt(agent99_dispatch_receipt)
|
|
if dispatch["runtime_closure_verified"]:
|
|
identity_binding_status = "bound_closed_verified_learning_written"
|
|
elif dispatch["post_verifier_passed"]:
|
|
identity_binding_status = "bound_verifier_passed_learning_pending"
|
|
elif dispatch["post_verifier_failed"]:
|
|
identity_binding_status = "bound_verifier_failed_retry_required"
|
|
elif dispatch["dispatched"]:
|
|
identity_binding_status = "bound_dispatch_verifier_pending"
|
|
elif dispatch["receipt_present"]:
|
|
identity_binding_status = "binding_incomplete_fail_closed"
|
|
else:
|
|
identity_binding_status = "dispatch_receipt_missing"
|
|
identity_binding = {
|
|
"schema_version": "backup_restore_dispatch_identity_binding_v1",
|
|
"status": identity_binding_status,
|
|
"complete": bool(dispatch["dispatched"]),
|
|
"source_message_receipt_id": str(receipt_id or "") or None,
|
|
"source_message_run_id": str(run_id or "") or None,
|
|
"brr_work_item_id": work_item_id,
|
|
"brr_candidate_id": candidate_id,
|
|
"brr_verifier_id": verifier_id,
|
|
"agent99_run_id": dispatch.get("agent99_run_id"),
|
|
"agent99_trace_id": dispatch.get("trace_id"),
|
|
"agent99_work_item_id": dispatch.get("agent99_work_item_id"),
|
|
"same_agent99_identity_required_for_outcome": True,
|
|
"runtime_closure_verified": bool(
|
|
dispatch["runtime_closure_verified"]
|
|
),
|
|
}
|
|
|
|
checks = _normalize_observations(
|
|
evidence_observations,
|
|
evidence_text=evidence_text,
|
|
)
|
|
failed_fields = [item["field"] for item in checks if item["status"] == "failed"]
|
|
missing_fields = [
|
|
item["field"]
|
|
for item in checks
|
|
if item["status"] in {"missing", "unverified"}
|
|
]
|
|
if failed_fields:
|
|
evidence_result = "failed"
|
|
domain_work_item_status = "open_evidence_failed"
|
|
evidence_next_action = (
|
|
"collect_failed_domain_readback_then_rerun_no_false_green_verifier"
|
|
)
|
|
elif missing_fields:
|
|
evidence_result = "degraded"
|
|
domain_work_item_status = "open_evidence_incomplete"
|
|
evidence_next_action = (
|
|
"collect_missing_backup_readback_then_rerun_no_false_green_verifier"
|
|
)
|
|
else:
|
|
# A firing signal cannot be closed only because all point-in-time checks
|
|
# look green. The source-resolution receipt remains mandatory.
|
|
evidence_result = "degraded"
|
|
domain_work_item_status = "open_source_resolution_receipt_required"
|
|
evidence_next_action = (
|
|
"persist_source_resolution_receipt_before_incident_closure"
|
|
)
|
|
|
|
if dispatch["runtime_closure_verified"]:
|
|
domain_work_item_status = "closed_verified_learning_written"
|
|
candidate_domain_status = "completed_verified_readback"
|
|
verifier_domain_status = "success_terminal"
|
|
next_action = "monitor_backup_restore_recurrence"
|
|
elif dispatch["post_verifier_passed"]:
|
|
candidate_domain_status = "verifier_passed_learning_writeback_pending"
|
|
verifier_domain_status = "success_learning_writeback_pending"
|
|
next_action = "complete_km_playbook_and_lifecycle_writeback"
|
|
elif dispatch["post_verifier_failed"]:
|
|
domain_work_item_status = "open_verifier_failed"
|
|
candidate_domain_status = "verifier_failed_retry_required"
|
|
verifier_domain_status = "failed_terminal"
|
|
next_action = (
|
|
"collect_failed_verifier_evidence_then_retry_bounded_readback"
|
|
)
|
|
elif dispatch["dispatched"]:
|
|
candidate_domain_status = "dispatched_verifier_pending"
|
|
verifier_domain_status = "dispatched_verifier_pending"
|
|
next_action = (
|
|
"ingest_agent99_backupcheck_outcome_then_run_independent_dr_verifier"
|
|
)
|
|
elif dispatch["receipt_present"]:
|
|
candidate_domain_status = "dispatch_blocked_fail_closed"
|
|
verifier_domain_status = "dispatch_blocked_verifier_not_started"
|
|
next_action = str(dispatch["next_action"])
|
|
elif source_row_exists:
|
|
candidate_domain_status = "source_row_observed_dispatch_receipt_missing"
|
|
verifier_domain_status = "dispatch_receipt_missing_verifier_not_started"
|
|
next_action = "persist_agent99_dispatch_receipt_before_verifier"
|
|
else:
|
|
candidate_domain_status = "pending_source_message_persistence"
|
|
verifier_domain_status = "source_message_pending_verifier_not_started"
|
|
next_action = "persist_source_message_then_dispatch_backupcheck_readonly"
|
|
|
|
def asset_status(value: str) -> str:
|
|
return f"projected_{value}" if synthesized_at_read else value
|
|
|
|
work_item_status = asset_status(domain_work_item_status)
|
|
candidate_status = asset_status(candidate_domain_status)
|
|
verifier_status = asset_status(verifier_domain_status)
|
|
if synthesized_at_read and source_row_exists:
|
|
source_receipt_status = "projected_from_historical_source_row"
|
|
elif synthesized_at_read:
|
|
source_receipt_status = "projected_source_row_missing"
|
|
elif not source_row_exists:
|
|
source_receipt_status = "pending_source_message_persistence"
|
|
elif contract_persisted_at_send:
|
|
source_receipt_status = "source_row_observed_contract_persisted_at_send"
|
|
else:
|
|
source_receipt_status = "source_row_observed_contract_projected_at_read"
|
|
|
|
if dispatch["runtime_closure_verified"]:
|
|
outcome_status = "terminal_verified_learning_written"
|
|
outcome_next_action = "monitor_backup_restore_recurrence"
|
|
elif dispatch["post_verifier_passed"]:
|
|
outcome_status = "verifier_passed_learning_writeback_pending"
|
|
outcome_next_action = "complete_km_playbook_and_lifecycle_writeback"
|
|
elif dispatch["post_verifier_failed"]:
|
|
outcome_status = "verifier_failed_no_runtime_closure"
|
|
outcome_next_action = (
|
|
"collect_failed_verifier_evidence_then_retry_bounded_readback"
|
|
)
|
|
elif dispatch["dispatched"]:
|
|
outcome_status = "dispatch_accepted_outcome_pending"
|
|
outcome_next_action = (
|
|
"persist_agent99_outcome_contract_v1_with_dispatch_identity_then_"
|
|
"run_independent_backup_dr_evidence_verifier"
|
|
)
|
|
else:
|
|
outcome_status = "outcome_source_unavailable"
|
|
outcome_next_action = str(dispatch["next_action"])
|
|
|
|
outcome_readback = {
|
|
"schema_version": "backup_restore_agent99_outcome_readback_v1",
|
|
"required_source_schema": "agent99_outcome_contract_v1",
|
|
"status": outcome_status,
|
|
"terminal": bool(
|
|
dispatch["runtime_closure_verified"]
|
|
or dispatch["post_verifier_failed"]
|
|
),
|
|
"same_identity_required": True,
|
|
"agent99_run_id": dispatch.get("agent99_run_id"),
|
|
"agent99_trace_id": dispatch.get("trace_id"),
|
|
"agent99_work_item_id": dispatch.get("agent99_work_item_id"),
|
|
"brr_work_item_id": work_item_id,
|
|
"brr_verifier_id": verifier_id,
|
|
"dispatch_alert_id": dispatch.get("alert_id"),
|
|
"backup_evidence_refs_complete": dispatch.get(
|
|
"backup_evidence_refs_complete"
|
|
),
|
|
"missing_backup_evidence_refs": dispatch.get(
|
|
"missing_backup_evidence_refs"
|
|
),
|
|
"required_mode": "BackupCheck",
|
|
"required_fields": [
|
|
"outcome.state",
|
|
"outcome.resolved",
|
|
"transportOk",
|
|
"verifierName",
|
|
"verifierPassed",
|
|
"evidenceFresh",
|
|
"evidenceModeMatched",
|
|
"sourceEventResolutionRequired",
|
|
"sourceEventResolutionPolicy",
|
|
"sourceEventResolved",
|
|
"sourceEventResolutionEvidence",
|
|
"failedChecks",
|
|
"checks",
|
|
"verifiedAt",
|
|
],
|
|
"next_action": outcome_next_action,
|
|
}
|
|
|
|
immutable_declared = {
|
|
"gates": list(declared_gates or []),
|
|
"runtime_write_gate_count": int(declared_runtime_write_gate_count or 0),
|
|
"runtime_write_gate_state": str(
|
|
declared_runtime_write_gate_state or "unknown"
|
|
),
|
|
"runtime_write_allowed": bool(declared_runtime_write_gate_count),
|
|
"controlled_playbook_queue": bool(declared_controlled_playbook_queue),
|
|
"candidate_only": bool(declared_candidate_only),
|
|
"learning_writeback_status": str(
|
|
declared_learning_writeback_status or "not_evaluated"
|
|
),
|
|
}
|
|
if recorded_policy is None and contract_persisted_at_send:
|
|
recorded_policy_snapshot = dict(policy)
|
|
else:
|
|
policy_source = recorded_policy or {}
|
|
recorded_policy_snapshot = {
|
|
key: policy_source[key]
|
|
for key in (
|
|
"schema_version",
|
|
"policy_version",
|
|
"effective_at",
|
|
"source_ref",
|
|
"source_hash",
|
|
"historical_projection_only",
|
|
"runtime_write_requires_terminal_verifier",
|
|
"metadata_refs_are_learning_writeback",
|
|
"agent99_dispatch_is_runtime_closure",
|
|
)
|
|
if key in policy_source
|
|
}
|
|
if recorded_immutable_declared is None and contract_persisted_at_send:
|
|
recorded_immutable_snapshot = dict(immutable_declared)
|
|
else:
|
|
immutable_source = recorded_immutable_declared or {}
|
|
recorded_immutable_snapshot = {
|
|
key: immutable_source[key]
|
|
for key in (
|
|
"gates",
|
|
"runtime_write_gate_count",
|
|
"runtime_write_gate_state",
|
|
"runtime_write_allowed",
|
|
"controlled_playbook_queue",
|
|
"candidate_only",
|
|
"learning_writeback_status",
|
|
)
|
|
if key in immutable_source
|
|
}
|
|
contract_provenance = {
|
|
"schema_version": "backup_restore_contract_provenance_v1",
|
|
"source_contract_present_at_send": bool(contract_persisted_at_send),
|
|
"recorded_contract_origin": (
|
|
str(recorded_contract_origin or "").strip()
|
|
or (
|
|
contract_origin
|
|
if contract_persisted_at_send
|
|
else "source_contract_missing"
|
|
)
|
|
),
|
|
"recorded_policy_present": bool(recorded_policy_snapshot),
|
|
"recorded_policy_effective_at": recorded_policy_snapshot.get(
|
|
"effective_at"
|
|
),
|
|
"recorded_policy_source_hash": recorded_policy_snapshot.get(
|
|
"source_hash"
|
|
),
|
|
"recorded_immutable_declared_present": bool(
|
|
recorded_immutable_snapshot
|
|
),
|
|
"recorded_immutable_declared": recorded_immutable_snapshot,
|
|
"effective_projection_policy_effective_at": policy["effective_at"],
|
|
"effective_projection_policy_source_hash": policy["source_hash"],
|
|
"synthesized_at_read": synthesized_at_read,
|
|
}
|
|
effective_policy_projection = {
|
|
"schema_version": "backup_restore_effective_policy_projection_v1",
|
|
"policy": policy,
|
|
"contract_origin": contract_origin,
|
|
"effective_gates": [
|
|
"controlled_playbook_queue=readback_only",
|
|
"runtime_write_gate=0_until_verifier",
|
|
],
|
|
"runtime_write_gate_count": 0,
|
|
"runtime_write_allowed": False,
|
|
"controlled_playbook_queue": False,
|
|
"candidate_only": True,
|
|
"learning_writeback_ready": bool(
|
|
dispatch["learning_writeback_complete"]
|
|
),
|
|
"learning_writeback_status": (
|
|
"closed_verified_learning_written"
|
|
if dispatch["runtime_closure_verified"]
|
|
else "verifier_passed_learning_writeback_pending"
|
|
if dispatch["post_verifier_passed"]
|
|
else "blocked_backup_restore_outcome_and_independent_verifier"
|
|
),
|
|
"candidate_status": candidate_status,
|
|
"verifier_status": verifier_status,
|
|
"production_write_performed": False,
|
|
}
|
|
|
|
asset_ledger = [
|
|
{
|
|
"asset_type": "WorkItem",
|
|
"asset_id": work_item_id,
|
|
"owner": "backup_dr",
|
|
"status": work_item_status,
|
|
"domain_status": domain_work_item_status,
|
|
"contract_origin": contract_origin,
|
|
"next_action": next_action,
|
|
},
|
|
{
|
|
"asset_type": "ReadbackCandidate",
|
|
"asset_id": candidate_id,
|
|
"owner": "Agent99",
|
|
"status": candidate_status,
|
|
"domain_status": candidate_domain_status,
|
|
"contract_origin": contract_origin,
|
|
"next_action": next_action,
|
|
},
|
|
{
|
|
"asset_type": "DurableReceipt",
|
|
"asset_id": durable_receipt_id,
|
|
"owner": "AwoooP",
|
|
"status": source_receipt_status,
|
|
"contract_origin": contract_origin,
|
|
"next_action": next_action,
|
|
},
|
|
{
|
|
"asset_type": "Verifier",
|
|
"asset_id": verifier_id,
|
|
"owner": "PostExecutionVerifier",
|
|
"status": verifier_status,
|
|
"domain_status": verifier_domain_status,
|
|
"contract_origin": contract_origin,
|
|
"next_action": next_action,
|
|
},
|
|
{
|
|
"asset_type": "Agent99DispatchBinding",
|
|
"asset_id": (
|
|
str(dispatch.get("agent99_run_id") or "")
|
|
or f"AGENT99-BIND-{cluster_digest.upper()}"
|
|
),
|
|
"owner": "Agent99",
|
|
"status": asset_status(identity_binding_status),
|
|
"contract_origin": contract_origin,
|
|
"next_action": next_action,
|
|
},
|
|
{
|
|
"asset_type": "DRScorecard",
|
|
"asset_id": f"DR-SCORECARD-{cluster_digest.upper()}",
|
|
"owner": "backup_dr",
|
|
"status": asset_status(
|
|
"closed_verified_learning_written"
|
|
if dispatch["runtime_closure_verified"]
|
|
else "writeback_blocked_until_verifier_terminal"
|
|
),
|
|
"contract_origin": contract_origin,
|
|
"next_action": next_action,
|
|
},
|
|
]
|
|
|
|
return {
|
|
"schema_version": "backup_restore_readback_automation_v1",
|
|
"event_type": event_type,
|
|
"lane": lane,
|
|
"target": target,
|
|
"source_fingerprint": source_fingerprint,
|
|
"contract_origin": contract_origin,
|
|
"contract_persisted_at_send": bool(contract_persisted_at_send),
|
|
"policy": policy,
|
|
"contract_provenance": contract_provenance,
|
|
"immutable_declared": immutable_declared,
|
|
"effective_policy_projection": effective_policy_projection,
|
|
"work_item": {
|
|
"work_item_id": work_item_id,
|
|
"kind": "backup_restore_readback",
|
|
"owner": "backup_dr",
|
|
"status": work_item_status,
|
|
"domain_status": domain_work_item_status,
|
|
"contract_origin": contract_origin,
|
|
"next_action": next_action,
|
|
},
|
|
"candidate": {
|
|
"candidate_id": candidate_id,
|
|
"action": "collect_readonly_backup_restore_readback",
|
|
"mode": "BackupCheck",
|
|
"risk": "low",
|
|
"status": candidate_status,
|
|
"domain_status": candidate_domain_status,
|
|
"contract_origin": contract_origin,
|
|
"execution_state": (
|
|
"completed_verified"
|
|
if dispatch["runtime_closure_verified"]
|
|
else "dispatched"
|
|
if dispatch["dispatched"]
|
|
else "not_started"
|
|
),
|
|
"work_item_id": work_item_id,
|
|
"source_message_run_id": str(run_id or "") or None,
|
|
"agent99_dispatch": dispatch,
|
|
"identity_binding": identity_binding,
|
|
"allowed_operations": [
|
|
"read_backup_status_metadata",
|
|
"read_freshness_metrics",
|
|
"read_offsite_verifier_receipt",
|
|
"read_escrow_missing_count",
|
|
"read_restore_drill_receipt",
|
|
],
|
|
},
|
|
"receipt": {
|
|
"receipt_id": durable_receipt_id,
|
|
"source": "awooop_outbound_message",
|
|
"source_receipt_id": str(receipt_id or ""),
|
|
"source_message_run_id": str(run_id or ""),
|
|
"status": source_receipt_status,
|
|
"source_row_exists": bool(source_row_exists),
|
|
"contract_origin": contract_origin,
|
|
"contract_persisted_at_send": bool(contract_persisted_at_send),
|
|
"is_agent99_execution_receipt": False,
|
|
},
|
|
"verifier": {
|
|
"verifier_id": verifier_id,
|
|
"status": verifier_status,
|
|
"domain_status": verifier_domain_status,
|
|
"evidence_result": evidence_result,
|
|
"result": (
|
|
"success"
|
|
if dispatch["runtime_closure_verified"]
|
|
else "failed"
|
|
if dispatch["post_verifier_failed"]
|
|
else "degraded"
|
|
if dispatch["post_verifier_passed"]
|
|
else evidence_result
|
|
),
|
|
"terminal": bool(
|
|
dispatch["runtime_closure_verified"]
|
|
or dispatch["post_verifier_failed"]
|
|
),
|
|
"no_false_green": True,
|
|
"checks": checks,
|
|
"failed_fields": failed_fields,
|
|
"missing_fields": missing_fields,
|
|
"required_fields": list(_REQUIRED_EVIDENCE),
|
|
"agent99_dispatch": dispatch,
|
|
"identity_binding": identity_binding,
|
|
"agent99_outcome": outcome_readback,
|
|
"backup_evidence_refs_complete": dispatch.get(
|
|
"backup_evidence_refs_complete"
|
|
),
|
|
"missing_backup_evidence_refs": dispatch.get(
|
|
"missing_backup_evidence_refs"
|
|
),
|
|
"evidence_next_action": evidence_next_action,
|
|
},
|
|
"asset_ledger": asset_ledger,
|
|
"declared_gates": list(declared_gates or []),
|
|
"effective_gates": list(effective_policy_projection["effective_gates"]),
|
|
"prohibited_actions": list(_PROHIBITED_ACTIONS),
|
|
"controlled_repair_queue": False,
|
|
"read_only_candidate_allowed": True,
|
|
"runtime_write_allowed": False,
|
|
"runtime_execution_authorized": False,
|
|
"production_write_performed": False,
|
|
"human_intervention_required": False,
|
|
"closure_state": (
|
|
"closed_verified_learning_written"
|
|
if dispatch["runtime_closure_verified"]
|
|
else "partial_degraded_safe_next_action"
|
|
),
|
|
"next_action": next_action,
|
|
}
|