820 lines
30 KiB
Python
820 lines
30 KiB
Python
"""Single-owner Alertmanager ingress for read-only BackupCheck automation.
|
|
|
|
The raw Alertmanager signal owns dispatch. Telegram is only a projection of
|
|
the durable Agent99 receipt. This lane may collect backup/restore evidence,
|
|
but it must never run backup/restore, delete remote data, change retention,
|
|
write escrow markers, or read secrets.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from typing import Any
|
|
from uuid import NAMESPACE_URL, UUID, uuid5
|
|
|
|
from src.core.logging import get_logger
|
|
from src.models.approval import (
|
|
ApprovalRequestCreate,
|
|
BlastRadius,
|
|
DataImpact,
|
|
DryRunCheck,
|
|
RiskLevel,
|
|
)
|
|
from src.services.agent99_controlled_dispatch_ledger import (
|
|
read_agent99_dispatch_receipt,
|
|
)
|
|
from src.services.agent99_sre_bridge import (
|
|
bridge_alertmanager_to_agent99,
|
|
resolve_agent99_durable_route,
|
|
)
|
|
from src.services.approval_db import get_approval_service
|
|
from src.services.channel_hub import record_alertmanager_event
|
|
from src.services.incident_service import (
|
|
create_incident_for_approval,
|
|
get_incident_service,
|
|
)
|
|
|
|
logger = get_logger("awoooi.backup_restore_alertmanager_ingress")
|
|
|
|
BACKUP_RESTORE_AGENT99_ROUTE_ID = "agent99:backup_health:BackupCheck"
|
|
BACKUP_RESTORE_DISPATCH_OWNER = "alertmanager_raw_signal"
|
|
BACKUP_RESTORE_LEGACY_OWNER_SCHEMA_VERSION = "backup_restore_ingress_owner_v1"
|
|
BACKUP_RESTORE_OWNER_SCHEMA_VERSION = "backup_restore_ingress_owner_v2"
|
|
BACKUP_RESTORE_OWNER_KEY_ENCODING = "canonical_json_v1"
|
|
BACKUP_RESTORE_LEGACY_BACKFILL_PREFIX = "legacy-backup-occurrence:"
|
|
APPROVAL_FINGERPRINT_MAX_LENGTH = 64
|
|
APPROVAL_FINGERPRINT_HASH_DOMAIN = "awoooi.backup_restore.approval_fingerprint.v2"
|
|
APPROVAL_OWNER_FINGERPRINT_HASH_DOMAIN = (
|
|
"awoooi.backup_restore.approval_owner_fingerprint.v3"
|
|
)
|
|
BACKUP_RESTORE_PROHIBITED_ACTIONS = (
|
|
"run_backup",
|
|
"run_restore",
|
|
"remote_delete",
|
|
"prune",
|
|
"change_retention",
|
|
"write_escrow_marker",
|
|
"read_secret",
|
|
)
|
|
|
|
|
|
def _stable_digest(*parts: object) -> str:
|
|
value = "\x1f".join(str(part or "").strip() for part in parts)
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _canonical_json(value: Any) -> str:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
|
|
|
|
def _canonical_source_occurrence(
|
|
*,
|
|
source_fingerprint: str,
|
|
source_event_fingerprint: str,
|
|
source_started_at: str,
|
|
) -> dict[str, str]:
|
|
event_fingerprint = str(source_event_fingerprint or "").strip()
|
|
started_at = str(source_started_at or "").strip()
|
|
return {
|
|
"source_event_fingerprint": event_fingerprint,
|
|
"source_started_at": started_at,
|
|
"fallback_source_fingerprint": (
|
|
""
|
|
if event_fingerprint or started_at
|
|
else str(source_fingerprint or "").strip()
|
|
),
|
|
}
|
|
|
|
|
|
def _legacy_backup_restore_ingress_owner(
|
|
*,
|
|
project_id: str,
|
|
source_fingerprint: str,
|
|
source_event_fingerprint: str,
|
|
source_started_at: str,
|
|
) -> dict[str, str]:
|
|
"""Rebuild the exact v1 IDs solely for bounded compatibility lookup."""
|
|
|
|
source_occurrence = (
|
|
"|".join(
|
|
value for value in (source_event_fingerprint, source_started_at) if value
|
|
)
|
|
or source_fingerprint
|
|
)
|
|
owner_key = (
|
|
f"{BACKUP_RESTORE_LEGACY_OWNER_SCHEMA_VERSION}:{project_id}:"
|
|
f"{source_fingerprint}:{source_occurrence}"
|
|
)
|
|
digest = _stable_digest(owner_key)
|
|
return {
|
|
"schema_version": BACKUP_RESTORE_LEGACY_OWNER_SCHEMA_VERSION,
|
|
"project_id": project_id,
|
|
"source_fingerprint": source_fingerprint,
|
|
"source_occurrence": source_occurrence,
|
|
"source_occurrence_hash": digest,
|
|
"approval_id": str(uuid5(NAMESPACE_URL, owner_key)),
|
|
"incident_id": f"INC-BRR-{digest[:16].upper()}",
|
|
"work_item_id": f"backupcheck:{project_id}:{digest[:32]}",
|
|
}
|
|
|
|
|
|
def _approval_storage_fingerprint(source_fingerprint: str) -> str:
|
|
"""Return the transitional v2 source-only storage fingerprint.
|
|
|
|
This representation remains a lookup alias for rows written by the first
|
|
bounded-fingerprint rollout. New rows use a v3 fingerprint over the full
|
|
canonical owner identity instead of only the source fingerprint.
|
|
"""
|
|
|
|
normalized = str(source_fingerprint or "").strip()
|
|
payload = f"{APPROVAL_FINGERPRINT_HASH_DOMAIN}\x1f{normalized}"
|
|
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _approval_owner_storage_fingerprint(owner: dict[str, Any]) -> str:
|
|
identity = owner.get("owner_identity")
|
|
if not isinstance(identity, dict):
|
|
raise ValueError("backup_restore_canonical_owner_identity_required")
|
|
payload = f"{APPROVAL_OWNER_FINGERPRINT_HASH_DOMAIN}\x1f{_canonical_json(identity)}"
|
|
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _approval_fingerprint_lookup_candidates(
|
|
owner: dict[str, Any],
|
|
) -> tuple[str, ...]:
|
|
"""Return full-owner v3, source-only v2, then the v1 source encoding.
|
|
|
|
Every source-only candidate is only an index probe. A row is reusable
|
|
solely after its complete deterministic owner metadata also matches.
|
|
"""
|
|
|
|
normalized = str(owner.get("source_fingerprint") or "").strip()
|
|
canonical = _approval_owner_storage_fingerprint(owner)
|
|
source_only_v2 = _approval_storage_fingerprint(normalized)
|
|
legacy = (
|
|
normalized
|
|
if len(normalized) <= APPROVAL_FINGERPRINT_MAX_LENGTH
|
|
else hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
|
)
|
|
return tuple(dict.fromkeys((canonical, source_only_v2, legacy)))
|
|
|
|
|
|
def _legacy_owner_reuse_is_unambiguous(owner: dict[str, Any]) -> bool:
|
|
"""Reject v1 occurrence shapes that cannot be losslessly decoded."""
|
|
|
|
event_fingerprint = str(owner.get("source_event_fingerprint") or "")
|
|
started_at = str(owner.get("source_started_at") or "")
|
|
if "|" in event_fingerprint or "|" in started_at:
|
|
return False
|
|
# With exactly one component, v1 cannot tell whether the value was the
|
|
# event fingerprint or startsAt because it discarded empty fields.
|
|
return bool(event_fingerprint) == bool(started_at)
|
|
|
|
|
|
def _approval_matches_owner_identity(
|
|
approval: Any,
|
|
*,
|
|
owner: dict[str, Any],
|
|
legacy_owner: dict[str, str],
|
|
) -> bool:
|
|
"""Require the complete deterministic owner before approval reuse."""
|
|
|
|
metadata = getattr(approval, "metadata", None)
|
|
if not isinstance(metadata, dict):
|
|
return False
|
|
approval_id = _value(getattr(approval, "id", ""))
|
|
allowed_ids = {
|
|
_value(owner.get("canonical_approval_id")),
|
|
_value(owner.get("legacy_approval_id")),
|
|
}
|
|
if approval_id not in allowed_ids:
|
|
return False
|
|
|
|
persisted_schema = _value(metadata.get("schema_version"))
|
|
if persisted_schema == BACKUP_RESTORE_OWNER_SCHEMA_VERSION:
|
|
persisted_identity = metadata.get("owner_identity")
|
|
return bool(
|
|
isinstance(persisted_identity, dict)
|
|
and persisted_identity == owner.get("owner_identity")
|
|
and _value(metadata.get("owner_key_encoding"))
|
|
== BACKUP_RESTORE_OWNER_KEY_ENCODING
|
|
and _value(metadata.get("owner_key_sha256"))
|
|
== _value(owner.get("owner_key_sha256"))
|
|
and _value(metadata.get("approval_id")) == approval_id
|
|
)
|
|
|
|
if persisted_schema != BACKUP_RESTORE_LEGACY_OWNER_SCHEMA_VERSION:
|
|
return False
|
|
if not _legacy_owner_reuse_is_unambiguous(owner):
|
|
return False
|
|
return bool(
|
|
approval_id == legacy_owner["approval_id"]
|
|
and _value(metadata.get("approval_id")) == legacy_owner["approval_id"]
|
|
and _value(metadata.get("project_id")) == legacy_owner["project_id"]
|
|
and _value(metadata.get("source_fingerprint"))
|
|
== legacy_owner["source_fingerprint"]
|
|
and _value(metadata.get("source_occurrence"))
|
|
== legacy_owner["source_occurrence"]
|
|
and _value(metadata.get("source_occurrence_hash"))
|
|
== legacy_owner["source_occurrence_hash"]
|
|
and _value(metadata.get("incident_id")) == legacy_owner["incident_id"]
|
|
and _value(metadata.get("work_item_id")) == legacy_owner["work_item_id"]
|
|
)
|
|
|
|
|
|
def _owner_with_id_scheme(
|
|
owner: dict[str, Any],
|
|
*,
|
|
scheme: str,
|
|
) -> dict[str, Any]:
|
|
if scheme not in {"canonical_json_v2", "legacy_colon_v1"}:
|
|
raise ValueError("backup_restore_owner_id_scheme_invalid")
|
|
prefix = "canonical" if scheme == "canonical_json_v2" else "legacy"
|
|
selected = dict(owner)
|
|
selected.update(
|
|
{
|
|
"approval_id": selected[f"{prefix}_approval_id"],
|
|
"incident_id": selected[f"{prefix}_incident_id"],
|
|
"work_item_id": selected[f"{prefix}_work_item_id"],
|
|
"identity_id_scheme": scheme,
|
|
"legacy_identity_compatibility": scheme == "legacy_colon_v1",
|
|
}
|
|
)
|
|
return selected
|
|
|
|
|
|
def _owner_for_approval(
|
|
owner: dict[str, Any],
|
|
approval: Any,
|
|
) -> dict[str, Any]:
|
|
approval_id = _value(getattr(approval, "id", ""))
|
|
if approval_id == _value(owner.get("canonical_approval_id")):
|
|
return _owner_with_id_scheme(owner, scheme="canonical_json_v2")
|
|
if approval_id == _value(owner.get("legacy_approval_id")):
|
|
return _owner_with_id_scheme(owner, scheme="legacy_colon_v1")
|
|
raise ValueError("backup_restore_approval_owner_id_not_deterministic")
|
|
|
|
|
|
async def _find_compatible_owner_approval(
|
|
approval_service: Any,
|
|
*,
|
|
owner: dict[str, Any],
|
|
legacy_owner: dict[str, str],
|
|
) -> Any | None:
|
|
"""Probe all storage generations without accepting an owner collision."""
|
|
|
|
for storage_fingerprint in _approval_fingerprint_lookup_candidates(owner):
|
|
recent = await approval_service.find_by_fingerprint(
|
|
fingerprint=storage_fingerprint,
|
|
debounce_minutes=30,
|
|
)
|
|
if recent is None:
|
|
continue
|
|
if _approval_matches_owner_identity(
|
|
recent,
|
|
owner=owner,
|
|
legacy_owner=legacy_owner,
|
|
):
|
|
return recent
|
|
logger.warning(
|
|
"backup_restore_approval_owner_identity_mismatch",
|
|
storage_fingerprint=storage_fingerprint,
|
|
approval_id=_value(getattr(recent, "id", "")),
|
|
)
|
|
return None
|
|
|
|
|
|
def build_backup_restore_ingress_owner(
|
|
*,
|
|
project_id: str,
|
|
source_fingerprint: str,
|
|
source_event_fingerprint: str = "",
|
|
source_started_at: str = "",
|
|
) -> dict[str, Any]:
|
|
"""Return the deterministic durable owner identity for one firing episode."""
|
|
|
|
project = str(project_id or "awoooi").strip() or "awoooi"
|
|
fingerprint = str(source_fingerprint or "").strip()
|
|
if not fingerprint:
|
|
raise ValueError("backup_restore_source_fingerprint_required")
|
|
event_fingerprint = str(source_event_fingerprint or "").strip()
|
|
started_at = str(source_started_at or "").strip()
|
|
source_occurrence = _canonical_source_occurrence(
|
|
source_fingerprint=fingerprint,
|
|
source_event_fingerprint=event_fingerprint,
|
|
source_started_at=started_at,
|
|
)
|
|
owner_identity = {
|
|
"schema_version": BACKUP_RESTORE_OWNER_SCHEMA_VERSION,
|
|
"project_id": project,
|
|
"source_fingerprint": fingerprint,
|
|
"source_occurrence": source_occurrence,
|
|
}
|
|
owner_key = _canonical_json(owner_identity)
|
|
digest = hashlib.sha256(owner_key.encode("utf-8")).hexdigest()
|
|
legacy_owner = _legacy_backup_restore_ingress_owner(
|
|
project_id=project,
|
|
source_fingerprint=fingerprint,
|
|
source_event_fingerprint=event_fingerprint,
|
|
source_started_at=started_at,
|
|
)
|
|
owner = {
|
|
**owner_identity,
|
|
"owner_identity": owner_identity,
|
|
"owner_key_encoding": BACKUP_RESTORE_OWNER_KEY_ENCODING,
|
|
"owner_key_sha256": digest,
|
|
"source_event_fingerprint": event_fingerprint,
|
|
"source_started_at": started_at,
|
|
"source_occurrence_hash": digest,
|
|
"canonical_approval_id": str(uuid5(NAMESPACE_URL, owner_key)),
|
|
"canonical_incident_id": f"INC-BRR-{digest[:16].upper()}",
|
|
"canonical_work_item_id": f"backupcheck:{project}:{digest[:32]}",
|
|
"legacy_approval_id": legacy_owner["approval_id"],
|
|
"legacy_incident_id": legacy_owner["incident_id"],
|
|
"legacy_work_item_id": legacy_owner["work_item_id"],
|
|
"route_id": BACKUP_RESTORE_AGENT99_ROUTE_ID,
|
|
"kind": "backup_health",
|
|
"mode": "BackupCheck",
|
|
"single_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
|
|
"telegram_role": "receipt_projection_only",
|
|
"controlled_apply": False,
|
|
"read_only": True,
|
|
"prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
|
|
}
|
|
# Production legacy-backfill claims already persist v1 IDs in their item
|
|
# payload. Preserve those IDs while upgrading their metadata and storage
|
|
# fingerprint; all other new episodes use collision-safe canonical IDs.
|
|
scheme = (
|
|
"legacy_colon_v1"
|
|
if fingerprint.startswith(BACKUP_RESTORE_LEGACY_BACKFILL_PREFIX)
|
|
else "canonical_json_v2"
|
|
)
|
|
return _owner_with_id_scheme(owner, scheme=scheme)
|
|
|
|
|
|
def _backup_restore_approval_request(
|
|
*,
|
|
owner: dict[str, Any],
|
|
target_resource: str,
|
|
message: str,
|
|
incident_id: str = "",
|
|
) -> ApprovalRequestCreate:
|
|
return ApprovalRequestCreate(
|
|
action=(
|
|
"READ_ONLY_BACKUPCHECK - collect backup status, freshness, offsite, "
|
|
"escrow count, restore-drill and source-resolution receipts"
|
|
),
|
|
description=(
|
|
"Alertmanager raw backup/restore signal selected the non-mutating "
|
|
f"BackupCheck evidence lane. {str(message or '')[:500]}"
|
|
),
|
|
risk_level=RiskLevel.LOW,
|
|
blast_radius=BlastRadius(
|
|
affected_pods=0,
|
|
estimated_downtime="0",
|
|
related_services=[target_resource] if target_resource else [],
|
|
data_impact=DataImpact.READ_ONLY,
|
|
),
|
|
dry_run_checks=[
|
|
DryRunCheck(
|
|
name="single dispatch owner",
|
|
passed=True,
|
|
message=BACKUP_RESTORE_DISPATCH_OWNER,
|
|
),
|
|
DryRunCheck(
|
|
name="non-mutating BackupCheck boundary",
|
|
passed=True,
|
|
message="backup/restore/delete/retention/escrow writes prohibited",
|
|
),
|
|
],
|
|
requested_by="Alertmanager BackupCheck durable owner",
|
|
incident_id=incident_id or None,
|
|
metadata={
|
|
**owner,
|
|
"preallocated_approval_id": owner["approval_id"],
|
|
"owner_policy": "global_product_governance_v2",
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": False,
|
|
},
|
|
)
|
|
|
|
|
|
def _value(value: Any) -> str:
|
|
return str(getattr(value, "value", value) or "").strip()
|
|
|
|
|
|
def _incident_risk_from_source_severity(severity: str) -> str:
|
|
normalized = str(severity or "warning").strip().lower()
|
|
if normalized in {"critical", "error"}:
|
|
return "high"
|
|
if normalized == "warning":
|
|
return "medium"
|
|
return "low"
|
|
|
|
|
|
def _dispatch_receipt_from_readback(
|
|
readback: dict[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
if not isinstance(readback, dict):
|
|
return {}
|
|
receipt = readback.get("dispatch_receipt")
|
|
return dict(receipt) if isinstance(receipt, dict) else {}
|
|
|
|
|
|
def _identity_from(value: dict[str, Any] | None) -> dict[str, Any]:
|
|
if not isinstance(value, dict):
|
|
return {}
|
|
identity = value.get("identity")
|
|
return dict(identity) if isinstance(identity, dict) else {}
|
|
|
|
|
|
async def process_backup_restore_alertmanager_signal(
|
|
*,
|
|
project_id: str,
|
|
alert_id: str,
|
|
alertname: str,
|
|
severity: str,
|
|
namespace: str,
|
|
target_resource: str,
|
|
message: str,
|
|
labels: dict[str, Any] | None,
|
|
annotations: dict[str, Any] | None,
|
|
source_fingerprint: str,
|
|
source_event_fingerprint: str = "",
|
|
source_started_at: str = "",
|
|
alert_category: str = "backup",
|
|
notification_type: str = "TYPE-1",
|
|
source_url: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Create/bind a canonical Incident, dispatch once, and read the receipt.
|
|
|
|
The function fails closed before transport whenever owner persistence or
|
|
canonical Incident readback is missing. A repeated invocation reuses the
|
|
same approval, Incident, work item, and PostgreSQL dispatch identity.
|
|
"""
|
|
|
|
safe_labels = dict(labels or {})
|
|
safe_annotations = dict(annotations or {})
|
|
route = resolve_agent99_durable_route(
|
|
alertname=alertname,
|
|
severity=severity,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
message=message,
|
|
labels=safe_labels,
|
|
annotations=safe_annotations,
|
|
)
|
|
if not isinstance(route, dict) or (
|
|
route.get("kind") != "backup_health"
|
|
or route.get("suggested_mode") != "BackupCheck"
|
|
or route.get("route_id") != BACKUP_RESTORE_AGENT99_ROUTE_ID
|
|
):
|
|
return {
|
|
"schema_version": "backup_restore_ingress_result_v1",
|
|
"status": "not_backup_restore_route_no_dispatch",
|
|
"dispatch_performed": False,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": False,
|
|
}
|
|
|
|
owner = build_backup_restore_ingress_owner(
|
|
project_id=project_id,
|
|
source_fingerprint=source_fingerprint,
|
|
source_event_fingerprint=source_event_fingerprint,
|
|
source_started_at=source_started_at,
|
|
)
|
|
legacy_owner = _legacy_backup_restore_ingress_owner(
|
|
project_id=str(owner["project_id"]),
|
|
source_fingerprint=str(owner["source_fingerprint"]),
|
|
source_event_fingerprint=str(owner["source_event_fingerprint"]),
|
|
source_started_at=str(owner["source_started_at"]),
|
|
)
|
|
approval_service = get_approval_service()
|
|
approval = None
|
|
mismatched_owner_ids: set[str] = set()
|
|
owner_candidate_ids = tuple(
|
|
dict.fromkeys(
|
|
(
|
|
_value(owner.get("approval_id")),
|
|
_value(owner.get("canonical_approval_id")),
|
|
_value(owner.get("legacy_approval_id")),
|
|
)
|
|
)
|
|
)
|
|
for candidate_id in owner_candidate_ids:
|
|
candidate = await approval_service.get_approval(UUID(candidate_id))
|
|
if candidate is None:
|
|
continue
|
|
if _approval_matches_owner_identity(
|
|
candidate,
|
|
owner=owner,
|
|
legacy_owner=legacy_owner,
|
|
):
|
|
approval = candidate
|
|
owner = _owner_for_approval(owner, candidate)
|
|
break
|
|
mismatched_owner_ids.add(candidate_id)
|
|
logger.warning(
|
|
"backup_restore_deterministic_owner_identity_mismatch",
|
|
approval_id=candidate_id,
|
|
)
|
|
|
|
if approval is None:
|
|
recent = await _find_compatible_owner_approval(
|
|
approval_service,
|
|
owner=owner,
|
|
legacy_owner=legacy_owner,
|
|
)
|
|
if recent is not None:
|
|
approval = recent
|
|
owner = _owner_for_approval(owner, recent)
|
|
|
|
created_owner = False
|
|
if approval is None:
|
|
# A v1 delimiter collision may occupy the legacy UUID. New work uses
|
|
# the canonical JSON UUID; a mismatch on that UUID itself fails closed.
|
|
canonical_id = _value(owner.get("canonical_approval_id"))
|
|
if canonical_id in mismatched_owner_ids:
|
|
return {
|
|
"schema_version": "backup_restore_ingress_result_v1",
|
|
"status": "durable_owner_identity_mismatch_no_dispatch",
|
|
"owner": owner,
|
|
"dispatch_performed": False,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": False,
|
|
}
|
|
if _value(owner.get("approval_id")) in mismatched_owner_ids:
|
|
owner = _owner_with_id_scheme(
|
|
owner,
|
|
scheme="canonical_json_v2",
|
|
)
|
|
owner_approval_id = UUID(str(owner["approval_id"]))
|
|
request = _backup_restore_approval_request(
|
|
owner=owner,
|
|
target_resource=target_resource,
|
|
message=message,
|
|
)
|
|
try:
|
|
approval = await approval_service.create_approval_with_fingerprint(
|
|
request=request,
|
|
fingerprint=_approval_owner_storage_fingerprint(owner),
|
|
)
|
|
created_owner = True
|
|
except Exception:
|
|
# A concurrent producer may have inserted the deterministic owner.
|
|
# Read it back; never create another random owner or dispatch twice.
|
|
approval = await approval_service.get_approval(owner_approval_id)
|
|
if approval is None:
|
|
raise
|
|
else:
|
|
owner_approval_id = UUID(str(owner["approval_id"]))
|
|
incremented = await approval_service.increment_hit_count(owner_approval_id)
|
|
if incremented is not None:
|
|
approval = incremented
|
|
|
|
if _value(getattr(approval, "id", "")) != str(
|
|
owner_approval_id
|
|
) or not _approval_matches_owner_identity(
|
|
approval,
|
|
owner=owner,
|
|
legacy_owner=legacy_owner,
|
|
):
|
|
return {
|
|
"schema_version": "backup_restore_ingress_result_v1",
|
|
"status": "durable_owner_identity_mismatch_no_dispatch",
|
|
"owner": owner,
|
|
"dispatch_performed": False,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": False,
|
|
}
|
|
|
|
incident_id = _value(getattr(approval, "incident_id", ""))
|
|
if not incident_id:
|
|
incident_id = str(owner["incident_id"])
|
|
incident_id = await create_incident_for_approval(
|
|
approval_id=str(owner_approval_id),
|
|
risk_level=_incident_risk_from_source_severity(severity),
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
alert_type="custom",
|
|
message=message,
|
|
source="alertmanager",
|
|
alertname=alertname,
|
|
alert_labels={
|
|
**safe_labels,
|
|
"fingerprint": source_fingerprint,
|
|
"alert_id": alert_id,
|
|
"backup_restore_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
|
|
},
|
|
notification_type=notification_type,
|
|
alert_category=alert_category,
|
|
canonical_incident_id=incident_id,
|
|
)
|
|
await approval_service.update_incident_id(owner_approval_id, incident_id)
|
|
|
|
bound_approval = await approval_service.get_approval(owner_approval_id)
|
|
if (
|
|
bound_approval is None
|
|
or _value(getattr(bound_approval, "incident_id", "")) != incident_id
|
|
or not _approval_matches_owner_identity(
|
|
bound_approval,
|
|
owner=owner,
|
|
legacy_owner=legacy_owner,
|
|
)
|
|
):
|
|
return {
|
|
"schema_version": "backup_restore_ingress_result_v1",
|
|
"status": "canonical_incident_binding_missing_no_dispatch",
|
|
"owner": owner,
|
|
"incident_id": incident_id or None,
|
|
"dispatch_performed": False,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": False,
|
|
"safe_next_action": (
|
|
"persist_owner_to_canonical_incident_binding_then_retry"
|
|
),
|
|
}
|
|
approval = bound_approval
|
|
|
|
canonical_incident = await get_incident_service().get_for_readback(
|
|
incident_id,
|
|
project_id=project_id,
|
|
)
|
|
canonical_status = _value(getattr(canonical_incident, "status", ""))
|
|
canonical_durable = bool(
|
|
canonical_incident is not None
|
|
and getattr(canonical_incident, "persisted_to_pg", False) is True
|
|
)
|
|
if not canonical_durable or canonical_status not in {
|
|
"investigating",
|
|
"mitigating",
|
|
}:
|
|
return {
|
|
"schema_version": "backup_restore_ingress_result_v1",
|
|
"status": "canonical_active_incident_missing_no_dispatch",
|
|
"owner": owner,
|
|
"incident_id": incident_id or None,
|
|
"canonical_incident_status": canonical_status or "missing",
|
|
"canonical_incident_durable": canonical_durable,
|
|
"dispatch_performed": False,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": False,
|
|
"safe_next_action": (
|
|
"repair_or_create_active_canonical_incident_then_retry_same_source_owner"
|
|
),
|
|
}
|
|
|
|
await record_alertmanager_event(
|
|
project_id=project_id,
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
severity=severity,
|
|
namespace=namespace,
|
|
target_resource=target_resource,
|
|
fingerprint=source_fingerprint,
|
|
stage="backup_restore_incident_owner_bound",
|
|
notification_type=notification_type,
|
|
alert_category=alert_category,
|
|
incident_id=incident_id,
|
|
approval_id=str(owner_approval_id),
|
|
repeat_count=int(getattr(approval, "hit_count", 1) or 1),
|
|
source_url=source_url,
|
|
labels={
|
|
**safe_labels,
|
|
"backup_restore_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
|
|
},
|
|
annotations=safe_annotations,
|
|
)
|
|
|
|
dispatch = await bridge_alertmanager_to_agent99(
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
severity=severity,
|
|
namespace=namespace,
|
|
target_resource=target_resource,
|
|
message=message,
|
|
labels=safe_labels,
|
|
annotations=safe_annotations,
|
|
fingerprint=source_fingerprint,
|
|
alert_category=alert_category,
|
|
notification_type=notification_type,
|
|
source_url=source_url,
|
|
project_id=project_id,
|
|
incident_id=incident_id,
|
|
approval_id=str(owner_approval_id),
|
|
route_id=BACKUP_RESTORE_AGENT99_ROUTE_ID,
|
|
work_item_id=str(owner["work_item_id"]),
|
|
)
|
|
readback = await read_agent99_dispatch_receipt(
|
|
project_id=project_id,
|
|
incident_id=incident_id,
|
|
)
|
|
dispatch_identity = _identity_from(dispatch)
|
|
readback_identity = _identity_from(readback)
|
|
readback_receipt = _dispatch_receipt_from_readback(readback)
|
|
identity_matches = bool(
|
|
dispatch_identity
|
|
and readback_identity
|
|
and all(
|
|
_value(dispatch_identity.get(field)) == _value(readback_identity.get(field))
|
|
for field in ("incident_id", "run_id", "trace_id", "work_item_id")
|
|
)
|
|
)
|
|
accepted = bool(
|
|
readback_receipt.get("accepted") is True
|
|
and readback_receipt.get("inbox_triggered") is True
|
|
)
|
|
receipt_persisted = bool(readback and identity_matches)
|
|
verifier = (
|
|
dict(readback.get("verifier"))
|
|
if isinstance(readback, dict) and isinstance(readback.get("verifier"), dict)
|
|
else {}
|
|
)
|
|
verifier_status = _value(verifier.get("status")) or "not_started"
|
|
durable = bool(accepted and receipt_persisted)
|
|
result_status = (
|
|
"backupcheck_dispatched_verifier_pending"
|
|
if durable and dispatch.get("dispatchPerformed") is True
|
|
else "backupcheck_deduplicated_verifier_pending"
|
|
if durable and dispatch.get("status") == "deduplicated"
|
|
else "backupcheck_dispatch_or_readback_failed"
|
|
)
|
|
|
|
await record_alertmanager_event(
|
|
project_id=project_id,
|
|
alert_id=alert_id,
|
|
alertname=alertname,
|
|
severity=severity,
|
|
namespace=namespace,
|
|
target_resource=target_resource,
|
|
fingerprint=source_fingerprint,
|
|
stage=(
|
|
"backup_restore_dispatch_receipt_persisted"
|
|
if durable
|
|
else "backup_restore_dispatch_receipt_missing"
|
|
),
|
|
notification_type=notification_type,
|
|
alert_category=alert_category,
|
|
incident_id=incident_id,
|
|
approval_id=str(owner_approval_id),
|
|
repeat_count=int(getattr(approval, "hit_count", 1) or 1),
|
|
source_url=source_url,
|
|
labels={
|
|
**safe_labels,
|
|
"backup_restore_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
|
|
},
|
|
annotations=safe_annotations,
|
|
)
|
|
logger.info(
|
|
"backup_restore_alertmanager_dispatch_result",
|
|
status=result_status,
|
|
incident_id=incident_id,
|
|
approval_id=str(owner_approval_id),
|
|
automation_run_id=readback_identity.get("run_id"),
|
|
dispatch_performed=bool(dispatch.get("dispatchPerformed") is True),
|
|
receipt_persisted=receipt_persisted,
|
|
accepted=accepted,
|
|
verifier_status=verifier_status,
|
|
runtime_execution_authorized=False,
|
|
runtime_closure_verified=False,
|
|
)
|
|
return {
|
|
"schema_version": "backup_restore_ingress_result_v1",
|
|
"status": result_status,
|
|
"owner": owner,
|
|
"owner_created": created_owner,
|
|
"approval_id": str(owner_approval_id),
|
|
"incident_id": incident_id,
|
|
"canonical_incident_status": canonical_status,
|
|
"canonical_incident_durable": canonical_durable,
|
|
"dispatch_performed": bool(dispatch.get("dispatchPerformed") is True),
|
|
"dispatch_status": _value(dispatch.get("status")),
|
|
"receipt_persisted": receipt_persisted,
|
|
"accepted": accepted,
|
|
"identity": readback_identity or dispatch_identity,
|
|
"verifier": {
|
|
"status": verifier_status,
|
|
"terminal": verifier_status in {"success", "failed"},
|
|
"receipt": verifier,
|
|
"outcome_source": "agent99_outcome_contract_v1",
|
|
"safe_next_action": (
|
|
"ingest_same_identity_agent99_outcome_then_run_independent_dr_verifier"
|
|
if verifier_status in {"pending", "pending_dispatch", "not_started"}
|
|
else "complete_learning_writeback_only_after_verifier_success"
|
|
),
|
|
},
|
|
"runtime_execution_authorized": False,
|
|
"runtime_closure_verified": bool(
|
|
isinstance(readback, dict)
|
|
and readback.get("runtime_closure_verified") is True
|
|
),
|
|
"production_write_performed": False,
|
|
"prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
|
|
}
|