6cf8429d17
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s
Restore D037 durable incident readback without weakening database failure semantics. Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints. Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls. Keep SSH service refusal visible while separating it from broker network-policy reachability.
282 lines
10 KiB
Python
282 lines
10 KiB
Python
"""Domain-aware routing for non-Kubernetes controlled alert targets.
|
|
|
|
The Alertmanager namespace label is a source attribute, not proof that every
|
|
alert is a Kubernetes workload. Cold-start/reboot recovery is a host control
|
|
plane lane. It must never fall through to a guessed ``kubectl`` action or the
|
|
generic Ansible keyword catalog merely because the source payload mentions a
|
|
host, backup, or container.
|
|
|
|
This module is deliberately pure. It only builds machine-readable routing
|
|
and receipt contracts; it does not dispatch Agent99, run a command, or write
|
|
runtime state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
_RECOVERY_MARKERS = (
|
|
"cold-start",
|
|
"cold_start",
|
|
"cold start",
|
|
"full-stack-cold-start",
|
|
"full_stack_cold_start",
|
|
"full stack cold start",
|
|
"reboot-auto-recovery",
|
|
"reboot_auto_recovery",
|
|
"reboot auto recovery",
|
|
)
|
|
|
|
|
|
def _text_token(value: str | None) -> str:
|
|
return " ".join(str(value or "").strip().lower().split())
|
|
|
|
|
|
def _asset_token(value: str) -> str:
|
|
normalized = re.sub(r"[^a-z0-9_.-]+", "-", value.lower()).strip("-.")
|
|
return normalized[:96] or "unbound"
|
|
|
|
|
|
def resolve_controlled_alert_target(
|
|
*,
|
|
alertname: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
) -> dict[str, Any] | None:
|
|
"""Return the dedicated executor route for a known non-K8s target."""
|
|
|
|
text = " ".join(
|
|
(
|
|
_text_token(alertname),
|
|
_text_token(target_resource),
|
|
)
|
|
)
|
|
if not any(marker in text for marker in _RECOVERY_MARKERS):
|
|
return None
|
|
|
|
return {
|
|
"schema_version": "controlled_alert_target_route_v1",
|
|
"target_kind": "control_plane_recovery",
|
|
"target_resource": target_resource or "cold-start-gate",
|
|
"source_namespace": namespace or None,
|
|
"normalized_execution_namespace": "host_recovery",
|
|
"kubernetes_namespace_applicable": False,
|
|
"executor": "Agent99",
|
|
"route_id": "agent99_recover_after_owner_review",
|
|
"check_route": "agent99-mode Status controlledApply=false",
|
|
"controlled_apply_route": "agent99-mode Recover controlledApply=true",
|
|
"rollback_route": "agent99-mode Status controlledApply=false",
|
|
"runtime_execution_authorized": False,
|
|
"runtime_write_allowed": False,
|
|
}
|
|
|
|
|
|
def build_controlled_recovery_promotion_contract(
|
|
*,
|
|
alertname: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
incident_id: str = "",
|
|
) -> dict[str, Any] | None:
|
|
"""Build the no-false-green check/apply/verifier/learning contract."""
|
|
|
|
route = resolve_controlled_alert_target(
|
|
alertname=alertname,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
)
|
|
if route is None:
|
|
return None
|
|
|
|
source_id = _asset_token(incident_id or f"{alertname}:{target_resource}")
|
|
fields = [
|
|
{
|
|
"field": "target_selector",
|
|
"status": "ready",
|
|
"source": "domain_target_normalizer",
|
|
"value": {
|
|
"target_kind": route["target_kind"],
|
|
"target_resource": route["target_resource"],
|
|
"normalized_execution_namespace": route[
|
|
"normalized_execution_namespace"
|
|
],
|
|
},
|
|
},
|
|
{
|
|
"field": "source_sensor_receipt",
|
|
"status": "pending",
|
|
"source": "alertmanager+cold_start_scorecard",
|
|
"value": "require_current_firing_or_recurrence_and_scorecard_ref",
|
|
},
|
|
{
|
|
"field": "check_mode_receipt",
|
|
"status": "pending",
|
|
"source": "Agent99 Status + read-only cold-start sensor",
|
|
"value": route["check_route"],
|
|
},
|
|
{
|
|
"field": "controlled_apply_route",
|
|
"status": "ready",
|
|
"source": "domain_target_normalizer",
|
|
"value": route["controlled_apply_route"],
|
|
},
|
|
{
|
|
"field": "dispatch_receipt",
|
|
"status": "pending",
|
|
"source": "agent99_sre_dispatch_receipt_v1",
|
|
"value": "accepted_and_inbox_triggered_required",
|
|
},
|
|
{
|
|
"field": "rollback_route",
|
|
"status": "ready",
|
|
"source": "domain_target_normalizer",
|
|
"value": route["rollback_route"],
|
|
},
|
|
{
|
|
"field": "post_apply_verifier",
|
|
"status": "pending",
|
|
"source": "Agent99 outcome + cold-start scorecard",
|
|
"value": [
|
|
"agent99_recover_outcome_terminal_readback",
|
|
"full_stack_cold_start_scorecard_rerun",
|
|
"required_hosts_harbor_k3s_routes_green",
|
|
"original_fingerprint_resolved_or_recurrence_decreased",
|
|
],
|
|
},
|
|
{
|
|
"field": "incident_closure_receipt",
|
|
"status": "pending",
|
|
"source": "incident_timeline",
|
|
"value": "terminal_only_after_independent_verifier",
|
|
},
|
|
{
|
|
"field": "telegram_receipt",
|
|
"status": "pending",
|
|
"source": "telegram_outbound_mirror",
|
|
"value": "lifecycle_result_delivery_ack_required",
|
|
},
|
|
{
|
|
"field": "km_writeback",
|
|
"status": "pending",
|
|
"source": "Hermes",
|
|
"value": f"km:cold-start-recovery:{source_id}",
|
|
},
|
|
{
|
|
"field": "playbook_trust_writeback",
|
|
"status": "pending",
|
|
"source": "OpenClaw/NemoTron",
|
|
"value": f"playbook-trust:cold-start-recovery:{source_id}",
|
|
},
|
|
]
|
|
ready_fields = [row["field"] for row in fields if row["status"] == "ready"]
|
|
blocked_fields = [row["field"] for row in fields if row["status"] != "ready"]
|
|
return {
|
|
"schema_version": "repair_candidate_promotion_contract_v1",
|
|
"status": "controlled_route_candidate_pending_receipts",
|
|
"route_id": route["route_id"],
|
|
"executor": route["executor"],
|
|
"incident_id": incident_id or None,
|
|
"target_route": route,
|
|
"ready_count": len(ready_fields),
|
|
"total_count": len(fields),
|
|
"blocked_count": len(blocked_fields),
|
|
"ready_fields": ready_fields,
|
|
"blocked_fields": blocked_fields,
|
|
"fields": fields,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_write_allowed": False,
|
|
"controlled_playbook_queue": True,
|
|
"owner_review_required": False,
|
|
"owner_review_gate": "auto_waived_for_low_medium_high",
|
|
"needs_human": False,
|
|
"completion_status": "partial",
|
|
"completion_blocker": "dispatch_verifier_and_learning_receipts_missing",
|
|
"forbidden_operations": [
|
|
"kubectl_for_host_control_plane_target",
|
|
"generic_ansible_keyword_fallback",
|
|
"host_reboot",
|
|
"node_drain",
|
|
"backup_restore",
|
|
"secret_read",
|
|
"mark_resolved_without_post_verifier",
|
|
],
|
|
"controlled_automation_closure": {
|
|
"schema_version": "controlled_recovery_closure_contract_v1",
|
|
"trace_id_required": True,
|
|
"status": "pending_source_check_dispatch_verify_writeback",
|
|
"stages": [
|
|
{"stage": "sensor", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "normalize", "status": "passed", "writes_runtime_state": False},
|
|
{"stage": "check", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "controlled_apply", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "verify", "status": "pending", "writes_runtime_state": False},
|
|
{"stage": "learn_writeback", "status": "pending", "writes_runtime_state": False},
|
|
],
|
|
"required_receipts": [row["field"] for row in fields],
|
|
"automation_asset_ledger": [
|
|
{
|
|
"asset_type": "Sensor",
|
|
"asset_id": "scripts/reboot-recovery/full-stack-cold-start-check.sh",
|
|
"owner": "ColdStartMonitor",
|
|
"status": "read_only_receipt_pending",
|
|
},
|
|
{
|
|
"asset_type": "PlayBook",
|
|
"asset_id": "agent99:Recover",
|
|
"owner": "Agent99",
|
|
"status": "dispatch_receipt_pending",
|
|
},
|
|
{
|
|
"asset_type": "Verifier",
|
|
"asset_id": f"verifier:cold-start-recovery:{source_id}",
|
|
"owner": "PostExecutionVerifier",
|
|
"status": "pending",
|
|
},
|
|
{
|
|
"asset_type": "KM",
|
|
"asset_id": f"km:cold-start-recovery:{source_id}",
|
|
"owner": "Hermes",
|
|
"status": "pending_after_verifier",
|
|
},
|
|
],
|
|
},
|
|
}
|
|
|
|
|
|
def build_controlled_recovery_handoff(
|
|
*,
|
|
alertname: str,
|
|
target_resource: str,
|
|
namespace: str,
|
|
incident_id: str = "",
|
|
) -> dict[str, Any] | None:
|
|
contract = build_controlled_recovery_promotion_contract(
|
|
alertname=alertname,
|
|
target_resource=target_resource,
|
|
namespace=namespace,
|
|
incident_id=incident_id,
|
|
)
|
|
if contract is None:
|
|
return None
|
|
return {
|
|
"schema_version": "ai_decision_controlled_executor_handoff_v1",
|
|
"status": "agent99_dispatch_receipt_readback_pending",
|
|
"queued": False,
|
|
"side_effect_performed": False,
|
|
"single_writer_executor": "Agent99",
|
|
"route_id": contract["route_id"],
|
|
"active_blockers": [
|
|
"agent99_dispatch_receipt_readback_pending",
|
|
"post_apply_verifier_missing",
|
|
"km_playbook_writeback_missing",
|
|
],
|
|
"safe_next_action": "read_agent99_dispatch_then_run_independent_cold_start_verifier",
|
|
"repair_candidate_promotion_contract": contract,
|
|
"runtime_execution_authorized": False,
|
|
"runtime_write_allowed": False,
|
|
"owner_review_required": False,
|
|
"owner_review_gate": "auto_waived_for_low_medium_high",
|
|
"needs_human": False,
|
|
}
|