fix(agent99): bind control plane health selfcheck
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 4m2s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
ogt
2026-07-17 00:31:34 +08:00
parent 65c7174336
commit 31a5fe2b94
7 changed files with 833 additions and 19 deletions

View File

@@ -303,6 +303,36 @@ def attach_agent99_dispatch_identity(
return enriched
def build_agent99_dispatch_scope(
*,
identity: Agent99DispatchIdentity,
kind: Any,
suggested_mode: Any,
target_resource: Any,
controlled_apply_requested: Any,
canonical_asset_id: Any = "",
typed_domain: Any = "",
executor: Any = "",
verifier: Any = "",
) -> dict[str, Any]:
"""Build the bounded typed scope persisted with one durable run."""
return {
"schema_version": "agent99_dispatch_scope_v1",
"kind": str(kind or "")[:120],
"suggested_mode": str(suggested_mode or "")[:80],
"target_resource": str(target_resource or "")[:240],
"controlled_apply_requested": bool(controlled_apply_requested is True),
"canonical_asset_id": str(canonical_asset_id or "")[:240],
"typed_domain": str(typed_domain or "")[:120],
"executor": str(executor or "")[:160],
"verifier": str(verifier or "")[:160],
# Never trust a transport-supplied route projection. The identity is
# the same-run source of truth for route binding.
"route_id": identity.route_id,
}
def build_agent99_dispatch_receipt_envelope(
*,
identity: Agent99DispatchIdentity,
@@ -337,6 +367,24 @@ def build_agent99_dispatch_receipt_envelope(
and delivery_certainty == "not_delivered"
)
public_identity = identity.public_dict()
supplied_scope = (
dispatch_receipt.get("dispatch_scope")
if isinstance(dispatch_receipt.get("dispatch_scope"), dict)
else {}
)
dispatch_scope = build_agent99_dispatch_scope(
identity=identity,
kind=dispatch_receipt.get("kind"),
suggested_mode=dispatch_receipt.get("suggested_mode"),
target_resource=dispatch_receipt.get("target_resource"),
controlled_apply_requested=dispatch_receipt.get(
"controlled_apply_requested"
),
canonical_asset_id=supplied_scope.get("canonical_asset_id"),
typed_domain=supplied_scope.get("typed_domain"),
executor=supplied_scope.get("executor"),
verifier=supplied_scope.get("verifier"),
)
stage_identity = {
"run_id": str(identity.run_id),
"trace_id": identity.trace_id,
@@ -352,15 +400,7 @@ def build_agent99_dispatch_receipt_envelope(
else "dispatch_delivery_unknown_reconcile_only"
),
"identity": public_identity,
"dispatch_scope": {
"schema_version": "agent99_dispatch_scope_v1",
"kind": str(dispatch_receipt.get("kind") or ""),
"suggested_mode": str(dispatch_receipt.get("suggested_mode") or ""),
"target_resource": str(dispatch_receipt.get("target_resource") or ""),
"controlled_apply_requested": bool(
dispatch_receipt.get("controlled_apply_requested") is True
),
},
"dispatch_scope": dispatch_scope,
"dispatch_receipt": {
"schema_version": str(
dispatch_receipt.get("schema_version")
@@ -508,13 +548,40 @@ class PostgresAgent99DispatchLedger:
payload: dict[str, Any],
) -> dict[str, Any]:
identity_payload = identity.public_dict()
routing = (
payload.get("routing")
if isinstance(payload.get("routing"), dict)
else {}
)
typed_target_route = (
routing.get("typedTargetRoute")
if isinstance(routing.get("typedTargetRoute"), dict)
else {}
)
awoooi = (
payload.get("awoooi")
if isinstance(payload.get("awoooi"), dict)
else {}
)
dispatch_scope = build_agent99_dispatch_scope(
identity=identity,
kind=payload.get("kind"),
suggested_mode=payload.get("suggestedMode") or "Status",
target_resource=awoooi.get("targetResource"),
controlled_apply_requested=payload.get("controlledApply"),
canonical_asset_id=typed_target_route.get("canonical_asset_id"),
typed_domain=typed_target_route.get("target_kind"),
executor=typed_target_route.get("executor"),
verifier=typed_target_route.get("verifier"),
)
input_payload = {
"schema_version": "agent99_controlled_dispatch_input_v1",
"identity": identity_payload,
"kind": str(payload.get("kind") or ""),
"suggested_mode": str(payload.get("suggestedMode") or "Status"),
"controlled_apply": bool(payload.get("controlledApply") is True),
"target_resource": str((payload.get("awoooi") or {}).get("targetResource") or ""),
"target_resource": str(awoooi.get("targetResource") or ""),
"dispatch_scope": dispatch_scope,
"stores_secrets": False,
}
input_json = _stable_json(input_payload)
@@ -529,13 +596,7 @@ class PostgresAgent99DispatchLedger:
"schema_version": "agent99_controlled_dispatch_receipt_v1",
"status": "dispatch_reserved",
"identity": identity_payload,
"dispatch_scope": {
"schema_version": "agent99_dispatch_scope_v1",
"kind": input_payload["kind"],
"suggested_mode": input_payload["suggested_mode"],
"target_resource": input_payload["target_resource"],
"controlled_apply_requested": input_payload["controlled_apply"],
},
"dispatch_scope": dispatch_scope,
"dispatch_accepted": False,
"controlled_apply_requested": bool(input_payload["controlled_apply"]),
"controlled_apply_authorized": False,

View File

@@ -29,7 +29,13 @@ from src.utils.timezone import now_taipei
logger = get_logger("awoooi.agent99_sre_bridge")
AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS = 600
AGENT99_CONTROL_PLANE_HEALTH_KIND = "agent99_control_plane_health"
AGENT99_CONTROL_PLANE_HEALTH_MODE = "SelfCheck"
AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID = (
"agent99:agent99_control_plane_health:SelfCheck"
)
AGENT99_DURABLE_ROUTE_KINDS = {
AGENT99_CONTROL_PLANE_HEALTH_KIND,
"provider_freshness_signal",
"public_route_502",
"performance_pressure",
@@ -116,6 +122,12 @@ def resolve_agent99_alert_kind(
annotations=annotations or {},
)
if (
re.sub(r"[^a-z0-9]+", "", alertname.lower())
== "agent99serviceunhealthy"
):
return AGENT99_CONTROL_PLANE_HEALTH_KIND
# Explicit backup identities win over generic freshness words. Backup
# failure cards intentionally mention freshness, but must stay in the
# read-only BackupCheck lane with its stricter no-delete/no-restore policy.
@@ -204,6 +216,7 @@ def resolve_agent99_alert_kind(
def _suggested_mode_for_kind(kind: str) -> str | None:
return {
AGENT99_CONTROL_PLANE_HEALTH_KIND: AGENT99_CONTROL_PLANE_HEALTH_MODE,
"provider_freshness_signal": "ProviderFreshness",
"public_route_502": "Recover",
"performance_pressure": "Perf",
@@ -254,6 +267,7 @@ def resolve_agent99_durable_route(
def _resolution_policy_for_kind(kind: str) -> str:
if kind in {
AGENT99_CONTROL_PLANE_HEALTH_KIND,
"provider_freshness_signal",
"public_route_502",
"performance_pressure",
@@ -479,6 +493,13 @@ def build_agent99_sre_alert(
"LightDM, VMware Tools, Wazuh services and recovery timer; do not reboot "
"the host, change VM power, reset the VM, read secrets or change firewall."
)
elif kind == AGENT99_CONTROL_PLANE_HEALTH_KIND:
instruction = (
"Run Agent99 SelfCheck only for canonical host_99 in the "
"windows_vmware domain; controlledApply=false; do not reboot, "
"change VM power, execute arbitrary shell, or fall back to another "
"host or domain."
)
payload: dict[str, Any] = {
"id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}",
@@ -490,7 +511,7 @@ def build_agent99_sre_alert(
"title": _safe_text(f"Alertmanager {alertname}", max_length=240),
"message": _safe_text(message, max_length=2000),
"labels": safe_labels,
"force": True,
"force": kind != AGENT99_CONTROL_PLANE_HEALTH_KIND,
"controlledApply": suggested_mode
in {
"Recover",
@@ -552,6 +573,17 @@ def build_agent99_sre_alert(
}
if suggested_mode:
payload["suggestedMode"] = suggested_mode
if kind == AGENT99_CONTROL_PLANE_HEALTH_KIND:
payload["routing"]["actionPolicy"] = {
"schemaVersion": "agent99_control_plane_action_policy_v1",
"allowedModes": [AGENT99_CONTROL_PLANE_HEALTH_MODE],
"controlledApplyAllowed": False,
"rebootAllowed": False,
"vmPowerChangeAllowed": False,
"arbitraryShellAllowed": False,
"crossHostFallbackAllowed": False,
"crossDomainFallbackAllowed": False,
}
if instruction:
payload["instruction"] = instruction
return payload
@@ -862,6 +894,55 @@ def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str:
return str(receipt["transport"])
def _agent99_dispatch_scope_from_payload(
payload: dict[str, Any],
) -> dict[str, Any]:
"""Project the public-safe typed scope carried by one Agent99 run."""
routing = (
payload.get("routing")
if isinstance(payload.get("routing"), dict)
else {}
)
typed_target = (
routing.get("typedTargetRoute")
if isinstance(routing.get("typedTargetRoute"), dict)
else {}
)
awoooi = (
payload.get("awoooi")
if isinstance(payload.get("awoooi"), dict)
else {}
)
identity = (
awoooi.get("agent99DispatchIdentity")
if isinstance(awoooi.get("agent99DispatchIdentity"), dict)
else {}
)
return {
"schema_version": "agent99_dispatch_scope_v1",
"kind": _safe_text(payload.get("kind"), max_length=120),
"suggested_mode": _safe_text(
payload.get("suggestedMode"), max_length=80
),
"target_resource": _safe_text(
awoooi.get("targetResource"), max_length=240
),
"controlled_apply_requested": bool(
payload.get("controlledApply") is True
),
"canonical_asset_id": _safe_text(
typed_target.get("canonical_asset_id"), max_length=240
),
"typed_domain": _safe_text(
typed_target.get("target_kind"), max_length=120
),
"executor": _safe_text(typed_target.get("executor"), max_length=160),
"verifier": _safe_text(typed_target.get("verifier"), max_length=160),
"route_id": _safe_text(identity.get("route_id"), max_length=240),
}
def dispatch_agent99_sre_alert_with_receipt(
payload: dict[str, Any],
) -> dict[str, Any]:
@@ -877,6 +958,7 @@ def dispatch_agent99_sre_alert_with_receipt(
target_resource = _safe_text(awoooi.get("targetResource"), max_length=240)
suggested_mode = _safe_text(payload.get("suggestedMode"), max_length=80)
controlled_apply_requested = bool(payload.get("controlledApply") is True)
dispatch_scope = _agent99_dispatch_scope_from_payload(payload)
if settings.AGENT99_SRE_ALERT_RELAY_URL:
raw_receipt = post_agent99_sre_alert(payload) or {}
response_payload: dict[str, Any] = {}
@@ -966,6 +1048,7 @@ def dispatch_agent99_sre_alert_with_receipt(
"target_resource": target_resource,
"suggested_mode": suggested_mode,
"controlled_apply_requested": controlled_apply_requested,
"dispatch_scope": dispatch_scope,
"http_status": http_status,
"transport_accepted": transport_accepted,
"accepted": accepted,
@@ -1001,6 +1084,7 @@ def dispatch_agent99_sre_alert_with_receipt(
"target_resource": target_resource,
"suggested_mode": suggested_mode,
"controlled_apply_requested": controlled_apply_requested,
"dispatch_scope": dispatch_scope,
"transport_accepted": True,
"accepted": False,
"inbox_triggered": False,
@@ -1031,6 +1115,7 @@ def dispatch_agent99_sre_alert_with_receipt(
"target_resource": target_resource,
"suggested_mode": suggested_mode,
"controlled_apply_requested": controlled_apply_requested,
"dispatch_scope": dispatch_scope,
"transport_accepted": False,
"accepted": False,
"inbox_triggered": False,
@@ -1130,6 +1215,48 @@ async def bridge_alertmanager_to_agent99(
str(route_id or "").strip()
or f"agent99:{str(payload.get('kind') or 'monitoring_alert')}:{suggested_mode}"
)
if payload.get("kind") == AGENT99_CONTROL_PLANE_HEALTH_KIND:
typed_target_route = (
routing.get("typedTargetRoute")
if isinstance(routing.get("typedTargetRoute"), dict)
else {}
)
exact_control_plane_scope = bool(
suggested_mode == AGENT99_CONTROL_PLANE_HEALTH_MODE
and mutating is False
and payload.get("force") is False
and resolved_route_id == AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID
and typed_target_route.get("resolution_status") == "resolved"
and typed_target_route.get("canonical_asset_id") == "host_99"
and typed_target_route.get("target_kind") == "windows_vmware"
and typed_target_route.get("executor") == "Agent99"
and typed_target_route.get("verifier")
== "agent99_control_plane_selfcheck_independent_verifier"
and typed_target_route.get("cross_domain_fallback_allowed")
is False
)
if not exact_control_plane_scope:
logger.warning(
"agent99_control_plane_scope_denied_fail_closed",
alert_id=alert_id,
alertname=alertname,
route_id=resolved_route_id,
canonical_asset_id=typed_target_route.get(
"canonical_asset_id"
),
target_kind=typed_target_route.get("target_kind"),
reason="agent99_control_plane_exact_scope_required",
)
return {
"status": "failed",
"reason": "agent99_control_plane_exact_scope_required",
"dispatchPerformed": False,
"runtimeClosureVerified": False,
"typedTargetRoute": typed_target_route,
"driftWorkItemId": typed_target_route.get(
"drift_work_item_id"
),
}
durable_route = bool(
str(incident_id or "").strip()
or str(route_id or "").strip()
@@ -1453,7 +1580,19 @@ async def bridge_alertmanager_to_agent99(
"status": "transport_exception",
"transport": "unknown",
"alert_id": str(dispatch_identity.run_id),
"kind": "host_recovery",
"kind": str(payload.get("kind") or "monitoring_alert"),
"suggested_mode": str(
payload.get("suggestedMode") or "Status"
),
"target_resource": str(
(payload.get("awoooi") or {}).get("targetResource") or ""
),
"controlled_apply_requested": bool(
payload.get("controlledApply") is True
),
"dispatch_scope": _agent99_dispatch_scope_from_payload(
payload
),
"accepted": False,
"inbox_triggered": False,
"delivery_certainty": "unknown",

View File

@@ -51,6 +51,7 @@ _WINDOWS_VMWARE_MARKERS = (
"vmware",
"vmx",
)
_AGENT99_CONTROL_PLANE_HEALTH_ALERTS = {"agent99serviceunhealthy"}
_DISK_ALERTS = {
"hostoutofdiskspace",
"hostdiskusagehigh",
@@ -489,6 +490,53 @@ def resolve_typed_alert_target(
execution_role="single_writer_host111_launchagent_executor",
)
if compact_alert in _AGENT99_CONTROL_PLANE_HEALTH_ALERTS:
# Agent99's own health signal is a read-only control-plane route. It
# is never permission to recover a guest, power a VM, reboot a host,
# or fall through to an arbitrary Windows command. An explicit host
# identity that is not host_99 is drift and must fail closed.
explicit_host_scope = _host_scope(
str(labels.get("host") or ""),
str(labels.get("instance") or ""),
str(labels.get("node") or ""),
target_resource,
)
if (
explicit_host_scope is not None
and explicit_host_scope != _HOSTS["99"]
):
return _typed_route(
resolution_status="asset_identity_unresolved",
target_kind="unknown",
target_resource=target_resource,
namespace=namespace,
canonical_asset_id=None,
host=None,
executor=None,
verifier=None,
risk_class="high",
drift_work_item_id=_kubernetes_drift_work_item_id(
alertname,
target_resource,
explicit_host_scope[0],
),
stateful_level=StatefulLevel.UNRESOLVED.value,
)
return _typed_route(
resolution_status="resolved",
target_kind="windows_vmware",
target_resource=target_resource or "Agent99",
namespace=namespace,
canonical_asset_id="host_99",
host=_HOSTS["99"][1],
executor="Agent99",
verifier="agent99_control_plane_selfcheck_independent_verifier",
risk_class="high",
allowed_inventory_hosts=["host_99"],
controlled_apply_allowed=False,
execution_role="single_writer_agent99_control_plane_selfcheck_executor",
)
if any(marker in text for marker in _BACKUP_RESTORE_MARKERS):
identity = registry.resolve_identity(target_resource)
generic_backup_signal = _compact_token(target_resource) in {