wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(sre): bind Agent99 closure to typed verifier

Browse Source
This commit is contained in:
Your Name
2026-07-19 00:14:43 +08:00
parent 39685e9a08
commit 373ad979f4
5 changed files with 799 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
apps/api/src/services/agent99_completion_callback.py
View File

@@ -12,12 +12,20 @@ from src.models.agent99_completion import Agent99CompletionCallbackRequest
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
from src.services.agent99_controlled_dispatch_ledger import (
parse_agent99_dispatch_identity,
read_agent99_dispatch_receipt_for_run,
)
from src.services.channel_hub import (
build_external_alert_provider_event_id,
record_external_alert_event,
)
_OPERATION_INCIDENT_ID_MAX_LENGTH = 30
_AGENT99_CLOSURE_DOMAINS = frozenset({
"control_plane_recovery",
"windows_vmware",
})
def _operation_incident_id(alert_id: str | None) -> str | None:
@@ -40,31 +48,37 @@ async def _load_callback_readback(
result = await db.execute(
text(
"""
WITH event_readback AS (
SELECT
event_row.event_id::text AS event_id,
event_row.run_id::text AS run_id
FROM awooop_conversation_event event_row
WHERE event_row.project_id = :project_id
AND event_row.channel_type = 'internal'
AND event_row.provider_event_id = :provider_event_id
LIMIT 1
),
operation_counts AS (
SELECT
count(*) AS operation_count,
count(*) FILTER (
WHERE operation_row.event_type = 'EXECUTION_COMPLETED'
) AS execution_count,
count(*) FILTER (
WHERE operation_row.event_type = 'RESOLVED'
) AS resolved_count
FROM alert_operation_log operation_row
WHERE operation_row.context ->> 'callback_id' = :callback_id
AND operation_row.context ->> 'project_id' = :project_id
)
SELECT
event_row.event_id::text AS event_id,
event_row.run_id::text AS run_id,
(
SELECT count(*)
FROM alert_operation_log operation_row
WHERE operation_row.context ->> 'callback_id' = :callback_id
) AS operation_count,
(
SELECT count(*)
FROM alert_operation_log operation_row
WHERE operation_row.context ->> 'callback_id' = :callback_id
AND operation_row.event_type = 'EXECUTION_COMPLETED'
) AS execution_count,
(
SELECT count(*)
FROM alert_operation_log operation_row
WHERE operation_row.context ->> 'callback_id' = :callback_id
AND operation_row.event_type = 'RESOLVED'
) AS resolved_count
FROM awooop_conversation_event event_row
WHERE event_row.project_id = :project_id
AND event_row.channel_type = 'internal'
AND event_row.provider_event_id = :provider_event_id
LIMIT 1
event_readback.event_id,
event_readback.run_id,
operation_counts.operation_count,
operation_counts.execution_count,
operation_counts.resolved_count
FROM operation_counts
LEFT JOIN event_readback ON TRUE
"""
),
{
@@ -77,7 +91,12 @@ async def _load_callback_readback(
return dict(row) if row else None
def _operation_context(payload: Agent99CompletionCallbackRequest) -> dict[str, Any]:
def _operation_context(
payload: Agent99CompletionCallbackRequest,
*,
effective_outcome_state: str,
closure_authority: dict[str, Any],
) -> dict[str, Any]:
return {
"schema_version": payload.schema_version,
"callback_id": payload.callback_id,
@@ -90,7 +109,8 @@ def _operation_context(payload: Agent99CompletionCallbackRequest) -> dict[str, A
"correlation_key": payload.correlation_key,
"source": payload.source,
"mode": payload.mode,
"outcome_state": payload.outcome_state,
"outcome_state": effective_outcome_state,
"claimed_outcome_state": payload.outcome_state,
"controlled_apply": payload.controlled_apply,
"transport_ok": payload.transport_ok,
"verifier_name": payload.verifier_name,
@@ -105,6 +125,17 @@ def _operation_context(payload: Agent99CompletionCallbackRequest) -> dict[str, A
"telegram": payload.telegram.model_dump(mode="json"),
"problem": payload.problem.model_dump(mode="json"),
"occurred_at": payload.occurred_at.isoformat(),
"callback_authority_verified": bool(
closure_authority.get("verified") is True
),
"closure_allowed": bool(closure_authority.get("verified") is True),
"closure_blocker": str(
closure_authority.get("blocker") or "unknown"
)[:120],
"typed_domain": str(
closure_authority.get("typed_domain") or ""
)[:120],
"route_id": str(closure_authority.get("route_id") or "")[:180],
"stores_secret": False,
"raw_log_stored": False,
}
@@ -117,12 +148,139 @@ def _owner_plane(alert_kind: str | None) -> str:
return "awoooi"
def _blocked_closure_authority(blocker: str) -> dict[str, Any]:
return {
"required": True,
"verified": False,
"blocker": blocker,
"typed_domain": "",
"route_id": "",
}
async def _load_callback_closure_authority(
payload: Agent99CompletionCallbackRequest,
) -> dict[str, Any]:
"""Bind a terminal callback to the typed dispatch and verifier ledger."""
receipt = await read_agent99_dispatch_receipt_for_run(
project_id=payload.project_id,
run_id=payload.run_id,
)
if not isinstance(receipt, dict):
return _blocked_closure_authority("dispatch_run_not_found")
public_identity = receipt.get("identity")
if not isinstance(public_identity, dict):
return _blocked_closure_authority("dispatch_identity_missing")
try:
identity = parse_agent99_dispatch_identity(public_identity)
except ValueError:
return _blocked_closure_authority("dispatch_identity_invalid")
expected_incident_ids = {
identity.incident_id,
_operation_incident_id(identity.incident_id),
}
if (
identity.project_id != payload.project_id
or str(identity.run_id) != payload.run_id
or identity.trace_id != payload.trace_id
or identity.work_item_id != payload.work_item_id
or payload.alert_id not in expected_incident_ids
or str(receipt.get("run_id") or "") != payload.run_id
or str(receipt.get("trace_id") or "") != payload.trace_id
):
return _blocked_closure_authority("dispatch_callback_identity_mismatch")
scope = receipt.get("dispatch_scope")
if not isinstance(scope, dict):
return _blocked_closure_authority("dispatch_scope_missing")
typed_domain = str(scope.get("typed_domain") or "")
route_id = str(scope.get("route_id") or "")
expected_verifier = str(scope.get("verifier") or "")
if (
scope.get("schema_version") != "agent99_dispatch_scope_v1"
or typed_domain not in _AGENT99_CLOSURE_DOMAINS
or scope.get("executor") != "Agent99"
or route_id != identity.route_id
or not expected_verifier
):
return _blocked_closure_authority("typed_dispatch_scope_mismatch")
if (
bool(scope.get("controlled_apply_requested") is True)
!= payload.controlled_apply
or (
payload.controlled_apply
and receipt.get("controlled_apply_authorized") is not True
)
or receipt.get("dispatch_promoted") is not True
):
return _blocked_closure_authority("controlled_dispatch_not_authorized")
if (
payload.verifier_name != expected_verifier
or (
str(scope.get("suggested_mode") or "")
and str(scope.get("suggested_mode")) != payload.mode
)
):
return _blocked_closure_authority("callback_route_projection_mismatch")
verifier = receipt.get("verifier")
if not isinstance(verifier, dict):
return _blocked_closure_authority("independent_verifier_missing")
if (
verifier.get("schema_version")
!= "agent99_independent_verifier_receipt_v1"
or verifier.get("status") != "success"
or str(verifier.get("run_id") or "") != payload.run_id
or str(verifier.get("trace_id") or "") != payload.trace_id
or str(verifier.get("work_item_id") or "") != payload.work_item_id
or verifier.get("outcome_state") != "resolved"
or verifier.get("transport_ok") is not True
or verifier.get("verifier_passed") is not True
or verifier.get("source_event_resolved") is not True
or receipt.get("post_verifier_passed") is not True
or str(receipt.get("run_state") or "") not in {"waiting_tool", "completed"}
):
return _blocked_closure_authority("independent_verifier_not_verified")
if (
payload.transport_ok is not True
or payload.verifier_passed is not True
or payload.source_event_resolved is not True
):
return _blocked_closure_authority("callback_verifier_projection_mismatch")
return {
"required": True,
"verified": True,
"blocker": "none",
"typed_domain": typed_domain,
"route_id": route_id,
}
async def record_agent99_completion_callback(
payload: Agent99CompletionCallbackRequest,
) -> dict[str, Any]:
"""Write one callback to the AWOOOI/AwoooP truth chain and read it back."""
stage = payload.outcome_state
closure_authority = (
await _load_callback_closure_authority(payload)
if payload.outcome_state == "resolved"
else {
"required": False,
"verified": False,
"blocker": "not_terminal_claim",
"typed_domain": "",
"route_id": "",
}
)
terminal_resolved = bool(
payload.outcome_state == "resolved"
and closure_authority.get("verified") is True
)
stage = "resolved" if terminal_resolved else (
"verifying" if payload.outcome_state == "resolved" else payload.outcome_state
)
provider_event_id = build_external_alert_provider_event_id(
"agent99",
payload.callback_id,
@@ -133,27 +291,32 @@ async def record_agent99_completion_callback(
provider_event_id=provider_event_id,
callback_id=payload.callback_id,
)
duplicate = before is not None
terminal_resolved = bool(
payload.outcome_state == "resolved"
and payload.verifier_passed
and payload.source_event_resolved
duplicate = bool(
before
and (
before.get("event_id")
or int(before.get("operation_count") or 0) > 0
)
)
severity = (
"info"
if payload.outcome_state == "resolved"
if stage == "resolved"
else "critical"
if payload.outcome_state == "failed"
if stage == "failed"
else "warning"
)
context = _operation_context(payload)
context = _operation_context(
payload,
effective_outcome_state=stage,
closure_authority=closure_authority,
)
operation_incident_id = _operation_incident_id(payload.alert_id)
event_id = await record_external_alert_event(
project_id=payload.project_id,
provider="agent99",
event_id=payload.callback_id,
stage=stage,
title=f"Agent99 {payload.mode} {payload.outcome_state}",
title=f"Agent99 {payload.mode} {stage}",
severity=severity,
namespace="agent99",
target_resource=payload.alert_service or payload.mode,
@@ -162,10 +325,12 @@ async def record_agent99_completion_callback(
labels={
"agent": "agent99",
"mode": payload.mode,
"outcome_state": payload.outcome_state,
"outcome_state": stage,
"claimed_outcome_state": payload.outcome_state,
"verifier_passed": str(payload.verifier_passed).lower(),
"source_event_resolved": str(payload.source_event_resolved).lower(),
"owner_plane": _owner_plane(payload.alert_kind),
"callback_authority_verified": str(terminal_resolved).lower(),
},
annotations={
"run_id": payload.run_id,
@@ -184,7 +349,7 @@ async def record_agent99_completion_callback(
incident_id=operation_incident_id,
actor="agent99_completion_callback",
action_detail=f"{payload.mode}:{payload.outcome_state}",
success=payload.verifier_passed,
success=bool(payload.verifier_passed and stage != "verifying"),
context=context,
)
if terminal_resolved and int((before or {}).get("resolved_count") or 0) == 0:
@@ -212,17 +377,30 @@ async def record_agent99_completion_callback(
and execution_count > 0
and (not terminal_resolved or resolved_count > 0)
)
closure_pending = bool(
payload.outcome_state == "resolved" and not terminal_resolved
)
return {
"schema_version": "agent99_completion_callback_receipt_v1",
"ok": durable_readback,
"status": "duplicate_verified" if duplicate else "recorded_verified",
"ok": bool(durable_readback and not closure_pending),
"status": (
"closure_authority_pending"
if closure_pending
else "duplicate_verified"
if duplicate
else "recorded_verified"
),
"accepted": True,
"duplicate": duplicate,
"callback_id": payload.callback_id,
"run_id": payload.run_id,
"trace_id": payload.trace_id,
"work_item_id": payload.work_item_id,
"outcome_state": payload.outcome_state,
"outcome_state": stage,
"claimed_outcome_state": payload.outcome_state,
"callback_authority_verified": terminal_resolved,
"closure_allowed": terminal_resolved,
"closure_blocker": str(closure_authority.get("blocker") or "unknown"),
"conversation_event_id": str(event_id),
"awooop_run_id": (readback or {}).get("run_id"),
"operation_receipt_count": operation_count,

110
apps/api/src/services/agent99_controlled_dispatch_ledger.py
View File

@@ -1271,6 +1271,57 @@ class PostgresAgent99DispatchLedger:
"runtime_closure_verified": False,
}
dispatch_scope = (
envelope.get("dispatch_scope")
if isinstance(envelope.get("dispatch_scope"), dict)
else {}
)
expected_mode = str(
dispatch_scope.get("suggested_mode") or ""
)
expected_verifier = str(
dispatch_scope.get("verifier") or ""
)
requested_controlled_apply = bool(
dispatch_scope.get("controlled_apply_requested") is True
)
supplied_controlled_apply = outcome_receipt.get(
"controlledApply"
)
if (
dispatch_scope.get("schema_version")
!= "agent99_dispatch_scope_v1"
or dispatch_scope.get("route_id") != identity.route_id
or not str(
dispatch_scope.get("canonical_asset_id") or ""
)
or not str(dispatch_scope.get("typed_domain") or "")
or dispatch_scope.get("executor") != "Agent99"
or not expected_mode
or not expected_verifier
or mode != expected_mode
or str(outcome.get("verifierName") or "")
!= expected_verifier
or not isinstance(supplied_controlled_apply, bool)
or supplied_controlled_apply != requested_controlled_apply
):
return {
"status": "verifier_dispatch_scope_mismatch_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
if (
requested_controlled_apply
and envelope.get("controlled_apply_authorized") is not True
):
return {
"status": (
"verifier_controlled_apply_not_authorized_fail_closed"
),
"receipt_persisted": False,
"runtime_closure_verified": False,
}
if (
current.state in {"pending", "running"}
or envelope.get("dispatch_promoted") is not True
@@ -1282,8 +1333,10 @@ class PostgresAgent99DispatchLedger:
"dispatch_accepted": True,
"inbox_triggered": True,
"dispatch_promoted": True,
# Outcome readback may prove delivery, but it cannot
# retroactively grant mutation authority.
"controlled_apply_authorized": bool(
outcome_receipt.get("controlledApply") is True
envelope.get("controlled_apply_authorized") is True
),
"dispatch_receipt": {
"schema_version": "agent99_sre_dispatch_receipt_v1",
@@ -2476,6 +2529,50 @@ class PostgresAgent99DispatchLedger:
)
return None
async def read_for_run(
self,
*,
project_id: str,
run_id: str,
) -> dict[str, Any] | None:
"""Read one exact Agent99 dispatch run; malformed identities fail closed."""
try:
resolved_run_id = UUID(str(run_id or "").strip())
except (TypeError, ValueError, AttributeError):
return None
try:
async with get_db_context(project_id or "awoooi") as db:
result = await db.execute(
select(
AwoooPRunState.run_id,
AwoooPRunState.state,
AwoooPRunState.trace_id,
AwoooPRunState.error_detail,
).where(
AwoooPRunState.project_id == (project_id or "awoooi"),
AwoooPRunState.agent_id == AGENT99_DISPATCH_AGENT_ID,
AwoooPRunState.run_id == resolved_run_id,
)
)
row = result.one_or_none()
envelope = _parse_envelope(row.error_detail if row else None)
if not isinstance(envelope, dict) or row is None:
return None
return {
**envelope,
"run_state": str(row.state),
"run_id": str(row.run_id),
"trace_id": str(row.trace_id or ""),
}
except Exception as exc:
logger.warning(
"agent99_dispatch_exact_receipt_read_failed",
run_id=str(resolved_run_id),
error=str(exc),
)
return None
async def list_reconcilable(
self,
*,
@@ -2600,6 +2697,17 @@ async def read_agent99_dispatch_receipt(
)
async def read_agent99_dispatch_receipt_for_run(
*,
project_id: str,
run_id: str,
) -> dict[str, Any] | None:
return await _ledger.read_for_run(
project_id=project_id,
run_id=run_id,
)
async def list_agent99_reconcilable_dispatches(
*,
project_id: str = "awoooi",