wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent99): reconcile existing run without writes
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 3m23s
Details
CD Pipeline / build-and-deploy (push) Successful in 14m17s
Details
CD Pipeline / post-deploy-checks (push) Successful in 1m57s
Details

Browse Source
This commit is contained in:
ogt
2026-07-14 17:59:43 +08:00
parent 9c44c5ea09
commit f4cf465676
10 changed files with 2633 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

375
apps/api/src/services/agent99_controlled_dispatch_ledger.py
View File

@@ -10,6 +10,7 @@ from __future__ import annotations
import hashlib
import json
import math
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -31,6 +32,7 @@ from src.services.agent99_public_receipts import (
AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS,
AGENT99_LEARNING_RECEIPT_REFS,
AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS,
normalize_agent99_public_receipt_ref,
sanitize_agent99_public_receipt_refs,
)
@@ -59,6 +61,19 @@ def _sha256(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _agent99_exact_nonnegative_int(value: Any) -> int | None:
"""Normalize a JSON number only when it is an exact non-negative int."""
if isinstance(value, bool) or not isinstance(value, int | float):
return None
if not math.isfinite(float(value)):
return None
normalized = int(value)
if normalized < 0 or float(value) != float(normalized):
return None
return normalized
@dataclass(frozen=True)
class Agent99DispatchIdentity:
project_id: str
@@ -1329,6 +1344,366 @@ class PostgresAgent99DispatchLedger:
"runtime_closure_verified": False,
}
async def record_external_no_write_reconciliation(
self,
*,
identity: Agent99DispatchIdentity,
outcome_receipt: dict[str, Any],
source_receipt: dict[str, Any],
evidence_refs: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Terminalize an externally recovered run without automation credit.
This is intentionally separate from ``record_verifier`` and
``record_learning_writeback``. A no-write Status proves current
posture only; it never proves that the original Recover execution
succeeded and must not resolve the incident or increase PlayBook
trust.
"""
if not validate_agent99_outcome_identity(identity, outcome_receipt):
return {
"status": "external_recovery_identity_mismatch_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
outcome = (
outcome_receipt.get("outcome")
if isinstance(outcome_receipt.get("outcome"), dict)
else {}
)
expected_top_level = {
"automationRunId": str(identity.run_id),
"traceId": identity.trace_id,
"workItemId": identity.work_item_id,
"projectId": identity.project_id,
"incidentId": identity.incident_id,
"routeId": identity.route_id,
"executionGeneration": identity.execution_generation,
"approvalId": identity.approval_id,
}
if any(
str(outcome_receipt.get(key) or "") != expected
for key, expected in expected_top_level.items()
):
return {
"status": "external_recovery_top_level_identity_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
source_ref = normalize_agent99_public_receipt_ref(
source_receipt.get("receipt_ref")
)
source_values = (
source_receipt.get("values")
if isinstance(source_receipt.get("values"), dict)
else {}
)
source_result = str(source_receipt.get("source_result") or "")
source_counts = {
key: _agent99_exact_nonnegative_int(source_values.get(key))
for key in (
"monitor_up",
"pass_gates",
"warn_gates",
"blocked_gates",
"green_result",
"degraded_result",
"blocked_result",
"check_failed_result",
"firing_blocking_alerts",
)
}
source_age = source_receipt.get("age_seconds")
source_age_exact = bool(
not isinstance(source_age, bool)
and isinstance(source_age, int | float)
and float(source_age) >= -60
and float(source_age) <= 900
)
source_observed_at = _agent99_exact_nonnegative_int(
source_receipt.get("observed_at_epoch")
)
expected_evidence_id = (
f"same-run-status-{identity.run_id}-"
f"{identity.public_dict()['canonical_digest'][:12]}"
)
exact_outcome = bool(
outcome_receipt.get("schemaVersion")
== "agent99_outcome_readback_v1"
and outcome_receipt.get("sameRunContractFieldsComplete") is True
and outcome_receipt.get("mode") == "Status"
and outcome_receipt.get("controlledApply") is False
and outcome_receipt.get("reconcileOnly") is True
and outcome_receipt.get("runtimeWritePerformed") is False
and outcome_receipt.get("evidenceReceiptId")
== expected_evidence_id
and outcome.get("schemaVersion") == "agent99_outcome_contract_v1"
and outcome.get("state") == "resolved"
and outcome.get("resolved") is True
and outcome.get("transportOk") is True
and outcome.get("verifierPassed") is True
and outcome.get("verifierName")
== "agent99-status-same-run-no-write-post-condition-v1"
and outcome.get("sourceEventResolved") is True
and source_ref is not None
and outcome.get("sourceEventEvidence") == source_ref
)
source_exact = bool(
source_receipt.get("schema_version")
== "agent99_cold_start_source_resolution_v1"
and source_receipt.get("resolved") is True
and source_result in {"green", "degraded"}
and source_receipt.get("degraded_provenance")
is (source_result == "degraded")
and source_age_exact
and source_observed_at is not None
and source_observed_at > 0
and source_counts["monitor_up"] == 1
and source_counts["pass_gates"] is not None
and source_counts["pass_gates"] > 0
and source_counts["warn_gates"] is not None
and source_counts["blocked_gates"] == 0
and source_counts["blocked_result"] == 0
and source_counts["check_failed_result"] == 0
and source_counts["firing_blocking_alerts"] == 0
and (
(
source_result == "degraded"
and source_counts["warn_gates"] > 0
and source_counts["green_result"] == 0
and source_counts["degraded_result"] == 1
)
or (
source_result == "green"
and source_counts["warn_gates"] == 0
and source_counts["green_result"] == 1
and source_counts["degraded_result"] == 0
)
)
)
safe_refs = _safe_receipt_refs(evidence_refs)
missing_refs = sorted(
set(AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS) - set(safe_refs)
)
if not exact_outcome or not source_exact or missing_refs:
return {
"status": "external_recovery_contract_invalid_fail_closed",
"missing_evidence_refs": missing_refs,
"receipt_persisted": False,
"runtime_closure_verified": False,
}
if safe_refs.get("source_event_evidence_ref") != source_ref:
return {
"status": "external_recovery_source_receipt_mismatch_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
now = _utc_now_naive()
terminal_receipt = {
"schema_version": (
"agent99_external_recovery_no_write_reconciliation_v1"
),
"status": "external_recovery_no_write_reconciled",
**_stage_identity(identity),
"mode": "Status",
"verifier_name": (
"agent99-status-same-run-no-write-post-condition-v1"
),
"evidence_receipt_id": expected_evidence_id,
"source": {
"receipt_ref": source_ref,
"result": source_result,
"degraded_provenance": bool(
source_receipt.get("degraded_provenance")
),
"pass_gates": source_counts["pass_gates"],
"warn_gates": source_counts["warn_gates"],
"blocked_gates": 0,
"age_seconds": source_age,
"observed_at_epoch": source_observed_at,
},
"transport_ok": True,
"status_verifier_passed": True,
"runtime_write_performed": False,
"recover_execution_success": False,
"incident_resolution_authorized": False,
"learning_writeback_authorized": False,
"telegram_authorized": False,
"evidence_refs": safe_refs,
"stores_raw_evidence": False,
}
try:
async with get_db_context(identity.project_id) as db:
current_result = await db.execute(
select(AwoooPRunState.state, AwoooPRunState.error_detail)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
)
.with_for_update()
)
current = current_result.one_or_none()
envelope = _parse_envelope(
current.error_detail if current is not None else None
)
if (
current is not None
and current.state == "cancelled"
and isinstance(envelope, dict)
and envelope.get("external_no_write_reconciliation_terminal")
is True
and _stage_identity_matches(
identity,
envelope.get("identity")
if isinstance(envelope.get("identity"), dict)
else {},
)
):
return {
**envelope,
"status": (
"external_recovery_no_write_terminal_idempotent"
),
"receipt_persisted": True,
"runtime_closure_verified": False,
}
if (
current is None
or current.state
not in {"pending", "running", "waiting_tool"}
or not isinstance(envelope, dict)
or not _stage_identity_matches(
identity,
envelope.get("identity")
if isinstance(envelope.get("identity"), dict)
else {},
)
):
return {
"status": (
"external_recovery_run_not_ready_fail_closed"
),
"receipt_persisted": False,
"runtime_closure_verified": False,
}
prior_runtime_execution_attempted = bool(
envelope.get("runtime_execution_attempted") is True
)
envelope.update({
"status": "external_recovery_no_write_reconciled_terminal",
"run_terminal_state": "cancelled",
"controlled_apply_authorized": False,
"runtime_execution_authorized": False,
"runtime_execution_attempted": (
prior_runtime_execution_attempted
),
"same_run_status_runtime_execution_attempted": False,
"runtime_write_performed": False,
"automation_execution_success": False,
"post_verifier_passed": False,
"external_status_verifier_passed": True,
"external_recovery_source_resolved": True,
"runtime_closure_verified": False,
"closure_complete": False,
"incident_resolution_authorized": False,
"external_no_write_reconciliation_terminal": True,
"external_recovery_reconciliation": terminal_receipt,
"learning_writeback": {
**_stage_identity(identity),
"step_seq": 3,
"status": (
"not_applicable_external_recovery_no_write"
),
"required_receipts": [],
"receipt_refs": {},
"stores_raw_evidence": False,
},
})
envelope_json = _stable_json(envelope)
updated = await db.execute(
update(AwoooPRunState)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
AwoooPRunState.state.in_([
"pending",
"running",
"waiting_tool",
]),
)
.values(
state="cancelled",
output_sha256=_sha256(envelope_json),
error_code=None,
error_detail=envelope_json,
heartbeat_at=now,
completed_at=now,
lease_until=None,
next_attempt_at=None,
worker_id=None,
)
.returning(AwoooPRunState.run_id)
)
persisted = updated.scalar_one_or_none() is not None
if persisted:
journal_states = {
1: (
"compensated",
True,
"external_recovery_superseded_runtime_execution",
),
2: ("success", False, None),
3: (
"compensated",
True,
"learning_not_applicable_external_recovery_no_write",
),
}
for step_seq, (
result_status,
was_blocked,
block_reason,
) in journal_states.items():
await db.execute(
update(AwoooPRunStepJournal)
.where(
AwoooPRunStepJournal.run_id == identity.run_id,
AwoooPRunStepJournal.step_seq == step_seq,
)
.values(
output_hash=_sha256(
_stable_json(terminal_receipt)
),
compensation_json=terminal_receipt,
result_status=result_status,
error_code=None,
was_blocked=was_blocked,
block_reason=block_reason,
completed_at=now,
)
)
except Exception as exc:
logger.warning(
"agent99_external_no_write_reconciliation_failed",
incident_id=identity.incident_id,
run_id=str(identity.run_id),
error=str(exc),
)
return {
"status": "external_recovery_persistence_failed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
return {
**envelope,
"receipt_persisted": persisted,
"runtime_closure_verified": False,
}
async def record_learning_writeback(
self,
*,