wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(agent99): reconcile existing run without writes
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 3m23s
Details
CD Pipeline / build-and-deploy (push) Successful in 14m17s
Details
CD Pipeline / post-deploy-checks (push) Successful in 1m57s
Details

Browse Source
This commit is contained in:
ogt
2026-07-14 17:59:43 +08:00
parent 9c44c5ea09
commit f4cf465676
10 changed files with 2633 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py
View File

@@ -22,11 +22,19 @@ from src.services.agent99_controlled_dispatch_ledger import (
Agent99DispatchIdentity,
get_agent99_dispatch_ledger,
record_agent99_learning_writeback,
validate_agent99_outcome_identity,
)
from src.services.agent99_public_receipts import (
extract_agent99_outcome_evidence_refs,
sanitize_agent99_public_receipt_refs,
)
from src.services.agent99_same_run_reconcile import (
AGENT99_SAME_RUN_STATUS_VERIFIER,
agent99_same_run_evidence_id,
is_cold_start_same_run_identity,
read_cold_start_source_resolution,
request_agent99_same_run_status_reconcile,
)
from src.services.agent99_sre_bridge import (
bridge_alertmanager_to_agent99,
read_agent99_sre_outcome,
@@ -46,6 +54,51 @@ LegacyIncidentFetcher = Callable[..., Awaitable[list[dict[str, Any]]]]
Agent99Dispatcher = Callable[..., Awaitable[dict[str, Any]]]
def _is_bound_no_write_status_outcome(
outcome: dict[str, Any] | None,
*,
identity: Agent99DispatchIdentity,
source_receipt_ref: str,
) -> bool:
"""Require the fresh Status outcome to prove its source/no-write fences."""
if not isinstance(outcome, dict):
return False
contract = (
outcome.get("outcome")
if isinstance(outcome.get("outcome"), dict)
else {}
)
return bool(
validate_agent99_outcome_identity(identity, outcome)
and outcome.get("schemaVersion") == "agent99_outcome_readback_v1"
and outcome.get("sameRunContractFieldsComplete") is True
and outcome.get("automationRunId") == str(identity.run_id)
and outcome.get("traceId") == identity.trace_id
and outcome.get("workItemId") == identity.work_item_id
and outcome.get("projectId") == identity.project_id
and outcome.get("incidentId") == identity.incident_id
and outcome.get("routeId") == identity.route_id
and outcome.get("executionGeneration")
== identity.execution_generation
and outcome.get("approvalId") == identity.approval_id
and outcome.get("evidenceReceiptId")
== agent99_same_run_evidence_id(identity)
and contract.get("schemaVersion") == "agent99_outcome_contract_v1"
and outcome.get("mode") == "Status"
and outcome.get("controlledApply") is False
and outcome.get("reconcileOnly") is True
and outcome.get("runtimeWritePerformed") is False
and contract.get("state") == "resolved"
and contract.get("resolved") is True
and contract.get("transportOk") is True
and contract.get("verifierPassed") is True
and contract.get("verifierName") == AGENT99_SAME_RUN_STATUS_VERIFIER
and contract.get("sourceEventResolved") is True
and contract.get("sourceEventEvidence") == source_receipt_ref
)
async def _fetch_legacy_cold_start_failures(
*,
project_id: str,
@@ -535,7 +588,15 @@ async def reconcile_agent99_controlled_dispatches_once(
) -> dict[str, int]:
ledger = get_agent99_dispatch_ledger()
items = await ledger.list_reconcilable(project_id=project_id, limit=limit)
counters = {"scanned": len(items), "verifier_written": 0, "closed": 0}
counters = {
"scanned": len(items),
"source_not_resolved": 0,
"status_reconcile_requested": 0,
"status_reconcile_pending": 0,
"external_no_write_reconciled": 0,
"verifier_written": 0,
"closed": 0,
}
for item in items:
identity = item["identity"]
receipt = item.get("receipt") if isinstance(item.get("receipt"), dict) else {}
@@ -552,6 +613,60 @@ async def reconcile_agent99_controlled_dispatches_once(
else {}
)
}
if is_cold_start_same_run_identity(identity):
source_receipt = await asyncio.to_thread(
read_cold_start_source_resolution,
)
source_receipt_ref = str(source_receipt.get("receipt_ref") or "")
if (
source_receipt.get("resolved") is not True
or not source_receipt_ref
):
counters["source_not_resolved"] += 1
continue
outcome = await asyncio.to_thread(
read_agent99_sre_outcome,
str(identity.run_id),
)
if not _is_bound_no_write_status_outcome(
outcome,
identity=identity,
source_receipt_ref=source_receipt_ref,
):
request_receipt = await asyncio.to_thread(
request_agent99_same_run_status_reconcile,
identity,
source_receipt,
)
if request_receipt.get("accepted") is True:
counters["status_reconcile_requested"] += 1
else:
counters["status_reconcile_pending"] += 1
continue
evidence_refs = extract_agent99_outcome_evidence_refs(
outcome,
{
"agent99_outcome_receipt_id": (
f"agent99-relay:outcome:{identity.run_id}"
),
"post_verifier_evidence_ref": (
"agent99:Status:no-write:"
f"{agent99_same_run_evidence_id(identity)}"
),
"source_event_evidence_ref": source_receipt_ref,
},
)
terminal = await ledger.record_external_no_write_reconciliation(
identity=identity,
outcome_receipt=outcome,
source_receipt=source_receipt,
evidence_refs=evidence_refs,
)
if terminal.get("receipt_persisted") is True:
counters["external_no_write_reconciled"] += 1
# This path is deliberately terminal without automation success,
# incident RESOLVED, Telegram, KM, or PlayBook trust writeback.
continue
if receipt.get("post_verifier_passed") is not True:
outcome = await asyncio.to_thread(
read_agent99_sre_outcome,
@@ -573,8 +688,12 @@ async def reconcile_agent99_controlled_dispatches_once(
"post_verifier_evidence_ref": (
f"agent99:{mode}:verified:{verified_at}"
),
"source_event_evidence_ref": (
f"incident:{identity.incident_id}:source-resolved:{verified_at}"
"source_event_evidence_ref": str(
outcome_contract.get("sourceEventEvidence")
or (
f"incident:{identity.incident_id}:"
f"source-resolved:{verified_at}"
)
),
}
evidence_refs = extract_agent99_outcome_evidence_refs(

375
apps/api/src/services/agent99_controlled_dispatch_ledger.py
View File

@@ -10,6 +10,7 @@ from __future__ import annotations
import hashlib
import json
import math
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -31,6 +32,7 @@ from src.services.agent99_public_receipts import (
AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS,
AGENT99_LEARNING_RECEIPT_REFS,
AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS,
normalize_agent99_public_receipt_ref,
sanitize_agent99_public_receipt_refs,
)
@@ -59,6 +61,19 @@ def _sha256(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _agent99_exact_nonnegative_int(value: Any) -> int | None:
"""Normalize a JSON number only when it is an exact non-negative int."""
if isinstance(value, bool) or not isinstance(value, int | float):
return None
if not math.isfinite(float(value)):
return None
normalized = int(value)
if normalized < 0 or float(value) != float(normalized):
return None
return normalized
@dataclass(frozen=True)
class Agent99DispatchIdentity:
project_id: str
@@ -1329,6 +1344,366 @@ class PostgresAgent99DispatchLedger:
"runtime_closure_verified": False,
}
async def record_external_no_write_reconciliation(
self,
*,
identity: Agent99DispatchIdentity,
outcome_receipt: dict[str, Any],
source_receipt: dict[str, Any],
evidence_refs: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Terminalize an externally recovered run without automation credit.
This is intentionally separate from ``record_verifier`` and
``record_learning_writeback``. A no-write Status proves current
posture only; it never proves that the original Recover execution
succeeded and must not resolve the incident or increase PlayBook
trust.
"""
if not validate_agent99_outcome_identity(identity, outcome_receipt):
return {
"status": "external_recovery_identity_mismatch_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
outcome = (
outcome_receipt.get("outcome")
if isinstance(outcome_receipt.get("outcome"), dict)
else {}
)
expected_top_level = {
"automationRunId": str(identity.run_id),
"traceId": identity.trace_id,
"workItemId": identity.work_item_id,
"projectId": identity.project_id,
"incidentId": identity.incident_id,
"routeId": identity.route_id,
"executionGeneration": identity.execution_generation,
"approvalId": identity.approval_id,
}
if any(
str(outcome_receipt.get(key) or "") != expected
for key, expected in expected_top_level.items()
):
return {
"status": "external_recovery_top_level_identity_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
source_ref = normalize_agent99_public_receipt_ref(
source_receipt.get("receipt_ref")
)
source_values = (
source_receipt.get("values")
if isinstance(source_receipt.get("values"), dict)
else {}
)
source_result = str(source_receipt.get("source_result") or "")
source_counts = {
key: _agent99_exact_nonnegative_int(source_values.get(key))
for key in (
"monitor_up",
"pass_gates",
"warn_gates",
"blocked_gates",
"green_result",
"degraded_result",
"blocked_result",
"check_failed_result",
"firing_blocking_alerts",
)
}
source_age = source_receipt.get("age_seconds")
source_age_exact = bool(
not isinstance(source_age, bool)
and isinstance(source_age, int | float)
and float(source_age) >= -60
and float(source_age) <= 900
)
source_observed_at = _agent99_exact_nonnegative_int(
source_receipt.get("observed_at_epoch")
)
expected_evidence_id = (
f"same-run-status-{identity.run_id}-"
f"{identity.public_dict()['canonical_digest'][:12]}"
)
exact_outcome = bool(
outcome_receipt.get("schemaVersion")
== "agent99_outcome_readback_v1"
and outcome_receipt.get("sameRunContractFieldsComplete") is True
and outcome_receipt.get("mode") == "Status"
and outcome_receipt.get("controlledApply") is False
and outcome_receipt.get("reconcileOnly") is True
and outcome_receipt.get("runtimeWritePerformed") is False
and outcome_receipt.get("evidenceReceiptId")
== expected_evidence_id
and outcome.get("schemaVersion") == "agent99_outcome_contract_v1"
and outcome.get("state") == "resolved"
and outcome.get("resolved") is True
and outcome.get("transportOk") is True
and outcome.get("verifierPassed") is True
and outcome.get("verifierName")
== "agent99-status-same-run-no-write-post-condition-v1"
and outcome.get("sourceEventResolved") is True
and source_ref is not None
and outcome.get("sourceEventEvidence") == source_ref
)
source_exact = bool(
source_receipt.get("schema_version")
== "agent99_cold_start_source_resolution_v1"
and source_receipt.get("resolved") is True
and source_result in {"green", "degraded"}
and source_receipt.get("degraded_provenance")
is (source_result == "degraded")
and source_age_exact
and source_observed_at is not None
and source_observed_at > 0
and source_counts["monitor_up"] == 1
and source_counts["pass_gates"] is not None
and source_counts["pass_gates"] > 0
and source_counts["warn_gates"] is not None
and source_counts["blocked_gates"] == 0
and source_counts["blocked_result"] == 0
and source_counts["check_failed_result"] == 0
and source_counts["firing_blocking_alerts"] == 0
and (
(
source_result == "degraded"
and source_counts["warn_gates"] > 0
and source_counts["green_result"] == 0
and source_counts["degraded_result"] == 1
)
or (
source_result == "green"
and source_counts["warn_gates"] == 0
and source_counts["green_result"] == 1
and source_counts["degraded_result"] == 0
)
)
)
safe_refs = _safe_receipt_refs(evidence_refs)
missing_refs = sorted(
set(AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS) - set(safe_refs)
)
if not exact_outcome or not source_exact or missing_refs:
return {
"status": "external_recovery_contract_invalid_fail_closed",
"missing_evidence_refs": missing_refs,
"receipt_persisted": False,
"runtime_closure_verified": False,
}
if safe_refs.get("source_event_evidence_ref") != source_ref:
return {
"status": "external_recovery_source_receipt_mismatch_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
now = _utc_now_naive()
terminal_receipt = {
"schema_version": (
"agent99_external_recovery_no_write_reconciliation_v1"
),
"status": "external_recovery_no_write_reconciled",
**_stage_identity(identity),
"mode": "Status",
"verifier_name": (
"agent99-status-same-run-no-write-post-condition-v1"
),
"evidence_receipt_id": expected_evidence_id,
"source": {
"receipt_ref": source_ref,
"result": source_result,
"degraded_provenance": bool(
source_receipt.get("degraded_provenance")
),
"pass_gates": source_counts["pass_gates"],
"warn_gates": source_counts["warn_gates"],
"blocked_gates": 0,
"age_seconds": source_age,
"observed_at_epoch": source_observed_at,
},
"transport_ok": True,
"status_verifier_passed": True,
"runtime_write_performed": False,
"recover_execution_success": False,
"incident_resolution_authorized": False,
"learning_writeback_authorized": False,
"telegram_authorized": False,
"evidence_refs": safe_refs,
"stores_raw_evidence": False,
}
try:
async with get_db_context(identity.project_id) as db:
current_result = await db.execute(
select(AwoooPRunState.state, AwoooPRunState.error_detail)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
)
.with_for_update()
)
current = current_result.one_or_none()
envelope = _parse_envelope(
current.error_detail if current is not None else None
)
if (
current is not None
and current.state == "cancelled"
and isinstance(envelope, dict)
and envelope.get("external_no_write_reconciliation_terminal")
is True
and _stage_identity_matches(
identity,
envelope.get("identity")
if isinstance(envelope.get("identity"), dict)
else {},
)
):
return {
**envelope,
"status": (
"external_recovery_no_write_terminal_idempotent"
),
"receipt_persisted": True,
"runtime_closure_verified": False,
}
if (
current is None
or current.state
not in {"pending", "running", "waiting_tool"}
or not isinstance(envelope, dict)
or not _stage_identity_matches(
identity,
envelope.get("identity")
if isinstance(envelope.get("identity"), dict)
else {},
)
):
return {
"status": (
"external_recovery_run_not_ready_fail_closed"
),
"receipt_persisted": False,
"runtime_closure_verified": False,
}
prior_runtime_execution_attempted = bool(
envelope.get("runtime_execution_attempted") is True
)
envelope.update({
"status": "external_recovery_no_write_reconciled_terminal",
"run_terminal_state": "cancelled",
"controlled_apply_authorized": False,
"runtime_execution_authorized": False,
"runtime_execution_attempted": (
prior_runtime_execution_attempted
),
"same_run_status_runtime_execution_attempted": False,
"runtime_write_performed": False,
"automation_execution_success": False,
"post_verifier_passed": False,
"external_status_verifier_passed": True,
"external_recovery_source_resolved": True,
"runtime_closure_verified": False,
"closure_complete": False,
"incident_resolution_authorized": False,
"external_no_write_reconciliation_terminal": True,
"external_recovery_reconciliation": terminal_receipt,
"learning_writeback": {
**_stage_identity(identity),
"step_seq": 3,
"status": (
"not_applicable_external_recovery_no_write"
),
"required_receipts": [],
"receipt_refs": {},
"stores_raw_evidence": False,
},
})
envelope_json = _stable_json(envelope)
updated = await db.execute(
update(AwoooPRunState)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
AwoooPRunState.state.in_([
"pending",
"running",
"waiting_tool",
]),
)
.values(
state="cancelled",
output_sha256=_sha256(envelope_json),
error_code=None,
error_detail=envelope_json,
heartbeat_at=now,
completed_at=now,
lease_until=None,
next_attempt_at=None,
worker_id=None,
)
.returning(AwoooPRunState.run_id)
)
persisted = updated.scalar_one_or_none() is not None
if persisted:
journal_states = {
1: (
"compensated",
True,
"external_recovery_superseded_runtime_execution",
),
2: ("success", False, None),
3: (
"compensated",
True,
"learning_not_applicable_external_recovery_no_write",
),
}
for step_seq, (
result_status,
was_blocked,
block_reason,
) in journal_states.items():
await db.execute(
update(AwoooPRunStepJournal)
.where(
AwoooPRunStepJournal.run_id == identity.run_id,
AwoooPRunStepJournal.step_seq == step_seq,
)
.values(
output_hash=_sha256(
_stable_json(terminal_receipt)
),
compensation_json=terminal_receipt,
result_status=result_status,
error_code=None,
was_blocked=was_blocked,
block_reason=block_reason,
completed_at=now,
)
)
except Exception as exc:
logger.warning(
"agent99_external_no_write_reconciliation_failed",
incident_id=identity.incident_id,
run_id=str(identity.run_id),
error=str(exc),
)
return {
"status": "external_recovery_persistence_failed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
return {
**envelope,
"receipt_persisted": persisted,
"runtime_closure_verified": False,
}
async def record_learning_writeback(
self,
*,

358
apps/api/src/services/agent99_same_run_reconcile.py Normal file
View File

@@ -0,0 +1,358 @@
"""Fail-closed, no-write Agent99 same-run Status reconciliation.
This module deliberately keeps the production source verifier separate from
the Agent99 verifier. A missing historical Recover outcome may be refreshed
only after the live cold-start source reports a fresh, unblocked state with no
firing ColdStart alert. The refresh keeps the existing deterministic
dispatch identity and asks the authenticated relay for ``Status`` in
``reconcileOnly`` mode; it never reserves a new run, advances a generation,
or authorizes controlled apply.
"""
from __future__ import annotations
import json
import math
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
from src.core.config import settings
from src.core.logging import get_logger
from src.services.agent99_controlled_dispatch_ledger import (
Agent99DispatchIdentity,
)
from src.services.agent99_public_receipts import (
normalize_agent99_public_receipt_ref,
)
logger = get_logger("awoooi.agent99_same_run_reconcile")
AGENT99_SAME_RUN_RECONCILE_SCHEMA = "agent99_same_run_status_reconcile_v1"
AGENT99_COLD_START_ROUTE_ID = "agent99:host_recovery:Recover"
AGENT99_COLD_START_INCIDENT_ID = "INC-20260711-11C751"
AGENT99_COLD_START_RUN_ID = "bf30ca01-1080-5725-b2d1-3e1534dfc811"
AGENT99_SAME_RUN_STATUS_VERIFIER = (
"agent99-status-same-run-no-write-post-condition-v1"
)
AGENT99_COLD_START_METRIC_HOST = "110"
AGENT99_COLD_START_METRIC_SCOPE = "110_120_121_188"
AGENT99_COLD_START_MAX_AGE_SECONDS = 900
_COLD_START_QUERIES = {
"monitor_up": (
'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",'
'mode="read_only"}'
),
"pass_gates": (
'awoooi_cold_start_pass_gates{host="110",scope="110_120_121_188"}'
),
"warn_gates": (
'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}'
),
"blocked_gates": (
'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}'
),
"last_run_timestamp": (
'awoooi_cold_start_last_run_timestamp{host="110",'
'scope="110_120_121_188"}'
),
"green_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="green"}'
),
"degraded_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="degraded"}'
),
"blocked_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="blocked"}'
),
"check_failed_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="check_failed"}'
),
"firing_blocking_alerts": (
'sum(ALERTS{alertname=~"ColdStart.*",alertstate="firing",'
'severity="critical"}) or vector(0)'
),
}
def is_cold_start_same_run_identity(identity: Agent99DispatchIdentity) -> bool:
"""Limit this transport to the existing cold-start Recover identity."""
return bool(
identity.route_id == AGENT99_COLD_START_ROUTE_ID
and identity.incident_id == AGENT99_COLD_START_INCIDENT_ID
and str(identity.run_id) == AGENT99_COLD_START_RUN_ID
and identity.source_fingerprint.startswith("legacy-cold-start:")
and identity.execution_generation == "1"
)
def agent99_same_run_evidence_id(identity: Agent99DispatchIdentity) -> str:
"""Return the only evidence identifier accepted for this bounded run."""
digest = identity.public_dict()["canonical_digest"]
return f"same-run-status-{identity.run_id}-{digest[:12]}"
def _query_prometheus_scalar(
query_url: str,
query: str,
*,
timeout_seconds: float,
) -> float | None:
target = f"{query_url}?{urllib.parse.urlencode({'query': query})}"
request = urllib.request.Request(
target,
headers={
"Accept": "application/json",
"User-Agent": "awoooi-agent99-source-verifier/1",
},
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
if int(response.getcode()) != 200:
return None
body = response.read(65536).decode("utf-8", errors="replace")
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError):
return None
try:
payload = json.loads(body)
data = payload.get("data") if isinstance(payload, dict) else None
result = data.get("result") if isinstance(data, dict) else None
if payload.get("status") != "success" or not isinstance(result, list):
return None
if len(result) != 1 or not isinstance(result[0], dict):
return None
sample = result[0].get("value")
if not isinstance(sample, list) or len(sample) != 2:
return None
value = float(sample[1])
return value if math.isfinite(value) else None
except (AttributeError, TypeError, ValueError, json.JSONDecodeError):
return None
def read_cold_start_source_resolution(
*,
prometheus_url: str | None = None,
timeout_seconds: float = 3.0,
now_epoch: float | None = None,
) -> dict[str, Any]:
"""Read a bounded production source receipt; missing evidence fails closed."""
base_url = str(prometheus_url or settings.PROMETHEUS_URL).rstrip("/")
query_url = f"{base_url}/api/v1/query"
values: dict[str, float] = {}
missing: list[str] = []
for name, query in _COLD_START_QUERIES.items():
value = _query_prometheus_scalar(
query_url,
query,
timeout_seconds=timeout_seconds,
)
if value is None:
missing.append(name)
else:
values[name] = value
blockers: list[str] = []
if missing:
blockers.append("source_metrics_missing:" + ",".join(sorted(missing)))
observed_at = float(now_epoch if now_epoch is not None else time.time())
last_run = values.get("last_run_timestamp")
age_seconds: float | None = None
if last_run is not None:
age_seconds = observed_at - last_run
if age_seconds < -60:
blockers.append("source_clock_skew_future")
elif age_seconds > AGENT99_COLD_START_MAX_AGE_SECONDS:
blockers.append("source_metrics_stale")
if values.get("monitor_up") != 1:
blockers.append("cold_start_monitor_not_up")
for count_name in ("pass_gates", "warn_gates", "blocked_gates"):
count_value = values.get(count_name)
if count_value is not None and (
count_value < 0 or not count_value.is_integer()
):
blockers.append(f"cold_start_{count_name}_invalid")
if values.get("pass_gates", 0) <= 0:
blockers.append("cold_start_pass_gates_missing")
if values.get("blocked_gates") != 0:
blockers.append("cold_start_blocked_gates_present")
if values.get("blocked_result") != 0:
blockers.append("cold_start_result_blocked")
if values.get("check_failed_result") != 0:
blockers.append("cold_start_result_check_failed")
warn_gates = values.get("warn_gates")
expected_result = "degraded" if warn_gates is not None and warn_gates > 0 else "green"
if expected_result == "degraded":
if values.get("degraded_result") != 1 or values.get("green_result") != 0:
blockers.append("cold_start_degraded_result_not_one_hot")
elif values.get("green_result") != 1 or values.get("degraded_result") != 0:
blockers.append("cold_start_green_result_not_one_hot")
if values.get("firing_blocking_alerts") != 0:
blockers.append("cold_start_blocking_alert_still_firing")
resolved = bool(not blockers and last_run is not None)
receipt_ref: str | None = None
if resolved:
receipt_ref = normalize_agent99_public_receipt_ref(
"prometheus:cold-start:"
f"{int(last_run)}:pass-{int(values['pass_gates'])}:"
f"warn-{int(values['warn_gates'])}:blocked-0:"
f"result-{expected_result}:blocking-alerts-0"
)
if receipt_ref is None:
blockers.append("source_receipt_ref_invalid")
resolved = False
return {
"schema_version": "agent99_cold_start_source_resolution_v1",
"status": "source_resolved" if resolved else "source_not_resolved",
"resolved": resolved,
"source_result": expected_result if resolved else "unresolved",
"degraded_provenance": bool(resolved and expected_result == "degraded"),
"receipt_ref": receipt_ref,
"observed_at_epoch": int(observed_at),
"age_seconds": round(age_seconds, 3) if age_seconds is not None else None,
"values": {
key: int(value) if float(value).is_integer() else value
for key, value in sorted(values.items())
},
"blockers": blockers,
"stores_raw_evidence": False,
}
def request_agent99_same_run_status_reconcile(
identity: Agent99DispatchIdentity,
source_receipt: dict[str, Any],
*,
relay_url: str | None = None,
relay_token: str | None = None,
timeout_seconds: float | None = None,
) -> dict[str, Any]:
"""Ask Agent99 for one idempotent, no-write Status on the existing run."""
if not is_cold_start_same_run_identity(identity):
return {
"status": "identity_not_eligible_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
receipt_ref = normalize_agent99_public_receipt_ref(
source_receipt.get("receipt_ref")
)
if source_receipt.get("resolved") is not True or not receipt_ref:
return {
"status": "source_not_resolved_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
if not receipt_ref.startswith("prometheus:cold-start:"):
return {
"status": "source_receipt_not_cold_start_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
target_url = str(
relay_url
if relay_url is not None
else settings.AGENT99_SRE_ALERT_RELAY_URL
)
token = str(
relay_token
if relay_token is not None
else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
)
if not target_url or not token:
return {
"status": "relay_auth_not_configured_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
timeout = float(
timeout_seconds
if timeout_seconds is not None
else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS
)
payload = {
"schemaVersion": AGENT99_SAME_RUN_RECONCILE_SCHEMA,
"action": "same_run_status_reconcile",
"identity": identity.public_dict(),
"mode": "Status",
"controlledApply": False,
"reconcileOnly": True,
"sourceEventResolved": True,
"sourceEventEvidence": receipt_ref,
"sourceEventResolutionPolicy": "external_source_receipt_required",
"suppressTelegram": True,
"evidenceReceiptId": agent99_same_run_evidence_id(identity),
}
request = urllib.request.Request(
target_url,
data=json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8"),
headers={
"Accept": "application/json",
"Content-Type": "application/json; charset=utf-8",
"User-Agent": "awoooi-agent99-same-run-reconciler/1",
"X-Agent99-Relay-Token": token,
},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
status_code = int(response.getcode())
body = response.read(65536).decode("utf-8", errors="replace")
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
logger.info(
"agent99_same_run_status_reconcile_pending",
run_id=str(identity.run_id),
error_type=type(exc).__name__,
)
return {
"status": "relay_request_failed_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
try:
response_payload = json.loads(body)
except json.JSONDecodeError:
response_payload = {}
accepted = bool(
200 <= status_code < 300
and isinstance(response_payload, dict)
and response_payload.get("ok") is True
and response_payload.get("reconcileOnly") is True
and response_payload.get("controlledApply") is False
and str(response_payload.get("automationRunId") or "")
== str(identity.run_id)
and str(response_payload.get("evidenceReceiptId") or "")
== agent99_same_run_evidence_id(identity)
)
return {
"schema_version": "agent99_same_run_status_reconcile_request_receipt_v1",
"status": (
str(response_payload.get("status") or "accepted")
if accepted
else "relay_response_rejected_fail_closed"
),
"accepted": accepted,
"http_status": status_code,
"identity": identity.public_dict(),
"source_event_evidence_ref": receipt_ref,
"evidence_receipt_id": agent99_same_run_evidence_id(identity),
"reconcile_only": True,
"controlled_apply": False,
"runtime_write_authorized": False,
"stores_raw_response": False,
}

86
apps/api/src/services/agent99_sre_bridge.py
View File

@@ -21,6 +21,7 @@ from src.services.agent99_controlled_dispatch_ledger import (
)
from src.services.agent99_public_receipts import (
extract_agent99_outcome_evidence_refs,
normalize_agent99_public_receipt_ref,
)
from src.utils.timezone import now_taipei
@@ -697,17 +698,101 @@ def read_agent99_sre_outcome(
):
return None
safe_evidence_refs = extract_agent99_outcome_evidence_refs(payload)
source_event_evidence = normalize_agent99_public_receipt_ref(
outcome.get("sourceEventEvidence")
)
expected_evidence_receipt_id = (
f"same-run-status-{identity.run_id}-"
f"{identity.public_dict()['canonical_digest'][:12]}"
)
evidence_receipt_id = str(payload.get("evidenceReceiptId") or "")
if evidence_receipt_id != expected_evidence_receipt_id:
evidence_receipt_id = ""
same_run_contract_fields_complete = bool(
payload.get("sameRunContractFieldsComplete") is True
and all(
key in payload
for key in (
"automationRunId",
"traceId",
"workItemId",
"projectId",
"incidentId",
"routeId",
"executionGeneration",
"approvalId",
"evidenceReceiptId",
"controlledApply",
"reconcileOnly",
"runtimeWritePerformed",
"mode",
)
)
and all(
isinstance(payload.get(key), str)
for key in (
"automationRunId",
"traceId",
"workItemId",
"projectId",
"incidentId",
"routeId",
"executionGeneration",
"approvalId",
"evidenceReceiptId",
"mode",
)
)
and isinstance(payload.get("controlledApply"), bool)
and isinstance(payload.get("reconcileOnly"), bool)
and isinstance(payload.get("runtimeWritePerformed"), bool)
and all(
key in outcome
for key in (
"schemaVersion",
"state",
"resolved",
"transportOk",
"verifierName",
"verifierPassed",
"sourceEventResolved",
"sourceEventEvidence",
"identity",
)
)
and all(
isinstance(outcome.get(key), str)
for key in (
"schemaVersion",
"state",
"verifierName",
"sourceEventEvidence",
)
)
and isinstance(outcome.get("resolved"), bool)
and isinstance(outcome.get("transportOk"), bool)
and isinstance(outcome.get("verifierPassed"), bool)
and isinstance(outcome.get("sourceEventResolved"), bool)
)
return {
"schemaVersion": "agent99_outcome_readback_v1",
"automationRunId": str(payload.get("automationRunId") or ""),
"traceId": str(payload.get("traceId") or ""),
"workItemId": str(payload.get("workItemId") or ""),
"projectId": str(payload.get("projectId") or ""),
"routeId": str(payload.get("routeId") or ""),
"idempotencyKey": str(payload.get("idempotencyKey") or ""),
"executionGeneration": str(payload.get("executionGeneration") or "1"),
"incidentId": str(payload.get("incidentId") or ""),
"approvalId": str(payload.get("approvalId") or ""),
"evidenceReceiptId": evidence_receipt_id,
"sameRunContractFieldsComplete": same_run_contract_fields_complete,
"identity": identity.public_dict(),
"controlledApply": bool(payload.get("controlledApply") is True),
"reconcileOnly": bool(payload.get("reconcileOnly") is True),
"runtimeWritePerformed": bool(
payload.get("runtimeWritePerformed") is True
),
"mode": str(payload.get("mode") or ""),
"outcome": {
"schemaVersion": str(outcome.get("schemaVersion") or ""),
@@ -719,6 +804,7 @@ def read_agent99_sre_outcome(
"sourceEventResolved": bool(
outcome.get("sourceEventResolved") is True
),
"sourceEventEvidence": source_event_evidence or "",
"failedChecks": [
str(value)[:120]
for value in (outcome.get("failedChecks") or [])[:32]