wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(agent99): reconcile existing run without writes
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 3m23s
Details
CD Pipeline / build-and-deploy (push) Successful in 14m17s
Details
CD Pipeline / post-deploy-checks (push) Successful in 1m57s
Details

Browse Source
This commit is contained in:
ogt
2026-07-14 17:59:43 +08:00
parent 9c44c5ea09
commit f4cf465676
10 changed files with 2633 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py
View File

@@ -22,11 +22,19 @@ from src.services.agent99_controlled_dispatch_ledger import (
Agent99DispatchIdentity,
get_agent99_dispatch_ledger,
record_agent99_learning_writeback,
validate_agent99_outcome_identity,
)
from src.services.agent99_public_receipts import (
extract_agent99_outcome_evidence_refs,
sanitize_agent99_public_receipt_refs,
)
from src.services.agent99_same_run_reconcile import (
AGENT99_SAME_RUN_STATUS_VERIFIER,
agent99_same_run_evidence_id,
is_cold_start_same_run_identity,
read_cold_start_source_resolution,
request_agent99_same_run_status_reconcile,
)
from src.services.agent99_sre_bridge import (
bridge_alertmanager_to_agent99,
read_agent99_sre_outcome,
@@ -46,6 +54,51 @@ LegacyIncidentFetcher = Callable[..., Awaitable[list[dict[str, Any]]]]
Agent99Dispatcher = Callable[..., Awaitable[dict[str, Any]]]
def _is_bound_no_write_status_outcome(
outcome: dict[str, Any] | None,
*,
identity: Agent99DispatchIdentity,
source_receipt_ref: str,
) -> bool:
"""Require the fresh Status outcome to prove its source/no-write fences."""
if not isinstance(outcome, dict):
return False
contract = (
outcome.get("outcome")
if isinstance(outcome.get("outcome"), dict)
else {}
)
return bool(
validate_agent99_outcome_identity(identity, outcome)
and outcome.get("schemaVersion") == "agent99_outcome_readback_v1"
and outcome.get("sameRunContractFieldsComplete") is True
and outcome.get("automationRunId") == str(identity.run_id)
and outcome.get("traceId") == identity.trace_id
and outcome.get("workItemId") == identity.work_item_id
and outcome.get("projectId") == identity.project_id
and outcome.get("incidentId") == identity.incident_id
and outcome.get("routeId") == identity.route_id
and outcome.get("executionGeneration")
== identity.execution_generation
and outcome.get("approvalId") == identity.approval_id
and outcome.get("evidenceReceiptId")
== agent99_same_run_evidence_id(identity)
and contract.get("schemaVersion") == "agent99_outcome_contract_v1"
and outcome.get("mode") == "Status"
and outcome.get("controlledApply") is False
and outcome.get("reconcileOnly") is True
and outcome.get("runtimeWritePerformed") is False
and contract.get("state") == "resolved"
and contract.get("resolved") is True
and contract.get("transportOk") is True
and contract.get("verifierPassed") is True
and contract.get("verifierName") == AGENT99_SAME_RUN_STATUS_VERIFIER
and contract.get("sourceEventResolved") is True
and contract.get("sourceEventEvidence") == source_receipt_ref
)
async def _fetch_legacy_cold_start_failures(
*,
project_id: str,
@@ -535,7 +588,15 @@ async def reconcile_agent99_controlled_dispatches_once(
) -> dict[str, int]:
ledger = get_agent99_dispatch_ledger()
items = await ledger.list_reconcilable(project_id=project_id, limit=limit)
counters = {"scanned": len(items), "verifier_written": 0, "closed": 0}
counters = {
"scanned": len(items),
"source_not_resolved": 0,
"status_reconcile_requested": 0,
"status_reconcile_pending": 0,
"external_no_write_reconciled": 0,
"verifier_written": 0,
"closed": 0,
}
for item in items:
identity = item["identity"]
receipt = item.get("receipt") if isinstance(item.get("receipt"), dict) else {}
@@ -552,6 +613,60 @@ async def reconcile_agent99_controlled_dispatches_once(
else {}
)
}
if is_cold_start_same_run_identity(identity):
source_receipt = await asyncio.to_thread(
read_cold_start_source_resolution,
)
source_receipt_ref = str(source_receipt.get("receipt_ref") or "")
if (
source_receipt.get("resolved") is not True
or not source_receipt_ref
):
counters["source_not_resolved"] += 1
continue
outcome = await asyncio.to_thread(
read_agent99_sre_outcome,
str(identity.run_id),
)
if not _is_bound_no_write_status_outcome(
outcome,
identity=identity,
source_receipt_ref=source_receipt_ref,
):
request_receipt = await asyncio.to_thread(
request_agent99_same_run_status_reconcile,
identity,
source_receipt,
)
if request_receipt.get("accepted") is True:
counters["status_reconcile_requested"] += 1
else:
counters["status_reconcile_pending"] += 1
continue
evidence_refs = extract_agent99_outcome_evidence_refs(
outcome,
{
"agent99_outcome_receipt_id": (
f"agent99-relay:outcome:{identity.run_id}"
),
"post_verifier_evidence_ref": (
"agent99:Status:no-write:"
f"{agent99_same_run_evidence_id(identity)}"
),
"source_event_evidence_ref": source_receipt_ref,
},
)
terminal = await ledger.record_external_no_write_reconciliation(
identity=identity,
outcome_receipt=outcome,
source_receipt=source_receipt,
evidence_refs=evidence_refs,
)
if terminal.get("receipt_persisted") is True:
counters["external_no_write_reconciled"] += 1
# This path is deliberately terminal without automation success,
# incident RESOLVED, Telegram, KM, or PlayBook trust writeback.
continue
if receipt.get("post_verifier_passed") is not True:
outcome = await asyncio.to_thread(
read_agent99_sre_outcome,
@@ -573,8 +688,12 @@ async def reconcile_agent99_controlled_dispatches_once(
"post_verifier_evidence_ref": (
f"agent99:{mode}:verified:{verified_at}"
),
"source_event_evidence_ref": (
f"incident:{identity.incident_id}:source-resolved:{verified_at}"
"source_event_evidence_ref": str(
outcome_contract.get("sourceEventEvidence")
or (
f"incident:{identity.incident_id}:"
f"source-resolved:{verified_at}"
)
),
}
evidence_refs = extract_agent99_outcome_evidence_refs(

375
apps/api/src/services/agent99_controlled_dispatch_ledger.py
View File

@@ -10,6 +10,7 @@ from __future__ import annotations
import hashlib
import json
import math
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -31,6 +32,7 @@ from src.services.agent99_public_receipts import (
AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS,
AGENT99_LEARNING_RECEIPT_REFS,
AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS,
normalize_agent99_public_receipt_ref,
sanitize_agent99_public_receipt_refs,
)
@@ -59,6 +61,19 @@ def _sha256(value: str) -> str:
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _agent99_exact_nonnegative_int(value: Any) -> int | None:
"""Normalize a JSON number only when it is an exact non-negative int."""
if isinstance(value, bool) or not isinstance(value, int | float):
return None
if not math.isfinite(float(value)):
return None
normalized = int(value)
if normalized < 0 or float(value) != float(normalized):
return None
return normalized
@dataclass(frozen=True)
class Agent99DispatchIdentity:
project_id: str
@@ -1329,6 +1344,366 @@ class PostgresAgent99DispatchLedger:
"runtime_closure_verified": False,
}
async def record_external_no_write_reconciliation(
self,
*,
identity: Agent99DispatchIdentity,
outcome_receipt: dict[str, Any],
source_receipt: dict[str, Any],
evidence_refs: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Terminalize an externally recovered run without automation credit.
This is intentionally separate from ``record_verifier`` and
``record_learning_writeback``. A no-write Status proves current
posture only; it never proves that the original Recover execution
succeeded and must not resolve the incident or increase PlayBook
trust.
"""
if not validate_agent99_outcome_identity(identity, outcome_receipt):
return {
"status": "external_recovery_identity_mismatch_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
outcome = (
outcome_receipt.get("outcome")
if isinstance(outcome_receipt.get("outcome"), dict)
else {}
)
expected_top_level = {
"automationRunId": str(identity.run_id),
"traceId": identity.trace_id,
"workItemId": identity.work_item_id,
"projectId": identity.project_id,
"incidentId": identity.incident_id,
"routeId": identity.route_id,
"executionGeneration": identity.execution_generation,
"approvalId": identity.approval_id,
}
if any(
str(outcome_receipt.get(key) or "") != expected
for key, expected in expected_top_level.items()
):
return {
"status": "external_recovery_top_level_identity_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
source_ref = normalize_agent99_public_receipt_ref(
source_receipt.get("receipt_ref")
)
source_values = (
source_receipt.get("values")
if isinstance(source_receipt.get("values"), dict)
else {}
)
source_result = str(source_receipt.get("source_result") or "")
source_counts = {
key: _agent99_exact_nonnegative_int(source_values.get(key))
for key in (
"monitor_up",
"pass_gates",
"warn_gates",
"blocked_gates",
"green_result",
"degraded_result",
"blocked_result",
"check_failed_result",
"firing_blocking_alerts",
)
}
source_age = source_receipt.get("age_seconds")
source_age_exact = bool(
not isinstance(source_age, bool)
and isinstance(source_age, int | float)
and float(source_age) >= -60
and float(source_age) <= 900
)
source_observed_at = _agent99_exact_nonnegative_int(
source_receipt.get("observed_at_epoch")
)
expected_evidence_id = (
f"same-run-status-{identity.run_id}-"
f"{identity.public_dict()['canonical_digest'][:12]}"
)
exact_outcome = bool(
outcome_receipt.get("schemaVersion")
== "agent99_outcome_readback_v1"
and outcome_receipt.get("sameRunContractFieldsComplete") is True
and outcome_receipt.get("mode") == "Status"
and outcome_receipt.get("controlledApply") is False
and outcome_receipt.get("reconcileOnly") is True
and outcome_receipt.get("runtimeWritePerformed") is False
and outcome_receipt.get("evidenceReceiptId")
== expected_evidence_id
and outcome.get("schemaVersion") == "agent99_outcome_contract_v1"
and outcome.get("state") == "resolved"
and outcome.get("resolved") is True
and outcome.get("transportOk") is True
and outcome.get("verifierPassed") is True
and outcome.get("verifierName")
== "agent99-status-same-run-no-write-post-condition-v1"
and outcome.get("sourceEventResolved") is True
and source_ref is not None
and outcome.get("sourceEventEvidence") == source_ref
)
source_exact = bool(
source_receipt.get("schema_version")
== "agent99_cold_start_source_resolution_v1"
and source_receipt.get("resolved") is True
and source_result in {"green", "degraded"}
and source_receipt.get("degraded_provenance")
is (source_result == "degraded")
and source_age_exact
and source_observed_at is not None
and source_observed_at > 0
and source_counts["monitor_up"] == 1
and source_counts["pass_gates"] is not None
and source_counts["pass_gates"] > 0
and source_counts["warn_gates"] is not None
and source_counts["blocked_gates"] == 0
and source_counts["blocked_result"] == 0
and source_counts["check_failed_result"] == 0
and source_counts["firing_blocking_alerts"] == 0
and (
(
source_result == "degraded"
and source_counts["warn_gates"] > 0
and source_counts["green_result"] == 0
and source_counts["degraded_result"] == 1
)
or (
source_result == "green"
and source_counts["warn_gates"] == 0
and source_counts["green_result"] == 1
and source_counts["degraded_result"] == 0
)
)
)
safe_refs = _safe_receipt_refs(evidence_refs)
missing_refs = sorted(
set(AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS) - set(safe_refs)
)
if not exact_outcome or not source_exact or missing_refs:
return {
"status": "external_recovery_contract_invalid_fail_closed",
"missing_evidence_refs": missing_refs,
"receipt_persisted": False,
"runtime_closure_verified": False,
}
if safe_refs.get("source_event_evidence_ref") != source_ref:
return {
"status": "external_recovery_source_receipt_mismatch_fail_closed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
now = _utc_now_naive()
terminal_receipt = {
"schema_version": (
"agent99_external_recovery_no_write_reconciliation_v1"
),
"status": "external_recovery_no_write_reconciled",
**_stage_identity(identity),
"mode": "Status",
"verifier_name": (
"agent99-status-same-run-no-write-post-condition-v1"
),
"evidence_receipt_id": expected_evidence_id,
"source": {
"receipt_ref": source_ref,
"result": source_result,
"degraded_provenance": bool(
source_receipt.get("degraded_provenance")
),
"pass_gates": source_counts["pass_gates"],
"warn_gates": source_counts["warn_gates"],
"blocked_gates": 0,
"age_seconds": source_age,
"observed_at_epoch": source_observed_at,
},
"transport_ok": True,
"status_verifier_passed": True,
"runtime_write_performed": False,
"recover_execution_success": False,
"incident_resolution_authorized": False,
"learning_writeback_authorized": False,
"telegram_authorized": False,
"evidence_refs": safe_refs,
"stores_raw_evidence": False,
}
try:
async with get_db_context(identity.project_id) as db:
current_result = await db.execute(
select(AwoooPRunState.state, AwoooPRunState.error_detail)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
)
.with_for_update()
)
current = current_result.one_or_none()
envelope = _parse_envelope(
current.error_detail if current is not None else None
)
if (
current is not None
and current.state == "cancelled"
and isinstance(envelope, dict)
and envelope.get("external_no_write_reconciliation_terminal")
is True
and _stage_identity_matches(
identity,
envelope.get("identity")
if isinstance(envelope.get("identity"), dict)
else {},
)
):
return {
**envelope,
"status": (
"external_recovery_no_write_terminal_idempotent"
),
"receipt_persisted": True,
"runtime_closure_verified": False,
}
if (
current is None
or current.state
not in {"pending", "running", "waiting_tool"}
or not isinstance(envelope, dict)
or not _stage_identity_matches(
identity,
envelope.get("identity")
if isinstance(envelope.get("identity"), dict)
else {},
)
):
return {
"status": (
"external_recovery_run_not_ready_fail_closed"
),
"receipt_persisted": False,
"runtime_closure_verified": False,
}
prior_runtime_execution_attempted = bool(
envelope.get("runtime_execution_attempted") is True
)
envelope.update({
"status": "external_recovery_no_write_reconciled_terminal",
"run_terminal_state": "cancelled",
"controlled_apply_authorized": False,
"runtime_execution_authorized": False,
"runtime_execution_attempted": (
prior_runtime_execution_attempted
),
"same_run_status_runtime_execution_attempted": False,
"runtime_write_performed": False,
"automation_execution_success": False,
"post_verifier_passed": False,
"external_status_verifier_passed": True,
"external_recovery_source_resolved": True,
"runtime_closure_verified": False,
"closure_complete": False,
"incident_resolution_authorized": False,
"external_no_write_reconciliation_terminal": True,
"external_recovery_reconciliation": terminal_receipt,
"learning_writeback": {
**_stage_identity(identity),
"step_seq": 3,
"status": (
"not_applicable_external_recovery_no_write"
),
"required_receipts": [],
"receipt_refs": {},
"stores_raw_evidence": False,
},
})
envelope_json = _stable_json(envelope)
updated = await db.execute(
update(AwoooPRunState)
.where(
AwoooPRunState.run_id == identity.run_id,
AwoooPRunState.project_id == identity.project_id,
AwoooPRunState.state.in_([
"pending",
"running",
"waiting_tool",
]),
)
.values(
state="cancelled",
output_sha256=_sha256(envelope_json),
error_code=None,
error_detail=envelope_json,
heartbeat_at=now,
completed_at=now,
lease_until=None,
next_attempt_at=None,
worker_id=None,
)
.returning(AwoooPRunState.run_id)
)
persisted = updated.scalar_one_or_none() is not None
if persisted:
journal_states = {
1: (
"compensated",
True,
"external_recovery_superseded_runtime_execution",
),
2: ("success", False, None),
3: (
"compensated",
True,
"learning_not_applicable_external_recovery_no_write",
),
}
for step_seq, (
result_status,
was_blocked,
block_reason,
) in journal_states.items():
await db.execute(
update(AwoooPRunStepJournal)
.where(
AwoooPRunStepJournal.run_id == identity.run_id,
AwoooPRunStepJournal.step_seq == step_seq,
)
.values(
output_hash=_sha256(
_stable_json(terminal_receipt)
),
compensation_json=terminal_receipt,
result_status=result_status,
error_code=None,
was_blocked=was_blocked,
block_reason=block_reason,
completed_at=now,
)
)
except Exception as exc:
logger.warning(
"agent99_external_no_write_reconciliation_failed",
incident_id=identity.incident_id,
run_id=str(identity.run_id),
error=str(exc),
)
return {
"status": "external_recovery_persistence_failed",
"receipt_persisted": False,
"runtime_closure_verified": False,
}
return {
**envelope,
"receipt_persisted": persisted,
"runtime_closure_verified": False,
}
async def record_learning_writeback(
self,
*,

358
apps/api/src/services/agent99_same_run_reconcile.py Normal file
View File

@@ -0,0 +1,358 @@
"""Fail-closed, no-write Agent99 same-run Status reconciliation.
This module deliberately keeps the production source verifier separate from
the Agent99 verifier. A missing historical Recover outcome may be refreshed
only after the live cold-start source reports a fresh, unblocked state with no
firing ColdStart alert. The refresh keeps the existing deterministic
dispatch identity and asks the authenticated relay for ``Status`` in
``reconcileOnly`` mode; it never reserves a new run, advances a generation,
or authorizes controlled apply.
"""
from __future__ import annotations
import json
import math
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Any
from src.core.config import settings
from src.core.logging import get_logger
from src.services.agent99_controlled_dispatch_ledger import (
Agent99DispatchIdentity,
)
from src.services.agent99_public_receipts import (
normalize_agent99_public_receipt_ref,
)
logger = get_logger("awoooi.agent99_same_run_reconcile")
AGENT99_SAME_RUN_RECONCILE_SCHEMA = "agent99_same_run_status_reconcile_v1"
AGENT99_COLD_START_ROUTE_ID = "agent99:host_recovery:Recover"
AGENT99_COLD_START_INCIDENT_ID = "INC-20260711-11C751"
AGENT99_COLD_START_RUN_ID = "bf30ca01-1080-5725-b2d1-3e1534dfc811"
AGENT99_SAME_RUN_STATUS_VERIFIER = (
"agent99-status-same-run-no-write-post-condition-v1"
)
AGENT99_COLD_START_METRIC_HOST = "110"
AGENT99_COLD_START_METRIC_SCOPE = "110_120_121_188"
AGENT99_COLD_START_MAX_AGE_SECONDS = 900
_COLD_START_QUERIES = {
"monitor_up": (
'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",'
'mode="read_only"}'
),
"pass_gates": (
'awoooi_cold_start_pass_gates{host="110",scope="110_120_121_188"}'
),
"warn_gates": (
'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}'
),
"blocked_gates": (
'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}'
),
"last_run_timestamp": (
'awoooi_cold_start_last_run_timestamp{host="110",'
'scope="110_120_121_188"}'
),
"green_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="green"}'
),
"degraded_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="degraded"}'
),
"blocked_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="blocked"}'
),
"check_failed_result": (
'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",'
'result="check_failed"}'
),
"firing_blocking_alerts": (
'sum(ALERTS{alertname=~"ColdStart.*",alertstate="firing",'
'severity="critical"}) or vector(0)'
),
}
def is_cold_start_same_run_identity(identity: Agent99DispatchIdentity) -> bool:
"""Limit this transport to the existing cold-start Recover identity."""
return bool(
identity.route_id == AGENT99_COLD_START_ROUTE_ID
and identity.incident_id == AGENT99_COLD_START_INCIDENT_ID
and str(identity.run_id) == AGENT99_COLD_START_RUN_ID
and identity.source_fingerprint.startswith("legacy-cold-start:")
and identity.execution_generation == "1"
)
def agent99_same_run_evidence_id(identity: Agent99DispatchIdentity) -> str:
"""Return the only evidence identifier accepted for this bounded run."""
digest = identity.public_dict()["canonical_digest"]
return f"same-run-status-{identity.run_id}-{digest[:12]}"
def _query_prometheus_scalar(
query_url: str,
query: str,
*,
timeout_seconds: float,
) -> float | None:
target = f"{query_url}?{urllib.parse.urlencode({'query': query})}"
request = urllib.request.Request(
target,
headers={
"Accept": "application/json",
"User-Agent": "awoooi-agent99-source-verifier/1",
},
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
if int(response.getcode()) != 200:
return None
body = response.read(65536).decode("utf-8", errors="replace")
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError):
return None
try:
payload = json.loads(body)
data = payload.get("data") if isinstance(payload, dict) else None
result = data.get("result") if isinstance(data, dict) else None
if payload.get("status") != "success" or not isinstance(result, list):
return None
if len(result) != 1 or not isinstance(result[0], dict):
return None
sample = result[0].get("value")
if not isinstance(sample, list) or len(sample) != 2:
return None
value = float(sample[1])
return value if math.isfinite(value) else None
except (AttributeError, TypeError, ValueError, json.JSONDecodeError):
return None
def read_cold_start_source_resolution(
*,
prometheus_url: str | None = None,
timeout_seconds: float = 3.0,
now_epoch: float | None = None,
) -> dict[str, Any]:
"""Read a bounded production source receipt; missing evidence fails closed."""
base_url = str(prometheus_url or settings.PROMETHEUS_URL).rstrip("/")
query_url = f"{base_url}/api/v1/query"
values: dict[str, float] = {}
missing: list[str] = []
for name, query in _COLD_START_QUERIES.items():
value = _query_prometheus_scalar(
query_url,
query,
timeout_seconds=timeout_seconds,
)
if value is None:
missing.append(name)
else:
values[name] = value
blockers: list[str] = []
if missing:
blockers.append("source_metrics_missing:" + ",".join(sorted(missing)))
observed_at = float(now_epoch if now_epoch is not None else time.time())
last_run = values.get("last_run_timestamp")
age_seconds: float | None = None
if last_run is not None:
age_seconds = observed_at - last_run
if age_seconds < -60:
blockers.append("source_clock_skew_future")
elif age_seconds > AGENT99_COLD_START_MAX_AGE_SECONDS:
blockers.append("source_metrics_stale")
if values.get("monitor_up") != 1:
blockers.append("cold_start_monitor_not_up")
for count_name in ("pass_gates", "warn_gates", "blocked_gates"):
count_value = values.get(count_name)
if count_value is not None and (
count_value < 0 or not count_value.is_integer()
):
blockers.append(f"cold_start_{count_name}_invalid")
if values.get("pass_gates", 0) <= 0:
blockers.append("cold_start_pass_gates_missing")
if values.get("blocked_gates") != 0:
blockers.append("cold_start_blocked_gates_present")
if values.get("blocked_result") != 0:
blockers.append("cold_start_result_blocked")
if values.get("check_failed_result") != 0:
blockers.append("cold_start_result_check_failed")
warn_gates = values.get("warn_gates")
expected_result = "degraded" if warn_gates is not None and warn_gates > 0 else "green"
if expected_result == "degraded":
if values.get("degraded_result") != 1 or values.get("green_result") != 0:
blockers.append("cold_start_degraded_result_not_one_hot")
elif values.get("green_result") != 1 or values.get("degraded_result") != 0:
blockers.append("cold_start_green_result_not_one_hot")
if values.get("firing_blocking_alerts") != 0:
blockers.append("cold_start_blocking_alert_still_firing")
resolved = bool(not blockers and last_run is not None)
receipt_ref: str | None = None
if resolved:
receipt_ref = normalize_agent99_public_receipt_ref(
"prometheus:cold-start:"
f"{int(last_run)}:pass-{int(values['pass_gates'])}:"
f"warn-{int(values['warn_gates'])}:blocked-0:"
f"result-{expected_result}:blocking-alerts-0"
)
if receipt_ref is None:
blockers.append("source_receipt_ref_invalid")
resolved = False
return {
"schema_version": "agent99_cold_start_source_resolution_v1",
"status": "source_resolved" if resolved else "source_not_resolved",
"resolved": resolved,
"source_result": expected_result if resolved else "unresolved",
"degraded_provenance": bool(resolved and expected_result == "degraded"),
"receipt_ref": receipt_ref,
"observed_at_epoch": int(observed_at),
"age_seconds": round(age_seconds, 3) if age_seconds is not None else None,
"values": {
key: int(value) if float(value).is_integer() else value
for key, value in sorted(values.items())
},
"blockers": blockers,
"stores_raw_evidence": False,
}
def request_agent99_same_run_status_reconcile(
identity: Agent99DispatchIdentity,
source_receipt: dict[str, Any],
*,
relay_url: str | None = None,
relay_token: str | None = None,
timeout_seconds: float | None = None,
) -> dict[str, Any]:
"""Ask Agent99 for one idempotent, no-write Status on the existing run."""
if not is_cold_start_same_run_identity(identity):
return {
"status": "identity_not_eligible_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
receipt_ref = normalize_agent99_public_receipt_ref(
source_receipt.get("receipt_ref")
)
if source_receipt.get("resolved") is not True or not receipt_ref:
return {
"status": "source_not_resolved_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
if not receipt_ref.startswith("prometheus:cold-start:"):
return {
"status": "source_receipt_not_cold_start_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
target_url = str(
relay_url
if relay_url is not None
else settings.AGENT99_SRE_ALERT_RELAY_URL
)
token = str(
relay_token
if relay_token is not None
else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
)
if not target_url or not token:
return {
"status": "relay_auth_not_configured_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
timeout = float(
timeout_seconds
if timeout_seconds is not None
else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS
)
payload = {
"schemaVersion": AGENT99_SAME_RUN_RECONCILE_SCHEMA,
"action": "same_run_status_reconcile",
"identity": identity.public_dict(),
"mode": "Status",
"controlledApply": False,
"reconcileOnly": True,
"sourceEventResolved": True,
"sourceEventEvidence": receipt_ref,
"sourceEventResolutionPolicy": "external_source_receipt_required",
"suppressTelegram": True,
"evidenceReceiptId": agent99_same_run_evidence_id(identity),
}
request = urllib.request.Request(
target_url,
data=json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8"),
headers={
"Accept": "application/json",
"Content-Type": "application/json; charset=utf-8",
"User-Agent": "awoooi-agent99-same-run-reconciler/1",
"X-Agent99-Relay-Token": token,
},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
status_code = int(response.getcode())
body = response.read(65536).decode("utf-8", errors="replace")
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
logger.info(
"agent99_same_run_status_reconcile_pending",
run_id=str(identity.run_id),
error_type=type(exc).__name__,
)
return {
"status": "relay_request_failed_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
try:
response_payload = json.loads(body)
except json.JSONDecodeError:
response_payload = {}
accepted = bool(
200 <= status_code < 300
and isinstance(response_payload, dict)
and response_payload.get("ok") is True
and response_payload.get("reconcileOnly") is True
and response_payload.get("controlledApply") is False
and str(response_payload.get("automationRunId") or "")
== str(identity.run_id)
and str(response_payload.get("evidenceReceiptId") or "")
== agent99_same_run_evidence_id(identity)
)
return {
"schema_version": "agent99_same_run_status_reconcile_request_receipt_v1",
"status": (
str(response_payload.get("status") or "accepted")
if accepted
else "relay_response_rejected_fail_closed"
),
"accepted": accepted,
"http_status": status_code,
"identity": identity.public_dict(),
"source_event_evidence_ref": receipt_ref,
"evidence_receipt_id": agent99_same_run_evidence_id(identity),
"reconcile_only": True,
"controlled_apply": False,
"runtime_write_authorized": False,
"stores_raw_response": False,
}

86
apps/api/src/services/agent99_sre_bridge.py
View File

@@ -21,6 +21,7 @@ from src.services.agent99_controlled_dispatch_ledger import (
)
from src.services.agent99_public_receipts import (
extract_agent99_outcome_evidence_refs,
normalize_agent99_public_receipt_ref,
)
from src.utils.timezone import now_taipei
@@ -697,17 +698,101 @@ def read_agent99_sre_outcome(
):
return None
safe_evidence_refs = extract_agent99_outcome_evidence_refs(payload)
source_event_evidence = normalize_agent99_public_receipt_ref(
outcome.get("sourceEventEvidence")
)
expected_evidence_receipt_id = (
f"same-run-status-{identity.run_id}-"
f"{identity.public_dict()['canonical_digest'][:12]}"
)
evidence_receipt_id = str(payload.get("evidenceReceiptId") or "")
if evidence_receipt_id != expected_evidence_receipt_id:
evidence_receipt_id = ""
same_run_contract_fields_complete = bool(
payload.get("sameRunContractFieldsComplete") is True
and all(
key in payload
for key in (
"automationRunId",
"traceId",
"workItemId",
"projectId",
"incidentId",
"routeId",
"executionGeneration",
"approvalId",
"evidenceReceiptId",
"controlledApply",
"reconcileOnly",
"runtimeWritePerformed",
"mode",
)
)
and all(
isinstance(payload.get(key), str)
for key in (
"automationRunId",
"traceId",
"workItemId",
"projectId",
"incidentId",
"routeId",
"executionGeneration",
"approvalId",
"evidenceReceiptId",
"mode",
)
)
and isinstance(payload.get("controlledApply"), bool)
and isinstance(payload.get("reconcileOnly"), bool)
and isinstance(payload.get("runtimeWritePerformed"), bool)
and all(
key in outcome
for key in (
"schemaVersion",
"state",
"resolved",
"transportOk",
"verifierName",
"verifierPassed",
"sourceEventResolved",
"sourceEventEvidence",
"identity",
)
)
and all(
isinstance(outcome.get(key), str)
for key in (
"schemaVersion",
"state",
"verifierName",
"sourceEventEvidence",
)
)
and isinstance(outcome.get("resolved"), bool)
and isinstance(outcome.get("transportOk"), bool)
and isinstance(outcome.get("verifierPassed"), bool)
and isinstance(outcome.get("sourceEventResolved"), bool)
)
return {
"schemaVersion": "agent99_outcome_readback_v1",
"automationRunId": str(payload.get("automationRunId") or ""),
"traceId": str(payload.get("traceId") or ""),
"workItemId": str(payload.get("workItemId") or ""),
"projectId": str(payload.get("projectId") or ""),
"routeId": str(payload.get("routeId") or ""),
"idempotencyKey": str(payload.get("idempotencyKey") or ""),
"executionGeneration": str(payload.get("executionGeneration") or "1"),
"incidentId": str(payload.get("incidentId") or ""),
"approvalId": str(payload.get("approvalId") or ""),
"evidenceReceiptId": evidence_receipt_id,
"sameRunContractFieldsComplete": same_run_contract_fields_complete,
"identity": identity.public_dict(),
"controlledApply": bool(payload.get("controlledApply") is True),
"reconcileOnly": bool(payload.get("reconcileOnly") is True),
"runtimeWritePerformed": bool(
payload.get("runtimeWritePerformed") is True
),
"mode": str(payload.get("mode") or ""),
"outcome": {
"schemaVersion": str(outcome.get("schemaVersion") or ""),
@@ -719,6 +804,7 @@ def read_agent99_sre_outcome(
"sourceEventResolved": bool(
outcome.get("sourceEventResolved") is True
),
"sourceEventEvidence": source_event_evidence or "",
"failedChecks": [
str(value)[:120]
for value in (outcome.get("failedChecks") or [])[:32]

2
apps/api/tests/test_agent99_alert_dispatch_order_contract.py
View File

@@ -134,7 +134,7 @@ def test_agent99_relay_exposes_authenticated_public_safe_outcome_readback() -> N
assert "function Get-AgentOutcomeReadback" in source
assert '$context.Request.HttpMethod -eq "GET"' in source
assert "outcome_readback_auth_not_configured" in source
assert "relay_auth_not_configured" in source
assert 'Headers["X-Agent99-Relay-Token"]' in source
assert "storesRawEvidence = $false" in source

26
apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py
View File

@@ -95,7 +95,15 @@ async def test_reconciler_consumes_authenticated_outcome_and_closes_same_run(
result = await job.reconcile_agent99_controlled_dispatches_once()
assert result == {"scanned": 1, "verifier_written": 1, "closed": 1}
assert result == {
"scanned": 1,
"source_not_resolved": 0,
"status_reconcile_requested": 0,
"status_reconcile_pending": 0,
"external_no_write_reconciled": 0,
"verifier_written": 1,
"closed": 1,
}
assert len(ledger.verifier_calls) == 1
refs = ledger.verifier_calls[0]["evidence_refs"]
assert refs["agent99_outcome_receipt_id"].endswith(str(IDENTITY.run_id))
@@ -116,7 +124,15 @@ async def test_reconciler_does_not_finalize_without_outcome(monkeypatch) -> None
result = await job.reconcile_agent99_controlled_dispatches_once()
assert result == {"scanned": 1, "verifier_written": 0, "closed": 0}
assert result == {
"scanned": 1,
"source_not_resolved": 0,
"status_reconcile_requested": 0,
"status_reconcile_pending": 0,
"external_no_write_reconciled": 0,
"verifier_written": 0,
"closed": 0,
}
assert ledger.verifier_calls == []
@@ -125,8 +141,12 @@ def test_windows_relay_outcome_readback_is_authenticated_and_public_safe() -> No
text = source.read_text(encoding="utf-8")
assert '$context.Request.HttpMethod -eq "GET"' in text
assert "outcome_readback_auth_not_configured" in text
assert "relay_auth_not_configured" in text
assert 'Headers["X-Agent99-Relay-Token"]' in text
token_gate = text.index("if (-not $expectedToken)")
get_branch = text.index('if ($context.Request.HttpMethod -eq "GET")')
post_branch = text.index('if ($context.Request.HttpMethod -ne "POST")')
assert token_gate < get_branch < post_branch
assert "storesRawEvidence = $false" in text
assert "stdout" not in text[text.index("function Get-AgentOutcomeReadback"):text.index("function Write-AgentRelayEvent")]

820
apps/api/tests/test_agent99_same_run_reconcile.py Normal file
View File

@@ -0,0 +1,820 @@
from __future__ import annotations
import json
from copy import deepcopy
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from src.jobs import agent99_controlled_dispatch_reconciler_job as job
from src.services import (
agent99_controlled_dispatch_ledger as ledger_module,
)
from src.services import agent99_same_run_reconcile as reconcile
from src.services import agent99_sre_bridge as bridge
from src.services.agent99_controlled_dispatch_ledger import (
PostgresAgent99DispatchLedger,
build_agent99_dispatch_identity,
build_agent99_dispatch_receipt_envelope,
)
IDENTITY = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260711-11C751",
source_fingerprint="legacy-cold-start:" + "a" * 64,
route_id="agent99:host_recovery:Recover",
approval_id="00000000-0000-0000-0000-000000000751",
work_item_id=(
"agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start"
),
)
@pytest.fixture(autouse=True)
def _bind_test_identity_to_bounded_run(monkeypatch) -> None:
monkeypatch.setattr(
reconcile,
"AGENT99_COLD_START_RUN_ID",
str(IDENTITY.run_id),
)
def _source_receipt() -> dict[str, object]:
return {
"schema_version": "agent99_cold_start_source_resolution_v1",
"status": "source_resolved",
"resolved": True,
"source_result": "degraded",
"degraded_provenance": True,
"receipt_ref": (
"prometheus:cold-start:1784020000:pass-95:"
"warn-1:blocked-0:result-degraded:blocking-alerts-0"
),
"observed_at_epoch": 1784020030,
"age_seconds": 30.0,
"values": {
"monitor_up": 1,
"pass_gates": 95,
"warn_gates": 1,
"blocked_gates": 0,
"green_result": 0,
"degraded_result": 1,
"blocked_result": 0,
"check_failed_result": 0,
"firing_blocking_alerts": 0,
},
"blockers": [],
"stores_raw_evidence": False,
}
def _valid_outcome() -> dict[str, object]:
identity = IDENTITY.public_dict()
return {
"schemaVersion": "agent99_outcome_readback_v1",
"sameRunContractFieldsComplete": True,
"automationRunId": str(IDENTITY.run_id),
"traceId": IDENTITY.trace_id,
"workItemId": IDENTITY.work_item_id,
"projectId": IDENTITY.project_id,
"incidentId": IDENTITY.incident_id,
"routeId": IDENTITY.route_id,
"executionGeneration": IDENTITY.execution_generation,
"approvalId": IDENTITY.approval_id,
"evidenceReceiptId": reconcile.agent99_same_run_evidence_id(IDENTITY),
"identity": identity,
"mode": "Status",
"controlledApply": False,
"reconcileOnly": True,
"runtimeWritePerformed": False,
"outcome": {
"schemaVersion": "agent99_outcome_contract_v1",
"state": "resolved",
"resolved": True,
"transportOk": True,
"verifierName": reconcile.AGENT99_SAME_RUN_STATUS_VERIFIER,
"verifierPassed": True,
"sourceEventResolved": True,
"sourceEventEvidence": _source_receipt()["receipt_ref"],
"identity": identity,
"verifiedAt": "2026-07-14T16:30:00+08:00",
},
}
def _evidence_refs() -> dict[str, str]:
return {
"agent99_outcome_receipt_id": (
f"agent99-relay:outcome:{IDENTITY.run_id}"
),
"post_verifier_evidence_ref": (
"agent99:Status:no-write:"
f"{reconcile.agent99_same_run_evidence_id(IDENTITY)}"
),
"source_event_evidence_ref": str(_source_receipt()["receipt_ref"]),
}
def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero(
monkeypatch,
) -> None:
values = {
"monitor_up": 1.0,
"pass_gates": 95.0,
"warn_gates": 1.0,
"blocked_gates": 0.0,
"last_run_timestamp": 1784020000.0,
"green_result": 0.0,
"degraded_result": 1.0,
"blocked_result": 0.0,
"check_failed_result": 0.0,
"firing_blocking_alerts": 0.0,
}
def fake_query(_url, query, *, timeout_seconds): # type: ignore[no-untyped-def]
del timeout_seconds
for name, expected_query in reconcile._COLD_START_QUERIES.items():
if query == expected_query:
return values[name]
raise AssertionError(query)
monkeypatch.setattr(reconcile, "_query_prometheus_scalar", fake_query)
receipt = reconcile.read_cold_start_source_resolution(
prometheus_url="http://prometheus.invalid",
now_epoch=1784020030,
)
assert receipt["resolved"] is True
assert receipt["status"] == "source_resolved"
assert receipt["values"]["warn_gates"] == 1
assert receipt["values"]["blocked_gates"] == 0
assert receipt["receipt_ref"] == (
"prometheus:cold-start:1784020000:pass-95:"
"warn-1:blocked-0:result-degraded:blocking-alerts-0"
)
assert receipt["stores_raw_evidence"] is False
@pytest.mark.parametrize(
("field", "value", "blocker"),
[
("blocked_gates", 1.0, "cold_start_blocked_gates_present"),
(
"firing_blocking_alerts",
1.0,
"cold_start_blocking_alert_still_firing",
),
("monitor_up", 0.0, "cold_start_monitor_not_up"),
],
)
def test_cold_start_source_resolution_fails_closed(
monkeypatch,
field: str,
value: float,
blocker: str,
) -> None:
values = {
"monitor_up": 1.0,
"pass_gates": 95.0,
"warn_gates": 0.0,
"blocked_gates": 0.0,
"last_run_timestamp": 1784020000.0,
"green_result": 1.0,
"degraded_result": 0.0,
"blocked_result": 0.0,
"check_failed_result": 0.0,
"firing_blocking_alerts": 0.0,
}
values[field] = value
query_to_name = {
query: name for name, query in reconcile._COLD_START_QUERIES.items()
}
monkeypatch.setattr(
reconcile,
"_query_prometheus_scalar",
lambda _url, query, **_kwargs: values[query_to_name[query]],
)
receipt = reconcile.read_cold_start_source_resolution(
prometheus_url="http://prometheus.invalid",
now_epoch=1784020030,
)
assert receipt["resolved"] is False
assert blocker in receipt["blockers"]
assert receipt["receipt_ref"] is None
def test_same_run_status_request_preserves_identity_and_no_write_fences(
monkeypatch,
) -> None:
seen: dict[str, object] = {}
class FakeResponse:
def getcode(self) -> int:
return 202
def read(self, _limit: int) -> bytes:
return json.dumps({
"ok": True,
"status": "same_run_status_reconcile_queued",
"automationRunId": str(IDENTITY.run_id),
"reconcileOnly": True,
"controlledApply": False,
"evidenceReceiptId": reconcile.agent99_same_run_evidence_id(
IDENTITY
),
}).encode()
def __enter__(self): # type: ignore[no-untyped-def]
return self
def __exit__(self, *_args): # type: ignore[no-untyped-def]
return None
def fake_urlopen(request, timeout): # type: ignore[no-untyped-def]
seen["url"] = request.full_url
seen["timeout"] = timeout
seen["payload"] = json.loads(request.data.decode())
return FakeResponse()
monkeypatch.setattr(reconcile.urllib.request, "urlopen", fake_urlopen)
result = reconcile.request_agent99_same_run_status_reconcile(
IDENTITY,
_source_receipt(),
relay_url="http://agent99.invalid/agent99/sre-alert/",
relay_token="configured-not-returned",
timeout_seconds=2,
)
assert result["accepted"] is True
assert result["runtime_write_authorized"] is False
payload = seen["payload"]
assert payload["schemaVersion"] == "agent99_same_run_status_reconcile_v1"
assert payload["mode"] == "Status"
assert payload["controlledApply"] is False
assert payload["reconcileOnly"] is True
assert payload["suppressTelegram"] is True
assert payload["evidenceReceiptId"] == reconcile.agent99_same_run_evidence_id(
IDENTITY
)
assert payload["identity"] == IDENTITY.public_dict()
assert payload["identity"]["execution_generation"] == "1"
assert "configured-not-returned" not in json.dumps(result)
def test_same_run_status_request_rejects_unresolved_source_without_transport(
monkeypatch,
) -> None:
monkeypatch.setattr(
reconcile.urllib.request,
"urlopen",
lambda *_args, **_kwargs: pytest.fail("transport must not run"),
)
result = reconcile.request_agent99_same_run_status_reconcile(
IDENTITY,
{"resolved": False, "receipt_ref": None},
relay_url="http://agent99.invalid/agent99/sre-alert/",
relay_token="configured",
)
assert result == {
"status": "source_not_resolved_fail_closed",
"accepted": False,
"runtime_write_authorized": False,
}
@pytest.mark.parametrize(
("relay_complete", "remove_runtime_field", "invalid_generation_type"),
[(False, False, False), (True, True, False), (True, False, True)],
)
def test_outcome_readback_rejects_incomplete_relay_contract(
monkeypatch,
relay_complete: bool,
remove_runtime_field: bool,
invalid_generation_type: bool,
) -> None:
payload = deepcopy(_valid_outcome())
payload["sameRunContractFieldsComplete"] = relay_complete
if remove_runtime_field:
payload.pop("runtimeWritePerformed")
if invalid_generation_type:
payload["executionGeneration"] = 1
class FakeResponse:
def getcode(self) -> int:
return 200
def read(self, _limit: int) -> bytes:
return json.dumps({"found": True, **payload}).encode()
def __enter__(self): # type: ignore[no-untyped-def]
return self
def __exit__(self, *_args): # type: ignore[no-untyped-def]
return None
monkeypatch.setattr(
bridge.urllib.request,
"urlopen",
lambda *_args, **_kwargs: FakeResponse(),
)
result = bridge.read_agent99_sre_outcome(
str(IDENTITY.run_id),
relay_url="http://agent99.invalid/agent99/sre-alert/",
relay_token="configured-not-returned",
timeout_seconds=2,
)
assert result is not None
assert result["sameRunContractFieldsComplete"] is False
assert job._is_bound_no_write_status_outcome(
result,
identity=IDENTITY,
source_receipt_ref=str(_source_receipt()["receipt_ref"]),
) is False
class _Ledger:
def __init__(self) -> None:
self.verifier_calls: list[dict[str, object]] = []
self.external_calls: list[dict[str, object]] = []
async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def]
return [{
"identity": IDENTITY,
"receipt": {
"status": "dispatch_accepted_verifier_pending",
"post_verifier_passed": False,
},
}]
async def record_verifier(self, **kwargs): # type: ignore[no-untyped-def]
self.verifier_calls.append(kwargs)
return {
"post_verifier_passed": True,
"receipt_persisted": True,
"verifier": {"evidence_refs": kwargs["evidence_refs"]},
}
async def record_external_no_write_reconciliation(
self,
**kwargs,
): # type: ignore[no-untyped-def]
self.external_calls.append(kwargs)
return {
"status": "external_recovery_no_write_reconciled_terminal",
"receipt_persisted": True,
"runtime_closure_verified": False,
}
@pytest.mark.asyncio
async def test_reconciler_requests_status_only_after_production_source_resolves(
monkeypatch,
) -> None:
ledger = _Ledger()
requests: list[tuple[object, object]] = []
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: None)
monkeypatch.setattr(
job,
"read_cold_start_source_resolution",
lambda: _source_receipt(),
)
monkeypatch.setattr(
job,
"request_agent99_same_run_status_reconcile",
lambda identity, source: requests.append((identity, source))
or {"accepted": True},
)
result = await job.reconcile_agent99_controlled_dispatches_once()
assert result["status_reconcile_requested"] == 1
assert result["verifier_written"] == 0
assert requests == [(IDENTITY, _source_receipt())]
assert ledger.verifier_calls == []
@pytest.mark.asyncio
async def test_reconciler_consumes_only_bound_no_write_status_outcome(
monkeypatch,
) -> None:
ledger = _Ledger()
source = _source_receipt()
outcome = _valid_outcome()
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome)
monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source)
monkeypatch.setattr(
job,
"request_agent99_same_run_status_reconcile",
lambda *_args: pytest.fail("fresh bound outcome must be consumed"),
)
finalize = AsyncMock(side_effect=AssertionError("learning must not run"))
telegram = AsyncMock(side_effect=AssertionError("Telegram must not run"))
playbook = AsyncMock(side_effect=AssertionError("PlayBook must not run"))
km = AsyncMock(side_effect=AssertionError("KM must not run"))
monkeypatch.setattr(job, "_finalize_learning", finalize)
monkeypatch.setattr(job, "_ensure_telegram_receipt", telegram)
monkeypatch.setattr(job, "_ensure_playbook_trust", playbook)
monkeypatch.setattr(job, "_ensure_km_writeback", km)
result = await job.reconcile_agent99_controlled_dispatches_once()
assert result["external_no_write_reconciled"] == 1
assert result["verifier_written"] == 0
assert len(ledger.external_calls) == 1
assert (
ledger.external_calls[0]["evidence_refs"]["source_event_evidence_ref"]
== source["receipt_ref"]
)
assert ledger.verifier_calls == []
finalize.assert_not_awaited()
telegram.assert_not_awaited()
playbook.assert_not_awaited()
km.assert_not_awaited()
@pytest.mark.asyncio
async def test_reconciler_rejects_status_outcome_with_mismatched_identity(
monkeypatch,
) -> None:
ledger = _Ledger()
source = _source_receipt()
other = build_agent99_dispatch_identity(
project_id=IDENTITY.project_id,
incident_id=IDENTITY.incident_id,
source_fingerprint=IDENTITY.source_fingerprint,
route_id=IDENTITY.route_id,
work_item_id="different-work-item",
)
outcome = {
"identity": other.public_dict(),
"mode": "Status",
"controlledApply": False,
"reconcileOnly": True,
"runtimeWritePerformed": False,
"outcome": {
"schemaVersion": "agent99_outcome_contract_v1",
"sourceEventResolved": True,
"sourceEventEvidence": source["receipt_ref"],
"identity": other.public_dict(),
},
}
requests: list[tuple[object, object]] = []
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome)
monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source)
monkeypatch.setattr(
job,
"request_agent99_same_run_status_reconcile",
lambda identity, receipt: requests.append((identity, receipt))
or {"accepted": True},
)
result = await job.reconcile_agent99_controlled_dispatches_once()
assert result["status_reconcile_requested"] == 1
assert result["verifier_written"] == 0
assert requests == [(IDENTITY, source)]
assert ledger.verifier_calls == []
def _mutate_path(payload: dict[str, object], path: str, value: object) -> None:
target: dict[str, object] = payload
parts = path.split(".")
for part in parts[:-1]:
nested = target[part]
assert isinstance(nested, dict)
target = nested
target[parts[-1]] = value
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "value"),
[
("outcome.state", "degraded"),
("outcome.state", "failed"),
("outcome.resolved", False),
("outcome.transportOk", False),
("outcome.verifierPassed", False),
("outcome.verifierName", "agent99-status-post-condition-v1"),
("reconcileOnly", False),
("runtimeWritePerformed", True),
("sameRunContractFieldsComplete", False),
("traceId", "00-00000000000000000000000000000000-0000000000000000-01"),
("workItemId", "different-work-item"),
("projectId", "other-project"),
("incidentId", "INC-OTHER"),
("automationRunId", "00000000-0000-0000-0000-000000000000"),
("routeId", "agent99:host_recovery:Status"),
("executionGeneration", "2"),
("evidenceReceiptId", "same-run-status-wrong"),
("outcome.sourceEventEvidence", "prometheus:cold-start:stale"),
],
)
async def test_reconciler_rejects_synthetic_bad_same_run_outcomes(
monkeypatch,
path: str,
value: object,
) -> None:
ledger = _Ledger()
source = _source_receipt()
outcome = deepcopy(_valid_outcome())
_mutate_path(outcome, path, value)
requests: list[tuple[object, object]] = []
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome)
monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source)
monkeypatch.setattr(
job,
"request_agent99_same_run_status_reconcile",
lambda identity, receipt: requests.append((identity, receipt))
or {"accepted": True},
)
finalize = AsyncMock(side_effect=AssertionError("learning must not run"))
monkeypatch.setattr(job, "_finalize_learning", finalize)
result = await job.reconcile_agent99_controlled_dispatches_once()
assert result["status_reconcile_requested"] == 1
assert result["external_no_write_reconciled"] == 0
assert result["verifier_written"] == 0
assert requests == [(IDENTITY, source)]
assert ledger.external_calls == []
assert ledger.verifier_calls == []
finalize.assert_not_awaited()
class _ScalarResult:
def __init__(self, *, value=None, row=None) -> None:
self.value = value
self.row = row
def scalar_one_or_none(self): # type: ignore[no-untyped-def]
return self.value
def one_or_none(self): # type: ignore[no-untyped-def]
return self.row
class _Context:
def __init__(self, db) -> None: # type: ignore[no-untyped-def]
self.db = db
async def __aenter__(self): # type: ignore[no-untyped-def]
return self.db
async def __aexit__(self, *_args): # type: ignore[no-untyped-def]
return False
@pytest.mark.asyncio
async def test_external_no_write_terminal_never_records_recover_success_or_closure(
monkeypatch,
) -> None:
envelope = build_agent99_dispatch_receipt_envelope(
identity=IDENTITY,
dispatch_receipt={
"accepted": True,
"inbox_triggered": True,
"delivery_certainty": "delivered",
},
controlled_apply_authorized=True,
)
envelope["runtime_execution_attempted"] = True
class ReconcileDB:
def __init__(self) -> None:
self.call = 0
self.statements: list[str] = []
async def execute(self, statement): # type: ignore[no-untyped-def]
self.call += 1
self.statements.append(str(statement))
if self.call == 1:
return _ScalarResult(
row=SimpleNamespace(
state="waiting_tool",
error_detail=json.dumps(envelope),
)
)
if self.call == 2:
return _ScalarResult(value=IDENTITY.run_id)
return _ScalarResult()
db = ReconcileDB()
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: _Context(db),
)
result = await PostgresAgent99DispatchLedger().record_external_no_write_reconciliation(
identity=IDENTITY,
outcome_receipt=_valid_outcome(),
source_receipt=_source_receipt(),
evidence_refs=_evidence_refs(),
)
assert result["receipt_persisted"] is True
assert result["run_terminal_state"] == "cancelled"
assert result["automation_execution_success"] is False
assert result["runtime_execution_attempted"] is True
assert result["same_run_status_runtime_execution_attempted"] is False
assert result["post_verifier_passed"] is False
assert result["external_status_verifier_passed"] is True
assert result["runtime_closure_verified"] is False
assert result["closure_complete"] is False
assert result["incident_resolution_authorized"] is False
assert result["external_recovery_reconciliation"][
"recover_execution_success"
] is False
assert result["external_recovery_reconciliation"][
"telegram_authorized"
] is False
sql = "\n".join(db.statements).lower()
assert "incident_records" not in sql
assert "alert_operation_logs" not in sql
assert "playbook" not in sql
assert "telegram" not in sql
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "value"),
[
("runtimeWritePerformed", True),
("outcome.verifierPassed", False),
("outcome.state", "degraded"),
],
)
async def test_external_no_write_terminal_rejects_invalid_contract_before_db(
monkeypatch,
path: str,
value: object,
) -> None:
outcome = deepcopy(_valid_outcome())
_mutate_path(outcome, path, value)
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: pytest.fail("invalid contract must not touch DB"),
)
result = await PostgresAgent99DispatchLedger().record_external_no_write_reconciliation(
identity=IDENTITY,
outcome_receipt=outcome,
source_receipt=_source_receipt(),
evidence_refs=_evidence_refs(),
)
assert result["status"] == "external_recovery_contract_invalid_fail_closed"
assert result["receipt_persisted"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("malformed", ["95", float("nan"), float("inf"), -float("inf")])
async def test_external_no_write_terminal_rejects_malformed_source_counts(
monkeypatch,
malformed: object,
) -> None:
source = deepcopy(_source_receipt())
values = source["values"]
assert isinstance(values, dict)
values["pass_gates"] = malformed
monkeypatch.setattr(
ledger_module,
"get_db_context",
lambda _project_id: pytest.fail("malformed source must not touch DB"),
)
result = await PostgresAgent99DispatchLedger().record_external_no_write_reconciliation(
identity=IDENTITY,
outcome_receipt=_valid_outcome(),
source_receipt=source,
evidence_refs=_evidence_refs(),
)
assert result["status"] == "external_recovery_contract_invalid_fail_closed"
assert result["receipt_persisted"] is False
def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> None:
root = Path(__file__).resolve().parents[3]
relay = (root / "agent99-sre-alert-relay.ps1").read_text(encoding="utf-8")
control = (root / "agent99-control-plane.ps1").read_text(encoding="utf-8")
bootstrap = (root / "agent99-bootstrap.ps1").read_text(encoding="utf-8")
reconcile_source = (
root / "apps/api/src/services/agent99_same_run_reconcile.py"
).read_text(encoding="utf-8")
assert (
'AGENT99_COLD_START_RUN_ID = "bf30ca01-1080-5725-b2d1-3e1534dfc811"'
in reconcile_source
)
assert "agent99_same_run_status_reconcile_v1" in relay
assert "function Test-AgentSameRunDispatchIdentity" in relay
assert 'route_id" "") -ne "agent99:host_recovery:Recover"' in relay
assert 'incident_id" "") -ne "INC-20260711-11C751"' in relay
assert 'runId -ne "bf30ca01-1080-5725-b2d1-3e1534dfc811"' in relay
assert 'execution_generation" "") -ne "1"' in relay
assert 'project_id" "") -ne "awoooi"' in relay
assert 'work_item_id" "") -ne "agent99-dispatch:awoooi:' in relay
assert 'trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in relay
assert "function Get-AgentExistingRecoverIdentity" in relay
assert "existing_dispatch_identity_not_found" in relay
assert "existing_dispatch_identity_mismatch" in relay
assert "function Test-AgentSameRunOutcomeEligible" in relay
assert 'sameRunContractFieldsComplete = $sameRunContractFieldsComplete' in relay
assert 'sameRunContractFieldsComplete" $false' in relay
assert "existing_same_run_status_outcome_invalid_fail_closed" in relay
assert "[IO.FileMode]::CreateNew" in relay
assert "agent99_same_run_single_flight_lock_v1" in relay
assert "relay_auth_not_configured" in relay
assert 'mode = "Status"' in relay
assert 'controlledApply = $false' in relay
assert 'reconcileOnly = $true' in relay
assert 'suppressTelegram = $true' in relay
assert 'sourceEventResolutionPolicy = "external_source_receipt_required"' in relay
assert '"-ReconcileQueueOnly", "-ReconcileQueueId", $queueId' in relay
assert '"-SuppressAlerts", "-SuppressReason"' in relay
assert "[switch]$ReconcileOnly" in control
assert '[string]$ReconcileEvidenceId = ""' in control
assert "[switch]$ReconcileQueueOnly" in control
assert '[string]$ReconcileQueueId = ""' in control
assert 'param([string]$OnlyCommandId = "")' in control
assert 'Get-Item -LiteralPath $onlyPath' in control
assert 'reason = "same_run_reconcile_exact_queue_contract_required"' in control
assert "$canonicalDigestMatches" in control
assert "$sameRunReconcileContractExact" in control
assert '$incidentId -eq "INC-20260711-11C751"' in control
assert (
'$automationRunId -eq "bf30ca01-1080-5725-b2d1-3e1534dfc811"'
in control
)
assert '$dispatchRouteId -eq "agent99:host_recovery:Recover"' in control
assert '$executionGeneration -eq "1"' in control
assert '$lockOwner -eq $automationRunId' in control
assert '$projectId -eq "awoooi"' in control
assert '$workItemId -eq "agent99-dispatch:awoooi:' in control
assert '$traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in control
assert '$reconcileEvidenceId -eq $expectedReconcileEvidenceId' in control
assert "function Get-AgentEvidenceAtPath" in control
assert 'Get-AgentEvidenceAtPath $exactEvidencePath' in control
assert 'Get-AgentBootState -Persist:(-not $ReconcileOnly)' in control
assert '$Mode -eq "Status" -and\n -not $ReconcileOnly -and' in control
assert '"reconcile_only_no_runtime_write"' in control
assert '$data.runtimeWritePerformed -is [bool]' in control
assert '$data.host112Recovery.runtimeWritePerformed -is [bool]' in control
assert '$data.reconcileEvidenceId -is [string]' in control
assert '"-ReconcileOnly", "-ReconcileEvidenceId", $reconcileEvidenceId' in control
assert 'status = "not_written_same_run_reconcile"' in control
assert 'status = "suppressed_same_run_reconcile"' in control
assert (
'status = "suppressed_same_run_reconcile_source_recheck_required"'
in control
)
assert 'Invoke-AgentControlTick -QueueOnly:$ReconcileQueueOnly' in control
assert "-ReconcileOnly:`$ReconcileOnly" in bootstrap
assert "-ReconcileEvidenceId `$ReconcileEvidenceId" in bootstrap
assert "-ReconcileQueueOnly:`$ReconcileQueueOnly" in bootstrap
reconcile_job = (
root
/ "apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py"
).read_text(encoding="utf-8")
cold_branch = reconcile_job[
reconcile_job.index("if is_cold_start_same_run_identity(identity):") :
reconcile_job.index(
'if receipt.get("post_verifier_passed") is not True:'
)
]
assert "record_external_no_write_reconciliation" in cold_branch
assert "_finalize_learning" not in cold_branch
assert "_ensure_telegram_receipt" not in cold_branch
assert "_ensure_playbook_trust" not in cold_branch
assert "_ensure_km_writeback" not in cold_branch
queue_function = control.index("function Invoke-AgentQueuedCommands")
launch_args = control.index("$args = @(", queue_function)
queue_branch = control[
control.index("if ($reconcileOnly) {", launch_args) :
control.index("$exitCode = if", launch_args)
]
assert '"-ReconcileOnly", "-ReconcileEvidenceId"' in queue_branch
assert "Get-AgentEvidenceAtPath $exactEvidencePath" in queue_branch
assert "Send-AgentTelegram" not in queue_branch
assert "Invoke-AgentCompletionCallback" not in queue_branch