fix(agent99): reconcile live cold-start dispatches
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m8s
CD Pipeline / build-and-deploy (push) Successful in 13m33s
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
ogt
2026-07-15 03:18:09 +08:00
parent 66aeb34bf6
commit 29072e4004
11 changed files with 300 additions and 73 deletions

View File

@@ -37,6 +37,10 @@ def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() ->
identity=identity,
dispatch_receipt={
"status": "accepted_inbox_pending",
"kind": "host_recovery",
"suggested_mode": "Recover",
"target_resource": "cold-start-gate",
"controlled_apply_requested": True,
"accepted": True,
"inbox_triggered": False,
"delivery_certainty": "unknown",
@@ -48,6 +52,13 @@ def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() ->
assert envelope["dispatch_promoted"] is False
assert envelope["controlled_apply_authorized"] is False
assert envelope["retry_policy"]["reconcile_only"] is True
assert envelope["dispatch_scope"] == {
"schema_version": "agent99_dispatch_scope_v1",
"kind": "host_recovery",
"suggested_mode": "Recover",
"target_resource": "cold-start-gate",
"controlled_apply_requested": True,
}
@pytest.mark.asyncio

View File

@@ -128,6 +128,44 @@ def test_cold_start_source_queries_include_host112_scope() -> None:
assert 'scope="110_120_121_188"' not in query
def test_live_cold_start_identity_requires_canonical_dispatch_scope() -> None:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260715-A1B2C3",
source_fingerprint="a" * 32,
route_id="agent99:host_recovery:Recover",
work_item_id=(
"agent99-dispatch:awoooi:INC-20260715-A1B2C3:"
"agent99:host_recovery:Recover"
),
)
scope = {
"dispatch_scope": {
"schema_version": "agent99_dispatch_scope_v1",
"kind": "host_recovery",
"suggested_mode": "Recover",
"target_resource": "cold-start-gate",
"controlled_apply_requested": True,
}
}
assert reconcile.is_cold_start_same_run_identity(identity, scope) is True
wrong_target = deepcopy(scope)
wrong_target["dispatch_scope"]["target_resource"] = "other-host"
assert reconcile.is_cold_start_same_run_identity(identity, wrong_target) is False
wrong_work_item = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260715-A1B2C3",
source_fingerprint="a" * 32,
route_id="agent99:host_recovery:Recover",
work_item_id="agent99-dispatch:awoooi:INC-20260715-A1B2C3:other",
)
assert (
reconcile.is_cold_start_same_run_identity(wrong_work_item, scope) is False
)
assert reconcile.is_cold_start_same_run_identity(identity, {}) is False
def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero(
monkeypatch,
) -> None:
@@ -301,6 +339,48 @@ def test_same_run_status_request_rejects_unresolved_source_without_transport(
}
def test_live_same_run_request_carries_dispatch_scope_into_identity_gate(
monkeypatch,
) -> None:
identity = build_agent99_dispatch_identity(
project_id="awoooi",
incident_id="INC-20260715-D4E5F6",
source_fingerprint="b" * 64,
route_id="agent99:host_recovery:Recover",
)
dispatch_receipt = {
"dispatch_scope": {
"schema_version": "agent99_dispatch_scope_v1",
"kind": "host_recovery",
"suggested_mode": "Recover",
"target_resource": "cold-start-gate",
"controlled_apply_requested": True,
}
}
monkeypatch.setattr(
reconcile.urllib.request,
"urlopen",
lambda *_args, **_kwargs: pytest.fail("transport must not run"),
)
eligible = reconcile.request_agent99_same_run_status_reconcile(
identity,
{"resolved": False, "receipt_ref": None},
dispatch_receipt,
relay_url="http://agent99.invalid/agent99/sre-alert/",
relay_token="configured",
)
missing_scope = reconcile.request_agent99_same_run_status_reconcile(
identity,
{"resolved": False, "receipt_ref": None},
relay_url="http://agent99.invalid/agent99/sre-alert/",
relay_token="configured",
)
assert eligible["status"] == "source_not_resolved_fail_closed"
assert missing_scope["status"] == "identity_not_eligible_fail_closed"
@pytest.mark.parametrize(
("relay_complete", "remove_runtime_field", "invalid_generation_type"),
[(False, False, False), (True, True, False), (True, False, True)],
@@ -357,14 +437,15 @@ class _Ledger:
def __init__(self) -> None:
self.verifier_calls: list[dict[str, object]] = []
self.external_calls: list[dict[str, object]] = []
self.list_receipt: dict[str, object] = {
"status": "dispatch_accepted_verifier_pending",
"post_verifier_passed": False,
}
async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def]
return [{
"identity": IDENTITY,
"receipt": {
"status": "dispatch_accepted_verifier_pending",
"post_verifier_passed": False,
},
"receipt": self.list_receipt,
}]
async def record_verifier(self, **kwargs): # type: ignore[no-untyped-def]
@@ -392,7 +473,7 @@ async def test_reconciler_requests_status_only_after_production_source_resolves(
monkeypatch,
) -> None:
ledger = _Ledger()
requests: list[tuple[object, object]] = []
requests: list[tuple[object, object, object]] = []
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: None)
monkeypatch.setattr(
@@ -403,7 +484,9 @@ async def test_reconciler_requests_status_only_after_production_source_resolves(
monkeypatch.setattr(
job,
"request_agent99_same_run_status_reconcile",
lambda identity, source: requests.append((identity, source))
lambda identity, source, receipt: requests.append(
(identity, source, receipt)
)
or {"accepted": True},
)
@@ -411,7 +494,11 @@ async def test_reconciler_requests_status_only_after_production_source_resolves(
assert result["status_reconcile_requested"] == 1
assert result["verifier_written"] == 0
assert requests == [(IDENTITY, _source_receipt())]
assert requests == [(
IDENTITY,
_source_receipt(),
ledger.list_receipt,
)]
assert ledger.verifier_calls == []
@@ -481,14 +568,16 @@ async def test_reconciler_rejects_status_outcome_with_mismatched_identity(
"identity": other.public_dict(),
},
}
requests: list[tuple[object, object]] = []
requests: list[tuple[object, object, object]] = []
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome)
monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source)
monkeypatch.setattr(
job,
"request_agent99_same_run_status_reconcile",
lambda identity, receipt: requests.append((identity, receipt))
lambda identity, source, receipt: requests.append(
(identity, source, receipt)
)
or {"accepted": True},
)
@@ -496,7 +585,7 @@ async def test_reconciler_rejects_status_outcome_with_mismatched_identity(
assert result["status_reconcile_requested"] == 1
assert result["verifier_written"] == 0
assert requests == [(IDENTITY, source)]
assert requests == [(IDENTITY, source, ledger.list_receipt)]
assert ledger.verifier_calls == []
@@ -543,14 +632,16 @@ async def test_reconciler_rejects_synthetic_bad_same_run_outcomes(
source = _source_receipt()
outcome = deepcopy(_valid_outcome())
_mutate_path(outcome, path, value)
requests: list[tuple[object, object]] = []
requests: list[tuple[object, object, object]] = []
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome)
monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source)
monkeypatch.setattr(
job,
"request_agent99_same_run_status_reconcile",
lambda identity, receipt: requests.append((identity, receipt))
lambda identity, source, receipt: requests.append(
(identity, source, receipt)
)
or {"accepted": True},
)
finalize = AsyncMock(side_effect=AssertionError("learning must not run"))
@@ -561,7 +652,7 @@ async def test_reconciler_rejects_synthetic_bad_same_run_outcomes(
assert result["status_reconcile_requested"] == 1
assert result["external_no_write_reconciled"] == 0
assert result["verifier_written"] == 0
assert requests == [(IDENTITY, source)]
assert requests == [(IDENTITY, source, ledger.list_receipt)]
assert ledger.external_calls == []
assert ledger.verifier_calls == []
finalize.assert_not_awaited()
@@ -738,22 +829,21 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
assert "agent99_same_run_status_reconcile_v1" in relay
assert "function Test-AgentSameRunDispatchIdentity" in relay
assert 'route_id" "") -ne "agent99:host_recovery:Recover"' in relay
assert 'incident_id" "") -ne "INC-20260711-11C751"' in relay
assert 'runId -ne "bf30ca01-1080-5725-b2d1-3e1534dfc811"' in relay
assert 'execution_generation" "") -ne "1"' in relay
assert 'project_id" "") -ne "awoooi"' in relay
assert 'work_item_id" "") -ne "agent99-dispatch:awoooi:' in relay
assert 'trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in relay
assert '$incidentId -match "^INC-[0-9]{8}-[0-9A-F]{6}$"' in relay
assert '$executionGeneration -match "^[1-3]$"' in relay
assert '$sourceFingerprint -match "^[0-9a-f]{32,64}$"' in relay
assert 'function Test-AgentStoredColdStartRecoverEnvelope' in relay
assert "function Get-AgentExistingRecoverIdentity" in relay
assert '$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"' in relay
assert '$acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json"' in relay
assert '$files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)' in relay
assert 'Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern' in relay
assert 'Get-AgentField $awoooi "agent99DispatchIdentity"' in relay
assert 'Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId"' in relay
assert 'Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager"' in relay
assert 'Get-AgentField $stored "kind" "") -ne "host_recovery"' in relay
assert 'Get-AgentField $stored "service" "") -ne "cold-start-gate"' in relay
assert '$source -eq "awoooi-api-alertmanager"' in relay
assert '$kind -eq "host_recovery"' in relay
assert '$service -eq "cold-start-gate"' in relay
assert "existing_dispatch_identity_not_found" in relay
assert "existing_dispatch_identity_mismatch" in relay
assert "function Test-AgentSameRunOutcomeEligible" in relay
@@ -782,16 +872,15 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
assert 'reason = "same_run_reconcile_exact_queue_contract_required"' in control
assert "$canonicalDigestMatches" in control
assert "$sameRunReconcileContractExact" in control
assert '$incidentId -eq "INC-20260711-11C751"' in control
assert (
'$automationRunId -eq "bf30ca01-1080-5725-b2d1-3e1534dfc811"'
in control
)
assert "$automationRunIdValid" in control
assert "$legacyColdStartIdentity" in control
assert "$liveColdStartIdentity" in control
assert "$coldStartIdentityEligible" in control
assert '$dispatchRouteId -eq "agent99:host_recovery:Recover"' in control
assert '$executionGeneration -eq "1"' in control
assert '$lockOwner -eq $automationRunId' in control
assert '$projectId -eq "awoooi"' in control
assert '$workItemId -eq "agent99-dispatch:awoooi:' in control
assert '$commandSource -eq "agent99-same-run-reconcile"' in control
assert '$alertSource -eq "awoooi-production-source-verifier"' in control
assert '$traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in control
assert '$reconcileEvidenceId -eq $expectedReconcileEvidenceId' in control
assert "function Get-AgentEvidenceAtPath" in control
@@ -819,7 +908,7 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
/ "apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py"
).read_text(encoding="utf-8")
cold_branch = reconcile_job[
reconcile_job.index("if is_cold_start_same_run_identity(identity):") :
reconcile_job.index("if is_cold_start_same_run_identity(identity, receipt):") :
reconcile_job.index(
'if receipt.get("post_verifier_passed") is not True:'
)

View File

@@ -362,6 +362,9 @@ def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -
"transport": "relay",
"alert_id": "awoooi-alertmanager-receipt-smoke",
"kind": "host_recovery",
"target_resource": "cold-start-gate",
"suggested_mode": "Recover",
"controlled_apply_requested": True,
"http_status": 202,
"accepted": True,
"inbox_triggered": True,

View File

@@ -225,6 +225,13 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start(
incident_lookup.assert_not_awaited()
bridge.assert_awaited_once()
assert handoff["single_writer_executor"] == "Agent99"
assert handoff["policy_route_id"] == "agent99_recover_after_owner_review"
assert handoff["route_id"] == "agent99:host_recovery:Recover"
assert bridge.await_args.kwargs["route_id"] == "agent99:host_recovery:Recover"
assert bridge.await_args.kwargs["work_item_id"] == (
"agent99-dispatch:awoooi:INC-20260711-11C751:"
"agent99:host_recovery:Recover"
)
assert handoff["execution_priority"] == 30
assert handoff["queued"] is True
assert handoff["runtime_execution_authorized"] is False