fix(agent99): reconcile live cold-start dispatches
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m8s
CD Pipeline / build-and-deploy (push) Successful in 13m33s
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m8s
CD Pipeline / build-and-deploy (push) Successful in 13m33s
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
@@ -37,6 +37,10 @@ def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() ->
|
||||
identity=identity,
|
||||
dispatch_receipt={
|
||||
"status": "accepted_inbox_pending",
|
||||
"kind": "host_recovery",
|
||||
"suggested_mode": "Recover",
|
||||
"target_resource": "cold-start-gate",
|
||||
"controlled_apply_requested": True,
|
||||
"accepted": True,
|
||||
"inbox_triggered": False,
|
||||
"delivery_certainty": "unknown",
|
||||
@@ -48,6 +52,13 @@ def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() ->
|
||||
assert envelope["dispatch_promoted"] is False
|
||||
assert envelope["controlled_apply_authorized"] is False
|
||||
assert envelope["retry_policy"]["reconcile_only"] is True
|
||||
assert envelope["dispatch_scope"] == {
|
||||
"schema_version": "agent99_dispatch_scope_v1",
|
||||
"kind": "host_recovery",
|
||||
"suggested_mode": "Recover",
|
||||
"target_resource": "cold-start-gate",
|
||||
"controlled_apply_requested": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -128,6 +128,44 @@ def test_cold_start_source_queries_include_host112_scope() -> None:
|
||||
assert 'scope="110_120_121_188"' not in query
|
||||
|
||||
|
||||
def test_live_cold_start_identity_requires_canonical_dispatch_scope() -> None:
|
||||
identity = build_agent99_dispatch_identity(
|
||||
project_id="awoooi",
|
||||
incident_id="INC-20260715-A1B2C3",
|
||||
source_fingerprint="a" * 32,
|
||||
route_id="agent99:host_recovery:Recover",
|
||||
work_item_id=(
|
||||
"agent99-dispatch:awoooi:INC-20260715-A1B2C3:"
|
||||
"agent99:host_recovery:Recover"
|
||||
),
|
||||
)
|
||||
scope = {
|
||||
"dispatch_scope": {
|
||||
"schema_version": "agent99_dispatch_scope_v1",
|
||||
"kind": "host_recovery",
|
||||
"suggested_mode": "Recover",
|
||||
"target_resource": "cold-start-gate",
|
||||
"controlled_apply_requested": True,
|
||||
}
|
||||
}
|
||||
|
||||
assert reconcile.is_cold_start_same_run_identity(identity, scope) is True
|
||||
wrong_target = deepcopy(scope)
|
||||
wrong_target["dispatch_scope"]["target_resource"] = "other-host"
|
||||
assert reconcile.is_cold_start_same_run_identity(identity, wrong_target) is False
|
||||
wrong_work_item = build_agent99_dispatch_identity(
|
||||
project_id="awoooi",
|
||||
incident_id="INC-20260715-A1B2C3",
|
||||
source_fingerprint="a" * 32,
|
||||
route_id="agent99:host_recovery:Recover",
|
||||
work_item_id="agent99-dispatch:awoooi:INC-20260715-A1B2C3:other",
|
||||
)
|
||||
assert (
|
||||
reconcile.is_cold_start_same_run_identity(wrong_work_item, scope) is False
|
||||
)
|
||||
assert reconcile.is_cold_start_same_run_identity(identity, {}) is False
|
||||
|
||||
|
||||
def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
@@ -301,6 +339,48 @@ def test_same_run_status_request_rejects_unresolved_source_without_transport(
|
||||
}
|
||||
|
||||
|
||||
def test_live_same_run_request_carries_dispatch_scope_into_identity_gate(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
identity = build_agent99_dispatch_identity(
|
||||
project_id="awoooi",
|
||||
incident_id="INC-20260715-D4E5F6",
|
||||
source_fingerprint="b" * 64,
|
||||
route_id="agent99:host_recovery:Recover",
|
||||
)
|
||||
dispatch_receipt = {
|
||||
"dispatch_scope": {
|
||||
"schema_version": "agent99_dispatch_scope_v1",
|
||||
"kind": "host_recovery",
|
||||
"suggested_mode": "Recover",
|
||||
"target_resource": "cold-start-gate",
|
||||
"controlled_apply_requested": True,
|
||||
}
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
reconcile.urllib.request,
|
||||
"urlopen",
|
||||
lambda *_args, **_kwargs: pytest.fail("transport must not run"),
|
||||
)
|
||||
|
||||
eligible = reconcile.request_agent99_same_run_status_reconcile(
|
||||
identity,
|
||||
{"resolved": False, "receipt_ref": None},
|
||||
dispatch_receipt,
|
||||
relay_url="http://agent99.invalid/agent99/sre-alert/",
|
||||
relay_token="configured",
|
||||
)
|
||||
missing_scope = reconcile.request_agent99_same_run_status_reconcile(
|
||||
identity,
|
||||
{"resolved": False, "receipt_ref": None},
|
||||
relay_url="http://agent99.invalid/agent99/sre-alert/",
|
||||
relay_token="configured",
|
||||
)
|
||||
|
||||
assert eligible["status"] == "source_not_resolved_fail_closed"
|
||||
assert missing_scope["status"] == "identity_not_eligible_fail_closed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("relay_complete", "remove_runtime_field", "invalid_generation_type"),
|
||||
[(False, False, False), (True, True, False), (True, False, True)],
|
||||
@@ -357,14 +437,15 @@ class _Ledger:
|
||||
def __init__(self) -> None:
|
||||
self.verifier_calls: list[dict[str, object]] = []
|
||||
self.external_calls: list[dict[str, object]] = []
|
||||
self.list_receipt: dict[str, object] = {
|
||||
"status": "dispatch_accepted_verifier_pending",
|
||||
"post_verifier_passed": False,
|
||||
}
|
||||
|
||||
async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def]
|
||||
return [{
|
||||
"identity": IDENTITY,
|
||||
"receipt": {
|
||||
"status": "dispatch_accepted_verifier_pending",
|
||||
"post_verifier_passed": False,
|
||||
},
|
||||
"receipt": self.list_receipt,
|
||||
}]
|
||||
|
||||
async def record_verifier(self, **kwargs): # type: ignore[no-untyped-def]
|
||||
@@ -392,7 +473,7 @@ async def test_reconciler_requests_status_only_after_production_source_resolves(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
ledger = _Ledger()
|
||||
requests: list[tuple[object, object]] = []
|
||||
requests: list[tuple[object, object, object]] = []
|
||||
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
|
||||
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: None)
|
||||
monkeypatch.setattr(
|
||||
@@ -403,7 +484,9 @@ async def test_reconciler_requests_status_only_after_production_source_resolves(
|
||||
monkeypatch.setattr(
|
||||
job,
|
||||
"request_agent99_same_run_status_reconcile",
|
||||
lambda identity, source: requests.append((identity, source))
|
||||
lambda identity, source, receipt: requests.append(
|
||||
(identity, source, receipt)
|
||||
)
|
||||
or {"accepted": True},
|
||||
)
|
||||
|
||||
@@ -411,7 +494,11 @@ async def test_reconciler_requests_status_only_after_production_source_resolves(
|
||||
|
||||
assert result["status_reconcile_requested"] == 1
|
||||
assert result["verifier_written"] == 0
|
||||
assert requests == [(IDENTITY, _source_receipt())]
|
||||
assert requests == [(
|
||||
IDENTITY,
|
||||
_source_receipt(),
|
||||
ledger.list_receipt,
|
||||
)]
|
||||
assert ledger.verifier_calls == []
|
||||
|
||||
|
||||
@@ -481,14 +568,16 @@ async def test_reconciler_rejects_status_outcome_with_mismatched_identity(
|
||||
"identity": other.public_dict(),
|
||||
},
|
||||
}
|
||||
requests: list[tuple[object, object]] = []
|
||||
requests: list[tuple[object, object, object]] = []
|
||||
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
|
||||
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome)
|
||||
monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source)
|
||||
monkeypatch.setattr(
|
||||
job,
|
||||
"request_agent99_same_run_status_reconcile",
|
||||
lambda identity, receipt: requests.append((identity, receipt))
|
||||
lambda identity, source, receipt: requests.append(
|
||||
(identity, source, receipt)
|
||||
)
|
||||
or {"accepted": True},
|
||||
)
|
||||
|
||||
@@ -496,7 +585,7 @@ async def test_reconciler_rejects_status_outcome_with_mismatched_identity(
|
||||
|
||||
assert result["status_reconcile_requested"] == 1
|
||||
assert result["verifier_written"] == 0
|
||||
assert requests == [(IDENTITY, source)]
|
||||
assert requests == [(IDENTITY, source, ledger.list_receipt)]
|
||||
assert ledger.verifier_calls == []
|
||||
|
||||
|
||||
@@ -543,14 +632,16 @@ async def test_reconciler_rejects_synthetic_bad_same_run_outcomes(
|
||||
source = _source_receipt()
|
||||
outcome = deepcopy(_valid_outcome())
|
||||
_mutate_path(outcome, path, value)
|
||||
requests: list[tuple[object, object]] = []
|
||||
requests: list[tuple[object, object, object]] = []
|
||||
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
|
||||
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome)
|
||||
monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source)
|
||||
monkeypatch.setattr(
|
||||
job,
|
||||
"request_agent99_same_run_status_reconcile",
|
||||
lambda identity, receipt: requests.append((identity, receipt))
|
||||
lambda identity, source, receipt: requests.append(
|
||||
(identity, source, receipt)
|
||||
)
|
||||
or {"accepted": True},
|
||||
)
|
||||
finalize = AsyncMock(side_effect=AssertionError("learning must not run"))
|
||||
@@ -561,7 +652,7 @@ async def test_reconciler_rejects_synthetic_bad_same_run_outcomes(
|
||||
assert result["status_reconcile_requested"] == 1
|
||||
assert result["external_no_write_reconciled"] == 0
|
||||
assert result["verifier_written"] == 0
|
||||
assert requests == [(IDENTITY, source)]
|
||||
assert requests == [(IDENTITY, source, ledger.list_receipt)]
|
||||
assert ledger.external_calls == []
|
||||
assert ledger.verifier_calls == []
|
||||
finalize.assert_not_awaited()
|
||||
@@ -738,22 +829,21 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
|
||||
assert "agent99_same_run_status_reconcile_v1" in relay
|
||||
assert "function Test-AgentSameRunDispatchIdentity" in relay
|
||||
assert 'route_id" "") -ne "agent99:host_recovery:Recover"' in relay
|
||||
assert 'incident_id" "") -ne "INC-20260711-11C751"' in relay
|
||||
assert 'runId -ne "bf30ca01-1080-5725-b2d1-3e1534dfc811"' in relay
|
||||
assert 'execution_generation" "") -ne "1"' in relay
|
||||
assert 'project_id" "") -ne "awoooi"' in relay
|
||||
assert 'work_item_id" "") -ne "agent99-dispatch:awoooi:' in relay
|
||||
assert 'trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in relay
|
||||
assert '$incidentId -match "^INC-[0-9]{8}-[0-9A-F]{6}$"' in relay
|
||||
assert '$executionGeneration -match "^[1-3]$"' in relay
|
||||
assert '$sourceFingerprint -match "^[0-9a-f]{32,64}$"' in relay
|
||||
assert 'function Test-AgentStoredColdStartRecoverEnvelope' in relay
|
||||
assert "function Get-AgentExistingRecoverIdentity" in relay
|
||||
assert '$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"' in relay
|
||||
assert '$acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json"' in relay
|
||||
assert '$files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)' in relay
|
||||
assert 'Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern' in relay
|
||||
assert 'Get-AgentField $awoooi "agent99DispatchIdentity"' in relay
|
||||
assert 'Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId"' in relay
|
||||
assert 'Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager"' in relay
|
||||
assert 'Get-AgentField $stored "kind" "") -ne "host_recovery"' in relay
|
||||
assert 'Get-AgentField $stored "service" "") -ne "cold-start-gate"' in relay
|
||||
assert '$source -eq "awoooi-api-alertmanager"' in relay
|
||||
assert '$kind -eq "host_recovery"' in relay
|
||||
assert '$service -eq "cold-start-gate"' in relay
|
||||
assert "existing_dispatch_identity_not_found" in relay
|
||||
assert "existing_dispatch_identity_mismatch" in relay
|
||||
assert "function Test-AgentSameRunOutcomeEligible" in relay
|
||||
@@ -782,16 +872,15 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
|
||||
assert 'reason = "same_run_reconcile_exact_queue_contract_required"' in control
|
||||
assert "$canonicalDigestMatches" in control
|
||||
assert "$sameRunReconcileContractExact" in control
|
||||
assert '$incidentId -eq "INC-20260711-11C751"' in control
|
||||
assert (
|
||||
'$automationRunId -eq "bf30ca01-1080-5725-b2d1-3e1534dfc811"'
|
||||
in control
|
||||
)
|
||||
assert "$automationRunIdValid" in control
|
||||
assert "$legacyColdStartIdentity" in control
|
||||
assert "$liveColdStartIdentity" in control
|
||||
assert "$coldStartIdentityEligible" in control
|
||||
assert '$dispatchRouteId -eq "agent99:host_recovery:Recover"' in control
|
||||
assert '$executionGeneration -eq "1"' in control
|
||||
assert '$lockOwner -eq $automationRunId' in control
|
||||
assert '$projectId -eq "awoooi"' in control
|
||||
assert '$workItemId -eq "agent99-dispatch:awoooi:' in control
|
||||
assert '$commandSource -eq "agent99-same-run-reconcile"' in control
|
||||
assert '$alertSource -eq "awoooi-production-source-verifier"' in control
|
||||
assert '$traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in control
|
||||
assert '$reconcileEvidenceId -eq $expectedReconcileEvidenceId' in control
|
||||
assert "function Get-AgentEvidenceAtPath" in control
|
||||
@@ -819,7 +908,7 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
|
||||
/ "apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py"
|
||||
).read_text(encoding="utf-8")
|
||||
cold_branch = reconcile_job[
|
||||
reconcile_job.index("if is_cold_start_same_run_identity(identity):") :
|
||||
reconcile_job.index("if is_cold_start_same_run_identity(identity, receipt):") :
|
||||
reconcile_job.index(
|
||||
'if receipt.get("post_verifier_passed") is not True:'
|
||||
)
|
||||
|
||||
@@ -362,6 +362,9 @@ def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -
|
||||
"transport": "relay",
|
||||
"alert_id": "awoooi-alertmanager-receipt-smoke",
|
||||
"kind": "host_recovery",
|
||||
"target_resource": "cold-start-gate",
|
||||
"suggested_mode": "Recover",
|
||||
"controlled_apply_requested": True,
|
||||
"http_status": 202,
|
||||
"accepted": True,
|
||||
"inbox_triggered": True,
|
||||
|
||||
@@ -225,6 +225,13 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start(
|
||||
incident_lookup.assert_not_awaited()
|
||||
bridge.assert_awaited_once()
|
||||
assert handoff["single_writer_executor"] == "Agent99"
|
||||
assert handoff["policy_route_id"] == "agent99_recover_after_owner_review"
|
||||
assert handoff["route_id"] == "agent99:host_recovery:Recover"
|
||||
assert bridge.await_args.kwargs["route_id"] == "agent99:host_recovery:Recover"
|
||||
assert bridge.await_args.kwargs["work_item_id"] == (
|
||||
"agent99-dispatch:awoooi:INC-20260711-11C751:"
|
||||
"agent99:host_recovery:Recover"
|
||||
)
|
||||
assert handoff["execution_priority"] == 30
|
||||
assert handoff["queued"] is True
|
||||
assert handoff["runtime_execution_authorized"] is False
|
||||
|
||||
Reference in New Issue
Block a user