1b551e0d33
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 8m17s
CD Pipeline / build-and-deploy (push) Successful in 19m28s
CD Pipeline / post-deploy-checks (push) Successful in 5m6s
AI 技術雷達監控 / ai-technology-watch (push) Successful in 39s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 11s
310 lines
12 KiB
Python
310 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from src.services.agent99_controlled_dispatch_ledger import (
|
|
attach_agent99_dispatch_identity,
|
|
build_agent99_dispatch_identity,
|
|
build_agent99_dispatch_receipt_envelope,
|
|
)
|
|
from src.services.agent99_sre_bridge import (
|
|
AGENT99_CONTROL_PLANE_HEALTH_KIND,
|
|
AGENT99_CONTROL_PLANE_HEALTH_MODE,
|
|
AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID,
|
|
bridge_alertmanager_to_agent99,
|
|
build_agent99_sre_alert,
|
|
dispatch_agent99_sre_alert_with_receipt,
|
|
resolve_agent99_alert_kind,
|
|
resolve_agent99_durable_route,
|
|
)
|
|
from src.services.controlled_alert_target_router import resolve_typed_alert_target
|
|
|
|
|
|
def _agent99_health_payload(*, host: str = "192.168.0.99") -> dict[str, object]:
|
|
return build_agent99_sre_alert(
|
|
alert_id="agent99-health-1",
|
|
alertname="Agent99ServiceUnhealthy",
|
|
severity="critical",
|
|
namespace="windows-control-plane",
|
|
target_resource="Agent99",
|
|
message="Agent99 control plane health check failed",
|
|
labels={"host": host, "service": "Agent99"},
|
|
fingerprint="fp-agent99-control-plane-health",
|
|
)
|
|
|
|
|
|
def test_agent99_service_unhealthy_has_exact_durable_selfcheck_route() -> None:
|
|
kind = resolve_agent99_alert_kind(
|
|
alertname="Agent99ServiceUnhealthy",
|
|
severity="critical",
|
|
namespace="windows-control-plane",
|
|
target_resource="Agent99",
|
|
message="control plane health check failed",
|
|
)
|
|
durable_route = resolve_agent99_durable_route(
|
|
alertname="Agent99ServiceUnhealthy",
|
|
severity="critical",
|
|
namespace="windows-control-plane",
|
|
target_resource="Agent99",
|
|
message="control plane health check failed",
|
|
)
|
|
|
|
assert kind == AGENT99_CONTROL_PLANE_HEALTH_KIND
|
|
assert durable_route == {
|
|
"kind": AGENT99_CONTROL_PLANE_HEALTH_KIND,
|
|
"suggested_mode": AGENT99_CONTROL_PLANE_HEALTH_MODE,
|
|
"route_id": AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID,
|
|
}
|
|
assert (
|
|
resolve_agent99_alert_kind(
|
|
alertname="Agent99ServiceDegraded",
|
|
severity="warning",
|
|
namespace="windows-control-plane",
|
|
target_resource="Agent99",
|
|
message="observation only",
|
|
)
|
|
== "monitoring_alert"
|
|
)
|
|
|
|
|
|
def test_agent99_control_plane_payload_is_host99_selfcheck_only() -> None:
|
|
payload = _agent99_health_payload()
|
|
typed_route = payload["routing"]["typedTargetRoute"] # type: ignore[index]
|
|
action_policy = payload["routing"]["actionPolicy"] # type: ignore[index]
|
|
|
|
assert payload["kind"] == AGENT99_CONTROL_PLANE_HEALTH_KIND
|
|
assert payload["suggestedMode"] == AGENT99_CONTROL_PLANE_HEALTH_MODE
|
|
assert payload["controlledApply"] is False
|
|
assert payload["force"] is False
|
|
assert typed_route["resolution_status"] == "resolved"
|
|
assert typed_route["canonical_asset_id"] == "windows-vmware:host_99"
|
|
assert typed_route["target_kind"] == "windows_vmware"
|
|
assert typed_route["executor"] == "Agent99"
|
|
assert typed_route["verifier"] == (
|
|
"agent99_control_plane_selfcheck_independent_verifier"
|
|
)
|
|
assert typed_route["allowed_inventory_hosts"] == ["host_99"]
|
|
assert typed_route["controlled_apply_allowed"] is False
|
|
assert typed_route["cross_domain_fallback_allowed"] is False
|
|
assert action_policy == {
|
|
"schemaVersion": "agent99_control_plane_action_policy_v1",
|
|
"allowedModes": ["SelfCheck"],
|
|
"controlledApplyAllowed": False,
|
|
"rebootAllowed": False,
|
|
"vmPowerChangeAllowed": False,
|
|
"arbitraryShellAllowed": False,
|
|
"crossHostFallbackAllowed": False,
|
|
"crossDomainFallbackAllowed": False,
|
|
}
|
|
assert "do not reboot" in payload["instruction"]
|
|
assert "change VM power" in payload["instruction"]
|
|
assert "arbitrary shell" in payload["instruction"]
|
|
|
|
|
|
def test_agent99_control_plane_rejects_cross_host_identity() -> None:
|
|
route = resolve_typed_alert_target(
|
|
alertname="Agent99ServiceUnhealthy",
|
|
namespace="windows-control-plane",
|
|
target_resource="Agent99 on host 110",
|
|
labels={"host": "192.168.0.110"},
|
|
)
|
|
|
|
assert route["resolution_status"] == "asset_identity_unresolved"
|
|
assert route["target_kind"] == "unknown"
|
|
assert route["canonical_asset_id"] is None
|
|
assert route["agent99_bridge"]["dispatch_allowed"] is False
|
|
assert route["cross_domain_fallback_allowed"] is False
|
|
|
|
|
|
def test_agent99_dispatch_scope_is_bound_to_same_run(monkeypatch) -> None:
|
|
payload = _agent99_health_payload()
|
|
identity = build_agent99_dispatch_identity(
|
|
project_id="awoooi",
|
|
incident_id="INC-AGENT99-HEALTH-1",
|
|
source_fingerprint="fp-agent99-control-plane-health",
|
|
route_id=AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID,
|
|
work_item_id="AIA-SRE-013",
|
|
)
|
|
payload = attach_agent99_dispatch_identity(payload, identity)
|
|
monkeypatch.setattr(
|
|
"src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_RELAY_URL",
|
|
"",
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_INBOX_PATH",
|
|
"",
|
|
)
|
|
|
|
transport_receipt = dispatch_agent99_sre_alert_with_receipt(payload)
|
|
envelope = build_agent99_dispatch_receipt_envelope(
|
|
identity=identity,
|
|
dispatch_receipt=transport_receipt,
|
|
)
|
|
|
|
assert envelope["dispatch_scope"] == {
|
|
"schema_version": "agent99_dispatch_scope_v1",
|
|
"kind": AGENT99_CONTROL_PLANE_HEALTH_KIND,
|
|
"suggested_mode": "SelfCheck",
|
|
"target_resource": "Agent99",
|
|
"controlled_apply_requested": False,
|
|
"canonical_asset_id": "windows-vmware:host_99",
|
|
"typed_domain": "windows_vmware",
|
|
"executor": "Agent99",
|
|
"verifier": "agent99_control_plane_selfcheck_independent_verifier",
|
|
"route_id": AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID,
|
|
}
|
|
assert envelope["identity"]["run_id"] == str(identity.run_id)
|
|
assert envelope["controlled_apply_authorized"] is False
|
|
assert envelope["runtime_execution_authorized"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_agent99_control_plane_dispatch_stays_selfcheck_no_write(
|
|
monkeypatch,
|
|
) -> None:
|
|
captured: list[dict[str, object]] = []
|
|
|
|
class Ledger:
|
|
async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def]
|
|
captured.append(payload)
|
|
return {
|
|
"status": "reserved",
|
|
"claimed": True,
|
|
"identity": identity.public_dict(),
|
|
"receipt": None,
|
|
"claim_token": "agent99-control-plane-claim",
|
|
}
|
|
|
|
async def mark_delivery_attempt_started( # type: ignore[no-untyped-def]
|
|
self, *, identity, claim_token
|
|
) -> dict[str, object]:
|
|
return {
|
|
"status": "delivery_attempt_fenced",
|
|
"identity": identity.public_dict(),
|
|
"claim_token": claim_token,
|
|
"receipt_persisted": True,
|
|
}
|
|
|
|
async def complete( # type: ignore[no-untyped-def]
|
|
self, *, identity, dispatch_receipt, **_kwargs
|
|
) -> dict[str, object]:
|
|
return {
|
|
**build_agent99_dispatch_receipt_envelope(
|
|
identity=identity,
|
|
dispatch_receipt=dispatch_receipt,
|
|
),
|
|
"receipt_persisted": True,
|
|
}
|
|
|
|
async def acquire(*_args, **_kwargs) -> dict[str, object]:
|
|
return {"acquired": True, "reason": "acquired"}
|
|
|
|
def dispatch(payload: dict[str, object]) -> dict[str, object]:
|
|
routing = payload["routing"]
|
|
awoooi = payload["awoooi"]
|
|
typed = routing["typedTargetRoute"] # type: ignore[index]
|
|
identity = awoooi["agent99DispatchIdentity"] # type: ignore[index]
|
|
return {
|
|
"schema_version": "agent99_sre_dispatch_receipt_v2",
|
|
"status": "accepted_queue_persisted",
|
|
"transport": "relay",
|
|
"alert_id": payload["id"],
|
|
"kind": payload["kind"],
|
|
"target_resource": awoooi["targetResource"], # type: ignore[index]
|
|
"suggested_mode": payload["suggestedMode"],
|
|
"controlled_apply_requested": payload["controlledApply"],
|
|
"dispatch_scope": {
|
|
"schema_version": "agent99_dispatch_scope_v1",
|
|
"canonical_asset_id": typed["canonical_asset_id"],
|
|
"typed_domain": typed["target_kind"],
|
|
"executor": typed["executor"],
|
|
"verifier": typed["verifier"],
|
|
"route_id": identity["route_id"],
|
|
},
|
|
"accepted": True,
|
|
"inbox_triggered": True,
|
|
"queue_accepted": True,
|
|
"dispatch_identity_matched": True,
|
|
"delivery_certainty": "delivered",
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
"src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
|
|
Ledger,
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
|
|
acquire,
|
|
)
|
|
monkeypatch.setattr(
|
|
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
|
|
dispatch,
|
|
)
|
|
|
|
result = await bridge_alertmanager_to_agent99(
|
|
alert_id="agent99-health-valid",
|
|
alertname="Agent99ServiceUnhealthy",
|
|
severity="critical",
|
|
namespace="windows-control-plane",
|
|
target_resource="Agent99",
|
|
message="Agent99 control plane health check failed",
|
|
labels={"host": "192.168.0.99", "service": "Agent99"},
|
|
fingerprint="fp-agent99-control-plane-health-valid",
|
|
incident_id="INC-AGENT99-HEALTH-VALID",
|
|
route_id=AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID,
|
|
work_item_id="AIA-SRE-013",
|
|
)
|
|
|
|
assert result["status"] == "dispatched"
|
|
assert result["kind"] == AGENT99_CONTROL_PLANE_HEALTH_KIND
|
|
assert result["suggestedMode"] == "SelfCheck"
|
|
assert len(captured) == 1
|
|
assert captured[0]["controlledApply"] is False
|
|
assert captured[0]["force"] is False
|
|
assert (
|
|
result["correlatedReceipt"]["dispatch_scope"]["route_id"] # type: ignore[index]
|
|
== AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID
|
|
)
|
|
assert result["correlatedReceipt"]["controlled_apply_authorized"] is False # type: ignore[index]
|
|
assert result["correlatedReceipt"]["runtime_execution_authorized"] is False # type: ignore[index]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("host", "route_id"),
|
|
[
|
|
("192.168.0.110", AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID),
|
|
("192.168.0.99", "agent99:host_recovery:Recover"),
|
|
],
|
|
)
|
|
async def test_agent99_control_plane_scope_mismatch_performs_zero_dispatch(
|
|
monkeypatch,
|
|
host: str,
|
|
route_id: str,
|
|
) -> None:
|
|
dispatched: list[dict[str, object]] = []
|
|
monkeypatch.setattr(
|
|
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
|
|
lambda payload: dispatched.append(payload),
|
|
)
|
|
|
|
result = await bridge_alertmanager_to_agent99(
|
|
alert_id="agent99-health-mismatch",
|
|
alertname="Agent99ServiceUnhealthy",
|
|
severity="critical",
|
|
namespace="windows-control-plane",
|
|
target_resource="Agent99",
|
|
message="Agent99 control plane health check failed",
|
|
labels={"host": host, "service": "Agent99"},
|
|
fingerprint="fp-agent99-control-plane-health-mismatch",
|
|
incident_id="INC-AGENT99-HEALTH-MISMATCH",
|
|
route_id=route_id,
|
|
work_item_id="AIA-SRE-013",
|
|
)
|
|
|
|
assert result["status"] == "failed"
|
|
assert result["reason"] == "agent99_control_plane_exact_scope_required"
|
|
assert result["dispatchPerformed"] is False
|
|
assert result["runtimeClosureVerified"] is False
|
|
assert dispatched == []
|