fix(agent99): dispatch actionable alerts with receipts
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m27s
CD Pipeline / build-and-deploy (push) Successful in 6m39s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s

This commit is contained in:
ogt
2026-07-11 09:43:48 +08:00
parent f102616f02
commit 34e32f6ed6
16 changed files with 870 additions and 89 deletions

View File

@@ -23,7 +23,9 @@ def test_agent99_bridge_has_fingerprint_single_flight_before_transport() -> None
)
bridge_at = source.index("async def bridge_alertmanager_to_agent99")
lock_at = source.index("try_acquire_agent99_sre_single_flight", bridge_at)
dispatch_at = source.index("dispatch_agent99_sre_alert, payload", lock_at)
dispatch_at = source.index(
"dispatch_agent99_sre_alert_with_receipt, payload", lock_at
)
assert lock_at < dispatch_at
@@ -39,6 +41,14 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
assert '"database_health" = "Status"' in source
assert '"alert_chain_health" = "Status"' in source
assert '"host_recovery" = "Recover"' in source
assert '"cicd_failure" = "Status"' in source
assert "sourceEventResolutionPolicy = $sourceEventResolutionPolicy" in source
assert "sourceEventResolutionRequired = $sourceEventResolutionRequired" in source
telegram_source = TELEGRAM_INBOX.read_text(encoding="utf-8")
assert "function Get-AgentTelegramSourceEventResolutionPolicy" in telegram_source
assert '"cicd_failure" = "Status"' in telegram_source
assert 'policy = $sourceEventResolutionPolicy' in telegram_source
def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None:

View File

@@ -54,6 +54,11 @@ def test_agent99_outcome_contract_has_runtime_self_test() -> None:
assert "function Invoke-AgentOutcomeContractSelfTest" in source
assert '$operatorResolved.state -eq "resolved"' in source
assert '$alertVerifying.state -eq "verifying"' in source
assert '$alertResolvedByVerifier.state -eq "resolved"' in source
assert '$alertTransportFailedVerifier.state -eq "failed"' in source
assert "-not $alertTransportFailedVerifier.sourceEventResolved" in source
assert '"mode_verifier_can_resolve"' in source
assert "sourceEventResolutionPolicy" in source
assert '$failedPostCondition.state -eq "degraded"' in source
assert '$transportFailed.state -eq "failed"' in source
assert '$securityCandidate.state -eq "candidate_ready"' in source

View File

@@ -8,6 +8,7 @@ from src.services.agent99_sre_bridge import (
agent99_sre_single_flight_key,
bridge_alertmanager_to_agent99,
build_agent99_sre_alert,
dispatch_agent99_sre_alert_with_receipt,
post_agent99_sre_alert,
release_agent99_sre_single_flight,
resolve_agent99_alert_kind,
@@ -112,6 +113,22 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
"awoooi_k3s",
"AwoooRepair",
),
(
"ColdStartGateBlocked",
"cold-start-gate",
"full-stack cold-start recovery gate requires remediation",
{"service": "awoooi"},
"host_recovery",
"Recover",
),
(
"AWOOOICICDFailed",
"AWOOOI deployment",
"build-and-deploy deployment failed",
{"service": "awoooi"},
"cicd_failure",
"Status",
),
],
)
def test_agent99_routing_replay_corpus(
@@ -137,6 +154,28 @@ def test_agent99_routing_replay_corpus(
assert payload["suggestedMode"] == expected_mode
def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts() -> None:
cold_start = build_agent99_sre_alert(
alert_id="cold-start",
alertname="ColdStartGateBlocked",
severity="critical",
namespace="awoooi-prod",
target_resource="cold-start-gate",
message="cold-start gate blocked",
)
cicd = build_agent99_sre_alert(
alert_id="cicd-failure",
alertname="AWOOOICICDFailed",
severity="critical",
namespace="awoooi-prod",
target_resource="AWOOOI deployment",
message="build-and-deploy failed",
)
assert cold_start["resolution"]["policy"] == "mode_verifier_can_resolve"
assert cicd["resolution"]["policy"] == "external_source_receipt_required"
def test_alert_kind_mapping_for_route_and_performance() -> None:
assert (
resolve_agent99_alert_kind(
@@ -234,6 +273,43 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None:
assert seen["headers"]["X-agent99-relay-token"] == "relay-token"
def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -> None:
payload = build_agent99_sre_alert(
alert_id="receipt-smoke",
alertname="ColdStartGateBlocked",
severity="critical",
namespace="awoooi-prod",
target_resource="cold-start-gate",
message="cold-start gate blocked",
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_RELAY_URL",
"http://192.168.0.99:8787/agent99/sre-alert/",
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.post_agent99_sre_alert",
lambda _payload: {
"status": 202,
"body": '{"ok":true,"inboxTriggered":true,"incomingFile":"private-path"}',
},
)
receipt = dispatch_agent99_sre_alert_with_receipt(payload)
assert receipt == {
"schema_version": "agent99_sre_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"transport": "relay",
"alert_id": "awoooi-alertmanager-receipt-smoke",
"kind": "host_recovery",
"http_status": 202,
"accepted": True,
"inbox_triggered": True,
"stores_raw_response": False,
}
assert "private-path" not in str(receipt)
def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None:
key = agent99_sre_single_flight_key("private/source/fingerprint")
@@ -252,8 +328,14 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) ->
duplicate,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
lambda payload: dispatched.append(payload) or "relay",
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload)
or {
"status": "accepted_inbox_triggered",
"transport": "relay",
"accepted": True,
"inbox_triggered": True,
},
)
result = await bridge_alertmanager_to_agent99(
@@ -282,8 +364,14 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
winner,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
lambda payload: dispatched.append(payload) or "relay",
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload)
or {
"status": "accepted_inbox_triggered",
"transport": "relay",
"accepted": True,
"inbox_triggered": True,
},
)
result = await bridge_alertmanager_to_agent99(
@@ -298,10 +386,58 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
assert result["status"] == "dispatched"
assert result["dispatch"] == "relay"
assert result["dispatchReceipt"]["accepted"] is True
assert result["dispatchReceipt"]["inbox_triggered"] is True
assert result["suggestedMode"] == "Recover"
assert len(dispatched) == 1
@pytest.mark.asyncio
async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
monkeypatch,
) -> None:
async def winner(*_args, **_kwargs) -> bool:
return True
released: list[tuple[str, str]] = []
async def release(fingerprint: str, alert_id: str) -> None:
released.append((fingerprint, alert_id))
monkeypatch.setattr(
"src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
winner,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.release_agent99_sre_single_flight",
release,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda _payload: {
"status": "relay_response_not_accepted",
"transport": "relay",
"accepted": False,
"inbox_triggered": False,
},
)
result = await bridge_alertmanager_to_agent99(
alert_id="receipt-rejected",
alertname="HostDown",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="host_down",
fingerprint="receipt-rejected-fingerprint",
)
assert result["status"] == "failed"
assert result["reason"] == "relay_response_not_accepted"
assert result["dispatchReceipt"]["accepted"] is False
assert released == [("receipt-rejected-fingerprint", "receipt-rejected")]
@pytest.mark.asyncio
async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None:
async def winner(*_args, **_kwargs) -> bool:
@@ -312,7 +448,7 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
async def release(fingerprint: str, alert_id: str) -> None:
released.append((fingerprint, alert_id))
def fail_transport(_payload: dict[str, object]) -> str:
def fail_transport(_payload: dict[str, object]) -> dict[str, object]:
raise RuntimeError("relay unavailable")
monkeypatch.setattr(
@@ -324,7 +460,7 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
release,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
fail_transport,
)

View File

@@ -290,6 +290,80 @@ async def test_candidate_blocked_when_playbook_is_generic_fallback() -> None:
assert "https://awoooi.wooo.work/zh-TW/awooop/work-items?" in work_item["work_item_url"]
@pytest.mark.asyncio
async def test_cold_start_gate_uses_agent99_recover_not_fake_k8s_deployment() -> None:
incident = _incident()
service = RepairCandidateService(
incident_service=FakeIncidentService(),
investigator=FakeInvestigator(_evidence(incident.incident_id)),
playbook_repository=FakePlaybookRepository(_generic_fallback_playbook()),
auto_repair_service=FakeAutoRepairService(),
)
result = await service.build_from_incident(
incident=incident,
alertname="ColdStartGateBlocked",
target_resource="cold-start-gate",
namespace="default",
message="full-stack cold-start recovery gate is blocked",
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
matched_playbook_id="PB-GENERIC-FALLBACK",
severity="low",
)
assert result.candidate_found is False
assert result.draft_ready_for_owner_review is True
draft_package = result.metadata["repair_candidate_draft_package"]
assert draft_package["coverage_gap"]["target_kind"] == "control_plane"
template = draft_package["playbook_draft_template"]
assert template["suggested_route"] == "agent99_recover_after_owner_review"
assert template["repair_command_template"] == (
"agent99-mode Recover controlledApply=true"
)
assert "kubectl rollout restart deployment/cold-start-gate" not in str(template)
handoff = result.metadata["controlled_executor_handoff"]
assert handoff["executor"] == "Agent99"
assert handoff["suggested_mode"] == "Recover"
assert handoff["status"] == (
"agent99_dispatch_required_on_actionable_card_send"
)
assert handoff["dispatch_receipt_required"] is True
@pytest.mark.asyncio
async def test_cicd_failure_namespace_uses_agent99_status_not_k8s_restart() -> None:
incident = _incident()
service = RepairCandidateService(
incident_service=FakeIncidentService(),
investigator=FakeInvestigator(_evidence(incident.incident_id)),
playbook_repository=FakePlaybookRepository(_generic_fallback_playbook()),
auto_repair_service=FakeAutoRepairService(),
)
result = await service.build_from_incident(
incident=incident,
alertname="AWOOOICICDFailed",
target_resource="AWOOOI 部署失敗",
namespace="build-and-deploy",
message="Gitea CI/CD deployment failed",
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
matched_playbook_id="PB-GENERIC-FALLBACK",
severity="critical",
)
draft_package = result.metadata["repair_candidate_draft_package"]
assert draft_package["coverage_gap"]["target_kind"] == "control_plane"
template = draft_package["playbook_draft_template"]
assert template["suggested_route"] == "agent99_status_after_owner_review"
assert template["repair_command_template"] == (
"agent99-mode Status controlledApply=false"
)
assert "kubectl rollout restart" not in str(template)
handoff = result.metadata["controlled_executor_handoff"]
assert handoff["executor"] == "Agent99"
assert handoff["suggested_mode"] == "Status"
@pytest.mark.asyncio
async def test_candidate_blocked_observe_only_prompts_repair_playbook_draft() -> None:
incident = _incident()

View File

@@ -1,7 +1,10 @@
from __future__ import annotations
from src.services.telegram_gateway import (
_agent99_actionable_outbound_metadata,
_agent99_alert_from_actionable_outbound,
_agent99_alert_from_ai_automation_card,
_merge_outbound_source_envelope_extra,
_outbound_source_envelope,
_sanitize_telegram_error,
format_aiops_signal_alert_card,
@@ -213,3 +216,91 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz
assert "provider-freshness" in payload["labels"]
assert "no-false-green" in payload["labels"]
assert "abcdefghijklmnopqrstuvwxyz" not in str(payload)
def test_action_required_cold_start_card_routes_to_agent99_recover() -> None:
card = (
" ACTION REQUIRED | <b>低風險</b>\n"
"──────────────────────\n"
"📋 <code>INC-20260710-5700F7</code>\n"
"🎯 資源:<code>cold-start-gate</code>\n"
"🏷️ 分類:<b>general</b>\n"
"🧭 處置狀態:<b>AI 選擇不執行修復</b>\n"
"🤖 <b>AI 自動化鏈路</b>"
)
metadata = _agent99_actionable_outbound_metadata(card)
payload = _agent99_alert_from_actionable_outbound(card, source="unit-test")
assert metadata is not None
assert metadata["card_schema"] == "awooop_action_required_v1"
assert metadata["incident_id"] == "INC-20260710-5700F7"
assert metadata["target"] == "cold-start-gate"
assert payload is not None
assert payload["kind"] == "host_recovery"
assert payload["suggestedMode"] == "Recover"
assert payload["controlledApply"] is True
assert payload["resolution"]["policy"] == "mode_verifier_can_resolve"
assert payload["awoooi"]["incidentId"] == "INC-20260710-5700F7"
def test_failed_cicd_card_routes_to_agent99_status_with_commit_receipt() -> None:
card = (
"❌ <b>[AWOOOI CI/CD]</b> | build-and-deploy\n"
"📦 AWOOOI 部署失敗\n"
"📋 <code>85159f25</code>\n"
"📝 fix(security): restrict executor SSH egress\n"
"🔗 <a href='https://gitea.example.invalid/run'>Workflow</a>"
)
metadata = _agent99_actionable_outbound_metadata(card)
payload = _agent99_alert_from_actionable_outbound(card, source="unit-test")
assert metadata is not None
assert metadata["card_schema"] == "awoooi_cicd_failure_v1"
assert metadata["commit_sha"] == "85159f25"
assert payload is not None
assert payload["kind"] == "cicd_failure"
assert payload["suggestedMode"] == "Status"
assert payload["controlledApply"] is False
assert payload["resolution"]["policy"] == "external_source_receipt_required"
assert payload["awoooi"]["commitSha"] == "85159f25"
assert "gitea.example.invalid" not in str(payload)
def test_agent99_dispatch_receipt_is_persisted_without_raw_transport_body() -> None:
envelope = _outbound_source_envelope(
"sendMessage",
{
"chat_id": "chat",
"text": (
" ACTION REQUIRED | 低風險\n"
"📋 INC-20260710-5700F7\n"
"🎯 資源cold-start-gate\n"
"🤖 AI 自動化鏈路"
),
},
)
receipt = {
"schema_version": "telegram_agent99_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"transport": "relay",
"alert_id": "awoooi-telegram-actionable-awooop_action_required-safehash",
"kind": "host_recovery",
"suggested_mode": "Recover",
"accepted": True,
"inbox_triggered": True,
"stores_raw_response": False,
}
merged = _merge_outbound_source_envelope_extra(
envelope,
{"agent99_dispatch_receipt": receipt},
)
assert merged["agent99_dispatch_receipt"] == receipt
assert merged["agent99_actionable_outbound"]["incident_id"] == (
"INC-20260710-5700F7"
)
assert "incomingFile" not in str(merged)
assert "private-path" not in str(merged)

View File

@@ -468,6 +468,7 @@ def test_send_request_payload_normalizer_blocks_direct_host_raw_dump() -> None:
async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) -> None:
"""direct sendMessage AI cards must reach Agent99 even without wrapper helpers."""
mirrored: list[dict[str, str]] = []
outbound_receipts: list[dict[str, object]] = []
gateway = TelegramGateway()
gateway._initialized = True
@@ -476,15 +477,25 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch)
return None
def json(self) -> dict:
return {"ok": True, "result": True}
return {"ok": True, "result": {"message_id": 991}}
class FakeHttpClient:
async def post(self, url, json): # type: ignore[no-untyped-def]
return FakeResponse()
async def fake_mirror(text: str, *, source: str = "telegram_gateway") -> str:
async def fake_mirror(
text: str, *, source: str = "telegram_gateway"
) -> dict[str, object]:
mirrored.append({"text": text, "source": source})
return "mirrored"
return {
"schema_version": "telegram_agent99_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"accepted": True,
"inbox_triggered": True,
}
async def fake_outbound_message(**kwargs) -> None: # type: ignore[no-untyped-def]
outbound_receipts.append(kwargs)
monkeypatch.setattr(TelegramGateway, "api_url", property(lambda _self: "https://telegram.test/botx"))
monkeypatch.setattr(
@@ -497,6 +508,7 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch)
"_mirror_ai_automation_alert_card_to_agent99",
fake_mirror,
)
monkeypatch.setattr(gateway, "_mirror_outbound_message", fake_outbound_message)
gateway._http_client = FakeHttpClient()
result = await gateway._send_request(
@@ -513,12 +525,18 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch)
)
await asyncio.sleep(0)
assert result == {"ok": True, "result": True}
assert result == {"ok": True, "result": {"message_id": 991}}
assert mirrored
assert mirrored[0]["source"] == "send_request"
assert "ai_automation_alert_card_v1" in mirrored[0]["text"]
assert "provider_freshness_signal" in mirrored[0]["text"]
assert "runtime_write_gate=controlled" in mirrored[0]["text"]
assert outbound_receipts
receipt = outbound_receipts[0]["source_envelope_extra"][
"agent99_dispatch_receipt"
]
assert receipt["status"] == "accepted_inbox_triggered"
assert receipt["accepted"] is True
def test_weekly_report_marks_all_zero_as_low_trust_anomaly() -> None: