fix(agent99): dispatch actionable alerts with receipts
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m27s
CD Pipeline / build-and-deploy (push) Successful in 6m39s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s

This commit is contained in:
ogt
2026-07-11 09:43:48 +08:00
parent f102616f02
commit 34e32f6ed6
16 changed files with 870 additions and 89 deletions

View File

@@ -124,6 +124,7 @@ def resolve_agent99_alert_kind(
return "alert_chain_health"
if re.search(
r"reboot-auto-recovery|reboot_auto_recovery|reboot recovery|"
r"cold[-_ ]start(?:[-_ ]gate)?|full[-_ ]stack[-_ ]cold[-_ ]start|"
r"host_down|hostdown|host_reboot|vm_not_running|powered_off|"
r"all_required_hosts_not_in_10_minute_reboot_window",
text,
@@ -135,6 +136,13 @@ def resolve_agent99_alert_kind(
return "performance_pressure"
if re.search(r"harbor|registry", text):
return "harbor_registry"
if re.search(
r"\bci/?cd\b|\bcicd\b|build-and-deploy|build_and_deploy|"
r"deploy(?:ment)?[_ -](?:failed|failure)|build[_ -](?:failed|failure)|"
r"部署失敗|構建失敗",
text,
):
return "cicd_failure"
if re.search(
r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text
):
@@ -161,9 +169,25 @@ def _suggested_mode_for_kind(kind: str) -> str | None:
"awoooi_k3s": "AwoooRepair",
"host_recovery": "Recover",
"application_ai_config_error": "Status",
"cicd_failure": "Status",
"monitoring_alert": "Status",
}.get(kind)
def _resolution_policy_for_kind(kind: str) -> str:
if kind in {
"provider_freshness_signal",
"public_route_502",
"performance_pressure",
"backup_health",
"harbor_registry",
"awoooi_k3s",
"host_recovery",
}:
return "mode_verifier_can_resolve"
return "external_source_receipt_required"
def _agent99_route_group_key(
*,
kind: str,
@@ -343,6 +367,11 @@ def build_agent99_sre_alert(
"sourceFingerprint": _safe_text(fingerprint, max_length=160),
"singleFlightWindowSeconds": AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
},
"resolution": {
"schemaVersion": "agent99_source_event_resolution_v1",
"policy": _resolution_policy_for_kind(kind),
"required": True,
},
"awoooi": {
"alertId": _safe_text(alert_id, max_length=160),
"alertname": _safe_text(alertname, max_length=160),
@@ -453,14 +482,60 @@ def post_agent99_sre_alert(
def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str:
receipt = dispatch_agent99_sre_alert_with_receipt(payload)
return str(receipt["transport"])
def dispatch_agent99_sre_alert_with_receipt(
payload: dict[str, Any],
) -> dict[str, Any]:
"""Dispatch one alert and retain only public-safe acceptance evidence."""
alert_id = _safe_text(payload.get("id"), max_length=160)
kind = _safe_text(payload.get("kind"), max_length=120)
if settings.AGENT99_SRE_ALERT_RELAY_URL:
post_agent99_sre_alert(payload)
return "relay"
raw_receipt = post_agent99_sre_alert(payload) or {}
response_payload: dict[str, Any] = {}
try:
parsed = json.loads(str(raw_receipt.get("body") or ""))
if isinstance(parsed, dict):
response_payload = parsed
except json.JSONDecodeError:
response_payload = {}
http_status = int(raw_receipt.get("status") or 0)
accepted = bool(200 <= http_status < 300 and response_payload.get("ok") is True)
inbox_triggered = bool(response_payload.get("inboxTriggered") is True)
return {
"schema_version": "agent99_sre_dispatch_receipt_v1",
"status": (
"accepted_inbox_triggered"
if accepted and inbox_triggered
else "accepted_inbox_pending"
if accepted
else "relay_response_not_accepted"
),
"transport": "relay",
"alert_id": alert_id,
"kind": kind,
"http_status": http_status,
"accepted": accepted,
"inbox_triggered": inbox_triggered,
"stores_raw_response": False,
}
if settings.AGENT99_SRE_ALERT_INBOX_PATH:
written = write_agent99_sre_alert(payload)
if written is not None:
return "file"
return {
"schema_version": "agent99_sre_dispatch_receipt_v1",
"status": "file_written",
"transport": "file",
"alert_id": alert_id,
"kind": kind,
"accepted": True,
"inbox_triggered": False,
"stores_raw_response": False,
}
logger.info(
"agent99_sre_bridge_disabled",
@@ -468,7 +543,16 @@ def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str:
alert_id=payload.get("id"),
kind=payload.get("kind"),
)
return "disabled"
return {
"schema_version": "agent99_sre_dispatch_receipt_v1",
"status": "bridge_disabled",
"transport": "disabled",
"alert_id": alert_id,
"kind": kind,
"accepted": False,
"inbox_triggered": False,
"stores_raw_response": False,
}
async def bridge_alertmanager_to_agent99(
@@ -527,19 +611,23 @@ async def bridge_alertmanager_to_agent99(
"fingerprint": single_flight_fingerprint,
"groupKey": routing.get("groupKey"),
}
dispatch_result = await asyncio.to_thread(dispatch_agent99_sre_alert, payload)
if dispatch_result == "disabled":
dispatch_receipt = await asyncio.to_thread(
dispatch_agent99_sre_alert_with_receipt, payload
)
if not dispatch_receipt.get("accepted"):
await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id)
single_flight_acquired = False
return {
"status": "failed",
"reason": "bridge_disabled",
"reason": dispatch_receipt.get("status") or "dispatch_not_accepted",
"dispatchReceipt": dispatch_receipt,
"fingerprint": single_flight_fingerprint,
"groupKey": routing.get("groupKey"),
}
return {
"status": "dispatched",
"dispatch": dispatch_result,
"dispatch": dispatch_receipt.get("transport"),
"dispatchReceipt": dispatch_receipt,
"fingerprint": single_flight_fingerprint,
"groupKey": routing.get("groupKey"),
"kind": payload.get("kind"),

View File

@@ -314,6 +314,28 @@ class RepairCandidateService:
if not isinstance(promotion_contract, dict):
return
route_id = str(promotion_contract.get("route_id") or "")
if route_id.startswith("agent99_"):
suggested_mode = "Recover" if route_id.startswith("agent99_recover") else "Status"
handoff = {
"schema_version": "repair_candidate_controlled_executor_handoff_v1",
"status": "agent99_dispatch_required_on_actionable_card_send",
"operation_type": "agent99_actionable_dispatch",
"executor": "Agent99",
"route_id": route_id,
"suggested_mode": suggested_mode,
"dispatch_receipt_required": True,
"post_condition_verifier_required": True,
"check_mode_queued": False,
"runtime_execution_authorized": False,
}
result.metadata["controlled_executor_handoff"] = handoff
draft_package["controlled_executor_handoff"] = handoff
work_item = draft_package.get("awooop_work_item")
if isinstance(work_item, dict):
work_item["controlled_executor_handoff"] = handoff
return
proposal_data = {
"source": "repair_candidate_controlled_queue",
"risk_level": severity or getattr(getattr(incident, "severity", None), "value", ""),
@@ -1194,13 +1216,23 @@ class RepairCandidateService:
return False
if "mcp_evidence_missing" in set(blockers):
return False
if "playbook_command_not_safely_routable" in set(blockers):
route = str(template.get("suggested_route") or "").strip()
repair_template = str(template.get("repair_command_template") or "").strip()
safe_agent99_route = bool(
route in {
"agent99_recover_after_owner_review",
"agent99_status_after_owner_review",
}
and repair_template.startswith("agent99-mode ")
)
if (
"playbook_command_not_safely_routable" in set(blockers)
and not safe_agent99_route
):
return False
if not coverage_gap.get("mcp_evidence_ready"):
return False
repair_template = str(template.get("repair_command_template") or "").strip()
route = str(template.get("suggested_route") or "").strip()
if not repair_template or not route:
return False
if "<" in repair_template:
@@ -1241,7 +1273,35 @@ class RepairCandidateService:
"publish_assets_to_runs_work_items_and_knowledge_base",
]
if target_kind == "k8s_workload":
if target_kind == "control_plane":
normalized_target = target.lower()
if any(
marker in normalized_target
for marker in (
"cold-start",
"cold_start",
"reboot-auto-recovery",
"reboot_auto_recovery",
)
):
command_template = "agent99-mode Recover controlledApply=true"
rollback_template = "agent99-mode Status controlledApply=false"
route = "agent99_recover_after_owner_review"
verifier_plan.extend([
"agent99 Recover outcome verifier must resolve",
"rerun reboot-auto-recovery-slo-scorecard",
"confirm required hosts, Harbor, K3s, and public routes are green",
])
else:
command_template = "agent99-mode Status controlledApply=false"
rollback_template = "no runtime write; retain previous production deployment"
route = "agent99_status_after_owner_review"
verifier_plan.extend([
"compare failed commit with current production deploy marker",
"confirm public routes and current production services remain green",
"keep source CI failure open until a later successful run is observed",
])
elif target_kind == "k8s_workload":
workload = target.replace("deployment/", "").replace("pod/", "").replace("svc/", "")
workload_ref = workload or "<deployment>"
namespace_ref = ns or "<namespace>"
@@ -1442,6 +1502,25 @@ class RepairCandidateService:
def _infer_target_kind(self, *, target_resource: str, namespace: str) -> str:
target = (target_resource or "").lower()
control_plane_context = f"{target} {(namespace or '').lower()}"
if any(
marker in control_plane_context
for marker in (
"cold-start",
"cold_start",
"reboot-auto-recovery",
"reboot_auto_recovery",
"provider_freshness",
"alert-chain",
"alert_chain",
"ci/cd",
"cicd",
"build-and-deploy",
"deployment failed",
"部署失敗",
)
):
return "control_plane"
if any(marker in target for marker in ("postgres", "postgresql", "pgbouncer", "database", "db-")):
return "database"
if any(marker in target for marker in ("node-exporter", "host", "188", "111", "168")):

View File

@@ -53,7 +53,7 @@ from src.services.approval_action_classifier import (
)
from src.services.agent99_sre_bridge import (
build_agent99_sre_alert,
dispatch_agent99_sre_alert,
dispatch_agent99_sre_alert_with_receipt,
)
from src.services.chat_manager import get_chat_manager
from src.services.operator_outcome import build_operator_outcome, normalize_operator_outcome
@@ -163,6 +163,17 @@ _AI_SIGNAL_TARGET_LABEL_RE = re.compile(
_ALERT_CARD_CODE_VALUE_RE = re.compile(
r"(?P<label>事件類型|Target|Lane)<code>(?P<value>[^<]+)</code>"
)
_ACTION_REQUIRED_RESOURCE_RE = re.compile(
r"資源\s*[:]\s*(?P<value>[^\r\n]+)", re.IGNORECASE
)
_ACTION_REQUIRED_CATEGORY_RE = re.compile(
r"分類\s*[:]\s*(?P<value>[^\r\n]+)", re.IGNORECASE
)
_CICD_STAGE_RE = re.compile(
r"\[AWOOOI CI/CD\]\s*(?:\|\s*(?P<value>[^\r\n]+))?", re.IGNORECASE
)
_CICD_JOB_RE = re.compile(r"📦\s*(?P<value>[^\r\n]+)")
_CICD_COMMIT_RE = re.compile(r"📋\s*(?P<value>[0-9a-f]{7,40})\b", re.IGNORECASE)
@dataclass(frozen=True)
@@ -2285,6 +2296,60 @@ def _ai_automation_alert_card_metadata(text: str) -> dict[str, object] | None:
return metadata
def _agent99_actionable_outbound_metadata(text: str) -> dict[str, object] | None:
"""Normalize every actionable SRE Telegram card into one Agent99 contract."""
ai_card = _ai_automation_alert_card_metadata(text)
if ai_card:
return ai_card
plain = _plain_text_from_html(text, limit=8000)
incident_match = _INCIDENT_ID_RE.search(plain)
if "ACTION REQUIRED" in plain and incident_match:
resource_match = _ACTION_REQUIRED_RESOURCE_RE.search(plain)
category_match = _ACTION_REQUIRED_CATEGORY_RE.search(plain)
target = (
resource_match.group("value").strip()
if resource_match
else "awooop-action-required"
)
lane = category_match.group("value").strip() if category_match else "general"
return {
"schema_version": "agent99_actionable_outbound_mirror_v1",
"card_schema": "awooop_action_required_v1",
"event_type": "awooop_action_required",
"lane": lane[:120] or "general",
"target": target[:240] or "awooop-action-required",
"incident_id": incident_match.group(0),
"runtime_write_gate_state": "controlled",
"delivery_receipt_readback_required": True,
"mirror_source": "legacy_telegram_gateway_outbound_message",
}
cicd_match = _CICD_STAGE_RE.search(plain)
failed_cicd = bool(
cicd_match
and any(token in plain.lower() for token in ("failed", "failure", "失敗"))
)
if failed_cicd:
job_match = _CICD_JOB_RE.search(plain)
commit_match = _CICD_COMMIT_RE.search(plain)
stage = (cicd_match.group("value") or "").strip() if cicd_match else ""
job = job_match.group("value").strip() if job_match else "AWOOOI CI/CD"
return {
"schema_version": "agent99_actionable_outbound_mirror_v1",
"card_schema": "awoooi_cicd_failure_v1",
"event_type": "cicd_failure",
"lane": stage[:120] or "build-and-deploy",
"target": job[:240] or "AWOOOI CI/CD",
"commit_sha": commit_match.group("value") if commit_match else "",
"runtime_write_gate_state": "external_source_receipt_required",
"delivery_receipt_readback_required": True,
"mirror_source": "legacy_telegram_gateway_outbound_message",
}
return None
def _agent99_ai_alert_card_mirror_configured() -> bool:
return bool(
settings.AGENT99_SRE_ALERT_RELAY_URL
@@ -2302,20 +2367,20 @@ def _agent99_alert_card_severity(text: str) -> str:
return "warning"
return "info"
lowered = text.lower()
if any(token in lowered for token in ("critical", "p0", "p1")):
if any(token in lowered for token in ("critical", "failed", "failure", "p0", "p1", "失敗")):
return "critical"
if any(token in lowered for token in ("warning", "warn", "p2")):
return "warning"
return "info"
def _agent99_alert_from_ai_automation_card(
def _agent99_alert_from_actionable_outbound(
text: str,
*,
source: str = "telegram_gateway",
) -> dict[str, object] | None:
"""Convert a Telegram AI alert card into a sanitized Agent99 action signal."""
metadata = _ai_automation_alert_card_metadata(text)
"""Convert one actionable Telegram card into a sanitized Agent99 signal."""
metadata = _agent99_actionable_outbound_metadata(text)
if not metadata:
return None
@@ -2327,13 +2392,16 @@ def _agent99_alert_from_ai_automation_card(
card_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
runtime_gate = str(metadata.get("runtime_write_gate_state") or "unknown")
card_schema = str(metadata.get("card_schema") or "ai_automation_alert_card_v1")
alert_id = f"telegram-ai-card-{event_type}-{card_hash}"
fingerprint = f"telegram-ai-card:{event_type}:{lane}:{target}:{card_hash}"
card_schema = str(metadata.get("card_schema") or "agent99_actionable_outbound_v1")
incident_id = str(metadata.get("incident_id") or "")
commit_sha = str(metadata.get("commit_sha") or "")
alert_id = f"telegram-actionable-{event_type}-{card_hash}"
fingerprint = f"telegram-actionable:{event_type}:{lane}:{target}:{card_hash}"
message = (
"Telegram AI automation alert card mirrored to Agent99; "
"Telegram actionable SRE card mirrored to Agent99; "
f"event_type={event_type} lane={lane} target={target} "
f"runtime_write_gate={runtime_gate}"
f"runtime_write_gate={runtime_gate} incident_id={incident_id or '-'} "
f"commit_sha={commit_sha[:12] or '-'}"
)
payload = build_agent99_sre_alert(
alert_id=alert_id,
@@ -2351,9 +2419,11 @@ def _agent99_alert_from_ai_automation_card(
"target": target,
"card_schema": card_schema,
"runtime_write_gate_state": runtime_gate,
"incident_id": incident_id,
"commit_sha": commit_sha[:40],
},
annotations={
"summary": "AI automation alert card mirrored from TelegramGateway.",
"summary": "Actionable SRE card mirrored from TelegramGateway.",
"lane": lane,
"runtime_write_gate_state": runtime_gate,
},
@@ -2362,27 +2432,49 @@ def _agent99_alert_from_ai_automation_card(
notification_type=card_schema,
)
payload["source"] = "awoooi-api-telegram-gateway"
payload["title"] = f"Telegram AI alert card {event_type}"
payload["id"] = f"awoooi-telegram-alert-card-{event_type}-{card_hash}"
payload["title"] = f"Telegram actionable alert {event_type}"
payload["id"] = (
f"awoooi-telegram-alert-card-{event_type}-{card_hash}"
if card_schema == "ai_automation_alert_card_v1"
else f"awoooi-telegram-actionable-{event_type}-{card_hash}"
)
if isinstance(payload.get("awoooi"), dict):
payload["awoooi"]["alertId"] = alert_id
payload["awoooi"]["telegramGatewaySource"] = source
payload["awoooi"]["cardHash"] = card_hash
payload["awoooi"]["incidentId"] = incident_id
payload["awoooi"]["commitSha"] = commit_sha[:40]
payload["awoooi"]["cardSchema"] = card_schema
return payload
def _agent99_alert_from_ai_automation_card(
text: str,
*,
source: str = "telegram_gateway",
) -> dict[str, object] | None:
"""Backward-compatible AI-card-only adapter used by older call sites/tests."""
if not _ai_automation_alert_card_metadata(text):
return None
return _agent99_alert_from_actionable_outbound(text, source=source)
async def _mirror_ai_automation_alert_card_to_agent99(
text: str,
*,
source: str = "telegram_gateway",
) -> str | None:
payload = _agent99_alert_from_ai_automation_card(text, source=source)
) -> dict[str, object] | None:
payload = _agent99_alert_from_actionable_outbound(text, source=source)
if not payload:
return None
alert_id = str(payload.get("id") or "")
try:
dispatch_result = await asyncio.to_thread(dispatch_agent99_sre_alert, payload)
dispatch_receipt = await asyncio.to_thread(
dispatch_agent99_sre_alert_with_receipt,
payload,
)
except Exception as exc:
logger.warning(
"telegram_agent99_alert_card_mirror_failed_fail_open",
@@ -2390,16 +2482,38 @@ async def _mirror_ai_automation_alert_card_to_agent99(
kind=payload.get("kind"),
error=str(exc),
)
return None
return {
"schema_version": "telegram_agent99_dispatch_receipt_v1",
"status": "dispatch_failed",
"alert_id": alert_id,
"kind": str(payload.get("kind") or "monitoring_alert"),
"suggested_mode": str(payload.get("suggestedMode") or "Status"),
"accepted": False,
"inbox_triggered": False,
"error_type": type(exc).__name__,
}
logger.info(
"telegram_agent99_alert_card_mirrored",
alert_id=alert_id,
kind=payload.get("kind"),
suggested_mode=payload.get("suggestedMode"),
dispatch_result=dispatch_result,
dispatch_result=dispatch_receipt.get("status"),
)
return alert_id
awoooi = payload.get("awoooi") if isinstance(payload.get("awoooi"), dict) else {}
return {
"schema_version": "telegram_agent99_dispatch_receipt_v1",
"status": str(dispatch_receipt.get("status") or "unknown"),
"transport": str(dispatch_receipt.get("transport") or "unknown"),
"alert_id": alert_id,
"kind": str(payload.get("kind") or "monitoring_alert"),
"suggested_mode": str(payload.get("suggestedMode") or "Status"),
"accepted": bool(dispatch_receipt.get("accepted") is True),
"inbox_triggered": bool(dispatch_receipt.get("inbox_triggered") is True),
"incident_id": str(awoooi.get("incidentId") or ""),
"commit_sha": str(awoooi.get("commitSha") or ""),
"stores_raw_response": False,
}
def _schedule_agent99_ai_alert_card_mirror(
@@ -2409,7 +2523,7 @@ def _schedule_agent99_ai_alert_card_mirror(
) -> bool:
if not _agent99_ai_alert_card_mirror_configured():
return False
if not _ai_automation_alert_card_metadata(text):
if not _agent99_actionable_outbound_metadata(text):
return False
try:
asyncio.create_task(
@@ -2429,6 +2543,7 @@ def _outbound_source_envelope(method: str, payload: dict) -> dict[str, object]:
"""Build a redaction-friendly source envelope for Channel Hub replay."""
text = str(payload.get("text") or payload.get("caption") or "")
alert_card_metadata = _ai_automation_alert_card_metadata(text)
actionable_metadata = _agent99_actionable_outbound_metadata(text)
incident_ids = sorted(
set(_INCIDENT_ID_RE.findall(text))
| set(_reply_markup_incident_ids(payload))
@@ -2448,19 +2563,24 @@ def _outbound_source_envelope(method: str, payload: dict) -> dict[str, object]:
if key == "event_ids":
continue
source_refs[key] = values[:20]
if alert_card_metadata:
if actionable_metadata:
fingerprint_namespace = (
"ai_automation_alert_card"
if actionable_metadata.get("card_schema") == "ai_automation_alert_card_v1"
else "agent99_actionable_outbound"
)
_append_source_ref(
source_refs,
"alert_ids",
alert_card_metadata.get("event_type"),
actionable_metadata.get("event_type"),
)
_append_source_ref(
source_refs,
"fingerprints",
(
"ai_automation_alert_card:"
f"{alert_card_metadata.get('event_type')}:"
f"{alert_card_metadata.get('lane')}"
f"{fingerprint_namespace}:"
f"{actionable_metadata.get('event_type')}:"
f"{actionable_metadata.get('lane')}"
),
)
@@ -2477,6 +2597,8 @@ def _outbound_source_envelope(method: str, payload: dict) -> dict[str, object]:
}
if alert_card_metadata:
envelope["ai_automation_alert_card"] = alert_card_metadata
if actionable_metadata:
envelope["agent99_actionable_outbound"] = actionable_metadata
return envelope
@@ -3066,6 +3188,10 @@ def _merge_outbound_source_envelope_extra(
if isinstance(alert_notification, dict):
envelope["alert_notification"] = alert_notification
agent99_dispatch_receipt = extra.get("agent99_dispatch_receipt")
if isinstance(agent99_dispatch_receipt, dict):
envelope["agent99_dispatch_receipt"] = agent99_dispatch_receipt
km_stale_completion_summary = extra.get("km_stale_completion_summary")
if isinstance(km_stale_completion_summary, dict):
envelope["km_stale_completion_summary"] = km_stale_completion_summary
@@ -4947,10 +5073,18 @@ class TelegramGateway:
payload = normalize_telegram_send_message_payload(method, payload)
source_envelope_extra = payload.pop(_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY, None)
await self._attach_incident_thread_reply(method, payload)
_schedule_agent99_ai_alert_card_mirror(
str(payload.get("text") or payload.get("caption") or ""),
source="send_request",
)
actionable_text = str(payload.get("text") or payload.get("caption") or "")
if (
_agent99_ai_alert_card_mirror_configured()
and _agent99_actionable_outbound_metadata(actionable_text)
):
agent99_receipt = await _mirror_ai_automation_alert_card_to_agent99(
actionable_text,
source="send_request",
)
if isinstance(agent99_receipt, dict):
source_envelope_extra = dict(source_envelope_extra or {})
source_envelope_extra["agent99_dispatch_receipt"] = agent99_receipt
url = f"{self.api_url}/{method}"

View File

@@ -23,7 +23,9 @@ def test_agent99_bridge_has_fingerprint_single_flight_before_transport() -> None
)
bridge_at = source.index("async def bridge_alertmanager_to_agent99")
lock_at = source.index("try_acquire_agent99_sre_single_flight", bridge_at)
dispatch_at = source.index("dispatch_agent99_sre_alert, payload", lock_at)
dispatch_at = source.index(
"dispatch_agent99_sre_alert_with_receipt, payload", lock_at
)
assert lock_at < dispatch_at
@@ -39,6 +41,14 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
assert '"database_health" = "Status"' in source
assert '"alert_chain_health" = "Status"' in source
assert '"host_recovery" = "Recover"' in source
assert '"cicd_failure" = "Status"' in source
assert "sourceEventResolutionPolicy = $sourceEventResolutionPolicy" in source
assert "sourceEventResolutionRequired = $sourceEventResolutionRequired" in source
telegram_source = TELEGRAM_INBOX.read_text(encoding="utf-8")
assert "function Get-AgentTelegramSourceEventResolutionPolicy" in telegram_source
assert '"cicd_failure" = "Status"' in telegram_source
assert 'policy = $sourceEventResolutionPolicy' in telegram_source
def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None:

View File

@@ -54,6 +54,11 @@ def test_agent99_outcome_contract_has_runtime_self_test() -> None:
assert "function Invoke-AgentOutcomeContractSelfTest" in source
assert '$operatorResolved.state -eq "resolved"' in source
assert '$alertVerifying.state -eq "verifying"' in source
assert '$alertResolvedByVerifier.state -eq "resolved"' in source
assert '$alertTransportFailedVerifier.state -eq "failed"' in source
assert "-not $alertTransportFailedVerifier.sourceEventResolved" in source
assert '"mode_verifier_can_resolve"' in source
assert "sourceEventResolutionPolicy" in source
assert '$failedPostCondition.state -eq "degraded"' in source
assert '$transportFailed.state -eq "failed"' in source
assert '$securityCandidate.state -eq "candidate_ready"' in source

View File

@@ -8,6 +8,7 @@ from src.services.agent99_sre_bridge import (
agent99_sre_single_flight_key,
bridge_alertmanager_to_agent99,
build_agent99_sre_alert,
dispatch_agent99_sre_alert_with_receipt,
post_agent99_sre_alert,
release_agent99_sre_single_flight,
resolve_agent99_alert_kind,
@@ -112,6 +113,22 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
"awoooi_k3s",
"AwoooRepair",
),
(
"ColdStartGateBlocked",
"cold-start-gate",
"full-stack cold-start recovery gate requires remediation",
{"service": "awoooi"},
"host_recovery",
"Recover",
),
(
"AWOOOICICDFailed",
"AWOOOI deployment",
"build-and-deploy deployment failed",
{"service": "awoooi"},
"cicd_failure",
"Status",
),
],
)
def test_agent99_routing_replay_corpus(
@@ -137,6 +154,28 @@ def test_agent99_routing_replay_corpus(
assert payload["suggestedMode"] == expected_mode
def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts() -> None:
cold_start = build_agent99_sre_alert(
alert_id="cold-start",
alertname="ColdStartGateBlocked",
severity="critical",
namespace="awoooi-prod",
target_resource="cold-start-gate",
message="cold-start gate blocked",
)
cicd = build_agent99_sre_alert(
alert_id="cicd-failure",
alertname="AWOOOICICDFailed",
severity="critical",
namespace="awoooi-prod",
target_resource="AWOOOI deployment",
message="build-and-deploy failed",
)
assert cold_start["resolution"]["policy"] == "mode_verifier_can_resolve"
assert cicd["resolution"]["policy"] == "external_source_receipt_required"
def test_alert_kind_mapping_for_route_and_performance() -> None:
assert (
resolve_agent99_alert_kind(
@@ -234,6 +273,43 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None:
assert seen["headers"]["X-agent99-relay-token"] == "relay-token"
def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -> None:
payload = build_agent99_sre_alert(
alert_id="receipt-smoke",
alertname="ColdStartGateBlocked",
severity="critical",
namespace="awoooi-prod",
target_resource="cold-start-gate",
message="cold-start gate blocked",
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_RELAY_URL",
"http://192.168.0.99:8787/agent99/sre-alert/",
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.post_agent99_sre_alert",
lambda _payload: {
"status": 202,
"body": '{"ok":true,"inboxTriggered":true,"incomingFile":"private-path"}',
},
)
receipt = dispatch_agent99_sre_alert_with_receipt(payload)
assert receipt == {
"schema_version": "agent99_sre_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"transport": "relay",
"alert_id": "awoooi-alertmanager-receipt-smoke",
"kind": "host_recovery",
"http_status": 202,
"accepted": True,
"inbox_triggered": True,
"stores_raw_response": False,
}
assert "private-path" not in str(receipt)
def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None:
key = agent99_sre_single_flight_key("private/source/fingerprint")
@@ -252,8 +328,14 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) ->
duplicate,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
lambda payload: dispatched.append(payload) or "relay",
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload)
or {
"status": "accepted_inbox_triggered",
"transport": "relay",
"accepted": True,
"inbox_triggered": True,
},
)
result = await bridge_alertmanager_to_agent99(
@@ -282,8 +364,14 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
winner,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
lambda payload: dispatched.append(payload) or "relay",
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda payload: dispatched.append(payload)
or {
"status": "accepted_inbox_triggered",
"transport": "relay",
"accepted": True,
"inbox_triggered": True,
},
)
result = await bridge_alertmanager_to_agent99(
@@ -298,10 +386,58 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
assert result["status"] == "dispatched"
assert result["dispatch"] == "relay"
assert result["dispatchReceipt"]["accepted"] is True
assert result["dispatchReceipt"]["inbox_triggered"] is True
assert result["suggestedMode"] == "Recover"
assert len(dispatched) == 1
@pytest.mark.asyncio
async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
monkeypatch,
) -> None:
async def winner(*_args, **_kwargs) -> bool:
return True
released: list[tuple[str, str]] = []
async def release(fingerprint: str, alert_id: str) -> None:
released.append((fingerprint, alert_id))
monkeypatch.setattr(
"src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
winner,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.release_agent99_sre_single_flight",
release,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda _payload: {
"status": "relay_response_not_accepted",
"transport": "relay",
"accepted": False,
"inbox_triggered": False,
},
)
result = await bridge_alertmanager_to_agent99(
alert_id="receipt-rejected",
alertname="HostDown",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="host_down",
fingerprint="receipt-rejected-fingerprint",
)
assert result["status"] == "failed"
assert result["reason"] == "relay_response_not_accepted"
assert result["dispatchReceipt"]["accepted"] is False
assert released == [("receipt-rejected-fingerprint", "receipt-rejected")]
@pytest.mark.asyncio
async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None:
async def winner(*_args, **_kwargs) -> bool:
@@ -312,7 +448,7 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
async def release(fingerprint: str, alert_id: str) -> None:
released.append((fingerprint, alert_id))
def fail_transport(_payload: dict[str, object]) -> str:
def fail_transport(_payload: dict[str, object]) -> dict[str, object]:
raise RuntimeError("relay unavailable")
monkeypatch.setattr(
@@ -324,7 +460,7 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
release,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
fail_transport,
)

View File

@@ -290,6 +290,80 @@ async def test_candidate_blocked_when_playbook_is_generic_fallback() -> None:
assert "https://awoooi.wooo.work/zh-TW/awooop/work-items?" in work_item["work_item_url"]
@pytest.mark.asyncio
async def test_cold_start_gate_uses_agent99_recover_not_fake_k8s_deployment() -> None:
incident = _incident()
service = RepairCandidateService(
incident_service=FakeIncidentService(),
investigator=FakeInvestigator(_evidence(incident.incident_id)),
playbook_repository=FakePlaybookRepository(_generic_fallback_playbook()),
auto_repair_service=FakeAutoRepairService(),
)
result = await service.build_from_incident(
incident=incident,
alertname="ColdStartGateBlocked",
target_resource="cold-start-gate",
namespace="default",
message="full-stack cold-start recovery gate is blocked",
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
matched_playbook_id="PB-GENERIC-FALLBACK",
severity="low",
)
assert result.candidate_found is False
assert result.draft_ready_for_owner_review is True
draft_package = result.metadata["repair_candidate_draft_package"]
assert draft_package["coverage_gap"]["target_kind"] == "control_plane"
template = draft_package["playbook_draft_template"]
assert template["suggested_route"] == "agent99_recover_after_owner_review"
assert template["repair_command_template"] == (
"agent99-mode Recover controlledApply=true"
)
assert "kubectl rollout restart deployment/cold-start-gate" not in str(template)
handoff = result.metadata["controlled_executor_handoff"]
assert handoff["executor"] == "Agent99"
assert handoff["suggested_mode"] == "Recover"
assert handoff["status"] == (
"agent99_dispatch_required_on_actionable_card_send"
)
assert handoff["dispatch_receipt_required"] is True
@pytest.mark.asyncio
async def test_cicd_failure_namespace_uses_agent99_status_not_k8s_restart() -> None:
incident = _incident()
service = RepairCandidateService(
incident_service=FakeIncidentService(),
investigator=FakeInvestigator(_evidence(incident.incident_id)),
playbook_repository=FakePlaybookRepository(_generic_fallback_playbook()),
auto_repair_service=FakeAutoRepairService(),
)
result = await service.build_from_incident(
incident=incident,
alertname="AWOOOICICDFailed",
target_resource="AWOOOI 部署失敗",
namespace="build-and-deploy",
message="Gitea CI/CD deployment failed",
fallback_action="NO_ACTION - REPAIR_CANDIDATE_MISSING",
matched_playbook_id="PB-GENERIC-FALLBACK",
severity="critical",
)
draft_package = result.metadata["repair_candidate_draft_package"]
assert draft_package["coverage_gap"]["target_kind"] == "control_plane"
template = draft_package["playbook_draft_template"]
assert template["suggested_route"] == "agent99_status_after_owner_review"
assert template["repair_command_template"] == (
"agent99-mode Status controlledApply=false"
)
assert "kubectl rollout restart" not in str(template)
handoff = result.metadata["controlled_executor_handoff"]
assert handoff["executor"] == "Agent99"
assert handoff["suggested_mode"] == "Status"
@pytest.mark.asyncio
async def test_candidate_blocked_observe_only_prompts_repair_playbook_draft() -> None:
incident = _incident()

View File

@@ -1,7 +1,10 @@
from __future__ import annotations
from src.services.telegram_gateway import (
_agent99_actionable_outbound_metadata,
_agent99_alert_from_actionable_outbound,
_agent99_alert_from_ai_automation_card,
_merge_outbound_source_envelope_extra,
_outbound_source_envelope,
_sanitize_telegram_error,
format_aiops_signal_alert_card,
@@ -213,3 +216,91 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz
assert "provider-freshness" in payload["labels"]
assert "no-false-green" in payload["labels"]
assert "abcdefghijklmnopqrstuvwxyz" not in str(payload)
def test_action_required_cold_start_card_routes_to_agent99_recover() -> None:
card = (
" ACTION REQUIRED | <b>低風險</b>\n"
"──────────────────────\n"
"📋 <code>INC-20260710-5700F7</code>\n"
"🎯 資源:<code>cold-start-gate</code>\n"
"🏷️ 分類:<b>general</b>\n"
"🧭 處置狀態:<b>AI 選擇不執行修復</b>\n"
"🤖 <b>AI 自動化鏈路</b>"
)
metadata = _agent99_actionable_outbound_metadata(card)
payload = _agent99_alert_from_actionable_outbound(card, source="unit-test")
assert metadata is not None
assert metadata["card_schema"] == "awooop_action_required_v1"
assert metadata["incident_id"] == "INC-20260710-5700F7"
assert metadata["target"] == "cold-start-gate"
assert payload is not None
assert payload["kind"] == "host_recovery"
assert payload["suggestedMode"] == "Recover"
assert payload["controlledApply"] is True
assert payload["resolution"]["policy"] == "mode_verifier_can_resolve"
assert payload["awoooi"]["incidentId"] == "INC-20260710-5700F7"
def test_failed_cicd_card_routes_to_agent99_status_with_commit_receipt() -> None:
card = (
"❌ <b>[AWOOOI CI/CD]</b> | build-and-deploy\n"
"📦 AWOOOI 部署失敗\n"
"📋 <code>85159f25</code>\n"
"📝 fix(security): restrict executor SSH egress\n"
"🔗 <a href='https://gitea.example.invalid/run'>Workflow</a>"
)
metadata = _agent99_actionable_outbound_metadata(card)
payload = _agent99_alert_from_actionable_outbound(card, source="unit-test")
assert metadata is not None
assert metadata["card_schema"] == "awoooi_cicd_failure_v1"
assert metadata["commit_sha"] == "85159f25"
assert payload is not None
assert payload["kind"] == "cicd_failure"
assert payload["suggestedMode"] == "Status"
assert payload["controlledApply"] is False
assert payload["resolution"]["policy"] == "external_source_receipt_required"
assert payload["awoooi"]["commitSha"] == "85159f25"
assert "gitea.example.invalid" not in str(payload)
def test_agent99_dispatch_receipt_is_persisted_without_raw_transport_body() -> None:
envelope = _outbound_source_envelope(
"sendMessage",
{
"chat_id": "chat",
"text": (
" ACTION REQUIRED | 低風險\n"
"📋 INC-20260710-5700F7\n"
"🎯 資源cold-start-gate\n"
"🤖 AI 自動化鏈路"
),
},
)
receipt = {
"schema_version": "telegram_agent99_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"transport": "relay",
"alert_id": "awoooi-telegram-actionable-awooop_action_required-safehash",
"kind": "host_recovery",
"suggested_mode": "Recover",
"accepted": True,
"inbox_triggered": True,
"stores_raw_response": False,
}
merged = _merge_outbound_source_envelope_extra(
envelope,
{"agent99_dispatch_receipt": receipt},
)
assert merged["agent99_dispatch_receipt"] == receipt
assert merged["agent99_actionable_outbound"]["incident_id"] == (
"INC-20260710-5700F7"
)
assert "incomingFile" not in str(merged)
assert "private-path" not in str(merged)

View File

@@ -468,6 +468,7 @@ def test_send_request_payload_normalizer_blocks_direct_host_raw_dump() -> None:
async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch) -> None:
"""direct sendMessage AI cards must reach Agent99 even without wrapper helpers."""
mirrored: list[dict[str, str]] = []
outbound_receipts: list[dict[str, object]] = []
gateway = TelegramGateway()
gateway._initialized = True
@@ -476,15 +477,25 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch)
return None
def json(self) -> dict:
return {"ok": True, "result": True}
return {"ok": True, "result": {"message_id": 991}}
class FakeHttpClient:
async def post(self, url, json): # type: ignore[no-untyped-def]
return FakeResponse()
async def fake_mirror(text: str, *, source: str = "telegram_gateway") -> str:
async def fake_mirror(
text: str, *, source: str = "telegram_gateway"
) -> dict[str, object]:
mirrored.append({"text": text, "source": source})
return "mirrored"
return {
"schema_version": "telegram_agent99_dispatch_receipt_v1",
"status": "accepted_inbox_triggered",
"accepted": True,
"inbox_triggered": True,
}
async def fake_outbound_message(**kwargs) -> None: # type: ignore[no-untyped-def]
outbound_receipts.append(kwargs)
monkeypatch.setattr(TelegramGateway, "api_url", property(lambda _self: "https://telegram.test/botx"))
monkeypatch.setattr(
@@ -497,6 +508,7 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch)
"_mirror_ai_automation_alert_card_to_agent99",
fake_mirror,
)
monkeypatch.setattr(gateway, "_mirror_outbound_message", fake_outbound_message)
gateway._http_client = FakeHttpClient()
result = await gateway._send_request(
@@ -513,12 +525,18 @@ async def test_send_request_mirrors_direct_ai_alert_card_to_agent99(monkeypatch)
)
await asyncio.sleep(0)
assert result == {"ok": True, "result": True}
assert result == {"ok": True, "result": {"message_id": 991}}
assert mirrored
assert mirrored[0]["source"] == "send_request"
assert "ai_automation_alert_card_v1" in mirrored[0]["text"]
assert "provider_freshness_signal" in mirrored[0]["text"]
assert "runtime_write_gate=controlled" in mirrored[0]["text"]
assert outbound_receipts
receipt = outbound_receipts[0]["source_envelope_extra"][
"agent99_dispatch_receipt"
]
assert receipt["status"] == "accepted_inbox_triggered"
assert receipt["accepted"] is True
def test_weekly_report_marks_all_zero_as_low_trust_anomaly() -> None: