414 lines
13 KiB
Python
414 lines
13 KiB
Python
from datetime import datetime
|
||
from types import SimpleNamespace
|
||
|
||
from src.services.platform_operator_service import (
|
||
_collect_run_incident_ids,
|
||
_legacy_mcp_timeline_status,
|
||
_legacy_mcp_timeline_summary,
|
||
_list_filter_context_limit,
|
||
_outbound_timeline_title,
|
||
_outbound_timeline_status,
|
||
_outbound_timeline_summary,
|
||
_remediation_summary_matches_incident_id,
|
||
_remediation_summary_matches_status,
|
||
_remediation_timeline_summary,
|
||
_run_callback_reply_summary,
|
||
_run_remediation_list_summary,
|
||
_timeline_sort_key,
|
||
)
|
||
|
||
|
||
def test_outbound_timeline_title_labels_runbook_review() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"approval_request",
|
||
"📄 <b>RUNBOOK REVIEW｜待審核</b>\nIncident：INC-1",
|
||
)
|
||
|
||
assert title == "TELEGRAM：Runbook 待人工審核"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_governance_alert() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"⚠️ *AI 治理警報｜知識庫劣化*",
|
||
)
|
||
|
||
assert title == "TELEGRAM：AI 治理警報"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_cicd_status() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"✅ <b>[AWOOOI CI/CD]</b> | code-review\n📦 Code Review 完成・LOW",
|
||
)
|
||
|
||
assert title == "TELEGRAM：CI/CD 狀態通知"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_auto_repair_handoff() -> None:
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"error",
|
||
"🤖❌ HANDOFF REQUIRED｜AI 自動修復失敗，已轉人工",
|
||
)
|
||
|
||
assert title == "TELEGRAM：AI 自動修復失敗，已轉人工"
|
||
|
||
|
||
def test_outbound_timeline_title_falls_back_to_human_label() -> None:
|
||
title = _outbound_timeline_title("telegram", "interim", "正在調用 MCP 工具")
|
||
|
||
assert title == "TELEGRAM：漸進式狀態回饋"
|
||
|
||
|
||
def test_outbound_timeline_title_labels_callback_reply_fallback() -> None:
|
||
callback_reply = {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"parse_mode": "plain_text",
|
||
}
|
||
|
||
title = _outbound_timeline_title(
|
||
"telegram",
|
||
"final",
|
||
"事件歷史統計 INC-20260513-79ED5E",
|
||
callback_reply,
|
||
)
|
||
summary = _outbound_timeline_summary(
|
||
content_preview="事件歷史統計 INC-20260513-79ED5E",
|
||
send_error=None,
|
||
callback_reply=callback_reply,
|
||
)
|
||
|
||
assert title == "TELEGRAM：歷史回覆 fallback 已送出"
|
||
assert _outbound_timeline_status("sent", callback_reply) == (
|
||
"callback_reply_fallback_sent"
|
||
)
|
||
assert "callback=history" in summary
|
||
assert "incident=INC-20260513-79ED5E" in summary
|
||
assert "parse_mode=plain_text" in summary
|
||
|
||
|
||
def test_outbound_timeline_title_labels_callback_reply_failure() -> None:
|
||
callback_reply = {
|
||
"status": "callback_reply_failed",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
"error": "HTTP error: 400",
|
||
}
|
||
|
||
assert _outbound_timeline_title("telegram", "error", None, callback_reply) == (
|
||
"TELEGRAM：詳情回覆送出失敗"
|
||
)
|
||
assert _outbound_timeline_status("failed", callback_reply) == "callback_reply_failed"
|
||
assert "error=HTTP error: 400" in _outbound_timeline_summary(
|
||
content_preview=None,
|
||
send_error="HTTP error: 400",
|
||
callback_reply=callback_reply,
|
||
)
|
||
|
||
|
||
def test_collect_run_incident_ids_reads_source_refs_and_legacy_text() -> None:
|
||
run = SimpleNamespace(
|
||
trigger_ref="not-an-incident",
|
||
error_detail=None,
|
||
)
|
||
inbound_events = [
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"source_refs": {
|
||
"incident_ids": ["INC-20260514-F85F21", "INC-20260514-F85F21"],
|
||
}
|
||
},
|
||
content_preview="Alertmanager inbound converged",
|
||
content_redacted=None,
|
||
)
|
||
]
|
||
outbound_messages = [
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"source_refs": {
|
||
"incident_ids": ["INC-20260518-CB0001"],
|
||
},
|
||
},
|
||
content_preview="詳情：INC-20260513-79ED5E",
|
||
send_error=None,
|
||
)
|
||
]
|
||
|
||
incident_ids = _collect_run_incident_ids(
|
||
run=run,
|
||
inbound_events=inbound_events,
|
||
outbound_messages=outbound_messages,
|
||
)
|
||
|
||
assert incident_ids == [
|
||
"INC-20260514-F85F21",
|
||
"INC-20260518-CB0001",
|
||
"INC-20260513-79ED5E",
|
||
]
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_latest_fallback() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_sent",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
provider_message_id="100",
|
||
),
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_fallback_sent",
|
||
"action": "history",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=datetime(2026, 5, 18, 6, 2, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 2, 0),
|
||
provider_message_id="101",
|
||
),
|
||
])
|
||
|
||
assert summary["status"] == "fallback_sent"
|
||
assert summary["total"] == 2
|
||
assert summary["sent"] == 1
|
||
assert summary["fallback_sent"] == 1
|
||
assert summary["latest_action"] == "history"
|
||
assert summary["latest_incident_id"] == "INC-20260513-79ED5E"
|
||
assert summary["latest_provider_message_id"] == "101"
|
||
assert summary["needs_human"] is False
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_failed_as_human_attention() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={
|
||
"callback_reply": {
|
||
"status": "callback_reply_failed",
|
||
"action": "detail",
|
||
"incident_id": "INC-20260513-79ED5E",
|
||
}
|
||
},
|
||
sent_at=None,
|
||
queued_at=datetime(2026, 5, 18, 6, 3, 0),
|
||
provider_message_id="telegram_callback_reply:failed",
|
||
)
|
||
])
|
||
|
||
assert summary["status"] == "failed"
|
||
assert summary["failed"] == 1
|
||
assert summary["needs_human"] is True
|
||
|
||
|
||
def test_run_callback_reply_summary_marks_no_callback() -> None:
|
||
summary = _run_callback_reply_summary([
|
||
SimpleNamespace(
|
||
source_envelope={},
|
||
sent_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
queued_at=datetime(2026, 5, 18, 6, 1, 0),
|
||
provider_message_id="100",
|
||
)
|
||
])
|
||
|
||
assert summary["status"] == "no_callback"
|
||
assert summary["total"] == 0
|
||
|
||
|
||
def test_remediation_timeline_summary_surfaces_route_and_write_flags() -> None:
|
||
summary = _remediation_timeline_summary({
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"mode": "replay",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
})
|
||
|
||
assert "incident=INC-20260514-F85F21" in summary
|
||
assert "route=auto_repair_executor/ssh_diagnose/read" in summary
|
||
assert "writes_incident=False" in summary
|
||
assert "writes_auto_repair=False" in summary
|
||
|
||
|
||
def test_legacy_mcp_timeline_summary_surfaces_tool_context() -> None:
|
||
record = {
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"agent_role": "pre_decision_investigator",
|
||
"flywheel_node": "investigator",
|
||
"duration_ms": 127,
|
||
"success": True,
|
||
"error_message": None,
|
||
}
|
||
|
||
assert _legacy_mcp_timeline_status(record) == "success"
|
||
summary = _legacy_mcp_timeline_summary(record)
|
||
|
||
assert "incident=INC-20260514-F85F21" in summary
|
||
assert "agent=pre_decision_investigator" in summary
|
||
assert "node=investigator" in summary
|
||
assert "duration_ms=127" in summary
|
||
|
||
|
||
def test_legacy_mcp_timeline_status_marks_failed_and_unknown() -> None:
|
||
assert _legacy_mcp_timeline_status({"success": False}) == "failed"
|
||
assert _legacy_mcp_timeline_status({"success": None}) == "warning"
|
||
|
||
|
||
def test_run_remediation_list_summary_marks_read_only_dry_run() -> None:
|
||
run = SimpleNamespace(state="waiting_approval")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260514-F85F21"],
|
||
items=[
|
||
{
|
||
"created_at": "2026-05-14T23:04:00+00:00",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"mode": "replay",
|
||
"verification_result_preview": "degraded",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "ssh_diagnose",
|
||
"required_scope": "read",
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "read_only_dry_run"
|
||
assert summary["has_dry_run"] is True
|
||
assert summary["is_read_only"] is True
|
||
assert summary["human_gate_open"] is True
|
||
assert summary["latest_route"] == "auto_repair_executor/ssh_diagnose/read"
|
||
|
||
|
||
def test_run_remediation_list_summary_marks_mcp_observed_without_dry_run() -> None:
|
||
run = SimpleNamespace(state="completed")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260518-792684"],
|
||
items=[],
|
||
legacy_mcp_records=[
|
||
{
|
||
"created_at": "2026-05-18T04:31:30+00:00",
|
||
"incident_id": "INC-20260518-792684",
|
||
"agent_role": "pre_decision_investigator",
|
||
"mcp_server": "ssh_host",
|
||
"tool_name": "ssh_diagnose",
|
||
"success": True,
|
||
},
|
||
{
|
||
"created_at": "2026-05-18T04:31:29+00:00",
|
||
"incident_id": "INC-20260518-792684",
|
||
"agent_role": "pre_decision_investigator",
|
||
"mcp_server": "signoz",
|
||
"tool_name": "query_logs",
|
||
"success": False,
|
||
},
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "mcp_observed"
|
||
assert summary["source"] == "mcp_audit_log"
|
||
assert summary["total"] == 0
|
||
assert summary["evidence_total"] == 2
|
||
assert summary["has_dry_run"] is False
|
||
assert summary["has_mcp_investigation"] is True
|
||
assert summary["mcp_observation_total"] == 2
|
||
assert summary["mcp_observation_success"] == 1
|
||
assert summary["mcp_observation_failed"] == 1
|
||
assert summary["latest_route"] == "pre_decision_investigator/ssh_host.ssh_diagnose/read"
|
||
|
||
|
||
def test_run_remediation_list_summary_flags_write_observed() -> None:
|
||
run = SimpleNamespace(state="completed")
|
||
|
||
summary = _run_remediation_list_summary(
|
||
run=run,
|
||
incident_ids=["INC-20260514-F85F21"],
|
||
items=[
|
||
{
|
||
"created_at": "2026-05-14T23:05:00+00:00",
|
||
"incident_id": "INC-20260514-F85F21",
|
||
"agent_id": "auto_repair_executor",
|
||
"tool_name": "state_update",
|
||
"required_scope": "write",
|
||
"writes_incident_state": True,
|
||
"writes_auto_repair_result": False,
|
||
}
|
||
],
|
||
)
|
||
|
||
assert summary["status"] == "write_observed"
|
||
assert summary["is_read_only"] is False
|
||
assert summary["writes_incident_state"] is True
|
||
|
||
|
||
def test_remediation_summary_matches_status_filter() -> None:
|
||
assert _remediation_summary_matches_status(
|
||
{"status": "mcp_observed"},
|
||
"mcp_observed",
|
||
)
|
||
assert _remediation_summary_matches_status(
|
||
{"status": "read_only_dry_run"},
|
||
"read_only_dry_run",
|
||
)
|
||
assert not _remediation_summary_matches_status(
|
||
{"status": "write_observed"},
|
||
"read_only_dry_run",
|
||
)
|
||
assert _remediation_summary_matches_status(None, "no_evidence")
|
||
|
||
|
||
def test_remediation_summary_matches_incident_id_filter() -> None:
|
||
assert _remediation_summary_matches_incident_id(
|
||
{"incident_ids": ["INC-20260514-F85F21"]},
|
||
"INC-20260514-F85F21",
|
||
)
|
||
assert not _remediation_summary_matches_incident_id(
|
||
{"incident_ids": ["INC-20260514-F85F21"]},
|
||
"INC-20260513-79ED5E",
|
||
)
|
||
assert _remediation_summary_matches_incident_id(None, None)
|
||
|
||
|
||
def test_list_filter_context_limit_scales_with_candidate_rows() -> None:
|
||
assert _list_filter_context_limit(2) == 500
|
||
assert _list_filter_context_limit(4176) == 16704
|
||
assert _list_filter_context_limit(10000) == 20000
|
||
|
||
|
||
def test_timeline_sort_key_normalizes_datetime_and_iso_string() -> None:
|
||
fallback = datetime(2026, 5, 14, 10, 0, 0)
|
||
keys = [
|
||
_timeline_sort_key({"ts": datetime(2026, 5, 14, 10, 0, 1)}, fallback),
|
||
_timeline_sort_key({"ts": "2026-05-14T10:00:02+00:00"}, fallback),
|
||
_timeline_sort_key({"ts": None}, fallback),
|
||
]
|
||
|
||
assert keys == [
|
||
"2026-05-14T10:00:01",
|
||
"2026-05-14T10:00:02+00:00",
|
||
"2026-05-14T10:00:00",
|
||
]
|
||
assert sorted(keys) == [
|
||
"2026-05-14T10:00:00",
|
||
"2026-05-14T10:00:01",
|
||
"2026-05-14T10:00:02+00:00",
|
||
]
|