from datetime import datetime from decimal import Decimal from types import SimpleNamespace from uuid import UUID import pytest from fastapi import HTTPException from src.api.v1.platform.operator_runs import ( AiRouteStatusResponse, ListApprovalsResponse, ListCallbackRepliesResponse, ListRunsResponse, ) from src.services.ollama_failover_manager import OllamaEndpoint, OllamaRoutingResult from src.services.ollama_health_monitor import HealthReport, HealthStatus from src.services.platform_operator_service import ( _ai_route_health_map, _ai_route_policy_order, _build_awooop_status_chain, _callback_reply_event_item, _callback_reply_summary_matches_status, _collect_run_incident_ids, _legacy_mcp_timeline_status, _legacy_mcp_timeline_summary, _list_filter_context_limit, _outbound_timeline_status, _outbound_timeline_summary, _outbound_timeline_title, _remediation_summary_matches_incident_id, _remediation_summary_matches_status, _remediation_timeline_summary, _run_callback_reply_summary, _run_remediation_list_summary, _timeline_sort_key, _validate_ai_route_workload, _validate_callback_reply_action_filter, _validate_callback_reply_status_filter, ) def test_outbound_timeline_title_labels_runbook_review() -> None: title = _outbound_timeline_title( "telegram", "approval_request", "📄 RUNBOOK REVIEW|待審核\nIncident:INC-1", ) assert title == "TELEGRAM:Runbook 待人工審核" def test_outbound_timeline_title_labels_governance_alert() -> None: title = _outbound_timeline_title( "telegram", "final", "⚠️ *AI 治理警報|知識庫劣化*", ) assert title == "TELEGRAM:AI 治理警報" def test_outbound_timeline_title_labels_cicd_status() -> None: title = _outbound_timeline_title( "telegram", "final", "✅ [AWOOOI CI/CD] | code-review\n📦 Code Review 完成・LOW", ) assert title == "TELEGRAM:CI/CD 狀態通知" def test_outbound_timeline_title_labels_auto_repair_handoff() -> None: title = _outbound_timeline_title( "telegram", "error", "🤖❌ HANDOFF REQUIRED|AI 自動修復失敗,已轉人工", ) assert title == "TELEGRAM:AI 自動修復失敗,已轉人工" def test_outbound_timeline_title_falls_back_to_human_label() -> None: title = _outbound_timeline_title("telegram", "interim", "正在調用 MCP 工具") assert title == "TELEGRAM:漸進式狀態回饋" def test_outbound_timeline_title_labels_callback_reply_fallback() -> None: callback_reply = { "status": "callback_reply_fallback_sent", "action": "history", "incident_id": "INC-20260513-79ED5E", "parse_mode": "plain_text", } title = _outbound_timeline_title( "telegram", "final", "事件歷史統計 INC-20260513-79ED5E", callback_reply, ) summary = _outbound_timeline_summary( content_preview="事件歷史統計 INC-20260513-79ED5E", send_error=None, callback_reply=callback_reply, ) assert title == "TELEGRAM:歷史回覆 fallback 已送出" assert _outbound_timeline_status("sent", callback_reply) == ( "callback_reply_fallback_sent" ) assert "callback=history" in summary assert "incident=INC-20260513-79ED5E" in summary assert "parse_mode=plain_text" in summary def test_outbound_timeline_title_labels_callback_reply_failure() -> None: callback_reply = { "status": "callback_reply_failed", "action": "detail", "incident_id": "INC-20260513-79ED5E", "error": "HTTP error: 400", } assert _outbound_timeline_title("telegram", "error", None, callback_reply) == ( "TELEGRAM:詳情回覆送出失敗" ) assert _outbound_timeline_status("failed", callback_reply) == "callback_reply_failed" assert "error=HTTP error: 400" in _outbound_timeline_summary( content_preview=None, send_error="HTTP error: 400", callback_reply=callback_reply, ) def test_collect_run_incident_ids_reads_source_refs_and_legacy_text() -> None: run = SimpleNamespace( trigger_ref="not-an-incident", error_detail=None, ) inbound_events = [ SimpleNamespace( source_envelope={ "source_refs": { "incident_ids": ["INC-20260514-F85F21", "INC-20260514-F85F21"], } }, content_preview="Alertmanager inbound converged", content_redacted=None, ) ] outbound_messages = [ SimpleNamespace( source_envelope={ "source_refs": { "incident_ids": ["INC-20260518-CB0001"], }, }, content_preview="詳情:INC-20260513-79ED5E", send_error=None, ) ] incident_ids = _collect_run_incident_ids( run=run, inbound_events=inbound_events, outbound_messages=outbound_messages, ) assert incident_ids == [ "INC-20260514-F85F21", "INC-20260518-CB0001", "INC-20260513-79ED5E", ] def test_run_callback_reply_summary_marks_latest_fallback() -> None: summary = _run_callback_reply_summary([ SimpleNamespace( source_envelope={ "callback_reply": { "status": "callback_reply_sent", "action": "detail", "incident_id": "INC-20260513-79ED5E", } }, sent_at=datetime(2026, 5, 18, 6, 1, 0), queued_at=datetime(2026, 5, 18, 6, 1, 0), provider_message_id="100", ), SimpleNamespace( source_envelope={ "callback_reply": { "status": "callback_reply_fallback_sent", "action": "history", "incident_id": "INC-20260513-79ED5E", } }, sent_at=datetime(2026, 5, 18, 6, 2, 0), queued_at=datetime(2026, 5, 18, 6, 2, 0), provider_message_id="101", ), ]) assert summary["status"] == "fallback_sent" assert summary["total"] == 2 assert summary["sent"] == 1 assert summary["fallback_sent"] == 1 assert summary["latest_action"] == "history" assert summary["latest_incident_id"] == "INC-20260513-79ED5E" assert summary["latest_provider_message_id"] == "101" assert summary["needs_human"] is False def test_run_callback_reply_summary_marks_failed_as_human_attention() -> None: summary = _run_callback_reply_summary([ SimpleNamespace( source_envelope={ "callback_reply": { "status": "callback_reply_failed", "action": "detail", "incident_id": "INC-20260513-79ED5E", } }, sent_at=None, queued_at=datetime(2026, 5, 18, 6, 3, 0), provider_message_id="telegram_callback_reply:failed", ) ]) assert summary["status"] == "failed" assert summary["failed"] == 1 assert summary["needs_human"] is True def test_run_callback_reply_summary_marks_no_callback() -> None: summary = _run_callback_reply_summary([ SimpleNamespace( source_envelope={}, sent_at=datetime(2026, 5, 18, 6, 1, 0), queued_at=datetime(2026, 5, 18, 6, 1, 0), provider_message_id="100", ) ]) assert summary["status"] == "no_callback" assert summary["total"] == 0 def test_list_runs_response_preserves_callback_reply_summary() -> None: run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38") response = ListRunsResponse.model_validate({ "runs": [ { "run_id": run_id, "project_id": "awoooi", "agent_id": "legacy-telegram-gateway", "state": "completed", "is_shadow": True, "cost_usd": Decimal("0.0000"), "step_count": 0, "created_at": datetime(2026, 5, 18, 7, 31, 37), "timeout_at": None, "remediation_summary": None, "callback_reply_summary": { "schema_version": "awooop_run_callback_reply_summary_v1", "status": "failed", "total": 1, "sent": 0, "fallback_sent": 0, "rescue_sent": 0, "failed": 1, "needs_human": True, "latest_status": "callback_reply_failed", "latest_action": "detail", "latest_incident_id": "INC-20260513-79ED5E", "latest_at": "2026-05-18T07:31:37", "latest_provider_message_id": "telegram_callback_reply:failed", }, } ], "total": 1, "page": 1, "per_page": 1, }) dumped = response.model_dump(mode="json") assert dumped["runs"][0]["callback_reply_summary"]["status"] == "failed" assert dumped["runs"][0]["callback_reply_summary"]["needs_human"] is True def test_callback_reply_event_item_surfaces_run_link_and_human_flag() -> None: run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38") message_id = UUID("56cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92") item = _callback_reply_event_item({ "message_id": message_id, "run_id": run_id, "project_id": "awoooi", "channel_type": "telegram", "message_type": "error", "send_status": "failed", "send_error": "HTTP error: 400", "provider_message_id": "telegram_callback_reply:failed", "queued_at": datetime(2026, 5, 18, 7, 31, 37), "sent_at": None, "triggered_by_state": "callback_reply", "content_preview": "無法取得歷史統計", "run_state": "completed", "agent_id": "legacy-telegram-gateway", "run_created_at": datetime(2026, 5, 18, 7, 30, 0), "callback_reply": { "status": "callback_reply_failed", "action": "history", "incident_id": "INC-20260513-79ED5E", "error": "HTTP error: 400", }, }) assert item["status"] == "failed" assert item["needs_human"] is True assert item["action"] == "history" assert item["incident_id"] == "INC-20260513-79ED5E" assert item["event_at"] == datetime(2026, 5, 18, 7, 31, 37) assert item["run_detail_href"] == ( "/awooop/runs/5c0306e0-591a-5445-9a33-80f499426b38?project_id=awoooi" ) def test_list_callback_replies_response_preserves_callback_evidence() -> None: run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38") message_id = UUID("56cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92") response = ListCallbackRepliesResponse.model_validate({ "items": [ { "message_id": message_id, "run_id": run_id, "project_id": "awoooi", "status": "fallback_sent", "needs_human": False, "action": "detail", "incident_id": "INC-20260513-79ED5E", "event_at": datetime(2026, 5, 18, 7, 31, 37), "channel_type": "telegram", "message_type": "final", "send_status": "sent", "send_error": None, "provider_message_id": "123", "triggered_by_state": "callback_reply", "content_preview": "事件詳情", "run_state": "completed", "agent_id": "legacy-telegram-gateway", "run_created_at": datetime(2026, 5, 18, 7, 30, 0), "callback_reply": { "status": "callback_reply_fallback_sent", "action": "detail", "incident_id": "INC-20260513-79ED5E", }, "awooop_status_chain": { "schema_version": "awooop_status_chain_v1", "repair_state": "read_only_dry_run", "needs_human": True, }, "run_detail_href": ( "/awooop/runs/5c0306e0-591a-5445-9a33-80f499426b38" "?project_id=awoooi" ), } ], "total": 1, "page": 1, "per_page": 20, }) dumped = response.model_dump(mode="json") assert dumped["items"][0]["status"] == "fallback_sent" assert dumped["items"][0]["callback_reply"]["action"] == "detail" assert dumped["items"][0]["awooop_status_chain"]["repair_state"] == ( "read_only_dry_run" ) assert dumped["items"][0]["run_detail_href"].endswith("project_id=awoooi") def test_list_approvals_response_preserves_status_chain() -> None: run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38") response = ListApprovalsResponse.model_validate({ "items": [ { "run_id": run_id, "project_id": "awoooi", "agent_id": "hermes-approval-router", "created_at": datetime(2026, 5, 18, 7, 30, 0), "timeout_at": datetime(2026, 5, 18, 7, 45, 0), "remediation_summary": { "status": "read_only_dry_run", "incident_ids": ["INC-20260513-79ED5E"], }, "awooop_status_chain": { "schema_version": "awooop_status_chain_v1", "source_id": "INC-20260513-79ED5E", "repair_state": "read_only_dry_run", "needs_human": True, "next_step": "approve_or_escalate_from_awooop", }, } ], "total": 1, }) dumped = response.model_dump(mode="json") assert dumped["items"][0]["remediation_summary"]["status"] == ( "read_only_dry_run" ) assert dumped["items"][0]["awooop_status_chain"]["source_id"] == ( "INC-20260513-79ED5E" ) assert dumped["items"][0]["awooop_status_chain"]["needs_human"] is True def test_callback_reply_action_filter_normalizes_safe_actions() -> None: assert _validate_callback_reply_action_filter(" History ") == "history" assert _validate_callback_reply_action_filter("incident:detail-2") == ( "incident:detail-2" ) assert _validate_callback_reply_action_filter("") is None def test_callback_reply_action_filter_rejects_unsafe_values() -> None: with pytest.raises(HTTPException): _validate_callback_reply_action_filter("detail;drop") def test_remediation_timeline_summary_surfaces_route_and_write_flags() -> None: summary = _remediation_timeline_summary({ "incident_id": "INC-20260514-F85F21", "mode": "replay", "verification_result_preview": "degraded", "agent_id": "auto_repair_executor", "tool_name": "ssh_diagnose", "required_scope": "read", "writes_incident_state": False, "writes_auto_repair_result": False, }) assert "incident=INC-20260514-F85F21" in summary assert "route=auto_repair_executor/ssh_diagnose/read" in summary assert "writes_incident=False" in summary assert "writes_auto_repair=False" in summary def test_awooop_status_chain_marks_verified_repair() -> None: chain = _build_awooop_status_chain( incident_ids=["INC-20260513-79ED5E"], source_id="INC-20260513-79ED5E", truth_chain={ "truth_status": { "current_stage": "execution_succeeded", "stage_status": "success", "needs_human": False, "blockers": [], }, "automation_quality": { "verdict": "auto_repaired_verified", "facts": { "auto_repair_execution_records": 1, "automation_operation_records": 1, "verification_result": "healthy", "mcp_gateway_total": 2, "knowledge_entries": 1, }, "blockers": [], }, }, remediation_history={ "total": 1, "items": [ { "incident_id": "INC-20260513-79ED5E", "agent_id": "auto_repair_executor", "tool_name": "rollout_restart", "required_scope": "write", "verification_result_preview": "healthy", "writes_incident_state": True, "writes_auto_repair_result": True, } ], }, ) assert chain["repair_state"] == "auto_repaired_verified" assert chain["verification"] == "healthy" assert chain["needs_human"] is False assert chain["next_step"] == "monitor_for_regression" assert chain["evidence"]["latest_route"] == "auto_repair_executor/rollout_restart/write" def test_awooop_status_chain_marks_read_only_manual_gate() -> None: chain = _build_awooop_status_chain( incident_ids=["INC-20260513-79ED5E"], source_id="INC-20260513-79ED5E", truth_chain={ "truth_status": { "current_stage": "approval_required", "stage_status": "waiting", "needs_human": True, "blockers": ["pending_human_approval"], }, "automation_quality": { "verdict": "approval_required", "facts": { "auto_repair_execution_records": 0, "automation_operation_records": 0, "verification_result": "missing", "mcp_gateway_total": 1, "knowledge_entries": 0, }, "blockers": [], }, }, remediation_history={ "total": 2, "items": [ { "incident_id": "INC-20260513-79ED5E", "agent_id": "investigator", "tool_name": "ssh_diagnose", "required_scope": "read", "verification_result_preview": "degraded", "writes_incident_state": False, "writes_auto_repair_result": False, } ], }, ) assert chain["repair_state"] == "read_only_dry_run" assert chain["needs_human"] is True assert chain["next_step"] == "approve_or_escalate_from_awooop" assert chain["blockers"] == ["pending_human_approval"] def test_legacy_mcp_timeline_summary_surfaces_tool_context() -> None: record = { "incident_id": "INC-20260514-F85F21", "agent_role": "pre_decision_investigator", "flywheel_node": "investigator", "duration_ms": 127, "success": True, "error_message": None, } assert _legacy_mcp_timeline_status(record) == "success" summary = _legacy_mcp_timeline_summary(record) assert "incident=INC-20260514-F85F21" in summary assert "agent=pre_decision_investigator" in summary assert "node=investigator" in summary assert "duration_ms=127" in summary def test_legacy_mcp_timeline_status_marks_failed_and_unknown() -> None: assert _legacy_mcp_timeline_status({"success": False}) == "failed" assert _legacy_mcp_timeline_status({"success": None}) == "warning" def test_run_remediation_list_summary_marks_read_only_dry_run() -> None: run = SimpleNamespace(state="waiting_approval") summary = _run_remediation_list_summary( run=run, incident_ids=["INC-20260514-F85F21"], items=[ { "created_at": "2026-05-14T23:04:00+00:00", "incident_id": "INC-20260514-F85F21", "mode": "replay", "verification_result_preview": "degraded", "agent_id": "auto_repair_executor", "tool_name": "ssh_diagnose", "required_scope": "read", "writes_incident_state": False, "writes_auto_repair_result": False, } ], ) assert summary["status"] == "read_only_dry_run" assert summary["has_dry_run"] is True assert summary["is_read_only"] is True assert summary["human_gate_open"] is True assert summary["latest_route"] == "auto_repair_executor/ssh_diagnose/read" def test_run_remediation_list_summary_marks_mcp_observed_without_dry_run() -> None: run = SimpleNamespace(state="completed") summary = _run_remediation_list_summary( run=run, incident_ids=["INC-20260518-792684"], items=[], legacy_mcp_records=[ { "created_at": "2026-05-18T04:31:30+00:00", "incident_id": "INC-20260518-792684", "agent_role": "pre_decision_investigator", "mcp_server": "ssh_host", "tool_name": "ssh_diagnose", "success": True, }, { "created_at": "2026-05-18T04:31:29+00:00", "incident_id": "INC-20260518-792684", "agent_role": "pre_decision_investigator", "mcp_server": "signoz", "tool_name": "query_logs", "success": False, }, ], ) assert summary["status"] == "mcp_observed" assert summary["source"] == "mcp_audit_log" assert summary["total"] == 0 assert summary["evidence_total"] == 2 assert summary["has_dry_run"] is False assert summary["has_mcp_investigation"] is True assert summary["mcp_observation_total"] == 2 assert summary["mcp_observation_success"] == 1 assert summary["mcp_observation_failed"] == 1 assert summary["latest_route"] == "pre_decision_investigator/ssh_host.ssh_diagnose/read" def test_run_remediation_list_summary_flags_write_observed() -> None: run = SimpleNamespace(state="completed") summary = _run_remediation_list_summary( run=run, incident_ids=["INC-20260514-F85F21"], items=[ { "created_at": "2026-05-14T23:05:00+00:00", "incident_id": "INC-20260514-F85F21", "agent_id": "auto_repair_executor", "tool_name": "state_update", "required_scope": "write", "writes_incident_state": True, "writes_auto_repair_result": False, } ], ) assert summary["status"] == "write_observed" assert summary["is_read_only"] is False assert summary["writes_incident_state"] is True def test_remediation_summary_matches_status_filter() -> None: assert _remediation_summary_matches_status( {"status": "mcp_observed"}, "mcp_observed", ) assert _remediation_summary_matches_status( {"status": "read_only_dry_run"}, "read_only_dry_run", ) assert not _remediation_summary_matches_status( {"status": "write_observed"}, "read_only_dry_run", ) assert _remediation_summary_matches_status(None, "no_evidence") def test_callback_reply_summary_matches_status_filter() -> None: assert _callback_reply_summary_matches_status( {"status": "failed"}, "failed", ) assert _callback_reply_summary_matches_status( {"status": "fallback_sent"}, "fallback_sent", ) assert not _callback_reply_summary_matches_status( {"status": "sent"}, "failed", ) assert _callback_reply_summary_matches_status(None, "no_callback") def test_callback_reply_status_filter_rejects_unknown_value() -> None: _validate_callback_reply_status_filter("failed") with pytest.raises(HTTPException) as exc_info: _validate_callback_reply_status_filter("telegram_error") assert exc_info.value.status_code == 422 assert "callback_reply_status" in str(exc_info.value.detail) def test_remediation_summary_matches_incident_id_filter() -> None: assert _remediation_summary_matches_incident_id( {"incident_ids": ["INC-20260514-F85F21"]}, "INC-20260514-F85F21", ) assert not _remediation_summary_matches_incident_id( {"incident_ids": ["INC-20260514-F85F21"]}, "INC-20260513-79ED5E", ) assert _remediation_summary_matches_incident_id(None, None) def test_list_filter_context_limit_scales_with_candidate_rows() -> None: assert _list_filter_context_limit(2) == 500 assert _list_filter_context_limit(4176) == 16704 assert _list_filter_context_limit(10000) == 20000 def test_timeline_sort_key_normalizes_datetime_and_iso_string() -> None: fallback = datetime(2026, 5, 14, 10, 0, 0) keys = [ _timeline_sort_key({"ts": datetime(2026, 5, 14, 10, 0, 1)}, fallback), _timeline_sort_key({"ts": "2026-05-14T10:00:02+00:00"}, fallback), _timeline_sort_key({"ts": None}, fallback), ] assert keys == [ "2026-05-14T10:00:01", "2026-05-14T10:00:02+00:00", "2026-05-14T10:00:00", ] assert sorted(keys) == [ "2026-05-14T10:00:00", "2026-05-14T10:00:01", "2026-05-14T10:00:02+00:00", ] def test_ai_route_policy_order_exposes_global_ollama_then_gemini() -> None: policy = _ai_route_policy_order("deep_rca") assert [item["provider_name"] for item in policy] == [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini", ] assert policy[-1]["role"] == "final_fallback" assert policy[-1]["runtime"] == "cloud" def test_ai_route_health_map_marks_standby_as_not_checked() -> None: route = OllamaRoutingResult( primary=OllamaEndpoint( url="http://gcp-a:11434", provider_name="ollama_gcp_a", model="qwen3:14b", ), fallback_chain=[ OllamaEndpoint( url="http://gcp-b:11434", provider_name="ollama_gcp_b", model="qwen3:14b", ), OllamaEndpoint( url="http://local-111:11434", provider_name="ollama_local", model="qwen3:14b", ), ], routing_reason="primary healthy", health_gcp_a=HealthReport( status=HealthStatus.HEALTHY, host="http://gcp-a:11434", latency_ms=123.4, reason="ok", ), health_gcp_b=None, health_local=None, ) health = _ai_route_health_map(route) assert health["ollama_gcp_a"]["status"] == "healthy" assert health["ollama_gcp_a"]["checked"] is True assert health["ollama_gcp_b"]["status"] == "not_checked" assert health["ollama_local"]["reason"] == "standby_not_checked_primary_healthy" def test_ai_route_status_response_preserves_route_fields() -> None: response = AiRouteStatusResponse.model_validate({ "schema_version": "awooop_ai_route_status_v1", "workload_type": "deep_rca", "policy_order": _ai_route_policy_order("deep_rca"), "selected_provider": "ollama_gcp_a", "selected_url": "http://gcp-a:11434", "selected_model": "qwen3:14b", "fallback_chain": [ { "priority": 2, "provider_name": "ollama_gcp_b", "url": "http://gcp-b:11434", "model": "qwen3:14b", "runtime": "ollama", } ], "route_reason": "primary healthy", "route_source": "ollama_failover_manager", "route_error": None, "health": { "ollama_gcp_a": { "status": "healthy", "host": "http://gcp-a:11434", "latency_ms": 123.4, "reason": "ok", "checked_at": 0, "from_cache": False, "checked": True, }, }, "checked_at": datetime(2026, 5, 19, 12, 0, 0), }) dumped = response.model_dump(mode="json") assert dumped["policy_order"][-1]["provider_name"] == "gemini" assert dumped["selected_provider"] == "ollama_gcp_a" def test_ai_route_workload_validation_rejects_unknown_value() -> None: assert _validate_ai_route_workload(" hermes ") == "hermes" with pytest.raises(HTTPException) as exc_info: _validate_ai_route_workload("charge_money") assert "Unsupported workload_type" in str(exc_info.value.detail)