From 4e4c56cae59d3005f63c76a167c0a92c303e2c1d Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 27 Jun 2026 20:49:32 +0800 Subject: [PATCH] fix(awooop): send controlled apply telegram receipts --- .../services/awooop_ansible_audit_service.py | 14 ++- .../awooop_ansible_check_mode_service.py | 43 ++++++++ apps/api/src/services/telegram_gateway.py | 101 ++++++++++++++++++ .../tests/test_awooop_truth_chain_service.py | 54 ++++++++++ .../tests/test_telegram_message_templates.py | 99 +++++++++++++++++ 5 files changed, 310 insertions(+), 1 deletion(-) diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index b748fd763..06617cc42 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -217,6 +217,18 @@ def _int_or_none(value: Any) -> int | None: return None +def _is_controlled_apply_record(row: dict[str, Any]) -> bool: + if not row: + return False + if row.get("approval_source"): + return True + if str(row.get("execution_mode") or "").strip().lower() == "controlled_apply": + return True + if "controlled_apply" in _tags(row): + return True + return str(row.get("actor") or "").strip().lower() == "ansible_controlled_apply_worker" + + def _is_ansible_operation(row: dict[str, Any]) -> bool: operation_type = str(_get(row, "operation_type") or "").lower() if operation_type in ANSIBLE_OPERATION_TYPES: @@ -341,7 +353,7 @@ def summarize_ansible_execution(records: list[dict[str, Any]]) -> dict[str, Any] "pending_check_mode_total": pending_check_mode_total, "applied_success_total": applied_success_total, "applied": applied_success_total > 0, - "controlled_apply": bool(latest_apply) and bool(approval_source), + "controlled_apply": bool(latest_apply) and _is_controlled_apply_record(latest_apply), "latest_operation_type": focused.get("operation_type"), "latest_status": focused.get("status"), "latest_catalog_id": focused.get("catalog_id"), diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index d44b4c694..a88679b84 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -805,6 +805,41 @@ async def _record_post_apply_verifier_and_learning( return status +async def _send_controlled_apply_telegram_receipt( + claim: AnsibleCheckModeClaim, + result: AnsibleRunResult, + *, + apply_op_id: str, + writeback: dict[str, bool], + project_id: str, +) -> bool: + try: + from src.services.telegram_gateway import get_telegram_gateway + + response = await get_telegram_gateway().send_controlled_apply_result_receipt( + incident_id=claim.incident_id, + catalog_id=claim.catalog_id, + apply_op_id=apply_op_id, + playbook_path=claim.apply_playbook_path, + verification_result=_post_apply_verification_result(result), + returncode=result.returncode, + duration_ms=result.duration_ms, + verifier_written=bool(writeback.get("verification")), + learning_written=bool(writeback.get("learning")), + project_id=project_id, + ) + return bool(response) + except Exception as exc: + logger.warning( + "ansible_controlled_apply_telegram_receipt_failed", + incident_id=claim.incident_id, + catalog_id=claim.catalog_id, + apply_op_id=apply_op_id, + error=str(exc), + ) + return False + + async def backfill_missing_auto_repair_execution_receipts_once( *, project_id: str = "awoooi", @@ -1322,6 +1357,13 @@ async def run_controlled_apply_for_claim( apply_op_id=apply_op_id, project_id=project_id, ) + telegram_receipt_sent = await _send_controlled_apply_telegram_receipt( + claim, + result, + apply_op_id=apply_op_id, + writeback=writeback, + project_id=project_id, + ) logger.info( "ansible_controlled_apply_completed", @@ -1335,6 +1377,7 @@ async def run_controlled_apply_for_claim( auto_repair_receipt_written=receipt_written, post_apply_verification_written=writeback.get("verification"), post_apply_learning_written=writeback.get("learning"), + telegram_receipt_sent=telegram_receipt_sent, ) return result diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index a8a7aad94..fa0af992f 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -9182,6 +9182,107 @@ class TelegramGateway: } return await self._send_request("sendMessage", payload) + async def send_controlled_apply_result_receipt( + self, + *, + incident_id: str, + catalog_id: str, + apply_op_id: str, + playbook_path: str, + verification_result: str, + returncode: int, + duration_ms: int, + verifier_written: bool, + learning_written: bool, + project_id: str = "awoooi", + ) -> dict: + """Send and mirror the AI Agent controlled-apply result receipt.""" + + if not self.alert_chat_id: + logger.warning( + "controlled_apply_result_receipt_skipped_no_chat", + incident_id=incident_id, + apply_op_id=apply_op_id, + ) + return {} + + success = verification_result == "success" and returncode == 0 + next_step = ( + "monitor_for_regression" + if success + else "queue_ai_rollback_or_playbook_repair" + ) + title = ( + "CONTROLLED APPLY RESULT｜AI Agent 受控執行完成" + if success + else "CONTROLLED APPLY RESULT｜AI Agent 受控執行待修復" + ) + truth_chain: dict[str, object] | None = None + try: + from src.services.awooop_truth_chain_service import fetch_truth_chain + + truth_chain = await fetch_truth_chain( + source_id=incident_id, + project_id=project_id or "awoooi", + ) + except Exception as exc: + logger.warning( + "controlled_apply_result_truth_chain_snapshot_failed", + incident_id=incident_id, + apply_op_id=apply_op_id, + error=str(exc), + ) + + status_snapshot = _callback_reply_awooop_status_chain_snapshot( + incident_id=incident_id, + truth_chain=truth_chain, + ) + source_extra = _callback_reply_source_envelope_extra( + incident_id=incident_id, + failure_context="controlled_apply_result", + status="callback_reply_sent", + chunk_index=0, + chunk_count=1, + callback_action="controlled_apply_result", + parse_mode="HTML", + awooop_status_chain=status_snapshot, + ) + lines = [ + f"{html.escape(title)}", + f"Incident: {html.escape(str(incident_id))}", + f"Catalog: {html.escape(str(catalog_id or '--'))}", + f"Apply op: {html.escape(str(apply_op_id or '')[:8])}", + f"PlayBook: {html.escape(str(playbook_path or '--'))}", + ( + "Result: " + f"{html.escape(str(verification_result or 'missing'))} " + f"/ rc {html.escape(str(returncode))} " + f"/ {html.escape(str(duration_ms))}ms" + ), + ( + "Receipts: verifier " + f"{html.escape(_bool_code(verifier_written))} / KM " + f"{html.escape(_bool_code(learning_written))}" + ), + f"Next: {html.escape(next_step)}", + f"Runs: {html.escape(incident_runs_url(incident_id, project_id=project_id or 'awoooi'))}", + ] + payload: dict = { + "chat_id": self.alert_chat_id, + "text": "\n".join(lines)[:4096], + "parse_mode": "HTML", + "disable_web_page_preview": True, + } + reply_markup = incident_truth_chain_reply_markup( + incident_id, + project_id=project_id or "awoooi", + ) + if reply_markup: + payload["reply_markup"] = reply_markup + if source_extra: + payload[_AWOOOP_SOURCE_ENVELOPE_EXTRA_KEY] = source_extra + return await self._send_request("sendMessage", payload) + # ========================================================================= # 2026-04-24 Claude Sonnet 4.6 (ADR-095 WS4): Hermes NL 回覆 # ========================================================================= diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 563ae56f4..d2e0a9a76 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -21,6 +21,7 @@ from src.services.awooop_ansible_check_mode_service import ( _post_apply_km_path_type, _post_apply_verification_result, _record_auto_repair_execution_receipt, + _send_controlled_apply_telegram_receipt, build_ansible_apply_command, build_ansible_check_mode_claim_input, build_ansible_check_mode_command, @@ -28,6 +29,7 @@ from src.services.awooop_ansible_check_mode_service import ( claim_pending_check_modes, detect_ansible_transport_blockers, recent_ansible_transport_blockers, + run_controlled_apply_for_claim, ) from src.services.awooop_truth_chain_service import ( _ansible_playbook_roots, @@ -1175,6 +1177,48 @@ def test_ansible_truth_surfaces_audited_check_mode_record() -> None: ) +def test_ansible_truth_marks_worker_apply_as_controlled_apply() -> None: + truth = build_ansible_truth( + [ + { + "op_id": "apply-op-1", + "operation_type": "ansible_apply_executed", + "status": "success", + "actor": "ansible_controlled_apply_worker", + "input_catalog_id": "ansible:188-momo-backup-user", + "input_execution_mode": "controlled_apply", + "input_playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml", + "input_apply_enabled": "true", + "output_apply_executed": "true", + "output_returncode": "0", + "tags": ["ansible", "controlled_apply", "low", "ai_agent_auto_execution"], + }, + { + "op_id": "check-op-1", + "operation_type": "ansible_check_mode_executed", + "status": "success", + "actor": "ansible_check_mode_worker", + "input_catalog_id": "ansible:188-momo-backup-user", + "input_execution_mode": "check_mode", + "input_playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml", + "dry_run_result": {"check_mode_executed": True, "apply_executed": False}, + "tags": ["ansible", "check_mode", "controlled_apply_allowed"], + }, + ], + incident={"incident_id": "INC-MOMO", "alertname": "MomoPostgresBackupFailed"}, + drift=None, + ) + + summary = truth["summary"] + assert summary["check_mode_total"] == 1 + assert summary["apply_total"] == 1 + assert summary["applied"] is True + assert summary["controlled_apply"] is True + assert summary["latest_actor"] == "ansible_controlled_apply_worker" + assert summary["latest_execution_mode"] == "controlled_apply" + assert summary["latest_apply_executed"] is True + + def test_ansible_truth_keeps_catalog_hint_separate_from_runtime_use() -> None: truth = build_ansible_truth( [], @@ -1548,6 +1592,16 @@ def test_ansible_apply_receipt_backfill_includes_verifier_and_km_gaps() -> None: assert "_record_post_apply_verifier_and_learning" in source +def test_ansible_live_controlled_apply_sends_telegram_receipt_but_backfill_does_not() -> None: + live_source = inspect.getsource(run_controlled_apply_for_claim) + backfill_source = inspect.getsource(backfill_missing_auto_repair_execution_receipts_once) + + assert "_send_controlled_apply_telegram_receipt" in live_source + assert "telegram_receipt_sent" in live_source + assert "_send_controlled_apply_telegram_receipt" not in backfill_source + assert inspect.iscoroutinefunction(_send_controlled_apply_telegram_receipt) + + def test_ansible_post_apply_verifier_helpers_are_deterministic() -> None: assert _post_apply_km_path_type("03ca6836-1b76-4da2-8e3e-6d3b6df9254a") == ( "ansible_apply_receipt:03ca6836" diff --git a/apps/api/tests/test_telegram_message_templates.py b/apps/api/tests/test_telegram_message_templates.py index 887f6a948..303965020 100644 --- a/apps/api/tests/test_telegram_message_templates.py +++ b/apps/api/tests/test_telegram_message_templates.py @@ -1030,6 +1030,105 @@ def test_awooop_status_chain_lines_show_ansible_apply_proof() -> None: assert ansible["approval_source"] == "user_chat_approved_continue" +@pytest.mark.asyncio +async def test_controlled_apply_result_receipt_marks_callback_reply_evidence(monkeypatch) -> None: + gateway = TelegramGateway() + sent_requests = [] + + async def fake_send_request(method, payload): + sent_requests.append((method, payload)) + return {"ok": True, "result": {"message_id": 12345}} + + async def fake_fetch_truth_chain(source_id, project_id): + assert source_id == "INC-20260627-64472B" + assert project_id == "awoooi" + return { + "truth_status": { + "current_stage": "execution_succeeded", + "stage_status": "success", + "needs_human": False, + "blockers": [], + }, + "automation_quality": { + "verdict": "auto_repaired_verified", + "facts": { + "auto_repair_execution_records": 1, + "automation_operation_records": 3, + "effective_execution_records": 2, + "verification_result": "success", + "mcp_gateway_total": 8, + "knowledge_entries": 1, + }, + "blockers": [], + }, + "execution": { + "automation_operation_log": [ + { + "operation_type": "ansible_apply_executed", + "status": "success", + "actor": "ansible_controlled_apply_worker", + "input_catalog_id": "ansible:188-momo-backup-user", + "input_execution_mode": "controlled_apply", + "input_playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml", + } + ], + "ansible": { + "considered": True, + "records": [ + { + "operation_type": "ansible_apply_executed", + "status": "success", + "actor": "ansible_controlled_apply_worker", + "catalog_id": "ansible:188-momo-backup-user", + "playbook_path": "infra/ansible/playbooks/188-momo-backup-user.yml", + "execution_mode": "controlled_apply", + "apply_executed": True, + "returncode": 0, + "tags": ["ansible", "controlled_apply", "ai_agent_auto_execution"], + } + ], + "candidate_catalog": {"candidates": []}, + }, + }, + } + + monkeypatch.setattr(TelegramGateway, "alert_chat_id", property(lambda _self: "chat")) + monkeypatch.setattr(gateway, "_send_request", fake_send_request) + monkeypatch.setattr( + "src.services.awooop_truth_chain_service.fetch_truth_chain", + fake_fetch_truth_chain, + ) + + result = await gateway.send_controlled_apply_result_receipt( + incident_id="INC-20260627-64472B", + catalog_id="ansible:188-momo-backup-user", + apply_op_id="73b7a95c-3652-4c0d-bb4c-729e500acedb", + playbook_path="infra/ansible/playbooks/188-momo-backup-user.yml", + verification_result="success", + returncode=0, + duration_ms=7727, + verifier_written=True, + learning_written=True, + project_id="awoooi", + ) + + assert result["ok"] is True + assert sent_requests + method, payload = sent_requests[0] + assert method == "sendMessage" + assert "CONTROLLED APPLY RESULT" in payload["text"] + assert "INC-20260627-64472B" in payload["text"] + assert "ansible:188-momo-backup-user" in payload["text"] + source_extra = payload["_awooop_source_envelope_extra"] + assert source_extra["callback_reply"]["action"] == "controlled_apply_result" + assert source_extra["callback_reply"]["status"] == "callback_reply_sent" + assert source_extra["source_refs"]["incident_ids"] == ["INC-20260627-64472B"] + snapshot = source_extra["awooop_status_chain"] + assert snapshot["repair_state"] == "auto_repaired_verified" + assert snapshot["operator_outcome"]["state"] == "completed_verified" + assert snapshot["execution"]["ansible"]["controlled_apply"] is True + + def test_callback_reply_awooop_status_chain_snapshot_marks_manual_gate() -> None: """Callback evidence 要保存當下 AwoooP 狀態鏈，不只保存 live query 結果。""" snapshot = telegram_gateway_module._callback_reply_awooop_status_chain_snapshot(