diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 693d177e0..106ce34b3 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -94,6 +94,9 @@ from src.services.ai_agent_receipt_readback_owner_review import ( from src.services.ai_agent_report_no_write_analysis_runtime import ( load_latest_ai_agent_report_no_write_analysis_runtime, ) +from src.services.host_runaway_aiops_loop_readiness import ( + load_latest_host_runaway_aiops_loop_readiness, +) from src.services.ai_agent_matched_playbook_learning_gap import ( load_latest_ai_agent_matched_playbook_learning_gap, ) @@ -863,6 +866,35 @@ async def get_agent_report_no_write_analysis_runtime() -> dict[str, Any]: ) from exc +@router.get( + "/agent-host-runaway-aiops-loop-readiness", + response_model=dict[str, Any], + summary="取得 P3-009 Host runaway AIOps loop readiness", + description=( + "讀取最新已提交的 P3-009 Host runaway AIOps loop readiness 只讀快照；" + "此端點只呈現 110 runaway process 的監控、告警、AI event packet、PlayBook、KM / Verifier " + "與 gated remediation 合約，不送 Telegram、不寫 Gateway queue、不呼叫 Bot API、不讀 secret、" + "不 kill process、不重啟 Docker / systemd / Nginx、不改 firewall、不執行 kubectl、不寫 production。" + ), +) +async def get_agent_host_runaway_aiops_loop_readiness() -> dict[str, Any]: + """回傳最新 P3-009 Host runaway AIOps loop readiness 只讀快照。""" + try: + payload = await asyncio.to_thread(load_latest_host_runaway_aiops_loop_readiness) + return redact_public_lan_topology(payload) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + logger.error("host_runaway_aiops_loop_readiness_invalid", error=str(exc)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="P3-009 Host runaway AIOps loop readiness 快照無效", + ) from exc + + @router.get( "/agent-communication-learning-contract", response_model=dict[str, Any], diff --git a/apps/api/src/services/host_runaway_aiops_loop_readiness.py b/apps/api/src/services/host_runaway_aiops_loop_readiness.py new file mode 100644 index 000000000..277f26c99 --- /dev/null +++ b/apps/api/src/services/host_runaway_aiops_loop_readiness.py @@ -0,0 +1,313 @@ +""" +Host runaway AIOps loop readiness snapshot. + +This loader exposes a committed, read-only product surface for the 110 CPU +runaway process loop. It validates that monitor, alert, event-packet, PlayBook, +KM / Verifier, and gated remediation contracts are complete while every runtime +write or process termination boundary remains closed. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from src.services.snapshot_paths import default_evaluations_dir + +_DEFAULT_EVALUATIONS_DIR = default_evaluations_dir(Path(__file__)) +_SNAPSHOT_PATTERN = "host_runaway_aiops_loop_readiness_*.json" +_SCHEMA_VERSION = "host_runaway_aiops_loop_readiness_v1" +_RUNTIME_AUTHORITY = "host_runaway_aiops_loop_readiness_only_no_host_write" +_EXPECTED_CURRENT_TASK = "P3-009" +_EXPECTED_NEXT_TASK = "P3-010" +_EXPECTED_ALERT_LANES = { + "HostOrphanBrowserSmokeHighCpu": "orphan_browser_smoke_runaway_process", + "HostCiRunnerLoadSaturation": "ci_runner_load_saturation", +} +_ZERO_ROLLUP_FIELDS = { + "runtime_remediation_authorized_count", + "telegram_send_count", + "gateway_queue_write_count", + "bot_api_call_count", + "host_write_count", + "process_termination_count", + "docker_restart_count", + "systemd_restart_count", + "nginx_reload_count", + "firewall_change_count", + "kubectl_action_count", + "production_write_count", +} +_FALSE_BOUNDARY_FLAGS = { + "runtime_remediation_enabled", + "process_termination_authorized", + "telegram_send_enabled", + "gateway_queue_write_enabled", + "bot_api_call_enabled", + "host_write_enabled", + "docker_restart_enabled", + "systemd_restart_enabled", + "nginx_reload_enabled", + "firewall_change_enabled", + "kubectl_action_enabled", + "production_write_enabled", + "secret_read_enabled", +} +_TRUE_BOUNDARY_FLAGS = { + "read_only_readback_allowed", + "ai_triage_packet_allowed", + "dry_run_generation_allowed", +} +_FORBIDDEN_PUBLIC_TERMS = { + "工作視窗", + "批准！", + "codex_delegation", + "source_thread_id", + "My request for Codex", + "authorization_header", + "secret_value", + "raw secret", + "private key", + "token value", + "chain_of_thought", +} + + +def load_latest_host_runaway_aiops_loop_readiness( + evaluations_dir: Path | None = None, +) -> dict[str, Any]: + """Load the newest committed host runaway AIOps loop readiness snapshot.""" + directory = evaluations_dir or _DEFAULT_EVALUATIONS_DIR + candidates = sorted(directory.glob(_SNAPSHOT_PATTERN)) + if not candidates: + raise FileNotFoundError( + f"no host runaway AIOps loop readiness snapshots found in {directory}" + ) + + latest = candidates[-1] + with latest.open(encoding="utf-8") as handle: + payload = json.load(handle) + + if not isinstance(payload, dict): + raise ValueError(f"{latest}: expected JSON object") + + label = str(latest) + _require_schema(payload, label) + _require_rollups(payload, label) + _require_loop_stages(payload, label) + _require_alert_lanes(payload, label) + _require_asset_writeback_contract(payload, label) + _require_live_readback(payload, label) + _require_remediation_gate(payload, label) + _require_activation_boundaries(payload, label) + _require_no_forbidden_public_terms(payload, label) + return payload + + +def _require_schema(payload: dict[str, Any], label: str) -> None: + if payload.get("schema_version") != _SCHEMA_VERSION: + raise ValueError(f"{label}: expected schema_version={_SCHEMA_VERSION}") + + status = payload.get("program_status") or {} + expected = { + "overall_completion_percent": 100, + "current_priority": "P3", + "current_task_id": _EXPECTED_CURRENT_TASK, + "next_task_id": _EXPECTED_NEXT_TASK, + "read_only_mode": True, + "runtime_authority": _RUNTIME_AUTHORITY, + } + mismatches = _mismatches(status, expected) + if mismatches: + raise ValueError(f"{label}: program_status mismatch: {mismatches}") + if not status.get("status_note"): + raise ValueError(f"{label}: program_status.status_note is required") + + source_refs = payload.get("source_refs") or [] + if len(source_refs) != 7: + raise ValueError(f"{label}: source_refs must contain 7 refs") + + +def _require_rollups(payload: dict[str, Any], label: str) -> None: + rollups = payload.get("rollups") or {} + expected_counts = { + "loop_stage_count": 6, + "alert_lane_count": 2, + "asset_writeback_contract_count": 5, + "source_ref_count": 7, + "live_readback_metric_count": 8, + "blocked_runtime_action_count": 12, + } + mismatches = _mismatches(rollups, expected_counts) + if mismatches: + raise ValueError(f"{label}: rollup counts mismatch: {mismatches}") + + nonzero = { + field: rollups.get(field) + for field in _ZERO_ROLLUP_FIELDS + if rollups.get(field) != 0 + } + if nonzero: + raise ValueError(f"{label}: zero rollup fields must remain 0: {nonzero}") + + +def _require_loop_stages(payload: dict[str, Any], label: str) -> None: + stages = payload.get("loop_stages") or [] + if len(stages) != 6: + raise ValueError(f"{label}: loop_stages must contain 6 stages") + + required_stage_ids = { + "read_only_host_textfile_exporter", + "prometheus_alert_rules", + "telegram_ai_event_packet", + "playbook_contract", + "km_verifier_writeback_contract", + "gated_remediation_helper", + } + stage_ids = {stage.get("stage_id") for stage in stages} + missing = sorted(required_stage_ids - stage_ids) + if missing: + raise ValueError(f"{label}: missing loop stages: {missing}") + + for stage in stages: + stage_id = stage.get("stage_id") or "" + if stage.get("completion_percent") != 100: + raise ValueError(f"{label}: stage {stage_id} must be 100 percent") + for field in ("display_name", "owner_agent", "status", "next_action"): + if not stage.get(field): + raise ValueError(f"{label}: stage {stage_id} missing {field}") + if not stage.get("evidence_refs"): + raise ValueError(f"{label}: stage {stage_id} missing evidence_refs") + if not stage.get("blocked_runtime_actions"): + raise ValueError(f"{label}: stage {stage_id} missing blocked_runtime_actions") + + +def _require_alert_lanes(payload: dict[str, Any], label: str) -> None: + lanes = payload.get("alert_lanes") or [] + if len(lanes) != 2: + raise ValueError(f"{label}: alert_lanes must contain 2 lanes") + + lane_map = {lane.get("alertname"): lane for lane in lanes} + missing = sorted(set(_EXPECTED_ALERT_LANES) - set(lane_map)) + if missing: + raise ValueError(f"{label}: missing alert lanes: {missing}") + + for alertname, expected_lane in _EXPECTED_ALERT_LANES.items(): + lane = lane_map[alertname] + if lane.get("lane_id") != expected_lane: + raise ValueError(f"{label}: {alertname} lane_id mismatch") + if lane.get("runtime_write_gate") != 0: + raise ValueError(f"{label}: {alertname} runtime_write_gate must be 0") + if lane.get("apply_allowed_without_owner_gate") is not False: + raise ValueError(f"{label}: {alertname} apply must require owner gate") + if not lane.get("next_action"): + raise ValueError(f"{label}: {alertname} next_action is required") + + +def _require_asset_writeback_contract(payload: dict[str, Any], label: str) -> None: + contracts = payload.get("asset_writeback_contract") or [] + if len(contracts) != 5: + raise ValueError(f"{label}: asset_writeback_contract must contain 5 contracts") + + for contract in contracts: + asset_id = contract.get("asset_id") or "" + if contract.get("required_on_real_incident") is not True: + raise ValueError(f"{label}: {asset_id} must be required on real incidents") + if contract.get("live_write_enabled") is not False: + raise ValueError(f"{label}: {asset_id} live_write_enabled must remain false") + if not contract.get("required_fields"): + raise ValueError(f"{label}: {asset_id} required_fields must not be empty") + + +def _require_live_readback(payload: dict[str, Any], label: str) -> None: + readback = payload.get("live_readback") or {} + expected = { + "host_label": "110", + "monitor_up": 1, + "orphan_browser_group_count": 0, + "active_ci_container_count": 2, + "remediation_authorized_count": 0, + "alerts_firing_count": 0, + "deploy_marker": "2d278568", + "argocd_sync": "Synced", + "argocd_health": "Healthy", + "production_route_count": 3, + "forbidden_public_hit_count": 0, + } + mismatches = _mismatches(readback, expected) + if mismatches: + raise ValueError(f"{label}: live_readback mismatch: {mismatches}") + if not str(readback.get("runtime_revision", "")).startswith("f358a0f6"): + raise ValueError(f"{label}: live_readback.runtime_revision must start with f358a0f6") + + +def _require_remediation_gate(payload: dict[str, Any], label: str) -> None: + gate = payload.get("remediation_gate") or {} + expected = { + "dry_run_required": True, + "owner_approval_required": True, + "maintenance_window_required": True, + "evidence_ref_required": True, + "post_check_required": True, + "allowed_signal_after_gate": "SIGTERM", + "process_termination_authorized": False, + } + mismatches = _mismatches(gate, expected) + if mismatches: + raise ValueError(f"{label}: remediation_gate mismatch: {mismatches}") + if len(gate.get("disallowed_actions") or []) != 8: + raise ValueError(f"{label}: remediation_gate.disallowed_actions must contain 8 actions") + + +def _require_activation_boundaries(payload: dict[str, Any], label: str) -> None: + boundaries = payload.get("activation_boundaries") or {} + true_mismatches = { + flag: boundaries.get(flag) + for flag in _TRUE_BOUNDARY_FLAGS + if boundaries.get(flag) is not True + } + if true_mismatches: + raise ValueError(f"{label}: true activation boundaries mismatch: {true_mismatches}") + + false_mismatches = { + flag: boundaries.get(flag) + for flag in _FALSE_BOUNDARY_FLAGS + if boundaries.get(flag) is not False + } + if false_mismatches: + raise ValueError(f"{label}: false activation boundaries mismatch: {false_mismatches}") + + for step in payload.get("next_steps") or []: + if step.get("runtime_write_allowed") is not False: + raise ValueError(f"{label}: next_steps runtime_write_allowed must remain false") + + +def _require_no_forbidden_public_terms(payload: Any, label: str) -> None: + hits: list[str] = [] + + def walk(value: Any, path: str) -> None: + if isinstance(value, dict): + for key, nested in value.items(): + walk(nested, f"{path}.{key}" if path else str(key)) + return + if isinstance(value, list): + for index, nested in enumerate(value): + walk(nested, f"{path}[{index}]") + return + if isinstance(value, str): + for term in _FORBIDDEN_PUBLIC_TERMS: + if term.lower() in value.lower(): + hits.append(f"{path}: {term}") + + walk(payload, "") + if hits: + raise ValueError(f"{label}: forbidden public terms: {hits[:5]}") + + +def _mismatches(actual: dict[str, Any], expected: dict[str, Any]) -> dict[str, dict[str, Any]]: + return { + key: {"expected": expected_value, "actual": actual.get(key)} + for key, expected_value in expected.items() + if actual.get(key) != expected_value + } diff --git a/apps/api/tests/test_host_runaway_aiops_loop_readiness.py b/apps/api/tests/test_host_runaway_aiops_loop_readiness.py new file mode 100644 index 000000000..ba56e78a3 --- /dev/null +++ b/apps/api/tests/test_host_runaway_aiops_loop_readiness.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import json + +import pytest + +from src.services.host_runaway_aiops_loop_readiness import ( + load_latest_host_runaway_aiops_loop_readiness, +) + + +def test_load_latest_host_runaway_aiops_loop_readiness_reads_newest_file(tmp_path): + older = _snapshot(generated_at="2026-06-17T00:00:00+08:00") + newer = _snapshot(generated_at="2026-06-18T15:08:00+08:00") + (tmp_path / "host_runaway_aiops_loop_readiness_2026-06-17.json").write_text( + json.dumps(older), + encoding="utf-8", + ) + _write_snapshot(tmp_path, newer) + + loaded = load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + assert loaded["generated_at"] == "2026-06-18T15:08:00+08:00" + assert loaded["program_status"]["current_task_id"] == "P3-009" + assert loaded["program_status"]["next_task_id"] == "P3-010" + assert loaded["rollups"]["loop_stage_count"] == 6 + assert loaded["rollups"]["alert_lane_count"] == 2 + assert loaded["rollups"]["asset_writeback_contract_count"] == 5 + assert loaded["live_readback"]["host_label"] == "110" + assert loaded["live_readback"]["deploy_marker"] == "2d278568" + assert loaded["activation_boundaries"]["runtime_remediation_enabled"] is False + + +def test_host_runaway_aiops_loop_readiness_requires_read_only_mode(tmp_path): + snapshot = _snapshot() + snapshot["program_status"]["read_only_mode"] = False + _write_snapshot(tmp_path, snapshot) + + with pytest.raises(ValueError, match="read_only_mode"): + load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + +def test_host_runaway_aiops_loop_readiness_blocks_process_termination(tmp_path): + snapshot = _snapshot() + snapshot["activation_boundaries"]["process_termination_authorized"] = True + _write_snapshot(tmp_path, snapshot) + + with pytest.raises(ValueError, match="false activation boundaries"): + load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + +def test_host_runaway_aiops_loop_readiness_blocks_host_write_rollup(tmp_path): + snapshot = _snapshot() + snapshot["rollups"]["host_write_count"] = 1 + _write_snapshot(tmp_path, snapshot) + + with pytest.raises(ValueError, match="zero rollup fields"): + load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + +def test_host_runaway_aiops_loop_readiness_requires_alert_lane_contract(tmp_path): + snapshot = _snapshot() + snapshot["alert_lanes"][0]["runtime_write_gate"] = 1 + _write_snapshot(tmp_path, snapshot) + + with pytest.raises(ValueError, match="runtime_write_gate"): + load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + +def test_host_runaway_aiops_loop_readiness_requires_asset_writeback_fields(tmp_path): + snapshot = _snapshot() + snapshot["asset_writeback_contract"][0]["required_fields"] = [] + _write_snapshot(tmp_path, snapshot) + + with pytest.raises(ValueError, match="required_fields"): + load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + +def test_host_runaway_aiops_loop_readiness_rejects_private_terms(tmp_path): + snapshot = _snapshot() + snapshot["loop_stages"][0]["next_action"] = "不要顯示工作視窗內容" + _write_snapshot(tmp_path, snapshot) + + with pytest.raises(ValueError, match="forbidden public terms"): + load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + +def test_host_runaway_aiops_loop_readiness_fails_when_missing(tmp_path): + with pytest.raises(FileNotFoundError): + load_latest_host_runaway_aiops_loop_readiness(tmp_path) + + +def _write_snapshot(tmp_path, payload: dict) -> None: + (tmp_path / "host_runaway_aiops_loop_readiness_2026-06-18.json").write_text( + json.dumps(payload), + encoding="utf-8", + ) + + +def _snapshot(*, generated_at: str = "2026-06-18T15:08:00+08:00") -> dict: + return { + "schema_version": "host_runaway_aiops_loop_readiness_v1", + "generated_at": generated_at, + "program_status": { + "overall_completion_percent": 100, + "current_priority": "P3", + "current_task_id": "P3-009", + "next_task_id": "P3-010", + "read_only_mode": True, + "runtime_authority": "host_runaway_aiops_loop_readiness_only_no_host_write", + "status_note": "只讀 readiness。", + }, + "source_refs": [ + "scripts/ops/host-runaway-process-exporter.py", + "scripts/ops/host-runaway-process-remediation.py", + "ops/monitoring/alerts-unified.yml", + "apps/api/src/services/telegram_gateway.py", + "docs/runbooks/HOST-RUNAWAY-PROCESS-AIOPS-PLAYBOOK.md", + "docs/runbooks/FULL-STACK-COLD-START-SOP.md", + "docs/LOGBOOK.md", + ], + "rollups": { + "loop_stage_count": 6, + "alert_lane_count": 2, + "asset_writeback_contract_count": 5, + "source_ref_count": 7, + "live_readback_metric_count": 8, + "blocked_runtime_action_count": 12, + "runtime_remediation_authorized_count": 0, + "telegram_send_count": 0, + "gateway_queue_write_count": 0, + "bot_api_call_count": 0, + "host_write_count": 0, + "process_termination_count": 0, + "docker_restart_count": 0, + "systemd_restart_count": 0, + "nginx_reload_count": 0, + "firewall_change_count": 0, + "kubectl_action_count": 0, + "production_write_count": 0, + }, + "loop_stages": [ + _stage("read_only_host_textfile_exporter"), + _stage("prometheus_alert_rules"), + _stage("telegram_ai_event_packet"), + _stage("playbook_contract"), + _stage("km_verifier_writeback_contract"), + _stage("gated_remediation_helper"), + ], + "alert_lanes": [ + { + "alertname": "HostOrphanBrowserSmokeHighCpu", + "lane_id": "orphan_browser_smoke_runaway_process", + "classification": "host_resource_runaway_process", + "action_policy": "triage_packet_then_dry_run_then_gated_sigterm", + "dry_run_allowed": True, + "apply_allowed_without_owner_gate": False, + "runtime_write_gate": 0, + "next_action": "prepare packet", + }, + { + "alertname": "HostCiRunnerLoadSaturation", + "lane_id": "ci_runner_load_saturation", + "classification": "host_resource_capacity", + "action_policy": "capacity_triage_no_process_remediation", + "dry_run_allowed": False, + "apply_allowed_without_owner_gate": False, + "runtime_write_gate": 0, + "next_action": "capacity triage", + }, + ], + "asset_writeback_contract": [ + _asset("knowledge_base_incident_summary"), + _asset("playbook_trust_evidence"), + _asset("awooop_work_item_truth_chain"), + _asset("verifier_post_check"), + _asset("recurrence_guard"), + ], + "live_readback": { + "host_label": "110", + "monitor_up": 1, + "orphan_browser_group_count": 0, + "active_ci_container_count": 2, + "load5_per_core_upper_observed": 0.81, + "swap_used_ratio_upper_observed": 1.0, + "remediation_authorized_count": 0, + "alerts_firing_count": 0, + "deploy_marker": "2d278568", + "runtime_revision": "f358a0f6c3e614e407dedb6eee89bf10b2bc8173", + "argocd_sync": "Synced", + "argocd_health": "Healthy", + "production_route_count": 3, + "forbidden_public_hit_count": 0, + }, + "remediation_gate": { + "dry_run_required": True, + "owner_approval_required": True, + "maintenance_window_required": True, + "evidence_ref_required": True, + "post_check_required": True, + "allowed_signal_after_gate": "SIGTERM", + "process_termination_authorized": False, + "disallowed_actions": [ + "SIGKILL", + "docker restart", + "systemctl restart", + "nginx reload", + "firewall change", + "kubectl action", + "secret read", + "production write", + ], + }, + "activation_boundaries": { + "read_only_readback_allowed": True, + "ai_triage_packet_allowed": True, + "dry_run_generation_allowed": True, + "runtime_remediation_enabled": False, + "process_termination_authorized": False, + "telegram_send_enabled": False, + "gateway_queue_write_enabled": False, + "bot_api_call_enabled": False, + "host_write_enabled": False, + "docker_restart_enabled": False, + "systemd_restart_enabled": False, + "nginx_reload_enabled": False, + "firewall_change_enabled": False, + "kubectl_action_enabled": False, + "production_write_enabled": False, + "secret_read_enabled": False, + }, + "next_steps": [ + { + "step_id": "real_alert_receipt_fixture", + "description": "verify packet", + "runtime_write_allowed": False, + } + ], + } + + +def _stage(stage_id: str) -> dict: + return { + "stage_id": stage_id, + "display_name": stage_id, + "owner_agent": "openclaw", + "status": "ready", + "completion_percent": 100, + "evidence_refs": ["docs/LOGBOOK.md"], + "next_action": "keep readback visible", + "blocked_runtime_actions": ["host_write"], + } + + +def _asset(asset_id: str) -> dict: + return { + "asset_id": asset_id, + "display_name": asset_id, + "required_on_real_incident": True, + "live_write_enabled": False, + "required_fields": ["incident_id"], + } diff --git a/apps/api/tests/test_host_runaway_aiops_loop_readiness_api.py b/apps/api/tests/test_host_runaway_aiops_loop_readiness_api.py new file mode 100644 index 000000000..f5ea1eaf0 --- /dev/null +++ b/apps/api/tests/test_host_runaway_aiops_loop_readiness_api.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.v1.agents import router + + +def test_host_runaway_aiops_loop_readiness_endpoint_returns_committed_snapshot(): + app = FastAPI() + app.include_router(router, prefix="/api/v1") + client = TestClient(app) + + response = client.get("/api/v1/agents/agent-host-runaway-aiops-loop-readiness") + + assert response.status_code == 200 + data = response.json() + assert data["schema_version"] == "host_runaway_aiops_loop_readiness_v1" + assert data["program_status"]["current_task_id"] == "P3-009" + assert data["program_status"]["next_task_id"] == "P3-010" + assert data["program_status"]["read_only_mode"] is True + assert data["rollups"]["loop_stage_count"] == len(data["loop_stages"]) == 6 + assert data["rollups"]["alert_lane_count"] == len(data["alert_lanes"]) == 2 + assert data["rollups"]["asset_writeback_contract_count"] == len(data["asset_writeback_contract"]) == 5 + assert data["rollups"]["runtime_remediation_authorized_count"] == 0 + assert data["rollups"]["process_termination_count"] == 0 + assert data["rollups"]["host_write_count"] == 0 + assert data["rollups"]["docker_restart_count"] == 0 + assert data["live_readback"]["host_label"] == "110" + assert data["live_readback"]["monitor_up"] == 1 + assert data["live_readback"]["orphan_browser_group_count"] == 0 + assert data["live_readback"]["remediation_authorized_count"] == 0 + assert data["live_readback"]["deploy_marker"] == "2d278568" + assert data["remediation_gate"]["dry_run_required"] is True + assert data["remediation_gate"]["owner_approval_required"] is True + assert data["remediation_gate"]["process_termination_authorized"] is False + assert data["activation_boundaries"]["runtime_remediation_enabled"] is False + assert data["activation_boundaries"]["process_termination_authorized"] is False + assert data["activation_boundaries"]["host_write_enabled"] is False diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 6c343eb76..9ee54c3ab 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -3747,6 +3747,46 @@ "critical": "critical" } }, + "hostRunawayAiops": { + "title": "P3-009 Host Runaway AIOps 閉環", + "subtitle": "{current} → {next}；閉環階段 {stages}；阻擋 runtime 操作 {blocked}。", + "source": "{generated} · {current} → {next}", + "badges": { + "host": "host {host}", + "deploy": "deploy {marker}", + "runtime": "runtime writes {value}" + }, + "metrics": { + "overall": "完成度", + "stages": "閉環階段", + "alertLanes": "告警 lane", + "assets": "資產回寫", + "liveWrites": "Live writes", + "orphans": "Orphan groups" + }, + "sections": { + "loopStages": "Monitor / Alert / PlayBook / KM 閉環", + "alertLanes": "告警分流 lane", + "boundaries": "仍維持關閉的 runtime 操作" + }, + "labels": { + "owner": "Owner", + "evidence": "證據", + "blockedActions": "阻擋操作", + "boundaryClosed": "保持 false / 0", + "boundaryOpen": "異常開啟", + "monitorUp": "monitor_up={value}", + "ciContainers": "CI containers={count}" + }, + "boundaries": { + "runtime": "Runtime remediation", + "process": "Process termination", + "host": "Host write", + "restart": "Docker / systemd restart", + "network": "Nginx / firewall / kubectl", + "telegram": "Telegram / Gateway / Bot API" + } + }, "overview": { "title": "決策指揮摘要", "mode": "只讀決策支援", diff --git a/apps/web/messages/zh-TW.json b/apps/web/messages/zh-TW.json index 6c343eb76..9ee54c3ab 100644 --- a/apps/web/messages/zh-TW.json +++ b/apps/web/messages/zh-TW.json @@ -3747,6 +3747,46 @@ "critical": "critical" } }, + "hostRunawayAiops": { + "title": "P3-009 Host Runaway AIOps 閉環", + "subtitle": "{current} → {next}；閉環階段 {stages}；阻擋 runtime 操作 {blocked}。", + "source": "{generated} · {current} → {next}", + "badges": { + "host": "host {host}", + "deploy": "deploy {marker}", + "runtime": "runtime writes {value}" + }, + "metrics": { + "overall": "完成度", + "stages": "閉環階段", + "alertLanes": "告警 lane", + "assets": "資產回寫", + "liveWrites": "Live writes", + "orphans": "Orphan groups" + }, + "sections": { + "loopStages": "Monitor / Alert / PlayBook / KM 閉環", + "alertLanes": "告警分流 lane", + "boundaries": "仍維持關閉的 runtime 操作" + }, + "labels": { + "owner": "Owner", + "evidence": "證據", + "blockedActions": "阻擋操作", + "boundaryClosed": "保持 false / 0", + "boundaryOpen": "異常開啟", + "monitorUp": "monitor_up={value}", + "ciContainers": "CI containers={count}" + }, + "boundaries": { + "runtime": "Runtime remediation", + "process": "Process termination", + "host": "Host write", + "restart": "Docker / systemd restart", + "network": "Nginx / firewall / kubectl", + "telegram": "Telegram / Gateway / Bot API" + } + }, "overview": { "title": "決策指揮摘要", "mode": "只讀決策支援", diff --git a/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx b/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx index d91737c4b..c94ce8073 100644 --- a/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx +++ b/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx @@ -48,6 +48,7 @@ import { type AiAgentProfessionalTaskExpansionSnapshot, type AiAgentReceiptReadbackOwnerReviewSnapshot, type AiAgentReportNoWriteAnalysisRuntimeSnapshot, + type HostRunawayAiopsLoopReadinessSnapshot, type AiAgentCandidateOperationDryRunEvidenceSnapshot, type AiAgentCriticReviewerResultCaptureSnapshot, type AiAgentDeploymentLayoutSnapshot, @@ -720,6 +721,7 @@ export function AutomationInventoryTab() { const [professionalTaskExpansion, setProfessionalTaskExpansion] = useState(null) const [receiptReadbackOwnerReview, setReceiptReadbackOwnerReview] = useState(null) const [reportNoWriteAnalysisRuntime, setReportNoWriteAnalysisRuntime] = useState(null) + const [hostRunawayAiops, setHostRunawayAiops] = useState(null) const [proactiveOperations, setProactiveOperations] = useState(null) const [interactionLearningProof, setInteractionLearningProof] = useState(null) const [liveReadModelGate, setLiveReadModelGate] = useState(null) @@ -812,6 +814,7 @@ export function AutomationInventoryTab() { apiClient.getAiAgentProfessionalTaskExpansion(), apiClient.getAiAgentReceiptReadbackOwnerReview(), apiClient.getAiAgentReportNoWriteAnalysisRuntime(), + apiClient.getHostRunawayAiopsLoopReadiness(), apiClient.getAiAgentProactiveOperationsContract(), apiClient.getAiAgentInteractionLearningProof(), apiClient.getAiAgentLiveReadModelGate(), @@ -897,6 +900,7 @@ export function AutomationInventoryTab() { professionalTaskExpansionResult, receiptReadbackOwnerReviewResult, reportNoWriteAnalysisRuntimeResult, + hostRunawayAiopsResult, proactiveOperationsResult, interactionLearningProofResult, liveReadModelGateResult, @@ -979,6 +983,7 @@ export function AutomationInventoryTab() { setProfessionalTaskExpansion(professionalTaskExpansionResult.status === 'fulfilled' ? professionalTaskExpansionResult.value : null) setReceiptReadbackOwnerReview(receiptReadbackOwnerReviewResult.status === 'fulfilled' ? receiptReadbackOwnerReviewResult.value : null) setReportNoWriteAnalysisRuntime(reportNoWriteAnalysisRuntimeResult.status === 'fulfilled' ? reportNoWriteAnalysisRuntimeResult.value : null) + setHostRunawayAiops(hostRunawayAiopsResult.status === 'fulfilled' ? hostRunawayAiopsResult.value : null) setProactiveOperations(proactiveOperationsResult.status === 'fulfilled' ? proactiveOperationsResult.value : null) setInteractionLearningProof(interactionLearningProofResult.status === 'fulfilled' ? interactionLearningProofResult.value : null) setLiveReadModelGate(liveReadModelGateResult.status === 'fulfilled' ? liveReadModelGateResult.value : null) @@ -1063,6 +1068,7 @@ export function AutomationInventoryTab() { professionalTaskExpansionResult, receiptReadbackOwnerReviewResult, reportNoWriteAnalysisRuntimeResult, + hostRunawayAiopsResult, proactiveOperationsResult, interactionLearningProofResult, liveReadModelGateResult, @@ -2512,7 +2518,7 @@ export function AutomationInventoryTab() { ) } - if (error || !snapshot || !backlog || !backupTargets || !backupReadiness || !backupPolicy || !offsiteEscrow || !giteaHealth || !observabilityMatrix || !providerRouteMatrix || !deploymentLayout || !warRoom || !professionalTaskExpansion || !receiptReadbackOwnerReview || !reportNoWriteAnalysisRuntime || !proactiveOperations || !interactionLearningProof || !liveReadModelGate || !redisDryRunGate || !learningWritebackPackage || !telegramReceiptPackage || !ownerApprovedLearningDryRun || !runtimeWriteGateReview || !postWriteVerifierPackage || !runtimeVerifierEvidenceReview || !reportAutomationReview || !reportStatusBoard || !reportRuntimeReadiness || !reportRuntimeDryRun || !reportRuntimeFixtureReadback || !runtimeWorkerShadowGate || !operationPermissionModel || !candidateOperationDryRunEvidence || !taskResultAuditTrail || !matchedPlaybookLearningGap || !criticReviewerResultCapture || !ownerApprovedResultCaptureDryRun || !ownerApprovedResultCaptureReadback || !runtimeReadbackApprovalPackage || !runtimeReadbackImplementationReview || !reportLiveDeliveryApprovalPackage || !runtimeReadbackFixtureApproval || !runtimeReadbackPromotionGate || !ownerApprovedFixturePromotionGate || !canonicalRuntimeReadbackOwnerAcceptance || !failureReceiptNoSendReplay || !reviewerQueueNoWriteReadback || !resultCaptureNoWriteReadback || !resultCapturePromotionApprovalGate || !ownerApprovedResultCapturePromotionDryRun || !resultCaptureWriteGateReview || !resultCaptureWriterImplementationReview || !resultCaptureWriterDryRunFixture || !resultCaptureWriterDryRunReadback || !resultCaptureOwnerPromotionReview || !resultCaptureOwnerApprovedExecutionRehearsal || !resultCaptureOwnerAcceptanceMaintenanceGate || !resultCaptureOwnerAcceptanceReadbackPreflightHold || !resultCaptureOwnerApprovedPreflightReleasePackage || !resultCaptureOwnerApprovedReleaseReadinessReadback || !resultCaptureOwnerReleaseApprovalGate || !resultCapturePostReleaseVerifierRollbackGate || !resultCaptureFinalReleaseCandidateReadback || !resultCaptureReleaseAuthorizationHold || !resultCaptureReleaseAuthorizationReadbackGate || !resultCaptureReleaseVerifierPreflightGate || !resultCaptureReleaseVerifierOwnerReviewPacket || !resultCaptureReleaseDecisionHold || !resultCaptureReleaseDecisionReadback || !resultCaptureReleaseDecisionNextHandoff || !resultCaptureReleaseDecisionInputPrep || !resultCaptureReleaseDecisionOwnerResponsePreflight || !resultCaptureReleaseDecisionOwnerResponseReadback || !resultCaptureReleaseDecisionOwnerResponseAcceptanceGate || !reportTruthActionabilityReview || !ownerDryRunPackage || !hostStatefulInventory || !dependencySupplyChainDriftMonitor || !serviceHealthGapMatrix || !serviceHealthNotificationPolicy) { + if (error || !snapshot || !backlog || !backupTargets || !backupReadiness || !backupPolicy || !offsiteEscrow || !giteaHealth || !observabilityMatrix || !providerRouteMatrix || !deploymentLayout || !warRoom || !professionalTaskExpansion || !receiptReadbackOwnerReview || !reportNoWriteAnalysisRuntime || !hostRunawayAiops || !proactiveOperations || !interactionLearningProof || !liveReadModelGate || !redisDryRunGate || !learningWritebackPackage || !telegramReceiptPackage || !ownerApprovedLearningDryRun || !runtimeWriteGateReview || !postWriteVerifierPackage || !runtimeVerifierEvidenceReview || !reportAutomationReview || !reportStatusBoard || !reportRuntimeReadiness || !reportRuntimeDryRun || !reportRuntimeFixtureReadback || !runtimeWorkerShadowGate || !operationPermissionModel || !candidateOperationDryRunEvidence || !taskResultAuditTrail || !matchedPlaybookLearningGap || !criticReviewerResultCapture || !ownerApprovedResultCaptureDryRun || !ownerApprovedResultCaptureReadback || !runtimeReadbackApprovalPackage || !runtimeReadbackImplementationReview || !reportLiveDeliveryApprovalPackage || !runtimeReadbackFixtureApproval || !runtimeReadbackPromotionGate || !ownerApprovedFixturePromotionGate || !canonicalRuntimeReadbackOwnerAcceptance || !failureReceiptNoSendReplay || !reviewerQueueNoWriteReadback || !resultCaptureNoWriteReadback || !resultCapturePromotionApprovalGate || !ownerApprovedResultCapturePromotionDryRun || !resultCaptureWriteGateReview || !resultCaptureWriterImplementationReview || !resultCaptureWriterDryRunFixture || !resultCaptureWriterDryRunReadback || !resultCaptureOwnerPromotionReview || !resultCaptureOwnerApprovedExecutionRehearsal || !resultCaptureOwnerAcceptanceMaintenanceGate || !resultCaptureOwnerAcceptanceReadbackPreflightHold || !resultCaptureOwnerApprovedPreflightReleasePackage || !resultCaptureOwnerApprovedReleaseReadinessReadback || !resultCaptureOwnerReleaseApprovalGate || !resultCapturePostReleaseVerifierRollbackGate || !resultCaptureFinalReleaseCandidateReadback || !resultCaptureReleaseAuthorizationHold || !resultCaptureReleaseAuthorizationReadbackGate || !resultCaptureReleaseVerifierPreflightGate || !resultCaptureReleaseVerifierOwnerReviewPacket || !resultCaptureReleaseDecisionHold || !resultCaptureReleaseDecisionReadback || !resultCaptureReleaseDecisionNextHandoff || !resultCaptureReleaseDecisionInputPrep || !resultCaptureReleaseDecisionOwnerResponsePreflight || !resultCaptureReleaseDecisionOwnerResponseReadback || !resultCaptureReleaseDecisionOwnerResponseAcceptanceGate || !reportTruthActionabilityReview || !ownerDryRunPackage || !hostStatefulInventory || !dependencySupplyChainDriftMonitor || !serviceHealthGapMatrix || !serviceHealthNotificationPolicy) { return (
@@ -2778,6 +2784,62 @@ export function AutomationInventoryTab() { + reportNoWriteAnalysisRuntime.rollups.host_write_count + reportNoWriteAnalysisRuntime.rollups.kubectl_action_count ) + const hostRunawayOverall = hostRunawayAiops.program_status.overall_completion_percent + const hostRunawayStages = hostRunawayAiops.rollups.loop_stage_count + const hostRunawayAlertLanes = hostRunawayAiops.rollups.alert_lane_count + const hostRunawayAssets = hostRunawayAiops.rollups.asset_writeback_contract_count + const hostRunawayBlocked = hostRunawayAiops.rollups.blocked_runtime_action_count + const hostRunawayLiveWrites = ( + hostRunawayAiops.rollups.runtime_remediation_authorized_count + + hostRunawayAiops.rollups.telegram_send_count + + hostRunawayAiops.rollups.gateway_queue_write_count + + hostRunawayAiops.rollups.bot_api_call_count + + hostRunawayAiops.rollups.host_write_count + + hostRunawayAiops.rollups.process_termination_count + + hostRunawayAiops.rollups.docker_restart_count + + hostRunawayAiops.rollups.systemd_restart_count + + hostRunawayAiops.rollups.nginx_reload_count + + hostRunawayAiops.rollups.firewall_change_count + + hostRunawayAiops.rollups.kubectl_action_count + + hostRunawayAiops.rollups.production_write_count + ) + const hostRunawayBoundaryRows = [ + { + key: 'runtime', + label: t('hostRunawayAiops.boundaries.runtime'), + active: hostRunawayAiops.activation_boundaries.runtime_remediation_enabled, + }, + { + key: 'process', + label: t('hostRunawayAiops.boundaries.process'), + active: hostRunawayAiops.activation_boundaries.process_termination_authorized, + }, + { + key: 'host', + label: t('hostRunawayAiops.boundaries.host'), + active: hostRunawayAiops.activation_boundaries.host_write_enabled, + }, + { + key: 'restart', + label: t('hostRunawayAiops.boundaries.restart'), + active: hostRunawayAiops.activation_boundaries.docker_restart_enabled + || hostRunawayAiops.activation_boundaries.systemd_restart_enabled, + }, + { + key: 'network', + label: t('hostRunawayAiops.boundaries.network'), + active: hostRunawayAiops.activation_boundaries.nginx_reload_enabled + || hostRunawayAiops.activation_boundaries.firewall_change_enabled + || hostRunawayAiops.activation_boundaries.kubectl_action_enabled, + }, + { + key: 'telegram', + label: t('hostRunawayAiops.boundaries.telegram'), + active: hostRunawayAiops.activation_boundaries.telegram_send_enabled + || hostRunawayAiops.activation_boundaries.gateway_queue_write_enabled + || hostRunawayAiops.activation_boundaries.bot_api_call_enabled, + }, + ] const reportRuntimeOverall = reportRuntimeReadiness.program_status.overall_completion_percent const reportRuntimeLanes = reportRuntimeReadiness.rollups.runtime_lane_count const reportRuntimeReady = reportRuntimeReadiness.rollups.ready_contract_count @@ -4762,6 +4824,121 @@ export function AutomationInventoryTab() {
+ +
+
+
+
+ +
+
+ + {t('hostRunawayAiops.title')} + + + {t('hostRunawayAiops.subtitle', { + current: hostRunawayAiops.program_status.current_task_id, + next: hostRunawayAiops.program_status.next_task_id, + stages: hostRunawayStages, + blocked: hostRunawayBlocked, + })} + +
+
+
+ + + +
+
+ +
+ } /> + } /> + } /> + } /> + } /> + } /> +
+ +
+
+ {t('hostRunawayAiops.sections.loopStages')} + {hostRunawayAiops.loop_stages.map(stage => ( +
+
+ + {redactPublicText(stage.display_name)} + + +
+
+ {redactPublicText(stage.next_action)} +
+
+ + + +
+
+ ))} +
+ +
+
+ {t('hostRunawayAiops.sections.alertLanes')} +
+ {hostRunawayAiops.alert_lanes.map(lane => ( + + ))} +
+
+ +
+ {t('hostRunawayAiops.sections.boundaries')} +
+ {hostRunawayBoundaryRows.map(row => ( + + ))} +
+
+ + + +
+
+
+
+
+ +
diff --git a/apps/web/src/lib/api-client.ts b/apps/web/src/lib/api-client.ts index efb988b41..ea65aa121 100644 --- a/apps/web/src/lib/api-client.ts +++ b/apps/web/src/lib/api-client.ts @@ -349,6 +349,11 @@ export const apiClient = { return handleResponse(res) }, + async getHostRunawayAiopsLoopReadiness() { + const res = await fetch(`${API_BASE_URL}/agents/agent-host-runaway-aiops-loop-readiness`) + return handleResponse(res) + }, + async getAiAgentProactiveOperationsContract() { const res = await fetch(`${API_BASE_URL}/agents/agent-proactive-operations-contract`) return handleResponse(res) @@ -2150,6 +2155,117 @@ export interface AiAgentProactiveOperationsContractSnapshot { } } +export interface HostRunawayAiopsLoopReadinessSnapshot { + schema_version: 'host_runaway_aiops_loop_readiness_v1' + generated_at: string + program_status: { + overall_completion_percent: number + current_priority: 'P3' + current_task_id: 'P3-009' + next_task_id: 'P3-010' + read_only_mode: true + runtime_authority: 'host_runaway_aiops_loop_readiness_only_no_host_write' + status_note: string + } + source_refs: string[] + rollups: { + loop_stage_count: number + alert_lane_count: number + asset_writeback_contract_count: number + source_ref_count: number + live_readback_metric_count: number + blocked_runtime_action_count: number + runtime_remediation_authorized_count: number + telegram_send_count: number + gateway_queue_write_count: number + bot_api_call_count: number + host_write_count: number + process_termination_count: number + docker_restart_count: number + systemd_restart_count: number + nginx_reload_count: number + firewall_change_count: number + kubectl_action_count: number + production_write_count: number + } + loop_stages: Array<{ + stage_id: string + display_name: string + owner_agent: 'openclaw' | 'hermes' | 'nemotron' + status: string + completion_percent: number + evidence_refs: string[] + next_action: string + blocked_runtime_actions: string[] + }> + alert_lanes: Array<{ + alertname: 'HostOrphanBrowserSmokeHighCpu' | 'HostCiRunnerLoadSaturation' + lane_id: 'orphan_browser_smoke_runaway_process' | 'ci_runner_load_saturation' + classification: string + action_policy: string + dry_run_allowed: boolean + apply_allowed_without_owner_gate: boolean + runtime_write_gate: number + next_action: string + }> + asset_writeback_contract: Array<{ + asset_id: string + display_name: string + required_on_real_incident: boolean + live_write_enabled: boolean + required_fields: string[] + }> + live_readback: { + host_label: string + monitor_up: number + orphan_browser_group_count: number + active_ci_container_count: number + load5_per_core_upper_observed: number + swap_used_ratio_upper_observed: number + remediation_authorized_count: number + alerts_firing_count: number + deploy_marker: string + runtime_revision: string + argocd_sync: string + argocd_health: string + production_route_count: number + forbidden_public_hit_count: number + } + remediation_gate: { + dry_run_required: boolean + owner_approval_required: boolean + maintenance_window_required: boolean + evidence_ref_required: boolean + post_check_required: boolean + allowed_signal_after_gate: 'SIGTERM' + process_termination_authorized: boolean + disallowed_actions: string[] + } + activation_boundaries: { + read_only_readback_allowed: boolean + ai_triage_packet_allowed: boolean + dry_run_generation_allowed: boolean + runtime_remediation_enabled: boolean + process_termination_authorized: boolean + telegram_send_enabled: boolean + gateway_queue_write_enabled: boolean + bot_api_call_enabled: boolean + host_write_enabled: boolean + docker_restart_enabled: boolean + systemd_restart_enabled: boolean + nginx_reload_enabled: boolean + firewall_change_enabled: boolean + kubectl_action_enabled: boolean + production_write_enabled: boolean + secret_read_enabled: boolean + } + next_steps: Array<{ + step_id: string + description: string + runtime_write_allowed: boolean + }> +} + export interface AiAgentInteractionLearningProofSnapshot { schema_version: 'ai_agent_interaction_learning_proof_v1' generated_at: string diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 38dec3dc7..cebf6c367 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -178,6 +178,27 @@ **下一步**：若未來 `HostOrphanBrowserSmokeHighCpu` 真正 firing，處理順序必須是 AI triage packet -> remediation dry-run evidence -> owner approval / maintenance window / evidence ref -> gated `SIGTERM` -> post-check -> KM / PlayBook / Verifier 回寫；合法 CI load 則走容量 / 排程 / runner queue 分流，不走 process remediation。 +### 2026-06-18 15:08 台北｜P3-009 Host Runaway AIOps 閉環產品面讀模型 + +**背景**：前段已完成 110 runaway process 的 exporter、Prometheus alert、Telegram / AI event packet 與 production deploy readback；但若治理頁沒有呈現 monitor -> alert -> event packet -> PlayBook -> KM / Verifier -> gated remediation 的完整狀態，使用者仍會看到黑盒。此段把閉環變成可讀 API 與 governance 卡片。 + +**完成內容**： +- 新增 `host_runaway_aiops_loop_readiness_v1` committed snapshot 與 schema：`docs/evaluations/host_runaway_aiops_loop_readiness_2026-06-18.json`、`docs/schemas/host_runaway_aiops_loop_readiness_v1.schema.json`。 +- 新增 `apps/api/src/services/host_runaway_aiops_loop_readiness.py`，嚴格驗證 6 個 loop stages、2 條 alert lanes、5 個 asset writeback contracts、110 live readback、remediation gate 與所有 runtime false boundaries。 +- 新增 API：`GET /api/v1/agents/agent-host-runaway-aiops-loop-readiness`；此 endpoint 只讀 committed snapshot，不送 Telegram、不寫 Gateway queue、不呼叫 Bot API、不 kill process、不改 host / Docker / systemd / Nginx / firewall / K8s。 +- `/zh-TW/governance?tab=automation-inventory` 新增 P3-009 卡片，顯示閉環階段 `6`、告警 lane `2`、資產回寫契約 `5`、orphan groups `0`、runtime writes `0`、host `110`、deploy marker `2d278568`。 + +**驗證**： +- `python3 -m json.tool`：host runaway snapshot / schema / zh-TW / en messages 通過。 +- `PYTHONPATH=apps/api python3 -m py_compile apps/api/src/services/host_runaway_aiops_loop_readiness.py apps/api/src/api/v1/agents.py`：通過。 +- `DATABASE_URL=postgresql+asyncpg://ci:ci@localhost/ci PYTHONPATH=apps/api /Users/ogt/.pyenv/shims/python3.11 -m pytest apps/api/tests/test_host_runaway_aiops_loop_readiness.py apps/api/tests/test_host_runaway_aiops_loop_readiness_api.py -q -p no:cacheprovider`：`9 passed`。 +- `pnpm --filter @awoooi/web typecheck`：通過；本 worktree 使用臨時 `node_modules` symlink 到同機既有 AWOOOI 依賴，測完已移除，未提交依賴或 `tsconfig.tsbuildinfo`。 + +**完成度同步**： +- Host runaway AIOps loop product readback：`0% -> 100%`。 +- Monitoring / alert / event packet / PlayBook / KM contract / governance visibility：`100%`。 +- Runtime remediation：仍 `0% / false`；這不是缺件，而是安全 gate。真正 `SIGTERM` 仍需 owner approval、maintenance window、evidence ref、dry-run、post-check 與 KM / PlayBook / Verifier 回寫。 + ## 2026-06-18｜P2-406B Receipt Readback Owner Review 本地完成與正式驗證 **背景**：P2-004 已把依賴 / 供應鏈漂移收斂成只讀監控讀回；統帥要求每次推進都不能忘記目標與方向，因此本段把日報 / 週報 / 月報、Telegram receipt owner review、P2-004 drift monitor 與 P2-403J 報表真相串成同一個 owner review surface，讓治理頁可以直接看到 AI Agent 分工、互審與仍被關閉的 runtime 邊界。 diff --git a/docs/evaluations/host_runaway_aiops_loop_readiness_2026-06-18.json b/docs/evaluations/host_runaway_aiops_loop_readiness_2026-06-18.json new file mode 100644 index 000000000..ed4a403c3 --- /dev/null +++ b/docs/evaluations/host_runaway_aiops_loop_readiness_2026-06-18.json @@ -0,0 +1,290 @@ +{ + "schema_version": "host_runaway_aiops_loop_readiness_v1", + "generated_at": "2026-06-18T15:08:00+08:00", + "program_status": { + "overall_completion_percent": 100, + "current_priority": "P3", + "current_task_id": "P3-009", + "next_task_id": "P3-010", + "read_only_mode": true, + "runtime_authority": "host_runaway_aiops_loop_readiness_only_no_host_write", + "status_note": "110 CPU runaway AIOps loop 已完成監控、告警、AI event packet、PlayBook / KM contract 與 production readback；runtime remediation 仍需 owner approval / maintenance window / evidence ref / dry-run / post-check。" + }, + "source_refs": [ + "scripts/ops/host-runaway-process-exporter.py", + "scripts/ops/host-runaway-process-remediation.py", + "ops/monitoring/alerts-unified.yml", + "apps/api/src/services/telegram_gateway.py", + "docs/runbooks/HOST-RUNAWAY-PROCESS-AIOPS-PLAYBOOK.md", + "docs/runbooks/FULL-STACK-COLD-START-SOP.md", + "docs/LOGBOOK.md" + ], + "rollups": { + "loop_stage_count": 6, + "alert_lane_count": 2, + "asset_writeback_contract_count": 5, + "source_ref_count": 7, + "live_readback_metric_count": 8, + "blocked_runtime_action_count": 12, + "runtime_remediation_authorized_count": 0, + "telegram_send_count": 0, + "gateway_queue_write_count": 0, + "bot_api_call_count": 0, + "host_write_count": 0, + "process_termination_count": 0, + "docker_restart_count": 0, + "systemd_restart_count": 0, + "nginx_reload_count": 0, + "firewall_change_count": 0, + "kubectl_action_count": 0, + "production_write_count": 0 + }, + "loop_stages": [ + { + "stage_id": "read_only_host_textfile_exporter", + "display_name": "110 textfile exporter", + "owner_agent": "openclaw", + "status": "production_readback_complete", + "completion_percent": 100, + "evidence_refs": [ + "scripts/ops/host-runaway-process-exporter.py", + "/home/wooo/node_exporter_textfiles/host_runaway_process.prom" + ], + "next_action": "保持每 2 分鐘 cron scrape，若 monitor missing / stale 則先修 observability，不宣稱 AI Ops 綠燈。", + "blocked_runtime_actions": [ + "process_termination", + "docker_restart" + ] + }, + { + "stage_id": "prometheus_alert_rules", + "display_name": "Prometheus alert lanes", + "owner_agent": "hermes", + "status": "deployed", + "completion_percent": 100, + "evidence_refs": [ + "ops/monitoring/alerts-unified.yml", + "deploy-alerts #3147" + ], + "next_action": "用 HostOrphanBrowserSmokeHighCpu 分流 orphan smoke，用 HostCiRunnerLoadSaturation 分流合法 CI load。", + "blocked_runtime_actions": [ + "auto_silence", + "auto_restart" + ] + }, + { + "stage_id": "telegram_ai_event_packet", + "display_name": "Telegram / AI event packet", + "owner_agent": "hermes", + "status": "production_deployed", + "completion_percent": 100, + "evidence_refs": [ + "apps/api/src/services/telegram_gateway.py", + "f358a0f6", + "2d278568" + ], + "next_action": "告警實際 firing 時產生專屬 lane 與 dry-run / owner / evidence gate，不只輸出泛用 CPU 文本。", + "blocked_runtime_actions": [ + "telegram_direct_send_without_gate", + "bot_api_call_without_receipt" + ] + }, + { + "stage_id": "playbook_contract", + "display_name": "Runaway PlayBook", + "owner_agent": "openclaw", + "status": "ready", + "completion_percent": 100, + "evidence_refs": [ + "docs/runbooks/HOST-RUNAWAY-PROCESS-AIOPS-PLAYBOOK.md", + "docs/runbooks/FULL-STACK-COLD-START-SOP.md" + ], + "next_action": "下一次事故依照 monitor -> alert -> triage -> dry-run -> approval -> post-check 順序執行。", + "blocked_runtime_actions": [ + "skip_dry_run", + "skip_post_check" + ] + }, + { + "stage_id": "km_verifier_writeback_contract", + "display_name": "KM / Verifier writeback contract", + "owner_agent": "nemotron", + "status": "contract_ready", + "completion_percent": 100, + "evidence_refs": [ + "docs/LOGBOOK.md", + "docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md" + ], + "next_action": "真實事故需回寫 KM 條目、PlayBook 信任證據、Verifier post-check 與 recurrence guard。", + "blocked_runtime_actions": [ + "claim_closed_without_km", + "claim_closed_without_verifier" + ] + }, + { + "stage_id": "gated_remediation_helper", + "display_name": "Gated remediation helper", + "owner_agent": "openclaw", + "status": "dry_run_ready_apply_blocked", + "completion_percent": 100, + "evidence_refs": [ + "scripts/ops/host-runaway-process-remediation.py", + "scripts/ops/tests/test_host_runaway_process_exporter.py" + ], + "next_action": "只有在 owner approval、maintenance window、evidence ref、rule、dry-run 與 post-check 都齊備後，才允許最小 SIGTERM。", + "blocked_runtime_actions": [ + "sigkill", + "restart_service" + ] + } + ], + "alert_lanes": [ + { + "alertname": "HostOrphanBrowserSmokeHighCpu", + "lane_id": "orphan_browser_smoke_runaway_process", + "classification": "host_resource_runaway_process", + "action_policy": "triage_packet_then_dry_run_then_gated_sigterm", + "dry_run_allowed": true, + "apply_allowed_without_owner_gate": false, + "runtime_write_gate": 0, + "next_action": "建立 triage packet，列出 process group、CPU、age、source command、dry-run target 與 post-check。" + }, + { + "alertname": "HostCiRunnerLoadSaturation", + "lane_id": "ci_runner_load_saturation", + "classification": "host_resource_capacity", + "action_policy": "capacity_triage_no_process_remediation", + "dry_run_allowed": false, + "apply_allowed_without_owner_gate": false, + "runtime_write_gate": 0, + "next_action": "關聯 Gitea Actions run、runner queue、load/core、swap trend；合法 CI 不走 process kill。" + } + ], + "asset_writeback_contract": [ + { + "asset_id": "knowledge_base_incident_summary", + "display_name": "Knowledge Base incident summary", + "required_on_real_incident": true, + "live_write_enabled": false, + "required_fields": [ + "incident_id", + "root_cause", + "evidence_refs", + "post_check_result" + ] + }, + { + "asset_id": "playbook_trust_evidence", + "display_name": "PlayBook trust evidence", + "required_on_real_incident": true, + "live_write_enabled": false, + "required_fields": [ + "playbook_id", + "matched_rule", + "success_or_failure", + "negative_reinforcement_note" + ] + }, + { + "asset_id": "awooop_work_item_truth_chain", + "display_name": "AwoooP Work Item truth-chain", + "required_on_real_incident": true, + "live_write_enabled": false, + "required_fields": [ + "owner", + "status", + "approval_ref", + "maintenance_window" + ] + }, + { + "asset_id": "verifier_post_check", + "display_name": "Verifier post-check", + "required_on_real_incident": true, + "live_write_enabled": false, + "required_fields": [ + "before_metrics", + "after_metrics", + "alert_state", + "service_health" + ] + }, + { + "asset_id": "recurrence_guard", + "display_name": "Recurrence guard", + "required_on_real_incident": true, + "live_write_enabled": false, + "required_fields": [ + "prometheus_metric", + "alert_lane", + "owner_followup", + "next_review_at" + ] + } + ], + "live_readback": { + "host_label": "110", + "monitor_up": 1, + "orphan_browser_group_count": 0, + "active_ci_container_count": 2, + "load5_per_core_upper_observed": 0.81, + "swap_used_ratio_upper_observed": 1.0, + "remediation_authorized_count": 0, + "alerts_firing_count": 0, + "deploy_marker": "2d278568", + "runtime_revision": "f358a0f6c3e614e407dedb6eee89bf10b2bc8173", + "argocd_sync": "Synced", + "argocd_health": "Healthy", + "production_route_count": 3, + "forbidden_public_hit_count": 0 + }, + "remediation_gate": { + "dry_run_required": true, + "owner_approval_required": true, + "maintenance_window_required": true, + "evidence_ref_required": true, + "post_check_required": true, + "allowed_signal_after_gate": "SIGTERM", + "process_termination_authorized": false, + "disallowed_actions": [ + "SIGKILL", + "docker restart", + "systemctl restart", + "nginx reload", + "firewall change", + "kubectl action", + "secret read", + "production write" + ] + }, + "activation_boundaries": { + "read_only_readback_allowed": true, + "ai_triage_packet_allowed": true, + "dry_run_generation_allowed": true, + "runtime_remediation_enabled": false, + "process_termination_authorized": false, + "telegram_send_enabled": false, + "gateway_queue_write_enabled": false, + "bot_api_call_enabled": false, + "host_write_enabled": false, + "docker_restart_enabled": false, + "systemd_restart_enabled": false, + "nginx_reload_enabled": false, + "firewall_change_enabled": false, + "kubectl_action_enabled": false, + "production_write_enabled": false, + "secret_read_enabled": false + }, + "next_steps": [ + { + "step_id": "real_alert_receipt_fixture", + "description": "下一次真實 firing 或脫敏 fixture 應驗證 Telegram card、AwoooP Work Item、KM / PlayBook / Verifier 欄位一致。", + "runtime_write_allowed": false + }, + { + "step_id": "owner_approved_apply_drill", + "description": "只有在人工批准維護窗口後，才可用非生產 fixture 演練 gated SIGTERM 與 post-check。", + "runtime_write_allowed": false + } + ] +} diff --git a/docs/schemas/host_runaway_aiops_loop_readiness_v1.schema.json b/docs/schemas/host_runaway_aiops_loop_readiness_v1.schema.json new file mode 100644 index 000000000..073b5d3d4 --- /dev/null +++ b/docs/schemas/host_runaway_aiops_loop_readiness_v1.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:awoooi:host-runaway-aiops-loop-readiness-v1", + "title": "Host Runaway AIOps Loop Readiness", + "type": "object", + "required": [ + "schema_version", + "generated_at", + "program_status", + "source_refs", + "rollups", + "loop_stages", + "alert_lanes", + "asset_writeback_contract", + "live_readback", + "remediation_gate", + "activation_boundaries", + "next_steps" + ], + "properties": { + "schema_version": { "const": "host_runaway_aiops_loop_readiness_v1" }, + "generated_at": { "type": "string" }, + "program_status": { + "type": "object", + "required": [ + "overall_completion_percent", + "current_priority", + "current_task_id", + "next_task_id", + "read_only_mode", + "runtime_authority", + "status_note" + ], + "properties": { + "overall_completion_percent": { "type": "integer", "const": 100 }, + "current_priority": { "type": "string", "const": "P3" }, + "current_task_id": { "type": "string", "const": "P3-009" }, + "next_task_id": { "type": "string", "const": "P3-010" }, + "read_only_mode": { "type": "boolean", "const": true }, + "runtime_authority": { "type": "string", "const": "host_runaway_aiops_loop_readiness_only_no_host_write" }, + "status_note": { "type": "string", "minLength": 1 } + } + }, + "source_refs": { "type": "array", "minItems": 7, "items": { "type": "string" } }, + "rollups": { "type": "object" }, + "loop_stages": { "type": "array", "minItems": 6 }, + "alert_lanes": { "type": "array", "minItems": 2 }, + "asset_writeback_contract": { "type": "array", "minItems": 5 }, + "live_readback": { "type": "object" }, + "remediation_gate": { "type": "object" }, + "activation_boundaries": { "type": "object" }, + "next_steps": { "type": "array", "minItems": 1 } + }, + "additionalProperties": true +} diff --git a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md index 5835686c5..00e9c58e8 100644 --- a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md +++ b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md @@ -5159,3 +5159,16 @@ Trigger commit `f5cd37b7` 與 deploy marker `0ba92357` 已把 governance UI 的 - Prometheus 仍讀到 110 runaway exporter：`monitor_up=1`、orphan browser group count `0`、active CI containers `2`、`remediation_authorized=0`，missing / orphan alerts 未 firing。 **裁決：** 110 CPU runaway 的 observe -> classify -> alert -> AI event packet -> PlayBook / KM contract 已經正式站可用。這仍不是 runtime 修復授權：Telegram 實發、Bot API call、Gateway queue write、host write、process kill、Docker / systemd restart、Nginx reload、firewall / K8s action 仍維持 `0 / false`。未來若 alert firing，AI 只能先產生 triage packet、dry-run evidence、owner / maintenance / evidence gate 與 post-check 計畫，再由 gated remediation helper 執行最小 `SIGTERM`。 + +### 2026-06-18 15:08 (台北) — §8 / Host CPU AIOps — P3-009 閉環產品面讀模型 + +**觸發**：前段已完成 110 host runaway 的 monitor / alert / event packet / PlayBook / KM contract 與 production deploy readback；但治理頁仍缺單一卡片呈現完整閉環，會讓自動化產品看起來仍是分散文件與黑盒。 + +**已推進：** +- 新增 `host_runaway_aiops_loop_readiness_v1` committed snapshot 與 JSON schema。 +- 新增 `apps/api/src/services/host_runaway_aiops_loop_readiness.py`，驗證 6 個 loop stages、2 條 alert lanes、5 個 asset writeback contracts、live readback、remediation gate 與所有 runtime false boundaries。 +- 新增 API `GET /api/v1/agents/agent-host-runaway-aiops-loop-readiness`，只讀 committed snapshot，不送 Telegram、不寫 Gateway queue、不呼叫 Bot API、不 kill process、不改 host / Docker / systemd / Nginx / firewall / K8s。 +- `/zh-TW/governance?tab=automation-inventory` 新增 P3-009 卡片，顯示閉環階段、告警 lane、資產回寫、110 live readback、deploy marker、orphan groups 與 runtime writes 0。 +- 精準 API / service tests `9 passed`，web typecheck 通過。 + +**裁決：** P3-009 把 110 CPU runaway automation 從「工程機制已存在」提升成「產品可觀測閉環」。這仍不是 runtime 自動修復啟用；真正 remediation 仍須 owner approval、maintenance window、evidence ref、dry-run、post-check 與 KM / PlayBook / Verifier 回寫。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index aec54f802..e69eebaa0 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -245,6 +245,15 @@ Blocked: No for production alert-to-event packet deployment; yes for runtime rem Next: Future firing alert must produce AI triage packet and dry-run evidence first. `HostCiRunnerLoadSaturation` remains capacity / runner scheduling triage, not process kill. Runtime remediation remains `0` until owner approval, maintenance window, evidence ref, gated SIGTERM, post-check, and KM / PlayBook / Verifier writeback exist. Completion: host runaway monitoring / alert / PlayBook / Telegram event packet / production deploy readback 100%; runtime auto-remediation remains safely gated at 0. +2026-06-18 15:08 Asia/Taipei +Phase: P3-009 Host runaway AIOps loop product readback +Before: Monitoring, alert rules, event packet routing, live scrape, and production deploy readback were complete, but governance UI still lacked a single product-visible loop state for monitor -> alert -> event packet -> PlayBook -> KM / Verifier -> gated remediation. +After: Added `host_runaway_aiops_loop_readiness_v1` committed snapshot, schema, strict API loader, endpoint `/api/v1/agents/agent-host-runaway-aiops-loop-readiness`, regression tests, API client type, and governance automation-inventory card. The card shows 6 loop stages, 2 alert lanes, 5 asset writeback contracts, host 110 live readback, deploy marker 2d278568, orphan groups 0, and runtime writes 0. +Evidence: `apps/api/tests/test_host_runaway_aiops_loop_readiness.py` + API test `9 passed`; web typecheck passed using a temporary existing node_modules symlink that was removed before commit; snapshot/schema/messages JSON parse and py_compile passed. +Blocked: No for product readback; yes for runtime remediation by design. +Next: If a real or fixture alert fires, verify Telegram card, AwoooP Work Item, KM / PlayBook / Verifier fields agree before considering any owner-approved non-production gated SIGTERM drill. +Completion: host runaway AIOps product-visible loop readback 100%; runtime auto-remediation remains safely gated at 0. + 2026-06-18 13:43 Asia/Taipei Phase: P1/P2/P3 live readback Before: live cold-start was `PASS=83 WARN=1 BLOCKED=0`, result `DEGRADED`, because retained stale `km-vectorize-29689620` failed Job evidence was still counted as a service warning.