From 579d12c3fa65b08c37aabde787bc1d60857e6704 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 15:20:23 +0800 Subject: [PATCH] feat(agent99): establish enterprise AI automation work ledger --- AGENT99-AI-WORKSTATION-SOP.md | 11 +- HOST-REBOOT-AI-AUTOMATION-SOP.md | 9 +- apps/api/src/api/v1/agents.py | 37 + ...t99_enterprise_ai_automation_work_items.py | 177 +++++ .../awoooi_priority_work_order_readback.py | 31 + ...enterprise_ai_automation_work_items_api.py | 109 +++ ...ise-ai-automation-work-items.snapshot.json | 747 ++++++++++++++++++ ...oot-cold-start-backup-recovery-workplan.md | 2 + ...r-inserted-requirements-priority-ledger.md | 2 + ...99-enterprise-ai-automation-master-plan.md | 160 ++++ 10 files changed, 1283 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/services/agent99_enterprise_ai_automation_work_items.py create mode 100644 apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py create mode 100644 docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json create mode 100644 docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md diff --git a/AGENT99-AI-WORKSTATION-SOP.md b/AGENT99-AI-WORKSTATION-SOP.md index e4e009b23..b20123aec 100644 --- a/AGENT99-AI-WORKSTATION-SOP.md +++ b/AGENT99-AI-WORKSTATION-SOP.md @@ -1,11 +1,20 @@ # Agent99 AI Workstation SOP -Last updated: 2026-07-09 +Last updated: 2026-07-10 + +Canonical enterprise scope and work order: + +- `docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md` +- `docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` + +This SOP defines how Agent99 operates. The canonical work ledger defines what remains incomplete across all hosts, products, sites, services, tools and packages. A transport or command `exitCode=0` never means an incident is resolved; the mode-specific verifier, source-event resolution and callback receipt are required. ## Objective 192.168.0.99 is the AI Agent workstation and local operations control node. +Host 99 is not the only availability observer. An external watcher and maintenance surface must remain available when 99 or the entire local failure domain is down. + It must act as: - Secretary: reminders, status summaries, evidence packets, operator updates. diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index a319b24bd..23221ed32 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -1,6 +1,13 @@ # Host Reboot AI Automation SOP -Last updated: 2026-07-09 +Last updated: 2026-07-10 + +Canonical enterprise scope and current work order: + +- `docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md` +- `docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` + +This SOP remains the detailed reboot procedure. It cannot mark recovery complete from command success, route HTTP 200, or a Telegram delivery receipt. Completion requires the reboot state machine, all required host/service/product/data gates, mode-specific verifiers and AWOOOI/AwoooP callback readback. ## P0 Goal diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 373441920..c81a0036e 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -35,6 +35,9 @@ from pydantic import BaseModel, Field from src.core.logging import get_logger from src.core.sse import get_publisher +from src.services.agent99_enterprise_ai_automation_work_items import ( + load_latest_agent99_enterprise_ai_automation_work_items, +) from src.services.agent_market_governance_snapshot import ( load_latest_agent_market_governance_snapshot, ) @@ -1133,6 +1136,40 @@ async def get_delivery_closure_workbench() -> dict[str, Any]: ) from exc +@router.get( + "/agent99-enterprise-ai-automation-work-items", + response_model=dict[str, Any], + summary="取得 Agent99 全域 AI 自動化工作總帳", + description=( + "讀取 Agent99 對全部主機、產品、網站、服務、工具、套件與 AI 控制面" + "的唯一工作總帳，包含 current P0、完整排序、完成契約、verifier、rollback " + "與 coverage。此端點只讀 committed snapshot，不連主機、不執行修復、不讀 secret、" + "不呼叫 GitHub，也不觸發 workflow。" + ), +) +async def get_agent99_enterprise_ai_automation_work_items() -> dict[str, Any]: + """回傳 Agent99 全域 AI 自動化工作總帳。""" + try: + payload = await asyncio.to_thread( + load_latest_agent99_enterprise_ai_automation_work_items + ) + return redact_public_lan_topology(payload) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + logger.error( + "agent99_enterprise_ai_automation_work_items_invalid", + error=str(exc), + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Agent99 全域 AI 自動化工作總帳無效", + ) from exc + + @router.get( "/awoooi-priority-work-order-readback", response_model=dict[str, Any], diff --git a/apps/api/src/services/agent99_enterprise_ai_automation_work_items.py b/apps/api/src/services/agent99_enterprise_ai_automation_work_items.py new file mode 100644 index 000000000..fb2a7142f --- /dev/null +++ b/apps/api/src/services/agent99_enterprise_ai_automation_work_items.py @@ -0,0 +1,177 @@ +"""Agent99 enterprise AI-automation work-item readback.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from src.services.snapshot_paths import default_operations_dir + +_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) +_SNAPSHOT_FILE = "agent99-enterprise-ai-automation-work-items.snapshot.json" +_SCHEMA_VERSION = "agent99_enterprise_ai_automation_work_items_v1" +_VALID_PRIORITIES = {"P0", "P1", "P2"} +_VALID_STATUSES = {"planned", "in_progress", "in_progress_blocked", "complete"} + + +def load_latest_agent99_enterprise_ai_automation_work_items( + operations_dir: Path | None = None, +) -> dict[str, Any]: + """Load and validate the canonical Agent99 enterprise work ledger.""" + directory = operations_dir or _DEFAULT_OPERATIONS_DIR + path = directory / _SNAPSHOT_FILE + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + + if not isinstance(payload, dict): + raise ValueError(f"{path}: expected JSON object") + if payload.get("schema_version") != _SCHEMA_VERSION: + raise ValueError(f"{path}: unexpected schema_version") + + coverage = _require_dict(payload, "coverage", path) + hosts = _require_list(coverage, "hosts", path) + products = _require_list(coverage, "products", path) + public_surfaces = _require_list(coverage, "public_surfaces", path) + work_items = _require_list(payload, "work_items", path) + summary = _require_dict(payload, "summary", path) + current_p0 = _require_dict(payload, "current_p0", path) + promotion_contract = _require_dict(payload, "promotion_contract", path) + required_p0_fields = _require_list( + promotion_contract, + "required_item_fields", + path, + ) + + _require_count(summary, "host_count", len(hosts), path) + _require_count(summary, "product_count", len(products), path) + _require_count(summary, "public_surface_count", len(public_surfaces), path) + _require_count(summary, "work_item_count", len(work_items), path) + + ids: list[str] = [] + orders: list[int] = [] + priority_counts = {priority: 0 for priority in _VALID_PRIORITIES} + status_counts = {status: 0 for status in _VALID_STATUSES} + p0_ids: list[str] = [] + + for index, item in enumerate(work_items, start=1): + if not isinstance(item, dict): + raise ValueError(f"{path}: work_items[{index - 1}] must be an object") + item_id = item.get("id") + order = item.get("order") + priority = item.get("priority") + item_status = item.get("status") + if not isinstance(item_id, str) or not item_id: + raise ValueError(f"{path}: work_items[{index - 1}].id is required") + if not isinstance(order, int): + raise ValueError(f"{path}: {item_id}.order must be an integer") + if priority not in _VALID_PRIORITIES: + raise ValueError(f"{path}: {item_id}.priority is invalid") + if item_status not in _VALID_STATUSES: + raise ValueError(f"{path}: {item_id}.status is invalid") + if order != index: + raise ValueError(f"{path}: {item_id}.order must equal {index}") + + ids.append(item_id) + orders.append(order) + priority_counts[priority] += 1 + status_counts[item_status] += 1 + if priority == "P0": + p0_ids.append(item_id) + missing = [field for field in required_p0_fields if item.get(field) is None] + if missing: + raise ValueError(f"{path}: {item_id} missing P0 fields: {missing}") + + if len(ids) != len(set(ids)): + raise ValueError(f"{path}: duplicate work item id") + if orders != list(range(1, len(work_items) + 1)): + raise ValueError(f"{path}: work item order is not contiguous") + + current_p0_id = current_p0.get("id") + if current_p0_id not in p0_ids: + raise ValueError(f"{path}: current_p0.id is not a P0 work item") + if not p0_ids or p0_ids[0] != current_p0_id: + raise ValueError(f"{path}: current_p0 must be the first ordered P0") + if payload.get("next_execution_order") != p0_ids: + raise ValueError(f"{path}: next_execution_order must equal ordered P0 ids") + + _require_count(summary, "p0_count", priority_counts["P0"], path) + _require_count(summary, "p1_count", priority_counts["P1"], path) + _require_count(summary, "p2_count", priority_counts["P2"], path) + _require_count(summary, "planned_count", status_counts["planned"], path) + _require_count( + summary, + "in_progress_count", + status_counts["in_progress"], + path, + ) + _require_count( + summary, + "in_progress_blocked_count", + status_counts["in_progress_blocked"], + path, + ) + _require_count(summary, "complete_count", status_counts["complete"], path) + _require_count(summary, "current_p0_count", 1, path) + + return payload + + +def build_agent99_enterprise_priority_projection( + payload: dict[str, Any], +) -> dict[str, Any]: + """Build the bounded projection embedded in the main priority endpoint.""" + coverage = payload["coverage"] + summary = payload["summary"] + return { + "schema_version": payload["schema_version"], + "generated_at": payload["generated_at"], + "status": payload["status"], + "scope_complete": payload["scope_complete"], + "current_p0": payload["current_p0"], + "coverage": { + "host_count": len(coverage["hosts"]), + "product_count": len(coverage["products"]), + "public_surface_count": len(coverage["public_surfaces"]), + "service_domain_count": len(coverage["service_domains"]), + "package_domain_count": len(coverage["package_domains"]), + }, + "work_item_counts": { + "total": summary["work_item_count"], + "p0": summary["p0_count"], + "p1": summary["p1_count"], + "p2": summary["p2_count"], + "in_progress": summary["in_progress_count"], + "in_progress_blocked": summary["in_progress_blocked_count"], + "planned": summary["planned_count"], + "complete": summary["complete_count"], + }, + "next_execution_order": payload["next_execution_order"], + "full_readback_api": "/api/v1/agents/agent99-enterprise-ai-automation-work-items", + } + + +def _require_dict(payload: dict[str, Any], key: str, path: Path) -> dict[str, Any]: + value = payload.get(key) + if not isinstance(value, dict): + raise ValueError(f"{path}: {key} must be an object") + return value + + +def _require_list(payload: dict[str, Any], key: str, path: Path) -> list[Any]: + value = payload.get(key) + if not isinstance(value, list): + raise ValueError(f"{path}: {key} must be an array") + return value + + +def _require_count( + summary: dict[str, Any], + key: str, + expected: int, + path: Path, +) -> None: + if summary.get(key) != expected: + raise ValueError( + f"{path}: summary.{key}={summary.get(key)!r}, expected {expected}" + ) diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index bbdf3deb7..202631dfb 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -17,6 +17,10 @@ from pathlib import Path from typing import Any from zoneinfo import ZoneInfo +from src.services.agent99_enterprise_ai_automation_work_items import ( + build_agent99_enterprise_priority_projection, + load_latest_agent99_enterprise_ai_automation_work_items, +) from src.services.snapshot_paths import default_operations_dir _DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) @@ -870,11 +874,38 @@ def load_latest_awoooi_priority_work_order_readback( ) _require_mainline_consistency(payload, str(path)) _apply_commander_inserted_requirement_work_items(payload) + _apply_agent99_enterprise_ai_automation_projection(payload, directory) _mark_runtime_generated_at(payload) apply_ai_automation_node_receipts(payload) return payload +def _apply_agent99_enterprise_ai_automation_projection( + payload: dict[str, Any], + operations_dir: Path, +) -> None: + """Attach a bounded projection while keeping fixture-only snapshots usable.""" + try: + enterprise = load_latest_agent99_enterprise_ai_automation_work_items( + operations_dir + ) + except FileNotFoundError: + return + + payload["agent99_enterprise_ai_automation"] = ( + build_agent99_enterprise_priority_projection(enterprise) + ) + source_refs = payload.setdefault("source_refs", {}) + if isinstance(source_refs, dict): + source_refs["agent99_enterprise_ai_automation_work_items"] = ( + "docs/operations/" + "agent99-enterprise-ai-automation-work-items.snapshot.json" + ) + source_refs["agent99_enterprise_ai_automation_work_items_api"] = ( + "/api/v1/agents/agent99-enterprise-ai-automation-work-items" + ) + + def load_controlled_cd_lane_live_metric_readback( metrics_url: str = _CONTROLLED_CD_LANE_NODE_EXPORTER_URL, prometheus_query_url: str = _CONTROLLED_CD_LANE_PROMETHEUS_QUERY_URL, diff --git a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py new file mode 100644 index 000000000..21cecd349 --- /dev/null +++ b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +# ruff: noqa: E402 +import json +import os +from pathlib import Path + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.v1.agents import router +from src.services.agent99_enterprise_ai_automation_work_items import ( + build_agent99_enterprise_priority_projection, + load_latest_agent99_enterprise_ai_automation_work_items, +) +from src.services.awoooi_priority_work_order_readback import ( + load_latest_awoooi_priority_work_order_readback, +) + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_OPERATIONS_DIR = _REPO_ROOT / "docs" / "operations" + + +def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None: + payload = load_latest_agent99_enterprise_ai_automation_work_items() + + assert payload["schema_version"] == ( + "agent99_enterprise_ai_automation_work_items_v1" + ) + assert payload["scope_complete"] is False + assert payload["current_p0"]["id"] == "AG99-P0-001" + assert payload["summary"] == { + "host_count": 7, + "product_count": 12, + "public_surface_count": 23, + "work_item_count": 25, + "p0_count": 14, + "p1_count": 8, + "p2_count": 3, + "in_progress_count": 5, + "in_progress_blocked_count": 3, + "planned_count": 17, + "complete_count": 0, + "current_p0_count": 1, + } + assert payload["next_execution_order"][0] == "AG99-P0-001" + assert payload["next_execution_order"][-1] == "AG99-P0-014" + + +def test_agent99_enterprise_projection_is_bounded() -> None: + payload = load_latest_agent99_enterprise_ai_automation_work_items() + projection = build_agent99_enterprise_priority_projection(payload) + + assert projection["current_p0"]["id"] == "AG99-P0-001" + assert projection["coverage"] == { + "host_count": 7, + "product_count": 12, + "public_surface_count": 23, + "service_domain_count": 12, + "package_domain_count": 7, + } + assert projection["work_item_counts"]["total"] == 25 + assert "work_items" not in projection + assert projection["full_readback_api"] == ( + "/api/v1/agents/agent99-enterprise-ai-automation-work-items" + ) + + +def test_priority_readback_projects_agent99_enterprise_order() -> None: + payload = load_latest_awoooi_priority_work_order_readback() + projection = payload["agent99_enterprise_ai_automation"] + + assert projection["current_p0"]["id"] == "AG99-P0-001" + assert projection["work_item_counts"]["p0"] == 14 + assert ( + payload["source_refs"]["agent99_enterprise_ai_automation_work_items_api"] + == "/api/v1/agents/agent99-enterprise-ai-automation-work-items" + ) + + +def test_agent99_enterprise_loader_rejects_duplicate_ids(tmp_path: Path) -> None: + source = _OPERATIONS_DIR / ( + "agent99-enterprise-ai-automation-work-items.snapshot.json" + ) + payload = json.loads(source.read_text(encoding="utf-8")) + payload["work_items"][1]["id"] = payload["work_items"][0]["id"] + target = tmp_path / source.name + target.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="duplicate work item id"): + load_latest_agent99_enterprise_ai_automation_work_items(tmp_path) + + +def test_agent99_enterprise_work_items_endpoint_returns_snapshot() -> None: + app = FastAPI() + app.include_router(router, prefix="/api/v1/agents") + client = TestClient(app) + + response = client.get("/api/v1/agents/agent99-enterprise-ai-automation-work-items") + + assert response.status_code == 200 + payload = response.json() + assert payload["current_p0"]["id"] == "AG99-P0-001" + assert payload["summary"]["host_count"] == 7 + assert payload["summary"]["product_count"] == 12 + assert payload["summary"]["public_surface_count"] == 23 diff --git a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json new file mode 100644 index 000000000..90694affb --- /dev/null +++ b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json @@ -0,0 +1,747 @@ +{ + "schema_version": "agent99_enterprise_ai_automation_work_items_v1", + "generated_at": "2026-07-10T15:05:00+08:00", + "status": "in_progress_p0_agent_outcome_truth", + "scope_complete": false, + "north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.", + "canonical_sources": { + "human_plan": "docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md", + "historical_requirement_ledger": "docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md", + "agent99_sop": "AGENT99-AI-WORKSTATION-SOP.md", + "reboot_sop": "HOST-REBOOT-AI-AUTOMATION-SOP.md", + "cold_start_baseline": "ops/reboot-recovery/full-stack-cold-start-baseline.yml", + "backup_baseline": "ops/reboot-recovery/full-stack-backup-baseline.yml", + "product_repo_inventory": "docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json", + "runtime_enterprise_work_items_readback": "https://awoooi.wooo.work/api/v1/agents/agent99-enterprise-ai-automation-work-items", + "runtime_priority_readback": "https://awoooi.wooo.work/api/v1/agents/awoooi-priority-work-order-readback", + "runtime_reboot_slo_readback": "https://awoooi.wooo.work/api/v1/agents/reboot-auto-recovery-slo-scorecard" + }, + "current_p0": { + "id": "AG99-P0-001", + "title": "Agent outcome truth and source-runtime reconciliation", + "reason": "Agent99 currently treats command exit success as incident resolution and the Windows runtime scripts do not consistently match the current Gitea source. All later automation depends on trustworthy outcomes.", + "next_action": "Implement mode-specific outcome evaluation, require verifier and source-event resolution for resolved, add regression tests, then deploy and compare source/runtime hashes on host 99." + }, + "current_runtime_truth": { + "observed_at": "2026-07-10T15:04:27+08:00", + "awoooi_health_http_status": 200, + "awoooi_health_state": "degraded", + "reboot_slo_status": "blocked_reboot_auto_recovery_slo_not_ready", + "reboot_slo_readiness_percent": 60, + "reboot_slo_primary_blocker": "reboot_event_required_host_unreachable", + "reboot_slo_active_blockers": [ + "all_required_hosts_not_in_10_minute_reboot_window", + "fresh_all_host_reboot_event_missing", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "reboot_event_required_host_unreachable", + "windows99_vmware_guest_power_not_ready", + "edge_public_maintenance_live_config_drift", + "windows99_vmware_vmx_missing", + "windows99_update_no_auto_reboot_policy_not_ready", + "privileged_edge_apply_channel_unavailable", + "windows99_vmware_autostart_config_not_ready" + ], + "claims_forbidden": [ + "all_services_recovered_within_10_minutes", + "all_products_are_latest", + "agent99_is_a_complete_ai_agent", + "http_200_means_full_stack_healthy", + "command_exit_zero_means_incident_resolved" + ] + }, + "coverage": { + "hosts": [ + { + "id": "99", + "address": "192.168.0.99", + "role": "Windows AI execution and VMware control node", + "required_surfaces": ["Agent99", "VMware Workstation", "vmrun", "Task Scheduler", "PowerShell", "OpenSSH", "Windows Update policy", "desktop control center"] + }, + { + "id": "110", + "address": "192.168.0.110", + "role": "DevOps, SCM, registry, observability and backup node", + "required_surfaces": ["Gitea", "Gitea Actions runner", "Docker", "Harbor", "Registry", "Prometheus", "Alertmanager", "Sentry", "backup"] + }, + { + "id": "111", + "address": "192.168.0.111", + "role": "Required VMware guest with role manifest pending", + "required_surfaces": ["verified VMX inventory", "VM power", "boot identity", "uptime", "SSH", "exporter", "service role manifest"] + }, + { + "id": "112", + "address": "192.168.0.112", + "role": "Kali and IWOOOS security node", + "required_surfaces": ["VM power", "boot identity", "SSH", "security tool health", "IWOOOS evidence collector"] + }, + { + "id": "120", + "address": "192.168.0.120", + "role": "K3s control plane, VIP and AWOOOI workload node", + "required_surfaces": ["K3s", "containerd", "kubectl", "VIP", "AWOOOI deployments", "Velero", "CronJobs"] + }, + { + "id": "121", + "address": "192.168.0.121", + "role": "K3s peer and DR drill node", + "required_surfaces": ["K3s", "containerd", "peer readiness", "workload balance", "Velero restore drill"] + }, + { + "id": "188", + "address": "192.168.0.188", + "role": "Data, AI, observability and web workload node", + "required_surfaces": ["PostgreSQL", "Redis", "MOMO", "Nginx", "OpenClaw", "Ollama", "SignOz", "Langfuse", "AI providers", "backup"] + } + ], + "products": [ + {"id": "awoooi", "gitea_repo": "wooo/awoooi"}, + {"id": "ewoooc", "gitea_repo": "wooo/ewoooc"}, + {"id": "2026fifa", "gitea_repo": "wooo/2026FIFAWorldCup"}, + {"id": "agent-bounty-protocol", "gitea_repo": "wooo/agent-bounty-protocol"}, + {"id": "awooogo", "gitea_repo": "wooo/AwoooGo"}, + {"id": "stockplatform-v2", "gitea_repo": "wooo/stockplatform-v2"}, + {"id": "vibework", "gitea_repo": "wooo/vibework"}, + {"id": "momo-pro-system", "gitea_repo": "wooo/momo-pro-system"}, + {"id": "tsenyang-website", "gitea_repo": "wooo/tsenyang-website"}, + {"id": "vtuber", "gitea_repo": "wooo/vtuber"}, + {"id": "bitan-pharmacy", "gitea_repo": "wooo/bitan-pharmacy"}, + {"id": "clawbot-openclaw", "gitea_repo": "wooo/clawbot-v5"} + ], + "platform_planes": ["AWOOOI", "AwoooP", "IWOOOS", "AISOC", "AgentOps", "OpenClaw", "KM", "RAG", "MCP", "PlayBook"], + "public_surfaces": [ + {"id": "awoooi-api", "url": "https://awoooi.wooo.work/api/v1/health", "owner": "awoooi"}, + {"id": "awoooi-web", "url": "https://awoooi.wooo.work/", "owner": "awoooi"}, + {"id": "iwooos", "url": "https://awoooi.wooo.work/zh-TW/iwooos", "owner": "awoooi"}, + {"id": "vibework", "url": "https://vibework.wooo.work/", "owner": "vibework"}, + {"id": "awooogo", "url": "https://awooogo.wooo.work/", "owner": "awooogo"}, + {"id": "2026fifa", "url": "https://2026fifa.wooo.work/", "owner": "2026fifa"}, + {"id": "agent", "url": "https://agent.wooo.work/", "owner": "ownership_contract_pending"}, + {"id": "momo", "url": "https://mo.wooo.work/", "owner": "momo-pro-system"}, + {"id": "momo-health", "url": "https://mo.wooo.work/health", "owner": "momo-pro-system"}, + {"id": "stock", "url": "https://stock.wooo.work/", "owner": "stockplatform-v2"}, + {"id": "stock-health", "url": "https://stock.wooo.work/healthz", "owner": "stockplatform-v2"}, + {"id": "stock-api", "url": "https://stock.wooo.work/api/healthz", "owner": "stockplatform-v2"}, + {"id": "bitan", "url": "https://bitan.wooo.work/", "owner": "bitan-pharmacy"}, + {"id": "tsenyang", "url": "https://tsenyang.com/", "owner": "tsenyang-website"}, + {"id": "tsenyang-www", "url": "https://www.tsenyang.com/", "owner": "tsenyang-website"}, + {"id": "vtuber", "url": "https://vtuber.wooo.work/", "owner": "vtuber"}, + {"id": "gitea", "url": "https://gitea.wooo.work/", "owner": "platform-110"}, + {"id": "harbor", "url": "https://harbor.wooo.work/", "owner": "platform-110"}, + {"id": "registry", "url": "https://registry.wooo.work/", "owner": "platform-110"}, + {"id": "sentry", "url": "https://sentry.wooo.work/", "owner": "platform-observability"}, + {"id": "signoz", "url": "https://signoz.wooo.work/", "owner": "platform-observability"}, + {"id": "langfuse", "url": "https://langfuse.wooo.work/", "owner": "platform-ai"}, + {"id": "aiops", "url": "https://aiops.wooo.work/", "owner": "ownership_contract_pending"} + ], + "service_domains": [ + "windows_and_vmware", + "network_and_dns", + "linux_systemd_and_schedulers", + "docker_containerd_and_k3s", + "gitea_runner_harbor_registry_and_cd", + "postgresql_redis_and_object_data", + "nginx_tls_edge_and_maintenance", + "prometheus_alertmanager_sentry_signoz_langfuse_and_wazuh", + "openclaw_hermes_nemotron_elephantalpha_ollama_litellm_and_models", + "backup_restic_rclone_velero_offsite_integrity_and_restore", + "km_rag_mcp_playbook_and_agentops", + "telegram_sre_and_operator_command_surfaces" + ], + "package_domains": [ + "windows_updates_and_drivers", + "linux_os_packages_and_kernels", + "container_images_and_digests", + "python_node_and_application_dependencies", + "vmware_tools_and_guest_agents", + "ai_models_adapters_and_provider_clients", + "security_rules_signatures_and_sbom" + ] + }, + "control_plane": [ + {"node": "external_watcher_edge", "responsibility": "Detect host 99 or all-site outage and keep alerts/status/maintenance available outside the local failure domain."}, + {"node": "awoooi", "responsibility": "Canonical events, incidents, fingerprints, recurrence, product and SLO read models."}, + {"node": "awooop", "responsibility": "Work Items, Runs, approvals, candidates, rollback, verifier and asset ledger."}, + {"node": "iwooos_aisoc", "responsibility": "Security evidence, policy, investigation and controlled response."}, + {"node": "agent99", "responsibility": "Evidence collection, planning, policy-bound execution, verification and callbacks for Windows, VMs, hosts, services and sites."}, + {"node": "km_rag_mcp_playbook", "responsibility": "Replayable evidence, knowledge, tools, trusted remediation and learning."}, + {"node": "telegram_sre", "responsibility": "Human-readable command and incident summary surface, never the only database or an arbitrary shell."} + ], + "automation_flow": [ + "sense", + "normalize", + "group_and_dedupe", + "collect_evidence", + "plan", + "policy_decision", + "controlled_execute", + "mode_specific_verify", + "callback_and_notify", + "learn_and_prevent_recurrence" + ], + "outcome_contract": { + "states": ["received", "diagnosing", "candidate_ready", "executing", "verifying", "resolved", "degraded", "blocked", "failed"], + "resolved_requires": ["verifier_passed", "source_event_resolved", "evidence_ref", "verified_at", "callback_written"], + "exit_code_zero_semantics": "command_transport_completed_only", + "observation_window_required_after_resolved": true + }, + "professional_controls": [ + "external_failure_domain_monitoring", + "zero_trust_authenticated_ingress", + "event_sourcing_and_idempotency", + "single_flight_and_concurrency_locks", + "dependency_dag_and_recovery_state_machine", + "mode_specific_verifiers", + "progressive_remediation_and_canary", + "circuit_breakers_timeouts_and_real_rollback", + "immutable_redacted_evidence", + "source_cd_runtime_and_data_freshness_provenance", + "slo_error_budget_mtta_mttr_and_recurrence", + "playbook_replay_shadow_canary_and_trust_promotion", + "sbom_cve_eol_package_and_model_drift", + "backup_integrity_offsite_and_restore_drills", + "human_readable_traditional_chinese_incident_cards" + ], + "hard_boundaries": [ + "secret_token_private_key_cookie_session_or_authorization_header_plaintext_access_or_exfiltration", + "drop_truncate_destructive_migration_restore_prune_or_remote_delete", + "host_reboot_vm_power_change_node_drain_or_irreversible_firewall_cutover", + "credentialed_exploit_or_external_attack_scan", + "paid_provider_cost_change_or_production_provider_route_switch", + "unvalidated_openclaw_core_replacement", + "force_push_repo_ref_visibility_or_source_history_destruction", + "raw_runtime_secret_volume_session_sqlite_or_auth_database_access" + ], + "shared_acceptance_contract": [ + "target_selector_present", + "source_of_truth_diff_present", + "before_evidence_present", + "check_mode_or_dry_run_present", + "rollback_present", + "mode_specific_post_verifier_present", + "timeout_and_single_flight_present", + "callback_and_human_readable_notification_present", + "km_rag_playbook_and_recurrence_writeback_present" + ], + "promotion_contract": { + "applies_before_states": ["candidate_ready", "in_progress", "executing"], + "required_item_fields": [ + "id", + "order", + "priority", + "status", + "title", + "owner_lane", + "dependencies", + "scope_refs", + "problem", + "trigger", + "ai_role", + "action_mode", + "evidence_sources", + "controlled_actions", + "prohibited_actions", + "rollback", + "verifier", + "callback", + "acceptance", + "next_action" + ] + }, + "work_items": [ + { + "id": "AG99-P0-001", + "order": 1, + "priority": "P0", + "status": "in_progress", + "title": "Agent outcome truth and source-runtime reconciliation", + "owner_lane": "agent99_control_plane", + "dependencies": [], + "scope_refs": ["host:99", "platform:Agent99", "platform:AWOOOI", "platform:AwoooP"], + "problem": "Command exit success is counted as resolved and deployed Windows scripts can drift from Gitea source.", + "trigger": "Any Agent99 command completion or source/runtime manifest mismatch.", + "ai_role": "Outcome evaluator and runtime provenance verifier", + "action_mode": "controlled_apply", + "evidence_sources": ["Agent99 event store", "mode evidence JSON", "source file hashes", "host 99 runtime file hashes", "AWOOOI incident state"], + "controlled_actions": ["replace outcome mapping", "add verifier contract", "deploy signed script set", "write runtime manifest"], + "prohibited_actions": ["mark resolved from exit code alone", "read secrets", "reboot host 99"], + "rollback": "Restore previous signed Agent99 script set and keep incidents degraded rather than resolved.", + "verifier": "For every mode, require post-condition pass plus source event resolution; compare source and runtime manifest hashes.", + "callback": "Write final outcome to AWOOOI incident and AwoooP Run/Verifier using the same trace id.", + "acceptance": "No false-resolved regression cases; source/runtime drift count is zero; unresolved evidence stays degraded or blocked.", + "next_action": "Patch agent99-control-plane.ps1 outcome evaluation and add focused regression tests." + }, + { + "id": "AG99-P0-002", + "order": 2, + "priority": "P0", + "status": "planned", + "title": "Alert taxonomy, routing, grouping and dedupe", + "owner_lane": "awoooi_alert_router_agent99_ingress", + "dependencies": ["AG99-P0-001"], + "scope_refs": ["all:alerts", "platform:AWOOOI", "host:99"], + "problem": "Source text contaminates routing and the bridge forwards before grouping, creating wrong modes and duplicate incidents.", + "trigger": "Any normalized alert or Telegram monitoring event entering the AWOOOI-to-Agent99 bridge.", + "ai_role": "Normalized incident classifier", + "action_mode": "controlled_apply", + "evidence_sources": ["Alertmanager payload", "suggestedMode", "event kind", "fingerprint", "five-minute grouping window"], + "controlled_actions": ["prioritize explicit kind and suggested mode", "group before bridge", "single-flight by incident fingerprint"], + "prohibited_actions": ["route from source name alone", "emit duplicate same-state incidents"], + "rollback": "Restore previous router while keeping dedupe store and queued events intact.", + "verifier": "Replay routing corpus and assert one active incident and expected mode per fingerprint.", + "callback": "Publish normalized route decision and dedupe reason to AWOOOI and AwoooP.", + "acceptance": "Known backup, provider, reboot, PostgreSQL, host and K3s samples route to the correct lane with no duplicate same-state dispatch.", + "next_action": "Add routing precedence and pre-bridge grouping regression corpus." + }, + { + "id": "AG99-P0-003", + "order": 3, + "priority": "P0", + "status": "planned", + "title": "Authenticated and policy-preserving Agent99 ingress", + "owner_lane": "agent99_security", + "dependencies": ["AG99-P0-002"], + "scope_refs": ["host:99", "surface:Telegram", "surface:LAN relay", "platform:IWOOOS"], + "problem": "Relay authentication is optional, the LAN is broadly trusted, producer repair policy can be overwritten and Telegram lacks a sender allowlist.", + "trigger": "Any LAN relay, Telegram command or monitoring-event ingress request.", + "ai_role": "Zero-trust command gate", + "action_mode": "controlled_apply", + "evidence_sources": ["relay auth metadata", "Telegram chat and sender metadata", "request policy", "idempotency key"], + "controlled_actions": ["require relay token presence without exposing it", "enforce chat and sender allowlists", "preserve producer auto-repair policy", "single consumer lease"], + "prohibited_actions": ["accept unauthenticated LAN writes", "log token or raw chat id", "force controlled apply at producer"], + "rollback": "Disable remote ingress and retain local signed queue until authentication is healthy.", + "verifier": "Negative auth tests, replay protection, allowlist tests and policy preservation tests.", + "callback": "Record accepted or rejected ingress receipt without secret values.", + "acceptance": "Unauthenticated, replayed or unauthorized requests are rejected; authorized requests preserve source policy and trace identity.", + "next_action": "Make relay authentication mandatory and add Telegram sender authorization." + }, + { + "id": "AG99-P0-004", + "order": 4, + "priority": "P0", + "status": "planned", + "title": "EvidenceCollect, MCP and RepairCandidate lanes", + "owner_lane": "agent99_investigator_awooop_candidate", + "dependencies": ["AG99-P0-001", "AG99-P0-002", "AG99-P0-003"], + "scope_refs": ["all:incidents", "platform:MCP", "platform:AwoooP"], + "problem": "Events with missing evidence are sent to unrelated fixed modes and repair candidates are not materialized or traceable.", + "trigger": "An actionable incident lacks a trusted PlayBook, complete evidence or a safe executable candidate.", + "ai_role": "Evidence investigator and controlled remediation planner", + "action_mode": "observe_then_controlled_apply", + "evidence_sources": ["metrics", "logs", "runtime state", "recent changes", "source and deploy provenance", "MCP references"], + "controlled_actions": ["collect redacted evidence", "create candidate", "attach repair rollback and verifier", "queue owner review when required"], + "prohibited_actions": ["execute generic fallback repair", "invent missing evidence", "switch provider"], + "rollback": "Expire the candidate without runtime action and preserve the evidence packet.", + "verifier": "Incident-to-candidate trace and required-field completeness checker.", + "callback": "Create or update AwoooP Work Item and AWOOOI incident candidate status.", + "acceptance": "Every actionable missing-evidence incident has a candidate or an explicit blocked reason, not a false resolved result.", + "next_action": "Add first-class EvidenceCollect and RepairCandidate modes and state directories." + }, + { + "id": "AG99-P0-005", + "order": 5, + "priority": "P0", + "status": "planned", + "title": "Mode-specific verifiers, rollback, cooldown and circuit breakers", + "owner_lane": "agent99_remediation_runtime", + "dependencies": ["AG99-P0-004"], + "scope_refs": ["all:remediation_modes"], + "problem": "Generic execution success and permissive verifier logic can write cooldown markers after failed repairs; rollback is often only recorded.", + "trigger": "Any remediation mode reaches execution, verification, rollback or cooldown evaluation.", + "ai_role": "Remediation executor and post-condition verifier", + "action_mode": "controlled_apply", + "evidence_sources": ["before state", "command receipt", "post state", "source alert", "observation window"], + "controlled_actions": ["run allowlisted repair", "perform actual rollback", "open circuit on repeated failures", "write cooldown only after verified success"], + "prohibited_actions": ["generic stateful restart", "cooldown on failed verify", "delete healthy pods"], + "rollback": "Mode-specific executable rollback with its own verifier.", + "verifier": "Recover, Perf, LoadShed, Harbor, AWOOOI, backup and provider modes each define independent post-conditions.", + "callback": "Report execution and verification separately to AWOOOI and AwoooP.", + "acceptance": "Failure, rollback and resolved states are distinguishable and regression-tested for every executable mode.", + "next_action": "Replace OR-based load-shed success and add verifier interfaces per mode." + }, + { + "id": "AG99-P0-006", + "order": 6, + "priority": "P0", + "status": "planned", + "title": "Agent99 completion callback and incident closure", + "owner_lane": "agent99_awoooi_awooop_integration", + "dependencies": ["AG99-P0-001", "AG99-P0-005"], + "scope_refs": ["platform:AWOOOI", "platform:AwoooP", "platform:IWOOOS", "surface:Telegram"], + "problem": "AWOOOI-to-Agent99 dispatch is not matched by a canonical Agent99-to-AWOOOI completion path.", + "trigger": "Any Agent99 run changes state or produces a verifier result.", + "ai_role": "Cross-system incident closer", + "action_mode": "controlled_apply", + "evidence_sources": ["incident id", "run id", "work item id", "verifier receipt", "recurrence state"], + "controlled_actions": ["write signed callback", "update Run and Work Item", "render human notification", "reopen on recurrence"], + "prohibited_actions": ["close by Telegram send success", "create unrelated incident ids"], + "rollback": "Queue callback retry idempotently without repeating the repair.", + "verifier": "Read back identical trace, final state and verifier evidence from AWOOOI and AwoooP.", + "callback": "This work item creates the canonical callback contract.", + "acceptance": "One trace is visible from alert receipt through final verifier, notification and recurrence counters.", + "next_action": "Define callback schema and idempotent authenticated endpoint." + }, + { + "id": "AG99-P0-007", + "order": 7, + "priority": "P0", + "status": "in_progress_blocked", + "title": "All-host ten-minute reboot recovery state machine", + "owner_lane": "reboot_auto_recovery", + "dependencies": ["AG99-P0-001", "AG99-P0-005", "AG99-P0-006"], + "scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188", "all:public_surfaces"], + "problem": "Reboot detection and recovery remain one-shot or incomplete, with stale uptime, unreachable host and VM power blockers.", + "trigger": "Fresh boot identity, host down/up transition, Windows startup task or all-host reboot correlation.", + "ai_role": "Dependency-aware reboot recovery orchestrator", + "action_mode": "controlled_apply", + "evidence_sources": ["boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness"], + "controlled_actions": ["detect reboot epoch", "start dependency phases", "retry bounded stateless recovery", "emit ETA and blockers"], + "prohibited_actions": ["blind host reboot", "declare green from route 200", "skip data and backup checks"], + "rollback": "Stop at last verified phase, activate maintenance state and preserve dependent workloads.", + "verifier": "Measure detection time, first availability, full green time and every required host/service/product gate.", + "callback": "Write reboot incident phase and SLO result to AWOOOI, AwoooP and Telegram.", + "acceptance": "A fresh all-host reboot drill is detected and all required gates recover within 600 seconds or report a precise blocker and ETA.", + "next_action": "After AG99-P0-001 through AG99-P0-004, rerun no-secret collector and state-machine scorecard without VM power changes." + }, + { + "id": "AG99-P0-008", + "order": 8, + "priority": "P0", + "status": "in_progress_blocked", + "title": "Windows 99 VMware autostart, VMX, lock and update policy", + "owner_lane": "windows99_vmware", + "dependencies": ["AG99-P0-003", "AG99-P0-005"], + "scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188"], + "problem": "Verified VMX inventory, guest power readback, VMware autostart and Windows no-auto-reboot policy are not all green.", + "trigger": "Windows startup, VMware exclusive-lock error, guest power mismatch, VMX drift or Windows Update policy drift.", + "ai_role": "Windows and VMware control agent", + "action_mode": "check_mode_then_controlled_apply", + "evidence_sources": ["VMX inventory", "vmrun list", "VMware services", "scheduled tasks", "lock owners", "Windows Update policy"], + "controlled_actions": ["repair stale lock after owner proof", "configure autostart order", "verify guest tools and network", "apply reversible update policy"], + "prohibited_actions": ["delete unknown lock", "guess VMX path", "power off VM", "reboot Windows without break-glass"], + "rollback": "Restore previous task, service and policy exports; keep guest power unchanged during check mode.", + "verifier": "All required VMX paths exist, guests report running and reachable, tasks survive reboot, update policy drift is zero.", + "callback": "Update reboot SLO VMware and Windows policy gates.", + "acceptance": "99 startup automatically launches VMware and required guests in order, with no exclusive-lock failure and no surprise Windows Update reboot.", + "next_action": "Complete 111 role/VMX inventory and build the guest-power readback check mode." + }, + { + "id": "AG99-P0-009", + "order": 9, + "priority": "P0", + "status": "in_progress_blocked", + "title": "External outage watcher and maintenance failover", + "owner_lane": "edge_external_observability", + "dependencies": ["AG99-P0-002", "AG99-P0-006"], + "scope_refs": ["host:99", "all:hosts", "all:public_surfaces"], + "problem": "Host 99 cannot detect its own outage and edge fallback has live configuration drift, allowing raw 502 during local recovery.", + "trigger": "External synthetic failure, local recovery incident or raw public 5xx threshold breach.", + "ai_role": "External availability observer and maintenance-state controller", + "action_mode": "controlled_apply", + "evidence_sources": ["external synthetic probes", "edge health", "maintenance headers", "local recovery state"], + "controlled_actions": ["send down and up incidents", "activate pre-provisioned maintenance origin", "withdraw fallback after verifier"], + "prohibited_actions": ["daily DNS mutation", "hide partial outage", "depend on 99 for all-host-down detection"], + "rollback": "Restore previous edge origin and keep status page incident open until public verifier passes.", + "verifier": "External probe sees maintenance content instead of raw 502 during outage and normal routes after recovery.", + "callback": "Correlate external outage with reboot incident and Telegram lifecycle card.", + "acceptance": "Down, recovering and recovered remain observable outside the LAN, and public users never receive unmanaged 502 for a known full-stack recovery.", + "next_action": "Resolve edge live-config drift and define an independent watcher runtime outside 99/110/188." + }, + { + "id": "AG99-P0-010", + "order": 10, + "priority": "P0", + "status": "in_progress", + "title": "All-product version and data freshness matrix", + "owner_lane": "product_runtime_governance", + "dependencies": ["AG99-P0-006"], + "scope_refs": ["all:products", "all:public_surfaces", "all:data_sources"], + "problem": "Repository presence and HTTP 200 do not prove that every product runtime and dataset is current.", + "trigger": "Source change, CD completion, scheduled freshness audit or product stale/degraded alert.", + "ai_role": "Release and freshness auditor", + "action_mode": "observe_then_controlled_apply", + "evidence_sources": ["Gitea main and dev SHA", "Gitea CD run", "deploy marker", "runtime image digest", "public smoke", "data last seen"], + "controlled_actions": ["flag drift", "queue product-specific deploy candidate", "rerun freshness ingestion verifier"], + "prohibited_actions": ["claim latest from repo alone", "force deploy without product verifier", "switch paid provider"], + "rollback": "Use product-specific previous image or release marker and rerun functional/data verifiers.", + "verifier": "One row per product and route with source, CD, runtime, HTTP, feature and data-freshness evidence in the same window.", + "callback": "Project matrix status to AWOOOI product governance and AwoooP Work Items.", + "acceptance": "All 12 product rows and all public surfaces are either verified current or explicitly degraded with owner and next action.", + "next_action": "Generate a fresh matrix and close route ownership_contract_pending rows." + }, + { + "id": "AG99-P0-011", + "order": 11, + "priority": "P0", + "status": "planned", + "title": "Global log registry, anomaly analysis and correlation", + "owner_lane": "aiops_log_intelligence", + "dependencies": ["AG99-P0-002", "AG99-P0-004"], + "scope_refs": ["all:hosts", "all:products", "all:sites", "all:services", "all:tools", "all:packages"], + "problem": "Logs are distributed and noisy; Agent99 lacks a complete freshness-aware source registry and cross-signal causal correlation.", + "trigger": "New log source registration, source freshness breach or incident evidence collection.", + "ai_role": "Log analyst and root-cause correlator", + "action_mode": "observe_then_controlled_apply", + "evidence_sources": ["redacted log metadata", "metrics", "traces", "deploy events", "backup events", "host events"], + "controlled_actions": ["tag and cluster", "identify primary and secondary signals", "create evidence packet", "route to playbook"], + "prohibited_actions": ["ingest secrets", "read raw sessions or SQLite", "auto-execute from untrusted log text"], + "rollback": "Disable faulty adapter or model route while retaining raw-source pointers and normalized receipts.", + "verifier": "Coverage, freshness, redaction, cluster precision and incident linkage readback.", + "callback": "Write cluster, root-cause confidence and evidence refs to incident and KM/RAG.", + "acceptance": "Every registered domain has owner, adapter, freshness and redaction status; primary issue and noise are distinguishable.", + "next_action": "Build metadata-only registry and first adapters for Telegram, reboot, CPU, backup and provider freshness." + }, + { + "id": "AG99-P0-012", + "order": 12, + "priority": "P0", + "status": "in_progress", + "title": "Backup observability, integrity, offsite and restore drills", + "owner_lane": "backup_dr", + "dependencies": ["AG99-P0-006", "AG99-P0-011"], + "scope_refs": ["all:backup_domains", "all:hosts", "all:products"], + "problem": "Backup activity and restore readiness are not consistently visible or understandable across host, DB, site, service, package, tool, log and AI artifacts.", + "trigger": "Backup schedule window, freshness breach, integrity failure, offsite failure or restore drill due date.", + "ai_role": "Backup auditor and safe restore planner", + "action_mode": "observe_then_controlled_apply", + "evidence_sources": ["schedule", "last success", "snapshot metadata", "integrity result", "offsite receipt", "restore drill"], + "controlled_actions": ["rerun non-destructive verifier", "repair stale status marker", "queue isolated restore drill"], + "prohibited_actions": ["read backup contents or secrets", "prune", "restore over production", "delete remote copies"], + "rollback": "Stop verifier or isolated drill without changing source repositories.", + "verifier": "Each baseline domain has schedule, freshness, integrity and isolated restore evidence.", + "callback": "Report one lifecycle incident per failed backup domain and update DR scorecard.", + "acceptance": "No unknown backup target; failures name the domain, age, SLA, owner, evidence and safe next action.", + "next_action": "Reconcile failed readbacks and add missing full Gitea, DB, packages, LFS and tool/log restore receipts." + }, + { + "id": "AG99-P0-013", + "order": 13, + "priority": "P0", + "status": "in_progress", + "title": "All-host performance AIOps and safe load shedding", + "owner_lane": "performance_aiops", + "dependencies": ["AG99-P0-001", "AG99-P0-005", "AG99-P0-011"], + "scope_refs": ["all:hosts", "all:services", "host:110", "host:188"], + "problem": "110 and 188 have recurring pressure, some readbacks are empty, and one-time load drops are treated as closure without causal proof.", + "trigger": "Sustained CPU, memory, disk, IO, queue or database threshold breach, or a performance readback failure.", + "ai_role": "Performance diagnostician and load-shed controller", + "action_mode": "controlled_apply", + "evidence_sources": ["CPU per core", "memory", "disk", "IO", "process and container attribution", "DB queries", "queue", "backup and crawler schedules"], + "controlled_actions": ["throttle allowlisted batch work", "pause duplicate jobs", "apply existing resource limits", "resume after verifier"], + "prohibited_actions": ["kill unknown process", "restart stateful service generically", "emit empty-value alerts"], + "rollback": "Restore prior quotas or resume paused jobs after health and queue checks.", + "verifier": "Sustained observation window plus service SLO, queue, data freshness and process attribution.", + "callback": "Update performance incident and Telegram only on state change or action result.", + "acceptance": "Pressure is attributed, safely reduced and stays within threshold without service or freshness regression.", + "next_action": "Fix perf readback failures and complete 110/188 process, DB and backup/crawler correlation." + }, + { + "id": "AG99-P0-014", + "order": 14, + "priority": "P0", + "status": "in_progress", + "title": "Gitea, Harbor, Registry, Gitea-only CD and complete DR", + "owner_lane": "source_control_registry_cd_dr", + "dependencies": ["AG99-P0-005", "AG99-P0-006", "AG99-P0-012"], + "scope_refs": ["host:110", "all:products", "surface:gitea", "surface:harbor", "surface:registry"], + "problem": "Repository refs are currently present, but complete Gitea state, registry, runner/CD and full restore coverage must be proven continuously.", + "trigger": "SCM, registry or CD health failure; scheduled backup/restore audit; repository visibility complaint; release drift.", + "ai_role": "SCM, registry and release recovery engineer", + "action_mode": "controlled_apply", + "evidence_sources": ["Gitea SSH refs", "main and dev branches", "Gitea full dump", "DB settings issues packages LFS", "Harbor manifest", "runner queue", "deploy marker", "sample restore"], + "controlled_actions": ["repair stateless exporter", "queue bounded runner action", "verify repository and registry", "perform isolated restore dry run"], + "prohibited_actions": ["create replacement repo to hide loss", "change visibility", "force push", "restore production", "use GitHub"], + "rollback": "Restore previous runner/CD config and keep registry/source incident open; never rewrite repository history.", + "verifier": "All expected repos and refs, full dump components, registry artifacts, CD run and runtime markers read back successfully.", + "callback": "Update source-control, release and DR scorecards with one trace.", + "acceptance": "All 12 products have source and release proof and the platform can be reconstructed in isolation from verified backups.", + "next_action": "Add full Gitea state and registry restore receipts to the existing repo-bundle proof." + }, + { + "id": "AG99-P1-001", + "order": 15, + "priority": "P1", + "status": "planned", + "title": "Model-backed planner, reviewer and tool loop", + "owner_lane": "agent_ai_runtime", + "dependencies": ["AG99-P0-004", "AG99-P0-005"], + "scope_refs": ["platform:OpenClaw", "platform:Hermes", "platform:NemoTron", "platform:ElephantAlpha", "platform:Ollama"], + "action_mode": "shadow_then_canary", + "verifier": "Replay, shadow agreement, unsafe-action rejection, canary success and cost/latency limits.", + "acceptance": "The planner emits structured allowlisted tool plans with evidence, risk, rollback and verifier, and cannot bypass policy.", + "next_action": "Define planner tool schema and replay corpus from verified incidents." + }, + { + "id": "AG99-P1-002", + "order": 16, + "priority": "P1", + "status": "planned", + "title": "PlayBook trust, promotion and recurrence prevention", + "owner_lane": "knowledge_governance", + "dependencies": ["AG99-P0-006", "AG99-P1-001"], + "scope_refs": ["platform:KM", "platform:RAG", "platform:MCP", "platform:PlayBook"], + "action_mode": "controlled_apply", + "verifier": "Trust score is derived from replay, execution, rollback, recurrence and human override receipts.", + "acceptance": "Repeated incidents promote verified automation or create a permanent-fix work item; failed playbooks are demoted automatically.", + "next_action": "Define trust and promotion schema with minimum replay/shadow/canary evidence." + }, + { + "id": "AG99-P1-003", + "order": 17, + "priority": "P1", + "status": "planned", + "title": "Human-readable Traditional Chinese Telegram incident cards", + "owner_lane": "telegram_incident_experience", + "dependencies": ["AG99-P0-006"], + "scope_refs": ["surface:Telegram", "all:incidents"], + "action_mode": "controlled_apply", + "verifier": "Message snapshot tests, duplicate suppression, state-change delivery and evidence-link checks.", + "acceptance": "Every card explains impact, AI action, verified result and next step in Traditional Chinese; charts or screenshots are attached when they improve understanding.", + "next_action": "Replace raw technical completion messages with one lifecycle card per incident state change." + }, + { + "id": "AG99-P1-004", + "order": 18, + "priority": "P1", + "status": "planned", + "title": "Agent99 desktop, Telegram and API command surfaces", + "owner_lane": "agent99_operator_experience", + "dependencies": ["AG99-P0-003", "AG99-P0-006"], + "scope_refs": ["host:99", "surface:Telegram", "surface:Agent99 desktop"], + "action_mode": "controlled_apply", + "verifier": "The same request has identical policy, trace, execution and result across all command surfaces.", + "acceptance": "Operators can issue natural-language tasks without Codex quota and cannot escape allowlisted tools or audit.", + "next_action": "Unify Submit Request, Telegram and API behind the authenticated task schema." + }, + { + "id": "AG99-P1-005", + "order": 19, + "priority": "P1", + "status": "planned", + "title": "AWOOOI, AwoooP and IWOOOS product integration", + "owner_lane": "platform_product_integration", + "dependencies": ["AG99-P0-006", "AG99-P1-002"], + "scope_refs": ["platform:AWOOOI", "platform:AwoooP", "platform:IWOOOS", "platform:AISOC"], + "action_mode": "controlled_apply", + "verifier": "Runs, Work Items, Incidents, Observability and Knowledge show the same asset ids, owners, states and next actions.", + "acceptance": "No remediation exists only in Telegram or Windows files; all lifecycle assets are visible and queryable in the products.", + "next_action": "Add shared incident/run/work-item/asset projections and deep links." + }, + { + "id": "AG99-P1-006", + "order": 20, + "priority": "P1", + "status": "planned", + "title": "Tool, package, image and model lifecycle governance", + "owner_lane": "supply_chain_governance", + "dependencies": ["AG99-P0-010", "AG99-P0-014"], + "scope_refs": ["all:tools", "all:packages", "all:container_images", "all:ai_models"], + "action_mode": "observe_then_controlled_apply", + "verifier": "Version, digest, SBOM, CVE, EOL, compatibility, rollback and runtime readback per managed artifact.", + "acceptance": "Drift and vulnerable or unsupported artifacts are visible, prioritized and updated only through canary and rollback-aware plans.", + "next_action": "Build package-domain inventory and freshness policy without automatic major upgrades." + }, + { + "id": "AG99-P1-007", + "order": 21, + "priority": "P1", + "status": "planned", + "title": "AISOC security triage and controlled response", + "owner_lane": "iwooos_aisoc", + "dependencies": ["AG99-P0-003", "AG99-P0-011", "AG99-P1-001"], + "scope_refs": ["host:112", "platform:IWOOOS", "platform:AISOC", "all:security_signals"], + "action_mode": "observe_then_controlled_apply", + "verifier": "Evidence integrity, rule/model agreement, containment scope, rollback and service impact checks.", + "acceptance": "Security incidents receive evidence-backed triage and safe containment candidates without autonomous attack scans or credential actions.", + "next_action": "Map IWOOOS/Wazuh signal taxonomy to the shared incident and candidate schema." + }, + { + "id": "AG99-P1-008", + "order": 22, + "priority": "P1", + "status": "planned", + "title": "Automation metrics, incident reporting and permanent fixes", + "owner_lane": "agentops_reporting", + "dependencies": ["AG99-P0-006", "AG99-P1-002", "AG99-P1-005"], + "scope_refs": ["all:incidents", "all:work_items", "platform:AgentOps"], + "action_mode": "observe", + "verifier": "Counts reconcile with event and verifier receipts; reports distinguish current, resolved, failed, avoided and recurrent problems.", + "acceptance": "Daily, incident and weekly reports expose MTTA, MTTR, SLO, recurrence, false positives, coverage and human intervention.", + "next_action": "Define counters and report schema from canonical event and verifier data." + }, + { + "id": "AG99-P2-001", + "order": 23, + "priority": "P2", + "status": "planned", + "title": "Cold-start, restore and chaos game days", + "owner_lane": "resilience_engineering", + "dependencies": ["AG99-P0-007", "AG99-P0-012", "AG99-P1-008"], + "scope_refs": ["all:hosts", "all:products", "all:backup_domains"], + "action_mode": "isolated_then_controlled_apply", + "verifier": "Preflight, abort criteria, SLO timing, rollback and post-drill data integrity.", + "acceptance": "Recovery claims are proven by scheduled isolated drills and explicitly approved production game days.", + "next_action": "Create isolated drill calendar and evidence templates." + }, + { + "id": "AG99-P2-002", + "order": 24, + "priority": "P2", + "status": "planned", + "title": "Capacity, cost and model quality optimization", + "owner_lane": "capacity_finops_agentops", + "dependencies": ["AG99-P0-013", "AG99-P1-001", "AG99-P1-008"], + "scope_refs": ["all:hosts", "all:models", "all:automation_lanes"], + "action_mode": "observe_then_controlled_apply", + "verifier": "Capacity headroom, service SLO, automation success, model quality, latency and approved cost limits.", + "acceptance": "Resource and model routing changes improve measured outcomes without hidden provider or cost changes.", + "next_action": "Establish baselines before any optimization action." + }, + { + "id": "AG99-P2-003", + "order": 25, + "priority": "P2", + "status": "planned", + "title": "Controlled GUI, VM Console and browser automation", + "owner_lane": "agent99_desktop_automation", + "dependencies": ["AG99-P0-003", "AG99-P0-008", "AG99-P1-004"], + "scope_refs": ["host:99", "surface:VMware Console", "surface:Windows desktop", "surface:browser"], + "action_mode": "controlled_apply", + "verifier": "App identity, allowed action, before/after screenshot, target-state readback and rollback.", + "acceptance": "GUI automation is used only where no reliable API or CLI exists and remains fully policy-bound and auditable.", + "next_action": "Define an allowlisted desktop tool catalog and screenshot evidence contract." + } + ], + "summary": { + "host_count": 7, + "product_count": 12, + "public_surface_count": 23, + "work_item_count": 25, + "p0_count": 14, + "p1_count": 8, + "p2_count": 3, + "in_progress_count": 5, + "in_progress_blocked_count": 3, + "planned_count": 17, + "complete_count": 0, + "current_p0_count": 1 + }, + "next_execution_order": [ + "AG99-P0-001", + "AG99-P0-002", + "AG99-P0-003", + "AG99-P0-004", + "AG99-P0-005", + "AG99-P0-006", + "AG99-P0-007", + "AG99-P0-008", + "AG99-P0-009", + "AG99-P0-010", + "AG99-P0-011", + "AG99-P0-012", + "AG99-P0-013", + "AG99-P0-014" + ] +} diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 0f91f32c9..5e2789810 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -1,5 +1,7 @@ # 2026-06-04 Reboot / Cold-Start / Backup Recovery Workplan +> **2026-07-10 執行入口：** 本文件保留 cold-start、backup 與事故經驗細節；目前工作順序、Agent99 控制面、全產品/網站/工具覆蓋與完成契約，以 `docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md` 及其機器可讀總帳為準。重啟恢復仍是 P0，但必須先修正 Agent99 假 resolved、routing/dedupe、secure ingress 與 verifier/callback，否則不能信任自動結案。 + > Owner: SRE / DevOps commander > Timezone: Asia/Taipei > Baseline: 2026-06-04 15:00 live read-only checks. Do not reuse the 2026-05-29 baseline without rerunning checks. diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 42ff4fb8a..91fc54403 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -1,5 +1,7 @@ # 統帥插入需求主線優先序台帳 +> **2026-07-10 排序更新：** 本文件保留為完整需求與歷史證據來源；目前唯一執行順序、全主機/產品/網站/服務/工具/套件覆蓋與 Agent99 AI 自動化完成契約，改由 `docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md` 與 `docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` 管理。下方舊狀態不得覆蓋新總帳，也不得再以歷史 `Done` 宣稱目前 runtime 完成。 + > 建立時間：2026-07-02 13:04 CST > 範圍：本輪 AWOOOI / Gitea / Product Governance 工作中，使用者在主線推進期間插入的要求。 > 目的：把插入要求轉成可排序、可驗證、可接續的工作項目，避免後續 Codex 把它們當成聊天雜訊或另開支線。 diff --git a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md new file mode 100644 index 000000000..a7ec66904 --- /dev/null +++ b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md @@ -0,0 +1,160 @@ +# Agent99 全域 AI 自動化主計畫 + +> 建立時間：2026-07-10 15:05 CST +> 狀態：P0 執行中，尚未達成全域 AI 自動化 +> 機器可讀總帳：`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` +> 歷史需求來源：`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md` + +## 1. 北極星與完成定義 + +192.168.0.99 是內網 AI 執行與協調節點。它必須能接收 AWOOOI、AwoooP、IWOOOS、Telegram、主機、VM、服務與網站訊號，完成診斷、受控修復、驗證、回報及知識沉澱；不能只做排程腳本、固定 regex 分流或告警轉送。 + +「完整操控」的專業定義是能力範圍完整、政策受控且全程可稽核，不是任意遠端 shell： + +- 低風險、可回滾、已有 verifier 的動作由 Agent99 自動執行。 +- 中風險動作先產生 candidate、check-mode、rollback 與 verifier，再由 controlled apply 執行。 +- secret 明文、破壞性 DB、主機 reboot、VM power change、firewall/DNS/provider 切換等事故級動作維持 break-glass。 +- 任何 `exitCode=0` 只表示命令完成，不能代表問題已解決；只有目標狀態 verifier 通過且事件來源回讀 resolved，才能標記 `resolved`。 +- 99 關機、LAN 中斷或全站失聯時，由獨立外部 watcher 告警並提供 L1 維護頁；不得讓 99 成為自己唯一的監視器。 + +全域完成必須同時符合： + +1. 7 台主機、12 個產品、全部公開路由、基礎服務、工具與套件都有 inventory、owner、freshness 與 verifier。 +2. 每個可處置告警都有 incident、evidence、candidate、policy、execution、rollback、verifier、callback 與 learning receipt。 +3. 所有產品都有 Gitea source SHA、CD run、deploy marker、runtime image/version、public route 與資料 freshness 的同一時間窗證據。 +4. 全主機重啟可被自動偵測，依相依順序恢復，並以首次可用時間與完整綠燈時間證明 10 分鐘 SLO；未達標時自動報 blocker 與 ETA。 +5. Telegram 只送人看得懂的繁體中文事件卡：發生什麼、影響、Agent 做了什麼、結果、下一步；技術證據放按鈕或連結，適合時附圖表或截圖。 +6. 相同問題的發生、解決、失敗與避免次數可查；重複問題必須提升 PlayBook trust 或產生永久修復工作項。 + +## 2. 覆蓋範圍 + +### 2.1 主機與 VM + +| 主機 | 角色與必要能力 | Agent99 責任 | +| --- | --- | --- | +| 192.168.0.99 | Windows、VMware Workstation、Task Scheduler、PowerShell/OpenSSH、Agent99 | Windows/VMware 狀態、VM inventory/autostart、Windows Update 政策、桌面與命令入口、全程 audit | +| 192.168.0.110 | Gitea、Gitea Actions runner、Docker、Harbor/Registry、Prometheus、Alertmanager、Sentry、backup | Source/CD/registry/observability 恢復、CPU/queue 降載、備份與 restore verifier | +| 192.168.0.111 | 必要 VMware guest；目前角色與 verified VMX inventory 未完整 | 補 role manifest、VMX、boot/uptime/SSH/exporter、autostart 與服務 verifier | +| 192.168.0.112 | Kali / IWOOOS 安全節點 | 開機、SSH、安全工具健康、受控調查與 AISOC evidence；禁止外部攻擊型掃描自動化 | +| 192.168.0.120 | K3s control plane、VIP、AWOOOI workloads | node/pod/rollout/image-pull/cron/Velero 恢復與 verifier | +| 192.168.0.121 | K3s peer、DR drill | peer readiness、workload balance、DR/Velero drill 與 verifier | +| 192.168.0.188 | PostgreSQL、Redis、MOMO、AI/observability/web workloads | data/AI/provider/SignOz/OpenClaw/Ollama/Nginx 健康、效能與備份修復 | + +### 2.2 產品與原始碼 + +目前 Gitea inventory 的 12 個產品全部納入：`awoooi`、`ewoooc`、`2026fifa`、`agent-bounty-protocol`、`awooogo`、`stockplatform-v2`、`vibework`、`momo-pro-system`、`tsenyang-website`、`vtuber`、`bitan-pharmacy`、`clawbot-openclaw`。AWOOOI 內的 AwoooP、IWOOOS、AgentOps、AISOC、KM/RAG/MCP 與 OpenClaw 是平台控制面，也必須納入版本、runtime、告警與回寫契約。 + +### 2.3 公開網站與 API + +以 cold-start baseline 為最低完整清單：AWOOOI web/API/IWOOOS、VibeWork、AwoooGo、2026 FIFA、Agent、MOMO web/health、StockPlatform web/health/API、碧潭大藥局、天揚、VTuber、Gitea、Harbor、Registry、Sentry、SignOz、Langfuse、AIOps。每列都要有 owner product、expected status、L0 maintenance、L1 external fallback、TLS、runtime version 與資料 freshness；不能只看 HTTP 200。 + +### 2.4 服務、工具、套件與資料 + +- Runtime：Windows services、VMware、systemd、Docker、containerd、K3s、Nginx、PostgreSQL、Redis、MinIO/物件資料、CronJob/Task Scheduler。 +- DevSecOps：Gitea、Actions runner、Harbor/Registry、ArgoCD/部署標記、Ansible、Velero、restic/rclone、SBOM 與 package inventory。 +- Observability：Prometheus、Alertmanager、node exporter、cAdvisor、Sentry、SignOz、Langfuse、Wazuh/IWOOOS、Telegram receipt。 +- AI：Agent99、OpenClaw、Hermes、NemoTron、ElephantAlpha、Ollama、LiteLLM、MCP、KM、RAG、PlayBook、模型與 provider freshness。 +- 資料與備份：主機設定、DB、repo、registry、網站、套件清單、工具設定、logs、AI artifacts、K3s/Velero、offsite、integrity check 與 restore drill。 + +## 3. 目標控制面 + +| 節點 | 權責 | +| --- | --- | +| 外部 watcher / edge | 99 或全內網失聯時仍可產生 down/up 告警、狀態頁與 L1 維護頁；不依賴 99/110/188 | +| AWOOOI | 事件與 incident source of truth、alert normalization、fingerprint、recurrence、產品與 SLO read model | +| AwoooP | Work Item、Run、approval、candidate、rollback、verifier、資產與 owner 總帳 | +| IWOOOS / AISOC | 安全事件 evidence、policy、調查與受控 response；高風險行為不得繞過安全邊界 | +| Agent99 | 99/VM/host/service/website 的 evidence collector、planner executor、controlled remediation 與 callback client | +| KM/RAG/MCP/PlayBook | 可重播 evidence、知識、工具介面、已驗證修復與 trust/recurrence 統計 | +| Telegram SRE 戰情室 | 人類命令與事件摘要介面，不是資料庫、任意 shell 或唯一事件來源 | + +## 4. 標準 AI 閉環 + +所有告警與指令必須走同一條鏈： + +1. **Sense**：外部 watcher、Alertmanager、Sentry/SignOz、Windows/VMware、host/exporter、logs、backup、public synthetic 收件。 +2. **Normalize**：產生 `event_id`、`incident_id`、fingerprint、source、target、severity、product、host、service、first_seen/last_seen。 +3. **Group/Dedupe**：先做 5 分鐘 grouping、single-flight 與同狀態抑噪，再交給 Agent99；狀態改變、修復完成與 SLO breach 不抑制。 +4. **Collect Evidence**：執行模式專屬 MCP/readback；收 metrics、logs、最近變更、source/runtime/version/freshness 與 dependency state。 +5. **Plan**：規則 PlayBook 優先；AI planner 只產出 allowlisted tool calls、風險、check-mode、rollback、verifier 與 confidence。 +6. **Policy**：依動作與 blast radius 判斷 observe、auto-apply、controlled apply 或 break-glass，不可由 producer 自行硬設 `controlledApply=true`。 +7. **Execute**：idempotency key、single-flight、timeout、circuit breaker、before evidence、限定 target selector。 +8. **Verify**：模式專屬 post-condition、來源事件 resolved、產品/資料 freshness 與回歸 smoke；失敗時執行真正 rollback 或明確升級。 +9. **Write Back**：Agent99 回寫 AWOOOI incident、AwoooP Run/Work Item/Verifier、IWOOOS evidence 與 Telegram 人類事件卡。 +10. **Learn**：更新 occurrence/resolved/failed/avoided 計數、MTTA/MTTR、KM/RAG chunk、PlayBook trust、永久修復候選。 + +合法狀態只有：`received`、`diagnosing`、`candidate_ready`、`executing`、`verifying`、`resolved`、`degraded`、`blocked`、`failed`。`resolved` 必須有 `verifier_passed=true`、`source_event_resolved=true` 及 evidence reference。 + +## 5. 更新後的唯一執行順序 + +### Wave 0：先修正 AI 決策真相 + +| 順序 | ID | 工作 | 狀態 | 驗收 | +| --- | --- | --- | --- | --- | +| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved` 假綠，對齊 99 runtime script 與 Gitea source SHA | **進行中，current P0** | 每種模式以 verifier 決定 outcome；舊 runtime/script drift 歸零；不得再把缺證據算 resolved | +| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping 與重複事件 | 待辦 | `suggestedMode`/kind 優先於 source 字串；5 分鐘先 group；同 fingerprint 只建立一個 active incident | +| 3 | AG99-P0-003 | 強化 ingress：relay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕；producer `auto_repair`/policy 被保留；所有 request 可追 trace | +| 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate，不再錯送 Perf/AwoooRepair；`INC-*` 可從收件追到 candidate | + +### Wave 1：完成可驗證修復閉環 + +| 順序 | ID | 工作 | 狀態 | 驗收 | +| --- | --- | --- | --- | --- | +| 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition；失敗不寫成功 cooldown | +| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | 待辦 | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂；Telegram 可讀回報只從結果物件產生 | +| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | 進行中，live readiness 60% | boot_id/uptime/VM power 建立 reboot event；依 dependency DAG 恢復；首可用/全綠時間與 blocker/ETA 可查 | +| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | 進行中 | 111/112/120/121/188 VMX inventory、開機順序、lock repair、guest running、政策 drift 全部 verifier green | + +### Wave 2：擴至所有產品與運維面 + +| 順序 | ID | 工作 | 狀態 | 驗收 | +| --- | --- | --- | --- | --- | +| 9 | AG99-P0-009 | 外部 down/up watcher、L0/L1 維護頁與 502 failover | 進行中 | 99/內網全斷仍能告警；edge 失聯自動顯示維護頁；恢復後撤回並驗證，不改正式 DNS 作為日常修復 | +| 10 | AG99-P0-010 | 12 產品、全部網站/API 的版本與資料 freshness 矩陣 | 進行中 | 每列具 source SHA、CD、deploy marker、runtime、HTTP、功能 smoke、data timestamp；任何缺欄不宣稱最新 | +| 11 | AG99-P0-011 | 全域 log source registry、AI 異常判讀與 correlation | 待辦 | host/product/site/service/package/tool logs 有 freshness、redaction、tag、cluster、primary/secondary/noise 判定 | +| 12 | AG99-P0-012 | 備份 observability、integrity、offsite 與 restore drill | 進行中 | 所有 backup domain 有 schedule/last success/freshness/restore receipt；失敗可定位且 TG 只報一次狀態變化 | +| 13 | AG99-P0-013 | 110/188 及全主機效能 AIOps 與安全降載 | 進行中 | CPU/memory/disk/IO/queue/DB attribution；allowlisted action 後 verifier，無空值 perf alert、無單次下降假結案 | +| 14 | AG99-P0-014 | Gitea/Harbor/Registry/Gitea-only CD 與完整 DR | 進行中 | repo/main/dev、full dump、DB/settings/issues/packages/LFS、registry、sample restore、runner/CD/runtime marker 全部可回讀 | + +### Wave 3：真正 AI Agent 與產品化 + +| 順序 | ID | 優先序 | 工作與驗收 | +| --- | --- | --- | --- | +| 15 | AG99-P1-001 | P1 | 接入 OpenClaw/Hermes/NemoTron/ElephantAlpha 的 planner/reviewer；先 replay、shadow、canary，再提升執行權限 | +| 16 | AG99-P1-002 | P1 | PlayBook trust/promotion：成功、失敗、rollback、人工介入與 recurrence 決定自動化等級 | +| 17 | AG99-P1-003 | P1 | Telegram 繁中事件卡：一眼摘要、影響、AI 動作、驗證結果、下一步；長證據摺疊，圖表/截圖按事件需要產生 | +| 18 | AG99-P1-004 | P1 | 99 桌面控制中心、Telegram `/agent99`、本機 Submit Request、API command surface 共用同一 policy/audit，不依賴 Codex 額度 | +| 19 | AG99-P1-005 | P1 | AwoooP Runs/Work Items/Knowledge 與 AWOOOI Incident/Observability、IWOOOS/AISOC 完整產品化整合 | +| 20 | AG99-P1-006 | P1 | 工具/套件版本、CVE、SBOM、EOL、更新與 rollback；Windows/Linux/container/model/package drift 皆可查 | +| 21 | AG99-P1-007 | P1 | 安全事件 AI triage、evidence preservation、containment candidate；外部攻擊、credential 與不可逆隔離維持 break-glass | +| 22 | AG99-P1-008 | P1 | 每日/事故/週報：MTTA、MTTR、SLO、recurrence、avoided、false-positive、automation coverage、人工介入與永久修復 | +| 23 | AG99-P2-001 | P2 | 定期 cold-start/restore/chaos drill 與 game day，先隔離環境再 production controlled drill | +| 24 | AG99-P2-002 | P2 | capacity、成本、模型品質與自動化成功率趨勢，據此調整資源與 agent routing | +| 25 | AG99-P2-003 | P2 | 受控 GUI/VM Console/browser automation；只能用 allowlisted app/action、畫面證據與 rollback，不以像素點擊取代 API/CLI | + +## 6. 目前真相與禁止誤報 + +2026-07-10 15:04 production readback： + +- AWOOOI health HTTP 200，但整體狀態為 `degraded`，不能稱完整正常。 +- Reboot SLO 為 `blocked_reboot_auto_recovery_slo_not_ready`，readiness 60%，12 個 active blockers，primary 是 `reboot_event_required_host_unreachable`。 +- Service/data/backup 的部分 readback 已綠，不代表 reboot event、VMware、edge fallback 與 Windows policy 已完成。 +- Gitea inventory 有 12 個產品的 main/dev SSH refs；這只證明 repo/branch 存在，不證明每個產品 production 都是最新版本。 +- Agent99 現況仍是規則式 SRE automation；在 planner、tool loop、mode verifier、callback、secure ingress 完成前，不得宣稱完整 AI Agent。 + +## 7. 工作項強制欄位與治理 + +每個工作項必須先有：`id`、priority/order/status、scope、owner lane、dependencies、acceptance 與 next action。任何工作項升到 `candidate_ready`、`in_progress` 或 `executing` 前，還必須補齊 problem、trigger、evidence sources、AI role、action mode、controlled actions、prohibited actions、rollback、mode-specific verifier、callback、evidence refs 與 last update；缺一不可執行。 + +執行規則： + +1. 同一時間只有一個 `current_p0`；可平行收集證據，但不能改變執行順序。 +2. 所有狀態改變寫入 JSON 總帳，再投影到 AWOOOI/AwoooP；Markdown 是人類說明，不是 runtime truth。 +3. source/test、Gitea CD、runtime deploy、public/data readback 分層保存，不能用其中一層替代其他層。 +4. controlled apply 前必須有 target selector、before evidence、check-mode、rollback、verifier、timeout、single-flight。 +5. `resolved` 後仍要跑 freshness/recurrence observation window；復發則 reopen 同 fingerprint 並增加 recurrence。 +6. GitHub 全面停用；所有 source、CD 與 restore truth 只走 Gitea、本機與 production readback。 + +## 8. 立即下一步 + +目前唯一 P0 是 `AG99-P0-001`：先修正 Agent99 假 resolved 與 99 runtime/source drift，建立 outcome contract 與 regression tests。完成後依序做 routing/dedupe、secure ingress、RepairCandidate，再回到 reboot state machine 的 live rerun；如此不是離開重啟主線，而是先修正會把重啟與告警錯誤結案的控制核心。