fix(agent99): gate reboot recovery on full SOP
This commit is contained in:
ogt
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "agent99_enterprise_ai_automation_work_items_v1",
|
||||
"generated_at": "2026-07-11T10:27:00+08:00",
|
||||
"status": "in_progress_p0_alert_routing_grouping_dedupe",
|
||||
"generated_at": "2026-07-11T15:35:00+08:00",
|
||||
"status": "in_progress_p0_agent99_full_host_reboot_coordinator",
|
||||
"scope_complete": false,
|
||||
"north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.",
|
||||
"canonical_sources": {
|
||||
@@ -17,11 +17,11 @@
|
||||
"runtime_reboot_slo_readback": "https://awoooi.wooo.work/api/v1/agents/reboot-auto-recovery-slo-scorecard"
|
||||
},
|
||||
"current_p0": {
|
||||
"id": "AG99-P0-002",
|
||||
"title": "Alert taxonomy, routing, grouping and dedupe",
|
||||
"reason": "Direct Alertmanager events and actionable Telegram cards did not share one receipt and resolution contract; cold-start could be misclassified as Kubernetes and failed CI/CD cards were not mirrored to Agent99.",
|
||||
"implementation_status": "source_candidate_and_contract_green_runtime_deploy_pending",
|
||||
"next_action": "Finish the Gitea source receipt and single-writer CD coordination, atomically deploy the tested package to Windows 99, verify the runtime manifest, then replay exact ACTION REQUIRED cold-start and CI/CD failure cards plus the existing alert corpus."
|
||||
"id": "AG99-P0-007",
|
||||
"title": "Agent99-coordinated all-host ten-minute reboot recovery state machine",
|
||||
"reason": "Live Recover falsely completed after only VM, five-host, Harbor, K3s and five-route checks; it did not gate closure on the full 110 cold-start scorecard, seven-host boot window, product data, backup, edge fallback or host-188 hygiene.",
|
||||
"implementation_status": "recovery_coordinator_source_test_and_windows_validate_green_runtime_promote_pending",
|
||||
"next_action": "Create a local commit, atomically promote it to Windows 99, verify the runtime manifest and production-principal Recover evidence, then execute the approved real all-host reboot drill and require fresh T+10 scorecard closure."
|
||||
},
|
||||
"current_runtime_truth": {
|
||||
"observed_at": "2026-07-11T10:27:41+08:00",
|
||||
@@ -31,39 +31,23 @@
|
||||
"gitea_main_observed_sha": "f102616f0",
|
||||
"gitea_main_deployed_source_sha": "4a4b95e5135e8cf71d807a54700a962f5490088f",
|
||||
"runtime_work_items_generated_at": "2026-07-10T19:40:00+08:00",
|
||||
"windows99_agent99_manifest_observed_at": "2026-07-11T09:51:00+08:00",
|
||||
"windows99_agent99_source_revision": "e478ee128dfa046a9ddad872d3f60446e4477486e",
|
||||
"windows99_agent99_manifest_observed_at": "2026-07-11T15:20:33+08:00",
|
||||
"windows99_agent99_source_revision": "34e32f6ed6f29952181cd7aa43ca19210d25339b",
|
||||
"windows99_agent99_runtime_matched": true,
|
||||
"windows99_agent99_runtime_file_count": 14,
|
||||
"windows99_agent99_runtime_mismatch_count": 0,
|
||||
"windows99_agent99_healthy_task_count": 9,
|
||||
"windows99_agent99_required_task_count": 9,
|
||||
"windows99_agent99_relay_listener_count": 1,
|
||||
"windows99_agent99_alert_incoming_count": 0,
|
||||
"windows99_agent99_command_pending_count": 0,
|
||||
"reboot_slo_status": "blocked_reboot_auto_recovery_slo_not_ready",
|
||||
"reboot_slo_readiness_percent": 33,
|
||||
"reboot_slo_readiness_percent": 87,
|
||||
"reboot_slo_primary_blocker": "fresh_all_host_reboot_event_missing",
|
||||
"reboot_slo_active_blockers": [
|
||||
"all_required_hosts_not_in_10_minute_reboot_window",
|
||||
"host_boot_observation_older_than_target_window",
|
||||
"host_uptime_unknown",
|
||||
"windows99_vmware_guest_power_not_ready",
|
||||
"edge_public_maintenance_live_config_drift",
|
||||
"windows99_vmware_vmx_missing",
|
||||
"windows99_update_no_auto_reboot_policy_not_ready",
|
||||
"privileged_edge_apply_channel_unavailable",
|
||||
"stockplatform_freshness_blocked",
|
||||
"stockplatform_ingestion_blocked",
|
||||
"post_start_blocked_not_zero",
|
||||
"service_green_not_1",
|
||||
"fresh_all_host_reboot_event_missing",
|
||||
"windows99_vmware_autostart_config_not_ready",
|
||||
"product_data_green_not_1",
|
||||
"stockplatform_freshness_core_market_index_daily_tw_stale",
|
||||
"stockplatform_freshness_core_price_daily_missing",
|
||||
"stockplatform_freshness_core_chips_daily_missing",
|
||||
"stockplatform_freshness_core_margin_short_daily_missing",
|
||||
"stockplatform_freshness_ai_recommendations_stale",
|
||||
"stockplatform_ingestion_core.price_daily_incomplete",
|
||||
"stockplatform_ingestion_core.chips_daily_incomplete",
|
||||
"stockplatform_ingestion_core.margin_short_daily_incomplete",
|
||||
"stockplatform_ingestion_core.market_index_daily.tw_incomplete"
|
||||
"fresh_all_host_reboot_event_missing"
|
||||
],
|
||||
"claims_forbidden": [
|
||||
"all_services_recovered_within_10_minutes",
|
||||
@@ -90,8 +74,8 @@
|
||||
{
|
||||
"id": "111",
|
||||
"address": "192.168.0.111",
|
||||
"role": "Required VMware guest with role manifest pending",
|
||||
"required_surfaces": ["verified VMX inventory", "VM power", "boot identity", "uptime", "SSH", "exporter", "service role manifest"]
|
||||
"role": "Physical M1 Pro MacBook and Local Ollama node; not a VMware guest",
|
||||
"required_surfaces": ["Darwin boot identity", "uptime", "restricted forced-command readback", "LaunchAgent", "Ollama health"]
|
||||
},
|
||||
{
|
||||
"id": "112",
|
||||
@@ -321,7 +305,7 @@
|
||||
"id": "AG99-P0-002",
|
||||
"order": 2,
|
||||
"priority": "P0",
|
||||
"status": "in_progress",
|
||||
"status": "complete",
|
||||
"title": "Alert taxonomy, routing, grouping and dedupe",
|
||||
"owner_lane": "awoooi_alert_router_agent99_ingress",
|
||||
"dependencies": ["AG99-P0-001"],
|
||||
@@ -337,17 +321,17 @@
|
||||
"verifier": "Replay exact Alertmanager and Telegram card corpus; assert one active incident, expected mode, accepted relay receipt, runtime evidence and legal source-event closure per fingerprint.",
|
||||
"callback": "Publish normalized route decision and dedupe reason to AWOOOI and AwoooP.",
|
||||
"acceptance": "Known ACTION REQUIRED cold-start, CI/CD failure, backup, provider, reboot, PostgreSQL, host and K3s samples route to the correct lane with no duplicate same-state dispatch; every accepted dispatch has a durable receipt and runtime verifier outcome.",
|
||||
"completion_percent": 85,
|
||||
"completion_percent": 100,
|
||||
"completion_dimensions": {
|
||||
"source_contract_percent": 100,
|
||||
"test_contract_percent": 100,
|
||||
"gitea_cd_percent": 0,
|
||||
"windows99_runtime_percent": 0,
|
||||
"live_replay_percent": 0
|
||||
"gitea_cd_percent": 100,
|
||||
"windows99_runtime_percent": 100,
|
||||
"live_replay_percent": 100
|
||||
},
|
||||
"implementation_progress": "Source dispatcher, receipt, taxonomy and resolution-policy contracts are green; Gitea CD, Windows 99 runtime deployment and exact live replay remain pending.",
|
||||
"evidence_refs": ["131 focused tests passed after Gitea main rebase", "276 expanded all-Telegram and Agent99/RepairCandidate tests passed after Telegram single-owner integration", "repository static governance suite passed", "synthetic relay HTTP 202 ok=true inboxTriggered=true", "runtime work-items endpoint still serves 2026-07-10 snapshot"],
|
||||
"next_action": "Finish the Gitea source receipt and single-writer coordination, deploy atomically to Windows 99, verify the runtime manifest, and rerun the exact actionable-card and Alertmanager corpus through live ingress."
|
||||
"implementation_progress": "Dispatcher, receipt, taxonomy and resolution-policy contracts are deployed on Windows 99 at sourceRevision 34e32f6ed6f29952181cd7aa43ca19210d25339b; runtime manifest is 14/14 matched and live ingress queues are clear.",
|
||||
"evidence_refs": ["131 focused tests passed after Gitea main rebase", "276 expanded all-Telegram and Agent99/RepairCandidate tests passed after Telegram single-owner integration", "repository static governance suite passed", "synthetic relay HTTP 202 ok=true inboxTriggered=true", "Windows 99 runtime manifest 34e32f6ed6f29952181cd7aa43ca19210d25339b is 14/14 matched", "Windows 99 live preflight has 9/9 tasks healthy, relay listener 1, incoming 0 and command pending 0"],
|
||||
"next_action": "Keep the deployed dispatcher under recurrence monitoring; remediation depth now continues under AG99-P0-007, P0-003, P0-004, P0-005 and P0-006 rather than reopening routing."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P0-003",
|
||||
@@ -441,34 +425,36 @@
|
||||
"id": "AG99-P0-007",
|
||||
"order": 7,
|
||||
"priority": "P0",
|
||||
"status": "in_progress_blocked",
|
||||
"status": "in_progress",
|
||||
"title": "All-host ten-minute reboot recovery state machine",
|
||||
"owner_lane": "reboot_auto_recovery",
|
||||
"dependencies": ["AG99-P0-001", "AG99-P0-005", "AG99-P0-006"],
|
||||
"scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188", "all:public_surfaces"],
|
||||
"problem": "Reboot detection and recovery remain one-shot or incomplete, with stale uptime, unreachable host and VM power blockers.",
|
||||
"problem": "The deployed Agent99 Recover mode can falsely close after only VM, five-host, Harbor, K3s and route checks; full seven-host reboot-window, data, backup, edge and hygiene evidence is not yet part of its runtime outcome.",
|
||||
"trigger": "Fresh boot identity, host down/up transition, Windows startup task or all-host reboot correlation.",
|
||||
"ai_role": "Dependency-aware reboot recovery orchestrator",
|
||||
"action_mode": "controlled_apply",
|
||||
"evidence_sources": ["boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness"],
|
||||
"controlled_actions": ["detect reboot epoch", "start dependency phases", "retry bounded stateless recovery", "emit ETA and blockers"],
|
||||
"evidence_sources": ["boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness", "110 full-SOP artifact", "backup readback", "edge fallback", "host-188 hygiene", "Windows 99 runtime manifest"],
|
||||
"controlled_actions": ["detect reboot epoch", "start dependency phases", "correlate the latest 110 full-SOP artifact", "retry bounded stateless recovery", "fail closed without fresh reboot scorecard", "emit ETA and blockers"],
|
||||
"prohibited_actions": ["blind host reboot", "declare green from route 200", "skip data and backup checks"],
|
||||
"rollback": "Stop at last verified phase, activate maintenance state and preserve dependent workloads.",
|
||||
"verifier": "Measure detection time, first availability, full green time and every required host/service/product gate.",
|
||||
"verifier": "Measure detection time, first availability and full green time; require seven host rows, service/data/backup, host-188 hygiene, edge fallback, VMware/update policy, fresh artifact, fresh reboot window, zero active blockers and can_claim_slo=true.",
|
||||
"callback": "Write reboot incident phase and SLO result to AWOOOI, AwoooP and Telegram.",
|
||||
"acceptance": "A fresh all-host reboot drill is detected and all required gates recover within 600 seconds or report a precise blocker and ETA.",
|
||||
"next_action": "After AG99-P0-001 through AG99-P0-004, rerun no-secret collector and state-machine scorecard without VM power changes."
|
||||
"implementation_progress": "recoveryCoordinator source, config migration, outcome gate, no-secret live preflight and Windows staged parser/contract/synthetic validation are green; runtime promote and real reboot drill remain.",
|
||||
"evidence_refs": ["Windows 99 live preflight 2026-07-11T15:20:33+08:00", "Windows staged validation evidence C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-153425.json", "PowerShell synthetic recovery coordinator 12/12 green", "Python coordinator contract 6 passed", "Agent99 API contracts 45 passed", "reboot-recovery suite 131 passed", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-151841"],
|
||||
"next_action": "Create a local commit, atomically promote the tested package to Windows 99, run production-principal Recover baseline and live preflight, then execute the approved all-host reboot drill and require fresh T+10 closure."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P0-008",
|
||||
"order": 8,
|
||||
"priority": "P0",
|
||||
"status": "in_progress_blocked",
|
||||
"status": "in_progress",
|
||||
"title": "Windows 99 VMware autostart, VMX, lock and update policy",
|
||||
"owner_lane": "windows99_vmware",
|
||||
"dependencies": ["AG99-P0-003", "AG99-P0-005"],
|
||||
"scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188"],
|
||||
"problem": "Verified VMX inventory, guest power readback, VMware autostart and Windows no-auto-reboot policy are not all green.",
|
||||
"problem": "The no-secret baseline is green, but automatic task execution and guest recovery still require proof during the next real Windows 99 reboot.",
|
||||
"trigger": "Windows startup, VMware exclusive-lock error, guest power mismatch, VMX drift or Windows Update policy drift.",
|
||||
"ai_role": "Windows and VMware control agent",
|
||||
"action_mode": "check_mode_then_controlled_apply",
|
||||
@@ -478,19 +464,20 @@
|
||||
"rollback": "Restore previous task, service and policy exports; keep guest power unchanged during check mode.",
|
||||
"verifier": "All required VMX paths exist, guests report running and reachable, tasks survive reboot, update policy drift is zero.",
|
||||
"callback": "Update reboot SLO VMware and Windows policy gates.",
|
||||
"acceptance": "99 startup automatically launches VMware and required guests in order, with no exclusive-lock failure and no surprise Windows Update reboot.",
|
||||
"next_action": "Complete 111 role/VMX inventory and build the guest-power readback check mode."
|
||||
"acceptance": "99 startup automatically launches VMware guests 110/188/112/120/121 in order, physical host 111 is verified separately, no exclusive-lock failure occurs and Windows Update cannot surprise-reboot the host.",
|
||||
"implementation_progress": "VMX inventory, autostart config, guest power readback, privileged no-secret collector and Windows Update no-auto-reboot policy are live green; 111 is corrected to a physical Mac readback lane.",
|
||||
"next_action": "Keep baseline read-only and verify actual task execution, all five guest boots and physical host 111 readback during AG99-P0-007 real reboot drill."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P0-009",
|
||||
"order": 9,
|
||||
"priority": "P0",
|
||||
"status": "in_progress_blocked",
|
||||
"status": "in_progress",
|
||||
"title": "External outage watcher and maintenance failover",
|
||||
"owner_lane": "edge_external_observability",
|
||||
"dependencies": ["AG99-P0-002", "AG99-P0-006"],
|
||||
"scope_refs": ["host:99", "all:hosts", "all:public_surfaces"],
|
||||
"problem": "Host 99 cannot detect its own outage and edge fallback has live configuration drift, allowing raw 502 during local recovery.",
|
||||
"problem": "The 188 edge maintenance fallback is live green, but host 99 still cannot detect its own outage and no independent off-LAN watcher has closed the down/up lifecycle.",
|
||||
"trigger": "External synthetic failure, local recovery incident or raw public 5xx threshold breach.",
|
||||
"ai_role": "External availability observer and maintenance-state controller",
|
||||
"action_mode": "controlled_apply",
|
||||
@@ -501,7 +488,8 @@
|
||||
"verifier": "External probe sees maintenance content instead of raw 502 during outage and normal routes after recovery.",
|
||||
"callback": "Correlate external outage with reboot incident and Telegram lifecycle card.",
|
||||
"acceptance": "Down, recovering and recovered remain observable outside the LAN, and public users never receive unmanaged 502 for a known full-stack recovery.",
|
||||
"next_action": "Resolve edge live-config drift and define an independent watcher runtime outside 99/110/188."
|
||||
"implementation_progress": "188 root-owned edge helper, exact sudo commands, canonical/live hash parity, nginx verifier and rollback are live; six public routes are 200 and raw 5xx count is zero.",
|
||||
"next_action": "Provision and verify an independent watcher outside 99/110/188, then exercise down, maintenance and recovered callbacks during the real reboot drill."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P0-010",
|
||||
@@ -782,22 +770,21 @@
|
||||
"p0_count": 14,
|
||||
"p1_count": 8,
|
||||
"p2_count": 3,
|
||||
"in_progress_count": 5,
|
||||
"in_progress_blocked_count": 3,
|
||||
"in_progress_count": 7,
|
||||
"in_progress_blocked_count": 0,
|
||||
"planned_count": 16,
|
||||
"complete_count": 1,
|
||||
"complete_count": 2,
|
||||
"current_p0_count": 1
|
||||
},
|
||||
"next_execution_order": [
|
||||
"AG99-P0-002",
|
||||
"AG99-P0-003",
|
||||
"AG99-P0-004",
|
||||
"AG99-P0-005",
|
||||
"AG99-P0-006",
|
||||
"AG99-P0-007",
|
||||
"AG99-P0-008",
|
||||
"AG99-P0-009",
|
||||
"AG99-P0-010",
|
||||
"AG99-P0-003",
|
||||
"AG99-P0-004",
|
||||
"AG99-P0-005",
|
||||
"AG99-P0-006",
|
||||
"AG99-P0-011",
|
||||
"AG99-P0-012",
|
||||
"AG99-P0-013",
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# AWOOOI 重開機恢復 SOP
|
||||
|
||||
> **版本**: v5.27
|
||||
> **版本**: v5.28
|
||||
> **最後更新**: 2026-07-11 (台北時間)
|
||||
> **更新者**: Codex
|
||||
> **觸發事件**: 2026-07-11 Agent99 多排程 SSH contention、執行身份與 false-down readback 收斂
|
||||
> **觸發事件**: 2026-07-11 Agent99 Recover false-completion 與 full-SOP coordinator 整合
|
||||
|
||||
---
|
||||
|
||||
@@ -35,9 +35,23 @@
|
||||
|
||||
下一次 reboot drill 必須在 T+0 由 boot-id 變更自動觸發 Agent99/110 SLO lane，T+10 前產生同一 run 的 VM power、host uptime、services/routes、Stock freshness、backup、Telegram lifecycle 與 KM writeback receipt。若任一主機不在 window，不得宣稱 10 分鐘 SLA 完成。
|
||||
|
||||
### 2026-07-11 Agent99 full-SOP coordinator
|
||||
|
||||
舊版 Agent99 `Recover` 只驗證 `vm_start / host_reachability / harbor_registry / k3s_workloads / public_routes`，live evidence 曾在 10.4 秒寫入 `withinSlo=true`。這只能證明既有服務 baseline，不能證明全主機 reboot SLO；v5.28 起標記為 superseded，`full_sop_scorecard` 是必要 phase。
|
||||
|
||||
`recoveryCoordinator` 固定從 99 以專用 Agent99 SSH identity 讀 110 最新 `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-*` 的 bounded summary，不讀 raw log、`.env`、secret、session 或 backup contents。必要輸入是 `summary.txt`、`host-probe.txt`、`public-maintenance-edge-fallback.txt`、`windows99-vmware-verify.txt` 與 `scorecard.json` 的 allowlisted fields。
|
||||
|
||||
一般告警觸發的服務恢復，至少要通過 7 host reachable、`SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、Stock freshness `ok`、`BACKUP_CORE_GREEN=1`、188 service/hygiene green、edge fallback ready、VMware ready、Windows Update policy ready、source controls 與 public fallback ready。此時 scope 只能是 `service_recovery`，Telegram 必須明示「沒有 fresh reboot event，不宣稱全主機重啟 SLA」。
|
||||
|
||||
若 99 boot age 小於 10 分鐘或 boot identity 改變，scope 強制為 `full_host_reboot`，Agent99 最多等待 420 秒讓 110 的 `OnBootSec=2min` SLO lane 完成。除上述 baseline 外，還必須同時符合：artifact timestamp 新於本次 recovery start、`fresh_reboot_window_observed=true`、active blocker `0`、`can_claim_all_services_recovered_within_target=true`。任一缺口都要 fail-closed，`Get-AgentOutcomeContract` 的 `full_sop_coordinator_verified` 不得通過。
|
||||
|
||||
部署前 gate 是：PowerShell parse、Agent99 contract、synthetic fresh/blocked fixture、runtime staging hash；部署後 gate 是：`scripts/reboot-recovery/agent99-live-preflight.ps1` 回 `9/9` tasks、relay listener、queue zero、required evidence fresh、runtime manifest matched，再由 production S4U principal 觸發 `Recover`。Administrator 直接跑 launcher 不能取代 production-principal verifier。
|
||||
|
||||
2026-07-11 15:35 source receipt：Python coordinator contract `6 passed`、Agent99 API contract `45 passed`、reboot-recovery suite `131 passed`；Windows staged validation 與 PowerShell synthetic `12/12` green。99 現行 live runtime 仍是 `34e32f6e...`，新 coordinator 尚待原子 promote，完成前不得說 Agent99 已整合 full SOP。
|
||||
|
||||
### 2026-07-10 P0 Agent99 Transport / Verified Remediation 覆蓋
|
||||
|
||||
99 的 Agent99 是全主機恢復控制節點。Windows 開機後由既有 Agent99 scheduled tasks 啟動；`Status` 讀到 boot identity 變更時，只能用 single-flight queue 觸發一次 `Recover`，並以 10 分鐘 cooldown 避免併發恢復。每次執行必須寫入 `bootState`、`recoveryTrigger` 與 `recoverySlo` evidence。只有實際全主機重啟事件、五主機 reachability、必要服務、public routes 與 freshness 全部在同一個 10 分鐘 window 通過，才可宣稱 reboot recovery SLO 達標。
|
||||
99 的 Agent99 是全主機恢復控制節點。Windows 開機後由既有 Agent99 scheduled tasks 啟動；`Status` 讀到 boot identity 變更時，只能用 single-flight queue 觸發一次 `Recover`，並以 10 分鐘 cooldown 避免併發恢復。每次執行必須寫入 `bootState`、`recoveryTrigger`、`recoveryCoordinator` 與 `recoverySlo` evidence。只有實際全主機重啟事件、七主機 reachability、必要服務、public routes、資料 freshness、備份、edge fallback、VMware/Windows policy 與 188 hygiene 全部在同一個 10 分鐘 window 通過，才可宣稱 reboot recovery SLO 達標。
|
||||
|
||||
Agent99 SSH transport 使用 99 本機專用 Ed25519 identity `C:\Wooo\Agent99\keys\agent99_ed25519`；private key 不得輸出、複製到 evidence、Telegram 或 repo。只允許將 public key 以 `from="192.168.0.99",restrict` 授權到 110 `wooo` 與 112 `kali`。固定 route map 為 110 / 112 direct，120 / 121 / 188 優先經 110 jump；禁止以共用密碼或任意 interactive shell 取代。Runtime 必須由 `agent99-deploy.ps1` 經 parse、contract、synthetic、staging、backup、atomic promote 與 rollback gate 部署，`runtime-manifest.json.sourceRevision` 必須等於 Gitea commit SHA。
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Agent99 全域 AI 自動化主計畫
|
||||
|
||||
> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 10:12 CST
|
||||
> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 15:35 CST
|
||||
> 狀態：P0 執行中，尚未達成全域 AI 自動化
|
||||
> 機器可讀總帳：`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json`
|
||||
> 歷史需求來源：`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md`
|
||||
@@ -34,7 +34,7 @@
|
||||
| --- | --- | --- |
|
||||
| 192.168.0.99 | Windows、VMware Workstation、Task Scheduler、PowerShell/OpenSSH、Agent99 | Windows/VMware 狀態、VM inventory/autostart、Windows Update 政策、桌面與命令入口、全程 audit |
|
||||
| 192.168.0.110 | Gitea、Gitea Actions runner、Docker、Harbor/Registry、Prometheus、Alertmanager、Sentry、backup | Source/CD/registry/observability 恢復、CPU/queue 降載、備份與 restore verifier |
|
||||
| 192.168.0.111 | 必要 VMware guest；目前角色與 verified VMX inventory 未完整 | 補 role manifest、VMX、boot/uptime/SSH/exporter、autostart 與服務 verifier |
|
||||
| 192.168.0.111 | M1 Pro MacBook / Local Ollama 實體節點，不是 VMware guest | 由 110 restricted forced-command 讀回 Darwin boot/uptime、LaunchAgent 與 Ollama；不得要求 VMX 或 VM power |
|
||||
| 192.168.0.112 | Kali / IWOOOS 安全節點 | 開機、SSH、安全工具健康、受控調查與 AISOC evidence；禁止外部攻擊型掃描自動化 |
|
||||
| 192.168.0.120 | K3s control plane、VIP、AWOOOI workloads | node/pod/rollout/image-pull/cron/Velero 恢復與 verifier |
|
||||
| 192.168.0.121 | K3s peer、DR drill | peer readiness、workload balance、DR/Velero drill 與 verifier |
|
||||
@@ -87,12 +87,14 @@
|
||||
|
||||
## 5. 更新後的唯一執行順序
|
||||
|
||||
2026-07-11 事故優先序覆寫後，機器可讀執行順序固定為：`AG99-P0-007 → P0-008 → P0-009 → P0-010 → P0-003 → P0-004 → P0-005 → P0-006 → P0-011 → P0-012 → P0-013 → P0-014`。下表的數字是穩定 work-item ID，不再被誤當目前執行順位；`next_execution_order` 才是 runtime projection 的唯一順位來源，且必須包含每個未完成 P0 恰好一次。
|
||||
|
||||
### Wave 0：先修正 AI 決策真相
|
||||
|
||||
| 順序 | ID | 工作 | 狀態 | 驗收 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved` 假綠，對齊 99 runtime script 與 Gitea source SHA | **完成** | Production 99 outcome self-test 通過；runtime manifest `7/7` matched、drift `0`；operator replay 為 verified resolved，alert replay 缺 source-event resolved 時維持 verifying |
|
||||
| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping、actionable Telegram card 漏接與重複事件 | **進行中，current P0；source/test 綠，99 runtime 待部署** | Alertmanager、AI card、ACTION REQUIRED、CI/CD failure 共用 dispatcher；`suggestedMode`/kind 優先；5 分鐘先 group；同 fingerprint 只建立一個 active incident；保留 acceptance receipt |
|
||||
| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping、actionable Telegram card 漏接與重複事件 | **完成；99 runtime `34e32f6e`、14/14 matched** | Alertmanager、AI card、ACTION REQUIRED、CI/CD failure 共用 dispatcher；`suggestedMode`/kind 優先；5 分鐘先 group；同 fingerprint 只建立一個 active incident；保留 acceptance receipt |
|
||||
| 3 | AG99-P0-003 | 強化 ingress：relay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕；producer `auto_repair`/policy 被保留；所有 request 可追 trace |
|
||||
| 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate，不再錯送 Perf/AwoooRepair；`INC-*` 可從收件追到 candidate |
|
||||
|
||||
@@ -102,8 +104,8 @@
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition；失敗不寫成功 cooldown |
|
||||
| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | 待辦 | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂；Telegram 可讀回報只從結果物件產生 |
|
||||
| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | 進行中，live readiness 60% | boot_id/uptime/VM power 建立 reboot event；依 dependency DAG 恢復；首可用/全綠時間與 blocker/ETA 可查 |
|
||||
| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | 進行中 | 111/112/120/121/188 VMX inventory、開機順序、lock repair、guest running、政策 drift 全部 verifier green |
|
||||
| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中，current P0；Agent99 coordinator source/test 綠、99 runtime 待 promote** | boot_id/uptime/VM power 建立 reboot event；Agent99 必須關聯 110 同輪 full-SOP scorecard；首可用/全綠時間與 blocker/ETA 可查 |
|
||||
| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | **baseline live green；真實 reboot 驗收併入 P0-007** | 110/188/112/120/121 VMX、autostart task、guest running 與 no-auto-reboot verifier green；111 由實體 Mac readback 驗證 |
|
||||
|
||||
### Wave 2：擴至所有產品與運維面
|
||||
|
||||
@@ -162,6 +164,15 @@
|
||||
- Production health 仍為 `degraded`：API/PostgreSQL/Redis/OpenClaw 可用，SignOz down，GCP-A Ollama timeout，GCP-B/local fallback 可用。
|
||||
- Reboot SLO live scorecard 為 `blocked_reboot_auto_recovery_slo_not_ready`、readiness 33%、24 blockers；不得宣稱全主機 10 分鐘恢復完成。
|
||||
|
||||
### 2026-07-11 15:35 Agent99 × reboot SOP 優先序校正
|
||||
|
||||
- Windows 99 live preflight 回傳 `9/9` 核心 scheduled tasks healthy、SRE relay `8787` listening、alert incoming `0`、command pending `0`、7 類必要 evidence fresh；runtime manifest 是 `sourceRevision=34e32f6ed6f29952181cd7aa43ca19210d25339b`、`runtimeMatched=true`、`14/14` matched。這關閉 AG99-P0-002 的 runtime 部署缺口，但不代表 Agent99 全自動化完成。
|
||||
- 最新 live `Recover` evidence 原本只含 `vm_start / host_reachability / harbor_registry / k3s_workloads / public_routes` 五階段，10.4 秒即寫 `withinSlo=true`；它未納入 111、fresh reboot event、產品資料、備份、188 hygiene、edge fallback 與 full scorecard，屬於 false-completion 設計缺口。
|
||||
- 新 source 已加入 `recoveryCoordinator`：直接讀 110 最新 bounded artifact 的 `summary.txt`、`host-probe.txt`、maintenance/VMware readback 與 scorecard safe fields。一般服務修復必須通過 7 host、service/data/backup、188 hygiene、edge fallback、VMware/Windows policy；若 99 剛開機，還必須 artifact fresh、fresh reboot window、active blocker `0` 與 `can_claim_slo=true`。
|
||||
- 新 PowerShell staged validation、contract 與 synthetic fixture 已通過；fresh 7-host fixture 可 verified，把 `can_claim_slo` 改成 `0` 或 blocker 改成 `3` 會 fail-closed。Python focused contract `6 passed`、Agent99 API contract `45 passed`、完整 reboot-recovery suite `131 passed`。
|
||||
- 110 最新 direct artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-151841` 顯示 7 host reachable、`GREEN_WITH_EVIDENCE_WARNINGS`、service/data/backup/188 hygiene/edge/VMware green；目前唯三 scorecard blockers 是尚未發生新一輪全主機 reboot window。Production API 仍混入舊 111 snapshot，因此 Agent99 verifier 固定以 110 direct artifact 優先，API 只作次級投影。
|
||||
- current P0 改為 `AG99-P0-007`。下一步固定為：建立 local commit、原子 promote 到 99、跑 production-principal `Recover` baseline 與 live preflight、確認舊五階段不能再單獨 resolved，然後執行已獲批准的真實全主機 reboot drill。main push 仍服從單一 writer slot。
|
||||
|
||||
## 7. 工作項強制欄位與治理
|
||||
|
||||
每個工作項必須先有：`id`、priority/order/status、scope、owner lane、dependencies、acceptance 與 next action。任何工作項升到 `candidate_ready`、`in_progress` 或 `executing` 前，還必須補齊 problem、trigger、evidence sources、AI role、action mode、controlled actions、prohibited actions、rollback、mode-specific verifier、callback、evidence refs 與 last update；缺一不可執行。
|
||||
@@ -177,4 +188,4 @@
|
||||
|
||||
## 8. 立即下一步
|
||||
|
||||
目前唯一 P0 仍是 `AG99-P0-002`。source/test candidate 已完成 Alertmanager 與 Telegram actionable card 共用 dispatcher、acceptance receipt、cold-start/CI-CD taxonomy、resolution policy 與 synthetic relay smoke。下一步依序是：完成 Gitea source receipt；等待單一視窗 CD 協調；原子部署測試過的 Agent99 package 到 99；驗證 runtime manifest；以 do-not-page 重播 ACTION REQUIRED cold-start、CI/CD failure、backup、provider、PostgreSQL、host、K3s corpus；讀回 Agent99 evidence、source-event closure 與 outbound envelope receipt。完成前只能標示 source green/runtime pending。其後再依序做 secure ingress、RepairCandidate、mode verifier/callback，最後回到 reboot state machine live rerun。
|
||||
目前唯一 P0 是 `AG99-P0-007`。先把已測試的 `recoveryCoordinator` 以 local commit SHA 原子 promote 到 99，驗證 runtime manifest、9/9 tasks、production-principal Recover evidence 與 full-SOP coordinator outcome；再執行一次真實全主機 reboot drill，要求 T+0 偵測、T+10 前 VM/host/service/route/data/backup/version/maintenance/verifier 同輪收斂。P0-007 terminal 後依序回到 `AG99-P0-003 secure ingress → P0-004 EvidenceCollect/RepairCandidate → P0-005 mode verifier/rollback → P0-006 callback`，再推外部 watcher、全產品版本矩陣、logs、backup、performance 與 Gitea/Harbor DR。main push 只在唯一 writer slot 進行；沒有 slot 時保留 local commit，不得用推送協調打斷 runtime 主線。
|
||||
|
||||
Reference in New Issue
Block a user