feat(agent99): add durable completion callback

This commit is contained in:
ogt
2026-07-11 20:45:52 +08:00
parent df26f43b77
commit e8b0244ee7
15 changed files with 965 additions and 41 deletions

View File

@@ -1,7 +1,7 @@
{
"schema_version": "agent99_enterprise_ai_automation_work_items_v1",
"generated_at": "2026-07-11T20:14:33+08:00",
"status": "in_progress_p0_agent99_false_green_and_lifecycle_closure",
"generated_at": "2026-07-11T20:34:00+08:00",
"status": "in_progress_p0_agent99_canonical_completion_callback",
"scope_complete": false,
"north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.",
"canonical_sources": {
@@ -17,22 +17,22 @@
"runtime_reboot_slo_readback": "https://awoooi.wooo.work/api/v1/agents/reboot-auto-recovery-slo-scorecard"
},
"current_p0": {
"id": "AG99-P0-001",
"title": "Agent99 runtime truth, false-green prevention and visible lifecycle receipt",
"reason": "The 2026-07-11 live audit proved that Agent99 runs real recovery work, but the operator cannot reliably see the closure: required inbox evidence can fail safe-summary parsing while preflight remains green, a successful Recover produces zero Telegram attempts because info lifecycle events are filtered out, and dedupe suppression leaves no explicit delivery receipt. Runtime truth and one-run lifecycle visibility must be corrected before adding more automation breadth.",
"implementation_status": "regression_reopened_source_fix_in_progress_runtime_deploy_and_recover_verifier_pending",
"next_action": "Deploy the parse-aware preflight and recovery lifecycle card to Windows 99, run one production-principal Recover, and require one explicit Telegram sent-or-dedupe receipt plus a parse-clean live preflight."
"id": "AG99-P0-006",
"title": "Agent99 canonical completion callback and cross-system incident closure",
"reason": "P0-001 is now runtime-verified on Windows 99: required evidence is fresh, parseable and content-healthy; a production-principal Recover passed all seven phases and produced a real Telegram visual receipt. The next missing link is canonical Agent99-to-AWOOOI/AwoooP/IWOOOS writeback, because Telegram delivery alone is not a durable incident closure.",
"implementation_status": "api_and_agent99_callback_source_ready_39_tests_passed_production_cd_and_runtime_callback_pending",
"next_action": "Commit the authenticated idempotent callback source, wait for the coordinated Gitea push slot, deploy through Gitea CD, then replay one SRE alert and require identical callback/run/trace plus AWOOOI operation and AwoooP run readback."
},
"current_runtime_truth": {
"observed_at": "2026-07-11T17:37:00+08:00",
"observed_at": "2026-07-11T20:25:26+08:00",
"awoooi_health_http_status": 200,
"awoooi_health_state": "recovered_late_currently_healthy",
"awoooi_runtime_build_commit_sha": "4a4b95e5135e8cf71d807a54700a962f5490088f",
"gitea_main_observed_sha": "f102616f0",
"gitea_main_deployed_source_sha": "4a4b95e5135e8cf71d807a54700a962f5490088f",
"runtime_work_items_generated_at": "2026-07-10T19:40:00+08:00",
"windows99_agent99_manifest_observed_at": "2026-07-11T17:31:00+08:00",
"windows99_agent99_source_revision": "442b319bb80e009291ced7afa4f704cbe3b64886",
"windows99_agent99_manifest_observed_at": "2026-07-11T20:20:51+08:00",
"windows99_agent99_source_revision": "06509e071c8b7c538feb112e4e4774ee22abf7d1",
"windows99_agent99_runtime_matched": true,
"windows99_agent99_runtime_file_count": 14,
"windows99_agent99_runtime_mismatch_count": 0,
@@ -41,11 +41,23 @@
"windows99_agent99_relay_listener_count": 1,
"windows99_agent99_alert_incoming_count": 0,
"windows99_agent99_command_pending_count": 0,
"windows99_agent99_live_preflight_green": true,
"windows99_agent99_required_evidence_fresh_count": 7,
"windows99_agent99_required_evidence_usable_count": 7,
"windows99_agent99_required_evidence_parse_failure_count": 0,
"windows99_agent99_required_evidence_content_failure_count": 0,
"windows99_agent99_recovery_run_id": "agent99-recovery-20260711-202159-3404b5b3",
"windows99_agent99_recovery_completed": true,
"windows99_agent99_recovery_elapsed_seconds": 42.9,
"windows99_agent99_recovery_phase_count": 7,
"windows99_agent99_recovery_telegram_sent": true,
"windows99_agent99_recovery_visual_sent": true,
"agent99_completion_callback_runtime_status": "source_ready_production_api_and_windows_client_not_yet_deployed",
"windows99_agent99_startup_recovery_execution_limit": "PT20M",
"windows99_vmware_running_guests": "5/5",
"public_surface_matrix": "23/23_http_2xx_or_expected_3xx",
"reboot_slo_status": "slo_breached_recovered_late",
"reboot_slo_readiness_percent": 87,
"reboot_slo_readiness_percent": 89,
"reboot_slo_primary_blocker": "all_required_hosts_not_in_10_minute_reboot_window",
"reboot_slo_active_blockers": [
"all_required_hosts_not_in_10_minute_reboot_window",
@@ -272,7 +284,7 @@
"id": "AG99-P0-001",
"order": 1,
"priority": "P0",
"status": "in_progress",
"status": "complete",
"title": "Agent outcome truth, source-runtime reconciliation and false-green prevention",
"owner_lane": "agent99_control_plane",
"dependencies": [],
@@ -289,9 +301,9 @@
"callback": "Write final outcome to AWOOOI incident and AwoooP Run/Verifier using the same trace id.",
"acceptance": "No false-resolved regression cases; source/runtime drift count is zero; unresolved evidence stays degraded or blocked.",
"completion_receipt": {
"source_revision": "5ea4405c2a75b51244da168a4834d27e24d74ee6",
"source_revision": "06509e071c8b7c538feb112e4e4774ee22abf7d1",
"runtime_manifest_matched": true,
"runtime_manifest_file_count": 7,
"runtime_manifest_file_count": 14,
"runtime_manifest_mismatch_count": 0,
"self_health_runtime_manifest_severity": "ok",
"outcome_self_test_ok": true,
@@ -300,19 +312,31 @@
"alert_replay_state": "verifying",
"alert_replay_verified_resolved": 0,
"alert_replay_source_event_resolved": false,
"telegram_sent_count": 0,
"rollback": "C:\\Wooo\\Agent99\\rollback\\p0-001-pre-5ea4405c",
"telegram_sent_count": 1,
"telegram_visual_sent_count": 1,
"required_evidence_usable_count": 7,
"required_evidence_parse_failure_count": 0,
"required_evidence_content_failure_count": 0,
"recovery_run_id": "agent99-recovery-20260711-202159-3404b5b3",
"recovery_completed": true,
"recovery_elapsed_seconds": 42.9,
"recovery_phase_count": 7,
"rollback": "C:\\Wooo\\Agent99\\deploy\\backup-20260711-202021",
"evidence": [
"C:\\Wooo\\Agent99\\evidence\\agent99-OutcomeContractSelfTest-20260710-161302.json",
"C:\\Wooo\\Agent99\\evidence\\agent99-SelfCheck-20260710-161350.json",
"C:\\Wooo\\Agent99\\queue\\processed\\processed-p0-001-operator-verifier-20260710-1615.json",
"C:\\Wooo\\Agent99\\queue\\processed\\processed-p0-001-alert-verifier-20260710-1615.json",
"C:\\Wooo\\Agent99\\evidence\\agent99-ControlTick-20260710-161919.json"
"C:\\Wooo\\Agent99\\evidence\\agent99-ControlTick-20260710-161919.json",
"C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-202021.json",
"C:\\Wooo\\Agent99\\evidence\\agent99-Recover-20260711-202159.json",
"C:\\Wooo\\Agent99\\evidence\\telegram-cards\\agent99-card-recovery_slo_result-20260711-202159.png"
]
},
"regression_reopened_at": "2026-07-11T20:02:00+08:00",
"regression_evidence": ["Windows 99 live preflight returned green while latest SreAlertInbox and TelegramInbox safe summaries reported RuntimeException", "latest production-principal Recover was verifier-green but telegramAttemptCount=0", "dedupe suppression returned without a Telegram attempt receipt"],
"next_action": "Deploy the parse-aware evidence gate, explicit lifecycle info alert and dedupe receipt; rerun production-principal Recover and keep this item open until runtime readback is parse-clean and notification-visible."
"closed_at": "2026-07-11T20:25:26+08:00",
"next_action": "Keep parse/content health and lifecycle notification receipts under regression monitoring; continue with AG99-P0-006."
},
{
"id": "AG99-P0-002",
@@ -432,8 +456,8 @@
"verifier": "Read back identical trace, final state and verifier evidence from AWOOOI and AwoooP.",
"callback": "This work item creates the canonical callback contract.",
"acceptance": "One trace is visible from alert receipt through final verifier, notification and recurrence counters.",
"implementation_progress": "The first closure slice is being implemented in Agent99 runtime: Recover now has a dedicated Traditional-Chinese lifecycle summary, a phase timeline image, a successful info-event delivery policy and an explicit dedupe receipt. Canonical authenticated AWOOOI/AwoooP/IWOOOS writeback remains open.",
"next_action": "Verify the Windows 99 recovery lifecycle receipt, then define the canonical callback schema and idempotent authenticated AWOOOI/AwoooP/IWOOOS endpoint."
"implementation_progress": "Recover lifecycle visibility is runtime verified on Windows 99. Source now adds an authenticated callback endpoint using the existing Agent99 relay token, idempotent AWOOOI operation receipts, an AwoooP completed shadow run, durable DB readback, bounded Agent99 pending retries and no-secret payload validation. Focused callback/recovery tests pass 39/39; production Gitea CD and one live same-trace replay remain open.",
"next_action": "Commit the callback source, obtain the coordinated single-writer push slot, deploy API and Agent99 client in order, then replay one SRE alert and verify AWOOOI/AwoooP/IWOOOS plus Telegram receipts share the same trace."
},
{
"id": "AG99-P0-007",
@@ -791,14 +815,13 @@
"p0_count": 14,
"p1_count": 8,
"p2_count": 3,
"in_progress_count": 10,
"in_progress_count": 9,
"in_progress_blocked_count": 0,
"planned_count": 14,
"complete_count": 1,
"complete_count": 2,
"current_p0_count": 1
},
"next_execution_order": [
"AG99-P0-001",
"AG99-P0-006",
"AG99-P0-003",
"AG99-P0-004",

View File

@@ -79,6 +79,8 @@ Windows 99 VMware task 已經由 `scripts/reboot-recovery/windows99-vmware-autos
2026-07-11 runtime closureinstaller runs `host112-guest-recovery-20260711T193137+0800-35698``host112-guest-recovery-20260711T195449+0800-43505` 已完成備份、受限公鑰、exact-command sudo、timer 與 durable rollback installer未 reboot、未改 VM power、未讀 secret。110 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-200338` 讀回 112 `systemd=running`、Console/Wazuh/timer/guest `1/1/1/1`、五項 Prometheus gauges `5/5``host112_* blocker=0`。兩條 112 告警規則 health `2/2 ok` 且 healthy state 為 inactivesource/canonical/active/runtime rule SHA 均為 `86fd0e56cb7633da2a5a09960b8bf2dd61ee917c55072b5a16ba68577e8379cc`。Windows99 verifier 同輪讀到 `VM_PROCESS alias=112 process_id=22288 session_id=0 owner=NT_AUTHORITY\SYSTEM ui_truth=runtime_running_gui_session_isolated`,因此 operator power truth 固定信任 VMX process + guest readiness不信任跨 principal 的互動式 VMware UI。Agent99 `Recover` 也已把 112 納入必要 phase 並完成 production-principal verifier完整 receipt 見 `docs/operations/host112-guest-recovery-20260711.snapshot.json`。Agent99 tests `49 passed`、reboot-recovery tests `142 passed`。目前 scorecard 僅保留 `all_required_hosts_not_in_10_minute_reboot_window``fresh_all_host_reboot_event_missing``host_boot_observation_older_than_target_window`;它們只能由下一次真實全主機 600 秒 drill 關閉。
2026-07-11 20:25 Agent99 false-green closureWindows runtime 已原子部署 `06509e071c8b7c538feb112e4e4774ee22abf7d1`deploy receipt `agent99-deploy-20260711-202021.json`、rollback `backup-20260711-202021`、manifest `14/14` matched。新版 preflight 為 tasks `9/9`、required evidence fresh/usable `7/7`、parse/content failure `0`。production-principal run `agent99-recovery-20260711-202159-3404b5b3` 在 42.9 秒完成 5/5 VM、5/5 host、5/5 public 與七個 phaseTG `sent=1``visualSent=1`,實際圖卡 `agent99-card-recovery_slo_result-20260711-202159.png` 為 1200×760 phase timeline下一輪同狀態以 `dedupe_window` 抑制且留下 receipt。這關閉 Agent99 可見性假綠,不改寫原始全主機 T+10 breach。
### 2026-07-10 P0 Agent99 Transport / Verified Remediation 覆蓋
99 的 Agent99 是全主機恢復控制節點。Windows 開機後由既有 Agent99 scheduled tasks 啟動;`Status` 讀到 boot identity 變更時,只能用 single-flight queue 觸發一次 `Recover`,並以 10 分鐘 cooldown 避免併發恢復。每次執行必須寫入 `bootState``recoveryTrigger``recoveryCoordinator``recoverySlo` evidence。只有實際全主機重啟事件、七主機 reachability、必要服務、public routes、資料 freshness、備份、edge fallback、VMware/Windows policy 與 188 hygiene 全部在同一個 10 分鐘 window 通過,才可宣稱 reboot recovery SLO 達標。

View File

@@ -1,6 +1,6 @@
# Agent99 全域 AI 自動化主計畫
> 建立時間2026-07-10 15:05 CST最近更新2026-07-11 20:14 CST
> 建立時間2026-07-10 15:05 CST最近更新2026-07-11 20:34 CST
> 狀態P0 執行中,尚未達成全域 AI 自動化
> 機器可讀總帳:`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json`
> 歷史需求來源:`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md`
@@ -97,13 +97,13 @@
## 5. 更新後的唯一執行順序
2026-07-11 20:14 live audit 重新校正後,機器可讀執行順序固定為:`AG99-P0-001 → P0-006 → P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011`先關閉 Agent99 假綠與看不見的執行結果,再擴充 ingress、evidence/candidate、verifier 與各 domain 自動化。下表的數字是穩定 work-item ID不再被誤當目前執行順位`next_execution_order` 才是 runtime projection 的唯一順位來源,且必須包含每個未完成 P0 恰好一次。
2026-07-11 20:34 runtime 驗證後,機器可讀執行順序固定為:`AG99-P0-006 → P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011``P0-001` 的假綠與看不見結果已由 99 runtime receipt 關閉,現在先完成 canonical callback,再擴充 ingress、evidence/candidate、verifier 與各 domain 自動化。下表的數字是穩定 work-item ID不再被誤當目前執行順位`next_execution_order` 才是 runtime projection 的唯一順位來源,且必須包含每個未完成 P0 恰好一次。
### Wave 0先修正 AI 決策真相
| 順序 | ID | 工作 | 狀態 | 驗收 |
| --- | --- | --- | --- | --- |
| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved`、evidence parse 假綠,對齊 99 runtime script 與 source SHA | **重新開啟current P0** | Production 99 outcome self-test 通過required evidence 必須 fresh 且可解析Recover 必須有 sent 或明確 suppressed receiptruntime manifest matched、drift `0` |
| 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved`、evidence parse 假綠,對齊 99 runtime script 與 source SHA | **完成99 runtime `06509e071`** | required evidence `7/7` fresh+parsed+content healthyRecover 7/7 phase verifiedTG `sent=1``visualSent=1`runtime manifest 14/14 matched |
| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping、actionable Telegram card 漏接與重複事件 | **完成99 runtime `34e32f6e`、14/14 matched** | Alertmanager、AI card、ACTION REQUIRED、CI/CD failure 共用 dispatcher`suggestedMode`/kind 優先5 分鐘先 group同 fingerprint 只建立一個 active incident保留 acceptance receipt |
| 3 | AG99-P0-003 | 強化 ingressrelay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕producer `auto_repair`/policy 被保留;所有 request 可追 trace |
| 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate不再錯送 Perf/AwoooRepair`INC-*` 可從收件追到 candidate |
@@ -113,7 +113,7 @@
| 順序 | ID | 工作 | 狀態 | 驗收 |
| --- | --- | --- | --- | --- |
| 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition失敗不寫成功 cooldown |
| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | **進行中;先關閉 Recover 可見生命週期 receipt** | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂Telegram 可讀回報只從結果物件產生 |
| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | **進行中;current P0** | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂Telegram 可讀回報只從結果物件產生 |
| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中;本輪 breach 後已 late recoveryL1/L2 runtime 已整合L0/600 秒重測待完成** | L0 off-LAN detection/OOB power、L1 Agent99/VMware、L2 110 full-SOP 必須以同一 incident 閉環;首可用/全綠時間與 blocker/ETA 可查 |
| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | **本次 boot runtime greencontrolled apply/rollback/verifier 完成,待下次 reboot 驗證 startup 時序** | 110/188/112/120/121 VMX、autostart task、guest running 與 no-auto-reboot verifier green111 由實體 Mac readback 驗證 |
@@ -200,6 +200,13 @@
- 同一 preflight 對最新 SRE/Telegram inbox safe summary 顯示 `RuntimeException`,卻仍因檔案時間新而回傳綠燈。這是明確 false-green regression不可用更多 dashboard 掩蓋。
- current P0 因此重新定位為 `AG99-P0-001`,第二順位是 `AG99-P0-006`:先把 required evidence 改成 fresh + parseable、Recover 成功事件形成繁中生命週期卡與 phase timeline、dedupe 也留下 receipt驗證後才回到 secure ingress、candidate、verifier 與 reboot/各 domain 主線。
### 2026-07-11 20:34 P0-001 runtime closure 與 P0-006 source
- Agent99 原子部署 receipt `agent99-deploy-20260711-202021.json` 回報 source revision `06509e071c8b7c538feb112e4e4774ee22abf7d1`、14/14 runtime files matchedrollback 是 `backup-20260711-202021`
- 新版 live preflight 在 20:20:51 回報 9/9 tasks、relay=1、queue=0required evidence `fresh=7/7``usable=7/7`、parse/content failure 都是 0不再用檔案時間掩蓋 inbox JSON 解析失敗。
- Production-principal Recover `agent99-recovery-20260711-202159-3404b5b3` 完成 5/5 VM、5/5 host、5/5 public 與 7/7 recovery phases42.9 秒、coordinator verified、service-recovery only。TG receipt 是 `sent=1``visualSent=1`,實際 1200×760 圖卡顯示七階段時間軸。後續同狀態 run 以 `dedupe_window` suppressed 並保留 receipt。
- `P0-006` source 已新增 token-authenticated callback、idempotent channel event、AWOOOI operation receipts、AwoooP completed shadow run、durable DB readback以及 99 pending/processed retry queuepayload 禁止 secret、raw log 與完整 evidence path。Focused tests 39 passed、ruff green。此段尚未經 Gitea CD 與 production replay所以維持 in progress。
## 7. 工作項強制欄位與治理
每個工作項必須先有:`id`、priority/order/status、scope、owner lane、dependencies、acceptance 與 next action。任何工作項升到 `candidate_ready``in_progress``executing` 前,還必須補齊 problem、trigger、evidence sources、AI role、action mode、controlled actions、prohibited actions、rollback、mode-specific verifier、callback、evidence refs 與 last update缺一不可執行。
@@ -215,4 +222,4 @@
## 8. 立即下一步
目前唯一 P0 是 `AG99-P0-001`。立即動作是把 parse-aware preflight、Recover 生命週期卡、phase timeline 與 dedupe receipt 原子部署 99,再由 production principal 跑一次 Recover只有 runtime manifest matched、preflight parse-clean、Telegram sent 或 dedupe-suppressed receipt 明確存在才前進。後續固定依序:`P0-006 → P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011`。main push 只在唯一 writer slot 進行;沒有 slot 時保留 local commit不讓推送協調打斷 runtime 主線。
目前唯一 P0 是 `AG99-P0-006`。立即動作是提交 callback source依單一 writer 協調取得 main push slot讓 API 先經 Gitea CD 上線;再部署 99 callback client重播一個受控 SRE alert要求同一 callback/run/trace 在 AWOOOI operation log、AwoooP Run、IWOOOS owner plane、Agent99 local receipt 與 Telegram 都可讀回。後續固定依序:`P0-003 → P0-004 → P0-005 → P0-007 → P0-008 → P0-009 → P0-013 → P0-012 → P0-014 → P0-010 → P0-011`。main push 只在唯一 writer slot 進行;沒有 slot 時保留 local commit不讓推送協調打斷 runtime 主線。