diff --git a/AGENT99-AI-WORKSTATION-SOP.md b/AGENT99-AI-WORKSTATION-SOP.md index 089e2f233..bbb4baba4 100644 --- a/AGENT99-AI-WORKSTATION-SOP.md +++ b/AGENT99-AI-WORKSTATION-SOP.md @@ -735,6 +735,9 @@ Agent99 now uses a source-trusted deployment path, a dedicated bounded SSH trans - Every SSH call has a timeout; timeout cleanup terminates the SSH process tree and verifies that the client exited. - Every Agent99 process acquires the shared `C:\Wooo\Agent99\state\ssh-transport.lock` with an exclusive file handle before starting SSH. This serializes scheduled tasks and operator runs across Windows sessions; the OS releases the handle if a process exits. - Lock acquisition waits at most 90 seconds. Timeout evidence is `ssh_transport_lock_timeout`; it must not start another SSH client or be reported as a service outage. + - Each performance row records `transportSerialized` and `transportLockWaitMs`, so lock behavior is verifier evidence rather than an implementation-only claim. + - Production scheduled tasks run as the local `WOOO` S4U principal. An interactive `Administrator` run that cannot read `keys\agent99_ed25519` is an identity mismatch, not host downtime; do not broaden the private-key ACL just to make an operator shell pass. + - A long-running SRE alert relay remains healthy while `Wooo-Agent99-SRE-Alert-Relay` is `Running/ok`; stale process-start evidence is annotated as `running_task_supersedes_stale_start_evidence` and does not page. - Before controlled `Perf` or `LoadShed`, the stale-process guard only considers Agent99 allowlisted targets using `BatchMode=yes` and explicitly excludes `-N`, `-L`, `-R`, and `-D` tunnels. - A process is eligible after 15 minutes with a missing parent, or after the 60-minute hard limit. Evidence records selected process metadata but never the command line. - Live receipts: diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index c14b7a7a0..37ac06a52 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -1657,8 +1657,10 @@ df -P / 2>/dev/null slowReadTimeoutSeconds = $slowTimeoutSeconds readbackSlowRetry = $slowRetryUsed readbackElapsedMs = $readback.elapsedMs + transportSerialized = if ($readback.PSObject.Properties["transportSerialized"]) { [bool]$readback.transportSerialized } else { $false } + transportLockWaitMs = if ($readback.PSObject.Properties["transportLockWaitMs"]) { [long]$readback.transportLockWaitMs } else { $null } } - Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" + Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) { Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert } @@ -1840,7 +1842,14 @@ function Get-AgentSelfHealthConfig { [pscustomobject]@{ name = "backup_health"; pattern = "agent99-BackupCheck-*.json"; maxAgeMinutes = 30; required = $false }, [pscustomobject]@{ name = "startup_recovery"; pattern = "agent99-Recover-*.json"; maxAgeMinutes = 1440; required = $false }, [pscustomobject]@{ name = "telegram_inbox"; pattern = "agent99-TelegramInbox-*.json"; maxAgeMinutes = 15; required = $false }, - [pscustomobject]@{ name = "sre_alert_inbox"; pattern = "agent99-SreAlertInbox-*.json"; maxAgeMinutes = 15; required = $false } + [pscustomobject]@{ name = "sre_alert_inbox"; pattern = "agent99-SreAlertInbox-*.json"; maxAgeMinutes = 15; required = $false }, + [pscustomobject]@{ + name = "sre_alert_relay" + pattern = "agent99-SreAlertRelay-start-*.json" + maxAgeMinutes = 1440 + required = $false + runningTaskName = "Wooo-Agent99-SRE-Alert-Relay" + } ) [pscustomobject]@{ @@ -1930,7 +1939,10 @@ function Test-AgentScheduledTaskHealth { } function Test-AgentEvidenceFreshness { - param([object[]]$Contracts) + param( + [object[]]$Contracts, + [object[]]$TaskHealth = @() + ) $results = @() foreach ($contract in $Contracts) { @@ -1940,8 +1952,39 @@ function Test-AgentEvidenceFreshness { $required = Get-AgentBooleanSetting $contract "required" $true $latest = Get-AgentLatestEvidence $pattern $fresh = [bool]($latest.exists -and $null -eq $latest.parseError -and $latest.ageMinutes -le $maxAgeMinutes) - $ok = [bool](((-not $required) -and (-not $latest.exists)) -or $fresh) + $runningTaskName = if ($contract.PSObject.Properties["runningTaskName"]) { + [string]$contract.runningTaskName + } elseif ($name -eq "sre_alert_relay") { + # Backward-compatible migration for live configs written before runningTaskName existed. + "Wooo-Agent99-SRE-Alert-Relay" + } else { + $null + } + $runningTask = if ($runningTaskName) { + $TaskHealth | Where-Object { $_.name -eq $runningTaskName } | Select-Object -First 1 + } else { + $null + } + $runningTaskHealthy = [bool]($runningTask -and $runningTask.ok -and $runningTask.state -eq "Running") + $staleStartEvidence = [bool]( + $latest.exists -and + $null -eq $latest.parseError -and + $latest.ageMinutes -gt $maxAgeMinutes + ) + $freshnessSupersededByRunningTask = [bool]($runningTaskName -and $runningTaskHealthy -and $staleStartEvidence) + $ok = [bool](((-not $required) -and (-not $latest.exists)) -or $fresh -or $freshnessSupersededByRunningTask) $severity = if ($ok) { "ok" } elseif ($required) { "critical" } else { "warning" } + $reason = if ($fresh) { + "evidence_fresh" + } elseif ($freshnessSupersededByRunningTask) { + "running_task_supersedes_stale_start_evidence" + } elseif ((-not $required) -and (-not $latest.exists)) { + "optional_evidence_missing" + } elseif ($latest.parseError) { + "evidence_parse_failed" + } else { + "evidence_stale_or_missing" + } $results += [pscustomobject]@{ name = $name @@ -1955,8 +1998,12 @@ function Test-AgentEvidenceFreshness { ageMinutes = $latest.ageMinutes maxAgeMinutes = $maxAgeMinutes parseError = $latest.parseError + reason = $reason + runningTaskName = $runningTaskName + runningTaskHealthy = $runningTaskHealthy + freshnessSupersededByRunningTask = $freshnessSupersededByRunningTask } - Write-AgentLog "selfcheck evidence name=$name ok=$ok severity=$severity ageMinutes=$($latest.ageMinutes) maxAgeMinutes=$maxAgeMinutes file=$($latest.fileName)" + Write-AgentLog "selfcheck evidence name=$name ok=$ok severity=$severity reason=$reason ageMinutes=$($latest.ageMinutes) maxAgeMinutes=$maxAgeMinutes file=$($latest.fileName) runningTask=$runningTaskName runningTaskHealthy=$runningTaskHealthy" } $results @@ -2167,7 +2214,7 @@ function Test-AgentRuntimeManifest { function Test-AgentSelfHealth { $selfConfig = Get-AgentSelfHealthConfig $tasks = Test-AgentScheduledTaskHealth $selfConfig.scheduledTasks - $evidence = Test-AgentEvidenceFreshness $selfConfig.evidenceFreshness + $evidence = Test-AgentEvidenceFreshness $selfConfig.evidenceFreshness $tasks $latestPerf = Test-AgentLatestPerformanceEvidence $selfConfig.requiredHosts $runtimeManifest = Test-AgentRuntimeManifest $telegram = if ($selfConfig.requireRecentTelegramDelivery) { diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 4d90f5567..627aedc97 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -290,7 +290,8 @@ "name": "sre_alert_relay", "pattern": "agent99-SreAlertRelay-start-*.json", "maxAgeMinutes": 1440, - "required": false + "required": false, + "runningTaskName": "Wooo-Agent99-SRE-Alert-Relay" } ] }, diff --git a/agent99.config.example.json b/agent99.config.example.json index 411cc6ca1..c168af891 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -226,7 +226,8 @@ "name": "sre_alert_relay", "pattern": "agent99-SreAlertRelay-start-*.json", "maxAgeMinutes": 1440, - "required": false + "required": false, + "runningTaskName": "Wooo-Agent99-SRE-Alert-Relay" } ] }, diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index bdc47a1df..8f3f17666 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -136,3 +136,20 @@ def test_agent99_serializes_ssh_across_scheduled_task_processes() -> None: "lockTimeoutSeconds": 90, "lockPollMilliseconds": 200, } + + +def test_agent99_emits_transport_wait_and_keeps_running_relay_healthy() -> None: + source = CONTROL.read_text(encoding="utf-8") + config = json.loads((ROOT / "agent99.config.99.example.json").read_text()) + relay = next( + item + for item in config["selfHealth"]["evidenceFreshness"] + if item["name"] == "sre_alert_relay" + ) + + assert "transportLockWaitMs = if" in source + assert "transportSerialized = if" in source + assert "running_task_supersedes_stale_start_evidence" in source + assert "freshnessSupersededByRunningTask" in source + assert "Test-AgentEvidenceFreshness $selfConfig.evidenceFreshness $tasks" in source + assert relay["runningTaskName"] == "Wooo-Agent99-SRE-Alert-Relay" diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index c15ccbe08..aeab166b4 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.25 +> **版本**: v5.26 > **最後更新**: 2026-07-10 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-11 Agent99 多排程 SSH contention 與 false-down readback 收斂 +> **觸發事件**: 2026-07-11 Agent99 多排程 SSH contention、執行身份與 false-down readback 收斂 --- @@ -29,12 +29,16 @@ Agent99 SSH transport 使用 99 本機專用 Ed25519 identity `C:\Wooo\Agent99\k 所有 SSH command 都必須 bounded timeout。Timeout 時先用 `taskkill /T` 終止 process tree,再確認 process 是否仍存活;`Perf` / `LoadShed` controlled apply 執行前會清理只符合以下全部條件的 stale client:Agent99 allowlisted target、`BatchMode=yes`、沒有 `-N/-L/-R/-D` tunnel、超過 15 分鐘且 parent 已消失,或超過 60 分鐘 hard limit。guard 不記錄 command line,不碰互動式 SSH 或 tunnel,結果寫入 `sshProcessGuard` evidence。 -Agent99 的 Heartbeat、Performance、手動 Status 或告警修復可能由不同 Windows process 同時啟動。所有 `Invoke-SshText` 必須先取得 `C:\Wooo\Agent99\state\ssh-transport.lock` 的 exclusive file handle,跨 SYSTEM scheduled task 與 RDP operator session 一次只允許一條 Agent99 SSH command;程序結束時由 OS 自動釋放 handle。預設最多等待 90 秒,逾時回 `ssh_transport_lock_timeout`,不得再開新 client。若同輪五主機 ping 全通,但 Harbor、AWOOOI 與所有 performance readback 同時空白,優先判定為 transport contention,必須等序列化後重驗,不得宣稱所有服務同時故障。 +Agent99 的 Heartbeat、Performance、手動 Status 或告警修復可能由不同 Windows process 同時啟動。所有 `Invoke-SshText` 必須先取得 `C:\Wooo\Agent99\state\ssh-transport.lock` 的 exclusive file handle,跨 scheduled task process 與 RDP operator session 一次只允許一條 Agent99 SSH command;程序結束時由 OS 自動釋放 handle。預設最多等待 90 秒,逾時回 `ssh_transport_lock_timeout`,不得再開新 client。若同輪五主機 ping 全通,但 Harbor、AWOOOI 與所有 performance readback 同時空白,優先判定為 transport contention,必須等序列化後重驗,不得宣稱所有服務同時故障。 + +99 的正式 Agent99 scheduled tasks 目前使用本機 `WOOO` 帳號的 S4U principal;專用 private key ACL 不授權 RDP 的 `Administrator`。因此 operator 直接執行 `agent99-run.ps1` 若出現 `Load key ... Permission denied`,只能判定為 execution-identity mismatch,不得判定五台主機或服務故障,也不得為了測試放寬 private-key ACL。正式 verifier 必須用 `schtasks /run /tn Wooo-Agent99-Heartbeat` 與 `Wooo-Agent99-Performance-Guard` 走既有 principal;evidence 必須包含 `transportSerialized` 與 `transportLockWaitMs`。長駐 `Wooo-Agent99-SRE-Alert-Relay` 若 task 為 `Running/ok`,不得因 start evidence 超過 1440 分鐘就製造 freshness warning;應記錄 `running_task_supersedes_stale_start_evidence`。 效能修復不得以 command exit code 取代 post-verifier。只要 action 配置 verifier,成功判定完全由 verifier 決定;verifier 失敗必須記為 failed,僅建立 10 分鐘 failure backoff,不得建立 success cooldown。policy 變更必須提升 `policyVersion`,由 deployer migration 更新 99 live config,避免舊 command 或 marker 繼續造成 false green。 本輪 live receipt:`agent99-load-shed-marker-correction.json` 已封存 188 舊 false-success marker,`agent99-stale-ssh-cleanup-v2.json` 已受控終止 7 個逾時 orphan SSH client;`agent99-Perf-20260710-224454.json` 顯示 110、112、120、121、188 全部 readback OK,route 分別為 direct、direct、via110、via110、via110。110 load/core `0.60`、disk `84%`;188 load/core `0.18`、disk 由 `90%` 降至 `88%`。這證明 transport、效能讀回與 verifier 修復 lane 已恢復,不等於已完成下一次實際全主機 cold-start 的 10 分鐘 SLA 驗收。 +2026-07-11 production-principal concurrency receipt:同時觸發 Heartbeat 與 Performance Guard 後,`agent99-Status-20260711-004148.json` 顯示五台 `ping=true`、AWOOOI `coreHealthy=true` / `redisUp=true`、Harbor `registryHealthy=true` / `deployReady=true`;`agent99-Perf-20260711-004149.json` 顯示五台 `severity=ok`,110 / 112 / 120 / 121 / 188 load/core 為 `0.54 / 0.01 / 0.06 / 0.01 / 0.11`,disk 為 `84 / 53 / 49 / 27 / 89%`。同輪 public routes 全部 `200`。這是排程身份與並行 readback closure,仍不得取代下一次真實全主機重啟的 10 分鐘 SLA drill。 + ### 2026-07-10 P0 備份/Gitea 自動驗證覆蓋 每次 host reboot 後,備份 lane 必須依序執行 `/backup/scripts/backup-status.sh --no-notify`、`/backup/scripts/verify-gitea-backup.sh --no-notify`,再由 110 的每日排程執行 `/backup/scripts/gitea-bundle-backup-sync-188.sh`。最後讀回 110/188 `backup_health.prom`、Gitea restore status、bundle sync status、188 sample restore metric 與 Prometheus Backup/Gitea alerts;不得只看檔案 mtime、HTTP 200 或命令 exit code。