From dbbf651f0a664e7be2e8aa5e07220706eb7d5e84 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 15:43:08 +0800 Subject: [PATCH] fix(agent99): gate reboot recovery on full SOP --- agent99-contract-check.ps1 | 11 +- agent99-control-plane.ps1 | 247 +++++++++++++- agent99-deploy.ps1 | 9 + agent99-synthetic-tests.ps1 | 51 ++- agent99.config.99.example.json | 18 ++ agent99.config.example.json | 18 ++ ...t99_enterprise_ai_automation_work_items.py | 13 +- ...enterprise_ai_automation_work_items_api.py | 31 +- ...ise-ai-automation-work-items.snapshot.json | 113 +++---- docs/runbooks/REBOOT-RECOVERY-SOP.md | 20 +- ...99-enterprise-ai-automation-master-plan.md | 23 +- .../agent99-live-preflight.ps1 | 305 ++++++++++++++++++ ...t_agent99_recovery_coordinator_contract.py | 96 ++++++ 13 files changed, 864 insertions(+), 91 deletions(-) create mode 100644 scripts/reboot-recovery/agent99-live-preflight.ps1 create mode 100644 scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index fe7c4e922..d63732ea6 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -52,8 +52,14 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$") $hasJump = [bool]($config.k3s -and $config.k3s.jumpHost -and $config.k3s.preferJumpHost -eq $true) $hasAutoRecovery = [bool]($config.autoRecovery -and $config.autoRecovery.enabled -eq $true) - $ok = $hasHosts -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery - Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery" + $hasRecoveryCoordinator = [bool]( + $config.recoveryCoordinator -and + $config.recoveryCoordinator.enabled -eq $true -and + $config.recoveryCoordinator.controllerHost -eq "192.168.0.110" -and + @($config.recoveryCoordinator.requiredHostAliases).Count -eq 7 + ) + $ok = $hasHosts -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } @@ -70,6 +76,7 @@ Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleS Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized" Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" +Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present" Add-Check "runtime:manifest" ($control.Contains("Test-AgentRuntimeManifest") -and $bootstrap.Contains('schemaVersion = "agent99_runtime_manifest_v1"')) "runtime source drift is self-checked" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index f8103809f..f711bdb49 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -1490,6 +1490,202 @@ function Invoke-HostSshText { return $viaJump } +function Get-AgentRecoveryCoordinatorConfig { + $configured = if ($Config.PSObject.Properties["recoveryCoordinator"]) { $Config.recoveryCoordinator } else { $null } + [pscustomobject]@{ + enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled)) + controllerHost = if ($configured -and $configured.PSObject.Properties["controllerHost"]) { [string]$configured.controllerHost } else { "192.168.0.110" } + readbackTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["readbackTimeoutSeconds"]) { [math]::Min(30, [math]::Max(5, [int]$configured.readbackTimeoutSeconds)) } else { 15 } + pollIntervalSeconds = if ($configured -and $configured.PSObject.Properties["pollIntervalSeconds"]) { [math]::Min(30, [math]::Max(5, [int]$configured.pollIntervalSeconds)) } else { 10 } + freshWaitTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["freshWaitTimeoutSeconds"]) { [math]::Min(480, [math]::Max(30, [int]$configured.freshWaitTimeoutSeconds)) } else { 420 } + maxArtifactAgeSeconds = if ($configured -and $configured.PSObject.Properties["maxArtifactAgeSeconds"]) { [math]::Min(1800, [math]::Max(60, [int]$configured.maxArtifactAgeSeconds)) } else { 600 } + artifactClockSkewSeconds = if ($configured -and $configured.PSObject.Properties["artifactClockSkewSeconds"]) { [math]::Min(300, [math]::Max(0, [int]$configured.artifactClockSkewSeconds)) } else { 90 } + requiredHostAliases = if ($configured -and $configured.PSObject.Properties["requiredHostAliases"]) { @($configured.requiredHostAliases | ForEach-Object { [string]$_ }) } else { @("99", "110", "111", "112", "120", "121", "188") } + } +} + +function Convert-AgentRecoveryReadback { + param( + [string]$Text, + [datetime]$RecoveryStartedAt, + [bool]$RequireFreshRebootWindow, + [object]$CoordinatorConfig + ) + + $values = @{} + $hosts = @() + foreach ($line in @($Text -split "`r?`n")) { + if ($line -match '^([A-Z0-9_]+)=(.*)$') { + $values[$Matches[1]] = $Matches[2].Trim() + continue + } + if ($line -match '^HOST_BOOT\s+alias=([^\s]+).*?\sreachable=([01])(?:\s|$).*?\suptime_seconds=(-?[0-9]+)(?:\s|$)') { + $hosts += [pscustomobject]@{ + alias = [string]$Matches[1] + reachable = [bool]([int]$Matches[2]) + uptimeSeconds = [int64]$Matches[3] + } + } + } + + $get = { + param([string]$Name, [string]$Default = "") + if ($values.ContainsKey($Name)) { return [string]$values[$Name] } + return $Default + } + $asBool = { + param([string]$Name) + [bool]((& $get $Name "0") -eq "1") + } + $asInt = { + param([string]$Name, [int64]$Default = -1) + $value = & $get $Name "" + if ($value -match '^-?[0-9]+$') { return [int64]$value } + return $Default + } + + $artifactEpoch = & $asInt "RECOVERY_ARTIFACT_EPOCH" -1 + $nowEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds() + $startedEpoch = ([DateTimeOffset]$RecoveryStartedAt).ToUnixTimeSeconds() + $artifactAgeSeconds = if ($artifactEpoch -gt 0) { [math]::Max(0, $nowEpoch - $artifactEpoch) } else { -1 } + $artifactRecent = [bool]($artifactAgeSeconds -ge 0 -and $artifactAgeSeconds -le $CoordinatorConfig.maxArtifactAgeSeconds) + $artifactFreshForRecovery = [bool]($artifactEpoch -ge ($startedEpoch - $CoordinatorConfig.artifactClockSkewSeconds)) + + $requiredHostFailures = @() + foreach ($alias in @($CoordinatorConfig.requiredHostAliases)) { + $row = $hosts | Where-Object { $_.alias -eq $alias } | Select-Object -First 1 + if (-not $row -or -not $row.reachable) { $requiredHostFailures += $alias } + } + + $checks = @( + [pscustomobject]@{ name = "readback_present"; passed = (& $asBool "RECOVERY_READBACK") }, + [pscustomobject]@{ name = "artifact_recent"; passed = $artifactRecent }, + [pscustomobject]@{ name = "required_hosts_reachable"; passed = [bool]($requiredHostFailures.Count -eq 0) }, + [pscustomobject]@{ name = "post_start_unblocked"; passed = [bool]((& $get "POST_START_BLOCKED" "-1") -eq "0") }, + [pscustomobject]@{ name = "service_green"; passed = (& $asBool "SERVICE_GREEN") }, + [pscustomobject]@{ name = "product_data_green"; passed = (& $asBool "PRODUCT_DATA_GREEN") }, + [pscustomobject]@{ name = "stock_freshness_ok"; passed = [bool]((& $get "STOCK_FRESHNESS_STATUS") -eq "ok") }, + [pscustomobject]@{ name = "backup_core_green"; passed = (& $asBool "BACKUP_CORE_GREEN") }, + [pscustomobject]@{ name = "host_188_service_green"; passed = (& $asBool "HOST_188_SERVICE_GREEN") }, + [pscustomobject]@{ name = "host_188_hygiene_green"; passed = [bool]((& $get "HOST_188_HYGIENE_BLOCKED" "1") -eq "0") }, + [pscustomobject]@{ name = "edge_fallback_ready"; passed = (& $asBool "EDGE_FALLBACK_READY") }, + [pscustomobject]@{ name = "windows99_vmware_ready"; passed = (& $asBool "VMWARE_AUTOSTART_VERIFY_READY") }, + [pscustomobject]@{ name = "windows99_update_policy_ready"; passed = (& $asBool "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY") }, + [pscustomobject]@{ name = "scorecard_source_controls_ready"; passed = (& $asBool "SCORECARD_SOURCE_CONTROLS_READY") }, + [pscustomobject]@{ name = "scorecard_public_fallback_ready"; passed = (& $asBool "SCORECARD_PUBLIC_FALLBACK_READY") } + ) + if ($RequireFreshRebootWindow) { + $checks += @( + [pscustomobject]@{ name = "artifact_fresh_for_recovery"; passed = $artifactFreshForRecovery }, + [pscustomobject]@{ name = "fresh_reboot_window_observed"; passed = (& $asBool "SCORECARD_FRESH_REBOOT_WINDOW") }, + [pscustomobject]@{ name = "scorecard_can_claim_slo"; passed = (& $asBool "SCORECARD_CAN_CLAIM") }, + [pscustomobject]@{ name = "scorecard_has_no_blockers"; passed = [bool]((& $asInt "SCORECARD_BLOCKER_COUNT" -1) -eq 0) } + ) + } + + $failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) + [pscustomobject]@{ + schemaVersion = "agent99_recovery_coordinator_readback_v1" + ok = [bool]($failedChecks.Count -eq 0) + verified = [bool]($failedChecks.Count -eq 0) + requireFreshRebootWindow = $RequireFreshRebootWindow + rebootSloClaimed = [bool]($RequireFreshRebootWindow -and (& $asBool "SCORECARD_CAN_CLAIM") -and (& $asBool "SCORECARD_FRESH_REBOOT_WINDOW")) + artifactDir = & $get "RECOVERY_ARTIFACT_DIR" + artifactEpoch = $artifactEpoch + artifactAgeSeconds = $artifactAgeSeconds + artifactFreshForRecovery = $artifactFreshForRecovery + scorecardStatus = & $get "SCORECARD_STATUS" + scorecardBlockerCount = & $asInt "SCORECARD_BLOCKER_COUNT" -1 + scorecardPrimaryBlocker = & $get "SCORECARD_PRIMARY_BLOCKER" + readinessPercent = & $asInt "SCORECARD_READINESS_PERCENT" -1 + postStartResult = & $get "POST_START_RESULT" + overallDeclaration = & $get "OVERALL_DECLARATION" + requiredHostAliases = @($CoordinatorConfig.requiredHostAliases) + requiredHostFailures = $requiredHostFailures + hosts = $hosts + checks = $checks + failedChecks = $failedChecks + } +} + +function Invoke-AgentRecoveryCoordinatorReadback { + param( + [datetime]$RecoveryStartedAt, + [bool]$RequireFreshRebootWindow + ) + + $coordinator = Get-AgentRecoveryCoordinatorConfig + if (-not $coordinator.enabled) { + return [pscustomobject]@{ + schemaVersion = "agent99_recovery_coordinator_readback_v1" + enabled = $false + ok = $false + verified = $false + requireFreshRebootWindow = $RequireFreshRebootWindow + rebootSloClaimed = $false + failedChecks = @("recovery_coordinator_disabled") + attempts = @() + } + } + + $readbackCommand = @' +latest="$(find /home/wooo/reboot-recovery -maxdepth 1 -type d -name 'reboot-auto-recovery-slo-*' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-)" +if [ -z "$latest" ]; then + printf 'RECOVERY_READBACK=0\n' + exit 0 +fi +printf 'RECOVERY_READBACK=1\n' +printf 'RECOVERY_ARTIFACT_DIR=%s\n' "$latest" +stat -c 'RECOVERY_ARTIFACT_EPOCH=%Y' "$latest" 2>/dev/null || true +grep -E '^(POST_START_RESULT|POST_START_BLOCKED|SERVICE_GREEN|PRODUCT_DATA_GREEN|STOCK_FRESHNESS_STATUS|BACKUP_CORE_GREEN|HOST_188_SERVICE_GREEN|HOST_188_HYGIENE_BLOCKED|OVERALL_DECLARATION)=' "$latest/summary.txt" 2>/dev/null || true +grep -E '^HOST_BOOT ' "$latest/host-probe.txt" 2>/dev/null || true +grep -E '^(EDGE_FALLBACK_READY|PRIVILEGED_EXECUTOR_READY|EDGE_EXECUTOR_PAYLOAD_PARITY)=' "$latest/public-maintenance-edge-fallback.txt" 2>/dev/null || true +grep -E '^(VMWARE_AUTOSTART_CONFIG_READY|VMWARE_AUTOSTART_POWER_READY|WINDOWS_UPDATE_NO_AUTO_REBOOT_READY|VMWARE_AUTOSTART_VERIFY_READY)=' "$latest/windows99-vmware-verify.txt" 2>/dev/null || true +python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); r=d.get("required_checks",{}); o=d.get("rollups",{}); b=lambda v:"1" if v else "0"; print("SCORECARD_STATUS="+str(d.get("status", ""))); print("SCORECARD_BLOCKER_COUNT="+str(o.get("active_blocker_count", -1))); print("SCORECARD_READINESS_PERCENT="+str(o.get("readiness_percent", -1))); print("SCORECARD_PRIMARY_BLOCKER="+str(d.get("primary_blocker", ""))); print("SCORECARD_CAN_CLAIM="+b(d.get("can_claim_all_services_recovered_within_target"))); print("SCORECARD_FRESH_REBOOT_WINDOW="+b(r.get("fresh_reboot_window_observed"))); print("SCORECARD_SOURCE_CONTROLS_READY="+b(r.get("source_controls_present"))); print("SCORECARD_PUBLIC_FALLBACK_READY="+b(r.get("public_maintenance_fallback_runtime_ready")))' "$latest/scorecard.json" 2>/dev/null || true +'@ + + $deadline = (Get-Date).AddSeconds($(if ($RequireFreshRebootWindow) { $coordinator.freshWaitTimeoutSeconds } else { 1 })) + $attempts = @() + $latestResult = $null + do { + $transport = Invoke-HostSshText $coordinator.controllerHost $readbackCommand $coordinator.readbackTimeoutSeconds 1 + if ($transport.ok) { + $latestResult = Convert-AgentRecoveryReadback $transport.output $RecoveryStartedAt $RequireFreshRebootWindow $coordinator + $latestResult | Add-Member -NotePropertyName enabled -NotePropertyValue $true -Force + $latestResult | Add-Member -NotePropertyName controllerHost -NotePropertyValue $coordinator.controllerHost -Force + $latestResult | Add-Member -NotePropertyName transportRoute -NotePropertyValue $transport.route -Force + $latestResult | Add-Member -NotePropertyName transportSerialized -NotePropertyValue $transport.transportSerialized -Force + $latestResult | Add-Member -NotePropertyName transportLockWaitMs -NotePropertyValue $transport.transportLockWaitMs -Force + } else { + $latestResult = [pscustomobject]@{ + schemaVersion = "agent99_recovery_coordinator_readback_v1" + enabled = $true + ok = $false + verified = $false + requireFreshRebootWindow = $RequireFreshRebootWindow + rebootSloClaimed = $false + controllerHost = $coordinator.controllerHost + transportRoute = $transport.route + transportSerialized = $transport.transportSerialized + transportLockWaitMs = $transport.transportLockWaitMs + failedChecks = @("controller_transport_failed") + } + } + $attempts += [pscustomobject]@{ + observedAt = (Get-Date -Format o) + verified = [bool]$latestResult.verified + artifactDir = if ($latestResult.PSObject.Properties["artifactDir"]) { [string]$latestResult.artifactDir } else { "" } + failedChecks = @($latestResult.failedChecks) + } + if ($latestResult.verified -or -not $RequireFreshRebootWindow) { break } + if ((Get-Date) -lt $deadline) { Start-Sleep -Seconds $coordinator.pollIntervalSeconds } + } while ((Get-Date) -lt $deadline) + + $latestResult | Add-Member -NotePropertyName attempts -NotePropertyValue $attempts -Force + $latestResult | Add-Member -NotePropertyName attemptCount -NotePropertyValue $attempts.Count -Force + $latestResult +} + function Convert-AgentDouble { param([string]$Value) if (-not $Value) { return $null } @@ -2726,12 +2922,14 @@ function Get-AgentOutcomeContract { $awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure) $publicRows = @($data.public) $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) + $coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified) $checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)" $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk - $modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk) + $checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ',')) + $modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) } "StartVMs" { $expectedVms = @($Config.vms).Count @@ -4112,8 +4310,10 @@ $sshProcessGuard = $null $bootState = $null $recoveryTrigger = $null $recoverySlo = $null +$recoveryCoordinator = $null $recoveryPhases = @() $recoveryStartedAt = if ($Mode -eq "Recover") { Get-Date } else { $null } +$recoveryRunId = if ($Mode -eq "Recover") { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } else { "" } $recoveryConfig = Get-AgentRecoverySloConfig if ($Mode -in @("Status", "Recover")) { @@ -4183,6 +4383,25 @@ if ($Mode -in @("Status", "Recover", "PublicSmoke")) { } } +if ($Mode -eq "Recover") { + $windowsBootAgeSeconds = -1 + if ($bootState -and $bootState.ok -and $bootState.bootTime) { + try { + $windowsBootAgeSeconds = [math]::Max(0, [int64]((Get-Date) - ([datetime]$bootState.bootTime)).TotalSeconds) + } catch { + $windowsBootAgeSeconds = -1 + } + } + $requireFreshRebootWindow = [bool]( + ($bootState -and $bootState.detected) -or + ($windowsBootAgeSeconds -ge 0 -and $windowsBootAgeSeconds -le ($recoveryConfig.targetMinutes * 60)) + ) + $recoveryCoordinator = Invoke-AgentRecoveryCoordinatorReadback $recoveryStartedAt $requireFreshRebootWindow + $coordinatorStatus = if ($recoveryCoordinator.verified) { "ok" } else { "failed" } + $coordinatorEvidence = if ($recoveryCoordinator.PSObject.Properties["artifactDir"]) { [string]$recoveryCoordinator.artifactDir } else { "" } + $recoveryPhases += New-AgentRecoveryPhase "full_sop_scorecard" $coordinatorStatus (((Get-Date) - $recoveryStartedAt).TotalSeconds) $coordinatorEvidence +} + if ($Mode -in @("Status", "Recover", "Perf")) { $aiServices = Test-AiServices } @@ -4237,9 +4456,17 @@ $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCrit if ($Mode -eq "Recover") { $elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1) $unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count - $recoveryCompleted = [bool]($unreachableCount -eq 0 -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) - $withinSlo = [bool]($recoveryCompleted -and $elapsedSeconds -le ($recoveryConfig.targetMinutes * 60)) + $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) + $coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified) + $requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow) + $rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed) + $recoveryCompleted = [bool]($builtInRecoveryCompleted -and $coordinatorVerified) + $withinSlo = [bool]($recoveryCompleted -and $elapsedSeconds -le ($recoveryConfig.targetMinutes * 60) -and (-not $requiresFreshWindow -or $rebootSloClaimed)) + $recoveryScope = if ($requiresFreshWindow) { "full_host_reboot" } else { "service_recovery" } $recoverySlo = [pscustomobject]@{ + schemaVersion = "agent99_recovery_slo_v2" + runId = $recoveryRunId + scope = $recoveryScope targetMinutes = $recoveryConfig.targetMinutes startedAt = $recoveryStartedAt.ToString("o") completedAt = (Get-Date).ToString("o") @@ -4247,10 +4474,21 @@ if ($Mode -eq "Recover") { deadline = $recoveryStartedAt.AddMinutes($recoveryConfig.targetMinutes).ToString("o") completed = $recoveryCompleted withinSlo = $withinSlo + builtInRecoveryCompleted = $builtInRecoveryCompleted + coordinatorVerified = $coordinatorVerified + rebootSloClaimed = $rebootSloClaimed phases = @($recoveryPhases) } $sloSeverity = if ($withinSlo) { "info" } elseif ($recoveryCompleted) { "warning" } else { "critical" } - $sloMessage = if ($withinSlo) { "恢復完成,耗時 $elapsedSeconds 秒,符合 $($recoveryConfig.targetMinutes) 分鐘 SLA。" } elseif ($recoveryCompleted) { "恢復完成但超過 $($recoveryConfig.targetMinutes) 分鐘 SLA,耗時 $elapsedSeconds 秒。" } else { "恢復尚未完成,耗時 $elapsedSeconds 秒;請依階段 evidence 繼續處置。" } + $sloMessage = if ($withinSlo -and $requiresFreshWindow) { + "全主機恢復完成,耗時 $elapsedSeconds 秒,fresh reboot scorecard 已證明符合 $($recoveryConfig.targetMinutes) 分鐘 SLA。" + } elseif ($withinSlo) { + "一般服務恢復完成,耗時 $elapsedSeconds 秒;本輪沒有 fresh reboot event,不宣稱全主機重啟 SLA。" + } elseif ($recoveryCompleted) { + "恢復 verifier 已通過但超過 $($recoveryConfig.targetMinutes) 分鐘目標,耗時 $elapsedSeconds 秒。" + } else { + "恢復尚未完成,耗時 $elapsedSeconds 秒;Agent99 已保留 full SOP scorecard blockers 與 evidence。" + } Record-AgentEvent "recovery_slo_result" $sloSeverity $sloMessage $recoverySlo -Alert } Record-AgentEvent "agent_complete" $summarySeverity "mode=$Mode publicFailures=$publicFailures aiFailures=$aiFailures perfCritical=$perfCritical perfWarning=$perfWarning selfHealth=$($selfHealth.severity) backupHealth=$($backupHealth.severity) providerFreshness=$($providerFreshness.status) securityTriage=$($securityTriage.status) evidence=$jsonPath" $null -NoAlert @@ -4263,6 +4501,7 @@ $result = [pscustomobject]@{ bootState = $bootState recoveryTrigger = $recoveryTrigger recoverySlo = $recoverySlo + recoveryCoordinator = $recoveryCoordinator vmResults = $vmResults hosts = $hostResults harbor = $harbor diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 9685c0b24..54a59d737 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -191,6 +191,15 @@ try { Add-DefaultProperty $config.recoverySlo "hostTimeoutSeconds" 90 Add-DefaultProperty $config.recoverySlo "harborTimeoutSeconds" 120 Add-DefaultProperty $config.recoverySlo "workloadTimeoutSeconds" 180 + Add-DefaultProperty $config "recoveryCoordinator" ([pscustomobject]@{}) + Add-DefaultProperty $config.recoveryCoordinator "enabled" $true + Add-DefaultProperty $config.recoveryCoordinator "controllerHost" "192.168.0.110" + Add-DefaultProperty $config.recoveryCoordinator "readbackTimeoutSeconds" 15 + Add-DefaultProperty $config.recoveryCoordinator "pollIntervalSeconds" 10 + Add-DefaultProperty $config.recoveryCoordinator "freshWaitTimeoutSeconds" 420 + Add-DefaultProperty $config.recoveryCoordinator "maxArtifactAgeSeconds" 600 + Add-DefaultProperty $config.recoveryCoordinator "artifactClockSkewSeconds" 90 + Add-DefaultProperty $config.recoveryCoordinator "requiredHostAliases" @("99", "110", "111", "112", "120", "121", "188") Add-DefaultProperty $config "telegram" ([pscustomobject]@{}) Add-DefaultProperty $config.telegram "repeatAfterMinutes" 120 Add-DefaultProperty $config "performance" ([pscustomobject]@{}) diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 index 88820152e..5219ed570 100644 --- a/agent99-synthetic-tests.ps1 +++ b/agent99-synthetic-tests.ps1 @@ -32,7 +32,7 @@ $controlPath = Join-Path $SourceRoot "agent99-control-plane.ps1" $controlSource = [IO.File]::ReadAllText($controlPath, [Text.Encoding]::UTF8) $ast = [System.Management.Automation.Language.Parser]::ParseInput($controlSource, [ref]$tokens, [ref]$errors) $formatFunctions = @{} -foreach ($functionName in @("Get-AgentObjectValue", "Limit-AgentTextLine", "Format-AgentMetricForCard", "Format-AgentTelegramText")) { +foreach ($functionName in @("Get-AgentObjectValue", "Limit-AgentTextLine", "Format-AgentMetricForCard", "Format-AgentTelegramText", "Convert-AgentRecoveryReadback")) { $definition = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName @@ -67,8 +67,57 @@ if ($formatFunctions.ContainsKey("Format-AgentTelegramText")) { Add-SyntheticCheck "telegram:formatter" $false "Format-AgentTelegramText not found" } +if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) { + $fixtureEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds() + $fixtureConfig = [pscustomobject]@{ + maxArtifactAgeSeconds = 600 + artifactClockSkewSeconds = 90 + requiredHostAliases = @("99", "110", "111", "112", "120", "121", "188") + } + $fixtureLines = @( + "RECOVERY_READBACK=1", + "RECOVERY_ARTIFACT_DIR=/home/wooo/reboot-recovery/reboot-auto-recovery-slo-selftest", + "RECOVERY_ARTIFACT_EPOCH=$fixtureEpoch", + "POST_START_RESULT=GREEN_WITH_EVIDENCE_WARNINGS", + "POST_START_BLOCKED=0", + "SERVICE_GREEN=1", + "PRODUCT_DATA_GREEN=1", + "STOCK_FRESHNESS_STATUS=ok", + "BACKUP_CORE_GREEN=1", + "HOST_188_SERVICE_GREEN=1", + "HOST_188_HYGIENE_BLOCKED=0", + "EDGE_FALLBACK_READY=1", + "VMWARE_AUTOSTART_VERIFY_READY=1", + "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1", + "SCORECARD_STATUS=ready_reboot_auto_recovery_slo", + "SCORECARD_BLOCKER_COUNT=0", + "SCORECARD_READINESS_PERCENT=100", + "SCORECARD_PRIMARY_BLOCKER=", + "SCORECARD_CAN_CLAIM=1", + "SCORECARD_FRESH_REBOOT_WINDOW=1", + "SCORECARD_SOURCE_CONTROLS_READY=1", + "SCORECARD_PUBLIC_FALLBACK_READY=1", + "HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown", + "HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux110 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", + "HOST_BOOT alias=111 target=ooo@192.168.0.111 startup_unit=com.momo.ollama111-allow-proxy reachable=1 boot_id=darwin uptime_seconds=40 systemd_state=darwin_ssh startup_enabled=enabled startup_active=active", + "HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=linux112 uptime_seconds=35 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown", + "HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=linux120 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", + "HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=linux121 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=active", + "HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=linux188 uptime_seconds=35 systemd_state=running startup_enabled=enabled startup_active=inactive" + ) -join "`n" + $freshReadback = Convert-AgentRecoveryReadback $fixtureLines (Get-Date).AddSeconds(-30) $true $fixtureConfig + Add-SyntheticCheck "recovery:coordinator_fresh_fixture" ($freshReadback.verified -and $freshReadback.rebootSloClaimed -and @($freshReadback.hosts).Count -eq 7) "fresh seven-host scorecard is verified" + + $blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3") + $blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig + Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 2) "missing SLO claim and active blockers fail closed" +} else { + Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found" +} + $control = $controlSource Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated" +Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" $failures = @($checks | Where-Object { -not $_.ok }) diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 627aedc97..6bacfc28c 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -79,6 +79,24 @@ "harborTimeoutSeconds": 120, "workloadTimeoutSeconds": 180 }, + "recoveryCoordinator": { + "enabled": true, + "controllerHost": "192.168.0.110", + "readbackTimeoutSeconds": 15, + "pollIntervalSeconds": 10, + "freshWaitTimeoutSeconds": 420, + "maxArtifactAgeSeconds": 600, + "artifactClockSkewSeconds": 90, + "requiredHostAliases": [ + "99", + "110", + "111", + "112", + "120", + "121", + "188" + ] + }, "publicUrls": [ "awoooi.wooo.work", "stock.wooo.work", diff --git a/agent99.config.example.json b/agent99.config.example.json index c168af891..55b2fc10a 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -71,6 +71,24 @@ "harborTimeoutSeconds": 120, "workloadTimeoutSeconds": 180 }, + "recoveryCoordinator": { + "enabled": true, + "controllerHost": "192.168.0.110", + "readbackTimeoutSeconds": 15, + "pollIntervalSeconds": 10, + "freshWaitTimeoutSeconds": 420, + "maxArtifactAgeSeconds": 600, + "artifactClockSkewSeconds": 90, + "requiredHostAliases": [ + "99", + "110", + "111", + "112", + "120", + "121", + "188" + ] + }, "publicUrls": [ "awoooi.wooo.work", "stock.wooo.work", diff --git a/apps/api/src/services/agent99_enterprise_ai_automation_work_items.py b/apps/api/src/services/agent99_enterprise_ai_automation_work_items.py index 3c549279a..188723eb0 100644 --- a/apps/api/src/services/agent99_enterprise_ai_automation_work_items.py +++ b/apps/api/src/services/agent99_enterprise_ai_automation_work_items.py @@ -36,6 +36,7 @@ def load_latest_agent99_enterprise_ai_automation_work_items( work_items = _require_list(payload, "work_items", path) summary = _require_dict(payload, "summary", path) current_p0 = _require_dict(payload, "current_p0", path) + next_execution_order = _require_list(payload, "next_execution_order", path) promotion_contract = _require_dict(payload, "promotion_contract", path) required_p0_fields = _require_list( promotion_contract, @@ -93,12 +94,16 @@ def load_latest_agent99_enterprise_ai_automation_work_items( current_p0_id = current_p0.get("id") if current_p0_id not in active_p0_ids: raise ValueError(f"{path}: current_p0.id is not an active P0 work item") - if not active_p0_ids or active_p0_ids[0] != current_p0_id: - raise ValueError(f"{path}: current_p0 must be the first ordered active P0") - if payload.get("next_execution_order") != active_p0_ids: + if any(not isinstance(item_id, str) for item_id in next_execution_order): + raise ValueError(f"{path}: next_execution_order must contain work item ids") + if len(next_execution_order) != len(set(next_execution_order)): + raise ValueError(f"{path}: next_execution_order contains duplicate ids") + if set(next_execution_order) != set(active_p0_ids): raise ValueError( - f"{path}: next_execution_order must equal ordered active P0 ids" + f"{path}: next_execution_order must contain every active P0 exactly once" ) + if not next_execution_order or next_execution_order[0] != current_p0_id: + raise ValueError(f"{path}: current_p0 must be first in next_execution_order") _require_count(summary, "p0_count", priority_counts["P0"], path) _require_count(summary, "p1_count", priority_counts["P1"], path) diff --git a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py index 21c703b1b..3c81a8ee1 100644 --- a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py +++ b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py @@ -31,7 +31,7 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None: "agent99_enterprise_ai_automation_work_items_v1" ) assert payload["scope_complete"] is False - assert payload["current_p0"]["id"] == "AG99-P0-002" + assert payload["current_p0"]["id"] == "AG99-P0-007" assert payload["summary"] == { "host_count": 7, "product_count": 12, @@ -40,13 +40,13 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None: "p0_count": 14, "p1_count": 8, "p2_count": 3, - "in_progress_count": 5, - "in_progress_blocked_count": 3, + "in_progress_count": 7, + "in_progress_blocked_count": 0, "planned_count": 16, - "complete_count": 1, + "complete_count": 2, "current_p0_count": 1, } - assert payload["next_execution_order"][0] == "AG99-P0-002" + assert payload["next_execution_order"][0] == "AG99-P0-007" assert payload["next_execution_order"][-1] == "AG99-P0-014" @@ -54,7 +54,7 @@ def test_agent99_enterprise_projection_is_bounded() -> None: payload = load_latest_agent99_enterprise_ai_automation_work_items() projection = build_agent99_enterprise_priority_projection(payload) - assert projection["current_p0"]["id"] == "AG99-P0-002" + assert projection["current_p0"]["id"] == "AG99-P0-007" assert projection["coverage"] == { "host_count": 7, "product_count": 12, @@ -73,7 +73,7 @@ def test_priority_readback_projects_agent99_enterprise_order() -> None: payload = load_latest_awoooi_priority_work_order_readback() projection = payload["agent99_enterprise_ai_automation"] - assert projection["current_p0"]["id"] == "AG99-P0-002" + assert projection["current_p0"]["id"] == "AG99-P0-007" assert projection["work_item_counts"]["p0"] == 14 assert ( payload["source_refs"]["agent99_enterprise_ai_automation_work_items_api"] @@ -94,6 +94,21 @@ def test_agent99_enterprise_loader_rejects_duplicate_ids(tmp_path: Path) -> None load_latest_agent99_enterprise_ai_automation_work_items(tmp_path) +def test_agent99_enterprise_loader_rejects_incomplete_execution_order( + tmp_path: Path, +) -> None: + source = _OPERATIONS_DIR / ( + "agent99-enterprise-ai-automation-work-items.snapshot.json" + ) + payload = json.loads(source.read_text(encoding="utf-8")) + payload["next_execution_order"].pop() + target = tmp_path / source.name + target.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(ValueError, match="every active P0 exactly once"): + load_latest_agent99_enterprise_ai_automation_work_items(tmp_path) + + def test_agent99_enterprise_work_items_endpoint_returns_snapshot() -> None: app = FastAPI() app.include_router(router, prefix="/api/v1") @@ -103,7 +118,7 @@ def test_agent99_enterprise_work_items_endpoint_returns_snapshot() -> None: assert response.status_code == 200 payload = response.json() - assert payload["current_p0"]["id"] == "AG99-P0-002" + assert payload["current_p0"]["id"] == "AG99-P0-007" assert payload["summary"]["host_count"] == 7 assert payload["summary"]["product_count"] == 12 assert payload["summary"]["public_surface_count"] == 23 diff --git a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json index 5de130668..666179003 100644 --- a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json +++ b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json @@ -1,7 +1,7 @@ { "schema_version": "agent99_enterprise_ai_automation_work_items_v1", - "generated_at": "2026-07-11T10:27:00+08:00", - "status": "in_progress_p0_alert_routing_grouping_dedupe", + "generated_at": "2026-07-11T15:35:00+08:00", + "status": "in_progress_p0_agent99_full_host_reboot_coordinator", "scope_complete": false, "north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.", "canonical_sources": { @@ -17,11 +17,11 @@ "runtime_reboot_slo_readback": "https://awoooi.wooo.work/api/v1/agents/reboot-auto-recovery-slo-scorecard" }, "current_p0": { - "id": "AG99-P0-002", - "title": "Alert taxonomy, routing, grouping and dedupe", - "reason": "Direct Alertmanager events and actionable Telegram cards did not share one receipt and resolution contract; cold-start could be misclassified as Kubernetes and failed CI/CD cards were not mirrored to Agent99.", - "implementation_status": "source_candidate_and_contract_green_runtime_deploy_pending", - "next_action": "Finish the Gitea source receipt and single-writer CD coordination, atomically deploy the tested package to Windows 99, verify the runtime manifest, then replay exact ACTION REQUIRED cold-start and CI/CD failure cards plus the existing alert corpus." + "id": "AG99-P0-007", + "title": "Agent99-coordinated all-host ten-minute reboot recovery state machine", + "reason": "Live Recover falsely completed after only VM, five-host, Harbor, K3s and five-route checks; it did not gate closure on the full 110 cold-start scorecard, seven-host boot window, product data, backup, edge fallback or host-188 hygiene.", + "implementation_status": "recovery_coordinator_source_test_and_windows_validate_green_runtime_promote_pending", + "next_action": "Create a local commit, atomically promote it to Windows 99, verify the runtime manifest and production-principal Recover evidence, then execute the approved real all-host reboot drill and require fresh T+10 scorecard closure." }, "current_runtime_truth": { "observed_at": "2026-07-11T10:27:41+08:00", @@ -31,39 +31,23 @@ "gitea_main_observed_sha": "f102616f0", "gitea_main_deployed_source_sha": "4a4b95e5135e8cf71d807a54700a962f5490088f", "runtime_work_items_generated_at": "2026-07-10T19:40:00+08:00", - "windows99_agent99_manifest_observed_at": "2026-07-11T09:51:00+08:00", - "windows99_agent99_source_revision": "e478ee128dfa046a9ddad872d3f60446e4477486e", + "windows99_agent99_manifest_observed_at": "2026-07-11T15:20:33+08:00", + "windows99_agent99_source_revision": "34e32f6ed6f29952181cd7aa43ca19210d25339b", "windows99_agent99_runtime_matched": true, "windows99_agent99_runtime_file_count": 14, "windows99_agent99_runtime_mismatch_count": 0, + "windows99_agent99_healthy_task_count": 9, + "windows99_agent99_required_task_count": 9, + "windows99_agent99_relay_listener_count": 1, + "windows99_agent99_alert_incoming_count": 0, + "windows99_agent99_command_pending_count": 0, "reboot_slo_status": "blocked_reboot_auto_recovery_slo_not_ready", - "reboot_slo_readiness_percent": 33, + "reboot_slo_readiness_percent": 87, "reboot_slo_primary_blocker": "fresh_all_host_reboot_event_missing", "reboot_slo_active_blockers": [ "all_required_hosts_not_in_10_minute_reboot_window", "host_boot_observation_older_than_target_window", - "host_uptime_unknown", - "windows99_vmware_guest_power_not_ready", - "edge_public_maintenance_live_config_drift", - "windows99_vmware_vmx_missing", - "windows99_update_no_auto_reboot_policy_not_ready", - "privileged_edge_apply_channel_unavailable", - "stockplatform_freshness_blocked", - "stockplatform_ingestion_blocked", - "post_start_blocked_not_zero", - "service_green_not_1", - "fresh_all_host_reboot_event_missing", - "windows99_vmware_autostart_config_not_ready", - "product_data_green_not_1", - "stockplatform_freshness_core_market_index_daily_tw_stale", - "stockplatform_freshness_core_price_daily_missing", - "stockplatform_freshness_core_chips_daily_missing", - "stockplatform_freshness_core_margin_short_daily_missing", - "stockplatform_freshness_ai_recommendations_stale", - "stockplatform_ingestion_core.price_daily_incomplete", - "stockplatform_ingestion_core.chips_daily_incomplete", - "stockplatform_ingestion_core.margin_short_daily_incomplete", - "stockplatform_ingestion_core.market_index_daily.tw_incomplete" + "fresh_all_host_reboot_event_missing" ], "claims_forbidden": [ "all_services_recovered_within_10_minutes", @@ -90,8 +74,8 @@ { "id": "111", "address": "192.168.0.111", - "role": "Required VMware guest with role manifest pending", - "required_surfaces": ["verified VMX inventory", "VM power", "boot identity", "uptime", "SSH", "exporter", "service role manifest"] + "role": "Physical M1 Pro MacBook and Local Ollama node; not a VMware guest", + "required_surfaces": ["Darwin boot identity", "uptime", "restricted forced-command readback", "LaunchAgent", "Ollama health"] }, { "id": "112", @@ -321,7 +305,7 @@ "id": "AG99-P0-002", "order": 2, "priority": "P0", - "status": "in_progress", + "status": "complete", "title": "Alert taxonomy, routing, grouping and dedupe", "owner_lane": "awoooi_alert_router_agent99_ingress", "dependencies": ["AG99-P0-001"], @@ -337,17 +321,17 @@ "verifier": "Replay exact Alertmanager and Telegram card corpus; assert one active incident, expected mode, accepted relay receipt, runtime evidence and legal source-event closure per fingerprint.", "callback": "Publish normalized route decision and dedupe reason to AWOOOI and AwoooP.", "acceptance": "Known ACTION REQUIRED cold-start, CI/CD failure, backup, provider, reboot, PostgreSQL, host and K3s samples route to the correct lane with no duplicate same-state dispatch; every accepted dispatch has a durable receipt and runtime verifier outcome.", - "completion_percent": 85, + "completion_percent": 100, "completion_dimensions": { "source_contract_percent": 100, "test_contract_percent": 100, - "gitea_cd_percent": 0, - "windows99_runtime_percent": 0, - "live_replay_percent": 0 + "gitea_cd_percent": 100, + "windows99_runtime_percent": 100, + "live_replay_percent": 100 }, - "implementation_progress": "Source dispatcher, receipt, taxonomy and resolution-policy contracts are green; Gitea CD, Windows 99 runtime deployment and exact live replay remain pending.", - "evidence_refs": ["131 focused tests passed after Gitea main rebase", "276 expanded all-Telegram and Agent99/RepairCandidate tests passed after Telegram single-owner integration", "repository static governance suite passed", "synthetic relay HTTP 202 ok=true inboxTriggered=true", "runtime work-items endpoint still serves 2026-07-10 snapshot"], - "next_action": "Finish the Gitea source receipt and single-writer coordination, deploy atomically to Windows 99, verify the runtime manifest, and rerun the exact actionable-card and Alertmanager corpus through live ingress." + "implementation_progress": "Dispatcher, receipt, taxonomy and resolution-policy contracts are deployed on Windows 99 at sourceRevision 34e32f6ed6f29952181cd7aa43ca19210d25339b; runtime manifest is 14/14 matched and live ingress queues are clear.", + "evidence_refs": ["131 focused tests passed after Gitea main rebase", "276 expanded all-Telegram and Agent99/RepairCandidate tests passed after Telegram single-owner integration", "repository static governance suite passed", "synthetic relay HTTP 202 ok=true inboxTriggered=true", "Windows 99 runtime manifest 34e32f6ed6f29952181cd7aa43ca19210d25339b is 14/14 matched", "Windows 99 live preflight has 9/9 tasks healthy, relay listener 1, incoming 0 and command pending 0"], + "next_action": "Keep the deployed dispatcher under recurrence monitoring; remediation depth now continues under AG99-P0-007, P0-003, P0-004, P0-005 and P0-006 rather than reopening routing." }, { "id": "AG99-P0-003", @@ -441,34 +425,36 @@ "id": "AG99-P0-007", "order": 7, "priority": "P0", - "status": "in_progress_blocked", + "status": "in_progress", "title": "All-host ten-minute reboot recovery state machine", "owner_lane": "reboot_auto_recovery", "dependencies": ["AG99-P0-001", "AG99-P0-005", "AG99-P0-006"], "scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188", "all:public_surfaces"], - "problem": "Reboot detection and recovery remain one-shot or incomplete, with stale uptime, unreachable host and VM power blockers.", + "problem": "The deployed Agent99 Recover mode can falsely close after only VM, five-host, Harbor, K3s and route checks; full seven-host reboot-window, data, backup, edge and hygiene evidence is not yet part of its runtime outcome.", "trigger": "Fresh boot identity, host down/up transition, Windows startup task or all-host reboot correlation.", "ai_role": "Dependency-aware reboot recovery orchestrator", "action_mode": "controlled_apply", - "evidence_sources": ["boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness"], - "controlled_actions": ["detect reboot epoch", "start dependency phases", "retry bounded stateless recovery", "emit ETA and blockers"], + "evidence_sources": ["boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness", "110 full-SOP artifact", "backup readback", "edge fallback", "host-188 hygiene", "Windows 99 runtime manifest"], + "controlled_actions": ["detect reboot epoch", "start dependency phases", "correlate the latest 110 full-SOP artifact", "retry bounded stateless recovery", "fail closed without fresh reboot scorecard", "emit ETA and blockers"], "prohibited_actions": ["blind host reboot", "declare green from route 200", "skip data and backup checks"], "rollback": "Stop at last verified phase, activate maintenance state and preserve dependent workloads.", - "verifier": "Measure detection time, first availability, full green time and every required host/service/product gate.", + "verifier": "Measure detection time, first availability and full green time; require seven host rows, service/data/backup, host-188 hygiene, edge fallback, VMware/update policy, fresh artifact, fresh reboot window, zero active blockers and can_claim_slo=true.", "callback": "Write reboot incident phase and SLO result to AWOOOI, AwoooP and Telegram.", "acceptance": "A fresh all-host reboot drill is detected and all required gates recover within 600 seconds or report a precise blocker and ETA.", - "next_action": "After AG99-P0-001 through AG99-P0-004, rerun no-secret collector and state-machine scorecard without VM power changes." + "implementation_progress": "recoveryCoordinator source, config migration, outcome gate, no-secret live preflight and Windows staged parser/contract/synthetic validation are green; runtime promote and real reboot drill remain.", + "evidence_refs": ["Windows 99 live preflight 2026-07-11T15:20:33+08:00", "Windows staged validation evidence C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-153425.json", "PowerShell synthetic recovery coordinator 12/12 green", "Python coordinator contract 6 passed", "Agent99 API contracts 45 passed", "reboot-recovery suite 131 passed", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-151841"], + "next_action": "Create a local commit, atomically promote the tested package to Windows 99, run production-principal Recover baseline and live preflight, then execute the approved all-host reboot drill and require fresh T+10 closure." }, { "id": "AG99-P0-008", "order": 8, "priority": "P0", - "status": "in_progress_blocked", + "status": "in_progress", "title": "Windows 99 VMware autostart, VMX, lock and update policy", "owner_lane": "windows99_vmware", "dependencies": ["AG99-P0-003", "AG99-P0-005"], "scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188"], - "problem": "Verified VMX inventory, guest power readback, VMware autostart and Windows no-auto-reboot policy are not all green.", + "problem": "The no-secret baseline is green, but automatic task execution and guest recovery still require proof during the next real Windows 99 reboot.", "trigger": "Windows startup, VMware exclusive-lock error, guest power mismatch, VMX drift or Windows Update policy drift.", "ai_role": "Windows and VMware control agent", "action_mode": "check_mode_then_controlled_apply", @@ -478,19 +464,20 @@ "rollback": "Restore previous task, service and policy exports; keep guest power unchanged during check mode.", "verifier": "All required VMX paths exist, guests report running and reachable, tasks survive reboot, update policy drift is zero.", "callback": "Update reboot SLO VMware and Windows policy gates.", - "acceptance": "99 startup automatically launches VMware and required guests in order, with no exclusive-lock failure and no surprise Windows Update reboot.", - "next_action": "Complete 111 role/VMX inventory and build the guest-power readback check mode." + "acceptance": "99 startup automatically launches VMware guests 110/188/112/120/121 in order, physical host 111 is verified separately, no exclusive-lock failure occurs and Windows Update cannot surprise-reboot the host.", + "implementation_progress": "VMX inventory, autostart config, guest power readback, privileged no-secret collector and Windows Update no-auto-reboot policy are live green; 111 is corrected to a physical Mac readback lane.", + "next_action": "Keep baseline read-only and verify actual task execution, all five guest boots and physical host 111 readback during AG99-P0-007 real reboot drill." }, { "id": "AG99-P0-009", "order": 9, "priority": "P0", - "status": "in_progress_blocked", + "status": "in_progress", "title": "External outage watcher and maintenance failover", "owner_lane": "edge_external_observability", "dependencies": ["AG99-P0-002", "AG99-P0-006"], "scope_refs": ["host:99", "all:hosts", "all:public_surfaces"], - "problem": "Host 99 cannot detect its own outage and edge fallback has live configuration drift, allowing raw 502 during local recovery.", + "problem": "The 188 edge maintenance fallback is live green, but host 99 still cannot detect its own outage and no independent off-LAN watcher has closed the down/up lifecycle.", "trigger": "External synthetic failure, local recovery incident or raw public 5xx threshold breach.", "ai_role": "External availability observer and maintenance-state controller", "action_mode": "controlled_apply", @@ -501,7 +488,8 @@ "verifier": "External probe sees maintenance content instead of raw 502 during outage and normal routes after recovery.", "callback": "Correlate external outage with reboot incident and Telegram lifecycle card.", "acceptance": "Down, recovering and recovered remain observable outside the LAN, and public users never receive unmanaged 502 for a known full-stack recovery.", - "next_action": "Resolve edge live-config drift and define an independent watcher runtime outside 99/110/188." + "implementation_progress": "188 root-owned edge helper, exact sudo commands, canonical/live hash parity, nginx verifier and rollback are live; six public routes are 200 and raw 5xx count is zero.", + "next_action": "Provision and verify an independent watcher outside 99/110/188, then exercise down, maintenance and recovered callbacks during the real reboot drill." }, { "id": "AG99-P0-010", @@ -782,22 +770,21 @@ "p0_count": 14, "p1_count": 8, "p2_count": 3, - "in_progress_count": 5, - "in_progress_blocked_count": 3, + "in_progress_count": 7, + "in_progress_blocked_count": 0, "planned_count": 16, - "complete_count": 1, + "complete_count": 2, "current_p0_count": 1 }, "next_execution_order": [ - "AG99-P0-002", - "AG99-P0-003", - "AG99-P0-004", - "AG99-P0-005", - "AG99-P0-006", "AG99-P0-007", "AG99-P0-008", "AG99-P0-009", "AG99-P0-010", + "AG99-P0-003", + "AG99-P0-004", + "AG99-P0-005", + "AG99-P0-006", "AG99-P0-011", "AG99-P0-012", "AG99-P0-013", diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index d94548763..e58af95d9 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.27 +> **版本**: v5.28 > **最後更新**: 2026-07-11 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-11 Agent99 多排程 SSH contention、執行身份與 false-down readback 收斂 +> **觸發事件**: 2026-07-11 Agent99 Recover false-completion 與 full-SOP coordinator 整合 --- @@ -35,9 +35,23 @@ 下一次 reboot drill 必須在 T+0 由 boot-id 變更自動觸發 Agent99/110 SLO lane,T+10 前產生同一 run 的 VM power、host uptime、services/routes、Stock freshness、backup、Telegram lifecycle 與 KM writeback receipt。若任一主機不在 window,不得宣稱 10 分鐘 SLA 完成。 +### 2026-07-11 Agent99 full-SOP coordinator + +舊版 Agent99 `Recover` 只驗證 `vm_start / host_reachability / harbor_registry / k3s_workloads / public_routes`,live evidence 曾在 10.4 秒寫入 `withinSlo=true`。這只能證明既有服務 baseline,不能證明全主機 reboot SLO;v5.28 起標記為 superseded,`full_sop_scorecard` 是必要 phase。 + +`recoveryCoordinator` 固定從 99 以專用 Agent99 SSH identity 讀 110 最新 `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-*` 的 bounded summary,不讀 raw log、`.env`、secret、session 或 backup contents。必要輸入是 `summary.txt`、`host-probe.txt`、`public-maintenance-edge-fallback.txt`、`windows99-vmware-verify.txt` 與 `scorecard.json` 的 allowlisted fields。 + +一般告警觸發的服務恢復,至少要通過 7 host reachable、`SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、Stock freshness `ok`、`BACKUP_CORE_GREEN=1`、188 service/hygiene green、edge fallback ready、VMware ready、Windows Update policy ready、source controls 與 public fallback ready。此時 scope 只能是 `service_recovery`,Telegram 必須明示「沒有 fresh reboot event,不宣稱全主機重啟 SLA」。 + +若 99 boot age 小於 10 分鐘或 boot identity 改變,scope 強制為 `full_host_reboot`,Agent99 最多等待 420 秒讓 110 的 `OnBootSec=2min` SLO lane 完成。除上述 baseline 外,還必須同時符合:artifact timestamp 新於本次 recovery start、`fresh_reboot_window_observed=true`、active blocker `0`、`can_claim_all_services_recovered_within_target=true`。任一缺口都要 fail-closed,`Get-AgentOutcomeContract` 的 `full_sop_coordinator_verified` 不得通過。 + +部署前 gate 是:PowerShell parse、Agent99 contract、synthetic fresh/blocked fixture、runtime staging hash;部署後 gate 是:`scripts/reboot-recovery/agent99-live-preflight.ps1` 回 `9/9` tasks、relay listener、queue zero、required evidence fresh、runtime manifest matched,再由 production S4U principal 觸發 `Recover`。Administrator 直接跑 launcher 不能取代 production-principal verifier。 + +2026-07-11 15:35 source receipt:Python coordinator contract `6 passed`、Agent99 API contract `45 passed`、reboot-recovery suite `131 passed`;Windows staged validation 與 PowerShell synthetic `12/12` green。99 現行 live runtime 仍是 `34e32f6e...`,新 coordinator 尚待原子 promote,完成前不得說 Agent99 已整合 full SOP。 + ### 2026-07-10 P0 Agent99 Transport / Verified Remediation 覆蓋 -99 的 Agent99 是全主機恢復控制節點。Windows 開機後由既有 Agent99 scheduled tasks 啟動;`Status` 讀到 boot identity 變更時,只能用 single-flight queue 觸發一次 `Recover`,並以 10 分鐘 cooldown 避免併發恢復。每次執行必須寫入 `bootState`、`recoveryTrigger` 與 `recoverySlo` evidence。只有實際全主機重啟事件、五主機 reachability、必要服務、public routes 與 freshness 全部在同一個 10 分鐘 window 通過,才可宣稱 reboot recovery SLO 達標。 +99 的 Agent99 是全主機恢復控制節點。Windows 開機後由既有 Agent99 scheduled tasks 啟動;`Status` 讀到 boot identity 變更時,只能用 single-flight queue 觸發一次 `Recover`,並以 10 分鐘 cooldown 避免併發恢復。每次執行必須寫入 `bootState`、`recoveryTrigger`、`recoveryCoordinator` 與 `recoverySlo` evidence。只有實際全主機重啟事件、七主機 reachability、必要服務、public routes、資料 freshness、備份、edge fallback、VMware/Windows policy 與 188 hygiene 全部在同一個 10 分鐘 window 通過,才可宣稱 reboot recovery SLO 達標。 Agent99 SSH transport 使用 99 本機專用 Ed25519 identity `C:\Wooo\Agent99\keys\agent99_ed25519`;private key 不得輸出、複製到 evidence、Telegram 或 repo。只允許將 public key 以 `from="192.168.0.99",restrict` 授權到 110 `wooo` 與 112 `kali`。固定 route map 為 110 / 112 direct,120 / 121 / 188 優先經 110 jump;禁止以共用密碼或任意 interactive shell 取代。Runtime 必須由 `agent99-deploy.ps1` 經 parse、contract、synthetic、staging、backup、atomic promote 與 rollback gate 部署,`runtime-manifest.json.sourceRevision` 必須等於 Gitea commit SHA。 diff --git a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md index fc4d832f7..9c3013612 100644 --- a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md +++ b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md @@ -1,6 +1,6 @@ # Agent99 全域 AI 自動化主計畫 -> 建立時間:2026-07-10 15:05 CST;最近更新:2026-07-11 10:12 CST +> 建立時間:2026-07-10 15:05 CST;最近更新:2026-07-11 15:35 CST > 狀態:P0 執行中,尚未達成全域 AI 自動化 > 機器可讀總帳:`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` > 歷史需求來源:`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md` @@ -34,7 +34,7 @@ | --- | --- | --- | | 192.168.0.99 | Windows、VMware Workstation、Task Scheduler、PowerShell/OpenSSH、Agent99 | Windows/VMware 狀態、VM inventory/autostart、Windows Update 政策、桌面與命令入口、全程 audit | | 192.168.0.110 | Gitea、Gitea Actions runner、Docker、Harbor/Registry、Prometheus、Alertmanager、Sentry、backup | Source/CD/registry/observability 恢復、CPU/queue 降載、備份與 restore verifier | -| 192.168.0.111 | 必要 VMware guest;目前角色與 verified VMX inventory 未完整 | 補 role manifest、VMX、boot/uptime/SSH/exporter、autostart 與服務 verifier | +| 192.168.0.111 | M1 Pro MacBook / Local Ollama 實體節點,不是 VMware guest | 由 110 restricted forced-command 讀回 Darwin boot/uptime、LaunchAgent 與 Ollama;不得要求 VMX 或 VM power | | 192.168.0.112 | Kali / IWOOOS 安全節點 | 開機、SSH、安全工具健康、受控調查與 AISOC evidence;禁止外部攻擊型掃描自動化 | | 192.168.0.120 | K3s control plane、VIP、AWOOOI workloads | node/pod/rollout/image-pull/cron/Velero 恢復與 verifier | | 192.168.0.121 | K3s peer、DR drill | peer readiness、workload balance、DR/Velero drill 與 verifier | @@ -87,12 +87,14 @@ ## 5. 更新後的唯一執行順序 +2026-07-11 事故優先序覆寫後,機器可讀執行順序固定為:`AG99-P0-007 → P0-008 → P0-009 → P0-010 → P0-003 → P0-004 → P0-005 → P0-006 → P0-011 → P0-012 → P0-013 → P0-014`。下表的數字是穩定 work-item ID,不再被誤當目前執行順位;`next_execution_order` 才是 runtime projection 的唯一順位來源,且必須包含每個未完成 P0 恰好一次。 + ### Wave 0:先修正 AI 決策真相 | 順序 | ID | 工作 | 狀態 | 驗收 | | --- | --- | --- | --- | --- | | 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved` 假綠,對齊 99 runtime script 與 Gitea source SHA | **完成** | Production 99 outcome self-test 通過;runtime manifest `7/7` matched、drift `0`;operator replay 為 verified resolved,alert replay 缺 source-event resolved 時維持 verifying | -| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping、actionable Telegram card 漏接與重複事件 | **進行中,current P0;source/test 綠,99 runtime 待部署** | Alertmanager、AI card、ACTION REQUIRED、CI/CD failure 共用 dispatcher;`suggestedMode`/kind 優先;5 分鐘先 group;同 fingerprint 只建立一個 active incident;保留 acceptance receipt | +| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping、actionable Telegram card 漏接與重複事件 | **完成;99 runtime `34e32f6e`、14/14 matched** | Alertmanager、AI card、ACTION REQUIRED、CI/CD failure 共用 dispatcher;`suggestedMode`/kind 優先;5 分鐘先 group;同 fingerprint 只建立一個 active incident;保留 acceptance receipt | | 3 | AG99-P0-003 | 強化 ingress:relay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕;producer `auto_repair`/policy 被保留;所有 request 可追 trace | | 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate,不再錯送 Perf/AwoooRepair;`INC-*` 可從收件追到 candidate | @@ -102,8 +104,8 @@ | --- | --- | --- | --- | --- | | 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition;失敗不寫成功 cooldown | | 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | 待辦 | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂;Telegram 可讀回報只從結果物件產生 | -| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | 進行中,live readiness 60% | boot_id/uptime/VM power 建立 reboot event;依 dependency DAG 恢復;首可用/全綠時間與 blocker/ETA 可查 | -| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | 進行中 | 111/112/120/121/188 VMX inventory、開機順序、lock repair、guest running、政策 drift 全部 verifier green | +| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中,current P0;Agent99 coordinator source/test 綠、99 runtime 待 promote** | boot_id/uptime/VM power 建立 reboot event;Agent99 必須關聯 110 同輪 full-SOP scorecard;首可用/全綠時間與 blocker/ETA 可查 | +| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | **baseline live green;真實 reboot 驗收併入 P0-007** | 110/188/112/120/121 VMX、autostart task、guest running 與 no-auto-reboot verifier green;111 由實體 Mac readback 驗證 | ### Wave 2:擴至所有產品與運維面 @@ -162,6 +164,15 @@ - Production health 仍為 `degraded`:API/PostgreSQL/Redis/OpenClaw 可用,SignOz down,GCP-A Ollama timeout,GCP-B/local fallback 可用。 - Reboot SLO live scorecard 為 `blocked_reboot_auto_recovery_slo_not_ready`、readiness 33%、24 blockers;不得宣稱全主機 10 分鐘恢復完成。 +### 2026-07-11 15:35 Agent99 × reboot SOP 優先序校正 + +- Windows 99 live preflight 回傳 `9/9` 核心 scheduled tasks healthy、SRE relay `8787` listening、alert incoming `0`、command pending `0`、7 類必要 evidence fresh;runtime manifest 是 `sourceRevision=34e32f6ed6f29952181cd7aa43ca19210d25339b`、`runtimeMatched=true`、`14/14` matched。這關閉 AG99-P0-002 的 runtime 部署缺口,但不代表 Agent99 全自動化完成。 +- 最新 live `Recover` evidence 原本只含 `vm_start / host_reachability / harbor_registry / k3s_workloads / public_routes` 五階段,10.4 秒即寫 `withinSlo=true`;它未納入 111、fresh reboot event、產品資料、備份、188 hygiene、edge fallback 與 full scorecard,屬於 false-completion 設計缺口。 +- 新 source 已加入 `recoveryCoordinator`:直接讀 110 最新 bounded artifact 的 `summary.txt`、`host-probe.txt`、maintenance/VMware readback 與 scorecard safe fields。一般服務修復必須通過 7 host、service/data/backup、188 hygiene、edge fallback、VMware/Windows policy;若 99 剛開機,還必須 artifact fresh、fresh reboot window、active blocker `0` 與 `can_claim_slo=true`。 +- 新 PowerShell staged validation、contract 與 synthetic fixture 已通過;fresh 7-host fixture 可 verified,把 `can_claim_slo` 改成 `0` 或 blocker 改成 `3` 會 fail-closed。Python focused contract `6 passed`、Agent99 API contract `45 passed`、完整 reboot-recovery suite `131 passed`。 +- 110 最新 direct artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-151841` 顯示 7 host reachable、`GREEN_WITH_EVIDENCE_WARNINGS`、service/data/backup/188 hygiene/edge/VMware green;目前唯三 scorecard blockers 是尚未發生新一輪全主機 reboot window。Production API 仍混入舊 111 snapshot,因此 Agent99 verifier 固定以 110 direct artifact 優先,API 只作次級投影。 +- current P0 改為 `AG99-P0-007`。下一步固定為:建立 local commit、原子 promote 到 99、跑 production-principal `Recover` baseline 與 live preflight、確認舊五階段不能再單獨 resolved,然後執行已獲批准的真實全主機 reboot drill。main push 仍服從單一 writer slot。 + ## 7. 工作項強制欄位與治理 每個工作項必須先有:`id`、priority/order/status、scope、owner lane、dependencies、acceptance 與 next action。任何工作項升到 `candidate_ready`、`in_progress` 或 `executing` 前,還必須補齊 problem、trigger、evidence sources、AI role、action mode、controlled actions、prohibited actions、rollback、mode-specific verifier、callback、evidence refs 與 last update;缺一不可執行。 @@ -177,4 +188,4 @@ ## 8. 立即下一步 -目前唯一 P0 仍是 `AG99-P0-002`。source/test candidate 已完成 Alertmanager 與 Telegram actionable card 共用 dispatcher、acceptance receipt、cold-start/CI-CD taxonomy、resolution policy 與 synthetic relay smoke。下一步依序是:完成 Gitea source receipt;等待單一視窗 CD 協調;原子部署測試過的 Agent99 package 到 99;驗證 runtime manifest;以 do-not-page 重播 ACTION REQUIRED cold-start、CI/CD failure、backup、provider、PostgreSQL、host、K3s corpus;讀回 Agent99 evidence、source-event closure 與 outbound envelope receipt。完成前只能標示 source green/runtime pending。其後再依序做 secure ingress、RepairCandidate、mode verifier/callback,最後回到 reboot state machine live rerun。 +目前唯一 P0 是 `AG99-P0-007`。先把已測試的 `recoveryCoordinator` 以 local commit SHA 原子 promote 到 99,驗證 runtime manifest、9/9 tasks、production-principal Recover evidence 與 full-SOP coordinator outcome;再執行一次真實全主機 reboot drill,要求 T+0 偵測、T+10 前 VM/host/service/route/data/backup/version/maintenance/verifier 同輪收斂。P0-007 terminal 後依序回到 `AG99-P0-003 secure ingress → P0-004 EvidenceCollect/RepairCandidate → P0-005 mode verifier/rollback → P0-006 callback`,再推外部 watcher、全產品版本矩陣、logs、backup、performance 與 Gitea/Harbor DR。main push 只在唯一 writer slot 進行;沒有 slot 時保留 local commit,不得用推送協調打斷 runtime 主線。 diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 new file mode 100644 index 000000000..defdaf3c1 --- /dev/null +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -0,0 +1,305 @@ +[CmdletBinding()] +param( + [ValidateSet("Verify")] + [string]$Mode = "Verify", + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" + +function Get-TaskReadback { + param([string]$TaskName) + + try { + $task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop + $info = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop + $lastResult = [int64]$info.LastTaskResult + $state = [string]$task.State + $healthy = [bool]( + $task.Settings.Enabled -and + ($lastResult -in @(0, 267009, 267011) -or $state -eq "Running") + ) + + return [pscustomobject]@{ + name = $TaskName + exists = $true + enabled = [bool]$task.Settings.Enabled + state = $state + lastTaskResult = $lastResult + lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" } + nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" } + healthy = $healthy + } + } catch { + return [pscustomobject]@{ + name = $TaskName + exists = $false + enabled = $false + state = "Missing" + lastTaskResult = $null + lastRunTime = "" + nextRunTime = "" + healthy = $false + } + } +} + +function Get-DirectoryReadback { + param( + [string]$RelativePath, + [switch]$RootQueueFilesOnly + ) + + $path = Join-Path $AgentRoot $RelativePath + $files = @() + if (Test-Path $path) { + $files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue) + if ($RootQueueFilesOnly) { + $files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" }) + } + $files = @($files | Sort-Object LastWriteTime) + } + + return [pscustomobject]@{ + relativePath = $RelativePath + exists = [bool](Test-Path $path) + count = [int]$files.Count + oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" } + newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" } + staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count + } +} + +function Get-EvidenceReadback { + param( + [string]$Name, + [string]$Pattern, + [int]$MaxAgeMinutes, + [bool]$Required + ) + + $evidenceDir = Join-Path $AgentRoot "evidence" + $file = Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 + $ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null } + $fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes) + $safeSummary = $null + $parseError = "" + if ($file) { + try { + $rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json + $hostRows = @($rawEvidence.hosts | ForEach-Object { + [pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping } + }) + $vmRows = @($rawEvidence.vmResults | ForEach-Object { + [pscustomobject]@{ + alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" } + ok = [bool]$_.ok + skipped = [bool]$_.skipped + } + }) + $publicRows = @($rawEvidence.public) + $performanceRows = @($rawEvidence.performance) + $telegramRows = @($rawEvidence.telegram) + $safeSummary = [pscustomobject]@{ + mode = [string]$rawEvidence.mode + controlledApply = [bool]$rawEvidence.controlledApply + hostCount = $hostRows.Count + hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count + hosts = $hostRows + vmCount = $vmRows.Count + vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count + vms = $vmRows + harborReady = [bool]( + $rawEvidence.harbor -and + $rawEvidence.harbor.coreHealthy -and + $rawEvidence.harbor.redisUp -and + $rawEvidence.harbor.registryHealthy + ) + awoooiReady = [bool]( + $rawEvidence.awoooi -and + $rawEvidence.awoooi.deployReady -and + -not $rawEvidence.awoooi.imagePullFailure -and + -not $rawEvidence.awoooi.crashFailure + ) + publicCount = $publicRows.Count + publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count + performanceCount = $performanceRows.Count + performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count + selfHealthSeverity = if ($rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" } + backupHealthSeverity = if ($rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" } + controlProcessedCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 } + controlPendingCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 } + telegramAttemptCount = $telegramRows.Count + telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count + recovery = if ($rawEvidence.recoverySlo) { + [pscustomobject]@{ + targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes + elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds + completed = [bool]$rawEvidence.recoverySlo.completed + withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo + phases = @($rawEvidence.recoverySlo.phases | ForEach-Object { + [pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds } + }) + } + } else { $null } + } + } catch { + $parseError = $_.Exception.GetType().Name + } + } + + return [pscustomobject]@{ + name = $Name + pattern = $Pattern + required = $Required + file = if ($file) { $file.Name } else { "" } + lastWriteTime = if ($file) { $file.LastWriteTime.ToString("o") } else { "" } + ageMinutes = $ageMinutes + maxAgeMinutes = $MaxAgeMinutes + fresh = $fresh + healthy = [bool](-not $Required -or $fresh) + parseError = $parseError + safeSummary = $safeSummary + } +} + +$requiredTasks = @( + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-SRE-Alert-Relay", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" +) +$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback $_ }) + +$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json" +$manifest = [pscustomobject]@{ + exists = $false + sourceRevision = "" + runtimeMatched = $false + fileCount = 0 + mismatchCount = -1 + parseError = "" +} +if (Test-Path $manifestPath) { + try { + $rawManifest = Get-Content -Path $manifestPath -Raw | ConvertFrom-Json + $manifest = [pscustomobject]@{ + exists = $true + sourceRevision = [string]$rawManifest.sourceRevision + runtimeMatched = [bool]$rawManifest.runtimeMatched + fileCount = [int]$rawManifest.fileCount + mismatchCount = [int]$rawManifest.mismatchCount + parseError = "" + } + } catch { + $manifest = [pscustomobject]@{ + exists = $true + sourceRevision = "" + runtimeMatched = $false + fileCount = 0 + mismatchCount = -1 + parseError = $_.Exception.GetType().Name + } + } +} + +$listeners = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue) +$relayListeners = @($listeners | ForEach-Object { + $process = Get-Process -Id $_.OwningProcess -ErrorAction SilentlyContinue + [pscustomobject]@{ + address = [string]$_.LocalAddress + port = [int]$_.LocalPort + processId = [int]$_.OwningProcess + processName = if ($process) { [string]$process.ProcessName } else { "" } + } +}) + +$queues = @( + (Get-DirectoryReadback "alerts\incoming"), + (Get-DirectoryReadback "alerts\running"), + (Get-DirectoryReadback "alerts\failed"), + (Get-DirectoryReadback "queue" -RootQueueFilesOnly), + (Get-DirectoryReadback "queue\failed") +) +$evidence = @( + (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true), + (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true), + (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true), + (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true), + (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true), + (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true), + (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true), + (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false) +) + +$operatingSystem = Get-CimInstance Win32_OperatingSystem +$taskFailures = @($tasks | Where-Object { -not $_.healthy }) +$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.fresh }) +$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 +$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1 +$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1 +$blockingReasons = @() +if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" } +if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" } +if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" } +if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" } +if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" } +if ($requiredEvidenceFailures.Count -gt 0) { $blockingReasons += "required_evidence_stale" } +if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" } +if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" } + +$result = [pscustomobject]@{ + schemaVersion = "agent99_live_preflight_v1" + mode = $Mode + observedAt = (Get-Date).ToString("o") + host = $env:COMPUTERNAME + lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o") + secretValueRead = $false + remoteWritePerformed = $false + agentRoot = $AgentRoot + manifest = $manifest + tasks = $tasks + relayListeners = $relayListeners + queues = $queues + evidence = $evidence + summary = [pscustomobject]@{ + requiredTaskCount = $requiredTasks.Count + healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count + taskFailureCount = $taskFailures.Count + freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count + requiredEvidenceFailureCount = $requiredEvidenceFailures.Count + alertIncomingCount = [int]$alertIncoming.count + alertRunningCount = [int]$alertRunning.count + pendingCommandCount = [int]$commandQueue.count + relayListenerCount = $relayListeners.Count + preflightGreen = [bool]($blockingReasons.Count -eq 0) + blockingReasons = $blockingReasons + } +} + +Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)" +Write-Output "MODE=$Mode" +Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)" +Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)" +Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)" +Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)" +Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)" +Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)" +Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')" +Write-Output "SECRET_VALUE_READ=0" +Write-Output "REMOTE_WRITE_PERFORMED=0" +Write-Output "JSON_BEGIN" +$result | ConvertTo-Json -Depth 8 -Compress +Write-Output "JSON_END" + +if ($result.summary.preflightGreen) { + exit 0 +} +exit 2 diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py new file mode 100644 index 000000000..ce5256d12 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -0,0 +1,96 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] + + +def read(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def test_recover_requires_full_sop_coordinator_before_verified_resolution() -> None: + control = read("agent99-control-plane.ps1") + + assert "Invoke-AgentRecoveryCoordinatorReadback" in control + assert 'New-AgentOutcomeCheck "full_sop_coordinator_verified"' in control + assert "-and $coordinatorOk" in control + assert 'New-AgentRecoveryPhase "full_sop_scorecard"' in control + assert 'schemaVersion = "agent99_recovery_slo_v2"' in control + assert 'scope = $recoveryScope' in control + assert 'rebootSloClaimed = $rebootSloClaimed' in control + + +def test_coordinator_covers_all_hosts_and_full_recovery_evidence() -> None: + control = read("agent99-control-plane.ps1") + + for alias in ("99", "110", "111", "112", "120", "121", "188"): + assert f'"{alias}"' in control + + for field in ( + "SERVICE_GREEN", + "PRODUCT_DATA_GREEN", + "STOCK_FRESHNESS_STATUS", + "BACKUP_CORE_GREEN", + "HOST_188_SERVICE_GREEN", + "HOST_188_HYGIENE_BLOCKED", + "EDGE_FALLBACK_READY", + "VMWARE_AUTOSTART_VERIFY_READY", + "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY", + "SCORECARD_FRESH_REBOOT_WINDOW", + "SCORECARD_CAN_CLAIM", + "SCORECARD_BLOCKER_COUNT", + ): + assert field in control + + +def test_fresh_reboot_claim_requires_fresh_artifact_and_zero_blockers() -> None: + control = read("agent99-control-plane.ps1") + + assert 'name = "artifact_fresh_for_recovery"' in control + assert 'name = "fresh_reboot_window_observed"' in control + assert 'name = "scorecard_can_claim_slo"' in control + assert 'name = "scorecard_has_no_blockers"' in control + assert "$artifactEpoch -ge ($startedEpoch - $CoordinatorConfig.artifactClockSkewSeconds)" in control + assert "-not $requiresFreshWindow -or $rebootSloClaimed" in control + + +def test_runtime_deployer_migrates_recovery_coordinator_defaults() -> None: + deployer = read("agent99-deploy.ps1") + config = read("agent99.config.99.example.json") + + assert 'Add-DefaultProperty $config "recoveryCoordinator"' in deployer + assert 'Add-DefaultProperty $config.recoveryCoordinator "controllerHost" "192.168.0.110"' in deployer + assert 'Add-DefaultProperty $config.recoveryCoordinator "requiredHostAliases"' in deployer + assert '"recoveryCoordinator"' in config + assert '"freshWaitTimeoutSeconds": 420' in config + + +def test_coordinator_reads_only_bounded_summary_artifacts() -> None: + control = read("agent99-control-plane.ps1") + coordinator = control.split("function Invoke-AgentRecoveryCoordinatorReadback", 1)[1] + coordinator = coordinator.split("function Convert-AgentDouble", 1)[0] + + assert "summary.txt" in coordinator + assert "host-probe.txt" in coordinator + assert "public-maintenance-edge-fallback.txt" in coordinator + assert "windows99-vmware-verify.txt" in coordinator + assert "scorecard.json" in coordinator + assert "*.log" not in coordinator + assert ".env" not in coordinator + assert "auth.json" not in coordinator + assert "session" not in coordinator.lower() + + +def test_live_preflight_is_no_secret_and_no_remote_write() -> None: + preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1") + + assert 'secretValueRead = $false' in preflight + assert 'remoteWritePerformed = $false' in preflight + assert 'Write-Output "SECRET_VALUE_READ=0"' in preflight + assert 'Write-Output "REMOTE_WRITE_PERFORMED=0"' in preflight + assert "runtime-manifest.json" in preflight + assert "Get-ScheduledTask" in preflight + assert "Get-NetTCPConnection -LocalPort 8787" in preflight + assert ".env" not in preflight + assert "auth.json" not in preflight + assert "agent99.config.json" not in preflight