diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 4325a64ba..1ad66b9e6 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -309,7 +309,15 @@ if (-not (Test-Path $configPath)) { "preferJumpHost": true, "directHosts": ["192.168.0.110", "192.168.0.112"] }, - "publicUrls": ["awoooi.wooo.work", "stock.wooo.work", "2026fifa.wooo.work", "gitea.wooo.work", "harbor.wooo.work"], + "edgeServiceRecovery": { + "enabled": true, + "host": "192.168.0.188", + "scriptPath": "/home/ollama/bin/agent99-host188-edge-services-recover.sh", + "checkTimeoutSeconds": 45, + "applyTimeoutSeconds": 900, + "requiredServices": ["n8n", "open-webui"] + }, + "publicUrls": ["https://awoooi.wooo.work", "https://stock.wooo.work", "https://2026fifa.wooo.work", "https://gitea.wooo.work", "https://harbor.wooo.work", "https://n8n.wooo.work", "https://ollama.wooo.work"], "sreAlertRelay": { "enabled": true, "prefix": "http://+:8787/agent99/sre-alert/", diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 79c481480..da8ccdcc4 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -56,7 +56,11 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js @($host111External[0].macAddresses).Count -ge 1 -and 22 -in @($host111External[0].verifyTcpPorts) ) - $hasRoutes = @($config.publicUrls).Count -ge 5 + $hasRoutes = [bool]( + @($config.publicUrls).Count -ge 7 -and + "https://n8n.wooo.work" -in @($config.publicUrls) -and + "https://ollama.wooo.work" -in @($config.publicUrls) + ) $hasSlo = [bool]($config.recoverySlo -and [int]$config.recoverySlo.targetMinutes -eq 10) $hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$") $hasCanonicalHost111User = [bool]( @@ -72,6 +76,14 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $config.recoveryCoordinator.controllerHost -eq "192.168.0.110" -and @($config.recoveryCoordinator.requiredHostAliases).Count -eq 7 ) + $hasHost188EdgeRecovery = [bool]( + $config.edgeServiceRecovery -and + $config.edgeServiceRecovery.enabled -eq $true -and + $config.edgeServiceRecovery.host -eq "192.168.0.188" -and + $config.edgeServiceRecovery.scriptPath -eq "/home/ollama/bin/agent99-host188-edge-services-recover.sh" -and + "n8n" -in @($config.edgeServiceRecovery.requiredServices) -and + "open-webui" -in @($config.edgeServiceRecovery.requiredServices) + ) $hasCompletionCallback = [bool]( $config.completionCallback -and $config.completionCallback.enabled -eq $true -and @@ -85,8 +97,8 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js [string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" }).Count -eq 1) } else { $true } - $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx - Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" + $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasHost188EdgeRecovery -and $hasCompletionCallback -and $hasCanonicalHost112Vmx + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator host188EdgeRecovery=$hasHost188EdgeRecovery completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } @@ -116,6 +128,7 @@ Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("manager Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" Add-Check "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf') -and $bootstrap.Contains('Get-AgentHost112CanonicalVm $PolicyPath') -and $bootstrap.Contains('Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))')) "Host112 VM identity is normalized from one staged policy row and fails closed when absent" Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves" +Add-Check "recovery:host188_edge_services" ($control.Contains("Invoke-AgentHost188EdgeRecovery") -and $control.Contains('New-AgentRecoveryPhase "host188_edge_services"') -and $control.Contains('New-AgentOutcomeCheck "host188_edge_services_ready"') -and $control.Contains("container_local_upstream_and_public_https") -and $deployer.Contains('name = "canonical_public_route_coverage"')) "Host188 n8n and Open WebUI use a pinned controlled apply and independent public-route verifier" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present" Add-Check "callback:durable_completion" ($control.Contains('schema_version = "agent99_completion_callback_v1"') -and $control.Contains("durable_readback") -and $control.Contains("completion-callbacks\pending")) "Agent99 completion callbacks are queued until durable AWOOOI/AwoooP readback" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 7fc0ab43f..be8f0c8bc 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -4450,6 +4450,7 @@ function Get-AgentOutcomeContract { $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0) $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) + $host188EdgeOk = [bool]($data.host188EdgeRecovery -and $data.host188EdgeRecovery.verified) $evidenceReconcileOnly = [bool]( $data.PSObject.Properties["reconcileOnly"] -and $data.reconcileOnly -is [bool] -and @@ -4493,6 +4494,7 @@ function Get-AgentOutcomeContract { $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) + $checks += New-AgentOutcomeCheck "host188_edge_services_ready" $host188EdgeOk $true $([string]$data.host188EdgeRecovery.terminal) if ($RequireReconcileOnly) { $checks += New-AgentOutcomeCheck "reconcile_only_evidence_bound" $evidenceReceiptBound $true "expected=$ExpectedReconcileEvidenceId" $checks += New-AgentOutcomeCheck "reconcile_only_no_runtime_write" $reconcileNoWrite $true @@ -4503,7 +4505,7 @@ function Get-AgentOutcomeContract { $checks += New-AgentOutcomeCheck "unexpected_reconcile_only_evidence" $false $true $reconcileNoWrite = $false } - $modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $reconcileNoWrite) + $modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $host188EdgeOk -and $harborOk -and $awoooOk -and $publicOk -and $reconcileNoWrite) } "Recover" { $expectedVms = @($Config.vms).Count @@ -4521,6 +4523,7 @@ function Get-AgentOutcomeContract { $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified) $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) + $host188EdgeOk = [bool]($data.host188EdgeRecovery -and $data.host188EdgeRecovery.verified) $checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)" $checks += New-AgentOutcomeCheck "external_hosts_recovered" $externalHostsOk $true "expected=$expectedExternalHosts actual=$($externalHostRows.Count)" $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" @@ -4528,8 +4531,9 @@ function Get-AgentOutcomeContract { $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) + $checks += New-AgentOutcomeCheck "host188_edge_services_ready" $host188EdgeOk $true $([string]$data.host188EdgeRecovery.terminal) $checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ',')) - $modeVerifierPassed = [bool]($vmsOk -and $externalHostsOk -and $hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) + $modeVerifierPassed = [bool]($vmsOk -and $externalHostsOk -and $hostsOk -and $host112Ok -and $host188EdgeOk -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) } "StartVMs" { $expectedVms = @($Config.vms).Count @@ -6945,6 +6949,180 @@ function Invoke-AgentExternalHostRecovery { $results } +function Get-AgentHost188EdgeRecoveryConfig { + $configured = if ($Config.PSObject.Properties["edgeServiceRecovery"]) { $Config.edgeServiceRecovery } else { $null } + [pscustomobject]@{ + enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled)) + host = if ($configured -and $configured.PSObject.Properties["host"]) { [string]$configured.host } else { "192.168.0.188" } + scriptPath = if ($configured -and $configured.PSObject.Properties["scriptPath"]) { [string]$configured.scriptPath } else { "/home/ollama/bin/agent99-host188-edge-services-recover.sh" } + checkTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["checkTimeoutSeconds"]) { [math]::Max(15, [math]::Min(120, [int]$configured.checkTimeoutSeconds)) } else { 45 } + applyTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["applyTimeoutSeconds"]) { [math]::Max(120, [math]::Min(1200, [int]$configured.applyTimeoutSeconds)) } else { 900 } + requiredServices = if ($configured -and $configured.PSObject.Properties["requiredServices"]) { @($configured.requiredServices | ForEach-Object { [string]$_ }) } else { @("n8n", "open-webui") } + } +} + +function Convert-AgentHost188EdgeReceipt { + param([object]$Transport) + + if (-not $Transport) { + return [pscustomobject]@{ + receiptPresent = $false + parseError = "transport_missing" + overallHealthy = $false + runtimeWritePerformed = $false + services = @() + } + } + + $jsonLines = @(([string]$Transport.output -split "`r?`n") | Where-Object { + $line = $_.Trim() + $line.StartsWith("{") -and $line.EndsWith("}") + }) + if ($jsonLines.Count -eq 0) { + return [pscustomobject]@{ + receiptPresent = $false + parseError = "json_receipt_missing" + overallHealthy = $false + runtimeWritePerformed = $false + services = @() + } + } + + try { + $receipt = $jsonLines[-1] | ConvertFrom-Json + $receipt | Add-Member -NotePropertyName receiptPresent -NotePropertyValue $true -Force + $receipt | Add-Member -NotePropertyName parseError -NotePropertyValue "" -Force + return $receipt + } catch { + return [pscustomobject]@{ + receiptPresent = $false + parseError = "json_receipt_invalid" + overallHealthy = $false + runtimeWritePerformed = $false + services = @() + } + } +} + +function Get-AgentSshReceiptSummary { + param([object]$Transport) + if (-not $Transport) { return $null } + [pscustomobject]@{ + ok = [bool]$Transport.ok + exitCode = [int]$Transport.exitCode + elapsedMs = $Transport.elapsedMs + route = if ($Transport.PSObject.Properties["route"]) { [string]$Transport.route } else { "" } + transportSerialized = [bool]($Transport.PSObject.Properties["transportSerialized"] -and $Transport.transportSerialized) + transportLockWaitMs = if ($Transport.PSObject.Properties["transportLockWaitMs"]) { $Transport.transportLockWaitMs } else { $null } + } +} + +function Invoke-AgentHost188EdgeRecovery { + param( + [string]$RequestedRunId, + [string]$RequestedTraceId, + [string]$RequestedWorkItemId, + [string]$FallbackRunId, + [switch]$Apply + ) + + $settings = Get-AgentHost188EdgeRecoveryConfig + $canonicalHost = "192.168.0.188" + $canonicalScript = "/home/ollama/bin/agent99-host188-edge-services-recover.sh" + $contractReady = [bool]( + $settings.enabled -and + $settings.host -eq $canonicalHost -and + $settings.scriptPath -eq $canonicalScript -and + "n8n" -in $settings.requiredServices -and + "open-webui" -in $settings.requiredServices + ) + if (-not $contractReady) { + return [pscustomobject]@{ + schemaVersion = "agent99_host188_edge_recovery_v1" + enabled = [bool]$settings.enabled + contractReady = $false + controlledApply = [bool]$Apply + applyAttempted = $false + runtimeWritePerformed = $false + changed = $false + verified = $false + terminal = "canonical_contract_invalid" + before = $null + applyReceipt = $null + after = $null + verifier = "container_local_upstream_and_public_https" + } + } + + $safeRunId = if ($RequestedRunId -match '^[A-Za-z0-9._:-]+$') { $RequestedRunId } elseif ($FallbackRunId -match '^[A-Za-z0-9._:-]+$') { $FallbackRunId } else { "agent99-edge-$([guid]::NewGuid().ToString('N'))" } + $safeTraceId = if ($RequestedTraceId -match '^[A-Za-z0-9._:-]+$') { $RequestedTraceId } else { "trace-$safeRunId" } + $safeWorkItemId = if ($RequestedWorkItemId -match '^[A-Za-z0-9._:-]+$') { $RequestedWorkItemId } else { "AIA-P0-006-EDGE-502-188" } + $quotedScript = Quote-ShSingle $settings.scriptPath + $beforeTransport = Invoke-HostSshText $settings.host "$quotedScript --check" $settings.checkTimeoutSeconds 1 + $before = Convert-AgentHost188EdgeReceipt $beforeTransport + $beforeHealthy = [bool]($before.receiptPresent -and (Convert-AgentBool $before.overallHealthy)) + $applyTransport = $null + $applyReceipt = $null + $applyAttempted = $false + $afterTransport = $beforeTransport + $after = $before + $terminal = if ($beforeHealthy) { "idempotent_already_verified_healthy" } else { "controlled_apply_required" } + + if (-not $beforeHealthy -and $Apply) { + $applyAttempted = $true + $applyCommand = "$quotedScript --apply --trace-id $(Quote-ShSingle $safeTraceId) --run-id $(Quote-ShSingle $safeRunId) --work-item-id $(Quote-ShSingle $safeWorkItemId)" + Write-AgentLog "CONTROLLED_APPLY host188_edge_services run=$safeRunId workItem=$safeWorkItemId" + $applyTransport = Invoke-HostSshText $settings.host $applyCommand $settings.applyTimeoutSeconds 1 + $applyReceipt = Convert-AgentHost188EdgeReceipt $applyTransport + $afterTransport = Invoke-HostSshText $settings.host "$quotedScript --check" $settings.checkTimeoutSeconds 2 + $after = Convert-AgentHost188EdgeReceipt $afterTransport + $afterHealthy = [bool]($after.receiptPresent -and (Convert-AgentBool $after.overallHealthy)) + if ($afterHealthy) { + $terminal = "deployed_verified" + } elseif (-not $applyTransport.ok) { + $terminal = "apply_transport_failed" + } else { + $terminal = "post_verifier_failed_snapshot_preserved" + } + } + + $verified = [bool]($after.receiptPresent -and (Convert-AgentBool $after.overallHealthy)) + $runtimeWritePerformed = [bool]( + $applyAttempted -and + $applyReceipt -and + $applyReceipt.PSObject.Properties["runtimeWritePerformed"] -and + (Convert-AgentBool $applyReceipt.runtimeWritePerformed) + ) + $result = [pscustomobject]@{ + schemaVersion = "agent99_host188_edge_recovery_v1" + enabled = $true + contractReady = $true + targetHost = $settings.host + requiredServices = @($settings.requiredServices) + controlledApply = [bool]$Apply + applyAttempted = $applyAttempted + runtimeWritePerformed = $runtimeWritePerformed + changed = [bool](-not $beforeHealthy -and $verified -and $runtimeWritePerformed) + verified = $verified + terminal = $terminal + beforeTransport = Get-AgentSshReceiptSummary $beforeTransport + before = $before + applyTransport = Get-AgentSshReceiptSummary $applyTransport + applyReceipt = $applyReceipt + afterTransport = Get-AgentSshReceiptSummary $afterTransport + after = $after + rollback = "run_scoped_snapshots_preserved_break_glass_restore_only" + verifier = "container_local_upstream_and_public_https" + prohibitedActions = @("host_reboot", "vm_power_change", "firewall_change", "database_content_read", "secret_read", "external_registry_direct_pull") + } + if ($result.changed) { + Record-AgentEvent "host188_edge_services_recovered" "info" "188 的 n8n 與 Open WebUI 已由 Agent99 受控恢復,並通過容器、本機 upstream、公開 HTTPS verifier。" $result -Alert + } elseif ($applyAttempted -and -not $verified) { + Record-AgentEvent "host188_edge_services_recovery_failed" "critical" "188 的 n8n 或 Open WebUI 仍未通過 verifier;快照已保留,未執行主機重啟或資料還原。" $result -Alert + } + $result +} + function Test-HostReachability { $results = @() foreach ($hostIp in $Config.hosts) { @@ -7350,6 +7528,7 @@ $hostResults = @() $externalHostRecovery = @() $requiredExternalHostFailures = @() $host112Recovery = $null +$host188EdgeRecovery = $null $harbor = $null $awooo = $null $public = @() @@ -7456,6 +7635,22 @@ if ($Mode -in @("Recover", "AwoooRepair") -and $awooo) { } } +if ($Mode -in @("Status", "Recover")) { + $host188EdgeRecovery = Invoke-AgentHost188EdgeRecovery ` + -RequestedRunId $AutomationRunId ` + -RequestedTraceId $TraceId ` + -RequestedWorkItemId $WorkItemId ` + -FallbackRunId $recoveryRunId ` + -Apply:($Mode -eq "Recover" -and $ControlledApply) + if ($Mode -eq "Recover") { + $edgeStatus = if ($host188EdgeRecovery.verified) { "ok" } else { "failed" } + $edgeEvidence = if ($host188EdgeRecovery.after -and $host188EdgeRecovery.after.PSObject.Properties["receiptPath"]) { [string]$host188EdgeRecovery.after.receiptPath } else { "" } + $recoveryPhases += New-AgentRecoveryPhase "host188_edge_services" $edgeStatus (((Get-Date) - $recoveryStartedAt).TotalSeconds) $edgeEvidence + } elseif (-not $ReconcileOnly -and -not $host188EdgeRecovery.verified -and -not $recoveryTrigger) { + $recoveryTrigger = Start-AgentRecoveryFromObservation "host188_edge_services_not_ready" $host188EdgeRecovery.before + } +} + if ($Mode -in @("Status", "Recover", "PublicSmoke")) { $public = Test-PublicRoutes if ($Mode -eq "Recover") { @@ -7541,7 +7736,7 @@ $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCrit if ($Mode -eq "Recover") { $elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1) $unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count - $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $requiredExternalHostFailures.Count -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) + $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $requiredExternalHostFailures.Count -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $host188EdgeRecovery -and $host188EdgeRecovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) $coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified) $requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow) $rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed) @@ -7590,7 +7785,8 @@ $result = [pscustomobject]@{ reconcileQueueId = $ReconcileQueueId runtimeWritePerformed = [bool]( ($recoveryTrigger -and $recoveryTrigger.queued) -or - ($host112Recovery -and $host112Recovery.runtimeWritePerformed) + ($host112Recovery -and $host112Recovery.runtimeWritePerformed) -or + ($host188EdgeRecovery -and $host188EdgeRecovery.runtimeWritePerformed) ) sshProcessGuard = $sshProcessGuard bootState = $bootState @@ -7601,6 +7797,7 @@ $result = [pscustomobject]@{ externalHostRecovery = $externalHostRecovery hosts = $hostResults host112Recovery = $host112Recovery + host188EdgeRecovery = $host188EdgeRecovery harbor = $harbor awoooi = $awooo public = $public diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index c286130fb..f336da167 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -456,6 +456,37 @@ try { Add-DefaultProperty $config.recoveryCoordinator "maxArtifactAgeSeconds" 600 Add-DefaultProperty $config.recoveryCoordinator "artifactClockSkewSeconds" 90 Add-DefaultProperty $config.recoveryCoordinator "requiredHostAliases" @("99", "110", "111", "112", "120", "121", "188") + Add-DefaultProperty $config "edgeServiceRecovery" ([pscustomobject]@{}) + Add-DefaultProperty $config.edgeServiceRecovery "enabled" $true + Add-DefaultProperty $config.edgeServiceRecovery "host" "192.168.0.188" + Add-DefaultProperty $config.edgeServiceRecovery "scriptPath" "/home/ollama/bin/agent99-host188-edge-services-recover.sh" + Add-DefaultProperty $config.edgeServiceRecovery "checkTimeoutSeconds" 45 + Add-DefaultProperty $config.edgeServiceRecovery "applyTimeoutSeconds" 900 + Add-DefaultProperty $config.edgeServiceRecovery "requiredServices" @("n8n", "open-webui") + Add-DefaultProperty $config "publicUrls" @() + $previousPublicUrls = @($config.publicUrls | ForEach-Object { [string]$_ } | Where-Object { $_ }) + $canonicalPublicUrls = @($previousPublicUrls | ForEach-Object { + $url = $_.Trim() + if ($url -match '^https?://') { $url } else { "https://$url" } + }) + $requiredPublicUrls = @( + "https://awoooi.wooo.work", + "https://stock.wooo.work", + "https://2026fifa.wooo.work", + "https://gitea.wooo.work", + "https://harbor.wooo.work", + "https://n8n.wooo.work", + "https://ollama.wooo.work" + ) + $canonicalPublicUrls = @($canonicalPublicUrls + $requiredPublicUrls | Select-Object -Unique) + if (($previousPublicUrls -join ',') -cne ($canonicalPublicUrls -join ',')) { + Set-AgentProperty $config "publicUrls" $canonicalPublicUrls + $script:ConfigMigrations += [pscustomobject]@{ + name = "canonical_public_route_coverage" + from = ($previousPublicUrls -join ",") + to = ($canonicalPublicUrls -join ",") + } + } Add-DefaultProperty $config "selfHealth" ([pscustomobject]@{}) Add-DefaultProperty $config.selfHealth "requiredHosts" @("192.168.0.110", "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188") $selfHealthHosts = @($config.selfHealth.requiredHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) @@ -509,6 +540,16 @@ try { $persistedHost112Vmx = if ($persistedHost112Vm.Count -eq 1) { [string]$persistedHost112Vm[0].vmx } else { "" } $persistedHosts = @($persistedConfig.hosts | ForEach-Object { [string]$_ }) $persistedSelfHealthHosts = @($persistedConfig.selfHealth.requiredHosts | ForEach-Object { [string]$_ }) + $persistedPublicUrls = @($persistedConfig.publicUrls | ForEach-Object { [string]$_ }) + $persistedEdgeServices = if ($persistedConfig.PSObject.Properties["edgeServiceRecovery"]) { $persistedConfig.edgeServiceRecovery } else { $null } + $persistedEdgeServicesReady = [bool]( + $persistedEdgeServices -and + [bool]$persistedEdgeServices.enabled -and + [string]$persistedEdgeServices.host -eq "192.168.0.188" -and + [string]$persistedEdgeServices.scriptPath -eq "/home/ollama/bin/agent99-host188-edge-services-recover.sh" -and + "n8n" -in @($persistedEdgeServices.requiredServices | ForEach-Object { [string]$_ }) -and + "open-webui" -in @($persistedEdgeServices.requiredServices | ForEach-Object { [string]$_ }) + ) $persistedHost111External = @($persistedConfig.externalHosts | Where-Object { [string]$_.host -eq "192.168.0.111" }) $host111ExternalReady = [bool]( $persistedHost111External.Count -eq 1 -and @@ -524,6 +565,9 @@ try { "192.168.0.112" -notin $persistedDirectHosts -or "192.168.0.111" -notin $persistedHosts -or "192.168.0.111" -notin $persistedSelfHealthHosts -or + "https://n8n.wooo.work" -notin $persistedPublicUrls -or + "https://ollama.wooo.work" -notin $persistedPublicUrls -or + -not $persistedEdgeServicesReady -or -not $host111ExternalReady -or $persistedHost112Vm.Count -ne 1 -or [string]$persistedHost112Vm[0].name -ne [string]$canonicalHost112Vm.name -or @@ -531,7 +575,7 @@ try { $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed." + throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Host188 edge recovery config post-verifier also failed." } $manifestRows = @() diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 48b481e49..7474b5fb3 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -126,12 +126,25 @@ "188" ] }, + "edgeServiceRecovery": { + "enabled": true, + "host": "192.168.0.188", + "scriptPath": "/home/ollama/bin/agent99-host188-edge-services-recover.sh", + "checkTimeoutSeconds": 45, + "applyTimeoutSeconds": 900, + "requiredServices": [ + "n8n", + "open-webui" + ] + }, "publicUrls": [ - "awoooi.wooo.work", - "stock.wooo.work", - "2026fifa.wooo.work", - "gitea.wooo.work", - "harbor.wooo.work" + "https://awoooi.wooo.work", + "https://stock.wooo.work", + "https://2026fifa.wooo.work", + "https://gitea.wooo.work", + "https://harbor.wooo.work", + "https://n8n.wooo.work", + "https://ollama.wooo.work" ], "aiServices": [ { diff --git a/agent99.config.example.json b/agent99.config.example.json index d970d69fe..744ab7088 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -120,12 +120,25 @@ "188" ] }, + "edgeServiceRecovery": { + "enabled": true, + "host": "192.168.0.188", + "scriptPath": "/home/ollama/bin/agent99-host188-edge-services-recover.sh", + "checkTimeoutSeconds": 45, + "applyTimeoutSeconds": 900, + "requiredServices": [ + "n8n", + "open-webui" + ] + }, "publicUrls": [ - "awoooi.wooo.work", - "stock.wooo.work", - "2026fifa.wooo.work", - "gitea.wooo.work", - "harbor.wooo.work" + "https://awoooi.wooo.work", + "https://stock.wooo.work", + "https://2026fifa.wooo.work", + "https://gitea.wooo.work", + "https://harbor.wooo.work", + "https://n8n.wooo.work", + "https://ollama.wooo.work" ], "aiServices": [ { diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 4ac798562..647a0d9b7 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.134 +> Version: v1.136 > Last updated: 2026-07-14 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號,但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal;此納入不代表授權任何破壞性 runtime apply。 @@ -10,6 +10,20 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天,必須先重跑 live check,再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +### v1.136 2026-07-14 Host112 正式納入 cold-start P0 release gate + +`full-stack-cold-start-check.sh` 舊版仍顯示 `112 Kali is intentionally skipped`,與 Windows99 五台 required VM、Agent99 `host112_guest_readiness` phase 及全主機 10 分鐘 SLO 不一致。現行 baseline 已把 112 從 excluded host 移入 included host;P0 network 固定檢查 ping / TCP 22,並透過 `kali@192.168.0.112` 執行 canonical `/usr/local/sbin/awoooi-host112-guest-readiness --check`。release gate 必須同時驗證 readback schema、systemd running、guest/console/services/recovery timer、Wazuh 1514/1515/55000 independent verifier 與 `runtime_write_performed=0`。任何一項失敗均維持 BLOCKED;不得因其他四台 Linux host 或 public routes 為 200 而宣稱全主機恢復。110 textfile exporter scope 固定為 `110_112_120_121_188`,並分開輸出 112 host unreachable 與 guest readiness failed blocker series;`verify-cold-start-monitor-deploy.sh` 必須驗證 source/live hash、scope 與 blocker series parity,避免 checker 已納管但 Prometheus 仍漏掉 112。 + +### v1.135 2026-07-14 Host188 n8n / Open WebUI 502 自動恢復 + +本輪全資產 public smoke 發現 `https://n8n.wooo.work` 與 `https://ollama.wooo.work` 均為 raw `502`。188 的 Nginx 正常,但 upstream `127.0.0.1:5678` / `127.0.0.1:3010` 沒有可用服務;Ansible 宣稱 `/opt/n8n`、`/opt/open-webui` 可直接 `docker compose up`,live 目錄卻不存在。n8n 的實際資料與 base compose 在 `/home/ollama/n8n`,Open WebUI 只剩 `open-webui` volume;兩個本機映像都已遺失。這類事件必須分類為「服務部署資產漂移」,不能只重啟 Nginx、不能把 502 當 warmup,也不能從外部 registry 拉 mutable `latest`。 + +- 受控恢復固定使用內部 Harbor immutable digest:n8n `sha256:dbe732489e5b8941aa89ca71b320ccb7f80319fcccd0f2c9e05f561562d804e5`;Open WebUI `sha256:a673af6e7fab29822a0d32d71184d3f0277347a693633c86372643590c921915`。apply 前先建立 run-scoped n8n data / Open WebUI volume 快照,任何資料 restore 仍是 break-glass,不得自動執行。 +- 188 Docker 啟用 `userns-remap`,`dockremap` 起始 UID/GID 為 `100000`。n8n image 的 `node` UID/GID `1000` 必須映射成 host `101000:101000`;直接加 `userns_mode: host` 會讓 image layer 內 `/home/node/.cache` 再次 EACCES,因此正式修法是收斂 bind data ownership、維持預設 user namespace。 +- canonical executor 是 `scripts/reboot-recovery/host188-edge-services-recover.sh`,live path `/home/ollama/bin/agent99-host188-edge-services-recover.sh`。`--check` 僅讀 container/image/restart/health、本機 upstream 與 local-Nginx HTTPS;`--apply` 才可拉固定 Harbor digest、建立快照、原子寫入 managed compose、修正 n8n remap ownership、啟動服務,再跑獨立 `--check`。 +- live apply `trace-agent99-edge-services-20260714 / run-agent99-edge-services-20260714-02 / AIA-P0-006-EDGE-502-188` 終態為 `deployed_verified`:n8n 與 Open WebUI 均 `running`、restart count `0`、local/public HTTP `200`、config managed、image pinned;receipt 在 `/home/ollama/agent99-runtime-recovery/run-agent99-edge-services-20260714-02/receipt.json`。後續 no-write check 為 exit `0`、`runtimeWritePerformed=false`,兩份 compose mtime 不變。 +- Agent99 `Status` 必須偵測此 lane 並排入單一 Recover;Recover 固定執行 Detect -> check -> bounded apply -> independent verifier -> callback/TG/KM。`full-stack-cold-start-check.sh` 與 AI log triage public routes 必須同時包含 n8n / Open WebUI,兩者任一未通過時不得宣稱所有網站恢復。 + ### v1.134 2026-07-14 Agent99 canonical host readback closure 本輪釐清 111 的 `perf_readback_failed` 不是 CPU 壓力,而是 Agent99 沿用預設 `wooo`,但 110 到 111 的 canonical restricted identity 是 `ooo`。部署合約固定 `sshUsers["192.168.0.111"]="ooo"`,舊 live config 必須由 `host111_canonical_ssh_user` migration 寫入後重新讀回;帳號、transport 或四項指標任一缺失時,只能標示 readback degraded,不得產生 CPU/記憶體/磁碟故障結論或執行降載。 diff --git a/infra/ansible/inventory/group_vars/host_188.yml b/infra/ansible/inventory/group_vars/host_188.yml index 0bc802d51..63d96345b 100644 --- a/infra/ansible/inventory/group_vars/host_188.yml +++ b/infra/ansible/inventory/group_vars/host_188.yml @@ -19,10 +19,10 @@ docker_compose_services: dir: /opt/minio expected_port: 9000 n8n: - dir: /opt/n8n + dir: /home/ollama/n8n expected_port: 5678 open_webui: - dir: /opt/open-webui + dir: /home/ollama/open-webui expected_port: 3010 docker_registry: dir: /opt/docker-registry diff --git a/infra/ansible/playbooks/188-ai-web.yml b/infra/ansible/playbooks/188-ai-web.yml index 1091574b8..24da34aae 100644 --- a/infra/ansible/playbooks/188-ai-web.yml +++ b/infra/ansible/playbooks/188-ai-web.yml @@ -177,35 +177,72 @@ # ======================================================================== # n8n / open-webui (Sprint A 新啟動) # ======================================================================== - - name: "N8N | 確認容器運作中" + - name: "Edge services | 確認受控恢復腳本目錄存在" + ansible.builtin.file: + path: /home/ollama/bin + state: directory + owner: ollama + group: ollama + mode: "0755" + tags: + - n8n + - open_webui + - edge_services + + - name: "Edge services | 安裝 Agent99 受控恢復腳本" + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../../scripts/reboot-recovery/host188-edge-services-recover.sh" + dest: /home/ollama/bin/agent99-host188-edge-services-recover.sh + owner: ollama + group: ollama + mode: "0755" + tags: + - n8n + - open_webui + - edge_services + + - name: "Edge services | 讀取 n8n 與 Open WebUI 前置狀態" + become: false ansible.builtin.command: - cmd: "docker ps --filter name=n8n --filter status=running --format '{{ '{{' }}.Names{{ '}}' }}'" - register: n8n_status + cmd: /home/ollama/bin/agent99-host188-edge-services-recover.sh --check + register: edge_services_before changed_when: false - tags: n8n + failed_when: edge_services_before.rc not in [0, 2] + check_mode: false + tags: + - n8n + - open_webui + - edge_services - - name: "N8N | 若停止則啟動" + - name: "Edge services | 執行有界恢復與獨立 verifier" + become: false ansible.builtin.command: - cmd: "docker compose up -d" - chdir: /opt/n8n + cmd: >- + /home/ollama/bin/agent99-host188-edge-services-recover.sh + --apply + --trace-id ansible-host188-edge-{{ ansible_date_time.epoch }} + --run-id ansible-host188-edge-{{ ansible_date_time.epoch }} + --work-item-id AIA-P0-006-EDGE-502-188 + when: + - not ansible_check_mode + - edge_services_before.rc == 2 changed_when: true - when: n8n_status.stdout == "" - tags: n8n + tags: + - n8n + - open_webui + - edge_services - - name: "Open-webui | 確認容器運作中" + - name: "Edge services | 驗證容器、本機 upstream 與公開 HTTPS" + become: false ansible.builtin.command: - cmd: "docker ps --filter name=open-webui --filter status=running --format '{{ '{{' }}.Names{{ '}}' }}'" - register: openwebui_status + cmd: /home/ollama/bin/agent99-host188-edge-services-recover.sh --check + register: edge_services_after changed_when: false - tags: open_webui - - - name: "Open-webui | 若停止則啟動" - ansible.builtin.command: - cmd: "docker compose up -d" - chdir: /opt/open-webui - changed_when: true - when: openwebui_status.stdout == "" - tags: open_webui + when: not ansible_check_mode or edge_services_before.rc == 0 + tags: + - n8n + - open_webui + - edge_services # ======================================================================== # Nginx 狀態確認 diff --git a/ops/reboot-recovery/full-stack-cold-start-baseline.yml b/ops/reboot-recovery/full-stack-cold-start-baseline.yml index e17f42d2f..d0782f042 100644 --- a/ops/reboot-recovery/full-stack-cold-start-baseline.yml +++ b/ops/reboot-recovery/full-stack-cold-start-baseline.yml @@ -1,12 +1,12 @@ -version: 2026-05-06.v1 +version: 2026-07-14.v2 scope: included_hosts: "110": "DevOps, registry, observability, Sentry, runners" + "112": "Kali security, Wazuh, guest readiness and bounded recovery" "120": "K3s control plane and VIP" "121": "K3s peer node and DR drill cron" "188": "Data, AI, web, momo, SignOz, public nginx gateway" - excluded_hosts: - "112": "Kali security host; recorded but not part of cold-start release gate" + excluded_hosts: {} principles: - recover_dependency_chain_before_workloads @@ -19,8 +19,8 @@ phases: - id: P0-NETWORK order: 0 gates: - - ping_110_120_121_188 - - ssh_port_110_120_121_188 + - ping_110_112_120_121_188 + - ssh_port_110_112_120_121_188 - arp_evidence_or_monitor_mode_fallback - id: P0-188-DATA @@ -54,6 +54,16 @@ phases: - cadvisor_image_v0_47_0 - cadvisor_cpu_cap_0_3 + - id: P0-112-SECURITY + order: 25 + gates: + - canonical_guest_readback_schema + - systemd_running + - graphical_console_and_vmware_tools_ready + - wazuh_stack_and_independent_ports_ready + - guest_recovery_timer_ready + - check_mode_performs_no_runtime_write + - id: P1-K3S order: 30 gates: diff --git a/scripts/ops/ansible-validate.sh b/scripts/ops/ansible-validate.sh index 5427b5c13..c84cd6d90 100755 --- a/scripts/ops/ansible-validate.sh +++ b/scripts/ops/ansible-validate.sh @@ -31,6 +31,7 @@ python3 ops/runner/guard-gitea-runner-pressure.py --root "$ROOT_DIR" echo "== Shell 語法 ==" bash -n \ scripts/reboot-recovery/full-stack-cold-start-check.sh \ + scripts/reboot-recovery/host188-edge-services-recover.sh \ scripts/reboot-recovery/full-stack-recovery-scorecard.sh \ scripts/reboot-recovery/dr-offsite-operator-checklist.sh \ scripts/reboot-recovery/wait-dr-offsite-ready.sh \ diff --git a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh index a330ef026..6a535ecf5 100755 --- a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh +++ b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh @@ -231,7 +231,9 @@ probe_public_routes() { "https://stock.wooo.work/api/v1/system/freshness" \ "https://gitea.wooo.work/" \ "https://registry.wooo.work/v2/" \ - "https://harbor.wooo.work/" + "https://harbor.wooo.work/" \ + "https://n8n.wooo.work/" \ + "https://ollama.wooo.work/" do code="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)" [ -n "$code" ] || code="000" diff --git a/scripts/reboot-recovery/cold-start-textfile-exporter.sh b/scripts/reboot-recovery/cold-start-textfile-exporter.sh index 2798f8775..2477b41e9 100755 --- a/scripts/reboot-recovery/cold-start-textfile-exporter.sh +++ b/scripts/reboot-recovery/cold-start-textfile-exporter.sh @@ -14,7 +14,7 @@ CHECK_TIMEOUT_SECONDS="${CHECK_TIMEOUT_SECONDS:-240}" CHECK_WATCH_INTERVAL_SECONDS="${CHECK_WATCH_INTERVAL_SECONDS:-10}" CHECK_WATCH_MAX_ATTEMPTS="${CHECK_WATCH_MAX_ATTEMPTS:-3}" HOST_LABEL="${AIOPS_HOST_LABEL:-110}" -SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_120_121_188}" +SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}" LOCK_FILE="${LOCK_FILE:-/tmp/awoooi-cold-start-textfile-exporter.lock}" escape_label() { @@ -39,6 +39,8 @@ write_metric_file() { local public_route_tls_blocker="${15}" local host_120_unreachable_blocker="${16}" local backup_health_blocker="${17}" + local host_112_unreachable_blocker="${18}" + local host_112_guest_not_ready_blocker="${19}" local host scope host=$(escape_label "$HOST_LABEL") scope=$(escape_label "$SCOPE_LABEL") @@ -80,6 +82,8 @@ awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="k3s_node_fi awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="public_route_tls_failure",target="public_https"} $public_route_tls_blocker awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="120"} $host_120_unreachable_blocker awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="backup_health_blocked",target="110"} $backup_health_blocker +awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="112"} $host_112_unreachable_blocker +awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="guest_readiness_failed",target="112"} $host_112_guest_not_ready_blocker METRICS } @@ -102,7 +106,7 @@ if [ ! -x "$CHECK_SCRIPT" ]; then tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX") last_green=$(cat "$state_file" 2>/dev/null || echo 0) printf 'CHECK_SCRIPT not executable: %s\n' "$CHECK_SCRIPT" >"$log_file" - write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0 + write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0 0 0 chmod 0644 "$tmp_metric" mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME" exit 0 @@ -131,6 +135,8 @@ k3s_node_fs_blocker=0 public_route_tls_blocker=0 host_120_unreachable_blocker=0 backup_health_blocker=0 +host_112_unreachable_blocker=0 +host_112_guest_not_ready_blocker=0 if [ -n "$summary_line" ]; then monitor_up=1 @@ -166,6 +172,14 @@ if grep -Eq 'BLOCKED 110 backup health has stale expected jobs' "$log_file"; the backup_health_blocker=1 fi +if grep -Eq 'BLOCKED (ping 192\.168\.0\.112|ssh port 192\.168\.0\.112:22|112 canonical guest readback unavailable)' "$log_file"; then + host_112_unreachable_blocker=1 +fi + +if grep -Eq 'BLOCKED 112 (canonical guest readback schema missing|systemd not running|guest readiness failed|console or required services not ready|Wazuh independent verifier failed|guest recovery timer not ready|cold-start probe write boundary violated)' "$log_file"; then + host_112_guest_not_ready_blocker=1 +fi + end_ts=$(date +%s) if [ "$green" -eq 1 ]; then printf '%s\n' "$end_ts" >"$state_file" @@ -173,6 +187,6 @@ fi last_green=$(cat "$state_file" 2>/dev/null || echo 0) tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX") -write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker" +write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker" "$host_112_unreachable_blocker" "$host_112_guest_not_ready_blocker" chmod 0644 "$tmp_metric" mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME" diff --git a/scripts/reboot-recovery/full-stack-cold-start-check.sh b/scripts/reboot-recovery/full-stack-cold-start-check.sh index be248399f..acaa22d5d 100755 --- a/scripts/reboot-recovery/full-stack-cold-start-check.sh +++ b/scripts/reboot-recovery/full-stack-cold-start-check.sh @@ -246,6 +246,13 @@ registry_code_ready() { [ "$code" = "200" ] || [ "$code" = "401" ] } +readback_value() { + local text="$1" key="$2" rest + rest="${text#*${key}=}" + [ "$rest" != "$text" ] || return 1 + printf '%s' "${rest%% *}" +} + probe_tcp() { local host="$1" local port="$2" @@ -254,11 +261,11 @@ probe_tcp() { print_neighbor_rows() { if command -v arp >/dev/null 2>&1; then - arp -an | grep -E '192\.168\.0\.(110|120|121|188)' + arp -an | grep -E '192\.168\.0\.(110|112|120|121|188)' return $? fi if command -v ip >/dev/null 2>&1; then - ip neigh show | grep -E '192\.168\.0\.(110|120|121|188)' + ip neigh show | grep -E '192\.168\.0\.(110|112|120|121|188)' return $? fi return 1 @@ -267,14 +274,14 @@ print_neighbor_rows() { print_header() { echo "AWOOOI full-stack cold-start check" date '+%Y-%m-%d %H:%M:%S %Z' - echo "Scope: 110 / 120 / 121 / 188. 112 Kali is intentionally skipped." + echo "Scope: 110 / 112 / 120 / 121 / 188." echo "Baseline: ops/reboot-recovery/full-stack-cold-start-baseline.yml" } check_network() { log_section "P0-NETWORK" local host - for host in 110 120 121 188; do + for host in 110 112 120 121 188; do if ping -c 1 -W 2 "192.168.0.$host" >/dev/null 2>&1; then ok "ping 192.168.0.$host" else @@ -297,6 +304,50 @@ check_network() { fi } +check_112() { + log_section "P0-112-SECURITY" + local out schema systemd_state guest_ready console_ready services_ready + local recovery_timer_ready manager_verifier_ready runtime_write_performed + if ! out=$(host_cmd "kali@192.168.0.112" \ + '/usr/local/sbin/awoooi-host112-guest-readiness --check' 2>&1); then + fail "112 canonical guest readback unavailable" + echo "$out" + return + fi + + schema="$(readback_value "$out" schema_version || true)" + systemd_state="$(readback_value "$out" systemd_state || true)" + guest_ready="$(readback_value "$out" guest_ready || true)" + console_ready="$(readback_value "$out" console_ready || true)" + services_ready="$(readback_value "$out" services_ready || true)" + recovery_timer_ready="$(readback_value "$out" recovery_timer_ready || true)" + manager_verifier_ready="$(readback_value "$out" manager_independent_verifier_ready || true)" + runtime_write_performed="$(readback_value "$out" runtime_write_performed || true)" + echo "HOST112_READBACK schema=${schema:-missing} systemd=${systemd_state:-missing} guest=${guest_ready:-missing} console=${console_ready:-missing} services=${services_ready:-missing} timer=${recovery_timer_ready:-missing} manager_verifier=${manager_verifier_ready:-missing} runtime_write=${runtime_write_performed:-missing}" + + [ "$schema" = "host112_guest_recovery_v2" ] \ + && ok "112 canonical guest readback schema verified" \ + || fail "112 canonical guest readback schema missing" + [ "$systemd_state" = "running" ] \ + && ok "112 systemd running" \ + || fail "112 systemd not running" + [ "$guest_ready" = "1" ] \ + && ok "112 guest readiness verified" \ + || fail "112 guest readiness failed" + [ "$console_ready" = "1" ] && [ "$services_ready" = "1" ] \ + && ok "112 console and required services ready" \ + || fail "112 console or required services not ready" + [ "$manager_verifier_ready" = "1" ] \ + && ok "112 Wazuh independent verifier ready" \ + || fail "112 Wazuh independent verifier failed" + [ "$recovery_timer_ready" = "1" ] \ + && ok "112 guest recovery timer ready" \ + || fail "112 guest recovery timer not ready" + [ "$runtime_write_performed" = "0" ] \ + && ok "112 cold-start probe remained read-only" \ + || fail "112 cold-start probe write boundary violated" +} + check_188() { log_section "P0-188-DATA" local out @@ -660,6 +711,8 @@ check_public_routes() { "langfuse|https://langfuse.wooo.work/" "bitan|https://bitan.wooo.work/" "aiops|https://aiops.wooo.work/" + "n8n|https://n8n.wooo.work/" + "open_webui|https://ollama.wooo.work/" ) for item in "${routes[@]}"; do @@ -1067,6 +1120,7 @@ fi print_header check_network +check_112 check_188 check_110 check_k3s diff --git a/scripts/reboot-recovery/host188-edge-services-recover.sh b/scripts/reboot-recovery/host188-edge-services-recover.sh new file mode 100755 index 000000000..77e839ca9 --- /dev/null +++ b/scripts/reboot-recovery/host188-edge-services-recover.sh @@ -0,0 +1,288 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="check" +TRACE_ID="" +RUN_ID="" +WORK_ITEM_ID="" + +HOST_ID="192.168.0.188" +N8N_IMAGE="harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:dbe732489e5b8941aa89ca71b320ccb7f80319fcccd0f2c9e05f561562d804e5" +N8N_IMAGE_ID="sha256:19c1ad26c0c285fddd7fc1300ff670b7977afebf52a21de4dfb2a5a2b2a75e2b" +OPEN_WEBUI_IMAGE="harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:a673af6e7fab29822a0d32d71184d3f0277347a693633c86372643590c921915" +OPEN_WEBUI_IMAGE_ID="sha256:5bddf2d93bc307b99c1fde8b8d0c2dc3c563e0e24fc3eebb6488c539fa55d080" +BACKUP_IMAGE="harbor.wooo.work/dockerhub-cache/library/alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc" +N8N_DIR="/home/ollama/n8n" +N8N_OVERRIDE="${N8N_DIR}/docker-compose.agent99.yml" +OPEN_WEBUI_DIR="/home/ollama/open-webui" +OPEN_WEBUI_COMPOSE="${OPEN_WEBUI_DIR}/docker-compose.yml" +RECEIPT_ROOT="/home/ollama/agent99-runtime-recovery" + +usage() { + printf 'usage: %s --check | --apply --trace-id ID --run-id ID --work-item-id ID\n' "$0" >&2 + exit 64 +} + +valid_id() { + [[ "$1" =~ ^[A-Za-z0-9._:-]+$ ]] +} + +while (($#)); do + case "$1" in + --check) + MODE="check" + shift + ;; + --apply) + MODE="apply" + shift + ;; + --trace-id) + (($# >= 2)) || usage + TRACE_ID="$2" + shift 2 + ;; + --run-id) + (($# >= 2)) || usage + RUN_ID="$2" + shift 2 + ;; + --work-item-id) + (($# >= 2)) || usage + WORK_ITEM_ID="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +if [[ "$MODE" == "apply" ]]; then + [[ -n "$TRACE_ID" && -n "$RUN_ID" && -n "$WORK_ITEM_ID" ]] || usage + valid_id "$TRACE_ID" && valid_id "$RUN_ID" && valid_id "$WORK_ITEM_ID" || usage +fi + +bool_json() { + if [[ "$1" == "true" ]]; then printf 'true'; else printf 'false'; fi +} + +http_status() { + local url="$1" + local resolve_arg="${2:-}" + local status + if [[ -n "$resolve_arg" ]]; then + status="$(curl -k -sS -L -o /dev/null -w '%{http_code}' --resolve "$resolve_arg" --max-time 12 "$url" 2>/dev/null || true)" + else + status="$(curl -sS -L -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)" + fi + [[ "$status" =~ ^[0-9]{3}$ ]] || status="000" + printf '%s' "$status" +} + +non_5xx() { + local status="$1" + [[ "$status" =~ ^[0-9]{3}$ ]] && ((10#$status >= 200 && 10#$status < 500)) +} + +container_value() { + local name="$1" + local format="$2" + docker inspect "$name" --format "$format" 2>/dev/null || true +} + +collect_state() { + local runtime_write="${1:-false}" + local terminal="${2:-observed}" + local receipt_path="${3:-}" + + local n8n_status n8n_image n8n_restart n8n_restarts n8n_local n8n_public + local webui_status webui_health webui_image webui_restart webui_restarts webui_local webui_public + local n8n_config_ok=false webui_config_ok=false n8n_ok=false webui_ok=false overall=false + + n8n_status="$(container_value n8n '{{.State.Status}}')" + n8n_image="$(container_value n8n '{{.Image}}')" + n8n_restart="$(container_value n8n '{{.HostConfig.RestartPolicy.Name}}')" + n8n_restarts="$(container_value n8n '{{.RestartCount}}')" + [[ "$n8n_restarts" =~ ^[0-9]+$ ]] || n8n_restarts=0 + n8n_local="$(http_status 'http://127.0.0.1:5678/')" + n8n_public="$(http_status 'https://n8n.wooo.work/' 'n8n.wooo.work:443:127.0.0.1')" + + webui_status="$(container_value open-webui '{{.State.Status}}')" + webui_health="$(container_value open-webui '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')" + webui_image="$(container_value open-webui '{{.Image}}')" + webui_restart="$(container_value open-webui '{{.HostConfig.RestartPolicy.Name}}')" + webui_restarts="$(container_value open-webui '{{.RestartCount}}')" + [[ "$webui_restarts" =~ ^[0-9]+$ ]] || webui_restarts=0 + webui_local="$(http_status 'http://127.0.0.1:3010/')" + webui_public="$(http_status 'https://ollama.wooo.work/' 'ollama.wooo.work:443:127.0.0.1')" + + if [[ -f "$N8N_OVERRIDE" ]] && grep -Fqx " image: ${N8N_IMAGE}" "$N8N_OVERRIDE"; then + n8n_config_ok=true + fi + if [[ -f "$OPEN_WEBUI_COMPOSE" ]] && grep -Fqx " image: ${OPEN_WEBUI_IMAGE}" "$OPEN_WEBUI_COMPOSE"; then + webui_config_ok=true + fi + + if [[ "$n8n_config_ok" == true && "$n8n_status" == running && "$n8n_image" == "$N8N_IMAGE_ID" && "$n8n_restart" == always ]] && non_5xx "$n8n_local" && non_5xx "$n8n_public"; then + n8n_ok=true + fi + if [[ "$webui_config_ok" == true && "$webui_status" == running && "$webui_health" == healthy && "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" && "$webui_restart" == always ]] && non_5xx "$webui_local" && non_5xx "$webui_public"; then + webui_ok=true + fi + if [[ "$n8n_ok" == true && "$webui_ok" == true ]]; then + overall=true + fi + + printf '{"schemaVersion":"agent99_host188_edge_services_v1","mode":"%s","traceId":"%s","runId":"%s","workItemId":"%s","host":"%s","overallHealthy":%s,"runtimeWritePerformed":%s,"secretValueRead":false,"rawDataRead":false,"terminal":"%s","receiptPath":"%s","services":[{"name":"n8n","publicUrl":"https://n8n.wooo.work","containerStatus":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s},{"name":"open-webui","publicUrl":"https://ollama.wooo.work","containerStatus":"%s","health":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s}],"rollback":{"automatic":false,"snapshotPreserved":true,"procedure":"stop_changed_containers_then_restore_run_scoped_snapshot_after_owner_break_glass"},"prohibitedActions":["host_reboot","vm_power_change","firewall_change","database_content_read","secret_read","external_registry_direct_pull"]}\n' \ + "$MODE" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$HOST_ID" \ + "$(bool_json "$overall")" "$(bool_json "$runtime_write")" "$terminal" "$receipt_path" \ + "$n8n_status" "$(bool_json "$([[ "$n8n_image" == "$N8N_IMAGE_ID" ]] && echo true || echo false)")" "$n8n_restart" "$n8n_restarts" "$n8n_local" "$n8n_public" "$(bool_json "$n8n_config_ok")" "$(bool_json "$n8n_ok")" \ + "$webui_status" "$webui_health" "$(bool_json "$([[ "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" ]] && echo true || echo false)")" "$webui_restart" "$webui_restarts" "$webui_local" "$webui_public" "$(bool_json "$webui_config_ok")" "$(bool_json "$webui_ok")" + + [[ "$overall" == true ]] +} + +if [[ "$MODE" == "check" ]]; then + if collect_state false observed ""; then + exit 0 + fi + exit 2 +fi + +exec 9>/tmp/agent99-host188-edge-services.lock +if ! flock -w 30 9; then + collect_state false single_flight_lock_timeout "" || true + exit 75 +fi + +RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" +OPERATION_LOG="${RECEIPT_DIR}/operation.log" +RECEIPT_PATH="${RECEIPT_DIR}/receipt.json" + +emit_apply_failure() { + local exit_code="$1" + local result + trap - ERR + set +e + result="$(collect_state true "apply_failed_exit_${exit_code}" "$RECEIPT_PATH" || true)" + printf '%s\n' "$result" | tee "$RECEIPT_PATH" + chmod 600 "$RECEIPT_PATH" 2>/dev/null || true + exit "$exit_code" +} + +mkdir -p "$RECEIPT_DIR" "$N8N_DIR" "$OPEN_WEBUI_DIR" +chmod 700 "$RECEIPT_DIR" +: >"$OPERATION_LOG" +chmod 600 "$OPERATION_LOG" +trap 'emit_apply_failure $?' ERR + +for image in "$BACKUP_IMAGE" "$N8N_IMAGE" "$OPEN_WEBUI_IMAGE"; do + timeout 1200 docker pull "$image" >>"$OPERATION_LOG" 2>&1 +done + +snapshot_with_helper() { + local source_path="$1" + local archive_name="$2" + if [[ -s "${RECEIPT_DIR}/${archive_name}" ]]; then + return 0 + fi + docker run --rm --pull never --userns=host \ + -v "${source_path}:/data:ro" -v "${RECEIPT_DIR}:/backup" \ + "$BACKUP_IMAGE" sh -c "umask 077; tar czf /backup/${archive_name} -C /data ." \ + >>"$OPERATION_LOG" 2>&1 +} + +[[ -d "${N8N_DIR}/data" && -f "${N8N_DIR}/docker-compose.yml" ]] || { + collect_state false n8n_base_assets_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true + exit 3 +} + +if ! docker volume inspect open-webui >/dev/null 2>&1; then + docker volume create open-webui >>"$OPERATION_LOG" +fi + +snapshot_with_helper "${N8N_DIR}/data" "n8n-data-before.tgz" +docker run --rm --pull never --userns=host \ + -v open-webui:/data:ro -v "${RECEIPT_DIR}:/backup" \ + "$BACKUP_IMAGE" sh -c 'umask 077; tar czf /backup/open-webui-volume-before.tgz -C /data .' \ + >>"$OPERATION_LOG" 2>&1 +docker run --rm --pull never --userns=host -v "${RECEIPT_DIR}:/backup" "$BACKUP_IMAGE" \ + chown -R "$(id -u):$(id -g)" /backup >>"$OPERATION_LOG" 2>&1 +chmod 600 "${RECEIPT_DIR}/n8n-data-before.tgz" "${RECEIPT_DIR}/open-webui-volume-before.tgz" + +tmp_n8n="$(mktemp "${N8N_OVERRIDE}.XXXXXX")" +cat >"$tmp_n8n" <"$tmp_webui" </dev/null || true)" +remap_gid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subgid 2>/dev/null || true)" +if [[ "$remap_uid" =~ ^[0-9]+$ && "$remap_gid" =~ ^[0-9]+$ ]]; then + node_uid=$((remap_uid + 1000)) + node_gid=$((remap_gid + 1000)) + docker run --rm --pull never --userns=host -v "${N8N_DIR}/data:/data" "$BACKUP_IMAGE" \ + chown -R "${node_uid}:${node_gid}" /data >>"$OPERATION_LOG" 2>&1 +else + collect_state true docker_userns_mapping_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true + exit 4 +fi + +docker compose --project-directory "$N8N_DIR" \ + -f "${N8N_DIR}/docker-compose.yml" -f "$N8N_OVERRIDE" \ + up -d --pull never >>"$OPERATION_LOG" 2>&1 +webui_project="$(docker inspect open-webui --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || true)" +if [[ -n "$(docker ps -aq --filter name='^/open-webui$')" && "$webui_project" != "open-webui" ]]; then + docker rm -f open-webui >>"$OPERATION_LOG" 2>&1 +fi +docker compose --project-directory "$OPEN_WEBUI_DIR" -f "$OPEN_WEBUI_COMPOSE" \ + up -d --pull never >>"$OPERATION_LOG" 2>&1 + +deadline=$((SECONDS + 360)) +while ((SECONDS < deadline)); do + if collect_state true verifying "$RECEIPT_PATH" >/dev/null; then + break + fi + sleep 5 +done + +if result="$(collect_state true deployed_verified "$RECEIPT_PATH")"; then + trap - ERR + printf '%s\n' "$result" | tee "$RECEIPT_PATH" + chmod 600 "$RECEIPT_PATH" + exit 0 +fi + +result="$(collect_state true verifier_failed_snapshot_preserved "$RECEIPT_PATH" || true)" +trap - ERR +printf '%s\n' "$result" | tee "$RECEIPT_PATH" +chmod 600 "$RECEIPT_PATH" +exit 5 diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index 91f6fa0ce..05270aaa4 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -63,6 +63,31 @@ def test_full_stack_cold_start_check_bounds_ssh_probes() -> None: assert "SSH_110_RECOVERY_PACKAGE_NEXT_ACTION verify_or_preinstall_local_recovery_package_from_console_before_harbor_repair_retry" in text +def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None: + text = COLD_START_CHECK.read_text(encoding="utf-8") + baseline = (ROOT / "ops" / "reboot-recovery" / "full-stack-cold-start-baseline.yml").read_text( + encoding="utf-8" + ) + exporter = (ROOT / "scripts" / "reboot-recovery" / "cold-start-textfile-exporter.sh").read_text( + encoding="utf-8" + ) + deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8") + + assert "Scope: 110 / 112 / 120 / 121 / 188." in text + assert "for host in 110 112 120 121 188; do" in text + assert 'log_section "P0-112-SECURITY"' in text + assert "awoooi-host112-guest-readiness --check" in text + assert 'check_112\ncheck_188' in text + assert '"112": "Kali security, Wazuh, guest readiness and bounded recovery"' in baseline + assert "- id: P0-112-SECURITY" in baseline + assert "112 Kali is intentionally skipped" not in text + assert "not part of cold-start release gate" not in baseline + assert 'SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}"' in exporter + assert 'reason="host_unreachable",target="112"' in exporter + assert 'reason="guest_readiness_failed",target="112"' in exporter + assert 'scope="110_112_120_121_188"' in deploy_verifier + + def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None: text = COLD_START_CHECK.read_text(encoding="utf-8") diff --git a/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py b/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py new file mode 100644 index 000000000..6cb365c4d --- /dev/null +++ b/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py @@ -0,0 +1,79 @@ +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] + + +def read(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def test_agent99_config_covers_host188_edge_services_and_public_routes() -> None: + for name in ("agent99.config.example.json", "agent99.config.99.example.json"): + config = json.loads(read(name)) + edge = config["edgeServiceRecovery"] + + assert edge["enabled"] is True + assert edge["host"] == "192.168.0.188" + assert edge["scriptPath"] == "/home/ollama/bin/agent99-host188-edge-services-recover.sh" + assert set(edge["requiredServices"]) == {"n8n", "open-webui"} + assert "https://n8n.wooo.work" in config["publicUrls"] + assert "https://ollama.wooo.work" in config["publicUrls"] + + +def test_host188_recovery_uses_only_internal_pinned_images() -> None: + script = read("scripts/reboot-recovery/host188-edge-services-recover.sh") + + assert "harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:" in script + assert "harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:" in script + assert "harbor.wooo.work/dockerhub-cache/library/alpine@sha256:" in script + assert "ghcr.io/" not in script + assert "docker.n8n.io/" not in script + assert ":latest" not in script + assert "external_registry_direct_pull" in script + + +def test_host188_check_mode_is_read_only_and_apply_is_bounded() -> None: + script = read("scripts/reboot-recovery/host188-edge-services-recover.sh") + check_block = script.split('if [[ "$MODE" == "check" ]]', 1)[1].split( + "exec 9>", 1 + )[0] + + for forbidden in ( + "docker pull", + "docker run", + "docker compose", + "mkdir", + "mv ", + "chown", + "chmod", + "tee ", + ): + assert forbidden not in check_block + + assert "flock -w 30" in script + assert "snapshot_with_helper" in script + assert "--pull never" in script + assert '"secretValueRead":false' in script + assert '"rawDataRead":false' in script + assert '"automatic":false' in script + assert "host_reboot" in script + assert "vm_power_change" in script + + +def test_agent99_recover_binds_check_apply_verify_and_completion() -> None: + control = read("agent99-control-plane.ps1") + deploy = read("agent99-deploy.ps1") + playbook = read("infra/ansible/playbooks/188-ai-web.yml") + + assert "function Invoke-AgentHost188EdgeRecovery" in control + assert 'New-AgentRecoveryPhase "host188_edge_services"' in control + assert 'New-AgentOutcomeCheck "host188_edge_services_ready"' in control + assert "$host188EdgeRecovery.runtimeWritePerformed" in control + assert "$host188EdgeRecovery.verified" in control + assert "container_local_upstream_and_public_https" in control + assert 'name = "canonical_public_route_coverage"' in deploy + assert "agent99-host188-edge-services-recover.sh --check" in playbook + assert "--work-item-id AIA-P0-006-EDGE-502-188" in playbook + assert "not ansible_check_mode" in playbook diff --git a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh index d55face23..c968df054 100755 --- a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh +++ b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh @@ -129,10 +129,15 @@ require_remote_pattern \ "110 deployed check script carries SSH host-key policy" require_remote_pattern \ - 'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",mode="read_only"} 1' \ + 'awoooi_cold_start_monitor_up{host="110",scope="110_112_120_121_188",mode="read_only"} 1' \ "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \ "110 cold-start monitor produced parseable metrics" +require_remote_pattern \ + 'awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="guest_readiness_failed",target="112"}' \ + "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \ + "110 cold-start monitor exports host112 readiness blocker series" + report_runtime_state report_cold_start_alerts