param( [switch]$ControlledApply, [string]$SshUser = "wooo", [string]$EvidenceDir = "$env:ProgramData\Wooo\host-reboot-scorecard", [string[]]$Hosts = @("192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"), [string[]]$PublicUrls = @( "awoooi.wooo.work", "stock.wooo.work", "2026fifa.wooo.work", "gitea.wooo.work", "harbor.wooo.work" ) ) $ErrorActionPreference = "Continue" New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $jsonPath = Join-Path $EvidenceDir "scorecard-$stamp.json" $logPath = Join-Path $EvidenceDir "scorecard-$stamp.log" function Write-Evidence { param([string]$Message) $line = "$(Get-Date -Format o) $Message" $line | Tee-Object -FilePath $logPath -Append } function Test-PingHost { param([string]$Host) $ok = Test-Connection -ComputerName $Host -Count 1 -Quiet -ErrorAction SilentlyContinue [pscustomobject]@{ host = $Host ping = [bool]$ok } } function Invoke-SshText { param( [string]$Host, [string]$Command, [int]$TimeoutSeconds = 20 ) $args = @( "-o", "BatchMode=yes", "-o", "ConnectTimeout=$TimeoutSeconds", "-l", $SshUser, $Host, $Command ) try { $output = & ssh @args 2>&1 [pscustomobject]@{ ok = ($LASTEXITCODE -eq 0) exit_code = $LASTEXITCODE output = ($output -join "`n") } } catch { [pscustomobject]@{ ok = $false exit_code = -1 output = $_.Exception.Message } } } function Test-HttpHead { param([string]$Url) try { $output = & curl.exe --location --head --max-time 12 $Url 2>&1 $text = $output -join "`n" $statusMatches = [regex]::Matches($text, "HTTP/\S+\s+(\d+)") $lastStatus = if ($statusMatches.Count -gt 0) { [int]$statusMatches[$statusMatches.Count - 1].Groups[1].Value } else { $null } [pscustomobject]@{ url = $Url status = $lastStatus non_502 = ($lastStatus -ne 502 -and $null -ne $lastStatus) ok = ($lastStatus -ge 200 -and $lastStatus -lt 500) output = $text } } catch { [pscustomobject]@{ url = $Url status = $null non_502 = $false ok = $false output = $_.Exception.Message } } } function Get-HarborState { $core = Invoke-SshText "192.168.0.110" "docker ps --filter name=harbor-core" $registry = Invoke-SshText "192.168.0.110" "docker ps --filter name=registry" $redis = Invoke-SshText "192.168.0.110" "docker ps --filter name=redis" [pscustomobject]@{ core_ok = ($core.output -match "harbor-core" -and $core.output -match "healthy") registry_ok = ($registry.output -match "registry" -and $registry.output -match "healthy") redis_ok = ($redis.output -match "redis" -and $redis.output -match "Up") core = $core.output registry = $registry.output redis = $redis.output } } function Get-AwoooDeployState { $deploy = Invoke-SshText "192.168.0.120" "sudo -n kubectl get deploy -n awoooi-prod" $pods = Invoke-SshText "192.168.0.120" "sudo -n kubectl get pods -n awoooi-prod" [pscustomobject]@{ deploy_ok = ( $deploy.output -match "awoooi-api\s+1/1" -and $deploy.output -match "awoooi-web\s+2/2" -and $deploy.output -match "awoooi-worker\s+1/1" ) image_pull_failure = ($pods.output -match "ImagePullBackOff|ErrImagePull") crash_failure = ($pods.output -match "CrashLoopBackOff|Pending") deploy = $deploy.output pods = $pods.output } } function Repair-HarborIfAllowed { param([object]$Harbor) if (-not $ControlledApply) { return } if (-not $Harbor.redis_ok) { Write-Evidence "CONTROLLED_APPLY restart harbor redis" Invoke-SshText "192.168.0.110" "docker restart redis" | Out-Null } if (-not $Harbor.core_ok) { Write-Evidence "CONTROLLED_APPLY restart harbor-core" Invoke-SshText "192.168.0.110" "docker restart harbor-core" | Out-Null } } function Repair-AwoooPodsIfAllowed { param([object]$Awooo, [object]$Harbor) if (-not $ControlledApply) { return } if (-not $Harbor.core_ok) { return } if (-not $Awooo.image_pull_failure) { return } Write-Evidence "CONTROLLED_APPLY delete AWOOOI ImagePullBackOff pods after Harbor core healthy" Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-api" | Out-Null Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-web" | Out-Null Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-worker" | Out-Null } Write-Evidence "scorecard_start controlled_apply=$ControlledApply" $hostResults = @() foreach ($h in $Hosts) { $result = Test-PingHost $h $hostResults += $result Write-Evidence "host_ping host=$($result.host) ok=$($result.ping)" } $harbor = Get-HarborState Write-Evidence "harbor core_ok=$($harbor.core_ok) registry_ok=$($harbor.registry_ok) redis_ok=$($harbor.redis_ok)" Repair-HarborIfAllowed $harbor $awooo = Get-AwoooDeployState Write-Evidence "awoooi deploy_ok=$($awooo.deploy_ok) image_pull_failure=$($awooo.image_pull_failure) crash_failure=$($awooo.crash_failure)" Repair-AwoooPodsIfAllowed $awooo $harbor $httpResults = @() foreach ($u in $PublicUrls) { $result = Test-HttpHead $u $httpResults += $result Write-Evidence "http url=$($result.url) status=$($result.status) non_502=$($result.non_502) ok=$($result.ok)" } $summary = [pscustomobject]@{ timestamp = (Get-Date -Format o) controlled_apply = [bool]$ControlledApply hosts = $hostResults harbor = $harbor awoooi = $awooo public_http = $httpResults completed = ( ($hostResults | Where-Object { -not $_.ping }).Count -eq 0 -and $harbor.core_ok -and $awooo.deploy_ok -and ($httpResults | Where-Object { -not $_.non_502 }).Count -eq 0 ) } $summary | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath Write-Evidence "scorecard_complete completed=$($summary.completed) json=$jsonPath" if (-not $summary.completed) { exit 2 } exit 0