awoooi/host-reboot-scorecard.ps1

param(
  [switch]$ControlledApply,
  [string]$SshUser = "wooo",
  [string]$EvidenceDir = "$env:ProgramData\Wooo\host-reboot-scorecard",
  [string[]]$Hosts = @("192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"),
  [string[]]$PublicUrls = @(
    "awoooi.wooo.work",
    "stock.wooo.work",
    "2026fifa.wooo.work",
    "gitea.wooo.work",
    "harbor.wooo.work"
  )
)

$ErrorActionPreference = "Continue"
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null

$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
$jsonPath = Join-Path $EvidenceDir "scorecard-$stamp.json"
$logPath = Join-Path $EvidenceDir "scorecard-$stamp.log"

function Write-Evidence {
  param([string]$Message)
  $line = "$(Get-Date -Format o) $Message"
  $line | Tee-Object -FilePath $logPath -Append
}

function Test-PingHost {
  param([string]$Host)
  $ok = Test-Connection -ComputerName $Host -Count 1 -Quiet -ErrorAction SilentlyContinue
  [pscustomobject]@{
    host = $Host
    ping = [bool]$ok
  }
}

function Invoke-SshText {
  param(
    [string]$Host,
    [string]$Command,
    [int]$TimeoutSeconds = 20
  )

  $args = @(
    "-o", "BatchMode=yes",
    "-o", "ConnectTimeout=$TimeoutSeconds",
    "-l", $SshUser,
    $Host,
    $Command
  )

  try {
    $output = & ssh @args 2>&1
    [pscustomobject]@{
      ok = ($LASTEXITCODE -eq 0)
      exit_code = $LASTEXITCODE
      output = ($output -join "`n")
    }
  } catch {
    [pscustomobject]@{
      ok = $false
      exit_code = -1
      output = $_.Exception.Message
    }
  }
}

function Test-HttpHead {
  param([string]$Url)
  try {
    $output = & curl.exe --location --head --max-time 12 $Url 2>&1
    $text = $output -join "`n"
    $statusMatches = [regex]::Matches($text, "HTTP/\S+\s+(\d+)")
    $lastStatus = if ($statusMatches.Count -gt 0) { [int]$statusMatches[$statusMatches.Count - 1].Groups[1].Value } else { $null }
    [pscustomobject]@{
      url = $Url
      status = $lastStatus
      non_502 = ($lastStatus -ne 502 -and $null -ne $lastStatus)
      ok = ($lastStatus -ge 200 -and $lastStatus -lt 500)
      output = $text
    }
  } catch {
    [pscustomobject]@{
      url = $Url
      status = $null
      non_502 = $false
      ok = $false
      output = $_.Exception.Message
    }
  }
}

function Get-HarborState {
  $core = Invoke-SshText "192.168.0.110" "docker ps --filter name=harbor-core"
  $registry = Invoke-SshText "192.168.0.110" "docker ps --filter name=registry"
  $redis = Invoke-SshText "192.168.0.110" "docker ps --filter name=redis"

  [pscustomobject]@{
    core_ok = ($core.output -match "harbor-core" -and $core.output -match "healthy")
    registry_ok = ($registry.output -match "registry" -and $registry.output -match "healthy")
    redis_ok = ($redis.output -match "redis" -and $redis.output -match "Up")
    core = $core.output
    registry = $registry.output
    redis = $redis.output
  }
}

function Get-AwoooDeployState {
  $deploy = Invoke-SshText "192.168.0.120" "sudo -n kubectl get deploy -n awoooi-prod"
  $pods = Invoke-SshText "192.168.0.120" "sudo -n kubectl get pods -n awoooi-prod"
  [pscustomobject]@{
    deploy_ok = (
      $deploy.output -match "awoooi-api\s+1/1" -and
      $deploy.output -match "awoooi-web\s+2/2" -and
      $deploy.output -match "awoooi-worker\s+1/1"
    )
    image_pull_failure = ($pods.output -match "ImagePullBackOff|ErrImagePull")
    crash_failure = ($pods.output -match "CrashLoopBackOff|Pending")
    deploy = $deploy.output
    pods = $pods.output
  }
}

function Repair-HarborIfAllowed {
  param([object]$Harbor)
  if (-not $ControlledApply) { return }

  if (-not $Harbor.redis_ok) {
    Write-Evidence "CONTROLLED_APPLY restart harbor redis"
    Invoke-SshText "192.168.0.110" "docker restart redis" | Out-Null
  }

  if (-not $Harbor.core_ok) {
    Write-Evidence "CONTROLLED_APPLY restart harbor-core"
    Invoke-SshText "192.168.0.110" "docker restart harbor-core" | Out-Null
  }
}

function Repair-AwoooPodsIfAllowed {
  param([object]$Awooo, [object]$Harbor)
  if (-not $ControlledApply) { return }
  if (-not $Harbor.core_ok) { return }
  if (-not $Awooo.image_pull_failure) { return }

  Write-Evidence "CONTROLLED_APPLY delete AWOOOI ImagePullBackOff pods after Harbor core healthy"
  Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-api" | Out-Null
  Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-web" | Out-Null
  Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-worker" | Out-Null
}

Write-Evidence "scorecard_start controlled_apply=$ControlledApply"

$hostResults = @()
foreach ($h in $Hosts) {
  $result = Test-PingHost $h
  $hostResults += $result
  Write-Evidence "host_ping host=$($result.host) ok=$($result.ping)"
}

$harbor = Get-HarborState
Write-Evidence "harbor core_ok=$($harbor.core_ok) registry_ok=$($harbor.registry_ok) redis_ok=$($harbor.redis_ok)"
Repair-HarborIfAllowed $harbor

$awooo = Get-AwoooDeployState
Write-Evidence "awoooi deploy_ok=$($awooo.deploy_ok) image_pull_failure=$($awooo.image_pull_failure) crash_failure=$($awooo.crash_failure)"
Repair-AwoooPodsIfAllowed $awooo $harbor

$httpResults = @()
foreach ($u in $PublicUrls) {
  $result = Test-HttpHead $u
  $httpResults += $result
  Write-Evidence "http url=$($result.url) status=$($result.status) non_502=$($result.non_502) ok=$($result.ok)"
}

$summary = [pscustomobject]@{
  timestamp = (Get-Date -Format o)
  controlled_apply = [bool]$ControlledApply
  hosts = $hostResults
  harbor = $harbor
  awoooi = $awooo
  public_http = $httpResults
  completed = (
    ($hostResults | Where-Object { -not $_.ping }).Count -eq 0 -and
    $harbor.core_ok -and
    $awooo.deploy_ok -and
    ($httpResults | Where-Object { -not $_.non_502 }).Count -eq 0
  )
}

$summary | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath
Write-Evidence "scorecard_complete completed=$($summary.completed) json=$jsonPath"

if (-not $summary.completed) {
  exit 2
}

exit 0