4610 lines
207 KiB
PowerShell
4610 lines
207 KiB
PowerShell
param(
|
||
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
|
||
[string]$Mode = "Status",
|
||
[switch]$ControlledApply,
|
||
[switch]$SelfTestTelegramCard,
|
||
[switch]$SelfTestTelegramDelivery,
|
||
[switch]$SelfTestSensorGate,
|
||
[switch]$SelfTestOutcomeContract,
|
||
[switch]$SuppressAlerts,
|
||
[string]$SuppressReason = "",
|
||
[string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json",
|
||
[string]$EvidenceDir = "C:\Wooo\Agent99\evidence"
|
||
)
|
||
|
||
$ErrorActionPreference = "Continue"
|
||
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
|
||
|
||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||
$logPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.log"
|
||
$jsonPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.json"
|
||
|
||
function Write-AgentLog {
|
||
param([string]$Message)
|
||
$line = "$(Get-Date -Format o) $Message"
|
||
$line | Tee-Object -FilePath $logPath -Append | Out-Null
|
||
}
|
||
|
||
function Load-Config {
|
||
if (-not (Test-Path $ConfigPath)) {
|
||
throw "Config not found: $ConfigPath"
|
||
}
|
||
Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
||
}
|
||
|
||
$Config = Load-Config
|
||
$SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" }
|
||
$SshIdentityFile = if ($Config.PSObject.Properties["sshIdentityFile"] -and $Config.sshIdentityFile) {
|
||
[Environment]::ExpandEnvironmentVariables([string]$Config.sshIdentityFile)
|
||
} else {
|
||
""
|
||
}
|
||
$script:Events = @()
|
||
$script:TelegramAttempts = @()
|
||
$script:SuppressAgentAlerts = [bool]$SuppressAlerts
|
||
$script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" }
|
||
|
||
function Convert-AgentBool {
|
||
param([object]$Value)
|
||
if ($null -eq $Value) { return $false }
|
||
if ($Value -is [bool]) { return [bool]$Value }
|
||
$text = ([string]$Value).Trim()
|
||
if (-not $text) { return $false }
|
||
return [bool]($text -match "^(?i:true|1|yes|y|on)$")
|
||
}
|
||
|
||
function Get-HostSshUser {
|
||
param([string]$TargetHost)
|
||
if ($Config.sshUsers -and $Config.sshUsers.PSObject.Properties[$TargetHost]) {
|
||
return [string]$Config.sshUsers.PSObject.Properties[$TargetHost].Value
|
||
}
|
||
$SshUser
|
||
}
|
||
|
||
function Get-AgentSshIdentityArguments {
|
||
if (-not $SshIdentityFile) { return @() }
|
||
@("-i", $SshIdentityFile, "-o", "IdentitiesOnly=yes")
|
||
}
|
||
|
||
function Get-AgentSshTransportConfig {
|
||
$configured = if ($Config.PSObject.Properties["sshTransport"]) { $Config.sshTransport } else { $null }
|
||
[pscustomobject]@{
|
||
serialized = [bool](-not $configured -or -not $configured.PSObject.Properties["serialized"] -or (Convert-AgentBool $configured.serialized))
|
||
lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Max(10, [int]$configured.lockTimeoutSeconds) } else { 90 }
|
||
lockPollMilliseconds = if ($configured -and $configured.PSObject.Properties["lockPollMilliseconds"]) { [math]::Min(2000, [math]::Max(50, [int]$configured.lockPollMilliseconds)) } else { 200 }
|
||
lockPath = Join-Path (Join-Path $Config.agentRoot "state") "ssh-transport.lock"
|
||
}
|
||
}
|
||
|
||
function Enter-AgentSshTransportLock {
|
||
$transport = Get-AgentSshTransportConfig
|
||
if (-not $transport.serialized) {
|
||
return [pscustomobject]@{ acquired = $true; disabled = $true; reason = "serialization_disabled"; waitMs = 0; stream = $null }
|
||
}
|
||
|
||
$lockDir = Split-Path -Parent $transport.lockPath
|
||
New-Item -ItemType Directory -Force -Path $lockDir | Out-Null
|
||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||
while ($stopwatch.Elapsed.TotalSeconds -lt $transport.lockTimeoutSeconds) {
|
||
try {
|
||
$stream = [IO.File]::Open(
|
||
$transport.lockPath,
|
||
[IO.FileMode]::OpenOrCreate,
|
||
[IO.FileAccess]::ReadWrite,
|
||
[IO.FileShare]::None
|
||
)
|
||
$stopwatch.Stop()
|
||
return [pscustomobject]@{
|
||
acquired = $true
|
||
disabled = $false
|
||
reason = "acquired"
|
||
waitMs = $stopwatch.ElapsedMilliseconds
|
||
stream = $stream
|
||
}
|
||
} catch [System.IO.IOException] {
|
||
Start-Sleep -Milliseconds $transport.lockPollMilliseconds
|
||
} catch [System.UnauthorizedAccessException] {
|
||
$stopwatch.Stop()
|
||
return [pscustomobject]@{
|
||
acquired = $false
|
||
disabled = $false
|
||
reason = "access_denied"
|
||
waitMs = $stopwatch.ElapsedMilliseconds
|
||
stream = $null
|
||
}
|
||
}
|
||
}
|
||
|
||
$stopwatch.Stop()
|
||
[pscustomobject]@{
|
||
acquired = $false
|
||
disabled = $false
|
||
reason = "timeout"
|
||
waitMs = $stopwatch.ElapsedMilliseconds
|
||
stream = $null
|
||
}
|
||
}
|
||
|
||
function Exit-AgentSshTransportLock {
|
||
param([object]$TransportLock)
|
||
if ($TransportLock -and $TransportLock.stream) {
|
||
try { $TransportLock.stream.Dispose() } catch {}
|
||
}
|
||
}
|
||
|
||
function Add-AgentSshTransportMetadata {
|
||
param([object]$Result, [object]$TransportLock)
|
||
if ($Result) {
|
||
$Result | Add-Member -NotePropertyName transportSerialized -NotePropertyValue ([bool](-not $TransportLock.disabled)) -Force
|
||
$Result | Add-Member -NotePropertyName transportLockWaitMs -NotePropertyValue ([long]$TransportLock.waitMs) -Force
|
||
}
|
||
$Result
|
||
}
|
||
|
||
function Get-AgentSshProcessGuardConfig {
|
||
$configured = if ($Config.PSObject.Properties["sshProcessGuard"]) { $Config.sshProcessGuard } else { $null }
|
||
[pscustomobject]@{
|
||
enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled))
|
||
staleMinutes = if ($configured -and $configured.PSObject.Properties["staleMinutes"]) { [math]::Max(5, [int]$configured.staleMinutes) } else { 15 }
|
||
hardStaleMinutes = if ($configured -and $configured.PSObject.Properties["hardStaleMinutes"]) { [math]::Max(15, [int]$configured.hardStaleMinutes) } else { 60 }
|
||
}
|
||
}
|
||
|
||
function Invoke-AgentStaleSshProcessGuard {
|
||
$guard = Get-AgentSshProcessGuardConfig
|
||
if (-not $guard.enabled -or -not $ControlledApply) {
|
||
return [pscustomobject]@{ ok = $true; skipped = $true; reason = "guard_disabled_or_observe_mode"; stoppedCount = 0; remainingCount = $null; processes = @() }
|
||
}
|
||
|
||
$targets = @($Config.hosts)
|
||
if ($Config.k3s -and $Config.k3s.PSObject.Properties["jumpHost"] -and $Config.k3s.jumpHost) {
|
||
$targets += [string]$Config.k3s.jumpHost
|
||
}
|
||
$targets = @($targets | Select-Object -Unique)
|
||
$rows = @()
|
||
foreach ($processInfo in @(Get-CimInstance Win32_Process -Filter "Name='ssh.exe'" -ErrorAction SilentlyContinue)) {
|
||
try {
|
||
$process = Get-Process -Id $processInfo.ProcessId -ErrorAction Stop
|
||
$ageMinutes = [math]::Round(((Get-Date) - $process.StartTime).TotalMinutes, 1)
|
||
$target = $targets | Where-Object { $processInfo.CommandLine -match [regex]::Escape($_) } | Select-Object -First 1
|
||
$batchMode = [bool]($processInfo.CommandLine -match "BatchMode=yes")
|
||
$portForward = [bool]($processInfo.CommandLine -cmatch "(^|\s)-[NLRD](\s|$)")
|
||
$parentAlive = [bool](Get-Process -Id $processInfo.ParentProcessId -ErrorAction SilentlyContinue)
|
||
$eligible = [bool](
|
||
$ageMinutes -ge $guard.staleMinutes -and
|
||
$target -and
|
||
$batchMode -and
|
||
-not $portForward -and
|
||
(-not $parentAlive -or $ageMinutes -ge $guard.hardStaleMinutes)
|
||
)
|
||
$stopped = $false
|
||
if ($eligible) {
|
||
Stop-Process -Id $processInfo.ProcessId -Force -ErrorAction Stop
|
||
$stopped = $true
|
||
}
|
||
$rows += [pscustomobject]@{
|
||
pid = $processInfo.ProcessId
|
||
ageMinutes = $ageMinutes
|
||
target = $target
|
||
batchMode = $batchMode
|
||
portForward = $portForward
|
||
parentAlive = $parentAlive
|
||
eligible = $eligible
|
||
stopped = $stopped
|
||
}
|
||
} catch {
|
||
$rows += [pscustomobject]@{ pid = $processInfo.ProcessId; eligible = $false; stopped = $false; error = "process_disappeared_or_stop_failed" }
|
||
}
|
||
}
|
||
|
||
$stoppedCount = @($rows | Where-Object { $_.stopped }).Count
|
||
$failures = @($rows | Where-Object { $_.eligible -and -not $_.stopped }).Count
|
||
$remainingCount = @(Get-Process ssh -ErrorAction SilentlyContinue).Count
|
||
$result = [pscustomobject]@{
|
||
ok = [bool]($failures -eq 0)
|
||
skipped = $false
|
||
stoppedCount = $stoppedCount
|
||
failureCount = $failures
|
||
remainingCount = $remainingCount
|
||
staleMinutes = $guard.staleMinutes
|
||
hardStaleMinutes = $guard.hardStaleMinutes
|
||
processes = $rows
|
||
}
|
||
if ($stoppedCount -gt 0 -or $failures -gt 0) {
|
||
Record-AgentEvent "ssh_stale_process_guard" $(if ($failures -gt 0) { "warning" } else { "info" }) "stopped=$stoppedCount failures=$failures remaining=$remainingCount" $result -NoAlert
|
||
}
|
||
$result
|
||
}
|
||
|
||
function Send-AgentTelegramRelay {
|
||
param(
|
||
[string]$Message,
|
||
[object]$Relay,
|
||
[string]$ChatIdOverride = "",
|
||
[string]$PhotoPath = ""
|
||
)
|
||
|
||
if (-not ($Relay -and $Relay.enabled)) {
|
||
return [pscustomobject]@{ ok = $false; error = "relay_disabled" }
|
||
}
|
||
|
||
$relayHost = if ($Relay.host) { [string]$Relay.host } else { "192.168.0.110" }
|
||
$container = if ($Relay.container) { [string]$Relay.container } else { "stockplatform-v2-api-1" }
|
||
if ($container -notmatch "^[A-Za-z0-9_.-]+$") {
|
||
return [pscustomobject]@{ ok = $false; error = "invalid_relay_container" }
|
||
}
|
||
|
||
$messageBytes = [Text.Encoding]::UTF8.GetBytes($Message)
|
||
$messageB64 = [Convert]::ToBase64String($messageBytes)
|
||
$chatOverrideB64 = if ($ChatIdOverride) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($ChatIdOverride)) } else { "" }
|
||
$photoSent = $false
|
||
$photoError = $null
|
||
$containerPhotoPath = ""
|
||
if ($PhotoPath -and (Test-Path -LiteralPath $PhotoPath)) {
|
||
$safePhotoName = "agent99-card-" + ([guid]::NewGuid().ToString("N")) + ".png"
|
||
$remotePhotoPath = "/tmp/" + $safePhotoName
|
||
$containerPhotoPath = "/tmp/" + $safePhotoName
|
||
$relayUser = Get-HostSshUser $relayHost
|
||
$scpArgs = @(Get-AgentSshIdentityArguments) + @(
|
||
"-o", "BatchMode=yes",
|
||
"-o", "StrictHostKeyChecking=accept-new",
|
||
"-o", "NumberOfPasswordPrompts=0",
|
||
"-o", "ConnectTimeout=10",
|
||
$PhotoPath,
|
||
"${relayUser}@${relayHost}:$remotePhotoPath"
|
||
)
|
||
try {
|
||
$scp = Start-Process -FilePath "scp.exe" -ArgumentList $scpArgs -NoNewWindow -Wait -PassThru
|
||
$scp.Refresh()
|
||
if ($null -ne $scp.ExitCode -and $scp.ExitCode -eq 0) {
|
||
$copyRun = Invoke-SshText $relayHost "docker cp $remotePhotoPath $container`:$containerPhotoPath && rm -f $remotePhotoPath" 45 1
|
||
if (-not $copyRun.ok) {
|
||
$photoError = "relay_docker_cp_failed: $($copyRun.output)"
|
||
$containerPhotoPath = ""
|
||
}
|
||
} else {
|
||
$photoError = "relay_scp_failed exitCode=$($scp.ExitCode)"
|
||
$containerPhotoPath = ""
|
||
}
|
||
} catch {
|
||
$photoError = "relay_scp_exception: $($_.Exception.Message)"
|
||
$containerPhotoPath = ""
|
||
}
|
||
}
|
||
|
||
$containerPhotoB64 = if ($containerPhotoPath) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($containerPhotoPath)) } else { "" }
|
||
$python = 'import base64, mimetypes, os, sys, urllib.error, urllib.parse, urllib.request
|
||
text = base64.b64decode(sys.argv[2]).decode()
|
||
chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] and sys.argv[3] != "-" else ""
|
||
photo_path = base64.b64decode(sys.argv[4]).decode() if len(sys.argv) > 4 and sys.argv[4] and sys.argv[4] != "-" else ""
|
||
token = os.environ.get("TELEGRAM_BOT_TOKEN")
|
||
chat = chat_override or os.environ.get("TELEGRAM_CHAT_ID")
|
||
assert token and chat
|
||
if photo_path and os.path.exists(photo_path):
|
||
caption = text
|
||
if len(caption) > 900:
|
||
caption = caption[:900] + "\n...(truncated; see evidence)"
|
||
filename = os.path.basename(photo_path)
|
||
ctype = mimetypes.guess_type(filename)[0] or "image/png"
|
||
try:
|
||
import requests
|
||
with open(photo_path, "rb") as fh:
|
||
response = requests.post(
|
||
"https://api.telegram.org/bot" + token + "/sendPhoto",
|
||
data={"chat_id": chat, "caption": caption},
|
||
files={"photo": (filename, fh, ctype)},
|
||
timeout=30,
|
||
)
|
||
if not response.ok:
|
||
print("photo_send_failed status=%s body=%s" % (response.status_code, response.text[:500]))
|
||
sys.exit(2)
|
||
print("photo_sent_ok")
|
||
except ImportError:
|
||
boundary = "----agent99boundary"
|
||
fields = [("chat_id", chat), ("caption", caption)]
|
||
body = bytearray()
|
||
for name, value in fields:
|
||
body.extend(("--" + boundary + "\r\n").encode())
|
||
body.extend(("Content-Disposition: form-data; name=\"" + name + "\"\r\n\r\n").encode())
|
||
body.extend(str(value).encode())
|
||
body.extend(b"\r\n")
|
||
body.extend(("--" + boundary + "\r\n").encode())
|
||
body.extend(("Content-Disposition: form-data; name=\"photo\"; filename=\"" + filename + "\"\r\n").encode())
|
||
body.extend(("Content-Type: " + ctype + "\r\n\r\n").encode())
|
||
with open(photo_path, "rb") as fh:
|
||
body.extend(fh.read())
|
||
body.extend(b"\r\n")
|
||
body.extend(("--" + boundary + "--\r\n").encode())
|
||
req = urllib.request.Request("https://api.telegram.org/bot" + token + "/sendPhoto", data=bytes(body))
|
||
req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary)
|
||
try:
|
||
urllib.request.urlopen(req, timeout=30).read()
|
||
print("photo_sent_ok")
|
||
except urllib.error.HTTPError as exc:
|
||
print("photo_send_failed status=%s body=%s" % (exc.code, exc.read().decode(errors="replace")[:500]))
|
||
sys.exit(2)
|
||
else:
|
||
data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode()
|
||
urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read()
|
||
print("sent_ok")
|
||
'
|
||
$pythonB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($python))
|
||
$bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))'
|
||
$chatOverrideArg = if ($chatOverrideB64) { $chatOverrideB64 } else { "-" }
|
||
$containerPhotoArg = if ($containerPhotoB64) { $containerPhotoB64 } else { "-" }
|
||
$remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideArg $containerPhotoArg"
|
||
$run = Invoke-SshText $relayHost $remoteCommand 45 1
|
||
$photoSent = [bool]($run.ok -and $run.output -match "photo_sent_ok")
|
||
if ($containerPhotoPath) {
|
||
Invoke-SshText $relayHost "docker exec $container rm -f $containerPhotoPath" 20 1 | Out-Null
|
||
}
|
||
|
||
[pscustomobject]@{
|
||
ok = [bool]($run.ok -and ($run.output -match "sent_ok|photo_sent_ok"))
|
||
host = $relayHost
|
||
container = $container
|
||
exitCode = $run.exitCode
|
||
chatOverride = if ($ChatIdOverride) { "telegram-origin" } else { $null }
|
||
photoSent = $photoSent
|
||
photoError = $photoError
|
||
output = $run.output
|
||
error = if ($run.ok) { $photoError } else { $run.output }
|
||
}
|
||
}
|
||
|
||
function Get-AgentObjectValue {
|
||
param(
|
||
[object]$Data,
|
||
[string]$Name,
|
||
[object]$Default = $null
|
||
)
|
||
if ($Data -and $Data.PSObject.Properties[$Name]) {
|
||
$value = $Data.PSObject.Properties[$Name].Value
|
||
if ($null -ne $value -and [string]$value -ne "") {
|
||
return $value
|
||
}
|
||
}
|
||
$Default
|
||
}
|
||
|
||
function Limit-AgentTextLine {
|
||
param(
|
||
[string]$Text,
|
||
[int]$MaxLength = 96
|
||
)
|
||
if (-not $Text) { return "" }
|
||
$clean = (($Text -replace "\s+", " ").Trim())
|
||
if ($clean.Length -le $MaxLength) { return $clean }
|
||
$clean.Substring(0, $MaxLength - 14) + "...(truncated)"
|
||
}
|
||
|
||
function Format-AgentMetricForCard {
|
||
param(
|
||
[object]$Value,
|
||
[string]$Suffix = ""
|
||
)
|
||
if ($null -eq $Value -or [string]$Value -eq "") {
|
||
return "readback_failed"
|
||
}
|
||
"$Value$Suffix"
|
||
}
|
||
|
||
function Format-AgentTelegramText {
|
||
param(
|
||
[string]$Severity,
|
||
[string]$EventType,
|
||
[string]$Message,
|
||
[object]$Data = $null
|
||
)
|
||
|
||
$severityLabel = switch ($Severity) {
|
||
"critical" { "CRITICAL" }
|
||
"warning" { "WARNING" }
|
||
"info" { "INFO" }
|
||
default { $Severity }
|
||
}
|
||
$lines = @()
|
||
$lines += "Agent99 事件卡 / Incident Card"
|
||
$lines += "嚴重度 Severity: $severityLabel"
|
||
|
||
if ($EventType -match "^performance_") {
|
||
$hostName = Get-AgentObjectValue $Data "host" "unknown"
|
||
$reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() }
|
||
$disk = Get-AgentObjectValue $Data "diskUsedPercent" $null
|
||
$load = Get-AgentObjectValue $Data "loadPerCore" $null
|
||
$mem = Get-AgentObjectValue $Data "memAvailablePercent" $null
|
||
$lines += "事件 Event: 主機效能 / host performance"
|
||
$lines += "主機 Host: $hostName"
|
||
if ($reasons -contains "perf_readback_failed" -or ($null -eq $disk -and $null -eq $load -and $null -eq $mem)) {
|
||
$lines += "原因 Reason: 效能指標 readback 失敗。"
|
||
$lines += "影響 Impact: CPU load、memory、disk 數值尚未可信，不能直接當成真故障。"
|
||
$lines += "Agent99 動作 Action: 維持噪音 sensor 暫停；修復前必須先補 verifier readback。"
|
||
} elseif ($reasons -contains "disk_used_warning") {
|
||
$lines += "原因 Reason: 磁碟使用率 $disk percent，已達 warning threshold；critical threshold 是 95 percent。"
|
||
$lines += "Agent99 動作 Action: 執行 allowlisted disk remediation，並在 30 分鐘內 suppress duplicates。"
|
||
} elseif ($reasons -contains "disk_used_high") {
|
||
$lines += "原因 Reason: 磁碟使用率 $disk percent，已達 critical threshold。"
|
||
$lines += "Agent99 動作 Action: 先執行 allowlisted cleanup；reboot 不是第一反應。"
|
||
} elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") {
|
||
$lines += "原因 Reason: CPU load/core=$load。"
|
||
$lines += "Agent99 動作 Action: 僅使用 allowlisted load reduction；不得任意停止服務。"
|
||
} elseif ($reasons -contains "memory_available_low") {
|
||
$lines += "原因 Reason: 可用記憶體為 $mem percent。"
|
||
$lines += "Agent99 動作 Action: 先檢查 memory pressure，再進行 allowlisted remediation。"
|
||
} else {
|
||
$lines += "原因 Reason: $($reasons -join ', ')"
|
||
$lines += "Agent99 動作 Action: 檢查 Agent99 evidence，完成 full readback 後再處理。"
|
||
}
|
||
$metricParts = @()
|
||
if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" }
|
||
if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" }
|
||
if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" }
|
||
if ($metricParts.Count -gt 0) {
|
||
$lines += "目前狀態 Current: $($metricParts -join ', ')."
|
||
} else {
|
||
$lines += "目前狀態 Current: metrics unavailable；空白指標不視為可信告警。"
|
||
}
|
||
} elseif ($EventType -match "^backup_") {
|
||
$targetName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" }
|
||
$reason = if ($Data -and $Data.PSObject.Properties["reason"]) { $Data.reason } else { $EventType }
|
||
$age = if ($Data -and $Data.PSObject.Properties["ageMinutes"]) { $Data.ageMinutes } else { $null }
|
||
$maxAge = if ($Data -and $Data.PSObject.Properties["maxAgeMinutes"]) { $Data.maxAgeMinutes } else { $null }
|
||
$lines += "事件 Event: 備份健康 / backup health"
|
||
$lines += "目標 Target: $targetName"
|
||
$lines += "原因 Reason: $reason"
|
||
if ($age -ne $null -or $maxAge -ne $null) {
|
||
$lines += "新鮮度 Freshness: age=$age minutes / max=$maxAge minutes"
|
||
}
|
||
$lines += "Agent99 動作 Action: 只檢查 /backup/status、/backup/logs、snapshot metadata；不得讀取備份內容或 secrets。"
|
||
} elseif ($EventType -eq "provider_freshness_triage") {
|
||
$candidateId = if ($Data -and $Data.PSObject.Properties["candidateId"]) { $Data.candidateId } else { "unknown" }
|
||
$targetName = if ($Data -and $Data.PSObject.Properties["target"]) { $Data.target } else { "provider_freshness_signal" }
|
||
$hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 }
|
||
$missing = if ($Data -and $Data.PSObject.Properties["missingFields"]) { @($Data.missingFields) } else { @() }
|
||
$candidatePath = if ($Data -and $Data.PSObject.Properties["candidatePath"]) { $Data.candidatePath } else { $null }
|
||
$readback = if ($Data -and $Data.PSObject.Properties["verifierReadback"]) { $Data.verifierReadback } else { $null }
|
||
$status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" }
|
||
$lines += "事件 Event: Provider freshness 自動判讀 / provider freshness auto triage"
|
||
$lines += "目標 Target: $targetName"
|
||
$lines += "狀態 Status: $status"
|
||
$lines += "影響 Impact: source freshness 未證明前，dashboard green 不可視為健康。"
|
||
$lines += "證據命中 Evidence hits: $hitCount"
|
||
if ($readback) {
|
||
$lines += "驗證 Verifier: directReadback ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)"
|
||
}
|
||
if (@($missing).Count -gt 0) {
|
||
$lines += "仍缺欄位 Still missing: $($missing -join ', ')"
|
||
} else {
|
||
$lines += "缺漏狀態 Missing: direct readback fields completed"
|
||
}
|
||
$lines += "Agent99 動作 Action: 已寫入 candidate 與 KM/RAG；相同狀態 duplicate alerts 會被 suppress。"
|
||
$lines += "限制 Guard: 不切 provider、不呼叫 paid model、不改 env、不 reload。"
|
||
if ($candidatePath) {
|
||
$lines += "證據 Evidence: $candidatePath"
|
||
}
|
||
} elseif ($EventType -match "^load_shed_") {
|
||
$actionName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" }
|
||
$hostName = if ($Data -and $Data.PSObject.Properties["host"]) { $Data.host } else { "unknown" }
|
||
$ok = if ($Data -and $Data.PSObject.Properties["ok"]) { $Data.ok } else { $null }
|
||
$executionOk = if ($Data -and $Data.PSObject.Properties["executionOk"]) { $Data.executionOk } else { $null }
|
||
$postVerifyOk = if ($Data -and $Data.PSObject.Properties["postVerifyOk"]) { $Data.postVerifyOk } else { $null }
|
||
$exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null }
|
||
$rollback = if ($Data -and $Data.PSObject.Properties["rollback"]) { $Data.rollback } else { $null }
|
||
$lines += "事件 Event: 受控修復 / controlled remediation"
|
||
$lines += "主機 Host: $hostName"
|
||
$lines += "Agent99 動作 Action: $actionName"
|
||
if ($ok -ne $null) {
|
||
$lines += "結果 Result: ok=$ok exitCode=$exitCode"
|
||
if ($executionOk -ne $null -or $postVerifyOk -ne $null) {
|
||
$lines += "驗證 Verifier: executionOk=$executionOk postVerifyOk=$postVerifyOk"
|
||
}
|
||
} else {
|
||
$lines += "結果 Result: 請查看 Agent99 evidence。"
|
||
}
|
||
if ($rollback) {
|
||
$lines += "回滾 Rollback: $rollback"
|
||
}
|
||
} elseif ($EventType -match "^operator_command_") {
|
||
$commandId = if ($Data -and $Data.PSObject.Properties["id"]) { $Data.id } else { "unknown" }
|
||
$modeName = if ($Data -and $Data.PSObject.Properties["mode"]) { $Data.mode } else { "unknown" }
|
||
$source = if ($Data -and $Data.PSObject.Properties["source"]) { $Data.source } else { "unknown" }
|
||
$instruction = if ($Data -and $Data.PSObject.Properties["instruction"]) { $Data.instruction } else { $null }
|
||
$exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null }
|
||
$evidencePath = if ($Data -and $Data.PSObject.Properties["evidence"]) { $Data.evidence } else { $null }
|
||
$outcome = if ($Data -and $Data.PSObject.Properties["outcome"]) { $Data.outcome } else { $null }
|
||
$outcomeState = if ($outcome -and $outcome.PSObject.Properties["state"]) { [string]$outcome.state } else { "failed" }
|
||
$resultLabel = switch ($outcomeState) {
|
||
"resolved" { "已驗證解決 / verified resolved" }
|
||
"candidate_ready" { "修復候選已建立 / candidate ready" }
|
||
"verifying" { "等待來源事件驗證 / verifying" }
|
||
"degraded" { "仍有異常 / degraded" }
|
||
"blocked" { "受阻擋 / blocked" }
|
||
default { "執行失敗 / failed" }
|
||
}
|
||
$actionText = switch ($modeName) {
|
||
"ProviderFreshness" { "建立 provider freshness candidate，要求 provider window、last seen、direct reference、verifier readback；不切 provider。" }
|
||
"BackupCheck" { "讀回備份 freshness、status/log、snapshot metadata、cron contract；不讀備份內容或 secrets。" }
|
||
"Perf" { "讀回主機 CPU/記憶體/磁碟；必要時只做 allowlisted 降載。" }
|
||
"LoadShed" { "執行 allowlisted 降載並做 post-apply verifier。" }
|
||
"Recover" { "依序檢查 VM、SSH、Harbor/K3s/public route，必要時執行受控恢復。" }
|
||
"StartVMs" { "檢查並啟動 allowlisted VMware VM。" }
|
||
"HarborRepair" { "檢查 Harbor core/redis/registry，必要時執行受控修復。" }
|
||
"AwoooRepair" { "檢查 AWOOOI/K3s pod、deployment、image pull 狀態，必要時受控修復。" }
|
||
"SecurityTriage" { "建立資安 triage candidate，分類 blast radius、affected host/service、source log reference，不任意 shell、不暴露 secrets。" }
|
||
default { "執行 allowlisted Agent99 mode 並寫入 evidence/KM。" }
|
||
}
|
||
$lines += "事件 Event: Agent99 自動化結果 / automation result"
|
||
$lines += "模式 Mode: $modeName"
|
||
$lines += "來源 Source: $source"
|
||
$lines += "結果 Result: $resultLabel, transportExitCode=$exitCode"
|
||
if ($outcome) {
|
||
$lines += "驗證 Verifier: name=$($outcome.verifierName), passed=$($outcome.verifierPassed), sourceEventResolved=$($outcome.sourceEventResolved)"
|
||
}
|
||
$lines += "Agent99 動作 Action: $actionText"
|
||
if ($Data -and $Data.PSObject.Properties["problem"]) {
|
||
$problem = $Data.problem
|
||
if ($problem -and $problem.PSObject.Properties["occurrences"]) {
|
||
$lines += "問題計數 Problem count: occurrences=$($problem.occurrences), verifiedResolved=$($problem.verifiedResolved), failed=$($problem.failed), degraded=$($problem.degraded), blocked=$($problem.blocked), verifying=$($problem.verifying), candidateReady=$($problem.candidateReady)"
|
||
if ($problem.PSObject.Properties["kmPath"]) {
|
||
$lines += "KM: $($problem.kmPath)"
|
||
}
|
||
}
|
||
}
|
||
if ($instruction -and $exitCode -ne 0) {
|
||
$lines += "失敗指令 Failure instruction: $instruction"
|
||
} elseif ($instruction) {
|
||
$lines += "處理摘要 Summary: $(Limit-AgentTextLine $instruction 130)"
|
||
}
|
||
if ($evidencePath) {
|
||
$lines += "證據 Evidence: $evidencePath"
|
||
}
|
||
if ($exitCode -eq 0) {
|
||
$lines += "下一步 Next: 若相同問題再發生，依 problem count 追蹤重複率並回寫 PlayBook/KM。"
|
||
} else {
|
||
$lines += "下一步 Next: 保留 failed queue/evidence，升級到對應 playbook，不做破壞性動作。"
|
||
}
|
||
} elseif ($EventType -eq "agent_control_tick") {
|
||
$dashboardPath = if ($Data -and $Data.PSObject.Properties["dashboardPath"]) { $Data.dashboardPath } else { $null }
|
||
$processed = if ($Data -and $Data.PSObject.Properties["processedCount"]) { $Data.processedCount } else { 0 }
|
||
$pending = if ($Data -and $Data.PSObject.Properties["pendingCount"]) { $Data.pendingCount } else { 0 }
|
||
$lines += "事件 Event: Agent99 控制迴圈 / control loop"
|
||
$lines += "儀表板 Dashboard: $dashboardPath"
|
||
$lines += "佇列 Queue: processed=$processed pending=$pending"
|
||
} elseif ($EventType -eq "telegram_delivery_test") {
|
||
$testId = if ($Data -and $Data.PSObject.Properties["testId"]) { $Data.testId } else { "unknown" }
|
||
$lines += "事件 Event: Telegram 低噪音送達測試 / delivery test"
|
||
$lines += "測試 Test: TEST_DO_NOT_PAGE"
|
||
$lines += "測試 ID Test ID: $testId"
|
||
$lines += "影響 Impact: 無使用者影響；此訊息只驗證 Agent99 繁中事件卡與 TG delivery。"
|
||
$lines += "Agent99 動作 Action: 驗證送達後寫入 evidence；不觸發修復、不恢復 sensor、不建立事故。"
|
||
} elseif ($EventType -match "^ai_service_") {
|
||
$serviceName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" }
|
||
$lines += "事件 Event: AI 服務健康 / AI service health"
|
||
$lines += "服務 Service: $serviceName"
|
||
$lines += "Agent99 動作 Action: 先檢查 health endpoint 與 service/container state，再做 controlled repair。"
|
||
} elseif ($EventType -eq "security_triage") {
|
||
$status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" }
|
||
$hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 }
|
||
$evidencePath = if ($Data -and $Data.PSObject.Properties["evidencePath"]) { $Data.evidencePath } else { $null }
|
||
$lines += "事件 Event: 資安判讀 / security triage"
|
||
$lines += "狀態 Status: $status"
|
||
$lines += "證據命中 Evidence hits: $hitCount"
|
||
$lines += "Agent99 動作 Action: 分類 blast radius、affected host/service、source log reference、repeated count、allowlisted next action。"
|
||
$lines += "限制 Guard: 不執行 arbitrary shell、不刪資料、不停用 security controls、不暴露 secrets。"
|
||
if ($evidencePath) {
|
||
$lines += "證據 Evidence: $evidencePath"
|
||
}
|
||
} elseif ($EventType -eq "agent_selfcheck") {
|
||
$lines += "事件 Event: Agent99 自我健康 / self health"
|
||
$lines += "摘要 Summary: $Message"
|
||
$lines += "Agent99 動作 Action: 檢查 task state、evidence freshness、Telegram delivery；reboot 不是第一反應。"
|
||
} else {
|
||
$lines += "事件 Event: $EventType"
|
||
$lines += "摘要 Summary: $Message"
|
||
}
|
||
|
||
$lines += "時間 Time: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')"
|
||
$text = ($lines -join "`n")
|
||
if ($text.Length -gt 3500) {
|
||
$text = $text.Substring(0, 3500) + "`n...(truncated)"
|
||
}
|
||
$text
|
||
}
|
||
|
||
function Get-AgentTelegramVisualPath {
|
||
param([object]$Data)
|
||
if (-not $Data) { return $null }
|
||
foreach ($propName in @("visualPath", "cardImagePath", "imagePath", "screenshotPath")) {
|
||
if ($Data.PSObject.Properties[$propName]) {
|
||
$path = [string]$Data.PSObject.Properties[$propName].Value
|
||
if ($path -and (Test-Path -LiteralPath $path)) {
|
||
return $path
|
||
}
|
||
}
|
||
}
|
||
$null
|
||
}
|
||
|
||
function Test-AgentSreOperatorResultAlertEnabled {
|
||
param([object]$Data)
|
||
if (-not $Data) { return $false }
|
||
$source = [string](Get-AgentObjectValue $Data "source" "")
|
||
$alertId = [string](Get-AgentObjectValue $Data "alertId" "")
|
||
$enabled = $true
|
||
if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSreCommandResults"]) {
|
||
$enabled = Convert-AgentBool $Config.telegram.alertSreCommandResults
|
||
}
|
||
[bool]($enabled -and $source -eq "agent99-sre-alert-inbox" -and $alertId)
|
||
}
|
||
|
||
function New-AgentTelegramCardImage {
|
||
param(
|
||
[string]$Severity,
|
||
[string]$EventType,
|
||
[string]$Text,
|
||
[object]$Data = $null
|
||
)
|
||
|
||
$visualCardsEnabled = $true
|
||
if ($Config.telegram -and $Config.telegram.PSObject.Properties["visualCardsEnabled"]) {
|
||
$visualCardsEnabled = [bool]$Config.telegram.visualCardsEnabled
|
||
}
|
||
if (-not $visualCardsEnabled) { return $null }
|
||
$forceVisual = [bool]($EventType -match "^operator_command_" -and (Test-AgentSreOperatorResultAlertEnabled $Data))
|
||
if (($Severity -notin @("warning", "critical")) -and -not $forceVisual) { return $null }
|
||
|
||
$cardDir = Join-Path $EvidenceDir "telegram-cards"
|
||
New-Item -ItemType Directory -Force -Path $cardDir | Out-Null
|
||
$safeEvent = ($EventType -replace "[^A-Za-z0-9_.-]", "-")
|
||
$imagePath = Join-Path $cardDir "agent99-card-$safeEvent-$stamp.png"
|
||
|
||
try {
|
||
Add-Type -AssemblyName System.Drawing -ErrorAction Stop
|
||
$width = 1040
|
||
$height = 620
|
||
$bitmap = New-Object System.Drawing.Bitmap -ArgumentList $width, $height
|
||
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
|
||
$graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias
|
||
$graphics.TextRenderingHint = [System.Drawing.Text.TextRenderingHint]::ClearTypeGridFit
|
||
|
||
$bgBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(246, 248, 250))
|
||
$panelBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(255, 255, 255))
|
||
$textBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(28, 35, 43))
|
||
$mutedBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(91, 103, 112))
|
||
$accentColor = if ($Severity -eq "critical") {
|
||
[System.Drawing.Color]::FromArgb(190, 18, 60)
|
||
} elseif ($Severity -eq "warning") {
|
||
[System.Drawing.Color]::FromArgb(217, 119, 6)
|
||
} else {
|
||
[System.Drawing.Color]::FromArgb(37, 99, 235)
|
||
}
|
||
$accentBrush = New-Object System.Drawing.SolidBrush -ArgumentList $accentColor
|
||
$fontFamily = "Microsoft JhengHei UI"
|
||
$titleFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 25, ([System.Drawing.FontStyle]::Bold)
|
||
$labelFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 13, ([System.Drawing.FontStyle]::Bold)
|
||
$bodyFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 16, ([System.Drawing.FontStyle]::Regular)
|
||
$smallFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 11, ([System.Drawing.FontStyle]::Regular)
|
||
|
||
$graphics.FillRectangle($bgBrush, 0, 0, $width, $height)
|
||
$graphics.FillRectangle($panelBrush, 36, 34, $width - 72, $height - 68)
|
||
$graphics.FillRectangle($accentBrush, 36, 34, 14, $height - 68)
|
||
$graphics.DrawString("Agent99 事件卡 / Incident Card", $titleFont, $textBrush, 72, 58)
|
||
$graphics.DrawString(("嚴重度 Severity: " + $Severity.ToUpperInvariant()), $labelFont, $accentBrush, 72, 106)
|
||
$graphics.DrawString(("事件 Event: " + (Limit-AgentTextLine $EventType 76)), $labelFont, $mutedBrush, 72, 134)
|
||
|
||
$bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 事件卡 / Incident Card$" } | Select-Object -First 12)
|
||
$y = 184
|
||
foreach ($line in $bodyLines) {
|
||
$graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y)
|
||
$y += 32
|
||
if ($y -gt 545) { break }
|
||
}
|
||
$graphics.DrawString(("產生時間 Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, 570)
|
||
$bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png)
|
||
|
||
$graphics.Dispose()
|
||
$bitmap.Dispose()
|
||
$bgBrush.Dispose()
|
||
$panelBrush.Dispose()
|
||
$textBrush.Dispose()
|
||
$mutedBrush.Dispose()
|
||
$accentBrush.Dispose()
|
||
$titleFont.Dispose()
|
||
$labelFont.Dispose()
|
||
$bodyFont.Dispose()
|
||
$smallFont.Dispose()
|
||
return $imagePath
|
||
} catch {
|
||
Write-AgentLog "telegram_card_image_failed event=$EventType error=$($_.Exception.Message)"
|
||
return $null
|
||
}
|
||
}
|
||
|
||
function Send-AgentTelegramPhotoDirect {
|
||
param(
|
||
[string]$Token,
|
||
[string]$ChatId,
|
||
[string]$Caption,
|
||
[string]$PhotoPath
|
||
)
|
||
|
||
Add-Type -AssemblyName System.Net.Http -ErrorAction Stop
|
||
$client = New-Object System.Net.Http.HttpClient
|
||
$content = New-Object System.Net.Http.MultipartFormDataContent
|
||
$fileContent = $null
|
||
try {
|
||
$content.Add([System.Net.Http.StringContent]::new($ChatId), "chat_id")
|
||
$captionToSend = $Caption
|
||
if ($captionToSend.Length -gt 900) {
|
||
$captionToSend = $captionToSend.Substring(0, 900) + "`n...(truncated; see evidence)"
|
||
}
|
||
$content.Add([System.Net.Http.StringContent]::new($captionToSend), "caption")
|
||
$fileBytes = [IO.File]::ReadAllBytes($PhotoPath)
|
||
$fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes)
|
||
$fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png")
|
||
$content.Add($fileContent, "photo", [IO.Path]::GetFileName($PhotoPath))
|
||
$response = $client.PostAsync("https://api.telegram.org/bot$Token/sendPhoto", $content).GetAwaiter().GetResult()
|
||
$body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
||
if (-not $response.IsSuccessStatusCode) {
|
||
throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body"
|
||
}
|
||
} finally {
|
||
if ($fileContent) { $fileContent.Dispose() }
|
||
if ($content) { $content.Dispose() }
|
||
if ($client) { $client.Dispose() }
|
||
}
|
||
}
|
||
|
||
function Invoke-AgentTelegramCardSelfTest {
|
||
$sample = [pscustomobject]@{
|
||
host = "192.168.0.110"
|
||
reasons = @("perf_readback_failed")
|
||
evidencePath = $jsonPath
|
||
}
|
||
$text = Format-AgentTelegramText "warning" "performance_warning" "selftest visual card" $sample
|
||
$imagePath = New-AgentTelegramCardImage "warning" "performance_warning" $text
|
||
$ok = [bool]($imagePath -and (Test-Path -LiteralPath $imagePath))
|
||
$operatorSample = [pscustomobject]@{
|
||
id = "selftest-sre-result"
|
||
mode = "ProviderFreshness"
|
||
source = "agent99-sre-alert-inbox"
|
||
reason = "sre_alert"
|
||
instruction = "Provider freshness candidate required; alertId=selftest-provider; require providerWindow,lastSeen,directReference,verifierReadback; do not switch provider."
|
||
alertId = "selftest-provider"
|
||
alertKind = "provider_freshness_signal"
|
||
alertSeverity = "warning"
|
||
alertService = "provider_freshness_signal"
|
||
ok = $true
|
||
exitCode = 0
|
||
evidence = $jsonPath
|
||
outcome = [pscustomobject]@{
|
||
state = "resolved"
|
||
verifierName = "selftest-verifier"
|
||
verifierPassed = $true
|
||
sourceEventResolved = $true
|
||
}
|
||
problem = [pscustomobject]@{
|
||
key = "selftest-provider-freshness"
|
||
occurrences = 2
|
||
resolved = 2
|
||
verifiedResolved = 2
|
||
failed = 0
|
||
degraded = 0
|
||
blocked = 0
|
||
verifying = 0
|
||
candidateReady = 0
|
||
kmPath = (Join-Path $Config.agentRoot "playbooks\agent99-incident-km.md")
|
||
}
|
||
}
|
||
$operatorText = Format-AgentTelegramText "info" "operator_command_result" "selftest SRE completion card" $operatorSample
|
||
$operatorImagePath = New-AgentTelegramCardImage "info" "operator_command_result" $operatorText $operatorSample
|
||
$operatorDecisionOk = Test-AgentSreOperatorResultAlertEnabled $operatorSample
|
||
$operatorTextOk = [bool](
|
||
$operatorText -match "已驗證解決 / verified resolved" -and
|
||
$operatorText -match "Agent99 動作 Action" -and
|
||
$operatorText -match "問題計數 Problem count" -and
|
||
$operatorText -match "KM:"
|
||
)
|
||
$operatorVisualOk = [bool]($operatorImagePath -and (Test-Path -LiteralPath $operatorImagePath))
|
||
$ok = [bool]($ok -and $operatorTextOk -and $operatorVisualOk -and $operatorDecisionOk)
|
||
$result = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
ok = $ok
|
||
textPreview = @($text -split "`n" | Select-Object -First 12)
|
||
visualPath = $imagePath
|
||
visualExists = $ok
|
||
operatorTextPreview = @($operatorText -split "`n" | Select-Object -First 14)
|
||
operatorVisualPath = $operatorImagePath
|
||
operatorVisualExists = $operatorVisualOk
|
||
operatorTextOk = $operatorTextOk
|
||
operatorDecisionOk = $operatorDecisionOk
|
||
sentTelegram = $false
|
||
messageFormat = "incident_card_zh_tw_v4"
|
||
}
|
||
$selfTestPath = Join-Path $EvidenceDir "agent99-TelegramCardSelfTest-$stamp.json"
|
||
$result | ConvertTo-Json -Depth 6 | Set-Content -Path $selfTestPath -Encoding UTF8
|
||
Write-AgentLog "telegram_card_selftest ok=$ok path=$imagePath evidence=$selfTestPath"
|
||
$result | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath
|
||
$result | ConvertTo-Json -Depth 6
|
||
if (-not $ok) { exit 1 }
|
||
exit 0
|
||
}
|
||
|
||
function Invoke-AgentTelegramDeliverySelfTest {
|
||
$testId = "telegram-delivery-test-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||
$data = [pscustomobject]@{
|
||
testId = $testId
|
||
visualPath = $null
|
||
}
|
||
Send-AgentTelegram "warning" "telegram_delivery_test" "TEST_DO_NOT_PAGE bilingual Telegram incident card delivery self-test" $data
|
||
$sent = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0)
|
||
$result = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
ok = $sent
|
||
testId = $testId
|
||
sentTelegram = $sent
|
||
messageFormat = "incident_card_zh_tw_v4"
|
||
attempts = $script:TelegramAttempts
|
||
}
|
||
$selfTestPath = Join-Path $EvidenceDir "agent99-TelegramDeliverySelfTest-$stamp.json"
|
||
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $selfTestPath -Encoding UTF8
|
||
Write-AgentLog "telegram_delivery_selftest ok=$sent evidence=$selfTestPath"
|
||
$result | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 $jsonPath
|
||
$result | ConvertTo-Json -Depth 10
|
||
if (-not $sent) { exit 1 }
|
||
exit 0
|
||
}
|
||
|
||
function Invoke-AgentSensorGateSelfTest {
|
||
$previousSuppress = $script:SuppressAgentAlerts
|
||
$script:SuppressAgentAlerts = $true
|
||
$gateError = $null
|
||
$performance = @()
|
||
$backupHealth = $null
|
||
$selfHealth = $null
|
||
|
||
try {
|
||
Write-AgentLog "sensor_gate start no_telegram=true controlled_apply=false remediation=false"
|
||
$performance = @(Get-HostPerformance)
|
||
$backupHealth = Test-AgentBackupHealth
|
||
$selfHealth = Test-AgentSelfHealth
|
||
} catch {
|
||
$gateError = $_.Exception.Message
|
||
Write-AgentLog "sensor_gate error=$gateError"
|
||
} finally {
|
||
$script:SuppressAgentAlerts = $previousSuppress
|
||
}
|
||
|
||
$perfReadbackFailed = @($performance | Where-Object { $_ -and ((-not $_.ok) -or (@($_.reasons) -contains "perf_readback_failed")) })
|
||
$perfMissingMetrics = @($performance | Where-Object {
|
||
$_ -and $_.ok -and (($null -eq $_.loadPerCore) -or ($null -eq $_.memAvailablePercent) -or ($null -eq $_.diskUsedPercent))
|
||
})
|
||
$perfCritical = @($performance | Where-Object { $_ -and $_.severity -eq "critical" })
|
||
$perfWarning = @($performance | Where-Object { $_ -and $_.severity -eq "warning" })
|
||
$sentTelegram = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0)
|
||
$suppressedTelegram = @($script:TelegramAttempts | Where-Object { $_.PSObject.Properties["suppressed"] -and $_.suppressed })
|
||
$perfReadbackOk = [bool](@($performance).Count -gt 0 -and @($perfReadbackFailed).Count -eq 0 -and @($perfMissingMetrics).Count -eq 0)
|
||
$backupOk = [bool]($backupHealth -and $backupHealth.severity -eq "ok")
|
||
$selfOk = [bool]($selfHealth -and $selfHealth.severity -eq "ok")
|
||
|
||
$performanceDecision = if (-not $perfReadbackOk) {
|
||
"hold_readback_or_metric_gap"
|
||
} elseif (@($perfCritical).Count -gt 0) {
|
||
"hold_critical_needs_remediation"
|
||
} elseif (@($perfWarning).Count -gt 0) {
|
||
"hold_warning_needs_summary_or_remediation"
|
||
} else {
|
||
"eligible"
|
||
}
|
||
$backupDecision = if ($backupOk) { "eligible" } elseif ($backupHealth) { "hold_$($backupHealth.severity)" } else { "hold_readback_failed" }
|
||
$selfDecision = if ($selfOk) { "eligible" } elseif ($selfHealth) { "hold_$($selfHealth.severity)" } else { "hold_readback_failed" }
|
||
$readyToEnableAll = [bool]($performanceDecision -eq "eligible" -and $backupDecision -eq "eligible" -and $selfDecision -eq "eligible")
|
||
$gateSafe = [bool]((-not $sentTelegram) -and (-not $ControlledApply) -and ($null -eq $gateError))
|
||
|
||
$result = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
ok = $gateSafe
|
||
readyToEnableAll = $readyToEnableAll
|
||
sentTelegram = $sentTelegram
|
||
suppressedTelegramCount = @($suppressedTelegram).Count
|
||
controlledApply = [bool]$ControlledApply
|
||
ranRemediation = $false
|
||
error = $gateError
|
||
decisions = [pscustomobject]@{
|
||
performanceGuard = $performanceDecision
|
||
backupHealth = $backupDecision
|
||
selfHealth = $selfDecision
|
||
}
|
||
performance = [pscustomobject]@{
|
||
ok = $perfReadbackOk
|
||
hostCount = @($performance).Count
|
||
readbackFailedHosts = @($perfReadbackFailed | ForEach-Object { $_.host })
|
||
missingMetricHosts = @($perfMissingMetrics | ForEach-Object { $_.host })
|
||
warningHosts = @($perfWarning | ForEach-Object { $_.host })
|
||
criticalHosts = @($perfCritical | ForEach-Object { $_.host })
|
||
rows = $performance
|
||
}
|
||
backupHealth = $backupHealth
|
||
selfHealth = $selfHealth
|
||
telegramAttempts = $script:TelegramAttempts
|
||
events = $script:Events
|
||
evidenceLog = $logPath
|
||
}
|
||
|
||
$selfTestPath = Join-Path $EvidenceDir "agent99-SensorGateSelfTest-$stamp.json"
|
||
$result | ConvertTo-Json -Depth 12 | Set-Content -Path $selfTestPath -Encoding UTF8
|
||
$result | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 $jsonPath
|
||
Write-AgentLog "sensor_gate complete ok=$($result.ok) readyToEnableAll=$readyToEnableAll evidence=$selfTestPath"
|
||
$result | ConvertTo-Json -Depth 12
|
||
if (-not $gateSafe) { exit 1 }
|
||
exit 0
|
||
}
|
||
|
||
function Send-AgentTelegram {
|
||
param(
|
||
[string]$Severity,
|
||
[string]$EventType,
|
||
[string]$Message,
|
||
[object]$Data = $null
|
||
)
|
||
|
||
$telegram = $Config.telegram
|
||
$enabled = ($telegram -and $telegram.enabled)
|
||
$botTokenEnv = if ($telegram.botTokenEnv) { $telegram.botTokenEnv } else { "AGENT99_TELEGRAM_BOT_TOKEN" }
|
||
$chatIdEnv = if ($telegram.chatIdEnv) { $telegram.chatIdEnv } else { "AGENT99_TELEGRAM_CHAT_ID" }
|
||
$tokenEnvAliases = @($botTokenEnv)
|
||
if ($telegram.botTokenEnvAliases) { $tokenEnvAliases += @($telegram.botTokenEnvAliases) }
|
||
$chatIdEnvAliases = @($chatIdEnv)
|
||
if ($telegram.chatIdEnvAliases) { $chatIdEnvAliases += @($telegram.chatIdEnvAliases) }
|
||
|
||
function Get-AgentEnvSecret {
|
||
param([string[]]$Names)
|
||
foreach ($name in ($Names | Where-Object { $_ } | Select-Object -Unique)) {
|
||
foreach ($scope in @("Machine", "User", "Process")) {
|
||
$value = [Environment]::GetEnvironmentVariable($name, $scope)
|
||
if ($value) {
|
||
return [pscustomobject]@{ value = $value; name = $name; scope = $scope }
|
||
}
|
||
}
|
||
}
|
||
return $null
|
||
}
|
||
|
||
$tokenSecret = Get-AgentEnvSecret $tokenEnvAliases
|
||
$chatSecret = Get-AgentEnvSecret $chatIdEnvAliases
|
||
$token = if ($tokenSecret) { $tokenSecret.value } else { $null }
|
||
$chatId = if ($chatSecret) { $chatSecret.value } else { $null }
|
||
$chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" }
|
||
|
||
$attempt = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
enabled = [bool]$enabled
|
||
eventType = $EventType
|
||
severity = $Severity
|
||
messageFormat = "incident_card_zh_tw_v3"
|
||
configured = [bool]($token -and $chatId)
|
||
tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null }
|
||
chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null }
|
||
chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null }
|
||
visualPath = $null
|
||
visualSent = $false
|
||
visualError = $null
|
||
relay = $null
|
||
sent = $false
|
||
error = $null
|
||
}
|
||
|
||
if (-not $enabled) {
|
||
$attempt.error = "telegram_disabled"
|
||
$script:TelegramAttempts += $attempt
|
||
return
|
||
}
|
||
$text = Format-AgentTelegramText $Severity $EventType $Message $Data
|
||
$visualPath = Get-AgentTelegramVisualPath $Data
|
||
if (-not $visualPath) {
|
||
$visualPath = New-AgentTelegramCardImage $Severity $EventType $text $Data
|
||
}
|
||
$attempt.visualPath = $visualPath
|
||
if (-not ($token -and ($chatId -or $chatIdOverride))) {
|
||
$relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride $visualPath
|
||
$attempt.relay = $relayResult
|
||
if ($relayResult.ok) {
|
||
$attempt.sent = $true
|
||
$attempt.error = $null
|
||
$attempt.visualSent = [bool]$relayResult.photoSent
|
||
if ($visualPath -and -not $relayResult.photoSent) {
|
||
$attempt.visualError = if ($relayResult.photoError) { $relayResult.photoError } else { "relay_sendPhoto_not_available" }
|
||
}
|
||
} else {
|
||
$attempt.error = "telegram_not_configured"
|
||
}
|
||
$script:TelegramAttempts += $attempt
|
||
return
|
||
}
|
||
|
||
try {
|
||
$targetChat = if ($chatIdOverride) { $chatIdOverride } else { $chatId }
|
||
if ($visualPath -and (Test-Path -LiteralPath $visualPath)) {
|
||
try {
|
||
Send-AgentTelegramPhotoDirect $token $targetChat $text $visualPath
|
||
$attempt.sent = $true
|
||
$attempt.visualSent = $true
|
||
} catch {
|
||
$attempt.visualError = $_.Exception.Message
|
||
Write-AgentLog "telegram_sendPhoto_failed event=$EventType fallback=sendMessage error=$($_.Exception.Message)"
|
||
Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{
|
||
chat_id = $targetChat
|
||
text = $text
|
||
disable_web_page_preview = "true"
|
||
} | Out-Null
|
||
$attempt.sent = $true
|
||
}
|
||
} else {
|
||
Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{
|
||
chat_id = $targetChat
|
||
text = $text
|
||
disable_web_page_preview = "true"
|
||
} | Out-Null
|
||
$attempt.sent = $true
|
||
}
|
||
} catch {
|
||
$attempt.error = $_.Exception.Message
|
||
}
|
||
|
||
$script:TelegramAttempts += $attempt
|
||
}
|
||
|
||
function Record-AgentEvent {
|
||
param(
|
||
[string]$EventType,
|
||
[string]$Severity,
|
||
[string]$Message,
|
||
[object]$Data = $null,
|
||
[switch]$Alert,
|
||
[switch]$NoAlert
|
||
)
|
||
|
||
Write-AgentLog "event type=$EventType severity=$Severity message=$Message"
|
||
$eventData = $Data
|
||
if ($Data -and $Data.PSObject.Properties["replyChatId"]) {
|
||
$eventData = $Data | Select-Object * -ExcludeProperty replyChatId
|
||
$eventData | Add-Member -MemberType NoteProperty -Name replyChat -Value "telegram-origin" -Force
|
||
}
|
||
$event = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
type = $EventType
|
||
severity = $Severity
|
||
message = $Message
|
||
data = $eventData
|
||
}
|
||
$script:Events += $event
|
||
|
||
if ($NoAlert) {
|
||
return
|
||
}
|
||
|
||
$eventSuppressTelegram = $false
|
||
if ($Data -and $Data.PSObject.Properties["suppressTelegram"]) {
|
||
$eventSuppressTelegram = Convert-AgentBool $Data.suppressTelegram
|
||
}
|
||
if ($script:SuppressAgentAlerts -or $eventSuppressTelegram) {
|
||
$suppressReason = if ($eventSuppressTelegram) {
|
||
"do_not_page"
|
||
} elseif ($script:SuppressAgentAlertReason) {
|
||
$script:SuppressAgentAlertReason
|
||
} else {
|
||
"sensor_gate_no_telegram"
|
||
}
|
||
Write-AgentLog "telegram_suppressed reason=$suppressReason type=$EventType severity=$Severity"
|
||
$script:TelegramAttempts += [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
eventType = $EventType
|
||
severity = $Severity
|
||
sent = $false
|
||
suppressed = $true
|
||
reason = $suppressReason
|
||
}
|
||
return
|
||
}
|
||
|
||
$alertAll = ($Config.telegram -and $Config.telegram.alertAllOperations)
|
||
$alertInfo = ($Config.telegram -and $Config.telegram.alertInfoEvents)
|
||
$alertOperatorCommands = $true
|
||
if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertOperatorCommands"]) {
|
||
$alertOperatorCommands = [bool]$Config.telegram.alertOperatorCommands
|
||
}
|
||
$operatorCommandAlert = [bool]($alertOperatorCommands -and $EventType -match "^operator_command_")
|
||
if ($operatorCommandAlert -and $Data) {
|
||
$operatorOk = $false
|
||
$operatorMode = ""
|
||
$operatorId = ""
|
||
$operatorSource = ""
|
||
$operatorAlertId = ""
|
||
$operatorOkValue = $null
|
||
$operatorExitCode = $null
|
||
if ($Data -is [System.Collections.IDictionary]) {
|
||
if ($Data.Contains("mode")) { $operatorMode = [string]$Data["mode"] }
|
||
if ($Data.Contains("id")) { $operatorId = [string]$Data["id"] }
|
||
if ($Data.Contains("source")) { $operatorSource = [string]$Data["source"] }
|
||
if ($Data.Contains("alertId")) { $operatorAlertId = [string]$Data["alertId"] }
|
||
if ($Data.Contains("ok")) { $operatorOkValue = $Data["ok"] }
|
||
if ($Data.Contains("exitCode")) { $operatorExitCode = $Data["exitCode"] }
|
||
} else {
|
||
$modeProp = $Data.PSObject.Properties.Match("mode") | Select-Object -First 1
|
||
$idProp = $Data.PSObject.Properties.Match("id") | Select-Object -First 1
|
||
$sourceProp = $Data.PSObject.Properties.Match("source") | Select-Object -First 1
|
||
$alertIdProp = $Data.PSObject.Properties.Match("alertId") | Select-Object -First 1
|
||
$okProp = $Data.PSObject.Properties.Match("ok") | Select-Object -First 1
|
||
$exitCodeProp = $Data.PSObject.Properties.Match("exitCode") | Select-Object -First 1
|
||
if ($modeProp) { $operatorMode = [string]$modeProp.Value }
|
||
if ($idProp) { $operatorId = [string]$idProp.Value }
|
||
if ($sourceProp) { $operatorSource = [string]$sourceProp.Value }
|
||
if ($alertIdProp) { $operatorAlertId = [string]$alertIdProp.Value }
|
||
if ($okProp) { $operatorOkValue = $okProp.Value }
|
||
if ($exitCodeProp) { $operatorExitCode = $exitCodeProp.Value }
|
||
}
|
||
if ($null -ne $operatorOkValue -and [string]$operatorOkValue -match "^(?i:true|1)$") { $operatorOk = $true }
|
||
$parsedExitCode = -999
|
||
if ($null -ne $operatorExitCode -and [int]::TryParse([string]$operatorExitCode, [ref]$parsedExitCode) -and $parsedExitCode -eq 0) { $operatorOk = $true }
|
||
$alertSuccessfulOperatorCommands = $false
|
||
if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSuccessfulOperatorCommands"]) {
|
||
$alertSuccessfulOperatorCommands = [bool]$Config.telegram.alertSuccessfulOperatorCommands
|
||
}
|
||
$operatorIsSreAlertResult = Test-AgentSreOperatorResultAlertEnabled $Data
|
||
if ($operatorOk -and -not $alertSuccessfulOperatorCommands -and -not $operatorIsSreAlertResult) {
|
||
Write-AgentLog "telegram_operator_success_suppressed type=$EventType mode=$operatorMode id=$operatorId"
|
||
$operatorCommandAlert = $false
|
||
} elseif ($operatorOk -and $operatorIsSreAlertResult) {
|
||
Write-AgentLog "telegram_sre_operator_success_enabled type=$EventType mode=$operatorMode id=$operatorId alertId=$operatorAlertId"
|
||
}
|
||
}
|
||
$shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo))
|
||
if ($shouldAlert) {
|
||
$dedupeMinutes = 30
|
||
if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) {
|
||
$dedupeMinutes = [int]$Config.telegram.dedupeMinutes
|
||
}
|
||
$dedupeParts = @($EventType, $Severity)
|
||
if ($Data) {
|
||
foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source")) {
|
||
if ($Data.PSObject.Properties[$propName]) {
|
||
$dedupeParts += "$propName=$($Data.PSObject.Properties[$propName].Value)"
|
||
}
|
||
}
|
||
if ($Data.PSObject.Properties["reasons"]) {
|
||
$dedupeParts += "reasons=$(@($Data.reasons) -join ',')"
|
||
}
|
||
if ($Data.PSObject.Properties["missingFields"]) {
|
||
$dedupeParts += "missing=$(@($Data.missingFields) -join ',')"
|
||
}
|
||
if ($Data.PSObject.Properties["problem"] -and $Data.problem -and $Data.problem.PSObject.Properties["key"]) {
|
||
$dedupeParts += "problem=$($Data.problem.key)"
|
||
}
|
||
}
|
||
if ($dedupeParts.Count -le 2) {
|
||
$normalizedMessage = $Message
|
||
foreach ($pattern in @("candidate=[^ ]+", "evidence=[^ ]+", "loadPerCore=[^ ]+", "memAvailPct=[^ ]+", "diskUsedPct=[^ ]+", "ageMinutes=[^ ]+")) {
|
||
$normalizedMessage = $normalizedMessage -replace $pattern, ($pattern.Split("=")[0] + "=<value>")
|
||
}
|
||
$dedupeParts += $normalizedMessage
|
||
}
|
||
$dedupeSource = ($dedupeParts -join "|")
|
||
$dedupeHashBytes = [Security.Cryptography.SHA256]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($dedupeSource))
|
||
$dedupeHash = [BitConverter]::ToString($dedupeHashBytes).Replace("-", "").ToLowerInvariant()
|
||
$dedupePath = Join-Path $EvidenceDir "telegram-dedupe-$dedupeHash.marker"
|
||
if ($dedupeMinutes -gt 0 -and (Test-Path $dedupePath)) {
|
||
$ageMinutes = ((Get-Date) - (Get-Item $dedupePath).LastWriteTime).TotalMinutes
|
||
if ($ageMinutes -lt $dedupeMinutes) {
|
||
Write-AgentLog "telegram_dedupe_skip type=$EventType severity=$Severity ageMinutes=$([math]::Round($ageMinutes, 2)) cooldown=$dedupeMinutes"
|
||
return
|
||
}
|
||
}
|
||
Set-Content -Path $dedupePath -Value (Get-Date -Format o) -Encoding UTF8
|
||
Send-AgentTelegram $Severity $EventType $Message $Data
|
||
}
|
||
}
|
||
|
||
function Get-AgentBootState {
|
||
$stateDir = Join-Path $Config.agentRoot "state"
|
||
$statePath = Join-Path $stateDir "boot-state.json"
|
||
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
||
try {
|
||
$os = Get-CimInstance -ClassName Win32_OperatingSystem -ErrorAction Stop
|
||
$bootTime = ([datetime]$os.LastBootUpTime).ToString("o")
|
||
} catch {
|
||
Write-AgentLog "boot_state_read_failed error=$($_.Exception.Message)"
|
||
return [pscustomobject]@{ ok = $false; detected = $false; reason = "boot_time_unavailable"; statePath = $statePath }
|
||
}
|
||
|
||
$observedAt = (Get-Date).ToString("o")
|
||
$previous = $null
|
||
if (Test-Path $statePath) {
|
||
try { $previous = Get-Content $statePath -Raw | ConvertFrom-Json } catch { $previous = $null }
|
||
}
|
||
$previousBootTime = if ($previous -and $previous.PSObject.Properties["bootTime"]) { [string]$previous.bootTime } else { $null }
|
||
$detected = [bool]($previousBootTime -and $previousBootTime -ne $bootTime)
|
||
$sameBoot = [bool]($previousBootTime -and $previousBootTime -eq $bootTime)
|
||
$previousPending = [bool](
|
||
$sameBoot -and
|
||
$previous.PSObject.Properties["pendingRecovery"] -and
|
||
(Convert-AgentBool $previous.pendingRecovery)
|
||
)
|
||
$pendingRecovery = [bool]($detected -or $previousPending)
|
||
$eventId = if ($detected) {
|
||
"windows99-boot-" + ($bootTime -replace '[^0-9]', '')
|
||
} elseif ($sameBoot -and $previous.PSObject.Properties["eventId"]) {
|
||
[string]$previous.eventId
|
||
} else { "" }
|
||
$result = [pscustomobject]@{
|
||
ok = $true
|
||
detected = $detected
|
||
pendingRecovery = $pendingRecovery
|
||
eventId = $eventId
|
||
detectedAt = if ($detected) { $observedAt } elseif ($sameBoot -and $previous.PSObject.Properties["detectedAt"]) { [string]$previous.detectedAt } else { "" }
|
||
bootTime = $bootTime
|
||
previousBootTime = $previousBootTime
|
||
terminalState = if ($sameBoot -and $previous.PSObject.Properties["terminalState"]) { [string]$previous.terminalState } elseif ($pendingRecovery) { "pending" } else { "observed" }
|
||
terminalAt = if ($sameBoot -and $previous.PSObject.Properties["terminalAt"]) { [string]$previous.terminalAt } else { "" }
|
||
lastRecoveryRunId = if ($sameBoot -and $previous.PSObject.Properties["lastRecoveryRunId"]) { [string]$previous.lastRecoveryRunId } else { "" }
|
||
lastRecoveryCompletedAt = if ($sameBoot -and $previous.PSObject.Properties["lastRecoveryCompletedAt"]) { [string]$previous.lastRecoveryCompletedAt } else { "" }
|
||
lastRecoveryCompleted = [bool]($sameBoot -and $previous.PSObject.Properties["lastRecoveryCompleted"] -and (Convert-AgentBool $previous.lastRecoveryCompleted))
|
||
lastRecoveryWithinSlo = [bool]($sameBoot -and $previous.PSObject.Properties["lastRecoveryWithinSlo"] -and (Convert-AgentBool $previous.lastRecoveryWithinSlo))
|
||
lastObservedAt = $observedAt
|
||
statePath = $statePath
|
||
}
|
||
$result | ConvertTo-Json -Depth 5 | Set-Content -Path $statePath -Encoding UTF8
|
||
if ($detected) {
|
||
Record-AgentEvent "host_reboot_detected" "warning" "偵測到 99 主機重新啟動；Agent99 已進入受控恢復主線。" $result -Alert
|
||
}
|
||
$result
|
||
}
|
||
|
||
function Update-AgentBootRecoveryState {
|
||
param([object]$RecoverySlo)
|
||
|
||
$statePath = Join-Path (Join-Path $Config.agentRoot "state") "boot-state.json"
|
||
if (-not $RecoverySlo -or -not (Test-Path $statePath)) { return $null }
|
||
try {
|
||
$state = Get-Content $statePath -Raw | ConvertFrom-Json
|
||
if (-not $state.PSObject.Properties["pendingRecovery"] -or -not (Convert-AgentBool $state.pendingRecovery)) {
|
||
return $state
|
||
}
|
||
$state | Add-Member -NotePropertyName lastRecoveryRunId -NotePropertyValue ([string]$RecoverySlo.runId) -Force
|
||
$state | Add-Member -NotePropertyName lastRecoveryCompletedAt -NotePropertyValue ([string]$RecoverySlo.completedAt) -Force
|
||
$state | Add-Member -NotePropertyName lastRecoveryCompleted -NotePropertyValue ([bool]$RecoverySlo.completed) -Force
|
||
$state | Add-Member -NotePropertyName lastRecoveryWithinSlo -NotePropertyValue ([bool]$RecoverySlo.withinSlo) -Force
|
||
if ($RecoverySlo.completed) {
|
||
$state | Add-Member -NotePropertyName pendingRecovery -NotePropertyValue $false -Force
|
||
$terminalState = if ($RecoverySlo.withinSlo) { "recovered_within_slo" } else { "recovered_late" }
|
||
$state | Add-Member -NotePropertyName terminalState -NotePropertyValue $terminalState -Force
|
||
$state | Add-Member -NotePropertyName terminalAt -NotePropertyValue ((Get-Date).ToString("o")) -Force
|
||
} else {
|
||
$bootAgeSeconds = try { [math]::Max(0, [int64]((Get-Date) - ([datetime]$state.bootTime)).TotalSeconds) } catch { -1 }
|
||
if ($bootAgeSeconds -gt ([int64]$RecoverySlo.targetMinutes * 60)) {
|
||
$state | Add-Member -NotePropertyName terminalState -NotePropertyValue "slo_breached_recovery_pending" -Force
|
||
}
|
||
}
|
||
$state | Add-Member -NotePropertyName lastObservedAt -NotePropertyValue ((Get-Date).ToString("o")) -Force
|
||
$state | ConvertTo-Json -Depth 6 | Set-Content -Path $statePath -Encoding UTF8
|
||
return $state
|
||
} catch {
|
||
Write-AgentLog "boot_recovery_state_write_failed error=$($_.Exception.GetType().Name)"
|
||
return $null
|
||
}
|
||
}
|
||
|
||
function Get-AgentAutoRecoveryConfig {
|
||
$configured = if ($Config.PSObject.Properties["autoRecovery"]) { $Config.autoRecovery } else { $null }
|
||
[pscustomobject]@{
|
||
enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled))
|
||
triggerOnHostUnreachable = [bool](-not $configured -or -not $configured.PSObject.Properties["triggerOnHostUnreachable"] -or (Convert-AgentBool $configured.triggerOnHostUnreachable))
|
||
minimumUnreachableHosts = if ($configured -and $configured.PSObject.Properties["minimumUnreachableHosts"]) { [math]::Max(1, [int]$configured.minimumUnreachableHosts) } else { 1 }
|
||
cooldownMinutes = if ($configured -and $configured.PSObject.Properties["cooldownMinutes"]) { [math]::Max(1, [int]$configured.cooldownMinutes) } else { 10 }
|
||
}
|
||
}
|
||
|
||
function Start-AgentRecoveryFromObservation {
|
||
param(
|
||
[string]$Reason,
|
||
[object]$Observation = $null
|
||
)
|
||
|
||
$autoRecovery = Get-AgentAutoRecoveryConfig
|
||
if (-not $autoRecovery.enabled) {
|
||
return [pscustomobject]@{ ok = $false; queued = $false; reason = "auto_recovery_disabled"; trigger = $Reason }
|
||
}
|
||
|
||
$queueDir = Join-Path $Config.agentRoot "queue"
|
||
$stateDir = Join-Path $Config.agentRoot "state"
|
||
$statePath = Join-Path $stateDir "recovery-trigger-state.json"
|
||
New-Item -ItemType Directory -Force -Path $queueDir, $stateDir | Out-Null
|
||
|
||
foreach ($file in @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue)) {
|
||
try {
|
||
$pending = Get-Content $file.FullName -Raw | ConvertFrom-Json
|
||
if ([string]$pending.mode -eq "Recover") {
|
||
return [pscustomobject]@{ ok = $true; queued = $false; suppressed = $true; reason = "recovery_already_queued"; queuePath = $file.FullName; trigger = $Reason }
|
||
}
|
||
} catch {
|
||
Write-AgentLog "recovery_queue_probe_failed path=$($file.FullName) error=$($_.Exception.Message)"
|
||
}
|
||
}
|
||
|
||
if (Test-Path $statePath) {
|
||
try {
|
||
$state = Get-Content $statePath -Raw | ConvertFrom-Json
|
||
$lastRequestedAt = [datetime]$state.lastRequestedAt
|
||
$ageMinutes = ((Get-Date) - $lastRequestedAt).TotalMinutes
|
||
if ($ageMinutes -lt $autoRecovery.cooldownMinutes) {
|
||
return [pscustomobject]@{ ok = $true; queued = $false; suppressed = $true; reason = "recovery_trigger_cooldown"; ageMinutes = [math]::Round($ageMinutes, 2); trigger = $Reason }
|
||
}
|
||
} catch {
|
||
Write-AgentLog "recovery_trigger_state_read_failed error=$($_.Exception.Message)"
|
||
}
|
||
}
|
||
|
||
$id = "auto-recover-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8))
|
||
$queuePath = Join-Path $queueDir "$id.json"
|
||
$temporaryPath = "$queuePath.tmp"
|
||
$createdAt = Get-Date -Format o
|
||
$request = [pscustomobject]@{
|
||
id = $id
|
||
mode = "Recover"
|
||
controlledApply = $true
|
||
requestedBy = "Agent99"
|
||
source = "agent99-recovery-observer"
|
||
externalId = $Reason
|
||
reason = $Reason
|
||
instruction = "Agent99 detected $Reason; run the allowlisted VM, host, Harbor, K3s, public-route, and freshness recovery chain."
|
||
observation = $Observation
|
||
createdAt = $createdAt
|
||
}
|
||
$request | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8
|
||
Move-Item -Force $temporaryPath $queuePath
|
||
[pscustomobject]@{
|
||
schemaVersion = "agent99_recovery_trigger_state_v1"
|
||
lastRequestedAt = $createdAt
|
||
reason = $Reason
|
||
queueId = $id
|
||
queuePath = $queuePath
|
||
} | ConvertTo-Json -Depth 6 | Set-Content -Path $statePath -Encoding UTF8
|
||
|
||
$result = [pscustomobject]@{ ok = $true; queued = $true; suppressed = $false; reason = $Reason; id = $id; queuePath = $queuePath }
|
||
Record-AgentEvent "recovery_auto_triggered" "warning" "偵測到 $Reason；已排入單一受控 Recover 佇列。" $result -Alert
|
||
$result
|
||
}
|
||
|
||
function Get-AgentRecoverySloConfig {
|
||
$configured = if ($Config.PSObject.Properties["recoverySlo"]) { $Config.recoverySlo } else { $null }
|
||
[pscustomobject]@{
|
||
targetMinutes = if ($configured -and $configured.PSObject.Properties["targetMinutes"]) { [int]$configured.targetMinutes } else { 10 }
|
||
hostTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["hostTimeoutSeconds"]) { [int]$configured.hostTimeoutSeconds } else { 90 }
|
||
harborTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["harborTimeoutSeconds"]) { [int]$configured.harborTimeoutSeconds } else { 120 }
|
||
workloadTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["workloadTimeoutSeconds"]) { [int]$configured.workloadTimeoutSeconds } else { 180 }
|
||
}
|
||
}
|
||
|
||
function New-AgentRecoveryPhase {
|
||
param(
|
||
[string]$Name,
|
||
[string]$Status,
|
||
[double]$ElapsedSeconds,
|
||
[string]$Evidence = ""
|
||
)
|
||
[pscustomobject]@{
|
||
name = $Name
|
||
status = $Status
|
||
elapsedSeconds = [math]::Round($ElapsedSeconds, 1)
|
||
evidence = if ($Evidence) { $Evidence } else { $null }
|
||
}
|
||
}
|
||
|
||
function Invoke-SshText {
|
||
param(
|
||
[string]$TargetHost,
|
||
[string]$Command,
|
||
[int]$TimeoutSeconds = 12,
|
||
[int]$Retries = 2,
|
||
[string]$User = $null
|
||
)
|
||
|
||
$transportLock = Enter-AgentSshTransportLock
|
||
if (-not $transportLock.acquired) {
|
||
Write-AgentLog "ssh_transport_lock_failed host=$TargetHost reason=$($transportLock.reason) waitMs=$($transportLock.waitMs)"
|
||
return [pscustomobject]@{
|
||
ok = $false
|
||
exitCode = -4
|
||
output = "ssh_transport_lock_$($transportLock.reason)"
|
||
elapsedMs = $transportLock.waitMs
|
||
transportSerialized = $true
|
||
transportLockWaitMs = $transportLock.waitMs
|
||
}
|
||
}
|
||
|
||
try {
|
||
$last = $null
|
||
$userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost }
|
||
for ($attempt = 1; $attempt -le $Retries; $attempt++) {
|
||
$args = @(Get-AgentSshIdentityArguments) + @(
|
||
"-o", "BatchMode=yes",
|
||
"-o", "StrictHostKeyChecking=accept-new",
|
||
"-o", "NumberOfPasswordPrompts=0",
|
||
"-o", "ConnectTimeout=$TimeoutSeconds",
|
||
"-o", "ServerAliveInterval=20",
|
||
"-o", "ServerAliveCountMax=3",
|
||
"-l", $userToUse,
|
||
$TargetHost,
|
||
$Command
|
||
)
|
||
|
||
$stopwatch = $null
|
||
try {
|
||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||
$id = [guid]::NewGuid().ToString("N")
|
||
$stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out"
|
||
$stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err"
|
||
$process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
|
||
|
||
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
||
$stopwatch.Stop()
|
||
try {
|
||
& taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null
|
||
} catch {}
|
||
if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
|
||
$process.Kill()
|
||
$process.WaitForExit(2000) | Out-Null
|
||
}
|
||
Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
|
||
$last = [pscustomobject]@{
|
||
ok = $false
|
||
exitCode = -2
|
||
output = "timeout_after_${TimeoutSeconds}s"
|
||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||
}
|
||
} else {
|
||
$process.WaitForExit() | Out-Null
|
||
$stopwatch.Stop()
|
||
$process.Refresh()
|
||
$stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" }
|
||
$stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" }
|
||
Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
|
||
$text = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n"
|
||
$exitCode = $process.ExitCode
|
||
if ($null -eq $exitCode) {
|
||
$exitCode = if ($stderr) { -3 } else { 0 }
|
||
}
|
||
$last = [pscustomobject]@{
|
||
ok = ($exitCode -eq 0)
|
||
exitCode = $exitCode
|
||
output = $text
|
||
elapsedMs = $stopwatch.ElapsedMilliseconds
|
||
}
|
||
}
|
||
} catch {
|
||
$last = [pscustomobject]@{
|
||
ok = $false
|
||
exitCode = -1
|
||
output = $_.Exception.Message
|
||
elapsedMs = if ($stopwatch) { $stopwatch.ElapsedMilliseconds } else { $null }
|
||
}
|
||
}
|
||
|
||
if ($last.ok) {
|
||
if ($attempt -gt 1) {
|
||
Write-AgentLog "ssh_retry_recovered host=$TargetHost attempt=$attempt"
|
||
}
|
||
return (Add-AgentSshTransportMetadata $last $transportLock)
|
||
}
|
||
|
||
if ($attempt -lt $Retries) {
|
||
Write-AgentLog "ssh_retry host=$TargetHost attempt=$attempt exit=$($last.exitCode)"
|
||
Start-Sleep -Seconds 2
|
||
}
|
||
}
|
||
|
||
Add-AgentSshTransportMetadata $last $transportLock
|
||
} finally {
|
||
Exit-AgentSshTransportLock $transportLock
|
||
}
|
||
}
|
||
|
||
function Quote-ShSingle {
|
||
param([string]$Text)
|
||
"'" + $Text.Replace("'", "'`"`"'") + "'"
|
||
}
|
||
|
||
function Invoke-HostSshText {
|
||
param(
|
||
[string]$TargetHost,
|
||
[string]$Command,
|
||
[int]$TimeoutSeconds = 30,
|
||
[int]$Retries = 1
|
||
)
|
||
|
||
$jumpHost = if ($Config.k3s -and $Config.k3s.PSObject.Properties["jumpHost"]) { [string]$Config.k3s.jumpHost } else { "" }
|
||
$preferJumpHost = [bool]($Config.k3s -and $Config.k3s.PSObject.Properties["preferJumpHost"] -and (Convert-AgentBool $Config.k3s.preferJumpHost))
|
||
$directHosts = if ($Config.k3s -and $Config.k3s.PSObject.Properties["directHosts"]) { @($Config.k3s.directHosts) } else { @() }
|
||
$canUseJump = [bool]($jumpHost -and $TargetHost -ne $jumpHost -and $TargetHost -notin $directHosts)
|
||
|
||
$quotedCommand = Quote-ShSingle $Command
|
||
$targetUser = Get-HostSshUser $TargetHost
|
||
$jumpCommand = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=10 -l $targetUser $TargetHost $quotedCommand"
|
||
|
||
if ($preferJumpHost -and $canUseJump) {
|
||
$viaJump = Invoke-SshText $jumpHost $jumpCommand $TimeoutSeconds $Retries
|
||
$viaJump | Add-Member -NotePropertyName route -NotePropertyValue "via_$jumpHost" -Force
|
||
if ($viaJump.ok) { return $viaJump }
|
||
Write-AgentLog "ssh_jump_fallback host=$TargetHost jumpHost=$jumpHost exit=$($viaJump.exitCode)"
|
||
}
|
||
|
||
$direct = Invoke-SshText $TargetHost $Command $TimeoutSeconds $Retries
|
||
if ($direct.ok -or -not $canUseJump -or $preferJumpHost) {
|
||
$direct | Add-Member -NotePropertyName route -NotePropertyValue "direct" -Force
|
||
return $direct
|
||
}
|
||
|
||
$viaJump = Invoke-SshText $jumpHost $jumpCommand $TimeoutSeconds $Retries
|
||
$viaJump | Add-Member -NotePropertyName route -NotePropertyValue "via_$jumpHost" -Force
|
||
return $viaJump
|
||
}
|
||
|
||
function Get-AgentRecoveryCoordinatorConfig {
|
||
$configured = if ($Config.PSObject.Properties["recoveryCoordinator"]) { $Config.recoveryCoordinator } else { $null }
|
||
[pscustomobject]@{
|
||
enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled))
|
||
controllerHost = if ($configured -and $configured.PSObject.Properties["controllerHost"]) { [string]$configured.controllerHost } else { "192.168.0.110" }
|
||
readbackTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["readbackTimeoutSeconds"]) { [math]::Min(30, [math]::Max(5, [int]$configured.readbackTimeoutSeconds)) } else { 15 }
|
||
pollIntervalSeconds = if ($configured -and $configured.PSObject.Properties["pollIntervalSeconds"]) { [math]::Min(30, [math]::Max(5, [int]$configured.pollIntervalSeconds)) } else { 10 }
|
||
freshWaitTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["freshWaitTimeoutSeconds"]) { [math]::Min(480, [math]::Max(30, [int]$configured.freshWaitTimeoutSeconds)) } else { 420 }
|
||
maxArtifactAgeSeconds = if ($configured -and $configured.PSObject.Properties["maxArtifactAgeSeconds"]) { [math]::Min(1800, [math]::Max(60, [int]$configured.maxArtifactAgeSeconds)) } else { 600 }
|
||
artifactClockSkewSeconds = if ($configured -and $configured.PSObject.Properties["artifactClockSkewSeconds"]) { [math]::Min(300, [math]::Max(0, [int]$configured.artifactClockSkewSeconds)) } else { 90 }
|
||
requiredHostAliases = if ($configured -and $configured.PSObject.Properties["requiredHostAliases"]) { @($configured.requiredHostAliases | ForEach-Object { [string]$_ }) } else { @("99", "110", "111", "112", "120", "121", "188") }
|
||
}
|
||
}
|
||
|
||
function Convert-AgentRecoveryReadback {
|
||
param(
|
||
[string]$Text,
|
||
[datetime]$RecoveryStartedAt,
|
||
[bool]$RequireFreshRebootWindow,
|
||
[object]$CoordinatorConfig
|
||
)
|
||
|
||
$values = @{}
|
||
$hosts = @()
|
||
foreach ($line in @($Text -split "`r?`n")) {
|
||
if ($line -match '^([A-Z0-9_]+)=(.*)$') {
|
||
$values[$Matches[1]] = $Matches[2].Trim()
|
||
continue
|
||
}
|
||
if ($line -match '^HOST_BOOT\s+alias=([^\s]+).*?\sreachable=([01])(?:\s|$).*?\suptime_seconds=(-?[0-9]+)(?:\s|$)') {
|
||
$hosts += [pscustomobject]@{
|
||
alias = [string]$Matches[1]
|
||
reachable = [bool]([int]$Matches[2])
|
||
uptimeSeconds = [int64]$Matches[3]
|
||
}
|
||
}
|
||
}
|
||
|
||
$get = {
|
||
param([string]$Name, [string]$Default = "")
|
||
if ($values.ContainsKey($Name)) { return [string]$values[$Name] }
|
||
return $Default
|
||
}
|
||
$asBool = {
|
||
param([string]$Name)
|
||
[bool]((& $get $Name "0") -eq "1")
|
||
}
|
||
$asInt = {
|
||
param([string]$Name, [int64]$Default = -1)
|
||
$value = & $get $Name ""
|
||
if ($value -match '^-?[0-9]+$') { return [int64]$value }
|
||
return $Default
|
||
}
|
||
|
||
$artifactEpoch = & $asInt "RECOVERY_ARTIFACT_EPOCH" -1
|
||
$nowEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds()
|
||
$evidenceNotBeforeEpoch = ([DateTimeOffset]$RecoveryStartedAt).ToUnixTimeSeconds()
|
||
$artifactAgeSeconds = if ($artifactEpoch -gt 0) { [math]::Max(0, $nowEpoch - $artifactEpoch) } else { -1 }
|
||
$artifactRecent = [bool]($artifactAgeSeconds -ge 0 -and $artifactAgeSeconds -le $CoordinatorConfig.maxArtifactAgeSeconds)
|
||
$artifactFreshForRecovery = [bool]($artifactEpoch -ge ($evidenceNotBeforeEpoch - $CoordinatorConfig.artifactClockSkewSeconds))
|
||
|
||
$requiredHostFailures = @()
|
||
foreach ($alias in @($CoordinatorConfig.requiredHostAliases)) {
|
||
$row = $hosts | Where-Object { $_.alias -eq $alias } | Select-Object -First 1
|
||
if (-not $row -or -not $row.reachable) { $requiredHostFailures += $alias }
|
||
}
|
||
|
||
$checks = @(
|
||
[pscustomobject]@{ name = "readback_present"; passed = (& $asBool "RECOVERY_READBACK") },
|
||
[pscustomobject]@{ name = "artifact_recent"; passed = $artifactRecent },
|
||
[pscustomobject]@{ name = "required_hosts_reachable"; passed = [bool]($requiredHostFailures.Count -eq 0) },
|
||
[pscustomobject]@{ name = "post_start_unblocked"; passed = [bool]((& $get "POST_START_BLOCKED" "-1") -eq "0") },
|
||
[pscustomobject]@{ name = "service_green"; passed = (& $asBool "SERVICE_GREEN") },
|
||
[pscustomobject]@{ name = "product_data_green"; passed = (& $asBool "PRODUCT_DATA_GREEN") },
|
||
[pscustomobject]@{ name = "stock_freshness_ok"; passed = [bool]((& $get "STOCK_FRESHNESS_STATUS") -eq "ok") },
|
||
[pscustomobject]@{ name = "backup_core_green"; passed = (& $asBool "BACKUP_CORE_GREEN") },
|
||
[pscustomobject]@{ name = "host_188_service_green"; passed = (& $asBool "HOST_188_SERVICE_GREEN") },
|
||
[pscustomobject]@{ name = "host_188_hygiene_green"; passed = [bool]((& $get "HOST_188_HYGIENE_BLOCKED" "1") -eq "0") },
|
||
[pscustomobject]@{ name = "edge_fallback_ready"; passed = (& $asBool "EDGE_FALLBACK_READY") },
|
||
[pscustomobject]@{ name = "windows99_vmware_ready"; passed = (& $asBool "VMWARE_AUTOSTART_VERIFY_READY") },
|
||
[pscustomobject]@{ name = "windows99_update_policy_ready"; passed = (& $asBool "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY") },
|
||
[pscustomobject]@{ name = "scorecard_source_controls_ready"; passed = (& $asBool "SCORECARD_SOURCE_CONTROLS_READY") },
|
||
[pscustomobject]@{ name = "scorecard_public_fallback_ready"; passed = (& $asBool "SCORECARD_PUBLIC_FALLBACK_READY") }
|
||
)
|
||
$sloChecks = @()
|
||
if ($RequireFreshRebootWindow) {
|
||
$checks += @(
|
||
[pscustomobject]@{ name = "artifact_fresh_for_recovery"; passed = $artifactFreshForRecovery },
|
||
[pscustomobject]@{
|
||
name = "scorecard_non_slo_blockers_clear"
|
||
passed = [bool](
|
||
((& $asInt "SCORECARD_BLOCKER_COUNT" -1) -eq 0) -or
|
||
(& $asBool "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY")
|
||
)
|
||
}
|
||
)
|
||
$sloChecks = @(
|
||
[pscustomobject]@{ name = "fresh_reboot_window_observed"; passed = (& $asBool "SCORECARD_FRESH_REBOOT_WINDOW") },
|
||
[pscustomobject]@{ name = "scorecard_can_claim_slo"; passed = (& $asBool "SCORECARD_CAN_CLAIM") },
|
||
[pscustomobject]@{ name = "scorecard_has_no_blockers"; passed = [bool]((& $asInt "SCORECARD_BLOCKER_COUNT" -1) -eq 0) }
|
||
)
|
||
}
|
||
|
||
$failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||
$sloFailedChecks = @($sloChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||
[pscustomobject]@{
|
||
schemaVersion = "agent99_recovery_coordinator_readback_v1"
|
||
ok = [bool]($failedChecks.Count -eq 0)
|
||
verified = [bool]($failedChecks.Count -eq 0)
|
||
requireFreshRebootWindow = $RequireFreshRebootWindow
|
||
rebootSloClaimed = [bool]($RequireFreshRebootWindow -and $sloFailedChecks.Count -eq 0)
|
||
artifactDir = & $get "RECOVERY_ARTIFACT_DIR"
|
||
artifactEpoch = $artifactEpoch
|
||
artifactAgeSeconds = $artifactAgeSeconds
|
||
artifactFreshForRecovery = $artifactFreshForRecovery
|
||
scorecardStatus = & $get "SCORECARD_STATUS"
|
||
scorecardBlockerCount = & $asInt "SCORECARD_BLOCKER_COUNT" -1
|
||
scorecardPrimaryBlocker = & $get "SCORECARD_PRIMARY_BLOCKER"
|
||
readinessPercent = & $asInt "SCORECARD_READINESS_PERCENT" -1
|
||
postStartResult = & $get "POST_START_RESULT"
|
||
overallDeclaration = & $get "OVERALL_DECLARATION"
|
||
requiredHostAliases = @($CoordinatorConfig.requiredHostAliases)
|
||
requiredHostFailures = $requiredHostFailures
|
||
hosts = $hosts
|
||
checks = $checks
|
||
failedChecks = $failedChecks
|
||
sloChecks = $sloChecks
|
||
sloFailedChecks = $sloFailedChecks
|
||
}
|
||
}
|
||
|
||
function Invoke-AgentRecoveryCoordinatorReadback {
|
||
param(
|
||
[datetime]$RecoveryStartedAt,
|
||
[bool]$RequireFreshRebootWindow
|
||
)
|
||
|
||
$coordinator = Get-AgentRecoveryCoordinatorConfig
|
||
if (-not $coordinator.enabled) {
|
||
return [pscustomobject]@{
|
||
schemaVersion = "agent99_recovery_coordinator_readback_v1"
|
||
enabled = $false
|
||
ok = $false
|
||
verified = $false
|
||
requireFreshRebootWindow = $RequireFreshRebootWindow
|
||
rebootSloClaimed = $false
|
||
failedChecks = @("recovery_coordinator_disabled")
|
||
attempts = @()
|
||
}
|
||
}
|
||
|
||
$readbackCommand = @'
|
||
latest="$(find /home/wooo/reboot-recovery -mindepth 2 -maxdepth 2 -type f -name 'scorecard.json' -printf '%T@ %h\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-)"
|
||
if [ -z "$latest" ]; then
|
||
printf 'RECOVERY_READBACK=0\n'
|
||
exit 0
|
||
fi
|
||
printf 'RECOVERY_READBACK=1\n'
|
||
printf 'RECOVERY_ARTIFACT_DIR=%s\n' "$latest"
|
||
stat -c 'RECOVERY_ARTIFACT_EPOCH=%Y' "$latest" 2>/dev/null || true
|
||
grep -E '^(POST_START_RESULT|POST_START_BLOCKED|SERVICE_GREEN|PRODUCT_DATA_GREEN|STOCK_FRESHNESS_STATUS|BACKUP_CORE_GREEN|HOST_188_SERVICE_GREEN|HOST_188_HYGIENE_BLOCKED|OVERALL_DECLARATION)=' "$latest/summary.txt" 2>/dev/null || true
|
||
grep -E '^HOST_BOOT ' "$latest/host-probe.txt" 2>/dev/null || true
|
||
grep -E '^(EDGE_FALLBACK_READY|PRIVILEGED_EXECUTOR_READY|EDGE_EXECUTOR_PAYLOAD_PARITY)=' "$latest/public-maintenance-edge-fallback.txt" 2>/dev/null || true
|
||
grep -E '^(VMWARE_AUTOSTART_CONFIG_READY|VMWARE_AUTOSTART_POWER_READY|WINDOWS_UPDATE_NO_AUTO_REBOOT_READY|VMWARE_AUTOSTART_VERIFY_READY)=' "$latest/windows99-vmware-verify.txt" 2>/dev/null || true
|
||
printf %s aW1wb3J0IGpzb24sc3lzCmQ9anNvbi5sb2FkKG9wZW4oc3lzLmFyZ3ZbMV0pKQpyPWQuZ2V0KCJyZXF1aXJlZF9jaGVja3MiLHt9KQpvPWQuZ2V0KCJyb2xsdXBzIix7fSkKYj1sYW1iZGEgdjoiMSIgaWYgdiBlbHNlICIwIgpwcmludCgiU0NPUkVDQVJEX1NUQVRVUz0iK3N0cihkLmdldCgic3RhdHVzIiwiIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRVJfQ09VTlQ9IitzdHIoby5nZXQoImFjdGl2ZV9ibG9ja2VyX2NvdW50IiwtMSkpKQpwcmludCgiU0NPUkVDQVJEX1JFQURJTkVTU19QRVJDRU5UPSIrc3RyKG8uZ2V0KCJyZWFkaW5lc3NfcGVyY2VudCIsLTEpKSkKcHJpbnQoIlNDT1JFQ0FSRF9QUklNQVJZX0JMT0NLRVI9IitzdHIoZC5nZXQoInByaW1hcnlfYmxvY2tlciIsIiIpKSkKcHJpbnQoIlNDT1JFQ0FSRF9DQU5fQ0xBSU09IitiKGQuZ2V0KCJjYW5fY2xhaW1fYWxsX3NlcnZpY2VzX3JlY292ZXJlZF93aXRoaW5fdGFyZ2V0IikpKQpwcmludCgiU0NPUkVDQVJEX0ZSRVNIX1JFQk9PVF9XSU5ET1c9IitiKHIuZ2V0KCJmcmVzaF9yZWJvb3Rfd2luZG93X29ic2VydmVkIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRURfQllfUkVCT09UX1dJTkRPV19PTkxZPSIrYihvLmdldCgiYmxvY2tlZF9ieV9mcmVzaF9yZWJvb3Rfd2luZG93X29ubHkiKSkpCnByaW50KCJTQ09SRUNBUkRfU09VUkNFX0NPTlRST0xTX1JFQURZPSIrYihyLmdldCgic291cmNlX2NvbnRyb2xzX3ByZXNlbnQiKSkpCnByaW50KCJTQ09SRUNBUkRfUFVCTElDX0ZBTExCQUNLX1JFQURZPSIrYihyLmdldCgicHVibGljX21haW50ZW5hbmNlX2ZhbGxiYWNrX3J1bnRpbWVfcmVhZHkiKSkpCg== | base64 -d | python3 - "$latest/scorecard.json" 2>/dev/null || true
|
||
'@
|
||
|
||
$deadline = (Get-Date).AddSeconds($(if ($RequireFreshRebootWindow) { $coordinator.freshWaitTimeoutSeconds } else { 1 }))
|
||
$attempts = @()
|
||
$latestResult = $null
|
||
do {
|
||
$transport = Invoke-HostSshText $coordinator.controllerHost $readbackCommand $coordinator.readbackTimeoutSeconds 1
|
||
if ($transport.ok) {
|
||
$latestResult = Convert-AgentRecoveryReadback $transport.output $RecoveryStartedAt $RequireFreshRebootWindow $coordinator
|
||
$latestResult | Add-Member -NotePropertyName enabled -NotePropertyValue $true -Force
|
||
$latestResult | Add-Member -NotePropertyName controllerHost -NotePropertyValue $coordinator.controllerHost -Force
|
||
$latestResult | Add-Member -NotePropertyName transportRoute -NotePropertyValue $transport.route -Force
|
||
$latestResult | Add-Member -NotePropertyName transportSerialized -NotePropertyValue $transport.transportSerialized -Force
|
||
$latestResult | Add-Member -NotePropertyName transportLockWaitMs -NotePropertyValue $transport.transportLockWaitMs -Force
|
||
} else {
|
||
$latestResult = [pscustomobject]@{
|
||
schemaVersion = "agent99_recovery_coordinator_readback_v1"
|
||
enabled = $true
|
||
ok = $false
|
||
verified = $false
|
||
requireFreshRebootWindow = $RequireFreshRebootWindow
|
||
rebootSloClaimed = $false
|
||
controllerHost = $coordinator.controllerHost
|
||
transportRoute = $transport.route
|
||
transportSerialized = $transport.transportSerialized
|
||
transportLockWaitMs = $transport.transportLockWaitMs
|
||
failedChecks = @("controller_transport_failed")
|
||
}
|
||
}
|
||
$attempts += [pscustomobject]@{
|
||
observedAt = (Get-Date -Format o)
|
||
verified = [bool]$latestResult.verified
|
||
artifactDir = if ($latestResult.PSObject.Properties["artifactDir"]) { [string]$latestResult.artifactDir } else { "" }
|
||
failedChecks = @($latestResult.failedChecks)
|
||
}
|
||
if ($latestResult.verified -or -not $RequireFreshRebootWindow) { break }
|
||
if ((Get-Date) -lt $deadline) { Start-Sleep -Seconds $coordinator.pollIntervalSeconds }
|
||
} while ((Get-Date) -lt $deadline)
|
||
|
||
$latestResult | Add-Member -NotePropertyName attempts -NotePropertyValue $attempts -Force
|
||
$latestResult | Add-Member -NotePropertyName attemptCount -NotePropertyValue $attempts.Count -Force
|
||
$latestResult
|
||
}
|
||
|
||
function Convert-AgentDouble {
|
||
param([string]$Value)
|
||
if (-not $Value) { return $null }
|
||
try {
|
||
return [double]::Parse($Value.Replace(",", "."), [Globalization.CultureInfo]::InvariantCulture)
|
||
} catch {
|
||
return $null
|
||
}
|
||
}
|
||
|
||
function Get-PerfThresholds {
|
||
$perf = $Config.performance
|
||
[pscustomobject]@{
|
||
loadPerCoreWarning = if ($perf.loadPerCoreWarning) { [double]$perf.loadPerCoreWarning } else { 0.9 }
|
||
loadPerCoreCritical = if ($perf.loadPerCoreCritical) { [double]$perf.loadPerCoreCritical } else { 1.5 }
|
||
memoryAvailablePercentCritical = if ($perf.memoryAvailablePercentCritical) { [double]$perf.memoryAvailablePercentCritical } else { 10 }
|
||
diskUsedPercentWarning = if ($perf.diskUsedPercentWarning) { [double]$perf.diskUsedPercentWarning } else { 90 }
|
||
diskUsedPercentCritical = if ($perf.diskUsedPercentCritical) { [double]$perf.diskUsedPercentCritical } else { 95 }
|
||
}
|
||
}
|
||
|
||
function Get-HostPerformance {
|
||
param([switch]$SuppressAlerts)
|
||
|
||
$thresholds = Get-PerfThresholds
|
||
$readTimeoutSeconds = 45
|
||
if ($Config.performance -and $Config.performance.PSObject.Properties["readTimeoutSeconds"]) {
|
||
$readTimeoutSeconds = [int]$Config.performance.readTimeoutSeconds
|
||
}
|
||
$readRetries = 1
|
||
if ($Config.performance -and $Config.performance.PSObject.Properties["readRetries"]) {
|
||
$readRetries = [int]$Config.performance.readRetries
|
||
}
|
||
$results = @()
|
||
$command = @'
|
||
echo __AGENT99_PERF__
|
||
echo HOST=$(hostname)
|
||
echo CORES=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)
|
||
uptime
|
||
echo __MEM__
|
||
free -m 2>/dev/null
|
||
echo __DISK__
|
||
df -P / 2>/dev/null
|
||
'@
|
||
|
||
foreach ($hostIp in $Config.hosts) {
|
||
$hostTimeoutSeconds = $readTimeoutSeconds
|
||
if ($Config.performance -and $Config.performance.PSObject.Properties["hostReadTimeoutSeconds"] -and $Config.performance.hostReadTimeoutSeconds.PSObject.Properties[$hostIp]) {
|
||
$hostTimeoutSeconds = [int]$Config.performance.hostReadTimeoutSeconds.PSObject.Properties[$hostIp].Value
|
||
}
|
||
$slowRetryUsed = $false
|
||
$slowTimeoutSeconds = $null
|
||
$readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries
|
||
if (-not $readback.ok -and $readback.exitCode -eq -2) {
|
||
$slowTimeoutSeconds = 90
|
||
if ($Config.performance -and $Config.performance.PSObject.Properties["slowReadTimeoutSeconds"]) {
|
||
$slowTimeoutSeconds = [int]$Config.performance.slowReadTimeoutSeconds
|
||
}
|
||
if ($Config.performance -and $Config.performance.PSObject.Properties["hostSlowReadTimeoutSeconds"] -and $Config.performance.hostSlowReadTimeoutSeconds.PSObject.Properties[$hostIp]) {
|
||
$slowTimeoutSeconds = [int]$Config.performance.hostSlowReadTimeoutSeconds.PSObject.Properties[$hostIp].Value
|
||
}
|
||
if ($slowTimeoutSeconds -gt $hostTimeoutSeconds) {
|
||
Write-AgentLog "perf slow_retry host=$hostIp firstTimeoutSeconds=$hostTimeoutSeconds slowTimeoutSeconds=$slowTimeoutSeconds"
|
||
$slowReadback = Invoke-HostSshText $hostIp $command $slowTimeoutSeconds 1
|
||
$slowReadback | Add-Member -NotePropertyName firstAttemptExitCode -NotePropertyValue $readback.exitCode -Force
|
||
$slowReadback | Add-Member -NotePropertyName firstAttemptElapsedMs -NotePropertyValue $readback.elapsedMs -Force
|
||
$readback = $slowReadback
|
||
$slowRetryUsed = $true
|
||
}
|
||
}
|
||
$text = $readback.output
|
||
$cores = 1
|
||
$coresMatch = [regex]::Match($text, "CORES=(\d+)")
|
||
if ($coresMatch.Success) { $cores = [int]$coresMatch.Groups[1].Value }
|
||
|
||
$load1 = $null
|
||
$loadMatch = [regex]::Match($text, "load averages?:\s*([0-9]+(?:[\.,][0-9]+)?)")
|
||
if (-not $loadMatch.Success) {
|
||
$loadMatch = [regex]::Match($text, "load average:\s*([0-9]+(?:[\.,][0-9]+)?)")
|
||
}
|
||
if ($loadMatch.Success) {
|
||
$load1 = Convert-AgentDouble $loadMatch.Groups[1].Value
|
||
}
|
||
|
||
$memAvailablePercent = $null
|
||
$memLine = (($text -split "`n") | Where-Object { $_ -match "^\s*Mem:" } | Select-Object -First 1)
|
||
if ($memLine) {
|
||
$parts = @($memLine -split "\s+" | Where-Object { $_ })
|
||
if ($parts.Count -ge 7) {
|
||
$memTotal = Convert-AgentDouble $parts[1]
|
||
$memAvailable = Convert-AgentDouble $parts[6]
|
||
if ($memTotal -and $memTotal -gt 0 -and $null -ne $memAvailable) {
|
||
$memAvailablePercent = [math]::Round(($memAvailable / $memTotal) * 100, 2)
|
||
}
|
||
}
|
||
}
|
||
|
||
$diskUsedPercent = $null
|
||
$diskLine = (($text -split "`n") | Where-Object { $_ -match "\s+\d+%\s+/$" } | Select-Object -First 1)
|
||
if ($diskLine) {
|
||
$diskMatch = [regex]::Match($diskLine, "\s(\d+)%\s+/$")
|
||
if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value }
|
||
}
|
||
|
||
$topCpu = @()
|
||
|
||
$loadPerCore = if ($null -ne $load1 -and $cores -gt 0) { [math]::Round($load1 / $cores, 2) } else { $null }
|
||
$severity = "ok"
|
||
$reasons = @()
|
||
if (-not $readback.ok) {
|
||
$severity = "warning"
|
||
$reasons += "perf_readback_failed"
|
||
}
|
||
if ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreCritical) {
|
||
$severity = "critical"
|
||
$reasons += "load_per_core_critical"
|
||
} elseif ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreWarning -and $severity -ne "critical") {
|
||
$severity = "warning"
|
||
$reasons += "load_per_core_warning"
|
||
}
|
||
if ($null -ne $memAvailablePercent -and $memAvailablePercent -le $thresholds.memoryAvailablePercentCritical) {
|
||
$severity = "critical"
|
||
$reasons += "memory_available_low"
|
||
}
|
||
if ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentCritical) {
|
||
$severity = "critical"
|
||
$reasons += "disk_used_high"
|
||
} elseif ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentWarning -and $severity -ne "critical") {
|
||
$severity = "warning"
|
||
$reasons += "disk_used_warning"
|
||
}
|
||
|
||
$captureTopCpuOnWarning = $true
|
||
if ($Config.performance -and $Config.performance.PSObject.Properties["captureTopCpuOnWarning"]) {
|
||
$captureTopCpuOnWarning = [bool]$Config.performance.captureTopCpuOnWarning
|
||
}
|
||
$topCpuTimeoutSeconds = 10
|
||
if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuTimeoutSeconds"]) {
|
||
$topCpuTimeoutSeconds = [int]$Config.performance.topCpuTimeoutSeconds
|
||
}
|
||
$shouldCaptureTopCpu = ($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning)
|
||
if ($shouldCaptureTopCpu -and $readback.ok) {
|
||
$topReadback = Invoke-HostSshText $hostIp "ps -eo pid,ppid,comm,pcpu,pmem --sort=-pcpu" $topCpuTimeoutSeconds 1
|
||
if ($topReadback.ok) {
|
||
$topCpu = @($topReadback.output -split "`n" | Select-Object -First 8)
|
||
} else {
|
||
$topCpu = @("top_cpu_readback_failed: $($topReadback.output)")
|
||
}
|
||
}
|
||
|
||
$result = [pscustomobject]@{
|
||
host = $hostIp
|
||
ok = [bool]$readback.ok
|
||
route = $readback.route
|
||
severity = $severity
|
||
reasons = $reasons
|
||
cores = $cores
|
||
load1 = $load1
|
||
loadPerCore = $loadPerCore
|
||
memAvailablePercent = $memAvailablePercent
|
||
diskUsedPercent = $diskUsedPercent
|
||
topCpu = $topCpu
|
||
output = $text
|
||
readTimeoutSeconds = $hostTimeoutSeconds
|
||
slowReadTimeoutSeconds = $slowTimeoutSeconds
|
||
readbackSlowRetry = $slowRetryUsed
|
||
readbackElapsedMs = $readback.elapsedMs
|
||
transportSerialized = if ($readback.PSObject.Properties["transportSerialized"]) { [bool]$readback.transportSerialized } else { $false }
|
||
transportLockWaitMs = if ($readback.PSObject.Properties["transportLockWaitMs"]) { [long]$readback.transportLockWaitMs } else { $null }
|
||
}
|
||
Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed"
|
||
if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) {
|
||
Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert
|
||
}
|
||
$results += $result
|
||
}
|
||
|
||
$results
|
||
}
|
||
|
||
function Record-PerformanceIssuesWithoutRemediation {
|
||
param(
|
||
[object[]]$Performance,
|
||
[object[]]$LoadShedding
|
||
)
|
||
|
||
foreach ($perf in @($Performance | Where-Object { $_ -and $_.severity -in @("warning", "critical") })) {
|
||
$attempts = @($LoadShedding | Where-Object { $_ -and $_.host -eq $perf.host })
|
||
$executed = @($attempts | Where-Object { $_.PSObject.Properties["skipped"] -and $_.skipped -eq $false })
|
||
if ($executed.Count -gt 0) {
|
||
continue
|
||
}
|
||
|
||
$message = "host=$($perf.host) loadPerCore=$($perf.loadPerCore) memAvailPct=$($perf.memAvailablePercent) diskUsedPct=$($perf.diskUsedPercent) reasons=$(@($perf.reasons) -join ',')"
|
||
Record-AgentEvent "performance_$($perf.severity)" $perf.severity $message $perf -Alert
|
||
}
|
||
}
|
||
|
||
function Invoke-LoadShedding {
|
||
param([object[]]$Performance)
|
||
|
||
$results = @()
|
||
$loadShedding = $Config.performance.loadShedding
|
||
if (-not ($loadShedding -and $loadShedding.enabled)) {
|
||
Write-AgentLog "load_shedding skipped enabled=false"
|
||
return $results
|
||
}
|
||
if (-not $ControlledApply) {
|
||
Write-AgentLog "load_shedding skipped controlled_apply=false"
|
||
return $results
|
||
}
|
||
|
||
foreach ($action in @($loadShedding.actions)) {
|
||
if ($action.enabled -eq $false) { continue }
|
||
$hostPerf = $Performance | Where-Object { $_.host -eq $action.host } | Select-Object -First 1
|
||
if (-not $hostPerf) { continue }
|
||
|
||
$severityRank = @{ ok = 0; warning = 1; critical = 2 }
|
||
$minSeverity = if ($action.minSeverity) { [string]$action.minSeverity } else { "critical" }
|
||
if (-not $severityRank.ContainsKey($minSeverity)) { $minSeverity = "critical" }
|
||
$hostSeverity = if ($hostPerf.severity -and $severityRank.ContainsKey([string]$hostPerf.severity)) { [string]$hostPerf.severity } else { "ok" }
|
||
if ($severityRank[$hostSeverity] -lt $severityRank[$minSeverity]) {
|
||
$results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "host_below_min_severity"; hostSeverity = $hostSeverity; minSeverity = $minSeverity }
|
||
continue
|
||
}
|
||
|
||
$reasonMatches = @($action.reasonMatches)
|
||
if ($reasonMatches.Count -eq 0) { $reasonMatches = @("load_per_core_critical") }
|
||
$matchedReasons = @($hostPerf.reasons | Where-Object { $_ -in $reasonMatches })
|
||
if ($matchedReasons.Count -eq 0) {
|
||
$results += [pscustomobject]@{
|
||
name = $action.name
|
||
host = $action.host
|
||
skipped = $true
|
||
reason = "critical_reason_not_allowlisted"
|
||
hostReasons = $hostPerf.reasons
|
||
allowedReasons = $reasonMatches
|
||
}
|
||
continue
|
||
}
|
||
|
||
$cooldownMinutes = if ($action.cooldownMinutes) { [int]$action.cooldownMinutes } else { 30 }
|
||
$failureBackoffMinutes = if ($action.PSObject.Properties["failureBackoffMinutes"]) { [int]$action.failureBackoffMinutes } else { 10 }
|
||
$safeName = ($action.name -replace "[^A-Za-z0-9_.-]", "_")
|
||
$markerPath = Join-Path $EvidenceDir "loadshedding-$safeName.marker"
|
||
$failureMarkerPath = Join-Path $EvidenceDir "loadshedding-$safeName.failure.marker"
|
||
if (Test-Path $markerPath) {
|
||
$ageMinutes = ((Get-Date) - (Get-Item $markerPath).LastWriteTime).TotalMinutes
|
||
if ($ageMinutes -lt $cooldownMinutes) {
|
||
$results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "cooldown"; ageMinutes = [math]::Round($ageMinutes, 1) }
|
||
continue
|
||
}
|
||
}
|
||
if (Test-Path $failureMarkerPath) {
|
||
$failureAgeMinutes = ((Get-Date) - (Get-Item $failureMarkerPath).LastWriteTime).TotalMinutes
|
||
if ($failureAgeMinutes -lt $failureBackoffMinutes) {
|
||
$results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "failure_backoff"; ageMinutes = [math]::Round($failureAgeMinutes, 1) }
|
||
continue
|
||
}
|
||
}
|
||
|
||
Record-AgentEvent "load_shed_attempt" $hostSeverity "host=$($action.host) action=$($action.name) command=$($action.command)" $action -NoAlert
|
||
$timeoutSeconds = if ($action.PSObject.Properties["timeoutSeconds"]) { [int]$action.timeoutSeconds } else { 60 }
|
||
$run = Invoke-HostSshText $action.host $action.command $timeoutSeconds 1
|
||
|
||
$postVerify = $null
|
||
$postVerifyOk = $false
|
||
if ($action.PSObject.Properties["postVerifyCommand"] -and $action.postVerifyCommand) {
|
||
$verifyRun = Invoke-HostSshText $action.host ([string]$action.postVerifyCommand) 30 1
|
||
$postVerify = [pscustomobject]@{
|
||
ok = [bool]$verifyRun.ok
|
||
exitCode = $verifyRun.exitCode
|
||
output = $verifyRun.output
|
||
route = $verifyRun.route
|
||
}
|
||
if ($verifyRun.ok) {
|
||
$postVerifyOk = $true
|
||
if ($action.PSObject.Properties["postVerifyMaxDiskUsedPercent"]) {
|
||
$match = [regex]::Match([string]$verifyRun.output, "__DISK_USED__=(\d+)")
|
||
if ($match.Success) {
|
||
$postVerifyOk = ([int]$match.Groups[1].Value -le [int]$action.postVerifyMaxDiskUsedPercent)
|
||
} else {
|
||
$postVerifyOk = $false
|
||
}
|
||
}
|
||
if ($action.PSObject.Properties["postVerifyOkRegex"] -and $action.postVerifyOkRegex) {
|
||
$postVerifyOk = [bool]([string]$verifyRun.output -match [string]$action.postVerifyOkRegex)
|
||
}
|
||
}
|
||
}
|
||
|
||
$postVerifyRequired = [bool]($action.PSObject.Properties["postVerifyCommand"] -and $action.postVerifyCommand)
|
||
$effectiveOk = if ($postVerifyRequired) { [bool]$postVerifyOk } else { [bool]$run.ok }
|
||
if ($effectiveOk) {
|
||
Set-Content -Path $markerPath -Value (Get-Date -Format o) -Encoding UTF8
|
||
Remove-Item -Force $failureMarkerPath -ErrorAction SilentlyContinue | Out-Null
|
||
} else {
|
||
Set-Content -Path $failureMarkerPath -Value (Get-Date -Format o) -Encoding UTF8
|
||
}
|
||
$result = [pscustomobject]@{
|
||
name = $action.name
|
||
host = $action.host
|
||
skipped = $false
|
||
ok = $effectiveOk
|
||
executionOk = [bool]$run.ok
|
||
postVerifyRequired = $postVerifyRequired
|
||
postVerifyOk = [bool]$postVerifyOk
|
||
postVerify = $postVerify
|
||
route = $run.route
|
||
exitCode = $run.exitCode
|
||
output = $run.output
|
||
rollback = $action.rollback
|
||
}
|
||
Record-AgentEvent "load_shed_result" $(if ($effectiveOk) { "warning" } else { "critical" }) "host=$($action.host) action=$($action.name) ok=$effectiveOk executionOk=$($run.ok) postVerifyOk=$postVerifyOk" $result -Alert
|
||
$results += $result
|
||
}
|
||
|
||
$results
|
||
}
|
||
|
||
function Get-AgentBooleanSetting {
|
||
param(
|
||
[object]$Object,
|
||
[string]$Name,
|
||
[bool]$Default
|
||
)
|
||
|
||
if ($Object -and $Object.PSObject.Properties[$Name]) {
|
||
return [bool]$Object.PSObject.Properties[$Name].Value
|
||
}
|
||
return $Default
|
||
}
|
||
|
||
function Get-AgentSelfHealthConfig {
|
||
$selfHealth = $Config.selfHealth
|
||
$defaultTasks = @(
|
||
"Wooo-Agent99-Startup-Recovery",
|
||
"Wooo-Agent99-Heartbeat",
|
||
"Wooo-Agent99-Performance-Guard",
|
||
"Wooo-Agent99-Control-Loop",
|
||
"Wooo-Agent99-Telegram-Inbox",
|
||
"Wooo-Agent99-SRE-Alert-Inbox",
|
||
"Wooo-Agent99-Self-Health",
|
||
"Wooo-Agent99-Backup-Health"
|
||
)
|
||
$defaultEvidence = @(
|
||
[pscustomobject]@{ name = "performance_guard"; pattern = "agent99-Perf-*.json"; maxAgeMinutes = 5; required = $true },
|
||
[pscustomobject]@{ name = "heartbeat_status"; pattern = "agent99-Status-*.json"; maxAgeMinutes = 15; required = $true },
|
||
[pscustomobject]@{ name = "self_health"; pattern = "agent99-SelfCheck-*.json"; maxAgeMinutes = 15; required = $false },
|
||
[pscustomobject]@{ name = "backup_health"; pattern = "agent99-BackupCheck-*.json"; maxAgeMinutes = 30; required = $false },
|
||
[pscustomobject]@{ name = "startup_recovery"; pattern = "agent99-Recover-*.json"; maxAgeMinutes = 1440; required = $false },
|
||
[pscustomobject]@{ name = "telegram_inbox"; pattern = "agent99-TelegramInbox-*.json"; maxAgeMinutes = 15; required = $false },
|
||
[pscustomobject]@{ name = "sre_alert_inbox"; pattern = "agent99-SreAlertInbox-*.json"; maxAgeMinutes = 15; required = $false },
|
||
[pscustomobject]@{
|
||
name = "sre_alert_relay"
|
||
pattern = "agent99-SreAlertRelay-start-*.json"
|
||
maxAgeMinutes = 1440
|
||
required = $false
|
||
runningTaskName = "Wooo-Agent99-SRE-Alert-Relay"
|
||
}
|
||
)
|
||
|
||
[pscustomobject]@{
|
||
scheduledTasks = if ($selfHealth -and $selfHealth.scheduledTasks) { @($selfHealth.scheduledTasks) } else { $defaultTasks }
|
||
evidenceFreshness = if ($selfHealth -and $selfHealth.evidenceFreshness) { @($selfHealth.evidenceFreshness) } else { $defaultEvidence }
|
||
requiredHosts = if ($selfHealth -and $selfHealth.requiredHosts) { @($selfHealth.requiredHosts) } else { @($Config.hosts) }
|
||
requireRecentTelegramDelivery = Get-AgentBooleanSetting $selfHealth "requireRecentTelegramDelivery" $true
|
||
}
|
||
}
|
||
|
||
function Get-AgentLatestEvidence {
|
||
param([string]$Pattern)
|
||
|
||
$file = Get-ChildItem -Path $EvidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue |
|
||
Sort-Object LastWriteTime -Descending |
|
||
Select-Object -First 1
|
||
if (-not $file) {
|
||
return [pscustomobject]@{
|
||
exists = $false
|
||
path = $null
|
||
fileName = $null
|
||
lastWriteTime = $null
|
||
ageMinutes = $null
|
||
data = $null
|
||
parseError = $null
|
||
}
|
||
}
|
||
|
||
$data = $null
|
||
$parseError = $null
|
||
try {
|
||
$data = Get-Content $file.FullName -Raw | ConvertFrom-Json
|
||
} catch {
|
||
$parseError = $_.Exception.Message
|
||
}
|
||
|
||
[pscustomobject]@{
|
||
exists = $true
|
||
path = $file.FullName
|
||
fileName = $file.Name
|
||
lastWriteTime = $file.LastWriteTime.ToString("o")
|
||
ageMinutes = [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 2)
|
||
data = $data
|
||
parseError = $parseError
|
||
}
|
||
}
|
||
|
||
function Test-AgentScheduledTaskHealth {
|
||
param([string[]]$TaskNames)
|
||
|
||
$results = @()
|
||
foreach ($taskName in $TaskNames) {
|
||
try {
|
||
$task = Get-ScheduledTask -TaskName $taskName -ErrorAction Stop
|
||
$info = Get-ScheduledTaskInfo -TaskName $taskName -ErrorAction Stop
|
||
$state = [string]$task.State
|
||
$lastTaskResult = $info.LastTaskResult
|
||
$pendingFirstRun = ($lastTaskResult -eq 267011)
|
||
$ok = ($state -eq "Running") -or (($state -eq "Ready") -and ($lastTaskResult -in @(0, 267011)))
|
||
$results += [pscustomobject]@{
|
||
name = $taskName
|
||
exists = $true
|
||
ok = [bool]$ok
|
||
state = $state
|
||
lastTaskResult = $lastTaskResult
|
||
pendingFirstRun = [bool]$pendingFirstRun
|
||
lastRunTime = if ($info.LastRunTime) { $info.LastRunTime.ToString("o") } else { $null }
|
||
nextRunTime = if ($info.NextRunTime) { $info.NextRunTime.ToString("o") } else { $null }
|
||
}
|
||
Write-AgentLog "selfcheck task name=$taskName ok=$ok state=$state lastTaskResult=$lastTaskResult pendingFirstRun=$pendingFirstRun"
|
||
} catch {
|
||
$results += [pscustomobject]@{
|
||
name = $taskName
|
||
exists = $false
|
||
ok = $false
|
||
state = $null
|
||
lastTaskResult = $null
|
||
lastRunTime = $null
|
||
nextRunTime = $null
|
||
error = $_.Exception.Message
|
||
}
|
||
Write-AgentLog "selfcheck task_missing name=$taskName error=$($_.Exception.Message)"
|
||
}
|
||
}
|
||
|
||
$results
|
||
}
|
||
|
||
function Test-AgentEvidenceFreshness {
|
||
param(
|
||
[object[]]$Contracts,
|
||
[object[]]$TaskHealth = @()
|
||
)
|
||
|
||
$results = @()
|
||
foreach ($contract in $Contracts) {
|
||
$name = if ($contract.name) { [string]$contract.name } else { [string]$contract.pattern }
|
||
$pattern = [string]$contract.pattern
|
||
$maxAgeMinutes = if ($contract.maxAgeMinutes) { [double]$contract.maxAgeMinutes } else { 15 }
|
||
$required = Get-AgentBooleanSetting $contract "required" $true
|
||
$latest = Get-AgentLatestEvidence $pattern
|
||
$fresh = [bool]($latest.exists -and $null -eq $latest.parseError -and $latest.ageMinutes -le $maxAgeMinutes)
|
||
$runningTaskName = if ($contract.PSObject.Properties["runningTaskName"]) {
|
||
[string]$contract.runningTaskName
|
||
} elseif ($name -eq "sre_alert_relay") {
|
||
# Backward-compatible migration for live configs written before runningTaskName existed.
|
||
"Wooo-Agent99-SRE-Alert-Relay"
|
||
} else {
|
||
$null
|
||
}
|
||
$runningTask = if ($runningTaskName) {
|
||
$TaskHealth | Where-Object { $_.name -eq $runningTaskName } | Select-Object -First 1
|
||
} else {
|
||
$null
|
||
}
|
||
$runningTaskHealthy = [bool]($runningTask -and $runningTask.ok -and $runningTask.state -eq "Running")
|
||
$staleStartEvidence = [bool](
|
||
$latest.exists -and
|
||
$null -eq $latest.parseError -and
|
||
$latest.ageMinutes -gt $maxAgeMinutes
|
||
)
|
||
$freshnessSupersededByRunningTask = [bool]($runningTaskName -and $runningTaskHealthy -and $staleStartEvidence)
|
||
$ok = [bool](((-not $required) -and (-not $latest.exists)) -or $fresh -or $freshnessSupersededByRunningTask)
|
||
$severity = if ($ok) { "ok" } elseif ($required) { "critical" } else { "warning" }
|
||
$reason = if ($fresh) {
|
||
"evidence_fresh"
|
||
} elseif ($freshnessSupersededByRunningTask) {
|
||
"running_task_supersedes_stale_start_evidence"
|
||
} elseif ((-not $required) -and (-not $latest.exists)) {
|
||
"optional_evidence_missing"
|
||
} elseif ($latest.parseError) {
|
||
"evidence_parse_failed"
|
||
} else {
|
||
"evidence_stale_or_missing"
|
||
}
|
||
|
||
$results += [pscustomobject]@{
|
||
name = $name
|
||
pattern = $pattern
|
||
required = [bool]$required
|
||
ok = $ok
|
||
severity = $severity
|
||
latestPath = $latest.path
|
||
latestFile = $latest.fileName
|
||
lastWriteTime = $latest.lastWriteTime
|
||
ageMinutes = $latest.ageMinutes
|
||
maxAgeMinutes = $maxAgeMinutes
|
||
parseError = $latest.parseError
|
||
reason = $reason
|
||
runningTaskName = $runningTaskName
|
||
runningTaskHealthy = $runningTaskHealthy
|
||
freshnessSupersededByRunningTask = $freshnessSupersededByRunningTask
|
||
}
|
||
Write-AgentLog "selfcheck evidence name=$name ok=$ok severity=$severity reason=$reason ageMinutes=$($latest.ageMinutes) maxAgeMinutes=$maxAgeMinutes file=$($latest.fileName) runningTask=$runningTaskName runningTaskHealthy=$runningTaskHealthy"
|
||
}
|
||
|
||
$results
|
||
}
|
||
|
||
function Test-AgentLatestPerformanceEvidence {
|
||
param([string[]]$RequiredHosts)
|
||
|
||
$latest = Get-AgentLatestEvidence "agent99-Perf-*.json"
|
||
$hostChecks = @()
|
||
if (-not $latest.exists) {
|
||
return [pscustomobject]@{
|
||
ok = $false
|
||
severity = "critical"
|
||
reason = "missing_perf_evidence"
|
||
latestPath = $null
|
||
latestFile = $null
|
||
ageMinutes = $null
|
||
hosts = $hostChecks
|
||
}
|
||
}
|
||
if ($latest.parseError) {
|
||
return [pscustomobject]@{
|
||
ok = $false
|
||
severity = "critical"
|
||
reason = "perf_evidence_parse_error"
|
||
latestPath = $latest.path
|
||
latestFile = $latest.fileName
|
||
ageMinutes = $latest.ageMinutes
|
||
parseError = $latest.parseError
|
||
hosts = $hostChecks
|
||
}
|
||
}
|
||
|
||
$perf = @($latest.data.performance)
|
||
foreach ($hostIp in $RequiredHosts) {
|
||
$entry = $perf | Where-Object { $_.host -eq $hostIp } | Select-Object -First 1
|
||
$hostSeverity = if (-not $entry) {
|
||
"missing"
|
||
} elseif ($entry.severity) {
|
||
[string]$entry.severity
|
||
} elseif ($entry.ok -eq $false) {
|
||
"warning"
|
||
} else {
|
||
"ok"
|
||
}
|
||
$ok = [bool]($hostSeverity -eq "ok")
|
||
$hostChecks += [pscustomobject]@{
|
||
host = $hostIp
|
||
present = [bool]$entry
|
||
ok = $ok
|
||
severity = $hostSeverity
|
||
loadPerCore = if ($entry) { $entry.loadPerCore } else { $null }
|
||
memAvailablePercent = if ($entry) { $entry.memAvailablePercent } else { $null }
|
||
diskUsedPercent = if ($entry) { $entry.diskUsedPercent } else { $null }
|
||
reasons = if ($entry) { @($entry.reasons) } else { @("missing_perf_host") }
|
||
}
|
||
}
|
||
|
||
$missingOrCritical = @($hostChecks | Where-Object { $_.severity -in @("missing", "critical") }).Count
|
||
$warning = @($hostChecks | Where-Object { $_.severity -eq "warning" }).Count
|
||
$severity = if ($missingOrCritical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" }
|
||
|
||
[pscustomobject]@{
|
||
ok = [bool]($severity -eq "ok")
|
||
severity = $severity
|
||
reason = if ($severity -eq "ok") { "all_required_hosts_ok" } else { "required_host_perf_not_ok" }
|
||
latestPath = $latest.path
|
||
latestFile = $latest.fileName
|
||
ageMinutes = $latest.ageMinutes
|
||
hosts = $hostChecks
|
||
}
|
||
}
|
||
|
||
function Test-AgentRecentTelegramDelivery {
|
||
param([object[]]$Contracts)
|
||
|
||
$checks = @()
|
||
$lookbackMinutes = 60
|
||
if ($Config.selfHealth -and $Config.selfHealth.PSObject.Properties["telegramDeliveryLookbackMinutes"]) {
|
||
$lookbackMinutes = [int]$Config.selfHealth.telegramDeliveryLookbackMinutes
|
||
}
|
||
$cutoff = (Get-Date).AddMinutes(-1 * $lookbackMinutes)
|
||
foreach ($contract in $Contracts) {
|
||
$pattern = [string]$contract.pattern
|
||
$files = Get-ChildItem -Path $EvidenceDir -Filter $pattern -File -ErrorAction SilentlyContinue |
|
||
Where-Object { $_.LastWriteTime -ge $cutoff } |
|
||
Sort-Object LastWriteTime -Descending |
|
||
Select-Object -First 20
|
||
|
||
foreach ($file in @($files)) {
|
||
$data = $null
|
||
try {
|
||
$data = Get-Content $file.FullName -Raw | ConvertFrom-Json
|
||
} catch {
|
||
continue
|
||
}
|
||
if (-not $data) { continue }
|
||
$attempts = @($data.telegram)
|
||
if ($attempts.Count -eq 0) { continue }
|
||
|
||
$sent = @($attempts | Where-Object { $_.sent -eq $true -or ($_.relay -and $_.relay.ok -eq $true) })
|
||
$checks += [pscustomobject]@{
|
||
name = if ($contract.name) { [string]$contract.name } else { [string]$contract.pattern }
|
||
latestPath = $file.FullName
|
||
latestFile = $file.Name
|
||
ageMinutes = [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 2)
|
||
attempts = $attempts.Count
|
||
sent = $sent.Count
|
||
ok = [bool]($sent.Count -gt 0)
|
||
}
|
||
}
|
||
}
|
||
|
||
$attemptCount = 0
|
||
$sentCount = 0
|
||
foreach ($check in @($checks)) {
|
||
$attemptCount += [int]$check.attempts
|
||
$sentCount += [int]$check.sent
|
||
}
|
||
if ($attemptCount -eq 0) {
|
||
return [pscustomobject]@{
|
||
ok = $true
|
||
severity = "ok"
|
||
reason = "no_recent_alert_attempts"
|
||
checks = $checks
|
||
}
|
||
}
|
||
|
||
[pscustomobject]@{
|
||
ok = [bool]($sentCount -gt 0)
|
||
severity = if ($sentCount -gt 0) { "ok" } else { "critical" }
|
||
reason = if ($sentCount -gt 0) { "recent_alert_delivery_ok" } else { "recent_alert_delivery_failed" }
|
||
checks = $checks
|
||
}
|
||
}
|
||
|
||
function Test-AgentRuntimeManifest {
|
||
$manifestPath = Join-Path $Config.agentRoot "state\runtime-manifest.json"
|
||
if (-not (Test-Path $manifestPath)) {
|
||
return [pscustomobject]@{
|
||
ok = $false
|
||
severity = "critical"
|
||
reason = "runtime_manifest_missing"
|
||
manifestPath = $manifestPath
|
||
sourceRevision = $null
|
||
driftCount = 0
|
||
missingCount = 1
|
||
files = @()
|
||
}
|
||
}
|
||
|
||
try {
|
||
$manifest = Get-Content $manifestPath -Raw | ConvertFrom-Json
|
||
} catch {
|
||
return [pscustomobject]@{
|
||
ok = $false
|
||
severity = "critical"
|
||
reason = "runtime_manifest_parse_failed"
|
||
manifestPath = $manifestPath
|
||
sourceRevision = $null
|
||
driftCount = 0
|
||
missingCount = 0
|
||
files = @()
|
||
error = $_.Exception.Message
|
||
}
|
||
}
|
||
|
||
$rows = @()
|
||
foreach ($entry in @($manifest.files)) {
|
||
$name = [string]$entry.name
|
||
$runtimePath = Join-Path $PSScriptRoot $name
|
||
$exists = Test-Path $runtimePath
|
||
$actualSha256 = $null
|
||
if ($exists) {
|
||
try {
|
||
$actualSha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $runtimePath).Hash.ToLowerInvariant()
|
||
} catch {
|
||
$actualSha256 = $null
|
||
}
|
||
}
|
||
$expectedSha256 = if ($entry.runtimeSha256) { ([string]$entry.runtimeSha256).ToLowerInvariant() } elseif ($entry.sourceSha256) { ([string]$entry.sourceSha256).ToLowerInvariant() } else { "" }
|
||
$matched = [bool]($exists -and $actualSha256 -and $expectedSha256 -and $actualSha256 -eq $expectedSha256)
|
||
$rows += [pscustomobject]@{
|
||
name = $name
|
||
exists = $exists
|
||
matched = $matched
|
||
expectedSha256 = $expectedSha256
|
||
actualSha256 = $actualSha256
|
||
}
|
||
}
|
||
$missingCount = @($rows | Where-Object { -not $_.exists }).Count
|
||
$driftCount = @($rows | Where-Object { $_.exists -and -not $_.matched }).Count
|
||
$ready = [bool](@($rows).Count -gt 0 -and $missingCount -eq 0 -and $driftCount -eq 0 -and $manifest.runtimeMatched -eq $true)
|
||
[pscustomobject]@{
|
||
ok = $ready
|
||
severity = if ($ready) { "ok" } else { "critical" }
|
||
reason = if ($ready) { "runtime_manifest_matched" } else { "runtime_source_drift_or_missing" }
|
||
manifestPath = $manifestPath
|
||
sourceRevision = $manifest.sourceRevision
|
||
generatedAt = $manifest.generatedAt
|
||
driftCount = $driftCount
|
||
missingCount = $missingCount
|
||
files = $rows
|
||
}
|
||
}
|
||
|
||
function Test-AgentSelfHealth {
|
||
$selfConfig = Get-AgentSelfHealthConfig
|
||
$tasks = Test-AgentScheduledTaskHealth $selfConfig.scheduledTasks
|
||
$evidence = Test-AgentEvidenceFreshness $selfConfig.evidenceFreshness $tasks
|
||
$latestPerf = Test-AgentLatestPerformanceEvidence $selfConfig.requiredHosts
|
||
$runtimeManifest = Test-AgentRuntimeManifest
|
||
$telegram = if ($selfConfig.requireRecentTelegramDelivery) {
|
||
Test-AgentRecentTelegramDelivery $selfConfig.evidenceFreshness
|
||
} else {
|
||
[pscustomobject]@{ ok = $true; severity = "ok"; checks = @(); skipped = $true }
|
||
}
|
||
|
||
$taskFailures = @($tasks | Where-Object { -not $_.ok }).Count
|
||
$evidenceCritical = @($evidence | Where-Object { $_.severity -eq "critical" }).Count
|
||
$evidenceWarning = @($evidence | Where-Object { $_.severity -eq "warning" }).Count
|
||
$critical = $taskFailures + $evidenceCritical
|
||
if ($latestPerf.severity -eq "critical") { $critical += 1 }
|
||
if ($telegram.severity -eq "critical") { $critical += 1 }
|
||
if ($runtimeManifest.severity -eq "critical") { $critical += 1 }
|
||
$warning = $evidenceWarning
|
||
if ($latestPerf.severity -eq "warning") { $warning += 1 }
|
||
|
||
$severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" }
|
||
$message = "selfHealth severity=$severity taskFailures=$taskFailures evidenceCritical=$evidenceCritical evidenceWarning=$evidenceWarning perf=$($latestPerf.severity) telegram=$($telegram.severity) runtimeManifest=$($runtimeManifest.severity)"
|
||
Record-AgentEvent "agent_selfcheck" $(if ($severity -eq "ok") { "info" } else { $severity }) $message $null -Alert
|
||
|
||
[pscustomobject]@{
|
||
ok = [bool]($severity -eq "ok")
|
||
severity = $severity
|
||
tasks = $tasks
|
||
evidenceFreshness = $evidence
|
||
latestPerformance = $latestPerf
|
||
telegramDelivery = $telegram
|
||
runtimeManifest = $runtimeManifest
|
||
summary = [pscustomobject]@{
|
||
taskFailures = $taskFailures
|
||
evidenceCritical = $evidenceCritical
|
||
evidenceWarning = $evidenceWarning
|
||
performanceSeverity = $latestPerf.severity
|
||
telegramSeverity = $telegram.severity
|
||
runtimeManifestSeverity = $runtimeManifest.severity
|
||
runtimeManifestSourceRevision = $runtimeManifest.sourceRevision
|
||
runtimeManifestDriftCount = $runtimeManifest.driftCount
|
||
}
|
||
}
|
||
}
|
||
|
||
function Get-AgentUnixAgeMinutes {
|
||
param([double]$UnixSeconds)
|
||
|
||
$now = [DateTimeOffset]::Now.ToUnixTimeSeconds()
|
||
[math]::Round(($now - $UnixSeconds) / 60, 2)
|
||
}
|
||
|
||
function Convert-AgentUnixToIso {
|
||
param([double]$UnixSeconds)
|
||
|
||
try {
|
||
[DateTimeOffset]::FromUnixTimeSeconds([int64][math]::Floor($UnixSeconds)).LocalDateTime.ToString("o")
|
||
} catch {
|
||
$null
|
||
}
|
||
}
|
||
|
||
function Get-AgentBackupHealthConfig {
|
||
$backupHealth = $Config.backupHealth
|
||
$backupHost = if ($backupHealth -and $backupHealth.host) { [string]$backupHealth.host } else { "192.168.0.110" }
|
||
$targets = @(
|
||
[pscustomobject]@{ name = "awoooi"; path = "/backup/awoooi/snapshots"; maxAgeMinutes = 430; required = $true },
|
||
[pscustomobject]@{ name = "gitea"; path = "/backup/gitea/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "harbor"; path = "/backup/harbor/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "momo"; path = "/backup/momo/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "langfuse"; path = "/backup/langfuse/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "monitoring"; path = "/backup/monitoring/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "signoz"; path = "/backup/signoz/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "clawbot"; path = "/backup/clawbot/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "open-webui"; path = "/backup/open-webui/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "sentry"; path = "/backup/sentry/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "ai-artifacts"; path = "/backup/ai-artifacts/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "configs"; path = "/backup/configs/snapshots"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "public-routes"; path = "/backup/public-routes/snapshots"; maxAgeMinutes = 10080; required = $false }
|
||
)
|
||
$fileChecks = @(
|
||
[pscustomobject]@{ name = "backup_log"; path = "/backup/logs/backup.log"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "backup_status_log"; path = "/backup/logs/backup-status.log"; maxAgeMinutes = 1800; required = $true },
|
||
[pscustomobject]@{ name = "status_log"; path = "/backup/logs/status.log"; maxAgeMinutes = 10080; required = $false },
|
||
[pscustomobject]@{ name = "cron_log"; path = "/backup/logs/cron.log"; maxAgeMinutes = 10080; required = $false },
|
||
[pscustomobject]@{ name = "integrity_check_status"; path = "/backup/integrity/check.status"; maxAgeMinutes = 10080; required = $true },
|
||
[pscustomobject]@{ name = "restore_drill_status"; path = "/backup/integrity/restore-drill.status"; maxAgeMinutes = 45000; required = $false },
|
||
[pscustomobject]@{ name = "configs_last_status"; path = "/backup/status/backup-configs-last-status.json"; maxAgeMinutes = 1800; required = $false }
|
||
)
|
||
$cronPatterns = @(
|
||
"backup-all.sh",
|
||
"backup-awoooi-frequent.sh",
|
||
"backup-health-textfile-exporter.py",
|
||
"check-backup-integrity.sh",
|
||
"sync-offsite-backups.sh --mode status",
|
||
"backup-status.sh"
|
||
)
|
||
|
||
[pscustomobject]@{
|
||
host = $backupHost
|
||
targets = if ($backupHealth -and $backupHealth.targets) { @($backupHealth.targets) } else { $targets }
|
||
fileChecks = if ($backupHealth -and $backupHealth.fileChecks) { @($backupHealth.fileChecks) } else { $fileChecks }
|
||
cronPatterns = if ($backupHealth -and $backupHealth.cronPatterns) { @($backupHealth.cronPatterns) } else { $cronPatterns }
|
||
}
|
||
}
|
||
|
||
function Test-AgentBackupSnapshotTarget {
|
||
param(
|
||
[object]$Target,
|
||
[string]$DefaultHost
|
||
)
|
||
|
||
$name = [string]$Target.name
|
||
$hostIp = if ($Target.host) { [string]$Target.host } else { $DefaultHost }
|
||
$path = [string]$Target.path
|
||
$required = Get-AgentBooleanSetting $Target "required" $true
|
||
$maxAgeMinutes = if ($Target.maxAgeMinutes) { [double]$Target.maxAgeMinutes } else { 1800 }
|
||
|
||
if ($path -notmatch "^/backup/[A-Za-z0-9_.\/-]+$") {
|
||
return [pscustomobject]@{ name = $name; host = $hostIp; path = $path; ok = $false; severity = "critical"; reason = "invalid_backup_path" }
|
||
}
|
||
|
||
$quotedPath = Quote-ShSingle $path
|
||
$command = "if [ -d $quotedPath ]; then echo __COUNT__=`$(find $quotedPath -maxdepth 1 -type f 2>/dev/null | wc -l); find $quotedPath -maxdepth 1 -type f -printf '__LATEST__=%T@|%TY-%Tm-%TdT%TH:%TM:%TS|%s|%p\n' 2>/dev/null | sort -n | tail -1; else echo __MISSING_DIR__=$quotedPath; fi"
|
||
$timeoutSeconds = 60
|
||
if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["snapshotReadTimeoutSeconds"]) {
|
||
$timeoutSeconds = [int]$Config.backupHealth.snapshotReadTimeoutSeconds
|
||
}
|
||
if ($Target.PSObject.Properties["timeoutSeconds"]) {
|
||
$timeoutSeconds = [int]$Target.timeoutSeconds
|
||
}
|
||
$readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1
|
||
|
||
$count = 0
|
||
$latestUnix = $null
|
||
$latestSize = $null
|
||
$latestPath = $null
|
||
if ($readback.ok) {
|
||
$countMatch = [regex]::Match($readback.output, "__COUNT__=(\d+)")
|
||
if ($countMatch.Success) { $count = [int]$countMatch.Groups[1].Value }
|
||
$latestMatch = [regex]::Match($readback.output, "__LATEST__=([0-9.]+)\|([^|]+)\|(\d+)\|(.+)")
|
||
if ($latestMatch.Success) {
|
||
$latestUnix = [double]::Parse($latestMatch.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture)
|
||
$latestSize = [int64]$latestMatch.Groups[3].Value
|
||
$latestPath = $latestMatch.Groups[4].Value.Trim()
|
||
}
|
||
}
|
||
|
||
$ageMinutes = if ($null -ne $latestUnix) { Get-AgentUnixAgeMinutes $latestUnix } else { $null }
|
||
$severity = "ok"
|
||
$reason = "fresh"
|
||
if (-not $readback.ok) {
|
||
$severity = if ($required) { "critical" } else { "warning" }
|
||
$reason = "backup_readback_failed"
|
||
} elseif ($readback.output -match "__MISSING_DIR__" -or $count -eq 0 -or $null -eq $latestUnix) {
|
||
$severity = if ($required) { "critical" } else { "warning" }
|
||
$reason = "backup_snapshot_missing"
|
||
} elseif ($ageMinutes -gt $maxAgeMinutes) {
|
||
$severity = if ($required) { "critical" } else { "warning" }
|
||
$reason = "backup_snapshot_stale"
|
||
}
|
||
|
||
$result = [pscustomobject]@{
|
||
name = $name
|
||
host = $hostIp
|
||
path = $path
|
||
required = [bool]$required
|
||
ok = [bool]($severity -eq "ok")
|
||
severity = $severity
|
||
reason = $reason
|
||
count = $count
|
||
latestPath = $latestPath
|
||
latestSizeBytes = $latestSize
|
||
latestTime = if ($null -ne $latestUnix) { Convert-AgentUnixToIso $latestUnix } else { $null }
|
||
ageMinutes = $ageMinutes
|
||
maxAgeMinutes = $maxAgeMinutes
|
||
route = $readback.route
|
||
exitCode = $readback.exitCode
|
||
}
|
||
Write-AgentLog "backup target=$name severity=$severity reason=$reason ageMinutes=$ageMinutes count=$count"
|
||
if ($severity -in @("warning", "critical")) {
|
||
Record-AgentEvent "backup_$reason" $severity "target=$name host=$hostIp ageMinutes=$ageMinutes maxAgeMinutes=$maxAgeMinutes path=$path" $result -Alert
|
||
}
|
||
$result
|
||
}
|
||
|
||
function Test-AgentBackupFileCheck {
|
||
param(
|
||
[object]$FileCheck,
|
||
[string]$DefaultHost
|
||
)
|
||
|
||
$name = [string]$FileCheck.name
|
||
$hostIp = if ($FileCheck.host) { [string]$FileCheck.host } else { $DefaultHost }
|
||
$path = [string]$FileCheck.path
|
||
$required = Get-AgentBooleanSetting $FileCheck "required" $true
|
||
$maxAgeMinutes = if ($FileCheck.maxAgeMinutes) { [double]$FileCheck.maxAgeMinutes } else { 1800 }
|
||
|
||
if ($path -notmatch "^/backup/[A-Za-z0-9_.\/-]+$") {
|
||
return [pscustomobject]@{ name = $name; host = $hostIp; path = $path; ok = $false; severity = "critical"; reason = "invalid_backup_file_path" }
|
||
}
|
||
|
||
$quotedPath = Quote-ShSingle $path
|
||
$command = "if [ -f $quotedPath ]; then stat -c '__STAT__=%Y|%y|%s|%n' $quotedPath; else echo __MISSING_FILE__=$quotedPath; fi"
|
||
$timeoutSeconds = 45
|
||
if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["fileReadTimeoutSeconds"]) {
|
||
$timeoutSeconds = [int]$Config.backupHealth.fileReadTimeoutSeconds
|
||
}
|
||
if ($FileCheck.PSObject.Properties["timeoutSeconds"]) {
|
||
$timeoutSeconds = [int]$FileCheck.timeoutSeconds
|
||
}
|
||
$readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1
|
||
$unix = $null
|
||
$size = $null
|
||
if ($readback.ok) {
|
||
$statMatch = [regex]::Match($readback.output, "__STAT__=([0-9]+)\|([^|]+)\|(\d+)\|(.+)")
|
||
if ($statMatch.Success) {
|
||
$unix = [double]$statMatch.Groups[1].Value
|
||
$size = [int64]$statMatch.Groups[3].Value
|
||
}
|
||
}
|
||
|
||
$ageMinutes = if ($null -ne $unix) { Get-AgentUnixAgeMinutes $unix } else { $null }
|
||
$severity = "ok"
|
||
$reason = "fresh"
|
||
if (-not $readback.ok) {
|
||
$severity = if ($required) { "critical" } else { "warning" }
|
||
$reason = "backup_file_readback_failed"
|
||
} elseif ($readback.output -match "__MISSING_FILE__" -or $null -eq $unix) {
|
||
$severity = if ($required) { "critical" } else { "warning" }
|
||
$reason = "backup_file_missing"
|
||
} elseif ($ageMinutes -gt $maxAgeMinutes) {
|
||
$severity = if ($required) { "critical" } else { "warning" }
|
||
$reason = "backup_file_stale"
|
||
}
|
||
|
||
$result = [pscustomobject]@{
|
||
name = $name
|
||
host = $hostIp
|
||
path = $path
|
||
required = [bool]$required
|
||
ok = [bool]($severity -eq "ok")
|
||
severity = $severity
|
||
reason = $reason
|
||
sizeBytes = $size
|
||
lastWriteTime = if ($null -ne $unix) { Convert-AgentUnixToIso $unix } else { $null }
|
||
ageMinutes = $ageMinutes
|
||
maxAgeMinutes = $maxAgeMinutes
|
||
route = $readback.route
|
||
exitCode = $readback.exitCode
|
||
readTimeoutSeconds = $timeoutSeconds
|
||
readbackElapsedMs = $readback.elapsedMs
|
||
}
|
||
Write-AgentLog "backup file=$name severity=$severity reason=$reason ageMinutes=$ageMinutes elapsedMs=$($readback.elapsedMs) timeoutSeconds=$timeoutSeconds"
|
||
if ($severity -in @("warning", "critical")) {
|
||
Record-AgentEvent "backup_$reason" $severity "file=$name host=$hostIp ageMinutes=$ageMinutes maxAgeMinutes=$maxAgeMinutes path=$path" $result -Alert
|
||
}
|
||
$result
|
||
}
|
||
|
||
function Test-AgentBackupCronPattern {
|
||
param(
|
||
[string]$Pattern,
|
||
[string]$DefaultHost
|
||
)
|
||
|
||
$quotedPattern = Quote-ShSingle $Pattern
|
||
$command = "crontab -l 2>/dev/null | grep -F -- $quotedPattern >/dev/null && echo __PRESENT__ || echo __MISSING__"
|
||
$timeoutSeconds = 45
|
||
if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["cronReadTimeoutSeconds"]) {
|
||
$timeoutSeconds = [int]$Config.backupHealth.cronReadTimeoutSeconds
|
||
}
|
||
$readback = Invoke-HostSshText $DefaultHost $command $timeoutSeconds 2
|
||
$present = [bool]($readback.ok -and $readback.output -match "__PRESENT__")
|
||
$result = [pscustomobject]@{
|
||
host = $DefaultHost
|
||
pattern = $Pattern
|
||
ok = $present
|
||
severity = if ($present) { "ok" } else { "critical" }
|
||
reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" }
|
||
route = $readback.route
|
||
exitCode = $readback.exitCode
|
||
readTimeoutSeconds = $timeoutSeconds
|
||
readbackElapsedMs = $readback.elapsedMs
|
||
}
|
||
Write-AgentLog "backup cron pattern=$Pattern ok=$present"
|
||
if (-not $present) {
|
||
Record-AgentEvent "backup_cron_pattern_missing" "critical" "host=$DefaultHost pattern=$Pattern" $result -Alert
|
||
}
|
||
$result
|
||
}
|
||
|
||
function Test-AgentBackupCronPatterns {
|
||
param(
|
||
[string[]]$Patterns,
|
||
[string]$DefaultHost
|
||
)
|
||
|
||
$timeoutSeconds = 45
|
||
if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["cronReadTimeoutSeconds"]) {
|
||
$timeoutSeconds = [int]$Config.backupHealth.cronReadTimeoutSeconds
|
||
}
|
||
$readback = Invoke-HostSshText $DefaultHost "crontab -l 2>/dev/null" $timeoutSeconds 2
|
||
$results = @()
|
||
if (-not $readback.ok) {
|
||
foreach ($pattern in $Patterns) {
|
||
$result = [pscustomobject]@{
|
||
host = $DefaultHost
|
||
pattern = $pattern
|
||
ok = $false
|
||
severity = "warning"
|
||
reason = "cron_readback_failed"
|
||
route = $readback.route
|
||
exitCode = $readback.exitCode
|
||
readTimeoutSeconds = $timeoutSeconds
|
||
readbackElapsedMs = $readback.elapsedMs
|
||
}
|
||
Record-AgentEvent "backup_cron_readback_failed" "warning" "host=$DefaultHost pattern=$pattern" $result -Alert
|
||
$results += $result
|
||
}
|
||
return $results
|
||
}
|
||
|
||
$crontab = [string]$readback.output
|
||
foreach ($pattern in $Patterns) {
|
||
$present = [bool]($crontab -match [regex]::Escape($pattern))
|
||
$result = [pscustomobject]@{
|
||
host = $DefaultHost
|
||
pattern = $pattern
|
||
ok = $present
|
||
severity = if ($present) { "ok" } else { "critical" }
|
||
reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" }
|
||
route = $readback.route
|
||
exitCode = $readback.exitCode
|
||
readTimeoutSeconds = $timeoutSeconds
|
||
readbackElapsedMs = $readback.elapsedMs
|
||
}
|
||
Write-AgentLog "backup cron pattern=$pattern ok=$present"
|
||
if (-not $present) {
|
||
Record-AgentEvent "backup_cron_pattern_missing" "critical" "host=$DefaultHost pattern=$pattern" $result -Alert
|
||
}
|
||
$results += $result
|
||
}
|
||
|
||
$results
|
||
}
|
||
|
||
function Test-AgentBackupHealth {
|
||
$backupConfig = Get-AgentBackupHealthConfig
|
||
$targets = @()
|
||
foreach ($target in @($backupConfig.targets)) {
|
||
$targets += Test-AgentBackupSnapshotTarget $target $backupConfig.host
|
||
}
|
||
|
||
$files = @()
|
||
foreach ($fileCheck in @($backupConfig.fileChecks)) {
|
||
$files += Test-AgentBackupFileCheck $fileCheck $backupConfig.host
|
||
}
|
||
|
||
$cron = Test-AgentBackupCronPatterns @($backupConfig.cronPatterns) $backupConfig.host
|
||
|
||
$targetCritical = @($targets | Where-Object { $_.severity -eq "critical" }).Count
|
||
$targetWarning = @($targets | Where-Object { $_.severity -eq "warning" }).Count
|
||
$fileCritical = @($files | Where-Object { $_.severity -eq "critical" }).Count
|
||
$fileWarning = @($files | Where-Object { $_.severity -eq "warning" }).Count
|
||
$cronCritical = @($cron | Where-Object { $_.severity -eq "critical" }).Count
|
||
$cronWarning = @($cron | Where-Object { $_.severity -eq "warning" }).Count
|
||
$critical = $targetCritical + $fileCritical + $cronCritical
|
||
$warning = $targetWarning + $fileWarning + $cronWarning
|
||
$severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" }
|
||
|
||
$summary = [pscustomobject]@{
|
||
targetCritical = $targetCritical
|
||
targetWarning = $targetWarning
|
||
fileCritical = $fileCritical
|
||
fileWarning = $fileWarning
|
||
cronCritical = $cronCritical
|
||
cronWarning = $cronWarning
|
||
targetCount = @($targets).Count
|
||
fileCheckCount = @($files).Count
|
||
cronPatternCount = @($cron).Count
|
||
}
|
||
if ($severity -in @("warning", "critical")) {
|
||
Record-AgentEvent "backup_health_degraded" $severity "severity=$severity targetCritical=$targetCritical fileCritical=$fileCritical cronCritical=$cronCritical" $summary -Alert
|
||
} else {
|
||
Write-AgentLog "backup_health ok targets=$($summary.targetCount) files=$($summary.fileCheckCount) cron=$($summary.cronPatternCount)"
|
||
}
|
||
|
||
[pscustomobject]@{
|
||
ok = [bool]($severity -eq "ok")
|
||
severity = $severity
|
||
host = $backupConfig.host
|
||
targets = $targets
|
||
files = $files
|
||
cron = $cron
|
||
summary = $summary
|
||
}
|
||
}
|
||
|
||
function Convert-AgentHtml {
|
||
param([object]$Value)
|
||
[System.Net.WebUtility]::HtmlEncode([string]$Value)
|
||
}
|
||
|
||
function Get-AgentLatestEvidenceSummary {
|
||
param(
|
||
[string]$Name,
|
||
[string]$Pattern
|
||
)
|
||
|
||
$latest = Get-AgentLatestEvidence $Pattern
|
||
[pscustomobject]@{
|
||
name = $Name
|
||
exists = $latest.exists
|
||
file = $latest.fileName
|
||
path = $latest.path
|
||
ageMinutes = $latest.ageMinutes
|
||
parseError = $latest.parseError
|
||
}
|
||
}
|
||
|
||
function Convert-AgentSafeKey {
|
||
param([string]$Value)
|
||
$safe = [regex]::Replace(([string]$Value), "[^A-Za-z0-9_.:-]", "_")
|
||
if (-not $safe) { return "unknown" }
|
||
if ($safe.Length -gt 140) { return $safe.Substring(0, 140) }
|
||
return $safe
|
||
}
|
||
|
||
function New-AgentOutcomeCheck {
|
||
param(
|
||
[string]$Name,
|
||
[bool]$Passed,
|
||
[bool]$Required = $true,
|
||
[string]$Detail = ""
|
||
)
|
||
[pscustomobject]@{
|
||
name = $Name
|
||
passed = $Passed
|
||
required = $Required
|
||
detail = $Detail
|
||
}
|
||
}
|
||
|
||
function Get-AgentOutcomeContract {
|
||
param(
|
||
[string]$ModeName,
|
||
[int]$TransportExitCode,
|
||
[object]$LatestEvidence,
|
||
[datetime]$StartedAt,
|
||
[bool]$SourceEventResolutionRequired,
|
||
[object]$SourceEventResolved = $null,
|
||
[string]$SourceEventEvidence = "",
|
||
[string]$SourceEventResolutionPolicy = "external_source_receipt_required"
|
||
)
|
||
|
||
$checks = @()
|
||
$transportOk = [bool]($TransportExitCode -eq 0)
|
||
$evidenceExists = [bool]($LatestEvidence -and $LatestEvidence.exists)
|
||
$evidenceParsed = [bool]($evidenceExists -and -not $LatestEvidence.parseError -and $LatestEvidence.data)
|
||
$evidenceFresh = $false
|
||
if ($evidenceExists -and $LatestEvidence.lastWriteTime) {
|
||
try {
|
||
$evidenceTime = [datetime]::Parse([string]$LatestEvidence.lastWriteTime)
|
||
$evidenceFresh = [bool]($evidenceTime -ge $StartedAt.AddSeconds(-5))
|
||
} catch {
|
||
$evidenceFresh = $false
|
||
}
|
||
}
|
||
$data = if ($evidenceParsed) { $LatestEvidence.data } else { $null }
|
||
$evidenceMode = if ($data -and $data.PSObject.Properties["mode"]) { [string]$data.mode } else { "" }
|
||
$modeMatches = [bool]($evidenceMode -eq $ModeName)
|
||
|
||
$checks += New-AgentOutcomeCheck "transport_completed" $transportOk $true "exitCode=$TransportExitCode"
|
||
$checks += New-AgentOutcomeCheck "evidence_exists" $evidenceExists $true ([string]$LatestEvidence.path)
|
||
$checks += New-AgentOutcomeCheck "evidence_parsed" $evidenceParsed $true ([string]$LatestEvidence.parseError)
|
||
$checks += New-AgentOutcomeCheck "evidence_fresh" $evidenceFresh $true ([string]$LatestEvidence.lastWriteTime)
|
||
$checks += New-AgentOutcomeCheck "evidence_mode_matches" $modeMatches $true "expected=$ModeName actual=$evidenceMode"
|
||
|
||
$verifierName = "agent99-$($ModeName.ToLowerInvariant())-post-condition-v1"
|
||
$modeVerifierPassed = $false
|
||
$successState = "resolved"
|
||
$failureState = "degraded"
|
||
|
||
if ($data) {
|
||
switch ($ModeName) {
|
||
"Status" {
|
||
$expectedHosts = @($Config.hosts).Count
|
||
$hostRows = @($data.hosts)
|
||
$hostsOk = [bool]($expectedHosts -gt 0 -and $hostRows.Count -ge $expectedHosts -and @($hostRows | Where-Object { -not $_.ping }).Count -eq 0)
|
||
$harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy)
|
||
$awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure)
|
||
$publicRows = @($data.public)
|
||
$publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0)
|
||
$aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0)
|
||
$checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)"
|
||
$checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk
|
||
$checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk
|
||
$checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk
|
||
$checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false
|
||
$modeVerifierPassed = [bool]($hostsOk -and $harborOk -and $awoooOk -and $publicOk)
|
||
}
|
||
"Recover" {
|
||
$expectedVms = @($Config.vms).Count
|
||
$vmRows = @($data.vmResults)
|
||
$vmsOk = [bool]($expectedVms -gt 0 -and $vmRows.Count -ge $expectedVms -and @($vmRows | Where-Object { -not $_.ok }).Count -eq 0)
|
||
$expectedHosts = @($Config.hosts).Count
|
||
$hostRows = @($data.hosts)
|
||
$hostsOk = [bool]($expectedHosts -gt 0 -and $hostRows.Count -ge $expectedHosts -and @($hostRows | Where-Object { -not $_.ping }).Count -eq 0)
|
||
$harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy)
|
||
$awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure)
|
||
$publicRows = @($data.public)
|
||
$publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0)
|
||
$coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified)
|
||
$checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)"
|
||
$checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)"
|
||
$checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk
|
||
$checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk
|
||
$checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk
|
||
$checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ','))
|
||
$modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk)
|
||
}
|
||
"StartVMs" {
|
||
$expectedVms = @($Config.vms).Count
|
||
$vmRows = @($data.vmResults)
|
||
$commandOk = [bool]($expectedVms -gt 0 -and $vmRows.Count -ge $expectedVms -and @($vmRows | Where-Object { -not $_.ok }).Count -eq 0)
|
||
$unreachable = @()
|
||
foreach ($vm in @($Config.vms)) {
|
||
if (-not $vm.host) { $unreachable += [string]$vm.name; continue }
|
||
$reachable = Test-Connection -ComputerName ([string]$vm.host) -Count 1 -Quiet -ErrorAction SilentlyContinue
|
||
if (-not $reachable) { $unreachable += [string]$vm.name }
|
||
}
|
||
$guestsReachable = [bool]($expectedVms -gt 0 -and @($unreachable).Count -eq 0)
|
||
$checks += New-AgentOutcomeCheck "vm_start_commands_ok" $commandOk $true "expected=$expectedVms actual=$($vmRows.Count)"
|
||
$checks += New-AgentOutcomeCheck "vm_guests_reachable" $guestsReachable $true "unreachable=$($unreachable -join ',')"
|
||
$modeVerifierPassed = [bool]($commandOk -and $guestsReachable)
|
||
}
|
||
"PublicSmoke" {
|
||
$rows = @($data.public)
|
||
$routesOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or -not $_.non502 }).Count -eq 0)
|
||
$checks += New-AgentOutcomeCheck "public_routes_non502" $routesOk $true "count=$($rows.Count)"
|
||
$modeVerifierPassed = $routesOk
|
||
}
|
||
"HarborRepair" {
|
||
$harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy)
|
||
$checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk
|
||
$modeVerifierPassed = $harborOk
|
||
}
|
||
"AwoooRepair" {
|
||
$awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure)
|
||
$checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk
|
||
$modeVerifierPassed = $awoooOk
|
||
}
|
||
"Perf" {
|
||
$rows = @($data.performance)
|
||
$readbackOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or (@($_.reasons) -contains "perf_readback_failed") }).Count -eq 0)
|
||
$pressureClear = [bool]($readbackOk -and @($rows | Where-Object { $_.severity -in @("warning", "critical") }).Count -eq 0)
|
||
$checks += New-AgentOutcomeCheck "performance_readback_complete" $readbackOk
|
||
$checks += New-AgentOutcomeCheck "performance_pressure_cleared" $pressureClear
|
||
$modeVerifierPassed = [bool]($readbackOk -and $pressureClear)
|
||
}
|
||
"LoadShed" {
|
||
$rows = @($data.performance)
|
||
$readbackOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or (@($_.reasons) -contains "perf_readback_failed") }).Count -eq 0)
|
||
$executed = @($data.loadShedding | Where-Object { -not $_.skipped })
|
||
$postVerifyOk = [bool]($executed.Count -gt 0 -and @($executed | Where-Object { -not $_.postVerifyOk }).Count -eq 0)
|
||
$alreadyClear = [bool]($readbackOk -and @($rows | Where-Object { $_.severity -in @("warning", "critical") }).Count -eq 0)
|
||
$checks += New-AgentOutcomeCheck "performance_readback_complete" $readbackOk
|
||
$checks += New-AgentOutcomeCheck "load_shed_post_verify" ([bool]($postVerifyOk -or $alreadyClear)) $true "executed=$($executed.Count)"
|
||
$modeVerifierPassed = [bool]($readbackOk -and ($postVerifyOk -or $alreadyClear))
|
||
}
|
||
"SelfCheck" {
|
||
$selfOk = [bool]($data.selfHealth -and $data.selfHealth.severity -eq "ok")
|
||
$checks += New-AgentOutcomeCheck "agent_self_health_ok" $selfOk
|
||
$modeVerifierPassed = $selfOk
|
||
}
|
||
"BackupCheck" {
|
||
$backupOk = [bool]($data.backupHealth -and $data.backupHealth.severity -eq "ok")
|
||
$checks += New-AgentOutcomeCheck "backup_health_ok" $backupOk
|
||
$modeVerifierPassed = $backupOk
|
||
}
|
||
"ProviderFreshness" {
|
||
$provider = $data.providerFreshness
|
||
$directOk = [bool]($provider -and $provider.status -eq "direct_readback_ok" -and @($provider.missingFields).Count -eq 0 -and $provider.lastSeen -and $provider.verifierReadback -and $provider.verifierReadback.ok -eq $true -and [int]$provider.verifierReadback.sourceEventTotal -gt 0)
|
||
$checks += New-AgentOutcomeCheck "provider_freshness_direct_readback" $directOk $true ([string]$provider.status)
|
||
$modeVerifierPassed = $directOk
|
||
}
|
||
"SecurityTriage" {
|
||
$candidateReady = [bool]($data.securityTriage -and $data.securityTriage.status -eq "security_triage_candidate_created")
|
||
$checks += New-AgentOutcomeCheck "security_candidate_created" $candidateReady
|
||
$modeVerifierPassed = $candidateReady
|
||
$successState = "candidate_ready"
|
||
}
|
||
"ControlTick" {
|
||
$controlOk = [bool]($data.controlTick -and $data.controlTick.ok)
|
||
$checks += New-AgentOutcomeCheck "control_tick_completed" $controlOk
|
||
$modeVerifierPassed = $controlOk
|
||
}
|
||
default {
|
||
$checks += New-AgentOutcomeCheck "mode_verifier_defined" $false
|
||
$failureState = "blocked"
|
||
}
|
||
}
|
||
}
|
||
|
||
$basePassed = [bool]($transportOk -and $evidenceExists -and $evidenceParsed -and $evidenceFresh -and $modeMatches)
|
||
$verifierPassed = [bool]($basePassed -and $modeVerifierPassed)
|
||
$sourceEventResolvedBool = if (-not $SourceEventResolutionRequired) {
|
||
$true
|
||
} elseif ($SourceEventResolved -eq $true) {
|
||
$true
|
||
} elseif ($SourceEventResolutionPolicy -eq "mode_verifier_can_resolve") {
|
||
[bool]$verifierPassed
|
||
} else {
|
||
$false
|
||
}
|
||
$effectiveSourceEventEvidence = if ($SourceEventEvidence) {
|
||
$SourceEventEvidence
|
||
} elseif ($SourceEventResolutionPolicy -eq "mode_verifier_can_resolve" -and $verifierPassed) {
|
||
[string]$LatestEvidence.path
|
||
} else {
|
||
""
|
||
}
|
||
if ($SourceEventResolutionRequired) {
|
||
$checks += New-AgentOutcomeCheck "source_event_resolved" $sourceEventResolvedBool $true "policy=$SourceEventResolutionPolicy evidence=$effectiveSourceEventEvidence"
|
||
}
|
||
$state = if (-not $transportOk) {
|
||
"failed"
|
||
} elseif (-not $basePassed) {
|
||
"blocked"
|
||
} elseif (-not $modeVerifierPassed) {
|
||
$failureState
|
||
} elseif ($successState -eq "candidate_ready") {
|
||
"candidate_ready"
|
||
} elseif ($SourceEventResolutionRequired -and -not $sourceEventResolvedBool) {
|
||
"verifying"
|
||
} else {
|
||
"resolved"
|
||
}
|
||
$failedChecks = @($checks | Where-Object { $_.required -and -not $_.passed } | ForEach-Object { $_.name })
|
||
|
||
[pscustomobject]@{
|
||
schemaVersion = "agent99_outcome_contract_v1"
|
||
state = $state
|
||
resolved = [bool]($state -eq "resolved")
|
||
transportOk = $transportOk
|
||
transportExitCode = $TransportExitCode
|
||
verifierName = $verifierName
|
||
verifierPassed = $verifierPassed
|
||
evidenceFresh = $evidenceFresh
|
||
evidenceModeMatched = $modeMatches
|
||
sourceEventResolutionRequired = $SourceEventResolutionRequired
|
||
sourceEventResolutionPolicy = $SourceEventResolutionPolicy
|
||
sourceEventResolved = $sourceEventResolvedBool
|
||
sourceEventEvidence = if ($effectiveSourceEventEvidence) { $effectiveSourceEventEvidence } else { $null }
|
||
failedChecks = $failedChecks
|
||
checks = $checks
|
||
verifiedAt = (Get-Date -Format o)
|
||
}
|
||
}
|
||
|
||
function Invoke-AgentOutcomeContractSelfTest {
|
||
$startedAt = (Get-Date).AddSeconds(-1)
|
||
$publicOkEvidence = [pscustomobject]@{
|
||
exists = $true
|
||
path = "selftest-public-ok.json"
|
||
lastWriteTime = (Get-Date -Format o)
|
||
parseError = $null
|
||
data = [pscustomobject]@{
|
||
mode = "PublicSmoke"
|
||
public = @([pscustomobject]@{ url = "https://example.invalid"; status = 200; non502 = $true; ok = $true })
|
||
}
|
||
}
|
||
$publicFailedEvidence = [pscustomobject]@{
|
||
exists = $true
|
||
path = "selftest-public-failed.json"
|
||
lastWriteTime = (Get-Date -Format o)
|
||
parseError = $null
|
||
data = [pscustomobject]@{
|
||
mode = "PublicSmoke"
|
||
public = @([pscustomobject]@{ url = "https://example.invalid"; status = 502; non502 = $false; ok = $false })
|
||
}
|
||
}
|
||
$providerEvidence = [pscustomobject]@{
|
||
exists = $true
|
||
path = "selftest-provider.json"
|
||
lastWriteTime = (Get-Date -Format o)
|
||
parseError = $null
|
||
data = [pscustomobject]@{
|
||
mode = "ProviderFreshness"
|
||
providerFreshness = [pscustomobject]@{
|
||
status = "direct_readback_ok_no_source_events"
|
||
missingFields = @("lastSeen")
|
||
lastSeen = $null
|
||
verifierReadback = [pscustomobject]@{ ok = $true; sourceEventTotal = 0 }
|
||
}
|
||
}
|
||
}
|
||
$securityEvidence = [pscustomobject]@{
|
||
exists = $true
|
||
path = "selftest-security.json"
|
||
lastWriteTime = (Get-Date -Format o)
|
||
parseError = $null
|
||
data = [pscustomobject]@{
|
||
mode = "SecurityTriage"
|
||
securityTriage = [pscustomobject]@{ status = "security_triage_candidate_created"; hitCount = 1 }
|
||
}
|
||
}
|
||
|
||
$operatorResolved = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $false
|
||
$alertVerifying = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $true $false "selftest-source-event"
|
||
$alertResolvedByVerifier = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $true $false "" "mode_verifier_can_resolve"
|
||
$alertTransportFailedVerifier = Get-AgentOutcomeContract "PublicSmoke" 7 $publicOkEvidence $startedAt $true $false "" "mode_verifier_can_resolve"
|
||
$failedPostCondition = Get-AgentOutcomeContract "PublicSmoke" 0 $publicFailedEvidence $startedAt $false
|
||
$transportFailed = Get-AgentOutcomeContract "PublicSmoke" 7 $publicOkEvidence $startedAt $false
|
||
$providerDegraded = Get-AgentOutcomeContract "ProviderFreshness" 0 $providerEvidence $startedAt $true $false
|
||
$securityCandidate = Get-AgentOutcomeContract "SecurityTriage" 0 $securityEvidence $startedAt $true $false
|
||
$ok = [bool](
|
||
$operatorResolved.state -eq "resolved" -and $operatorResolved.verifierPassed -and
|
||
$alertVerifying.state -eq "verifying" -and -not $alertVerifying.resolved -and
|
||
$alertResolvedByVerifier.state -eq "resolved" -and $alertResolvedByVerifier.sourceEventResolved -and
|
||
$alertTransportFailedVerifier.state -eq "failed" -and -not $alertTransportFailedVerifier.sourceEventResolved -and
|
||
$failedPostCondition.state -eq "degraded" -and -not $failedPostCondition.verifierPassed -and
|
||
$transportFailed.state -eq "failed" -and
|
||
$providerDegraded.state -eq "degraded" -and
|
||
$securityCandidate.state -eq "candidate_ready"
|
||
)
|
||
$result = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
ok = $ok
|
||
contract = "agent99_outcome_contract_v1"
|
||
cases = [pscustomobject]@{
|
||
operatorResolved = $operatorResolved
|
||
alertVerifying = $alertVerifying
|
||
alertResolvedByVerifier = $alertResolvedByVerifier
|
||
alertTransportFailedVerifier = $alertTransportFailedVerifier
|
||
failedPostCondition = $failedPostCondition
|
||
transportFailed = $transportFailed
|
||
providerDegraded = $providerDegraded
|
||
securityCandidate = $securityCandidate
|
||
}
|
||
}
|
||
$selfTestPath = Join-Path $EvidenceDir "agent99-OutcomeContractSelfTest-$stamp.json"
|
||
$result | ConvertTo-Json -Depth 12 | Set-Content -Path $selfTestPath -Encoding UTF8
|
||
$result | ConvertTo-Json -Depth 12 | Set-Content -Path $jsonPath -Encoding UTF8
|
||
Write-AgentLog "outcome_contract_selftest ok=$ok evidence=$selfTestPath"
|
||
$result | ConvertTo-Json -Depth 12
|
||
if (-not $ok) { exit 1 }
|
||
exit 0
|
||
}
|
||
|
||
function Get-AgentProblemStats {
|
||
$stateDir = Join-Path $Config.agentRoot "state"
|
||
$statsPath = Join-Path $stateDir "problem-counts.json"
|
||
if (-not (Test-Path $statsPath)) {
|
||
return [pscustomobject]@{
|
||
path = $statsPath
|
||
schemaVersion = "agent99_problem_counts_v2"
|
||
outcomeContractActivatedAt = $null
|
||
totalEvents = 0
|
||
totalResolved = 0
|
||
totalVerifiedResolved = 0
|
||
totalFailed = 0
|
||
totalDegraded = 0
|
||
totalBlocked = 0
|
||
totalVerifying = 0
|
||
totalCandidateReady = 0
|
||
problems = @()
|
||
}
|
||
}
|
||
try {
|
||
$data = Get-Content $statsPath -Raw | ConvertFrom-Json
|
||
return [pscustomobject]@{
|
||
path = $statsPath
|
||
schemaVersion = if ($data.schemaVersion) { [string]$data.schemaVersion } else { "agent99_problem_counts_legacy_v1" }
|
||
outcomeContractActivatedAt = if ($data.outcomeContractActivatedAt) { [string]$data.outcomeContractActivatedAt } else { $null }
|
||
totalEvents = if ($data.totalEvents) { [int]$data.totalEvents } else { 0 }
|
||
totalResolved = if ($data.totalResolved) { [int]$data.totalResolved } else { 0 }
|
||
totalVerifiedResolved = if ($data.totalVerifiedResolved) { [int]$data.totalVerifiedResolved } else { 0 }
|
||
totalFailed = if ($data.totalFailed) { [int]$data.totalFailed } else { 0 }
|
||
totalDegraded = if ($data.totalDegraded) { [int]$data.totalDegraded } else { 0 }
|
||
totalBlocked = if ($data.totalBlocked) { [int]$data.totalBlocked } else { 0 }
|
||
totalVerifying = if ($data.totalVerifying) { [int]$data.totalVerifying } else { 0 }
|
||
totalCandidateReady = if ($data.totalCandidateReady) { [int]$data.totalCandidateReady } else { 0 }
|
||
problems = @($data.problems)
|
||
}
|
||
} catch {
|
||
Write-AgentLog "problem_stats_parse_failed path=$statsPath error=$($_.Exception.Message)"
|
||
return [pscustomobject]@{
|
||
path = $statsPath
|
||
schemaVersion = "agent99_problem_counts_v2"
|
||
outcomeContractActivatedAt = $null
|
||
totalEvents = 0
|
||
totalResolved = 0
|
||
totalVerifiedResolved = 0
|
||
totalFailed = 0
|
||
totalDegraded = 0
|
||
totalBlocked = 0
|
||
totalVerifying = 0
|
||
totalCandidateReady = 0
|
||
problems = @()
|
||
parseError = $_.Exception.Message
|
||
}
|
||
}
|
||
}
|
||
|
||
function Update-AgentProblemKnowledge {
|
||
param([object]$Result)
|
||
|
||
$stateDir = Join-Path $Config.agentRoot "state"
|
||
$playbookDir = Join-Path $Config.agentRoot "playbooks"
|
||
$ragDir = Join-Path $playbookDir "rag-import"
|
||
New-Item -ItemType Directory -Force -Path $stateDir, $playbookDir, $ragDir | Out-Null
|
||
|
||
$statsPath = Join-Path $stateDir "problem-counts.json"
|
||
$eventsPath = Join-Path $playbookDir "agent99-incident-records.jsonl"
|
||
$kmPath = Join-Path $playbookDir "agent99-incident-km.md"
|
||
$ragPath = Join-Path $ragDir "agent99-incidents.jsonl"
|
||
|
||
$source = if ($Result.source) { [string]$Result.source } else { "agent99" }
|
||
$kind = if ($Result.alertKind) { [string]$Result.alertKind } elseif ($Result.reason) { [string]$Result.reason } else { [string]$Result.mode }
|
||
$service = if ($Result.alertService) { [string]$Result.alertService } else { "agent99" }
|
||
$hostName = if ($Result.alertHost) { [string]$Result.alertHost } else { "" }
|
||
$severity = if ($Result.alertSeverity) { [string]$Result.alertSeverity } elseif ($Result.ok) { "info" } else { "critical" }
|
||
$key = Convert-AgentSafeKey (($source, $service, $hostName, $kind | Where-Object { $_ }) -join "|")
|
||
$now = Get-Date -Format o
|
||
$outcome = if ($Result.PSObject.Properties["outcome"]) { $Result.outcome } else { $null }
|
||
$statusText = if ($outcome -and $outcome.PSObject.Properties["state"]) { [string]$outcome.state } elseif ($Result.ok) { "resolved" } else { "failed" }
|
||
$isResolved = [bool]($statusText -eq "resolved")
|
||
$isFailed = [bool]($statusText -eq "failed")
|
||
if (-not $Result.alertSeverity) {
|
||
$severity = if ($isResolved) { "info" } elseif ($isFailed) { "critical" } else { "warning" }
|
||
}
|
||
|
||
$event = [pscustomobject]@{
|
||
timestamp = $now
|
||
problemKey = $key
|
||
status = $statusText
|
||
commandId = $Result.id
|
||
source = $source
|
||
reason = $Result.reason
|
||
alertId = $Result.alertId
|
||
alertKind = $kind
|
||
alertSeverity = $severity
|
||
service = $service
|
||
host = $hostName
|
||
mode = $Result.mode
|
||
instruction = $Result.instruction
|
||
ok = $isResolved
|
||
transportOk = if ($Result.PSObject.Properties["transportOk"]) { [bool]$Result.transportOk } else { [bool]($Result.exitCode -eq 0) }
|
||
exitCode = $Result.exitCode
|
||
outcomeState = $statusText
|
||
verifierName = if ($outcome) { $outcome.verifierName } else { $null }
|
||
verifierPassed = if ($outcome) { [bool]$outcome.verifierPassed } else { $false }
|
||
sourceEventResolved = if ($outcome) { [bool]$outcome.sourceEventResolved } else { $false }
|
||
evidence = $Result.evidence
|
||
}
|
||
|
||
($event | ConvertTo-Json -Depth 8 -Compress) | Add-Content -Path $eventsPath -Encoding UTF8
|
||
($event | ConvertTo-Json -Depth 8 -Compress) | Add-Content -Path $ragPath -Encoding UTF8
|
||
|
||
$stats = Get-AgentProblemStats
|
||
$problems = @($stats.problems)
|
||
$existing = $problems | Where-Object { $_.key -eq $key } | Select-Object -First 1
|
||
if (-not $existing) {
|
||
$existing = [pscustomobject]@{
|
||
key = $key
|
||
source = $source
|
||
kind = $kind
|
||
service = $service
|
||
host = $hostName
|
||
severity = $severity
|
||
occurrences = 0
|
||
resolved = 0
|
||
verifiedResolved = 0
|
||
failed = 0
|
||
degraded = 0
|
||
blocked = 0
|
||
verifying = 0
|
||
candidateReady = 0
|
||
lastStatus = $null
|
||
firstSeenAt = $now
|
||
lastSeenAt = $null
|
||
lastResolvedAt = $null
|
||
lastEvidence = $null
|
||
lastMode = $null
|
||
lastInstruction = $null
|
||
}
|
||
$problems += $existing
|
||
}
|
||
|
||
foreach ($counterName in @("verifiedResolved", "degraded", "blocked", "verifying", "candidateReady")) {
|
||
if (-not $existing.PSObject.Properties[$counterName]) {
|
||
$existing | Add-Member -MemberType NoteProperty -Name $counterName -Value 0
|
||
}
|
||
}
|
||
|
||
$existing.occurrences = [int]$existing.occurrences + 1
|
||
if ($isResolved) {
|
||
$existing.resolved = [int]$existing.resolved + 1
|
||
$existing.verifiedResolved = [int]$existing.verifiedResolved + 1
|
||
$existing.lastResolvedAt = $now
|
||
} elseif ($isFailed) {
|
||
$existing.failed = [int]$existing.failed + 1
|
||
} elseif ($statusText -eq "degraded") {
|
||
$existing.degraded = [int]$existing.degraded + 1
|
||
} elseif ($statusText -eq "blocked") {
|
||
$existing.blocked = [int]$existing.blocked + 1
|
||
} elseif ($statusText -eq "verifying") {
|
||
$existing.verifying = [int]$existing.verifying + 1
|
||
} elseif ($statusText -eq "candidate_ready") {
|
||
$existing.candidateReady = [int]$existing.candidateReady + 1
|
||
}
|
||
$existing.lastStatus = $statusText
|
||
$existing.lastSeenAt = $now
|
||
$existing.lastEvidence = $Result.evidence
|
||
$existing.lastMode = $Result.mode
|
||
$existing.lastInstruction = $Result.instruction
|
||
$existing.severity = $severity
|
||
|
||
$newTotalEvents = [int]$stats.totalEvents + 1
|
||
$newTotalResolved = [int]$stats.totalResolved + $(if ($isResolved) { 1 } else { 0 })
|
||
$newTotalVerifiedResolved = [int]$stats.totalVerifiedResolved + $(if ($isResolved) { 1 } else { 0 })
|
||
$newTotalFailed = [int]$stats.totalFailed + $(if ($isFailed) { 1 } else { 0 })
|
||
$newTotalDegraded = [int]$stats.totalDegraded + $(if ($statusText -eq "degraded") { 1 } else { 0 })
|
||
$newTotalBlocked = [int]$stats.totalBlocked + $(if ($statusText -eq "blocked") { 1 } else { 0 })
|
||
$newTotalVerifying = [int]$stats.totalVerifying + $(if ($statusText -eq "verifying") { 1 } else { 0 })
|
||
$newTotalCandidateReady = [int]$stats.totalCandidateReady + $(if ($statusText -eq "candidate_ready") { 1 } else { 0 })
|
||
$outcomeContractActivatedAt = if ($stats.outcomeContractActivatedAt) { $stats.outcomeContractActivatedAt } else { $now }
|
||
|
||
$updated = [pscustomobject]@{
|
||
schemaVersion = "agent99_problem_counts_v2"
|
||
generatedAt = $now
|
||
outcomeContractActivatedAt = $outcomeContractActivatedAt
|
||
legacyResolvedCountMayIncludeTransportOnly = [bool]($stats.schemaVersion -ne "agent99_problem_counts_v2" -or ([int]$stats.totalResolved -gt [int]$stats.totalVerifiedResolved))
|
||
totalEvents = $newTotalEvents
|
||
totalResolved = $newTotalResolved
|
||
totalVerifiedResolved = $newTotalVerifiedResolved
|
||
totalFailed = $newTotalFailed
|
||
totalDegraded = $newTotalDegraded
|
||
totalBlocked = $newTotalBlocked
|
||
totalVerifying = $newTotalVerifying
|
||
totalCandidateReady = $newTotalCandidateReady
|
||
problems = @($problems | Sort-Object @{ Expression = { [int]$_.occurrences }; Descending = $true }, @{ Expression = { $_.lastSeenAt }; Descending = $true })
|
||
}
|
||
$updated | ConvertTo-Json -Depth 10 | Set-Content -Path $statsPath -Encoding UTF8
|
||
|
||
$rows = @($updated.problems | Select-Object -First 50 | ForEach-Object {
|
||
"- key=$($_.key); occurrences=$($_.occurrences); verifiedResolved=$($_.verifiedResolved); failed=$($_.failed); degraded=$($_.degraded); blocked=$($_.blocked); verifying=$($_.verifying); candidateReady=$($_.candidateReady); lastStatus=$($_.lastStatus); lastMode=$($_.lastMode); evidence=$($_.lastEvidence)"
|
||
})
|
||
$km = @(
|
||
"# Agent99 Incident KM",
|
||
"",
|
||
"Last updated: $now",
|
||
"",
|
||
"## Totals",
|
||
"",
|
||
"- totalEvents: $($updated.totalEvents)",
|
||
"- totalResolvedLegacyAndVerified: $($updated.totalResolved)",
|
||
"- totalVerifiedResolved: $($updated.totalVerifiedResolved)",
|
||
"- totalFailed: $($updated.totalFailed)",
|
||
"- totalDegraded: $($updated.totalDegraded)",
|
||
"- totalBlocked: $($updated.totalBlocked)",
|
||
"- totalVerifying: $($updated.totalVerifying)",
|
||
"- totalCandidateReady: $($updated.totalCandidateReady)",
|
||
"",
|
||
"## Recurring Problems",
|
||
"",
|
||
($rows -join "`n")
|
||
) -join "`n"
|
||
Set-Content -Path $kmPath -Value $km -Encoding UTF8
|
||
|
||
[pscustomobject]@{
|
||
key = $key
|
||
occurrences = $existing.occurrences
|
||
resolved = $existing.resolved
|
||
verifiedResolved = $existing.verifiedResolved
|
||
failed = $existing.failed
|
||
degraded = $existing.degraded
|
||
blocked = $existing.blocked
|
||
verifying = $existing.verifying
|
||
candidateReady = $existing.candidateReady
|
||
lastStatus = $existing.lastStatus
|
||
statsPath = $statsPath
|
||
eventsPath = $eventsPath
|
||
kmPath = $kmPath
|
||
ragPath = $ragPath
|
||
}
|
||
}
|
||
|
||
function Get-AgentDashboardSummary {
|
||
$status = Get-AgentLatestEvidence "agent99-Status-*.json"
|
||
$perf = Get-AgentLatestEvidence "agent99-Perf-*.json"
|
||
$self = Get-AgentLatestEvidence "agent99-SelfCheck-*.json"
|
||
$backup = Get-AgentLatestEvidence "agent99-BackupCheck-*.json"
|
||
$recover = Get-AgentLatestEvidence "agent99-Recover-*.json"
|
||
|
||
$taskNames = @(
|
||
"Wooo-Agent99-Startup-Recovery",
|
||
"Wooo-Agent99-Heartbeat",
|
||
"Wooo-Agent99-Performance-Guard",
|
||
"Wooo-Agent99-Control-Loop",
|
||
"Wooo-Agent99-Telegram-Inbox",
|
||
"Wooo-Agent99-SRE-Alert-Inbox",
|
||
"Wooo-Agent99-Self-Health",
|
||
"Wooo-Agent99-Backup-Health"
|
||
)
|
||
$tasks = @()
|
||
foreach ($taskName in $taskNames) {
|
||
try {
|
||
$task = Get-ScheduledTask -TaskName $taskName -ErrorAction Stop
|
||
$info = Get-ScheduledTaskInfo -TaskName $taskName -ErrorAction Stop
|
||
$tasks += [pscustomobject]@{
|
||
name = $taskName
|
||
exists = $true
|
||
state = [string]$task.State
|
||
lastTaskResult = $info.LastTaskResult
|
||
lastRunTime = if ($info.LastRunTime) { $info.LastRunTime.ToString("yyyy-MM-dd HH:mm:ss") } else { $null }
|
||
nextRunTime = if ($info.NextRunTime) { $info.NextRunTime.ToString("yyyy-MM-dd HH:mm:ss") } else { $null }
|
||
}
|
||
} catch {
|
||
$tasks += [pscustomobject]@{
|
||
name = $taskName
|
||
exists = $false
|
||
state = "missing"
|
||
lastTaskResult = $null
|
||
lastRunTime = $null
|
||
nextRunTime = $null
|
||
}
|
||
}
|
||
}
|
||
|
||
$queueDir = Join-Path $Config.agentRoot "queue"
|
||
if (-not (Test-Path $queueDir)) { New-Item -ItemType Directory -Force -Path $queueDir | Out-Null }
|
||
$pending = @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" })
|
||
$processedDir = Join-Path $queueDir "processed"
|
||
$failedDir = Join-Path $queueDir "failed"
|
||
$processedCount = if (Test-Path $processedDir) { @(Get-ChildItem -Path $processedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 }
|
||
$failedCount = if (Test-Path $failedDir) { @(Get-ChildItem -Path $failedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 }
|
||
$alertIncomingDir = Join-Path $Config.agentRoot "alerts\incoming"
|
||
$alertProcessedDir = Join-Path $Config.agentRoot "alerts\processed"
|
||
$alertFailedDir = Join-Path $Config.agentRoot "alerts\failed"
|
||
$alertPendingCount = if (Test-Path $alertIncomingDir) { @(Get-ChildItem -Path $alertIncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 }
|
||
$alertProcessedCount = if (Test-Path $alertProcessedDir) { @(Get-ChildItem -Path $alertProcessedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 }
|
||
$alertFailedCount = if (Test-Path $alertFailedDir) { @(Get-ChildItem -Path $alertFailedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 }
|
||
$problemStats = Get-AgentProblemStats
|
||
|
||
[pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
computer = $env:COMPUTERNAME
|
||
user = $env:USERNAME
|
||
agentRoot = $Config.agentRoot
|
||
evidenceDir = $EvidenceDir
|
||
queue = [pscustomobject]@{
|
||
path = $queueDir
|
||
pending = $pending.Count
|
||
processed = $processedCount
|
||
failed = $failedCount
|
||
}
|
||
alerts = [pscustomobject]@{
|
||
incomingPath = $alertIncomingDir
|
||
pending = $alertPendingCount
|
||
processed = $alertProcessedCount
|
||
failed = $alertFailedCount
|
||
}
|
||
problemCounts = [pscustomobject]@{
|
||
path = $problemStats.path
|
||
totalEvents = $problemStats.totalEvents
|
||
totalResolved = $problemStats.totalResolved
|
||
totalVerifiedResolved = $problemStats.totalVerifiedResolved
|
||
totalFailed = $problemStats.totalFailed
|
||
totalDegraded = $problemStats.totalDegraded
|
||
totalBlocked = $problemStats.totalBlocked
|
||
totalVerifying = $problemStats.totalVerifying
|
||
totalCandidateReady = $problemStats.totalCandidateReady
|
||
top = @($problemStats.problems | Select-Object -First 10)
|
||
}
|
||
tasks = $tasks
|
||
evidence = @(
|
||
Get-AgentLatestEvidenceSummary "status" "agent99-Status-*.json"
|
||
Get-AgentLatestEvidenceSummary "performance" "agent99-Perf-*.json"
|
||
Get-AgentLatestEvidenceSummary "self_health" "agent99-SelfCheck-*.json"
|
||
Get-AgentLatestEvidenceSummary "backup" "agent99-BackupCheck-*.json"
|
||
Get-AgentLatestEvidenceSummary "startup_recovery" "agent99-Recover-*.json"
|
||
)
|
||
public = if ($status.exists -and -not $status.parseError -and $status.data) { @($status.data.public | Select-Object url,status,non502,ok) } else { @() }
|
||
hosts = if ($status.exists -and -not $status.parseError -and $status.data) { @($status.data.hosts | Select-Object host,ping) } else { @() }
|
||
performance = if ($perf.exists -and -not $perf.parseError -and $perf.data) { @($perf.data.performance | Select-Object host,severity,loadPerCore,memAvailablePercent,diskUsedPercent,route) } else { @() }
|
||
selfHealth = if ($self.exists -and -not $self.parseError -and $self.data) { $self.data.selfHealth.summary } else { $null }
|
||
backupHealth = if ($backup.exists -and -not $backup.parseError -and $backup.data) { $backup.data.backupHealth.summary } else { $null }
|
||
latestSelfHealthSeverity = if ($self.exists -and -not $self.parseError -and $self.data) { $self.data.selfHealth.severity } else { "unknown" }
|
||
latestBackupSeverity = if ($backup.exists -and -not $backup.parseError -and $backup.data) { $backup.data.backupHealth.severity } else { "unknown" }
|
||
}
|
||
}
|
||
|
||
function Write-AgentDashboard {
|
||
param([object]$Summary)
|
||
|
||
$dashboardDir = Join-Path $Config.agentRoot "dashboard"
|
||
New-Item -ItemType Directory -Force -Path $dashboardDir | Out-Null
|
||
$jsonPath = Join-Path $dashboardDir "agent99-control-center.latest.json"
|
||
$htmlPath = Join-Path $dashboardDir "index.html"
|
||
$Summary | ConvertTo-Json -Depth 12 | Set-Content -Path $jsonPath -Encoding UTF8
|
||
|
||
$taskRows = foreach ($task in @($Summary.tasks)) {
|
||
"<tr><td>$(Convert-AgentHtml $task.name)</td><td>$(Convert-AgentHtml $task.state)</td><td>$(Convert-AgentHtml $task.lastTaskResult)</td><td>$(Convert-AgentHtml $task.lastRunTime)</td><td>$(Convert-AgentHtml $task.nextRunTime)</td></tr>"
|
||
}
|
||
$hostRows = foreach ($hostItem in @($Summary.hosts)) {
|
||
"<tr><td>$(Convert-AgentHtml $hostItem.host)</td><td>$(Convert-AgentHtml $hostItem.ping)</td></tr>"
|
||
}
|
||
$publicRows = foreach ($route in @($Summary.public)) {
|
||
"<tr><td>$(Convert-AgentHtml $route.url)</td><td>$(Convert-AgentHtml $route.status)</td><td>$(Convert-AgentHtml $route.non502)</td><td>$(Convert-AgentHtml $route.ok)</td></tr>"
|
||
}
|
||
$perfRows = foreach ($perfItem in @($Summary.performance)) {
|
||
"<tr><td>$(Convert-AgentHtml $perfItem.host)</td><td>$(Convert-AgentHtml $perfItem.severity)</td><td>$(Convert-AgentHtml $perfItem.loadPerCore)</td><td>$(Convert-AgentHtml $perfItem.memAvailablePercent)</td><td>$(Convert-AgentHtml $perfItem.diskUsedPercent)</td><td>$(Convert-AgentHtml $perfItem.route)</td></tr>"
|
||
}
|
||
$evidenceRows = foreach ($ev in @($Summary.evidence)) {
|
||
"<tr><td>$(Convert-AgentHtml $ev.name)</td><td>$(Convert-AgentHtml $ev.file)</td><td>$(Convert-AgentHtml $ev.ageMinutes)</td><td>$(Convert-AgentHtml $ev.parseError)</td></tr>"
|
||
}
|
||
$problemRows = foreach ($problem in @($Summary.problemCounts.top)) {
|
||
"<tr><td>$(Convert-AgentHtml $problem.key)</td><td>$(Convert-AgentHtml $problem.occurrences)</td><td>$(Convert-AgentHtml $problem.verifiedResolved)</td><td>$(Convert-AgentHtml $problem.failed)</td><td>$(Convert-AgentHtml $problem.degraded)</td><td>$(Convert-AgentHtml $problem.blocked)</td><td>$(Convert-AgentHtml $problem.verifying)</td><td>$(Convert-AgentHtml $problem.candidateReady)</td><td>$(Convert-AgentHtml $problem.lastStatus)</td><td>$(Convert-AgentHtml $problem.lastMode)</td></tr>"
|
||
}
|
||
|
||
$html = @"
|
||
<!doctype html>
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta http-equiv="refresh" content="30">
|
||
<title>Agent99 Control Center</title>
|
||
<style>
|
||
body { font-family: Segoe UI, Arial, sans-serif; margin: 24px; background: #f5f7fb; color: #16202a; }
|
||
h1 { margin: 0 0 6px; font-size: 28px; }
|
||
h2 { margin-top: 26px; font-size: 18px; }
|
||
.meta { color: #526170; margin-bottom: 18px; }
|
||
.grid { display: grid; grid-template-columns: repeat(4, minmax(160px, 1fr)); gap: 12px; }
|
||
.card { background: #fff; border: 1px solid #d8e0ea; border-radius: 8px; padding: 14px; }
|
||
.label { color: #637282; font-size: 12px; text-transform: uppercase; }
|
||
.value { font-size: 22px; margin-top: 4px; font-weight: 650; }
|
||
table { width: 100%; border-collapse: collapse; background: #fff; border: 1px solid #d8e0ea; }
|
||
th, td { padding: 8px 10px; border-bottom: 1px solid #e7edf5; text-align: left; font-size: 13px; }
|
||
th { background: #eef3f8; }
|
||
.ok { color: #087443; }
|
||
.path { font-family: Consolas, monospace; font-size: 12px; color: #465565; }
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<h1>Agent99 Control Center</h1>
|
||
<div class="meta">Last update: $(Convert-AgentHtml $Summary.timestamp) on $(Convert-AgentHtml $Summary.computer) as $(Convert-AgentHtml $Summary.user). Auto-refreshes every 30 seconds.</div>
|
||
<div class="grid">
|
||
<div class="card"><div class="label">Self Health</div><div class="value">$(Convert-AgentHtml $Summary.latestSelfHealthSeverity)</div></div>
|
||
<div class="card"><div class="label">Backup Health</div><div class="value">$(Convert-AgentHtml $Summary.latestBackupSeverity)</div></div>
|
||
<div class="card"><div class="label">Queue Pending</div><div class="value">$(Convert-AgentHtml $Summary.queue.pending)</div></div>
|
||
<div class="card"><div class="label">Alerts Pending</div><div class="value">$(Convert-AgentHtml $Summary.alerts.pending)</div></div>
|
||
<div class="card"><div class="label">Problem Events</div><div class="value">$(Convert-AgentHtml $Summary.problemCounts.totalEvents)</div></div>
|
||
<div class="card"><div class="label">Verified Resolved</div><div class="value">$(Convert-AgentHtml $Summary.problemCounts.totalVerifiedResolved)</div></div>
|
||
<div class="card"><div class="label">Failed</div><div class="value">$(Convert-AgentHtml $Summary.problemCounts.totalFailed)</div></div>
|
||
<div class="card"><div class="label">Degraded</div><div class="value">$(Convert-AgentHtml $Summary.problemCounts.totalDegraded)</div></div>
|
||
<div class="card"><div class="label">Verifying</div><div class="value">$(Convert-AgentHtml $Summary.problemCounts.totalVerifying)</div></div>
|
||
</div>
|
||
<p class="path">Agent root: $(Convert-AgentHtml $Summary.agentRoot)<br>Queue: $(Convert-AgentHtml $Summary.queue.path)<br>Alerts: $(Convert-AgentHtml $Summary.alerts.incomingPath)<br>Problem counts: $(Convert-AgentHtml $Summary.problemCounts.path)<br>Evidence: $(Convert-AgentHtml $Summary.evidenceDir)</p>
|
||
<h2>Problem Counts</h2>
|
||
<table><tr><th>Problem</th><th>Occurrences</th><th>Verified Resolved</th><th>Failed</th><th>Degraded</th><th>Blocked</th><th>Verifying</th><th>Candidate Ready</th><th>Last Status</th><th>Last Mode</th></tr>$($problemRows -join "`n")</table>
|
||
<h2>Scheduled Tasks</h2>
|
||
<table><tr><th>Task</th><th>State</th><th>Last Result</th><th>Last Run</th><th>Next Run</th></tr>$($taskRows -join "`n")</table>
|
||
<h2>Hosts</h2>
|
||
<table><tr><th>Host</th><th>Ping</th></tr>$($hostRows -join "`n")</table>
|
||
<h2>Public Routes</h2>
|
||
<table><tr><th>URL</th><th>Status</th><th>Non-502</th><th>OK</th></tr>$($publicRows -join "`n")</table>
|
||
<h2>Performance</h2>
|
||
<table><tr><th>Host</th><th>Severity</th><th>Load/Core</th><th>Mem Avail %</th><th>Disk Used %</th><th>Route</th></tr>$($perfRows -join "`n")</table>
|
||
<h2>Evidence</h2>
|
||
<table><tr><th>Name</th><th>File</th><th>Age Minutes</th><th>Parse Error</th></tr>$($evidenceRows -join "`n")</table>
|
||
</body>
|
||
</html>
|
||
"@
|
||
Set-Content -Path $htmlPath -Value $html -Encoding UTF8
|
||
[pscustomobject]@{
|
||
dashboardDir = $dashboardDir
|
||
htmlPath = $htmlPath
|
||
jsonPath = $jsonPath
|
||
}
|
||
}
|
||
|
||
function Ensure-AgentDesktopShortcuts {
|
||
param([string]$DashboardPath)
|
||
|
||
$created = @()
|
||
$targets = @(
|
||
[pscustomobject]@{ name = "Agent99 Control Center.lnk"; target = $DashboardPath },
|
||
[pscustomobject]@{ name = "Agent99 Submit Request.lnk"; target = (Join-Path $Config.agentRoot "agent99-submit-request.cmd") },
|
||
[pscustomobject]@{ name = "Agent99 Queue.lnk"; target = (Join-Path $Config.agentRoot "queue") },
|
||
[pscustomobject]@{ name = "Agent99 Evidence.lnk"; target = $EvidenceDir }
|
||
)
|
||
$desktopDirs = @(
|
||
[Environment]::GetFolderPath("CommonDesktopDirectory"),
|
||
[Environment]::GetFolderPath("Desktop")
|
||
) | Where-Object { $_ } | Select-Object -Unique
|
||
|
||
foreach ($desktop in $desktopDirs) {
|
||
if (-not (Test-Path $desktop)) { continue }
|
||
foreach ($item in $targets) {
|
||
$shortcutPath = Join-Path $desktop $item.name
|
||
try {
|
||
$shell = New-Object -ComObject WScript.Shell
|
||
$shortcut = $shell.CreateShortcut($shortcutPath)
|
||
$shortcut.TargetPath = $item.target
|
||
$shortcut.WorkingDirectory = Split-Path -Parent $item.target
|
||
$shortcut.Description = "Agent99 local control and evidence shortcut"
|
||
$shortcut.Save()
|
||
$created += $shortcutPath
|
||
} catch {
|
||
Write-AgentLog "desktop_shortcut_failed path=$shortcutPath error=$($_.Exception.Message)"
|
||
}
|
||
}
|
||
}
|
||
$created
|
||
}
|
||
|
||
function Invoke-AgentQueuedCommands {
|
||
$queueDir = Join-Path $Config.agentRoot "queue"
|
||
$processedDir = Join-Path $queueDir "processed"
|
||
$failedDir = Join-Path $queueDir "failed"
|
||
New-Item -ItemType Directory -Force -Path $queueDir, $processedDir, $failedDir | Out-Null
|
||
|
||
$allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage")
|
||
$commands = @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue |
|
||
Where-Object { $_.Name -notmatch "^(processed|failed|running)-" } |
|
||
Sort-Object LastWriteTime |
|
||
Select-Object -First 3)
|
||
$results = @()
|
||
foreach ($file in $commands) {
|
||
$command = $null
|
||
try {
|
||
$command = Get-Content $file.FullName -Raw | ConvertFrom-Json
|
||
} catch {
|
||
$target = Join-Path $failedDir ("failed-" + $file.Name)
|
||
Move-Item -Force $file.FullName $target
|
||
$results += [pscustomobject]@{ id = $file.BaseName; ok = $false; reason = "invalid_json"; file = $target }
|
||
continue
|
||
}
|
||
|
||
$id = if ($command.id) { [string]$command.id } else { $file.BaseName }
|
||
$modeName = if ($command.mode) { [string]$command.mode } else { "" }
|
||
$controlled = if ($command.PSObject.Properties["controlledApply"]) { [bool]$command.controlledApply } else { $false }
|
||
$commandSource = if ($command.PSObject.Properties["source"]) { [string]$command.source } else { "queue" }
|
||
$commandReason = if ($command.PSObject.Properties["reason"]) { [string]$command.reason } else { "operator_instruction" }
|
||
$commandInstruction = if ($command.PSObject.Properties["instruction"]) { [string]$command.instruction } else { "" }
|
||
$alertId = if ($command.PSObject.Properties["alertId"]) { [string]$command.alertId } else { $null }
|
||
$alertSource = if ($command.PSObject.Properties["alertSource"]) { [string]$command.alertSource } else { $null }
|
||
$alertKind = if ($command.PSObject.Properties["alertKind"]) { [string]$command.alertKind } else { $null }
|
||
$alertSeverity = if ($command.PSObject.Properties["alertSeverity"]) { [string]$command.alertSeverity } else { $null }
|
||
$alertService = if ($command.PSObject.Properties["alertService"]) { [string]$command.alertService } else { $null }
|
||
$alertHost = if ($command.PSObject.Properties["alertHost"]) { [string]$command.alertHost } else { $null }
|
||
$sourceEventResolutionRequired = if ($command.PSObject.Properties["sourceEventResolutionRequired"]) { Convert-AgentBool $command.sourceEventResolutionRequired } else { [bool]$alertId }
|
||
$sourceEventResolutionPolicy = if ($command.PSObject.Properties["sourceEventResolutionPolicy"]) { [string]$command.sourceEventResolutionPolicy } else { "external_source_receipt_required" }
|
||
$sourceEventResolved = if ($command.PSObject.Properties["sourceEventResolved"]) { [object](Convert-AgentBool $command.sourceEventResolved) } else { $null }
|
||
$sourceEventEvidence = if ($command.PSObject.Properties["sourceEventEvidence"]) { [string]$command.sourceEventEvidence } else { "" }
|
||
$replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" }
|
||
$suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false }
|
||
if ($modeName -notin $allowedModes) {
|
||
$target = Join-Path $failedDir ("failed-" + $file.Name)
|
||
Move-Item -Force $file.FullName $target
|
||
$result = [pscustomobject]@{ id = $id; mode = $modeName; source = $commandSource; instruction = $commandInstruction; ok = $false; reason = "mode_not_allowlisted"; file = $target }
|
||
Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=mode_not_allowlisted" $result -Alert
|
||
$results += $result
|
||
continue
|
||
}
|
||
|
||
$runningPath = Join-Path $queueDir ("running-" + $file.Name)
|
||
Move-Item -Force $file.FullName $runningPath
|
||
$startTime = Get-Date
|
||
$launcher = Join-Path $Config.agentRoot "agent99-run.ps1"
|
||
$stdoutPath = Join-Path $EvidenceDir "agent99-command-$id.out"
|
||
$stderrPath = Join-Path $EvidenceDir "agent99-command-$id.err"
|
||
$args = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", $modeName)
|
||
if ($controlled) { $args += "-ControlledApply" }
|
||
if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") }
|
||
$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
|
||
$completed = $process.WaitForExit(600000)
|
||
if (-not $completed) {
|
||
$process.Kill()
|
||
$process.WaitForExit(2000) | Out-Null
|
||
}
|
||
$process.WaitForExit() | Out-Null
|
||
$process.Refresh()
|
||
$latest = Get-AgentLatestEvidence "agent99-$modeName-*.json"
|
||
$exitCode = if (-not $completed) {
|
||
-2
|
||
} elseif ($null -ne $process.ExitCode) {
|
||
$process.ExitCode
|
||
} elseif ($latest.exists -and -not $latest.parseError) {
|
||
0
|
||
} else {
|
||
-3
|
||
}
|
||
$outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence -SourceEventResolutionPolicy $sourceEventResolutionPolicy
|
||
$result = [pscustomobject]@{
|
||
id = $id
|
||
mode = $modeName
|
||
controlledApply = $controlled
|
||
source = $commandSource
|
||
reason = $commandReason
|
||
instruction = $commandInstruction
|
||
alertId = $alertId
|
||
alertSource = $alertSource
|
||
alertKind = $alertKind
|
||
alertSeverity = $alertSeverity
|
||
alertService = $alertService
|
||
alertHost = $alertHost
|
||
suppressTelegram = $suppressTelegram
|
||
transportOk = [bool]($exitCode -eq 0)
|
||
ok = [bool]$outcome.resolved
|
||
outcomeState = $outcome.state
|
||
verifierPassed = [bool]$outcome.verifierPassed
|
||
sourceEventResolved = [bool]$outcome.sourceEventResolved
|
||
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
|
||
outcome = $outcome
|
||
exitCode = $exitCode
|
||
durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1)
|
||
evidence = $latest.path
|
||
stdout = $stdoutPath
|
||
stderr = $stderrPath
|
||
}
|
||
$problem = Update-AgentProblemKnowledge $result
|
||
$result | Add-Member -MemberType NoteProperty -Name problem -Value $problem -Force
|
||
$donePath = Join-Path $processedDir ("processed-$id.json")
|
||
$result | ConvertTo-Json -Depth 8 | Set-Content -Path $donePath -Encoding UTF8
|
||
Remove-Item $runningPath -Force -ErrorAction SilentlyContinue | Out-Null
|
||
$recordData = $result | Select-Object *
|
||
if ($replyChatId) {
|
||
$recordData | Add-Member -MemberType NoteProperty -Name replyChatId -Value $replyChatId -Force
|
||
}
|
||
$outcomeSeverity = if ($outcome.state -eq "resolved") { "info" } elseif ($outcome.state -eq "failed") { "critical" } else { "warning" }
|
||
Record-AgentEvent "operator_command_result" $outcomeSeverity "id=$id mode=$modeName outcome=$($outcome.state) transportExitCode=$exitCode verifierPassed=$($outcome.verifierPassed) sourceEventResolved=$($outcome.sourceEventResolved) evidence=$($latest.path)" $recordData -Alert
|
||
$results += $result
|
||
}
|
||
$results
|
||
}
|
||
|
||
function Invoke-AgentControlTick {
|
||
$processed = Invoke-AgentQueuedCommands
|
||
$summary = Get-AgentDashboardSummary
|
||
$dashboard = Write-AgentDashboard $summary
|
||
$shortcuts = Ensure-AgentDesktopShortcuts $dashboard.htmlPath
|
||
$pendingCount = if ($summary.queue) { $summary.queue.pending } else { 0 }
|
||
$result = [pscustomobject]@{
|
||
ok = $true
|
||
dashboardPath = $dashboard.htmlPath
|
||
dashboardJson = $dashboard.jsonPath
|
||
shortcutCount = @($shortcuts).Count
|
||
shortcuts = $shortcuts
|
||
processedCount = @($processed).Count
|
||
processed = $processed
|
||
pendingCount = $pendingCount
|
||
}
|
||
Record-AgentEvent "agent_control_tick" "info" "dashboard=$($dashboard.htmlPath) processed=$(@($processed).Count) pending=$pendingCount" $result -NoAlert
|
||
$result
|
||
}
|
||
|
||
function Invoke-AgentSecurityTriage {
|
||
$lookbackMinutes = 120
|
||
$since = (Get-Date).AddMinutes(-1 * $lookbackMinutes)
|
||
$triageId = "security-triage-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||
$triageDir = Join-Path $Config.agentRoot "state\security-triage"
|
||
New-Item -ItemType Directory -Force -Path $triageDir | Out-Null
|
||
$triagePath = Join-Path $triageDir "$triageId.json"
|
||
|
||
$patterns = @(
|
||
"wazuh",
|
||
"siem",
|
||
"security",
|
||
"auth_failed",
|
||
"bruteforce",
|
||
"brute force",
|
||
"intrusion",
|
||
"malware",
|
||
"privilege",
|
||
"vulnerability",
|
||
"cve",
|
||
"ioc",
|
||
"suspicious",
|
||
"unauthorized"
|
||
)
|
||
$scanDirs = @(
|
||
(Join-Path $Config.agentRoot "alerts\incoming"),
|
||
(Join-Path $Config.agentRoot "alerts\processed"),
|
||
(Join-Path $Config.agentRoot "queue"),
|
||
(Join-Path $Config.agentRoot "queue\processed"),
|
||
(Join-Path $Config.agentRoot "requests")
|
||
) | Where-Object { Test-Path $_ } | Select-Object -Unique
|
||
|
||
$hits = @()
|
||
foreach ($dir in $scanDirs) {
|
||
$files = @(Get-ChildItem -LiteralPath $dir -Recurse -File -ErrorAction SilentlyContinue |
|
||
Where-Object { $_.LastWriteTime -ge $since -and $_.Length -lt 2097152 })
|
||
foreach ($file in $files) {
|
||
$matchedPatterns = @()
|
||
foreach ($pattern in $patterns) {
|
||
if (Select-String -LiteralPath $file.FullName -Pattern $pattern -SimpleMatch -Quiet -ErrorAction SilentlyContinue) {
|
||
$matchedPatterns += $pattern
|
||
}
|
||
}
|
||
if (@($matchedPatterns).Count -gt 0) {
|
||
$excerpt = ""
|
||
$firstHit = Select-String -LiteralPath $file.FullName -Pattern $matchedPatterns[0] -SimpleMatch -ErrorAction SilentlyContinue | Select-Object -First 1
|
||
if ($firstHit) {
|
||
$excerpt = [string]$firstHit.Line
|
||
$excerpt = $excerpt -replace '("?(telegram|token|chat|authorization|password|secret|cookie)[^,}\r\n]*"?\s*:\s*)"[^"]+"', '$1"[redacted]"'
|
||
if ($excerpt.Length -gt 500) { $excerpt = $excerpt.Substring(0, 500) + "...(truncated)" }
|
||
}
|
||
$hits += [pscustomobject]@{
|
||
path = $file.FullName
|
||
lastWrite = $file.LastWriteTime.ToString("o")
|
||
patterns = @($matchedPatterns)
|
||
excerpt = $excerpt
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
$status = if (@($hits).Count -gt 0) { "security_triage_candidate_created" } else { "security_triage_no_recent_alert_body_seen" }
|
||
$triage = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
triageId = $triageId
|
||
status = $status
|
||
lookbackMinutes = $lookbackMinutes
|
||
since = $since.ToString("o")
|
||
hitCount = @($hits).Count
|
||
hits = @($hits | Sort-Object lastWrite -Descending | Select-Object -First 20)
|
||
requiredReadback = @("source_log_reference", "affected_host_or_service", "blast_radius", "repeated_count", "allowlisted_next_action")
|
||
allowedActions = @("classify", "read_existing_evidence", "write_km", "request_source_log_reference", "recommend_allowlisted_next_action")
|
||
prohibitedActions = @("arbitrary_shell", "data_delete", "disable_security_control", "secret_exposure")
|
||
evidencePath = $triagePath
|
||
}
|
||
$triage | ConvertTo-Json -Depth 10 | Set-Content -Path $triagePath -Encoding UTF8
|
||
Write-AgentLog "security_triage triage=$triageId hits=$(@($hits).Count) status=$status path=$triagePath"
|
||
$eventSeverity = if (@($hits).Count -gt 0) { "warning" } else { "info" }
|
||
Record-AgentEvent "security_triage" $eventSeverity "status=$status hits=$(@($hits).Count)" $triage -Alert
|
||
$triage
|
||
}
|
||
|
||
function Invoke-AgentProviderFreshnessDirectReadback {
|
||
param(
|
||
[int]$LookbackMinutes
|
||
)
|
||
|
||
$baseUrl = "https://awoooi.wooo.work/api/v1/platform/events/dossier/recurrence?project_id=awoooi&limit=20"
|
||
$startedAt = Get-Date
|
||
try {
|
||
$response = Invoke-RestMethod -Method Get -Uri $baseUrl -TimeoutSec 12
|
||
$summary = if ($response -and $response.PSObject.Properties["summary"]) { $response.summary } else { $null }
|
||
$latest = if ($summary -and $summary.PSObject.Properties["latest_received_at"]) { $summary.latest_received_at } else { $null }
|
||
$sourceTotal = if ($summary -and $summary.PSObject.Properties["source_event_total"]) { [int]$summary.source_event_total } else { 0 }
|
||
$itemsCount = if ($response -and $response.PSObject.Properties["items"]) { @($response.items).Count } else { 0 }
|
||
return [pscustomobject]@{
|
||
ok = $true
|
||
verifier = "awoooi-platform-events-recurrence"
|
||
directReference = $baseUrl
|
||
providerWindowMinutes = $LookbackMinutes
|
||
latestReceivedAt = $latest
|
||
sourceEventTotal = $sourceTotal
|
||
recurrenceGroupTotal = if ($summary -and $summary.PSObject.Properties["recurrence_group_total"]) { [int]$summary.recurrence_group_total } else { 0 }
|
||
itemsCount = $itemsCount
|
||
httpStatus = 200
|
||
durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2)
|
||
checkedAt = (Get-Date -Format o)
|
||
}
|
||
} catch {
|
||
return [pscustomobject]@{
|
||
ok = $false
|
||
verifier = "awoooi-platform-events-recurrence"
|
||
directReference = $baseUrl
|
||
providerWindowMinutes = $LookbackMinutes
|
||
latestReceivedAt = $null
|
||
sourceEventTotal = 0
|
||
recurrenceGroupTotal = 0
|
||
itemsCount = 0
|
||
httpStatus = $null
|
||
durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2)
|
||
checkedAt = (Get-Date -Format o)
|
||
error = ($_ | Out-String).Trim()
|
||
}
|
||
}
|
||
}
|
||
|
||
function Invoke-AgentProviderFreshnessTriage {
|
||
$providerConfig = if ($Config.PSObject.Properties["providerFreshness"]) { $Config.providerFreshness } else { $null }
|
||
$lookbackMinutes = if ($providerConfig -and $providerConfig.PSObject.Properties["lookbackMinutes"]) { [int]$providerConfig.lookbackMinutes } else { 120 }
|
||
$since = (Get-Date).AddMinutes(-1 * $lookbackMinutes)
|
||
$candidateId = "provider-freshness-" + (Get-Date -Format "yyyyMMdd-HHmmss")
|
||
$candidateDir = Join-Path $Config.agentRoot "state\provider-freshness-candidates"
|
||
New-Item -ItemType Directory -Force -Path $candidateDir | Out-Null
|
||
$candidatePath = Join-Path $candidateDir "$candidateId.json"
|
||
|
||
$patterns = @(
|
||
"provider_freshness_signal",
|
||
"source_provider_freshness_triage",
|
||
"raw_signal_normalized",
|
||
"controlled_runtime_candidate",
|
||
"freshness",
|
||
"last_seen",
|
||
"last seen",
|
||
"ingestion"
|
||
)
|
||
$scanDirs = @(
|
||
(Join-Path $Config.agentRoot "alerts\incoming"),
|
||
(Join-Path $Config.agentRoot "alerts\processed"),
|
||
(Join-Path $Config.agentRoot "queue"),
|
||
(Join-Path $Config.agentRoot "queue\processed"),
|
||
(Join-Path $Config.agentRoot "requests")
|
||
) | Where-Object { Test-Path $_ } | Select-Object -Unique
|
||
|
||
$hits = @()
|
||
foreach ($dir in $scanDirs) {
|
||
$files = @(Get-ChildItem -LiteralPath $dir -Recurse -File -ErrorAction SilentlyContinue |
|
||
Where-Object { $_.LastWriteTime -ge $since -and $_.Length -lt 2097152 })
|
||
foreach ($file in $files) {
|
||
$matchedPatterns = @()
|
||
foreach ($pattern in $patterns) {
|
||
if (Select-String -LiteralPath $file.FullName -Pattern $pattern -SimpleMatch -Quiet -ErrorAction SilentlyContinue) {
|
||
$matchedPatterns += $pattern
|
||
}
|
||
}
|
||
if (@($matchedPatterns).Count -gt 0) {
|
||
$excerpt = ""
|
||
$firstHit = Select-String -LiteralPath $file.FullName -Pattern $matchedPatterns[0] -SimpleMatch -ErrorAction SilentlyContinue | Select-Object -First 1
|
||
if ($firstHit) {
|
||
$excerpt = [string]$firstHit.Line
|
||
$excerpt = $excerpt -replace '("?(telegram|token|chat|authorization|password)[^,}\r\n]*"?\s*:\s*)"[^"]+"', '$1"[redacted]"'
|
||
if ($excerpt.Length -gt 500) { $excerpt = $excerpt.Substring(0, 500) + "...(truncated)" }
|
||
}
|
||
$hits += [pscustomobject]@{
|
||
path = $file.FullName
|
||
lastWrite = $file.LastWriteTime.ToString("o")
|
||
patterns = @($matchedPatterns)
|
||
excerpt = $excerpt
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
$targets = @()
|
||
if ($providerConfig -and $providerConfig.targets) {
|
||
foreach ($target in @($providerConfig.targets)) {
|
||
$targets += [pscustomobject]@{
|
||
name = if ($target.name) { [string]$target.name } else { "provider_freshness_signal" }
|
||
providerWindowMinutes = if ($target.PSObject.Properties["providerWindowMinutes"]) { $target.providerWindowMinutes } else { $null }
|
||
directReference = if ($target.directReference) { [string]$target.directReference } else { $null }
|
||
verifier = if ($target.verifier) { [string]$target.verifier } else { "agent99-provider-freshness-triage" }
|
||
}
|
||
}
|
||
}
|
||
if (@($targets).Count -eq 0) {
|
||
$targets += [pscustomobject]@{
|
||
name = "provider_freshness_signal"
|
||
providerWindowMinutes = $null
|
||
directReference = $null
|
||
verifier = "agent99-provider-freshness-triage"
|
||
}
|
||
}
|
||
|
||
$verifierReadback = Invoke-AgentProviderFreshnessDirectReadback -LookbackMinutes $lookbackMinutes
|
||
$directReference = if ($verifierReadback -and $verifierReadback.PSObject.Properties["directReference"]) { $verifierReadback.directReference } else { $null }
|
||
$latestSeen = if ($verifierReadback -and $verifierReadback.PSObject.Properties["latestReceivedAt"]) { $verifierReadback.latestReceivedAt } else { $null }
|
||
$missingFields = @()
|
||
if (-not $lookbackMinutes) { $missingFields += "providerWindow" }
|
||
if (-not $latestSeen) { $missingFields += "lastSeen" }
|
||
if (-not $directReference) { $missingFields += "directReference" }
|
||
if (-not ($verifierReadback -and $verifierReadback.ok -eq $true)) { $missingFields += "verifierReadback" }
|
||
$status = if ($verifierReadback -and $verifierReadback.ok -eq $true -and $latestSeen) {
|
||
"direct_readback_ok"
|
||
} elseif ($verifierReadback -and $verifierReadback.ok -eq $true) {
|
||
"direct_readback_ok_no_source_events"
|
||
} elseif (@($hits).Count -gt 0) {
|
||
"candidate_created_waiting_direct_readback"
|
||
} else {
|
||
"candidate_created_no_alert_body_seen"
|
||
}
|
||
$candidate = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
candidateId = $candidateId
|
||
target = "provider_freshness_signal"
|
||
status = $status
|
||
lookbackMinutes = $lookbackMinutes
|
||
since = $since.ToString("o")
|
||
hitCount = @($hits).Count
|
||
hits = @($hits | Sort-Object lastWrite -Descending | Select-Object -First 20)
|
||
targets = $targets
|
||
missingFields = $missingFields
|
||
providerWindowMinutes = $lookbackMinutes
|
||
lastSeen = $latestSeen
|
||
directReference = $directReference
|
||
verifierReadback = $verifierReadback
|
||
requiredVerifierReadback = @("provider_window", "last_seen", "direct_reference", "source_trace_or_sentry_signoz_ref")
|
||
noFalseGreenRisk = $true
|
||
allowedActions = @("create_candidate", "read_existing_evidence", "provider_freshness_direct_readback", "request_direct_reference", "write_km")
|
||
prohibitedActions = @("provider_switch", "paid_model_call", "env_change", "reload")
|
||
candidatePath = $candidatePath
|
||
}
|
||
$candidate | ConvertTo-Json -Depth 10 | Set-Content -Path $candidatePath -Encoding UTF8
|
||
Write-AgentLog "provider_freshness candidate=$candidateId hits=$(@($hits).Count) status=$status missing=$($missingFields -join ',') path=$candidatePath"
|
||
$eventSeverity = if (@($missingFields).Count -gt 0) { "warning" } else { "info" }
|
||
Record-AgentEvent "provider_freshness_triage" $eventSeverity "target=provider_freshness_signal status=$status missing=$($missingFields -join ',')" $candidate -Alert
|
||
$candidate
|
||
}
|
||
|
||
function Test-HostReachability {
|
||
$results = @()
|
||
foreach ($hostIp in $Config.hosts) {
|
||
$ping = Test-Connection -ComputerName $hostIp -Count 1 -Quiet -ErrorAction SilentlyContinue
|
||
Write-AgentLog "host host=$hostIp ping=$ping"
|
||
$results += [pscustomobject]@{
|
||
host = $hostIp
|
||
ping = [bool]$ping
|
||
}
|
||
}
|
||
$results
|
||
}
|
||
|
||
function Test-PublicRoutes {
|
||
$results = @()
|
||
foreach ($url in $Config.publicUrls) {
|
||
try {
|
||
$output = & curl.exe --location --head --max-time 12 $url 2>&1
|
||
$text = $output -join "`n"
|
||
$matches = [regex]::Matches($text, "HTTP/\S+\s+(\d+)")
|
||
$status = if ($matches.Count -gt 0) { [int]$matches[$matches.Count - 1].Groups[1].Value } else { $null }
|
||
$non502 = ($null -ne $status -and $status -ne 502)
|
||
$ok = ($null -ne $status -and $status -ge 200 -and $status -lt 500)
|
||
Write-AgentLog "public url=$url status=$status non502=$non502 ok=$ok"
|
||
$results += [pscustomobject]@{
|
||
url = $url
|
||
status = $status
|
||
non502 = $non502
|
||
ok = $ok
|
||
}
|
||
} catch {
|
||
Write-AgentLog "public url=$url error=$($_.Exception.Message)"
|
||
$results += [pscustomobject]@{
|
||
url = $url
|
||
status = $null
|
||
non502 = $false
|
||
ok = $false
|
||
}
|
||
}
|
||
}
|
||
$results
|
||
}
|
||
|
||
function Test-AiServices {
|
||
$results = @()
|
||
foreach ($svc in @($Config.aiServices)) {
|
||
if (-not $svc.url) { continue }
|
||
$name = if ($svc.name) { [string]$svc.name } else { [string]$svc.url }
|
||
try {
|
||
$output = & curl.exe --max-time 12 -sS $svc.url 2>&1
|
||
$text = ($output -join "`n").Trim()
|
||
$statusValue = $null
|
||
try {
|
||
$json = $text | ConvertFrom-Json
|
||
if ($json.status) { $statusValue = [string]$json.status }
|
||
} catch {
|
||
$statusValue = $null
|
||
}
|
||
$expectedStatus = if ($svc.expectedStatus) { [string]$svc.expectedStatus } else { "healthy" }
|
||
$ok = ($statusValue -eq $expectedStatus)
|
||
$severity = if ($ok) { "ok" } elseif ($svc.critical) { "critical" } else { "warning" }
|
||
$result = [pscustomobject]@{
|
||
name = $name
|
||
url = $svc.url
|
||
host = $svc.host
|
||
ok = $ok
|
||
severity = $severity
|
||
status = $statusValue
|
||
expectedStatus = $expectedStatus
|
||
output = $text
|
||
}
|
||
Write-AgentLog "ai_service name=$name ok=$ok severity=$severity status=$statusValue url=$($svc.url)"
|
||
if (-not $ok) {
|
||
Record-AgentEvent "ai_service_unhealthy" $severity "name=$name status=$statusValue expected=$expectedStatus url=$($svc.url)" $result -Alert
|
||
}
|
||
$results += $result
|
||
} catch {
|
||
$severity = if ($svc.critical) { "critical" } else { "warning" }
|
||
$result = [pscustomobject]@{
|
||
name = $name
|
||
url = $svc.url
|
||
host = $svc.host
|
||
ok = $false
|
||
severity = $severity
|
||
status = $null
|
||
expectedStatus = if ($svc.expectedStatus) { [string]$svc.expectedStatus } else { "healthy" }
|
||
output = $_.Exception.Message
|
||
}
|
||
Write-AgentLog "ai_service name=$name error=$($_.Exception.Message)"
|
||
Record-AgentEvent "ai_service_unhealthy" $severity "name=$name error=$($_.Exception.Message) url=$($svc.url)" $result -Alert
|
||
$results += $result
|
||
}
|
||
}
|
||
$results
|
||
}
|
||
|
||
function Get-HarborState {
|
||
$inspectCommand = "docker inspect --format '{{.Name}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' harbor-core redis registry"
|
||
$inspect = Invoke-SshText "192.168.0.110" $inspectCommand 30 1
|
||
$containerStates = @{}
|
||
foreach ($line in @(([string]$inspect.output) -split "`n")) {
|
||
$trimmed = $line.Trim()
|
||
if (-not $trimmed) { continue }
|
||
$parts = @($trimmed -split "\|")
|
||
if ($parts.Count -lt 3) { continue }
|
||
$name = $parts[0].Trim().TrimStart("/")
|
||
$containerStates[$name] = [pscustomobject]@{
|
||
name = $name
|
||
status = $parts[1].Trim()
|
||
health = $parts[2].Trim()
|
||
}
|
||
}
|
||
$core = $containerStates["harbor-core"]
|
||
$redis = $containerStates["redis"]
|
||
$registry = $containerStates["registry"]
|
||
$coreLine = if ($core) { "$($core.name) $($core.status) $($core.health)" } else { "harbor-core missing" }
|
||
$redisLine = if ($redis) { "$($redis.name) $($redis.status) $($redis.health)" } else { "redis missing" }
|
||
$registryLine = if ($registry) { "$($registry.name) $($registry.status) $($registry.health)" } else { "registry missing" }
|
||
|
||
$state = [pscustomobject]@{
|
||
coreHealthy = ($core -and $core.status -eq "running" -and $core.health -eq "healthy")
|
||
redisUp = ($redis -and $redis.status -eq "running")
|
||
registryHealthy = ($registry -and $registry.status -eq "running" -and $registry.health -eq "healthy")
|
||
coreOutput = $coreLine
|
||
redisOutput = $redisLine
|
||
registryOutput = $registryLine
|
||
inspectExitCode = $inspect.exitCode
|
||
healthOutput = $inspect.output
|
||
inspectOutput = "[redacted: raw docker inspect omitted]"
|
||
}
|
||
Write-AgentLog "harbor coreHealthy=$($state.coreHealthy) redisUp=$($state.redisUp) registryHealthy=$($state.registryHealthy)"
|
||
$state
|
||
}
|
||
|
||
function Get-AwoooState {
|
||
$nestedSsh = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $SshUser 192.168.0.120"
|
||
$readback = Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl get deploy,pods -n awoooi-prod" 60 1
|
||
|
||
$state = [pscustomobject]@{
|
||
deployReady = (
|
||
$readback.output -match "deployment.apps/awoooi-api\s+1/1" -and
|
||
$readback.output -match "deployment.apps/awoooi-web\s+2/2" -and
|
||
$readback.output -match "deployment.apps/awoooi-worker\s+1/1"
|
||
)
|
||
imagePullFailure = ($readback.output -match "ImagePullBackOff|ErrImagePull")
|
||
crashFailure = ($readback.output -match "CrashLoopBackOff|Pending")
|
||
deployOutput = $readback.output
|
||
podOutput = $readback.output
|
||
readbackExitCode = $readback.exitCode
|
||
}
|
||
Write-AgentLog "awoooi deployReady=$($state.deployReady) imagePullFailure=$($state.imagePullFailure) crashFailure=$($state.crashFailure)"
|
||
$state
|
||
}
|
||
|
||
function Find-Vmrun {
|
||
if ($Config.vmrunPath -and (Test-Path $Config.vmrunPath)) {
|
||
return $Config.vmrunPath
|
||
}
|
||
|
||
foreach ($candidate in @(
|
||
"C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe",
|
||
"C:\Program Files\VMware\VMware Workstation\vmrun.exe"
|
||
)) {
|
||
if (Test-Path $candidate) { return $candidate }
|
||
}
|
||
|
||
return $null
|
||
}
|
||
|
||
function Start-ConfiguredVMs {
|
||
$vmrun = Find-Vmrun
|
||
$results = @()
|
||
|
||
if (-not $vmrun) {
|
||
Write-AgentLog "vmrun missing"
|
||
return @([pscustomobject]@{ ok = $false; reason = "vmrun_missing" })
|
||
}
|
||
|
||
foreach ($vm in $Config.vms) {
|
||
if (-not $vm.vmx) {
|
||
$results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "missing_vmx_path" }
|
||
continue
|
||
}
|
||
|
||
$hostPing = $false
|
||
if ($vm.host) {
|
||
$hostPing = Test-Connection -ComputerName $vm.host -Count 1 -Quiet -ErrorAction SilentlyContinue
|
||
}
|
||
|
||
if ($hostPing) {
|
||
$results += [pscustomobject]@{ name = $vm.name; ok = $true; skipped = $true; reason = "host_already_reachable" }
|
||
continue
|
||
}
|
||
|
||
if (-not $ControlledApply) {
|
||
$results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "controlled_apply_required" }
|
||
continue
|
||
}
|
||
|
||
Write-AgentLog "CONTROLLED_APPLY vm_start name=$($vm.name) vmx=$($vm.vmx)"
|
||
$output = & $vmrun start $vm.vmx nogui 2>&1
|
||
$results += [pscustomobject]@{
|
||
name = $vm.name
|
||
ok = ($LASTEXITCODE -eq 0)
|
||
skipped = $false
|
||
output = ($output -join "`n")
|
||
}
|
||
}
|
||
|
||
$results
|
||
}
|
||
|
||
function Repair-Harbor {
|
||
param([object]$Harbor)
|
||
if (-not $ControlledApply) {
|
||
Write-AgentLog "harbor repair skipped controlled_apply=false"
|
||
return $false
|
||
}
|
||
|
||
$changed = $false
|
||
if (-not $Harbor.redisUp) {
|
||
Write-AgentLog "CONTROLLED_APPLY harbor restart redis"
|
||
Invoke-SshText "192.168.0.110" "docker restart redis" | Out-Null
|
||
$changed = $true
|
||
}
|
||
|
||
if (-not $Harbor.coreHealthy) {
|
||
Write-AgentLog "CONTROLLED_APPLY harbor restart harbor-core"
|
||
Invoke-SshText "192.168.0.110" "docker restart harbor-core" | Out-Null
|
||
$changed = $true
|
||
}
|
||
|
||
return $changed
|
||
}
|
||
|
||
function Repair-Awooo {
|
||
param([object]$Awooo, [object]$Harbor)
|
||
if (-not $ControlledApply) {
|
||
Write-AgentLog "awoooi repair skipped controlled_apply=false"
|
||
return $false
|
||
}
|
||
if (-not $Harbor.coreHealthy) {
|
||
Write-AgentLog "awoooi repair skipped harbor_core_not_healthy"
|
||
return $false
|
||
}
|
||
if (-not $Awooo.imagePullFailure) {
|
||
Write-AgentLog "awoooi repair skipped no_image_pull_failure"
|
||
return $false
|
||
}
|
||
|
||
Write-AgentLog "CONTROLLED_APPLY awoooi delete failed pods"
|
||
$nestedSsh = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $SshUser 192.168.0.120"
|
||
Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-api" 25 | Out-Null
|
||
Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-web" 25 | Out-Null
|
||
Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-worker" 25 | Out-Null
|
||
return $true
|
||
}
|
||
|
||
if ($SelfTestTelegramCard) {
|
||
Invoke-AgentTelegramCardSelfTest
|
||
}
|
||
|
||
if ($SelfTestTelegramDelivery) {
|
||
Invoke-AgentTelegramDeliverySelfTest
|
||
}
|
||
|
||
if ($SelfTestSensorGate) {
|
||
Invoke-AgentSensorGateSelfTest
|
||
}
|
||
|
||
if ($SelfTestOutcomeContract) {
|
||
Invoke-AgentOutcomeContractSelfTest
|
||
}
|
||
|
||
Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath"
|
||
Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert
|
||
|
||
$vmResults = @()
|
||
$hostResults = @()
|
||
$harbor = $null
|
||
$awooo = $null
|
||
$public = @()
|
||
$aiServices = @()
|
||
$performance = @()
|
||
$loadShedding = @()
|
||
$selfHealth = $null
|
||
$backupHealth = $null
|
||
$providerFreshness = $null
|
||
$securityTriage = $null
|
||
$controlTick = $null
|
||
$sshProcessGuard = $null
|
||
$bootState = $null
|
||
$recoveryTrigger = $null
|
||
$recoverySlo = $null
|
||
$recoveryCoordinator = $null
|
||
$recoveryPhases = @()
|
||
$recoveryStartedAt = if ($Mode -eq "Recover") { Get-Date } else { $null }
|
||
$recoveryRunId = if ($Mode -eq "Recover") { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } else { "" }
|
||
$recoveryConfig = Get-AgentRecoverySloConfig
|
||
|
||
if ($Mode -in @("Status", "Recover")) {
|
||
$bootState = Get-AgentBootState
|
||
}
|
||
if (
|
||
$Mode -eq "Status" -and
|
||
$bootState -and
|
||
($bootState.detected -or ($bootState.PSObject.Properties["pendingRecovery"] -and $bootState.pendingRecovery))
|
||
) {
|
||
$triggerReason = if ($bootState.detected) { "host_reboot_detected" } else { "host_reboot_recovery_pending" }
|
||
$recoveryTrigger = Start-AgentRecoveryFromObservation $triggerReason $bootState
|
||
}
|
||
|
||
if ($Mode -in @("Recover", "StartVMs")) {
|
||
$vmResults = Start-ConfiguredVMs
|
||
if ($Mode -eq "Recover") {
|
||
$vmFailures = @($vmResults | Where-Object { $_.ok -eq $false -and -not $_.skipped }).Count
|
||
$recoveryPhases += New-AgentRecoveryPhase "vm_start" $(if ($vmFailures -eq 0) { "ok" } else { "degraded" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
|
||
}
|
||
}
|
||
|
||
if ($Mode -in @("Status", "Recover")) {
|
||
$hostResults = Test-HostReachability
|
||
$unreachableHosts = @($hostResults | Where-Object { -not $_.ping })
|
||
if ($Mode -eq "Recover") {
|
||
$recoveryPhases += New-AgentRecoveryPhase "host_reachability" $(if ($unreachableHosts.Count -eq 0) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
|
||
}
|
||
$autoRecovery = Get-AgentAutoRecoveryConfig
|
||
if ($Mode -eq "Status" -and -not $recoveryTrigger -and $autoRecovery.triggerOnHostUnreachable -and $unreachableHosts.Count -ge $autoRecovery.minimumUnreachableHosts) {
|
||
$recoveryTrigger = Start-AgentRecoveryFromObservation "host_unreachable_detected" ([pscustomobject]@{ hosts = @($unreachableHosts.host); count = $unreachableHosts.Count })
|
||
}
|
||
}
|
||
|
||
if ($Mode -in @("Status", "Recover", "HarborRepair")) {
|
||
$harbor = Get-HarborState
|
||
}
|
||
|
||
if ($Mode -in @("Recover", "HarborRepair") -and $harbor) {
|
||
$harborChanged = Repair-Harbor $harbor
|
||
if ($harborChanged) {
|
||
Start-Sleep -Seconds 3
|
||
$harbor = Get-HarborState
|
||
}
|
||
if ($Mode -eq "Recover") {
|
||
$harborReady = [bool]($harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy)
|
||
$recoveryPhases += New-AgentRecoveryPhase "harbor_registry" $(if ($harborReady) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
|
||
}
|
||
}
|
||
|
||
if ($Mode -in @("Status", "Recover", "AwoooRepair")) {
|
||
$awooo = Get-AwoooState
|
||
}
|
||
|
||
if ($Mode -in @("Recover", "AwoooRepair") -and $awooo) {
|
||
if (-not $harbor) { $harbor = Get-HarborState }
|
||
$awoooChanged = Repair-Awooo $awooo $harbor
|
||
if ($awoooChanged) {
|
||
Start-Sleep -Seconds 3
|
||
$awooo = Get-AwoooState
|
||
}
|
||
if ($Mode -eq "Recover") {
|
||
$workloadsReady = [bool]($awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure)
|
||
$recoveryPhases += New-AgentRecoveryPhase "k3s_workloads" $(if ($workloadsReady) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
|
||
}
|
||
}
|
||
|
||
if ($Mode -in @("Status", "Recover", "PublicSmoke")) {
|
||
$public = Test-PublicRoutes
|
||
if ($Mode -eq "Recover") {
|
||
$recoveryPhases += New-AgentRecoveryPhase "public_routes" $(if (@($public | Where-Object { -not $_.ok }).Count -eq 0) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
|
||
}
|
||
}
|
||
|
||
if ($Mode -eq "Recover") {
|
||
$windowsBootAgeSeconds = -1
|
||
if ($bootState -and $bootState.ok -and $bootState.bootTime) {
|
||
try {
|
||
$windowsBootAgeSeconds = [math]::Max(0, [int64]((Get-Date) - ([datetime]$bootState.bootTime)).TotalSeconds)
|
||
} catch {
|
||
$windowsBootAgeSeconds = -1
|
||
}
|
||
}
|
||
$requireFreshRebootWindow = [bool](
|
||
($bootState -and $bootState.detected) -or
|
||
($bootState -and $bootState.PSObject.Properties["pendingRecovery"] -and $bootState.pendingRecovery) -or
|
||
($windowsBootAgeSeconds -ge 0 -and $windowsBootAgeSeconds -le ($recoveryConfig.targetMinutes * 60))
|
||
)
|
||
$coordinatorEvidenceNotBefore = $recoveryStartedAt
|
||
if ($requireFreshRebootWindow -and $bootState -and $bootState.bootTime) {
|
||
try { $coordinatorEvidenceNotBefore = [datetime]$bootState.bootTime } catch { $coordinatorEvidenceNotBefore = $recoveryStartedAt }
|
||
}
|
||
$recoveryCoordinator = Invoke-AgentRecoveryCoordinatorReadback $coordinatorEvidenceNotBefore $requireFreshRebootWindow
|
||
$coordinatorStatus = if ($recoveryCoordinator.verified) { "ok" } else { "failed" }
|
||
$coordinatorEvidence = if ($recoveryCoordinator.PSObject.Properties["artifactDir"]) { [string]$recoveryCoordinator.artifactDir } else { "" }
|
||
$recoveryPhases += New-AgentRecoveryPhase "full_sop_scorecard" $coordinatorStatus (((Get-Date) - $recoveryStartedAt).TotalSeconds) $coordinatorEvidence
|
||
}
|
||
|
||
if ($Mode -in @("Status", "Recover", "Perf")) {
|
||
$aiServices = Test-AiServices
|
||
}
|
||
|
||
if ($Mode -in @("Perf", "LoadShed") -and $ControlledApply) {
|
||
$sshProcessGuard = Invoke-AgentStaleSshProcessGuard
|
||
}
|
||
|
||
if ($Mode -in @("Status", "Recover", "Perf", "LoadShed")) {
|
||
$suppressPerfAlerts = [bool]($ControlledApply -and $Mode -in @("Perf", "LoadShed"))
|
||
$performance = Get-HostPerformance -SuppressAlerts:$suppressPerfAlerts
|
||
}
|
||
|
||
if ($Mode -in @("Perf", "LoadShed")) {
|
||
$loadShedding = Invoke-LoadShedding $performance
|
||
if ($ControlledApply) {
|
||
Record-PerformanceIssuesWithoutRemediation $performance $loadShedding
|
||
}
|
||
}
|
||
|
||
if ($Mode -in @("SelfCheck")) {
|
||
$selfHealth = Test-AgentSelfHealth
|
||
}
|
||
|
||
if ($Mode -in @("BackupCheck")) {
|
||
$backupHealth = Test-AgentBackupHealth
|
||
}
|
||
|
||
if ($Mode -in @("ProviderFreshness")) {
|
||
$providerFreshness = Invoke-AgentProviderFreshnessTriage
|
||
}
|
||
|
||
if ($Mode -in @("SecurityTriage")) {
|
||
$securityTriage = Invoke-AgentSecurityTriage
|
||
}
|
||
|
||
if ($Mode -in @("ControlTick")) {
|
||
$controlTick = Invoke-AgentControlTick
|
||
}
|
||
|
||
$publicFailures = @($public | Where-Object { $_ -and -not $_.ok }).Count
|
||
$aiFailures = @($aiServices | Where-Object { $_ -and -not $_.ok }).Count
|
||
$perfCritical = @($performance | Where-Object { $_ -and $_.severity -eq "critical" }).Count
|
||
$perfWarning = @($performance | Where-Object { $_ -and $_.severity -eq "warning" }).Count
|
||
$selfHealthCritical = if ($selfHealth -and $selfHealth.severity -eq "critical") { 1 } else { 0 }
|
||
$selfHealthWarning = if ($selfHealth -and $selfHealth.severity -eq "warning") { 1 } else { 0 }
|
||
$backupCritical = if ($backupHealth -and $backupHealth.severity -eq "critical") { 1 } else { 0 }
|
||
$backupWarning = if ($backupHealth -and $backupHealth.severity -eq "warning") { 1 } else { 0 }
|
||
$providerFreshnessWarning = if ($providerFreshness) { 1 } else { 0 }
|
||
$securityTriageWarning = if ($securityTriage -and $securityTriage.hitCount -gt 0) { 1 } else { 0 }
|
||
$summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCritical -gt 0 -or $selfHealthCritical -gt 0 -or $backupCritical -gt 0) { "critical" } elseif ($perfWarning -gt 0 -or $selfHealthWarning -gt 0 -or $backupWarning -gt 0 -or $providerFreshnessWarning -gt 0 -or $securityTriageWarning -gt 0) { "warning" } else { "info" }
|
||
if ($Mode -eq "Recover") {
|
||
$elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1)
|
||
$unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count
|
||
$builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0)
|
||
$coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified)
|
||
$requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow)
|
||
$rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed)
|
||
$recoveryCompleted = [bool]($builtInRecoveryCompleted -and $coordinatorVerified)
|
||
$withinSlo = [bool]($recoveryCompleted -and $elapsedSeconds -le ($recoveryConfig.targetMinutes * 60) -and (-not $requiresFreshWindow -or $rebootSloClaimed))
|
||
$recoveryScope = if ($requiresFreshWindow) { "full_host_reboot" } else { "service_recovery" }
|
||
$recoverySlo = [pscustomobject]@{
|
||
schemaVersion = "agent99_recovery_slo_v2"
|
||
runId = $recoveryRunId
|
||
scope = $recoveryScope
|
||
targetMinutes = $recoveryConfig.targetMinutes
|
||
startedAt = $recoveryStartedAt.ToString("o")
|
||
completedAt = (Get-Date).ToString("o")
|
||
elapsedSeconds = $elapsedSeconds
|
||
deadline = $recoveryStartedAt.AddMinutes($recoveryConfig.targetMinutes).ToString("o")
|
||
completed = $recoveryCompleted
|
||
withinSlo = $withinSlo
|
||
builtInRecoveryCompleted = $builtInRecoveryCompleted
|
||
coordinatorVerified = $coordinatorVerified
|
||
rebootSloClaimed = $rebootSloClaimed
|
||
phases = @($recoveryPhases)
|
||
}
|
||
$updatedBootState = Update-AgentBootRecoveryState $recoverySlo
|
||
if ($updatedBootState) { $bootState = $updatedBootState }
|
||
$sloSeverity = if ($withinSlo) { "info" } elseif ($recoveryCompleted) { "warning" } else { "critical" }
|
||
$sloMessage = if ($withinSlo -and $requiresFreshWindow) {
|
||
"全主機恢復完成，耗時 $elapsedSeconds 秒，fresh reboot scorecard 已證明符合 $($recoveryConfig.targetMinutes) 分鐘 SLA。"
|
||
} elseif ($withinSlo) {
|
||
"一般服務恢復完成，耗時 $elapsedSeconds 秒；本輪沒有 fresh reboot event，不宣稱全主機重啟 SLA。"
|
||
} elseif ($recoveryCompleted) {
|
||
"恢復 verifier 已通過但超過 $($recoveryConfig.targetMinutes) 分鐘目標，耗時 $elapsedSeconds 秒。"
|
||
} else {
|
||
"恢復尚未完成，耗時 $elapsedSeconds 秒；Agent99 已保留 full SOP scorecard blockers 與 evidence。"
|
||
}
|
||
Record-AgentEvent "recovery_slo_result" $sloSeverity $sloMessage $recoverySlo -Alert
|
||
}
|
||
Record-AgentEvent "agent_complete" $summarySeverity "mode=$Mode publicFailures=$publicFailures aiFailures=$aiFailures perfCritical=$perfCritical perfWarning=$perfWarning selfHealth=$($selfHealth.severity) backupHealth=$($backupHealth.severity) providerFreshness=$($providerFreshness.status) securityTriage=$($securityTriage.status) evidence=$jsonPath" $null -NoAlert
|
||
|
||
$result = [pscustomobject]@{
|
||
timestamp = (Get-Date -Format o)
|
||
mode = $Mode
|
||
controlledApply = [bool]$ControlledApply
|
||
sshProcessGuard = $sshProcessGuard
|
||
bootState = $bootState
|
||
recoveryTrigger = $recoveryTrigger
|
||
recoverySlo = $recoverySlo
|
||
recoveryCoordinator = $recoveryCoordinator
|
||
vmResults = $vmResults
|
||
hosts = $hostResults
|
||
harbor = $harbor
|
||
awoooi = $awooo
|
||
public = $public
|
||
aiServices = $aiServices
|
||
performance = $performance
|
||
loadShedding = $loadShedding
|
||
selfHealth = $selfHealth
|
||
backupHealth = $backupHealth
|
||
providerFreshness = $providerFreshness
|
||
securityTriage = $securityTriage
|
||
controlTick = $controlTick
|
||
events = $script:Events
|
||
telegram = $script:TelegramAttempts
|
||
evidenceLog = $logPath
|
||
}
|
||
|
||
$result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath
|
||
Write-AgentLog "agent99_complete json=$jsonPath"
|
||
|
||
exit 0
|