param( [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] [string]$Mode = "Status", [switch]$ControlledApply, [switch]$SelfTestTelegramCard, [switch]$SelfTestTelegramDelivery, [switch]$SelfTestSensorGate, [switch]$SelfTestOutcomeContract, [switch]$SuppressAlerts, [string]$SuppressReason = "", [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", [string]$EvidenceDir = "C:\Wooo\Agent99\evidence" ) $ErrorActionPreference = "Continue" New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $logPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.log" $jsonPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.json" function Write-AgentLog { param([string]$Message) $line = "$(Get-Date -Format o) $Message" $line | Tee-Object -FilePath $logPath -Append | Out-Null } function Load-Config { if (-not (Test-Path $ConfigPath)) { throw "Config not found: $ConfigPath" } Get-Content $ConfigPath -Raw | ConvertFrom-Json } $Config = Load-Config $SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" } $script:Events = @() $script:TelegramAttempts = @() $script:SuppressAgentAlerts = [bool]$SuppressAlerts $script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" } function Convert-AgentBool { param([object]$Value) if ($null -eq $Value) { return $false } if ($Value -is [bool]) { return [bool]$Value } $text = ([string]$Value).Trim() if (-not $text) { return $false } return [bool]($text -match "^(?i:true|1|yes|y|on)$") } function Get-HostSshUser { param([string]$TargetHost) if ($Config.sshUsers -and $Config.sshUsers.PSObject.Properties[$TargetHost]) { return [string]$Config.sshUsers.PSObject.Properties[$TargetHost].Value } $SshUser } function Send-AgentTelegramRelay { param( [string]$Message, [object]$Relay, [string]$ChatIdOverride = "", [string]$PhotoPath = "" ) if (-not ($Relay -and $Relay.enabled)) { return [pscustomobject]@{ ok = $false; error = "relay_disabled" } } $relayHost = if ($Relay.host) { [string]$Relay.host } else { "192.168.0.110" } $container = if ($Relay.container) { [string]$Relay.container } else { "stockplatform-v2-api-1" } if ($container -notmatch "^[A-Za-z0-9_.-]+$") { return [pscustomobject]@{ ok = $false; error = "invalid_relay_container" } } $messageBytes = [Text.Encoding]::UTF8.GetBytes($Message) $messageB64 = [Convert]::ToBase64String($messageBytes) $chatOverrideB64 = if ($ChatIdOverride) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($ChatIdOverride)) } else { "" } $photoSent = $false $photoError = $null $containerPhotoPath = "" if ($PhotoPath -and (Test-Path -LiteralPath $PhotoPath)) { $safePhotoName = "agent99-card-" + ([guid]::NewGuid().ToString("N")) + ".png" $remotePhotoPath = "/tmp/" + $safePhotoName $containerPhotoPath = "/tmp/" + $safePhotoName $relayUser = Get-HostSshUser $relayHost $scpArgs = @( "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "NumberOfPasswordPrompts=0", "-o", "ConnectTimeout=10", $PhotoPath, "${relayUser}@${relayHost}:$remotePhotoPath" ) try { $scp = Start-Process -FilePath "scp.exe" -ArgumentList $scpArgs -NoNewWindow -Wait -PassThru $scp.Refresh() if ($null -ne $scp.ExitCode -and $scp.ExitCode -eq 0) { $copyRun = Invoke-SshText $relayHost "docker cp $remotePhotoPath $container`:$containerPhotoPath && rm -f $remotePhotoPath" 45 1 if (-not $copyRun.ok) { $photoError = "relay_docker_cp_failed: $($copyRun.output)" $containerPhotoPath = "" } } else { $photoError = "relay_scp_failed exitCode=$($scp.ExitCode)" $containerPhotoPath = "" } } catch { $photoError = "relay_scp_exception: $($_.Exception.Message)" $containerPhotoPath = "" } } $containerPhotoB64 = if ($containerPhotoPath) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($containerPhotoPath)) } else { "" } $python = 'import base64, mimetypes, os, sys, urllib.error, urllib.parse, urllib.request text = base64.b64decode(sys.argv[2]).decode() chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] and sys.argv[3] != "-" else "" photo_path = base64.b64decode(sys.argv[4]).decode() if len(sys.argv) > 4 and sys.argv[4] and sys.argv[4] != "-" else "" token = os.environ.get("TELEGRAM_BOT_TOKEN") chat = chat_override or os.environ.get("TELEGRAM_CHAT_ID") assert token and chat if photo_path and os.path.exists(photo_path): caption = text if len(caption) > 900: caption = caption[:900] + "\n...(truncated; see evidence)" filename = os.path.basename(photo_path) ctype = mimetypes.guess_type(filename)[0] or "image/png" try: import requests with open(photo_path, "rb") as fh: response = requests.post( "https://api.telegram.org/bot" + token + "/sendPhoto", data={"chat_id": chat, "caption": caption}, files={"photo": (filename, fh, ctype)}, timeout=30, ) if not response.ok: print("photo_send_failed status=%s body=%s" % (response.status_code, response.text[:500])) sys.exit(2) print("photo_sent_ok") except ImportError: boundary = "----agent99boundary" fields = [("chat_id", chat), ("caption", caption)] body = bytearray() for name, value in fields: body.extend(("--" + boundary + "\r\n").encode()) body.extend(("Content-Disposition: form-data; name=\"" + name + "\"\r\n\r\n").encode()) body.extend(str(value).encode()) body.extend(b"\r\n") body.extend(("--" + boundary + "\r\n").encode()) body.extend(("Content-Disposition: form-data; name=\"photo\"; filename=\"" + filename + "\"\r\n").encode()) body.extend(("Content-Type: " + ctype + "\r\n\r\n").encode()) with open(photo_path, "rb") as fh: body.extend(fh.read()) body.extend(b"\r\n") body.extend(("--" + boundary + "--\r\n").encode()) req = urllib.request.Request("https://api.telegram.org/bot" + token + "/sendPhoto", data=bytes(body)) req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary) try: urllib.request.urlopen(req, timeout=30).read() print("photo_sent_ok") except urllib.error.HTTPError as exc: print("photo_send_failed status=%s body=%s" % (exc.code, exc.read().decode(errors="replace")[:500])) sys.exit(2) else: data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode() urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read() print("sent_ok") ' $pythonB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($python)) $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))' $chatOverrideArg = if ($chatOverrideB64) { $chatOverrideB64 } else { "-" } $containerPhotoArg = if ($containerPhotoB64) { $containerPhotoB64 } else { "-" } $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideArg $containerPhotoArg" $run = Invoke-SshText $relayHost $remoteCommand 45 1 $photoSent = [bool]($run.ok -and $run.output -match "photo_sent_ok") if ($containerPhotoPath) { Invoke-SshText $relayHost "docker exec $container rm -f $containerPhotoPath" 20 1 | Out-Null } [pscustomobject]@{ ok = [bool]($run.ok -and ($run.output -match "sent_ok|photo_sent_ok")) host = $relayHost container = $container exitCode = $run.exitCode chatOverride = if ($ChatIdOverride) { "telegram-origin" } else { $null } photoSent = $photoSent photoError = $photoError output = $run.output error = if ($run.ok) { $photoError } else { $run.output } } } function Get-AgentObjectValue { param( [object]$Data, [string]$Name, [object]$Default = $null ) if ($Data -and $Data.PSObject.Properties[$Name]) { $value = $Data.PSObject.Properties[$Name].Value if ($null -ne $value -and [string]$value -ne "") { return $value } } $Default } function Limit-AgentTextLine { param( [string]$Text, [int]$MaxLength = 96 ) if (-not $Text) { return "" } $clean = (($Text -replace "\s+", " ").Trim()) if ($clean.Length -le $MaxLength) { return $clean } $clean.Substring(0, $MaxLength - 14) + "...(truncated)" } function Format-AgentMetricForCard { param( [object]$Value, [string]$Suffix = "" ) if ($null -eq $Value -or [string]$Value -eq "") { return "readback_failed" } "$Value$Suffix" } function Format-AgentTelegramText { param( [string]$Severity, [string]$EventType, [string]$Message, [object]$Data = $null ) $severityLabel = switch ($Severity) { "critical" { "CRITICAL" } "warning" { "WARNING" } "info" { "INFO" } default { $Severity } } $lines = @() $lines += "Agent99 事件卡 / Incident Card" $lines += "嚴重度 Severity: $severityLabel" if ($EventType -match "^performance_") { $hostName = Get-AgentObjectValue $Data "host" "unknown" $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() } $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null $load = Get-AgentObjectValue $Data "loadPerCore" $null $mem = Get-AgentObjectValue $Data "memAvailablePercent" $null $lines += "事件 Event: 主機效能 / host performance" $lines += "主機 Host: $hostName" if ($reasons -contains "perf_readback_failed" -or ($null -eq $disk -and $null -eq $load -and $null -eq $mem)) { $lines += "原因 Reason: 效能指標 readback 失敗。" $lines += "影響 Impact: CPU load、memory、disk 數值尚未可信,不能直接當成真故障。" $lines += "Agent99 動作 Action: 維持噪音 sensor 暫停;修復前必須先補 verifier readback。" } elseif ($reasons -contains "disk_used_warning") { $lines += "原因 Reason: 磁碟使用率 $disk percent,已達 warning threshold;critical threshold 是 95 percent。" $lines += "Agent99 動作 Action: 執行 allowlisted disk remediation,並在 30 分鐘內 suppress duplicates。" } elseif ($reasons -contains "disk_used_high") { $lines += "原因 Reason: 磁碟使用率 $disk percent,已達 critical threshold。" $lines += "Agent99 動作 Action: 先執行 allowlisted cleanup;reboot 不是第一反應。" } elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") { $lines += "原因 Reason: CPU load/core=$load。" $lines += "Agent99 動作 Action: 僅使用 allowlisted load reduction;不得任意停止服務。" } elseif ($reasons -contains "memory_available_low") { $lines += "原因 Reason: 可用記憶體為 $mem percent。" $lines += "Agent99 動作 Action: 先檢查 memory pressure,再進行 allowlisted remediation。" } else { $lines += "原因 Reason: $($reasons -join ', ')" $lines += "Agent99 動作 Action: 檢查 Agent99 evidence,完成 full readback 後再處理。" } $metricParts = @() if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" } if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" } if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" } if ($metricParts.Count -gt 0) { $lines += "目前狀態 Current: $($metricParts -join ', ')." } else { $lines += "目前狀態 Current: metrics unavailable;空白指標不視為可信告警。" } } elseif ($EventType -match "^backup_") { $targetName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } $reason = if ($Data -and $Data.PSObject.Properties["reason"]) { $Data.reason } else { $EventType } $age = if ($Data -and $Data.PSObject.Properties["ageMinutes"]) { $Data.ageMinutes } else { $null } $maxAge = if ($Data -and $Data.PSObject.Properties["maxAgeMinutes"]) { $Data.maxAgeMinutes } else { $null } $lines += "事件 Event: 備份健康 / backup health" $lines += "目標 Target: $targetName" $lines += "原因 Reason: $reason" if ($age -ne $null -or $maxAge -ne $null) { $lines += "新鮮度 Freshness: age=$age minutes / max=$maxAge minutes" } $lines += "Agent99 動作 Action: 只檢查 /backup/status、/backup/logs、snapshot metadata;不得讀取備份內容或 secrets。" } elseif ($EventType -eq "provider_freshness_triage") { $candidateId = if ($Data -and $Data.PSObject.Properties["candidateId"]) { $Data.candidateId } else { "unknown" } $targetName = if ($Data -and $Data.PSObject.Properties["target"]) { $Data.target } else { "provider_freshness_signal" } $hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 } $missing = if ($Data -and $Data.PSObject.Properties["missingFields"]) { @($Data.missingFields) } else { @() } $candidatePath = if ($Data -and $Data.PSObject.Properties["candidatePath"]) { $Data.candidatePath } else { $null } $readback = if ($Data -and $Data.PSObject.Properties["verifierReadback"]) { $Data.verifierReadback } else { $null } $status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" } $lines += "事件 Event: Provider freshness 自動判讀 / provider freshness auto triage" $lines += "目標 Target: $targetName" $lines += "狀態 Status: $status" $lines += "影響 Impact: source freshness 未證明前,dashboard green 不可視為健康。" $lines += "證據命中 Evidence hits: $hitCount" if ($readback) { $lines += "驗證 Verifier: directReadback ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)" } if (@($missing).Count -gt 0) { $lines += "仍缺欄位 Still missing: $($missing -join ', ')" } else { $lines += "缺漏狀態 Missing: direct readback fields completed" } $lines += "Agent99 動作 Action: 已寫入 candidate 與 KM/RAG;相同狀態 duplicate alerts 會被 suppress。" $lines += "限制 Guard: 不切 provider、不呼叫 paid model、不改 env、不 reload。" if ($candidatePath) { $lines += "證據 Evidence: $candidatePath" } } elseif ($EventType -match "^load_shed_") { $actionName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } $hostName = if ($Data -and $Data.PSObject.Properties["host"]) { $Data.host } else { "unknown" } $ok = if ($Data -and $Data.PSObject.Properties["ok"]) { $Data.ok } else { $null } $executionOk = if ($Data -and $Data.PSObject.Properties["executionOk"]) { $Data.executionOk } else { $null } $postVerifyOk = if ($Data -and $Data.PSObject.Properties["postVerifyOk"]) { $Data.postVerifyOk } else { $null } $exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null } $rollback = if ($Data -and $Data.PSObject.Properties["rollback"]) { $Data.rollback } else { $null } $lines += "事件 Event: 受控修復 / controlled remediation" $lines += "主機 Host: $hostName" $lines += "Agent99 動作 Action: $actionName" if ($ok -ne $null) { $lines += "結果 Result: ok=$ok exitCode=$exitCode" if ($executionOk -ne $null -or $postVerifyOk -ne $null) { $lines += "驗證 Verifier: executionOk=$executionOk postVerifyOk=$postVerifyOk" } } else { $lines += "結果 Result: 請查看 Agent99 evidence。" } if ($rollback) { $lines += "回滾 Rollback: $rollback" } } elseif ($EventType -match "^operator_command_") { $commandId = if ($Data -and $Data.PSObject.Properties["id"]) { $Data.id } else { "unknown" } $modeName = if ($Data -and $Data.PSObject.Properties["mode"]) { $Data.mode } else { "unknown" } $source = if ($Data -and $Data.PSObject.Properties["source"]) { $Data.source } else { "unknown" } $instruction = if ($Data -and $Data.PSObject.Properties["instruction"]) { $Data.instruction } else { $null } $exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null } $evidencePath = if ($Data -and $Data.PSObject.Properties["evidence"]) { $Data.evidence } else { $null } $outcome = if ($Data -and $Data.PSObject.Properties["outcome"]) { $Data.outcome } else { $null } $outcomeState = if ($outcome -and $outcome.PSObject.Properties["state"]) { [string]$outcome.state } else { "failed" } $resultLabel = switch ($outcomeState) { "resolved" { "已驗證解決 / verified resolved" } "candidate_ready" { "修復候選已建立 / candidate ready" } "verifying" { "等待來源事件驗證 / verifying" } "degraded" { "仍有異常 / degraded" } "blocked" { "受阻擋 / blocked" } default { "執行失敗 / failed" } } $actionText = switch ($modeName) { "ProviderFreshness" { "建立 provider freshness candidate,要求 provider window、last seen、direct reference、verifier readback;不切 provider。" } "BackupCheck" { "讀回備份 freshness、status/log、snapshot metadata、cron contract;不讀備份內容或 secrets。" } "Perf" { "讀回主機 CPU/記憶體/磁碟;必要時只做 allowlisted 降載。" } "LoadShed" { "執行 allowlisted 降載並做 post-apply verifier。" } "Recover" { "依序檢查 VM、SSH、Harbor/K3s/public route,必要時執行受控恢復。" } "StartVMs" { "檢查並啟動 allowlisted VMware VM。" } "HarborRepair" { "檢查 Harbor core/redis/registry,必要時執行受控修復。" } "AwoooRepair" { "檢查 AWOOOI/K3s pod、deployment、image pull 狀態,必要時受控修復。" } "SecurityTriage" { "建立資安 triage candidate,分類 blast radius、affected host/service、source log reference,不任意 shell、不暴露 secrets。" } default { "執行 allowlisted Agent99 mode 並寫入 evidence/KM。" } } $lines += "事件 Event: Agent99 自動化結果 / automation result" $lines += "模式 Mode: $modeName" $lines += "來源 Source: $source" $lines += "結果 Result: $resultLabel, transportExitCode=$exitCode" if ($outcome) { $lines += "驗證 Verifier: name=$($outcome.verifierName), passed=$($outcome.verifierPassed), sourceEventResolved=$($outcome.sourceEventResolved)" } $lines += "Agent99 動作 Action: $actionText" if ($Data -and $Data.PSObject.Properties["problem"]) { $problem = $Data.problem if ($problem -and $problem.PSObject.Properties["occurrences"]) { $lines += "問題計數 Problem count: occurrences=$($problem.occurrences), verifiedResolved=$($problem.verifiedResolved), failed=$($problem.failed), degraded=$($problem.degraded), blocked=$($problem.blocked), verifying=$($problem.verifying), candidateReady=$($problem.candidateReady)" if ($problem.PSObject.Properties["kmPath"]) { $lines += "KM: $($problem.kmPath)" } } } if ($instruction -and $exitCode -ne 0) { $lines += "失敗指令 Failure instruction: $instruction" } elseif ($instruction) { $lines += "處理摘要 Summary: $(Limit-AgentTextLine $instruction 130)" } if ($evidencePath) { $lines += "證據 Evidence: $evidencePath" } if ($exitCode -eq 0) { $lines += "下一步 Next: 若相同問題再發生,依 problem count 追蹤重複率並回寫 PlayBook/KM。" } else { $lines += "下一步 Next: 保留 failed queue/evidence,升級到對應 playbook,不做破壞性動作。" } } elseif ($EventType -eq "agent_control_tick") { $dashboardPath = if ($Data -and $Data.PSObject.Properties["dashboardPath"]) { $Data.dashboardPath } else { $null } $processed = if ($Data -and $Data.PSObject.Properties["processedCount"]) { $Data.processedCount } else { 0 } $pending = if ($Data -and $Data.PSObject.Properties["pendingCount"]) { $Data.pendingCount } else { 0 } $lines += "事件 Event: Agent99 控制迴圈 / control loop" $lines += "儀表板 Dashboard: $dashboardPath" $lines += "佇列 Queue: processed=$processed pending=$pending" } elseif ($EventType -eq "telegram_delivery_test") { $testId = if ($Data -and $Data.PSObject.Properties["testId"]) { $Data.testId } else { "unknown" } $lines += "事件 Event: Telegram 低噪音送達測試 / delivery test" $lines += "測試 Test: TEST_DO_NOT_PAGE" $lines += "測試 ID Test ID: $testId" $lines += "影響 Impact: 無使用者影響;此訊息只驗證 Agent99 繁中事件卡與 TG delivery。" $lines += "Agent99 動作 Action: 驗證送達後寫入 evidence;不觸發修復、不恢復 sensor、不建立事故。" } elseif ($EventType -match "^ai_service_") { $serviceName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } $lines += "事件 Event: AI 服務健康 / AI service health" $lines += "服務 Service: $serviceName" $lines += "Agent99 動作 Action: 先檢查 health endpoint 與 service/container state,再做 controlled repair。" } elseif ($EventType -eq "security_triage") { $status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" } $hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 } $evidencePath = if ($Data -and $Data.PSObject.Properties["evidencePath"]) { $Data.evidencePath } else { $null } $lines += "事件 Event: 資安判讀 / security triage" $lines += "狀態 Status: $status" $lines += "證據命中 Evidence hits: $hitCount" $lines += "Agent99 動作 Action: 分類 blast radius、affected host/service、source log reference、repeated count、allowlisted next action。" $lines += "限制 Guard: 不執行 arbitrary shell、不刪資料、不停用 security controls、不暴露 secrets。" if ($evidencePath) { $lines += "證據 Evidence: $evidencePath" } } elseif ($EventType -eq "agent_selfcheck") { $lines += "事件 Event: Agent99 自我健康 / self health" $lines += "摘要 Summary: $Message" $lines += "Agent99 動作 Action: 檢查 task state、evidence freshness、Telegram delivery;reboot 不是第一反應。" } else { $lines += "事件 Event: $EventType" $lines += "摘要 Summary: $Message" } $lines += "時間 Time: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')" $text = ($lines -join "`n") if ($text.Length -gt 3500) { $text = $text.Substring(0, 3500) + "`n...(truncated)" } $text } function Get-AgentTelegramVisualPath { param([object]$Data) if (-not $Data) { return $null } foreach ($propName in @("visualPath", "cardImagePath", "imagePath", "screenshotPath")) { if ($Data.PSObject.Properties[$propName]) { $path = [string]$Data.PSObject.Properties[$propName].Value if ($path -and (Test-Path -LiteralPath $path)) { return $path } } } $null } function Test-AgentSreOperatorResultAlertEnabled { param([object]$Data) if (-not $Data) { return $false } $source = [string](Get-AgentObjectValue $Data "source" "") $alertId = [string](Get-AgentObjectValue $Data "alertId" "") $enabled = $true if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSreCommandResults"]) { $enabled = Convert-AgentBool $Config.telegram.alertSreCommandResults } [bool]($enabled -and $source -eq "agent99-sre-alert-inbox" -and $alertId) } function New-AgentTelegramCardImage { param( [string]$Severity, [string]$EventType, [string]$Text, [object]$Data = $null ) $visualCardsEnabled = $true if ($Config.telegram -and $Config.telegram.PSObject.Properties["visualCardsEnabled"]) { $visualCardsEnabled = [bool]$Config.telegram.visualCardsEnabled } if (-not $visualCardsEnabled) { return $null } $forceVisual = [bool]($EventType -match "^operator_command_" -and (Test-AgentSreOperatorResultAlertEnabled $Data)) if (($Severity -notin @("warning", "critical")) -and -not $forceVisual) { return $null } $cardDir = Join-Path $EvidenceDir "telegram-cards" New-Item -ItemType Directory -Force -Path $cardDir | Out-Null $safeEvent = ($EventType -replace "[^A-Za-z0-9_.-]", "-") $imagePath = Join-Path $cardDir "agent99-card-$safeEvent-$stamp.png" try { Add-Type -AssemblyName System.Drawing -ErrorAction Stop $width = 1040 $height = 620 $bitmap = New-Object System.Drawing.Bitmap -ArgumentList $width, $height $graphics = [System.Drawing.Graphics]::FromImage($bitmap) $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias $graphics.TextRenderingHint = [System.Drawing.Text.TextRenderingHint]::ClearTypeGridFit $bgBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(246, 248, 250)) $panelBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(255, 255, 255)) $textBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(28, 35, 43)) $mutedBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(91, 103, 112)) $accentColor = if ($Severity -eq "critical") { [System.Drawing.Color]::FromArgb(190, 18, 60) } elseif ($Severity -eq "warning") { [System.Drawing.Color]::FromArgb(217, 119, 6) } else { [System.Drawing.Color]::FromArgb(37, 99, 235) } $accentBrush = New-Object System.Drawing.SolidBrush -ArgumentList $accentColor $fontFamily = "Microsoft JhengHei UI" $titleFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 25, ([System.Drawing.FontStyle]::Bold) $labelFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 13, ([System.Drawing.FontStyle]::Bold) $bodyFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 16, ([System.Drawing.FontStyle]::Regular) $smallFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 11, ([System.Drawing.FontStyle]::Regular) $graphics.FillRectangle($bgBrush, 0, 0, $width, $height) $graphics.FillRectangle($panelBrush, 36, 34, $width - 72, $height - 68) $graphics.FillRectangle($accentBrush, 36, 34, 14, $height - 68) $graphics.DrawString("Agent99 事件卡 / Incident Card", $titleFont, $textBrush, 72, 58) $graphics.DrawString(("嚴重度 Severity: " + $Severity.ToUpperInvariant()), $labelFont, $accentBrush, 72, 106) $graphics.DrawString(("事件 Event: " + (Limit-AgentTextLine $EventType 76)), $labelFont, $mutedBrush, 72, 134) $bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 事件卡 / Incident Card$" } | Select-Object -First 12) $y = 184 foreach ($line in $bodyLines) { $graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y) $y += 32 if ($y -gt 545) { break } } $graphics.DrawString(("產生時間 Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, 570) $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png) $graphics.Dispose() $bitmap.Dispose() $bgBrush.Dispose() $panelBrush.Dispose() $textBrush.Dispose() $mutedBrush.Dispose() $accentBrush.Dispose() $titleFont.Dispose() $labelFont.Dispose() $bodyFont.Dispose() $smallFont.Dispose() return $imagePath } catch { Write-AgentLog "telegram_card_image_failed event=$EventType error=$($_.Exception.Message)" return $null } } function Send-AgentTelegramPhotoDirect { param( [string]$Token, [string]$ChatId, [string]$Caption, [string]$PhotoPath ) Add-Type -AssemblyName System.Net.Http -ErrorAction Stop $client = New-Object System.Net.Http.HttpClient $content = New-Object System.Net.Http.MultipartFormDataContent $fileContent = $null try { $content.Add([System.Net.Http.StringContent]::new($ChatId), "chat_id") $captionToSend = $Caption if ($captionToSend.Length -gt 900) { $captionToSend = $captionToSend.Substring(0, 900) + "`n...(truncated; see evidence)" } $content.Add([System.Net.Http.StringContent]::new($captionToSend), "caption") $fileBytes = [IO.File]::ReadAllBytes($PhotoPath) $fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes) $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png") $content.Add($fileContent, "photo", [IO.Path]::GetFileName($PhotoPath)) $response = $client.PostAsync("https://api.telegram.org/bot$Token/sendPhoto", $content).GetAwaiter().GetResult() $body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() if (-not $response.IsSuccessStatusCode) { throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body" } } finally { if ($fileContent) { $fileContent.Dispose() } if ($content) { $content.Dispose() } if ($client) { $client.Dispose() } } } function Invoke-AgentTelegramCardSelfTest { $sample = [pscustomobject]@{ host = "192.168.0.110" reasons = @("perf_readback_failed") evidencePath = $jsonPath } $text = Format-AgentTelegramText "warning" "performance_warning" "selftest visual card" $sample $imagePath = New-AgentTelegramCardImage "warning" "performance_warning" $text $ok = [bool]($imagePath -and (Test-Path -LiteralPath $imagePath)) $operatorSample = [pscustomobject]@{ id = "selftest-sre-result" mode = "ProviderFreshness" source = "agent99-sre-alert-inbox" reason = "sre_alert" instruction = "Provider freshness candidate required; alertId=selftest-provider; require providerWindow,lastSeen,directReference,verifierReadback; do not switch provider." alertId = "selftest-provider" alertKind = "provider_freshness_signal" alertSeverity = "warning" alertService = "provider_freshness_signal" ok = $true exitCode = 0 evidence = $jsonPath outcome = [pscustomobject]@{ state = "resolved" verifierName = "selftest-verifier" verifierPassed = $true sourceEventResolved = $true } problem = [pscustomobject]@{ key = "selftest-provider-freshness" occurrences = 2 resolved = 2 verifiedResolved = 2 failed = 0 degraded = 0 blocked = 0 verifying = 0 candidateReady = 0 kmPath = (Join-Path $Config.agentRoot "playbooks\agent99-incident-km.md") } } $operatorText = Format-AgentTelegramText "info" "operator_command_result" "selftest SRE completion card" $operatorSample $operatorImagePath = New-AgentTelegramCardImage "info" "operator_command_result" $operatorText $operatorSample $operatorDecisionOk = Test-AgentSreOperatorResultAlertEnabled $operatorSample $operatorTextOk = [bool]( $operatorText -match "已驗證解決 / verified resolved" -and $operatorText -match "Agent99 動作 Action" -and $operatorText -match "問題計數 Problem count" -and $operatorText -match "KM:" ) $operatorVisualOk = [bool]($operatorImagePath -and (Test-Path -LiteralPath $operatorImagePath)) $ok = [bool]($ok -and $operatorTextOk -and $operatorVisualOk -and $operatorDecisionOk) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $ok textPreview = @($text -split "`n" | Select-Object -First 12) visualPath = $imagePath visualExists = $ok operatorTextPreview = @($operatorText -split "`n" | Select-Object -First 14) operatorVisualPath = $operatorImagePath operatorVisualExists = $operatorVisualOk operatorTextOk = $operatorTextOk operatorDecisionOk = $operatorDecisionOk sentTelegram = $false messageFormat = "incident_card_zh_tw_v4" } $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramCardSelfTest-$stamp.json" $result | ConvertTo-Json -Depth 6 | Set-Content -Path $selfTestPath -Encoding UTF8 Write-AgentLog "telegram_card_selftest ok=$ok path=$imagePath evidence=$selfTestPath" $result | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath $result | ConvertTo-Json -Depth 6 if (-not $ok) { exit 1 } exit 0 } function Invoke-AgentTelegramDeliverySelfTest { $testId = "telegram-delivery-test-" + (Get-Date -Format "yyyyMMdd-HHmmss") $data = [pscustomobject]@{ testId = $testId visualPath = $null } Send-AgentTelegram "warning" "telegram_delivery_test" "TEST_DO_NOT_PAGE bilingual Telegram incident card delivery self-test" $data $sent = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $sent testId = $testId sentTelegram = $sent messageFormat = "incident_card_zh_tw_v4" attempts = $script:TelegramAttempts } $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramDeliverySelfTest-$stamp.json" $result | ConvertTo-Json -Depth 10 | Set-Content -Path $selfTestPath -Encoding UTF8 Write-AgentLog "telegram_delivery_selftest ok=$sent evidence=$selfTestPath" $result | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 $jsonPath $result | ConvertTo-Json -Depth 10 if (-not $sent) { exit 1 } exit 0 } function Invoke-AgentSensorGateSelfTest { $previousSuppress = $script:SuppressAgentAlerts $script:SuppressAgentAlerts = $true $gateError = $null $performance = @() $backupHealth = $null $selfHealth = $null try { Write-AgentLog "sensor_gate start no_telegram=true controlled_apply=false remediation=false" $performance = @(Get-HostPerformance) $backupHealth = Test-AgentBackupHealth $selfHealth = Test-AgentSelfHealth } catch { $gateError = $_.Exception.Message Write-AgentLog "sensor_gate error=$gateError" } finally { $script:SuppressAgentAlerts = $previousSuppress } $perfReadbackFailed = @($performance | Where-Object { $_ -and ((-not $_.ok) -or (@($_.reasons) -contains "perf_readback_failed")) }) $perfMissingMetrics = @($performance | Where-Object { $_ -and $_.ok -and (($null -eq $_.loadPerCore) -or ($null -eq $_.memAvailablePercent) -or ($null -eq $_.diskUsedPercent)) }) $perfCritical = @($performance | Where-Object { $_ -and $_.severity -eq "critical" }) $perfWarning = @($performance | Where-Object { $_ -and $_.severity -eq "warning" }) $sentTelegram = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0) $suppressedTelegram = @($script:TelegramAttempts | Where-Object { $_.PSObject.Properties["suppressed"] -and $_.suppressed }) $perfReadbackOk = [bool](@($performance).Count -gt 0 -and @($perfReadbackFailed).Count -eq 0 -and @($perfMissingMetrics).Count -eq 0) $backupOk = [bool]($backupHealth -and $backupHealth.severity -eq "ok") $selfOk = [bool]($selfHealth -and $selfHealth.severity -eq "ok") $performanceDecision = if (-not $perfReadbackOk) { "hold_readback_or_metric_gap" } elseif (@($perfCritical).Count -gt 0) { "hold_critical_needs_remediation" } elseif (@($perfWarning).Count -gt 0) { "hold_warning_needs_summary_or_remediation" } else { "eligible" } $backupDecision = if ($backupOk) { "eligible" } elseif ($backupHealth) { "hold_$($backupHealth.severity)" } else { "hold_readback_failed" } $selfDecision = if ($selfOk) { "eligible" } elseif ($selfHealth) { "hold_$($selfHealth.severity)" } else { "hold_readback_failed" } $readyToEnableAll = [bool]($performanceDecision -eq "eligible" -and $backupDecision -eq "eligible" -and $selfDecision -eq "eligible") $gateSafe = [bool]((-not $sentTelegram) -and (-not $ControlledApply) -and ($null -eq $gateError)) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $gateSafe readyToEnableAll = $readyToEnableAll sentTelegram = $sentTelegram suppressedTelegramCount = @($suppressedTelegram).Count controlledApply = [bool]$ControlledApply ranRemediation = $false error = $gateError decisions = [pscustomobject]@{ performanceGuard = $performanceDecision backupHealth = $backupDecision selfHealth = $selfDecision } performance = [pscustomobject]@{ ok = $perfReadbackOk hostCount = @($performance).Count readbackFailedHosts = @($perfReadbackFailed | ForEach-Object { $_.host }) missingMetricHosts = @($perfMissingMetrics | ForEach-Object { $_.host }) warningHosts = @($perfWarning | ForEach-Object { $_.host }) criticalHosts = @($perfCritical | ForEach-Object { $_.host }) rows = $performance } backupHealth = $backupHealth selfHealth = $selfHealth telegramAttempts = $script:TelegramAttempts events = $script:Events evidenceLog = $logPath } $selfTestPath = Join-Path $EvidenceDir "agent99-SensorGateSelfTest-$stamp.json" $result | ConvertTo-Json -Depth 12 | Set-Content -Path $selfTestPath -Encoding UTF8 $result | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 $jsonPath Write-AgentLog "sensor_gate complete ok=$($result.ok) readyToEnableAll=$readyToEnableAll evidence=$selfTestPath" $result | ConvertTo-Json -Depth 12 if (-not $gateSafe) { exit 1 } exit 0 } function Send-AgentTelegram { param( [string]$Severity, [string]$EventType, [string]$Message, [object]$Data = $null ) $telegram = $Config.telegram $enabled = ($telegram -and $telegram.enabled) $botTokenEnv = if ($telegram.botTokenEnv) { $telegram.botTokenEnv } else { "AGENT99_TELEGRAM_BOT_TOKEN" } $chatIdEnv = if ($telegram.chatIdEnv) { $telegram.chatIdEnv } else { "AGENT99_TELEGRAM_CHAT_ID" } $tokenEnvAliases = @($botTokenEnv) if ($telegram.botTokenEnvAliases) { $tokenEnvAliases += @($telegram.botTokenEnvAliases) } $chatIdEnvAliases = @($chatIdEnv) if ($telegram.chatIdEnvAliases) { $chatIdEnvAliases += @($telegram.chatIdEnvAliases) } function Get-AgentEnvSecret { param([string[]]$Names) foreach ($name in ($Names | Where-Object { $_ } | Select-Object -Unique)) { foreach ($scope in @("Machine", "User", "Process")) { $value = [Environment]::GetEnvironmentVariable($name, $scope) if ($value) { return [pscustomobject]@{ value = $value; name = $name; scope = $scope } } } } return $null } $tokenSecret = Get-AgentEnvSecret $tokenEnvAliases $chatSecret = Get-AgentEnvSecret $chatIdEnvAliases $token = if ($tokenSecret) { $tokenSecret.value } else { $null } $chatId = if ($chatSecret) { $chatSecret.value } else { $null } $chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" } $attempt = [pscustomobject]@{ timestamp = (Get-Date -Format o) enabled = [bool]$enabled eventType = $EventType severity = $Severity messageFormat = "incident_card_zh_tw_v3" configured = [bool]($token -and $chatId) tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null } chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null } chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null } visualPath = $null visualSent = $false visualError = $null relay = $null sent = $false error = $null } if (-not $enabled) { $attempt.error = "telegram_disabled" $script:TelegramAttempts += $attempt return } $text = Format-AgentTelegramText $Severity $EventType $Message $Data $visualPath = Get-AgentTelegramVisualPath $Data if (-not $visualPath) { $visualPath = New-AgentTelegramCardImage $Severity $EventType $text $Data } $attempt.visualPath = $visualPath if (-not ($token -and ($chatId -or $chatIdOverride))) { $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride $visualPath $attempt.relay = $relayResult if ($relayResult.ok) { $attempt.sent = $true $attempt.error = $null $attempt.visualSent = [bool]$relayResult.photoSent if ($visualPath -and -not $relayResult.photoSent) { $attempt.visualError = if ($relayResult.photoError) { $relayResult.photoError } else { "relay_sendPhoto_not_available" } } } else { $attempt.error = "telegram_not_configured" } $script:TelegramAttempts += $attempt return } try { $targetChat = if ($chatIdOverride) { $chatIdOverride } else { $chatId } if ($visualPath -and (Test-Path -LiteralPath $visualPath)) { try { Send-AgentTelegramPhotoDirect $token $targetChat $text $visualPath $attempt.sent = $true $attempt.visualSent = $true } catch { $attempt.visualError = $_.Exception.Message Write-AgentLog "telegram_sendPhoto_failed event=$EventType fallback=sendMessage error=$($_.Exception.Message)" Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ chat_id = $targetChat text = $text disable_web_page_preview = "true" } | Out-Null $attempt.sent = $true } } else { Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ chat_id = $targetChat text = $text disable_web_page_preview = "true" } | Out-Null $attempt.sent = $true } } catch { $attempt.error = $_.Exception.Message } $script:TelegramAttempts += $attempt } function Record-AgentEvent { param( [string]$EventType, [string]$Severity, [string]$Message, [object]$Data = $null, [switch]$Alert, [switch]$NoAlert ) Write-AgentLog "event type=$EventType severity=$Severity message=$Message" $eventData = $Data if ($Data -and $Data.PSObject.Properties["replyChatId"]) { $eventData = $Data | Select-Object * -ExcludeProperty replyChatId $eventData | Add-Member -MemberType NoteProperty -Name replyChat -Value "telegram-origin" -Force } $event = [pscustomobject]@{ timestamp = (Get-Date -Format o) type = $EventType severity = $Severity message = $Message data = $eventData } $script:Events += $event if ($NoAlert) { return } $eventSuppressTelegram = $false if ($Data -and $Data.PSObject.Properties["suppressTelegram"]) { $eventSuppressTelegram = Convert-AgentBool $Data.suppressTelegram } if ($script:SuppressAgentAlerts -or $eventSuppressTelegram) { $suppressReason = if ($eventSuppressTelegram) { "do_not_page" } elseif ($script:SuppressAgentAlertReason) { $script:SuppressAgentAlertReason } else { "sensor_gate_no_telegram" } Write-AgentLog "telegram_suppressed reason=$suppressReason type=$EventType severity=$Severity" $script:TelegramAttempts += [pscustomobject]@{ timestamp = (Get-Date -Format o) eventType = $EventType severity = $Severity sent = $false suppressed = $true reason = $suppressReason } return } $alertAll = ($Config.telegram -and $Config.telegram.alertAllOperations) $alertInfo = ($Config.telegram -and $Config.telegram.alertInfoEvents) $alertOperatorCommands = $true if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertOperatorCommands"]) { $alertOperatorCommands = [bool]$Config.telegram.alertOperatorCommands } $operatorCommandAlert = [bool]($alertOperatorCommands -and $EventType -match "^operator_command_") if ($operatorCommandAlert -and $Data) { $operatorOk = $false $operatorMode = "" $operatorId = "" $operatorSource = "" $operatorAlertId = "" $operatorOkValue = $null $operatorExitCode = $null if ($Data -is [System.Collections.IDictionary]) { if ($Data.Contains("mode")) { $operatorMode = [string]$Data["mode"] } if ($Data.Contains("id")) { $operatorId = [string]$Data["id"] } if ($Data.Contains("source")) { $operatorSource = [string]$Data["source"] } if ($Data.Contains("alertId")) { $operatorAlertId = [string]$Data["alertId"] } if ($Data.Contains("ok")) { $operatorOkValue = $Data["ok"] } if ($Data.Contains("exitCode")) { $operatorExitCode = $Data["exitCode"] } } else { $modeProp = $Data.PSObject.Properties.Match("mode") | Select-Object -First 1 $idProp = $Data.PSObject.Properties.Match("id") | Select-Object -First 1 $sourceProp = $Data.PSObject.Properties.Match("source") | Select-Object -First 1 $alertIdProp = $Data.PSObject.Properties.Match("alertId") | Select-Object -First 1 $okProp = $Data.PSObject.Properties.Match("ok") | Select-Object -First 1 $exitCodeProp = $Data.PSObject.Properties.Match("exitCode") | Select-Object -First 1 if ($modeProp) { $operatorMode = [string]$modeProp.Value } if ($idProp) { $operatorId = [string]$idProp.Value } if ($sourceProp) { $operatorSource = [string]$sourceProp.Value } if ($alertIdProp) { $operatorAlertId = [string]$alertIdProp.Value } if ($okProp) { $operatorOkValue = $okProp.Value } if ($exitCodeProp) { $operatorExitCode = $exitCodeProp.Value } } if ($null -ne $operatorOkValue -and [string]$operatorOkValue -match "^(?i:true|1)$") { $operatorOk = $true } $parsedExitCode = -999 if ($null -ne $operatorExitCode -and [int]::TryParse([string]$operatorExitCode, [ref]$parsedExitCode) -and $parsedExitCode -eq 0) { $operatorOk = $true } $alertSuccessfulOperatorCommands = $false if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSuccessfulOperatorCommands"]) { $alertSuccessfulOperatorCommands = [bool]$Config.telegram.alertSuccessfulOperatorCommands } $operatorIsSreAlertResult = Test-AgentSreOperatorResultAlertEnabled $Data if ($operatorOk -and -not $alertSuccessfulOperatorCommands -and -not $operatorIsSreAlertResult) { Write-AgentLog "telegram_operator_success_suppressed type=$EventType mode=$operatorMode id=$operatorId" $operatorCommandAlert = $false } elseif ($operatorOk -and $operatorIsSreAlertResult) { Write-AgentLog "telegram_sre_operator_success_enabled type=$EventType mode=$operatorMode id=$operatorId alertId=$operatorAlertId" } } $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo)) if ($shouldAlert) { $dedupeMinutes = 30 if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) { $dedupeMinutes = [int]$Config.telegram.dedupeMinutes } $dedupeParts = @($EventType, $Severity) if ($Data) { foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source")) { if ($Data.PSObject.Properties[$propName]) { $dedupeParts += "$propName=$($Data.PSObject.Properties[$propName].Value)" } } if ($Data.PSObject.Properties["reasons"]) { $dedupeParts += "reasons=$(@($Data.reasons) -join ',')" } if ($Data.PSObject.Properties["missingFields"]) { $dedupeParts += "missing=$(@($Data.missingFields) -join ',')" } if ($Data.PSObject.Properties["problem"] -and $Data.problem -and $Data.problem.PSObject.Properties["key"]) { $dedupeParts += "problem=$($Data.problem.key)" } } if ($dedupeParts.Count -le 2) { $normalizedMessage = $Message foreach ($pattern in @("candidate=[^ ]+", "evidence=[^ ]+", "loadPerCore=[^ ]+", "memAvailPct=[^ ]+", "diskUsedPct=[^ ]+", "ageMinutes=[^ ]+")) { $normalizedMessage = $normalizedMessage -replace $pattern, ($pattern.Split("=")[0] + "=") } $dedupeParts += $normalizedMessage } $dedupeSource = ($dedupeParts -join "|") $dedupeHashBytes = [Security.Cryptography.SHA256]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($dedupeSource)) $dedupeHash = [BitConverter]::ToString($dedupeHashBytes).Replace("-", "").ToLowerInvariant() $dedupePath = Join-Path $EvidenceDir "telegram-dedupe-$dedupeHash.marker" if ($dedupeMinutes -gt 0 -and (Test-Path $dedupePath)) { $ageMinutes = ((Get-Date) - (Get-Item $dedupePath).LastWriteTime).TotalMinutes if ($ageMinutes -lt $dedupeMinutes) { Write-AgentLog "telegram_dedupe_skip type=$EventType severity=$Severity ageMinutes=$([math]::Round($ageMinutes, 2)) cooldown=$dedupeMinutes" return } } Set-Content -Path $dedupePath -Value (Get-Date -Format o) -Encoding UTF8 Send-AgentTelegram $Severity $EventType $Message $Data } } function Invoke-SshText { param( [string]$TargetHost, [string]$Command, [int]$TimeoutSeconds = 12, [int]$Retries = 2, [string]$User = $null ) $last = $null $userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost } for ($attempt = 1; $attempt -le $Retries; $attempt++) { $args = @( "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "NumberOfPasswordPrompts=0", "-o", "ConnectTimeout=$TimeoutSeconds", "-o", "ServerAliveInterval=20", "-o", "ServerAliveCountMax=3", "-l", $userToUse, $TargetHost, $Command ) try { $stopwatch = [Diagnostics.Stopwatch]::StartNew() $id = [guid]::NewGuid().ToString("N") $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out" $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err" $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { $stopwatch.Stop() $process.Kill() $process.WaitForExit(2000) | Out-Null Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null $last = [pscustomobject]@{ ok = $false exitCode = -2 output = "timeout_after_${TimeoutSeconds}s" elapsedMs = $stopwatch.ElapsedMilliseconds } } else { $process.WaitForExit() | Out-Null $stopwatch.Stop() $process.Refresh() $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" } $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" } Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null $text = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" $exitCode = $process.ExitCode if ($null -eq $exitCode) { $exitCode = if ($stderr) { -3 } else { 0 } } $last = [pscustomobject]@{ ok = ($exitCode -eq 0) exitCode = $exitCode output = $text elapsedMs = $stopwatch.ElapsedMilliseconds } } } catch { $last = [pscustomobject]@{ ok = $false exitCode = -1 output = $_.Exception.Message elapsedMs = if ($stopwatch) { $stopwatch.ElapsedMilliseconds } else { $null } } } if ($last.ok) { if ($attempt -gt 1) { Write-AgentLog "ssh_retry_recovered host=$TargetHost attempt=$attempt" } return $last } if ($attempt -lt $Retries) { Write-AgentLog "ssh_retry host=$TargetHost attempt=$attempt exit=$($last.exitCode)" Start-Sleep -Seconds 2 } } $last } function Quote-ShSingle { param([string]$Text) "'" + $Text.Replace("'", "'`"`"'") + "'" } function Invoke-HostSshText { param( [string]$TargetHost, [string]$Command, [int]$TimeoutSeconds = 30, [int]$Retries = 1 ) $direct = Invoke-SshText $TargetHost $Command $TimeoutSeconds $Retries if ($direct.ok -or -not $Config.k3s.jumpHost -or $TargetHost -eq $Config.k3s.jumpHost) { $direct | Add-Member -NotePropertyName route -NotePropertyValue "direct" -Force return $direct } $quotedCommand = Quote-ShSingle $Command $targetUser = Get-HostSshUser $TargetHost $jumpCommand = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=10 -l $targetUser $TargetHost $quotedCommand" $viaJump = Invoke-SshText $Config.k3s.jumpHost $jumpCommand $TimeoutSeconds $Retries $viaJump | Add-Member -NotePropertyName route -NotePropertyValue "via_$($Config.k3s.jumpHost)" -Force return $viaJump } function Convert-AgentDouble { param([string]$Value) if (-not $Value) { return $null } try { return [double]::Parse($Value.Replace(",", "."), [Globalization.CultureInfo]::InvariantCulture) } catch { return $null } } function Get-PerfThresholds { $perf = $Config.performance [pscustomobject]@{ loadPerCoreWarning = if ($perf.loadPerCoreWarning) { [double]$perf.loadPerCoreWarning } else { 0.9 } loadPerCoreCritical = if ($perf.loadPerCoreCritical) { [double]$perf.loadPerCoreCritical } else { 1.5 } memoryAvailablePercentCritical = if ($perf.memoryAvailablePercentCritical) { [double]$perf.memoryAvailablePercentCritical } else { 10 } diskUsedPercentWarning = if ($perf.diskUsedPercentWarning) { [double]$perf.diskUsedPercentWarning } else { 90 } diskUsedPercentCritical = if ($perf.diskUsedPercentCritical) { [double]$perf.diskUsedPercentCritical } else { 95 } } } function Get-HostPerformance { param([switch]$SuppressAlerts) $thresholds = Get-PerfThresholds $readTimeoutSeconds = 45 if ($Config.performance -and $Config.performance.PSObject.Properties["readTimeoutSeconds"]) { $readTimeoutSeconds = [int]$Config.performance.readTimeoutSeconds } $readRetries = 1 if ($Config.performance -and $Config.performance.PSObject.Properties["readRetries"]) { $readRetries = [int]$Config.performance.readRetries } $results = @() $command = @' echo __AGENT99_PERF__ echo HOST=$(hostname) echo CORES=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1) uptime echo __MEM__ free -m 2>/dev/null echo __DISK__ df -P / 2>/dev/null '@ foreach ($hostIp in $Config.hosts) { $hostTimeoutSeconds = $readTimeoutSeconds if ($Config.performance -and $Config.performance.PSObject.Properties["hostReadTimeoutSeconds"] -and $Config.performance.hostReadTimeoutSeconds.PSObject.Properties[$hostIp]) { $hostTimeoutSeconds = [int]$Config.performance.hostReadTimeoutSeconds.PSObject.Properties[$hostIp].Value } $slowRetryUsed = $false $slowTimeoutSeconds = $null $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries if (-not $readback.ok -and $readback.exitCode -eq -2) { $slowTimeoutSeconds = 90 if ($Config.performance -and $Config.performance.PSObject.Properties["slowReadTimeoutSeconds"]) { $slowTimeoutSeconds = [int]$Config.performance.slowReadTimeoutSeconds } if ($Config.performance -and $Config.performance.PSObject.Properties["hostSlowReadTimeoutSeconds"] -and $Config.performance.hostSlowReadTimeoutSeconds.PSObject.Properties[$hostIp]) { $slowTimeoutSeconds = [int]$Config.performance.hostSlowReadTimeoutSeconds.PSObject.Properties[$hostIp].Value } if ($slowTimeoutSeconds -gt $hostTimeoutSeconds) { Write-AgentLog "perf slow_retry host=$hostIp firstTimeoutSeconds=$hostTimeoutSeconds slowTimeoutSeconds=$slowTimeoutSeconds" $slowReadback = Invoke-HostSshText $hostIp $command $slowTimeoutSeconds 1 $slowReadback | Add-Member -NotePropertyName firstAttemptExitCode -NotePropertyValue $readback.exitCode -Force $slowReadback | Add-Member -NotePropertyName firstAttemptElapsedMs -NotePropertyValue $readback.elapsedMs -Force $readback = $slowReadback $slowRetryUsed = $true } } $text = $readback.output $cores = 1 $coresMatch = [regex]::Match($text, "CORES=(\d+)") if ($coresMatch.Success) { $cores = [int]$coresMatch.Groups[1].Value } $load1 = $null $loadMatch = [regex]::Match($text, "load averages?:\s*([0-9]+(?:[\.,][0-9]+)?)") if (-not $loadMatch.Success) { $loadMatch = [regex]::Match($text, "load average:\s*([0-9]+(?:[\.,][0-9]+)?)") } if ($loadMatch.Success) { $load1 = Convert-AgentDouble $loadMatch.Groups[1].Value } $memAvailablePercent = $null $memLine = (($text -split "`n") | Where-Object { $_ -match "^\s*Mem:" } | Select-Object -First 1) if ($memLine) { $parts = @($memLine -split "\s+" | Where-Object { $_ }) if ($parts.Count -ge 7) { $memTotal = Convert-AgentDouble $parts[1] $memAvailable = Convert-AgentDouble $parts[6] if ($memTotal -and $memTotal -gt 0 -and $null -ne $memAvailable) { $memAvailablePercent = [math]::Round(($memAvailable / $memTotal) * 100, 2) } } } $diskUsedPercent = $null $diskLine = (($text -split "`n") | Where-Object { $_ -match "\s+\d+%\s+/$" } | Select-Object -First 1) if ($diskLine) { $diskMatch = [regex]::Match($diskLine, "\s(\d+)%\s+/$") if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } } $topCpu = @() $loadPerCore = if ($null -ne $load1 -and $cores -gt 0) { [math]::Round($load1 / $cores, 2) } else { $null } $severity = "ok" $reasons = @() if (-not $readback.ok) { $severity = "warning" $reasons += "perf_readback_failed" } if ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreCritical) { $severity = "critical" $reasons += "load_per_core_critical" } elseif ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreWarning -and $severity -ne "critical") { $severity = "warning" $reasons += "load_per_core_warning" } if ($null -ne $memAvailablePercent -and $memAvailablePercent -le $thresholds.memoryAvailablePercentCritical) { $severity = "critical" $reasons += "memory_available_low" } if ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentCritical) { $severity = "critical" $reasons += "disk_used_high" } elseif ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentWarning -and $severity -ne "critical") { $severity = "warning" $reasons += "disk_used_warning" } $captureTopCpuOnWarning = $true if ($Config.performance -and $Config.performance.PSObject.Properties["captureTopCpuOnWarning"]) { $captureTopCpuOnWarning = [bool]$Config.performance.captureTopCpuOnWarning } $topCpuTimeoutSeconds = 10 if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuTimeoutSeconds"]) { $topCpuTimeoutSeconds = [int]$Config.performance.topCpuTimeoutSeconds } $shouldCaptureTopCpu = ($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning) if ($shouldCaptureTopCpu -and $readback.ok) { $topReadback = Invoke-HostSshText $hostIp "ps -eo pid,ppid,comm,pcpu,pmem --sort=-pcpu" $topCpuTimeoutSeconds 1 if ($topReadback.ok) { $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) } else { $topCpu = @("top_cpu_readback_failed: $($topReadback.output)") } } $result = [pscustomobject]@{ host = $hostIp ok = [bool]$readback.ok route = $readback.route severity = $severity reasons = $reasons cores = $cores load1 = $load1 loadPerCore = $loadPerCore memAvailablePercent = $memAvailablePercent diskUsedPercent = $diskUsedPercent topCpu = $topCpu output = $text readTimeoutSeconds = $hostTimeoutSeconds slowReadTimeoutSeconds = $slowTimeoutSeconds readbackSlowRetry = $slowRetryUsed readbackElapsedMs = $readback.elapsedMs } Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) { Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert } $results += $result } $results } function Record-PerformanceIssuesWithoutRemediation { param( [object[]]$Performance, [object[]]$LoadShedding ) foreach ($perf in @($Performance | Where-Object { $_ -and $_.severity -in @("warning", "critical") })) { $attempts = @($LoadShedding | Where-Object { $_ -and $_.host -eq $perf.host }) $executed = @($attempts | Where-Object { $_.PSObject.Properties["skipped"] -and $_.skipped -eq $false }) if ($executed.Count -gt 0) { continue } $message = "host=$($perf.host) loadPerCore=$($perf.loadPerCore) memAvailPct=$($perf.memAvailablePercent) diskUsedPct=$($perf.diskUsedPercent) reasons=$(@($perf.reasons) -join ',')" Record-AgentEvent "performance_$($perf.severity)" $perf.severity $message $perf -Alert } } function Invoke-LoadShedding { param([object[]]$Performance) $results = @() $loadShedding = $Config.performance.loadShedding if (-not ($loadShedding -and $loadShedding.enabled)) { Write-AgentLog "load_shedding skipped enabled=false" return $results } if (-not $ControlledApply) { Write-AgentLog "load_shedding skipped controlled_apply=false" return $results } foreach ($action in @($loadShedding.actions)) { if ($action.enabled -eq $false) { continue } $hostPerf = $Performance | Where-Object { $_.host -eq $action.host } | Select-Object -First 1 if (-not $hostPerf) { continue } $severityRank = @{ ok = 0; warning = 1; critical = 2 } $minSeverity = if ($action.minSeverity) { [string]$action.minSeverity } else { "critical" } if (-not $severityRank.ContainsKey($minSeverity)) { $minSeverity = "critical" } $hostSeverity = if ($hostPerf.severity -and $severityRank.ContainsKey([string]$hostPerf.severity)) { [string]$hostPerf.severity } else { "ok" } if ($severityRank[$hostSeverity] -lt $severityRank[$minSeverity]) { $results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "host_below_min_severity"; hostSeverity = $hostSeverity; minSeverity = $minSeverity } continue } $reasonMatches = @($action.reasonMatches) if ($reasonMatches.Count -eq 0) { $reasonMatches = @("load_per_core_critical") } $matchedReasons = @($hostPerf.reasons | Where-Object { $_ -in $reasonMatches }) if ($matchedReasons.Count -eq 0) { $results += [pscustomobject]@{ name = $action.name host = $action.host skipped = $true reason = "critical_reason_not_allowlisted" hostReasons = $hostPerf.reasons allowedReasons = $reasonMatches } continue } $cooldownMinutes = if ($action.cooldownMinutes) { [int]$action.cooldownMinutes } else { 30 } $safeName = ($action.name -replace "[^A-Za-z0-9_.-]", "_") $markerPath = Join-Path $EvidenceDir "loadshedding-$safeName.marker" if (Test-Path $markerPath) { $ageMinutes = ((Get-Date) - (Get-Item $markerPath).LastWriteTime).TotalMinutes if ($ageMinutes -lt $cooldownMinutes) { $results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "cooldown"; ageMinutes = [math]::Round($ageMinutes, 1) } continue } } Record-AgentEvent "load_shed_attempt" $hostSeverity "host=$($action.host) action=$($action.name) command=$($action.command)" $action -NoAlert $timeoutSeconds = if ($action.PSObject.Properties["timeoutSeconds"]) { [int]$action.timeoutSeconds } else { 60 } $run = Invoke-HostSshText $action.host $action.command $timeoutSeconds 1 $postVerify = $null $postVerifyOk = $false if ($action.PSObject.Properties["postVerifyCommand"] -and $action.postVerifyCommand) { $verifyRun = Invoke-HostSshText $action.host ([string]$action.postVerifyCommand) 30 1 $postVerify = [pscustomobject]@{ ok = [bool]$verifyRun.ok exitCode = $verifyRun.exitCode output = $verifyRun.output route = $verifyRun.route } if ($verifyRun.ok) { $postVerifyOk = $true if ($action.PSObject.Properties["postVerifyMaxDiskUsedPercent"]) { $match = [regex]::Match([string]$verifyRun.output, "__DISK_USED__=(\d+)") if ($match.Success) { $postVerifyOk = ([int]$match.Groups[1].Value -le [int]$action.postVerifyMaxDiskUsedPercent) } else { $postVerifyOk = $false } } if ($action.PSObject.Properties["postVerifyOkRegex"] -and $action.postVerifyOkRegex) { $postVerifyOk = [bool]([string]$verifyRun.output -match [string]$action.postVerifyOkRegex) } } } $effectiveOk = [bool]($run.ok -or $postVerifyOk) Set-Content -Path $markerPath -Value (Get-Date -Format o) -Encoding UTF8 $result = [pscustomobject]@{ name = $action.name host = $action.host skipped = $false ok = $effectiveOk executionOk = [bool]$run.ok postVerifyOk = [bool]$postVerifyOk postVerify = $postVerify route = $run.route exitCode = $run.exitCode output = $run.output rollback = $action.rollback } Record-AgentEvent "load_shed_result" $(if ($effectiveOk) { "warning" } else { "critical" }) "host=$($action.host) action=$($action.name) ok=$effectiveOk executionOk=$($run.ok) postVerifyOk=$postVerifyOk" $result -Alert $results += $result } $results } function Get-AgentBooleanSetting { param( [object]$Object, [string]$Name, [bool]$Default ) if ($Object -and $Object.PSObject.Properties[$Name]) { return [bool]$Object.PSObject.Properties[$Name].Value } return $Default } function Get-AgentSelfHealthConfig { $selfHealth = $Config.selfHealth $defaultTasks = @( "Wooo-Agent99-Startup-Recovery", "Wooo-Agent99-Heartbeat", "Wooo-Agent99-Performance-Guard", "Wooo-Agent99-Control-Loop", "Wooo-Agent99-Telegram-Inbox", "Wooo-Agent99-SRE-Alert-Inbox", "Wooo-Agent99-Self-Health", "Wooo-Agent99-Backup-Health" ) $defaultEvidence = @( [pscustomobject]@{ name = "performance_guard"; pattern = "agent99-Perf-*.json"; maxAgeMinutes = 5; required = $true }, [pscustomobject]@{ name = "heartbeat_status"; pattern = "agent99-Status-*.json"; maxAgeMinutes = 15; required = $true }, [pscustomobject]@{ name = "self_health"; pattern = "agent99-SelfCheck-*.json"; maxAgeMinutes = 15; required = $false }, [pscustomobject]@{ name = "backup_health"; pattern = "agent99-BackupCheck-*.json"; maxAgeMinutes = 30; required = $false }, [pscustomobject]@{ name = "startup_recovery"; pattern = "agent99-Recover-*.json"; maxAgeMinutes = 1440; required = $false }, [pscustomobject]@{ name = "telegram_inbox"; pattern = "agent99-TelegramInbox-*.json"; maxAgeMinutes = 15; required = $false }, [pscustomobject]@{ name = "sre_alert_inbox"; pattern = "agent99-SreAlertInbox-*.json"; maxAgeMinutes = 15; required = $false } ) [pscustomobject]@{ scheduledTasks = if ($selfHealth -and $selfHealth.scheduledTasks) { @($selfHealth.scheduledTasks) } else { $defaultTasks } evidenceFreshness = if ($selfHealth -and $selfHealth.evidenceFreshness) { @($selfHealth.evidenceFreshness) } else { $defaultEvidence } requiredHosts = if ($selfHealth -and $selfHealth.requiredHosts) { @($selfHealth.requiredHosts) } else { @($Config.hosts) } requireRecentTelegramDelivery = Get-AgentBooleanSetting $selfHealth "requireRecentTelegramDelivery" $true } } function Get-AgentLatestEvidence { param([string]$Pattern) $file = Get-ChildItem -Path $EvidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 if (-not $file) { return [pscustomobject]@{ exists = $false path = $null fileName = $null lastWriteTime = $null ageMinutes = $null data = $null parseError = $null } } $data = $null $parseError = $null try { $data = Get-Content $file.FullName -Raw | ConvertFrom-Json } catch { $parseError = $_.Exception.Message } [pscustomobject]@{ exists = $true path = $file.FullName fileName = $file.Name lastWriteTime = $file.LastWriteTime.ToString("o") ageMinutes = [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 2) data = $data parseError = $parseError } } function Test-AgentScheduledTaskHealth { param([string[]]$TaskNames) $results = @() foreach ($taskName in $TaskNames) { try { $task = Get-ScheduledTask -TaskName $taskName -ErrorAction Stop $info = Get-ScheduledTaskInfo -TaskName $taskName -ErrorAction Stop $state = [string]$task.State $lastTaskResult = $info.LastTaskResult $pendingFirstRun = ($lastTaskResult -eq 267011) $ok = ($state -eq "Running") -or (($state -eq "Ready") -and ($lastTaskResult -in @(0, 267011))) $results += [pscustomobject]@{ name = $taskName exists = $true ok = [bool]$ok state = $state lastTaskResult = $lastTaskResult pendingFirstRun = [bool]$pendingFirstRun lastRunTime = if ($info.LastRunTime) { $info.LastRunTime.ToString("o") } else { $null } nextRunTime = if ($info.NextRunTime) { $info.NextRunTime.ToString("o") } else { $null } } Write-AgentLog "selfcheck task name=$taskName ok=$ok state=$state lastTaskResult=$lastTaskResult pendingFirstRun=$pendingFirstRun" } catch { $results += [pscustomobject]@{ name = $taskName exists = $false ok = $false state = $null lastTaskResult = $null lastRunTime = $null nextRunTime = $null error = $_.Exception.Message } Write-AgentLog "selfcheck task_missing name=$taskName error=$($_.Exception.Message)" } } $results } function Test-AgentEvidenceFreshness { param([object[]]$Contracts) $results = @() foreach ($contract in $Contracts) { $name = if ($contract.name) { [string]$contract.name } else { [string]$contract.pattern } $pattern = [string]$contract.pattern $maxAgeMinutes = if ($contract.maxAgeMinutes) { [double]$contract.maxAgeMinutes } else { 15 } $required = Get-AgentBooleanSetting $contract "required" $true $latest = Get-AgentLatestEvidence $pattern $fresh = [bool]($latest.exists -and $null -eq $latest.parseError -and $latest.ageMinutes -le $maxAgeMinutes) $ok = [bool](((-not $required) -and (-not $latest.exists)) -or $fresh) $severity = if ($ok) { "ok" } elseif ($required) { "critical" } else { "warning" } $results += [pscustomobject]@{ name = $name pattern = $pattern required = [bool]$required ok = $ok severity = $severity latestPath = $latest.path latestFile = $latest.fileName lastWriteTime = $latest.lastWriteTime ageMinutes = $latest.ageMinutes maxAgeMinutes = $maxAgeMinutes parseError = $latest.parseError } Write-AgentLog "selfcheck evidence name=$name ok=$ok severity=$severity ageMinutes=$($latest.ageMinutes) maxAgeMinutes=$maxAgeMinutes file=$($latest.fileName)" } $results } function Test-AgentLatestPerformanceEvidence { param([string[]]$RequiredHosts) $latest = Get-AgentLatestEvidence "agent99-Perf-*.json" $hostChecks = @() if (-not $latest.exists) { return [pscustomobject]@{ ok = $false severity = "critical" reason = "missing_perf_evidence" latestPath = $null latestFile = $null ageMinutes = $null hosts = $hostChecks } } if ($latest.parseError) { return [pscustomobject]@{ ok = $false severity = "critical" reason = "perf_evidence_parse_error" latestPath = $latest.path latestFile = $latest.fileName ageMinutes = $latest.ageMinutes parseError = $latest.parseError hosts = $hostChecks } } $perf = @($latest.data.performance) foreach ($hostIp in $RequiredHosts) { $entry = $perf | Where-Object { $_.host -eq $hostIp } | Select-Object -First 1 $hostSeverity = if (-not $entry) { "missing" } elseif ($entry.severity) { [string]$entry.severity } elseif ($entry.ok -eq $false) { "warning" } else { "ok" } $ok = [bool]($hostSeverity -eq "ok") $hostChecks += [pscustomobject]@{ host = $hostIp present = [bool]$entry ok = $ok severity = $hostSeverity loadPerCore = if ($entry) { $entry.loadPerCore } else { $null } memAvailablePercent = if ($entry) { $entry.memAvailablePercent } else { $null } diskUsedPercent = if ($entry) { $entry.diskUsedPercent } else { $null } reasons = if ($entry) { @($entry.reasons) } else { @("missing_perf_host") } } } $missingOrCritical = @($hostChecks | Where-Object { $_.severity -in @("missing", "critical") }).Count $warning = @($hostChecks | Where-Object { $_.severity -eq "warning" }).Count $severity = if ($missingOrCritical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } [pscustomobject]@{ ok = [bool]($severity -eq "ok") severity = $severity reason = if ($severity -eq "ok") { "all_required_hosts_ok" } else { "required_host_perf_not_ok" } latestPath = $latest.path latestFile = $latest.fileName ageMinutes = $latest.ageMinutes hosts = $hostChecks } } function Test-AgentRecentTelegramDelivery { param([object[]]$Contracts) $checks = @() $lookbackMinutes = 60 if ($Config.selfHealth -and $Config.selfHealth.PSObject.Properties["telegramDeliveryLookbackMinutes"]) { $lookbackMinutes = [int]$Config.selfHealth.telegramDeliveryLookbackMinutes } $cutoff = (Get-Date).AddMinutes(-1 * $lookbackMinutes) foreach ($contract in $Contracts) { $pattern = [string]$contract.pattern $files = Get-ChildItem -Path $EvidenceDir -Filter $pattern -File -ErrorAction SilentlyContinue | Where-Object { $_.LastWriteTime -ge $cutoff } | Sort-Object LastWriteTime -Descending | Select-Object -First 20 foreach ($file in @($files)) { $data = $null try { $data = Get-Content $file.FullName -Raw | ConvertFrom-Json } catch { continue } if (-not $data) { continue } $attempts = @($data.telegram) if ($attempts.Count -eq 0) { continue } $sent = @($attempts | Where-Object { $_.sent -eq $true -or ($_.relay -and $_.relay.ok -eq $true) }) $checks += [pscustomobject]@{ name = if ($contract.name) { [string]$contract.name } else { [string]$contract.pattern } latestPath = $file.FullName latestFile = $file.Name ageMinutes = [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 2) attempts = $attempts.Count sent = $sent.Count ok = [bool]($sent.Count -gt 0) } } } $attemptCount = 0 $sentCount = 0 foreach ($check in @($checks)) { $attemptCount += [int]$check.attempts $sentCount += [int]$check.sent } if ($attemptCount -eq 0) { return [pscustomobject]@{ ok = $true severity = "ok" reason = "no_recent_alert_attempts" checks = $checks } } [pscustomobject]@{ ok = [bool]($sentCount -gt 0) severity = if ($sentCount -gt 0) { "ok" } else { "critical" } reason = if ($sentCount -gt 0) { "recent_alert_delivery_ok" } else { "recent_alert_delivery_failed" } checks = $checks } } function Test-AgentRuntimeManifest { $manifestPath = Join-Path $Config.agentRoot "state\runtime-manifest.json" if (-not (Test-Path $manifestPath)) { return [pscustomobject]@{ ok = $false severity = "critical" reason = "runtime_manifest_missing" manifestPath = $manifestPath sourceRevision = $null driftCount = 0 missingCount = 1 files = @() } } try { $manifest = Get-Content $manifestPath -Raw | ConvertFrom-Json } catch { return [pscustomobject]@{ ok = $false severity = "critical" reason = "runtime_manifest_parse_failed" manifestPath = $manifestPath sourceRevision = $null driftCount = 0 missingCount = 0 files = @() error = $_.Exception.Message } } $rows = @() foreach ($entry in @($manifest.files)) { $name = [string]$entry.name $runtimePath = Join-Path $PSScriptRoot $name $exists = Test-Path $runtimePath $actualSha256 = $null if ($exists) { try { $actualSha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $runtimePath).Hash.ToLowerInvariant() } catch { $actualSha256 = $null } } $expectedSha256 = if ($entry.runtimeSha256) { ([string]$entry.runtimeSha256).ToLowerInvariant() } elseif ($entry.sourceSha256) { ([string]$entry.sourceSha256).ToLowerInvariant() } else { "" } $matched = [bool]($exists -and $actualSha256 -and $expectedSha256 -and $actualSha256 -eq $expectedSha256) $rows += [pscustomobject]@{ name = $name exists = $exists matched = $matched expectedSha256 = $expectedSha256 actualSha256 = $actualSha256 } } $missingCount = @($rows | Where-Object { -not $_.exists }).Count $driftCount = @($rows | Where-Object { $_.exists -and -not $_.matched }).Count $ready = [bool](@($rows).Count -gt 0 -and $missingCount -eq 0 -and $driftCount -eq 0 -and $manifest.runtimeMatched -eq $true) [pscustomobject]@{ ok = $ready severity = if ($ready) { "ok" } else { "critical" } reason = if ($ready) { "runtime_manifest_matched" } else { "runtime_source_drift_or_missing" } manifestPath = $manifestPath sourceRevision = $manifest.sourceRevision generatedAt = $manifest.generatedAt driftCount = $driftCount missingCount = $missingCount files = $rows } } function Test-AgentSelfHealth { $selfConfig = Get-AgentSelfHealthConfig $tasks = Test-AgentScheduledTaskHealth $selfConfig.scheduledTasks $evidence = Test-AgentEvidenceFreshness $selfConfig.evidenceFreshness $latestPerf = Test-AgentLatestPerformanceEvidence $selfConfig.requiredHosts $runtimeManifest = Test-AgentRuntimeManifest $telegram = if ($selfConfig.requireRecentTelegramDelivery) { Test-AgentRecentTelegramDelivery $selfConfig.evidenceFreshness } else { [pscustomobject]@{ ok = $true; severity = "ok"; checks = @(); skipped = $true } } $taskFailures = @($tasks | Where-Object { -not $_.ok }).Count $evidenceCritical = @($evidence | Where-Object { $_.severity -eq "critical" }).Count $evidenceWarning = @($evidence | Where-Object { $_.severity -eq "warning" }).Count $critical = $taskFailures + $evidenceCritical if ($latestPerf.severity -eq "critical") { $critical += 1 } if ($telegram.severity -eq "critical") { $critical += 1 } if ($runtimeManifest.severity -eq "critical") { $critical += 1 } $warning = $evidenceWarning if ($latestPerf.severity -eq "warning") { $warning += 1 } $severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } $message = "selfHealth severity=$severity taskFailures=$taskFailures evidenceCritical=$evidenceCritical evidenceWarning=$evidenceWarning perf=$($latestPerf.severity) telegram=$($telegram.severity) runtimeManifest=$($runtimeManifest.severity)" Record-AgentEvent "agent_selfcheck" $(if ($severity -eq "ok") { "info" } else { $severity }) $message $null -Alert [pscustomobject]@{ ok = [bool]($severity -eq "ok") severity = $severity tasks = $tasks evidenceFreshness = $evidence latestPerformance = $latestPerf telegramDelivery = $telegram runtimeManifest = $runtimeManifest summary = [pscustomobject]@{ taskFailures = $taskFailures evidenceCritical = $evidenceCritical evidenceWarning = $evidenceWarning performanceSeverity = $latestPerf.severity telegramSeverity = $telegram.severity runtimeManifestSeverity = $runtimeManifest.severity runtimeManifestSourceRevision = $runtimeManifest.sourceRevision runtimeManifestDriftCount = $runtimeManifest.driftCount } } } function Get-AgentUnixAgeMinutes { param([double]$UnixSeconds) $now = [DateTimeOffset]::Now.ToUnixTimeSeconds() [math]::Round(($now - $UnixSeconds) / 60, 2) } function Convert-AgentUnixToIso { param([double]$UnixSeconds) try { [DateTimeOffset]::FromUnixTimeSeconds([int64][math]::Floor($UnixSeconds)).LocalDateTime.ToString("o") } catch { $null } } function Get-AgentBackupHealthConfig { $backupHealth = $Config.backupHealth $backupHost = if ($backupHealth -and $backupHealth.host) { [string]$backupHealth.host } else { "192.168.0.110" } $targets = @( [pscustomobject]@{ name = "awoooi"; path = "/backup/awoooi/snapshots"; maxAgeMinutes = 430; required = $true }, [pscustomobject]@{ name = "gitea"; path = "/backup/gitea/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "harbor"; path = "/backup/harbor/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "momo"; path = "/backup/momo/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "langfuse"; path = "/backup/langfuse/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "monitoring"; path = "/backup/monitoring/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "signoz"; path = "/backup/signoz/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "clawbot"; path = "/backup/clawbot/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "open-webui"; path = "/backup/open-webui/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "sentry"; path = "/backup/sentry/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "ai-artifacts"; path = "/backup/ai-artifacts/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "configs"; path = "/backup/configs/snapshots"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "public-routes"; path = "/backup/public-routes/snapshots"; maxAgeMinutes = 10080; required = $false } ) $fileChecks = @( [pscustomobject]@{ name = "backup_log"; path = "/backup/logs/backup.log"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "backup_status_log"; path = "/backup/logs/backup-status.log"; maxAgeMinutes = 1800; required = $true }, [pscustomobject]@{ name = "status_log"; path = "/backup/logs/status.log"; maxAgeMinutes = 10080; required = $false }, [pscustomobject]@{ name = "cron_log"; path = "/backup/logs/cron.log"; maxAgeMinutes = 10080; required = $false }, [pscustomobject]@{ name = "integrity_check_status"; path = "/backup/integrity/check.status"; maxAgeMinutes = 10080; required = $true }, [pscustomobject]@{ name = "restore_drill_status"; path = "/backup/integrity/restore-drill.status"; maxAgeMinutes = 45000; required = $false }, [pscustomobject]@{ name = "configs_last_status"; path = "/backup/status/backup-configs-last-status.json"; maxAgeMinutes = 1800; required = $false } ) $cronPatterns = @( "backup-all.sh", "backup-awoooi-frequent.sh", "backup-health-textfile-exporter.py", "check-backup-integrity.sh", "sync-offsite-backups.sh --mode status", "backup-status.sh" ) [pscustomobject]@{ host = $backupHost targets = if ($backupHealth -and $backupHealth.targets) { @($backupHealth.targets) } else { $targets } fileChecks = if ($backupHealth -and $backupHealth.fileChecks) { @($backupHealth.fileChecks) } else { $fileChecks } cronPatterns = if ($backupHealth -and $backupHealth.cronPatterns) { @($backupHealth.cronPatterns) } else { $cronPatterns } } } function Test-AgentBackupSnapshotTarget { param( [object]$Target, [string]$DefaultHost ) $name = [string]$Target.name $hostIp = if ($Target.host) { [string]$Target.host } else { $DefaultHost } $path = [string]$Target.path $required = Get-AgentBooleanSetting $Target "required" $true $maxAgeMinutes = if ($Target.maxAgeMinutes) { [double]$Target.maxAgeMinutes } else { 1800 } if ($path -notmatch "^/backup/[A-Za-z0-9_.\/-]+$") { return [pscustomobject]@{ name = $name; host = $hostIp; path = $path; ok = $false; severity = "critical"; reason = "invalid_backup_path" } } $quotedPath = Quote-ShSingle $path $command = "if [ -d $quotedPath ]; then echo __COUNT__=`$(find $quotedPath -maxdepth 1 -type f 2>/dev/null | wc -l); find $quotedPath -maxdepth 1 -type f -printf '__LATEST__=%T@|%TY-%Tm-%TdT%TH:%TM:%TS|%s|%p\n' 2>/dev/null | sort -n | tail -1; else echo __MISSING_DIR__=$quotedPath; fi" $timeoutSeconds = 60 if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["snapshotReadTimeoutSeconds"]) { $timeoutSeconds = [int]$Config.backupHealth.snapshotReadTimeoutSeconds } if ($Target.PSObject.Properties["timeoutSeconds"]) { $timeoutSeconds = [int]$Target.timeoutSeconds } $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1 $count = 0 $latestUnix = $null $latestSize = $null $latestPath = $null if ($readback.ok) { $countMatch = [regex]::Match($readback.output, "__COUNT__=(\d+)") if ($countMatch.Success) { $count = [int]$countMatch.Groups[1].Value } $latestMatch = [regex]::Match($readback.output, "__LATEST__=([0-9.]+)\|([^|]+)\|(\d+)\|(.+)") if ($latestMatch.Success) { $latestUnix = [double]::Parse($latestMatch.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture) $latestSize = [int64]$latestMatch.Groups[3].Value $latestPath = $latestMatch.Groups[4].Value.Trim() } } $ageMinutes = if ($null -ne $latestUnix) { Get-AgentUnixAgeMinutes $latestUnix } else { $null } $severity = "ok" $reason = "fresh" if (-not $readback.ok) { $severity = if ($required) { "critical" } else { "warning" } $reason = "backup_readback_failed" } elseif ($readback.output -match "__MISSING_DIR__" -or $count -eq 0 -or $null -eq $latestUnix) { $severity = if ($required) { "critical" } else { "warning" } $reason = "backup_snapshot_missing" } elseif ($ageMinutes -gt $maxAgeMinutes) { $severity = if ($required) { "critical" } else { "warning" } $reason = "backup_snapshot_stale" } $result = [pscustomobject]@{ name = $name host = $hostIp path = $path required = [bool]$required ok = [bool]($severity -eq "ok") severity = $severity reason = $reason count = $count latestPath = $latestPath latestSizeBytes = $latestSize latestTime = if ($null -ne $latestUnix) { Convert-AgentUnixToIso $latestUnix } else { $null } ageMinutes = $ageMinutes maxAgeMinutes = $maxAgeMinutes route = $readback.route exitCode = $readback.exitCode } Write-AgentLog "backup target=$name severity=$severity reason=$reason ageMinutes=$ageMinutes count=$count" if ($severity -in @("warning", "critical")) { Record-AgentEvent "backup_$reason" $severity "target=$name host=$hostIp ageMinutes=$ageMinutes maxAgeMinutes=$maxAgeMinutes path=$path" $result -Alert } $result } function Test-AgentBackupFileCheck { param( [object]$FileCheck, [string]$DefaultHost ) $name = [string]$FileCheck.name $hostIp = if ($FileCheck.host) { [string]$FileCheck.host } else { $DefaultHost } $path = [string]$FileCheck.path $required = Get-AgentBooleanSetting $FileCheck "required" $true $maxAgeMinutes = if ($FileCheck.maxAgeMinutes) { [double]$FileCheck.maxAgeMinutes } else { 1800 } if ($path -notmatch "^/backup/[A-Za-z0-9_.\/-]+$") { return [pscustomobject]@{ name = $name; host = $hostIp; path = $path; ok = $false; severity = "critical"; reason = "invalid_backup_file_path" } } $quotedPath = Quote-ShSingle $path $command = "if [ -f $quotedPath ]; then stat -c '__STAT__=%Y|%y|%s|%n' $quotedPath; else echo __MISSING_FILE__=$quotedPath; fi" $timeoutSeconds = 45 if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["fileReadTimeoutSeconds"]) { $timeoutSeconds = [int]$Config.backupHealth.fileReadTimeoutSeconds } if ($FileCheck.PSObject.Properties["timeoutSeconds"]) { $timeoutSeconds = [int]$FileCheck.timeoutSeconds } $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1 $unix = $null $size = $null if ($readback.ok) { $statMatch = [regex]::Match($readback.output, "__STAT__=([0-9]+)\|([^|]+)\|(\d+)\|(.+)") if ($statMatch.Success) { $unix = [double]$statMatch.Groups[1].Value $size = [int64]$statMatch.Groups[3].Value } } $ageMinutes = if ($null -ne $unix) { Get-AgentUnixAgeMinutes $unix } else { $null } $severity = "ok" $reason = "fresh" if (-not $readback.ok) { $severity = if ($required) { "critical" } else { "warning" } $reason = "backup_file_readback_failed" } elseif ($readback.output -match "__MISSING_FILE__" -or $null -eq $unix) { $severity = if ($required) { "critical" } else { "warning" } $reason = "backup_file_missing" } elseif ($ageMinutes -gt $maxAgeMinutes) { $severity = if ($required) { "critical" } else { "warning" } $reason = "backup_file_stale" } $result = [pscustomobject]@{ name = $name host = $hostIp path = $path required = [bool]$required ok = [bool]($severity -eq "ok") severity = $severity reason = $reason sizeBytes = $size lastWriteTime = if ($null -ne $unix) { Convert-AgentUnixToIso $unix } else { $null } ageMinutes = $ageMinutes maxAgeMinutes = $maxAgeMinutes route = $readback.route exitCode = $readback.exitCode readTimeoutSeconds = $timeoutSeconds readbackElapsedMs = $readback.elapsedMs } Write-AgentLog "backup file=$name severity=$severity reason=$reason ageMinutes=$ageMinutes elapsedMs=$($readback.elapsedMs) timeoutSeconds=$timeoutSeconds" if ($severity -in @("warning", "critical")) { Record-AgentEvent "backup_$reason" $severity "file=$name host=$hostIp ageMinutes=$ageMinutes maxAgeMinutes=$maxAgeMinutes path=$path" $result -Alert } $result } function Test-AgentBackupCronPattern { param( [string]$Pattern, [string]$DefaultHost ) $quotedPattern = Quote-ShSingle $Pattern $command = "crontab -l 2>/dev/null | grep -F -- $quotedPattern >/dev/null && echo __PRESENT__ || echo __MISSING__" $timeoutSeconds = 45 if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["cronReadTimeoutSeconds"]) { $timeoutSeconds = [int]$Config.backupHealth.cronReadTimeoutSeconds } $readback = Invoke-HostSshText $DefaultHost $command $timeoutSeconds 2 $present = [bool]($readback.ok -and $readback.output -match "__PRESENT__") $result = [pscustomobject]@{ host = $DefaultHost pattern = $Pattern ok = $present severity = if ($present) { "ok" } else { "critical" } reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" } route = $readback.route exitCode = $readback.exitCode readTimeoutSeconds = $timeoutSeconds readbackElapsedMs = $readback.elapsedMs } Write-AgentLog "backup cron pattern=$Pattern ok=$present" if (-not $present) { Record-AgentEvent "backup_cron_pattern_missing" "critical" "host=$DefaultHost pattern=$Pattern" $result -Alert } $result } function Test-AgentBackupCronPatterns { param( [string[]]$Patterns, [string]$DefaultHost ) $timeoutSeconds = 45 if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["cronReadTimeoutSeconds"]) { $timeoutSeconds = [int]$Config.backupHealth.cronReadTimeoutSeconds } $readback = Invoke-HostSshText $DefaultHost "crontab -l 2>/dev/null" $timeoutSeconds 2 $results = @() if (-not $readback.ok) { foreach ($pattern in $Patterns) { $result = [pscustomobject]@{ host = $DefaultHost pattern = $pattern ok = $false severity = "warning" reason = "cron_readback_failed" route = $readback.route exitCode = $readback.exitCode readTimeoutSeconds = $timeoutSeconds readbackElapsedMs = $readback.elapsedMs } Record-AgentEvent "backup_cron_readback_failed" "warning" "host=$DefaultHost pattern=$pattern" $result -Alert $results += $result } return $results } $crontab = [string]$readback.output foreach ($pattern in $Patterns) { $present = [bool]($crontab -match [regex]::Escape($pattern)) $result = [pscustomobject]@{ host = $DefaultHost pattern = $pattern ok = $present severity = if ($present) { "ok" } else { "critical" } reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" } route = $readback.route exitCode = $readback.exitCode readTimeoutSeconds = $timeoutSeconds readbackElapsedMs = $readback.elapsedMs } Write-AgentLog "backup cron pattern=$pattern ok=$present" if (-not $present) { Record-AgentEvent "backup_cron_pattern_missing" "critical" "host=$DefaultHost pattern=$pattern" $result -Alert } $results += $result } $results } function Test-AgentBackupHealth { $backupConfig = Get-AgentBackupHealthConfig $targets = @() foreach ($target in @($backupConfig.targets)) { $targets += Test-AgentBackupSnapshotTarget $target $backupConfig.host } $files = @() foreach ($fileCheck in @($backupConfig.fileChecks)) { $files += Test-AgentBackupFileCheck $fileCheck $backupConfig.host } $cron = Test-AgentBackupCronPatterns @($backupConfig.cronPatterns) $backupConfig.host $targetCritical = @($targets | Where-Object { $_.severity -eq "critical" }).Count $targetWarning = @($targets | Where-Object { $_.severity -eq "warning" }).Count $fileCritical = @($files | Where-Object { $_.severity -eq "critical" }).Count $fileWarning = @($files | Where-Object { $_.severity -eq "warning" }).Count $cronCritical = @($cron | Where-Object { $_.severity -eq "critical" }).Count $cronWarning = @($cron | Where-Object { $_.severity -eq "warning" }).Count $critical = $targetCritical + $fileCritical + $cronCritical $warning = $targetWarning + $fileWarning + $cronWarning $severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } $summary = [pscustomobject]@{ targetCritical = $targetCritical targetWarning = $targetWarning fileCritical = $fileCritical fileWarning = $fileWarning cronCritical = $cronCritical cronWarning = $cronWarning targetCount = @($targets).Count fileCheckCount = @($files).Count cronPatternCount = @($cron).Count } if ($severity -in @("warning", "critical")) { Record-AgentEvent "backup_health_degraded" $severity "severity=$severity targetCritical=$targetCritical fileCritical=$fileCritical cronCritical=$cronCritical" $summary -Alert } else { Write-AgentLog "backup_health ok targets=$($summary.targetCount) files=$($summary.fileCheckCount) cron=$($summary.cronPatternCount)" } [pscustomobject]@{ ok = [bool]($severity -eq "ok") severity = $severity host = $backupConfig.host targets = $targets files = $files cron = $cron summary = $summary } } function Convert-AgentHtml { param([object]$Value) [System.Net.WebUtility]::HtmlEncode([string]$Value) } function Get-AgentLatestEvidenceSummary { param( [string]$Name, [string]$Pattern ) $latest = Get-AgentLatestEvidence $Pattern [pscustomobject]@{ name = $Name exists = $latest.exists file = $latest.fileName path = $latest.path ageMinutes = $latest.ageMinutes parseError = $latest.parseError } } function Convert-AgentSafeKey { param([string]$Value) $safe = [regex]::Replace(([string]$Value), "[^A-Za-z0-9_.:-]", "_") if (-not $safe) { return "unknown" } if ($safe.Length -gt 140) { return $safe.Substring(0, 140) } return $safe } function New-AgentOutcomeCheck { param( [string]$Name, [bool]$Passed, [bool]$Required = $true, [string]$Detail = "" ) [pscustomobject]@{ name = $Name passed = $Passed required = $Required detail = $Detail } } function Get-AgentOutcomeContract { param( [string]$ModeName, [int]$TransportExitCode, [object]$LatestEvidence, [datetime]$StartedAt, [bool]$SourceEventResolutionRequired, [object]$SourceEventResolved = $null, [string]$SourceEventEvidence = "" ) $checks = @() $transportOk = [bool]($TransportExitCode -eq 0) $evidenceExists = [bool]($LatestEvidence -and $LatestEvidence.exists) $evidenceParsed = [bool]($evidenceExists -and -not $LatestEvidence.parseError -and $LatestEvidence.data) $evidenceFresh = $false if ($evidenceExists -and $LatestEvidence.lastWriteTime) { try { $evidenceTime = [datetime]::Parse([string]$LatestEvidence.lastWriteTime) $evidenceFresh = [bool]($evidenceTime -ge $StartedAt.AddSeconds(-5)) } catch { $evidenceFresh = $false } } $data = if ($evidenceParsed) { $LatestEvidence.data } else { $null } $evidenceMode = if ($data -and $data.PSObject.Properties["mode"]) { [string]$data.mode } else { "" } $modeMatches = [bool]($evidenceMode -eq $ModeName) $checks += New-AgentOutcomeCheck "transport_completed" $transportOk $true "exitCode=$TransportExitCode" $checks += New-AgentOutcomeCheck "evidence_exists" $evidenceExists $true ([string]$LatestEvidence.path) $checks += New-AgentOutcomeCheck "evidence_parsed" $evidenceParsed $true ([string]$LatestEvidence.parseError) $checks += New-AgentOutcomeCheck "evidence_fresh" $evidenceFresh $true ([string]$LatestEvidence.lastWriteTime) $checks += New-AgentOutcomeCheck "evidence_mode_matches" $modeMatches $true "expected=$ModeName actual=$evidenceMode" $verifierName = "agent99-$($ModeName.ToLowerInvariant())-post-condition-v1" $modeVerifierPassed = $false $successState = "resolved" $failureState = "degraded" if ($data) { switch ($ModeName) { "Status" { $expectedHosts = @($Config.hosts).Count $hostRows = @($data.hosts) $hostsOk = [bool]($expectedHosts -gt 0 -and $hostRows.Count -ge $expectedHosts -and @($hostRows | Where-Object { -not $_.ping }).Count -eq 0) $harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy) $awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure) $publicRows = @($data.public) $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0) $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false $modeVerifierPassed = [bool]($hostsOk -and $harborOk -and $awoooOk -and $publicOk) } "Recover" { $expectedVms = @($Config.vms).Count $vmRows = @($data.vmResults) $vmsOk = [bool]($expectedVms -gt 0 -and $vmRows.Count -ge $expectedVms -and @($vmRows | Where-Object { -not $_.ok }).Count -eq 0) $expectedHosts = @($Config.hosts).Count $hostRows = @($data.hosts) $hostsOk = [bool]($expectedHosts -gt 0 -and $hostRows.Count -ge $expectedHosts -and @($hostRows | Where-Object { -not $_.ping }).Count -eq 0) $harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy) $awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure) $publicRows = @($data.public) $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)" $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk) } "StartVMs" { $expectedVms = @($Config.vms).Count $vmRows = @($data.vmResults) $commandOk = [bool]($expectedVms -gt 0 -and $vmRows.Count -ge $expectedVms -and @($vmRows | Where-Object { -not $_.ok }).Count -eq 0) $unreachable = @() foreach ($vm in @($Config.vms)) { if (-not $vm.host) { $unreachable += [string]$vm.name; continue } $reachable = Test-Connection -ComputerName ([string]$vm.host) -Count 1 -Quiet -ErrorAction SilentlyContinue if (-not $reachable) { $unreachable += [string]$vm.name } } $guestsReachable = [bool]($expectedVms -gt 0 -and @($unreachable).Count -eq 0) $checks += New-AgentOutcomeCheck "vm_start_commands_ok" $commandOk $true "expected=$expectedVms actual=$($vmRows.Count)" $checks += New-AgentOutcomeCheck "vm_guests_reachable" $guestsReachable $true "unreachable=$($unreachable -join ',')" $modeVerifierPassed = [bool]($commandOk -and $guestsReachable) } "PublicSmoke" { $rows = @($data.public) $routesOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or -not $_.non502 }).Count -eq 0) $checks += New-AgentOutcomeCheck "public_routes_non502" $routesOk $true "count=$($rows.Count)" $modeVerifierPassed = $routesOk } "HarborRepair" { $harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy) $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $modeVerifierPassed = $harborOk } "AwoooRepair" { $awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure) $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $modeVerifierPassed = $awoooOk } "Perf" { $rows = @($data.performance) $readbackOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or (@($_.reasons) -contains "perf_readback_failed") }).Count -eq 0) $pressureClear = [bool]($readbackOk -and @($rows | Where-Object { $_.severity -in @("warning", "critical") }).Count -eq 0) $checks += New-AgentOutcomeCheck "performance_readback_complete" $readbackOk $checks += New-AgentOutcomeCheck "performance_pressure_cleared" $pressureClear $modeVerifierPassed = [bool]($readbackOk -and $pressureClear) } "LoadShed" { $rows = @($data.performance) $readbackOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or (@($_.reasons) -contains "perf_readback_failed") }).Count -eq 0) $executed = @($data.loadShedding | Where-Object { -not $_.skipped }) $postVerifyOk = [bool]($executed.Count -gt 0 -and @($executed | Where-Object { -not $_.postVerifyOk }).Count -eq 0) $alreadyClear = [bool]($readbackOk -and @($rows | Where-Object { $_.severity -in @("warning", "critical") }).Count -eq 0) $checks += New-AgentOutcomeCheck "performance_readback_complete" $readbackOk $checks += New-AgentOutcomeCheck "load_shed_post_verify" ([bool]($postVerifyOk -or $alreadyClear)) $true "executed=$($executed.Count)" $modeVerifierPassed = [bool]($readbackOk -and ($postVerifyOk -or $alreadyClear)) } "SelfCheck" { $selfOk = [bool]($data.selfHealth -and $data.selfHealth.severity -eq "ok") $checks += New-AgentOutcomeCheck "agent_self_health_ok" $selfOk $modeVerifierPassed = $selfOk } "BackupCheck" { $backupOk = [bool]($data.backupHealth -and $data.backupHealth.severity -eq "ok") $checks += New-AgentOutcomeCheck "backup_health_ok" $backupOk $modeVerifierPassed = $backupOk } "ProviderFreshness" { $provider = $data.providerFreshness $directOk = [bool]($provider -and $provider.status -eq "direct_readback_ok" -and @($provider.missingFields).Count -eq 0 -and $provider.lastSeen -and $provider.verifierReadback -and $provider.verifierReadback.ok -eq $true -and [int]$provider.verifierReadback.sourceEventTotal -gt 0) $checks += New-AgentOutcomeCheck "provider_freshness_direct_readback" $directOk $true ([string]$provider.status) $modeVerifierPassed = $directOk } "SecurityTriage" { $candidateReady = [bool]($data.securityTriage -and $data.securityTriage.status -eq "security_triage_candidate_created") $checks += New-AgentOutcomeCheck "security_candidate_created" $candidateReady $modeVerifierPassed = $candidateReady $successState = "candidate_ready" } "ControlTick" { $controlOk = [bool]($data.controlTick -and $data.controlTick.ok) $checks += New-AgentOutcomeCheck "control_tick_completed" $controlOk $modeVerifierPassed = $controlOk } default { $checks += New-AgentOutcomeCheck "mode_verifier_defined" $false $failureState = "blocked" } } } $basePassed = [bool]($transportOk -and $evidenceExists -and $evidenceParsed -and $evidenceFresh -and $modeMatches) $verifierPassed = [bool]($basePassed -and $modeVerifierPassed) $sourceEventResolvedBool = if (-not $SourceEventResolutionRequired) { $true } else { [bool]($SourceEventResolved -eq $true) } $state = if (-not $transportOk) { "failed" } elseif (-not $basePassed) { "blocked" } elseif (-not $modeVerifierPassed) { $failureState } elseif ($successState -eq "candidate_ready") { "candidate_ready" } elseif ($SourceEventResolutionRequired -and -not $sourceEventResolvedBool) { "verifying" } else { "resolved" } $failedChecks = @($checks | Where-Object { $_.required -and -not $_.passed } | ForEach-Object { $_.name }) [pscustomobject]@{ schemaVersion = "agent99_outcome_contract_v1" state = $state resolved = [bool]($state -eq "resolved") transportOk = $transportOk transportExitCode = $TransportExitCode verifierName = $verifierName verifierPassed = $verifierPassed evidenceFresh = $evidenceFresh evidenceModeMatched = $modeMatches sourceEventResolutionRequired = $SourceEventResolutionRequired sourceEventResolved = $sourceEventResolvedBool sourceEventEvidence = if ($SourceEventEvidence) { $SourceEventEvidence } else { $null } failedChecks = $failedChecks checks = $checks verifiedAt = (Get-Date -Format o) } } function Invoke-AgentOutcomeContractSelfTest { $startedAt = (Get-Date).AddSeconds(-1) $publicOkEvidence = [pscustomobject]@{ exists = $true path = "selftest-public-ok.json" lastWriteTime = (Get-Date -Format o) parseError = $null data = [pscustomobject]@{ mode = "PublicSmoke" public = @([pscustomobject]@{ url = "https://example.invalid"; status = 200; non502 = $true; ok = $true }) } } $publicFailedEvidence = [pscustomobject]@{ exists = $true path = "selftest-public-failed.json" lastWriteTime = (Get-Date -Format o) parseError = $null data = [pscustomobject]@{ mode = "PublicSmoke" public = @([pscustomobject]@{ url = "https://example.invalid"; status = 502; non502 = $false; ok = $false }) } } $providerEvidence = [pscustomobject]@{ exists = $true path = "selftest-provider.json" lastWriteTime = (Get-Date -Format o) parseError = $null data = [pscustomobject]@{ mode = "ProviderFreshness" providerFreshness = [pscustomobject]@{ status = "direct_readback_ok_no_source_events" missingFields = @("lastSeen") lastSeen = $null verifierReadback = [pscustomobject]@{ ok = $true; sourceEventTotal = 0 } } } } $securityEvidence = [pscustomobject]@{ exists = $true path = "selftest-security.json" lastWriteTime = (Get-Date -Format o) parseError = $null data = [pscustomobject]@{ mode = "SecurityTriage" securityTriage = [pscustomobject]@{ status = "security_triage_candidate_created"; hitCount = 1 } } } $operatorResolved = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $false $alertVerifying = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $true $false "selftest-source-event" $failedPostCondition = Get-AgentOutcomeContract "PublicSmoke" 0 $publicFailedEvidence $startedAt $false $transportFailed = Get-AgentOutcomeContract "PublicSmoke" 7 $publicOkEvidence $startedAt $false $providerDegraded = Get-AgentOutcomeContract "ProviderFreshness" 0 $providerEvidence $startedAt $true $false $securityCandidate = Get-AgentOutcomeContract "SecurityTriage" 0 $securityEvidence $startedAt $true $false $ok = [bool]( $operatorResolved.state -eq "resolved" -and $operatorResolved.verifierPassed -and $alertVerifying.state -eq "verifying" -and -not $alertVerifying.resolved -and $failedPostCondition.state -eq "degraded" -and -not $failedPostCondition.verifierPassed -and $transportFailed.state -eq "failed" -and $providerDegraded.state -eq "degraded" -and $securityCandidate.state -eq "candidate_ready" ) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $ok contract = "agent99_outcome_contract_v1" cases = [pscustomobject]@{ operatorResolved = $operatorResolved alertVerifying = $alertVerifying failedPostCondition = $failedPostCondition transportFailed = $transportFailed providerDegraded = $providerDegraded securityCandidate = $securityCandidate } } $selfTestPath = Join-Path $EvidenceDir "agent99-OutcomeContractSelfTest-$stamp.json" $result | ConvertTo-Json -Depth 12 | Set-Content -Path $selfTestPath -Encoding UTF8 $result | ConvertTo-Json -Depth 12 | Set-Content -Path $jsonPath -Encoding UTF8 Write-AgentLog "outcome_contract_selftest ok=$ok evidence=$selfTestPath" $result | ConvertTo-Json -Depth 12 if (-not $ok) { exit 1 } exit 0 } function Get-AgentProblemStats { $stateDir = Join-Path $Config.agentRoot "state" $statsPath = Join-Path $stateDir "problem-counts.json" if (-not (Test-Path $statsPath)) { return [pscustomobject]@{ path = $statsPath schemaVersion = "agent99_problem_counts_v2" outcomeContractActivatedAt = $null totalEvents = 0 totalResolved = 0 totalVerifiedResolved = 0 totalFailed = 0 totalDegraded = 0 totalBlocked = 0 totalVerifying = 0 totalCandidateReady = 0 problems = @() } } try { $data = Get-Content $statsPath -Raw | ConvertFrom-Json return [pscustomobject]@{ path = $statsPath schemaVersion = if ($data.schemaVersion) { [string]$data.schemaVersion } else { "agent99_problem_counts_legacy_v1" } outcomeContractActivatedAt = if ($data.outcomeContractActivatedAt) { [string]$data.outcomeContractActivatedAt } else { $null } totalEvents = if ($data.totalEvents) { [int]$data.totalEvents } else { 0 } totalResolved = if ($data.totalResolved) { [int]$data.totalResolved } else { 0 } totalVerifiedResolved = if ($data.totalVerifiedResolved) { [int]$data.totalVerifiedResolved } else { 0 } totalFailed = if ($data.totalFailed) { [int]$data.totalFailed } else { 0 } totalDegraded = if ($data.totalDegraded) { [int]$data.totalDegraded } else { 0 } totalBlocked = if ($data.totalBlocked) { [int]$data.totalBlocked } else { 0 } totalVerifying = if ($data.totalVerifying) { [int]$data.totalVerifying } else { 0 } totalCandidateReady = if ($data.totalCandidateReady) { [int]$data.totalCandidateReady } else { 0 } problems = @($data.problems) } } catch { Write-AgentLog "problem_stats_parse_failed path=$statsPath error=$($_.Exception.Message)" return [pscustomobject]@{ path = $statsPath schemaVersion = "agent99_problem_counts_v2" outcomeContractActivatedAt = $null totalEvents = 0 totalResolved = 0 totalVerifiedResolved = 0 totalFailed = 0 totalDegraded = 0 totalBlocked = 0 totalVerifying = 0 totalCandidateReady = 0 problems = @() parseError = $_.Exception.Message } } } function Update-AgentProblemKnowledge { param([object]$Result) $stateDir = Join-Path $Config.agentRoot "state" $playbookDir = Join-Path $Config.agentRoot "playbooks" $ragDir = Join-Path $playbookDir "rag-import" New-Item -ItemType Directory -Force -Path $stateDir, $playbookDir, $ragDir | Out-Null $statsPath = Join-Path $stateDir "problem-counts.json" $eventsPath = Join-Path $playbookDir "agent99-incident-records.jsonl" $kmPath = Join-Path $playbookDir "agent99-incident-km.md" $ragPath = Join-Path $ragDir "agent99-incidents.jsonl" $source = if ($Result.source) { [string]$Result.source } else { "agent99" } $kind = if ($Result.alertKind) { [string]$Result.alertKind } elseif ($Result.reason) { [string]$Result.reason } else { [string]$Result.mode } $service = if ($Result.alertService) { [string]$Result.alertService } else { "agent99" } $hostName = if ($Result.alertHost) { [string]$Result.alertHost } else { "" } $severity = if ($Result.alertSeverity) { [string]$Result.alertSeverity } elseif ($Result.ok) { "info" } else { "critical" } $key = Convert-AgentSafeKey (($source, $service, $hostName, $kind | Where-Object { $_ }) -join "|") $now = Get-Date -Format o $outcome = if ($Result.PSObject.Properties["outcome"]) { $Result.outcome } else { $null } $statusText = if ($outcome -and $outcome.PSObject.Properties["state"]) { [string]$outcome.state } elseif ($Result.ok) { "resolved" } else { "failed" } $isResolved = [bool]($statusText -eq "resolved") $isFailed = [bool]($statusText -eq "failed") if (-not $Result.alertSeverity) { $severity = if ($isResolved) { "info" } elseif ($isFailed) { "critical" } else { "warning" } } $event = [pscustomobject]@{ timestamp = $now problemKey = $key status = $statusText commandId = $Result.id source = $source reason = $Result.reason alertId = $Result.alertId alertKind = $kind alertSeverity = $severity service = $service host = $hostName mode = $Result.mode instruction = $Result.instruction ok = $isResolved transportOk = if ($Result.PSObject.Properties["transportOk"]) { [bool]$Result.transportOk } else { [bool]($Result.exitCode -eq 0) } exitCode = $Result.exitCode outcomeState = $statusText verifierName = if ($outcome) { $outcome.verifierName } else { $null } verifierPassed = if ($outcome) { [bool]$outcome.verifierPassed } else { $false } sourceEventResolved = if ($outcome) { [bool]$outcome.sourceEventResolved } else { $false } evidence = $Result.evidence } ($event | ConvertTo-Json -Depth 8 -Compress) | Add-Content -Path $eventsPath -Encoding UTF8 ($event | ConvertTo-Json -Depth 8 -Compress) | Add-Content -Path $ragPath -Encoding UTF8 $stats = Get-AgentProblemStats $problems = @($stats.problems) $existing = $problems | Where-Object { $_.key -eq $key } | Select-Object -First 1 if (-not $existing) { $existing = [pscustomobject]@{ key = $key source = $source kind = $kind service = $service host = $hostName severity = $severity occurrences = 0 resolved = 0 verifiedResolved = 0 failed = 0 degraded = 0 blocked = 0 verifying = 0 candidateReady = 0 lastStatus = $null firstSeenAt = $now lastSeenAt = $null lastResolvedAt = $null lastEvidence = $null lastMode = $null lastInstruction = $null } $problems += $existing } foreach ($counterName in @("verifiedResolved", "degraded", "blocked", "verifying", "candidateReady")) { if (-not $existing.PSObject.Properties[$counterName]) { $existing | Add-Member -MemberType NoteProperty -Name $counterName -Value 0 } } $existing.occurrences = [int]$existing.occurrences + 1 if ($isResolved) { $existing.resolved = [int]$existing.resolved + 1 $existing.verifiedResolved = [int]$existing.verifiedResolved + 1 $existing.lastResolvedAt = $now } elseif ($isFailed) { $existing.failed = [int]$existing.failed + 1 } elseif ($statusText -eq "degraded") { $existing.degraded = [int]$existing.degraded + 1 } elseif ($statusText -eq "blocked") { $existing.blocked = [int]$existing.blocked + 1 } elseif ($statusText -eq "verifying") { $existing.verifying = [int]$existing.verifying + 1 } elseif ($statusText -eq "candidate_ready") { $existing.candidateReady = [int]$existing.candidateReady + 1 } $existing.lastStatus = $statusText $existing.lastSeenAt = $now $existing.lastEvidence = $Result.evidence $existing.lastMode = $Result.mode $existing.lastInstruction = $Result.instruction $existing.severity = $severity $newTotalEvents = [int]$stats.totalEvents + 1 $newTotalResolved = [int]$stats.totalResolved + $(if ($isResolved) { 1 } else { 0 }) $newTotalVerifiedResolved = [int]$stats.totalVerifiedResolved + $(if ($isResolved) { 1 } else { 0 }) $newTotalFailed = [int]$stats.totalFailed + $(if ($isFailed) { 1 } else { 0 }) $newTotalDegraded = [int]$stats.totalDegraded + $(if ($statusText -eq "degraded") { 1 } else { 0 }) $newTotalBlocked = [int]$stats.totalBlocked + $(if ($statusText -eq "blocked") { 1 } else { 0 }) $newTotalVerifying = [int]$stats.totalVerifying + $(if ($statusText -eq "verifying") { 1 } else { 0 }) $newTotalCandidateReady = [int]$stats.totalCandidateReady + $(if ($statusText -eq "candidate_ready") { 1 } else { 0 }) $outcomeContractActivatedAt = if ($stats.outcomeContractActivatedAt) { $stats.outcomeContractActivatedAt } else { $now } $updated = [pscustomobject]@{ schemaVersion = "agent99_problem_counts_v2" generatedAt = $now outcomeContractActivatedAt = $outcomeContractActivatedAt legacyResolvedCountMayIncludeTransportOnly = [bool]($stats.schemaVersion -ne "agent99_problem_counts_v2" -or ([int]$stats.totalResolved -gt [int]$stats.totalVerifiedResolved)) totalEvents = $newTotalEvents totalResolved = $newTotalResolved totalVerifiedResolved = $newTotalVerifiedResolved totalFailed = $newTotalFailed totalDegraded = $newTotalDegraded totalBlocked = $newTotalBlocked totalVerifying = $newTotalVerifying totalCandidateReady = $newTotalCandidateReady problems = @($problems | Sort-Object @{ Expression = { [int]$_.occurrences }; Descending = $true }, @{ Expression = { $_.lastSeenAt }; Descending = $true }) } $updated | ConvertTo-Json -Depth 10 | Set-Content -Path $statsPath -Encoding UTF8 $rows = @($updated.problems | Select-Object -First 50 | ForEach-Object { "- key=$($_.key); occurrences=$($_.occurrences); verifiedResolved=$($_.verifiedResolved); failed=$($_.failed); degraded=$($_.degraded); blocked=$($_.blocked); verifying=$($_.verifying); candidateReady=$($_.candidateReady); lastStatus=$($_.lastStatus); lastMode=$($_.lastMode); evidence=$($_.lastEvidence)" }) $km = @( "# Agent99 Incident KM", "", "Last updated: $now", "", "## Totals", "", "- totalEvents: $($updated.totalEvents)", "- totalResolvedLegacyAndVerified: $($updated.totalResolved)", "- totalVerifiedResolved: $($updated.totalVerifiedResolved)", "- totalFailed: $($updated.totalFailed)", "- totalDegraded: $($updated.totalDegraded)", "- totalBlocked: $($updated.totalBlocked)", "- totalVerifying: $($updated.totalVerifying)", "- totalCandidateReady: $($updated.totalCandidateReady)", "", "## Recurring Problems", "", ($rows -join "`n") ) -join "`n" Set-Content -Path $kmPath -Value $km -Encoding UTF8 [pscustomobject]@{ key = $key occurrences = $existing.occurrences resolved = $existing.resolved verifiedResolved = $existing.verifiedResolved failed = $existing.failed degraded = $existing.degraded blocked = $existing.blocked verifying = $existing.verifying candidateReady = $existing.candidateReady lastStatus = $existing.lastStatus statsPath = $statsPath eventsPath = $eventsPath kmPath = $kmPath ragPath = $ragPath } } function Get-AgentDashboardSummary { $status = Get-AgentLatestEvidence "agent99-Status-*.json" $perf = Get-AgentLatestEvidence "agent99-Perf-*.json" $self = Get-AgentLatestEvidence "agent99-SelfCheck-*.json" $backup = Get-AgentLatestEvidence "agent99-BackupCheck-*.json" $recover = Get-AgentLatestEvidence "agent99-Recover-*.json" $taskNames = @( "Wooo-Agent99-Startup-Recovery", "Wooo-Agent99-Heartbeat", "Wooo-Agent99-Performance-Guard", "Wooo-Agent99-Control-Loop", "Wooo-Agent99-Telegram-Inbox", "Wooo-Agent99-SRE-Alert-Inbox", "Wooo-Agent99-Self-Health", "Wooo-Agent99-Backup-Health" ) $tasks = @() foreach ($taskName in $taskNames) { try { $task = Get-ScheduledTask -TaskName $taskName -ErrorAction Stop $info = Get-ScheduledTaskInfo -TaskName $taskName -ErrorAction Stop $tasks += [pscustomobject]@{ name = $taskName exists = $true state = [string]$task.State lastTaskResult = $info.LastTaskResult lastRunTime = if ($info.LastRunTime) { $info.LastRunTime.ToString("yyyy-MM-dd HH:mm:ss") } else { $null } nextRunTime = if ($info.NextRunTime) { $info.NextRunTime.ToString("yyyy-MM-dd HH:mm:ss") } else { $null } } } catch { $tasks += [pscustomobject]@{ name = $taskName exists = $false state = "missing" lastTaskResult = $null lastRunTime = $null nextRunTime = $null } } } $queueDir = Join-Path $Config.agentRoot "queue" if (-not (Test-Path $queueDir)) { New-Item -ItemType Directory -Force -Path $queueDir | Out-Null } $pending = @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" }) $processedDir = Join-Path $queueDir "processed" $failedDir = Join-Path $queueDir "failed" $processedCount = if (Test-Path $processedDir) { @(Get-ChildItem -Path $processedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } $failedCount = if (Test-Path $failedDir) { @(Get-ChildItem -Path $failedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } $alertIncomingDir = Join-Path $Config.agentRoot "alerts\incoming" $alertProcessedDir = Join-Path $Config.agentRoot "alerts\processed" $alertFailedDir = Join-Path $Config.agentRoot "alerts\failed" $alertPendingCount = if (Test-Path $alertIncomingDir) { @(Get-ChildItem -Path $alertIncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } $alertProcessedCount = if (Test-Path $alertProcessedDir) { @(Get-ChildItem -Path $alertProcessedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } $alertFailedCount = if (Test-Path $alertFailedDir) { @(Get-ChildItem -Path $alertFailedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } $problemStats = Get-AgentProblemStats [pscustomobject]@{ timestamp = (Get-Date -Format o) computer = $env:COMPUTERNAME user = $env:USERNAME agentRoot = $Config.agentRoot evidenceDir = $EvidenceDir queue = [pscustomobject]@{ path = $queueDir pending = $pending.Count processed = $processedCount failed = $failedCount } alerts = [pscustomobject]@{ incomingPath = $alertIncomingDir pending = $alertPendingCount processed = $alertProcessedCount failed = $alertFailedCount } problemCounts = [pscustomobject]@{ path = $problemStats.path totalEvents = $problemStats.totalEvents totalResolved = $problemStats.totalResolved totalVerifiedResolved = $problemStats.totalVerifiedResolved totalFailed = $problemStats.totalFailed totalDegraded = $problemStats.totalDegraded totalBlocked = $problemStats.totalBlocked totalVerifying = $problemStats.totalVerifying totalCandidateReady = $problemStats.totalCandidateReady top = @($problemStats.problems | Select-Object -First 10) } tasks = $tasks evidence = @( Get-AgentLatestEvidenceSummary "status" "agent99-Status-*.json" Get-AgentLatestEvidenceSummary "performance" "agent99-Perf-*.json" Get-AgentLatestEvidenceSummary "self_health" "agent99-SelfCheck-*.json" Get-AgentLatestEvidenceSummary "backup" "agent99-BackupCheck-*.json" Get-AgentLatestEvidenceSummary "startup_recovery" "agent99-Recover-*.json" ) public = if ($status.exists -and -not $status.parseError -and $status.data) { @($status.data.public | Select-Object url,status,non502,ok) } else { @() } hosts = if ($status.exists -and -not $status.parseError -and $status.data) { @($status.data.hosts | Select-Object host,ping) } else { @() } performance = if ($perf.exists -and -not $perf.parseError -and $perf.data) { @($perf.data.performance | Select-Object host,severity,loadPerCore,memAvailablePercent,diskUsedPercent,route) } else { @() } selfHealth = if ($self.exists -and -not $self.parseError -and $self.data) { $self.data.selfHealth.summary } else { $null } backupHealth = if ($backup.exists -and -not $backup.parseError -and $backup.data) { $backup.data.backupHealth.summary } else { $null } latestSelfHealthSeverity = if ($self.exists -and -not $self.parseError -and $self.data) { $self.data.selfHealth.severity } else { "unknown" } latestBackupSeverity = if ($backup.exists -and -not $backup.parseError -and $backup.data) { $backup.data.backupHealth.severity } else { "unknown" } } } function Write-AgentDashboard { param([object]$Summary) $dashboardDir = Join-Path $Config.agentRoot "dashboard" New-Item -ItemType Directory -Force -Path $dashboardDir | Out-Null $jsonPath = Join-Path $dashboardDir "agent99-control-center.latest.json" $htmlPath = Join-Path $dashboardDir "index.html" $Summary | ConvertTo-Json -Depth 12 | Set-Content -Path $jsonPath -Encoding UTF8 $taskRows = foreach ($task in @($Summary.tasks)) { "$(Convert-AgentHtml $task.name)$(Convert-AgentHtml $task.state)$(Convert-AgentHtml $task.lastTaskResult)$(Convert-AgentHtml $task.lastRunTime)$(Convert-AgentHtml $task.nextRunTime)" } $hostRows = foreach ($hostItem in @($Summary.hosts)) { "$(Convert-AgentHtml $hostItem.host)$(Convert-AgentHtml $hostItem.ping)" } $publicRows = foreach ($route in @($Summary.public)) { "$(Convert-AgentHtml $route.url)$(Convert-AgentHtml $route.status)$(Convert-AgentHtml $route.non502)$(Convert-AgentHtml $route.ok)" } $perfRows = foreach ($perfItem in @($Summary.performance)) { "$(Convert-AgentHtml $perfItem.host)$(Convert-AgentHtml $perfItem.severity)$(Convert-AgentHtml $perfItem.loadPerCore)$(Convert-AgentHtml $perfItem.memAvailablePercent)$(Convert-AgentHtml $perfItem.diskUsedPercent)$(Convert-AgentHtml $perfItem.route)" } $evidenceRows = foreach ($ev in @($Summary.evidence)) { "$(Convert-AgentHtml $ev.name)$(Convert-AgentHtml $ev.file)$(Convert-AgentHtml $ev.ageMinutes)$(Convert-AgentHtml $ev.parseError)" } $problemRows = foreach ($problem in @($Summary.problemCounts.top)) { "$(Convert-AgentHtml $problem.key)$(Convert-AgentHtml $problem.occurrences)$(Convert-AgentHtml $problem.verifiedResolved)$(Convert-AgentHtml $problem.failed)$(Convert-AgentHtml $problem.degraded)$(Convert-AgentHtml $problem.blocked)$(Convert-AgentHtml $problem.verifying)$(Convert-AgentHtml $problem.candidateReady)$(Convert-AgentHtml $problem.lastStatus)$(Convert-AgentHtml $problem.lastMode)" } $html = @" Agent99 Control Center

Agent99 Control Center

Last update: $(Convert-AgentHtml $Summary.timestamp) on $(Convert-AgentHtml $Summary.computer) as $(Convert-AgentHtml $Summary.user). Auto-refreshes every 30 seconds.
Self Health
$(Convert-AgentHtml $Summary.latestSelfHealthSeverity)
Backup Health
$(Convert-AgentHtml $Summary.latestBackupSeverity)
Queue Pending
$(Convert-AgentHtml $Summary.queue.pending)
Alerts Pending
$(Convert-AgentHtml $Summary.alerts.pending)
Problem Events
$(Convert-AgentHtml $Summary.problemCounts.totalEvents)
Verified Resolved
$(Convert-AgentHtml $Summary.problemCounts.totalVerifiedResolved)
Failed
$(Convert-AgentHtml $Summary.problemCounts.totalFailed)
Degraded
$(Convert-AgentHtml $Summary.problemCounts.totalDegraded)
Verifying
$(Convert-AgentHtml $Summary.problemCounts.totalVerifying)

Agent root: $(Convert-AgentHtml $Summary.agentRoot)
Queue: $(Convert-AgentHtml $Summary.queue.path)
Alerts: $(Convert-AgentHtml $Summary.alerts.incomingPath)
Problem counts: $(Convert-AgentHtml $Summary.problemCounts.path)
Evidence: $(Convert-AgentHtml $Summary.evidenceDir)

Problem Counts

$($problemRows -join "`n")
ProblemOccurrencesVerified ResolvedFailedDegradedBlockedVerifyingCandidate ReadyLast StatusLast Mode

Scheduled Tasks

$($taskRows -join "`n")
TaskStateLast ResultLast RunNext Run

Hosts

$($hostRows -join "`n")
HostPing

Public Routes

$($publicRows -join "`n")
URLStatusNon-502OK

Performance

$($perfRows -join "`n")
HostSeverityLoad/CoreMem Avail %Disk Used %Route

Evidence

$($evidenceRows -join "`n")
NameFileAge MinutesParse Error
"@ Set-Content -Path $htmlPath -Value $html -Encoding UTF8 [pscustomobject]@{ dashboardDir = $dashboardDir htmlPath = $htmlPath jsonPath = $jsonPath } } function Ensure-AgentDesktopShortcuts { param([string]$DashboardPath) $created = @() $targets = @( [pscustomobject]@{ name = "Agent99 Control Center.lnk"; target = $DashboardPath }, [pscustomobject]@{ name = "Agent99 Submit Request.lnk"; target = (Join-Path $Config.agentRoot "agent99-submit-request.cmd") }, [pscustomobject]@{ name = "Agent99 Queue.lnk"; target = (Join-Path $Config.agentRoot "queue") }, [pscustomobject]@{ name = "Agent99 Evidence.lnk"; target = $EvidenceDir } ) $desktopDirs = @( [Environment]::GetFolderPath("CommonDesktopDirectory"), [Environment]::GetFolderPath("Desktop") ) | Where-Object { $_ } | Select-Object -Unique foreach ($desktop in $desktopDirs) { if (-not (Test-Path $desktop)) { continue } foreach ($item in $targets) { $shortcutPath = Join-Path $desktop $item.name try { $shell = New-Object -ComObject WScript.Shell $shortcut = $shell.CreateShortcut($shortcutPath) $shortcut.TargetPath = $item.target $shortcut.WorkingDirectory = Split-Path -Parent $item.target $shortcut.Description = "Agent99 local control and evidence shortcut" $shortcut.Save() $created += $shortcutPath } catch { Write-AgentLog "desktop_shortcut_failed path=$shortcutPath error=$($_.Exception.Message)" } } } $created } function Invoke-AgentQueuedCommands { $queueDir = Join-Path $Config.agentRoot "queue" $processedDir = Join-Path $queueDir "processed" $failedDir = Join-Path $queueDir "failed" New-Item -ItemType Directory -Force -Path $queueDir, $processedDir, $failedDir | Out-Null $allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage") $commands = @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" } | Sort-Object LastWriteTime | Select-Object -First 3) $results = @() foreach ($file in $commands) { $command = $null try { $command = Get-Content $file.FullName -Raw | ConvertFrom-Json } catch { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target $results += [pscustomobject]@{ id = $file.BaseName; ok = $false; reason = "invalid_json"; file = $target } continue } $id = if ($command.id) { [string]$command.id } else { $file.BaseName } $modeName = if ($command.mode) { [string]$command.mode } else { "" } $controlled = if ($command.PSObject.Properties["controlledApply"]) { [bool]$command.controlledApply } else { $false } $commandSource = if ($command.PSObject.Properties["source"]) { [string]$command.source } else { "queue" } $commandReason = if ($command.PSObject.Properties["reason"]) { [string]$command.reason } else { "operator_instruction" } $commandInstruction = if ($command.PSObject.Properties["instruction"]) { [string]$command.instruction } else { "" } $alertId = if ($command.PSObject.Properties["alertId"]) { [string]$command.alertId } else { $null } $alertSource = if ($command.PSObject.Properties["alertSource"]) { [string]$command.alertSource } else { $null } $alertKind = if ($command.PSObject.Properties["alertKind"]) { [string]$command.alertKind } else { $null } $alertSeverity = if ($command.PSObject.Properties["alertSeverity"]) { [string]$command.alertSeverity } else { $null } $alertService = if ($command.PSObject.Properties["alertService"]) { [string]$command.alertService } else { $null } $alertHost = if ($command.PSObject.Properties["alertHost"]) { [string]$command.alertHost } else { $null } $sourceEventResolutionRequired = if ($command.PSObject.Properties["sourceEventResolutionRequired"]) { Convert-AgentBool $command.sourceEventResolutionRequired } else { [bool]$alertId } $sourceEventResolved = if ($command.PSObject.Properties["sourceEventResolved"]) { [object](Convert-AgentBool $command.sourceEventResolved) } else { $null } $sourceEventEvidence = if ($command.PSObject.Properties["sourceEventEvidence"]) { [string]$command.sourceEventEvidence } else { "" } $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" } $suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false } if ($modeName -notin $allowedModes) { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target $result = [pscustomobject]@{ id = $id; mode = $modeName; source = $commandSource; instruction = $commandInstruction; ok = $false; reason = "mode_not_allowlisted"; file = $target } Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=mode_not_allowlisted" $result -Alert $results += $result continue } $runningPath = Join-Path $queueDir ("running-" + $file.Name) Move-Item -Force $file.FullName $runningPath $startTime = Get-Date $launcher = Join-Path $Config.agentRoot "agent99-run.ps1" $stdoutPath = Join-Path $EvidenceDir "agent99-command-$id.out" $stderrPath = Join-Path $EvidenceDir "agent99-command-$id.err" $args = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", $modeName) if ($controlled) { $args += "-ControlledApply" } if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") } $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru $completed = $process.WaitForExit(600000) if (-not $completed) { $process.Kill() $process.WaitForExit(2000) | Out-Null } $process.WaitForExit() | Out-Null $process.Refresh() $latest = Get-AgentLatestEvidence "agent99-$modeName-*.json" $exitCode = if (-not $completed) { -2 } elseif ($null -ne $process.ExitCode) { $process.ExitCode } elseif ($latest.exists -and -not $latest.parseError) { 0 } else { -3 } $outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence $result = [pscustomobject]@{ id = $id mode = $modeName controlledApply = $controlled source = $commandSource reason = $commandReason instruction = $commandInstruction alertId = $alertId alertSource = $alertSource alertKind = $alertKind alertSeverity = $alertSeverity alertService = $alertService alertHost = $alertHost suppressTelegram = $suppressTelegram transportOk = [bool]($exitCode -eq 0) ok = [bool]$outcome.resolved outcomeState = $outcome.state verifierPassed = [bool]$outcome.verifierPassed sourceEventResolved = [bool]$outcome.sourceEventResolved outcome = $outcome exitCode = $exitCode durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1) evidence = $latest.path stdout = $stdoutPath stderr = $stderrPath } $problem = Update-AgentProblemKnowledge $result $result | Add-Member -MemberType NoteProperty -Name problem -Value $problem -Force $donePath = Join-Path $processedDir ("processed-$id.json") $result | ConvertTo-Json -Depth 8 | Set-Content -Path $donePath -Encoding UTF8 Remove-Item $runningPath -Force -ErrorAction SilentlyContinue | Out-Null $recordData = $result | Select-Object * if ($replyChatId) { $recordData | Add-Member -MemberType NoteProperty -Name replyChatId -Value $replyChatId -Force } $outcomeSeverity = if ($outcome.state -eq "resolved") { "info" } elseif ($outcome.state -eq "failed") { "critical" } else { "warning" } Record-AgentEvent "operator_command_result" $outcomeSeverity "id=$id mode=$modeName outcome=$($outcome.state) transportExitCode=$exitCode verifierPassed=$($outcome.verifierPassed) sourceEventResolved=$($outcome.sourceEventResolved) evidence=$($latest.path)" $recordData -Alert $results += $result } $results } function Invoke-AgentControlTick { $processed = Invoke-AgentQueuedCommands $summary = Get-AgentDashboardSummary $dashboard = Write-AgentDashboard $summary $shortcuts = Ensure-AgentDesktopShortcuts $dashboard.htmlPath $pendingCount = if ($summary.queue) { $summary.queue.pending } else { 0 } $result = [pscustomobject]@{ ok = $true dashboardPath = $dashboard.htmlPath dashboardJson = $dashboard.jsonPath shortcutCount = @($shortcuts).Count shortcuts = $shortcuts processedCount = @($processed).Count processed = $processed pendingCount = $pendingCount } Record-AgentEvent "agent_control_tick" "info" "dashboard=$($dashboard.htmlPath) processed=$(@($processed).Count) pending=$pendingCount" $result -NoAlert $result } function Invoke-AgentSecurityTriage { $lookbackMinutes = 120 $since = (Get-Date).AddMinutes(-1 * $lookbackMinutes) $triageId = "security-triage-" + (Get-Date -Format "yyyyMMdd-HHmmss") $triageDir = Join-Path $Config.agentRoot "state\security-triage" New-Item -ItemType Directory -Force -Path $triageDir | Out-Null $triagePath = Join-Path $triageDir "$triageId.json" $patterns = @( "wazuh", "siem", "security", "auth_failed", "bruteforce", "brute force", "intrusion", "malware", "privilege", "vulnerability", "cve", "ioc", "suspicious", "unauthorized" ) $scanDirs = @( (Join-Path $Config.agentRoot "alerts\incoming"), (Join-Path $Config.agentRoot "alerts\processed"), (Join-Path $Config.agentRoot "queue"), (Join-Path $Config.agentRoot "queue\processed"), (Join-Path $Config.agentRoot "requests") ) | Where-Object { Test-Path $_ } | Select-Object -Unique $hits = @() foreach ($dir in $scanDirs) { $files = @(Get-ChildItem -LiteralPath $dir -Recurse -File -ErrorAction SilentlyContinue | Where-Object { $_.LastWriteTime -ge $since -and $_.Length -lt 2097152 }) foreach ($file in $files) { $matchedPatterns = @() foreach ($pattern in $patterns) { if (Select-String -LiteralPath $file.FullName -Pattern $pattern -SimpleMatch -Quiet -ErrorAction SilentlyContinue) { $matchedPatterns += $pattern } } if (@($matchedPatterns).Count -gt 0) { $excerpt = "" $firstHit = Select-String -LiteralPath $file.FullName -Pattern $matchedPatterns[0] -SimpleMatch -ErrorAction SilentlyContinue | Select-Object -First 1 if ($firstHit) { $excerpt = [string]$firstHit.Line $excerpt = $excerpt -replace '("?(telegram|token|chat|authorization|password|secret|cookie)[^,}\r\n]*"?\s*:\s*)"[^"]+"', '$1"[redacted]"' if ($excerpt.Length -gt 500) { $excerpt = $excerpt.Substring(0, 500) + "...(truncated)" } } $hits += [pscustomobject]@{ path = $file.FullName lastWrite = $file.LastWriteTime.ToString("o") patterns = @($matchedPatterns) excerpt = $excerpt } } } } $status = if (@($hits).Count -gt 0) { "security_triage_candidate_created" } else { "security_triage_no_recent_alert_body_seen" } $triage = [pscustomobject]@{ timestamp = (Get-Date -Format o) triageId = $triageId status = $status lookbackMinutes = $lookbackMinutes since = $since.ToString("o") hitCount = @($hits).Count hits = @($hits | Sort-Object lastWrite -Descending | Select-Object -First 20) requiredReadback = @("source_log_reference", "affected_host_or_service", "blast_radius", "repeated_count", "allowlisted_next_action") allowedActions = @("classify", "read_existing_evidence", "write_km", "request_source_log_reference", "recommend_allowlisted_next_action") prohibitedActions = @("arbitrary_shell", "data_delete", "disable_security_control", "secret_exposure") evidencePath = $triagePath } $triage | ConvertTo-Json -Depth 10 | Set-Content -Path $triagePath -Encoding UTF8 Write-AgentLog "security_triage triage=$triageId hits=$(@($hits).Count) status=$status path=$triagePath" $eventSeverity = if (@($hits).Count -gt 0) { "warning" } else { "info" } Record-AgentEvent "security_triage" $eventSeverity "status=$status hits=$(@($hits).Count)" $triage -Alert $triage } function Invoke-AgentProviderFreshnessDirectReadback { param( [int]$LookbackMinutes ) $baseUrl = "https://awoooi.wooo.work/api/v1/platform/events/dossier/recurrence?project_id=awoooi&limit=20" $startedAt = Get-Date try { $response = Invoke-RestMethod -Method Get -Uri $baseUrl -TimeoutSec 12 $summary = if ($response -and $response.PSObject.Properties["summary"]) { $response.summary } else { $null } $latest = if ($summary -and $summary.PSObject.Properties["latest_received_at"]) { $summary.latest_received_at } else { $null } $sourceTotal = if ($summary -and $summary.PSObject.Properties["source_event_total"]) { [int]$summary.source_event_total } else { 0 } $itemsCount = if ($response -and $response.PSObject.Properties["items"]) { @($response.items).Count } else { 0 } return [pscustomobject]@{ ok = $true verifier = "awoooi-platform-events-recurrence" directReference = $baseUrl providerWindowMinutes = $LookbackMinutes latestReceivedAt = $latest sourceEventTotal = $sourceTotal recurrenceGroupTotal = if ($summary -and $summary.PSObject.Properties["recurrence_group_total"]) { [int]$summary.recurrence_group_total } else { 0 } itemsCount = $itemsCount httpStatus = 200 durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2) checkedAt = (Get-Date -Format o) } } catch { return [pscustomobject]@{ ok = $false verifier = "awoooi-platform-events-recurrence" directReference = $baseUrl providerWindowMinutes = $LookbackMinutes latestReceivedAt = $null sourceEventTotal = 0 recurrenceGroupTotal = 0 itemsCount = 0 httpStatus = $null durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2) checkedAt = (Get-Date -Format o) error = ($_ | Out-String).Trim() } } } function Invoke-AgentProviderFreshnessTriage { $providerConfig = if ($Config.PSObject.Properties["providerFreshness"]) { $Config.providerFreshness } else { $null } $lookbackMinutes = if ($providerConfig -and $providerConfig.PSObject.Properties["lookbackMinutes"]) { [int]$providerConfig.lookbackMinutes } else { 120 } $since = (Get-Date).AddMinutes(-1 * $lookbackMinutes) $candidateId = "provider-freshness-" + (Get-Date -Format "yyyyMMdd-HHmmss") $candidateDir = Join-Path $Config.agentRoot "state\provider-freshness-candidates" New-Item -ItemType Directory -Force -Path $candidateDir | Out-Null $candidatePath = Join-Path $candidateDir "$candidateId.json" $patterns = @( "provider_freshness_signal", "source_provider_freshness_triage", "raw_signal_normalized", "controlled_runtime_candidate", "freshness", "last_seen", "last seen", "ingestion" ) $scanDirs = @( (Join-Path $Config.agentRoot "alerts\incoming"), (Join-Path $Config.agentRoot "alerts\processed"), (Join-Path $Config.agentRoot "queue"), (Join-Path $Config.agentRoot "queue\processed"), (Join-Path $Config.agentRoot "requests") ) | Where-Object { Test-Path $_ } | Select-Object -Unique $hits = @() foreach ($dir in $scanDirs) { $files = @(Get-ChildItem -LiteralPath $dir -Recurse -File -ErrorAction SilentlyContinue | Where-Object { $_.LastWriteTime -ge $since -and $_.Length -lt 2097152 }) foreach ($file in $files) { $matchedPatterns = @() foreach ($pattern in $patterns) { if (Select-String -LiteralPath $file.FullName -Pattern $pattern -SimpleMatch -Quiet -ErrorAction SilentlyContinue) { $matchedPatterns += $pattern } } if (@($matchedPatterns).Count -gt 0) { $excerpt = "" $firstHit = Select-String -LiteralPath $file.FullName -Pattern $matchedPatterns[0] -SimpleMatch -ErrorAction SilentlyContinue | Select-Object -First 1 if ($firstHit) { $excerpt = [string]$firstHit.Line $excerpt = $excerpt -replace '("?(telegram|token|chat|authorization|password)[^,}\r\n]*"?\s*:\s*)"[^"]+"', '$1"[redacted]"' if ($excerpt.Length -gt 500) { $excerpt = $excerpt.Substring(0, 500) + "...(truncated)" } } $hits += [pscustomobject]@{ path = $file.FullName lastWrite = $file.LastWriteTime.ToString("o") patterns = @($matchedPatterns) excerpt = $excerpt } } } } $targets = @() if ($providerConfig -and $providerConfig.targets) { foreach ($target in @($providerConfig.targets)) { $targets += [pscustomobject]@{ name = if ($target.name) { [string]$target.name } else { "provider_freshness_signal" } providerWindowMinutes = if ($target.PSObject.Properties["providerWindowMinutes"]) { $target.providerWindowMinutes } else { $null } directReference = if ($target.directReference) { [string]$target.directReference } else { $null } verifier = if ($target.verifier) { [string]$target.verifier } else { "agent99-provider-freshness-triage" } } } } if (@($targets).Count -eq 0) { $targets += [pscustomobject]@{ name = "provider_freshness_signal" providerWindowMinutes = $null directReference = $null verifier = "agent99-provider-freshness-triage" } } $verifierReadback = Invoke-AgentProviderFreshnessDirectReadback -LookbackMinutes $lookbackMinutes $directReference = if ($verifierReadback -and $verifierReadback.PSObject.Properties["directReference"]) { $verifierReadback.directReference } else { $null } $latestSeen = if ($verifierReadback -and $verifierReadback.PSObject.Properties["latestReceivedAt"]) { $verifierReadback.latestReceivedAt } else { $null } $missingFields = @() if (-not $lookbackMinutes) { $missingFields += "providerWindow" } if (-not $latestSeen) { $missingFields += "lastSeen" } if (-not $directReference) { $missingFields += "directReference" } if (-not ($verifierReadback -and $verifierReadback.ok -eq $true)) { $missingFields += "verifierReadback" } $status = if ($verifierReadback -and $verifierReadback.ok -eq $true -and $latestSeen) { "direct_readback_ok" } elseif ($verifierReadback -and $verifierReadback.ok -eq $true) { "direct_readback_ok_no_source_events" } elseif (@($hits).Count -gt 0) { "candidate_created_waiting_direct_readback" } else { "candidate_created_no_alert_body_seen" } $candidate = [pscustomobject]@{ timestamp = (Get-Date -Format o) candidateId = $candidateId target = "provider_freshness_signal" status = $status lookbackMinutes = $lookbackMinutes since = $since.ToString("o") hitCount = @($hits).Count hits = @($hits | Sort-Object lastWrite -Descending | Select-Object -First 20) targets = $targets missingFields = $missingFields providerWindowMinutes = $lookbackMinutes lastSeen = $latestSeen directReference = $directReference verifierReadback = $verifierReadback requiredVerifierReadback = @("provider_window", "last_seen", "direct_reference", "source_trace_or_sentry_signoz_ref") noFalseGreenRisk = $true allowedActions = @("create_candidate", "read_existing_evidence", "provider_freshness_direct_readback", "request_direct_reference", "write_km") prohibitedActions = @("provider_switch", "paid_model_call", "env_change", "reload") candidatePath = $candidatePath } $candidate | ConvertTo-Json -Depth 10 | Set-Content -Path $candidatePath -Encoding UTF8 Write-AgentLog "provider_freshness candidate=$candidateId hits=$(@($hits).Count) status=$status missing=$($missingFields -join ',') path=$candidatePath" $eventSeverity = if (@($missingFields).Count -gt 0) { "warning" } else { "info" } Record-AgentEvent "provider_freshness_triage" $eventSeverity "target=provider_freshness_signal status=$status missing=$($missingFields -join ',')" $candidate -Alert $candidate } function Test-HostReachability { $results = @() foreach ($hostIp in $Config.hosts) { $ping = Test-Connection -ComputerName $hostIp -Count 1 -Quiet -ErrorAction SilentlyContinue Write-AgentLog "host host=$hostIp ping=$ping" $results += [pscustomobject]@{ host = $hostIp ping = [bool]$ping } } $results } function Test-PublicRoutes { $results = @() foreach ($url in $Config.publicUrls) { try { $output = & curl.exe --location --head --max-time 12 $url 2>&1 $text = $output -join "`n" $matches = [regex]::Matches($text, "HTTP/\S+\s+(\d+)") $status = if ($matches.Count -gt 0) { [int]$matches[$matches.Count - 1].Groups[1].Value } else { $null } $non502 = ($null -ne $status -and $status -ne 502) $ok = ($null -ne $status -and $status -ge 200 -and $status -lt 500) Write-AgentLog "public url=$url status=$status non502=$non502 ok=$ok" $results += [pscustomobject]@{ url = $url status = $status non502 = $non502 ok = $ok } } catch { Write-AgentLog "public url=$url error=$($_.Exception.Message)" $results += [pscustomobject]@{ url = $url status = $null non502 = $false ok = $false } } } $results } function Test-AiServices { $results = @() foreach ($svc in @($Config.aiServices)) { if (-not $svc.url) { continue } $name = if ($svc.name) { [string]$svc.name } else { [string]$svc.url } try { $output = & curl.exe --max-time 12 -sS $svc.url 2>&1 $text = ($output -join "`n").Trim() $statusValue = $null try { $json = $text | ConvertFrom-Json if ($json.status) { $statusValue = [string]$json.status } } catch { $statusValue = $null } $expectedStatus = if ($svc.expectedStatus) { [string]$svc.expectedStatus } else { "healthy" } $ok = ($statusValue -eq $expectedStatus) $severity = if ($ok) { "ok" } elseif ($svc.critical) { "critical" } else { "warning" } $result = [pscustomobject]@{ name = $name url = $svc.url host = $svc.host ok = $ok severity = $severity status = $statusValue expectedStatus = $expectedStatus output = $text } Write-AgentLog "ai_service name=$name ok=$ok severity=$severity status=$statusValue url=$($svc.url)" if (-not $ok) { Record-AgentEvent "ai_service_unhealthy" $severity "name=$name status=$statusValue expected=$expectedStatus url=$($svc.url)" $result -Alert } $results += $result } catch { $severity = if ($svc.critical) { "critical" } else { "warning" } $result = [pscustomobject]@{ name = $name url = $svc.url host = $svc.host ok = $false severity = $severity status = $null expectedStatus = if ($svc.expectedStatus) { [string]$svc.expectedStatus } else { "healthy" } output = $_.Exception.Message } Write-AgentLog "ai_service name=$name error=$($_.Exception.Message)" Record-AgentEvent "ai_service_unhealthy" $severity "name=$name error=$($_.Exception.Message) url=$($svc.url)" $result -Alert $results += $result } } $results } function Get-HarborState { $inspectCommand = "docker inspect --format '{{.Name}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' harbor-core redis registry" $inspect = Invoke-SshText "192.168.0.110" $inspectCommand 30 1 $containerStates = @{} foreach ($line in @(([string]$inspect.output) -split "`n")) { $trimmed = $line.Trim() if (-not $trimmed) { continue } $parts = @($trimmed -split "\|") if ($parts.Count -lt 3) { continue } $name = $parts[0].Trim().TrimStart("/") $containerStates[$name] = [pscustomobject]@{ name = $name status = $parts[1].Trim() health = $parts[2].Trim() } } $core = $containerStates["harbor-core"] $redis = $containerStates["redis"] $registry = $containerStates["registry"] $coreLine = if ($core) { "$($core.name) $($core.status) $($core.health)" } else { "harbor-core missing" } $redisLine = if ($redis) { "$($redis.name) $($redis.status) $($redis.health)" } else { "redis missing" } $registryLine = if ($registry) { "$($registry.name) $($registry.status) $($registry.health)" } else { "registry missing" } $state = [pscustomobject]@{ coreHealthy = ($core -and $core.status -eq "running" -and $core.health -eq "healthy") redisUp = ($redis -and $redis.status -eq "running") registryHealthy = ($registry -and $registry.status -eq "running" -and $registry.health -eq "healthy") coreOutput = $coreLine redisOutput = $redisLine registryOutput = $registryLine inspectExitCode = $inspect.exitCode healthOutput = $inspect.output inspectOutput = "[redacted: raw docker inspect omitted]" } Write-AgentLog "harbor coreHealthy=$($state.coreHealthy) redisUp=$($state.redisUp) registryHealthy=$($state.registryHealthy)" $state } function Get-AwoooState { $nestedSsh = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $SshUser 192.168.0.120" $readback = Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl get deploy,pods -n awoooi-prod" 60 1 $state = [pscustomobject]@{ deployReady = ( $readback.output -match "deployment.apps/awoooi-api\s+1/1" -and $readback.output -match "deployment.apps/awoooi-web\s+2/2" -and $readback.output -match "deployment.apps/awoooi-worker\s+1/1" ) imagePullFailure = ($readback.output -match "ImagePullBackOff|ErrImagePull") crashFailure = ($readback.output -match "CrashLoopBackOff|Pending") deployOutput = $readback.output podOutput = $readback.output readbackExitCode = $readback.exitCode } Write-AgentLog "awoooi deployReady=$($state.deployReady) imagePullFailure=$($state.imagePullFailure) crashFailure=$($state.crashFailure)" $state } function Find-Vmrun { if ($Config.vmrunPath -and (Test-Path $Config.vmrunPath)) { return $Config.vmrunPath } foreach ($candidate in @( "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe", "C:\Program Files\VMware\VMware Workstation\vmrun.exe" )) { if (Test-Path $candidate) { return $candidate } } return $null } function Start-ConfiguredVMs { $vmrun = Find-Vmrun $results = @() if (-not $vmrun) { Write-AgentLog "vmrun missing" return @([pscustomobject]@{ ok = $false; reason = "vmrun_missing" }) } foreach ($vm in $Config.vms) { if (-not $vm.vmx) { $results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "missing_vmx_path" } continue } $hostPing = $false if ($vm.host) { $hostPing = Test-Connection -ComputerName $vm.host -Count 1 -Quiet -ErrorAction SilentlyContinue } if ($hostPing) { $results += [pscustomobject]@{ name = $vm.name; ok = $true; skipped = $true; reason = "host_already_reachable" } continue } if (-not $ControlledApply) { $results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "controlled_apply_required" } continue } Write-AgentLog "CONTROLLED_APPLY vm_start name=$($vm.name) vmx=$($vm.vmx)" $output = & $vmrun start $vm.vmx nogui 2>&1 $results += [pscustomobject]@{ name = $vm.name ok = ($LASTEXITCODE -eq 0) skipped = $false output = ($output -join "`n") } } $results } function Repair-Harbor { param([object]$Harbor) if (-not $ControlledApply) { Write-AgentLog "harbor repair skipped controlled_apply=false" return $false } $changed = $false if (-not $Harbor.redisUp) { Write-AgentLog "CONTROLLED_APPLY harbor restart redis" Invoke-SshText "192.168.0.110" "docker restart redis" | Out-Null $changed = $true } if (-not $Harbor.coreHealthy) { Write-AgentLog "CONTROLLED_APPLY harbor restart harbor-core" Invoke-SshText "192.168.0.110" "docker restart harbor-core" | Out-Null $changed = $true } return $changed } function Repair-Awooo { param([object]$Awooo, [object]$Harbor) if (-not $ControlledApply) { Write-AgentLog "awoooi repair skipped controlled_apply=false" return $false } if (-not $Harbor.coreHealthy) { Write-AgentLog "awoooi repair skipped harbor_core_not_healthy" return $false } if (-not $Awooo.imagePullFailure) { Write-AgentLog "awoooi repair skipped no_image_pull_failure" return $false } Write-AgentLog "CONTROLLED_APPLY awoooi delete failed pods" $nestedSsh = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $SshUser 192.168.0.120" Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-api" 25 | Out-Null Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-web" 25 | Out-Null Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-worker" 25 | Out-Null return $true } if ($SelfTestTelegramCard) { Invoke-AgentTelegramCardSelfTest } if ($SelfTestTelegramDelivery) { Invoke-AgentTelegramDeliverySelfTest } if ($SelfTestSensorGate) { Invoke-AgentSensorGateSelfTest } if ($SelfTestOutcomeContract) { Invoke-AgentOutcomeContractSelfTest } Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert $vmResults = @() $hostResults = @() $harbor = $null $awooo = $null $public = @() $aiServices = @() $performance = @() $loadShedding = @() $selfHealth = $null $backupHealth = $null $providerFreshness = $null $securityTriage = $null $controlTick = $null if ($Mode -in @("Recover", "StartVMs")) { $vmResults = Start-ConfiguredVMs } if ($Mode -in @("Status", "Recover")) { $hostResults = Test-HostReachability } if ($Mode -in @("Status", "Recover", "HarborRepair")) { $harbor = Get-HarborState } if ($Mode -in @("Recover", "HarborRepair") -and $harbor) { $harborChanged = Repair-Harbor $harbor if ($harborChanged) { Start-Sleep -Seconds 3 $harbor = Get-HarborState } } if ($Mode -in @("Status", "Recover", "AwoooRepair")) { $awooo = Get-AwoooState } if ($Mode -in @("Recover", "AwoooRepair") -and $awooo) { if (-not $harbor) { $harbor = Get-HarborState } $awoooChanged = Repair-Awooo $awooo $harbor if ($awoooChanged) { Start-Sleep -Seconds 3 $awooo = Get-AwoooState } } if ($Mode -in @("Status", "Recover", "PublicSmoke")) { $public = Test-PublicRoutes } if ($Mode -in @("Status", "Recover", "Perf")) { $aiServices = Test-AiServices } if ($Mode -in @("Status", "Recover", "Perf", "LoadShed")) { $suppressPerfAlerts = [bool]($ControlledApply -and $Mode -in @("Perf", "LoadShed")) $performance = Get-HostPerformance -SuppressAlerts:$suppressPerfAlerts } if ($Mode -in @("Perf", "LoadShed")) { $loadShedding = Invoke-LoadShedding $performance if ($ControlledApply) { Record-PerformanceIssuesWithoutRemediation $performance $loadShedding } } if ($Mode -in @("SelfCheck")) { $selfHealth = Test-AgentSelfHealth } if ($Mode -in @("BackupCheck")) { $backupHealth = Test-AgentBackupHealth } if ($Mode -in @("ProviderFreshness")) { $providerFreshness = Invoke-AgentProviderFreshnessTriage } if ($Mode -in @("SecurityTriage")) { $securityTriage = Invoke-AgentSecurityTriage } if ($Mode -in @("ControlTick")) { $controlTick = Invoke-AgentControlTick } $publicFailures = @($public | Where-Object { $_ -and -not $_.ok }).Count $aiFailures = @($aiServices | Where-Object { $_ -and -not $_.ok }).Count $perfCritical = @($performance | Where-Object { $_ -and $_.severity -eq "critical" }).Count $perfWarning = @($performance | Where-Object { $_ -and $_.severity -eq "warning" }).Count $selfHealthCritical = if ($selfHealth -and $selfHealth.severity -eq "critical") { 1 } else { 0 } $selfHealthWarning = if ($selfHealth -and $selfHealth.severity -eq "warning") { 1 } else { 0 } $backupCritical = if ($backupHealth -and $backupHealth.severity -eq "critical") { 1 } else { 0 } $backupWarning = if ($backupHealth -and $backupHealth.severity -eq "warning") { 1 } else { 0 } $providerFreshnessWarning = if ($providerFreshness) { 1 } else { 0 } $securityTriageWarning = if ($securityTriage -and $securityTriage.hitCount -gt 0) { 1 } else { 0 } $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCritical -gt 0 -or $selfHealthCritical -gt 0 -or $backupCritical -gt 0) { "critical" } elseif ($perfWarning -gt 0 -or $selfHealthWarning -gt 0 -or $backupWarning -gt 0 -or $providerFreshnessWarning -gt 0 -or $securityTriageWarning -gt 0) { "warning" } else { "info" } Record-AgentEvent "agent_complete" $summarySeverity "mode=$Mode publicFailures=$publicFailures aiFailures=$aiFailures perfCritical=$perfCritical perfWarning=$perfWarning selfHealth=$($selfHealth.severity) backupHealth=$($backupHealth.severity) providerFreshness=$($providerFreshness.status) securityTriage=$($securityTriage.status) evidence=$jsonPath" $null -NoAlert $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) mode = $Mode controlledApply = [bool]$ControlledApply vmResults = $vmResults hosts = $hostResults harbor = $harbor awoooi = $awooo public = $public aiServices = $aiServices performance = $performance loadShedding = $loadShedding selfHealth = $selfHealth backupHealth = $backupHealth providerFreshness = $providerFreshness securityTriage = $securityTriage controlTick = $controlTick events = $script:Events telegram = $script:TelegramAttempts evidenceLog = $logPath } $result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath Write-AgentLog "agent99_complete json=$jsonPath" exit 0