diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 9b974a2e2..a79d77d30 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -221,7 +221,8 @@ function Send-AgentTelegramRelay { [string]$Message, [object]$Relay, [string]$ChatIdOverride = "", - [string]$PhotoPath = "" + [string]$PhotoPath = "", + [string]$ReplyToMessageId = "" ) if (-not ($Relay -and $Relay.enabled)) { @@ -273,13 +274,19 @@ function Send-AgentTelegramRelay { } $containerPhotoB64 = if ($containerPhotoPath) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($containerPhotoPath)) } else { "" } - $python = 'import base64, mimetypes, os, sys, urllib.error, urllib.parse, urllib.request + $python = 'import base64, json, mimetypes, os, sys, urllib.error, urllib.parse, urllib.request text = base64.b64decode(sys.argv[2]).decode() chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] and sys.argv[3] != "-" else "" photo_path = base64.b64decode(sys.argv[4]).decode() if len(sys.argv) > 4 and sys.argv[4] and sys.argv[4] != "-" else "" +reply_to = int(sys.argv[5]) if len(sys.argv) > 5 and sys.argv[5].isdigit() else None token = os.environ.get("TELEGRAM_BOT_TOKEN") chat = chat_override or os.environ.get("TELEGRAM_CHAT_ID") assert token and chat +reply_parameters = json.dumps({"message_id": reply_to}) if reply_to else None +def emit_receipt(payload): + result = payload.get("result") or {} + safe = {"message_id": result.get("message_id"), "chat_type": (result.get("chat") or {}).get("type")} + print("telegram_receipt_b64=" + base64.b64encode(json.dumps(safe).encode()).decode()) if photo_path and os.path.exists(photo_path): caption = text if len(caption) > 900: @@ -288,20 +295,26 @@ if photo_path and os.path.exists(photo_path): ctype = mimetypes.guess_type(filename)[0] or "image/png" try: import requests + request_data = {"chat_id": chat, "caption": caption} + if reply_parameters: + request_data["reply_parameters"] = reply_parameters with open(photo_path, "rb") as fh: response = requests.post( "https://api.telegram.org/bot" + token + "/sendPhoto", - data={"chat_id": chat, "caption": caption}, + data=request_data, files={"photo": (filename, fh, ctype)}, timeout=30, ) if not response.ok: print("photo_send_failed status=%s body=%s" % (response.status_code, response.text[:500])) sys.exit(2) + emit_receipt(response.json()) print("photo_sent_ok") except ImportError: boundary = "----agent99boundary" fields = [("chat_id", chat), ("caption", caption)] + if reply_parameters: + fields.append(("reply_parameters", reply_parameters)) body = bytearray() for name, value in fields: body.extend(("--" + boundary + "\r\n").encode()) @@ -318,23 +331,39 @@ if photo_path and os.path.exists(photo_path): req = urllib.request.Request("https://api.telegram.org/bot" + token + "/sendPhoto", data=bytes(body)) req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary) try: - urllib.request.urlopen(req, timeout=30).read() + response_payload = json.loads(urllib.request.urlopen(req, timeout=30).read().decode()) + emit_receipt(response_payload) print("photo_sent_ok") except urllib.error.HTTPError as exc: print("photo_send_failed status=%s body=%s" % (exc.code, exc.read().decode(errors="replace")[:500])) sys.exit(2) else: - data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode() - urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read() + fields = {"chat_id": chat, "text": text, "disable_web_page_preview": "true"} + if reply_parameters: + fields["reply_parameters"] = reply_parameters + data = urllib.parse.urlencode(fields).encode() + response_payload = json.loads(urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read().decode()) + emit_receipt(response_payload) print("sent_ok") ' $pythonB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($python)) $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))' $chatOverrideArg = if ($chatOverrideB64) { $chatOverrideB64 } else { "-" } $containerPhotoArg = if ($containerPhotoB64) { $containerPhotoB64 } else { "-" } - $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideArg $containerPhotoArg" + $replyArg = if ($ReplyToMessageId -match "^\d+$") { $ReplyToMessageId } else { "-" } + $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideArg $containerPhotoArg $replyArg" $run = Invoke-SshText $relayHost $remoteCommand 45 1 $photoSent = [bool]($run.ok -and $run.output -match "photo_sent_ok") + $messageId = $null + $receiptMatch = [regex]::Match([string]$run.output, "telegram_receipt_b64=([A-Za-z0-9+/=]+)") + if ($receiptMatch.Success) { + try { + $receiptJson = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($receiptMatch.Groups[1].Value)) | ConvertFrom-Json + $messageId = $receiptJson.message_id + } catch { + $messageId = $null + } + } if ($containerPhotoPath) { Invoke-SshText $relayHost "docker exec $container rm -f $containerPhotoPath" 20 1 | Out-Null } @@ -345,6 +374,8 @@ else: container = $container exitCode = $run.exitCode chatOverride = if ($ChatIdOverride) { "telegram-origin" } else { $null } + replyToMessageId = if ($ReplyToMessageId) { $ReplyToMessageId } else { $null } + messageId = $messageId photoSent = $photoSent photoError = $photoError output = $run.output @@ -389,7 +420,7 @@ function Format-AgentMetricForCard { "$Value$Suffix" } -function Format-AgentTelegramText { +function Format-AgentTelegramLegacyText { param( [string]$Severity, [string]$EventType, @@ -647,13 +678,415 @@ function Format-AgentTelegramText { $text } +function Get-AgentSha256Hex { + param([string]$Text) + $sha = [Security.Cryptography.SHA256]::Create() + try { + $bytes = $sha.ComputeHash([Text.Encoding]::UTF8.GetBytes([string]$Text)) + [BitConverter]::ToString($bytes).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Get-AgentIncidentCardModel { + param( + [string]$Severity, + [string]$EventType, + [string]$Message, + [object]$Data = $null + ) + + $hostName = [string](Get-AgentObjectValue $Data "host" (Get-AgentObjectValue $Data "alertHost" "")) + $modeName = [string](Get-AgentObjectValue $Data "mode" "") + $targetName = [string](Get-AgentObjectValue $Data "target" (Get-AgentObjectValue $Data "alertService" "")) + $alertId = [string](Get-AgentObjectValue $Data "alertId" "") + $correlationKey = [string](Get-AgentObjectValue $Data "correlationKey" "") + $problem = if ($Data -and $Data.PSObject.Properties["problem"]) { $Data.problem } else { $null } + $outcome = if ($Data -and $Data.PSObject.Properties["outcome"]) { $Data.outcome } else { $null } + $problemKey = [string](Get-AgentObjectValue $problem "key" "") + $outcomeState = [string](Get-AgentObjectValue $outcome "state" (Get-AgentObjectValue $Data "outcomeState" "")) + $verifierPassed = Convert-AgentBool (Get-AgentObjectValue $outcome "verifierPassed" (Get-AgentObjectValue $Data "verifierPassed" $false)) + $sourceResolved = Convert-AgentBool (Get-AgentObjectValue $outcome "sourceEventResolved" (Get-AgentObjectValue $Data "sourceEventResolved" $false)) + $executionOk = Convert-AgentBool (Get-AgentObjectValue $Data "executionOk" (Get-AgentObjectValue $Data "ok" $false)) + $postVerifyOk = Convert-AgentBool (Get-AgentObjectValue $Data "postVerifyOk" $false) + + $family = switch -Regex ($EventType) { + "^performance_|^load_shed_" { "host_performance"; break } + "^backup_" { "backup"; break } + "provider_freshness" { "provider_freshness"; break } + "^recovery_|^host_reboot_|^host112_guest_" { "recovery"; break } + "^operator_command_" { "automation_result"; break } + "^security_" { "security"; break } + "^ai_service_" { "ai_service"; break } + "^agent_selfcheck" { "agent_health"; break } + default { "platform" } + } + + $identityTarget = if ($problemKey) { + $problemKey + } elseif ($correlationKey) { + $correlationKey + } elseif ($alertId) { + $alertId + } elseif ($hostName) { + "$family`:$hostName" + } elseif ($targetName) { + "$family`:$targetName" + } else { + "$family`:$EventType" + } + $fingerprint = Get-AgentSha256Hex $identityTarget + $incidentId = "INC-" + $fingerprint.Substring(0, 8).ToUpperInvariant() + + $completed = Convert-AgentBool (Get-AgentObjectValue $Data "completed" $false) + $lifecycle = "detected" + if ($EventType -eq "recovery_slo_result") { + $lifecycle = if ($completed) { "recovered" } else { "remediating" } + } elseif ($EventType -match "recovered|resolved") { + $lifecycle = "recovered" + } elseif ($EventType -eq "load_shed_result") { + $lifecycle = if ($executionOk -and $postVerifyOk) { "recovered" } else { "blocked" } + } elseif ($outcomeState) { + $lifecycle = switch ($outcomeState) { + "resolved" { "recovered" } + "candidate_ready" { "investigating" } + "verifying" { "verifying" } + "degraded" { "remediating" } + "blocked" { "blocked" } + "failed" { "blocked" } + default { "investigating" } + } + } elseif ($EventType -match "triage|selfcheck") { + $lifecycle = "investigating" + } elseif ($EventType -match "result|attempt") { + $lifecycle = if ($executionOk -and ($postVerifyOk -or $verifierPassed)) { "recovered" } else { "verifying" } + } + + $lifecycleLabel = switch ($lifecycle) { + "detected" { "已偵測 / DETECTED" } + "investigating" { "調查中 / INVESTIGATING" } + "remediating" { "修復中 / REMEDIATING" } + "verifying" { "驗證中 / VERIFYING" } + "recovered" { "已恢復 / RECOVERED" } + "blocked" { "需介入 / ACTION REQUIRED" } + default { "狀態未知 / UNKNOWN" } + } + $severityLabel = switch ($Severity) { + "critical" { "重大 / CRITICAL" } + "warning" { "警告 / WARNING" } + "info" { "資訊 / INFO" } + default { ([string]$Severity).ToUpperInvariant() } + } + + $title = "Agent99 平台事件" + $target = if ($targetName) { $targetName } elseif ($hostName) { $hostName } else { "AWOOOI platform" } + $impact = "Agent99 已接手判讀;目前影響仍在確認。" + $action = "收集受控 evidence,匹配 allowlisted PlayBook,再執行獨立 verifier。" + $verification = "尚未形成可關閉事件的 verifier 結果。" + $visualKind = "lifecycle" + $metrics = @() + $gates = @() + $phases = @() + + if ($EventType -eq "recovery_slo_result") { + $scope = [string](Get-AgentObjectValue $Data "scope" "service_recovery") + $target = if ($scope -eq "full_host_reboot") { "全主機重啟" } else { "主機與服務恢復" } + $title = if ($completed) { "主機與服務已通過恢復驗證" } else { "主機與服務恢復尚未完成" } + $impact = if ($completed) { "目前服務 verifier 已通過;使用者流量已恢復。" } else { "仍有恢復階段未通過,維護狀態不可撤除。" } + $coordinatorVerified = Convert-AgentBool (Get-AgentObjectValue $Data "coordinatorVerified" $false) + $rebootSloClaimed = Convert-AgentBool (Get-AgentObjectValue $Data "rebootSloClaimed" $false) + $verification = if ($coordinatorVerified) { + if ($rebootSloClaimed) { "協調器與 fresh reboot SLO verifier 均已通過。" } else { "服務協調器已通過;本輪不是 fresh reboot drill,不宣稱全主機 SLA。" } + } else { + "協調器尚未通過,事件保持開啟。" + } + $action = if ($completed) { "完成獨立驗證並寫回 incident/KM receipt。" } else { "持續處理失敗 phase,維持 maintenance fallback。" } + $phases = if ($Data -and $Data.PSObject.Properties["phases"]) { @($Data.phases) } else { @() } + $visualKind = "recovery" + } elseif ($EventType -match "^performance_") { + $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() } + $load = Get-AgentObjectValue $Data "loadPerCore" $null + $memory = Get-AgentObjectValue $Data "memAvailablePercent" $null + $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null + $title = if ($reasons -contains "perf_readback_failed") { "$hostName 效能讀回失敗" } else { "$hostName 主機效能異常" } + $target = if ($hostName) { $hostName } else { "managed host" } + if ($reasons -contains "perf_readback_failed" -or ($null -eq $load -and $null -eq $memory -and $null -eq $disk)) { + $impact = "CPU、記憶體與磁碟數值目前不可信,不能把空白讀回當成主機故障。" + $action = "先修復監控 transport/readback,再決定是否需要降載。" + $verification = "等待完整三項主機指標重新讀回。" + } else { + $impact = "資源壓力可能降低服務回應速度或可用性。" + $action = "只執行 allowlisted 降載或清理,禁止任意停服務或重啟主機。" + $verification = "修復後必須重新讀回 CPU、記憶體與磁碟才可關閉。" + } + $metrics = @( + [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "磁碟使用"; value = $disk; scale = 100.0; suffix = "%" } + ) + $visualKind = "performance" + } elseif ($EventType -match "^backup_") { + $age = Get-AgentObjectValue $Data "ageMinutes" $null + $maxAge = Get-AgentObjectValue $Data "maxAgeMinutes" $null + $target = [string](Get-AgentObjectValue $Data "name" (Get-AgentObjectValue $Data "target" "備份系統")) + $title = "$target 備份健康異常" + $impact = "備份新鮮度或可還原性未被證明,災難復原保障可能下降。" + $action = "檢查 status、log metadata、snapshot metadata 與 restore drill receipt;不讀備份內容或 secrets。" + $verification = if ($null -ne $age -and $null -ne $maxAge) { "目前 age=$age 分鐘,SLA 上限=$maxAge 分鐘;仍需 integrity/restore verifier。" } else { "備份讀回不完整,事件保持開啟。" } + $metrics = @([pscustomobject]@{ name = "備份 age / SLA"; value = $age; scale = $maxAge; suffix = " 分鐘" }) + $visualKind = "backup" + } elseif ($EventType -match "provider_freshness") { + $missing = if ($Data -and $Data.PSObject.Properties["missingFields"]) { @($Data.missingFields) } else { @() } + $target = [string](Get-AgentObjectValue $Data "target" "provider freshness") + $title = "Provider freshness 證據不完整" + $impact = "觀測來源可能過期或 ingestion 中斷;route 200 或 dashboard up 不能視為健康。" + $action = "建立 freshness candidate,補 provider window、last seen、direct reference 與 verifier readback。" + $verification = if ($missing.Count -eq 0) { "四項 freshness evidence 已齊,等待來源事件解除。" } else { "仍缺 $($missing.Count) 項 freshness evidence,事件保持開啟。" } + foreach ($gate in @("providerWindow", "lastSeen", "directReference", "verifierReadback")) { + $gates += [pscustomobject]@{ name = $gate; passed = [bool]($gate -notin $missing) } + } + $visualKind = "freshness" + } elseif ($EventType -match "^operator_command_") { + $target = if ($targetName) { $targetName } elseif ($hostName) { $hostName } elseif ($modeName) { $modeName } else { "Agent99 task" } + $title = switch ($modeName) { + "Recover" { "主機與服務自動恢復結果" } + "StartVMs" { "VMware 虛擬主機恢復結果" } + "ProviderFreshness" { "Provider freshness 自動判讀結果" } + "BackupCheck" { "備份健康自動稽核結果" } + "Perf" { "主機效能自動判讀結果" } + "LoadShed" { "主機受控降載結果" } + "HarborRepair" { "Harbor 受控修復結果" } + "AwoooRepair" { "AWOOOI 受控修復結果" } + "SecurityTriage" { "資安事件自動判讀結果" } + default { "Agent99 自動化處置結果" } + } + $impact = switch ($modeName) { + "ProviderFreshness" { "資料來源 freshness 未完整證明前,產品健康狀態不可判綠。" } + "BackupCheck" { "備份 freshness 或 restore readiness 可能不符合 SLA。" } + "Perf" { "主機資源壓力可能影響服務延遲與穩定性。" } + "LoadShed" { "正在降低主機資源壓力,避免擴大使用者影響。" } + "Recover" { "一或多個主機、服務或公開路由未通過恢復 gate。" } + "StartVMs" { "必要 VM 未就緒會阻斷後續服務與網站恢復。" } + "SecurityTriage" { "資安訊號需確認 blast radius 與受影響資產。" } + default { "Agent99 已接手來源告警並執行受控處置。" } + } + $action = switch ($modeName) { + "ProviderFreshness" { "補齊四項 freshness evidence;不切 provider、不呼叫付費模型、不改 env。" } + "BackupCheck" { "讀回 freshness、cron、integrity 與 restore drill receipt;不讀備份內容。" } + "Perf" { "讀回 CPU、記憶體、磁碟;必要時只做 allowlisted 降載。" } + "LoadShed" { "執行 allowlisted 降載並立即做 post-apply verifier。" } + "Recover" { "依序驗證 VM、SSH、112 readiness、Harbor/K3s、網站與 full-SOP scorecard。" } + "StartVMs" { "檢查 VMX/lock/guest identity 後,只啟動 allowlisted VM。" } + "HarborRepair" { "檢查 core、Redis、Registry 後執行可回滾修復。" } + "AwoooRepair" { "檢查 K3s workload/image pull 後執行受控修復。" } + "SecurityTriage" { "分類 blast radius 與 source evidence,不執行任意 shell。" } + default { "執行 allowlisted Agent99 mode 並保留 execution/verifier receipt。" } + } + if ($outcomeState -eq "resolved" -and $verifierPassed -and $sourceResolved) { + $verification = "獨立 verifier 已通過,來源告警已解除。" + } elseif ($verifierPassed -and -not $sourceResolved) { + $verification = "修復 verifier 已通過,但來源告警尚未解除;事件保持開啟。" + } elseif ($outcomeState -eq "candidate_ready") { + $verification = "已建立受控修復候選,尚未執行。" + } elseif ($outcomeState -eq "verifying") { + $verification = "執行已完成,正在等待獨立 verifier 與來源事件回讀。" + } else { + $verification = "尚未通過獨立 verifier;不可標示已解決。" + } + $gates = @( + [pscustomobject]@{ name = "受控執行"; passed = [bool]((Get-AgentObjectValue $Data "exitCode" -1) -eq 0) }, + [pscustomobject]@{ name = "獨立驗證"; passed = $verifierPassed }, + [pscustomobject]@{ name = "來源解除"; passed = $sourceResolved } + ) + $visualKind = "automation" + } elseif ($EventType -match "^load_shed_") { + $target = if ($hostName) { $hostName } else { "managed host" } + $title = if ($lifecycle -eq "recovered") { "$target 降載完成" } else { "$target 降載未通過驗證" } + $impact = "主機資源壓力可能影響產品延遲或可用性。" + $action = "執行 allowlisted 降載,保留 rollback 與 cooldown receipt。" + $verification = if ($executionOk -and $postVerifyOk) { "執行與 post-verifier 均通過。" } else { "post-verifier 未通過,事件保持開啟且不建立成功 cooldown。" } + $visualKind = "automation" + } elseif ($EventType -match "^host112_guest_") { + $target = "192.168.0.112 Kali" + $title = if ($lifecycle -eq "recovered") { "Kali 112 已完整恢復" } else { "Kali 112 尚未完整開機" } + $impact = "Console、VMware Tools 或 Wazuh 未就緒會阻斷 AISOC 與重啟 SLA。" + $action = "只執行 graphical/LightDM/Tools/Indexer 的 bounded recovery,不改 VM power。" + $verification = if ($lifecycle -eq "recovered") { "Console、Wazuh、timer 與 guest-ready verifier 已通過。" } else { "至少一項 guest readiness gate 仍未通過。" } + $visualKind = "recovery" + } elseif ($EventType -match "^security_") { + $title = "資安訊號調查中" + $impact = "可能涉及受影響主機或服務,需先確認 blast radius。" + $action = "建立 no-secret triage、來源證據與 allowlisted next action。" + $verification = "需由獨立 security verifier 確認來源事件解除。" + } elseif ($EventType -eq "agent_selfcheck") { + $title = "Agent99 自我健康異常" + $target = "Windows 99 Agent99" + $impact = "Agent99 任務、evidence 或 Telegram transport 可能降級。" + $action = "檢查 task、manifest、evidence freshness 與 delivery,不把 reboot 當第一反應。" + $verification = "所有必要 task、runtime parity 與新鮮 evidence 通過後才關閉。" + } + + $occurrences = 1 + if ($problem -and $problem.PSObject.Properties["occurrences"]) { + $occurrences = [math]::Max(1, [int]$problem.occurrences) + } elseif ($Data -and $Data.PSObject.Properties["occurrenceCount"]) { + $occurrences = [math]::Max(1, [int]$Data.occurrenceCount) + } + $durationSeconds = Get-AgentObjectValue $Data "durationSeconds" (Get-AgentObjectValue $Data "elapsedSeconds" $null) + $durationText = if ($null -ne $durationSeconds -and [string]$durationSeconds -ne "") { + "$([math]::Round([double]$durationSeconds, 1)) 秒" + } else { + "計時中" + } + $nextUpdate = switch ($lifecycle) { + "recovered" { "事件已完成 verifier;再次發生會累計 recurrence 並重開生命週期。" } + "blocked" { "待補受控執行條件或人工核准;狀態改變時更新。" } + default { "狀態改變時更新;相同 fingerprint、相同狀態不重複推播。" } + } + + [pscustomobject]@{ + schemaVersion = "agent99_incident_card_v5" + incidentId = $incidentId + fingerprint = $fingerprint + identityKey = $identityTarget + eventType = $EventType + family = $family + severity = $Severity + severityLabel = $severityLabel + lifecycle = $lifecycle + lifecycleLabel = $lifecycleLabel + stateKey = "$lifecycle|$Severity" + title = Limit-AgentTextLine $title 96 + target = Limit-AgentTextLine $target 96 + impact = Limit-AgentTextLine $impact 240 + action = Limit-AgentTextLine $action 280 + verification = Limit-AgentTextLine $verification 240 + durationText = $durationText + occurrences = $occurrences + nextUpdate = $nextUpdate + visualKind = $visualKind + metrics = $metrics + gates = $gates + phases = $phases + } +} + +function Format-AgentTelegramText { + param( + [string]$Severity, + [string]$EventType, + [string]$Message, + [object]$Data = $null + ) + + try { + $card = Get-AgentIncidentCardModel $Severity $EventType $Message $Data + @( + "$($card.severityLabel) | $($card.lifecycleLabel)", + "$($card.title)", + "資產:$($card.target)", + "影響:$($card.impact)", + "Agent99:$($card.action)", + "驗證:$($card.verification)", + "耗時:$($card.durationText) | 發生:$($card.occurrences) 次", + "事件:$($card.incidentId)", + "下一次更新:$($card.nextUpdate)", + "時間:$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')" + ) -join "`n" + } catch { + Write-AgentLog "telegram_card_model_failed event=$EventType error=$($_.Exception.Message)" + Format-AgentTelegramLegacyText $Severity $EventType $Message $Data + } +} + +function Format-AgentTelegramCaption { + param( + [string]$Severity, + [string]$EventType, + [string]$Message, + [object]$Data = $null + ) + $card = Get-AgentIncidentCardModel $Severity $EventType $Message $Data + @( + "$($card.severityLabel) | $($card.lifecycleLabel) | $($card.incidentId)", + "$($card.title)", + "影響:$($card.impact)", + "目前:$($card.verification)", + "下一次更新:$($card.nextUpdate)" + ) -join "`n" +} + +function Get-AgentTelegramIncidentState { + param([object]$Card) + $stateDir = Join-Path (Join-Path $Config.agentRoot "state") "telegram-incidents" + New-Item -ItemType Directory -Force -Path $stateDir | Out-Null + $path = Join-Path $stateDir ("incident-" + $Card.fingerprint + ".json") + $value = $null + if (Test-Path -LiteralPath $path) { + try { + $value = Get-Content -LiteralPath $path -Raw | ConvertFrom-Json + } catch { + Write-AgentLog "telegram_incident_state_parse_failed incident=$($Card.incidentId)" + } + } + [pscustomobject]@{ + path = $path + exists = [bool]($null -ne $value) + value = $value + } +} + +function Save-AgentTelegramIncidentState { + param( + [object]$Card, + [object]$PreviousState, + [object]$MessageId, + [object]$ReplyToMessageId + ) + $state = Get-AgentTelegramIncidentState $Card + $previous = if ($PreviousState -and $PreviousState.value) { $PreviousState.value } else { $state.value } + $rootMessageId = if ($previous -and $previous.PSObject.Properties["rootMessageId"] -and $previous.rootMessageId) { + $previous.rootMessageId + } elseif ($ReplyToMessageId) { + $ReplyToMessageId + } else { + $MessageId + } + $payload = [pscustomobject]@{ + schemaVersion = "agent99_telegram_incident_state_v1" + incidentId = $Card.incidentId + fingerprint = $Card.fingerprint + lastStateKey = $Card.stateKey + lastLifecycle = $Card.lifecycle + lastSeverity = $Card.severity + rootMessageId = $rootMessageId + lastMessageId = $MessageId + occurrences = $Card.occurrences + updatedAt = (Get-Date -Format o) + } + $tempPath = $state.path + ".tmp-" + ([guid]::NewGuid().ToString("N")) + try { + $payload | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $tempPath -Encoding UTF8 + Move-Item -LiteralPath $tempPath -Destination $state.path -Force + return $state.path + } finally { + Remove-Item -LiteralPath $tempPath -Force -ErrorAction SilentlyContinue | Out-Null + } +} + function Get-AgentTelegramVisualPath { param([object]$Data) if (-not $Data) { return $null } + $visualEvidenceVerified = [bool]( + $Data.PSObject.Properties["visualEvidenceVerified"] -and + (Convert-AgentBool $Data.visualEvidenceVerified) + ) foreach ($propName in @("visualPath", "cardImagePath", "imagePath", "screenshotPath")) { if ($Data.PSObject.Properties[$propName]) { $path = [string]$Data.PSObject.Properties[$propName].Value - if ($path -and (Test-Path -LiteralPath $path)) { + $agentGenerated = [bool]($propName -eq "cardImagePath") + if ($path -and ($agentGenerated -or $visualEvidenceVerified) -and (Test-Path -LiteralPath $path)) { return $path } } @@ -692,6 +1125,8 @@ function New-AgentTelegramCardImage { ) if (($Severity -notin @("warning", "critical")) -and -not $forceVisual) { return $null } + $card = Get-AgentIncidentCardModel $Severity $EventType $Text $Data + $cardDir = Join-Path $EvidenceDir "telegram-cards" New-Item -ItemType Directory -Force -Path $cardDir | Out-Null $safeEvent = ($EventType -replace "[^A-Za-z0-9_.-]", "-") @@ -699,19 +1134,21 @@ function New-AgentTelegramCardImage { try { Add-Type -AssemblyName System.Drawing -ErrorAction Stop - $isRecoveryCard = [bool]($EventType -eq "recovery_slo_result") - $width = if ($isRecoveryCard) { 1200 } else { 1040 } - $height = if ($isRecoveryCard) { 760 } else { 620 } - $bitmap = New-Object System.Drawing.Bitmap -ArgumentList $width, $height + $width = 1200 + $height = 760 + $bitmap = [System.Drawing.Bitmap]::new($width, $height, [System.Drawing.Imaging.PixelFormat]::Format24bppRgb) $graphics = [System.Drawing.Graphics]::FromImage($bitmap) $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias $graphics.TextRenderingHint = [System.Drawing.Text.TextRenderingHint]::ClearTypeGridFit + $graphics.Clear([System.Drawing.Color]::FromArgb(246, 248, 250)) $bgBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(246, 248, 250)) $panelBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(255, 255, 255)) $textBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(28, 35, 43)) $mutedBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(91, 103, 112)) - $accentColor = if ($Severity -eq "critical") { + $accentColor = if ($card.lifecycle -eq "recovered") { + [System.Drawing.Color]::FromArgb(22, 101, 52) + } elseif ($Severity -eq "critical") { [System.Drawing.Color]::FromArgb(190, 18, 60) } elseif ($Severity -eq "warning") { [System.Drawing.Color]::FromArgb(217, 119, 6) @@ -719,97 +1156,113 @@ function New-AgentTelegramCardImage { [System.Drawing.Color]::FromArgb(37, 99, 235) } $accentBrush = New-Object System.Drawing.SolidBrush -ArgumentList $accentColor + $softBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(244, 246, 248)) + $goodBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(22, 101, 52)) + $badBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(190, 18, 60)) + $neutralBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(148, 163, 184)) + $barBackBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(226, 232, 240)) + $borderPen = New-Object System.Drawing.Pen -ArgumentList ([System.Drawing.Color]::FromArgb(218, 223, 230)), 1 $fontFamily = "Microsoft JhengHei UI" - $titleFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 25, ([System.Drawing.FontStyle]::Bold) + $titleFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 24, ([System.Drawing.FontStyle]::Bold) + $statusFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 21, ([System.Drawing.FontStyle]::Bold) $labelFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 13, ([System.Drawing.FontStyle]::Bold) - $bodyFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 16, ([System.Drawing.FontStyle]::Regular) + $bodyFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 15, ([System.Drawing.FontStyle]::Regular) $smallFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 11, ([System.Drawing.FontStyle]::Regular) $graphics.FillRectangle($bgBrush, 0, 0, $width, $height) $graphics.FillRectangle($panelBrush, 36, 34, $width - 72, $height - 68) $graphics.FillRectangle($accentBrush, 36, 34, 14, $height - 68) - $cardTitle = if ($isRecoveryCard) { "Agent99 恢復狀態 / Recovery Status" } else { "Agent99 事件卡 / Incident Card" } - $graphics.DrawString($cardTitle, $titleFont, $textBrush, 72, 58) - $graphics.DrawString(("嚴重度 Severity: " + $Severity.ToUpperInvariant()), $labelFont, $accentBrush, 72, 106) - $graphics.DrawString(("事件 Event: " + (Limit-AgentTextLine $EventType 76)), $labelFont, $mutedBrush, 72, 134) + $graphics.DrawString("Agent99 事故狀態 / INCIDENT STATUS", $titleFont, $textBrush, 72, 54) + $graphics.DrawString($card.lifecycleLabel, $statusFont, $accentBrush, 72, 96) + $graphics.DrawString(("$($card.severityLabel) | $($card.incidentId) | 發生 $($card.occurrences) 次"), $labelFont, $mutedBrush, 72, 137) + $graphics.DrawString($card.title, $titleFont, $textBrush, [System.Drawing.RectangleF]::new(72, 176, 1048, 46)) + $graphics.DrawString(("受影響資產:" + $card.target), $labelFont, $mutedBrush, 72, 224) - if ($isRecoveryCard) { - $completed = Convert-AgentBool (Get-AgentObjectValue $Data "completed" $false) - $withinSlo = Convert-AgentBool (Get-AgentObjectValue $Data "withinSlo" $false) - $scope = [string](Get-AgentObjectValue $Data "scope" "service_recovery") - $elapsedSeconds = [double](Get-AgentObjectValue $Data "elapsedSeconds" 0) - $targetMinutes = [int](Get-AgentObjectValue $Data "targetMinutes" 10) - $runId = [string](Get-AgentObjectValue $Data "runId" "unknown") - $coordinatorVerified = Convert-AgentBool (Get-AgentObjectValue $Data "coordinatorVerified" $false) - $statusText = if ($completed -and $withinSlo) { - "已恢復 / RECOVERED" - } elseif ($completed) { - "已恢復但逾時 / RECOVERED LATE" - } else { - "尚未恢復 / ACTION REQUIRED" - } - $statusColor = if ($completed -and $withinSlo) { - [System.Drawing.Color]::FromArgb(22, 101, 52) - } elseif ($completed) { - [System.Drawing.Color]::FromArgb(180, 83, 9) - } else { - [System.Drawing.Color]::FromArgb(190, 18, 60) - } - $statusBrush = New-Object System.Drawing.SolidBrush -ArgumentList $statusColor - $statusFont = New-Object System.Drawing.Font -ArgumentList $fontFamily, 23, ([System.Drawing.FontStyle]::Bold) - $scopeText = if ($scope -eq "full_host_reboot") { "全主機重啟" } else { "服務恢復" } - $graphics.DrawString($statusText, $statusFont, $statusBrush, 72, 176) - $graphics.DrawString(("範圍 " + $scopeText + " 耗時 " + $elapsedSeconds + " 秒 目標 " + $targetMinutes + " 分鐘"), $bodyFont, $textBrush, 72, 220) - $graphics.DrawString(("Verifier: coordinator=" + $coordinatorVerified + " Run: " + (Limit-AgentTextLine $runId 72)), $smallFont, $mutedBrush, 72, 256) - $graphics.DrawString("恢復階段 / RECOVERY PHASES", $labelFont, $mutedBrush, 72, 296) + $graphics.FillRectangle($softBrush, 72, 258, 1056, 82) + $graphics.DrawString("使用者影響", $labelFont, $accentBrush, 92, 274) + $graphics.DrawString($card.impact, $bodyFont, $textBrush, [System.Drawing.RectangleF]::new(92, 301, 1016, 34)) - $phaseY = 330 - $phaseIndex = 0 - $phases = if ($Data -and $Data.PSObject.Properties["phases"]) { @($Data.phases) } else { @() } - foreach ($phase in @($phases | Select-Object -First 7)) { - $phaseIndex += 1 - $phaseName = [string]$phase.name - $phaseLabel = switch ($phaseName) { - "vm_start" { "VMware 虛擬主機" } - "host_reachability" { "主機網路與 SSH" } - "host112_guest_readiness" { "Kali 112 Console 與安全服務" } - "harbor_registry" { "Harbor 與 Registry" } - "k3s_workloads" { "K3s 與 AWOOOI workloads" } - "public_routes" { "網站與公開路由" } - "full_sop_scorecard" { "Full-stack SOP scorecard" } - default { $phaseName } + $graphics.DrawString("Agent99 實際動作", $labelFont, $mutedBrush, 72, 366) + $graphics.DrawString($card.action, $bodyFont, $textBrush, [System.Drawing.RectangleF]::new(72, 394, 1056, 58)) + $graphics.DrawLine($borderPen, 72, 462, 1128, 462) + $graphics.DrawString("獨立驗證", $labelFont, $mutedBrush, 72, 480) + $graphics.DrawString($card.verification, $bodyFont, $textBrush, [System.Drawing.RectangleF]::new(72, 508, 1056, 54)) + + $graphics.DrawString(("耗時 " + $card.durationText), $labelFont, $textBrush, 930, 100) + $graphics.DrawString("證據視圖", $labelFont, $mutedBrush, 72, 578) + + if ($card.visualKind -eq "performance") { + $metricIndex = 0 + foreach ($metric in @($card.metrics | Select-Object -First 3)) { + $x = 72 + ($metricIndex * 350) + $metricIndex += 1 + $rawValue = 0.0 + $valid = [double]::TryParse([string]$metric.value, [ref]$rawValue) + $scale = if ($metric.scale -and [double]$metric.scale -gt 0) { [double]$metric.scale } else { 100.0 } + $ratio = if ($valid) { [math]::Min(1.0, [math]::Max(0.0, $rawValue / $scale)) } else { 0.0 } + $graphics.DrawString([string]$metric.name, $smallFont, $mutedBrush, $x, 608) + $graphics.FillRectangle($barBackBrush, $x, 638, 300, 18) + if ($valid) { $graphics.FillRectangle($accentBrush, $x, 638, [int](300 * $ratio), 18) } + $valueText = if ($valid) { "$rawValue$([string]$metric.suffix)" } else { "N/A" } + $graphics.DrawString($valueText, $labelFont, $(if ($valid) { $textBrush } else { $badBrush }), $x, 664) + } + } elseif ($card.visualKind -in @("freshness", "automation")) { + $gateRows = @($card.gates | Select-Object -First 4) + $gateIndex = 0 + foreach ($gate in $gateRows) { + $x = 72 + ($gateIndex * 260) + $gateIndex += 1 + $passed = Convert-AgentBool $gate.passed + $gateBrush = if ($passed) { $goodBrush } else { $badBrush } + $gateLabel = switch ([string]$gate.name) { + "providerWindow" { "Provider window" } + "lastSeen" { "Last seen" } + "directReference" { "Direct reference" } + "verifierReadback" { "Verifier readback" } + default { [string]$gate.name } } - $phaseOk = [bool]([string]$phase.status -eq "ok") - $phaseColor = if ($phaseOk) { [System.Drawing.Color]::FromArgb(22, 101, 52) } else { [System.Drawing.Color]::FromArgb(190, 18, 60) } - $phaseBrush = New-Object System.Drawing.SolidBrush -ArgumentList $phaseColor - $rowBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(244, 246, 248)) - $graphics.FillRectangle($rowBrush, 72, $phaseY, $width - 144, 38) - $graphics.FillRectangle($phaseBrush, 72, $phaseY, 10, 38) - $graphics.DrawString(("$phaseIndex. " + (Limit-AgentTextLine $phaseLabel 48)), $bodyFont, $textBrush, 98, ($phaseY + 5)) - $phaseState = if ($phaseOk) { "VERIFIED" } else { "FAILED" } - $phaseDuration = [math]::Round([double]$phase.elapsedSeconds, 1) - $graphics.DrawString(("$phaseState ${phaseDuration}s"), $labelFont, $phaseBrush, ($width - 300), ($phaseY + 8)) - $phaseBrush.Dispose() - $rowBrush.Dispose() - $phaseY += 46 + $graphics.FillRectangle($softBrush, $x, 612, 230, 62) + $graphics.FillRectangle($gateBrush, $x, 612, 8, 62) + $graphics.DrawString((Limit-AgentTextLine $gateLabel 22), $smallFont, $textBrush, ($x + 20), 622) + $graphics.DrawString($(if ($passed) { "已通過 / PASS" } else { "未通過 / OPEN" }), $labelFont, $gateBrush, ($x + 20), 646) } - if ($phases.Count -eq 0) { - $graphics.DrawString("尚無 phase evidence,事件不可視為完成。", $bodyFont, $accentBrush, 72, 338) + } elseif ($card.visualKind -eq "backup" -and @($card.metrics).Count -gt 0) { + $metric = @($card.metrics)[0] + $rawValue = 0.0 + $scaleValue = 0.0 + $valid = [double]::TryParse([string]$metric.value, [ref]$rawValue) -and [double]::TryParse([string]$metric.scale, [ref]$scaleValue) -and $scaleValue -gt 0 + $ratio = if ($valid) { [math]::Min(1.0, [math]::Max(0.0, $rawValue / $scaleValue)) } else { 0.0 } + $graphics.FillRectangle($barBackBrush, 72, 630, 850, 22) + if ($valid) { $graphics.FillRectangle($accentBrush, 72, 630, [int](850 * $ratio), 22) } + $graphics.DrawString($(if ($valid) { "$rawValue / $scaleValue 分鐘" } else { "freshness readback unavailable" }), $labelFont, $textBrush, 72, 662) + } elseif ($card.visualKind -eq "recovery" -and @($card.phases).Count -gt 0) { + $phaseRows = @($card.phases | Select-Object -First 7) + $phaseWidth = [math]::Floor(1030 / [math]::Max(1, $phaseRows.Count)) + $phaseIndex = 0 + foreach ($phase in $phaseRows) { + $x = 72 + ($phaseIndex * $phaseWidth) + $phaseIndex += 1 + $passed = [bool]([string]$phase.status -eq "ok") + $phaseBrush = if ($passed) { $goodBrush } else { $badBrush } + $graphics.FillRectangle($phaseBrush, $x, 620, ($phaseWidth - 10), 12) + $graphics.DrawString((Limit-AgentTextLine ([string]$phase.name) 18), $smallFont, $textBrush, $x, 643) + $graphics.DrawString($(if ($passed) { "PASS" } else { "OPEN" }), $smallFont, $phaseBrush, $x, 668) } - $graphics.DrawString("同一 run 保留 Detect -> Apply -> Verify -> Notify receipt", $smallFont, $mutedBrush, 72, 706) - $statusBrush.Dispose() - $statusFont.Dispose() } else { - $bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 事件卡 / Incident Card$" } | Select-Object -First 12) - $y = 184 - foreach ($line in $bodyLines) { - $graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y) - $y += 32 - if ($y -gt 545) { break } + $states = @("detected", "investigating", "remediating", "verifying", "recovered") + $stateIndex = 0 + foreach ($state in $states) { + $x = 72 + ($stateIndex * 205) + $stateIndex += 1 + $active = [bool]($state -eq $card.lifecycle) + $stateBrush = if ($active) { $accentBrush } else { $neutralBrush } + $graphics.FillRectangle($stateBrush, $x, 630, 170, 10) + $graphics.DrawString($state.ToUpperInvariant(), $smallFont, $stateBrush, $x, 651) } } - $footerY = if ($isRecoveryCard) { 724 } else { 570 } - $graphics.DrawString(("產生時間 Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, $footerY) + + $graphics.DrawString(("下一次更新:" + $card.nextUpdate), $smallFont, $mutedBrush, [System.Drawing.RectangleF]::new(72, 698, 850, 30)) + $graphics.DrawString((Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz"), $smallFont, $mutedBrush, 930, 704) $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png) $graphics.Dispose() @@ -819,7 +1272,14 @@ function New-AgentTelegramCardImage { $textBrush.Dispose() $mutedBrush.Dispose() $accentBrush.Dispose() + $softBrush.Dispose() + $goodBrush.Dispose() + $badBrush.Dispose() + $neutralBrush.Dispose() + $barBackBrush.Dispose() + $borderPen.Dispose() $titleFont.Dispose() + $statusFont.Dispose() $labelFont.Dispose() $bodyFont.Dispose() $smallFont.Dispose() @@ -835,7 +1295,8 @@ function Send-AgentTelegramPhotoDirect { [string]$Token, [string]$ChatId, [string]$Caption, - [string]$PhotoPath + [string]$PhotoPath, + [string]$ReplyToMessageId = "" ) Add-Type -AssemblyName System.Net.Http -ErrorAction Stop @@ -849,6 +1310,10 @@ function Send-AgentTelegramPhotoDirect { $captionToSend = $captionToSend.Substring(0, 900) + "`n...(truncated; see evidence)" } $content.Add([System.Net.Http.StringContent]::new($captionToSend), "caption") + if ($ReplyToMessageId -match "^\d+$") { + $replyJson = @{ message_id = [long]$ReplyToMessageId } | ConvertTo-Json -Compress + $content.Add([System.Net.Http.StringContent]::new($replyJson), "reply_parameters") + } $fileBytes = [IO.File]::ReadAllBytes($PhotoPath) $fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes) $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png") @@ -858,6 +1323,12 @@ function Send-AgentTelegramPhotoDirect { if (-not $response.IsSuccessStatusCode) { throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body" } + $payload = $body | ConvertFrom-Json + return [pscustomobject]@{ + ok = [bool]$payload.ok + messageId = if ($payload.result) { $payload.result.message_id } else { $null } + replyToMessageId = if ($ReplyToMessageId) { $ReplyToMessageId } else { $null } + } } finally { if ($fileContent) { $fileContent.Dispose() } if ($content) { $content.Dispose() } @@ -872,7 +1343,7 @@ function Invoke-AgentTelegramCardSelfTest { evidencePath = $jsonPath } $text = Format-AgentTelegramText "warning" "performance_warning" "selftest visual card" $sample - $imagePath = New-AgentTelegramCardImage "warning" "performance_warning" $text + $imagePath = New-AgentTelegramCardImage "warning" "performance_warning" $text $sample $ok = [bool]($imagePath -and (Test-Path -LiteralPath $imagePath)) $operatorSample = [pscustomobject]@{ id = "selftest-sre-result" @@ -907,16 +1378,56 @@ function Invoke-AgentTelegramCardSelfTest { } } $operatorText = Format-AgentTelegramText "info" "operator_command_result" "selftest SRE completion card" $operatorSample + $operatorCaption = Format-AgentTelegramCaption "info" "operator_command_result" "selftest SRE completion card" $operatorSample + $operatorModel = Get-AgentIncidentCardModel "info" "operator_command_result" "selftest SRE completion card" $operatorSample $operatorImagePath = New-AgentTelegramCardImage "info" "operator_command_result" $operatorText $operatorSample $operatorDecisionOk = Test-AgentSreOperatorResultAlertEnabled $operatorSample $operatorTextOk = [bool]( - $operatorText -match "已驗證解決 / verified resolved" -and - $operatorText -match "Agent99 動作 Action" -and - $operatorText -match "問題計數 Problem count" -and - $operatorText -match "KM:" + $operatorText -match "已恢復 / RECOVERED" -and + $operatorText -match "Agent99:" -and + $operatorText -match "驗證:" -and + $operatorText -match "事件:INC-" -and + $operatorText -notmatch "C:\\" -and + $operatorText -notmatch "KM:" -and + $operatorText -notmatch "verifierName=" -and + $operatorCaption.Length -lt 900 -and + $operatorModel.lifecycle -eq "recovered" -and + $operatorModel.occurrences -eq 2 ) $operatorVisualOk = [bool]($operatorImagePath -and (Test-Path -LiteralPath $operatorImagePath)) - $ok = [bool]($ok -and $operatorTextOk -and $operatorVisualOk -and $operatorDecisionOk) + $stateTestOk = $false + $stateSuppressionReason = "" + $stateRootMessageId = $null + $originalAgentRoot = [string]$Config.agentRoot + $originalSuppressAlerts = $script:SuppressAgentAlerts + $originalTelegramEnabled = if ($Config.telegram -and $Config.telegram.PSObject.Properties["enabled"]) { [bool]$Config.telegram.enabled } else { $false } + $stateTestRoot = Join-Path $EvidenceDir "telegram-lifecycle-selftest" + try { + $Config.agentRoot = $stateTestRoot + $script:SuppressAgentAlerts = $false + if ($Config.telegram -and $Config.telegram.PSObject.Properties["enabled"]) { $Config.telegram.enabled = $false } + $stateBefore = Get-AgentTelegramIncidentState $operatorModel + Save-AgentTelegramIncidentState $operatorModel $stateBefore 888 777 | Out-Null + $stateAfter = Get-AgentTelegramIncidentState $operatorModel + $attemptCountBefore = @($script:TelegramAttempts).Count + Record-AgentEvent "operator_command_result" "info" "selftest lifecycle suppression" $operatorSample -Alert + $newAttempts = @($script:TelegramAttempts | Select-Object -Skip $attemptCountBefore) + $stateSuppressionReason = if ($newAttempts.Count -gt 0) { [string]$newAttempts[-1].reason } else { "" } + $stateRootMessageId = if ($stateAfter.value) { $stateAfter.value.rootMessageId } else { $null } + $stateTestOk = [bool]( + $stateAfter.exists -and + [string]$stateAfter.value.lastStateKey -eq $operatorModel.stateKey -and + [string]$stateRootMessageId -eq "777" -and + $stateSuppressionReason -eq "same_lifecycle_state" -and + @($newAttempts | Where-Object { $_.sent }).Count -eq 0 + ) + } finally { + $Config.agentRoot = $originalAgentRoot + $script:SuppressAgentAlerts = $originalSuppressAlerts + if ($Config.telegram -and $Config.telegram.PSObject.Properties["enabled"]) { $Config.telegram.enabled = $originalTelegramEnabled } + Remove-Item -LiteralPath $stateTestRoot -Recurse -Force -ErrorAction SilentlyContinue | Out-Null + } + $ok = [bool]($ok -and $operatorTextOk -and $operatorVisualOk -and $operatorDecisionOk -and $stateTestOk) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $ok @@ -924,12 +1435,18 @@ function Invoke-AgentTelegramCardSelfTest { visualPath = $imagePath visualExists = $ok operatorTextPreview = @($operatorText -split "`n" | Select-Object -First 14) + operatorCaptionPreview = @($operatorCaption -split "`n" | Select-Object -First 8) + operatorIncidentId = $operatorModel.incidentId + operatorLifecycle = $operatorModel.lifecycle operatorVisualPath = $operatorImagePath operatorVisualExists = $operatorVisualOk operatorTextOk = $operatorTextOk operatorDecisionOk = $operatorDecisionOk + lifecycleStateTestOk = $stateTestOk + lifecycleSuppressionReason = $stateSuppressionReason + lifecycleRootMessageId = $stateRootMessageId sentTelegram = $false - messageFormat = "incident_card_zh_tw_v4" + messageFormat = "incident_card_zh_tw_v5" } $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramCardSelfTest-$stamp.json" $result | ConvertTo-Json -Depth 6 | Set-Content -Path $selfTestPath -Encoding UTF8 @@ -946,14 +1463,15 @@ function Invoke-AgentTelegramDeliverySelfTest { testId = $testId visualPath = $null } - Send-AgentTelegram "warning" "telegram_delivery_test" "TEST_DO_NOT_PAGE bilingual Telegram incident card delivery self-test" $data + $deliveryAttempt = Send-AgentTelegram "warning" "telegram_delivery_test" "TEST_DO_NOT_PAGE bilingual Telegram incident card delivery self-test" $data $sent = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $sent testId = $testId sentTelegram = $sent - messageFormat = "incident_card_zh_tw_v4" + deliveryAttempt = $deliveryAttempt + messageFormat = "incident_card_zh_tw_v5" attempts = $script:TelegramAttempts } $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramDeliverySelfTest-$stamp.json" @@ -1085,17 +1603,30 @@ function Send-AgentTelegram { $token = if ($tokenSecret) { $tokenSecret.value } else { $null } $chatId = if ($chatSecret) { $chatSecret.value } else { $null } $chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" } + $card = Get-AgentIncidentCardModel $Severity $EventType $Message $Data + $incidentState = Get-AgentTelegramIncidentState $card + $explicitReplyMessageId = if ($Data -and $Data.PSObject.Properties["replyMessageId"]) { [string]$Data.replyMessageId } else { "" } + $storedRootMessageId = if ($incidentState.value -and $incidentState.value.PSObject.Properties["rootMessageId"]) { [string]$incidentState.value.rootMessageId } else { "" } + $replyToMessageId = if ($explicitReplyMessageId -match "^\d+$") { $explicitReplyMessageId } elseif ($storedRootMessageId -match "^\d+$") { $storedRootMessageId } else { "" } $attempt = [pscustomobject]@{ timestamp = (Get-Date -Format o) enabled = [bool]$enabled eventType = $EventType severity = $Severity - messageFormat = "incident_card_zh_tw_v4" + messageFormat = "incident_card_zh_tw_v5" + incidentId = $card.incidentId + fingerprint = $card.fingerprint + lifecycle = $card.lifecycle + stateKey = $card.stateKey configured = [bool]($token -and $chatId) tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null } chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null } chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null } + replyToMessageId = if ($replyToMessageId) { $replyToMessageId } else { $null } + messageId = $null + incidentStatePath = $incidentState.path + incidentStateWritten = $false visualPath = $null visualSent = $false visualError = $null @@ -1107,19 +1638,22 @@ function Send-AgentTelegram { if (-not $enabled) { $attempt.error = "telegram_disabled" $script:TelegramAttempts += $attempt - return + return $attempt } $text = Format-AgentTelegramText $Severity $EventType $Message $Data + $caption = Format-AgentTelegramCaption $Severity $EventType $Message $Data $visualPath = Get-AgentTelegramVisualPath $Data if (-not $visualPath) { $visualPath = New-AgentTelegramCardImage $Severity $EventType $text $Data } $attempt.visualPath = $visualPath if (-not ($token -and ($chatId -or $chatIdOverride))) { - $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride $visualPath + $relayText = if ($visualPath) { $caption } else { $text } + $relayResult = Send-AgentTelegramRelay $relayText $telegram.relay $chatIdOverride $visualPath $replyToMessageId $attempt.relay = $relayResult if ($relayResult.ok) { $attempt.sent = $true + $attempt.messageId = $relayResult.messageId $attempt.error = $null $attempt.visualSent = [bool]$relayResult.photoSent if ($visualPath -and -not $relayResult.photoSent) { @@ -1128,40 +1662,57 @@ function Send-AgentTelegram { } else { $attempt.error = "telegram_not_configured" } + if ($attempt.sent) { + $attempt.incidentStatePath = Save-AgentTelegramIncidentState $card $incidentState $attempt.messageId $replyToMessageId + $attempt.incidentStateWritten = $true + } $script:TelegramAttempts += $attempt - return + return $attempt } try { $targetChat = if ($chatIdOverride) { $chatIdOverride } else { $chatId } if ($visualPath -and (Test-Path -LiteralPath $visualPath)) { try { - Send-AgentTelegramPhotoDirect $token $targetChat $text $visualPath + $photoResult = Send-AgentTelegramPhotoDirect $token $targetChat $caption $visualPath $replyToMessageId $attempt.sent = $true $attempt.visualSent = $true + $attempt.messageId = $photoResult.messageId } catch { $attempt.visualError = $_.Exception.Message Write-AgentLog "telegram_sendPhoto_failed event=$EventType fallback=sendMessage error=$($_.Exception.Message)" - Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ + $fallbackBody = @{ chat_id = $targetChat text = $text disable_web_page_preview = "true" - } | Out-Null + } + if ($replyToMessageId) { $fallbackBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } + $fallbackResult = Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body $fallbackBody $attempt.sent = $true + $attempt.messageId = if ($fallbackResult.result) { $fallbackResult.result.message_id } else { $null } } } else { - Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ + $textBody = @{ chat_id = $targetChat text = $text disable_web_page_preview = "true" - } | Out-Null + } + if ($replyToMessageId) { $textBody.reply_parameters = (@{ message_id = [long]$replyToMessageId } | ConvertTo-Json -Compress) } + $textResult = Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body $textBody $attempt.sent = $true + $attempt.messageId = if ($textResult.result) { $textResult.result.message_id } else { $null } } } catch { $attempt.error = $_.Exception.Message } + if ($attempt.sent) { + $attempt.incidentStatePath = Save-AgentTelegramIncidentState $card $incidentState $attempt.messageId $replyToMessageId + $attempt.incidentStateWritten = $true + } + $script:TelegramAttempts += $attempt + return $attempt } function Record-AgentEvent { @@ -1271,11 +1822,32 @@ function Record-AgentEvent { $lifecycleInfoAlert = [bool]($Alert -and $Severity -eq "info" -and $EventType -in @("recovery_slo_result")) $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or $lifecycleInfoAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo)) if ($shouldAlert) { + $incidentCard = Get-AgentIncidentCardModel $Severity $EventType $Message $Data + $incidentState = Get-AgentTelegramIncidentState $incidentCard + $forceLifecycleNotification = [bool]($Data -and $Data.PSObject.Properties["forceLifecycleNotification"] -and (Convert-AgentBool $Data.forceLifecycleNotification)) + if (-not $forceLifecycleNotification -and $incidentState.value -and [string]$incidentState.value.lastStateKey -eq $incidentCard.stateKey) { + Write-AgentLog "telegram_lifecycle_suppressed incident=$($incidentCard.incidentId) state=$($incidentCard.stateKey)" + $script:TelegramAttempts += [pscustomobject]@{ + timestamp = (Get-Date -Format o) + eventType = $EventType + severity = $Severity + messageFormat = "incident_card_zh_tw_v5" + incidentId = $incidentCard.incidentId + fingerprint = $incidentCard.fingerprint + lifecycle = $incidentCard.lifecycle + stateKey = $incidentCard.stateKey + sent = $false + suppressed = $true + reason = "same_lifecycle_state" + } + return + } $dedupeMinutes = 30 if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) { $dedupeMinutes = [int]$Config.telegram.dedupeMinutes } $dedupeParts = @($EventType, $Severity) + $dedupeParts += "lifecycle=$($incidentCard.stateKey)" if ($Data) { foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source", "scope", "completed", "withinSlo", "rebootSloClaimed")) { if ($Data.PSObject.Properties[$propName]) { @@ -1311,7 +1883,10 @@ function Record-AgentEvent { timestamp = (Get-Date -Format o) eventType = $EventType severity = $Severity - messageFormat = "incident_card_zh_tw_v4" + messageFormat = "incident_card_zh_tw_v5" + incidentId = $incidentCard.incidentId + fingerprint = $incidentCard.fingerprint + lifecycle = $incidentCard.lifecycle sent = $false suppressed = $true reason = "dedupe_window" @@ -1321,8 +1896,10 @@ function Record-AgentEvent { return } } - Set-Content -Path $dedupePath -Value (Get-Date -Format o) -Encoding UTF8 - Send-AgentTelegram $Severity $EventType $Message $Data + $telegramAttempt = Send-AgentTelegram $Severity $EventType $Message $Data + if ($telegramAttempt -and $telegramAttempt.sent) { + Set-Content -Path $dedupePath -Value (Get-Date -Format o) -Encoding UTF8 + } } } @@ -3854,6 +4431,7 @@ function Invoke-AgentQueuedCommands { $commandSource = if ($command.PSObject.Properties["source"]) { [string]$command.source } else { "queue" } $commandReason = if ($command.PSObject.Properties["reason"]) { [string]$command.reason } else { "operator_instruction" } $commandInstruction = if ($command.PSObject.Properties["instruction"]) { [string]$command.instruction } else { "" } + $correlationKey = if ($command.PSObject.Properties["correlationKey"]) { [string]$command.correlationKey } else { "" } $alertId = if ($command.PSObject.Properties["alertId"]) { [string]$command.alertId } else { $null } $alertSource = if ($command.PSObject.Properties["alertSource"]) { [string]$command.alertSource } else { $null } $alertKind = if ($command.PSObject.Properties["alertKind"]) { [string]$command.alertKind } else { $null } @@ -3865,6 +4443,7 @@ function Invoke-AgentQueuedCommands { $sourceEventResolved = if ($command.PSObject.Properties["sourceEventResolved"]) { [object](Convert-AgentBool $command.sourceEventResolved) } else { $null } $sourceEventEvidence = if ($command.PSObject.Properties["sourceEventEvidence"]) { [string]$command.sourceEventEvidence } else { "" } $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" } + $replyMessageId = if ($command.PSObject.Properties["replyMessageId"]) { [string]$command.replyMessageId } else { "" } $suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false } if ($modeName -notin $allowedModes) { $target = Join-Path $failedDir ("failed-" + $file.Name) @@ -3910,6 +4489,7 @@ function Invoke-AgentQueuedCommands { source = $commandSource reason = $commandReason instruction = $commandInstruction + correlationKey = $correlationKey alertId = $alertId alertSource = $alertSource alertKind = $alertKind @@ -3939,6 +4519,9 @@ function Invoke-AgentQueuedCommands { if ($replyChatId) { $recordData | Add-Member -MemberType NoteProperty -Name replyChatId -Value $replyChatId -Force } + if ($replyMessageId) { + $recordData | Add-Member -MemberType NoteProperty -Name replyMessageId -Value $replyMessageId -Force + } $outcomeSeverity = if ($outcome.state -eq "resolved") { "info" } elseif ($outcome.state -eq "failed") { "critical" } else { "warning" } Record-AgentEvent "operator_command_result" $outcomeSeverity "id=$id mode=$modeName outcome=$($outcome.state) transportExitCode=$exitCode verifierPassed=$($outcome.verifierPassed) sourceEventResolved=$($outcome.sourceEventResolved) evidence=$($latest.path)" $recordData -Alert $results += $result diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index 2fba0304d..584ed271a 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -742,6 +742,7 @@ foreach ($file in $files) { $service = [string](Get-AgentField $alert "service" "unknown") $hostName = [string](Get-AgentField $alert "host" "") $replyChatId = [string](Get-AgentField $alert "replyChatId" "") + $replyMessageId = [string](Get-AgentField $alert "telegramMessageId" "") $suppressTelegram = Test-AgentAlertSuppressTelegram $alert $resolution = Get-AgentField $alert "resolution" $null $sourceEventResolutionPolicy = [string](Get-AgentField $resolution "policy" "external_source_receipt_required") @@ -768,6 +769,7 @@ foreach ($file in $files) { alertSeverity = $severity alertService = $service alertHost = $hostName + replyMessageId = $replyMessageId alertPath = $processedPath suppressTelegram = $suppressTelegram sourceEventResolutionPolicy = $sourceEventResolutionPolicy @@ -796,6 +798,7 @@ foreach ($file in $files) { alertHost = $hostName alertPath = $processedPath replyChatId = $replyChatId + replyMessageId = $replyMessageId suppressTelegram = $suppressTelegram sourceEventResolutionPolicy = $sourceEventResolutionPolicy sourceEventResolutionRequired = $sourceEventResolutionRequired diff --git a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py index 4d00674bd..dc1f64a4d 100644 --- a/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py +++ b/apps/api/tests/test_agent99_enterprise_ai_automation_work_items_api.py @@ -40,9 +40,9 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None: "p0_count": 14, "p1_count": 8, "p2_count": 3, - "in_progress_count": 9, + "in_progress_count": 10, "in_progress_blocked_count": 0, - "planned_count": 15, + "planned_count": 14, "complete_count": 1, "current_p0_count": 1, } diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index a2ac08670..25c589b56 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -8,6 +8,7 @@ ROOT = Path(__file__).resolve().parents[3] CONTROL = ROOT / "agent99-control-plane.ps1" DEPLOY = ROOT / "agent99-deploy.ps1" BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" +SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1" def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None: @@ -47,13 +48,48 @@ def test_agent99_recovery_completion_has_human_lifecycle_receipt() -> None: source = CONTROL.read_text(encoding="utf-8") assert '$EventType -eq "recovery_slo_result"' in source - assert '主機與服務恢復 / recovery lifecycle' in source - assert '已恢復且符合目標 / recovered within target' in source assert '$lifecycleInfoAlert' in source assert '$EventType -in @("recovery_slo_result")' in source - assert 'Agent99 恢復狀態 / Recovery Status' in source - assert '恢復階段 / RECOVERY PHASES' in source - assert 'Detect -> Apply -> Verify -> Notify receipt' in source + assert 'schemaVersion = "agent99_incident_card_v5"' in source + assert '主機與服務已通過恢復驗證' in source + assert '本輪不是 fresh reboot drill,不宣稱全主機 SLA' in source + assert 'Agent99 事故狀態 / INCIDENT STATUS' in source + assert '使用者影響' in source + assert 'Agent99 實際動作' in source + assert '獨立驗證' in source + + +def test_agent99_telegram_v5_hides_raw_evidence_and_has_operator_fields() -> None: + source = CONTROL.read_text(encoding="utf-8") + formatter = source[source.index("function Format-AgentTelegramText {") :] + formatter = formatter[: formatter.index("function Format-AgentTelegramCaption")] + + assert '"資產:$($card.target)"' in formatter + assert '"影響:$($card.impact)"' in formatter + assert '"Agent99:$($card.action)"' in formatter + assert '"驗證:$($card.verification)"' in formatter + assert '"事件:$($card.incidentId)"' in formatter + assert "evidencePath" not in formatter + assert "candidatePath" not in formatter + assert "kmPath" not in formatter + assert "verifierName=" not in formatter + assert 'messageFormat = "incident_card_zh_tw_v5"' in source + + +def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None: + source = CONTROL.read_text(encoding="utf-8") + inbox = SRE_INBOX.read_text(encoding="utf-8") + + assert 'Get-AgentField $alert "telegramMessageId"' in inbox + assert "replyMessageId = $replyMessageId" in inbox + assert 'PSObject.Properties["replyMessageId"]' in source + assert 'PSObject.Properties["correlationKey"]' in source + assert '"reply_parameters"' in source + assert 'telegram_receipt_b64=' in source + assert 'schemaVersion = "agent99_telegram_incident_state_v1"' in source + assert 'rootMessageId = $rootMessageId' in source + assert 'reason = "same_lifecycle_state"' in source + assert 'reason = "dedupe_window"' in source def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None: @@ -65,6 +101,7 @@ def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None: assert 'suppressed = $true' in dedupe assert 'reason = "dedupe_window"' in dedupe assert 'dedupeAgeMinutes' in dedupe + assert dedupe.index("Send-AgentTelegram") < len(dedupe) def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None: diff --git a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json index 2c1b63089..b7e123894 100644 --- a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json +++ b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json @@ -655,7 +655,7 @@ "id": "AG99-P1-003", "order": 17, "priority": "P1", - "status": "planned", + "status": "in_progress", "title": "Human-readable Traditional Chinese Telegram incident cards", "owner_lane": "telegram_incident_experience", "dependencies": ["AG99-P0-006"], @@ -666,8 +666,9 @@ "prohibited_actions": ["send raw verifier dumps", "repeat the same text as an image", "expose internal paths or secrets", "declare resolved from send success", "send periodic same-state noise"], "verifier": "Message snapshot tests, duplicate suppression, state-change delivery and evidence-link checks.", "acceptance": "Every card explains severity, affected service or host, user impact, lifecycle state, AI diagnosis, controlled action, verifier result, duration, recurrence count and next update in Traditional Chinese. Charts or real screenshots are attached only when they improve understanding; one fingerprint produces one lifecycle thread and no same-state spam.", - "implementation_progress": "The 2026-07-11 Agent99 incident-card screenshot was captured as a failing UX fixture and added to the P1 acceptance contract; runtime renderer replacement remains pending behind the current host 112 P0 closure.", - "next_action": "Replace the raw automation-result renderer with detected, progress and recovered card templates; add fingerprint edit/thread dedup, redaction, chart-or-screenshot evidence rules and Telegram delivery snapshot tests." + "implementation_progress": "The v5 source now normalizes every Agent99 alert into one Traditional Chinese incident model with deterministic incident ID/fingerprint, severity, lifecycle, affected asset, user impact, actual Agent99 action, independent verifier result, duration, recurrence and next update. Operator-command captions no longer expose Windows evidence paths, KM paths or raw verifier fields. Warning/critical and verified recovery events render 1200x760 evidence views for recovery phases, performance metrics, backup freshness and automation/freshness gates; external screenshots are accepted only when visualEvidenceVerified=true. Telegram source message IDs are preserved through the SRE inbox so Agent99 replies under the original alert, incident root/message state is persisted, same lifecycle state is suppressed until state changes, and failed sends do not create a dedupe marker. Windows 99 staging parser reports zero errors; the no-send self-test reports incident_card_zh_tw_v5, operator lifecycle recovered, visual generated, lifecycle state test passed, same_lifecycle_state suppression and root message preservation. Runtime atomic deployment and a production-principal lifecycle receipt remain open.", + "evidence_refs": ["C:\\Wooo\\Agent99\\staging\\codex-alert-v5-20260711-202853\\evidence\\agent99-TelegramCardSelfTest-*.json", "Windows PowerShell 5.1 parser errors=0", "Agent99 tests 59 passed", "reboot-recovery tests 145 passed", "apps/api/tests/test_agent99_transport_recovery_deploy_contract.py", "scripts/reboot-recovery/agent99-recover-receipt-readback.ps1"], + "next_action": "Create a local audited source commit, atomically deploy it to Windows 99, then run one production-principal Recover and require the v5 recovery lifecycle card, visual delivery or explicit same-state suppression, Telegram message receipt, persistent incident state and independent recovery verifier to pass. Do not send periodic test noise." }, { "id": "AG99-P1-004", @@ -790,9 +791,9 @@ "p0_count": 14, "p1_count": 8, "p2_count": 3, - "in_progress_count": 9, + "in_progress_count": 10, "in_progress_blocked_count": 0, - "planned_count": 15, + "planned_count": 14, "complete_count": 1, "current_p0_count": 1 }, diff --git a/scripts/reboot-recovery/agent99-recover-receipt-readback.ps1 b/scripts/reboot-recovery/agent99-recover-receipt-readback.ps1 new file mode 100644 index 000000000..e87cf7e1d --- /dev/null +++ b/scripts/reboot-recovery/agent99-recover-receipt-readback.ps1 @@ -0,0 +1,158 @@ +[CmdletBinding()] +param( + [string]$AgentRoot = "C:\Wooo\Agent99", + [string]$NotBefore = "" +) + +$ErrorActionPreference = "Stop" +$notBeforeTime = [datetime]::MinValue +if ($NotBefore) { + $notBeforeTime = [datetime]::Parse($NotBefore) +} + +$evidenceDir = Join-Path $AgentRoot "evidence" +$file = Get-ChildItem -Path $evidenceDir -Filter "agent99-Recover-*.json" -File -ErrorAction SilentlyContinue | + Where-Object { $_.LastWriteTime -ge $notBeforeTime } | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +if (-not $file) { + throw "new_recover_evidence_missing" +} + +$raw = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json +$phases = @($raw.recoverySlo.phases | ForEach-Object { + [pscustomobject]@{ + name = [string]$_.name + status = [string]$_.status + elapsedSeconds = [double]$_.elapsedSeconds + } +}) +$telegram = @($raw.telegram | ForEach-Object { + $visualPath = if ($_.PSObject.Properties["visualPath"]) { [string]$_.visualPath } else { "" } + [pscustomobject]@{ + eventType = [string]$_.eventType + severity = [string]$_.severity + sent = [bool]$_.sent + suppressed = if ($_.PSObject.Properties["suppressed"]) { [bool]$_.suppressed } else { $false } + reason = if ($_.PSObject.Properties["reason"]) { [string]$_.reason } else { "" } + messageFormat = if ($_.PSObject.Properties["messageFormat"]) { [string]$_.messageFormat } else { "" } + incidentId = if ($_.PSObject.Properties["incidentId"]) { [string]$_.incidentId } else { "" } + fingerprint = if ($_.PSObject.Properties["fingerprint"]) { [string]$_.fingerprint } else { "" } + lifecycle = if ($_.PSObject.Properties["lifecycle"]) { [string]$_.lifecycle } else { "" } + stateKey = if ($_.PSObject.Properties["stateKey"]) { [string]$_.stateKey } else { "" } + visualSent = if ($_.PSObject.Properties["visualSent"]) { [bool]$_.visualSent } else { $false } + messageIdPresent = [bool]($_.PSObject.Properties["messageId"] -and $_.messageId) + replyThreaded = [bool]($_.PSObject.Properties["replyToMessageId"] -and $_.replyToMessageId) + incidentStateWritten = [bool]($_.PSObject.Properties["incidentStateWritten"] -and $_.incidentStateWritten) + visualFile = if ($visualPath) { Split-Path -Leaf $visualPath } else { "" } + visualExists = [bool]($visualPath -and (Test-Path -LiteralPath $visualPath)) + } +}) + +$phaseFailures = @($phases | Where-Object { $_.status -ne "ok" }) +$sentCount = @($telegram | Where-Object { $_.sent }).Count +$suppressedCount = @($telegram | Where-Object { $_.suppressed }).Count +$lifecycleReceipts = @($telegram | Where-Object { $_.eventType -eq "recovery_slo_result" -and ($_.sent -or $_.suppressed) }) +$sentLifecycleReceipts = @($lifecycleReceipts | Where-Object { $_.sent }) +$notificationReceipted = [bool]($lifecycleReceipts.Count -gt 0) +$humanReadableCardReady = [bool]( + $lifecycleReceipts.Count -gt 0 -and + @($lifecycleReceipts | Where-Object { + $_.messageFormat -ne "incident_card_zh_tw_v5" -or + $_.incidentId -notmatch "^INC-[A-F0-9]{8}$" -or + $_.lifecycle -notin @("remediating", "recovered", "blocked") + }).Count -eq 0 +) +$visualLifecycleReady = [bool](@($sentLifecycleReceipts | Where-Object { -not $_.visualSent }).Count -eq 0) +$incidentStateReady = [bool](@($sentLifecycleReceipts | Where-Object { -not $_.messageIdPresent -or -not $_.incidentStateWritten }).Count -eq 0) +$recoveryVerified = [bool]( + $raw.mode -eq "Recover" -and + $raw.recoverySlo.completed -and + $raw.recoverySlo.coordinatorVerified -and + $phases.Count -gt 0 -and + $phaseFailures.Count -eq 0 +) +$recentReceipts = @(Get-ChildItem -Path $evidenceDir -Filter "agent99-Recover-*.json" -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | + Select-Object -First 5 | + ForEach-Object { + $historyFile = $_ + try { + $history = Get-Content -Path $historyFile.FullName -Raw | ConvertFrom-Json + $historyTelegram = @($history.telegram) + [pscustomobject]@{ + evidenceFile = $historyFile.Name + evidenceLastWriteTime = $historyFile.LastWriteTime.ToString("o") + runId = [string]$history.recoverySlo.runId + completed = [bool]$history.recoverySlo.completed + telegramAttemptCount = $historyTelegram.Count + telegramSentCount = @($historyTelegram | Where-Object { $_.sent }).Count + telegramSuppressedCount = @($historyTelegram | Where-Object { $_.PSObject.Properties["suppressed"] -and $_.suppressed }).Count + visualSentCount = @($historyTelegram | Where-Object { $_.PSObject.Properties["visualSent"] -and $_.visualSent }).Count + visualFiles = @($historyTelegram | Where-Object { $_.PSObject.Properties["visualPath"] -and $_.visualPath } | ForEach-Object { Split-Path -Leaf ([string]$_.visualPath) }) + } + } catch { + [pscustomobject]@{ + evidenceFile = $historyFile.Name + evidenceLastWriteTime = $historyFile.LastWriteTime.ToString("o") + parseError = $_.Exception.GetType().Name + } + } + }) + +$result = [pscustomobject]@{ + schemaVersion = "agent99_recover_receipt_readback_v1" + observedAt = (Get-Date).ToString("o") + evidenceFile = $file.Name + evidenceLastWriteTime = $file.LastWriteTime.ToString("o") + secretValueRead = $false + remoteWritePerformed = $false + mode = [string]$raw.mode + controlledApply = [bool]$raw.controlledApply + vmCount = @($raw.vmResults).Count + vmOkCount = @($raw.vmResults | Where-Object { $_.ok -or $_.skipped }).Count + hostCount = @($raw.hosts).Count + hostReachableCount = @($raw.hosts | Where-Object { $_.ping }).Count + publicCount = @($raw.public).Count + publicOkCount = @($raw.public | Where-Object { $_.ok }).Count + recovery = [pscustomobject]@{ + runId = [string]$raw.recoverySlo.runId + scope = [string]$raw.recoverySlo.scope + completed = [bool]$raw.recoverySlo.completed + withinSlo = [bool]$raw.recoverySlo.withinSlo + elapsedSeconds = [double]$raw.recoverySlo.elapsedSeconds + coordinatorVerified = [bool]$raw.recoverySlo.coordinatorVerified + rebootSloClaimed = [bool]$raw.recoverySlo.rebootSloClaimed + phases = $phases + } + telegramAttemptCount = $telegram.Count + telegramSentCount = $sentCount + telegramSuppressedCount = $suppressedCount + lifecycleReceiptCount = $lifecycleReceipts.Count + notificationReceipted = $notificationReceipted + humanReadableCardReady = $humanReadableCardReady + visualLifecycleReady = $visualLifecycleReady + incidentStateReady = $incidentStateReady + telegram = $telegram + recentReceipts = $recentReceipts + recoveryVerified = $recoveryVerified + ok = [bool]($recoveryVerified -and $notificationReceipted -and $humanReadableCardReady -and $visualLifecycleReady -and $incidentStateReady) +} + +Write-Output "RECOVER_RECEIPT_OK=$([int]$result.ok)" +Write-Output "RECOVERY_VERIFIED=$([int]$result.recoveryVerified)" +Write-Output "NOTIFICATION_RECEIPTED=$([int]$result.notificationReceipted)" +Write-Output "HUMAN_READABLE_CARD_READY=$([int]$result.humanReadableCardReady)" +Write-Output "VISUAL_LIFECYCLE_READY=$([int]$result.visualLifecycleReady)" +Write-Output "INCIDENT_STATE_READY=$([int]$result.incidentStateReady)" +Write-Output "TELEGRAM_ATTEMPTS=$($result.telegramAttemptCount)" +Write-Output "TELEGRAM_SENT=$($result.telegramSentCount)" +Write-Output "TELEGRAM_SUPPRESSED=$($result.telegramSuppressedCount)" +Write-Output "SECRET_VALUE_READ=0" +Write-Output "REMOTE_WRITE_PERFORMED=0" +Write-Output "JSON_BEGIN" +$result | ConvertTo-Json -Depth 10 -Compress +Write-Output "JSON_END" + +if ($result.ok) { exit 0 } +exit 2 diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index 4cbed804a..c37f08512 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -147,3 +147,23 @@ def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() -> assert safe_field in preflight for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"): assert forbidden not in preflight + + +def test_recover_receipt_readback_requires_verifier_and_notification() -> None: + readback = read("scripts/reboot-recovery/agent99-recover-receipt-readback.ps1") + + assert 'schemaVersion = "agent99_recover_receipt_readback_v1"' in readback + assert '$raw.recoverySlo.coordinatorVerified' in readback + assert '$phaseFailures.Count -eq 0' in readback + assert '$notificationReceipted' in readback + assert '$humanReadableCardReady' in readback + assert '$visualLifecycleReady' in readback + assert '$incidentStateReady' in readback + assert 'incident_card_zh_tw_v5' in readback + assert '$recoveryVerified -and $notificationReceipted -and $humanReadableCardReady' in readback + assert '$recentReceipts' in readback + assert 'visualSentCount' in readback + assert 'secretValueRead = $false' in readback + assert 'remoteWritePerformed = $false' in readback + for forbidden in ("botToken", "chatId", "messageText", "rawUpdate", ".env"): + assert forbidden not in readback