From cbcd9ccdd266e242f4909e67f6e70842a86dbed7 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 11:26:48 +0800 Subject: [PATCH] fix(agent99): verify telegram e2e photo delivery --- HOST-REBOOT-AI-AUTOMATION-SOP.md | 19 ++++ agent99-control-plane.ps1 | 152 ++++++++++++++++++++++++++++--- agent99-telegram-inbox.ps1 | 1 - 3 files changed, 159 insertions(+), 13 deletions(-) diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index d97148b08..b97220781 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -457,6 +457,8 @@ Recovery is complete only when all of the following are captured: - P0-6: route Gitea repository missing alerts to evidence-first `Status` until a dedicated Gitea recovery playbook is verified. - P0-7: run live TG-to-Agent99 end-to-end tests for each alert family before re-enabling the paused sensor schedules. - P0-8: all Agent99 Telegram alert messages must include Traditional Chinese user-facing labels and human-readable context; professional terms such as `readback_failed`, `SecurityTriage`, `directReadback`, `KM/RAG`, and provider names may remain English when translation would reduce precision. + - P0-9: Telegram inbox must not force `controlledApply=true` on every monitoring alert. `controlledApply` is resolved by SRE mode: remediation modes may run controlled apply, evidence-first modes such as `ProviderFreshness`, `BackupCheck`, `SecurityTriage`, and `Status` must remain non-apply unless explicitly allowlisted. + - P0-10: Telegram delivery must support actual image/photo delivery through the relay path, not only text plus a local evidence image. - 2026-07-10 10:54 visual card verifier: - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`. @@ -482,6 +484,23 @@ Recovery is complete only when all of the following are captured: - Result: `ok=True`, `visualExists=True`, `sentTelegram=False`, `messageFormat=incident_card_zh_tw_v3`. - UTF-8 readback confirmed local preview lines include: `Agent99 事件卡 / Incident Card`, `嚴重度 Severity`, `事件 Event`, `原因 Reason`, `影響 Impact`, `Agent99 動作 Action`, `目前狀態 Current`, and `時間 Time`. - Local visual inspection: PNG is `1040x620`, non-blank, and Traditional Chinese text renders correctly. +- 2026-07-10 11:17 synthetic TG-to-SRE e2e verifier: + - First run found a real bug: Telegram inbox wrote `controlledApply=True` for a `ProviderFreshness` alert. + - Unsafe test queue was quarantined before ControlTick execution: `C:\Wooo\Agent99\queue\failed\quarantined-e2e-controlled-apply-bug-sre-alert-20260710-111547-face168a.json`. + - Runtime fix deployed: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1`. + - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1.bak-20260710-111651`. + - Classifier self-test: `ok=True`, `caseCount=8`, `passedCount=8`, `failedCount=0`. + - Post-fix synthetic e2e evidence: + - Telegram inbox evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-111728.json`. + - SRE inbox evidence: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260710-111729.json`. + - Queue result: `mode=ProviderFreshness`, `controlledApply=False`, `wroteTelegram=False`, `ranControlTick=False`. + - Test queue quarantined after verification: `C:\Wooo\Agent99\queue\failed\quarantined-e2e-after-verify-sre-alert-20260710-111729-b2712df3.json`. +- 2026-07-10 11:25 Telegram relay photo delivery verifier: + - Runtime script deployed with relay `sendPhoto` support: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. + - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-112511`. + - Self-test evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramDeliverySelfTest-20260710-112512.json`. + - Result: `ok=True`, `sentTelegram=True`, `visualSent=True`, `relay.photoSent=True`, `relay.output=photo_sent_ok`, `messageFormat=incident_card_zh_tw_v3`. + - Delivery mode: Windows 99 generated the PNG, copied it to host 110, copied it into relay container `stockplatform-v2-api-1`, and sent it through Telegram `sendPhoto`; text-only fallback was not used. ## Agent99 Monitoring Alert Routing Self-Test Receipt diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 7f7d4847d..926dc87d9 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -3,6 +3,7 @@ param( [string]$Mode = "Status", [switch]$ControlledApply, [switch]$SelfTestTelegramCard, + [switch]$SelfTestTelegramDelivery, [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", [string]$EvidenceDir = "C:\Wooo\Agent99\evidence" ) @@ -44,7 +45,8 @@ function Send-AgentTelegramRelay { param( [string]$Message, [object]$Relay, - [string]$ChatIdOverride = "" + [string]$ChatIdOverride = "", + [string]$PhotoPath = "" ) if (-not ($Relay -and $Relay.enabled)) { @@ -60,29 +62,118 @@ function Send-AgentTelegramRelay { $messageBytes = [Text.Encoding]::UTF8.GetBytes($Message) $messageB64 = [Convert]::ToBase64String($messageBytes) $chatOverrideB64 = if ($ChatIdOverride) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($ChatIdOverride)) } else { "" } - $python = 'import base64, os, sys, urllib.parse, urllib.request + $photoSent = $false + $photoError = $null + $containerPhotoPath = "" + if ($PhotoPath -and (Test-Path -LiteralPath $PhotoPath)) { + $safePhotoName = "agent99-card-" + ([guid]::NewGuid().ToString("N")) + ".png" + $remotePhotoPath = "/tmp/" + $safePhotoName + $containerPhotoPath = "/tmp/" + $safePhotoName + $relayUser = Get-HostSshUser $relayHost + $scpArgs = @( + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=10", + $PhotoPath, + "${relayUser}@${relayHost}:$remotePhotoPath" + ) + try { + $scp = Start-Process -FilePath "scp.exe" -ArgumentList $scpArgs -NoNewWindow -Wait -PassThru + $scp.Refresh() + if ($null -ne $scp.ExitCode -and $scp.ExitCode -eq 0) { + $copyRun = Invoke-SshText $relayHost "docker cp $remotePhotoPath $container`:$containerPhotoPath && rm -f $remotePhotoPath" 45 1 + if (-not $copyRun.ok) { + $photoError = "relay_docker_cp_failed: $($copyRun.output)" + $containerPhotoPath = "" + } + } else { + $photoError = "relay_scp_failed exitCode=$($scp.ExitCode)" + $containerPhotoPath = "" + } + } catch { + $photoError = "relay_scp_exception: $($_.Exception.Message)" + $containerPhotoPath = "" + } + } + + $containerPhotoB64 = if ($containerPhotoPath) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($containerPhotoPath)) } else { "" } + $python = 'import base64, mimetypes, os, sys, urllib.error, urllib.parse, urllib.request text = base64.b64decode(sys.argv[2]).decode() -chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] else "" +chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] and sys.argv[3] != "-" else "" +photo_path = base64.b64decode(sys.argv[4]).decode() if len(sys.argv) > 4 and sys.argv[4] and sys.argv[4] != "-" else "" token = os.environ.get("TELEGRAM_BOT_TOKEN") chat = chat_override or os.environ.get("TELEGRAM_CHAT_ID") assert token and chat -data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode() -urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read() -print("sent_ok") +if photo_path and os.path.exists(photo_path): + caption = text + if len(caption) > 900: + caption = caption[:900] + "\n...(truncated; see evidence)" + filename = os.path.basename(photo_path) + ctype = mimetypes.guess_type(filename)[0] or "image/png" + try: + import requests + with open(photo_path, "rb") as fh: + response = requests.post( + "https://api.telegram.org/bot" + token + "/sendPhoto", + data={"chat_id": chat, "caption": caption}, + files={"photo": (filename, fh, ctype)}, + timeout=30, + ) + if not response.ok: + print("photo_send_failed status=%s body=%s" % (response.status_code, response.text[:500])) + sys.exit(2) + print("photo_sent_ok") + except ImportError: + boundary = "----agent99boundary" + fields = [("chat_id", chat), ("caption", caption)] + body = bytearray() + for name, value in fields: + body.extend(("--" + boundary + "\r\n").encode()) + body.extend(("Content-Disposition: form-data; name=\"" + name + "\"\r\n\r\n").encode()) + body.extend(str(value).encode()) + body.extend(b"\r\n") + body.extend(("--" + boundary + "\r\n").encode()) + body.extend(("Content-Disposition: form-data; name=\"photo\"; filename=\"" + filename + "\"\r\n").encode()) + body.extend(("Content-Type: " + ctype + "\r\n\r\n").encode()) + with open(photo_path, "rb") as fh: + body.extend(fh.read()) + body.extend(b"\r\n") + body.extend(("--" + boundary + "--\r\n").encode()) + req = urllib.request.Request("https://api.telegram.org/bot" + token + "/sendPhoto", data=bytes(body)) + req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary) + try: + urllib.request.urlopen(req, timeout=30).read() + print("photo_sent_ok") + except urllib.error.HTTPError as exc: + print("photo_send_failed status=%s body=%s" % (exc.code, exc.read().decode(errors="replace")[:500])) + sys.exit(2) +else: + data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode() + urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read() + print("sent_ok") ' $pythonB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($python)) $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))' - $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideB64" + $chatOverrideArg = if ($chatOverrideB64) { $chatOverrideB64 } else { "-" } + $containerPhotoArg = if ($containerPhotoB64) { $containerPhotoB64 } else { "-" } + $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideArg $containerPhotoArg" $run = Invoke-SshText $relayHost $remoteCommand 45 1 + $photoSent = [bool]($run.ok -and $run.output -match "photo_sent_ok") + if ($containerPhotoPath) { + Invoke-SshText $relayHost "docker exec $container rm -f $containerPhotoPath" 20 1 | Out-Null + } [pscustomobject]@{ - ok = [bool]($run.ok -and $run.output -match "sent_ok") + ok = [bool]($run.ok -and ($run.output -match "sent_ok|photo_sent_ok")) host = $relayHost container = $container exitCode = $run.exitCode chatOverride = if ($ChatIdOverride) { "telegram-origin" } else { $null } + photoSent = $photoSent + photoError = $photoError output = $run.output - error = if ($run.ok) { $null } else { $run.output } + error = if ($run.ok) { $photoError } else { $run.output } } } @@ -268,6 +359,13 @@ function Format-AgentTelegramText { $lines += "事件 Event: Agent99 控制迴圈 / control loop" $lines += "儀表板 Dashboard: $dashboardPath" $lines += "佇列 Queue: processed=$processed pending=$pending" + } elseif ($EventType -eq "telegram_delivery_test") { + $testId = if ($Data -and $Data.PSObject.Properties["testId"]) { $Data.testId } else { "unknown" } + $lines += "事件 Event: Telegram 低噪音送達測試 / delivery test" + $lines += "測試 Test: TEST_DO_NOT_PAGE" + $lines += "測試 ID Test ID: $testId" + $lines += "影響 Impact: 無使用者影響；此訊息只驗證 Agent99 繁中事件卡與 TG delivery。" + $lines += "Agent99 動作 Action: 驗證送達後寫入 evidence；不觸發修復、不恢復 sensor、不建立事故。" } elseif ($EventType -match "^ai_service_") { $serviceName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } $lines += "事件 Event: AI 服務健康 / AI service health" @@ -459,6 +557,31 @@ function Invoke-AgentTelegramCardSelfTest { exit 0 } +function Invoke-AgentTelegramDeliverySelfTest { + $testId = "telegram-delivery-test-" + (Get-Date -Format "yyyyMMdd-HHmmss") + $data = [pscustomobject]@{ + testId = $testId + visualPath = $null + } + Send-AgentTelegram "warning" "telegram_delivery_test" "TEST_DO_NOT_PAGE bilingual Telegram incident card delivery self-test" $data + $sent = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0) + $result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = $sent + testId = $testId + sentTelegram = $sent + messageFormat = "incident_card_zh_tw_v3" + attempts = $script:TelegramAttempts + } + $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramDeliverySelfTest-$stamp.json" + $result | ConvertTo-Json -Depth 10 | Set-Content -Path $selfTestPath -Encoding UTF8 + Write-AgentLog "telegram_delivery_selftest ok=$sent evidence=$selfTestPath" + $result | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 $jsonPath + $result | ConvertTo-Json -Depth 10 + if (-not $sent) { exit 1 } + exit 0 +} + function Send-AgentTelegram { param( [string]$Severity, @@ -525,13 +648,14 @@ function Send-AgentTelegram { } $attempt.visualPath = $visualPath if (-not ($token -and ($chatId -or $chatIdOverride))) { - $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride + $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride $visualPath $attempt.relay = $relayResult if ($relayResult.ok) { $attempt.sent = $true $attempt.error = $null - if ($visualPath) { - $attempt.visualError = "visual_card_generated_but_relay_sendPhoto_not_available" + $attempt.visualSent = [bool]$relayResult.photoSent + if ($visualPath -and -not $relayResult.photoSent) { + $attempt.visualError = if ($relayResult.photoError) { $relayResult.photoError } else { "relay_sendPhoto_not_available" } } } else { $attempt.error = "telegram_not_configured" @@ -2769,6 +2893,10 @@ if ($SelfTestTelegramCard) { Invoke-AgentTelegramCardSelfTest } +if ($SelfTestTelegramDelivery) { + Invoke-AgentTelegramDeliverySelfTest +} + Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 index 0391bff36..78a845925 100644 --- a/agent99-telegram-inbox.ps1 +++ b/agent99-telegram-inbox.ps1 @@ -338,7 +338,6 @@ function New-AgentMonitoringAlertFromTelegram { telegramChatType = $Update.chat_type replyChatId = [string]$Update.chat_id force = $true - controlledApply = $true createdAt = (Get-Date -Format o) } $alert | ConvertTo-Json -Depth 8 | Set-Content -Path $alertPath -Encoding UTF8