fix(agent99): verify telegram e2e photo delivery

2026-07-10 11:26:48 +08:00
parent 4e817dc21f
commit cbcd9ccdd2
3 changed files with 159 additions and 13 deletions
--- a/HOST-REBOOT-AI-AUTOMATION-SOP.md
+++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md
@@ -457,6 +457,8 @@ Recovery is complete only when all of the following are captured:
  - P0-6: route Gitea repository missing alerts to evidence-first `Status` until a dedicated Gitea recovery playbook is verified.
  - P0-7: run live TG-to-Agent99 end-to-end tests for each alert family before re-enabling the paused sensor schedules.
  - P0-8: all Agent99 Telegram alert messages must include Traditional Chinese user-facing labels and human-readable context; professional terms such as `readback_failed`, `SecurityTriage`, `directReadback`, `KM/RAG`, and provider names may remain English when translation would reduce precision.
+  - P0-9: Telegram inbox must not force `controlledApply=true` on every monitoring alert. `controlledApply` is resolved by SRE mode: remediation modes may run controlled apply, evidence-first modes such as `ProviderFreshness`, `BackupCheck`, `SecurityTriage`, and `Status` must remain non-apply unless explicitly allowlisted.
+  - P0-10: Telegram delivery must support actual image/photo delivery through the relay path, not only text plus a local evidence image.
 - 2026-07-10 10:54 visual card verifier:
  - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`.
  - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`.
@@ -482,6 +484,23 @@ Recovery is complete only when all of the following are captured:
  - Result: `ok=True`, `visualExists=True`, `sentTelegram=False`, `messageFormat=incident_card_zh_tw_v3`.
  - UTF-8 readback confirmed local preview lines include: `Agent99 事件卡 / Incident Card`, `嚴重度 Severity`, `事件 Event`, `原因 Reason`, `影響 Impact`, `Agent99 動作 Action`, `目前狀態 Current`, and `時間 Time`.
  - Local visual inspection: PNG is `1040x620`, non-blank, and Traditional Chinese text renders correctly.
+- 2026-07-10 11:17 synthetic TG-to-SRE e2e verifier:
+  - First run found a real bug: Telegram inbox wrote `controlledApply=True` for a `ProviderFreshness` alert.
+  - Unsafe test queue was quarantined before ControlTick execution: `C:\Wooo\Agent99\queue\failed\quarantined-e2e-controlled-apply-bug-sre-alert-20260710-111547-face168a.json`.
+  - Runtime fix deployed: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1`.
+  - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1.bak-20260710-111651`.
+  - Classifier self-test: `ok=True`, `caseCount=8`, `passedCount=8`, `failedCount=0`.
+  - Post-fix synthetic e2e evidence:
+    - Telegram inbox evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-111728.json`.
+    - SRE inbox evidence: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260710-111729.json`.
+    - Queue result: `mode=ProviderFreshness`, `controlledApply=False`, `wroteTelegram=False`, `ranControlTick=False`.
+    - Test queue quarantined after verification: `C:\Wooo\Agent99\queue\failed\quarantined-e2e-after-verify-sre-alert-20260710-111729-b2712df3.json`.
+- 2026-07-10 11:25 Telegram relay photo delivery verifier:
+  - Runtime script deployed with relay `sendPhoto` support: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`.
+  - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-112511`.
+  - Self-test evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramDeliverySelfTest-20260710-112512.json`.
+  - Result: `ok=True`, `sentTelegram=True`, `visualSent=True`, `relay.photoSent=True`, `relay.output=photo_sent_ok`, `messageFormat=incident_card_zh_tw_v3`.
+  - Delivery mode: Windows 99 generated the PNG, copied it to host 110, copied it into relay container `stockplatform-v2-api-1`, and sent it through Telegram `sendPhoto`; text-only fallback was not used.

 ## Agent99 Monitoring Alert Routing Self-Test Receipt

--- a/agent99-control-plane.ps1
+++ b/agent99-control-plane.ps1
@@ -3,6 +3,7 @@ param(
  [string]$Mode = "Status",
  [switch]$ControlledApply,
  [switch]$SelfTestTelegramCard,
+  [switch]$SelfTestTelegramDelivery,
  [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json",
  [string]$EvidenceDir = "C:\Wooo\Agent99\evidence"
 )
@@ -44,7 +45,8 @@ function Send-AgentTelegramRelay {
  param(
    [string]$Message,
    [object]$Relay,
-    [string]$ChatIdOverride = ""
+    [string]$ChatIdOverride = "",
+    [string]$PhotoPath = ""
  )

  if (-not ($Relay -and $Relay.enabled)) {
@@ -60,29 +62,118 @@ function Send-AgentTelegramRelay {
  $messageBytes = [Text.Encoding]::UTF8.GetBytes($Message)
  $messageB64 = [Convert]::ToBase64String($messageBytes)
  $chatOverrideB64 = if ($ChatIdOverride) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($ChatIdOverride)) } else { "" }
-  $python = 'import base64, os, sys, urllib.parse, urllib.request
+  $photoSent = $false
+  $photoError = $null
+  $containerPhotoPath = ""
+  if ($PhotoPath -and (Test-Path -LiteralPath $PhotoPath)) {
+    $safePhotoName = "agent99-card-" + ([guid]::NewGuid().ToString("N")) + ".png"
+    $remotePhotoPath = "/tmp/" + $safePhotoName
+    $containerPhotoPath = "/tmp/" + $safePhotoName
+    $relayUser = Get-HostSshUser $relayHost
+    $scpArgs = @(
+      "-o", "BatchMode=yes",
+      "-o", "StrictHostKeyChecking=accept-new",
+      "-o", "NumberOfPasswordPrompts=0",
+      "-o", "ConnectTimeout=10",
+      $PhotoPath,
+      "${relayUser}@${relayHost}:$remotePhotoPath"
+    )
+    try {
+      $scp = Start-Process -FilePath "scp.exe" -ArgumentList $scpArgs -NoNewWindow -Wait -PassThru
+      $scp.Refresh()
+      if ($null -ne $scp.ExitCode -and $scp.ExitCode -eq 0) {
+        $copyRun = Invoke-SshText $relayHost "docker cp $remotePhotoPath $container`:$containerPhotoPath && rm -f $remotePhotoPath" 45 1
+        if (-not $copyRun.ok) {
+          $photoError = "relay_docker_cp_failed: $($copyRun.output)"
+          $containerPhotoPath = ""
+        }
+      } else {
+        $photoError = "relay_scp_failed exitCode=$($scp.ExitCode)"
+        $containerPhotoPath = ""
+      }
+    } catch {
+      $photoError = "relay_scp_exception: $($_.Exception.Message)"
+      $containerPhotoPath = ""
+    }
+  }
+
+  $containerPhotoB64 = if ($containerPhotoPath) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($containerPhotoPath)) } else { "" }
+  $python = 'import base64, mimetypes, os, sys, urllib.error, urllib.parse, urllib.request
 text = base64.b64decode(sys.argv[2]).decode()
-chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] else ""
+chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] and sys.argv[3] != "-" else ""
+photo_path = base64.b64decode(sys.argv[4]).decode() if len(sys.argv) > 4 and sys.argv[4] and sys.argv[4] != "-" else ""
 token = os.environ.get("TELEGRAM_BOT_TOKEN")
 chat = chat_override or os.environ.get("TELEGRAM_CHAT_ID")
 assert token and chat
-data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode()
-urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read()
-print("sent_ok")
+if photo_path and os.path.exists(photo_path):
+  caption = text
+  if len(caption) > 900:
+    caption = caption[:900] + "\n...(truncated; see evidence)"
+  filename = os.path.basename(photo_path)
+  ctype = mimetypes.guess_type(filename)[0] or "image/png"
+  try:
+    import requests
+    with open(photo_path, "rb") as fh:
+      response = requests.post(
+        "https://api.telegram.org/bot" + token + "/sendPhoto",
+        data={"chat_id": chat, "caption": caption},
+        files={"photo": (filename, fh, ctype)},
+        timeout=30,
+      )
+    if not response.ok:
+      print("photo_send_failed status=%s body=%s" % (response.status_code, response.text[:500]))
+      sys.exit(2)
+    print("photo_sent_ok")
+  except ImportError:
+    boundary = "----agent99boundary"
+    fields = [("chat_id", chat), ("caption", caption)]
+    body = bytearray()
+    for name, value in fields:
+      body.extend(("--" + boundary + "\r\n").encode())
+      body.extend(("Content-Disposition: form-data; name=\"" + name + "\"\r\n\r\n").encode())
+      body.extend(str(value).encode())
+      body.extend(b"\r\n")
+    body.extend(("--" + boundary + "\r\n").encode())
+    body.extend(("Content-Disposition: form-data; name=\"photo\"; filename=\"" + filename + "\"\r\n").encode())
+    body.extend(("Content-Type: " + ctype + "\r\n\r\n").encode())
+    with open(photo_path, "rb") as fh:
+      body.extend(fh.read())
+    body.extend(b"\r\n")
+    body.extend(("--" + boundary + "--\r\n").encode())
+    req = urllib.request.Request("https://api.telegram.org/bot" + token + "/sendPhoto", data=bytes(body))
+    req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary)
+    try:
+      urllib.request.urlopen(req, timeout=30).read()
+      print("photo_sent_ok")
+    except urllib.error.HTTPError as exc:
+      print("photo_send_failed status=%s body=%s" % (exc.code, exc.read().decode(errors="replace")[:500]))
+      sys.exit(2)
+else:
+  data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode()
+  urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read()
+  print("sent_ok")
 '
  $pythonB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($python))
  $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))'
-  $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideB64"
+  $chatOverrideArg = if ($chatOverrideB64) { $chatOverrideB64 } else { "-" }
+  $containerPhotoArg = if ($containerPhotoB64) { $containerPhotoB64 } else { "-" }
+  $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideArg $containerPhotoArg"
  $run = Invoke-SshText $relayHost $remoteCommand 45 1
+  $photoSent = [bool]($run.ok -and $run.output -match "photo_sent_ok")
+  if ($containerPhotoPath) {
+    Invoke-SshText $relayHost "docker exec $container rm -f $containerPhotoPath" 20 1 | Out-Null
+  }

  [pscustomobject]@{
-    ok = [bool]($run.ok -and $run.output -match "sent_ok")
+    ok = [bool]($run.ok -and ($run.output -match "sent_ok|photo_sent_ok"))
    host = $relayHost
    container = $container
    exitCode = $run.exitCode
    chatOverride = if ($ChatIdOverride) { "telegram-origin" } else { $null }
+    photoSent = $photoSent
+    photoError = $photoError
    output = $run.output
-    error = if ($run.ok) { $null } else { $run.output }
+    error = if ($run.ok) { $photoError } else { $run.output }
  }
 }

@@ -268,6 +359,13 @@ function Format-AgentTelegramText {
    $lines += "事件 Event: Agent99 控制迴圈 / control loop"
    $lines += "儀表板 Dashboard: $dashboardPath"
    $lines += "佇列 Queue: processed=$processed pending=$pending"
+  } elseif ($EventType -eq "telegram_delivery_test") {
+    $testId = if ($Data -and $Data.PSObject.Properties["testId"]) { $Data.testId } else { "unknown" }
+    $lines += "事件 Event: Telegram 低噪音送達測試 / delivery test"
+    $lines += "測試 Test: TEST_DO_NOT_PAGE"
+    $lines += "測試 ID Test ID: $testId"
+    $lines += "影響 Impact: 無使用者影響；此訊息只驗證 Agent99 繁中事件卡與 TG delivery。"
+    $lines += "Agent99 動作 Action: 驗證送達後寫入 evidence；不觸發修復、不恢復 sensor、不建立事故。"
  } elseif ($EventType -match "^ai_service_") {
    $serviceName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" }
    $lines += "事件 Event: AI 服務健康 / AI service health"
@@ -459,6 +557,31 @@ function Invoke-AgentTelegramCardSelfTest {
  exit 0
 }

+function Invoke-AgentTelegramDeliverySelfTest {
+  $testId = "telegram-delivery-test-" + (Get-Date -Format "yyyyMMdd-HHmmss")
+  $data = [pscustomobject]@{
+    testId = $testId
+    visualPath = $null
+  }
+  Send-AgentTelegram "warning" "telegram_delivery_test" "TEST_DO_NOT_PAGE bilingual Telegram incident card delivery self-test" $data
+  $sent = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0)
+  $result = [pscustomobject]@{
+    timestamp = (Get-Date -Format o)
+    ok = $sent
+    testId = $testId
+    sentTelegram = $sent
+    messageFormat = "incident_card_zh_tw_v3"
+    attempts = $script:TelegramAttempts
+  }
+  $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramDeliverySelfTest-$stamp.json"
+  $result | ConvertTo-Json -Depth 10 | Set-Content -Path $selfTestPath -Encoding UTF8
+  Write-AgentLog "telegram_delivery_selftest ok=$sent evidence=$selfTestPath"
+  $result | ConvertTo-Json -Depth 10 | Set-Content -Encoding UTF8 $jsonPath
+  $result | ConvertTo-Json -Depth 10
+  if (-not $sent) { exit 1 }
+  exit 0
+}
+
 function Send-AgentTelegram {
  param(
    [string]$Severity,
@@ -525,13 +648,14 @@ function Send-AgentTelegram {
 	  }
 	  $attempt.visualPath = $visualPath
 	  if (-not ($token -and ($chatId -or $chatIdOverride))) {
-	    $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride
+	    $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride $visualPath
 	    $attempt.relay = $relayResult
 	    if ($relayResult.ok) {
 	      $attempt.sent = $true
 	      $attempt.error = $null
-	      if ($visualPath) {
-	        $attempt.visualError = "visual_card_generated_but_relay_sendPhoto_not_available"
+	      $attempt.visualSent = [bool]$relayResult.photoSent
+	      if ($visualPath -and -not $relayResult.photoSent) {
+	        $attempt.visualError = if ($relayResult.photoError) { $relayResult.photoError } else { "relay_sendPhoto_not_available" }
 	      }
 	    } else {
 	      $attempt.error = "telegram_not_configured"
@@ -2769,6 +2893,10 @@ if ($SelfTestTelegramCard) {
  Invoke-AgentTelegramCardSelfTest
 }

+if ($SelfTestTelegramDelivery) {
+  Invoke-AgentTelegramDeliverySelfTest
+}
+
 Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath"
 Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert

--- a/agent99-telegram-inbox.ps1
+++ b/agent99-telegram-inbox.ps1
@@ -338,7 +338,6 @@ function New-AgentMonitoringAlertFromTelegram {
    telegramChatType = $Update.chat_type
    replyChatId = [string]$Update.chat_id
    force = $true
-    controlledApply = $true
    createdAt = (Get-Date -Format o)
  }
  $alert | ConvertTo-Json -Depth 8 | Set-Content -Path $alertPath -Encoding UTF8