diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index 5b27fb3c9..074116255 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -574,6 +574,15 @@ Recovery is complete only when all of the following are captured: - `192.168.0.120`: `ok`, `loadPerCore=0.05`, `diskUsedPct=47`, `elapsedMs=699`. - `192.168.0.121`: `ok`, `loadPerCore=0.02`, `diskUsedPct=27`, `elapsedMs=1183`. - `192.168.0.188`: `ok`, `loadPerCore=0.09`, `diskUsedPct=88`, `elapsedMs=881`. + - 2026-07-10 12:12 Telegram inbox idle readback verifier: + - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1`. + - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1.bak-20260710-121231`. + - Parser readback: `parse_ok`. + - Fix: Telegram relay readback now records `timeoutSeconds`, `elapsedMs`, and `slowRetry`; empty update polling is `ok=True` with `reason=no_updates`, not a false failure. + - Self-test run: `C:\Wooo\Agent99\evidence\agent99-telegram-inbox-selftest-run-20260710-121249.txt`, exit code `0`. + - Real inbox run evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-121252.json`. + - Real inbox result: `ok=True`, `reason=no_updates`, `updatesSeen=0`, `processedCount=0`, `alertedCount=0`, `ignoredCount=0`, `sentTelegram=0`. + - Relay readback: host `192.168.0.110`, container `stockplatform-v2-api-1`, `ok=True`, `exitCode=0`, `elapsedMs=2176`, `timeoutSeconds=60`, `slowRetry=False`. ## Agent99 Monitoring Alert Routing Self-Test Receipt diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 index 78a845925..21e966dbf 100644 --- a/agent99-telegram-inbox.ps1 +++ b/agent99-telegram-inbox.ps1 @@ -46,8 +46,10 @@ function Invoke-AgentSshText { $Command ) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru $completed = $process.WaitForExit($TimeoutSeconds * 1000) + $stopwatch.Stop() if (-not $completed) { $process.Kill() $process.WaitForExit(2000) | Out-Null @@ -59,7 +61,9 @@ function Invoke-AgentSshText { [pscustomobject]@{ ok = [bool]($completed -and (($null -eq $process.ExitCode) -or $process.ExitCode -eq 0)) exitCode = if ($completed -and $null -ne $process.ExitCode) { $process.ExitCode } elseif ($completed) { 0 } else { -2 } - output = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" + output = if ((@($stdout, $stderr) | Where-Object { $_ }).Count -gt 0) { (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" } elseif (-not $completed) { "timeout_after_${TimeoutSeconds}s" } else { "" } + elapsedMs = $stopwatch.ElapsedMilliseconds + timeoutSeconds = $TimeoutSeconds } } @@ -175,9 +179,36 @@ except Exception as exc: $allowedTitleB64 = Convert-AgentBase64 $allowedTitlePattern $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))' $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $Offset $allowedTitleB64" - $run = Invoke-AgentSshText $relayHost $remoteCommand 35 + $timeoutSeconds = 60 + if ($Config.telegram -and $Config.telegram.PSObject.Properties["inboxRelayTimeoutSeconds"]) { + $timeoutSeconds = [int]$Config.telegram.inboxRelayTimeoutSeconds + } + $run = Invoke-AgentSshText $relayHost $remoteCommand $timeoutSeconds + $slowRetryUsed = $false + if (-not $run.ok -and $run.exitCode -eq -2) { + $slowTimeoutSeconds = 120 + if ($Config.telegram -and $Config.telegram.PSObject.Properties["inboxRelaySlowTimeoutSeconds"]) { + $slowTimeoutSeconds = [int]$Config.telegram.inboxRelaySlowTimeoutSeconds + } + if ($slowTimeoutSeconds -gt $timeoutSeconds) { + $slowRun = Invoke-AgentSshText $relayHost $remoteCommand $slowTimeoutSeconds + $slowRun | Add-Member -NotePropertyName firstAttemptExitCode -NotePropertyValue $run.exitCode -Force + $slowRun | Add-Member -NotePropertyName firstAttemptElapsedMs -NotePropertyValue $run.elapsedMs -Force + $run = $slowRun + $slowRetryUsed = $true + } + } + $relayReadback = [pscustomobject]@{ + host = $relayHost + container = $container + ok = [bool]$run.ok + exitCode = $run.exitCode + elapsedMs = $run.elapsedMs + timeoutSeconds = $run.timeoutSeconds + slowRetry = $slowRetryUsed + } if (-not $run.ok) { - return [pscustomobject]@{ ok = $false; error = $run.output; updates = @(); raw = $null } + return [pscustomobject]@{ ok = $false; error = "telegram_relay_readback_failed: $($run.output)"; updates = @(); raw = $null; relay = $relayReadback } } try { @@ -189,9 +220,9 @@ except Exception as exc: } else { [string]$parsed.error } - return [pscustomobject]@{ ok = [bool]$parsed.ok; error = $errorText; updates = @($parsed.updates); raw = $parsed } + return [pscustomobject]@{ ok = [bool]$parsed.ok; error = $errorText; updates = @($parsed.updates); raw = $parsed; relay = $relayReadback } } catch { - return [pscustomobject]@{ ok = $false; error = $_.Exception.Message; updates = @(); raw = $run.output } + return [pscustomobject]@{ ok = $false; error = "telegram_updates_parse_failed: $($_.Exception.Message)"; updates = @(); raw = $run.output; relay = $relayReadback } } } @@ -606,7 +637,9 @@ if ((-not $SyntheticUpdateText) -and $updatesResult.ok -and $maxUpdateId -ge 0) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = [bool]$updatesResult.ok + reason = if ($updatesResult.ok -and @($updatesResult.updates).Count -eq 0) { "no_updates" } elseif ($updatesResult.ok) { "updates_read" } else { "updates_read_failed" } error = $updatesResult.error + relay = $updatesResult.relay offsetBefore = $offset offsetAfter = if ($SyntheticUpdateText) { $offset } elseif ($updatesResult.ok) { $maxUpdateId + 1 } else { $offset } synthetic = [bool]$SyntheticUpdateText