34e32f6ed6
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m27s
CD Pipeline / build-and-deploy (push) Successful in 6m39s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
842 lines
33 KiB
PowerShell
842 lines
33 KiB
PowerShell
param(
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[switch]$RunNow,
|
|
[switch]$SelfTest,
|
|
[string]$SyntheticUpdateText = "",
|
|
[string]$SyntheticChatTitle = "AwoooI SRE",
|
|
[string]$SyntheticChatId = ""
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json"
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
$StateDir = Join-Path $AgentRoot "state"
|
|
$BinDir = Join-Path $AgentRoot "bin"
|
|
$SubmitScript = Join-Path $BinDir "agent99-submit-request.ps1"
|
|
$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1"
|
|
$AlertIncomingDir = Join-Path $AgentRoot "alerts\incoming"
|
|
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir, $StateDir, $AlertIncomingDir | Out-Null
|
|
|
|
function Load-AgentConfig {
|
|
if (-not (Test-Path $ConfigPath)) {
|
|
throw "Config not found: $ConfigPath"
|
|
}
|
|
Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
|
}
|
|
|
|
function Invoke-AgentSshText {
|
|
param(
|
|
[string]$TargetHost,
|
|
[string]$Command,
|
|
[int]$TimeoutSeconds = 30
|
|
)
|
|
|
|
$id = [guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "agent99-tg-ssh-$id.out"
|
|
$stderrPath = Join-Path $env:TEMP "agent99-tg-ssh-$id.err"
|
|
$args = @(
|
|
"-o", "BatchMode=yes",
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "ConnectTimeout=10",
|
|
"-l", "wooo",
|
|
$TargetHost,
|
|
$Command
|
|
)
|
|
|
|
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
|
$process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
|
|
$completed = $process.WaitForExit($TimeoutSeconds * 1000)
|
|
$stopwatch.Stop()
|
|
if (-not $completed) {
|
|
$process.Kill()
|
|
$process.WaitForExit(2000) | Out-Null
|
|
}
|
|
$process.Refresh()
|
|
$stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" }
|
|
$stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" }
|
|
Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
|
|
[pscustomobject]@{
|
|
ok = [bool]($completed -and (($null -eq $process.ExitCode) -or $process.ExitCode -eq 0))
|
|
exitCode = if ($completed -and $null -ne $process.ExitCode) { $process.ExitCode } elseif ($completed) { 0 } else { -2 }
|
|
output = if ((@($stdout, $stderr) | Where-Object { $_ }).Count -gt 0) { (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" } elseif (-not $completed) { "timeout_after_${TimeoutSeconds}s" } else { "" }
|
|
elapsedMs = $stopwatch.ElapsedMilliseconds
|
|
timeoutSeconds = $TimeoutSeconds
|
|
}
|
|
}
|
|
|
|
function Convert-AgentBase64 {
|
|
param([string]$Text)
|
|
[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Text))
|
|
}
|
|
|
|
function Get-AgentTelegramUpdates {
|
|
param(
|
|
[object]$Config,
|
|
[int64]$Offset
|
|
)
|
|
|
|
$relay = $Config.telegram.relay
|
|
if (-not ($relay -and $relay.enabled)) {
|
|
return [pscustomobject]@{ ok = $false; error = "relay_disabled"; updates = @(); raw = $null }
|
|
}
|
|
$relayHost = if ($relay.host) { [string]$relay.host } else { "192.168.0.110" }
|
|
$container = if ($relay.container) { [string]$relay.container } else { "stockplatform-v2-api-1" }
|
|
$allowedTitlePattern = if ($Config.telegram.allowedCommandChatTitlePattern) {
|
|
[string]$Config.telegram.allowedCommandChatTitlePattern
|
|
} else {
|
|
"AwoooI SRE|AWOOOI SRE|SRE"
|
|
}
|
|
if ($container -notmatch "^[A-Za-z0-9_.-]+$") {
|
|
return [pscustomobject]@{ ok = $false; error = "invalid_relay_container"; updates = @(); raw = $null }
|
|
}
|
|
|
|
$python = @'
|
|
import base64, json, os, re, sys, urllib.parse, urllib.request, urllib.error
|
|
|
|
token = os.environ.get("TELEGRAM_BOT_TOKEN") or ""
|
|
chat = os.environ.get("TELEGRAM_CHAT_ID") or ""
|
|
|
|
def redact(value):
|
|
text = str(value or "")
|
|
for secret in (token, chat):
|
|
if secret:
|
|
text = text.replace(secret, "<redacted>")
|
|
return text[-1200:]
|
|
|
|
def emit(payload):
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
|
|
try:
|
|
offset = int(sys.argv[2]) if len(sys.argv) > 2 else 0
|
|
allowed_title_pattern = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] else ""
|
|
allowed_title_re = re.compile(allowed_title_pattern, re.I) if allowed_title_pattern else None
|
|
if not token or not chat:
|
|
emit({"ok": False, "error": "telegram_env_missing", "updates": []})
|
|
sys.exit(0)
|
|
|
|
params = {
|
|
"timeout": "0",
|
|
"allowed_updates": json.dumps(["message"])
|
|
}
|
|
if offset > 0:
|
|
params["offset"] = str(offset)
|
|
url = "https://api.telegram.org/bot" + token + "/getUpdates?" + urllib.parse.urlencode(params)
|
|
|
|
try:
|
|
raw = urllib.request.urlopen(url, timeout=20).read().decode()
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read().decode(errors="replace")
|
|
emit({"ok": False, "error": "telegram_http_error", "status": exc.code, "description": redact(body), "updates": []})
|
|
sys.exit(0)
|
|
except Exception as exc:
|
|
emit({"ok": False, "error": "telegram_request_failed", "errorType": type(exc).__name__, "description": redact(exc), "updates": []})
|
|
sys.exit(0)
|
|
|
|
try:
|
|
data = json.loads(raw)
|
|
except Exception as exc:
|
|
emit({"ok": False, "error": "telegram_json_parse_failed", "errorType": type(exc).__name__, "description": redact(exc), "updates": []})
|
|
sys.exit(0)
|
|
|
|
if not data.get("ok", False):
|
|
emit({"ok": False, "error": "telegram_api_not_ok", "description": redact(data.get("description", "")), "errorCode": data.get("error_code"), "updates": []})
|
|
sys.exit(0)
|
|
|
|
items = []
|
|
for item in data.get("result", []):
|
|
msg = item.get("message") or {}
|
|
chat_obj = msg.get("chat") or {}
|
|
chat_id = str(chat_obj.get("id") or "")
|
|
chat_title = chat_obj.get("title") or chat_obj.get("username") or chat_obj.get("first_name") or ""
|
|
chat_matches_config = chat_id == str(chat)
|
|
chat_matches_title = bool(allowed_title_re and allowed_title_re.search(chat_title))
|
|
if not (chat_matches_config or chat_matches_title):
|
|
continue
|
|
text = msg.get("text") or ""
|
|
if not text:
|
|
continue
|
|
sender = msg.get("from") or {}
|
|
items.append({
|
|
"update_id": item.get("update_id"),
|
|
"message_id": msg.get("message_id"),
|
|
"date": msg.get("date"),
|
|
"text": text,
|
|
"chat_id": chat_id,
|
|
"chat_title": chat_title,
|
|
"chat_type": chat_obj.get("type"),
|
|
"chat_match": "configured_chat_id" if chat_matches_config else "allowed_title",
|
|
"from": sender.get("username") or sender.get("first_name") or str(sender.get("id") or "")
|
|
})
|
|
emit({"ok": True, "updates": items})
|
|
except Exception as exc:
|
|
emit({"ok": False, "error": "telegram_inbox_exception", "errorType": type(exc).__name__, "description": redact(exc), "updates": []})
|
|
'@
|
|
|
|
$pythonB64 = Convert-AgentBase64 $python
|
|
$allowedTitleB64 = Convert-AgentBase64 $allowedTitlePattern
|
|
$bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))'
|
|
$remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $Offset $allowedTitleB64"
|
|
$timeoutSeconds = 60
|
|
if ($Config.telegram -and $Config.telegram.PSObject.Properties["inboxRelayTimeoutSeconds"]) {
|
|
$timeoutSeconds = [int]$Config.telegram.inboxRelayTimeoutSeconds
|
|
}
|
|
$run = Invoke-AgentSshText $relayHost $remoteCommand $timeoutSeconds
|
|
$slowRetryUsed = $false
|
|
if (-not $run.ok -and $run.exitCode -eq -2) {
|
|
$slowTimeoutSeconds = 120
|
|
if ($Config.telegram -and $Config.telegram.PSObject.Properties["inboxRelaySlowTimeoutSeconds"]) {
|
|
$slowTimeoutSeconds = [int]$Config.telegram.inboxRelaySlowTimeoutSeconds
|
|
}
|
|
if ($slowTimeoutSeconds -gt $timeoutSeconds) {
|
|
$slowRun = Invoke-AgentSshText $relayHost $remoteCommand $slowTimeoutSeconds
|
|
$slowRun | Add-Member -NotePropertyName firstAttemptExitCode -NotePropertyValue $run.exitCode -Force
|
|
$slowRun | Add-Member -NotePropertyName firstAttemptElapsedMs -NotePropertyValue $run.elapsedMs -Force
|
|
$run = $slowRun
|
|
$slowRetryUsed = $true
|
|
}
|
|
}
|
|
$relayReadback = [pscustomobject]@{
|
|
host = $relayHost
|
|
container = $container
|
|
ok = [bool]$run.ok
|
|
exitCode = $run.exitCode
|
|
elapsedMs = $run.elapsedMs
|
|
timeoutSeconds = $run.timeoutSeconds
|
|
slowRetry = $slowRetryUsed
|
|
}
|
|
if (-not $run.ok) {
|
|
return [pscustomobject]@{ ok = $false; error = "telegram_relay_readback_failed: $($run.output)"; updates = @(); raw = $null; relay = $relayReadback }
|
|
}
|
|
|
|
try {
|
|
$parsed = $run.output | ConvertFrom-Json
|
|
$errorText = if ($parsed.ok) {
|
|
$null
|
|
} elseif ($parsed.description) {
|
|
"$($parsed.error): $($parsed.description)"
|
|
} else {
|
|
[string]$parsed.error
|
|
}
|
|
return [pscustomobject]@{ ok = [bool]$parsed.ok; error = $errorText; updates = @($parsed.updates); raw = $parsed; relay = $relayReadback }
|
|
} catch {
|
|
return [pscustomobject]@{ ok = $false; error = "telegram_updates_parse_failed: $($_.Exception.Message)"; updates = @(); raw = $run.output; relay = $relayReadback }
|
|
}
|
|
}
|
|
|
|
function Resolve-AgentTelegramInstruction {
|
|
param([string]$Text)
|
|
|
|
$trimmed = $Text.Trim()
|
|
$patterns = @(
|
|
"^/agent99\s+(.+)$",
|
|
"^!agent99\s+(.+)$",
|
|
"^#agent99\s+(.+)$",
|
|
"^agent99\s+(.+)$",
|
|
"^@agent99\s+(.+)$"
|
|
)
|
|
foreach ($pattern in $patterns) {
|
|
$match = [regex]::Match($trimmed, $pattern, [Text.RegularExpressions.RegexOptions]::IgnoreCase)
|
|
if ($match.Success) {
|
|
return $match.Groups[1].Value.Trim()
|
|
}
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Test-AgentMonitoringAlertText {
|
|
param([string]$Text)
|
|
|
|
$trimmed = $Text.Trim()
|
|
if (-not $trimmed) { return $false }
|
|
$lower = $trimmed.ToLowerInvariant()
|
|
|
|
if ($lower -match "^agent99 alert:|\[agent99\]|event:\s+operator command|operator_command_result") {
|
|
return $false
|
|
}
|
|
|
|
if ($lower -match "alertmanager|action required|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot-auto-recovery|cold[-_ ]start|\bci/?cd\b|\bcicd\b|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗|postgres|postgresql|database|alertchain|alert_chain|monitoring pipeline|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|ai support|locale guard|missing_key|e_ai_missing_key") {
|
|
return $true
|
|
}
|
|
|
|
foreach ($code in @(0x544A, 0x8B66, 0x7570, 0x5E38, 0x56B4, 0x91CD, 0x8CA0, 0x8F09, 0x78C1, 0x789F, 0x5099, 0x4EFD, 0x5BA2, 0x670D, 0x78A7, 0x6F6D)) {
|
|
if ($trimmed.IndexOf([string][char]$code) -ge 0) { return $true }
|
|
}
|
|
|
|
return $false
|
|
}
|
|
|
|
function Test-AgentTextContainsCode {
|
|
param(
|
|
[string]$Text,
|
|
[int[]]$Codes
|
|
)
|
|
foreach ($code in $Codes) {
|
|
if ($Text.IndexOf([string][char]$code) -ge 0) { return $true }
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Get-AgentTelegramAlertSeverity {
|
|
param([string]$Text)
|
|
$lower = $Text.ToLowerInvariant()
|
|
if ($lower -match "resolved|recovered" -or (Test-AgentTextContainsCode $Text @(0x6062, 0x89E3))) { return "resolved" }
|
|
if ($lower -match "critical|p0|emergency|fatal|down" -or (Test-AgentTextContainsCode $Text @(0x56B4, 0x91CD, 0x7DCA, 0x6025))) { return "critical" }
|
|
if ($lower -match "failed" -or $Text -match "失敗") { return "critical" }
|
|
if ($lower -match "warning|warn|degraded|readback_failed|missing_key|locale guard|ai support" -or (Test-AgentTextContainsCode $Text @(0x544A, 0x8B66, 0x7570, 0x5E38))) { return "warning" }
|
|
return "warning"
|
|
}
|
|
|
|
function Get-AgentTelegramAlertKind {
|
|
param([string]$Text)
|
|
$lower = $Text.ToLowerInvariant()
|
|
if ($lower -match "\bci/?cd\b|\bcicd\b|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") { return "cicd_failure" }
|
|
if ($lower -match "backup|snapshot|cron" -or (Test-AgentTextContainsCode $Text @(0x5099, 0x4EFD))) { return "backup_health" }
|
|
if ($lower -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc") { return "wazuh_security_alert" }
|
|
if ($lower -match "gitea|repo_missing|repository_missing|repo missing|repository missing") { return "repository_missing" }
|
|
if ($lower -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") { return "provider_freshness_signal" }
|
|
if ($lower -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag") { return "database_health" }
|
|
if ($lower -match "alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") { return "alert_chain_health" }
|
|
if ($lower -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot|restart|powered_off" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" }
|
|
if ($lower -match "502|site_502|public_route|website|route") { return "public_route_502" }
|
|
if ($lower -match "missing_key|e_ai_missing_key|ai support|locale guard" -or (Test-AgentTextContainsCode $Text @(0x5BA2, 0x670D))) { return "application_ai_config_error" }
|
|
if ($lower -match "cpu|load" -or (Test-AgentTextContainsCode $Text @(0x8CA0, 0x8F09))) { return "performance_cpu" }
|
|
if ($lower -match "disk" -or (Test-AgentTextContainsCode $Text @(0x78C1, 0x789F))) { return "performance_disk" }
|
|
if ($lower -match "memory|mem_|ram" -or (Test-AgentTextContainsCode $Text @(0x8A18, 0x61B6, 0x9AD4))) { return "performance_memory" }
|
|
if ($lower -match "harbor|registry") { return "harbor_registry" }
|
|
if ($lower -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return "awoooi_k3s" }
|
|
return "monitoring_alert"
|
|
}
|
|
|
|
function Get-AgentTelegramAlertService {
|
|
param([string]$Text)
|
|
$lower = $Text.ToLowerInvariant()
|
|
if ($lower -match "stock") { return "stockplatform" }
|
|
if ($lower -match "2026fifa|fifa") { return "2026fifa" }
|
|
if ($lower -match "bitan" -or (Test-AgentTextContainsCode $Text @(0x78A7, 0x6F6D))) { return "bitan-pharmacy" }
|
|
if ($lower -match "gitea") { return "gitea" }
|
|
if ($lower -match "harbor") { return "harbor" }
|
|
if ($lower -match "wazuh|siem|security") { return "iwooos-security" }
|
|
if ($lower -match "agent99") { return "agent99" }
|
|
if ($lower -match "awoooi|k3s|502|site|route|website") { return "awoooi" }
|
|
return "unknown"
|
|
}
|
|
|
|
function Get-AgentTelegramAlertHost {
|
|
param([string]$Text)
|
|
$match = [regex]::Match($Text, "(192\\.168\\.0\\.(110|112|120|121|188|99))")
|
|
if ($match.Success) { return $match.Groups[1].Value }
|
|
foreach ($hostSuffix in @("110", "112", "120", "121", "188", "99")) {
|
|
if ($Text -match "(^|\D)$hostSuffix(\D|$)") {
|
|
return "192.168.0.$hostSuffix"
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
function Get-AgentTelegramAlertTarget {
|
|
param(
|
|
[string]$Text,
|
|
[string]$Kind,
|
|
[string]$Service,
|
|
[string]$HostName
|
|
)
|
|
$targetMatch = [regex]::Match($Text, "(?im)(?:\bTarget|資源)\s*[:\uFF1A]\s*([^\r\n]+)")
|
|
if ($targetMatch.Success) {
|
|
return $targetMatch.Groups[1].Value.Trim()
|
|
}
|
|
if ($Kind -eq "provider_freshness_signal") { return "provider_freshness_signal" }
|
|
if ($Kind -eq "host_recovery" -and $HostName) { return $HostName }
|
|
if ($Service -and $Service -ne "unknown") { return $Service }
|
|
return $Kind
|
|
}
|
|
|
|
function Get-AgentTelegramSuggestedMode {
|
|
param([string]$Kind)
|
|
$modeByKind = @{
|
|
"provider_freshness_signal" = "ProviderFreshness"
|
|
"backup_health" = "BackupCheck"
|
|
"wazuh_security_alert" = "SecurityTriage"
|
|
"public_route_502" = "Recover"
|
|
"host_recovery" = "Recover"
|
|
"performance_cpu" = "Perf"
|
|
"performance_memory" = "Perf"
|
|
"performance_disk" = "Perf"
|
|
"harbor_registry" = "HarborRepair"
|
|
"awoooi_k3s" = "AwoooRepair"
|
|
"database_health" = "Status"
|
|
"alert_chain_health" = "Status"
|
|
"application_ai_config_error" = "Status"
|
|
"repository_missing" = "Status"
|
|
"cicd_failure" = "Status"
|
|
}
|
|
if ($modeByKind.ContainsKey($Kind)) { return [string]$modeByKind[$Kind] }
|
|
return "Status"
|
|
}
|
|
|
|
function Get-AgentTelegramSourceEventResolutionPolicy {
|
|
param([string]$Kind)
|
|
if ($Kind -in @(
|
|
"provider_freshness_signal",
|
|
"backup_health",
|
|
"public_route_502",
|
|
"host_recovery",
|
|
"performance_cpu",
|
|
"performance_memory",
|
|
"performance_disk",
|
|
"harbor_registry",
|
|
"awoooi_k3s"
|
|
)) {
|
|
return "mode_verifier_can_resolve"
|
|
}
|
|
return "external_source_receipt_required"
|
|
}
|
|
|
|
function Convert-AgentTelegramRouteToken {
|
|
param([string]$Value)
|
|
$safe = [regex]::Replace($Value.ToLowerInvariant(), "[^a-z0-9_.-]+", "_").Trim([char[]]".-_")
|
|
if ($safe) { return $safe }
|
|
return "unknown"
|
|
}
|
|
|
|
function Test-AgentTelegramSuppressPage {
|
|
param([object]$Update)
|
|
$text = if ($Update -and $Update.PSObject.Properties["text"]) { [string]$Update.text } else { "" }
|
|
$chatMatch = if ($Update -and $Update.PSObject.Properties["chat_match"]) { [string]$Update.chat_match } else { "" }
|
|
if ($chatMatch -eq "synthetic") { return $true }
|
|
if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true }
|
|
return $false
|
|
}
|
|
|
|
function New-AgentMonitoringAlertFromTelegram {
|
|
param([object]$Update)
|
|
|
|
$text = [string]$Update.text
|
|
$severity = Get-AgentTelegramAlertSeverity $text
|
|
if ($severity -eq "resolved") {
|
|
return [pscustomobject]@{
|
|
ok = $false
|
|
skipped = $true
|
|
reason = "resolved_alert_no_remediation"
|
|
}
|
|
}
|
|
|
|
$alertId = "telegram-monitoring-" + [string]$Update.update_id
|
|
$alertPath = Join-Path $AlertIncomingDir ($alertId + ".json")
|
|
$message = if ($text.Length -gt 2000) { $text.Substring(0, 2000) + "`n...(truncated)" } else { $text }
|
|
$kind = Get-AgentTelegramAlertKind $text
|
|
$service = Get-AgentTelegramAlertService $text
|
|
$hostName = Get-AgentTelegramAlertHost $text
|
|
$target = Get-AgentTelegramAlertTarget $text $kind $service $hostName
|
|
$suggestedMode = Get-AgentTelegramSuggestedMode $kind
|
|
$sourceEventResolutionPolicy = Get-AgentTelegramSourceEventResolutionPolicy $kind
|
|
$correlationKey = @(
|
|
(Convert-AgentTelegramRouteToken $kind),
|
|
(Convert-AgentTelegramRouteToken $target),
|
|
$(if ($hostName) { Convert-AgentTelegramRouteToken $hostName } else { "no-host" })
|
|
) -join ":"
|
|
$suppressTelegram = Test-AgentTelegramSuppressPage $Update
|
|
$labels = if ($kind -eq "provider_freshness_signal") {
|
|
@("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green")
|
|
} else {
|
|
@("telegram", "sre-war-room", "monitoring")
|
|
}
|
|
if ($suppressTelegram) {
|
|
$labels += @("synthetic", "do-not-page")
|
|
}
|
|
$alert = [pscustomobject]@{
|
|
id = $alertId
|
|
source = "telegram-sre-war-room-monitoring"
|
|
severity = $severity
|
|
kind = $kind
|
|
suggestedMode = $suggestedMode
|
|
controlledApply = [bool]($suggestedMode -in @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed"))
|
|
service = $service
|
|
host = $hostName
|
|
title = "AwoooI SRE Telegram monitoring alert"
|
|
message = $message
|
|
labels = $labels
|
|
routing = [pscustomobject]@{
|
|
schemaVersion = "agent99_alert_route_v1"
|
|
kind = $kind
|
|
suggestedMode = $suggestedMode
|
|
routeSource = "telegram_structured_kind"
|
|
groupKey = $correlationKey
|
|
correlationKey = $correlationKey
|
|
singleFlightWindowSeconds = 300
|
|
}
|
|
resolution = [pscustomobject]@{
|
|
schemaVersion = "agent99_source_event_resolution_v1"
|
|
policy = $sourceEventResolutionPolicy
|
|
required = $true
|
|
}
|
|
telegramUpdateId = $Update.update_id
|
|
telegramMessageId = $Update.message_id
|
|
telegramChatTitle = $Update.chat_title
|
|
telegramChatType = $Update.chat_type
|
|
replyChatId = [string]$Update.chat_id
|
|
suppressTelegram = $suppressTelegram
|
|
suppressReason = if ($suppressTelegram) { "telegram_synthetic_or_do_not_page" } else { "" }
|
|
force = $true
|
|
createdAt = (Get-Date -Format o)
|
|
}
|
|
$alert | ConvertTo-Json -Depth 8 | Set-Content -Path $alertPath -Encoding UTF8
|
|
|
|
$sreExitCode = $null
|
|
if ($RunNow -and (Test-Path $SreAlertInboxScript)) {
|
|
$process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $SreAlertInboxScript, "-AgentRoot", $AgentRoot, "-RunNow") -Wait -PassThru -WindowStyle Hidden
|
|
$process.Refresh()
|
|
$sreExitCode = $process.ExitCode
|
|
}
|
|
|
|
[pscustomobject]@{
|
|
ok = $true
|
|
skipped = $false
|
|
alertId = $alertId
|
|
alertPath = $alertPath
|
|
severity = $alert.severity
|
|
kind = $alert.kind
|
|
service = $alert.service
|
|
host = $alert.host
|
|
suppressTelegram = $suppressTelegram
|
|
sreInboxExitCode = $sreExitCode
|
|
}
|
|
}
|
|
|
|
function Invoke-AgentTelegramInboxSelfTest {
|
|
$selfStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$selfTestPath = Join-Path $EvidenceDir "agent99-TelegramInboxSelfTest-$selfStamp.json"
|
|
$cases = @(
|
|
[pscustomobject]@{
|
|
name = "agent99 echo ignored"
|
|
text = "Agent99 Alert: WARNING`nEvent: host performance"
|
|
expectedMonitoring = $false
|
|
expectedSeverity = $null
|
|
expectedKind = $null
|
|
expectedService = $null
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "provider freshness"
|
|
text = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate lastSeen missing"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "warning"
|
|
expectedKind = "provider_freshness_signal"
|
|
expectedService = "unknown"
|
|
expectedHost = ""
|
|
expectedSuppressTelegram = $false
|
|
},
|
|
[pscustomobject]@{
|
|
name = "synthetic provider freshness do not page"
|
|
text = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate lastSeen missing"
|
|
chatMatch = "synthetic"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "warning"
|
|
expectedKind = "provider_freshness_signal"
|
|
expectedService = "unknown"
|
|
expectedHost = ""
|
|
expectedSuppressTelegram = $true
|
|
},
|
|
[pscustomobject]@{
|
|
name = "host perf readback failed"
|
|
text = "Agent99 host performance warning Host: 192.168.0.110 Reason: perf_readback_failed Current: load/core= memAvail= disk="
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "warning"
|
|
expectedKind = "performance_cpu"
|
|
expectedService = "agent99"
|
|
expectedHost = "192.168.0.110"
|
|
},
|
|
[pscustomobject]@{
|
|
name = "backup health"
|
|
text = "backup health warning restore_drill_status backup_file_readback_failed snapshot metadata only"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "warning"
|
|
expectedKind = "backup_health"
|
|
expectedService = "unknown"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "bitan ai config"
|
|
text = "bitan ai support locale guard failed E_AI_MISSING_KEY missing_key /api/customer-support/chat"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "warning"
|
|
expectedKind = "application_ai_config_error"
|
|
expectedService = "bitan-pharmacy"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "security triage"
|
|
text = "Wazuh SIEM critical security alert auth_failed brute force suspicious login host 192.168.0.112"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "wazuh_security_alert"
|
|
expectedService = "iwooos-security"
|
|
expectedHost = "192.168.0.112"
|
|
},
|
|
[pscustomobject]@{
|
|
name = "gitea repo missing"
|
|
text = "critical gitea repository_missing repo_missing production project"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "repository_missing"
|
|
expectedService = "gitea"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "reboot slo wins over cpu text"
|
|
text = "critical Target: reboot-auto-recovery-slo host 192.168.0.110 all_required_hosts_not_in_10_minute_reboot_window cpu pressure"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "host_recovery"
|
|
expectedService = "unknown"
|
|
expectedHost = "192.168.0.110"
|
|
},
|
|
[pscustomobject]@{
|
|
name = "postgres does not become k3s"
|
|
text = "critical PostgreSQL database unavailable service awoooi deployment"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "database_health"
|
|
expectedService = "awoooi"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "alert chain does not become k3s"
|
|
text = "critical AlertChainBroken monitoring pipeline down service awoooi"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "alert_chain_health"
|
|
expectedService = "awoooi"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "k3s stays awooo repair domain"
|
|
text = "critical Target: awoooi-api k3s pod crashloopbackoff deployment"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "awoooi_k3s"
|
|
expectedService = "awoooi"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "public 502"
|
|
text = "critical public route 502 website https://awoooi.wooo.work"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "public_route_502"
|
|
expectedService = "awoooi"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "action required cold start"
|
|
text = "ACTION REQUIRED INC-20260710-5700F7 Target: cold-start-gate"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "warning"
|
|
expectedKind = "host_recovery"
|
|
expectedService = "unknown"
|
|
expectedHost = ""
|
|
},
|
|
[pscustomobject]@{
|
|
name = "failed ci cd stays diagnostic"
|
|
text = "[AWOOOI CI/CD] build-and-deploy AWOOOI 部署失敗"
|
|
expectedMonitoring = $true
|
|
expectedSeverity = "critical"
|
|
expectedKind = "cicd_failure"
|
|
expectedService = "awoooi"
|
|
expectedHost = ""
|
|
}
|
|
)
|
|
|
|
$results = @()
|
|
foreach ($case in $cases) {
|
|
$monitoring = Test-AgentMonitoringAlertText $case.text
|
|
$severity = if ($monitoring) { Get-AgentTelegramAlertSeverity $case.text } else { $null }
|
|
$kind = if ($monitoring) { Get-AgentTelegramAlertKind $case.text } else { $null }
|
|
$service = if ($monitoring) { Get-AgentTelegramAlertService $case.text } else { $null }
|
|
$hostName = if ($monitoring) { Get-AgentTelegramAlertHost $case.text } else { "" }
|
|
$chatMatch = if ($case.PSObject.Properties["chatMatch"]) { [string]$case.chatMatch } else { "configured_chat_id" }
|
|
$suppressTelegram = if ($monitoring) { Test-AgentTelegramSuppressPage ([pscustomobject]@{ text = $case.text; chat_match = $chatMatch }) } else { $false }
|
|
$expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false }
|
|
$passed = (
|
|
$monitoring -eq $case.expectedMonitoring -and
|
|
[string]$severity -eq [string]$case.expectedSeverity -and
|
|
[string]$kind -eq [string]$case.expectedKind -and
|
|
[string]$service -eq [string]$case.expectedService -and
|
|
[string]$hostName -eq [string]$case.expectedHost -and
|
|
$suppressTelegram -eq $expectedSuppressTelegram
|
|
)
|
|
$results += [pscustomobject]@{
|
|
name = $case.name
|
|
passed = $passed
|
|
monitoring = $monitoring
|
|
expectedMonitoring = $case.expectedMonitoring
|
|
severity = $severity
|
|
expectedSeverity = $case.expectedSeverity
|
|
kind = $kind
|
|
expectedKind = $case.expectedKind
|
|
service = $service
|
|
expectedService = $case.expectedService
|
|
host = $hostName
|
|
expectedHost = $case.expectedHost
|
|
suppressTelegram = $suppressTelegram
|
|
expectedSuppressTelegram = $expectedSuppressTelegram
|
|
}
|
|
}
|
|
|
|
$failed = @($results | Where-Object { -not $_.passed })
|
|
$summary = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
ok = (@($failed).Count -eq 0)
|
|
caseCount = @($results).Count
|
|
passedCount = @($results | Where-Object { $_.passed }).Count
|
|
failedCount = @($failed).Count
|
|
wroteAlerts = $false
|
|
sentTelegram = $false
|
|
results = $results
|
|
}
|
|
$summary | ConvertTo-Json -Depth 8 | Set-Content -Path $selfTestPath -Encoding UTF8
|
|
$summary | ConvertTo-Json -Depth 8
|
|
if (-not $summary.ok) { exit 1 }
|
|
exit 0
|
|
}
|
|
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$evidencePath = Join-Path $EvidenceDir "agent99-TelegramInbox-$stamp.json"
|
|
$offsetPath = Join-Path $StateDir "telegram-inbox-offset.json"
|
|
$offset = 0
|
|
if ((-not $SyntheticUpdateText) -and (Test-Path $offsetPath)) {
|
|
try {
|
|
$offsetState = Get-Content $offsetPath -Raw | ConvertFrom-Json
|
|
if ($offsetState.nextOffset) { $offset = [int64]$offsetState.nextOffset }
|
|
} catch {
|
|
$offset = 0
|
|
}
|
|
}
|
|
|
|
$config = Load-AgentConfig
|
|
if ($SelfTest) {
|
|
Invoke-AgentTelegramInboxSelfTest
|
|
}
|
|
|
|
$autoIngestMonitoringAlerts = $true
|
|
if ($config.telegram -and $config.telegram.PSObject.Properties["autoIngestMonitoringAlerts"]) {
|
|
$autoIngestMonitoringAlerts = [bool]$config.telegram.autoIngestMonitoringAlerts
|
|
}
|
|
if ($SyntheticUpdateText) {
|
|
$updatesResult = [pscustomobject]@{
|
|
ok = $true
|
|
error = $null
|
|
updates = @([pscustomobject]@{
|
|
update_id = [int64](Get-Date -UFormat %s)
|
|
message_id = 1
|
|
date = [int64](Get-Date -UFormat %s)
|
|
text = $SyntheticUpdateText
|
|
chat_id = $SyntheticChatId
|
|
chat_title = $SyntheticChatTitle
|
|
chat_type = "supergroup"
|
|
chat_match = "synthetic"
|
|
from = "synthetic-sre-monitor"
|
|
})
|
|
raw = $null
|
|
}
|
|
} else {
|
|
$updatesResult = Get-AgentTelegramUpdates $config $offset
|
|
}
|
|
$processed = @()
|
|
$alerted = @()
|
|
$ignored = @()
|
|
$maxUpdateId = if ($offset -gt 0) { $offset - 1 } else { 0 }
|
|
|
|
foreach ($update in @($updatesResult.updates)) {
|
|
if ($update.update_id -gt $maxUpdateId) { $maxUpdateId = [int64]$update.update_id }
|
|
$instruction = Resolve-AgentTelegramInstruction ([string]$update.text)
|
|
if (-not $instruction) {
|
|
if ($autoIngestMonitoringAlerts -and (Test-AgentMonitoringAlertText ([string]$update.text))) {
|
|
$alertResult = New-AgentMonitoringAlertFromTelegram $update
|
|
if ($alertResult.ok) {
|
|
$alerted += [pscustomobject]@{
|
|
updateId = $update.update_id
|
|
messageId = $update.message_id
|
|
from = $update.from
|
|
chatTitle = $update.chat_title
|
|
chatType = $update.chat_type
|
|
chatMatch = $update.chat_match
|
|
alertId = $alertResult.alertId
|
|
severity = $alertResult.severity
|
|
kind = $alertResult.kind
|
|
service = $alertResult.service
|
|
host = $alertResult.host
|
|
suppressTelegram = $alertResult.suppressTelegram
|
|
alertPath = $alertResult.alertPath
|
|
sreInboxExitCode = $alertResult.sreInboxExitCode
|
|
}
|
|
} else {
|
|
$ignored += [pscustomobject]@{
|
|
updateId = $update.update_id
|
|
reason = $alertResult.reason
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
$ignored += [pscustomobject]@{
|
|
updateId = $update.update_id
|
|
reason = "no_agent99_prefix"
|
|
}
|
|
continue
|
|
}
|
|
|
|
$submitArgs = @(
|
|
"-NoProfile", "-ExecutionPolicy", "Bypass",
|
|
"-File", $SubmitScript,
|
|
"-Instruction", $instruction,
|
|
"-ControlledApply",
|
|
"-Source", "telegram-sre-war-room",
|
|
"-RequestedBy", ([string]$update.from),
|
|
"-ExternalId", ("telegram-update-" + [string]$update.update_id),
|
|
"-ReplyChatId", ([string]$update.chat_id)
|
|
)
|
|
if ($RunNow) { $submitArgs += "-RunNow" }
|
|
$submit = Start-Process -FilePath "powershell.exe" -ArgumentList $submitArgs -Wait -PassThru -NoNewWindow -RedirectStandardOutput (Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).out") -RedirectStandardError (Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).err")
|
|
$submit.Refresh()
|
|
$processed += [pscustomobject]@{
|
|
updateId = $update.update_id
|
|
messageId = $update.message_id
|
|
from = $update.from
|
|
chatTitle = $update.chat_title
|
|
chatType = $update.chat_type
|
|
chatMatch = $update.chat_match
|
|
instruction = $instruction
|
|
submitExitCode = if ($null -ne $submit.ExitCode) { $submit.ExitCode } else { 0 }
|
|
}
|
|
}
|
|
|
|
if ((-not $SyntheticUpdateText) -and $updatesResult.ok -and $maxUpdateId -ge 0) {
|
|
[pscustomobject]@{
|
|
nextOffset = $maxUpdateId + 1
|
|
updatedAt = (Get-Date -Format o)
|
|
} | ConvertTo-Json -Depth 4 | Set-Content -Path $offsetPath -Encoding UTF8
|
|
}
|
|
|
|
$result = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
ok = [bool]$updatesResult.ok
|
|
reason = if ($updatesResult.ok -and @($updatesResult.updates).Count -eq 0) { "no_updates" } elseif ($updatesResult.ok) { "updates_read" } else { "updates_read_failed" }
|
|
error = $updatesResult.error
|
|
relay = $updatesResult.relay
|
|
offsetBefore = $offset
|
|
offsetAfter = if ($SyntheticUpdateText) { $offset } elseif ($updatesResult.ok) { $maxUpdateId + 1 } else { $offset }
|
|
synthetic = [bool]$SyntheticUpdateText
|
|
updatesSeen = @($updatesResult.updates).Count
|
|
processedCount = @($processed).Count
|
|
alertedCount = @($alerted).Count
|
|
ignoredCount = @($ignored).Count
|
|
processed = $processed
|
|
alerted = $alerted
|
|
ignored = $ignored
|
|
offsetPath = $offsetPath
|
|
}
|
|
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8
|
|
$result | ConvertTo-Json -Depth 10
|