From 8a766db5e2612fa587326f4d5752355e19d07a54 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 00:26:52 +0800 Subject: [PATCH] feat(agent99): bridge sre alerts to windows relay --- AGENT99-AI-WORKSTATION-SOP.md | 691 ++++++ HOST-REBOOT-AI-AUTOMATION-SOP.md | 589 +++++ agent99-bootstrap.ps1 | 162 ++ agent99-control-plane.ps1 | 2454 +++++++++++++++++++ agent99-install-sre-alert-relay.ps1 | 119 + agent99-register-tasks.ps1 | 182 ++ agent99-sre-alert-inbox.ps1 | 297 +++ agent99-sre-alert-relay.ps1 | 218 ++ agent99-submit-request.ps1 | 150 ++ agent99-telegram-inbox.ps1 | 488 ++++ agent99-windows-update-policy.ps1 | 104 + agent99.config.99.example.json | 273 +++ agent99.config.example.json | 205 ++ apps/api/src/api/v1/webhooks.py | 16 + apps/api/src/core/config.py | 22 + apps/api/src/services/agent99_sre_bridge.py | 357 +++ apps/api/tests/test_agent99_sre_bridge.py | 143 ++ host-reboot-scorecard.ps1 | 197 ++ k8s/awoooi-prod/04-configmap.yaml | 6 + 19 files changed, 6673 insertions(+) create mode 100644 AGENT99-AI-WORKSTATION-SOP.md create mode 100644 HOST-REBOOT-AI-AUTOMATION-SOP.md create mode 100644 agent99-bootstrap.ps1 create mode 100644 agent99-control-plane.ps1 create mode 100644 agent99-install-sre-alert-relay.ps1 create mode 100644 agent99-register-tasks.ps1 create mode 100644 agent99-sre-alert-inbox.ps1 create mode 100644 agent99-sre-alert-relay.ps1 create mode 100644 agent99-submit-request.ps1 create mode 100644 agent99-telegram-inbox.ps1 create mode 100644 agent99-windows-update-policy.ps1 create mode 100644 agent99.config.99.example.json create mode 100644 agent99.config.example.json create mode 100644 apps/api/src/services/agent99_sre_bridge.py create mode 100644 apps/api/tests/test_agent99_sre_bridge.py create mode 100644 host-reboot-scorecard.ps1 diff --git a/AGENT99-AI-WORKSTATION-SOP.md b/AGENT99-AI-WORKSTATION-SOP.md new file mode 100644 index 000000000..e4e009b23 --- /dev/null +++ b/AGENT99-AI-WORKSTATION-SOP.md @@ -0,0 +1,691 @@ +# Agent99 AI Workstation SOP + +Last updated: 2026-07-09 + +## Objective + +192.168.0.99 is the AI Agent workstation and local operations control node. + +It must act as: + +- Secretary: reminders, status summaries, evidence packets, operator updates. +- Assistant: routine checks, file organization, browser/readback checks, alert routing. +- Engineer: VM control, service recovery, deploy/readback verification, runbook execution. +- Incident operator: reboot detection, recovery orchestration, Telegram alerting, SLA countdown. + +The machine must be able to operate VMware, SSH to service hosts, run public HTTP readbacks, and produce evidence without depending on old chat context. + +## Agent99 Operating Modes + +Observe mode: + +- Read host, VM, service, and website state. +- Produce evidence only. +- Default mode for scheduled checks. + +Controlled apply mode: + +- Execute approved low-blast-radius actions. +- Must write evidence before and after each action. +- Requires explicit operator approval or scheduled startup recovery policy. + +Break-glass mode: + +- Reserved for destructive, irreversible, secret-bearing, cost-bearing, or firewall/provider cutover actions. +- Must not be hidden inside automatic tasks. + +## Required Local Capabilities On 99 + +Base runtime: + +- Windows PowerShell 5+ or PowerShell 7. +- Windows OpenSSH client. +- `curl.exe`. +- VMware Workstation. +- `vmrun.exe`. +- Windows Task Scheduler. + +Agent filesystem: + +- `C:\Wooo\Agent99\bin` +- `C:\Wooo\Agent99\config` +- `C:\Wooo\Agent99\evidence` +- `C:\Wooo\Agent99\logs` +- `C:\Wooo\Agent99\queue` +- `C:\Wooo\Agent99\requests` +- `C:\Wooo\Agent99\alerts\incoming` +- `C:\Wooo\Agent99\playbooks` + +Agent scripts: + +- `agent99-bootstrap.ps1` +- `agent99-control-plane.ps1` +- `agent99-submit-request.ps1` +- `agent99-telegram-inbox.ps1` +- `agent99-sre-alert-inbox.ps1` +- `host-reboot-scorecard.ps1` +- `agent99.config.json` + +## Command Ingress Without Codex Quota + +Codex is not the runtime command plane. Agent99 must accept operator work through local 99 channels even when Codex is unavailable. + +Supported command channels: + +- Local desktop: `Agent99 Submit Request.lnk`. +- Local queue: write allowlisted command JSON to `C:\Wooo\Agent99\queue`. +- AwoooI SRE Telegram group: send `/agent99 ...`, `!agent99 ...`, `#agent99 ...`, `agent99 ...`, or `@agent99 ...`. +- SRE war room alerts: write alert JSON to `C:\Wooo\Agent99\alerts\incoming`. +- AwoooI SRE Telegram monitoring alerts: non-command messages that look like Alertmanager/FIRING/critical/warning/502/CPU/disk/Harbor/K3s/backup/host-down/provider-freshness alerts are auto-ingested. + +Command processing requirements: + +- Every command resolves to an allowlisted mode only. +- Website plus 502 plus recover/repair intent resolves to `Recover`, not passive smoke-only checking. +- Alert-driven remediation uses controlled apply only for allowlisted modes such as `Recover`, `HarborRepair`, `AwoooRepair`, `Perf`, and `LoadShed`. +- Every accepted command writes a request record, queue record, execution result, and evidence pointer. +- Telegram messages must be human-readable: source, instruction, mode, result, and evidence path. Raw Python, raw stack trace, token, chat id, cookies, sessions, or `.env` must never be sent. +- Telegram SRE group commands must be recorded with `source=telegram-sre-war-room`. +- Telegram SRE monitoring alerts must be recorded with `source=telegram-sre-war-room-monitoring`. +- Telegram SRE group matching uses `allowedCommandChatTitlePattern=AwoooI SRE|AWOOOI SRE|SRE`, so the command path is not limited to the default private chat id. +- When a command originates from a matched Telegram group, completion replies are routed back to the same Telegram origin while evidence stores only `telegram-origin`, not the raw chat id. +- After automated recovery completes, Agent99 must report the result back to Telegram so the project AI Agents and human operators know whether the abnormality is resolved. +- Every queue execution must update problem counts, write a Playbook/KM record, and append a RAG-import JSONL line. +- Operator command and remediation result Telegram reports are not deduped; each completed repair emits a completion report. Noisy warning/critical signals still use dedupe. + +SRE Telegram command examples: + +- `/agent99 檢查所有網站是否正常，若502就自動修復` +- `/agent99 檢查110和188 CPU負載，必要時執行允許的降載` +- `/agent99 檢查AWOOOI K3s與Harbor，若ImagePullBackOff就修復` +- `/agent99 檢查備份健康狀態` +- `/agent99 建立 provider freshness candidate，要求 provider window、last seen、direct ref 與 verifier readback` + +Monitoring alert auto-ingest examples: + +- `Alertmanager FIRING critical public_route_502 service=awoooi host=192.168.0.120 HTTP 502` +- `HostHighCpuLoad warning host=192.168.0.110` +- `HarborRegistryDown critical host=192.168.0.110` +- `KubePodCrashLooping critical namespace=awoooi-prod` +- `provider_freshness_signal target=provider_freshness_signal freshness last_seen missing direct_ref required` + +Problem memory outputs: + +- Problem counters: `C:\Wooo\Agent99\state\problem-counts.json`. +- Full event ledger: `C:\Wooo\Agent99\playbooks\agent99-incident-records.jsonl`. +- Playbook/KM summary: `C:\Wooo\Agent99\playbooks\agent99-incident-km.md`. +- RAG import feed: `C:\Wooo\Agent99\playbooks\rag-import\agent99-incidents.jsonl`. +- Dashboard: `C:\Wooo\Agent99\dashboard\index.html`, section `Problem Counts`. + +## SRE War Room Alert Contract + +AWOOOI SRE monitors and other AI Agent work windows can hand alerts to Agent99 by writing JSON files to: + +`C:\Wooo\Agent99\alerts\incoming` + +Minimum fields: + +```json +{ + "id": "alert-unique-id", + "source": "awoooi-sre-war-room", + "severity": "critical", + "kind": "public_route_502", + "service": "awoooi", + "host": "192.168.0.120", + "title": "AWOOOI public route returned 502", + "message": "The public health route is returning 502.", + "createdAt": "2026-07-09T22:50:00+08:00" +} +``` + +Routing rules: + +- `502`, `site_502`, `public_route_502`, `route_down`, `host_down`, VM down, or reboot detected -> `Recover`. +- CPU, memory, disk, performance pressure -> `Perf` with controlled apply. +- Harbor or registry -> `HarborRepair`. +- AWOOOI, K3s, image pull, pod, deployment -> `AwoooRepair`. +- Backup, snapshot, restore drill, or cron -> `BackupCheck`. +- Unknown actionable alert -> `Status` evidence first. + +Alert processing evidence: + +- Inbox evidence: `agent99-SreAlertInbox-*.json`. +- Queued request: `C:\Wooo\Agent99\requests`. +- Execution evidence: `agent99-Recover-*.json`, `agent99-Perf-*.json`, or the resolved mode evidence file. + +## VM Control Requirements + +Agent99 must be able to operate these VM hosts: + +- 110 +- 112 +- 120 +- 121 +- 188 + +Required actions: + +- Detect VM powered-off state. +- Start missing VMs through `vmrun.exe`. +- Detect VMware config lock symptoms. +- Avoid deleting VM lock files unless VMware process state proves the VM is not running. +- Prefer guest shutdown/restart first; use reset only when the guest is stuck and SLA requires recovery. + +## Engineering Recovery Requirements + +Agent99 must support these engineering actions: + +- SSH readback to 110 and 120. +- Harbor core / Redis / registry checks on 110. +- K3s AWOOOI deployment and pod checks on 120. +- Public route readback for user-visible websites. +- Evidence capture after every action. +- Telegram alert hooks. + +## Main Scheduled Automations + +Startup recovery: + +- Trigger: 99 boot or operator logon. +- Frequency: once at startup, then repeat during first recovery window. +- Mode: controlled apply allowed for approved actions. +- Purpose: start VMs, restore Harbor, restore K3s pods, verify public routes. + +Heartbeat: + +- Trigger: every 5 minutes. +- Mode: observe by default. +- Purpose: detect host down, 502, service degraded, backup job missing, or stale freshness. + +Daily evidence summary: + +- Trigger: daily. +- Mode: observe. +- Purpose: produce a short host/service/site/backup status packet. + +## Telegram Alert Classes + +The agent must send alerts for: + +- `host_down` +- `host_reboot_detected` +- `vm_not_running` +- `vm_start_attempted` +- `service_recovering` +- `harbor_unhealthy` +- `k3s_image_pull_failure` +- `site_502` +- `site_recovered` +- `service_degraded` +- `backup_missing` +- `backup_failed` +- `recovery_sla_risk` +- `recovery_complete` + +Telegram credentials must be stored outside the repo, preferably in Windows Credential Manager or machine environment variables. + +## 2026-07-09 Recovery Logic To Embed + +When AWOOOI or other sites return 502: + +1. Do not only inspect nginx. +2. Check 120 K3s workloads first. +3. If AWOOOI pods are `ImagePullBackOff`, check 110 Harbor before deleting pods. +4. If Harbor token endpoint returns 500 or pulls fail, inspect: + - Docker engine. + - Harbor Redis. + - `harbor-core`. + - `registry`. + - `registryctl`. +5. If `harbor-core` is not healthy: + - Restart Redis and `harbor-core`. + - If Docker/systemd hangs, execute controlled 110 guest restart via VMware. +6. After `harbor-core` is healthy: + - Delete only failed AWOOOI pods. + - Wait for deployment readiness. + - Verify public HTTP. + +## First Implementation Milestone + +Milestone M1 is complete when: + +- Agent99 directory exists on 99. +- Config file exists. +- Startup and heartbeat scheduled tasks exist. +- Scorecard can run in observe mode. +- Recovery can run in controlled apply mode. +- Evidence JSON is written. +- No secrets are stored in repo files. + +## 2026-07-09 M1 Receipt + +M1 is operational on 99 as of 2026-07-09 16:49 Asia/Taipei. + +- `C:\Wooo\Agent99` exists on `WOOO-SUPER`. +- `agent99-control-plane.ps1`, `host-reboot-scorecard.ps1`, and task registration scripts are deployed. +- `agent99.config.json` includes VMX selectors for 110, 188, 112, 120, and 121. +- `Wooo-Agent99-Heartbeat` is registered and produced `LastTaskResult=0`. +- `Wooo-Agent99-Startup-Recovery` is registered and produced `LastTaskResult=0`. +- Latest heartbeat evidence showed Harbor healthy, AWOOOI deploy ready, and public routes non-502. +- Latest recovery evidence showed all five configured VM hosts already reachable and no recovery action required. + +Remaining M2 work: + +- Install Telegram credentials with environment variables or Windows Credential Manager, then verify real delivery. +- Add backup-job freshness readback and alert classes. +- Add Windows Update no-surprise-reboot policy verification. +- Promote this into a long-running Windows service after the scheduled-task lane stays clean. + +## 2026-07-09 M2 Performance Guard Receipt + +Agent99 performance guard is operational on 99 as of 2026-07-09 17:15 Asia/Taipei. + +- `Wooo-Agent99-Performance-Guard` is registered in Windows Task Scheduler. +- Trigger interval: every 1 minute. +- Last manual verification: `LastTaskResult=0`. +- Latest evidence: `C:\Wooo\Agent99\evidence\agent99-Perf-20260709-171502.json`. +- Readback coverage: + - 110: OK, load-per-core 0.35, memory available 62.93%, disk used 82%. + - 112: OK, load-per-core 0.07, memory available 61.06%, disk used 47%. + - 120: OK, load-per-core 0.05, memory available 91.76%, disk used 44%. + - 121: OK, load-per-core 0.00, memory available 92.79%, disk used 27%. + - 188: OK, load-per-core 0.08, memory available 90.19%, disk used 89%. +- 99 SSH access was completed for: + - 112 with user `kali`. + - 188 with user `ollama`. +- Load-shedding is controlled by allowlist only. + - Current allowlisted action: pause `vibework-dedicated-runner` on 110 only when `load_per_core_critical` is present. + - Disk or memory pressure only alerts; it does not trigger the CPU runner pause action. +- Telegram audit is enabled in config and called for Agent99 start, warnings, critical events, load-shed attempts/results, and completion. + - 99 local env aliases are supported for `AGENT99_*`, `AWOOOI_TG_*`, `OPENCLAW_TG_*`, `TELEGRAM_*`, and `TG_*`. + - If 99 does not have local Telegram env values, Agent99 relays through the existing 110 `stockplatform-v2-api-1` container and uses that container's existing `TELEGRAM_BOT_TOKEN` / `TELEGRAM_CHAT_ID`. + - 2026-07-09 17:27 verification: `Wooo-Agent99-Performance-Guard` produced `LastTaskResult=0`, `agent99-Perf-20260709-172712.json`, `tgSent=2`, and `relayOk=True`. + - No token or chat id is stored in repo files or Agent99 evidence. + +## 2026-07-09 OpenClaw Receipt + +OpenClaw is configured and now included in Agent99 evidence. + +- Source truth: + - AWOOOI prod config: `OPENCLAW_URL=http://192.168.0.188:8088`. + - 188 Ansible service target: OpenClaw on port `8088`. +- Runtime truth: + - `192.168.0.188:8088/health` returns `status=healthy`, `version=6.0`, `service=ClawBot`, `environment=production`, and `telegram_bot=connected`. + - `clawbot.service` on 188 is `active`. + - `192.168.0.188:8089` is not the current production endpoint. +- Agent99 truth: + - Latest status evidence: `C:\Wooo\Agent99\evidence\agent99-Status-20260709-173813.json`. + - `aiServices.openclaw`: `ok=True`, `status=healthy`, `url=http://192.168.0.188:8088/health`. + - Telegram relay evidence: `tgSent=2`, `relayOk=2`. + +## 2026-07-09 M3 Command And Alert Ingress Receipt + +Agent99 no longer depends on Codex as the only command channel. + +- Telegram command inbox deployed: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1`. +- SRE alert inbox deployed: `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1`. +- Scheduled tasks registered: + - `Wooo-Agent99-Telegram-Inbox` + - `Wooo-Agent99-SRE-Alert-Inbox` +- PowerShell parse readback: all five Agent99 scripts parsed successfully on 99. +- Live config includes both scheduled tasks and freshness contracts. +- Telegram inbox evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260709-230031.json`. + - `ok=True` + - `updatesSeen=0` + - `processedCount=0` + - no Python code or token text in evidence. +- SRE synthetic alert evidence: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260709-230107.json`. + - input kind: `public_route_502` + - source: `awoooi-sre-war-room` + - resolved mode: `Recover` + - queued count: `1` + - failed count: `0` + - `controlExitCode=0` +- Recovery execution evidence: `C:\Wooo\Agent99\evidence\agent99-Recover-20260709-230108.json`. + - public route failures: `0` + - host reachability failures: `0` + - Harbor core healthy: `True` + - AWOOOI deploy ready: `True` +- Queue processor evidence: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260709-230108.json`. + - processed count: `1` + - source: `agent99-sre-alert-inbox` + - reason: `sre_alert` + - mode: `Recover` + - exit code: `0` + - Telegram relay: `sent=True`, `relayOk=True` + - evidence contains no Python code. +- Self-health evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-230223.json`. + - severity: `ok` + - task failures: `0` + - evidence critical: `0` + - evidence warning: `0` + - Telegram inbox task: OK + - SRE alert inbox task: OK +- Dashboard JSON: `C:\Wooo\Agent99\dashboard\agent99-control-center.latest.json`. + - queue pending: `0` + - alert pending: `0` + +## 2026-07-09 M4 SRE Telegram Command And Problem Memory Receipt + +Agent99 now accepts AwoooI SRE Telegram-group commands as first-class work orders. + +- Telegram SRE command source mapping: + - source: `telegram-sre-war-room` + - request path: `C:\Wooo\Agent99\requests` + - queue path: `C:\Wooo\Agent99\queue` + - command result event: `operator_command_result` +- Synthetic SRE Telegram command verification: + - instruction: `檢查所有網站是否恢復正常，若502就自動修復` + - request id: `request-20260709-231157-d53f53d8` + - resolved mode: `Recover` + - control exit code: `0` + - evidence: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260709-231158.json` + - source: `telegram-sre-war-room` + - processed mode: `Recover` + - processed ok: `True` + - processed exit code: `0` + - Telegram relay: sent `1` + - Telegram evidence contains no Python code. +- Problem memory readback: + - problem key: `telegram-sre-war-room_agent99_operator_instruction` + - occurrences: `1` + - resolved: `1` + - failed: `0` + - stats path: `C:\Wooo\Agent99\state\problem-counts.json` + - KM exists: `True` + - RAG feed exists: `True` + - dashboard problem events: `1` + - dashboard resolved: `1` + - dashboard failed: `0` +- Post-check: + - Telegram inbox evidence: `ok=True`, `updatesSeen=0`, `processedCount=0` + - Self-health evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-231255.json` + - self-health severity: `ok` + - task failures: `0` + - queue pending: `0` + - alert pending: `0` +- SRE group routing hardening: + - Existing relay default chat readback was a private chat, so group commands must not depend only on `TELEGRAM_CHAT_ID`. + - Live config was updated with `allowedCommandChatTitlePattern=AwoooI SRE|AWOOOI SRE|SRE`. + - Telegram inbox accepts matched group commands and carries a redacted `telegram-origin` reply route for completion reports. + +## 2026-07-09 M5 SRE Telegram Monitoring Alert Auto-Ingest Receipt + +Agent99 now ingests monitoring alerts from the AwoooI SRE Telegram group, not only `/agent99` commands. + +- Runtime switch: + - `autoIngestMonitoringAlerts=True` + - `allowedCommandChatTitlePattern=AwoooI SRE|AWOOOI SRE|SRE` +- Encoding hardening: + - Runtime PowerShell scripts are ASCII-only. + - Chinese alert keyword support uses Unicode codepoint checks instead of raw Chinese strings, avoiding Windows PowerShell 5 parser failures. +- Synthetic SRE Telegram monitoring alert: + - text: `Alertmanager FIRING critical public_route_502 service=awoooi host=192.168.0.120 HTTP 502` + - no `/agent99` prefix. + - Telegram inbox evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260709-233230.json` + - `inboxOk=True` + - `alertedCount=1` +- SRE alert bridge evidence: + - `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260709-233231.json` + - queued count: `1` + - failed count: `0` +- Control/recovery evidence: + - `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260709-233232.json` + - source: `telegram-sre-war-room-monitoring` + - kind: `public_route_502` + - mode: `Recover` + - processed ok: `True` + - exit code: `0` + - Telegram attempts: `1` + - Telegram sent: `1` + - evidence contains no Python code. +- Problem memory: + - total events before: `2` + - total events after: `3` + - total resolved after: `3` + - total failed after: `0` + +## 2026-07-09 M6 Provider Freshness Signal Receipt + +Provider freshness alerts must not be treated as passive route health. + +- Gap found: + - Telegram Inbox and SRE Alert Inbox were running, but `provider_freshness_signal` was not present in Agent99 alerts, queue, or execution evidence after 2026-07-09 23:30 Asia/Taipei. + - Only SelfCheck `evidenceFreshness` fields matched the word `freshness`, so the original alert was not processed as an actionable provider freshness signal. +- Runtime behavior added: + - Telegram auto-ingest recognizes `provider_freshness_signal`, `source_provider_freshness_triage`, `raw_signal_normalized`, `controlled_runtime_candidate`, `freshness`, `last seen`, `last_seen`, and `ingestion`. + - SRE Alert Inbox routes freshness signals to `ProviderFreshness`. + - `ProviderFreshness` creates a read-only candidate under `C:\Wooo\Agent99\state\provider-freshness-candidates`. + - Candidate evidence must require provider window, last seen, direct reference, and verifier readback. + - Guardrail remains: no provider switch, no paid model call, no env change, no reload. + +## 2026-07-09 M7 AWOOI API To Agent99 Alert Bridge + +Agent99 must not depend only on Telegram text polling for AwoooI SRE alerts. + +- Producer-side bridge: + - AWOOI API `POST /api/v1/webhooks/alertmanager` now schedules a fail-open Agent99 SRE bridge after alert normalization and fingerprinting. + - The bridge POSTs the same Agent99 alert contract to `AGENT99_SRE_ALERT_RELAY_URL` when configured. + - If relay URL is empty, the bridge writes the contract expected by `C:\Wooo\Agent99\alerts\incoming` when `AGENT99_SRE_ALERT_INBOX_PATH` is configured. + - Empty relay/path means bridge disabled; Alertmanager response must not fail or retry-storm because Agent99 inbox is unavailable. +- Safety: + - Payloads are sanitized before writing. + - Labels with token, secret, password, authorization, cookie, session, or private-key names are redacted. + - Provider freshness signals map to `ProviderFreshness`. + - Route 502 maps to `Recover`; CPU/memory/disk/load maps to `Perf`; backup maps to `BackupCheck`; Harbor maps to `HarborRepair`; K3s/pod/image-pull maps to `AwoooRepair`. +- Guardrail: + - This bridge does not switch provider, call paid models, edit env, reload services, or read raw secrets. + - problem key: `agent99-sre-alert-inbox_awoooi_192.168.0.120_public_route_502` + - problem occurrences: `2` + +## 2026-07-10 M8 Agent99 Direct SRE Relay Contract + +Telegram text polling is not enough for SLA-grade SRE automation. + +- 99-side relay: + - Script: `C:\Wooo\Agent99\bin\agent99-sre-alert-relay.ps1`. + - Scheduled task: `Wooo-Agent99-SRE-Alert-Relay`. + - Default prefix: `http://+:8787/agent99/sre-alert/`. + - Accepted sources: local and `192.168.0.*` by default. + - Optional token env: `AGENT99_SRE_RELAY_TOKEN`. +- Production producer: + - `k8s/awoooi-prod/04-configmap.yaml` sets `AGENT99_SRE_ALERT_RELAY_URL=http://192.168.0.99:8787/agent99/sre-alert/`. + - `AGENT99_SRE_ALERT_RELAY_TOKEN` is optional and must come from Secret/env only. + - API bridge uses relay first, then mounted file inbox fallback. +- Runtime behavior: + - Relay validates POST + JSON, writes into `C:\Wooo\Agent99\alerts\incoming`, and triggers `agent99-sre-alert-inbox.ps1 -RunNow`. + - SRE Alert Inbox resolves mode and queues only allowlisted work. + - Control Loop writes evidence, Telegram completion, problem counts, KM, and RAG import records. + - Relay must not execute arbitrary shell, read secrets, switch providers, reload services, or perform destructive actions. + +## 2026-07-09 Windows Update Policy Receipt + +99 is configured to prevent surprise Windows Update reboots. + +- Evidence: `C:\Wooo\Agent99\evidence\agent99-windows-update-policy-20260709-173933.json`. +- `NoAutoRebootWithLoggedOnUsers=1`. +- `AlwaysAutoRebootAtScheduledTime=0`. +- `AUOptions=3`. +- `ActiveHoursStart=0`, `ActiveHoursEnd=23`. +- `gpupdateOk=True`. + +## 2026-07-09 M3 Self-Health Receipt + +Agent99 now monitors its own automation loop. + +- Scheduled task: `Wooo-Agent99-Self-Health`. +- Trigger interval: every 5 minutes. +- Task Scheduler verification: + - `LastTaskResult=0`. + - Latest triggered evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-175218.json`. + - `evidenceAfterTrigger=True`. +- Self-check coverage: + - Startup recovery task, heartbeat task, performance guard task, and self-health task. + - Evidence freshness for `agent99-Perf-*.json`, `agent99-Status-*.json`, `agent99-SelfCheck-*.json`, and `agent99-Recover-*.json`. + - Latest performance evidence must include all five hosts: 110, 112, 120, 121, and 188. + - Recent attempted Telegram delivery must be successful through direct env credentials or the 110 relay; a quiet interval with no alert attempts is OK and must not create a false critical self-check. +- Latest result: + - `selfHealth.severity=ok`. + - `taskFailures=0`. + - `performanceSeverity=ok`. + - `telegramSeverity=ok`. + - `tgSent=3`, `relayOk=3`. + +## 2026-07-09 M4 110 Load Auto-Reduction Receipt + +Agent99 now handles the 110 CPU/load pressure pattern found after reboot. + +- Incident findings: + - The high-load source was not the runner. + - Normal high consumers included Gitea and StockPlatform Postgres/API. + - Backup load came from `/backup/scripts/backup-awoooi.sh`, `gzip`, and remote `pg_dump`. + - A random `alpine "tar czf - /data"` container named `mystifying_napier` was consuming CPU and logging compressed binary output. +- Manual controlled action applied once: + - Backup PIDs were set to nice 15 / ionice best-effort 7. + - `mystifying_napier` was stopped and retained as a dead container for evidence. +- Automation added to `performance.loadShedding.actions`: + - `stop-110-runaway-alpine-tar-data`: warning-or-critical action for allowlisted runaway `alpine tar czf - /data` containers. + - `renice-110-awoooi-backup-pressure`: warning-or-critical action for backup/gzip/remote-pg-dump pressure. + - `pause-110-ci-runner-pressure`: critical-only action for runner pressure, unchanged. +- Verification: + - Evidence: `C:\Wooo\Agent99\evidence\agent99-Perf-20260709-180704.json`. + - 110: severity OK, load-per-core 0.67, memory available 62.02%, disk used 80%. + - 112, 120, 121, and 188: severity OK. + - `perfCritical=0`, `perfWarning=0`. + - Telegram relay evidence: `tgSent=2`, `relayOk=2`. + +## 2026-07-09 M5 Backup Health Receipt + +Agent99 now monitors backup freshness and backup-job contracts. + +- Scheduled task: `Wooo-Agent99-Backup-Health`. +- Trigger interval: every 15 minutes. +- Task Scheduler verification: + - `LastTaskResult=0`. + - Latest triggered evidence: `C:\Wooo\Agent99\evidence\agent99-BackupCheck-20260709-190617.json`. + - `evidenceAfterTrigger=True`. +- BackupCheck coverage: + - 13 snapshot targets: `awoooi`, `gitea`, `harbor`, `momo`, `langfuse`, `monitoring`, `signoz`, `clawbot`, `open-webui`, `sentry`, `ai-artifacts`, `configs`, and `public-routes`. + - 7 file/status checks: `backup.log`, `backup-status.log`, legacy `status.log`, legacy `cron.log`, integrity check status, restore drill status, and configs last-status JSON. + - 6 cron contract patterns: `backup-all.sh`, `backup-awoooi-frequent.sh`, `backup-health-textfile-exporter.py`, `check-backup-integrity.sh`, `sync-offsite-backups.sh --mode status`, and `backup-status.sh`. +- Latest result: + - `backupHealth.severity=ok`. + - `targetCritical=0`, `targetWarning=0`. + - `fileCritical=0`, `fileWarning=0`. + - `cronCritical=0`, `cronWarning=0`. + - `tgSent=2`, `relayOk=2`. +- Agent self-health now includes backup monitoring: + - Evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-190714.json`. + - `Wooo-Agent99-Backup-Health`: OK, `LastTaskResult=0`. + - Backup evidence freshness: OK, latest file `agent99-BackupCheck-20260709-190617.json`. + +## 2026-07-09 M6 Alert-to-Remediation Receipt + +Agent99 now treats alerts as action triggers, not as the final output. + +- Telegram relay bug fixed: + - The relay sender now decodes the message argument from `sys.argv[2]`, so Telegram receives the alert text instead of the Python relay source. + - Info/start/complete events are suppressed by default: `alertAllOperations=false`, `alertInfoEvents=false`. + - Duplicate warning/critical events are deduped for 30 minutes using stable event keys. +- Controlled remediation behavior: + - `Perf -ControlledApply` suppresses immediate performance spam, runs allowlisted load-shedding/remediation first, then alerts the controlled remediation result. + - If no remediation applies, Agent99 still alerts the original warning/critical performance issue. + - 188 disk pressure now has an allowlisted action: `prune-188-docker-build-cache-disk-pressure`. + - The 188 prune action uses `timeoutSeconds=240` and a post-apply verifier: `postVerifyMaxDiskUsedPercent=89`. + - A command timeout is not reported as final critical when the post-verifier proves the host is back under the disk threshold. +- False critical prevention: + - Self-health no longer treats "no recent alerts were needed" as Telegram failure. + - Telegram health is critical only when recent alert attempts exist and delivery failed. + - Telegram delivery health uses a 60-minute lookback across recent evidence files, not only the newest file per evidence family. + - Transient performance warnings are no longer promoted to self-health critical; only missing or critical host performance evidence is critical. +- Backup monitor false critical prevention: + - Backup snapshot readback now supports `snapshotReadTimeoutSeconds`. + - 99 live config sets `snapshotReadTimeoutSeconds=90`. + - Harbor backup timeout at 22:10 was verified as a readback timeout, not a missing backup: `/backup/harbor/snapshots` had latest metadata from 2026-07-09 17:48 Asia/Taipei and remained within the 1800-minute SLA. +- Evidence hygiene: + - Harbor status readback no longer stores raw `docker inspect` JSON. + - Agent99 now records only selected Harbor health fields and sets `inspectOutput=[redacted: raw docker inspect omitted]`. + - Existing Status evidence on 99 was sanitized: 77 files checked; `inspectOutputNotRedactedCount=0`; `unsafeHarborFieldCount=0`. +- Runtime readback after deployment: + - Performance evidence: `C:\Wooo\Agent99\evidence\agent99-Perf-20260709-220718.json`. + - Self-health evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-220829.json`, `selfHealth.severity=ok`. + - Status evidence: `C:\Wooo\Agent99\evidence\agent99-Status-20260709-221147.json`. + - Host reachability: 110, 112, 120, 121, and 188 all ping OK. + - Public routes: AWOOOI, StockPlatform, FIFA, Gitea, and Harbor all HTTP 200 / non-502. + - Harbor: core, Redis, and registry healthy with redacted evidence. + - AWOOOI: `deployReady=true`, no image pull failure, no crash failure. + - Backup evidence after timeout fix: `C:\Wooo\Agent99\evidence\agent99-BackupCheck-20260709-221759.json`, `backupHealth.severity=ok`, all target/file/cron critical and warning counts are 0. + - Final self-health evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-222236.json`, `selfHealth.severity=ok`, `taskFailures=0`, `performanceSeverity=ok`, `telegramSeverity=ok`. + +## 2026-07-09 M7 Visible Control Center Receipt + +Agent99 is now visible and operable on the 99 Windows desktop instead of being only background scheduled tasks. + +- New mode: + - `ControlTick`. + - Runs the Agent99 visible control loop. + - Updates dashboard HTML/JSON. + - Processes allowlisted operator command queue files. + - Writes evidence and alerts operator command results to Telegram. +- Scheduled task: + - `Wooo-Agent99-Control-Loop`. + - Trigger interval: every 1 minute. + - Self-health now checks this task. +- Desktop shortcuts created: + - `C:\Users\Public\Desktop\Agent99 Control Center.lnk`. + - `C:\Users\Public\Desktop\Agent99 Queue.lnk`. + - `C:\Users\Public\Desktop\Agent99 Evidence.lnk`. + - `D:\Desktop\Agent99 Control Center.lnk`. + - `D:\Desktop\Agent99 Queue.lnk`. + - `D:\Desktop\Agent99 Evidence.lnk`. +- Dashboard paths: + - HTML: `C:\Wooo\Agent99\dashboard\index.html`. + - JSON: `C:\Wooo\Agent99\dashboard\agent99-control-center.latest.json`. +- Queue processor: + - Input: `C:\Wooo\Agent99\queue\*.json`. + - Processed output: `C:\Wooo\Agent99\queue\processed\processed-.json`. + - Failed output: `C:\Wooo\Agent99\queue\failed\failed-.json`. + - Allowed modes: `Status`, `Recover`, `StartVMs`, `PublicSmoke`, `HarborRepair`, `AwoooRepair`, `Perf`, `LoadShed`, `SelfCheck`, `BackupCheck`. + - Operator command results are Telegram-alerted even when info/start/complete noise is suppressed. +- Verification: + - Control evidence: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260709-223309.json`. + - Processed command: `codex-visible-publicsmoke-20260709-223308`. + - Processed result: `ok=true`, `exitCode=0`, `durationSeconds=2.2`. + - Command evidence: `C:\Wooo\Agent99\evidence\agent99-PublicSmoke-20260709-223309.json`. + - Public smoke: AWOOOI, StockPlatform, FIFA, Gitea, and Harbor all HTTP 200 / non-502. + - Telegram result: `operator_command_result`, `severity=info`, `sent=true`, `relayOk=true`. + - Self-health evidence after control-loop install: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-223340.json`, `selfHealth.severity=ok`, `Wooo-Agent99-Control-Loop=ok`. + - Latest performance evidence: `C:\Wooo\Agent99\evidence\agent99-Perf-20260709-223419.json`, `perfCritical=0`, `perfWarning=0`. + +## 2026-07-09 M8 Operator Instruction Intake Receipt + +Agent99 now accepts operator text instructions and turns them into allowlisted queue commands. + +- New submit tool: + - `C:\Wooo\Agent99\bin\agent99-submit-request.ps1`. + - Interactive launcher: `C:\Wooo\Agent99\agent99-submit-request.cmd`. + - Desktop shortcut: `Agent99 Submit Request.lnk`. +- Request flow: + - Operator enters one text instruction. + - Agent99 writes the original request to `C:\Wooo\Agent99\requests`. + - Agent99 resolves the request into an allowlisted mode. + - Agent99 writes a queue command to `C:\Wooo\Agent99\queue`. + - `ControlTick` processes the command, writes evidence, and sends Telegram `operator_command_result`. +- Current allowlisted instruction targets: + - website / route / 502 checks -> `PublicSmoke`. + - reboot / restore / repair -> `Recover`. + - VM / VMware start -> `StartVMs`. + - CPU / load / memory / disk / performance -> `Perf`. + - backup / snapshot / cron -> `BackupCheck`. + - self / agent / health -> `SelfCheck`. + - Harbor / registry -> `HarborRepair`. + - AWOOOI / K3s / pod -> `AwoooRepair`. +- Verification: + - Test instruction: `check all websites are normal` equivalent in Traditional Chinese. + - Request ID: `request-20260709-224133-0dc045fe`. + - Resolved mode: `PublicSmoke`. + - Processed result: `ok=true`, `exitCode=0`, `durationSeconds=1.9`. + - Evidence: `C:\Wooo\Agent99\evidence\agent99-PublicSmoke-20260709-224134.json`. + - Public smoke: AWOOOI, StockPlatform, FIFA, Gitea, and Harbor all HTTP 200 / non-502. + - Telegram result: `operator_command_result`, `severity=info`, `sent=true`, `relayOk=true`. diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md new file mode 100644 index 000000000..87abaf693 --- /dev/null +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -0,0 +1,589 @@ +# Host Reboot AI Automation SOP + +Last updated: 2026-07-09 + +## P0 Goal + +After host 99 or any VM host is rebooted, the automation must detect the reboot, restore VM reachability, restore critical services, and prove public sites are no longer returning 502. + +Target recovery bar: + +- Detect 99 / 110 / 112 / 120 / 121 / 188 reboot or down state. +- Ensure VMware Workstation on 99 is running and required VMs are powered on. +- Restore Harbor / registry on 110 before forcing K3s workloads to repull images. +- Restore AWOOOI production pods on 120. +- Verify public HTTP readback for AWOOOI, StockPlatform, FIFA, Gitea, and Harbor. +- Emit operator-visible evidence and Telegram alerts for down, recovering, recovered, and degraded states. + +## 2026-07-09 Incident Findings To Automate + +The reboot incident was not a generic 502 problem. The confirmed chain was: + +1. 110 rebooted and Docker/Harbor did not fully recover. +2. `harbor-core` entered a failed/restarting state because Harbor Redis / Docker DNS dependency was not clean. +3. K3s on 120 could not pull AWOOOI images from `192.168.0.110:5000`. +4. AWOOOI prod pods stayed in `ImagePullBackOff`. +5. Public nginx returned 502 because upstream pods were unavailable. +6. Restarting / rebooting 110 cleared the Docker / systemd / Harbor dependency state. +7. Once `harbor-core` became healthy, 120 pods repulled images and AWOOOI recovered. + +This failure mode must be treated as a first-class automated recovery path. + +## Recovery Priority Order + +P0. Control-plane and VM reachability + +- 99 Windows host reachable. +- VMware Workstation process running on 99. +- VMs 110, 112, 120, 121, and 188 powered on. +- Hosts 110, 112, 120, 121, and 188 respond to ICMP or SSH. + +P1. 110 Harbor and registry dependency chain + +- Docker engine active. +- Harbor Redis running. +- `harbor-core` running and healthy. +- Harbor registry / registryctl running and healthy. +- `harbor.wooo.work` returns HTTP 200 after redirect. +- Internal registry pull path from K3s nodes is reachable. + +P2. 120 K3s production workloads + +- `awoooi-prod` deployments ready: + - `awoooi-api` 1/1 + - `awoooi-web` 2/2 + - `awoooi-worker` 1/1 + - `awoooi-auto-repair-canary` 1/1 +- No critical pods stuck in `ImagePullBackOff`, `ErrImagePull`, `CrashLoopBackOff`, or `Pending`. +- If image pull failures remain after Harbor is healthy, delete only the failed pods and let controllers recreate them. + +P3. Public user-visible surfaces + +- `awoooi.wooo.work` returns HTTP 200 for web. +- `awoooi.wooo.work/api/v1/health` returns JSON and not 502. +- `stock.wooo.work` returns HTTP 200. +- `2026fifa.wooo.work` returns HTTP 200. +- `gitea.wooo.work` returns HTTP 200. +- `harbor.wooo.work` returns HTTP 200. + +P4. Degraded provider follow-up + +- AWOOOI health can be `degraded` even when 502 is fixed. +- Treat AI provider health separately from user-visible 502 restoration. +- Follow-up checks include `ollama_gcp_a`, `ollama_gcp_b`, and `ollama_local`. +- Do not block public-site 502 recovery on non-primary AI provider cooldown unless the app is user-impacting. + +## Automated Decision Tree + +1. Detect reboot / outage. + - Ping 110, 112, 120, 121, 188. + - SSH to 110 and 120 if keys are available. + - Query VMware running VM list on 99 if `vmrun.exe` is available. + +2. If a VM is not reachable. + - Check VMware VM state. + - Start the VM from 99. + - Wait for guest ping / SSH. + - Alert Telegram: `host_recovering`. + +3. If AWOOOI public route is 502. + - Check 120 `awoooi-prod` deployments. + - If pods are `ImagePullBackOff`, check 110 Harbor first. + - Do not repeatedly delete pods before Harbor is healthy. + +4. If 110 Harbor is unhealthy. + - Check Docker active state. + - Check `redis`, `harbor-core`, `registry`, and `registryctl`. + - Restart Harbor Redis and `harbor-core` first. + - If Docker/systemd/cgroup operations hang, use controlled 110 guest reboot through VMware. + - After 110 returns, wait for Harbor core healthy before touching K3s pods. + +5. If Harbor is healthy and pods are still in image pull failure. + - Delete only failed AWOOOI pods in `awoooi-prod`. + - Wait for deployments to become ready. + - Read back public HTTP. + +6. If public sites are HTTP 200 but AWOOOI health is degraded. + - Record degraded providers. + - Alert `service_degraded`. + - Continue provider-specific remediation separately. + +## 99 Windows AI Agent Control Node + +99 must become the automation entrypoint for reboot recovery. + +Required capabilities: + +- Run a local AI Agent service on Windows 99 with least-privilege operator permissions. +- Execute the reboot scorecard on a schedule and on boot. +- Control VMware Workstation VM startup through `vmrun.exe`. +- Use SSH keys to 110 and 120 for service-level checks and controlled repair. +- Send Telegram alerts for down, recovering, recovered, degraded, failed, and controlled-remediation-result states. +- Do not stop at alerting. Every warning/critical alert must either trigger an allowlisted remediation, record that no remediation matched, or record that the action was skipped by cooldown/min-severity. +- Do not send start/complete/info noise to Telegram by default. Human-facing alerts must be deduped and action-oriented. +- Persist scorecard evidence to a local non-secret log directory. +- Never store raw passwords, tokens, cookies, sessions, or `.env` in the repo. +- Accept operator commands through the 99 desktop submitter, Telegram `/agent99 ...`, and local queue files without depending on Codex quota. +- Accept SRE war-room alert JSON at `C:\Wooo\Agent99\alerts\incoming`, resolve each alert into an allowlisted mode, and either run controlled remediation or record why no remediation matched. + +Candidate implementation layers: + +- P0: Windows Task Scheduler runs `host-reboot-scorecard.ps1` at startup and every minute for the first 15 minutes after 99 boot. +- P1: Install a local Windows service wrapper for the scorecard loop. +- P2: Add an AI Agent runtime that consumes scorecard JSON and chooses approved recovery actions. +- P3: Add a UI/console dashboard on 99 for operator evidence and manual override. + +Current command and alert loops: + +- `Wooo-Agent99-Control-Loop`: processes `C:\Wooo\Agent99\queue`, updates the dashboard, and writes execution evidence. +- `Wooo-Agent99-Telegram-Inbox`: reads Telegram group commands with Agent99 prefixes and submits allowlisted queue work. +- `Wooo-Agent99-SRE-Alert-Inbox`: reads `C:\Wooo\Agent99\alerts\incoming`, converts actionable alerts into queue work, and triggers the control loop. +- `Wooo-Agent99-SRE-Alert-Relay`: runs the 99-side HTTP relay for sanitized AWOOI Alertmanager JSON and writes accepted alerts into `C:\Wooo\Agent99\alerts\incoming`. +- `Wooo-Agent99-Performance-Guard`: runs every minute and performs allowlisted load shedding before escalating unresolved performance alerts. +- `Wooo-Agent99-Self-Health`: verifies the scheduled tasks and evidence freshness for the control plane itself. + +AwoooI SRE Telegram command loop: + +- Operators and project AI Agents can issue commands in the SRE Telegram group with `/agent99 ...`. +- Agent99 records these as `source=telegram-sre-war-room`. +- Completed automated repairs send `operator_command_result` back to Telegram. +- Results include source, instruction, mode, exit code, evidence path, problem count, and KM path. +- The command path must not depend on Codex quota. +- The command path must not depend only on the default `TELEGRAM_CHAT_ID`, because that can point to a private chat. Agent99 also accepts command updates from Telegram chats whose title matches `AwoooI SRE|AWOOOI SRE|SRE`. +- When the command comes from a matched Telegram group, the completion report is sent back to that same Telegram origin; stored evidence redacts the raw chat id as `telegram-origin`. +- Non-command monitoring alerts in the SRE Telegram group are auto-ingested when they look like Alertmanager/FIRING/critical/warning/502/CPU/disk/Harbor/K3s/backup/host-down/provider-freshness alerts. +- Auto-ingested monitoring alerts use `source=telegram-sre-war-room-monitoring` and enter the same SRE alert inbox / queue / recovery / Telegram completion / KM-RAG path. + +Alert-to-action routing: + +- Public 502 / route down / website down -> `Recover`. +- Host down / reboot detected / VM not running -> `Recover`. +- CPU, memory, disk, or load pressure -> `Perf` with controlled apply. +- Harbor or registry unhealthy -> `HarborRepair`. +- AWOOOI K3s, pod, deployment, or image pull failure -> `AwoooRepair`. +- Backup freshness, snapshot, restore drill, or cron issue -> `BackupCheck`. +- Provider freshness, last-seen, ingestion, raw signal, or no-false-green risk -> `ProviderFreshness`. +- Unknown actionable alert -> `Status` evidence first. + +Producer-side Agent99 bridge: + +- AWOOI API Alertmanager webhook schedules an Agent99 bridge after normalization and fingerprinting. +- Preferred transport: when `AGENT99_SRE_ALERT_RELAY_URL` is configured, the bridge POSTs sanitized alert JSON to the 99 relay. Production URL is `http://192.168.0.99:8787/agent99/sre-alert/`. +- Fallback transport: when `AGENT99_SRE_ALERT_INBOX_PATH` is configured and relay URL is empty, the bridge writes sanitized alert JSON directly to the Agent99 SRE inbox contract. +- If relay/path is empty or unavailable, the bridge fails open and logs; it must not cause Alertmanager retries. +- Relay safety: only LAN/local source prefixes are accepted by default; optional `AGENT99_SRE_RELAY_TOKEN` / `AGENT99_SRE_ALERT_RELAY_TOKEN` values stay in env/Secret and are never stored in repo or evidence. +- Provider freshness bridge output must require provider window, last seen, direct reference, and verifier readback before dashboard green is trusted. + +Approved controlled actions for the 99 agent: + +- Start a missing VM. +- Ping and SSH readback. +- Restart Harbor Redis / Harbor core after evidence of Harbor token failure. +- Delete K3s pods stuck in image pull failure only after Harbor is healthy. +- Run public route readback. +- Send Telegram status. + +Actions still requiring explicit break-glass: + +- Reading or exfiltrating secrets. +- Destructive database operations. +- Firewall cutover. +- Remote delete / prune / force push. +- Production provider route cutover with cost or billing impact. + +## Evidence Required Before Calling Recovery Complete + +Recovery is complete only when all of the following are captured: + +- Five VM hosts have reachability evidence. +- 110 Harbor core is healthy. +- 120 AWOOOI deployments are ready. +- Public routes return non-502, preferably HTTP 200. +- Any degraded health components are listed separately with owner / next action. + +## Current 2026-07-09 Receipts + +- 110 `harbor-core`: recovered to `Up ... (healthy)`. +- 120 `awoooi-prod`: + - `awoooi-api`: 1/1 + - `awoooi-web`: 2/2 + - `awoooi-worker`: 1/1 + - `awoooi-auto-repair-canary`: 1/1 +- Public readback: + - `awoooi.wooo.work`: HTTP 200 after redirect. + - `stock.wooo.work`: HTTP 200 after redirect. + - `2026fifa.wooo.work`: HTTP 200 after redirect. + - `gitea.wooo.work`: HTTP 200 after redirect. + - `harbor.wooo.work`: HTTP 200 after redirect. +- Host reachability: + - 110, 112, 120, 121, 188 responded to ping or SSH. +- Remaining degraded signal: + - AWOOOI health reported degraded AI provider signals including `ollama_gcp_b` timeout and `ollama_local` connection refused / unavailable. + +## Agent99 Installation Receipt + +2026-07-09 16:49 Asia/Taipei: + +- 99 host: `WOOO-SUPER`. +- Agent root: `C:\Wooo\Agent99`. +- VMware CLI: `C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe`. +- Live config: `C:\Wooo\Agent99\config\agent99.config.json`. +- Evidence directory: `C:\Wooo\Agent99\evidence`. +- Scheduled tasks: + - `Wooo-Agent99-Heartbeat`: `LastTaskResult=0`, latest status evidence `agent99-Status-20260709-164845.json`. + - `Wooo-Agent99-Startup-Recovery`: `LastTaskResult=0`, latest recovery evidence `agent99-Recover-20260709-164937.json`. +- Startup recovery readback: + - VM selector: 110, 188, 112, 120, 121 all `host_already_reachable`. + - Harbor: core, Redis, and registry all healthy. + - AWOOOI: deploy ready, no image pull failure, no crash failure. + - Public routes: `awoooi.wooo.work`, `stock.wooo.work`, `2026fifa.wooo.work`, `gitea.wooo.work`, and `harbor.wooo.work` all HTTP 200 / non-502. + +## Agent99 Performance Guard Receipt + +2026-07-09 17:15 Asia/Taipei: + +- Scheduled task: `Wooo-Agent99-Performance-Guard`. +- Trigger interval: every 1 minute. +- Last task result: `0`. +- Latest evidence: `C:\Wooo\Agent99\evidence\agent99-Perf-20260709-171502.json`. +- Host performance readback: + - 110: OK. + - 112: OK. + - 120: OK. + - 121: OK. + - 188: OK. +- Controlled load shedding: + - Only allowlisted actions can run. + - Current 110 runner action requires `load_per_core_critical`. + - Memory/disk pressure alerts do not execute the runner pause action. +- Telegram audit: + - Agent99 emits Telegram attempts for all operation start/complete events, warnings, critical events, and load-shed actions. + - Agent99 supports local env aliases and an existing-token relay. + - Current relay: 110 `stockplatform-v2-api-1`, using that container's existing `TELEGRAM_BOT_TOKEN` / `TELEGRAM_CHAT_ID`. + - 2026-07-09 17:27 verification: `Wooo-Agent99-Performance-Guard` `LastTaskResult=0`; latest evidence `agent99-Perf-20260709-172712.json`; `tgSent=2`; `relayOk=True`. + - Token and chat id values were not printed, copied into repo files, or written to evidence. + +## Agent99 Command And SRE Alert Integration Receipt + +2026-07-09 23:03 Asia/Taipei: + +- Telegram command inbox is deployed and scheduled: + - Script: `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1` + - Task: `Wooo-Agent99-Telegram-Inbox` + - Latest evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260709-230031.json` + - Result: `ok=True`, `updatesSeen=0`, `processedCount=0` + - Evidence check: no Python code, no token text. +- SRE war-room alert inbox is deployed and scheduled: + - Script: `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1` + - Task: `Wooo-Agent99-SRE-Alert-Inbox` + - Incoming path: `C:\Wooo\Agent99\alerts\incoming` + - Latest evidence: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260709-230226.json` +- Synthetic 502 alert test: + - Alert id: `synthetic-public-502-20260709-230107` + - Alert source: `awoooi-sre-war-room` + - Alert kind: `public_route_502` + - Resolved mode: `Recover` + - Queue processor evidence: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260709-230108.json` + - Processed count: `1` + - Mode: `Recover` + - Exit code: `0` + - Telegram relay: `sent=True`, `relayOk=True` +- Recovery verifier from the synthetic alert: + - Evidence: `C:\Wooo\Agent99\evidence\agent99-Recover-20260709-230108.json` + - Public route failures: `0` + - Host reachability failures: `0` + - Harbor core healthy: `True` + - AWOOOI deploy ready: `True` +- Agent99 self-health after integration: + - Evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-230223.json` + - Severity: `ok` + - Task failures: `0` + - Evidence critical: `0` + - Evidence warning: `0` + - Telegram inbox task: OK + - SRE alert inbox task: OK + - Dashboard queue pending: `0` + - Dashboard alert pending: `0` + +## Agent99 Problem Count / KM / RAG Receipt + +2026-07-09 23:12 Asia/Taipei: + +- Synthetic AwoooI SRE Telegram command: + - instruction: `檢查所有網站是否恢復正常，若502就自動修復` + - source: `telegram-sre-war-room` + - request id: `request-20260709-231157-d53f53d8` + - resolved mode: `Recover` + - control exit code: `0` +- Execution evidence: + - `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260709-231158.json` + - processed source: `telegram-sre-war-room` + - processed mode: `Recover` + - processed ok: `True` + - processed exit code: `0` + - Telegram relay sent: `1` + - Telegram evidence contains no Python code. +- Problem counters: + - path: `C:\Wooo\Agent99\state\problem-counts.json` + - total events: `1` + - total resolved: `1` + - total failed: `0` + - problem key: `telegram-sre-war-room_agent99_operator_instruction` + - occurrences: `1` + - resolved: `1` + - failed: `0` +- Knowledge outputs: + - full event ledger: `C:\Wooo\Agent99\playbooks\agent99-incident-records.jsonl` + - KM summary: `C:\Wooo\Agent99\playbooks\agent99-incident-km.md` + - RAG import feed: `C:\Wooo\Agent99\playbooks\rag-import\agent99-incidents.jsonl` +- Dashboard: + - problem events: `1` + - resolved: `1` + - failed: `0` + - queue pending: `0` + - alert pending: `0` +- SRE Telegram group routing: + - Default relay chat readback was private chat `OoO`, not the SRE group. + - Live Agent99 config now includes `allowedCommandChatTitlePattern=AwoooI SRE|AWOOOI SRE|SRE`. + - Telegram inbox readback after the change: `ok=True`, `updatesSeen=0`, `processedCount=0`. + - Next real `/agent99 ...` message from the SRE group will be accepted by title match and will reply back to the same Telegram origin. + +## Agent99 SRE Telegram Monitoring Alert Auto-Ingest Receipt + +2026-07-09 23:32 Asia/Taipei: + +- Runtime behavior: + - SRE Telegram group monitoring alerts are now consumed automatically. + - Prefix `/agent99` is no longer required for monitoring alerts. + - Auto-ingest is controlled by `autoIngestMonitoringAlerts=True`. + - Group matching uses `allowedCommandChatTitlePattern=AwoooI SRE|AWOOOI SRE|SRE`. +- Loop prevention: + - Agent99's own `Agent99 Alert` / `operator_command_result` messages are ignored by monitoring auto-ingest. + - Operator command result Telegram reports bypass dedupe so every completed repair is reported. + - Provider freshness signals are not passive dashboard health; they create a read-only freshness candidate and require provider window, last seen, direct reference, and verifier readback. + - Provider freshness guardrail: do not switch provider, call paid model, edit env, or reload. +- Synthetic monitoring alert: + - `Alertmanager FIRING critical public_route_502 service=awoooi host=192.168.0.120 HTTP 502` + - no `/agent99` prefix. +- Evidence chain: + - Telegram inbox: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260709-233230.json` + - `inboxOk=True` + - `alertedCount=1` + - SRE alert inbox: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260709-233231.json` + - queued count: `1` + - failed count: `0` + - Control loop: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260709-233232.json` + - alert source: `telegram-sre-war-room-monitoring` + - alert kind: `public_route_502` + - mode: `Recover` + - processed ok: `True` + - exit code: `0` + - Telegram attempts: `1` + - Telegram sent: `1` + - Recovery: `C:\Wooo\Agent99\evidence\agent99-Recover-20260709-233055.json` +- Problem counters: + - total events before: `2` + - total events after: `3` + - total resolved after: `3` + - total failed after: `0` + - problem key: `agent99-sre-alert-inbox_awoooi_192.168.0.120_public_route_502` + - problem occurrences: `2` + +## OpenClaw Cold-Start Readback + +2026-07-09 17:38 Asia/Taipei: + +- Production OpenClaw URL: `http://192.168.0.188:8088`. +- 188 service: `clawbot.service` is `active`. +- Health endpoint: `http://192.168.0.188:8088/health`. +- Health readback: `status=healthy`, `version=6.0`, `service=ClawBot`, `environment=production`, `telegram_bot=connected`. +- Agent99 latest status evidence: `C:\Wooo\Agent99\evidence\agent99-Status-20260709-173813.json`. +- Agent99 `aiServices.openclaw`: `ok=True`, `status=healthy`. +- Telegram relay from the Agent99 status run: `tgSent=2`, `relayOk=2`. +- Note: `192.168.0.188:8089` is not the current production OpenClaw endpoint. + +## Windows Update No-Surprise-Reboot Policy + +2026-07-09 17:39 Asia/Taipei: + +- Evidence: `C:\Wooo\Agent99\evidence\agent99-windows-update-policy-20260709-173933.json`. +- `NoAutoRebootWithLoggedOnUsers=1`. +- `AlwaysAutoRebootAtScheduledTime=0`. +- `AUOptions=3`. +- `ActiveHoursStart=0`, `ActiveHoursEnd=23`. +- `gpupdateOk=True`. + +## Agent99 Self-Health Receipt + +2026-07-09 17:52 Asia/Taipei: + +- Scheduled task: `Wooo-Agent99-Self-Health`. +- Trigger interval: every 5 minutes. +- Task Scheduler trigger verification: + - `LastTaskResult=0`. + - Evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-175218.json`. + - `evidenceAfterTrigger=True`. +- Self-health checks: + - `Wooo-Agent99-Startup-Recovery`: OK. + - `Wooo-Agent99-Heartbeat`: OK. + - `Wooo-Agent99-Performance-Guard`: OK. + - `Wooo-Agent99-Self-Health`: OK. + - Evidence freshness: performance, status, self-health, and startup recovery OK. + - Latest performance evidence: all five hosts OK. + - Telegram delivery: `tgSent=3`, `relayOk=3`. + - Updated rule: a quiet interval with no alert attempts is OK; only failed alert attempts are Telegram critical. + +## 110 Load Pressure Incident Readback + +2026-07-09 18:07 Asia/Taipei: + +- Symptom: 110 host load stayed above warning after all hosts rebooted. +- Initial findings: + - `gitea` and `stockplatform-v2-postgres-1` were the highest regular service containers. + - `/backup/scripts/backup-awoooi.sh` was running a remote `pg_dump` from 188 and local gzip compression. + - A random container `mystifying_napier` was running `alpine "tar czf - /data"` with no restart policy and no labels, consuming CPU and writing compressed binary output into Docker logs. +- Controlled actions taken: + - Backup pipeline PIDs were reduced to nice 15 / ionice best-effort 7. + - `mystifying_napier` was stopped and left as `Dead`; it was not deleted, so evidence remains available. +- Agent99 automation added: + - `stop-110-runaway-alpine-tar-data`: stops allowlisted `alpine` containers running `tar czf - /data` when 110 is at warning or critical load. + - `renice-110-awoooi-backup-pressure`: lowers priority for `backup-awoooi.sh`, `gzip`, and remote `pg_dump` pipeline PIDs when 110 is at warning or critical load. + - `pause-110-ci-runner-pressure`: still only runs at critical load with `load_per_core_critical`; runner is not touched for backup, disk, memory, or warning-only pressure. + - Warning-level top CPU capture is enabled with a 10-second timeout to avoid diagnostics becoming a new load source. +- Verification: + - Evidence: `C:\Wooo\Agent99\evidence\agent99-Perf-20260709-180704.json`. + - 110 load-per-core: `0.67`, severity `ok`. + - 112, 120, 121, and 188: severity `ok`. + - `perfCritical=0`, `perfWarning=0`. + - Load-shedding actions skipped because host severity was below threshold. + - Telegram relay: `tgSent=2`, `relayOk=2`. + +## Agent99 Backup Health Receipt + +2026-07-09 19:07 Asia/Taipei: + +- Scheduled task: `Wooo-Agent99-Backup-Health`. +- Trigger interval: every 15 minutes. +- Task Scheduler trigger verification: + - `LastTaskResult=0`. + - Evidence: `C:\Wooo\Agent99\evidence\agent99-BackupCheck-20260709-190617.json`. + - `evidenceAfterTrigger=True`. +- BackupCheck coverage: + - Snapshot freshness targets: `awoooi`, `gitea`, `harbor`, `momo`, `langfuse`, `monitoring`, `signoz`, `clawbot`, `open-webui`, `sentry`, `ai-artifacts`, `configs`, and `public-routes`. + - File/status checks: `backup.log`, `backup-status.log`, legacy `status.log`, legacy `cron.log`, integrity check status, restore drill status, and configs last-status JSON. + - Cron contract patterns: `backup-all.sh`, `backup-awoooi-frequent.sh`, `backup-health-textfile-exporter.py`, `check-backup-integrity.sh`, `sync-offsite-backups.sh --mode status`, and `backup-status.sh`. +- Latest result: + - `backupHealth.severity=ok`. + - `targetCritical=0`, `targetWarning=0`. + - `fileCritical=0`, `fileWarning=0`. + - `cronCritical=0`, `cronWarning=0`. + - `targetCount=13`, `fileCheckCount=7`, `cronPatternCount=6`. + - Telegram relay: `tgSent=2`, `relayOk=2`. +- Self-health cross-check: + - Evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-190714.json`. + - `Wooo-Agent99-Backup-Health`: OK, `LastTaskResult=0`. + - Backup evidence freshness: OK, latest file `agent99-BackupCheck-20260709-190617.json`, age `0.5` minutes, max age `30` minutes. + +## Alert-to-Remediation / Evidence Sanitization Readback + +2026-07-09 22:11 Asia/Taipei: + +- Root cause fixed for unreadable Telegram relay messages: + - Relay sender argument index corrected from `sys.argv[1]` to `sys.argv[2]`. + - Telegram now receives formatted Agent99 alert text, not embedded Python relay code. +- Noise control: + - `alertAllOperations=false`. + - `alertInfoEvents=false`. + - `dedupeMinutes=30`. + - `agent_complete` remains evidence-only and does not alert by default. +- Alert-to-action rule: + - `Perf -ControlledApply` suppresses immediate performance spam, performs allowlisted remediation first, then alerts `load_shed_result`. + - If no remediation applies, Agent99 alerts the unresolved performance issue. + - 188 disk pressure is now allowlisted through `prune-188-docker-build-cache-disk-pressure`. + - The 188 action uses `timeoutSeconds=240` plus a post-apply verifier; `postVerifyMaxDiskUsedPercent=89`. + - Timeout alone is not final truth; post-verifier success can mark remediation effective. +- Self-health false critical fixed: + - A quiet interval with no recent alerts is OK. + - Telegram health is critical only when recent alert attempts exist and all delivery attempts fail. + - Telegram delivery health now uses a 60-minute lookback across recent evidence files. + - Transient performance warnings are no longer promoted to self-health critical; only missing or critical host performance evidence is critical. +- Backup monitor false critical fixed: + - Backup snapshot readback now supports `snapshotReadTimeoutSeconds`. + - 99 live config sets `snapshotReadTimeoutSeconds=90`. + - Harbor backup was verified as present and fresh after a timeout false critical; latest metadata was from 2026-07-09 17:48 Asia/Taipei and within the 1800-minute SLA. +- Evidence security fixed: + - Harbor readback now stores selected health fields only: name, running state, health state. + - Raw `docker inspect` output is not stored. + - Existing Status evidence on 99 was sanitized: 77 files checked; `inspectOutputNotRedactedCount=0`; `unsafeHarborFieldCount=0`. +- Latest recovery proof: + - Status evidence: `C:\Wooo\Agent99\evidence\agent99-Status-20260709-221147.json`. + - Host reachability: 110, 112, 120, 121, and 188 all OK. + - Public routes: `awoooi.wooo.work`, `stock.wooo.work`, `2026fifa.wooo.work`, `gitea.wooo.work`, and `harbor.wooo.work` all HTTP 200 / non-502. + - Performance: `perfCritical=0`, `perfWarning=0`; 188 disk used 87%; 110 load-per-core 0.85. + - Harbor: core, Redis, and registry healthy with redacted evidence. + - AWOOOI: `deployReady=true`, `imagePullFailure=false`, `crashFailure=false`. + - Backup evidence after timeout fix: `C:\Wooo\Agent99\evidence\agent99-BackupCheck-20260709-221759.json`, `backupHealth.severity=ok`, all target/file/cron critical and warning counts are 0. + - Final self-health evidence: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-222236.json`, `selfHealth.severity=ok`, `taskFailures=0`, `performanceSeverity=ok`, `telegramSeverity=ok`. + +## 99 Visible Agent Control Center + +2026-07-09 22:33 Asia/Taipei: + +- Agent99 is visible and controllable from the 99 Windows desktop. +- New scheduled task: + - `Wooo-Agent99-Control-Loop`. + - Runs every 1 minute. + - Updates dashboard and processes queue commands. +- Desktop shortcuts: + - `C:\Users\Public\Desktop\Agent99 Control Center.lnk`. + - `C:\Users\Public\Desktop\Agent99 Queue.lnk`. + - `C:\Users\Public\Desktop\Agent99 Evidence.lnk`. + - `D:\Desktop\Agent99 Control Center.lnk`. + - `D:\Desktop\Agent99 Queue.lnk`. + - `D:\Desktop\Agent99 Evidence.lnk`. +- Dashboard: + - HTML: `C:\Wooo\Agent99\dashboard\index.html`. + - JSON: `C:\Wooo\Agent99\dashboard\agent99-control-center.latest.json`. + - Latest dashboard timestamp readback: `2026-07-09T22:33:26+08:00`. + - Queue readback: `pending=0`, `processed=2`, `failed=0`. +- Queue command contract: + - Add a JSON command to `C:\Wooo\Agent99\queue`. + - `ControlTick` processes only allowlisted modes. + - Results are written to `C:\Wooo\Agent99\queue\processed` or `C:\Wooo\Agent99\queue\failed`. + - Operator command results are sent to Telegram. +- Verification: + - Command ID: `codex-visible-publicsmoke-20260709-223308`. + - Mode: `PublicSmoke`. + - Result: `ok=true`, `exitCode=0`. + - Evidence: `C:\Wooo\Agent99\evidence\agent99-PublicSmoke-20260709-223309.json`. + - Telegram: `operator_command_result`, `severity=info`, `sent=true`, `relayOk=true`. + - Self-health: `C:\Wooo\Agent99\evidence\agent99-SelfCheck-20260709-223340.json`, `selfHealth.severity=ok`, `Wooo-Agent99-Control-Loop=ok`. + - Latest performance: `C:\Wooo\Agent99\evidence\agent99-Perf-20260709-223419.json`, `perfCritical=0`, `perfWarning=0`. + +## Operator Instruction Intake + +2026-07-09 22:41 Asia/Taipei: + +- Agent99 accepts text instructions on 99 through: + - `C:\Wooo\Agent99\bin\agent99-submit-request.ps1`. + - `C:\Wooo\Agent99\agent99-submit-request.cmd`. + - `Agent99 Submit Request.lnk` on the desktop. +- Flow: + - instruction -> request record -> queue command -> `ControlTick` -> evidence -> Telegram result. +- Current allowlisted request router: + - websites / routes / 502 -> `PublicSmoke`. + - restore / reboot / repair -> `Recover`. + - VM / VMware -> `StartVMs`. + - CPU / load / memory / disk / performance -> `Perf`. + - backup / snapshot / cron -> `BackupCheck`. + - self health / agent health -> `SelfCheck`. + - Harbor / registry -> `HarborRepair`. + - AWOOOI / K3s / pods -> `AwoooRepair`. +- Verification: + - Chinese instruction submitted from Codex to 99: check whether all websites are normal. + - Request ID: `request-20260709-224133-0dc045fe`. + - Resolved mode: `PublicSmoke`. + - Processed result: `ok=true`, `exitCode=0`. + - Evidence: `C:\Wooo\Agent99\evidence\agent99-PublicSmoke-20260709-224134.json`. + - Public routes: AWOOOI, StockPlatform, FIFA, Gitea, and Harbor all HTTP 200 / non-502. + - Telegram: `operator_command_result`, `severity=info`, `sent=true`, `relayOk=true`. diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 new file mode 100644 index 000000000..801ea1f57 --- /dev/null +++ b/agent99-bootstrap.ps1 @@ -0,0 +1,162 @@ +param( + [string]$AgentRoot = "C:\Wooo\Agent99", + [switch]$InstallScheduledTasks +) + +$ErrorActionPreference = "Stop" + +$SourceRoot = Split-Path -Parent $MyInvocation.MyCommand.Path +$BinDir = Join-Path $AgentRoot "bin" +$ConfigDir = Join-Path $AgentRoot "config" +$EvidenceDir = Join-Path $AgentRoot "evidence" +$LogDir = Join-Path $AgentRoot "logs" +$QueueDir = Join-Path $AgentRoot "queue" +$RequestDir = Join-Path $AgentRoot "requests" +$AlertDir = Join-Path $AgentRoot "alerts" +$PlaybookDir = Join-Path $AgentRoot "playbooks" +$DashboardDir = Join-Path $AgentRoot "dashboard" + +foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir, $EvidenceDir, $LogDir, $QueueDir, $RequestDir, $AlertDir, $PlaybookDir, $DashboardDir)) { + New-Item -ItemType Directory -Force -Path $dir | Out-Null +} + +function Copy-AgentFile { + param([string]$Name) + $src = Join-Path $SourceRoot $Name + $dst = Join-Path $BinDir $Name + if (Test-Path $src) { + $srcFull = [System.IO.Path]::GetFullPath($src) + $dstFull = [System.IO.Path]::GetFullPath($dst) + if ($srcFull -ne $dstFull) { + Copy-Item -Force $src $dst + } + } +} + +Copy-AgentFile "agent99-control-plane.ps1" +Copy-AgentFile "agent99-register-tasks.ps1" +Copy-AgentFile "agent99-submit-request.ps1" +Copy-AgentFile "agent99-telegram-inbox.ps1" +Copy-AgentFile "agent99-sre-alert-inbox.ps1" +Copy-AgentFile "agent99-sre-alert-relay.ps1" +Copy-AgentFile "host-reboot-scorecard.ps1" + +$exampleConfig = Join-Path $SourceRoot "agent99.config.example.json" +$configPath = Join-Path $ConfigDir "agent99.config.json" +if (-not (Test-Path $configPath)) { + if (Test-Path $exampleConfig) { + Copy-Item $exampleConfig $configPath + } else { + @" +{ + "agentName": "Agent99", + "agentRoot": "$($AgentRoot.Replace('\', '\\'))", + "sshUser": "wooo", + "sshUsers": {}, + "vmrunPath": "", + "hosts": ["192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"], + "vms": [], + "publicUrls": ["awoooi.wooo.work", "stock.wooo.work", "2026fifa.wooo.work", "gitea.wooo.work", "harbor.wooo.work"], + "sreAlertRelay": { + "enabled": true, + "prefix": "http://+:8787/agent99/sre-alert/", + "tokenEnv": "AGENT99_SRE_RELAY_TOKEN", + "allowedRemotePrefixes": ["127.", "::1", "192.168.0."] + }, + "telegram": { + "enabled": true, + "botTokenEnv": "AGENT99_TELEGRAM_BOT_TOKEN", + "chatIdEnv": "AGENT99_TELEGRAM_CHAT_ID", + "botTokenEnvAliases": ["AWOOOI_TG_BOT_TOKEN", "OPENCLAW_TG_BOT_TOKEN", "TELEGRAM_BOT_TOKEN", "TELEGRAM_TOKEN", "TG_BOT_TOKEN", "AIDER_WATCH_TELEGRAM_TOKEN"], + "chatIdEnvAliases": ["TELEGRAM_CHAT_ID", "TELEGRAM_ALERT_CHAT_ID", "TELEGRAM_P0_CHAT_ID", "TELEGRAM_SECURITY_CHAT_ID", "OPENCLAW_TG_CHAT_ID", "TG_CHAT_ID", "AIDER_WATCH_TELEGRAM_CHAT_ID"], + "relay": { "enabled": true, "host": "192.168.0.110", "container": "stockplatform-v2-api-1" }, + "alertAllOperations": false, + "alertInfoEvents": false, + "alertOperatorCommands": true, + "dedupeMinutes": 30 + }, + "selfHealth": { + "scheduledTasks": [ + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-SRE-Alert-Relay", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" + ] + }, + "performance": { + "enabled": true, + "intervalMinutes": 1, + "loadPerCoreWarning": 0.9, + "loadPerCoreCritical": 1.5, + "memoryAvailablePercentCritical": 10, + "diskUsedPercentWarning": 90, + "diskUsedPercentCritical": 95, + "captureTopCpuOnWarning": true, + "topCpuTimeoutSeconds": 10, + "loadShedding": { "enabled": true, "actions": [] } + }, + "backupHealth": { + "host": "192.168.0.110", + "targets": [], + "fileChecks": [], + "cronPatterns": [] + } +} +"@ | Set-Content -Encoding UTF8 $configPath + } +} + +$launcher = Join-Path $AgentRoot "agent99-run.ps1" +@" +param( + [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "ControlTick")] + [string]`$Mode = "Status", + [switch]`$ControlledApply +) + +& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" +"@ | Set-Content -Encoding UTF8 $launcher + +$submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd" +@" +@echo off +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "$BinDir\agent99-submit-request.ps1" -RunNow +pause +"@ | Set-Content -Encoding ASCII $submitCmd + +function Find-Vmrun { + $candidates = @( + "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe", + "C:\Program Files\VMware\VMware Workstation\vmrun.exe" + ) + foreach ($candidate in $candidates) { + if (Test-Path $candidate) { return $candidate } + } + return "" +} + +$vmrun = Find-Vmrun +if ($vmrun) { + $config = Get-Content $configPath -Raw | ConvertFrom-Json + $config.vmrunPath = $vmrun + $config | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $configPath +} + +if ($InstallScheduledTasks) { + & (Join-Path $BinDir "agent99-register-tasks.ps1") -AgentRoot $AgentRoot +} + +Write-Host "Agent99 bootstrap complete" +Write-Host "AgentRoot: $AgentRoot" +Write-Host "Config: $configPath" +Write-Host "Evidence: $EvidenceDir" +if ($vmrun) { + Write-Host "vmrun: $vmrun" +} else { + Write-Host "vmrun: not detected; update agent99.config.json" +} diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 new file mode 100644 index 000000000..7f3bad331 --- /dev/null +++ b/agent99-control-plane.ps1 @@ -0,0 +1,2454 @@ +param( + [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "ControlTick")] + [string]$Mode = "Status", + [switch]$ControlledApply, + [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", + [string]$EvidenceDir = "C:\Wooo\Agent99\evidence" +) + +$ErrorActionPreference = "Continue" +New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null + +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$logPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.log" +$jsonPath = Join-Path $EvidenceDir "agent99-$Mode-$stamp.json" + +function Write-AgentLog { + param([string]$Message) + $line = "$(Get-Date -Format o) $Message" + $line | Tee-Object -FilePath $logPath -Append | Out-Null +} + +function Load-Config { + if (-not (Test-Path $ConfigPath)) { + throw "Config not found: $ConfigPath" + } + Get-Content $ConfigPath -Raw | ConvertFrom-Json +} + +$Config = Load-Config +$SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" } +$script:Events = @() +$script:TelegramAttempts = @() + +function Get-HostSshUser { + param([string]$TargetHost) + if ($Config.sshUsers -and $Config.sshUsers.PSObject.Properties[$TargetHost]) { + return [string]$Config.sshUsers.PSObject.Properties[$TargetHost].Value + } + $SshUser +} + +function Send-AgentTelegramRelay { + param( + [string]$Message, + [object]$Relay, + [string]$ChatIdOverride = "" + ) + + if (-not ($Relay -and $Relay.enabled)) { + return [pscustomobject]@{ ok = $false; error = "relay_disabled" } + } + + $relayHost = if ($Relay.host) { [string]$Relay.host } else { "192.168.0.110" } + $container = if ($Relay.container) { [string]$Relay.container } else { "stockplatform-v2-api-1" } + if ($container -notmatch "^[A-Za-z0-9_.-]+$") { + return [pscustomobject]@{ ok = $false; error = "invalid_relay_container" } + } + + $messageBytes = [Text.Encoding]::UTF8.GetBytes($Message) + $messageB64 = [Convert]::ToBase64String($messageBytes) + $chatOverrideB64 = if ($ChatIdOverride) { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($ChatIdOverride)) } else { "" } + $python = 'import base64, os, sys, urllib.parse, urllib.request +text = base64.b64decode(sys.argv[2]).decode() +chat_override = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] else "" +token = os.environ.get("TELEGRAM_BOT_TOKEN") +chat = chat_override or os.environ.get("TELEGRAM_CHAT_ID") +assert token and chat +data = urllib.parse.urlencode({"chat_id": chat, "text": text, "disable_web_page_preview": "true"}).encode() +urllib.request.urlopen("https://api.telegram.org/bot" + token + "/sendMessage", data=data, timeout=15).read() +print("sent_ok") +' + $pythonB64 = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($python)) + $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))' + $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $messageB64 $chatOverrideB64" + $run = Invoke-SshText $relayHost $remoteCommand 45 1 + + [pscustomobject]@{ + ok = [bool]($run.ok -and $run.output -match "sent_ok") + host = $relayHost + container = $container + exitCode = $run.exitCode + chatOverride = if ($ChatIdOverride) { "telegram-origin" } else { $null } + output = $run.output + error = if ($run.ok) { $null } else { $run.output } + } +} + +function Format-AgentTelegramText { + param( + [string]$Severity, + [string]$EventType, + [string]$Message, + [object]$Data = $null + ) + + $severityLabel = switch ($Severity) { + "critical" { "CRITICAL" } + "warning" { "WARNING" } + "info" { "INFO" } + default { $Severity } + } + $lines = @() + $lines += "Agent99 Alert: $severityLabel" + + if ($EventType -match "^performance_") { + $hostName = if ($Data -and $Data.PSObject.Properties["host"]) { $Data.host } else { "unknown" } + $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() } + $disk = if ($Data -and $Data.PSObject.Properties["diskUsedPercent"]) { $Data.diskUsedPercent } else { $null } + $load = if ($Data -and $Data.PSObject.Properties["loadPerCore"]) { $Data.loadPerCore } else { $null } + $mem = if ($Data -and $Data.PSObject.Properties["memAvailablePercent"]) { $Data.memAvailablePercent } else { $null } + $lines += "Event: host performance" + $lines += "Host: $hostName" + if ($reasons -contains "disk_used_warning") { + $lines += "Reason: disk usage is $disk percent. Warning threshold reached; critical threshold is 95 percent." + $lines += "Action: Agent99 will run allowlisted disk remediation and suppress duplicates for 30 minutes." + } elseif ($reasons -contains "disk_used_high") { + $lines += "Reason: disk usage is $disk percent. Critical threshold reached." + $lines += "Action: Agent99 will run allowlisted cleanup first; reboot is not the first response." + } elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") { + $lines += "Reason: CPU load/core=$load." + $lines += "Action: Agent99 will use allowlisted load reduction only; no arbitrary service stop." + } elseif ($reasons -contains "memory_available_low") { + $lines += "Reason: available memory is $mem percent." + $lines += "Action: Agent99 will inspect memory pressure before allowlisted remediation." + } else { + $lines += "Reason: $($reasons -join ', ')" + $lines += "Action: check Agent99 evidence for full readback." + } + $lines += "Current: load/core=$load, memAvail=$mem percent, disk=$disk percent." + } elseif ($EventType -match "^backup_") { + $targetName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } + $reason = if ($Data -and $Data.PSObject.Properties["reason"]) { $Data.reason } else { $EventType } + $age = if ($Data -and $Data.PSObject.Properties["ageMinutes"]) { $Data.ageMinutes } else { $null } + $maxAge = if ($Data -and $Data.PSObject.Properties["maxAgeMinutes"]) { $Data.maxAgeMinutes } else { $null } + $lines += "Event: backup health" + $lines += "Target: $targetName" + $lines += "Reason: $reason" + if ($age -ne $null -or $maxAge -ne $null) { + $lines += "Freshness: age=$age minutes / max=$maxAge minutes" + } + $lines += "Action: inspect /backup/status, /backup/logs, and snapshot metadata only; do not read backup contents or secrets." + } elseif ($EventType -eq "provider_freshness_triage") { + $candidateId = if ($Data -and $Data.PSObject.Properties["candidateId"]) { $Data.candidateId } else { "unknown" } + $targetName = if ($Data -and $Data.PSObject.Properties["target"]) { $Data.target } else { "provider_freshness_signal" } + $hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 } + $missing = if ($Data -and $Data.PSObject.Properties["missingFields"]) { @($Data.missingFields) } else { @() } + $candidatePath = if ($Data -and $Data.PSObject.Properties["candidatePath"]) { $Data.candidatePath } else { $null } + $lines += "Event: provider freshness triage" + $lines += "Target: $targetName" + $lines += "Candidate: $candidateId" + $lines += "Evidence hits: $hitCount" + if (@($missing).Count -gt 0) { + $lines += "Missing: $($missing -join ', ')" + } + $lines += "Action: require provider window, last seen, direct reference, and verifier readback before trusting dashboard green." + $lines += "Guard: no provider switch, no paid model call, no env change, no reload." + if ($candidatePath) { + $lines += "Evidence: $candidatePath" + } + } elseif ($EventType -match "^load_shed_") { + $actionName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } + $hostName = if ($Data -and $Data.PSObject.Properties["host"]) { $Data.host } else { "unknown" } + $ok = if ($Data -and $Data.PSObject.Properties["ok"]) { $Data.ok } else { $null } + $executionOk = if ($Data -and $Data.PSObject.Properties["executionOk"]) { $Data.executionOk } else { $null } + $postVerifyOk = if ($Data -and $Data.PSObject.Properties["postVerifyOk"]) { $Data.postVerifyOk } else { $null } + $exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null } + $rollback = if ($Data -and $Data.PSObject.Properties["rollback"]) { $Data.rollback } else { $null } + $lines += "Event: controlled remediation" + $lines += "Host: $hostName" + $lines += "Action: $actionName" + if ($ok -ne $null) { + $lines += "Result: ok=$ok exitCode=$exitCode" + if ($executionOk -ne $null -or $postVerifyOk -ne $null) { + $lines += "Verifier: executionOk=$executionOk postVerifyOk=$postVerifyOk" + } + } else { + $lines += "Result: check Agent99 evidence." + } + if ($rollback) { + $lines += "Rollback: $rollback" + } + } elseif ($EventType -match "^operator_command_") { + $commandId = if ($Data -and $Data.PSObject.Properties["id"]) { $Data.id } else { "unknown" } + $modeName = if ($Data -and $Data.PSObject.Properties["mode"]) { $Data.mode } else { "unknown" } + $source = if ($Data -and $Data.PSObject.Properties["source"]) { $Data.source } else { "unknown" } + $instruction = if ($Data -and $Data.PSObject.Properties["instruction"]) { $Data.instruction } else { $null } + $exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null } + $evidencePath = if ($Data -and $Data.PSObject.Properties["evidence"]) { $Data.evidence } else { $null } + $lines += "Event: operator command" + $lines += "Command: $commandId" + $lines += "Source: $source" + $lines += "Mode: $modeName" + if ($instruction) { + $lines += "Instruction: $instruction" + } + $lines += "Result: exitCode=$exitCode" + if ($Data -and $Data.PSObject.Properties["problem"]) { + $problem = $Data.problem + if ($problem -and $problem.PSObject.Properties["occurrences"]) { + $lines += "Problem count: occurrences=$($problem.occurrences), resolved=$($problem.resolved), failed=$($problem.failed)" + } + if ($problem -and $problem.PSObject.Properties["kmPath"]) { + $lines += "KM: $($problem.kmPath)" + } + } + if ($evidencePath) { + $lines += "Evidence: $evidencePath" + } + } elseif ($EventType -eq "agent_control_tick") { + $dashboardPath = if ($Data -and $Data.PSObject.Properties["dashboardPath"]) { $Data.dashboardPath } else { $null } + $processed = if ($Data -and $Data.PSObject.Properties["processedCount"]) { $Data.processedCount } else { 0 } + $pending = if ($Data -and $Data.PSObject.Properties["pendingCount"]) { $Data.pendingCount } else { 0 } + $lines += "Event: Agent99 control loop" + $lines += "Dashboard: $dashboardPath" + $lines += "Queue: processed=$processed pending=$pending" + } elseif ($EventType -match "^ai_service_") { + $serviceName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } + $lines += "Event: AI service health" + $lines += "Service: $serviceName" + $lines += "Action: inspect health endpoint and service/container state before controlled repair." + } elseif ($EventType -eq "agent_selfcheck") { + $lines += "Event: Agent99 self health" + $lines += "Summary: $Message" + $lines += "Action: inspect task state, evidence freshness, and Telegram delivery; reboot is not the first response." + } else { + $lines += "Event: $EventType" + $lines += "Summary: $Message" + } + + $lines += "Time: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss zzz')" + $text = ($lines -join "`n") + if ($text.Length -gt 3500) { + $text = $text.Substring(0, 3500) + "`n...(truncated)" + } + $text +} + +function Send-AgentTelegram { + param( + [string]$Severity, + [string]$EventType, + [string]$Message, + [object]$Data = $null + ) + + $telegram = $Config.telegram + $enabled = ($telegram -and $telegram.enabled) + $botTokenEnv = if ($telegram.botTokenEnv) { $telegram.botTokenEnv } else { "AGENT99_TELEGRAM_BOT_TOKEN" } + $chatIdEnv = if ($telegram.chatIdEnv) { $telegram.chatIdEnv } else { "AGENT99_TELEGRAM_CHAT_ID" } + $tokenEnvAliases = @($botTokenEnv) + if ($telegram.botTokenEnvAliases) { $tokenEnvAliases += @($telegram.botTokenEnvAliases) } + $chatIdEnvAliases = @($chatIdEnv) + if ($telegram.chatIdEnvAliases) { $chatIdEnvAliases += @($telegram.chatIdEnvAliases) } + + function Get-AgentEnvSecret { + param([string[]]$Names) + foreach ($name in ($Names | Where-Object { $_ } | Select-Object -Unique)) { + foreach ($scope in @("Machine", "User", "Process")) { + $value = [Environment]::GetEnvironmentVariable($name, $scope) + if ($value) { + return [pscustomobject]@{ value = $value; name = $name; scope = $scope } + } + } + } + return $null + } + + $tokenSecret = Get-AgentEnvSecret $tokenEnvAliases + $chatSecret = Get-AgentEnvSecret $chatIdEnvAliases + $token = if ($tokenSecret) { $tokenSecret.value } else { $null } + $chatId = if ($chatSecret) { $chatSecret.value } else { $null } + $chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" } + + $attempt = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + enabled = [bool]$enabled + eventType = $EventType + severity = $Severity + configured = [bool]($token -and $chatId) + tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null } + chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null } + chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null } + relay = $null + sent = $false + error = $null + } + + if (-not $enabled) { + $attempt.error = "telegram_disabled" + $script:TelegramAttempts += $attempt + return + } + $text = Format-AgentTelegramText $Severity $EventType $Message $Data + if (-not ($token -and ($chatId -or $chatIdOverride))) { + $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride + $attempt.relay = $relayResult + if ($relayResult.ok) { + $attempt.sent = $true + $attempt.error = $null + } else { + $attempt.error = "telegram_not_configured" + } + $script:TelegramAttempts += $attempt + return + } + + try { + Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ + chat_id = if ($chatIdOverride) { $chatIdOverride } else { $chatId } + text = $text + disable_web_page_preview = "true" + } | Out-Null + $attempt.sent = $true + } catch { + $attempt.error = $_.Exception.Message + } + + $script:TelegramAttempts += $attempt +} + +function Record-AgentEvent { + param( + [string]$EventType, + [string]$Severity, + [string]$Message, + [object]$Data = $null, + [switch]$Alert, + [switch]$NoAlert + ) + + Write-AgentLog "event type=$EventType severity=$Severity message=$Message" + $eventData = $Data + if ($Data -and $Data.PSObject.Properties["replyChatId"]) { + $eventData = $Data | Select-Object * -ExcludeProperty replyChatId + $eventData | Add-Member -MemberType NoteProperty -Name replyChat -Value "telegram-origin" -Force + } + $event = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + type = $EventType + severity = $Severity + message = $Message + data = $eventData + } + $script:Events += $event + + if ($NoAlert) { + return + } + + $alertAll = ($Config.telegram -and $Config.telegram.alertAllOperations) + $alertInfo = ($Config.telegram -and $Config.telegram.alertInfoEvents) + $alertOperatorCommands = $true + if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertOperatorCommands"]) { + $alertOperatorCommands = [bool]$Config.telegram.alertOperatorCommands + } + $operatorCommandAlert = [bool]($alertOperatorCommands -and $EventType -match "^operator_command_") + $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo)) + if ($shouldAlert) { + if ($operatorCommandAlert) { + Send-AgentTelegram $Severity $EventType $Message $Data + return + } + $dedupeMinutes = 30 + if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) { + $dedupeMinutes = [int]$Config.telegram.dedupeMinutes + } + $dedupeParts = @($EventType, $Severity) + if ($Data) { + foreach ($propName in @("host", "name", "path", "reason")) { + if ($Data.PSObject.Properties[$propName]) { + $dedupeParts += "$propName=$($Data.PSObject.Properties[$propName].Value)" + } + } + if ($Data.PSObject.Properties["reasons"]) { + $dedupeParts += "reasons=$(@($Data.reasons) -join ',')" + } + } + if ($dedupeParts.Count -le 2) { + $normalizedMessage = $Message + foreach ($pattern in @("evidence=[^ ]+", "loadPerCore=[^ ]+", "memAvailPct=[^ ]+", "diskUsedPct=[^ ]+", "ageMinutes=[^ ]+")) { + $normalizedMessage = $normalizedMessage -replace $pattern, ($pattern.Split("=")[0] + "=") + } + $dedupeParts += $normalizedMessage + } + $dedupeSource = ($dedupeParts -join "|") + $dedupeHashBytes = [Security.Cryptography.SHA256]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($dedupeSource)) + $dedupeHash = [BitConverter]::ToString($dedupeHashBytes).Replace("-", "").ToLowerInvariant() + $dedupePath = Join-Path $EvidenceDir "telegram-dedupe-$dedupeHash.marker" + if ($dedupeMinutes -gt 0 -and (Test-Path $dedupePath)) { + $ageMinutes = ((Get-Date) - (Get-Item $dedupePath).LastWriteTime).TotalMinutes + if ($ageMinutes -lt $dedupeMinutes) { + Write-AgentLog "telegram_dedupe_skip type=$EventType severity=$Severity ageMinutes=$([math]::Round($ageMinutes, 2)) cooldown=$dedupeMinutes" + return + } + } + Set-Content -Path $dedupePath -Value (Get-Date -Format o) -Encoding UTF8 + Send-AgentTelegram $Severity $EventType $Message $Data + } +} + +function Invoke-SshText { + param( + [string]$TargetHost, + [string]$Command, + [int]$TimeoutSeconds = 12, + [int]$Retries = 2, + [string]$User = $null + ) + + $last = $null + $userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost } + for ($attempt = 1; $attempt -le $Retries; $attempt++) { + $args = @( + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=$TimeoutSeconds", + "-o", "ServerAliveInterval=20", + "-o", "ServerAliveCountMax=3", + "-l", $userToUse, + $TargetHost, + $Command + ) + + try { + $id = [guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out" + $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err" + $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru + + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $process.Kill() + $process.WaitForExit(2000) | Out-Null + Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null + $last = [pscustomobject]@{ + ok = $false + exitCode = -2 + output = "timeout_after_${TimeoutSeconds}s" + } + } else { + $process.WaitForExit() | Out-Null + $process.Refresh() + $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" } + $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" } + Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null + $text = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" + $exitCode = $process.ExitCode + if ($null -eq $exitCode) { + $exitCode = if ($stderr) { -3 } else { 0 } + } + $last = [pscustomobject]@{ + ok = ($exitCode -eq 0) + exitCode = $exitCode + output = $text + } + } + } catch { + $last = [pscustomobject]@{ + ok = $false + exitCode = -1 + output = $_.Exception.Message + } + } + + if ($last.ok) { + if ($attempt -gt 1) { + Write-AgentLog "ssh_retry_recovered host=$TargetHost attempt=$attempt" + } + return $last + } + + if ($attempt -lt $Retries) { + Write-AgentLog "ssh_retry host=$TargetHost attempt=$attempt exit=$($last.exitCode)" + Start-Sleep -Seconds 2 + } + } + + $last +} + +function Quote-ShSingle { + param([string]$Text) + "'" + $Text.Replace("'", "'`"`"'") + "'" +} + +function Invoke-HostSshText { + param( + [string]$TargetHost, + [string]$Command, + [int]$TimeoutSeconds = 30, + [int]$Retries = 1 + ) + + $direct = Invoke-SshText $TargetHost $Command $TimeoutSeconds $Retries + if ($direct.ok -or -not $Config.k3s.jumpHost -or $TargetHost -eq $Config.k3s.jumpHost) { + $direct | Add-Member -NotePropertyName route -NotePropertyValue "direct" -Force + return $direct + } + + $quotedCommand = Quote-ShSingle $Command + $targetUser = Get-HostSshUser $TargetHost + $jumpCommand = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=10 -l $targetUser $TargetHost $quotedCommand" + $viaJump = Invoke-SshText $Config.k3s.jumpHost $jumpCommand $TimeoutSeconds $Retries + $viaJump | Add-Member -NotePropertyName route -NotePropertyValue "via_$($Config.k3s.jumpHost)" -Force + return $viaJump +} + +function Convert-AgentDouble { + param([string]$Value) + if (-not $Value) { return $null } + try { + return [double]::Parse($Value.Replace(",", "."), [Globalization.CultureInfo]::InvariantCulture) + } catch { + return $null + } +} + +function Get-PerfThresholds { + $perf = $Config.performance + [pscustomobject]@{ + loadPerCoreWarning = if ($perf.loadPerCoreWarning) { [double]$perf.loadPerCoreWarning } else { 0.9 } + loadPerCoreCritical = if ($perf.loadPerCoreCritical) { [double]$perf.loadPerCoreCritical } else { 1.5 } + memoryAvailablePercentCritical = if ($perf.memoryAvailablePercentCritical) { [double]$perf.memoryAvailablePercentCritical } else { 10 } + diskUsedPercentWarning = if ($perf.diskUsedPercentWarning) { [double]$perf.diskUsedPercentWarning } else { 90 } + diskUsedPercentCritical = if ($perf.diskUsedPercentCritical) { [double]$perf.diskUsedPercentCritical } else { 95 } + } +} + +function Get-HostPerformance { + param([switch]$SuppressAlerts) + + $thresholds = Get-PerfThresholds + $results = @() + $command = @' +echo __AGENT99_PERF__ +echo HOST=$(hostname) +echo CORES=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1) +uptime +echo __MEM__ +free -m 2>/dev/null +echo __DISK__ +df -P / 2>/dev/null +'@ + + foreach ($hostIp in $Config.hosts) { + $readback = Invoke-HostSshText $hostIp $command 25 1 + $text = $readback.output + $cores = 1 + $coresMatch = [regex]::Match($text, "CORES=(\d+)") + if ($coresMatch.Success) { $cores = [int]$coresMatch.Groups[1].Value } + + $load1 = $null + $loadMatch = [regex]::Match($text, "load averages?:\s*([0-9]+(?:[\.,][0-9]+)?)") + if (-not $loadMatch.Success) { + $loadMatch = [regex]::Match($text, "load average:\s*([0-9]+(?:[\.,][0-9]+)?)") + } + if ($loadMatch.Success) { + $load1 = Convert-AgentDouble $loadMatch.Groups[1].Value + } + + $memAvailablePercent = $null + $memLine = (($text -split "`n") | Where-Object { $_ -match "^\s*Mem:" } | Select-Object -First 1) + if ($memLine) { + $parts = @($memLine -split "\s+" | Where-Object { $_ }) + if ($parts.Count -ge 7) { + $memTotal = Convert-AgentDouble $parts[1] + $memAvailable = Convert-AgentDouble $parts[6] + if ($memTotal -and $memTotal -gt 0 -and $null -ne $memAvailable) { + $memAvailablePercent = [math]::Round(($memAvailable / $memTotal) * 100, 2) + } + } + } + + $diskUsedPercent = $null + $diskLine = (($text -split "`n") | Where-Object { $_ -match "\s+\d+%\s+/$" } | Select-Object -First 1) + if ($diskLine) { + $diskMatch = [regex]::Match($diskLine, "\s(\d+)%\s+/$") + if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } + } + + $topCpu = @() + + $loadPerCore = if ($null -ne $load1 -and $cores -gt 0) { [math]::Round($load1 / $cores, 2) } else { $null } + $severity = "ok" + $reasons = @() + if (-not $readback.ok) { + $severity = "warning" + $reasons += "perf_readback_failed" + } + if ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreCritical) { + $severity = "critical" + $reasons += "load_per_core_critical" + } elseif ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreWarning -and $severity -ne "critical") { + $severity = "warning" + $reasons += "load_per_core_warning" + } + if ($null -ne $memAvailablePercent -and $memAvailablePercent -le $thresholds.memoryAvailablePercentCritical) { + $severity = "critical" + $reasons += "memory_available_low" + } + if ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentCritical) { + $severity = "critical" + $reasons += "disk_used_high" + } elseif ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentWarning -and $severity -ne "critical") { + $severity = "warning" + $reasons += "disk_used_warning" + } + + $captureTopCpuOnWarning = $true + if ($Config.performance -and $Config.performance.PSObject.Properties["captureTopCpuOnWarning"]) { + $captureTopCpuOnWarning = [bool]$Config.performance.captureTopCpuOnWarning + } + $topCpuTimeoutSeconds = 10 + if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuTimeoutSeconds"]) { + $topCpuTimeoutSeconds = [int]$Config.performance.topCpuTimeoutSeconds + } + $shouldCaptureTopCpu = ($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning) + if ($shouldCaptureTopCpu -and $readback.ok) { + $topReadback = Invoke-HostSshText $hostIp "ps -eo pid,ppid,comm,pcpu,pmem --sort=-pcpu" $topCpuTimeoutSeconds 1 + if ($topReadback.ok) { + $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) + } else { + $topCpu = @("top_cpu_readback_failed: $($topReadback.output)") + } + } + + $result = [pscustomobject]@{ + host = $hostIp + ok = [bool]$readback.ok + route = $readback.route + severity = $severity + reasons = $reasons + cores = $cores + load1 = $load1 + loadPerCore = $loadPerCore + memAvailablePercent = $memAvailablePercent + diskUsedPercent = $diskUsedPercent + topCpu = $topCpu + output = $text + } + Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route)" + if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) { + Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert + } + $results += $result + } + + $results +} + +function Record-PerformanceIssuesWithoutRemediation { + param( + [object[]]$Performance, + [object[]]$LoadShedding + ) + + foreach ($perf in @($Performance | Where-Object { $_ -and $_.severity -in @("warning", "critical") })) { + $attempts = @($LoadShedding | Where-Object { $_ -and $_.host -eq $perf.host }) + $executed = @($attempts | Where-Object { $_.PSObject.Properties["skipped"] -and $_.skipped -eq $false }) + if ($executed.Count -gt 0) { + continue + } + + $message = "host=$($perf.host) loadPerCore=$($perf.loadPerCore) memAvailPct=$($perf.memAvailablePercent) diskUsedPct=$($perf.diskUsedPercent) reasons=$(@($perf.reasons) -join ',')" + Record-AgentEvent "performance_$($perf.severity)" $perf.severity $message $perf -Alert + } +} + +function Invoke-LoadShedding { + param([object[]]$Performance) + + $results = @() + $loadShedding = $Config.performance.loadShedding + if (-not ($loadShedding -and $loadShedding.enabled)) { + Write-AgentLog "load_shedding skipped enabled=false" + return $results + } + if (-not $ControlledApply) { + Write-AgentLog "load_shedding skipped controlled_apply=false" + return $results + } + + foreach ($action in @($loadShedding.actions)) { + if ($action.enabled -eq $false) { continue } + $hostPerf = $Performance | Where-Object { $_.host -eq $action.host } | Select-Object -First 1 + if (-not $hostPerf) { continue } + + $severityRank = @{ ok = 0; warning = 1; critical = 2 } + $minSeverity = if ($action.minSeverity) { [string]$action.minSeverity } else { "critical" } + if (-not $severityRank.ContainsKey($minSeverity)) { $minSeverity = "critical" } + $hostSeverity = if ($hostPerf.severity -and $severityRank.ContainsKey([string]$hostPerf.severity)) { [string]$hostPerf.severity } else { "ok" } + if ($severityRank[$hostSeverity] -lt $severityRank[$minSeverity]) { + $results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "host_below_min_severity"; hostSeverity = $hostSeverity; minSeverity = $minSeverity } + continue + } + + $reasonMatches = @($action.reasonMatches) + if ($reasonMatches.Count -eq 0) { $reasonMatches = @("load_per_core_critical") } + $matchedReasons = @($hostPerf.reasons | Where-Object { $_ -in $reasonMatches }) + if ($matchedReasons.Count -eq 0) { + $results += [pscustomobject]@{ + name = $action.name + host = $action.host + skipped = $true + reason = "critical_reason_not_allowlisted" + hostReasons = $hostPerf.reasons + allowedReasons = $reasonMatches + } + continue + } + + $cooldownMinutes = if ($action.cooldownMinutes) { [int]$action.cooldownMinutes } else { 30 } + $safeName = ($action.name -replace "[^A-Za-z0-9_.-]", "_") + $markerPath = Join-Path $EvidenceDir "loadshedding-$safeName.marker" + if (Test-Path $markerPath) { + $ageMinutes = ((Get-Date) - (Get-Item $markerPath).LastWriteTime).TotalMinutes + if ($ageMinutes -lt $cooldownMinutes) { + $results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "cooldown"; ageMinutes = [math]::Round($ageMinutes, 1) } + continue + } + } + + Record-AgentEvent "load_shed_attempt" $hostSeverity "host=$($action.host) action=$($action.name) command=$($action.command)" $action -NoAlert + $timeoutSeconds = if ($action.PSObject.Properties["timeoutSeconds"]) { [int]$action.timeoutSeconds } else { 60 } + $run = Invoke-HostSshText $action.host $action.command $timeoutSeconds 1 + + $postVerify = $null + $postVerifyOk = $false + if ($action.PSObject.Properties["postVerifyCommand"] -and $action.postVerifyCommand) { + $verifyRun = Invoke-HostSshText $action.host ([string]$action.postVerifyCommand) 30 1 + $postVerify = [pscustomobject]@{ + ok = [bool]$verifyRun.ok + exitCode = $verifyRun.exitCode + output = $verifyRun.output + route = $verifyRun.route + } + if ($verifyRun.ok) { + $postVerifyOk = $true + if ($action.PSObject.Properties["postVerifyMaxDiskUsedPercent"]) { + $match = [regex]::Match([string]$verifyRun.output, "__DISK_USED__=(\d+)") + if ($match.Success) { + $postVerifyOk = ([int]$match.Groups[1].Value -le [int]$action.postVerifyMaxDiskUsedPercent) + } else { + $postVerifyOk = $false + } + } + if ($action.PSObject.Properties["postVerifyOkRegex"] -and $action.postVerifyOkRegex) { + $postVerifyOk = [bool]([string]$verifyRun.output -match [string]$action.postVerifyOkRegex) + } + } + } + + $effectiveOk = [bool]($run.ok -or $postVerifyOk) + Set-Content -Path $markerPath -Value (Get-Date -Format o) -Encoding UTF8 + $result = [pscustomobject]@{ + name = $action.name + host = $action.host + skipped = $false + ok = $effectiveOk + executionOk = [bool]$run.ok + postVerifyOk = [bool]$postVerifyOk + postVerify = $postVerify + route = $run.route + exitCode = $run.exitCode + output = $run.output + rollback = $action.rollback + } + Record-AgentEvent "load_shed_result" $(if ($effectiveOk) { "warning" } else { "critical" }) "host=$($action.host) action=$($action.name) ok=$effectiveOk executionOk=$($run.ok) postVerifyOk=$postVerifyOk" $result -Alert + $results += $result + } + + $results +} + +function Get-AgentBooleanSetting { + param( + [object]$Object, + [string]$Name, + [bool]$Default + ) + + if ($Object -and $Object.PSObject.Properties[$Name]) { + return [bool]$Object.PSObject.Properties[$Name].Value + } + return $Default +} + +function Get-AgentSelfHealthConfig { + $selfHealth = $Config.selfHealth + $defaultTasks = @( + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" + ) + $defaultEvidence = @( + [pscustomobject]@{ name = "performance_guard"; pattern = "agent99-Perf-*.json"; maxAgeMinutes = 5; required = $true }, + [pscustomobject]@{ name = "heartbeat_status"; pattern = "agent99-Status-*.json"; maxAgeMinutes = 15; required = $true }, + [pscustomobject]@{ name = "self_health"; pattern = "agent99-SelfCheck-*.json"; maxAgeMinutes = 15; required = $false }, + [pscustomobject]@{ name = "backup_health"; pattern = "agent99-BackupCheck-*.json"; maxAgeMinutes = 30; required = $false }, + [pscustomobject]@{ name = "startup_recovery"; pattern = "agent99-Recover-*.json"; maxAgeMinutes = 1440; required = $false }, + [pscustomobject]@{ name = "telegram_inbox"; pattern = "agent99-TelegramInbox-*.json"; maxAgeMinutes = 15; required = $false }, + [pscustomobject]@{ name = "sre_alert_inbox"; pattern = "agent99-SreAlertInbox-*.json"; maxAgeMinutes = 15; required = $false } + ) + + [pscustomobject]@{ + scheduledTasks = if ($selfHealth -and $selfHealth.scheduledTasks) { @($selfHealth.scheduledTasks) } else { $defaultTasks } + evidenceFreshness = if ($selfHealth -and $selfHealth.evidenceFreshness) { @($selfHealth.evidenceFreshness) } else { $defaultEvidence } + requiredHosts = if ($selfHealth -and $selfHealth.requiredHosts) { @($selfHealth.requiredHosts) } else { @($Config.hosts) } + requireRecentTelegramDelivery = Get-AgentBooleanSetting $selfHealth "requireRecentTelegramDelivery" $true + } +} + +function Get-AgentLatestEvidence { + param([string]$Pattern) + + $file = Get-ChildItem -Path $EvidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 + if (-not $file) { + return [pscustomobject]@{ + exists = $false + path = $null + fileName = $null + lastWriteTime = $null + ageMinutes = $null + data = $null + parseError = $null + } + } + + $data = $null + $parseError = $null + try { + $data = Get-Content $file.FullName -Raw | ConvertFrom-Json + } catch { + $parseError = $_.Exception.Message + } + + [pscustomobject]@{ + exists = $true + path = $file.FullName + fileName = $file.Name + lastWriteTime = $file.LastWriteTime.ToString("o") + ageMinutes = [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 2) + data = $data + parseError = $parseError + } +} + +function Test-AgentScheduledTaskHealth { + param([string[]]$TaskNames) + + $results = @() + foreach ($taskName in $TaskNames) { + try { + $task = Get-ScheduledTask -TaskName $taskName -ErrorAction Stop + $info = Get-ScheduledTaskInfo -TaskName $taskName -ErrorAction Stop + $state = [string]$task.State + $lastTaskResult = $info.LastTaskResult + $pendingFirstRun = ($lastTaskResult -eq 267011) + $ok = ($state -eq "Running") -or (($state -eq "Ready") -and ($lastTaskResult -in @(0, 267011))) + $results += [pscustomobject]@{ + name = $taskName + exists = $true + ok = [bool]$ok + state = $state + lastTaskResult = $lastTaskResult + pendingFirstRun = [bool]$pendingFirstRun + lastRunTime = if ($info.LastRunTime) { $info.LastRunTime.ToString("o") } else { $null } + nextRunTime = if ($info.NextRunTime) { $info.NextRunTime.ToString("o") } else { $null } + } + Write-AgentLog "selfcheck task name=$taskName ok=$ok state=$state lastTaskResult=$lastTaskResult pendingFirstRun=$pendingFirstRun" + } catch { + $results += [pscustomobject]@{ + name = $taskName + exists = $false + ok = $false + state = $null + lastTaskResult = $null + lastRunTime = $null + nextRunTime = $null + error = $_.Exception.Message + } + Write-AgentLog "selfcheck task_missing name=$taskName error=$($_.Exception.Message)" + } + } + + $results +} + +function Test-AgentEvidenceFreshness { + param([object[]]$Contracts) + + $results = @() + foreach ($contract in $Contracts) { + $name = if ($contract.name) { [string]$contract.name } else { [string]$contract.pattern } + $pattern = [string]$contract.pattern + $maxAgeMinutes = if ($contract.maxAgeMinutes) { [double]$contract.maxAgeMinutes } else { 15 } + $required = Get-AgentBooleanSetting $contract "required" $true + $latest = Get-AgentLatestEvidence $pattern + $fresh = [bool]($latest.exists -and $null -eq $latest.parseError -and $latest.ageMinutes -le $maxAgeMinutes) + $ok = [bool](((-not $required) -and (-not $latest.exists)) -or $fresh) + $severity = if ($ok) { "ok" } elseif ($required) { "critical" } else { "warning" } + + $results += [pscustomobject]@{ + name = $name + pattern = $pattern + required = [bool]$required + ok = $ok + severity = $severity + latestPath = $latest.path + latestFile = $latest.fileName + lastWriteTime = $latest.lastWriteTime + ageMinutes = $latest.ageMinutes + maxAgeMinutes = $maxAgeMinutes + parseError = $latest.parseError + } + Write-AgentLog "selfcheck evidence name=$name ok=$ok severity=$severity ageMinutes=$($latest.ageMinutes) maxAgeMinutes=$maxAgeMinutes file=$($latest.fileName)" + } + + $results +} + +function Test-AgentLatestPerformanceEvidence { + param([string[]]$RequiredHosts) + + $latest = Get-AgentLatestEvidence "agent99-Perf-*.json" + $hostChecks = @() + if (-not $latest.exists) { + return [pscustomobject]@{ + ok = $false + severity = "critical" + reason = "missing_perf_evidence" + latestPath = $null + latestFile = $null + ageMinutes = $null + hosts = $hostChecks + } + } + if ($latest.parseError) { + return [pscustomobject]@{ + ok = $false + severity = "critical" + reason = "perf_evidence_parse_error" + latestPath = $latest.path + latestFile = $latest.fileName + ageMinutes = $latest.ageMinutes + parseError = $latest.parseError + hosts = $hostChecks + } + } + + $perf = @($latest.data.performance) + foreach ($hostIp in $RequiredHosts) { + $entry = $perf | Where-Object { $_.host -eq $hostIp } | Select-Object -First 1 + $hostSeverity = if (-not $entry) { + "missing" + } elseif ($entry.severity) { + [string]$entry.severity + } elseif ($entry.ok -eq $false) { + "warning" + } else { + "ok" + } + $ok = [bool]($hostSeverity -eq "ok") + $hostChecks += [pscustomobject]@{ + host = $hostIp + present = [bool]$entry + ok = $ok + severity = $hostSeverity + loadPerCore = if ($entry) { $entry.loadPerCore } else { $null } + memAvailablePercent = if ($entry) { $entry.memAvailablePercent } else { $null } + diskUsedPercent = if ($entry) { $entry.diskUsedPercent } else { $null } + reasons = if ($entry) { @($entry.reasons) } else { @("missing_perf_host") } + } + } + + $missingOrCritical = @($hostChecks | Where-Object { $_.severity -in @("missing", "critical") }).Count + $warning = @($hostChecks | Where-Object { $_.severity -eq "warning" }).Count + $severity = if ($missingOrCritical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } + + [pscustomobject]@{ + ok = [bool]($severity -eq "ok") + severity = $severity + reason = if ($severity -eq "ok") { "all_required_hosts_ok" } else { "required_host_perf_not_ok" } + latestPath = $latest.path + latestFile = $latest.fileName + ageMinutes = $latest.ageMinutes + hosts = $hostChecks + } +} + +function Test-AgentRecentTelegramDelivery { + param([object[]]$Contracts) + + $checks = @() + $lookbackMinutes = 60 + if ($Config.selfHealth -and $Config.selfHealth.PSObject.Properties["telegramDeliveryLookbackMinutes"]) { + $lookbackMinutes = [int]$Config.selfHealth.telegramDeliveryLookbackMinutes + } + $cutoff = (Get-Date).AddMinutes(-1 * $lookbackMinutes) + foreach ($contract in $Contracts) { + $pattern = [string]$contract.pattern + $files = Get-ChildItem -Path $EvidenceDir -Filter $pattern -File -ErrorAction SilentlyContinue | + Where-Object { $_.LastWriteTime -ge $cutoff } | + Sort-Object LastWriteTime -Descending | + Select-Object -First 20 + + foreach ($file in @($files)) { + $data = $null + try { + $data = Get-Content $file.FullName -Raw | ConvertFrom-Json + } catch { + continue + } + if (-not $data) { continue } + $attempts = @($data.telegram) + if ($attempts.Count -eq 0) { continue } + + $sent = @($attempts | Where-Object { $_.sent -eq $true -or ($_.relay -and $_.relay.ok -eq $true) }) + $checks += [pscustomobject]@{ + name = if ($contract.name) { [string]$contract.name } else { [string]$contract.pattern } + latestPath = $file.FullName + latestFile = $file.Name + ageMinutes = [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 2) + attempts = $attempts.Count + sent = $sent.Count + ok = [bool]($sent.Count -gt 0) + } + } + } + + $attemptCount = 0 + $sentCount = 0 + foreach ($check in @($checks)) { + $attemptCount += [int]$check.attempts + $sentCount += [int]$check.sent + } + if ($attemptCount -eq 0) { + return [pscustomobject]@{ + ok = $true + severity = "ok" + reason = "no_recent_alert_attempts" + checks = $checks + } + } + + [pscustomobject]@{ + ok = [bool]($sentCount -gt 0) + severity = if ($sentCount -gt 0) { "ok" } else { "critical" } + reason = if ($sentCount -gt 0) { "recent_alert_delivery_ok" } else { "recent_alert_delivery_failed" } + checks = $checks + } +} + +function Test-AgentSelfHealth { + $selfConfig = Get-AgentSelfHealthConfig + $tasks = Test-AgentScheduledTaskHealth $selfConfig.scheduledTasks + $evidence = Test-AgentEvidenceFreshness $selfConfig.evidenceFreshness + $latestPerf = Test-AgentLatestPerformanceEvidence $selfConfig.requiredHosts + $telegram = if ($selfConfig.requireRecentTelegramDelivery) { + Test-AgentRecentTelegramDelivery $selfConfig.evidenceFreshness + } else { + [pscustomobject]@{ ok = $true; severity = "ok"; checks = @(); skipped = $true } + } + + $taskFailures = @($tasks | Where-Object { -not $_.ok }).Count + $evidenceCritical = @($evidence | Where-Object { $_.severity -eq "critical" }).Count + $evidenceWarning = @($evidence | Where-Object { $_.severity -eq "warning" }).Count + $critical = $taskFailures + $evidenceCritical + if ($latestPerf.severity -eq "critical") { $critical += 1 } + if ($telegram.severity -eq "critical") { $critical += 1 } + $warning = $evidenceWarning + if ($latestPerf.severity -eq "warning") { $warning += 1 } + + $severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } + $message = "selfHealth severity=$severity taskFailures=$taskFailures evidenceCritical=$evidenceCritical evidenceWarning=$evidenceWarning perf=$($latestPerf.severity) telegram=$($telegram.severity)" + Record-AgentEvent "agent_selfcheck" $(if ($severity -eq "ok") { "info" } else { $severity }) $message $null -Alert + + [pscustomobject]@{ + ok = [bool]($severity -eq "ok") + severity = $severity + tasks = $tasks + evidenceFreshness = $evidence + latestPerformance = $latestPerf + telegramDelivery = $telegram + summary = [pscustomobject]@{ + taskFailures = $taskFailures + evidenceCritical = $evidenceCritical + evidenceWarning = $evidenceWarning + performanceSeverity = $latestPerf.severity + telegramSeverity = $telegram.severity + } + } +} + +function Get-AgentUnixAgeMinutes { + param([double]$UnixSeconds) + + $now = [DateTimeOffset]::Now.ToUnixTimeSeconds() + [math]::Round(($now - $UnixSeconds) / 60, 2) +} + +function Convert-AgentUnixToIso { + param([double]$UnixSeconds) + + try { + [DateTimeOffset]::FromUnixTimeSeconds([int64][math]::Floor($UnixSeconds)).LocalDateTime.ToString("o") + } catch { + $null + } +} + +function Get-AgentBackupHealthConfig { + $backupHealth = $Config.backupHealth + $backupHost = if ($backupHealth -and $backupHealth.host) { [string]$backupHealth.host } else { "192.168.0.110" } + $targets = @( + [pscustomobject]@{ name = "awoooi"; path = "/backup/awoooi/snapshots"; maxAgeMinutes = 430; required = $true }, + [pscustomobject]@{ name = "gitea"; path = "/backup/gitea/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "harbor"; path = "/backup/harbor/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "momo"; path = "/backup/momo/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "langfuse"; path = "/backup/langfuse/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "monitoring"; path = "/backup/monitoring/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "signoz"; path = "/backup/signoz/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "clawbot"; path = "/backup/clawbot/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "open-webui"; path = "/backup/open-webui/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "sentry"; path = "/backup/sentry/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "ai-artifacts"; path = "/backup/ai-artifacts/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "configs"; path = "/backup/configs/snapshots"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "public-routes"; path = "/backup/public-routes/snapshots"; maxAgeMinutes = 10080; required = $false } + ) + $fileChecks = @( + [pscustomobject]@{ name = "backup_log"; path = "/backup/logs/backup.log"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "backup_status_log"; path = "/backup/logs/backup-status.log"; maxAgeMinutes = 1800; required = $true }, + [pscustomobject]@{ name = "status_log"; path = "/backup/logs/status.log"; maxAgeMinutes = 10080; required = $false }, + [pscustomobject]@{ name = "cron_log"; path = "/backup/logs/cron.log"; maxAgeMinutes = 10080; required = $false }, + [pscustomobject]@{ name = "integrity_check_status"; path = "/backup/integrity/check.status"; maxAgeMinutes = 10080; required = $true }, + [pscustomobject]@{ name = "restore_drill_status"; path = "/backup/integrity/restore-drill.status"; maxAgeMinutes = 45000; required = $false }, + [pscustomobject]@{ name = "configs_last_status"; path = "/backup/status/backup-configs-last-status.json"; maxAgeMinutes = 1800; required = $false } + ) + $cronPatterns = @( + "backup-all.sh", + "backup-awoooi-frequent.sh", + "backup-health-textfile-exporter.py", + "check-backup-integrity.sh", + "sync-offsite-backups.sh --mode status", + "backup-status.sh" + ) + + [pscustomobject]@{ + host = $backupHost + targets = if ($backupHealth -and $backupHealth.targets) { @($backupHealth.targets) } else { $targets } + fileChecks = if ($backupHealth -and $backupHealth.fileChecks) { @($backupHealth.fileChecks) } else { $fileChecks } + cronPatterns = if ($backupHealth -and $backupHealth.cronPatterns) { @($backupHealth.cronPatterns) } else { $cronPatterns } + } +} + +function Test-AgentBackupSnapshotTarget { + param( + [object]$Target, + [string]$DefaultHost + ) + + $name = [string]$Target.name + $hostIp = if ($Target.host) { [string]$Target.host } else { $DefaultHost } + $path = [string]$Target.path + $required = Get-AgentBooleanSetting $Target "required" $true + $maxAgeMinutes = if ($Target.maxAgeMinutes) { [double]$Target.maxAgeMinutes } else { 1800 } + + if ($path -notmatch "^/backup/[A-Za-z0-9_.\/-]+$") { + return [pscustomobject]@{ name = $name; host = $hostIp; path = $path; ok = $false; severity = "critical"; reason = "invalid_backup_path" } + } + + $quotedPath = Quote-ShSingle $path + $command = "if [ -d $quotedPath ]; then echo __COUNT__=`$(find $quotedPath -maxdepth 1 -type f 2>/dev/null | wc -l); find $quotedPath -maxdepth 1 -type f -printf '__LATEST__=%T@|%TY-%Tm-%TdT%TH:%TM:%TS|%s|%p\n' 2>/dev/null | sort -n | tail -1; else echo __MISSING_DIR__=$quotedPath; fi" + $timeoutSeconds = 60 + if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["snapshotReadTimeoutSeconds"]) { + $timeoutSeconds = [int]$Config.backupHealth.snapshotReadTimeoutSeconds + } + if ($Target.PSObject.Properties["timeoutSeconds"]) { + $timeoutSeconds = [int]$Target.timeoutSeconds + } + $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1 + + $count = 0 + $latestUnix = $null + $latestSize = $null + $latestPath = $null + if ($readback.ok) { + $countMatch = [regex]::Match($readback.output, "__COUNT__=(\d+)") + if ($countMatch.Success) { $count = [int]$countMatch.Groups[1].Value } + $latestMatch = [regex]::Match($readback.output, "__LATEST__=([0-9.]+)\|([^|]+)\|(\d+)\|(.+)") + if ($latestMatch.Success) { + $latestUnix = [double]::Parse($latestMatch.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture) + $latestSize = [int64]$latestMatch.Groups[3].Value + $latestPath = $latestMatch.Groups[4].Value.Trim() + } + } + + $ageMinutes = if ($null -ne $latestUnix) { Get-AgentUnixAgeMinutes $latestUnix } else { $null } + $severity = "ok" + $reason = "fresh" + if (-not $readback.ok) { + $severity = if ($required) { "critical" } else { "warning" } + $reason = "backup_readback_failed" + } elseif ($readback.output -match "__MISSING_DIR__" -or $count -eq 0 -or $null -eq $latestUnix) { + $severity = if ($required) { "critical" } else { "warning" } + $reason = "backup_snapshot_missing" + } elseif ($ageMinutes -gt $maxAgeMinutes) { + $severity = if ($required) { "critical" } else { "warning" } + $reason = "backup_snapshot_stale" + } + + $result = [pscustomobject]@{ + name = $name + host = $hostIp + path = $path + required = [bool]$required + ok = [bool]($severity -eq "ok") + severity = $severity + reason = $reason + count = $count + latestPath = $latestPath + latestSizeBytes = $latestSize + latestTime = if ($null -ne $latestUnix) { Convert-AgentUnixToIso $latestUnix } else { $null } + ageMinutes = $ageMinutes + maxAgeMinutes = $maxAgeMinutes + route = $readback.route + exitCode = $readback.exitCode + } + Write-AgentLog "backup target=$name severity=$severity reason=$reason ageMinutes=$ageMinutes count=$count" + if ($severity -in @("warning", "critical")) { + Record-AgentEvent "backup_$reason" $severity "target=$name host=$hostIp ageMinutes=$ageMinutes maxAgeMinutes=$maxAgeMinutes path=$path" $result -Alert + } + $result +} + +function Test-AgentBackupFileCheck { + param( + [object]$FileCheck, + [string]$DefaultHost + ) + + $name = [string]$FileCheck.name + $hostIp = if ($FileCheck.host) { [string]$FileCheck.host } else { $DefaultHost } + $path = [string]$FileCheck.path + $required = Get-AgentBooleanSetting $FileCheck "required" $true + $maxAgeMinutes = if ($FileCheck.maxAgeMinutes) { [double]$FileCheck.maxAgeMinutes } else { 1800 } + + if ($path -notmatch "^/backup/[A-Za-z0-9_.\/-]+$") { + return [pscustomobject]@{ name = $name; host = $hostIp; path = $path; ok = $false; severity = "critical"; reason = "invalid_backup_file_path" } + } + + $quotedPath = Quote-ShSingle $path + $command = "if [ -f $quotedPath ]; then stat -c '__STAT__=%Y|%y|%s|%n' $quotedPath; else echo __MISSING_FILE__=$quotedPath; fi" + $readback = Invoke-HostSshText $hostIp $command 15 1 + $unix = $null + $size = $null + if ($readback.ok) { + $statMatch = [regex]::Match($readback.output, "__STAT__=([0-9]+)\|([^|]+)\|(\d+)\|(.+)") + if ($statMatch.Success) { + $unix = [double]$statMatch.Groups[1].Value + $size = [int64]$statMatch.Groups[3].Value + } + } + + $ageMinutes = if ($null -ne $unix) { Get-AgentUnixAgeMinutes $unix } else { $null } + $severity = "ok" + $reason = "fresh" + if (-not $readback.ok) { + $severity = if ($required) { "critical" } else { "warning" } + $reason = "backup_file_readback_failed" + } elseif ($readback.output -match "__MISSING_FILE__" -or $null -eq $unix) { + $severity = if ($required) { "critical" } else { "warning" } + $reason = "backup_file_missing" + } elseif ($ageMinutes -gt $maxAgeMinutes) { + $severity = if ($required) { "critical" } else { "warning" } + $reason = "backup_file_stale" + } + + $result = [pscustomobject]@{ + name = $name + host = $hostIp + path = $path + required = [bool]$required + ok = [bool]($severity -eq "ok") + severity = $severity + reason = $reason + sizeBytes = $size + lastWriteTime = if ($null -ne $unix) { Convert-AgentUnixToIso $unix } else { $null } + ageMinutes = $ageMinutes + maxAgeMinutes = $maxAgeMinutes + route = $readback.route + exitCode = $readback.exitCode + } + Write-AgentLog "backup file=$name severity=$severity reason=$reason ageMinutes=$ageMinutes" + if ($severity -in @("warning", "critical")) { + Record-AgentEvent "backup_$reason" $severity "file=$name host=$hostIp ageMinutes=$ageMinutes maxAgeMinutes=$maxAgeMinutes path=$path" $result -Alert + } + $result +} + +function Test-AgentBackupCronPattern { + param( + [string]$Pattern, + [string]$DefaultHost + ) + + $quotedPattern = Quote-ShSingle $Pattern + $command = "crontab -l 2>/dev/null | grep -F -- $quotedPattern >/dev/null && echo __PRESENT__ || echo __MISSING__" + $readback = Invoke-HostSshText $DefaultHost $command 8 2 + $present = [bool]($readback.ok -and $readback.output -match "__PRESENT__") + $result = [pscustomobject]@{ + host = $DefaultHost + pattern = $Pattern + ok = $present + severity = if ($present) { "ok" } else { "critical" } + reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" } + route = $readback.route + exitCode = $readback.exitCode + } + Write-AgentLog "backup cron pattern=$Pattern ok=$present" + if (-not $present) { + Record-AgentEvent "backup_cron_pattern_missing" "critical" "host=$DefaultHost pattern=$Pattern" $result -Alert + } + $result +} + +function Test-AgentBackupCronPatterns { + param( + [string[]]$Patterns, + [string]$DefaultHost + ) + + $readback = Invoke-HostSshText $DefaultHost "crontab -l 2>/dev/null" 12 2 + $results = @() + if (-not $readback.ok) { + foreach ($pattern in $Patterns) { + $result = [pscustomobject]@{ + host = $DefaultHost + pattern = $pattern + ok = $false + severity = "warning" + reason = "cron_readback_failed" + route = $readback.route + exitCode = $readback.exitCode + } + Record-AgentEvent "backup_cron_readback_failed" "warning" "host=$DefaultHost pattern=$pattern" $result -Alert + $results += $result + } + return $results + } + + $crontab = [string]$readback.output + foreach ($pattern in $Patterns) { + $present = [bool]($crontab -match [regex]::Escape($pattern)) + $result = [pscustomobject]@{ + host = $DefaultHost + pattern = $pattern + ok = $present + severity = if ($present) { "ok" } else { "critical" } + reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" } + route = $readback.route + exitCode = $readback.exitCode + } + Write-AgentLog "backup cron pattern=$pattern ok=$present" + if (-not $present) { + Record-AgentEvent "backup_cron_pattern_missing" "critical" "host=$DefaultHost pattern=$pattern" $result -Alert + } + $results += $result + } + + $results +} + +function Test-AgentBackupHealth { + $backupConfig = Get-AgentBackupHealthConfig + $targets = @() + foreach ($target in @($backupConfig.targets)) { + $targets += Test-AgentBackupSnapshotTarget $target $backupConfig.host + } + + $files = @() + foreach ($fileCheck in @($backupConfig.fileChecks)) { + $files += Test-AgentBackupFileCheck $fileCheck $backupConfig.host + } + + $cron = Test-AgentBackupCronPatterns @($backupConfig.cronPatterns) $backupConfig.host + + $targetCritical = @($targets | Where-Object { $_.severity -eq "critical" }).Count + $targetWarning = @($targets | Where-Object { $_.severity -eq "warning" }).Count + $fileCritical = @($files | Where-Object { $_.severity -eq "critical" }).Count + $fileWarning = @($files | Where-Object { $_.severity -eq "warning" }).Count + $cronCritical = @($cron | Where-Object { $_.severity -eq "critical" }).Count + $cronWarning = @($cron | Where-Object { $_.severity -eq "warning" }).Count + $critical = $targetCritical + $fileCritical + $cronCritical + $warning = $targetWarning + $fileWarning + $cronWarning + $severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } + + $summary = [pscustomobject]@{ + targetCritical = $targetCritical + targetWarning = $targetWarning + fileCritical = $fileCritical + fileWarning = $fileWarning + cronCritical = $cronCritical + cronWarning = $cronWarning + targetCount = @($targets).Count + fileCheckCount = @($files).Count + cronPatternCount = @($cron).Count + } + if ($severity -in @("warning", "critical")) { + Record-AgentEvent "backup_health_degraded" $severity "severity=$severity targetCritical=$targetCritical fileCritical=$fileCritical cronCritical=$cronCritical" $summary -Alert + } else { + Write-AgentLog "backup_health ok targets=$($summary.targetCount) files=$($summary.fileCheckCount) cron=$($summary.cronPatternCount)" + } + + [pscustomobject]@{ + ok = [bool]($severity -eq "ok") + severity = $severity + host = $backupConfig.host + targets = $targets + files = $files + cron = $cron + summary = $summary + } +} + +function Convert-AgentHtml { + param([object]$Value) + [System.Net.WebUtility]::HtmlEncode([string]$Value) +} + +function Get-AgentLatestEvidenceSummary { + param( + [string]$Name, + [string]$Pattern + ) + + $latest = Get-AgentLatestEvidence $Pattern + [pscustomobject]@{ + name = $Name + exists = $latest.exists + file = $latest.fileName + path = $latest.path + ageMinutes = $latest.ageMinutes + parseError = $latest.parseError + } +} + +function Convert-AgentSafeKey { + param([string]$Value) + $safe = [regex]::Replace(([string]$Value), "[^A-Za-z0-9_.:-]", "_") + if (-not $safe) { return "unknown" } + if ($safe.Length -gt 140) { return $safe.Substring(0, 140) } + return $safe +} + +function Get-AgentProblemStats { + $stateDir = Join-Path $Config.agentRoot "state" + $statsPath = Join-Path $stateDir "problem-counts.json" + if (-not (Test-Path $statsPath)) { + return [pscustomobject]@{ + path = $statsPath + totalEvents = 0 + totalResolved = 0 + totalFailed = 0 + problems = @() + } + } + try { + $data = Get-Content $statsPath -Raw | ConvertFrom-Json + return [pscustomobject]@{ + path = $statsPath + totalEvents = if ($data.totalEvents) { [int]$data.totalEvents } else { 0 } + totalResolved = if ($data.totalResolved) { [int]$data.totalResolved } else { 0 } + totalFailed = if ($data.totalFailed) { [int]$data.totalFailed } else { 0 } + problems = @($data.problems) + } + } catch { + Write-AgentLog "problem_stats_parse_failed path=$statsPath error=$($_.Exception.Message)" + return [pscustomobject]@{ + path = $statsPath + totalEvents = 0 + totalResolved = 0 + totalFailed = 0 + problems = @() + parseError = $_.Exception.Message + } + } +} + +function Update-AgentProblemKnowledge { + param([object]$Result) + + $stateDir = Join-Path $Config.agentRoot "state" + $playbookDir = Join-Path $Config.agentRoot "playbooks" + $ragDir = Join-Path $playbookDir "rag-import" + New-Item -ItemType Directory -Force -Path $stateDir, $playbookDir, $ragDir | Out-Null + + $statsPath = Join-Path $stateDir "problem-counts.json" + $eventsPath = Join-Path $playbookDir "agent99-incident-records.jsonl" + $kmPath = Join-Path $playbookDir "agent99-incident-km.md" + $ragPath = Join-Path $ragDir "agent99-incidents.jsonl" + + $source = if ($Result.source) { [string]$Result.source } else { "agent99" } + $kind = if ($Result.alertKind) { [string]$Result.alertKind } elseif ($Result.reason) { [string]$Result.reason } else { [string]$Result.mode } + $service = if ($Result.alertService) { [string]$Result.alertService } else { "agent99" } + $hostName = if ($Result.alertHost) { [string]$Result.alertHost } else { "" } + $severity = if ($Result.alertSeverity) { [string]$Result.alertSeverity } elseif ($Result.ok) { "info" } else { "critical" } + $key = Convert-AgentSafeKey (($source, $service, $hostName, $kind | Where-Object { $_ }) -join "|") + $now = Get-Date -Format o + $statusText = if ($Result.ok) { "resolved" } else { "failed" } + + $event = [pscustomobject]@{ + timestamp = $now + problemKey = $key + status = $statusText + commandId = $Result.id + source = $source + reason = $Result.reason + alertId = $Result.alertId + alertKind = $kind + alertSeverity = $severity + service = $service + host = $hostName + mode = $Result.mode + instruction = $Result.instruction + ok = [bool]$Result.ok + exitCode = $Result.exitCode + evidence = $Result.evidence + } + + ($event | ConvertTo-Json -Depth 8 -Compress) | Add-Content -Path $eventsPath -Encoding UTF8 + ($event | ConvertTo-Json -Depth 8 -Compress) | Add-Content -Path $ragPath -Encoding UTF8 + + $stats = Get-AgentProblemStats + $problems = @($stats.problems) + $existing = $problems | Where-Object { $_.key -eq $key } | Select-Object -First 1 + if (-not $existing) { + $existing = [pscustomobject]@{ + key = $key + source = $source + kind = $kind + service = $service + host = $hostName + severity = $severity + occurrences = 0 + resolved = 0 + failed = 0 + lastStatus = $null + firstSeenAt = $now + lastSeenAt = $null + lastResolvedAt = $null + lastEvidence = $null + lastMode = $null + lastInstruction = $null + } + $problems += $existing + } + + $existing.occurrences = [int]$existing.occurrences + 1 + if ($Result.ok) { + $existing.resolved = [int]$existing.resolved + 1 + $existing.lastResolvedAt = $now + } else { + $existing.failed = [int]$existing.failed + 1 + } + $existing.lastStatus = $statusText + $existing.lastSeenAt = $now + $existing.lastEvidence = $Result.evidence + $existing.lastMode = $Result.mode + $existing.lastInstruction = $Result.instruction + $existing.severity = $severity + + $newTotalEvents = [int]$stats.totalEvents + 1 + $newTotalResolved = [int]$stats.totalResolved + $(if ($Result.ok) { 1 } else { 0 }) + $newTotalFailed = [int]$stats.totalFailed + $(if ($Result.ok) { 0 } else { 1 }) + + $updated = [pscustomobject]@{ + generatedAt = $now + totalEvents = $newTotalEvents + totalResolved = $newTotalResolved + totalFailed = $newTotalFailed + problems = @($problems | Sort-Object @{ Expression = { [int]$_.occurrences }; Descending = $true }, @{ Expression = { $_.lastSeenAt }; Descending = $true }) + } + $updated | ConvertTo-Json -Depth 10 | Set-Content -Path $statsPath -Encoding UTF8 + + $rows = @($updated.problems | Select-Object -First 50 | ForEach-Object { + "- key=$($_.key); occurrences=$($_.occurrences); resolved=$($_.resolved); failed=$($_.failed); lastStatus=$($_.lastStatus); lastMode=$($_.lastMode); evidence=$($_.lastEvidence)" + }) + $km = @( + "# Agent99 Incident KM", + "", + "Last updated: $now", + "", + "## Totals", + "", + "- totalEvents: $($updated.totalEvents)", + "- totalResolved: $($updated.totalResolved)", + "- totalFailed: $($updated.totalFailed)", + "", + "## Recurring Problems", + "", + ($rows -join "`n") + ) -join "`n" + Set-Content -Path $kmPath -Value $km -Encoding UTF8 + + [pscustomobject]@{ + key = $key + occurrences = $existing.occurrences + resolved = $existing.resolved + failed = $existing.failed + lastStatus = $existing.lastStatus + statsPath = $statsPath + eventsPath = $eventsPath + kmPath = $kmPath + ragPath = $ragPath + } +} + +function Get-AgentDashboardSummary { + $status = Get-AgentLatestEvidence "agent99-Status-*.json" + $perf = Get-AgentLatestEvidence "agent99-Perf-*.json" + $self = Get-AgentLatestEvidence "agent99-SelfCheck-*.json" + $backup = Get-AgentLatestEvidence "agent99-BackupCheck-*.json" + $recover = Get-AgentLatestEvidence "agent99-Recover-*.json" + + $taskNames = @( + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" + ) + $tasks = @() + foreach ($taskName in $taskNames) { + try { + $task = Get-ScheduledTask -TaskName $taskName -ErrorAction Stop + $info = Get-ScheduledTaskInfo -TaskName $taskName -ErrorAction Stop + $tasks += [pscustomobject]@{ + name = $taskName + exists = $true + state = [string]$task.State + lastTaskResult = $info.LastTaskResult + lastRunTime = if ($info.LastRunTime) { $info.LastRunTime.ToString("yyyy-MM-dd HH:mm:ss") } else { $null } + nextRunTime = if ($info.NextRunTime) { $info.NextRunTime.ToString("yyyy-MM-dd HH:mm:ss") } else { $null } + } + } catch { + $tasks += [pscustomobject]@{ + name = $taskName + exists = $false + state = "missing" + lastTaskResult = $null + lastRunTime = $null + nextRunTime = $null + } + } + } + + $queueDir = Join-Path $Config.agentRoot "queue" + if (-not (Test-Path $queueDir)) { New-Item -ItemType Directory -Force -Path $queueDir | Out-Null } + $pending = @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" }) + $processedDir = Join-Path $queueDir "processed" + $failedDir = Join-Path $queueDir "failed" + $processedCount = if (Test-Path $processedDir) { @(Get-ChildItem -Path $processedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } + $failedCount = if (Test-Path $failedDir) { @(Get-ChildItem -Path $failedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } + $alertIncomingDir = Join-Path $Config.agentRoot "alerts\incoming" + $alertProcessedDir = Join-Path $Config.agentRoot "alerts\processed" + $alertFailedDir = Join-Path $Config.agentRoot "alerts\failed" + $alertPendingCount = if (Test-Path $alertIncomingDir) { @(Get-ChildItem -Path $alertIncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } + $alertProcessedCount = if (Test-Path $alertProcessedDir) { @(Get-ChildItem -Path $alertProcessedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } + $alertFailedCount = if (Test-Path $alertFailedDir) { @(Get-ChildItem -Path $alertFailedDir -Filter "*.json" -File -ErrorAction SilentlyContinue).Count } else { 0 } + $problemStats = Get-AgentProblemStats + + [pscustomobject]@{ + timestamp = (Get-Date -Format o) + computer = $env:COMPUTERNAME + user = $env:USERNAME + agentRoot = $Config.agentRoot + evidenceDir = $EvidenceDir + queue = [pscustomobject]@{ + path = $queueDir + pending = $pending.Count + processed = $processedCount + failed = $failedCount + } + alerts = [pscustomobject]@{ + incomingPath = $alertIncomingDir + pending = $alertPendingCount + processed = $alertProcessedCount + failed = $alertFailedCount + } + problemCounts = [pscustomobject]@{ + path = $problemStats.path + totalEvents = $problemStats.totalEvents + totalResolved = $problemStats.totalResolved + totalFailed = $problemStats.totalFailed + top = @($problemStats.problems | Select-Object -First 10) + } + tasks = $tasks + evidence = @( + Get-AgentLatestEvidenceSummary "status" "agent99-Status-*.json" + Get-AgentLatestEvidenceSummary "performance" "agent99-Perf-*.json" + Get-AgentLatestEvidenceSummary "self_health" "agent99-SelfCheck-*.json" + Get-AgentLatestEvidenceSummary "backup" "agent99-BackupCheck-*.json" + Get-AgentLatestEvidenceSummary "startup_recovery" "agent99-Recover-*.json" + ) + public = if ($status.exists -and -not $status.parseError -and $status.data) { @($status.data.public | Select-Object url,status,non502,ok) } else { @() } + hosts = if ($status.exists -and -not $status.parseError -and $status.data) { @($status.data.hosts | Select-Object host,ping) } else { @() } + performance = if ($perf.exists -and -not $perf.parseError -and $perf.data) { @($perf.data.performance | Select-Object host,severity,loadPerCore,memAvailablePercent,diskUsedPercent,route) } else { @() } + selfHealth = if ($self.exists -and -not $self.parseError -and $self.data) { $self.data.selfHealth.summary } else { $null } + backupHealth = if ($backup.exists -and -not $backup.parseError -and $backup.data) { $backup.data.backupHealth.summary } else { $null } + latestSelfHealthSeverity = if ($self.exists -and -not $self.parseError -and $self.data) { $self.data.selfHealth.severity } else { "unknown" } + latestBackupSeverity = if ($backup.exists -and -not $backup.parseError -and $backup.data) { $backup.data.backupHealth.severity } else { "unknown" } + } +} + +function Write-AgentDashboard { + param([object]$Summary) + + $dashboardDir = Join-Path $Config.agentRoot "dashboard" + New-Item -ItemType Directory -Force -Path $dashboardDir | Out-Null + $jsonPath = Join-Path $dashboardDir "agent99-control-center.latest.json" + $htmlPath = Join-Path $dashboardDir "index.html" + $Summary | ConvertTo-Json -Depth 12 | Set-Content -Path $jsonPath -Encoding UTF8 + + $taskRows = foreach ($task in @($Summary.tasks)) { + "$(Convert-AgentHtml $task.name)$(Convert-AgentHtml $task.state)$(Convert-AgentHtml $task.lastTaskResult)$(Convert-AgentHtml $task.lastRunTime)$(Convert-AgentHtml $task.nextRunTime)" + } + $hostRows = foreach ($hostItem in @($Summary.hosts)) { + "$(Convert-AgentHtml $hostItem.host)$(Convert-AgentHtml $hostItem.ping)" + } + $publicRows = foreach ($route in @($Summary.public)) { + "$(Convert-AgentHtml $route.url)$(Convert-AgentHtml $route.status)$(Convert-AgentHtml $route.non502)$(Convert-AgentHtml $route.ok)" + } + $perfRows = foreach ($perfItem in @($Summary.performance)) { + "$(Convert-AgentHtml $perfItem.host)$(Convert-AgentHtml $perfItem.severity)$(Convert-AgentHtml $perfItem.loadPerCore)$(Convert-AgentHtml $perfItem.memAvailablePercent)$(Convert-AgentHtml $perfItem.diskUsedPercent)$(Convert-AgentHtml $perfItem.route)" + } + $evidenceRows = foreach ($ev in @($Summary.evidence)) { + "$(Convert-AgentHtml $ev.name)$(Convert-AgentHtml $ev.file)$(Convert-AgentHtml $ev.ageMinutes)$(Convert-AgentHtml $ev.parseError)" + } + $problemRows = foreach ($problem in @($Summary.problemCounts.top)) { + "$(Convert-AgentHtml $problem.key)$(Convert-AgentHtml $problem.occurrences)$(Convert-AgentHtml $problem.resolved)$(Convert-AgentHtml $problem.failed)$(Convert-AgentHtml $problem.lastStatus)$(Convert-AgentHtml $problem.lastMode)" + } + + $html = @" + + + + + + Agent99 Control Center + + + +

Agent99 Control Center

+
Last update: $(Convert-AgentHtml $Summary.timestamp) on $(Convert-AgentHtml $Summary.computer) as $(Convert-AgentHtml $Summary.user). Auto-refreshes every 30 seconds.
+
+
Self Health
$(Convert-AgentHtml $Summary.latestSelfHealthSeverity)
+
Backup Health
$(Convert-AgentHtml $Summary.latestBackupSeverity)
+
Queue Pending
$(Convert-AgentHtml $Summary.queue.pending)
+
Alerts Pending
$(Convert-AgentHtml $Summary.alerts.pending)
+
Problem Events
$(Convert-AgentHtml $Summary.problemCounts.totalEvents)
+
Resolved
$(Convert-AgentHtml $Summary.problemCounts.totalResolved)
+
Failed
$(Convert-AgentHtml $Summary.problemCounts.totalFailed)
+
+

Agent root: $(Convert-AgentHtml $Summary.agentRoot)
Queue: $(Convert-AgentHtml $Summary.queue.path)
Alerts: $(Convert-AgentHtml $Summary.alerts.incomingPath)
Problem counts: $(Convert-AgentHtml $Summary.problemCounts.path)
Evidence: $(Convert-AgentHtml $Summary.evidenceDir)

+

Problem Counts

+ $($problemRows -join "`n")
ProblemOccurrencesResolvedFailedLast StatusLast Mode
+

Scheduled Tasks

+ $($taskRows -join "`n")
TaskStateLast ResultLast RunNext Run
+

Hosts

+ $($hostRows -join "`n")
HostPing
+

Public Routes

+ $($publicRows -join "`n")
URLStatusNon-502OK
+

Performance

+ $($perfRows -join "`n")
HostSeverityLoad/CoreMem Avail %Disk Used %Route
+

Evidence

+ $($evidenceRows -join "`n")
NameFileAge MinutesParse Error
+ + +"@ + Set-Content -Path $htmlPath -Value $html -Encoding UTF8 + [pscustomobject]@{ + dashboardDir = $dashboardDir + htmlPath = $htmlPath + jsonPath = $jsonPath + } +} + +function Ensure-AgentDesktopShortcuts { + param([string]$DashboardPath) + + $created = @() + $targets = @( + [pscustomobject]@{ name = "Agent99 Control Center.lnk"; target = $DashboardPath }, + [pscustomobject]@{ name = "Agent99 Submit Request.lnk"; target = (Join-Path $Config.agentRoot "agent99-submit-request.cmd") }, + [pscustomobject]@{ name = "Agent99 Queue.lnk"; target = (Join-Path $Config.agentRoot "queue") }, + [pscustomobject]@{ name = "Agent99 Evidence.lnk"; target = $EvidenceDir } + ) + $desktopDirs = @( + [Environment]::GetFolderPath("CommonDesktopDirectory"), + [Environment]::GetFolderPath("Desktop") + ) | Where-Object { $_ } | Select-Object -Unique + + foreach ($desktop in $desktopDirs) { + if (-not (Test-Path $desktop)) { continue } + foreach ($item in $targets) { + $shortcutPath = Join-Path $desktop $item.name + try { + $shell = New-Object -ComObject WScript.Shell + $shortcut = $shell.CreateShortcut($shortcutPath) + $shortcut.TargetPath = $item.target + $shortcut.WorkingDirectory = Split-Path -Parent $item.target + $shortcut.Description = "Agent99 local control and evidence shortcut" + $shortcut.Save() + $created += $shortcutPath + } catch { + Write-AgentLog "desktop_shortcut_failed path=$shortcutPath error=$($_.Exception.Message)" + } + } + } + $created +} + +function Invoke-AgentQueuedCommands { + $queueDir = Join-Path $Config.agentRoot "queue" + $processedDir = Join-Path $queueDir "processed" + $failedDir = Join-Path $queueDir "failed" + New-Item -ItemType Directory -Force -Path $queueDir, $processedDir, $failedDir | Out-Null + + $allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness") + $commands = @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notmatch "^(processed|failed|running)-" } | + Sort-Object LastWriteTime | + Select-Object -First 3) + $results = @() + foreach ($file in $commands) { + $command = $null + try { + $command = Get-Content $file.FullName -Raw | ConvertFrom-Json + } catch { + $target = Join-Path $failedDir ("failed-" + $file.Name) + Move-Item -Force $file.FullName $target + $results += [pscustomobject]@{ id = $file.BaseName; ok = $false; reason = "invalid_json"; file = $target } + continue + } + + $id = if ($command.id) { [string]$command.id } else { $file.BaseName } + $modeName = if ($command.mode) { [string]$command.mode } else { "" } + $controlled = if ($command.PSObject.Properties["controlledApply"]) { [bool]$command.controlledApply } else { $false } + $commandSource = if ($command.PSObject.Properties["source"]) { [string]$command.source } else { "queue" } + $commandReason = if ($command.PSObject.Properties["reason"]) { [string]$command.reason } else { "operator_instruction" } + $commandInstruction = if ($command.PSObject.Properties["instruction"]) { [string]$command.instruction } else { "" } + $alertId = if ($command.PSObject.Properties["alertId"]) { [string]$command.alertId } else { $null } + $alertSource = if ($command.PSObject.Properties["alertSource"]) { [string]$command.alertSource } else { $null } + $alertKind = if ($command.PSObject.Properties["alertKind"]) { [string]$command.alertKind } else { $null } + $alertSeverity = if ($command.PSObject.Properties["alertSeverity"]) { [string]$command.alertSeverity } else { $null } + $alertService = if ($command.PSObject.Properties["alertService"]) { [string]$command.alertService } else { $null } + $alertHost = if ($command.PSObject.Properties["alertHost"]) { [string]$command.alertHost } else { $null } + $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" } + if ($modeName -notin $allowedModes) { + $target = Join-Path $failedDir ("failed-" + $file.Name) + Move-Item -Force $file.FullName $target + $result = [pscustomobject]@{ id = $id; mode = $modeName; source = $commandSource; instruction = $commandInstruction; ok = $false; reason = "mode_not_allowlisted"; file = $target } + Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=mode_not_allowlisted" $result -Alert + $results += $result + continue + } + + $runningPath = Join-Path $queueDir ("running-" + $file.Name) + Move-Item -Force $file.FullName $runningPath + $startTime = Get-Date + $launcher = Join-Path $Config.agentRoot "agent99-run.ps1" + $stdoutPath = Join-Path $EvidenceDir "agent99-command-$id.out" + $stderrPath = Join-Path $EvidenceDir "agent99-command-$id.err" + $args = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", $modeName) + if ($controlled) { $args += "-ControlledApply" } + $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru + $completed = $process.WaitForExit(600000) + if (-not $completed) { + $process.Kill() + $process.WaitForExit(2000) | Out-Null + } + $process.WaitForExit() | Out-Null + $process.Refresh() + $latest = Get-AgentLatestEvidence "agent99-$modeName-*.json" + $exitCode = if (-not $completed) { + -2 + } elseif ($null -ne $process.ExitCode) { + $process.ExitCode + } elseif ($latest.exists -and -not $latest.parseError) { + 0 + } else { + -3 + } + $result = [pscustomobject]@{ + id = $id + mode = $modeName + controlledApply = $controlled + source = $commandSource + reason = $commandReason + instruction = $commandInstruction + alertId = $alertId + alertSource = $alertSource + alertKind = $alertKind + alertSeverity = $alertSeverity + alertService = $alertService + alertHost = $alertHost + ok = [bool]($exitCode -eq 0) + exitCode = $exitCode + durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1) + evidence = $latest.path + stdout = $stdoutPath + stderr = $stderrPath + } + $problem = Update-AgentProblemKnowledge $result + $result | Add-Member -MemberType NoteProperty -Name problem -Value $problem -Force + $donePath = Join-Path $processedDir ("processed-$id.json") + $result | ConvertTo-Json -Depth 8 | Set-Content -Path $donePath -Encoding UTF8 + Remove-Item $runningPath -Force -ErrorAction SilentlyContinue | Out-Null + $recordData = $result | Select-Object * + if ($replyChatId) { + $recordData | Add-Member -MemberType NoteProperty -Name replyChatId -Value $replyChatId -Force + } + Record-AgentEvent "operator_command_result" $(if ($exitCode -eq 0) { "info" } else { "critical" }) "id=$id mode=$modeName exitCode=$exitCode evidence=$($latest.path)" $recordData -Alert + $results += $result + } + $results +} + +function Invoke-AgentControlTick { + $processed = Invoke-AgentQueuedCommands + $summary = Get-AgentDashboardSummary + $dashboard = Write-AgentDashboard $summary + $shortcuts = Ensure-AgentDesktopShortcuts $dashboard.htmlPath + $pendingCount = if ($summary.queue) { $summary.queue.pending } else { 0 } + $result = [pscustomobject]@{ + ok = $true + dashboardPath = $dashboard.htmlPath + dashboardJson = $dashboard.jsonPath + shortcutCount = @($shortcuts).Count + shortcuts = $shortcuts + processedCount = @($processed).Count + processed = $processed + pendingCount = $pendingCount + } + Record-AgentEvent "agent_control_tick" "info" "dashboard=$($dashboard.htmlPath) processed=$(@($processed).Count) pending=$pendingCount" $result -NoAlert + $result +} + +function Invoke-AgentProviderFreshnessTriage { + $providerConfig = if ($Config.PSObject.Properties["providerFreshness"]) { $Config.providerFreshness } else { $null } + $lookbackMinutes = if ($providerConfig -and $providerConfig.PSObject.Properties["lookbackMinutes"]) { [int]$providerConfig.lookbackMinutes } else { 120 } + $since = (Get-Date).AddMinutes(-1 * $lookbackMinutes) + $candidateId = "provider-freshness-" + (Get-Date -Format "yyyyMMdd-HHmmss") + $candidateDir = Join-Path $Config.agentRoot "state\provider-freshness-candidates" + New-Item -ItemType Directory -Force -Path $candidateDir | Out-Null + $candidatePath = Join-Path $candidateDir "$candidateId.json" + + $patterns = @( + "provider_freshness_signal", + "source_provider_freshness_triage", + "raw_signal_normalized", + "controlled_runtime_candidate", + "freshness", + "last_seen", + "last seen", + "ingestion" + ) + $scanDirs = @( + (Join-Path $Config.agentRoot "alerts\incoming"), + (Join-Path $Config.agentRoot "alerts\processed"), + (Join-Path $Config.agentRoot "queue"), + (Join-Path $Config.agentRoot "queue\processed"), + (Join-Path $Config.agentRoot "requests") + ) | Where-Object { Test-Path $_ } | Select-Object -Unique + + $hits = @() + foreach ($dir in $scanDirs) { + $files = @(Get-ChildItem -LiteralPath $dir -Recurse -File -ErrorAction SilentlyContinue | + Where-Object { $_.LastWriteTime -ge $since -and $_.Length -lt 2097152 }) + foreach ($file in $files) { + $matchedPatterns = @() + foreach ($pattern in $patterns) { + if (Select-String -LiteralPath $file.FullName -Pattern $pattern -SimpleMatch -Quiet -ErrorAction SilentlyContinue) { + $matchedPatterns += $pattern + } + } + if (@($matchedPatterns).Count -gt 0) { + $excerpt = "" + $firstHit = Select-String -LiteralPath $file.FullName -Pattern $matchedPatterns[0] -SimpleMatch -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($firstHit) { + $excerpt = [string]$firstHit.Line + $excerpt = $excerpt -replace '("?(telegram|token|chat|authorization|password)[^,}\r\n]*"?\s*:\s*)"[^"]+"', '$1"[redacted]"' + if ($excerpt.Length -gt 500) { $excerpt = $excerpt.Substring(0, 500) + "...(truncated)" } + } + $hits += [pscustomobject]@{ + path = $file.FullName + lastWrite = $file.LastWriteTime.ToString("o") + patterns = @($matchedPatterns) + excerpt = $excerpt + } + } + } + } + + $targets = @() + if ($providerConfig -and $providerConfig.targets) { + foreach ($target in @($providerConfig.targets)) { + $targets += [pscustomobject]@{ + name = if ($target.name) { [string]$target.name } else { "provider_freshness_signal" } + providerWindowMinutes = if ($target.PSObject.Properties["providerWindowMinutes"]) { $target.providerWindowMinutes } else { $null } + directReference = if ($target.directReference) { [string]$target.directReference } else { $null } + verifier = if ($target.verifier) { [string]$target.verifier } else { "agent99-provider-freshness-triage" } + } + } + } + if (@($targets).Count -eq 0) { + $targets += [pscustomobject]@{ + name = "provider_freshness_signal" + providerWindowMinutes = $null + directReference = $null + verifier = "agent99-provider-freshness-triage" + } + } + + $missingFields = @("providerWindow", "lastSeen", "directReference", "verifierReadback") + $status = if (@($hits).Count -gt 0) { "candidate_created_waiting_direct_readback" } else { "candidate_created_no_alert_body_seen" } + $candidate = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + candidateId = $candidateId + target = "provider_freshness_signal" + status = $status + lookbackMinutes = $lookbackMinutes + since = $since.ToString("o") + hitCount = @($hits).Count + hits = @($hits | Sort-Object lastWrite -Descending | Select-Object -First 20) + targets = $targets + missingFields = $missingFields + requiredVerifierReadback = @("provider_window", "last_seen", "direct_reference", "source_trace_or_sentry_signoz_ref") + noFalseGreenRisk = $true + allowedActions = @("create_candidate", "read_existing_evidence", "request_direct_reference", "write_km") + prohibitedActions = @("provider_switch", "paid_model_call", "env_change", "reload") + candidatePath = $candidatePath + } + $candidate | ConvertTo-Json -Depth 10 | Set-Content -Path $candidatePath -Encoding UTF8 + Write-AgentLog "provider_freshness candidate=$candidateId hits=$(@($hits).Count) status=$status path=$candidatePath" + Record-AgentEvent "provider_freshness_triage" "warning" "candidate=$candidateId hits=$(@($hits).Count) missing=$($missingFields -join ',') evidence=$candidatePath" $candidate -Alert + $candidate +} + +function Test-HostReachability { + $results = @() + foreach ($hostIp in $Config.hosts) { + $ping = Test-Connection -ComputerName $hostIp -Count 1 -Quiet -ErrorAction SilentlyContinue + Write-AgentLog "host host=$hostIp ping=$ping" + $results += [pscustomobject]@{ + host = $hostIp + ping = [bool]$ping + } + } + $results +} + +function Test-PublicRoutes { + $results = @() + foreach ($url in $Config.publicUrls) { + try { + $output = & curl.exe --location --head --max-time 12 $url 2>&1 + $text = $output -join "`n" + $matches = [regex]::Matches($text, "HTTP/\S+\s+(\d+)") + $status = if ($matches.Count -gt 0) { [int]$matches[$matches.Count - 1].Groups[1].Value } else { $null } + $non502 = ($null -ne $status -and $status -ne 502) + $ok = ($null -ne $status -and $status -ge 200 -and $status -lt 500) + Write-AgentLog "public url=$url status=$status non502=$non502 ok=$ok" + $results += [pscustomobject]@{ + url = $url + status = $status + non502 = $non502 + ok = $ok + } + } catch { + Write-AgentLog "public url=$url error=$($_.Exception.Message)" + $results += [pscustomobject]@{ + url = $url + status = $null + non502 = $false + ok = $false + } + } + } + $results +} + +function Test-AiServices { + $results = @() + foreach ($svc in @($Config.aiServices)) { + if (-not $svc.url) { continue } + $name = if ($svc.name) { [string]$svc.name } else { [string]$svc.url } + try { + $output = & curl.exe --max-time 12 -sS $svc.url 2>&1 + $text = ($output -join "`n").Trim() + $statusValue = $null + try { + $json = $text | ConvertFrom-Json + if ($json.status) { $statusValue = [string]$json.status } + } catch { + $statusValue = $null + } + $expectedStatus = if ($svc.expectedStatus) { [string]$svc.expectedStatus } else { "healthy" } + $ok = ($statusValue -eq $expectedStatus) + $severity = if ($ok) { "ok" } elseif ($svc.critical) { "critical" } else { "warning" } + $result = [pscustomobject]@{ + name = $name + url = $svc.url + host = $svc.host + ok = $ok + severity = $severity + status = $statusValue + expectedStatus = $expectedStatus + output = $text + } + Write-AgentLog "ai_service name=$name ok=$ok severity=$severity status=$statusValue url=$($svc.url)" + if (-not $ok) { + Record-AgentEvent "ai_service_unhealthy" $severity "name=$name status=$statusValue expected=$expectedStatus url=$($svc.url)" $result -Alert + } + $results += $result + } catch { + $severity = if ($svc.critical) { "critical" } else { "warning" } + $result = [pscustomobject]@{ + name = $name + url = $svc.url + host = $svc.host + ok = $false + severity = $severity + status = $null + expectedStatus = if ($svc.expectedStatus) { [string]$svc.expectedStatus } else { "healthy" } + output = $_.Exception.Message + } + Write-AgentLog "ai_service name=$name error=$($_.Exception.Message)" + Record-AgentEvent "ai_service_unhealthy" $severity "name=$name error=$($_.Exception.Message) url=$($svc.url)" $result -Alert + $results += $result + } + } + $results +} + +function Get-HarborState { + $inspectCommand = "docker inspect --format '{{.Name}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' harbor-core redis registry" + $inspect = Invoke-SshText "192.168.0.110" $inspectCommand 30 1 + $containerStates = @{} + foreach ($line in @(([string]$inspect.output) -split "`n")) { + $trimmed = $line.Trim() + if (-not $trimmed) { continue } + $parts = @($trimmed -split "\|") + if ($parts.Count -lt 3) { continue } + $name = $parts[0].Trim().TrimStart("/") + $containerStates[$name] = [pscustomobject]@{ + name = $name + status = $parts[1].Trim() + health = $parts[2].Trim() + } + } + $core = $containerStates["harbor-core"] + $redis = $containerStates["redis"] + $registry = $containerStates["registry"] + $coreLine = if ($core) { "$($core.name) $($core.status) $($core.health)" } else { "harbor-core missing" } + $redisLine = if ($redis) { "$($redis.name) $($redis.status) $($redis.health)" } else { "redis missing" } + $registryLine = if ($registry) { "$($registry.name) $($registry.status) $($registry.health)" } else { "registry missing" } + + $state = [pscustomobject]@{ + coreHealthy = ($core -and $core.status -eq "running" -and $core.health -eq "healthy") + redisUp = ($redis -and $redis.status -eq "running") + registryHealthy = ($registry -and $registry.status -eq "running" -and $registry.health -eq "healthy") + coreOutput = $coreLine + redisOutput = $redisLine + registryOutput = $registryLine + inspectExitCode = $inspect.exitCode + healthOutput = $inspect.output + inspectOutput = "[redacted: raw docker inspect omitted]" + } + Write-AgentLog "harbor coreHealthy=$($state.coreHealthy) redisUp=$($state.redisUp) registryHealthy=$($state.registryHealthy)" + $state +} + +function Get-AwoooState { + $nestedSsh = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $SshUser 192.168.0.120" + $readback = Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl get deploy,pods -n awoooi-prod" 60 1 + + $state = [pscustomobject]@{ + deployReady = ( + $readback.output -match "deployment.apps/awoooi-api\s+1/1" -and + $readback.output -match "deployment.apps/awoooi-web\s+2/2" -and + $readback.output -match "deployment.apps/awoooi-worker\s+1/1" + ) + imagePullFailure = ($readback.output -match "ImagePullBackOff|ErrImagePull") + crashFailure = ($readback.output -match "CrashLoopBackOff|Pending") + deployOutput = $readback.output + podOutput = $readback.output + readbackExitCode = $readback.exitCode + } + Write-AgentLog "awoooi deployReady=$($state.deployReady) imagePullFailure=$($state.imagePullFailure) crashFailure=$($state.crashFailure)" + $state +} + +function Find-Vmrun { + if ($Config.vmrunPath -and (Test-Path $Config.vmrunPath)) { + return $Config.vmrunPath + } + + foreach ($candidate in @( + "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe", + "C:\Program Files\VMware\VMware Workstation\vmrun.exe" + )) { + if (Test-Path $candidate) { return $candidate } + } + + return $null +} + +function Start-ConfiguredVMs { + $vmrun = Find-Vmrun + $results = @() + + if (-not $vmrun) { + Write-AgentLog "vmrun missing" + return @([pscustomobject]@{ ok = $false; reason = "vmrun_missing" }) + } + + foreach ($vm in $Config.vms) { + if (-not $vm.vmx) { + $results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "missing_vmx_path" } + continue + } + + $hostPing = $false + if ($vm.host) { + $hostPing = Test-Connection -ComputerName $vm.host -Count 1 -Quiet -ErrorAction SilentlyContinue + } + + if ($hostPing) { + $results += [pscustomobject]@{ name = $vm.name; ok = $true; skipped = $true; reason = "host_already_reachable" } + continue + } + + if (-not $ControlledApply) { + $results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "controlled_apply_required" } + continue + } + + Write-AgentLog "CONTROLLED_APPLY vm_start name=$($vm.name) vmx=$($vm.vmx)" + $output = & $vmrun start $vm.vmx nogui 2>&1 + $results += [pscustomobject]@{ + name = $vm.name + ok = ($LASTEXITCODE -eq 0) + skipped = $false + output = ($output -join "`n") + } + } + + $results +} + +function Repair-Harbor { + param([object]$Harbor) + if (-not $ControlledApply) { + Write-AgentLog "harbor repair skipped controlled_apply=false" + return $false + } + + $changed = $false + if (-not $Harbor.redisUp) { + Write-AgentLog "CONTROLLED_APPLY harbor restart redis" + Invoke-SshText "192.168.0.110" "docker restart redis" | Out-Null + $changed = $true + } + + if (-not $Harbor.coreHealthy) { + Write-AgentLog "CONTROLLED_APPLY harbor restart harbor-core" + Invoke-SshText "192.168.0.110" "docker restart harbor-core" | Out-Null + $changed = $true + } + + return $changed +} + +function Repair-Awooo { + param([object]$Awooo, [object]$Harbor) + if (-not $ControlledApply) { + Write-AgentLog "awoooi repair skipped controlled_apply=false" + return $false + } + if (-not $Harbor.coreHealthy) { + Write-AgentLog "awoooi repair skipped harbor_core_not_healthy" + return $false + } + if (-not $Awooo.imagePullFailure) { + Write-AgentLog "awoooi repair skipped no_image_pull_failure" + return $false + } + + Write-AgentLog "CONTROLLED_APPLY awoooi delete failed pods" + $nestedSsh = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $SshUser 192.168.0.120" + Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-api" 25 | Out-Null + Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-web" 25 | Out-Null + Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-worker" 25 | Out-Null + return $true +} + +Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" +Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert + +$vmResults = @() +$hostResults = @() +$harbor = $null +$awooo = $null +$public = @() +$aiServices = @() +$performance = @() +$loadShedding = @() +$selfHealth = $null +$backupHealth = $null +$providerFreshness = $null +$controlTick = $null + +if ($Mode -in @("Recover", "StartVMs")) { + $vmResults = Start-ConfiguredVMs +} + +if ($Mode -in @("Status", "Recover")) { + $hostResults = Test-HostReachability +} + +if ($Mode -in @("Status", "Recover", "HarborRepair")) { + $harbor = Get-HarborState +} + +if ($Mode -in @("Recover", "HarborRepair") -and $harbor) { + $harborChanged = Repair-Harbor $harbor + if ($harborChanged) { + Start-Sleep -Seconds 3 + $harbor = Get-HarborState + } +} + +if ($Mode -in @("Status", "Recover", "AwoooRepair")) { + $awooo = Get-AwoooState +} + +if ($Mode -in @("Recover", "AwoooRepair") -and $awooo) { + if (-not $harbor) { $harbor = Get-HarborState } + $awoooChanged = Repair-Awooo $awooo $harbor + if ($awoooChanged) { + Start-Sleep -Seconds 3 + $awooo = Get-AwoooState + } +} + +if ($Mode -in @("Status", "Recover", "PublicSmoke")) { + $public = Test-PublicRoutes +} + +if ($Mode -in @("Status", "Recover", "Perf")) { + $aiServices = Test-AiServices +} + +if ($Mode -in @("Status", "Recover", "Perf", "LoadShed")) { + $suppressPerfAlerts = [bool]($ControlledApply -and $Mode -in @("Perf", "LoadShed")) + $performance = Get-HostPerformance -SuppressAlerts:$suppressPerfAlerts +} + +if ($Mode -in @("Perf", "LoadShed")) { + $loadShedding = Invoke-LoadShedding $performance + if ($ControlledApply) { + Record-PerformanceIssuesWithoutRemediation $performance $loadShedding + } +} + +if ($Mode -in @("SelfCheck")) { + $selfHealth = Test-AgentSelfHealth +} + +if ($Mode -in @("BackupCheck")) { + $backupHealth = Test-AgentBackupHealth +} + +if ($Mode -in @("ProviderFreshness")) { + $providerFreshness = Invoke-AgentProviderFreshnessTriage +} + +if ($Mode -in @("ControlTick")) { + $controlTick = Invoke-AgentControlTick +} + +$publicFailures = @($public | Where-Object { $_ -and -not $_.ok }).Count +$aiFailures = @($aiServices | Where-Object { $_ -and -not $_.ok }).Count +$perfCritical = @($performance | Where-Object { $_ -and $_.severity -eq "critical" }).Count +$perfWarning = @($performance | Where-Object { $_ -and $_.severity -eq "warning" }).Count +$selfHealthCritical = if ($selfHealth -and $selfHealth.severity -eq "critical") { 1 } else { 0 } +$selfHealthWarning = if ($selfHealth -and $selfHealth.severity -eq "warning") { 1 } else { 0 } +$backupCritical = if ($backupHealth -and $backupHealth.severity -eq "critical") { 1 } else { 0 } +$backupWarning = if ($backupHealth -and $backupHealth.severity -eq "warning") { 1 } else { 0 } +$providerFreshnessWarning = if ($providerFreshness) { 1 } else { 0 } +$summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCritical -gt 0 -or $selfHealthCritical -gt 0 -or $backupCritical -gt 0) { "critical" } elseif ($perfWarning -gt 0 -or $selfHealthWarning -gt 0 -or $backupWarning -gt 0 -or $providerFreshnessWarning -gt 0) { "warning" } else { "info" } +Record-AgentEvent "agent_complete" $summarySeverity "mode=$Mode publicFailures=$publicFailures aiFailures=$aiFailures perfCritical=$perfCritical perfWarning=$perfWarning selfHealth=$($selfHealth.severity) backupHealth=$($backupHealth.severity) providerFreshness=$($providerFreshness.status) evidence=$jsonPath" $null -NoAlert + +$result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + mode = $Mode + controlledApply = [bool]$ControlledApply + vmResults = $vmResults + hosts = $hostResults + harbor = $harbor + awoooi = $awooo + public = $public + aiServices = $aiServices + performance = $performance + loadShedding = $loadShedding + selfHealth = $selfHealth + backupHealth = $backupHealth + providerFreshness = $providerFreshness + controlTick = $controlTick + events = $script:Events + telegram = $script:TelegramAttempts + evidenceLog = $logPath +} + +$result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath +Write-AgentLog "agent99_complete json=$jsonPath" + +exit 0 diff --git a/agent99-install-sre-alert-relay.ps1 b/agent99-install-sre-alert-relay.ps1 new file mode 100644 index 000000000..30b9f80d0 --- /dev/null +++ b/agent99-install-sre-alert-relay.ps1 @@ -0,0 +1,119 @@ +param( + [string]$AgentRoot = "C:\Wooo\Agent99", + [string]$StagingDir = "", + [string]$RelayPrefix = "http://+:8787/agent99/sre-alert/", + [string]$TokenEnv = "AGENT99_SRE_RELAY_TOKEN" +) + +$ErrorActionPreference = "Stop" + +if (-not $StagingDir) { + $StagingDir = Split-Path -Parent $MyInvocation.MyCommand.Path +} + +$BinDir = Join-Path $AgentRoot "bin" +$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json" +$PlaybookDir = Join-Path $AgentRoot "playbooks" +$EvidenceDir = Join-Path $AgentRoot "evidence" + +New-Item -ItemType Directory -Force -Path $BinDir, $PlaybookDir, $EvidenceDir | Out-Null + +foreach ($name in @("agent99-sre-alert-relay.ps1", "agent99-register-tasks.ps1", "agent99-bootstrap.ps1")) { + $src = Join-Path $StagingDir $name + if (-not (Test-Path $src)) { + throw "missing staged file: $name" + } + Copy-Item -Force $src (Join-Path $BinDir $name) +} + +foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.json")) { + $src = Join-Path $StagingDir $name + if (Test-Path $src) { + Copy-Item -Force $src (Join-Path $PlaybookDir $name) + } +} + +function Set-AgentProp { + param([object]$Object, [string]$Name, [object]$Value) + if ($Object.PSObject.Properties[$Name]) { + $Object.PSObject.Properties[$Name].Value = $Value + } else { + $Object | Add-Member -NotePropertyName $Name -NotePropertyValue $Value + } +} + +if (-not (Test-Path $ConfigPath)) { + throw "missing config: $ConfigPath" +} + +$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json +if (-not $config.PSObject.Properties["sreAlertRelay"]) { + $config | Add-Member -NotePropertyName "sreAlertRelay" -NotePropertyValue ([pscustomobject]@{}) +} +Set-AgentProp $config.sreAlertRelay "enabled" $true +Set-AgentProp $config.sreAlertRelay "prefix" $RelayPrefix +Set-AgentProp $config.sreAlertRelay "tokenEnv" $TokenEnv +Set-AgentProp $config.sreAlertRelay "allowedRemotePrefixes" @("127.", "::1", "192.168.0.") + +if (-not $config.PSObject.Properties["selfHealth"]) { + $config | Add-Member -NotePropertyName "selfHealth" -NotePropertyValue ([pscustomobject]@{}) +} + +$tasks = @() +if ($config.selfHealth.PSObject.Properties["scheduledTasks"]) { + $tasks = @($config.selfHealth.scheduledTasks) +} +foreach ($taskName in @( + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-SRE-Alert-Relay", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" +)) { + if ($tasks -notcontains $taskName) { + $tasks += $taskName + } +} +Set-AgentProp $config.selfHealth "scheduledTasks" $tasks + +$freshness = @() +if ($config.selfHealth.PSObject.Properties["evidenceFreshness"]) { + $freshness = @($config.selfHealth.evidenceFreshness) +} +if (-not ($freshness | Where-Object { $_.name -eq "sre_alert_relay" } | Select-Object -First 1)) { + $freshness += [pscustomobject]@{ + name = "sre_alert_relay" + pattern = "agent99-SreAlertRelay-start-*.json" + maxAgeMinutes = 1440 + required = $false + } +} +Set-AgentProp $config.selfHealth "evidenceFreshness" $freshness +$config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $ConfigPath + +& (Join-Path $BinDir "agent99-register-tasks.ps1") -AgentRoot $AgentRoot | Out-Null +Start-ScheduledTask -TaskName "Wooo-Agent99-SRE-Alert-Relay" +Start-Sleep -Seconds 4 + +$task = Get-ScheduledTask -TaskName "Wooo-Agent99-SRE-Alert-Relay" -ErrorAction Stop +$info = Get-ScheduledTaskInfo -TaskName "Wooo-Agent99-SRE-Alert-Relay" -ErrorAction Stop +$listener = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue) +$latestEvidence = Get-ChildItem -Path $EvidenceDir -Filter "agent99-SreAlertRelay-start-*.json" -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 + +[pscustomobject]@{ + ok = $true + taskState = [string]$task.State + lastTaskResult = $info.LastTaskResult + listenerCount = $listener.Count + listenerLocal = @($listener | Select-Object -First 3 | ForEach-Object { "$($_.LocalAddress):$($_.LocalPort)" }) + latestEvidence = if ($latestEvidence) { $latestEvidence.FullName } else { "" } + configRelayEnabled = [bool]$config.sreAlertRelay.enabled + configTokenEnv = [string]$config.sreAlertRelay.tokenEnv + scheduledTaskListed = @($config.selfHealth.scheduledTasks) -contains "Wooo-Agent99-SRE-Alert-Relay" +} | ConvertTo-Json -Depth 6 -Compress diff --git a/agent99-register-tasks.ps1 b/agent99-register-tasks.ps1 new file mode 100644 index 000000000..f54a62d15 --- /dev/null +++ b/agent99-register-tasks.ps1 @@ -0,0 +1,182 @@ +param( + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" + +$launcher = Join-Path $AgentRoot "agent99-run.ps1" +if (-not (Test-Path $launcher)) { + throw "Agent99 launcher not found: $launcher" +} + +$ps = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" +$principalUser = if ($env:COMPUTERNAME) { "$env:COMPUTERNAME\$env:USERNAME" } else { $env:USERNAME } +$principal = New-ScheduledTaskPrincipal -UserId $principalUser -LogonType S4U -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable ` + -MultipleInstances IgnoreNew ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 10) +$serviceSettings = New-ScheduledTaskSettingsSet ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable ` + -MultipleInstances IgnoreNew ` + -ExecutionTimeLimit ([TimeSpan]::Zero) + +$recoverArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode Recover -ControlledApply" +$recoverAction = New-ScheduledTaskAction -Execute $ps -Argument $recoverArgs +$startupTrigger = New-ScheduledTaskTrigger -AtStartup +$logonTrigger = New-ScheduledTaskTrigger -AtLogOn -User $principalUser +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-Startup-Recovery" ` + -Action $recoverAction ` + -Trigger @($startupTrigger, $logonTrigger) ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 startup recovery for VMs, Harbor, K3s, and public routes." ` + -Force | Out-Null + +$heartbeatArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode Status" +$heartbeatAction = New-ScheduledTaskAction -Execute $ps -Argument $heartbeatArgs +$heartbeatTrigger = New-ScheduledTaskTrigger ` + -Once ` + -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes 5) ` + -RepetitionDuration (New-TimeSpan -Days 3650) +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-Heartbeat" ` + -Action $heartbeatAction ` + -Trigger $heartbeatTrigger ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 observe-mode heartbeat for host, Harbor, K3s, and public route evidence." ` + -Force | Out-Null + +$perfArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode Perf -ControlledApply" +$perfAction = New-ScheduledTaskAction -Execute $ps -Argument $perfArgs +$perfTrigger = New-ScheduledTaskTrigger ` + -Once ` + -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes 1) ` + -RepetitionDuration (New-TimeSpan -Days 3650) +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-Performance-Guard" ` + -Action $perfAction ` + -Trigger $perfTrigger ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 minute-level performance guard with Telegram audit and allowlisted controlled load shedding." ` + -Force | Out-Null + +$controlArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode ControlTick" +$controlAction = New-ScheduledTaskAction -Execute $ps -Argument $controlArgs +$controlTrigger = New-ScheduledTaskTrigger ` + -Once ` + -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes 1) ` + -RepetitionDuration (New-TimeSpan -Days 3650) +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-Control-Loop" ` + -Action $controlAction ` + -Trigger $controlTrigger ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 visible control center, desktop shortcuts, and allowlisted operator command queue processor." ` + -Force | Out-Null + +$telegramInboxScript = Join-Path $AgentRoot "bin\agent99-telegram-inbox.ps1" +$telegramInboxArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$telegramInboxScript`" -RunNow" +$telegramInboxAction = New-ScheduledTaskAction -Execute $ps -Argument $telegramInboxArgs +$telegramInboxTrigger = New-ScheduledTaskTrigger ` + -Once ` + -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes 1) ` + -RepetitionDuration (New-TimeSpan -Days 3650) +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-Telegram-Inbox" ` + -Action $telegramInboxAction ` + -Trigger $telegramInboxTrigger ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 Telegram command inbox for allowlisted /agent99 operator instructions." ` + -Force | Out-Null + +$sreAlertInboxScript = Join-Path $AgentRoot "bin\agent99-sre-alert-inbox.ps1" +$sreAlertInboxArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$sreAlertInboxScript`" -RunNow" +$sreAlertInboxAction = New-ScheduledTaskAction -Execute $ps -Argument $sreAlertInboxArgs +$sreAlertInboxTrigger = New-ScheduledTaskTrigger ` + -Once ` + -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes 1) ` + -RepetitionDuration (New-TimeSpan -Days 3650) +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-SRE-Alert-Inbox" ` + -Action $sreAlertInboxAction ` + -Trigger $sreAlertInboxTrigger ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 SRE war-room alert inbox that converts actionable alerts into allowlisted remediation queue entries." ` + -Force | Out-Null + +$sreAlertRelayScript = Join-Path $AgentRoot "bin\agent99-sre-alert-relay.ps1" +if (Test-Path $sreAlertRelayScript) { + $sreAlertRelayArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$sreAlertRelayScript`"" + $sreAlertRelayAction = New-ScheduledTaskAction -Execute $ps -Argument $sreAlertRelayArgs + Register-ScheduledTask ` + -TaskName "Wooo-Agent99-SRE-Alert-Relay" ` + -Action $sreAlertRelayAction ` + -Trigger @($startupTrigger, $logonTrigger) ` + -Principal $principal ` + -Settings $serviceSettings ` + -Description "Agent99 HTTP relay for AWOOI Alertmanager alerts into the SRE alert inbox." ` + -Force | Out-Null +} + +$selfCheckArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode SelfCheck" +$selfCheckAction = New-ScheduledTaskAction -Execute $ps -Argument $selfCheckArgs +$selfCheckTrigger = New-ScheduledTaskTrigger ` + -Once ` + -At (Get-Date).AddMinutes(2) ` + -RepetitionInterval (New-TimeSpan -Minutes 5) ` + -RepetitionDuration (New-TimeSpan -Days 3650) +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-Self-Health" ` + -Action $selfCheckAction ` + -Trigger $selfCheckTrigger ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 self-health guard for scheduled tasks, evidence freshness, host perf readback, and Telegram delivery." ` + -Force | Out-Null + +$backupArgs = "-NoProfile -ExecutionPolicy Bypass -File `"$launcher`" -Mode BackupCheck" +$backupAction = New-ScheduledTaskAction -Execute $ps -Argument $backupArgs +$backupTrigger = New-ScheduledTaskTrigger ` + -Once ` + -At (Get-Date).AddMinutes(3) ` + -RepetitionInterval (New-TimeSpan -Minutes 15) ` + -RepetitionDuration (New-TimeSpan -Days 3650) +Register-ScheduledTask ` + -TaskName "Wooo-Agent99-Backup-Health" ` + -Action $backupAction ` + -Trigger $backupTrigger ` + -Principal $principal ` + -Settings $settings ` + -Description "Agent99 backup freshness, integrity-status, cron-contract, and Telegram alert guard." ` + -Force | Out-Null + +$taskNames = @( + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-SRE-Alert-Relay", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" +) + +Get-ScheduledTask -TaskName $taskNames -ErrorAction SilentlyContinue | + Select-Object TaskName, State, TaskPath diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 new file mode 100644 index 000000000..97128fdac --- /dev/null +++ b/agent99-sre-alert-inbox.ps1 @@ -0,0 +1,297 @@ +param( + [string]$AgentRoot = "C:\Wooo\Agent99", + [switch]$RunNow, + [int]$MaxAlerts = 10 +) + +$ErrorActionPreference = "Stop" + +$EvidenceDir = Join-Path $AgentRoot "evidence" +$AlertRoot = Join-Path $AgentRoot "alerts" +$IncomingDir = Join-Path $AlertRoot "incoming" +$RunningDir = Join-Path $AlertRoot "running" +$ProcessedDir = Join-Path $AlertRoot "processed" +$FailedDir = Join-Path $AlertRoot "failed" +$IgnoredDir = Join-Path $AlertRoot "ignored" +$QueueDir = Join-Path $AgentRoot "queue" +$RequestDir = Join-Path $AgentRoot "requests" + +New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir | Out-Null + +$allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness") +$remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed") + +function Get-AgentField { + param( + [object]$Object, + [string]$Name, + [object]$Default = $null + ) + if ($Object -and $Object.PSObject.Properties[$Name]) { + return $Object.PSObject.Properties[$Name].Value + } + return $Default +} + +function Convert-AgentSafeFileToken { + param([string]$Value) + $safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_") + if (-not $safe) { return "alert" } + if ($safe.Length -gt 80) { return $safe.Substring(0, 80) } + return $safe +} + +function Get-AgentAlertText { + param([object]$Alert) + $labels = @() + $rawLabels = Get-AgentField $Alert "labels" @() + foreach ($label in @($rawLabels)) { + if ($label -ne $null) { $labels += [string]$label } + } + @( + (Get-AgentField $Alert "source" ""), + (Get-AgentField $Alert "severity" ""), + (Get-AgentField $Alert "kind" ""), + (Get-AgentField $Alert "service" ""), + (Get-AgentField $Alert "host" ""), + (Get-AgentField $Alert "title" ""), + (Get-AgentField $Alert "message" ""), + ($labels -join " ") + ) -join " " +} + +function Test-AgentAlertActionable { + param([object]$Alert) + $force = [bool](Get-AgentField $Alert "force" $false) + if ($force) { return $true } + + $text = (Get-AgentAlertText $Alert).ToLowerInvariant() + $severity = ([string](Get-AgentField $Alert "severity" "")).ToLowerInvariant() + if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) { + return $true + } + if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s") { + return $true + } + return $false +} + +function Resolve-AgentAlertMode { + param([object]$Alert) + + $text = (Get-AgentAlertText $Alert).ToLowerInvariant() + $suggestedMode = [string](Get-AgentField $Alert "suggestedMode" "") + + if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion") { + return "ProviderFreshness" + } + if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") { + return "Recover" + } + if ($text -match "host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") { + return "Recover" + } + if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") { + return "Perf" + } + if ($text -match "backup|snapshot|restore drill|cron") { + return "BackupCheck" + } + if ($text -match "harbor|registry") { + return "HarborRepair" + } + if ($text -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { + return "AwoooRepair" + } + if ($suggestedMode -and $suggestedMode -in $allowedModes) { + return $suggestedMode + } + return "Status" +} + +function New-AgentInstructionFromAlert { + param( + [object]$Alert, + [string]$AlertId, + [string]$ModeName + ) + + $existing = [string](Get-AgentField $Alert "instruction" "") + if ($existing) { return $existing } + + $source = [string](Get-AgentField $Alert "source" "sre-war-room") + $severity = [string](Get-AgentField $Alert "severity" "unknown") + $kind = [string](Get-AgentField $Alert "kind" "unknown") + $service = [string](Get-AgentField $Alert "service" "unknown") + $hostName = [string](Get-AgentField $Alert "host" "") + $title = [string](Get-AgentField $Alert "title" "") + $message = [string](Get-AgentField $Alert "message" "") + + if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness") { + return "Provider freshness candidate required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; require providerWindow,lastSeen,directReference,verifierReadback; mark missing,expired,timeout,no_false_green; do not switch provider, call paid model, edit env, or reload." + } + + $parts = @("SRE alert", "id=$AlertId", "source=$source", "severity=$severity", "service=$service", "kind=$kind", "mode=$ModeName") + if ($hostName) { $parts += "host=$hostName" } + if ($title) { $parts += "title=$title" } + if ($message) { $parts += "message=$message" } + return ($parts -join "; ") +} + +function Get-AgentControlledApplyForAlert { + param( + [object]$Alert, + [string]$ModeName + ) + if ($Alert -and $Alert.PSObject.Properties["controlledApply"]) { + return [bool]$Alert.controlledApply + } + return [bool]($ModeName -in $remediationModes) +} + +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json" +$queued = @() +$ignored = @() +$failed = @() + +$files = @(Get-ChildItem -Path $IncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTime | + Select-Object -First $MaxAlerts) + +foreach ($file in $files) { + $runningPath = Join-Path $RunningDir $file.Name + Move-Item -Force $file.FullName $runningPath + + try { + $alert = Get-Content $runningPath -Raw | ConvertFrom-Json + } catch { + $failedPath = Join-Path $FailedDir ("failed-" + $file.Name) + Move-Item -Force $runningPath $failedPath + $failed += [pscustomobject]@{ + file = $failedPath + ok = $false + reason = "invalid_json" + error = $_.Exception.Message + } + continue + } + + $alertId = [string](Get-AgentField $alert "id" $file.BaseName) + $actionable = Test-AgentAlertActionable $alert + if (-not $actionable) { + $ignoredPath = Join-Path $IgnoredDir ("ignored-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) + Move-Item -Force $runningPath $ignoredPath + $ignored += [pscustomobject]@{ + alertId = $alertId + file = $ignoredPath + reason = "not_actionable" + } + continue + } + + $modeName = Resolve-AgentAlertMode $alert + if ($modeName -notin $allowedModes) { + $failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) + Move-Item -Force $runningPath $failedPath + $failed += [pscustomobject]@{ + alertId = $alertId + file = $failedPath + ok = $false + reason = "mode_not_allowlisted" + mode = $modeName + } + continue + } + + $queueId = "sre-alert-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) + $controlled = Get-AgentControlledApplyForAlert $alert $modeName + $instruction = New-AgentInstructionFromAlert $alert $alertId $modeName + $source = [string](Get-AgentField $alert "source" "sre-war-room") + $kind = [string](Get-AgentField $alert "kind" "unknown") + $severity = [string](Get-AgentField $alert "severity" "unknown") + $service = [string](Get-AgentField $alert "service" "unknown") + $hostName = [string](Get-AgentField $alert "host" "") + $replyChatId = [string](Get-AgentField $alert "replyChatId" "") + + $requestPath = Join-Path $RequestDir "$queueId.json" + $queuePath = Join-Path $QueueDir "$queueId.json" + $processedPath = Join-Path $ProcessedDir ("processed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) + + [pscustomobject]@{ + id = $queueId + instruction = $instruction + resolvedMode = $modeName + controlledApply = $controlled + requestedBy = $env:USERNAME + source = "agent99-sre-alert-inbox" + alertId = $alertId + alertSource = $source + alertKind = $kind + alertSeverity = $severity + alertService = $service + alertHost = $hostName + alertPath = $processedPath + createdAt = (Get-Date -Format o) + } | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8 + + [pscustomobject]@{ + id = $queueId + mode = $modeName + controlledApply = $controlled + requestedBy = $env:USERNAME + source = "agent99-sre-alert-inbox" + reason = "sre_alert" + instruction = $instruction + requestPath = $requestPath + alertId = $alertId + alertSource = $source + alertKind = $kind + alertSeverity = $severity + alertService = $service + alertHost = $hostName + alertPath = $processedPath + replyChatId = $replyChatId + createdAt = (Get-Date -Format o) + } | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 + + Move-Item -Force $runningPath $processedPath + $queued += [pscustomobject]@{ + alertId = $alertId + source = $source + severity = $severity + kind = $kind + service = $service + host = $hostName + mode = $modeName + controlledApply = $controlled + requestPath = $requestPath + queuePath = $queuePath + alertPath = $processedPath + } +} + +$controlExitCode = $null +if ($RunNow -and @($queued).Count -gt 0) { + $launcher = Join-Path $AgentRoot "agent99-run.ps1" + $process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden + $process.Refresh() + $controlExitCode = $process.ExitCode +} + +$result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = [bool](@($failed).Count -eq 0) + incomingPath = $IncomingDir + queuedCount = @($queued).Count + ignoredCount = @($ignored).Count + failedCount = @($failed).Count + queued = $queued + ignored = $ignored + failed = $failed + runNow = [bool]$RunNow + controlExitCode = $controlExitCode +} + +$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8 +$result | ConvertTo-Json -Depth 10 diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 new file mode 100644 index 000000000..1e5d4cc40 --- /dev/null +++ b/agent99-sre-alert-relay.ps1 @@ -0,0 +1,218 @@ +param( + [string]$AgentRoot = "C:\Wooo\Agent99", + [string]$Prefix = "http://+:8787/agent99/sre-alert/", + [string]$TokenEnv = "AGENT99_SRE_RELAY_TOKEN", + [string[]]$AllowedRemotePrefixes = @("127.", "::1", "192.168.0."), + [int]$MaxBodyBytes = 262144, + [switch]$RunOnce +) + +$ErrorActionPreference = "Stop" + +$EvidenceDir = Join-Path $AgentRoot "evidence" +$AlertRoot = Join-Path $AgentRoot "alerts" +$IncomingDir = Join-Path $AlertRoot "incoming" +$LogDir = Join-Path $AgentRoot "logs" +$BinDir = Join-Path $AgentRoot "bin" +$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" + +New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir | Out-Null + +function Convert-AgentSafeFileToken { + param([string]$Value) + $safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_") + if (-not $safe) { return "relay-alert" } + if ($safe.Length -gt 96) { return $safe.Substring(0, 96) } + return $safe +} + +function Get-AgentField { + param( + [object]$Object, + [string]$Name, + [object]$Default = $null + ) + if ($Object -and $Object.PSObject.Properties[$Name]) { + return $Object.PSObject.Properties[$Name].Value + } + return $Default +} + +function Get-AgentEnvValue { + param([string]$Name) + foreach ($target in @("Process", "User", "Machine")) { + $value = [Environment]::GetEnvironmentVariable($Name, $target) + if ($value) { return $value } + } + return "" +} + +function Write-AgentRelayEvent { + param([hashtable]$Event) + $path = Join-Path $LogDir "agent99-sre-alert-relay.jsonl" + $Event["timestamp"] = (Get-Date).ToString("o") + ($Event | ConvertTo-Json -Compress -Depth 8) | Add-Content -Encoding UTF8 $path +} + +function Send-AgentRelayResponse { + param( + [System.Net.HttpListenerContext]$Context, + [int]$StatusCode, + [hashtable]$Body + ) + $json = $Body | ConvertTo-Json -Compress -Depth 8 + $bytes = [System.Text.Encoding]::UTF8.GetBytes($json) + $Context.Response.StatusCode = $StatusCode + $Context.Response.ContentType = "application/json; charset=utf-8" + $Context.Response.ContentLength64 = $bytes.Length + $Context.Response.OutputStream.Write($bytes, 0, $bytes.Length) + $Context.Response.Close() +} + +function Test-AgentRelayRemoteAllowed { + param([System.Net.HttpListenerContext]$Context) + $remote = [string]$Context.Request.RemoteEndPoint.Address + foreach ($prefix in $AllowedRemotePrefixes) { + if ($remote.StartsWith($prefix)) { return $true } + } + return $false +} + +function Start-AgentSreAlertInbox { + if (-not (Test-Path $SreAlertInboxScript)) { + return [pscustomobject]@{ started = $false; reason = "sre_alert_inbox_script_missing" } + } + $ps = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $stamp = Get-Date -Format "yyyyMMdd-HHmmss" + $stdout = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.out" + $stderr = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.err" + $args = @( + "-NoProfile", + "-ExecutionPolicy", "Bypass", + "-File", $SreAlertInboxScript, + "-AgentRoot", $AgentRoot, + "-RunNow" + ) + $proc = Start-Process -FilePath $ps -ArgumentList $args -PassThru -WindowStyle Hidden -RedirectStandardOutput $stdout -RedirectStandardError $stderr + return [pscustomobject]@{ + started = $true + processId = $proc.Id + stdout = $stdout + stderr = $stderr + } +} + +if (-not $Prefix.EndsWith("/")) { + $Prefix = "$Prefix/" +} + +$listener = [System.Net.HttpListener]::new() +$listener.Prefixes.Add($Prefix) + +$startupStamp = Get-Date -Format "yyyyMMdd-HHmmss" +$startupEvidence = Join-Path $EvidenceDir "agent99-SreAlertRelay-start-$startupStamp.json" +@{ + ok = $true + prefix = $Prefix + tokenEnv = $TokenEnv + tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv) + allowedRemotePrefixes = $AllowedRemotePrefixes + maxBodyBytes = $MaxBodyBytes + runOnce = [bool]$RunOnce + startedAt = (Get-Date).ToString("o") +} | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $startupEvidence + +try { + $listener.Start() + Write-AgentRelayEvent @{ + event = "relay_started" + ok = $true + prefix = $Prefix + tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv) + evidence = $startupEvidence + } + + do { + $context = $listener.GetContext() + $remote = [string]$context.Request.RemoteEndPoint.Address + try { + if ($context.Request.HttpMethod -ne "POST") { + Send-AgentRelayResponse $context 405 @{ ok = $false; error = "method_not_allowed" } + Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "method_not_allowed" } + continue + } + + if (-not (Test-AgentRelayRemoteAllowed $context)) { + Send-AgentRelayResponse $context 403 @{ ok = $false; error = "remote_not_allowed" } + Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "remote_not_allowed" } + continue + } + + $expectedToken = Get-AgentEnvValue $TokenEnv + if ($expectedToken) { + $actualToken = [string]$context.Request.Headers["X-Agent99-Relay-Token"] + if ($actualToken -ne $expectedToken) { + Send-AgentRelayResponse $context 401 @{ ok = $false; error = "invalid_relay_token" } + Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "invalid_relay_token" } + continue + } + } + + if ($context.Request.ContentLength64 -gt $MaxBodyBytes) { + Send-AgentRelayResponse $context 413 @{ ok = $false; error = "payload_too_large" } + Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "payload_too_large" } + continue + } + + $reader = [System.IO.StreamReader]::new($context.Request.InputStream, $context.Request.ContentEncoding) + $body = $reader.ReadToEnd() + if ($body.Length -gt $MaxBodyBytes) { + Send-AgentRelayResponse $context 413 @{ ok = $false; error = "payload_too_large" } + Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "payload_too_large_readback" } + continue + } + + $alert = $body | ConvertFrom-Json + $alertId = [string](Get-AgentField $alert "id" ("relay-" + [guid]::NewGuid().ToString("N"))) + $safeId = Convert-AgentSafeFileToken $alertId + $fileStamp = Get-Date -Format "yyyyMMdd-HHmmss" + $tmpPath = Join-Path $IncomingDir ".$safeId-$fileStamp-$([guid]::NewGuid().ToString("N")).tmp" + $finalPath = Join-Path $IncomingDir "$safeId-$fileStamp.json" + $alert | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $tmpPath + Move-Item -Force $tmpPath $finalPath + + $trigger = Start-AgentSreAlertInbox + Send-AgentRelayResponse $context 202 @{ + ok = $true + alertId = $alertId + incomingFile = $finalPath + inboxTriggered = [bool]$trigger.started + inboxProcessId = $trigger.processId + } + Write-AgentRelayEvent @{ + event = "relay_accepted" + ok = $true + remote = $remote + alertId = $alertId + incomingFile = $finalPath + inboxTriggered = [bool]$trigger.started + inboxProcessId = $trigger.processId + } + } catch { + try { + Send-AgentRelayResponse $context 500 @{ ok = $false; error = "relay_exception"; errorType = $_.Exception.GetType().Name } + } catch { + } + Write-AgentRelayEvent @{ + event = "relay_exception" + ok = $false + remote = $remote + errorType = $_.Exception.GetType().Name + error = $_.Exception.Message + } + } + } while (-not $RunOnce) +} finally { + if ($listener.IsListening) { $listener.Stop() } + $listener.Close() +} diff --git a/agent99-submit-request.ps1 b/agent99-submit-request.ps1 new file mode 100644 index 000000000..679194c37 --- /dev/null +++ b/agent99-submit-request.ps1 @@ -0,0 +1,150 @@ +param( + [string]$Instruction = "", + [string]$Mode = "", + [switch]$ControlledApply, + [switch]$RunNow, + [string]$Source = "agent99-submit-request", + [string]$RequestedBy = "", + [string]$ExternalId = "", + [string]$ReplyChatId = "", + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" + +if (-not $Instruction) { + $Instruction = Read-Host "Agent99 request" +} + +if (-not $Instruction) { + throw "Instruction is required." +} + +$allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck") + +function Test-AgentContainsChar { + param( + [string]$Text, + [int[]]$Codes + ) + foreach ($code in $Codes) { + if ($Text.IndexOf([string][char]$code) -ge 0) { + return $true + } + } + return $false +} + +function Resolve-AgentRequestMode { + param( + [string]$Text, + [string]$ExplicitMode + ) + + if ($ExplicitMode) { + if ($ExplicitMode -notin $allowedModes) { + throw "Mode is not allowlisted: $ExplicitMode" + } + return $ExplicitMode + } + + $lower = $Text.ToLowerInvariant() + $mentionsPublic = ($lower -match "public|smoke|website|site|route|url|http|502" -or (Test-AgentContainsChar $Text @(0x7DB2, 0x7AD9))) + $wantsRecovery = ($lower -match "recover|reboot|restore|restart|startup|repair|fix|502" -or (Test-AgentContainsChar $Text @(0x6062, 0x5FA9, 0x91CD, 0x555F, 0x4FEE))) + + if ($mentionsPublic -and $wantsRecovery) { + return "Recover" + } + + if ($mentionsPublic) { + return "PublicSmoke" + } + + if ($wantsRecovery) { + return "Recover" + } + + if ($lower -match "vm|vmware|startvm|start vm|start vms" -or (Test-AgentContainsChar $Text @(0x865B, 0x64EC, 0x4E3B, 0x6A5F))) { + return "StartVMs" + } + + if ($lower -match "cpu|load|memory|ram|disk|perf|performance|pressure" -or (Test-AgentContainsChar $Text @(0x6548, 0x80FD, 0x78C1, 0x789F, 0x8CA0, 0x8F09))) { + return "Perf" + } + + if ($lower -match "backup|snapshot|restore drill|cron" -or (Test-AgentContainsChar $Text @(0x5099, 0x4EFD))) { + return "BackupCheck" + } + + if ($lower -match "self|agent|health|selfcheck" -or (Test-AgentContainsChar $Text @(0x81EA, 0x6AA2, 0x5065, 0x5EB7))) { + return "SelfCheck" + } + + if ($lower -match "harbor|registry") { + return "HarborRepair" + } + + if ($lower -match "awoooi|k3s|pod|pods") { + return "AwoooRepair" + } + + return "Status" +} + +$modeName = Resolve-AgentRequestMode $Instruction $Mode +$queueDir = Join-Path $AgentRoot "queue" +$requestDir = Join-Path $AgentRoot "requests" +$evidenceDir = Join-Path $AgentRoot "evidence" +New-Item -ItemType Directory -Force -Path $queueDir, $requestDir, $evidenceDir | Out-Null + +$id = "request-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) +$request = [pscustomobject]@{ + id = $id + instruction = $Instruction + resolvedMode = $modeName + controlledApply = [bool]$ControlledApply + requestedBy = if ($RequestedBy) { $RequestedBy } else { $env:USERNAME } + source = $Source + externalId = $ExternalId + createdAt = (Get-Date -Format o) +} + +$requestPath = Join-Path $requestDir "$id.json" +$queuePath = Join-Path $queueDir "$id.json" +$request | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8 +[pscustomobject]@{ + id = $id + mode = $modeName + controlledApply = [bool]$ControlledApply + requestedBy = if ($RequestedBy) { $RequestedBy } else { $env:USERNAME } + source = $Source + externalId = $ExternalId + reason = "operator_instruction" + instruction = $Instruction + requestPath = $requestPath + replyChatId = $ReplyChatId + createdAt = (Get-Date -Format o) +} | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 + +$result = [pscustomobject]@{ + ok = $true + id = $id + instruction = $Instruction + resolvedMode = $modeName + controlledApply = [bool]$ControlledApply + requestPath = $requestPath + queuePath = $queuePath + runNow = [bool]$RunNow + controlExitCode = $null +} + +if ($RunNow) { + $launcher = Join-Path $AgentRoot "agent99-run.ps1" + $process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden + $process.Refresh() + $result.controlExitCode = $process.ExitCode +} + +$submitEvidence = Join-Path $evidenceDir "agent99-submit-$id.json" +$result | ConvertTo-Json -Depth 8 | Set-Content -Path $submitEvidence -Encoding UTF8 +$result | ConvertTo-Json -Depth 8 diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 new file mode 100644 index 000000000..a70f03aaa --- /dev/null +++ b/agent99-telegram-inbox.ps1 @@ -0,0 +1,488 @@ +param( + [string]$AgentRoot = "C:\Wooo\Agent99", + [switch]$RunNow, + [string]$SyntheticUpdateText = "", + [string]$SyntheticChatTitle = "AwoooI SRE", + [string]$SyntheticChatId = "" +) + +$ErrorActionPreference = "Stop" + +$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json" +$EvidenceDir = Join-Path $AgentRoot "evidence" +$StateDir = Join-Path $AgentRoot "state" +$BinDir = Join-Path $AgentRoot "bin" +$SubmitScript = Join-Path $BinDir "agent99-submit-request.ps1" +$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" +$AlertIncomingDir = Join-Path $AgentRoot "alerts\incoming" + +New-Item -ItemType Directory -Force -Path $EvidenceDir, $StateDir, $AlertIncomingDir | Out-Null + +function Load-AgentConfig { + if (-not (Test-Path $ConfigPath)) { + throw "Config not found: $ConfigPath" + } + Get-Content $ConfigPath -Raw | ConvertFrom-Json +} + +function Invoke-AgentSshText { + param( + [string]$TargetHost, + [string]$Command, + [int]$TimeoutSeconds = 30 + ) + + $id = [guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-tg-ssh-$id.out" + $stderrPath = Join-Path $env:TEMP "agent99-tg-ssh-$id.err" + $args = @( + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=10", + "-l", "wooo", + $TargetHost, + $Command + ) + + $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru + $completed = $process.WaitForExit($TimeoutSeconds * 1000) + if (-not $completed) { + $process.Kill() + $process.WaitForExit(2000) | Out-Null + } + $process.Refresh() + $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" } + $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" } + Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null + [pscustomobject]@{ + ok = [bool]($completed -and (($null -eq $process.ExitCode) -or $process.ExitCode -eq 0)) + exitCode = if ($completed -and $null -ne $process.ExitCode) { $process.ExitCode } elseif ($completed) { 0 } else { -2 } + output = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" + } +} + +function Convert-AgentBase64 { + param([string]$Text) + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Text)) +} + +function Get-AgentTelegramUpdates { + param( + [object]$Config, + [int64]$Offset + ) + + $relay = $Config.telegram.relay + if (-not ($relay -and $relay.enabled)) { + return [pscustomobject]@{ ok = $false; error = "relay_disabled"; updates = @(); raw = $null } + } + $relayHost = if ($relay.host) { [string]$relay.host } else { "192.168.0.110" } + $container = if ($relay.container) { [string]$relay.container } else { "stockplatform-v2-api-1" } + $allowedTitlePattern = if ($Config.telegram.allowedCommandChatTitlePattern) { + [string]$Config.telegram.allowedCommandChatTitlePattern + } else { + "AwoooI SRE|AWOOOI SRE|SRE" + } + if ($container -notmatch "^[A-Za-z0-9_.-]+$") { + return [pscustomobject]@{ ok = $false; error = "invalid_relay_container"; updates = @(); raw = $null } + } + + $python = @' +import base64, json, os, re, sys, urllib.parse, urllib.request, urllib.error + +token = os.environ.get("TELEGRAM_BOT_TOKEN") or "" +chat = os.environ.get("TELEGRAM_CHAT_ID") or "" + +def redact(value): + text = str(value or "") + for secret in (token, chat): + if secret: + text = text.replace(secret, "") + return text[-1200:] + +def emit(payload): + print(json.dumps(payload, ensure_ascii=False)) + +try: + offset = int(sys.argv[2]) if len(sys.argv) > 2 else 0 + allowed_title_pattern = base64.b64decode(sys.argv[3]).decode() if len(sys.argv) > 3 and sys.argv[3] else "" + allowed_title_re = re.compile(allowed_title_pattern, re.I) if allowed_title_pattern else None + if not token or not chat: + emit({"ok": False, "error": "telegram_env_missing", "updates": []}) + sys.exit(0) + + params = { + "timeout": "0", + "allowed_updates": json.dumps(["message"]) + } + if offset > 0: + params["offset"] = str(offset) + url = "https://api.telegram.org/bot" + token + "/getUpdates?" + urllib.parse.urlencode(params) + + try: + raw = urllib.request.urlopen(url, timeout=20).read().decode() + except urllib.error.HTTPError as exc: + body = exc.read().decode(errors="replace") + emit({"ok": False, "error": "telegram_http_error", "status": exc.code, "description": redact(body), "updates": []}) + sys.exit(0) + except Exception as exc: + emit({"ok": False, "error": "telegram_request_failed", "errorType": type(exc).__name__, "description": redact(exc), "updates": []}) + sys.exit(0) + + try: + data = json.loads(raw) + except Exception as exc: + emit({"ok": False, "error": "telegram_json_parse_failed", "errorType": type(exc).__name__, "description": redact(exc), "updates": []}) + sys.exit(0) + + if not data.get("ok", False): + emit({"ok": False, "error": "telegram_api_not_ok", "description": redact(data.get("description", "")), "errorCode": data.get("error_code"), "updates": []}) + sys.exit(0) + + items = [] + for item in data.get("result", []): + msg = item.get("message") or {} + chat_obj = msg.get("chat") or {} + chat_id = str(chat_obj.get("id") or "") + chat_title = chat_obj.get("title") or chat_obj.get("username") or chat_obj.get("first_name") or "" + chat_matches_config = chat_id == str(chat) + chat_matches_title = bool(allowed_title_re and allowed_title_re.search(chat_title)) + if not (chat_matches_config or chat_matches_title): + continue + text = msg.get("text") or "" + if not text: + continue + sender = msg.get("from") or {} + items.append({ + "update_id": item.get("update_id"), + "message_id": msg.get("message_id"), + "date": msg.get("date"), + "text": text, + "chat_id": chat_id, + "chat_title": chat_title, + "chat_type": chat_obj.get("type"), + "chat_match": "configured_chat_id" if chat_matches_config else "allowed_title", + "from": sender.get("username") or sender.get("first_name") or str(sender.get("id") or "") + }) + emit({"ok": True, "updates": items}) +except Exception as exc: + emit({"ok": False, "error": "telegram_inbox_exception", "errorType": type(exc).__name__, "description": redact(exc), "updates": []}) +'@ + + $pythonB64 = Convert-AgentBase64 $python + $allowedTitleB64 = Convert-AgentBase64 $allowedTitlePattern + $bootstrap = 'import base64,sys;exec(base64.b64decode(sys.argv[1]))' + $remoteCommand = "docker exec $container python3 -c '$bootstrap' $pythonB64 $Offset $allowedTitleB64" + $run = Invoke-AgentSshText $relayHost $remoteCommand 35 + if (-not $run.ok) { + return [pscustomobject]@{ ok = $false; error = $run.output; updates = @(); raw = $null } + } + + try { + $parsed = $run.output | ConvertFrom-Json + $errorText = if ($parsed.ok) { + $null + } elseif ($parsed.description) { + "$($parsed.error): $($parsed.description)" + } else { + [string]$parsed.error + } + return [pscustomobject]@{ ok = [bool]$parsed.ok; error = $errorText; updates = @($parsed.updates); raw = $parsed } + } catch { + return [pscustomobject]@{ ok = $false; error = $_.Exception.Message; updates = @(); raw = $run.output } + } +} + +function Resolve-AgentTelegramInstruction { + param([string]$Text) + + $trimmed = $Text.Trim() + $patterns = @( + "^/agent99\s+(.+)$", + "^!agent99\s+(.+)$", + "^#agent99\s+(.+)$", + "^agent99\s+(.+)$", + "^@agent99\s+(.+)$" + ) + foreach ($pattern in $patterns) { + $match = [regex]::Match($trimmed, $pattern, [Text.RegularExpressions.RegexOptions]::IgnoreCase) + if ($match.Success) { + return $match.Groups[1].Value.Trim() + } + } + return $null +} + +function Test-AgentMonitoringAlertText { + param([string]$Text) + + $trimmed = $Text.Trim() + if (-not $trimmed) { return $false } + $lower = $trimmed.ToLowerInvariant() + + if ($lower -match "^agent99 alert:|\[agent99\]|event:\s+operator command|operator_command_result") { + return $false + } + + if ($lower -match "alertmanager|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running") { + return $true + } + + foreach ($code in @(0x544A, 0x8B66, 0x7570, 0x5E38, 0x56B4, 0x91CD, 0x8CA0, 0x8F09, 0x78C1, 0x789F, 0x5099, 0x4EFD)) { + if ($trimmed.IndexOf([string][char]$code) -ge 0) { return $true } + } + + return $false +} + +function Test-AgentTextContainsCode { + param( + [string]$Text, + [int[]]$Codes + ) + foreach ($code in $Codes) { + if ($Text.IndexOf([string][char]$code) -ge 0) { return $true } + } + return $false +} + +function Get-AgentTelegramAlertSeverity { + param([string]$Text) + $lower = $Text.ToLowerInvariant() + if ($lower -match "resolved|recovered" -or (Test-AgentTextContainsCode $Text @(0x6062, 0x89E3))) { return "resolved" } + if ($lower -match "critical|p0|emergency|fatal|down|failed" -or (Test-AgentTextContainsCode $Text @(0x56B4, 0x91CD, 0x7DCA, 0x6025))) { return "critical" } + if ($lower -match "warning|warn|degraded" -or (Test-AgentTextContainsCode $Text @(0x544A, 0x8B66, 0x7570, 0x5E38))) { return "warning" } + return "warning" +} + +function Get-AgentTelegramAlertKind { + param([string]$Text) + $lower = $Text.ToLowerInvariant() + if ($lower -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion") { return "provider_freshness_signal" } + if ($lower -match "502|site_502|public_route|website|route") { return "public_route_502" } + if ($lower -match "cpu|load" -or (Test-AgentTextContainsCode $Text @(0x8CA0, 0x8F09))) { return "performance_cpu" } + if ($lower -match "disk" -or (Test-AgentTextContainsCode $Text @(0x78C1, 0x789F))) { return "performance_disk" } + if ($lower -match "memory|mem_|ram" -or (Test-AgentTextContainsCode $Text @(0x8A18, 0x61B6, 0x9AD4))) { return "performance_memory" } + if ($lower -match "harbor|registry") { return "harbor_registry" } + if ($lower -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return "awoooi_k3s" } + if ($lower -match "backup|snapshot|cron" -or (Test-AgentTextContainsCode $Text @(0x5099, 0x4EFD))) { return "backup_health" } + if ($lower -match "hostdown|host_down|vm_not_running|reboot|restart|down" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" } + return "monitoring_alert" +} + +function Get-AgentTelegramAlertService { + param([string]$Text) + $lower = $Text.ToLowerInvariant() + if ($lower -match "stock") { return "stockplatform" } + if ($lower -match "2026fifa|fifa") { return "2026fifa" } + if ($lower -match "gitea") { return "gitea" } + if ($lower -match "harbor") { return "harbor" } + if ($lower -match "agent99") { return "agent99" } + if ($lower -match "awoooi|k3s|502|site|route|website") { return "awoooi" } + return "unknown" +} + +function Get-AgentTelegramAlertHost { + param([string]$Text) + $match = [regex]::Match($Text, "(192\\.168\\.0\\.(110|112|120|121|188|99))") + if ($match.Success) { return $match.Groups[1].Value } + foreach ($hostSuffix in @("110", "112", "120", "121", "188", "99")) { + if ($Text -match "(^|\D)$hostSuffix(\D|$)") { + return "192.168.0.$hostSuffix" + } + } + return "" +} + +function New-AgentMonitoringAlertFromTelegram { + param([object]$Update) + + $text = [string]$Update.text + $severity = Get-AgentTelegramAlertSeverity $text + if ($severity -eq "resolved") { + return [pscustomobject]@{ + ok = $false + skipped = $true + reason = "resolved_alert_no_remediation" + } + } + + $alertId = "telegram-monitoring-" + [string]$Update.update_id + $alertPath = Join-Path $AlertIncomingDir ($alertId + ".json") + $message = if ($text.Length -gt 2000) { $text.Substring(0, 2000) + "`n...(truncated)" } else { $text } + $alert = [pscustomobject]@{ + id = $alertId + source = "telegram-sre-war-room-monitoring" + severity = $severity + kind = Get-AgentTelegramAlertKind $text + service = Get-AgentTelegramAlertService $text + host = Get-AgentTelegramAlertHost $text + title = "AwoooI SRE Telegram monitoring alert" + message = $message + labels = if ((Get-AgentTelegramAlertKind $text) -eq "provider_freshness_signal") { + @("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green") + } else { + @("telegram", "sre-war-room", "monitoring") + } + telegramUpdateId = $Update.update_id + telegramMessageId = $Update.message_id + telegramChatTitle = $Update.chat_title + telegramChatType = $Update.chat_type + replyChatId = [string]$Update.chat_id + force = $true + controlledApply = $true + createdAt = (Get-Date -Format o) + } + $alert | ConvertTo-Json -Depth 8 | Set-Content -Path $alertPath -Encoding UTF8 + + $sreExitCode = $null + if ($RunNow -and (Test-Path $SreAlertInboxScript)) { + $process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $SreAlertInboxScript, "-AgentRoot", $AgentRoot, "-RunNow") -Wait -PassThru -WindowStyle Hidden + $process.Refresh() + $sreExitCode = $process.ExitCode + } + + [pscustomobject]@{ + ok = $true + skipped = $false + alertId = $alertId + alertPath = $alertPath + severity = $alert.severity + kind = $alert.kind + service = $alert.service + host = $alert.host + sreInboxExitCode = $sreExitCode + } +} + +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidencePath = Join-Path $EvidenceDir "agent99-TelegramInbox-$stamp.json" +$offsetPath = Join-Path $StateDir "telegram-inbox-offset.json" +$offset = 0 +if ((-not $SyntheticUpdateText) -and (Test-Path $offsetPath)) { + try { + $offsetState = Get-Content $offsetPath -Raw | ConvertFrom-Json + if ($offsetState.nextOffset) { $offset = [int64]$offsetState.nextOffset } + } catch { + $offset = 0 + } +} + +$config = Load-AgentConfig +$autoIngestMonitoringAlerts = $true +if ($config.telegram -and $config.telegram.PSObject.Properties["autoIngestMonitoringAlerts"]) { + $autoIngestMonitoringAlerts = [bool]$config.telegram.autoIngestMonitoringAlerts +} +if ($SyntheticUpdateText) { + $updatesResult = [pscustomobject]@{ + ok = $true + error = $null + updates = @([pscustomobject]@{ + update_id = [int64](Get-Date -UFormat %s) + message_id = 1 + date = [int64](Get-Date -UFormat %s) + text = $SyntheticUpdateText + chat_id = $SyntheticChatId + chat_title = $SyntheticChatTitle + chat_type = "supergroup" + chat_match = "synthetic" + from = "synthetic-sre-monitor" + }) + raw = $null + } +} else { + $updatesResult = Get-AgentTelegramUpdates $config $offset +} +$processed = @() +$alerted = @() +$ignored = @() +$maxUpdateId = if ($offset -gt 0) { $offset - 1 } else { 0 } + +foreach ($update in @($updatesResult.updates)) { + if ($update.update_id -gt $maxUpdateId) { $maxUpdateId = [int64]$update.update_id } + $instruction = Resolve-AgentTelegramInstruction ([string]$update.text) + if (-not $instruction) { + if ($autoIngestMonitoringAlerts -and (Test-AgentMonitoringAlertText ([string]$update.text))) { + $alertResult = New-AgentMonitoringAlertFromTelegram $update + if ($alertResult.ok) { + $alerted += [pscustomobject]@{ + updateId = $update.update_id + messageId = $update.message_id + from = $update.from + chatTitle = $update.chat_title + chatType = $update.chat_type + chatMatch = $update.chat_match + alertId = $alertResult.alertId + severity = $alertResult.severity + kind = $alertResult.kind + service = $alertResult.service + host = $alertResult.host + alertPath = $alertResult.alertPath + sreInboxExitCode = $alertResult.sreInboxExitCode + } + } else { + $ignored += [pscustomobject]@{ + updateId = $update.update_id + reason = $alertResult.reason + } + } + continue + } + $ignored += [pscustomobject]@{ + updateId = $update.update_id + reason = "no_agent99_prefix" + } + continue + } + + $submitArgs = @( + "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", $SubmitScript, + "-Instruction", $instruction, + "-ControlledApply", + "-Source", "telegram-sre-war-room", + "-RequestedBy", ([string]$update.from), + "-ExternalId", ("telegram-update-" + [string]$update.update_id), + "-ReplyChatId", ([string]$update.chat_id) + ) + if ($RunNow) { $submitArgs += "-RunNow" } + $submit = Start-Process -FilePath "powershell.exe" -ArgumentList $submitArgs -Wait -PassThru -NoNewWindow -RedirectStandardOutput (Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).out") -RedirectStandardError (Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).err") + $submit.Refresh() + $processed += [pscustomobject]@{ + updateId = $update.update_id + messageId = $update.message_id + from = $update.from + chatTitle = $update.chat_title + chatType = $update.chat_type + chatMatch = $update.chat_match + instruction = $instruction + submitExitCode = if ($null -ne $submit.ExitCode) { $submit.ExitCode } else { 0 } + } +} + +if ((-not $SyntheticUpdateText) -and $updatesResult.ok -and $maxUpdateId -ge 0) { + [pscustomobject]@{ + nextOffset = $maxUpdateId + 1 + updatedAt = (Get-Date -Format o) + } | ConvertTo-Json -Depth 4 | Set-Content -Path $offsetPath -Encoding UTF8 +} + +$result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = [bool]$updatesResult.ok + error = $updatesResult.error + offsetBefore = $offset + offsetAfter = if ($SyntheticUpdateText) { $offset } elseif ($updatesResult.ok) { $maxUpdateId + 1 } else { $offset } + synthetic = [bool]$SyntheticUpdateText + updatesSeen = @($updatesResult.updates).Count + processedCount = @($processed).Count + alertedCount = @($alerted).Count + ignoredCount = @($ignored).Count + processed = $processed + alerted = $alerted + ignored = $ignored + offsetPath = $offsetPath +} +$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8 +$result | ConvertTo-Json -Depth 10 diff --git a/agent99-windows-update-policy.ps1 b/agent99-windows-update-policy.ps1 new file mode 100644 index 000000000..7f20e6a31 --- /dev/null +++ b/agent99-windows-update-policy.ps1 @@ -0,0 +1,104 @@ +param( + [string]$EvidenceDir = "C:\Wooo\Agent99\evidence" +) + +$ErrorActionPreference = "Stop" +New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null + +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$evidencePath = Join-Path $EvidenceDir "agent99-windows-update-policy-$stamp.json" +$logPath = Join-Path $EvidenceDir "agent99-windows-update-policy-$stamp.log" + +function Write-PolicyLog { + param([string]$Message) + "$(Get-Date -Format o) $Message" | Tee-Object -FilePath $logPath -Append | Out-Null +} + +function Get-RegistrySnapshot { + param([string]$Path) + if (-not (Test-Path $Path)) { + return [pscustomobject]@{ path = $Path; exists = $false; values = @{} } + } + + $item = Get-ItemProperty -Path $Path + $values = @{} + foreach ($prop in $item.PSObject.Properties) { + if ($prop.Name -notmatch "^PS") { + $values[$prop.Name] = $prop.Value + } + } + + [pscustomobject]@{ path = $Path; exists = $true; values = $values } +} + +$windowsUpdatePolicy = "HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate" +$auPolicy = Join-Path $windowsUpdatePolicy "AU" +$uxSettings = "HKLM:\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" + +$before = @( + Get-RegistrySnapshot $windowsUpdatePolicy + Get-RegistrySnapshot $auPolicy + Get-RegistrySnapshot $uxSettings +) + +Write-PolicyLog "apply_start windows_update_no_surprise_reboot" +New-Item -Path $windowsUpdatePolicy -Force | Out-Null +New-Item -Path $auPolicy -Force | Out-Null +New-Item -Path $uxSettings -Force | Out-Null + +# Keep Windows Update enabled, but prevent unattended surprise restarts. +New-ItemProperty -Path $auPolicy -Name "NoAutoRebootWithLoggedOnUsers" -Value 1 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $auPolicy -Name "AlwaysAutoRebootAtScheduledTime" -Value 0 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $auPolicy -Name "AUOptions" -Value 3 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $auPolicy -Name "ScheduledInstallDay" -Value 0 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $auPolicy -Name "ScheduledInstallTime" -Value 3 -PropertyType DWord -Force | Out-Null + +# Make the interactive active-hours UI reflect an ops-safe window. +New-ItemProperty -Path $uxSettings -Name "ActiveHoursStart" -Value 0 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $uxSettings -Name "ActiveHoursEnd" -Value 23 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $uxSettings -Name "SmartActiveHoursState" -Value 0 -PropertyType DWord -Force | Out-Null + +$after = @( + Get-RegistrySnapshot $windowsUpdatePolicy + Get-RegistrySnapshot $auPolicy + Get-RegistrySnapshot $uxSettings +) + +$pendingRebootKeys = @( + "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending", + "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired", + "HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager" +) + +$pending = @() +foreach ($key in $pendingRebootKeys) { + if ($key -like "*Session Manager") { + $value = (Get-ItemProperty -Path $key -Name "PendingFileRenameOperations" -ErrorAction SilentlyContinue).PendingFileRenameOperations + $pending += [pscustomobject]@{ path = $key; signal = "PendingFileRenameOperations"; present = [bool]$value } + } else { + $pending += [pscustomobject]@{ path = $key; signal = "key_exists"; present = [bool](Test-Path $key) } + } +} + +try { + gpupdate.exe /target:computer /force | Out-File -FilePath $logPath -Append -Encoding UTF8 + $gpupdateOk = $true +} catch { + $gpupdateOk = $false + Write-PolicyLog "gpupdate_failed error=$($_.Exception.Message)" +} + +$result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + applied = $true + policy = "windows_update_no_surprise_reboot" + gpupdateOk = $gpupdateOk + before = $before + after = $after + pendingRebootSignals = $pending + evidenceLog = $logPath +} + +$result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $evidencePath +Write-PolicyLog "apply_complete evidence=$evidencePath" +Write-Host $evidencePath diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json new file mode 100644 index 000000000..f02888aa3 --- /dev/null +++ b/agent99.config.99.example.json @@ -0,0 +1,273 @@ +{ + "agentName": "Agent99", + "agentRoot": "C:\\Wooo\\Agent99", + "sshUser": "wooo", + "sshUsers": { + "192.168.0.112": "kali", + "192.168.0.188": "ollama" + }, + "vmrunPath": "C:\\Program Files (x86)\\VMware\\VMware Workstation\\vmrun.exe", + "hosts": [ + "192.168.0.110", + "192.168.0.112", + "192.168.0.120", + "192.168.0.121", + "192.168.0.188" + ], + "vms": [ + { + "name": "host110-devops", + "host": "192.168.0.110", + "vmx": "D:\\Documents\\Virtual Machines\\Ubuntu 64-bit (3)\\Ubuntu 64-bit (3).vmx", + "priority": 10 + }, + { + "name": "host188-ai", + "host": "192.168.0.188", + "vmx": "D:\\Documents\\Virtual Machines\\Ollama_Ubuntu_64-bit\\Ollama_Ubuntu_64-bit.vmx", + "priority": 20 + }, + { + "name": "host112-kali", + "host": "192.168.0.112", + "vmx": "D:\\Downloads\\kali-linux-2025.4-vmware-amd64\\kali-linux-2025.4-vmware-amd64.vmwarevm\\kali-linux-2025.4-vmware-amd64.vmx", + "priority": 30 + }, + { + "name": "host120-k3s", + "host": "192.168.0.120", + "vmx": "D:\\Documents\\Virtual Machines\\192.168.0.120_Ubuntu_64-bit\\192.168.0.120_Ubuntu_64-bit.vmx", + "priority": 40 + }, + { + "name": "host121-k3s", + "host": "192.168.0.121", + "vmx": "D:\\Documents\\Virtual Machines\\192.168.0.121_Ubuntu_64-bit\\192.168.0.121_Ubuntu_64-bit.vmx", + "priority": 50 + } + ], + "k3s": { + "jumpHost": "192.168.0.110", + "primary": "192.168.0.120", + "namespace": "awoooi-prod" + }, + "publicUrls": [ + "awoooi.wooo.work", + "stock.wooo.work", + "2026fifa.wooo.work", + "gitea.wooo.work", + "harbor.wooo.work" + ], + "aiServices": [ + { + "name": "openclaw", + "url": "http://192.168.0.188:8088/health", + "host": "192.168.0.188", + "expectedStatus": "healthy", + "critical": true + } + ], + "providerFreshness": { + "enabled": true, + "lookbackMinutes": 120, + "targets": [ + { + "name": "provider_freshness_signal", + "providerWindowMinutes": null, + "directReference": null, + "verifier": "agent99-provider-freshness-triage" + } + ] + }, + "sreAlertRelay": { + "enabled": true, + "prefix": "http://+:8787/agent99/sre-alert/", + "tokenEnv": "AGENT99_SRE_RELAY_TOKEN", + "allowedRemotePrefixes": [ + "127.", + "::1", + "192.168.0." + ] + }, + "telegram": { + "enabled": true, + "botTokenEnv": "AGENT99_TELEGRAM_BOT_TOKEN", + "chatIdEnv": "AGENT99_TELEGRAM_CHAT_ID", + "botTokenEnvAliases": [ + "AWOOOI_TG_BOT_TOKEN", + "OPENCLAW_TG_BOT_TOKEN", + "TELEGRAM_BOT_TOKEN", + "TELEGRAM_TOKEN", + "TG_BOT_TOKEN", + "AIDER_WATCH_TELEGRAM_TOKEN" + ], + "chatIdEnvAliases": [ + "TELEGRAM_CHAT_ID", + "TELEGRAM_ALERT_CHAT_ID", + "TELEGRAM_P0_CHAT_ID", + "TELEGRAM_SECURITY_CHAT_ID", + "OPENCLAW_TG_CHAT_ID", + "TG_CHAT_ID", + "AIDER_WATCH_TELEGRAM_CHAT_ID" + ], + "relay": { + "enabled": true, + "host": "192.168.0.110", + "container": "stockplatform-v2-api-1" + }, + "allowedCommandChatTitlePattern": "AwoooI SRE|AWOOOI SRE|SRE", + "autoIngestMonitoringAlerts": true, + "alertAllOperations": false, + "alertInfoEvents": false, + "alertOperatorCommands": true, + "dedupeMinutes": 30 + }, + "performance": { + "enabled": true, + "intervalMinutes": 1, + "loadPerCoreWarning": 0.9, + "loadPerCoreCritical": 1.5, + "memoryAvailablePercentCritical": 10, + "diskUsedPercentWarning": 90, + "diskUsedPercentCritical": 95, + "captureTopCpuOnWarning": true, + "topCpuTimeoutSeconds": 10, + "loadShedding": { + "enabled": true, + "actions": [ + { + "name": "prune-188-docker-build-cache-disk-pressure", + "host": "192.168.0.188", + "command": "echo BEFORE_DF=$(df -P / | awk 'NR==2{print $5}'); docker system df; docker builder prune -af --filter until=24h; echo AFTER_DF=$(df -P / | awk 'NR==2{print $5}'); docker system df", + "timeoutSeconds": 240, + "postVerifyCommand": "df -P / | awk 'NR==2{gsub(\"%\",\"\",$5); print \"__DISK_USED__=\"$5}'", + "postVerifyMaxDiskUsedPercent": 89, + "rollback": "echo rollback_not_applicable_for_build_cache_prune", + "cooldownMinutes": 120, + "minSeverity": "warning", + "reasonMatches": [ + "disk_used_warning", + "disk_used_high" + ], + "enabled": true + }, + { + "name": "stop-110-runaway-alpine-tar-data", + "host": "192.168.0.110", + "command": "ids=$(docker ps --filter ancestor=alpine --format '{{.ID}} {{.Command}}' | awk '/tar czf - \\/data/ {print $1}'); [ -z \"$ids\" ] && exit 0; for id in $ids; do docker stop \"$id\"; done", + "rollback": "echo manual_review_required_for_stopped_runaway_container", + "cooldownMinutes": 10, + "minSeverity": "warning", + "reasonMatches": [ + "load_per_core_warning", + "load_per_core_critical" + ], + "enabled": true + }, + { + "name": "renice-110-awoooi-backup-pressure", + "host": "192.168.0.110", + "command": "pids=$(ps -eo pid,args | awk '/backup\\/scripts\\/backup-awoooi\\.sh|gzip -c|pg_dump --no-password/ && !/awk/ {print $1}'); [ -z \"$pids\" ] && exit 0; for p in $pids; do sudo -n renice -n 15 -p $p >/dev/null 2>&1 || true; sudo -n ionice -c2 -n7 -p $p >/dev/null 2>&1 || true; done; ps -o pid,ni,pri,comm,args -p $pids", + "rollback": "pids=$(ps -eo pid,args | awk '/backup\\/scripts\\/backup-awoooi\\.sh|gzip -c|pg_dump --no-password/ && !/awk/ {print $1}'); [ -z \"$pids\" ] && exit 0; for p in $pids; do sudo -n renice -n 0 -p $p >/dev/null 2>&1 || true; sudo -n ionice -c2 -n4 -p $p >/dev/null 2>&1 || true; done", + "cooldownMinutes": 15, + "minSeverity": "warning", + "reasonMatches": [ + "load_per_core_warning", + "load_per_core_critical" + ], + "enabled": true + }, + { + "name": "pause-110-ci-runner-pressure", + "host": "192.168.0.110", + "command": "docker pause vibework-dedicated-runner", + "rollback": "docker unpause vibework-dedicated-runner", + "cooldownMinutes": 30, + "reasonMatches": [ + "load_per_core_critical" + ], + "enabled": true + } + ] + } + }, + "selfHealth": { + "scheduledTasks": [ + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-SRE-Alert-Relay", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" + ], + "requiredHosts": [ + "192.168.0.110", + "192.168.0.112", + "192.168.0.120", + "192.168.0.121", + "192.168.0.188" + ], + "requireRecentTelegramDelivery": true, + "telegramDeliveryLookbackMinutes": 60, + "evidenceFreshness": [ + { + "name": "performance_guard", + "pattern": "agent99-Perf-*.json", + "maxAgeMinutes": 5, + "required": true + }, + { + "name": "heartbeat_status", + "pattern": "agent99-Status-*.json", + "maxAgeMinutes": 15, + "required": true + }, + { + "name": "self_health", + "pattern": "agent99-SelfCheck-*.json", + "maxAgeMinutes": 15, + "required": false + }, + { + "name": "backup_health", + "pattern": "agent99-BackupCheck-*.json", + "maxAgeMinutes": 30, + "required": false + }, + { + "name": "startup_recovery", + "pattern": "agent99-Recover-*.json", + "maxAgeMinutes": 1440, + "required": false + }, + { + "name": "telegram_inbox", + "pattern": "agent99-TelegramInbox-*.json", + "maxAgeMinutes": 15, + "required": false + }, + { + "name": "sre_alert_inbox", + "pattern": "agent99-SreAlertInbox-*.json", + "maxAgeMinutes": 15, + "required": false + }, + { + "name": "sre_alert_relay", + "pattern": "agent99-SreAlertRelay-start-*.json", + "maxAgeMinutes": 1440, + "required": false + } + ] + }, + "backupHealth": { + "host": "192.168.0.110", + "snapshotReadTimeoutSeconds": 90, + "targets": [], + "fileChecks": [], + "cronPatterns": [] + } +} diff --git a/agent99.config.example.json b/agent99.config.example.json new file mode 100644 index 000000000..fbd64b721 --- /dev/null +++ b/agent99.config.example.json @@ -0,0 +1,205 @@ +{ + "agentName": "Agent99", + "agentRoot": "C:\\Wooo\\Agent99", + "sshUser": "wooo", + "sshUsers": {}, + "vmrunPath": "", + "hosts": [ + "192.168.0.110", + "192.168.0.112", + "192.168.0.120", + "192.168.0.121", + "192.168.0.188" + ], + "vms": [ + { + "name": "110", + "host": "192.168.0.110", + "vmx": "" + }, + { + "name": "112", + "host": "192.168.0.112", + "vmx": "" + }, + { + "name": "120", + "host": "192.168.0.120", + "vmx": "" + }, + { + "name": "121", + "host": "192.168.0.121", + "vmx": "" + }, + { + "name": "188", + "host": "192.168.0.188", + "vmx": "" + } + ], + "publicUrls": [ + "awoooi.wooo.work", + "stock.wooo.work", + "2026fifa.wooo.work", + "gitea.wooo.work", + "harbor.wooo.work" + ], + "aiServices": [ + { + "name": "openclaw", + "url": "http://192.168.0.188:8088/health", + "host": "192.168.0.188", + "expectedStatus": "healthy", + "critical": true + } + ], + "providerFreshness": { + "enabled": true, + "lookbackMinutes": 120, + "targets": [ + { + "name": "provider_freshness_signal", + "providerWindowMinutes": null, + "directReference": null, + "verifier": "agent99-provider-freshness-triage" + } + ] + }, + "sreAlertRelay": { + "enabled": true, + "prefix": "http://+:8787/agent99/sre-alert/", + "tokenEnv": "AGENT99_SRE_RELAY_TOKEN", + "allowedRemotePrefixes": [ + "127.", + "::1", + "192.168.0." + ] + }, + "telegram": { + "enabled": true, + "botTokenEnv": "AGENT99_TELEGRAM_BOT_TOKEN", + "chatIdEnv": "AGENT99_TELEGRAM_CHAT_ID", + "botTokenEnvAliases": [ + "AWOOOI_TG_BOT_TOKEN", + "OPENCLAW_TG_BOT_TOKEN", + "TELEGRAM_BOT_TOKEN", + "TELEGRAM_TOKEN", + "TG_BOT_TOKEN", + "AIDER_WATCH_TELEGRAM_TOKEN" + ], + "chatIdEnvAliases": [ + "TELEGRAM_CHAT_ID", + "TELEGRAM_ALERT_CHAT_ID", + "TELEGRAM_P0_CHAT_ID", + "TELEGRAM_SECURITY_CHAT_ID", + "OPENCLAW_TG_CHAT_ID", + "TG_CHAT_ID", + "AIDER_WATCH_TELEGRAM_CHAT_ID" + ], + "relay": { + "enabled": true, + "host": "192.168.0.110", + "container": "stockplatform-v2-api-1" + }, + "allowedCommandChatTitlePattern": "AwoooI SRE|AWOOOI SRE|SRE", + "autoIngestMonitoringAlerts": true, + "alertAllOperations": false, + "alertInfoEvents": false, + "alertOperatorCommands": true, + "dedupeMinutes": 30 + }, + "performance": { + "enabled": true, + "intervalMinutes": 1, + "loadPerCoreWarning": 0.9, + "loadPerCoreCritical": 1.5, + "memoryAvailablePercentCritical": 10, + "diskUsedPercentWarning": 90, + "diskUsedPercentCritical": 95, + "captureTopCpuOnWarning": true, + "topCpuTimeoutSeconds": 10, + "loadShedding": { + "enabled": true, + "actions": [] + } + }, + "selfHealth": { + "scheduledTasks": [ + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-SRE-Alert-Relay", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" + ], + "requiredHosts": [ + "192.168.0.110", + "192.168.0.112", + "192.168.0.120", + "192.168.0.121", + "192.168.0.188" + ], + "requireRecentTelegramDelivery": true, + "telegramDeliveryLookbackMinutes": 60, + "evidenceFreshness": [ + { + "name": "performance_guard", + "pattern": "agent99-Perf-*.json", + "maxAgeMinutes": 5, + "required": true + }, + { + "name": "heartbeat_status", + "pattern": "agent99-Status-*.json", + "maxAgeMinutes": 15, + "required": true + }, + { + "name": "self_health", + "pattern": "agent99-SelfCheck-*.json", + "maxAgeMinutes": 15, + "required": false + }, + { + "name": "backup_health", + "pattern": "agent99-BackupCheck-*.json", + "maxAgeMinutes": 30, + "required": false + }, + { + "name": "startup_recovery", + "pattern": "agent99-Recover-*.json", + "maxAgeMinutes": 1440, + "required": false + }, + { + "name": "telegram_inbox", + "pattern": "agent99-TelegramInbox-*.json", + "maxAgeMinutes": 15, + "required": false + }, + { + "name": "sre_alert_inbox", + "pattern": "agent99-SreAlertInbox-*.json", + "maxAgeMinutes": 15, + "required": false + }, + { + "name": "sre_alert_relay", + "pattern": "agent99-SreAlertRelay-start-*.json", + "maxAgeMinutes": 1440, + "required": false + } + ] + }, + "backupHealth": { + "host": "192.168.0.110", + "targets": [], + "fileChecks": [], + "cronPatterns": [] + } +} diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 9397e6887..e20a2b7d1 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -56,6 +56,7 @@ from src.services.alertmanager_llm_guard import ( from src.services.approval_db import get_approval_service from src.services.auto_approve import get_auto_approve_policy from src.services.auto_repair_service import AutoRepairService +from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99 from src.services.channel_hub import ( record_alertmanager_event, record_grouped_alert_event, @@ -2849,6 +2850,21 @@ async def alertmanager_webhook( labels=dict(alert.labels) if alert.labels else {}, annotations=dict(alert.annotations) if alert.annotations else {}, ) + background_tasks.add_task( + bridge_alertmanager_to_agent99, + alert_id=alert_id, + alertname=alertname, + severity=severity, + namespace=namespace, + target_resource=target_resource, + message=message, + labels=dict(alert.labels) if alert.labels else {}, + annotations=dict(alert.annotations) if alert.annotations else {}, + fingerprint=fingerprint, + alert_category=alert_category, + notification_type=notification_type, + source_url=alert.generatorURL, + ) # ========================================================================== # ADR-076: 告警聚合引擎 — 5 分鐘滑動視窗，防止告警風暴 diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index 2f008d451..38291d433 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -925,6 +925,28 @@ class Settings(BaseSettings): default="", description="Telegram Webhook Secret Token（setWebhook 設定的同一值）", ) + AGENT99_SRE_ALERT_INBOX_PATH: str = Field( + default="", + description=( + "Optional mounted path for Agent99 SRE alert inbox. Empty disables the " + "fail-open Alertmanager to Agent99 file bridge." + ), + ) + AGENT99_SRE_ALERT_RELAY_URL: str = Field( + default="", + description=( + "Optional Agent99 SRE alert relay endpoint. When set, Alertmanager " + "events are posted to Agent99 before falling back to the mounted file inbox." + ), + ) + AGENT99_SRE_ALERT_RELAY_TOKEN: str = Field( + default="", + description="Optional shared token for Agent99 SRE alert relay POSTs.", + ) + AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS: float = Field( + default=3.0, + description="Timeout for fail-open Agent99 SRE alert relay POSTs.", + ) # 2026-04-24 Claude Sonnet 4.6 (ADR-095 WS4): Hermes NL 自然語言閘道 # false=不啟用（預設），true=啟用 @mention 問答（需 ANTHROPIC_API_KEY） HERMES_NL_ENABLED: bool = Field( diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py new file mode 100644 index 000000000..2f176506b --- /dev/null +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +import asyncio +import json +import re +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path +from typing import Any + +from src.core.config import settings +from src.core.logging import get_logger +from src.utils.timezone import now_taipei + +logger = get_logger("awoooi.agent99_sre_bridge") + +_SENSITIVE_KEY_RE = re.compile( + r"(token|secret|password|passwd|authorization|cookie|session|private[_-]?key)", + re.IGNORECASE, +) +_SAFE_TOKEN_RE = re.compile(r"[^A-Za-z0-9_.-]+") + + +def _safe_text(value: Any, *, max_length: int = 1200) -> str: + text = str(value or "") + text = " ".join(text.split()) + if len(text) > max_length: + return text[: max_length - 14] + "...(truncated)" + return text + + +def _safe_label_value(key: str, value: Any) -> str: + if _SENSITIVE_KEY_RE.search(key): + return "[redacted]" + return _safe_text(value, max_length=240) + + +def _safe_file_token(value: str) -> str: + safe = _SAFE_TOKEN_RE.sub("_", value).strip("._-") + return (safe or "alert")[:96] + + +def _safe_url_for_log(value: str) -> str: + if not value: + return "" + parsed = urllib.parse.urlsplit(value) + return urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", "")) + + +def _alert_text( + *, + alertname: str, + severity: str, + target_resource: str, + namespace: str, + message: str, + labels: dict[str, Any], + annotations: dict[str, Any], +) -> str: + return " ".join( + [ + alertname, + severity, + target_resource, + namespace, + message, + " ".join(f"{k}={v}" for k, v in labels.items()), + " ".join(f"{k}={v}" for k, v in annotations.items()), + ] + ).lower() + + +def resolve_agent99_alert_kind( + *, + alertname: str, + severity: str, + target_resource: str, + namespace: str, + message: str, + labels: dict[str, Any] | None = None, + annotations: dict[str, Any] | None = None, +) -> str: + text = _alert_text( + alertname=alertname, + severity=severity, + target_resource=target_resource, + namespace=namespace, + message=message, + labels=labels or {}, + annotations=annotations or {}, + ) + + if re.search( + r"provider_freshness_signal|source_provider_freshness_triage|" + r"raw_signal_normalized|controlled_runtime_candidate|freshness|" + r"last_seen|last seen|ingestion", + text, + ): + return "provider_freshness_signal" + if re.search(r"502|site_502|public_route|website|route_down|nginx upstream", text): + return "public_route_502" + if re.search(r"cpu|load|memory|mem_|disk|pressure|performance", text): + return "performance_pressure" + if re.search(r"backup|snapshot|restore drill|cron", text): + return "backup_health" + if re.search(r"harbor|registry", text): + return "harbor_registry" + if re.search(r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text): + return "awoooi_k3s" + if re.search(r"host_down|hostdown|vm_not_running|reboot|restart|powered_off", text): + return "host_recovery" + return "monitoring_alert" + + +def _suggested_mode_for_kind(kind: str) -> str | None: + return { + "provider_freshness_signal": "ProviderFreshness", + "public_route_502": "Recover", + "performance_pressure": "Perf", + "backup_health": "BackupCheck", + "harbor_registry": "HarborRepair", + "awoooi_k3s": "AwoooRepair", + "host_recovery": "Recover", + }.get(kind) + + +def build_agent99_sre_alert( + *, + alert_id: str, + alertname: str, + severity: str, + namespace: str, + target_resource: str, + message: str, + labels: dict[str, Any] | None = None, + annotations: dict[str, Any] | None = None, + fingerprint: str = "", + alert_category: str = "", + notification_type: str = "", + source_url: str | None = None, +) -> dict[str, Any]: + labels = labels or {} + annotations = annotations or {} + normalized_severity = (severity or "warning").lower() + kind = resolve_agent99_alert_kind( + alertname=alertname, + severity=normalized_severity, + target_resource=target_resource, + namespace=namespace, + message=message, + labels=labels, + annotations=annotations, + ) + suggested_mode = _suggested_mode_for_kind(kind) + host = str(labels.get("host") or labels.get("node") or labels.get("instance") or "") + if ":" in host: + host = host.split(":", 1)[0] + service = str( + labels.get("service") + or labels.get("app") + or labels.get("component") + or target_resource + or "awoooi" + ) + safe_labels = [ + "alertmanager", + "awoooi-api", + f"alertname={_safe_label_value('alertname', alertname)}", + f"namespace={_safe_label_value('namespace', namespace)}", + f"target={_safe_label_value('target', target_resource)}", + ] + if fingerprint: + safe_labels.append(f"fingerprint={_safe_label_value('fingerprint', fingerprint)}") + for key, value in sorted(labels.items()): + safe_labels.append(f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}") + if kind == "provider_freshness_signal": + safe_labels.extend(["provider-freshness", "no-false-green"]) + + instruction = None + if kind == "provider_freshness_signal": + instruction = ( + "Provider freshness candidate required; require providerWindow,lastSeen," + "directReference,verifierReadback; mark missing,expired,timeout," + "no_false_green; do not switch provider, call paid model, edit env, or reload." + ) + + payload: dict[str, Any] = { + "id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}", + "source": "awoooi-api-alertmanager", + "severity": normalized_severity, + "kind": kind, + "service": _safe_text(service, max_length=160), + "host": _safe_text(host, max_length=160), + "title": _safe_text(f"Alertmanager {alertname}", max_length=240), + "message": _safe_text(message, max_length=2000), + "labels": safe_labels, + "force": True, + "controlledApply": True, + "createdAt": now_taipei().isoformat(), + "awoooi": { + "alertId": _safe_text(alert_id, max_length=160), + "alertname": _safe_text(alertname, max_length=160), + "namespace": _safe_text(namespace, max_length=160), + "targetResource": _safe_text(target_resource, max_length=240), + "fingerprint": _safe_text(fingerprint, max_length=160), + "alertCategory": _safe_text(alert_category, max_length=160), + "notificationType": _safe_text(notification_type, max_length=160), + "sourceUrl": _safe_text(source_url or "", max_length=600), + }, + } + if suggested_mode: + payload["suggestedMode"] = suggested_mode + if instruction: + payload["instruction"] = instruction + return payload + + +def write_agent99_sre_alert( + payload: dict[str, Any], + *, + inbox_path: str | None = None, +) -> Path | None: + target_dir_text = inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH + if not target_dir_text: + logger.info( + "agent99_sre_bridge_disabled", + reason="AGENT99_SRE_ALERT_INBOX_PATH not configured", + alert_id=payload.get("id"), + kind=payload.get("kind"), + ) + return None + + target_dir = Path(target_dir_text) + target_dir.mkdir(parents=True, exist_ok=True) + alert_id = _safe_file_token(str(payload.get("id") or uuid.uuid4().hex)) + final_path = target_dir / f"{alert_id}.json" + tmp_path = target_dir / f".{alert_id}.{uuid.uuid4().hex}.tmp" + tmp_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) + tmp_path.replace(final_path) + logger.info( + "agent99_sre_alert_written", + path=str(final_path), + alert_id=payload.get("id"), + kind=payload.get("kind"), + ) + return final_path + + +def post_agent99_sre_alert( + payload: dict[str, Any], + *, + relay_url: str | None = None, + relay_token: str | None = None, + timeout_seconds: float | None = None, +) -> dict[str, Any] | None: + target_url = relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL + if not target_url: + return None + + token = relay_token if relay_token is not None else settings.AGENT99_SRE_ALERT_RELAY_TOKEN + timeout = ( + timeout_seconds + if timeout_seconds is not None + else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS + ) + data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") + headers = { + "Content-Type": "application/json; charset=utf-8", + "User-Agent": "awoooi-agent99-sre-bridge/1", + } + if token: + headers["X-Agent99-Relay-Token"] = token + + request = urllib.request.Request(target_url, data=data, headers=headers, method="POST") + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + status = int(response.getcode()) + body = response.read(4096).decode("utf-8", errors="replace") + except urllib.error.HTTPError as exc: + body = exc.read(4096).decode("utf-8", errors="replace") + raise RuntimeError(f"Agent99 relay HTTP {exc.code}: {body[:300]}") from exc + + if status >= 400: + raise RuntimeError(f"Agent99 relay HTTP {status}: {body[:300]}") + + logger.info( + "agent99_sre_alert_relay_posted", + relay_url=_safe_url_for_log(target_url), + alert_id=payload.get("id"), + kind=payload.get("kind"), + status=status, + ) + return {"status": status, "body": body} + + +def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str: + if settings.AGENT99_SRE_ALERT_RELAY_URL: + post_agent99_sre_alert(payload) + return "relay" + + if settings.AGENT99_SRE_ALERT_INBOX_PATH: + written = write_agent99_sre_alert(payload) + if written is not None: + return "file" + + logger.info( + "agent99_sre_bridge_disabled", + reason="no relay URL or mounted inbox path configured", + alert_id=payload.get("id"), + kind=payload.get("kind"), + ) + return "disabled" + + +async def bridge_alertmanager_to_agent99( + *, + alert_id: str, + alertname: str, + severity: str, + namespace: str, + target_resource: str, + message: str, + labels: dict[str, Any] | None = None, + annotations: dict[str, Any] | None = None, + fingerprint: str = "", + alert_category: str = "", + notification_type: str = "", + source_url: str | None = None, +) -> None: + try: + payload = build_agent99_sre_alert( + alert_id=alert_id, + alertname=alertname, + severity=severity, + namespace=namespace, + target_resource=target_resource, + message=message, + labels=labels, + annotations=annotations, + fingerprint=fingerprint, + alert_category=alert_category, + notification_type=notification_type, + source_url=source_url, + ) + await asyncio.to_thread(dispatch_agent99_sre_alert, payload) + except Exception as exc: + logger.warning( + "agent99_sre_bridge_failed_fail_open", + alert_id=alert_id, + alertname=alertname, + error=str(exc), + ) diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py new file mode 100644 index 000000000..e62023656 --- /dev/null +++ b/apps/api/tests/test_agent99_sre_bridge.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import json + +from src.services.agent99_sre_bridge import ( + build_agent99_sre_alert, + post_agent99_sre_alert, + resolve_agent99_alert_kind, + write_agent99_sre_alert, +) + + +def test_provider_freshness_alert_builds_agent99_contract() -> None: + payload = build_agent99_sre_alert( + alert_id="alert-20260709233000", + alertname="provider_freshness_signal", + severity="warning", + namespace="awoooi-prod", + target_resource="provider_freshness_signal", + message=( + "raw_signal_normalized controlled_runtime_candidate freshness " + "last seen missing direct ref verifier readback" + ), + labels={ + "service": "awoooi", + "token": "must-not-leak", + }, + annotations={ + "summary": "Provider freshness evidence is missing", + }, + fingerprint="fp-provider-freshness", + alert_category="source_provider_freshness_triage", + notification_type="raw_signal_normalized", + source_url="https://awoooi.wooo.work/api/v1/webhooks/alertmanager", + ) + + assert payload["source"] == "awoooi-api-alertmanager" + assert payload["kind"] == "provider_freshness_signal" + assert payload["suggestedMode"] == "ProviderFreshness" + assert payload["controlledApply"] is True + assert payload["force"] is True + assert "provider-freshness" in payload["labels"] + assert "no-false-green" in payload["labels"] + assert "token=[redacted]" in payload["labels"] + assert "must-not-leak" not in json.dumps(payload) + assert "do not switch provider" in payload["instruction"] + + +def test_alert_kind_mapping_for_route_and_performance() -> None: + assert ( + resolve_agent99_alert_kind( + alertname="PublicRoute502", + severity="critical", + namespace="awoooi-prod", + target_resource="awoooi-web", + message="HTTP 502", + ) + == "public_route_502" + ) + assert ( + resolve_agent99_alert_kind( + alertname="HostHighCpuLoad", + severity="warning", + namespace="node", + target_resource="192.168.0.110", + message="load per core is high", + ) + == "performance_pressure" + ) + + +def test_write_agent99_sre_alert_to_configured_inbox(tmp_path) -> None: + payload = build_agent99_sre_alert( + alert_id="alert-write-smoke", + alertname="HostHighCpuLoad", + severity="warning", + namespace="node", + target_resource="192.168.0.110", + message="CPU load per core is high", + labels={"host": "192.168.0.110"}, + ) + + written = write_agent99_sre_alert(payload, inbox_path=str(tmp_path)) + + assert written is not None + data = json.loads(written.read_text(encoding="utf-8")) + assert data["id"] == "awoooi-alertmanager-alert-write-smoke" + assert data["kind"] == "performance_pressure" + assert data["suggestedMode"] == "Perf" + + +def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None: + payload = build_agent99_sre_alert( + alert_id="relay-smoke", + alertname="provider_freshness_signal", + severity="warning", + namespace="awoooi-prod", + target_resource="provider_freshness_signal", + message="freshness last_seen missing", + labels={"service": "awoooi", "authorization": "must-not-leak"}, + ) + seen: dict[str, object] = {} + + class FakeResponse: + def getcode(self) -> int: + return 202 + + def read(self, _limit: int) -> bytes: + return b'{"ok":true}' + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def fake_urlopen(request, timeout): # type: ignore[no-untyped-def] + seen["url"] = request.full_url + seen["timeout"] = timeout + seen["headers"] = dict(request.header_items()) + seen["payload"] = json.loads(request.data.decode("utf-8")) + return FakeResponse() + + monkeypatch.setattr( + "src.services.agent99_sre_bridge.urllib.request.urlopen", + fake_urlopen, + ) + + receipt = post_agent99_sre_alert( + payload, + relay_url="http://192.168.0.99:8787/agent99/sre-alert/", + relay_token="relay-token", + timeout_seconds=1.5, + ) + + assert receipt == {"status": 202, "body": '{"ok":true}'} + assert seen["url"] == "http://192.168.0.99:8787/agent99/sre-alert/" + assert seen["timeout"] == 1.5 + assert seen["payload"]["kind"] == "provider_freshness_signal" + assert seen["payload"]["suggestedMode"] == "ProviderFreshness" + assert "authorization=[redacted]" in seen["payload"]["labels"] + assert "must-not-leak" not in json.dumps(seen["payload"]) + assert seen["headers"]["X-agent99-relay-token"] == "relay-token" diff --git a/host-reboot-scorecard.ps1 b/host-reboot-scorecard.ps1 new file mode 100644 index 000000000..4c6fdb0d6 --- /dev/null +++ b/host-reboot-scorecard.ps1 @@ -0,0 +1,197 @@ +param( + [switch]$ControlledApply, + [string]$SshUser = "wooo", + [string]$EvidenceDir = "$env:ProgramData\Wooo\host-reboot-scorecard", + [string[]]$Hosts = @("192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"), + [string[]]$PublicUrls = @( + "awoooi.wooo.work", + "stock.wooo.work", + "2026fifa.wooo.work", + "gitea.wooo.work", + "harbor.wooo.work" + ) +) + +$ErrorActionPreference = "Continue" +New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null + +$stamp = Get-Date -Format "yyyyMMdd-HHmmss" +$jsonPath = Join-Path $EvidenceDir "scorecard-$stamp.json" +$logPath = Join-Path $EvidenceDir "scorecard-$stamp.log" + +function Write-Evidence { + param([string]$Message) + $line = "$(Get-Date -Format o) $Message" + $line | Tee-Object -FilePath $logPath -Append +} + +function Test-PingHost { + param([string]$Host) + $ok = Test-Connection -ComputerName $Host -Count 1 -Quiet -ErrorAction SilentlyContinue + [pscustomobject]@{ + host = $Host + ping = [bool]$ok + } +} + +function Invoke-SshText { + param( + [string]$Host, + [string]$Command, + [int]$TimeoutSeconds = 20 + ) + + $args = @( + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=$TimeoutSeconds", + "-l", $SshUser, + $Host, + $Command + ) + + try { + $output = & ssh @args 2>&1 + [pscustomobject]@{ + ok = ($LASTEXITCODE -eq 0) + exit_code = $LASTEXITCODE + output = ($output -join "`n") + } + } catch { + [pscustomobject]@{ + ok = $false + exit_code = -1 + output = $_.Exception.Message + } + } +} + +function Test-HttpHead { + param([string]$Url) + try { + $output = & curl.exe --location --head --max-time 12 $Url 2>&1 + $text = $output -join "`n" + $statusMatches = [regex]::Matches($text, "HTTP/\S+\s+(\d+)") + $lastStatus = if ($statusMatches.Count -gt 0) { [int]$statusMatches[$statusMatches.Count - 1].Groups[1].Value } else { $null } + [pscustomobject]@{ + url = $Url + status = $lastStatus + non_502 = ($lastStatus -ne 502 -and $null -ne $lastStatus) + ok = ($lastStatus -ge 200 -and $lastStatus -lt 500) + output = $text + } + } catch { + [pscustomobject]@{ + url = $Url + status = $null + non_502 = $false + ok = $false + output = $_.Exception.Message + } + } +} + +function Get-HarborState { + $core = Invoke-SshText "192.168.0.110" "docker ps --filter name=harbor-core" + $registry = Invoke-SshText "192.168.0.110" "docker ps --filter name=registry" + $redis = Invoke-SshText "192.168.0.110" "docker ps --filter name=redis" + + [pscustomobject]@{ + core_ok = ($core.output -match "harbor-core" -and $core.output -match "healthy") + registry_ok = ($registry.output -match "registry" -and $registry.output -match "healthy") + redis_ok = ($redis.output -match "redis" -and $redis.output -match "Up") + core = $core.output + registry = $registry.output + redis = $redis.output + } +} + +function Get-AwoooDeployState { + $deploy = Invoke-SshText "192.168.0.120" "sudo -n kubectl get deploy -n awoooi-prod" + $pods = Invoke-SshText "192.168.0.120" "sudo -n kubectl get pods -n awoooi-prod" + [pscustomobject]@{ + deploy_ok = ( + $deploy.output -match "awoooi-api\s+1/1" -and + $deploy.output -match "awoooi-web\s+2/2" -and + $deploy.output -match "awoooi-worker\s+1/1" + ) + image_pull_failure = ($pods.output -match "ImagePullBackOff|ErrImagePull") + crash_failure = ($pods.output -match "CrashLoopBackOff|Pending") + deploy = $deploy.output + pods = $pods.output + } +} + +function Repair-HarborIfAllowed { + param([object]$Harbor) + if (-not $ControlledApply) { return } + + if (-not $Harbor.redis_ok) { + Write-Evidence "CONTROLLED_APPLY restart harbor redis" + Invoke-SshText "192.168.0.110" "docker restart redis" | Out-Null + } + + if (-not $Harbor.core_ok) { + Write-Evidence "CONTROLLED_APPLY restart harbor-core" + Invoke-SshText "192.168.0.110" "docker restart harbor-core" | Out-Null + } +} + +function Repair-AwoooPodsIfAllowed { + param([object]$Awooo, [object]$Harbor) + if (-not $ControlledApply) { return } + if (-not $Harbor.core_ok) { return } + if (-not $Awooo.image_pull_failure) { return } + + Write-Evidence "CONTROLLED_APPLY delete AWOOOI ImagePullBackOff pods after Harbor core healthy" + Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-api" | Out-Null + Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-web" | Out-Null + Invoke-SshText "192.168.0.120" "sudo -n kubectl delete pod -n awoooi-prod --selector app=awoooi-worker" | Out-Null +} + +Write-Evidence "scorecard_start controlled_apply=$ControlledApply" + +$hostResults = @() +foreach ($h in $Hosts) { + $result = Test-PingHost $h + $hostResults += $result + Write-Evidence "host_ping host=$($result.host) ok=$($result.ping)" +} + +$harbor = Get-HarborState +Write-Evidence "harbor core_ok=$($harbor.core_ok) registry_ok=$($harbor.registry_ok) redis_ok=$($harbor.redis_ok)" +Repair-HarborIfAllowed $harbor + +$awooo = Get-AwoooDeployState +Write-Evidence "awoooi deploy_ok=$($awooo.deploy_ok) image_pull_failure=$($awooo.image_pull_failure) crash_failure=$($awooo.crash_failure)" +Repair-AwoooPodsIfAllowed $awooo $harbor + +$httpResults = @() +foreach ($u in $PublicUrls) { + $result = Test-HttpHead $u + $httpResults += $result + Write-Evidence "http url=$($result.url) status=$($result.status) non_502=$($result.non_502) ok=$($result.ok)" +} + +$summary = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + controlled_apply = [bool]$ControlledApply + hosts = $hostResults + harbor = $harbor + awoooi = $awooo + public_http = $httpResults + completed = ( + ($hostResults | Where-Object { -not $_.ping }).Count -eq 0 -and + $harbor.core_ok -and + $awooo.deploy_ok -and + ($httpResults | Where-Object { -not $_.non_502 }).Count -eq 0 + ) +} + +$summary | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath +Write-Evidence "scorecard_complete completed=$($summary.completed) json=$jsonPath" + +if (-not $summary.completed) { + exit 2 +} + +exit 0 diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index d62467579..f4b348df7 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -107,6 +107,12 @@ data: # OPENCLAW_BOT_TOKEN / NEMOTRON_BOT_TOKEN 在 Secrets 中配置 # ============================================================================ SRE_GROUP_CHAT_ID: "-1003711974679" + # 2026-07-10 Codex: Agent99 direct SRE relay. Alertmanager still lands in + # AWOOI API first; the sanitized bridge then POSTs to 99 so Agent99 can queue + # allowlisted recovery and write evidence/KM. Optional relay token stays in + # Secret/env only; no token value is stored here. + AGENT99_SRE_ALERT_RELAY_URL: "http://192.168.0.99:8787/agent99/sre-alert/" + AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS: "3" # 快取 TTL (秒) CACHE_TTL_DASHBOARD: "300"