fix(api): route runaway host alerts to ai event packets

apps/api/src/services/telegram_gateway.py

@@ -97,6 +97,9 @@ _HOST_RESOURCE_ALERT_HEADER_RE = re.compile(
 _HOST_RESOURCE_TARGET_RE = re.compile(
     r"\b(?:WARN|CRIT|INFO)\s+(?P<target>[A-Za-z0-9_.-]+)\b"
 )
+_HOST_RESOURCE_ALERTNAME_RE = re.compile(r"\balertname\s*=\s*\"?(?P<alertname>[A-Za-z0-9_.:-]+)\"?")
+_HOST_RESOURCE_HOST_LABEL_RE = re.compile(r"\bhost\s*=\s*\"?(?P<host>[A-Za-z0-9_.:-]+)\"?")
+_HOST_RESOURCE_RULE_LABEL_RE = re.compile(r"\brule\s*=\s*\"?(?P<rule>[A-Za-z0-9_.:-]+)\"?")
 _HOST_PROCESS_LINE_RE = re.compile(
     r"^\s*(?P<user>\S+)\s+"
     r"(?P<pid>\d+)\s+"
@@ -112,6 +115,10 @@ def _is_host_resource_alert_text(text: str) -> bool:
         "CPU 警告" in text
         or "容器內 root Node.js 進程" in text
         or ("ps aux" in text and ("next build" in text or "npm run build" in text))
+        or "HostOrphanBrowserSmokeHighCpu" in text
+        or "HostCiRunnerLoadSaturation" in text
+        or "awoooi_host_runaway_browser_orphan" in text
+        or "awoooi_host_gitea_actions_active_container_count" in text
     )
 
 
@@ -165,10 +172,15 @@ def _parse_host_process_lines(text: str) -> list[dict[str, str | float]]:
 
 
 def _host_resource_alert_impact(
+    text: str,
     cpu_text: str,
     load_text: str,
     processes: list[dict[str, str | float]],
 ) -> str:
+    if "HostOrphanBrowserSmokeHighCpu" in text or "awoooi_host_runaway_browser_orphan" in text:
+        return "orphan Chrome / Playwright smoke 疑似吃滿 CPU；先驗 pgid、age、cmdline 與 active CI 分流"
+    if "HostCiRunnerLoadSaturation" in text or "awoooi_host_gitea_actions_active_container_count" in text:
+        return "CI runner 正在造成主機負載；先確認 Actions run、queue、timeout 與服務 SLO"
     try:
         load = float(load_text)
     except (TypeError, ValueError):
@@ -186,7 +198,20 @@ def _host_resource_alert_impact(
     return "資源升高但尚未確認根因；先聚合觀察並補足 owner 判讀"
 
 
-def _host_resource_automation_lane(processes: list[dict[str, str | float]]) -> tuple[str, str]:
+def _host_resource_automation_lane(
+    text: str,
+    processes: list[dict[str, str | float]],
+) -> tuple[str, str]:
+    if "HostOrphanBrowserSmokeHighCpu" in text or "awoooi_host_runaway_browser_orphan" in text:
+        return (
+            "orphan_browser_smoke_runaway_process",
+            "建立 runaway process triage packet；先跑 remediation dry-run，待 owner / window / evidence 後才可 SIGTERM",
+        )
+    if "HostCiRunnerLoadSaturation" in text or "awoooi_host_gitea_actions_active_container_count" in text:
+        return (
+            "ci_runner_load_saturation",
+            "建立 CI load evidence packet，彙整 Gitea Actions run、runner queue、load/core 與 swap；不 kill process",
+        )
     commands = " ".join(str(item.get("command", "")).lower() for item in processes)
     if "build" in commands:
         return (
@@ -204,6 +229,69 @@ def _host_resource_automation_lane(processes: list[dict[str, str | float]]) -> t
     )
 
 
+def _host_resource_alert_label(name: str, text: str) -> str:
+    patterns = {
+        "alertname": _HOST_RESOURCE_ALERTNAME_RE,
+        "host": _HOST_RESOURCE_HOST_LABEL_RE,
+        "rule": _HOST_RESOURCE_RULE_LABEL_RE,
+    }
+    match = patterns[name].search(text)
+    return match.group(name) if match else ""
+
+
+def _host_resource_alert_evidence_lines(
+    text: str,
+    processes: list[dict[str, str | float]],
+) -> list[str]:
+    if processes:
+        lines: list[str] = []
+        for item in processes[:3]:
+            process_cpu = f"{float(item['cpu']):g}%"
+            lines.append(
+                "├ "
+                f"<code>PID {html.escape(str(item['pid']))}</code> "
+                f"CPU <code>{html.escape(process_cpu)}</code>："
+                f"<code>{html.escape(str(item['command']))}</code>"
+            )
+        lines[-1] = "└" + lines[-1][1:]
+        return lines
+
+    alertname = _host_resource_alert_label("alertname", text)
+    rule = _host_resource_alert_label("rule", text)
+    if alertname:
+        lines = [
+            f"├ Alert：<code>{html.escape(alertname)}</code>",
+            "├ Metric：<code>awoooi_host_runaway_process_*</code>",
+        ]
+        if rule:
+            lines.append(f"└ Rule：<code>{html.escape(rule)}</code>")
+        else:
+            lines[-1] = "└" + lines[-1][1:]
+        return lines
+
+    return ["└ 尚未收到可解析的 top process，請補只讀 evidence。"]
+
+
+def _host_resource_recommendation_lines(text: str) -> list[str]:
+    if "HostOrphanBrowserSmokeHighCpu" in text or "awoooi_host_runaway_browser_orphan" in text:
+        return [
+            "├ 先讀 Prometheus orphan group / CPU / age / cmdline 指標與 textfile timestamp",
+            "├ 執行 `host-runaway-process-remediation.py` dry-run 產生候選，不直接 apply",
+            "└ 若 owner approval、maintenance window、evidence ref 齊全，才可 gated SIGTERM 並回寫 KM / PlayBook / Verifier",
+        ]
+    if "HostCiRunnerLoadSaturation" in text or "awoooi_host_gitea_actions_active_container_count" in text:
+        return [
+            "├ 確認 Gitea Actions run、runner queue、build timeout、load/core 與 swap trend",
+            "├ 若是合法 CI，標記為 capacity / queue 事件，不做 process remediation",
+            "└ 若 CI 卡死，產出 owner packet 與 runner cleanup dry-run，再進維護窗口",
+        ]
+    return [
+        "├ 確認是否為 CI/CD / Actions / runner 正常建置窗口",
+        "├ 若持續超過門檻，先查 runner queue、build job、容器資源限制與服務 SLO",
+        "└ 同一 host/service 5 分鐘聚合一次，避免洗版",
+    ]
+
+
 def format_host_resource_alert_card(text: str) -> str:
     """把 host CPU/load raw dump 轉成值班者可讀的 Telegram HTML 卡。"""
     if not _is_host_resource_alert_text(text):
@@ -214,29 +302,21 @@ def format_host_resource_alert_card(text: str) -> str:
     target = (
         header.group("target")
         if header
-        else (target_match.group("target") if target_match else "unknown-host")
+        else (
+            target_match.group("target")
+            if target_match
+            else (_host_resource_alert_label("host", text) or "unknown-host")
+        )
     )
     cpu = header.group("cpu") if header else "-"
     load = header.group("load") if header else "-"
     processes = _parse_host_process_lines(text)
-    impact = _host_resource_alert_impact(cpu, load, processes)
-    automation_lane, automation_next_step = _host_resource_automation_lane(processes)
+    impact = _host_resource_alert_impact(text, cpu, load, processes)
+    automation_lane, automation_next_step = _host_resource_automation_lane(text, processes)
     load_bar = _resource_load_bar(load)
     severity = "🔴" if load != "-" and load_bar.count("■") >= 7 else "⚠️"
-
-    evidence_lines: list[str] = []
-    for item in processes[:3]:
-        process_cpu = f"{float(item['cpu']):g}%"
-        evidence_lines.append(
-            "├ "
-            f"<code>PID {html.escape(str(item['pid']))}</code> "
-            f"CPU <code>{html.escape(process_cpu)}</code>："
-            f"<code>{html.escape(str(item['command']))}</code>"
-        )
-    if evidence_lines:
-        evidence_lines[-1] = "└" + evidence_lines[-1][1:]
-    else:
-        evidence_lines.append("└ 尚未收到可解析的 top process，請補只讀 evidence。")
+    evidence_lines = _host_resource_alert_evidence_lines(text, processes)
+    recommendation_lines = _host_resource_recommendation_lines(text)
 
     return "\n".join(
         [
@@ -257,9 +337,7 @@ def format_host_resource_alert_card(text: str) -> str:
             *evidence_lines,
             "",
             "<b>建議下一步</b>",
-            "├ 確認是否為 CI/CD / Actions / runner 正常建置窗口",
-            "├ 若持續超過門檻，先查 runner queue、build job、容器資源限制與服務 SLO",
-            "└ 同一 host/service 5 分鐘聚合一次，避免洗版",
+            *recommendation_lines,
             "",
             "<b>禁止事項</b>",
             "└ 不 kill process、不 restart Docker / Gitea、不 reload Nginx、不改 firewall；除非已有維護窗口與 owner 批准。",