chore(observability): clarify quick review completion copy

services/elephant_alpha_autonomous_engine.py

@@ -8,8 +8,8 @@ AI 3.0 Autonomous Operations:
 - Continuous improvement loop
 
 ADR-012 Compliance:
-  §③ 單一 audit trail — 所有基行完畢後必發 triaged_alert Telegram
-  §⑤ 雙寫強制 — ai_insights (由 orchestrator._log_decision) + Telegram
+  §③ 單一 audit trail — 有實證的行動/升級需寫入 ai_insights 並發 triaged_alert Telegram
+  §⑤ 雙寫強制 — 無實證低信心升級只寫 suppressed cooldown，避免 human_review 噪音
 ADR-013 Compliance:
   resource_optimization trigger → auto_heal_service.handle_exception
 """
@@ -42,10 +42,18 @@ SSH_PORT = int(os.getenv("ELEPHANT_ALPHA_SSH_PORT", "22"))
 SSH_CONNECT_TIMEOUT = int(os.getenv("ELEPHANT_ALPHA_SSH_CONNECT_TIMEOUT", "10"))
 SSH_COMMAND_TIMEOUT = int(os.getenv("ELEPHANT_ALPHA_SSH_COMMAND_TIMEOUT", "60"))
 
-CACHE_DB_PATH = os.getenv("ELEPHANT_ALPHA_CACHE_DB", ":memory:")
+_DEFAULT_CACHE_DB_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(__file__)),
+    "data",
+    "elephant_alpha_cache.db",
+)
+CACHE_DB_PATH = os.getenv("ELEPHANT_ALPHA_CACHE_DB", _DEFAULT_CACHE_DB_PATH)
 ESCALATION_COOLDOWN_MIN = int(os.getenv("ELEPHANT_ALPHA_ESCALATION_COOLDOWN_MIN", "30"))
+NO_EVIDENCE_ESCALATION_COOLDOWN_MIN = int(os.getenv("ELEPHANT_ALPHA_NO_EVIDENCE_COOLDOWN_MIN", "360"))
 CONFIDENCE_THRESHOLD = float(os.getenv("ELEPHANT_ALPHA_CONFIDENCE_THRESHOLD", "0.7"))
 MAX_AUTONOMOUS_DECISIONS_PER_HOUR = int(os.getenv("ELEPHANT_ALPHA_MAX_AUTONOMOUS_DECISIONS_PER_HOUR", "10"))
+RESOURCE_QUEUE_THRESHOLD = int(os.getenv("ELEPHANT_ALPHA_RESOURCE_QUEUE_THRESHOLD", "10"))
+RESOURCE_LOAD_THRESHOLD = float(os.getenv("ELEPHANT_ALPHA_RESOURCE_LOAD_THRESHOLD", "80"))
 
 # ---- Constants ----
 _ALLOWED_ACTION_TYPES = frozenset({
@@ -123,6 +131,15 @@ _PRICE_RELATED_TRIGGERS = frozenset({
     "threat_escalation",
 })
 
+_OPERATIONAL_PENDING_ACTION_TYPES = frozenset({
+    "agent_safe_action",
+    "auto_heal",
+    "resource_optimization",
+    "scheduler_retry",
+})
+
+_NO_EVIDENCE_DEDUP_PREFIX = "no_evidence:"
+
 
 def _zh_trigger(trigger_type: str) -> str:
     return _TRIGGER_ZH.get(trigger_type, trigger_type)
@@ -210,6 +227,8 @@ class ElephantAlphaAutonomousEngine:
     # ---- DB ----
     def _init_cache_db(self) -> None:
         self._db_lock = threading.Lock()
+        if CACHE_DB_PATH != ":memory:":
+            os.makedirs(os.path.dirname(CACHE_DB_PATH), exist_ok=True)
         self._conn = sqlite3.connect(CACHE_DB_PATH, check_same_thread=False)
         self._conn.execute("""
             CREATE TABLE IF NOT EXISTS escalation_dedup (
@@ -315,9 +334,7 @@ class ElephantAlphaAutonomousEngine:
         for trigger in self.triggers:
             if not trigger.enabled:
                 continue
-            cooldown_min = self._get_cooldown(trigger.trigger_type)
-            last = trigger.last_triggered
-            if last and (datetime.now() - last).total_seconds() / 60 < cooldown_min:
+            if self._is_trigger_in_cooldown(trigger):
                 continue
             if await self._evaluate_trigger(trigger):
                 await self._execute_autonomous_decision(trigger)
@@ -327,8 +344,32 @@ class ElephantAlphaAutonomousEngine:
         return self._get_cooldown_min(trigger_type)
 
     def _get_cooldown_min(self, trigger_type: str) -> int:
+        if trigger_type.startswith(_NO_EVIDENCE_DEDUP_PREFIX):
+            return NO_EVIDENCE_ESCALATION_COOLDOWN_MIN
         return ESCALATION_COOLDOWN_MIN
 
+    def _is_trigger_in_cooldown(self, trigger: AutonomousTrigger) -> bool:
+        """同時使用記憶體與持久化冷卻，避免 thread/container 重啟後重送噪音告警。"""
+        def _within_cooldown(trigger_key: str, last_ts: Optional[float]) -> bool:
+            stored_ts = self._load_escalation(trigger_key)
+            if stored_ts:
+                last_ts = max(last_ts or 0, float(stored_ts))
+
+            if not last_ts:
+                return False
+
+            cooldown_sec = self._get_cooldown_min(trigger_key) * 60
+            return (datetime.now().timestamp() - last_ts) < cooldown_sec
+
+        last_ts: Optional[float] = None
+        if trigger.last_triggered:
+            last_ts = trigger.last_triggered.timestamp()
+
+        if _within_cooldown(trigger.trigger_type, last_ts):
+            return True
+
+        return _within_cooldown(f"{_NO_EVIDENCE_DEDUP_PREFIX}{trigger.trigger_type}", None)
+
     async def _evaluate_trigger(self, trigger: AutonomousTrigger) -> bool:
         try:
             if trigger.trigger_type == "price_drop_alert":
@@ -415,8 +456,10 @@ class ElephantAlphaAutonomousEngine:
             session.close()
 
     async def _check_resource_optimization_trigger(self, trigger: AutonomousTrigger) -> bool:
-        return (self._get_action_queue_size() > 10
-                or self._get_system_load_percentage() > 80)
+        operational_queue_size = self._get_action_queue_size()
+        system_load_pct = self._get_system_load_percentage()
+        return (operational_queue_size > RESOURCE_QUEUE_THRESHOLD
+                or system_load_pct > RESOURCE_LOAD_THRESHOLD)
 
     async def _check_code_exception_trigger(self, trigger: AutonomousTrigger) -> bool:
         containers = trigger.conditions.get("scan_containers", ["momo-pro-system", "momo-scheduler"])
@@ -696,19 +739,19 @@ class ElephantAlphaAutonomousEngine:
         將「步驟 1: [OpenClaw] 生成策略」這類元流程文字換成
         「[SKU] 商品｜MOMO $X / PChome $Y｜流失 NT$ Z｜建議 NT$ W」具體可決策行動。
 
-        失敗回 None，由呼叫端 fallback 至既有 execution_plan 文字。
-        本方法為 best-effort：任何例外都不阻斷 escalation 主流程。
+        失敗回 None，由呼叫端判斷是否 suppressed；不得 fallback 至 LLM plan 文字。
+        本方法為 best-effort：任何例外都不阻斷 escalation 判斷流程。
 
         Critic High-1 fix: 加 5 秒短超時防止阻塞 escalation cooldown 視窗
           （Hermes 完整 run 可能 30-60s，HITL 訊息應快速送出）
-        Critic High-2 fix: 若每筆都缺 loss/rec_price，視同無料、return None 觸發 fallback
+        Critic High-2 fix: 若每筆都缺 loss/rec_price，視同無料、return None 觸發 suppressed
         """
         # 使用 5s 短超時：Hermes 熱駐留時實測 < 10s，但若需冷啟動會拖到 30s+
-        # HITL 訊息延遲不可大於 10s（影響統帥決策時效性），寧可 fallback 到原 plan 文字
+        # HITL 訊息延遲不可大於 10s；timeout 後由呼叫端 suppressed，避免無實證文字洗版
         try:
             result = await asyncio.wait_for(self._hermes_analyze(), timeout=5)
         except asyncio.TimeoutError:
-            self._log.warning("Pre-fetch Hermes 5s timeout; falling back to plan text")
+            self._log.warning("Pre-fetch Hermes 5s timeout; no concrete data for escalation")
             return None
         except Exception as e:
             self._log.warning("Pre-fetch Hermes threats failed (non-blocking): %s", e)
@@ -716,7 +759,7 @@ class ElephantAlphaAutonomousEngine:
 
         threats = getattr(result, "threats", None) or []
         if not threats:
-            self._log.info("Pre-fetch Hermes returned 0 threats; falling back to plan text")
+            self._log.info("Pre-fetch Hermes returned 0 threats; no concrete data for escalation")
             return None
 
         # 模組頂部 import 較乾淨，但這裡保留 lazy import 避免兩服務循環依賴
@@ -756,8 +799,8 @@ class ElephantAlphaAutonomousEngine:
 
         if not any_concrete:
             # Critic High-2: 全部都只有「MOMO $X vs PChome $Y」乾巴巴兩行，
-            # 比原本「步驟 1:OpenClaw 生成策略」更空泛。返回 None 觸發 plan fallback
-            self._log.info("Pre-fetch threats lacked impact figures on all rows; falling back")
+            # 比原本「步驟 1:OpenClaw 生成策略」更空泛。返回 None 觸發 suppressed
+            self._log.info("Pre-fetch threats lacked impact figures on all rows; no concrete data for escalation")
             return None
 
         self._log.info("Pre-fetch Hermes threats produced %d concrete actions", len(lines))
@@ -881,8 +924,106 @@ class ElephantAlphaAutonomousEngine:
         except Exception as e:
             self._log.error("Telegram audit failed (non-blocking): %s", e)
 
+    def _build_resource_escalation_actions(self) -> Optional[List[str]]:
+        queue_size = self._safe_metric(self._get_action_queue_size, default=0)
+        system_load_pct = self._safe_metric(self._get_system_load_percentage, default=0.0)
+
+        evidence = []
+        if queue_size > RESOURCE_QUEUE_THRESHOLD:
+            evidence.append(f"auto action queue {queue_size} 筆 > 門檻 {RESOURCE_QUEUE_THRESHOLD} 筆")
+        if system_load_pct > RESOURCE_LOAD_THRESHOLD:
+            evidence.append(f"CPU load {system_load_pct:.1f}% > 門檻 {RESOURCE_LOAD_THRESHOLD:.0f}%")
+
+        if not evidence:
+            return None
+
+        return [
+            f"📊 實測資源訊號：{'；'.join(evidence)}",
+            "🔎 優先檢查 action_plans 的 auto_pending/running 任務是否卡住",
+            "🔧 僅允許處置 app/scheduler 類服務；禁止操作 momo-db 容器生命週期",
+        ]
+
+    @staticmethod
+    def _build_code_exception_escalation_actions(trigger: AutonomousTrigger) -> Optional[List[str]]:
+        error_msg = (trigger.temp_error_msg or "").strip()
+        if not error_msg:
+            return None
+
+        first_line = next((line.strip() for line in error_msg.splitlines() if line.strip()), "容器日誌偵測到 Python 例外")
+        target = trigger.temp_target_file or "未能自動定位檔案"
+        return [
+            f"🧾 例外摘要：{first_line[:180]}",
+            f"📍 疑似檔案：{target}",
+            "🔎 建議先查最近 5 分鐘 app/scheduler logs，確認是否仍持續重現",
+        ]
+
+    def _suppress_no_evidence_escalation(
+        self,
+        decision: StrategicDecision,
+        trigger: AutonomousTrigger,
+        reason: str,
+    ) -> None:
+        dedup_key = f"{_NO_EVIDENCE_DEDUP_PREFIX}{trigger.trigger_type}"
+        self._store_escalation(dedup_key)
+        self._log.warning(
+            "EA no-evidence escalation suppressed: trigger=%s confidence=%.2f reason=%s cooldown_min=%s",
+            trigger.trigger_type,
+            decision.confidence,
+            reason,
+            NO_EVIDENCE_ESCALATION_COOLDOWN_MIN,
+        )
+
     async def _escalate_to_human(self, decision: StrategicDecision, trigger: AutonomousTrigger) -> None:
         self._log.warning("Escalating to human: %s", trigger.trigger_type)
+        concrete_actions: Optional[List[str]] = None
+        ai_summary_text = (decision.reasoning or "")[:300]
+        ai_cause_text = (
+            f"觸發類型：{_zh_trigger(trigger.trigger_type)} | "
+            f"信心度：{decision.confidence:.2f} | "
+            f"參與模組：{', '.join(_AGENT_LABEL.get(a.lower(), a) for a in decision.agents_required)}"
+        )
+
+        if trigger.trigger_type in _PRICE_RELATED_TRIGGERS:
+            concrete_actions = (trigger.conditions or {}).get("_prefetched_hermes_threats")
+            if not concrete_actions:
+                try:
+                    concrete_actions = await self._fetch_hermes_threats_summary(top_n=5)
+                except Exception as e:
+                    self._log.warning("Pre-fetch threats raised (non-blocking): %s", e)
+                    concrete_actions = None
+            if not concrete_actions:
+                self._suppress_no_evidence_escalation(decision, trigger, "price_trigger_without_hermes_threats")
+                return
+        elif trigger.trigger_type == "resource_optimization":
+            concrete_actions = self._build_resource_escalation_actions()
+            if concrete_actions:
+                ai_summary_text = "資源類升級含實測 queue/load 訊號，請依安全邊界判斷是否處置。"
+                ai_cause_text = (
+                    f"觸發類型：{_zh_trigger(trigger.trigger_type)} | "
+                    f"信心度：{decision.confidence:.2f} | "
+                    "證據來源：action_plans queue / host CPU load"
+                )
+            else:
+                self._suppress_no_evidence_escalation(decision, trigger, "resource_trigger_without_operational_metrics")
+                return
+        elif trigger.trigger_type == "code_exception":
+            concrete_actions = self._build_code_exception_escalation_actions(trigger)
+            if concrete_actions:
+                ai_summary_text = "容器日誌偵測到具體例外，已轉人工審核避免自動修復風險擴大。"
+                ai_cause_text = (
+                    f"觸發類型：{_zh_trigger(trigger.trigger_type)} | "
+                    f"信心度：{decision.confidence:.2f} | "
+                    "證據來源：最近容器 logs"
+                )
+            else:
+                self._suppress_no_evidence_escalation(decision, trigger, "code_exception_without_log_context")
+                return
+        else:
+            if not decision.execution_plan and not (decision.reasoning or "").strip():
+                self._suppress_no_evidence_escalation(decision, trigger, "empty_decision_payload")
+                return
+            concrete_actions = [_zh_step(s) for s in decision.execution_plan[:3]]
+
         session = get_session()
         try:
             row = session.execute(
@@ -930,68 +1071,16 @@ class ElephantAlphaAutonomousEngine:
         if not dedup_ts or (datetime.now().timestamp() - dedup_ts) / 60 >= cooldown_min:
             self._store_escalation(trigger.trigger_type)
 
-            # A' 軌：價格類觸發前 pre-fetch Hermes 具體威脅清單，
-            # 取代「步驟 1:[OpenClaw] 生成策略」這類元流程文字。
-            # — Claude Opus 4.7 (2026-05-02)
-            concrete_actions: Optional[List[str]] = None
-            if trigger.trigger_type in _PRICE_RELATED_TRIGGERS:
-                try:
-                    concrete_actions = await self._fetch_hermes_threats_summary(top_n=5)
-                except Exception as e:
-                    self._log.warning("Pre-fetch threats raised (non-blocking): %s", e)
-                    concrete_actions = None
-
-            # ─── Operation Ollama-First v5.0 修補：消除空泛幻覺訊息 ───
-            # 統帥反饋（2026-05-03）：fallback 路徑帶 OpenClaw Gemini plan 文字 +
-            # decision.reasoning 全是「312 SKU / 23% / 14 項任務」幻覺數字，無 DB 鉤住，
-            # 嚴重誤導決策。修法：concrete=Hermes 實證 vs concrete=None 兩條路徑徹底分離。
-            #   - 有實證 → 完整訊息（含 SKU 流失金額）
-            #   - 無實證 → 極簡訊息「Hermes 即時數據不可用」+ 不再灌 LLM 幻覺
-
             from services.telegram_templates import triaged_alert, _send_telegram_raw
 
-            if concrete_actions:
-                # 有實證數據路徑：保留完整訊息
-                ai_actions_payload = concrete_actions
-                ai_summary_text = (decision.reasoning or "")[:300]
-                ai_cause_text = (
-                    f"觸發類型：{_zh_trigger(trigger.trigger_type)} | "
-                    f"信心度：{decision.confidence:.2f} | "
-                    f"參與模組：{', '.join(_AGENT_LABEL.get(a.lower(), a) for a in decision.agents_required)}"
-                )
-            else:
-                # 無實證數據路徑：極簡訊息，明確標註無數據
-                self._log.warning(
-                    "EA escalation 落入 no-concrete-data fallback (trigger=%s)；"
-                    "送極簡訊息避免 LLM 幻覺數字誤導統帥",
-                    trigger.trigger_type
-                )
-                ai_actions_payload = [
-                    "⚠️ Hermes 即時威脅清單不可用（5s timeout 或無 SKU 命中）",
-                    "📋 建議：手動下 SQL 查詢過去 24h competitor_price_history 確認狀況",
-                    "🔧 或：SSH 188 跑 docker exec momo-pro-system python -c "
-                    "'from services.hermes_analyst_service import HermesAnalystService;"
-                    " print(HermesAnalystService().run().threats[:5])'",
-                ]
-                ai_summary_text = (
-                    f"⚠️ 本訊息為**無實證**告警：Hermes pre-fetch 失敗，"
-                    f"以下原始決策內容含 LLM 自由發揮數字（非 DB 數據），請審慎參考。"
-                )
-                ai_cause_text = (
-                    f"觸發類型：{_zh_trigger(trigger.trigger_type)} | "
-                    f"信心度：{decision.confidence:.2f} | "
-                    f"⚠️ 無 Hermes SKU 數據（不顯示 LLM 幻覺 plan 文字）"
-                )
+            ai_actions_payload = concrete_actions
 
             try:
                 msg, keyboard = triaged_alert(
                     base_event={
                         "event_type": "ea_escalation",
                         "title": f"🐘 EA 升級審核 · {_zh_trigger(trigger.trigger_type)}",
-                        "summary": (
-                            f"自主決策信心度 {decision.confidence:.2f} 低於門檻，需人工批准"
-                            + ("" if concrete_actions else "（⚠️ 無實證數據）")
-                        ),
+                        "summary": f"自主決策信心度 {decision.confidence:.2f} 低於門檻，需人工批准",
                         "id": f"ea_review_{int(datetime.now().timestamp())}",
                     },
                     tier_label="🐘 Elephant Alpha · L3 HITL",
@@ -1059,10 +1148,19 @@ class ElephantAlphaAutonomousEngine:
     def _get_action_queue_size(self) -> int:
         session = get_session()
         try:
+            operational_types = "', '".join(sorted(_OPERATIONAL_PENDING_ACTION_TYPES))
             row = session.execute(
-                text("SELECT COUNT(*) AS count FROM action_plans WHERE status = 'pending'")
+                text(f"""
+                    SELECT COUNT(*) AS count
+                    FROM action_plans
+                    WHERE status IN ('auto_pending', 'running')
+                       OR (
+                           status = 'pending'
+                           AND COALESCE(action_type, '') IN ('{operational_types}')
+                       )
+                """)
             ).fetchone()
-            return row.count if row else 0
+            return int(row[0] or 0) if row else 0
         finally:
             session.close()