feat(sre): close provider and telegram automation receipts

apps/api/src/services/ai_control.py

@@ -22,6 +22,7 @@ Usage:
 """
 
 import hashlib
+import json
 import re
 from typing import Any
 
@@ -585,21 +586,144 @@ async def get_status_summary() -> str:
     primary = await get_primary_provider()
     primary_str = f"🎯 {primary}" if primary else "🤖 智慧路由 (AIRouter 決定)"
 
-    # Provider 狀態
+    # Provider 狀態：只讀既有 health/control receipts，不呼叫模型或端點。
+    ollama_specs = (
+        ("ollama_gcp_a", "GCP-A", "OLLAMA_URL"),
+        ("ollama_gcp_b", "GCP-B", "OLLAMA_SECONDARY_URL"),
+        ("ollama_local", "111", "OLLAMA_FALLBACK_URL"),
+    )
+    ollama_health = {provider: "not_observed" for provider, _, _ in ollama_specs}
+    try:
+        from src.services.ollama_health_monitor import make_cache_key
+
+        health_redis = await _get_redis()
+        for provider, _, setting_name in ollama_specs:
+            endpoint = str(getattr(settings, setting_name, "") or "").strip()
+            if not endpoint:
+                ollama_health[provider] = "not_configured"
+                continue
+            try:
+                raw = await health_redis.get(make_cache_key(endpoint))
+                if raw:
+                    if isinstance(raw, bytes):
+                        raw = raw.decode()
+                    receipt = json.loads(raw)
+                    ollama_health[provider] = str(
+                        receipt.get("status") or "invalid_receipt"
+                    )
+            except Exception as exc:
+                ollama_health[provider] = "readback_unavailable"
+                logger.warning(
+                    "ai_control_ollama_health_receipt_read_failed",
+                    provider=provider,
+                    error_type=type(exc).__name__,
+                )
+    except Exception as exc:
+        logger.warning(
+            "ai_control_ollama_health_readback_unavailable",
+            error_type=type(exc).__name__,
+        )
+
     provider_lines = []
-    for p in [
-        "ollama_gcp_a",
-        "ollama_gcp_b",
-        "ollama_local",
-        "claude",
-        "gemini",
-    ]:
+    for provider, label, _ in ollama_specs:
         try:
-            disabled = await is_provider_disabled(p)
+            control = "disabled" if await is_provider_disabled(provider) else "enabled"
         except Exception:
-            disabled = True
-        icon = "❌" if disabled else "✅"
-        provider_lines.append(f"  {icon} {p}")
+            control = "readback_unavailable"
+        health = ollama_health[provider]
+        if control == "disabled":
+            icon = "⏸️"
+        elif health == "healthy":
+            icon = "🟢"
+        elif health in {"slow", "degraded"}:
+            icon = "🟠"
+        elif health == "offline":
+            icon = "🔴"
+        else:
+            icon = "⚪"
+        provider_lines.append(
+            f"  {icon} {label} | health={health} | control={control}"
+        )
+
+    paid_control: dict[str, Any] = {}
+    route_lease = "readback_unavailable"
+    try:
+        paid_control = await get_paid_provider_canary_control_readback()
+        if paid_control.get("legacy_persistent_state_detected") is True:
+            route_lease = "legacy_persistent_state_detected"
+        elif (
+            paid_control.get("lock_present") is True
+            and paid_control.get("claude_disabled") is False
+            and paid_control.get("gemini_disabled") is False
+            and int(paid_control.get("lock_ttl_seconds") or 0) > 0
+        ):
+            route_lease = "active_run_owned_lease"
+        elif paid_control.get("disabled_verified") is True:
+            route_lease = "待單次 run-owned lease"
+        else:
+            route_lease = "inconsistent_fail_closed"
+    except Exception as exc:
+        logger.warning(
+            "ai_control_paid_control_status_readback_unavailable",
+            error_type=type(exc).__name__,
+        )
+
+    authentication: dict[str, dict[str, Any]] = {}
+    try:
+        from src.services.ai_rate_limiter import get_ai_rate_limiter
+
+        limiter = get_ai_rate_limiter()
+        for provider in ("claude", "gemini"):
+            try:
+                authentication[provider] = (
+                    await limiter.get_provider_authentication_status(provider)
+                )
+            except Exception:
+                authentication[provider] = {
+                    "configured": None,
+                    "authenticated": None,
+                    "authentication_verified": False,
+                    "verification_status": "authentication_readback_unavailable",
+                }
+    except Exception as exc:
+        logger.warning(
+            "ai_control_paid_authentication_status_readback_unavailable",
+            error_type=type(exc).__name__,
+        )
+
+    def _truth_text(value: Any) -> str:
+        if value is True:
+            return "true"
+        if value is False:
+            return "false"
+        return "unknown"
+
+    for provider, label in (("claude", "Claude"), ("gemini", "Gemini")):
+        auth = authentication.get(provider) or {}
+        configured = auth.get("configured")
+        mode = str(
+            getattr(settings, f"{provider.upper()}_EXECUTION_MODE", "shadow")
+        ).lower()
+        percent = int(
+            getattr(settings, f"{provider.upper()}_CANARY_PERCENT", 0) or 0
+        )
+        if configured is True and auth.get("authenticated") is False:
+            icon = "⚠️"
+        elif configured is True and route_lease == "待單次 run-owned lease":
+            icon = "🧪"
+        elif configured is True and route_lease == "active_run_owned_lease":
+            icon = "🟢"
+        elif configured is True:
+            icon = "⚠️"
+        else:
+            icon = "⚪"
+        provider_lines.append(
+            f"  {icon} {label} | configured={_truth_text(configured)} | "
+            f"authenticated={_truth_text(auth.get('authenticated'))} | "
+            "verification="
+            f"{auth.get('verification_status', 'authentication_readback_unavailable')} | "
+            f"canary={mode} {percent}% | route_lease={route_lease}"
+        )
     provider_lines.extend(
         f"  ⚪ {provider} (shadow metadata only)"
         for provider in sorted(SHADOW_METADATA_ONLY_PROVIDERS)
@@ -608,15 +732,31 @@ async def get_status_summary() -> str:
     # 費用統計
     cost_lines = []
     try:
-        r = await _get_redis()
-        for p in ["claude", "gemini"]:
-            val = await r.get(f"ai_rate:total_cost:{p}")
-            if val:
-                cost = float(val)
-                if cost > 0:
-                    cost_lines.append(f"  💰 {p}: ${cost:.4f}")
-    except Exception:
-        cost_lines.append("  (費用統計不可用)")
+        from src.services.ai_rate_limiter import get_ai_rate_limiter
+
+        limiter = get_ai_rate_limiter()
+        for provider in ("claude", "gemini"):
+            usage = await limiter.get_usage_stats(provider)
+            cost = float(
+                (usage.get("total_cost_usd") or {}).get("current") or 0.0
+            )
+            accounting = usage.get("accounting") or {}
+            receipt_backed = bool(
+                usage.get("readback_status") == "verified"
+                and accounting.get("complete") is True
+                and accounting.get("status") == "receipt_backed"
+                and accounting.get("degraded") is not True
+            )
+            cost_lines.append(
+                f"  💰 {provider}: ${cost:.6f} | "
+                f"receipt={'verified' if receipt_backed else 'degraded'}"
+            )
+    except Exception as exc:
+        logger.warning(
+            "ai_control_status_cost_readback_unavailable",
+            error_type=type(exc).__name__,
+        )
+        cost_lines.append("  (accounting receipt readback unavailable)")
 
     providers_str = "\n".join(provider_lines)
     costs_str = "\n".join(cost_lines) if cost_lines else "  (尚無費用記錄)"
@@ -629,7 +769,8 @@ async def get_status_summary() -> str:
         f"<b>費用統計</b>:\n{costs_str}\n\n"
         f"<code>/ai primary &lt;provider&gt;</code> 切換\n"
         f"<code>/ai router on/off</code> 開關 AIRouter\n"
-        f"<code>/ai enable/disable &lt;provider&gt;</code> 控制 Provider"
+        f"<code>/ai enable/disable &lt;ollama provider&gt;</code> 控制 Ollama\n"
+        f"Claude/Gemini 僅由單次 run-owned canary lease 啟用"
     )
 
 
@@ -709,23 +850,49 @@ async def handle_ai_command(text: str) -> str:
         return f"⚠️ {provider} 已停用" if ok else "❌ 操作失敗"
 
     elif sub == "cost":
-        lines = ["<b>💰 AI 費用統計</b>\n"]
+        lines = ["<b>💰 AI 費用 receipt</b>\n"]
         try:
-            r = await _get_redis()
+            from src.services.ai_rate_limiter import get_ai_rate_limiter
+
+            limiter = get_ai_rate_limiter()
             total = 0.0
-            for p in ["claude", "gemini"]:
-                val = await r.get(f"ai_rate:total_cost:{p}")
-                if val:
-                    cost = float(val)
-                    total += cost
-                    if cost > 0:
-                        lines.append(f"  {p}: ${cost:.4f}")
-            if total > 0:
-                lines.append(f"\n  <b>總計: ${total:.4f}</b>")
-            else:
-                lines.append("  (尚無費用記錄)")
-        except Exception as e:
-            lines.append(f"  ⚠️ 費用查詢失敗: {e}")
+            all_receipt_backed = True
+            for provider in ("claude", "gemini"):
+                usage = await limiter.get_usage_stats(provider)
+                cost = float(
+                    (usage.get("total_cost_usd") or {}).get("current") or 0.0
+                )
+                accounting = usage.get("accounting") or {}
+                receipt_backed = bool(
+                    usage.get("readback_status") == "verified"
+                    and accounting.get("complete") is True
+                    and accounting.get("status") == "receipt_backed"
+                    and accounting.get("degraded") is not True
+                )
+                total += cost
+                all_receipt_backed = all_receipt_backed and receipt_backed
+                lines.append(
+                    f"  {provider}: ${cost:.6f} | "
+                    f"receipt={'verified' if receipt_backed else 'degraded'}"
+                )
+            lines.extend(
+                [
+                    f"\n  <b>總計: ${total:.6f}</b>",
+                    (
+                        "  ✅ receipt-backed accounting"
+                        if all_receipt_backed
+                        else "  ⚠️ accounting receipt degraded；禁止假性估算"
+                    ),
+                ]
+            )
+        except Exception as exc:
+            logger.warning(
+                "ai_control_cost_readback_unavailable",
+                error_type=type(exc).__name__,
+            )
+            lines.append(
+                "  ⚠️ accounting readback unavailable；未以舊 Redis key 或估算值替代。"
+            )
         return "\n".join(lines)
 
     elif sub == "help":