diff --git a/apps/api/models.json b/apps/api/models.json index 1a766a2f7..a0c0943a9 100644 --- a/apps/api/models.json +++ b/apps/api/models.json @@ -2,18 +2,18 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "name": "OpenClaw AI Router Configuration", "version": "2.0.0", - "description": "Global production route: GCP-A Ollama -> GCP-B Ollama -> host111 Ollama -> Gemini API; legacy providers are non-executable shadow metadata only", - "updated_at": "2026-07-14", + "description": "Global production route: GCP-A Ollama -> GCP-B Ollama -> host111 Ollama -> Anthropic Claude API -> Gemini API; paid providers require durable cost and runtime gates", + "updated_at": "2026-07-15", "default_provider": "ollama", - "fallback_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], - "tool_calling_fallback_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], + "fallback_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "claude", "gemini"], + "tool_calling_fallback_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "claude", "gemini"], "production_route": { "schema_version": "ai_provider_production_route_v1", - "provider_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], - "route_label": "GCP-A -> GCP-B -> host111 Ollama -> Gemini API", + "provider_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "claude", "gemini"], + "route_label": "GCP-A -> GCP-B -> host111 Ollama -> Anthropic Claude API -> Gemini API", "applies_to": ["all_risk_levels", "all_complexity_levels", "tool_calling", "background_jobs", "code_review", "runbook_generation"], - "paid_provider_policy": "gemini_final_only_with_atomic_cost_guard", + "paid_provider_policy": "claude_then_gemini_with_independent_atomic_cost_guards", "disabled_provider_policy": "never_reinsert" }, @@ -64,7 +64,7 @@ "gemini": { "name": "Google Gemini", "enabled": true, - "priority": 4, + "priority": 5, "production_executable": true, "execution_gate": "configured_and_enabled_and_atomic_cost_guard_reserved", "endpoint": "https://generativelanguage.googleapis.com/v1beta", @@ -107,23 +107,29 @@ "claude": { "name": "Anthropic Claude", "enabled": false, - "priority": null, - "production_executable": false, - "mode": "non_executable_shadow_metadata_only", + "priority": 4, + "production_executable": true, + "mode": "paid_fallback_shadow_by_default", + "execution_gate": "configured_and_enabled_and_mode_selected_and_atomic_cost_guard_reserved", "endpoint": "https://api.anthropic.com/v1", "api_path": "/messages", "models": { - "default": "claude-haiku-4-5-20251001", - "rca": "claude-haiku-4-5-20251001", - "summary": "claude-haiku-4-5-20251001" + "default": "claude-sonnet-5", + "rca": "claude-sonnet-5", + "summary": "claude-sonnet-5" }, "options": { - "max_tokens": 2048 + "max_tokens": 4096 }, "timeout_seconds": 30, "cost": { - "per_1k_tokens": 0.005, - "currency": "USD" + "input_usd_per_million": 3.0, + "output_usd_per_million": 15.0, + "currency": "USD", + "pricing_basis": "conservative_standard_ceiling_even_during_introductory_discount", + "pricing_source": "https://platform.claude.com/docs/en/about-claude/models/overview", + "pricing_version": "anthropic-standard-ceiling-2026-07-15", + "pricing_checked_at": "2026-07-15" }, "auth": { "type": "header", @@ -131,8 +137,12 @@ "header_name": "x-api-key" }, "rate_limits": { - "daily_tokens": 35000, - "requests_per_minute": 50 + "daily_requests": 50, + "daily_tokens": 100000, + "requests_per_minute": 2, + "daily_cost_limit_usd": 1.0, + "total_cost_limit_usd": 5.0, + "cost_alert_threshold_usd": 4.0 }, "features": { "tool_use": true, @@ -231,14 +241,14 @@ "description": "Root Cause Analysis for alerts", "preferred_provider": "ollama_gcp_a", "fallback_enabled": true, - "fallback_order": ["ollama_gcp_b", "ollama_local", "gemini"], + "fallback_order": ["ollama_gcp_b", "ollama_local", "claude", "gemini"], "required_features": ["json_output"] }, "log_summary": { "description": "Summarize K8s logs for context gathering", "preferred_provider": "ollama_gcp_a", "fallback_enabled": true, - "fallback_order": ["ollama_gcp_b", "ollama_local", "gemini"], + "fallback_order": ["ollama_gcp_b", "ollama_local", "claude", "gemini"], "max_input_tokens": 4096 }, "telegram_compose": { @@ -251,7 +261,7 @@ "description": "K8s Tool Calling operations on the global production route", "preferred_provider": "ollama_gcp_a", "fallback_enabled": true, - "fallback_order": ["ollama_gcp_b", "ollama_local", "gemini"], + "fallback_order": ["ollama_gcp_b", "ollama_local", "claude", "gemini"], "required_features": ["tool_use"], "notes": "NVIDIA/Nemotron is non-executable shadow metadata only" } diff --git a/apps/api/src/api/v1/ai.py b/apps/api/src/api/v1/ai.py index 8502f8bd5..b0118e713 100644 --- a/apps/api/src/api/v1/ai.py +++ b/apps/api/src/api/v1/ai.py @@ -278,6 +278,7 @@ async def get_ai_status() -> dict: from src.services.ai_rate_limiter import get_ai_rate_limiter gemini_configured = bool(settings.GEMINI_API_KEY) + claude_configured = bool(settings.CLAUDE_API_KEY) limiter = get_ai_rate_limiter() try: gemini_usage = await limiter.get_usage_stats("gemini") @@ -301,6 +302,28 @@ async def get_ai_status() -> dict: }, } authentication = gemini_usage.get("authentication") or {} + try: + claude_usage = await limiter.get_usage_stats("claude") + claude_usage["readback_status"] = "verified" + except Exception as exc: + claude_usage = { + "provider": "claude", + "limited": True, + "readback_status": "unavailable", + "readback_error": type(exc).__name__, + "accounting": { + "complete": False, + "status": "blocked_cost_guard_readback_unavailable", + }, + "authentication": { + "provider": "claude", + "configured": claude_configured, + "authenticated": None, + "authentication_verified": False, + "verification_status": "authentication_readback_unavailable", + }, + } + claude_authentication = claude_usage.get("authentication") or {} try: gemini_disabled = await is_provider_disabled("gemini") disable_state_status = "verified" @@ -312,11 +335,28 @@ async def get_ai_status() -> dict: and is_provider_enabled_by_env("gemini") and not gemini_disabled ) + try: + claude_disabled = await is_provider_disabled("claude") + claude_disable_state_status = "verified" + except Exception as exc: + claude_disabled = True + claude_disable_state_status = f"unavailable:{type(exc).__name__}" + claude_enabled = bool( + claude_configured + and is_provider_enabled_by_env("claude") + and not claude_disabled + ) route = production_route_readback( gemini_configured=gemini_configured, gemini_enabled=gemini_enabled, gemini_authentication=authentication, cost_guard=gemini_usage, + claude_configured=claude_configured, + claude_enabled=claude_enabled, + claude_authentication=claude_authentication, + claude_cost_guard=claude_usage, + claude_execution_mode=settings.CLAUDE_EXECUTION_MODE, + claude_canary_percent=settings.CLAUDE_CANARY_PERCENT, ) return { @@ -333,5 +373,19 @@ async def get_ai_status() -> dict: ], "gemini_verification_status": authentication.get("verification_status"), "gemini_disable_state_status": disable_state_status, + "claude_configured": claude_configured, + "claude_enabled": claude_enabled, + "claude_authenticated": claude_authentication.get("authenticated"), + "claude_authentication_verified": bool( + claude_authentication.get("authentication_verified") is True + ), + "claude_production_executable": route["claude"][ + "production_executable" + ], + "claude_canary_executable": route["claude"]["canary_executable"], + "claude_verification_status": claude_authentication.get( + "verification_status" + ), + "claude_disable_state_status": claude_disable_state_status, "legacy_provider_policy": route["legacy_provider_policy"], } diff --git a/apps/api/src/api/v1/health.py b/apps/api/src/api/v1/health.py index 702d083d9..68a876df4 100644 --- a/apps/api/src/api/v1/health.py +++ b/apps/api/src/api/v1/health.py @@ -382,6 +382,7 @@ async def get_ai_usage() -> dict: from src.services.ai_rate_limiter import get_ai_rate_limiter gemini_configured = bool(settings.GEMINI_API_KEY) + claude_configured = bool(settings.CLAUDE_API_KEY) rate_limiter = get_ai_rate_limiter() try: gemini_stats = await rate_limiter.get_usage_stats("gemini") @@ -405,6 +406,28 @@ async def get_ai_usage() -> dict: }, } authentication = gemini_stats.get("authentication") or {} + try: + claude_stats = await rate_limiter.get_usage_stats("claude") + claude_stats["readback_status"] = "verified" + except Exception as exc: + claude_stats = { + "provider": "claude", + "limited": True, + "readback_status": "unavailable", + "readback_error": type(exc).__name__, + "accounting": { + "complete": False, + "status": "blocked_cost_guard_readback_unavailable", + }, + "authentication": { + "provider": "claude", + "configured": claude_configured, + "authenticated": None, + "authentication_verified": False, + "verification_status": "authentication_readback_unavailable", + }, + } + claude_authentication = claude_stats.get("authentication") or {} try: gemini_disabled = await is_provider_disabled("gemini") disable_state_status = "verified" @@ -416,20 +439,33 @@ async def get_ai_usage() -> dict: and is_provider_enabled_by_env("gemini") and not gemini_disabled ) + try: + claude_disabled = await is_provider_disabled("claude") + claude_disable_state_status = "verified" + except Exception as exc: + claude_disabled = True + claude_disable_state_status = f"unavailable:{type(exc).__name__}" + claude_enabled = bool( + claude_configured + and is_provider_enabled_by_env("claude") + and not claude_disabled + ) route = production_route_readback( gemini_configured=gemini_configured, gemini_enabled=gemini_enabled, gemini_authentication=authentication, cost_guard=gemini_stats, + claude_configured=claude_configured, + claude_enabled=claude_enabled, + claude_authentication=claude_authentication, + claude_cost_guard=claude_stats, + claude_execution_mode=settings.CLAUDE_EXECUTION_MODE, + claude_canary_percent=settings.CLAUDE_CANARY_PERCENT, ) return { "gemini": gemini_stats, - "claude": { - "provider": "claude", - "production_executable": False, - "mode": "non_executable_shadow_metadata_only", - }, + "claude": claude_stats, "fallback_order": route["provider_order"], "production_route": route, "gemini_status": { @@ -445,4 +481,22 @@ async def get_ai_usage() -> dict: "verification_status": authentication.get("verification_status"), "disable_state_status": disable_state_status, }, + "claude_status": { + "configured": claude_configured, + "enabled": claude_enabled, + "execution_mode": settings.CLAUDE_EXECUTION_MODE, + "canary_percent": settings.CLAUDE_CANARY_PERCENT, + "authenticated": claude_authentication.get("authenticated"), + "authentication_verified": bool( + claude_authentication.get("authentication_verified") is True + ), + "production_executable": route["claude"][ + "production_executable" + ], + "canary_executable": route["claude"]["canary_executable"], + "verification_status": claude_authentication.get( + "verification_status" + ), + "disable_state_status": claude_disable_state_status, + }, } diff --git a/apps/api/src/api/v1/platform/operator_runs.py b/apps/api/src/api/v1/platform/operator_runs.py index f4285b9aa..34ed1acf0 100644 --- a/apps/api/src/api/v1/platform/operator_runs.py +++ b/apps/api/src/api/v1/platform/operator_runs.py @@ -369,6 +369,7 @@ class AiRouteStatusResponse(BaseModel): health: dict[str, dict[str, Any]] lane_mode: str | None = None active_lane: dict[str, Any] | None = None + claude_activation_evidence: dict[str, Any] | None = None gemini_activation_evidence: dict[str, Any] | None = None skipped_lanes: list[dict[str, Any]] = Field(default_factory=list) operator_action: dict[str, Any] | None = None diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index b083aedea..9f44c1207 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -6,7 +6,7 @@ AWOOOI API - Sentry Webhook Handler 整合流程: 1. Sentry Alert → AWOOOI API Webhook 2. 組裝錯誤上下文 -3. 呼叫 global AI Router（GCP-A → GCP-B → 111 → Gemini） +3. 呼叫 global AI Router（GCP-A → GCP-B → 111 → Claude → Gemini） 4. 結果回寫 Sentry Issue Comment 5. 發送 Telegram 告警 (含截圖) 6. 建立 Approval 供人工審核 @@ -500,8 +500,7 @@ async def call_openclaw_analyzer(error_context: dict) -> ErrorAnalysisResult | N - Layer 1: Circuit Breaker（5 連續失敗 → 斷路 60 秒） - Layer 2: Concurrency Semaphore（最多 3 並發） - 優先使用 Ollama (本地，零成本) - Fallback: Claude (高嚴重性) + 固定順序先使用三段 Ollama（零 API 成本），再受控進入 Claude、Gemini。 Direct Host188/OpenClaw execution is forbidden for production Sentry analysis. """ diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index 053cc8646..593e1aa1c 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -4,7 +4,7 @@ AWOOOI API Configuration Pydantic Settings + Environment Variables ADR-005: BFF Architecture -ADR-006: AI Fallback Strategy (GCP-A -> GCP-B -> host111 -> Gemini) +ADR-006: AI Fallback Strategy (GCP-A -> GCP-B -> host111 -> Claude -> Gemini) Four Iron Laws: 1. Async-First @@ -428,8 +428,8 @@ class Settings(BaseSettings): default=["bge-m3:latest", "qwen2.5:7b-instruct", "qwen3:14b", "deepseek-r1:14b", "hermes3:latest"], description="HeartbeatReportService 探測必要模型是否載入", ) - # Gemini is the final paid fallback after GCP-A -> GCP-B -> host111. - # All generation calls atomically reserve requests/tokens/cost in Redis. + # Claude and Gemini are paid fallbacks after GCP-A -> GCP-B -> host111. + # All paid generation calls atomically reserve requests/tokens/cost in Redis. GEMINI_MODEL: str = Field( default="gemini-2.5-flash-lite", min_length=1, @@ -486,6 +486,78 @@ class Settings(BaseSettings): le=3600, description="Gemini failed generation work-item cooldown seconds", ) + CLAUDE_MODEL: str = Field( + default="claude-sonnet-5", + min_length=1, + description=( + "Anthropic SRE/architecture paid fallback model; exact model must " + "have a reviewed pricing policy before any generation" + ), + ) + CLAUDE_DAILY_QUOTA: int = Field( + default=50, + ge=1, + description="Claude daily generation request hard limit", + ) + CLAUDE_RPM_LIMIT: int = Field( + default=2, + ge=1, + description="Claude generation requests per minute hard limit", + ) + CLAUDE_DAILY_TOKEN_LIMIT: int = Field( + default=100_000, + ge=1, + description="Claude daily token hard limit including pending reservations", + ) + CLAUDE_DAILY_COST_LIMIT_USD: float = Field( + default=1.0, + gt=0, + description="Claude daily spend hard limit in USD", + ) + CLAUDE_TOTAL_COST_LIMIT_USD: float = Field( + default=5.0, + gt=0, + description="Claude cumulative spend hard limit in USD", + ) + CLAUDE_COST_ALERT_THRESHOLD_USD: float = Field( + default=4.0, + gt=0, + description="Claude cumulative spend warning threshold in USD", + ) + CLAUDE_MAX_OUTPUT_TOKENS: int = Field( + default=4096, + ge=1, + le=16_384, + description="Claude output ceiling and pre-call cost reservation basis", + ) + CLAUDE_USAGE_RECEIPT_TTL_SECONDS: int = Field( + default=604_800, + ge=86_400, + description=( + "Claude non-secret auth-status receipt TTL; generation and run " + "identity receipts remain durable" + ), + ) + CLAUDE_FAILURE_COOLDOWN_SECONDS: int = Field( + default=120, + ge=1, + le=3600, + description="Claude failed generation work-item cooldown seconds", + ) + CLAUDE_EXECUTION_MODE: str = Field( + default="shadow", + pattern="^(shadow|canary|production)$", + description=( + "Claude route mode: shadow performs no API call; canary uses a " + "deterministic run bucket; production enables the ordered fallback" + ), + ) + CLAUDE_CANARY_PERCENT: int = Field( + default=0, + ge=0, + le=100, + description="Deterministic Claude canary percentage when mode=canary", + ) # Deprecated: use OPENCLAW_URL instead CLAWBOT_URL: str = Field( default="http://192.168.0.188:8088", # 🔧 修正: OpenClaw 實際 port 是 8088 @@ -588,14 +660,25 @@ class Settings(BaseSettings): # ========================================================================== # Runtime fallback strategy. Alert/incident order is enforced as: - # GCP-A -> GCP-B -> host111 Ollama -> Gemini. No paid/provider insertion. + # GCP-A -> GCP-B -> host111 Ollama -> Claude -> Gemini. # ========================================================================== AI_FALLBACK_ORDER: list[str] = Field( - default=["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], + default=[ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ], description=( - "唯一 production provider chain: GCP-A -> GCP-B -> host111 -> Gemini" + "唯一 production provider chain: GCP-A -> GCP-B -> host111 -> " + "Claude -> Gemini" ), ) + ENABLE_CLAUDE: bool = Field( + default=False, + description="Claude paid fallback explicit opt-in; default disabled", + ) ENABLE_GEMINI: bool = Field( default=False, description="Gemini paid final fallback explicit opt-in; default disabled", @@ -615,30 +698,30 @@ class Settings(BaseSettings): description=( "Allow incident/alert OpenClaw analysis to use cloud fallback " "providers after the GCP-A/GCP-B/111 Ollama lane is exhausted. " - "Default true so Gemini can act as the final backup, after the " - "ordered Ollama lane is exhausted." + "Default true so Claude and then Gemini can act as paid backups " + "after the ordered Ollama lane is exhausted." ), ) ALERT_AI_ENFORCE_OLLAMA_FIRST: bool = Field( default=True, description=( "Force incident/alert OpenClaw analysis to try GCP-A, then GCP-B, " - "then local 111 before cloud backup providers such as Gemini." + "then local 111 before cloud backup providers Claude and Gemini." ), ) ALERT_AI_STRICT_PROVIDER_CHAIN: bool = Field( default=True, description=( - "Forbid direct OpenClaw/Nemo/NVIDIA/Claude incident delegation; " - "the only alert route is GCP-A, GCP-B, host111, then Gemini." + "Forbid direct OpenClaw/Nemo/NVIDIA incident delegation; the only " + "alert route is GCP-A, GCP-B, host111, Claude, then Gemini." ), ) ALERT_OLLAMA_MODEL: str = Field( default="qwen3:14b", description=( "Ollama model used for incident/alert deep diagnosis. Alert cards " - "may wait for this model; Gemini remains a backup after GCP-A, " - "GCP-B, and 111 fail." + "may wait for this model; Claude then Gemini remain paid backups " + "after GCP-A, GCP-B, and 111 fail." ), ) INCIDENT_LLM_TIMEOUT_SECONDS: int = Field( @@ -670,9 +753,9 @@ class Settings(BaseSettings): def parse_ai_fallback(cls, v: str | list[str]) -> list[str]: """ 解析 AI_FALLBACK_ORDER，支援三種格式: - 1. JSON: '["ollama_gcp_a","ollama_gcp_b","ollama_local","gemini"]' - 2. CSV: 'ollama_gcp_a,ollama_gcp_b,ollama_local,gemini' - 3. List: ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"] + 1. JSON: '["ollama_gcp_a","ollama_gcp_b","ollama_local","claude","gemini"]' + 2. CSV: 'ollama_gcp_a,ollama_gcp_b,ollama_local,claude,gemini' + 3. List: ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "claude", "gemini"] 2026-03-27 修復: ConfigMap 用 JSON 格式，原本只支援 CSV """ @@ -692,13 +775,19 @@ class Settings(BaseSettings): else: values = [p.strip().lower() for p in v] - expected = ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"] - if values == ["ollama", "gemini"]: + expected = [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ] + if values in (["ollama", "gemini"], ["ollama", "claude", "gemini"]): return expected if values != expected: raise ValueError( "AI_FALLBACK_ORDER must be exactly " - "ollama_gcp_a,ollama_gcp_b,ollama_local,gemini" + "ollama_gcp_a,ollama_gcp_b,ollama_local,claude,gemini" ) return values diff --git a/apps/api/src/main.py b/apps/api/src/main.py index ed734b331..c31af5d41 100644 --- a/apps/api/src/main.py +++ b/apps/api/src/main.py @@ -960,7 +960,8 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]: logger.warning("ollama_failover_system_start_failed", error=str(e)) # 2026-04-27 P3.2.2 by Claude — AI Provider 版本追蹤（每 1 小時） - # 探測 5 Provider（ollama/ollama_local/gemini/claude/openclaw_nemo）版本 + # 探測 5 個 provider inventory（ollama/ollama_local/claude/gemini/openclaw_nemo）； + # 這是版本盤點，不是可重排的 production execution route。 # 寫入 ai_provider_version_history；版本變更時 log warning，P3.2.3 alerter 後續整合 try: diff --git a/apps/api/src/services/ai_control.py b/apps/api/src/services/ai_control.py index 3323b4517..6e8a88e0a 100644 --- a/apps/api/src/services/ai_control.py +++ b/apps/api/src/services/ai_control.py @@ -8,8 +8,8 @@ Telegram /ai 指令動態控制 AI Router 狀態 Redis Keys: ai:control:use_router "true"/"false" (TTL: 30天) ai:control:primary_provider legacy compatibility; only "ollama"/"ollama_gcp_a" - ai:control:disabled: "1" disabled / Gemini "0" enabled - (Gemini 無 TTL；其他 provider 30 天) + ai:control:disabled: "1" disabled / paid provider "0" enabled + (paid providers 無 TTL；free providers 30 天) Usage: /ai status — 顯示所有 Provider 狀態 + 當前路由模式 @@ -31,19 +31,20 @@ _PRIMARY_PROVIDER_KEY = "ai:control:primary_provider" _DISABLED_KEY_PREFIX = "ai:control:disabled:" # Free-provider/router compatibility controls retain the historical 30-day -# expiry. Gemini's explicit paid-provider state is intentionally persistent. +# expiry. Paid-provider explicit control state is intentionally persistent. _CONTROL_KEY_TTL = 30 * 24 * 3600 # 30 days +PAID_PROVIDERS = {"claude", "gemini"} VALID_PROVIDERS = { "ollama", "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", } VALID_PRIMARY_PROVIDERS = {"ollama", "ollama_gcp_a"} SHADOW_METADATA_ONLY_PROVIDERS = { - "claude", "nvidia", "nemotron", "openclaw", @@ -130,15 +131,15 @@ async def is_provider_disabled(provider: str) -> bool: # durable control key is disabled until an operator explicitly enables # it; free Ollama lanes retain their historical missing-key=enabled # behaviour. - if provider == "gemini": + if provider in PAID_PROVIDERS: return val is None or ( val.decode() if isinstance(val, bytes) else str(val) ) != "0" return val is not None except Exception: # Paid-provider disable truth is fail-closed. Callers can catch this to - # publish an unavailable readback, but must never re-add Gemini. - if provider == "gemini": + # publish an unavailable readback, but must never re-add a paid lane. + if provider in PAID_PROVIDERS: raise return False @@ -151,13 +152,13 @@ async def set_provider_disabled(provider: str, disabled: bool) -> bool: r = await _get_redis() key = f"{_DISABLED_KEY_PREFIX}{provider}" if disabled: - if provider == "gemini": + if provider in PAID_PROVIDERS: # Paid-provider disable truth must not silently expire. await r.set(key, "1") else: await r.set(key, "1", ex=_CONTROL_KEY_TTL) else: - if provider == "gemini": + if provider in PAID_PROVIDERS: # Explicitly persist the enabled state. Deleting the key would # fail closed to disabled on the next read. await r.set(key, "0") @@ -190,7 +191,13 @@ async def get_status_summary() -> str: # Provider 狀態 provider_lines = [] - for p in ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"]: + for p in [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ]: try: disabled = await is_provider_disabled(p) except Exception: @@ -206,7 +213,7 @@ async def get_status_summary() -> str: cost_lines = [] try: r = await _get_redis() - for p in ["gemini"]: + for p in ["claude", "gemini"]: val = await r.get(f"ai_rate:total_cost:{p}") if val: cost = float(val) @@ -303,7 +310,7 @@ async def handle_ai_command(text: str) -> str: try: r = await _get_redis() total = 0.0 - for p in ["gemini"]: + for p in ["claude", "gemini"]: val = await r.get(f"ai_rate:total_cost:{p}") if val: cost = float(val) @@ -329,7 +336,7 @@ async def handle_ai_command(text: str) -> str: "/ai disable <p> — 停用 Provider\n" "/ai cost — 費用統計\n\n" f"可控 Provider: {', '.join(sorted(VALID_PROVIDERS))}\n" - "Legacy providers 僅保留 shadow metadata，不可執行" + "Legacy providers 僅保留 shadow metadata；Claude/Gemini 受 persistent paid gate 控制" ) else: diff --git a/apps/api/src/services/ai_provider_policy.py b/apps/api/src/services/ai_provider_policy.py index 023552828..0ee77d314 100644 --- a/apps/api/src/services/ai_provider_policy.py +++ b/apps/api/src/services/ai_provider_policy.py @@ -17,11 +17,11 @@ PRODUCTION_OLLAMA_ORDER: tuple[str, ...] = ( ) PRODUCTION_PROVIDER_ORDER: tuple[str, ...] = ( *PRODUCTION_OLLAMA_ORDER, + "claude", "gemini", ) NON_EXECUTABLE_SHADOW_PROVIDERS: frozenset[str] = frozenset( { - "claude", "nvidia", "nemotron", "openclaw", @@ -31,13 +31,23 @@ NON_EXECUTABLE_SHADOW_PROVIDERS: frozenset[str] = frozenset( ) -def production_provider_order(*, include_gemini: bool = True) -> list[str]: +PAID_PROVIDER_ORDER: tuple[str, ...] = ("claude", "gemini") + + +def production_provider_order( + *, + include_gemini: bool = True, + include_claude: bool | None = None, +) -> list[str]: """Return a new list so callers cannot mutate the global contract.""" - return list( - PRODUCTION_PROVIDER_ORDER - if include_gemini - else PRODUCTION_OLLAMA_ORDER - ) + if include_claude is None: + include_claude = include_gemini + order = list(PRODUCTION_OLLAMA_ORDER) + if include_claude: + order.append("claude") + if include_gemini: + order.append("gemini") + return order def normalize_production_execution_order( @@ -50,9 +60,9 @@ def normalize_production_execution_order( The legacy logical ``ollama`` alias expands to all three concrete Ollama identities. A production call must contain an Ollama lane; a lone Gemini request is rejected because it has no evidence that the free lane ran. - Gemini is appended only when it was explicitly requested and local-only - processing is not required. This also means a disabled/removed Gemini is - never silently re-added by normalization. + Paid providers are appended only when explicitly requested and local-only + processing is not required. This means a disabled/removed Claude or Gemini + lane is never silently re-added by normalization. """ values = [str(value).strip().lower() for value in requested if str(value).strip()] rejected = sorted( @@ -69,24 +79,25 @@ def normalize_production_execution_order( return [], rejected normalized = list(PRODUCTION_OLLAMA_ORDER) - if not require_local and "gemini" in values: - normalized.append("gemini") + if not require_local: + normalized.extend( + provider for provider in PAID_PROVIDER_ORDER if provider in values + ) return normalized, rejected -def production_route_readback( +def _paid_provider_readback( *, - gemini_configured: bool, - gemini_enabled: bool, - gemini_authentication: dict[str, Any] | None = None, - cost_guard: dict[str, Any] | None = None, + configured: bool, + enabled: bool, + authentication: dict[str, Any] | None, + cost_guard: dict[str, Any] | None, ) -> dict[str, Any]: - """Build a public-safe route/status payload without endpoint URLs or keys.""" - authentication = dict(gemini_authentication or {}) + authentication_payload = dict(authentication or {}) cost_guard_payload = dict(cost_guard or {}) - authenticated = authentication.get("authenticated") + authenticated = authentication_payload.get("authenticated") authentication_verified = bool( - authentication.get("authentication_verified") is True + authentication_payload.get("authentication_verified") is True ) pricing_policy = cost_guard_payload.get("pricing_policy") or {} accounting = cost_guard_payload.get("accounting") or {} @@ -103,9 +114,61 @@ def production_route_readback( and pricing_policy.get("supported") is True and cost_guard_payload.get("cost_exceeded") is False ) - gemini_production_executable = bool( - gemini_enabled and cost_guard_ready and authentication_verified + return { + "configured": bool(configured), + "enabled": bool(enabled), + "route_ready": bool(enabled and cost_guard_ready and authentication_verified), + "authenticated": authenticated, + "authentication_verified": authentication_verified, + "verification_status": authentication_payload.get( + "verification_status", "not_verified" + ), + "verification_source": authentication_payload.get("verification_source"), + "verified_at": authentication_payload.get("verified_at"), + "cost_guard_ready": cost_guard_ready, + "cost_guard": cost_guard_payload, + } + + +def production_route_readback( + *, + gemini_configured: bool, + gemini_enabled: bool, + gemini_authentication: dict[str, Any] | None = None, + cost_guard: dict[str, Any] | None = None, + claude_configured: bool = False, + claude_enabled: bool = False, + claude_authentication: dict[str, Any] | None = None, + claude_cost_guard: dict[str, Any] | None = None, + claude_execution_mode: str = "shadow", + claude_canary_percent: int = 0, +) -> dict[str, Any]: + """Build a public-safe route/status payload without endpoint URLs or keys.""" + gemini = _paid_provider_readback( + configured=gemini_configured, + enabled=gemini_enabled, + authentication=gemini_authentication, + cost_guard=cost_guard, ) + claude = _paid_provider_readback( + configured=claude_configured, + enabled=claude_enabled, + authentication=claude_authentication, + cost_guard=claude_cost_guard, + ) + execution_mode = str(claude_execution_mode or "shadow").strip().lower() + canary_percent = max(0, min(100, int(claude_canary_percent or 0))) + claude["execution_mode"] = execution_mode + claude["canary_percent"] = canary_percent + claude["production_executable"] = bool( + claude["route_ready"] and execution_mode == "production" + ) + claude["canary_executable"] = bool( + claude["route_ready"] + and execution_mode == "canary" + and canary_percent > 0 + ) + gemini["production_executable"] = bool(gemini["route_ready"]) hops = [ { "position": 1, @@ -133,40 +196,43 @@ def production_route_readback( }, { "position": 4, + "provider": "claude", + "identity": "Anthropic Claude API", + "runtime": "claude-sonnet-5", + "paid": True, + "production_executable": claude["production_executable"], + "canary_executable": claude["canary_executable"], + "execution_mode": execution_mode, + "configured": bool(claude_configured), + "authenticated": claude["authenticated"], + "authentication_verified": claude["authentication_verified"], + "verification_status": claude["verification_status"], + }, + { + "position": 5, "provider": "gemini", "identity": "Gemini API", "runtime": "gemini-2.5-flash-lite", "paid": True, - "production_executable": gemini_production_executable, + "production_executable": gemini["production_executable"], "configured": bool(gemini_configured), - "authenticated": authenticated, - "authentication_verified": authentication_verified, - "verification_status": authentication.get( - "verification_status", "not_verified" - ), + "authenticated": gemini["authenticated"], + "authentication_verified": gemini["authentication_verified"], + "verification_status": gemini["verification_status"], }, ] return { "schema_version": "ai_provider_production_route_v1", "provider_order": list(PRODUCTION_PROVIDER_ORDER), - "route_label": "GCP-A -> GCP-B -> host111 Ollama -> Gemini API", + "route_label": ( + "GCP-A -> GCP-B -> host111 Ollama -> " + "Anthropic Claude API -> Gemini API" + ), "hops": hops, "legacy_provider_policy": { provider: "non_executable_shadow_metadata_only" for provider in sorted(NON_EXECUTABLE_SHADOW_PROVIDERS) }, - "gemini": { - "configured": bool(gemini_configured), - "enabled": bool(gemini_enabled), - "production_executable": gemini_production_executable, - "authenticated": authenticated, - "authentication_verified": authentication_verified, - "verification_status": authentication.get( - "verification_status", "not_verified" - ), - "verification_source": authentication.get("verification_source"), - "verified_at": authentication.get("verified_at"), - "cost_guard_ready": cost_guard_ready, - "cost_guard": cost_guard_payload, - }, + "claude": claude, + "gemini": gemini, } diff --git a/apps/api/src/services/ai_provider_route_matrix.py b/apps/api/src/services/ai_provider_route_matrix.py index 1a9d2cbc5..9380fbdae 100644 --- a/apps/api/src/services/ai_provider_route_matrix.py +++ b/apps/api/src/services/ai_provider_route_matrix.py @@ -83,6 +83,7 @@ def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None: "shadow_or_canary_allowed", "openclaw_replacement_allowed", "nemotron_shadow_allowed", + "claude_direct_call_allowed", "gemini_direct_call_allowed", "secret_read_allowed", "secret_plaintext_allowed", @@ -164,6 +165,7 @@ def _require_route_evidence(payload: dict[str, Any], label: str) -> None: "alert_ai_ollama_first_lane", "openclaw_nemo_rca_lane", "nemotron_tool_calling_candidate", + "claude_paid_fallback_policy", "gemini_final_fallback_policy", } present = {route.get("route_id") for route in routes} @@ -202,7 +204,7 @@ def _require_candidate_gates(payload: dict[str, Any], label: str) -> None: def _require_global_production_route(payload: dict[str, Any], label: str) -> None: - """Ensure every executable route row publishes the same four-hop chain.""" + """Ensure every executable route row publishes the same five-hop chain.""" exceptions = {"nemotron_tool_calling_candidate"} invalid = sorted( route.get("route_id") diff --git a/apps/api/src/services/ai_providers/claude.py b/apps/api/src/services/ai_providers/claude.py index 8665e0c44..5f4b5a7f4 100644 --- a/apps/api/src/services/ai_providers/claude.py +++ b/apps/api/src/services/ai_providers/claude.py @@ -1,46 +1,217 @@ -""" -Claude Provider - Phase 24 ADR-052 -==================================== -Anthropic Claude API (Claude Haiku 4.5, Tool Use 強制 JSON) +"""Anthropic Claude paid fallback provider. -搬移自: openclaw.py _call_claude (L474-550) -特性: 最強推理、昂貴、CRITICAL/DELETE 強制使用 - -2026-04-02 ogt: Phase 24-A 從 openclaw.py 抽出 +The executable route is guarded in this order: +environment opt-in -> shadow/canary/production mode -> persistent runtime +disable state -> atomic token/cost reservation -> API call -> durable finalize. +No API key value, prompt, response, or provider error body is written into the +paid-provider receipt. """ from __future__ import annotations +import hashlib import json import time +from typing import Any import httpx import structlog from src.core.config import get_settings from src.plugins.mcp.interfaces import MCPTool -from src.services.ai_providers.interfaces import AIResult, is_provider_enabled_by_env +from src.services.ai_providers.interfaces import ( + AIResult, + cloud_context_block_reason, + is_provider_enabled_by_env, +) from src.services.ai_providers.tool_schema import anthropic_tools_for_agent -from src.services.model_registry import get_model_registry logger = structlog.get_logger(__name__) settings = get_settings() -class ClaudeProvider: - """ - Anthropic Claude Cloud Provider +def classify_claude_provider_error(error: Exception) -> str: + """Return a bounded non-secret receipt code for an Anthropic failure.""" + if isinstance(error, httpx.HTTPStatusError): + status_code = error.response.status_code + if status_code in {401, 403}: + return "authentication_rejected" + if status_code == 429: + return "provider_rate_limited" + return f"http_{status_code}" + if isinstance(error, httpx.TimeoutException | TimeoutError): + return "provider_timeout" + if isinstance(error, httpx.RequestError): + return "provider_transport_error" + return type(error).__name__[:80] - privacy_level: cloud (禁止機密資料) - capabilities: rca, chat, code_review + +def _structured_tool_result_envelope(result: Any) -> dict[str, Any] | None: + """Return one structured result without retaining or logging its content.""" + try: + envelope = result.to_dict() if hasattr(result, "to_dict") else result + except Exception: + return None + return envelope if isinstance(envelope, dict) else None + + +def _tool_result_block_reason(envelope: dict[str, Any] | None) -> str | None: + """Apply the producer-supplied cloud metadata contract to a tool result. + + MCPToolResult wraps provider output under ``output``. A safe classification + there can attest that wrapper, but only after the output itself passes the + same structured deny-key walk. This does not inspect arbitrary text or + claim DLP coverage. """ + if envelope is None: + return "cloud_context_classification_missing" + + reason = cloud_context_block_reason(envelope) + if reason != "cloud_context_classification_missing": + return reason + + output = envelope.get("output") + if not isinstance(output, dict): + return reason + output_reason = cloud_context_block_reason(output) + if output_reason: + return output_reason + + # The output just supplied an explicit safe classification. Promote only + # that attestation so the complete wrapper is still walked for deny keys. + wrapper_context = dict(envelope) + wrapper_context["data_classification"] = "sanitized" + return cloud_context_block_reason(wrapper_context) + + +def _tool_result_block_audit( + audit: dict[str, Any], + *, + reason: str, + iteration: int, + provider_call_count: int, + receipt_finalized: bool, +) -> dict[str, Any]: + receipt = dict(audit) + receipt["tool_result_privacy_receipt"] = { + "schema_version": "cloud_tool_result_privacy_v1", + "status": "blocked_no_forward", + "reason": reason, + "iteration": iteration, + "provider_call_count": provider_call_count, + "paid_iteration_terminal": "failed", + "receipt_finalized": receipt_finalized, + "content_logged": False, + "content_forwarded": False, + } + return receipt + + +async def _finalize_generation_once( + limiter: Any, + reservation: Any, + *, + success: bool, + prompt_tokens: int = 0, + completion_tokens: int = 0, + error_code: str = "", +) -> bool: + """Attempt exactly one terminal transition without logging exception text.""" + try: + finalize_kwargs: dict[str, Any] = {"success": success} + if success or prompt_tokens or completion_tokens: + finalize_kwargs.update( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + if error_code: + finalize_kwargs["error_code"] = error_code + return bool(await limiter.finalize_generation(reservation, **finalize_kwargs)) + except Exception: + return False + + +def _analysis_tool() -> dict[str, Any]: + return { + "name": "submit_analysis", + "description": "Submit the RCA analysis result in structured format", + "input_schema": { + "type": "object", + "properties": { + "action_title": {"type": "string"}, + "description": {"type": "string"}, + "suggested_action": { + "type": "string", + "enum": [ + "RESTART_DEPLOYMENT", + "DELETE_POD", + "SCALE_DEPLOYMENT", + "NO_ACTION", + ], + }, + "kubectl_command": {"type": "string"}, + "target_resource": {"type": "string"}, + "namespace": {"type": "string"}, + "risk_level": { + "type": "string", + "enum": ["low", "medium", "high", "critical"], + }, + "blast_radius": { + "type": "object", + "properties": { + "affected_pods": {"type": "integer"}, + "estimated_downtime": {"type": "string"}, + "related_services": { + "type": "array", + "items": {"type": "string"}, + }, + "data_impact": { + "type": "string", + "enum": ["NONE", "READ_ONLY", "WRITE", "DESTRUCTIVE"], + }, + }, + "required": [ + "affected_pods", + "estimated_downtime", + "related_services", + "data_impact", + ], + }, + "reasoning": {"type": "string"}, + "deviation_analysis": {"type": "string"}, + "confidence": {"type": "number"}, + "affected_services": { + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": [ + "action_title", + "description", + "suggested_action", + "kubectl_command", + "target_resource", + "namespace", + "risk_level", + "blast_radius", + "reasoning", + "confidence", + ], + }, + } + + +class ClaudeProvider: + """Anthropic Claude cloud provider with fail-closed paid execution gates.""" def __init__(self) -> None: self._http_client: httpx.AsyncClient | None = None async def _get_client(self) -> httpx.AsyncClient: if self._http_client is None or self._http_client.is_closed: - self._http_client = httpx.AsyncClient(timeout=httpx.Timeout(30.0, connect=10.0)) + self._http_client = httpx.AsyncClient( + timeout=httpx.Timeout(30.0, connect=10.0) + ) return self._http_client @property @@ -49,31 +220,151 @@ class ClaudeProvider: @property def is_enabled(self) -> bool: - if not settings.CLAUDE_API_KEY: - return False - return is_provider_enabled_by_env("claude") + mode = str(settings.CLAUDE_EXECUTION_MODE).strip().lower() + mode_enabled = mode == "production" or ( + mode == "canary" and int(settings.CLAUDE_CANARY_PERCENT) > 0 + ) + return bool( + settings.CLAUDE_API_KEY + and is_provider_enabled_by_env("claude") + and mode_enabled + ) @property def capabilities(self) -> set[str]: - return {"rca", "chat", "code_review"} + return {"rca", "chat", "code_review", "tool_calling"} @property def privacy_level(self) -> str: return "cloud" + @staticmethod + def _execution_mode_gate( + context: dict[str, Any] | None, + ) -> tuple[bool, str, str, int | None]: + mode = str(settings.CLAUDE_EXECUTION_MODE or "shadow").strip().lower() + if mode == "production": + return True, "production_route", mode, None + if mode == "shadow": + return False, "claude_shadow_metadata_only", mode, None + if mode != "canary": + return False, "claude_execution_mode_invalid", mode, None + + percent = max(0, min(100, int(settings.CLAUDE_CANARY_PERCENT or 0))) + run_id = str((context or {}).get("run_id") or "") + if percent <= 0 or not run_id: + return False, "claude_canary_not_selected", mode, None + bucket = int(hashlib.sha256(run_id.encode("utf-8")).hexdigest()[:8], 16) % 100 + return ( + bucket < percent, + "claude_canary_selected" if bucket < percent else "claude_canary_not_selected", + mode, + bucket, + ) + + async def _preflight( + self, + context: dict[str, Any] | None, + ) -> tuple[bool, str, str, int | None]: + allowed, reason, mode, bucket = self._execution_mode_gate(context) + if not allowed: + # Shadow/canary exclusion is intentionally evaluated before auth or + # runtime probes: an excluded request performs zero external calls. + return False, reason, mode, bucket + + privacy_block = cloud_context_block_reason(context) + if privacy_block: + return False, privacy_block, mode, bucket + if not settings.CLAUDE_API_KEY: + return False, "claude_not_configured", "disabled", None + if not is_provider_enabled_by_env("claude"): + return False, "claude_disabled_by_environment", "disabled", None + + try: + from src.services.ai_control import is_provider_disabled + + if await is_provider_disabled("claude"): + return False, "claude_disabled_by_runtime_control", mode, bucket + except Exception: + return False, "claude_disable_state_unavailable", mode, bucket + return True, reason, mode, bucket + + @staticmethod + def _audit_metadata( + *, + mode: str, + receipt_id: str | None = None, + pricing_version: str | None = None, + canary_bucket: int | None = None, + receipt_ids: list[str] | None = None, + ) -> dict[str, Any]: + return { + "schema_version": "paid_provider_audit_v1", + "paid_provider": True, + "provider": "claude", + "fallback_position": 4, + "execution_mode": mode, + "generation_receipt_id": receipt_id, + "generation_receipt_ids": list(receipt_ids or []), + "pricing_version": pricing_version, + "canary_bucket": canary_bucket, + } + async def analyze( self, prompt: str, - context: dict | None = None, + context: dict[str, Any] | None = None, ) -> AIResult: - if not settings.CLAUDE_API_KEY: - return AIResult(raw_response="", success=False, provider=self.name, error="CLAUDE_API_KEY not configured") + allowed, reason, mode, bucket = await self._preflight(context) + if not allowed: + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=reason, + audit_metadata=self._audit_metadata( + mode=mode, + canary_bucket=bucket, + ), + ) + + from src.services.ai_rate_limiter import get_ai_rate_limiter + + limiter = get_ai_rate_limiter() + model_name = str(settings.CLAUDE_MODEL).strip() + reservation = await limiter.reserve_generation( + "claude", + prompt, + model=model_name, + max_output_tokens=settings.CLAUDE_MAX_OUTPUT_TOKENS, + context=context, + ) + audit = self._audit_metadata( + mode=mode, + receipt_id=reservation.receipt_id, + pricing_version=reservation.pricing_version, + canary_bucket=bucket, + ) + if not reservation.allowed: + logger.warning( + "claude_provider_cost_guard_blocked", + receipt_id=reservation.receipt_id, + reason=reservation.reason, + execution_mode=mode, + ) + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=f"cost_guard_blocked:{reservation.reason}", + audit_metadata=audit, + ) start = time.perf_counter() + input_tokens = 0 + output_tokens = 0 try: - client = await self._get_client() - - response = await client.post( + response = await (await self._get_client()).post( "https://api.anthropic.com/v1/messages", headers={ "x-api-key": settings.CLAUDE_API_KEY, @@ -81,90 +372,181 @@ class ClaudeProvider: "content-type": "application/json", }, json={ - "model": get_model_registry().get_model("claude", "rca"), - "max_tokens": 2048, + "model": model_name, + "max_tokens": settings.CLAUDE_MAX_OUTPUT_TOKENS, "messages": [{"role": "user", "content": prompt}], - "tools": [{ - "name": "submit_analysis", - "description": "Submit the RCA analysis result in structured format", - "input_schema": { - "type": "object", - "properties": { - "action_title": {"type": "string"}, - "description": {"type": "string"}, - "suggested_action": {"type": "string", "enum": ["RESTART_DEPLOYMENT", "DELETE_POD", "SCALE_DEPLOYMENT", "NO_ACTION"]}, - "kubectl_command": {"type": "string"}, - "target_resource": {"type": "string"}, - "namespace": {"type": "string"}, - "risk_level": {"type": "string", "enum": ["low", "medium", "critical"]}, - "blast_radius": { - "type": "object", - "properties": { - "affected_pods": {"type": "integer"}, - "estimated_downtime": {"type": "string"}, - "related_services": {"type": "array", "items": {"type": "string"}}, - "data_impact": {"type": "string", "enum": ["NONE", "READ_ONLY", "WRITE", "DESTRUCTIVE"]}, - }, - "required": ["affected_pods", "estimated_downtime", "related_services", "data_impact"], - }, - "reasoning": {"type": "string"}, - "deviation_analysis": {"type": "string"}, - "confidence": {"type": "number"}, - "affected_services": {"type": "array", "items": {"type": "string"}}, - }, - "required": ["action_title", "description", "suggested_action", "kubectl_command", "target_resource", "namespace", "risk_level", "blast_radius", "reasoning", "confidence"], - }, - }], + "tools": [_analysis_tool()], "tool_choice": {"type": "tool", "name": "submit_analysis"}, }, timeout=30.0, ) response.raise_for_status() data = response.json() - - # I2 修復: 追蹤 tokens/cost - usage = data.get("usage", {}) - input_tokens = usage.get("input_tokens", 0) - output_tokens = usage.get("output_tokens", 0) + usage = data.get("usage") or {} + input_tokens = int(usage.get("input_tokens", 0) or 0) + output_tokens = int(usage.get("output_tokens", 0) or 0) total_tokens = input_tokens + output_tokens - # Claude Haiku 4.5: Input $1/1M, Output $5/1M - cost_usd = (input_tokens * 0.000001) + (output_tokens * 0.000005) + cost_usd = reservation.actual_cost_usd(input_tokens, output_tokens) + latency = (time.perf_counter() - start) * 1000 + + if total_tokens <= 0 or cost_usd <= 0: + total_tokens = ( + reservation.estimated_prompt_tokens + + reservation.estimated_completion_tokens + ) + cost_usd = reservation.estimated_cost_usd + + if str(data.get("stop_reason") or "").lower() == "refusal": + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code="provider_refusal", + ) + return AIResult( + raw_response="", + success=False, + provider=self.name, + tokens=total_tokens, + cost_usd=cost_usd, + latency_ms=latency, + error=( + "claude_provider_refusal" + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, + ) - # 從 Tool Use 回應中提取 JSON for block in data.get("content", []): - if block.get("type") == "tool_use" and block.get("name") == "submit_analysis": - tool_input = block.get("input", {}) - latency = (time.perf_counter() - start) * 1000 - logger.info("claude_provider_success", keys=list(tool_input.keys()), tokens=total_tokens, latency_ms=round(latency, 1)) + if not isinstance(block, dict): + continue + if ( + block.get("type") == "tool_use" + and block.get("name") == "submit_analysis" + ): + raw_response = json.dumps( + block.get("input", {}), + ensure_ascii=False, + ) + finalized = await _finalize_generation_once( + limiter, + reservation, + success=True, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + ) + if not finalized: + return AIResult( + raw_response="", + success=False, + provider=self.name, + latency_ms=latency, + error="claude_accounting_finalize_failed", + audit_metadata=audit, + ) + logger.info( + "claude_provider_success", + receipt_id=reservation.receipt_id, + tokens=total_tokens, + cost_usd=round(cost_usd, 6), + latency_ms=round(latency, 1), + execution_mode=mode, + ) return AIResult( - raw_response=json.dumps(tool_input), + raw_response=raw_response, success=True, provider=self.name, tokens=total_tokens, cost_usd=cost_usd, latency_ms=latency, + audit_metadata=audit, ) - # Fallback: text content for block in data.get("content", []): - if block.get("type") == "text": - latency = (time.perf_counter() - start) * 1000 + if not isinstance(block, dict) or block.get("type") != "text": + continue + text = str(block.get("text") or "") + if text.strip(): + finalized = await _finalize_generation_once( + limiter, + reservation, + success=True, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + ) + if not finalized: + return AIResult( + raw_response="", + success=False, + provider=self.name, + latency_ms=latency, + error="claude_accounting_finalize_failed", + audit_metadata=audit, + ) return AIResult( - raw_response=block.get("text", ""), + raw_response=text, success=True, provider=self.name, tokens=total_tokens, cost_usd=cost_usd, latency_ms=latency, + audit_metadata=audit, ) + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code="response_missing_content", + ) + return AIResult( + raw_response="", + success=False, + provider=self.name, + tokens=total_tokens, + cost_usd=cost_usd, + latency_ms=latency, + error=( + "claude_response_missing_content" + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, + ) + except Exception as error: latency = (time.perf_counter() - start) * 1000 - return AIResult(raw_response="", success=False, provider=self.name, latency_ms=latency, error="No valid response") - - except Exception as e: - latency = (time.perf_counter() - start) * 1000 - logger.warning("claude_provider_failed", error=str(e), latency_ms=round(latency, 1)) - return AIResult(raw_response="", success=False, provider=self.name, latency_ms=latency, error=str(e)) + error_code = classify_claude_provider_error(error) + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code=error_code, + ) + logger.warning( + "claude_provider_failed", + error_code=error_code, + latency_ms=round(latency, 1), + receipt_id=reservation.receipt_id, + receipt_finalized=finalized, + execution_mode=mode, + ) + return AIResult( + raw_response="", + success=False, + provider=self.name, + latency_ms=latency, + error=( + error_code if finalized else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, + ) async def analyze_with_tools( self, @@ -173,35 +555,87 @@ class ClaudeProvider: tool_executor, max_iterations: int = 5, agent_role: str = "openclaw", - context: dict | None = None, + context: dict[str, Any] | None = None, ) -> AIResult: - """Run Claude in Agent Loop mode with MCP tool_use.""" - + """Run a bounded paid Agent Loop; every iteration has its own receipt.""" if not available_tools: return await self.analyze(prompt, context=context) - if not settings.CLAUDE_API_KEY: - return AIResult(raw_response="", success=False, provider=self.name, error="CLAUDE_API_KEY not configured") + allowed, reason, mode, bucket = await self._preflight(context) + if not allowed: + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=reason, + audit_metadata=self._audit_metadata( + mode=mode, + canary_bucket=bucket, + ), + ) tools_schema = anthropic_tools_for_agent(available_tools, agent_role) if not tools_schema: return AIResult( raw_response="", success=False, provider=self.name, - error=f"No MCP tools allowed for agent_role={agent_role}", + error=f"no_mcp_tools_allowed:{agent_role}", + audit_metadata=self._audit_metadata(mode=mode, canary_bucket=bucket), ) - start = time.perf_counter() + from src.services.ai_rate_limiter import get_ai_rate_limiter + + limiter = get_ai_rate_limiter() + model_name = str(settings.CLAUDE_MODEL).strip() + messages: list[dict[str, Any]] = [{"role": "user", "content": prompt}] total_tokens = 0 total_cost = 0.0 - messages: list[dict] = [{"role": "user", "content": prompt}] + receipt_ids: list[str] = [] + last_text = "" + start = time.perf_counter() - try: - client = await self._get_client() - last_text = "" + for iteration in range(max(1, min(10, int(max_iterations)))): + iteration_context = dict(context or {}) + base_run_id = str(iteration_context.get("run_id") or "")[:142] + iteration_context["run_id"] = f"{base_run_id}:tool:{iteration + 1}" + reservation_prompt = json.dumps( + {"messages": messages, "tools": tools_schema}, + ensure_ascii=False, + default=str, + ) + reservation = await limiter.reserve_generation( + "claude", + reservation_prompt, + model=model_name, + max_output_tokens=settings.CLAUDE_MAX_OUTPUT_TOKENS, + context=iteration_context, + ) + receipt_ids.append(reservation.receipt_id) + audit = self._audit_metadata( + mode=mode, + receipt_id=reservation.receipt_id, + receipt_ids=receipt_ids, + pricing_version=reservation.pricing_version, + canary_bucket=bucket, + ) + if not reservation.allowed: + return AIResult( + raw_response=last_text, + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error=f"cost_guard_blocked:{reservation.reason}", + audit_metadata=audit, + ) - for iteration in range(max_iterations): - response = await client.post( + input_tokens = 0 + output_tokens = 0 + iteration_tokens = 0 + iteration_cost = 0.0 + try: + response = await (await self._get_client()).post( "https://api.anthropic.com/v1/messages", headers={ "x-api-key": settings.CLAUDE_API_KEY, @@ -209,8 +643,8 @@ class ClaudeProvider: "content-type": "application/json", }, json={ - "model": get_model_registry().get_model("claude", "rca"), - "max_tokens": 2048, + "model": model_name, + "max_tokens": settings.CLAUDE_MAX_OUTPUT_TOKENS, "messages": messages, "tools": tools_schema, }, @@ -218,93 +652,315 @@ class ClaudeProvider: ) response.raise_for_status() data = response.json() - - usage = data.get("usage", {}) - input_tokens = usage.get("input_tokens", 0) - output_tokens = usage.get("output_tokens", 0) - total_tokens += input_tokens + output_tokens - total_cost += (input_tokens * 0.000001) + (output_tokens * 0.000005) - - content_blocks = data.get("content", []) - text_blocks = [ - block.get("text", "") - for block in content_blocks - if block.get("type") == "text" - ] - if text_blocks: - last_text = "\n".join(text_blocks) - - tool_uses = [ - block for block in content_blocks - if block.get("type") == "tool_use" - ] - if not tool_uses: - latency = (time.perf_counter() - start) * 1000 - logger.info( - "claude_agent_loop_success", - agent_role=agent_role, - iterations=iteration + 1, - tokens=total_tokens, - latency_ms=round(latency, 1), + usage = data.get("usage") or {} + input_tokens = int(usage.get("input_tokens", 0) or 0) + output_tokens = int(usage.get("output_tokens", 0) or 0) + iteration_tokens = input_tokens + output_tokens + iteration_cost = reservation.actual_cost_usd( + input_tokens, + output_tokens, + ) + except Exception as error: + error_code = classify_claude_provider_error(error) + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code=error_code, + ) + if iteration_tokens <= 0 or iteration_cost <= 0: + iteration_tokens = ( + reservation.estimated_prompt_tokens + + reservation.estimated_completion_tokens ) + iteration_cost = reservation.estimated_cost_usd + total_tokens += iteration_tokens + total_cost += iteration_cost + return AIResult( + raw_response="", + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error=( + error_code + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, + ) + + if iteration_tokens <= 0 or iteration_cost <= 0: + iteration_tokens = ( + reservation.estimated_prompt_tokens + + reservation.estimated_completion_tokens + ) + iteration_cost = reservation.estimated_cost_usd + + if str(data.get("stop_reason") or "").lower() == "refusal": + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code="provider_refusal", + ) + total_tokens += iteration_tokens + total_cost += iteration_cost + return AIResult( + raw_response="", + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error=( + "claude_provider_refusal" + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, + ) + + raw_content = data.get("content", []) + content_blocks = [ + block for block in raw_content if isinstance(block, dict) + ] if isinstance(raw_content, list) else [] + text_blocks = [ + str(block.get("text") or "") + for block in content_blocks + if block.get("type") == "text" + and str(block.get("text") or "").strip() + ] + iteration_text = "\n".join(text_blocks) + if iteration_text: + last_text = iteration_text + tool_uses = [ + block for block in content_blocks if block.get("type") == "tool_use" + ] + if not tool_uses: + terminal_error = "" if iteration_text else "response_missing_content" + finalized = await _finalize_generation_once( + limiter, + reservation, + success=bool(iteration_text), + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code=terminal_error, + ) + total_tokens += iteration_tokens + total_cost += iteration_cost + if not iteration_text or not finalized: return AIResult( - raw_response=last_text or json.dumps(data, ensure_ascii=False), - success=True, + raw_response="", + success=False, provider=f"{self.name}_agent_loop", tokens=total_tokens, cost_usd=total_cost, - latency_ms=latency, + latency_ms=(time.perf_counter() - start) * 1000, + error=( + "claude_agent_loop_response_missing_content" + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, ) + return AIResult( + raw_response=iteration_text, + success=True, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + audit_metadata=audit, + ) - tool_results = [] - for block in tool_uses: - tool_name = str(block.get("name", "")) - tool_input = block.get("input") or {} - result = await tool_executor(tool_name, tool_input) - tool_results.append({ + tool_results = [] + for block in tool_uses: + tool_name = str(block.get("name") or "") + try: + result = await tool_executor(tool_name, block.get("input") or {}) + except Exception as error: + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code="tool_executor_failed", + ) + total_tokens += iteration_tokens + total_cost += iteration_cost + logger.warning( + "claude_agent_loop_tool_executor_failed", + receipt_id=reservation.receipt_id, + error_code="tool_executor_failed", + error_type=type(error).__name__[:80], + iteration=iteration + 1, + receipt_finalized=finalized, + ) + return AIResult( + raw_response="", + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error=( + "claude_tool_executor_failed" + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, + ) + result_envelope = _structured_tool_result_envelope(result) + if result_envelope and result_envelope.get("success") is False: + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code="tool_executor_failed", + ) + total_tokens += iteration_tokens + total_cost += iteration_cost + logger.warning( + "claude_agent_loop_tool_executor_failed", + receipt_id=reservation.receipt_id, + error_code="tool_executor_failed", + error_type="structured_failure_receipt", + iteration=iteration + 1, + receipt_finalized=finalized, + ) + return AIResult( + raw_response="", + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error=( + "claude_tool_executor_failed" + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=audit, + ) + tool_result_block = _tool_result_block_reason(result_envelope) + if tool_result_block: + finalized = await _finalize_generation_once( + limiter, + reservation, + success=False, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + error_code="tool_result_privacy_blocked", + ) + total_tokens += iteration_tokens + total_cost += iteration_cost + blocked_audit = _tool_result_block_audit( + audit, + reason=tool_result_block, + iteration=iteration + 1, + provider_call_count=len(receipt_ids), + receipt_finalized=finalized, + ) + logger.warning( + "claude_agent_loop_tool_result_blocked", + receipt_id=reservation.receipt_id, + reason=tool_result_block, + iteration=iteration + 1, + content_logged=False, + content_forwarded=False, + ) + return AIResult( + raw_response="", + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error=( + f"claude_tool_result_blocked:{tool_result_block}" + if finalized + else "claude_accounting_finalize_failed" + ), + audit_metadata=blocked_audit, + ) + tool_results.append( + { "type": "tool_result", "tool_use_id": block.get("id"), "content": json.dumps( - result.to_dict() if hasattr(result, "to_dict") else result, + result_envelope, ensure_ascii=False, default=str, ), - }) - - messages.append({"role": "assistant", "content": content_blocks}) - messages.append({"role": "user", "content": tool_results}) - - latency = (time.perf_counter() - start) * 1000 - return AIResult( - raw_response=last_text, - success=False, - provider=f"{self.name}_agent_loop", - tokens=total_tokens, - cost_usd=total_cost, - latency_ms=latency, - error=f"Agent loop exceeded max_iterations={max_iterations}", + } + ) + finalized = await _finalize_generation_once( + limiter, + reservation, + success=True, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, ) + total_tokens += iteration_tokens + total_cost += iteration_cost + if not finalized: + return AIResult( + raw_response="", + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error="claude_accounting_finalize_failed", + audit_metadata=audit, + ) + messages.append({"role": "assistant", "content": content_blocks}) + messages.append({"role": "user", "content": tool_results}) - except Exception as e: - latency = (time.perf_counter() - start) * 1000 - logger.warning( - "claude_agent_loop_failed", - agent_role=agent_role, - error=str(e), - latency_ms=round(latency, 1), - ) - return AIResult( - raw_response="", - success=False, - provider=f"{self.name}_agent_loop", - tokens=total_tokens, - cost_usd=total_cost, - latency_ms=latency, - error=str(e), - ) + return AIResult( + raw_response=last_text, + success=False, + provider=f"{self.name}_agent_loop", + tokens=total_tokens, + cost_usd=total_cost, + latency_ms=(time.perf_counter() - start) * 1000, + error=f"agent_loop_exceeded:{max_iterations}", + audit_metadata=self._audit_metadata( + mode=mode, + receipt_ids=receipt_ids, + canary_bucket=bucket, + ), + ) async def health_check(self) -> bool: - return bool(settings.CLAUDE_API_KEY) + if not self.is_enabled: + return False + try: + from src.services.ai_control import is_provider_disabled + from src.services.ai_rate_limiter import get_ai_rate_limiter + + if await is_provider_disabled("claude"): + return False + authentication = ( + await get_ai_rate_limiter().get_provider_authentication_status( + "claude" + ) + ) + return bool( + authentication.get("authenticated") is True + and authentication.get("authentication_verified") is True + ) + except Exception: + return False async def close(self) -> None: if self._http_client: diff --git a/apps/api/src/services/ai_providers/gemini.py b/apps/api/src/services/ai_providers/gemini.py index b35f184df..ea6c8b027 100644 --- a/apps/api/src/services/ai_providers/gemini.py +++ b/apps/api/src/services/ai_providers/gemini.py @@ -10,38 +10,29 @@ Google Gemini Cloud API (gemini-2.5-flash-lite) 2026-04-29 ogt + Claude Code: P0 SECRET LEAK 修復 發現 prod log 出現完整 API key 明碼（feedback_secret_debug_output_ban 鐵律） 根因：httpx HTTPStatusError str() 會包含完整 URL（含 ?key=... query string） - 修法：改用 x-goog-api-key header，並保留 _sanitize_error 當防線 + 修法：改用 x-goog-api-key header，exception log 只保留 bounded code/type """ from __future__ import annotations -import re import time +from typing import Any import httpx import structlog from src.core.config import get_settings -from src.services.ai_providers.interfaces import AIResult, is_provider_enabled_by_env +from src.services.ai_providers.interfaces import ( + AIResult, + cloud_context_block_reason, + is_provider_enabled_by_env, +) from src.services.model_registry import get_model_registry logger = structlog.get_logger(__name__) settings = get_settings() -# 2026-04-29 ogt + Claude Code: P0 SECRET LEAK -# httpx exception str() 會把完整 URL 含 ?key=AIzaSy... 寫進 error message -# 此 redact 函式在所有 logger 出口先過濾，防止 K8s pod log / Sentry / Telegram -# 任何下游接收端看到 key 明碼 -_KEY_REDACT_PATTERN = re.compile(r"([?&])key=[^&\s'\"]+") - - -def _sanitize_error(error: object) -> str: - """從錯誤訊息移除 ?key=xxx 等敏感 query string""" - msg = str(error) - return _KEY_REDACT_PATTERN.sub(r"\1key=", msg) - - def classify_gemini_provider_error(error: Exception) -> str: """Return a bounded, non-secret receipt code for a provider failure.""" if isinstance(error, httpx.HTTPStatusError): @@ -105,19 +96,51 @@ class GeminiProvider: def privacy_level(self) -> str: return "cloud" + @staticmethod + def _audit_metadata( + *, + receipt_id: str | None = None, + pricing_version: str | None = None, + ) -> dict[str, Any]: + return { + "schema_version": "paid_provider_audit_v1", + "paid_provider": True, + "provider": "gemini", + "fallback_position": 5, + "execution_mode": "production", + "generation_receipt_id": receipt_id, + "pricing_version": pricing_version, + } + async def analyze( self, prompt: str, context: dict | None = None, ) -> AIResult: + privacy_block = cloud_context_block_reason(context) + if privacy_block: + return AIResult( + raw_response="", + success=False, + provider=self.name, + error=privacy_block, + audit_metadata=self._audit_metadata(), + ) if not settings.GEMINI_API_KEY: - return AIResult(raw_response="", success=False, provider=self.name, error="GEMINI_API_KEY not configured") + return AIResult( + raw_response="", + success=False, + provider=self.name, + error="GEMINI_API_KEY not configured", + audit_metadata=self._audit_metadata(), + ) if not is_provider_enabled_by_env("gemini"): return AIResult( raw_response="", success=False, provider=self.name, error="gemini_disabled_by_environment", + audit_metadata=self._audit_metadata(), ) try: @@ -129,6 +152,7 @@ class GeminiProvider: success=False, provider=self.name, error="gemini_disabled_by_runtime_control", + audit_metadata=self._audit_metadata(), ) except Exception: # Paid fallback is fail-closed when its durable disable-state @@ -138,6 +162,7 @@ class GeminiProvider: success=False, provider=self.name, error="gemini_disable_state_unavailable", + audit_metadata=self._audit_metadata(), ) from src.services.ai_rate_limiter import get_ai_rate_limiter @@ -151,6 +176,10 @@ class GeminiProvider: max_output_tokens=settings.GEMINI_MAX_OUTPUT_TOKENS, context=context, ) + audit = self._audit_metadata( + receipt_id=reservation.receipt_id, + pricing_version=reservation.pricing_version, + ) if not reservation.allowed: logger.warning( "gemini_provider_cost_guard_blocked", @@ -162,6 +191,7 @@ class GeminiProvider: success=False, provider=self.name, error=f"cost_guard_blocked:{reservation.reason}", + audit_metadata=audit, ) start = time.perf_counter() @@ -223,6 +253,7 @@ class GeminiProvider: provider=self.name, latency_ms=latency, error="gemini_accounting_finalize_failed", + audit_metadata=audit, ) # Missing usageMetadata is explicitly estimated, never exposed as a @@ -248,27 +279,33 @@ class GeminiProvider: tokens=total_tokens, cost_usd=cost_usd, latency_ms=latency, + audit_metadata=audit, ) - except Exception as e: + except Exception as error: latency = (time.perf_counter() - start) * 1000 - # 2026-04-29 ogt + Claude Code: P0 SECRET LEAK 修復 - # 之前 str(e) 會洩漏 URL 中的 ?key=AIzaSy... 到 prod log - # 現用 _sanitize_error 過濾 ?key= query string - safe_error = _sanitize_error(e) + error_code = classify_gemini_provider_error(error) finalized = await rate_limiter.finalize_generation( reservation, success=False, - error_code=classify_gemini_provider_error(e), + error_code=error_code, ) logger.warning( "gemini_provider_failed", - error=safe_error, + error_code=error_code, + error_type=type(error).__name__[:80], latency_ms=round(latency, 1), receipt_id=reservation.receipt_id, receipt_finalized=finalized, ) - return AIResult(raw_response="", success=False, provider=self.name, latency_ms=latency, error=safe_error) + return AIResult( + raw_response="", + success=False, + provider=self.name, + latency_ms=latency, + error=error_code, + audit_metadata=audit, + ) async def health_check(self) -> bool: if not settings.GEMINI_API_KEY or not self.is_enabled: diff --git a/apps/api/src/services/ai_providers/interfaces.py b/apps/api/src/services/ai_providers/interfaces.py index a02909951..01619ea16 100644 --- a/apps/api/src/services/ai_providers/interfaces.py +++ b/apps/api/src/services/ai_providers/interfaces.py @@ -14,12 +14,130 @@ AI Provider Interfaces - Phase 24 ADR-052 from __future__ import annotations import os +import re from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Protocol, runtime_checkable from src.plugins.mcp.interfaces import MCPTool +_CLOUD_SAFE_DATA_CLASSIFICATIONS = frozenset( + { + "non_sensitive", + "operational_metadata", + "public", + "sanitized", + } +) +_EXPLICIT_SENSITIVE_CONTEXT_KEYS = frozenset( + { + "api_key", + "auth_header", + "authorization", + "authorization_header", + "access_token", + "bearer_token", + "client_secret", + "cookie", + "contains_private_data", + "contains_secret", + "contains_secret_value", + "contains_sensitive_context", + "credential", + "credentials", + "credentials_present", + "id_token", + "passphrase", + "password", + "payload_contains_secret", + "private_key", + "refresh_token", + "secret_value_exposed", + "session", + "session_id", + "set_cookie", + "sensitive_context", + "token_value", + } +) + + +def _canonical_context_key(value: Any) -> str: + """Normalize structured metadata keys without inspecting their values.""" + text = str(value).strip() + text = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", text) + text = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", text) + return re.sub(r"[^A-Za-z0-9]+", "_", text).strip("_").lower() + + +def _root_context_value(context: dict[str, Any], key: str) -> Any: + for raw_key, value in context.items(): + if _canonical_context_key(raw_key) == key: + return value + return None + + +def _marker_is_present(value: Any) -> bool: + """Interpret a structured context marker conservatively without prompt DLP.""" + if value is None or value is False: + return False + if isinstance(value, str): + return value.strip().lower() not in {"", "0", "false", "no", "none", "absent"} + if isinstance(value, list | tuple | set | dict): + return bool(value) + return bool(value) + + +def cloud_context_block_reason( + context: dict[str, Any] | None, + *, + require_local: bool = False, +) -> str | None: + """Fail closed on explicit structured privacy markers for cloud providers. + + This is an enforcement boundary for producer-supplied metadata, not a + claim that arbitrary prompt text has been inspected or sanitized by DLP. + Upstream normalizers must explicitly classify/redact content before using a + paid cloud lane. + """ + root_context = context or {} + if require_local or _marker_is_present( + _root_context_value(root_context, "require_local") + ): + return "cloud_context_requires_local" + + classification = _root_context_value(root_context, "data_classification") + if classification is None or not str(classification).strip(): + return "cloud_context_classification_missing" + if str(classification).strip().lower() not in _CLOUD_SAFE_DATA_CLASSIFICATIONS: + return "cloud_context_data_classification_blocked" + + def _walk(value: Any) -> str | None: + if isinstance(value, dict): + for raw_key, child in value.items(): + key = _canonical_context_key(raw_key) + if key == "data_classification" and child is not None: + classification = str(child).strip().lower() + if classification not in _CLOUD_SAFE_DATA_CLASSIFICATIONS: + return "cloud_context_data_classification_blocked" + if key == "require_local" and _marker_is_present(child): + return "cloud_context_requires_local" + if key in _EXPLICIT_SENSITIVE_CONTEXT_KEYS and _marker_is_present(child): + return "cloud_context_sensitive_marker_blocked" + if "raw_log" in key and _marker_is_present(child): + return "cloud_context_raw_log_blocked" + nested = _walk(child) + if nested: + return nested + elif isinstance(value, list | tuple | set): + for child in value: + nested = _walk(child) + if nested: + return nested + return None + + return _walk(context or {}) + # ============================================================================= # AIResult — 標準化 AI 分析結果 # ============================================================================= @@ -42,6 +160,7 @@ class AIResult: cost_usd: float = 0.0 latency_ms: float = 0.0 error: str | None = None + audit_metadata: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict: return { @@ -53,6 +172,7 @@ class AIResult: "cost_usd": self.cost_usd, "latency_ms": round(self.latency_ms, 2), "error": self.error, + "audit_metadata": self.audit_metadata, } @@ -194,5 +314,9 @@ def is_provider_enabled_by_env(provider_name: str) -> bool: bool: 是否啟用 (Gemini 預設 False，其他 provider 預設 True) """ env_key = f"ENABLE_{provider_name.upper()}" - default = "false" if provider_name.strip().lower() == "gemini" else "true" + default = ( + "false" + if provider_name.strip().lower() in {"claude", "gemini"} + else "true" + ) return os.getenv(env_key, default).lower() == "true" diff --git a/apps/api/src/services/ai_rate_limiter.py b/apps/api/src/services/ai_rate_limiter.py index 0b053485b..40d8e2c5a 100644 --- a/apps/api/src/services/ai_rate_limiter.py +++ b/apps/api/src/services/ai_rate_limiter.py @@ -1,5 +1,5 @@ """ -AI Rate Limiter - Gemini API 用量閥值控制 +AI Rate Limiter - paid AI provider usage and spend guard ========================================= 防止最終付費備援用量暴衝；超過任一硬閘即 no-write fail closed。 @@ -79,8 +79,8 @@ COST_LIMITS = { } @dataclass(frozen=True) -class GeminiPricingPolicy: - """Verified Standard text pricing used by the paid-fallback guard.""" +class PaidProviderPricingPolicy: + """Reviewed exact-model pricing used by the paid-fallback guard.""" model: str input_usd_per_million: float @@ -98,8 +98,8 @@ class GeminiPricingPolicy: # Unknown models are intentionally absent and therefore fail closed. Updating a # model requires an explicit, reviewed pricing policy update in the same change. -GEMINI_PRICING_POLICIES: dict[str, GeminiPricingPolicy] = { - "gemini-2.5-flash-lite": GeminiPricingPolicy( +GEMINI_PRICING_POLICIES: dict[str, PaidProviderPricingPolicy] = { + "gemini-2.5-flash-lite": PaidProviderPricingPolicy( model="gemini-2.5-flash-lite", input_usd_per_million=0.10, output_usd_per_million=0.40, @@ -112,11 +112,45 @@ GEMINI_PRICING_POLICIES: dict[str, GeminiPricingPolicy] = { ), } +# Sonnet 5 had an introductory $2/$10 price through 2026-08-31. The guard +# deliberately reserves the higher published standard $3/$15 rate so a +# promotion expiry cannot under-reserve spend without a source change. +CLAUDE_PRICING_POLICIES: dict[str, PaidProviderPricingPolicy] = { + "claude-sonnet-5": PaidProviderPricingPolicy( + model="claude-sonnet-5", + input_usd_per_million=3.0, + output_usd_per_million=15.0, + source="https://platform.claude.com/docs/en/about-claude/models/overview", + version="anthropic-standard-ceiling-2026-07-15", + checked_at="2026-07-15", + ), +} -def get_gemini_pricing_policy(model: str) -> GeminiPricingPolicy | None: +# Backward-compatible public type name used by existing Gemini tests/imports. +GeminiPricingPolicy = PaidProviderPricingPolicy + + +def get_gemini_pricing_policy(model: str) -> PaidProviderPricingPolicy | None: """Return a verified exact-model policy; no prefix or family matching.""" return GEMINI_PRICING_POLICIES.get(str(model).strip()) + +def get_claude_pricing_policy(model: str) -> PaidProviderPricingPolicy | None: + """Return the reviewed exact-model Anthropic policy; aliases fail closed.""" + return CLAUDE_PRICING_POLICIES.get(str(model).strip()) + + +def get_paid_provider_pricing_policy( + provider: str, + model: str, +) -> PaidProviderPricingPolicy | None: + """Resolve exact provider/model pricing without family or prefix matching.""" + if provider == "gemini": + return get_gemini_pricing_policy(model) + if provider == "claude": + return get_claude_pricing_policy(model) + return None + # Redis Keys REDIS_KEY_PREFIX = "ai_rate:" RPM_KEY = f"{REDIS_KEY_PREFIX}rpm:{{provider}}" @@ -156,7 +190,7 @@ class AIGenerationReservation: estimated_completion_tokens: int estimated_cost_usd: float date: str - model: str = "gemini-2.5-flash-lite" + model: str = "" pricing_source: str = "" pricing_version: str = "" pricing_checked_at: str = "" @@ -201,7 +235,7 @@ local function receipt(status, reason) terminal_state = 'pending_finalize' end redis.call('HSET', KEYS[9], - 'schema_version', 'gemini_generation_receipt_v1', + 'schema_version', 'ai_paid_generation_receipt_v1', 'provider', ARGV[9], 'receipt_id', ARGV[10], 'status', status, @@ -235,7 +269,7 @@ end -- One paid provider call is permitted for one canonical run_id; trace_id and -- work_item_id remain mandatory receipt correlation. The claim and the -- receipt/budget transition live in the same Lua execution so concurrent --- workers cannot double-reserve or double-call Gemini. +-- workers cannot double-reserve or double-call a paid provider. local existing_receipt_id = redis.call('GET', KEYS[10]) if existing_receipt_id then return {0, 'idempotent_replay', existing_receipt_id} @@ -400,7 +434,7 @@ elseif ARGV[6] == 'authentication_rejected' then end redis.call('HSET', KEYS[8], 'schema_version', 'ai_provider_authentication_status_v1', - 'provider', 'gemini', + 'provider', ARGV[10], 'authenticated', authentication_state, 'authentication_verified', success and 'true' or 'false', 'verification_status', verification_status, @@ -497,12 +531,12 @@ class AIRateLimiter: return hashlib.sha256(payload).hexdigest() @staticmethod - def _estimate_gemini_usage( + def _estimate_paid_usage( prompt: str, max_output_tokens: int, - pricing: GeminiPricingPolicy, + pricing: PaidProviderPricingPolicy, ) -> tuple[int, int, float]: - """Conservatively reserve Gemini tokens and spend before generation.""" + """Conservatively reserve paid-provider tokens and spend before generation.""" # One token per UTF-8 byte deliberately over-reserves mixed zh-TW/English # prompts. Completion reserves the configured hard output ceiling. prompt_tokens = max(1, len(prompt.encode("utf-8"))) @@ -512,23 +546,30 @@ class AIRateLimiter: @staticmethod def _provider_limits(provider: str) -> tuple[dict[str, int], dict[str, float]]: - """Use one settings-backed Gemini quota source across router and health.""" + """Use one settings-backed paid-provider quota source across runtime/readback.""" limits = dict(RATE_LIMITS[provider]) costs = dict(COST_LIMITS.get(provider, {})) - if provider != "gemini": + if provider not in {"claude", "gemini"}: return limits, costs from src.core.config import settings + prefix = provider.upper() limits.update( - rpm=int(settings.GEMINI_RPM_LIMIT), - daily_requests=int(settings.GEMINI_DAILY_QUOTA), - daily_tokens=int(settings.GEMINI_DAILY_TOKEN_LIMIT), + rpm=int(getattr(settings, f"{prefix}_RPM_LIMIT")), + daily_requests=int(getattr(settings, f"{prefix}_DAILY_QUOTA")), + daily_tokens=int(getattr(settings, f"{prefix}_DAILY_TOKEN_LIMIT")), ) costs.update( - daily_cost_usd=float(settings.GEMINI_DAILY_COST_LIMIT_USD), - total_cost_usd=float(settings.GEMINI_TOTAL_COST_LIMIT_USD), - alert_threshold_usd=float(settings.GEMINI_COST_ALERT_THRESHOLD_USD), + daily_cost_usd=float( + getattr(settings, f"{prefix}_DAILY_COST_LIMIT_USD") + ), + total_cost_usd=float( + getattr(settings, f"{prefix}_TOTAL_COST_LIMIT_USD") + ), + alert_threshold_usd=float( + getattr(settings, f"{prefix}_COST_ALERT_THRESHOLD_USD") + ), ) return limits, costs @@ -556,7 +597,7 @@ class AIRateLimiter: receipt_id=receipt_id, ) mapping = { - "schema_version": "gemini_generation_receipt_v1", + "schema_version": "ai_paid_generation_receipt_v1", "provider": provider, "receipt_id": receipt_id, "status": "blocked", @@ -625,14 +666,16 @@ class AIRateLimiter: Redis or configuration failure denies the call. A blocked attempt still gets a non-secret receipt when Redis is available. """ - if provider != "gemini": - raise ValueError("reserve_generation currently supports gemini only") + if provider not in {"claude", "gemini"}: + raise ValueError("reserve_generation requires a supported paid provider") from src.core.config import settings from src.utils.timezone import now_taipei_iso today = self._get_today() - model_name = str(model if model is not None else settings.GEMINI_MODEL).strip() + provider_prefix = provider.upper() + configured_model = getattr(settings, f"{provider_prefix}_MODEL") + model_name = str(model if model is not None else configured_model).strip() trace_id, run_id, work_item_id = self._safe_receipt_context(context) if not (trace_id and run_id and work_item_id): receipt_id = str(uuid4()) @@ -671,7 +714,7 @@ class AIRateLimiter: run_id, ) receipt_id = f"gen-{identity_hash[:32]}" - pricing = get_gemini_pricing_policy(model_name) + pricing = get_paid_provider_pricing_policy(provider, model_name) if pricing is None: receipt_written = await self._write_blocked_generation_receipt( provider=provider, @@ -708,9 +751,9 @@ class AIRateLimiter: output_limit = int( max_output_tokens if max_output_tokens is not None - else settings.GEMINI_MAX_OUTPUT_TOKENS + else getattr(settings, f"{provider_prefix}_MAX_OUTPUT_TOKENS") ) - prompt_tokens, completion_tokens, estimated_cost = self._estimate_gemini_usage( + prompt_tokens, completion_tokens, estimated_cost = self._estimate_paid_usage( prompt, output_limit, pricing, @@ -747,7 +790,7 @@ class AIRateLimiter: or costs["alert_threshold_usd"] <= 0 or costs["alert_threshold_usd"] > costs["total_cost_usd"] ): - raise ValueError("invalid_gemini_cost_guard_limits") + raise ValueError(f"invalid_{provider}_cost_guard_limits") redis = await self._get_redis() if redis is None: @@ -897,7 +940,10 @@ class AIRateLimiter: from src.utils.timezone import now_taipei_iso try: - pricing = get_gemini_pricing_policy(reservation.model) + pricing = get_paid_provider_pricing_policy( + reservation.provider, + reservation.model, + ) if ( pricing is None or reservation.pricing_source != pricing.source @@ -906,7 +952,7 @@ class AIRateLimiter: or reservation.input_usd_per_million != pricing.input_usd_per_million or reservation.output_usd_per_million != pricing.output_usd_per_million ): - raise ValueError("gemini_pricing_reservation_mismatch") + raise ValueError("paid_provider_pricing_reservation_mismatch") redis = await self._get_redis() if redis is None: @@ -925,7 +971,10 @@ class AIRateLimiter: else 0 ) daily_ttl = self._seconds_until_tomorrow() - auth_status_ttl = int(settings.GEMINI_USAGE_RECEIPT_TTL_SECONDS) + provider_prefix = reservation.provider.upper() + auth_status_ttl = int( + getattr(settings, f"{provider_prefix}_USAGE_RECEIPT_TTL_SECONDS") + ) receipt_key = GENERATION_RECEIPT_KEY.format( provider=reservation.provider, receipt_id=reservation.receipt_id, @@ -962,7 +1011,13 @@ class AIRateLimiter: str(error_code)[:120], auth_status_ttl, daily_ttl, - int(settings.GEMINI_FAILURE_COOLDOWN_SECONDS), + int( + getattr( + settings, + f"{provider_prefix}_FAILURE_COOLDOWN_SECONDS", + ) + ), + reservation.provider, ) result_code = int(result[0]) raw_status = result[1] @@ -1010,9 +1065,10 @@ class AIRateLimiter: Returns: tuple[bool, str | None]: (是否允許, 拒絕原因) """ - if provider == "gemini": + if provider in {"claude", "gemini"}: logger.error( - "gemini_legacy_rate_gate_rejected", + "paid_provider_legacy_rate_gate_rejected", + provider=provider, reason="reserve_generation_required", ) return False, "reserve_generation_required" @@ -1318,7 +1374,9 @@ class AIRateLimiter: """Read public-safe authentication evidence; configuration is separate.""" from src.core.config import settings - configured = bool(settings.GEMINI_API_KEY) if provider == "gemini" else False + configured = bool( + getattr(settings, f"{provider.upper()}_API_KEY", "") + ) if provider in {"claude", "gemini"} else False base = { "provider": provider, "configured": configured, @@ -1462,11 +1520,13 @@ class AIRateLimiter: pricing_info: dict[str, Any] = {} pricing_missing = False - if provider == "gemini": + if provider in {"claude", "gemini"}: from src.core.config import settings - configured_model = str(settings.GEMINI_MODEL).strip() - pricing = get_gemini_pricing_policy(configured_model) + configured_model = str( + getattr(settings, f"{provider.upper()}_MODEL") + ).strip() + pricing = get_paid_provider_pricing_policy(provider, configured_model) pricing_missing = pricing is None pricing_info = { "pricing_policy": { diff --git a/apps/api/src/services/ai_router.py b/apps/api/src/services/ai_router.py index b69238597..91c881120 100644 --- a/apps/api/src/services/ai_router.py +++ b/apps/api/src/services/ai_router.py @@ -8,8 +8,9 @@ AI Router - Phase 13.3 #87 延遲目標: < 50ms (規則引擎優先) 唯一 production 路由（所有風險、複雜度與工具路徑共用）： -GCP-A Ollama -> GCP-B Ollama -> host111 Ollama -> Gemini API。 -Claude/NVIDIA/Nemo/OpenClaw 僅保留 non-executable shadow metadata。 +GCP-A Ollama -> GCP-B Ollama -> host111 Ollama -> Anthropic Claude API +-> Gemini API。Claude/Gemini 都必須通過 persistent enablement、原子成本閘與 +per-provider circuit breaker；NVIDIA/Nemo/OpenClaw 僅保留 shadow metadata。 版本: v5.0 建立: 2026-03-26 (台北時區) @@ -28,7 +29,7 @@ Claude/NVIDIA/Nemo/OpenClaw 僅保留 non-executable shadow metadata。 | v4.2 | 2026-04-04 | Claude Code | Phase 25 P0 實測修正: _local_fallback_chain 移除 Nemotron(雲端)，僅留 Ollama(本地); timeout 依實測調整(NIM 60s/Ollama 200s) | | v4.3 | 2026-04-05 | Claude Code | Phase 25 P0 架構修正: 實測 Ollama CPU ~238s(不可用); NIM 實測 2-27s avg 10.6s; DIAGNOSE 改走 _full_fallback_chain(NIM 主力); _local_fallback_chain 廢棄 | | v4.4 | 2026-04-27 | Claude Sonnet 4.6 | A2 INC-20260425: DIAGNOSE fallback chain 移除 Ollama (CPU 238s 二次 timeout); 新增 _diagnose_fallback_chain (NEMO→GEMINI→CLAUDE); 新增 aiops_diagnose_fallback_total metric | -| v5.0 | 2026-07-14 | Codex | 舊路由全部 superseded；所有路徑固定 GCP-A→GCP-B→host111→Gemini，legacy providers 僅 shadow metadata | +| v5.1 | 2026-07-15 | Codex | Claude 以 paid/cost/canary gate 插入 host111 與 Gemini 之間；legacy providers 維持 shadow metadata | """ from __future__ import annotations @@ -36,8 +37,10 @@ from __future__ import annotations import asyncio import hashlib import json as _json +import re import time from dataclasses import dataclass, field +from datetime import UTC, datetime from enum import Enum from typing import TYPE_CHECKING, Protocol @@ -50,6 +53,8 @@ if TYPE_CHECKING: from src.core.config import get_settings from src.services.ai_provider_policy import ( NON_EXECUTABLE_SHADOW_PROVIDERS, + PAID_PROVIDER_ORDER, + PRODUCTION_OLLAMA_ORDER, PRODUCTION_PROVIDER_ORDER, normalize_production_execution_order, ) @@ -70,6 +75,24 @@ from src.services.model_registry import get_model_registry logger = structlog.get_logger(__name__) +_DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_SCHEMA = "ollama_unavailable_receipt_v1" +_DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_PREFIX = "ai:ollama:unavailable_receipt:" +_DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_ID = re.compile(r"^[A-Za-z0-9:_-]{1,128}$") +_DURABLE_OLLAMA_UNAVAILABLE_MAX_SECONDS = 300.0 + + +def _is_paid_provider_identity(provider: object) -> bool: + """Treat decorated paid identities as paid and never cache-executable.""" + + normalized = str(provider or "").strip().lower() + return any( + normalized == paid + or normalized.startswith(f"{paid}_") + or normalized.startswith(f"{paid}:") + or normalized.startswith(f"{paid}-") + for paid in PAID_PROVIDER_ORDER + ) + # ============================================================================= # Provider 定義 @@ -214,7 +237,7 @@ class AIRouter: 動態選擇最適合的 AI Provider 和模型。 Intent/risk/complexity only choose an Ollama model. They never choose a - provider. Provider placement is the global four-hop production contract. + provider. Provider placement is the global five-hop production contract. """ def __init__(self): @@ -252,6 +275,7 @@ class AIRouter: (AIProviderEnum.OLLAMA_GCP_A, self._ollama_default), (AIProviderEnum.OLLAMA_GCP_B, self._ollama_default), (AIProviderEnum.OLLAMA_LOCAL, self._ollama_default), + (AIProviderEnum.CLAUDE, self._claude_default), (AIProviderEnum.GEMINI, self._gemini_default), ] @@ -487,7 +511,7 @@ class AIRouter: 選擇 Provider 和模型 (Phase 13.3 #87 核心邏輯) 所有 intent / risk / complexity 只選擇 Ollama model；provider - execution order 永遠是 GCP-A、GCP-B、host111，最後才是 Gemini。 + execution order 永遠是 GCP-A、GCP-B、host111、Claude，最後才是 Gemini。 Args: intent: 正規化後的意圖 @@ -521,7 +545,7 @@ class AIRouter: model = self._ollama_summary if score <= 1 else self._ollama_default reason = ( f"複雜度={score}/5, 風險={risk.value}；全域 production chain " - "GCP-A -> GCP-B -> host111 -> Gemini" + "GCP-A -> GCP-B -> host111 -> Claude -> Gemini" ) return provider, model, reason @@ -556,7 +580,7 @@ class AIRouter: # DEPRECATED 2026-04-28 — 已由 _build_fallback_chain_for_intent 取代，無呼叫方 建立 Fallback 鏈 (排除已選 Provider) - Fallback 順序: GCP-A → GCP-B → host111 → Gemini + Fallback 順序: GCP-A → GCP-B → host111 → Claude → Gemini Args: selected_provider: 已選擇的 Provider @@ -719,7 +743,7 @@ class AIRouter: "condition": "all intents, risks, complexities, tools and background jobs", "provider": "ollama_gcp_a", "provider_order": list(PRODUCTION_PROVIDER_ORDER), - "reason": "single global production route; legacy providers are shadow metadata only", + "reason": "single global production route; paid providers require cost gates and legacy providers remain shadow metadata only", } ] @@ -917,6 +941,10 @@ class AIRouterExecutor: self._circuit_breakers[name] = _SimpleCircuitBreaker( name, failure_threshold=10, recovery_timeout=30.0 ) + elif name == "claude": + self._circuit_breakers[name] = _SimpleCircuitBreaker( + name, failure_threshold=3, recovery_timeout=120.0 + ) else: self._circuit_breakers[name] = _SimpleCircuitBreaker(name) return self._circuit_breakers[name] @@ -926,10 +954,203 @@ class AIRouterExecutor: """生成 Cache Key (與 openclaw.py 相容)""" ctx_hash = "" if context: - ctx_hash = f":{context.get('alert_type', '')}:{context.get('target_resource', '')}" + cloud_context = context.get("paid_cloud_context") or {} + receipt = ( + cloud_context.get("cloud_sanitization_receipt") or {} + if isinstance(cloud_context, dict) + else {} + ) + ctx_hash = ( + f":{context.get('alert_type', '')}:{context.get('target_resource', '')}:" + f"{(cloud_context or {}).get('data_classification', context.get('data_classification', ''))}:" + f"{receipt.get('receipt_id', '') if isinstance(receipt, dict) else ''}" + ) content = f"{prompt}{ctx_hash}" return f"llm_cache:{hashlib.sha256(content.encode()).hexdigest()[:16]}" + @staticmethod + def _paid_alert_execution_inputs( + prompt: str, + context: dict | None, + ) -> tuple[str | None, dict | None, str | None]: + """Validate an allowlist sanitizer receipt and isolate cloud inputs.""" + + is_alert = bool(context) and any( + key in context + for key in ("alert_type", "alertname", "alert_name", "incident_id", "signals") + ) + if not is_alert: + return prompt, context, None + cloud_context = (context or {}).get("paid_cloud_context") + paid_prompt = (context or {}).get("paid_cloud_prompt") + if not isinstance(cloud_context, dict) or not isinstance(paid_prompt, str): + return None, None, "alert_cloud_sanitization_receipt_missing" + receipt = cloud_context.get("cloud_sanitization_receipt") + payload = cloud_context.get("sanitized_payload") + if not isinstance(receipt, dict) or not isinstance(payload, dict): + return None, None, "alert_cloud_sanitization_receipt_missing" + if ( + receipt.get("schema_version") != "openclaw_paid_cloud_sanitization_v1" + or receipt.get("status") != "verified" + or receipt.get("sanitizer") != "src.services.sanitization_service.sanitize" + or receipt.get("classifier") != "allowlisted_structured_alert_fields_v1" + or receipt.get("dlp_policy") != "paid_cloud_allowlist_dlp_v1" + or receipt.get("raw_payload_forwarded") is not False + or receipt.get("raw_log_payload_forwarded") is not False + or receipt.get("secret_value_exposed") is not False + or receipt.get("private_network_value_exposed") is not False + ): + return None, None, "alert_cloud_sanitization_receipt_invalid" + canonical = _json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + payload_sha256 = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + prompt_sha256 = hashlib.sha256(paid_prompt.encode("utf-8")).hexdigest() + if receipt.get("payload_sha256") != payload_sha256: + return None, None, "alert_cloud_sanitization_payload_mismatch" + if cloud_context.get("paid_prompt_sha256") != prompt_sha256: + return None, None, "alert_cloud_sanitized_prompt_mismatch" + # Receipt fields are caller-controlled until they are rebound to the + # original alert through the canonical allowlist sanitizer. A matching + # self-signed hash alone is not an authority boundary. + from src.services.openclaw import build_paid_cloud_alert_context + + rebuilt_context, rebuilt_reason = build_paid_cloud_alert_context(context) + if rebuilt_context is None: + return None, None, rebuilt_reason + rebuilt_receipt = rebuilt_context.get("cloud_sanitization_receipt") or {} + if ( + rebuilt_context.get("sanitized_payload") != payload + or rebuilt_receipt.get("receipt_id") != receipt.get("receipt_id") + ): + return None, None, "alert_cloud_sanitization_receipt_untrusted" + restricted_context = dict(cloud_context) + for correlation_field in ("trace_id", "run_id", "work_item_id"): + if (context or {}).get(correlation_field): + restricted_context[correlation_field] = (context or {})[ + correlation_field + ] + return paid_prompt, restricted_context, None + + @staticmethod + async def _bounded_ollama_unavailable_receipts( + context: dict | None, + ) -> set[str]: + """Verify current-run free-lane unavailability against durable truth. + + Caller-provided JSON is only a lookup hint. The full receipt must exist + independently in Redis, carry a short expiry, and acknowledge a + completed failover-manager verifier before it can replace an attempt. + """ + + run_id = str((context or {}).get("run_id") or "") + if not run_id: + return set() + raw_receipts = (context or {}).get("ollama_unavailable_receipts") or [] + if isinstance(raw_receipts, dict): + raw_receipts = list(raw_receipts.values()) + valid: set[str] = set() + allowed_reasons = { + "endpoint_unreachable", + "health_probe_failed", + "provider_not_registered", + "provider_disabled", + "bounded_deadline_exhausted", + } + try: + from src.core.redis_client import get_redis + + redis = get_redis() + except Exception as redis_error: + logger.warning( + "ai_router_ollama_unavailable_receipt_store_unavailable", + error_type=type(redis_error).__name__, + ) + return set() + now = datetime.now(UTC) + bounded_receipts = ( + raw_receipts[: len(PRODUCTION_OLLAMA_ORDER)] + if isinstance(raw_receipts, list) + else [] + ) + for receipt in bounded_receipts: + if not isinstance(receipt, dict): + continue + provider = str(receipt.get("provider") or "") + receipt_id = str(receipt.get("receipt_id") or "") + if not ( + provider in PRODUCTION_OLLAMA_ORDER + and receipt.get("schema_version") + == _DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_SCHEMA + and receipt.get("source") == "ollama_failover_manager" + and receipt.get("status") == "bounded_unavailable" + and str(receipt.get("run_id") or "") == run_id + and receipt.get("check_completed") is True + and str(receipt.get("reason") or "") in allowed_reasons + and _DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_ID.fullmatch(receipt_id) + ): + continue + try: + durable_raw = await redis.get( + f"{_DURABLE_OLLAMA_UNAVAILABLE_RECEIPT_PREFIX}{receipt_id}" + ) + if isinstance(durable_raw, bytes): + durable_raw = durable_raw.decode("utf-8") + durable = _json.loads(durable_raw) if durable_raw else None + except Exception as receipt_error: + logger.warning( + "ai_router_ollama_unavailable_receipt_read_failed", + provider=provider, + error_type=type(receipt_error).__name__, + ) + continue + if not isinstance(durable, dict): + continue + contract_fields = ( + "schema_version", + "receipt_id", + "source", + "provider", + "run_id", + "status", + "check_completed", + "reason", + "observed_at", + "expires_at", + ) + if any(durable.get(field) != receipt.get(field) for field in contract_fields): + continue + if ( + durable.get("durable_write_ack") is not True + or durable.get("verifier_status") != "verified" + or durable.get("verified_by") + != "ollama_failover_manager.health_probe" + ): + continue + try: + observed_at = datetime.fromisoformat( + str(durable.get("observed_at") or "").replace("Z", "+00:00") + ) + expires_at = datetime.fromisoformat( + str(durable.get("expires_at") or "").replace("Z", "+00:00") + ) + except ValueError: + continue + if observed_at.tzinfo is None or expires_at.tzinfo is None: + continue + lifetime = (expires_at - observed_at).total_seconds() + if not ( + 0 < lifetime <= _DURABLE_OLLAMA_UNAVAILABLE_MAX_SECONDS + and observed_at <= now + and now < expires_at + ): + continue + valid.add(provider) + return valid + async def execute( self, prompt: str, @@ -990,39 +1211,89 @@ class AIRouterExecutor: error="production_route_missing_ollama_lane", ) - # Paid-provider runtime control is evaluated before cache lookup. This - # prevents a previously cached Gemini response from bypassing a newly - # asserted durable disable and prevents Redis read failures from - # silently re-adding the paid lane. + # Paid-provider runtime control is evaluated before cache lookup. This + # prevents a cached cloud response from bypassing a newly asserted + # durable disable and prevents Redis read failures from silently + # re-adding either paid lane. preflight_errors: list[str] = [] - if "gemini" in provider_order: - from src.services.ai_providers.interfaces import ( - is_provider_enabled_by_env, + from src.services.ai_control import is_provider_disabled + from src.services.ai_providers.interfaces import ( + cloud_context_block_reason, + is_provider_enabled_by_env, + ) + + paid_prompt, paid_context, paid_input_block = self._paid_alert_execution_inputs( + prompt, + context, + ) + cloud_privacy_block = paid_input_block or cloud_context_block_reason( + paid_context, + require_local=require_local, + ) + if cloud_privacy_block: + blocked_paid_providers = [ + provider + for provider in PAID_PROVIDER_ORDER + if provider in provider_order + ] + provider_order = [ + provider + for provider in provider_order + if provider not in PAID_PROVIDER_ORDER + ] + preflight_errors.extend( + f"{provider}: {cloud_privacy_block}" + for provider in blocked_paid_providers ) - - if not is_provider_enabled_by_env("gemini"): - provider_order = [ - provider for provider in provider_order if provider != "gemini" - ] - preflight_errors.append("gemini: disabled_by_environment") - logger.info("ai_router_provider_environment_disabled_pre_cache", provider="gemini") - else: + logger.warning( + "ai_router_cloud_context_blocked_pre_cache", + reason=cloud_privacy_block, + blocked_providers=blocked_paid_providers, + ) + if any(provider in provider_order for provider in PAID_PROVIDER_ORDER): + for paid_provider in PAID_PROVIDER_ORDER: + if paid_provider not in provider_order: + continue + if not is_provider_enabled_by_env(paid_provider): + provider_order = [ + provider + for provider in provider_order + if provider != paid_provider + ] + preflight_errors.append( + f"{paid_provider}: disabled_by_environment" + ) + logger.info( + "ai_router_provider_environment_disabled_pre_cache", + provider=paid_provider, + ) + continue try: - from src.services.ai_control import is_provider_disabled - - if await is_provider_disabled("gemini"): + if await is_provider_disabled(paid_provider): provider_order = [ - provider for provider in provider_order if provider != "gemini" + provider + for provider in provider_order + if provider != paid_provider ] - preflight_errors.append("gemini: disabled_by_runtime_control") - logger.info("ai_router_provider_disabled_pre_cache", provider="gemini") + preflight_errors.append( + f"{paid_provider}: disabled_by_runtime_control" + ) + logger.info( + "ai_router_provider_disabled_pre_cache", + provider=paid_provider, + ) except Exception as disable_check_error: provider_order = [ - provider for provider in provider_order if provider != "gemini" + provider + for provider in provider_order + if provider != paid_provider ] - preflight_errors.append("gemini: disabled_state_unavailable") + preflight_errors.append( + f"{paid_provider}: disabled_state_unavailable" + ) logger.warning( - "ai_router_gemini_disable_state_unavailable_pre_cache_blocked", + "ai_router_paid_disable_state_unavailable_pre_cache_blocked", + provider=paid_provider, error_type=type(disable_check_error).__name__, ) @@ -1035,6 +1306,14 @@ class AIRouterExecutor: if cached: data = _json.loads(cached) cached_provider = data.get("provider", "cache") + if _is_paid_provider_identity(cached_provider): + logger.info( + "ai_router_paid_cache_bypassed", + cache_key=cache_key[:30], + cached_provider=cached_provider, + reason="current_policy_and_ordered_attempt_recheck_required", + ) + raise ValueError("paid provider cache is non-executable") provider_allowed = cached_provider in provider_order ollama_first_required = ( bool(context) @@ -1082,20 +1361,27 @@ class AIRouterExecutor: from_cache=True, ) except Exception as e: - logger.debug("ai_router_cache_read_failed", error=str(e)) + logger.debug( + "ai_router_cache_read_failed", + error_type=type(e).__name__, + ) # ③ 遍歷 Provider + 閘門 (D3) # 2026-04-02 ogt: C1 修復 — 建立 Langfuse Trace (D5) # 包住整個執行鏈，記錄每個 Provider 的 generation try: from src.services.langfuse_client import langfuse_trace + alert_type_value = str((context or {}).get("alert_type", "")) _lf_trace_ctx = langfuse_trace( "ai_router_execute", metadata={ "provider_order": provider_order, "prompt_length": len(prompt), "require_local": require_local, - "alert_type": (context or {}).get("alert_type", ""), + "alert_type_length": len(alert_type_value), + "alert_type_sha256": hashlib.sha256( + alert_type_value.encode("utf-8") + ).hexdigest(), }, ) _lf_trace_ctx.__enter__() @@ -1104,6 +1390,10 @@ class AIRouterExecutor: errors: list[str] = list(preflight_errors) attempted_providers: set[str] = set() + bounded_unavailable_providers = ( + await self._bounded_ollama_unavailable_receipts(context) + ) + circuit_open_ollama_providers: set[str] = set() provider_timeout_seconds: float | None = None try: configured_provider_timeout = float( @@ -1170,6 +1460,40 @@ class AIRouterExecutor: errors.append(f"{provider_name}: not_registered") continue + if provider_name in PAID_PROVIDER_ORDER: + unresolved_ollama_hops = [ + hop + for hop in PRODUCTION_OLLAMA_ORDER + if hop not in attempted_providers + and hop not in bounded_unavailable_providers + ] + if circuit_open_ollama_providers: + errors.append( + f"{provider_name}: cloud_blocked_ollama_circuit_open(" + f"{','.join(sorted(circuit_open_ollama_providers))})" + ) + logger.warning( + "ai_router_cloud_blocked_by_ollama_circuit_open", + provider=provider_name, + circuit_open_providers=sorted(circuit_open_ollama_providers), + ) + continue + if unresolved_ollama_hops: + errors.append( + f"{provider_name}: cloud_blocked_ollama_attempt_receipts_missing(" + f"{','.join(unresolved_ollama_hops)})" + ) + logger.warning( + "ai_router_cloud_blocked_missing_ollama_attempt_receipts", + provider=provider_name, + unresolved_ollama_hops=unresolved_ollama_hops, + attempted_providers=sorted(attempted_providers), + bounded_unavailable_providers=sorted( + bounded_unavailable_providers + ), + ) + continue + # 隱私過濾 (D7) # 2026-04-27 Claude Sonnet 4.6: F6 — privacy_skip 不設 _last_attempted_provider（未嘗試） if require_local and provider.privacy_level != "local": @@ -1190,6 +1514,8 @@ class AIRouterExecutor: # 閘門 1: Circuit Breaker (per-provider, C2 修復) cb = self._get_circuit_breaker(provider_name) if cb.is_open(): + if provider_name in PRODUCTION_OLLAMA_ORDER: + circuit_open_ollama_providers.add(provider_name) if alert_requires_ollama_before_cloud and provider_name.startswith("ollama"): logger.warning( "ai_router_alert_ollama_circuit_bypassed", @@ -1204,9 +1530,10 @@ class AIRouterExecutor: # 閘門 2: Rate Limiter # 2026-04-02 Claude Code: Phase 24 B3 + C1 修復 — Rate Limiter (含 openclaw_nemo) - # Gemini owns an atomic requests/tokens/cost reservation inside the - # provider so every generation path is guarded exactly once. - if provider_name in ("openclaw_nemo", "nemotron", "claude"): + # Paid providers own atomic requests/tokens/cost reservations inside + # their provider implementations; legacy free/provider lanes retain + # this compatibility rate gate. + if provider_name in ("openclaw_nemo", "nemotron"): try: from src.services.ai_rate_limiter import get_ai_rate_limiter rate_limiter = get_ai_rate_limiter() @@ -1223,7 +1550,27 @@ class AIRouterExecutor: async with sem: try: attempted_providers.add(provider_name) - provider_call = provider.analyze(prompt, context) + provider_prompt = ( + paid_prompt + if provider_name in PAID_PROVIDER_ORDER + else prompt + ) + provider_context = ( + paid_context + if provider_name in PAID_PROVIDER_ORDER + else context + ) + if provider_name in PAID_PROVIDER_ORDER and ( + provider_prompt is None or provider_context is None + ): + errors.append( + f"{provider_name}: paid_cloud_execution_inputs_missing" + ) + continue + provider_call = provider.analyze( + provider_prompt, + provider_context, + ) result = ( await asyncio.wait_for( provider_call, @@ -1238,24 +1585,30 @@ class AIRouterExecutor: cb.record_success() # 記錄費用 - if result.cost_usd > 0 and provider_name != "gemini": + if ( + result.cost_usd > 0 + and provider_name not in PAID_PROVIDER_ORDER + ): try: rate_limiter = get_ai_rate_limiter() await rate_limiter.record_cost(provider_name, result.cost_usd) except Exception: pass - # 寫入 Cache (D4) - try: - redis = get_redis() - cache_data = _json.dumps({ - "response": result.raw_response, - "provider": result.provider, - "cached_at": time.strftime("%Y-%m-%dT%H:%M:%S+08:00"), - }) - await redis.set(cache_key, cache_data, ex=cache_ttl) - except Exception: - pass + # Paid responses are deliberately non-cacheable: every + # paid candidate must re-run current durable policy, + # privacy, canary, cost, and ordered-attempt gates. + if provider_name not in PAID_PROVIDER_ORDER: + try: + redis = get_redis() + cache_data = _json.dumps({ + "response": result.raw_response, + "provider": result.provider, + "cached_at": time.strftime("%Y-%m-%dT%H:%M:%S+08:00"), + }) + await redis.set(cache_key, cache_data, ex=cache_ttl) + except Exception: + pass logger.info( "ai_router_execute_success", @@ -1267,13 +1620,46 @@ class AIRouterExecutor: # D5: 記錄 Langfuse generation if _lf_trace_ctx: try: + paid_provider = provider_name in PAID_PROVIDER_ORDER + prompt_hash = hashlib.sha256( + provider_prompt.encode("utf-8") + ).hexdigest() + response_hash = hashlib.sha256( + result.raw_response.encode("utf-8") + ).hexdigest() + receipt = ( + (provider_context or {}).get( + "cloud_sanitization_receipt" + ) + or {} + ) _lf_trace_ctx.generation( name=f"{provider_name}_call", model=provider_name, - input=prompt[:500], - output=result.raw_response[:500], + input=None if paid_provider else provider_prompt[:500], + output=( + None + if paid_provider + else result.raw_response[:500] + ), usage={"total": result.tokens} if result.tokens else None, - metadata={"cost_usd": result.cost_usd, "latency_ms": round(result.latency_ms, 1)}, + metadata={ + "cost_usd": result.cost_usd, + "latency_ms": round(result.latency_ms, 1), + "content_redacted": paid_provider, + "prompt_length": len(provider_prompt), + "prompt_sha256": prompt_hash, + "response_length": len(result.raw_response), + "response_sha256": response_hash, + "sanitization_receipt_id": ( + receipt.get("receipt_id") + if isinstance(receipt, dict) + else None + ), + "generation_receipt_id": result.audit_metadata.get( + "generation_receipt_id" + ), + }, ) _lf_trace_ctx.__exit__(None, None, None) except Exception: @@ -1282,14 +1668,53 @@ class AIRouterExecutor: # Provider 回傳 success=False cb.record_failure() - errors.append(f"{provider_name}: {result.error}") - logger.warning("ai_router_provider_failed", provider=provider_name, error=result.error) + if provider_name in PAID_PROVIDER_ORDER: + error_text = str(result.error or "") + error_sha256 = hashlib.sha256( + error_text.encode("utf-8") + ).hexdigest() + errors.append( + f"{provider_name}: provider_failed(error_sha256={error_sha256})" + ) + logger.warning( + "ai_router_paid_provider_failed", + provider=provider_name, + error_length=len(error_text), + error_sha256=error_sha256, + ) + else: + errors.append(f"{provider_name}: {result.error}") + logger.warning( + "ai_router_provider_failed", + provider=provider_name, + error=result.error, + ) # 2026-04-27 A2: 記錄失敗的 provider，供下輪迭代計算 fallback metric _last_attempted_provider = provider_name except Exception as e: - errors.append(f"{provider_name}: {e}") - logger.warning("ai_router_provider_exception", provider=provider_name, error=str(e)) + if provider_name in PAID_PROVIDER_ORDER: + error_text = str(e) + error_sha256 = hashlib.sha256( + error_text.encode("utf-8") + ).hexdigest() + errors.append( + f"{provider_name}: provider_exception(error_sha256={error_sha256})" + ) + logger.warning( + "ai_router_paid_provider_exception", + provider=provider_name, + error_type=type(e).__name__, + error_length=len(error_text), + error_sha256=error_sha256, + ) + else: + errors.append(f"{provider_name}: {e}") + logger.warning( + "ai_router_provider_exception", + provider=provider_name, + error=str(e), + ) # 2026-04-05 Claude Code: v4.3 — Timeout 不計 CB 失敗 # NIM 偶爾 GPU 忙碌導致 27s，timeout 不代表 NIM 故障 # 只有明確連線錯誤（非 timeout）才累積 CB 失敗次數 diff --git a/apps/api/src/services/controlled_alert_target_router.py b/apps/api/src/services/controlled_alert_target_router.py index 7ae678a4d..558c12b43 100644 --- a/apps/api/src/services/controlled_alert_target_router.py +++ b/apps/api/src/services/controlled_alert_target_router.py @@ -17,7 +17,7 @@ import re from collections.abc import Mapping from typing import Any -from src.services.ai_provider_policy import PRODUCTION_OLLAMA_ORDER +from src.services.ai_provider_policy import PRODUCTION_PROVIDER_ORDER from src.services.service_registry import ( ServiceRegistryClient, StatefulLevel, @@ -197,7 +197,7 @@ def _typed_route( }, "reasoning_contract": { "rca": "ollama_provider_chain", - "provider_order": list(PRODUCTION_OLLAMA_ORDER), + "provider_order": list(PRODUCTION_PROVIDER_ORDER), "critic": "gemini_final_fallback_explicit_cost_gate", "paid_call_allowed": False, }, diff --git a/apps/api/src/services/model_registry.py b/apps/api/src/services/model_registry.py index 0c40b4b72..76d4c2927 100644 --- a/apps/api/src/services/model_registry.py +++ b/apps/api/src/services/model_registry.py @@ -138,9 +138,9 @@ class ModelRegistry: }, "claude": { "models": { - "default": "claude-haiku-4-5-20251001", - "rca": "claude-haiku-4-5-20251001", - "summary": "claude-haiku-4-5-20251001", + "default": "claude-sonnet-5", + "rca": "claude-sonnet-5", + "summary": "claude-sonnet-5", } }, # 2026-03-29 ogt: P2-3 加入 NVIDIA (ADR-036) @@ -192,7 +192,7 @@ class ModelRegistry: fallback_map = { "ollama": "qwen2.5:7b-instruct", "gemini": "gemini-2.5-flash-lite", - "claude": "claude-haiku-4-5-20251001", + "claude": "claude-sonnet-5", } model = fallback_map.get(provider, provider) logger.warning( diff --git a/apps/api/src/services/model_version_probe.py b/apps/api/src/services/model_version_probe.py index 8a3004c0f..c013fe1e6 100644 --- a/apps/api/src/services/model_version_probe.py +++ b/apps/api/src/services/model_version_probe.py @@ -6,8 +6,8 @@ AI Provider 版本探測 — 為每個 Provider 提供 get_version() Provider: - ollama : 34.143.170.20 GCP-A Ollama (primary) — 2026-05-03 ogt: ADR-110 GCP-A Primary - ollama_local : 192.168.0.111 / 110 proxy Ollama (local fallback) - - gemini : Google Gemini API (版本 = model name) - claude : Anthropic Claude (版本 = model name) + - gemini : Google Gemini API (版本 = model name) - openclaw_nemo : OpenClaw NemoTron (版本 = OPENCLAW_DEFAULT_MODEL) # 2026-04-27 P3.2.1 by Claude @@ -31,7 +31,7 @@ TAIPEI_TZ = timezone(timedelta(hours=8)) class ProviderVersionInfo: """AI Provider 版本快照""" - provider: str # "ollama" / "ollama_local" / "gemini" / "claude" / "openclaw_nemo" + provider: str # "ollama" / "ollama_local" / "claude" / "gemini" / "openclaw_nemo" model: str version: str # version string 或 tag（Ollama 用 modified_at，其他用 model name） digest: str | None = None # SHA256 digest（僅 Ollama 有） @@ -125,7 +125,7 @@ async def probe_gemini_version() -> ProviderVersionInfo: # ============================================================================= async def probe_claude_version() -> ProviderVersionInfo: - """Claude：model name 即版本識別（例如 "claude-haiku-4-5-20251001"） + """Claude：model name 即版本識別（由 CLAUDE_MODEL 精確設定） Anthropic 沒有 list models endpoint（截至 2026-04）， 以設定中的 claude model name 作為版本字串。 @@ -141,7 +141,9 @@ async def probe_claude_version() -> ProviderVersionInfo: if not api_key: raise RuntimeError("CLAUDE_API_KEY not configured") - model_name = "claude-haiku-4-5-20251001" + model_name = str(settings.CLAUDE_MODEL).strip() + if not model_name: + raise RuntimeError("CLAUDE_MODEL not configured") return ProviderVersionInfo( provider="claude", @@ -228,15 +230,15 @@ async def probe_all_providers() -> list[ProviderVersionInfo]: settings.OLLAMA_FALLBACK_URL or settings.OLLAMA_URL, settings.OLLAMA_HEALTH_CHECK_MODEL, ), - probe_gemini_version(), probe_claude_version(), + probe_gemini_version(), probe_openclaw_nemo_version(), ] raw = await asyncio.gather(*tasks, return_exceptions=True) results: list[ProviderVersionInfo] = [] - provider_labels = ["ollama", "ollama_local", "gemini", "claude", "openclaw_nemo"] + provider_labels = ["ollama", "ollama_local", "claude", "gemini", "openclaw_nemo"] for label, outcome in zip(provider_labels, raw, strict=True): if isinstance(outcome, ProviderVersionInfo): results.append(outcome) diff --git a/apps/api/src/services/ollama_auto_recovery.py b/apps/api/src/services/ollama_auto_recovery.py index 167eeb39d..82deb2bc0 100644 --- a/apps/api/src/services/ollama_auto_recovery.py +++ b/apps/api/src/services/ollama_auto_recovery.py @@ -31,26 +31,26 @@ from __future__ import annotations import asyncio import datetime -from datetime import timezone, timedelta +from datetime import timedelta, timezone from typing import Any, Protocol, runtime_checkable import structlog -# 台北時區 +8（標準庫保險絲，100% 可用） -# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2 -# 原 zoneinfo.ZoneInfo("Asia/Taipei") 失敗時 = None → datetime.now(None) 為 UTC -TAIPEI_TZ = timezone(timedelta(hours=8)) - from src.core.config import get_settings +from src.services.ollama_failover_manager import ( + OllamaFailoverManager, + get_ollama_failover_manager, +) from src.services.ollama_health_monitor import ( HealthStatus, OllamaHealthMonitor, get_ollama_health_monitor, ) -from src.services.ollama_failover_manager import ( - OllamaFailoverManager, - get_ollama_failover_manager, -) + +# 台北時區 +8（標準庫保險絲，100% 可用） +# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2 +# 原 zoneinfo.ZoneInfo("Asia/Taipei") 失敗時 = None → datetime.now(None) 為 UTC +TAIPEI_TZ = timezone(timedelta(hours=8)) logger = structlog.get_logger(__name__) @@ -111,7 +111,8 @@ class OllamaAutoRecoveryService: # 狀態追蹤 # 2026-05-04 ogt: B5 修復 — 改用與 failover_manager callback 一致的命名 - # failover_manager 傳入的是 "ollama_gcp_a"/"ollama_gcp_b"/"ollama_local"/"gemini" + # failover_manager 傳入的是固定五跳 provider identity： + # "ollama_gcp_a"/"ollama_gcp_b"/"ollama_local"/"claude"/"gemini" # 原設計用 "ollama" 造成 != "ollama" 永遠成立 → recovery loop 永遠以為在 Gemini # 修法：判斷改用 _is_ollama_primary()，比對 startswith("ollama_") self._current_primary: str = "ollama_gcp_a" # fallback_manager 傳入的 provider_name @@ -455,8 +456,8 @@ class OllamaAutoRecoveryService: # 2026-04-26 P2.3 by Claude Sonnet 4.6 (tool-expert) — 記錄 recovery Prometheus metric try: from src.core.metrics import ( - OLLAMA_RECOVERY_TRIGGERED_TOTAL, OLLAMA_CURRENT_PRIMARY_IS_OLLAMA, + OLLAMA_RECOVERY_TRIGGERED_TOTAL, ) OLLAMA_RECOVERY_TRIGGERED_TOTAL.labels(from_provider=from_provider).inc() OLLAMA_CURRENT_PRIMARY_IS_OLLAMA.set(1) diff --git a/apps/api/src/services/ollama_endpoint_resolver.py b/apps/api/src/services/ollama_endpoint_resolver.py index 5b30f05b0..c0736e45b 100644 --- a/apps/api/src/services/ollama_endpoint_resolver.py +++ b/apps/api/src/services/ollama_endpoint_resolver.py @@ -2,7 +2,7 @@ Ollama endpoint resolver for AWOOOI workload placement. ADR-110 gives AWOOOI three Ollama endpoints. Every runtime workload uses -GCP-A -> GCP-B -> host111. Gemini is owned by the caller/AI Router as the +GCP-A -> GCP-B -> host111. Claude then Gemini are owned by the caller/AI Router as the final paid fallback after all three Ollama endpoints are exhausted. """ diff --git a/apps/api/src/services/ollama_failover_manager.py b/apps/api/src/services/ollama_failover_manager.py index 2136574ff..955fee474 100644 --- a/apps/api/src/services/ollama_failover_manager.py +++ b/apps/api/src/services/ollama_failover_manager.py @@ -5,8 +5,8 @@ Ollama 自動容災管理 - P1.1b 路由邏輯（global_product_governance_v2）： selected primary 永遠是 GCP-A，executable chain 永遠是 - GCP-A → GCP-B → Local 111 → Gemini。健康探測只會產生 - observed_fallback，不得將 Gemini 呈現為 selected primary。 + GCP-A → GCP-B → Local 111 → Claude → Gemini。健康探測只會產生 + observed_fallback，不得將 paid provider 呈現為 selected primary。 設計說明： - GCP-A 主機：34.143.170.20（SSD，9x 載速 + 2x 推理） @@ -21,7 +21,7 @@ Ollama 自動容災管理 - P1.1b 版本: v3.0 建立: 2026-04-25 (台北時區) 建立者: Claude Engineer-C (P1.1b) -更新: 2026-05-03 ogt — GCP 三層容災（ADR-110），GCP-A → GCP-B → Local → Gemini +更新: 2026-07-15 Codex — 固定五段 route 加入 Claude Sonnet 5 # Created 2026-04-25 P1.1 by Claude Engineer-C # 2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復 # 2026-05-03 ogt: GCP 三層容災（ADR-110），GCP-A → GCP-B → Local → Gemini @@ -147,6 +147,11 @@ _GEMINI_ENDPOINT = OllamaEndpoint( provider_name="gemini", model="gemini-2.5-flash-lite", ) +_CLAUDE_ENDPOINT = OllamaEndpoint( + url="", + provider_name="claude", + model="claude-sonnet-5", +) _PAID_FALLBACK_BLOCKED_ENDPOINT = OllamaEndpoint( url="", provider_name="paid_fallback_blocked", @@ -196,8 +201,8 @@ class OllamaFailoverManager: ) -> OllamaRoutingResult: """ 三層 Ollama 容災路由（ADR-110 修正版 2026-05-04）。所有工作固定 - GCP-A → GCP-B → 111；Gemini 由實際 generation call 的原子費控 - 決定是否允許。 + GCP-A → GCP-B → 111 → Claude → Gemini；paid provider 由實際 + generation call 的原子費控決定是否允許。 Production URL slots are GCP-A primary, GCP-B secondary, and 111 local fallback; the ordered provider contract does not change by task type. @@ -240,6 +245,7 @@ class OllamaFailoverManager: fallback_chain = [ OllamaEndpoint(url=url_secondary, provider_name="ollama_gcp_b", model=model), OllamaEndpoint(url=url_tertiary, provider_name="ollama_local", model=model), + _CLAUDE_ENDPOINT, _GEMINI_ENDPOINT, ] result = OllamaRoutingResult( @@ -271,9 +277,8 @@ class OllamaFailoverManager: task_type=task_type, ) - # Provider selection never consumes paid quota. GeminiProvider performs - # the only atomic requests/tokens/cost reservation immediately before - # generateContent and fails closed when the guard cannot write a receipt. + # Provider selection never consumes paid quota. Claude/Gemini providers + # each reserve requests/tokens/cost atomically immediately before calls. # 寫入 audit_log（best-effort） await self._write_failover_audit(result) @@ -348,7 +353,12 @@ class OllamaFailoverManager: lbl_t = _short(url_local) # tertiary label _ = task_type - fixed_fallback_chain = [ep_gcp_b, ep_local, _GEMINI_ENDPOINT] + fixed_fallback_chain = [ + ep_gcp_b, + ep_local, + _CLAUDE_ENDPOINT, + _GEMINI_ENDPOINT, + ] def _result( *, @@ -420,16 +430,16 @@ class OllamaFailoverManager: ), ) - # Gemini is only an observed final policy candidate here. Actual - # selection occurs after all Ollama attempts and its own durable cost - # reservation; this readback must never label Gemini as primary. + # Claude is the first observed paid policy candidate. Actual selection + # occurs only after all Ollama attempts and its own durable cost + # reservation; this readback must never label it as primary. return _result( - observed_fallback=_GEMINI_ENDPOINT, + observed_fallback=_CLAUDE_ENDPOINT, reason=( f"selected_primary({lbl_p}) remains fixed; all Ollama health " f"observations unavailable ({health_gcp_a.status.value}, " f"{health_gcp_b.status.value}, {health_local.status.value}); " - f"observed_fallback=gemini policy candidate at {now_ts}" + f"observed_fallback=claude policy candidate at {now_ts}" ), ) @@ -501,6 +511,7 @@ class OllamaFailoverManager: fallback_chain=[ _endpoint("ollama_gcp_b"), _endpoint("ollama_local"), + _CLAUDE_ENDPOINT, _GEMINI_ENDPOINT, ], routing_reason=( diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 3dea0b0f1..649e8ee2d 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -5,7 +5,7 @@ Phase 5: OpenClaw 實體化升級 (2026-03-21) 統帥校正: SignOz 為唯一全能視力中心 Features: -- 真實 LLM SDK 整合 (告警預設 Ollama GCP-A → GCP-B → 111 → Gemini) +- 真實 LLM SDK 整合 (告警預設 Ollama GCP-A → GCP-B → 111 → Claude → Gemini) - SignOz Gold Metrics 即時擷取 (P99/Error/RPS) - AIOps Agent 專業人格 (K8s 維運 + SRE RCA 專精) - 強制結構化 JSON 輸出 (符合 API 契約) @@ -19,11 +19,13 @@ Features: """ import hashlib +import ipaddress import json import random import re import time from datetime import datetime +from typing import Any import httpx import structlog @@ -137,7 +139,250 @@ def _build_alert_cache_context_hash(alert_context: dict | None) -> str: target = alert_context.get("target_resource", "") severity = alert_context.get("severity", "") fingerprint = alert_context.get("fingerprint", "") - return f"{alertname}:{category}:{namespace}:{target}:{severity}:{fingerprint}" + classification = alert_context.get("data_classification", "") + receipt = alert_context.get("cloud_sanitization_receipt") or {} + receipt_id = receipt.get("receipt_id", "") if isinstance(receipt, dict) else "" + return ( + f"{alertname}:{category}:{namespace}:{target}:{severity}:{fingerprint}:" + f"{classification}:{receipt_id}" + ) + + +_PAID_CLOUD_PROVIDERS = frozenset({"claude", "gemini"}) +_OUTER_CACHE_LOCAL_PROVIDERS = frozenset( + {"ollama", "ollama_gcp_a", "ollama_gcp_b", "ollama_local", "mock", "mock_fallback"} +) +_PAID_ALERT_SCALAR_FIELDS = ( + "alert_type", + "alertname", + "alert_name", + "alert_category", + "severity", + "source", + "target_resource", + "namespace", + "fingerprint", + "incident_id", + "operation_type", + "initial_classification", +) +_PAID_ALERT_SIGNAL_FIELDS = ( + "alert_type", + "alert_name", + "alertname", + "severity", + "source", + "target_resource", + "namespace", +) +_PAID_CLOUD_RECEIPT_SCHEMA = "openclaw_paid_cloud_sanitization_v1" +_PAID_CLOUD_DLP_POLICY = "paid_cloud_allowlist_dlp_v1" +_PAID_CLOUD_SECRET_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"\bsk-ant-[A-Za-z0-9_-]{16,}\b", re.IGNORECASE), + re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b"), + re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b", re.IGNORECASE), + re.compile( + r"\b(?:authorization|proxy-authorization|cookie|set-cookie|x-api-key)" + r"\s*[:=]\s*[^\s,;]+(?:\s+[^\s,;]+)?", + re.IGNORECASE, + ), + re.compile(r"\bbearer\s+[A-Za-z0-9._~+/-]{8,}", re.IGNORECASE), +) +_PAID_CLOUD_IPV4_CANDIDATE = re.compile( + r"(? bool: + """Treat decorated paid-provider identities as paid and non-cacheable.""" + + normalized = str(provider or "").strip().lower() + return any( + normalized == paid + or normalized.startswith(f"{paid}_") + or normalized.startswith(f"{paid}:") + or normalized.startswith(f"{paid}-") + for paid in _PAID_CLOUD_PROVIDERS + ) + + +def _redact_private_network_addresses(value: str) -> str: + """Redact private/local IP literals from an allowlisted scalar.""" + + def _replace(match: re.Match[str]) -> str: + raw = match.group(0) + candidate = raw[1:-1] if raw.startswith("[") and raw.endswith("]") else raw + try: + address = ipaddress.ip_address(candidate) + except ValueError: + return raw + if ( + address.is_private + or address.is_loopback + or address.is_link_local + or address.is_reserved + or address.is_unspecified + ): + return "[PRIVATE_IP_REDACTED]" + return raw + + value = _PAID_CLOUD_IPV4_CANDIDATE.sub(_replace, value) + return _PAID_CLOUD_IPV6_CANDIDATE.sub(_replace, value) + + +def _content_receipt_metadata(value: str) -> dict[str, Any]: + """Return content-safe telemetry; never return the content itself.""" + + encoded = (value or "").encode("utf-8") + return { + "content_length": len(value or ""), + "content_sha256": hashlib.sha256(encoded).hexdigest(), + } + + +def _cloud_safe_scalar(value: Any, *, source_label: str) -> str | int | float | bool | None: + """Sanitize one allowlisted scalar without forwarding arbitrary structures.""" + + if value is None or isinstance(value, bool | int | float): + return value + if not isinstance(value, str): + return None + from src.services.sanitization_service import sanitize + + sanitized = sanitize(str(value), source_label=source_label) + # The general sanitizer intentionally preserves some operational values for + # local SRE readability. Paid-cloud input applies a narrower DLP boundary. + for secret_pattern in _PAID_CLOUD_SECRET_PATTERNS: + sanitized = secret_pattern.sub("[SECRET_REDACTED]", sanitized) + sanitized = re.sub( + r"\[PRIVATE_IP:(?:\d{1,3}\.){3}\d{1,3}\]", + "[PRIVATE_IP_REDACTED]", + sanitized, + ) + sanitized = _redact_private_network_addresses(sanitized) + # A future pattern change must fail closed rather than issue a false-green + # receipt for a still-recognizable credential. + if any(pattern.search(sanitized) for pattern in _PAID_CLOUD_SECRET_PATTERNS): + return None + return sanitized[:512] + + +def build_paid_cloud_alert_context(alert_context: dict | None) -> tuple[dict | None, str]: + """Build a separate allowlist-only cloud payload and attestation. + + The original alert context remains untouched for local Ollama. Free-form + message/labels/annotations/logs/headers and every non-allowlisted field are + intentionally absent from the returned cloud context. + """ + + source = dict(alert_context or {}) + payload: dict[str, Any] = {} + for field in _PAID_ALERT_SCALAR_FIELDS: + value = source.get(field) + if value is None or value == "": + continue + sanitized_value = _cloud_safe_scalar( + value, + source_label=f"openclaw_paid_alert.{field}", + ) + if sanitized_value is not None: + payload[field] = sanitized_value + + signals: list[dict[str, Any]] = [] + for index, raw_signal in enumerate(source.get("signals") or []): + if not isinstance(raw_signal, dict): + continue + signal: dict[str, Any] = {} + for field in _PAID_ALERT_SIGNAL_FIELDS: + value = raw_signal.get(field) + if value is None or value == "": + continue + sanitized_value = _cloud_safe_scalar( + value, + source_label=f"openclaw_paid_alert.signals.{index}.{field}", + ) + if sanitized_value is not None: + signal[field] = sanitized_value + if signal: + signals.append(signal) + if len(signals) >= 20: + break + if signals: + payload["signals"] = signals + + alert_identity = payload.get("alert_type") or payload.get("alertname") or payload.get("alert_name") + if not alert_identity: + return None, "alert_cloud_allowlisted_identity_missing" + + canonical_payload = json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + payload_sha256 = hashlib.sha256(canonical_payload.encode("utf-8")).hexdigest() + receipt_id = f"openclaw-cloud-sanitize:{payload_sha256[:24]}" + excluded_fields = sorted(set(source) - set(_PAID_ALERT_SCALAR_FIELDS) - {"signals"}) + receipt = { + "schema_version": _PAID_CLOUD_RECEIPT_SCHEMA, + "status": "verified", + "receipt_id": receipt_id, + "sanitizer": "src.services.sanitization_service.sanitize", + "classifier": "allowlisted_structured_alert_fields_v1", + "dlp_policy": _PAID_CLOUD_DLP_POLICY, + "payload_sha256": payload_sha256, + "raw_payload_forwarded": False, + "raw_log_payload_forwarded": False, + "secret_value_exposed": False, + "private_network_value_exposed": False, + "excluded_field_count": len(excluded_fields), + } + cloud_context = { + "data_classification": "sanitized", + "sanitized_payload": payload, + "cloud_sanitization_receipt": receipt, + "contains_secret": False, + "secret_value_exposed": False, + "raw_log_payload_forwarded": False, + } + return cloud_context, "verified_allowlisted_sanitization_receipt" + + +def _verified_paid_cloud_context_reason(cloud_context: dict | None) -> str | None: + """Verify the sanitizer receipt is bound to the exact sanitized payload.""" + + if not isinstance(cloud_context, dict): + return "alert_cloud_sanitization_receipt_missing" + receipt = cloud_context.get("cloud_sanitization_receipt") + payload = cloud_context.get("sanitized_payload") + if not isinstance(receipt, dict) or not isinstance(payload, dict): + return "alert_cloud_sanitization_receipt_missing" + if ( + receipt.get("schema_version") != _PAID_CLOUD_RECEIPT_SCHEMA + or receipt.get("status") != "verified" + or receipt.get("sanitizer") != "src.services.sanitization_service.sanitize" + or receipt.get("classifier") != "allowlisted_structured_alert_fields_v1" + or receipt.get("dlp_policy") != _PAID_CLOUD_DLP_POLICY + or receipt.get("raw_payload_forwarded") is not False + or receipt.get("raw_log_payload_forwarded") is not False + or receipt.get("secret_value_exposed") is not False + or receipt.get("private_network_value_exposed") is not False + ): + return "alert_cloud_sanitization_receipt_invalid" + canonical_payload = json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + payload_sha256 = hashlib.sha256(canonical_payload.encode("utf-8")).hexdigest() + if receipt.get("payload_sha256") != payload_sha256: + return "alert_cloud_sanitization_payload_mismatch" + return None class OpenClawService: @@ -146,7 +391,7 @@ class OpenClawService: 實作 AI_FALLBACK_ORDER 備援機制。 告警/incident 上下文預設套用成本與期限防線，依序嘗試 - Ollama GCP-A → GCP-B → host111，最後才可進入 Gemini 付費備援。 + Ollama GCP-A → GCP-B → host111，再依序進入 Claude、Gemini 付費備援。 新增 SignOz 整合: - 自動擷取 Gold Metrics @@ -157,6 +402,11 @@ class OpenClawService: def __init__(self): self._http_client: httpx.AsyncClient | None = None self._signoz = get_signoz_client() + self._paid_cloud_route_readback: dict[str, Any] = { + "status": "not_evaluated", + "eligible": False, + "reason": "not_evaluated", + } async def _get_client(self) -> httpx.AsyncClient: """取得 HTTP 客戶端 @@ -246,13 +496,132 @@ class OpenClawService: ) return payload + def get_paid_cloud_route_readback(self) -> dict[str, Any]: + """Return public-safe last-evaluated paid-cloud eligibility evidence.""" + + return dict( + getattr( + self, + "_paid_cloud_route_readback", + { + "status": "not_evaluated", + "eligible": False, + "reason": "not_evaluated", + }, + ) + ) + + def _prepare_paid_cloud_route_context( + self, + alert_context: dict | None, + ) -> dict | None: + """Attach a separate sanitized cloud envelope; preserve local context.""" + + if not self._is_incident_alert_context(alert_context): + return alert_context + prepared = dict(alert_context or {}) + # Caller-supplied sanitizer receipts are not an authority boundary. + # Rebuild from the original alert on every route evaluation so a forged + # payload/hash pair cannot smuggle content into a paid provider. + cloud_context, reason = build_paid_cloud_alert_context(prepared) + + if cloud_context is None: + prepared.pop("paid_cloud_context", None) + prepared.pop("paid_cloud_prompt", None) + prepared["paid_cloud_blocked_reason"] = reason + self._paid_cloud_route_readback = { + "status": "blocked_ollama_only", + "eligible": False, + "reason": reason, + "receipt_id": None, + } + return prepared + + receipt = cloud_context["cloud_sanitization_receipt"] + paid_prompt = ( + OPENCLAW_SYSTEM_PROMPT + + "\n\n## Sanitized Operational Alert Envelope\n" + + json.dumps( + cloud_context["sanitized_payload"], + ensure_ascii=False, + sort_keys=True, + indent=2, + ) + + "\n\nAnalyze only this sanitized envelope and return the required JSON schema." + ) + prompt_metadata = _content_receipt_metadata(paid_prompt) + cloud_context = dict(cloud_context) + cloud_context.update( + paid_prompt_sha256=prompt_metadata["content_sha256"], + paid_prompt_length=prompt_metadata["content_length"], + ) + prepared["paid_cloud_context"] = cloud_context + prepared["paid_cloud_prompt"] = paid_prompt + prepared.pop("paid_cloud_blocked_reason", None) + self._paid_cloud_route_readback = { + "status": "eligible_after_ollama_lane", + "eligible": True, + "reason": reason, + "receipt_id": receipt.get("receipt_id"), + "payload_sha256": receipt.get("payload_sha256"), + "excluded_field_count": receipt.get("excluded_field_count", 0), + } + return prepared + + @staticmethod + def _paid_cloud_execution_inputs( + prompt: str, + alert_context: dict | None, + ) -> tuple[str | None, dict | None, str | None]: + """Return isolated cloud inputs or a fail-closed reason.""" + + if not alert_context or not any( + key in alert_context + for key in ("alert_type", "alertname", "alert_name", "incident_id", "signals") + ): + return prompt, alert_context, None + cloud_context = alert_context.get("paid_cloud_context") + reason = _verified_paid_cloud_context_reason( + cloud_context if isinstance(cloud_context, dict) else None + ) + paid_prompt = alert_context.get("paid_cloud_prompt") + if reason: + return None, None, reason + rebuilt_cloud_context, rebuild_reason = build_paid_cloud_alert_context( + alert_context + ) + if rebuilt_cloud_context is None: + return None, None, rebuild_reason + if ( + rebuilt_cloud_context.get("sanitized_payload") + != (cloud_context or {}).get("sanitized_payload") + or rebuilt_cloud_context.get("cloud_sanitization_receipt", {}).get( + "receipt_id" + ) + != (cloud_context or {}).get("cloud_sanitization_receipt", {}).get( + "receipt_id" + ) + ): + return None, None, "alert_cloud_sanitization_receipt_untrusted" + if not isinstance(paid_prompt, str) or not paid_prompt: + return None, None, "alert_cloud_sanitized_prompt_missing" + expected = str((cloud_context or {}).get("paid_prompt_sha256") or "") + actual = hashlib.sha256(paid_prompt.encode("utf-8")).hexdigest() + if expected != actual: + return None, None, "alert_cloud_sanitized_prompt_mismatch" + restricted_context = dict(cloud_context or {}) + for field in ("trace_id", "run_id", "work_item_id"): + if alert_context.get(field): + restricted_context[field] = alert_context[field] + return paid_prompt, restricted_context, None + async def _resolve_alert_provider_order( self, task_type: str = "diagnose", alert_context: dict | None = None, cloud_provider_order: list[str] | None = None, ) -> list[str]: - """Resolve the only production route: GCP-A, GCP-B, 111, Gemini.""" + """Resolve the only production route: GCP-A, GCP-B, 111, Claude, Gemini.""" provider_order: list[str] = [] try: route = await get_ollama_failover_manager().select_provider(task_type=task_type) @@ -281,28 +650,97 @@ class OpenClawService: ordered_ollama = ["ollama_gcp_a", "ollama_gcp_b", "ollama_local"] if not self._cloud_fallback_allowed_for_alert(alert_context): + if self._is_incident_alert_context(alert_context): + self._paid_cloud_route_readback = { + "status": "blocked_ollama_only", + "eligible": False, + "reason": "cloud_fallback_disabled", + "receipt_id": ( + self.get_paid_cloud_route_readback().get("receipt_id") + ), + } return ordered_ollama - # Router/Redis primary overrides cannot insert NVIDIA, OpenClaw/Nemo, - # Claude, or any other provider into the production chain. Gemini is - # appended only when both configuration and durable runtime control say - # it is enabled; a disabled paid provider is never silently re-added. + if self._is_incident_alert_context(alert_context): + _paid_prompt, _paid_context, cloud_block_reason = ( + self._paid_cloud_execution_inputs("", alert_context) + ) + if cloud_block_reason: + self._paid_cloud_route_readback = { + "status": "blocked_ollama_only", + "eligible": False, + "reason": cloud_block_reason, + "receipt_id": None, + } + logger.info( + "alert_paid_cloud_route_blocked", + reason=cloud_block_reason, + ) + return ordered_ollama + + # Router/Redis primary overrides cannot insert NVIDIA, OpenClaw/Nemo, or + # any other provider. Paid lanes are appended only when configuration, + # execution mode, and durable runtime control all enable them. _ = cloud_provider_order try: from src.services.ai_control import is_provider_disabled + from src.services.ai_provider_policy import PAID_PROVIDER_ORDER from src.services.ai_providers.interfaces import is_provider_enabled_by_env - gemini_enabled = bool(settings.GEMINI_API_KEY) and is_provider_enabled_by_env( - "gemini" - ) - if gemini_enabled and not await is_provider_disabled("gemini"): - return ordered_ollama + ["gemini"] + paid_order: list[str] = [] + for provider_name in PAID_PROVIDER_ORDER: + configured = bool( + getattr(settings, f"{provider_name.upper()}_API_KEY", "") + ) + if not configured or not is_provider_enabled_by_env(provider_name): + continue + if provider_name == "claude": + mode = str(settings.CLAUDE_EXECUTION_MODE).strip().lower() + if mode == "shadow" or ( + mode == "canary" + and int(settings.CLAUDE_CANARY_PERCENT) <= 0 + ): + continue + try: + disabled = await is_provider_disabled(provider_name) + except Exception as error: + logger.warning( + "paid_provider_enablement_readback_unavailable_blocked", + provider=provider_name, + error_type=type(error).__name__, + ) + continue + if not disabled: + paid_order.append(provider_name) + current_readback = self.get_paid_cloud_route_readback() + self._paid_cloud_route_readback = { + **current_readback, + "status": ( + "eligible_after_ollama_lane" + if paid_order + else "blocked_ollama_only" + ), + "eligible": bool(paid_order), + "reason": ( + current_readback.get("reason") + if paid_order + else "paid_provider_not_enabled_or_runtime_disabled" + ), + "paid_provider_candidates": list(paid_order), + } + return ordered_ollama + paid_order except Exception as error: logger.warning( - "gemini_enablement_readback_unavailable_blocked", + "paid_provider_route_resolution_failed_closed", error_type=type(error).__name__, ) - return ordered_ollama + self._paid_cloud_route_readback = { + "status": "blocked_ollama_only", + "eligible": False, + "reason": "paid_provider_enablement_readback_unavailable", + "receipt_id": self.get_paid_cloud_route_readback().get("receipt_id"), + } + return ordered_ollama # ========================================================================= # SignOz Integration @@ -735,83 +1173,25 @@ class OpenClawService: finally: await provider.close() - async def _call_claude(self, prompt: str) -> tuple[str, bool]: - """ - 呼叫 Anthropic Claude (使用 Tool Use 強制 JSON) - """ - if not settings.CLAUDE_API_KEY: - return "CLAUDE_API_KEY not configured", False + async def _call_claude( + self, + prompt: str, + context: dict | None = None, + ) -> tuple[str, bool, int, float]: + """Call the single guarded ClaudeProvider path; never bypass cost receipts.""" + from src.services.ai_providers.claude import ClaudeProvider + provider = ClaudeProvider() try: - client = await self._get_client() - - # Claude 使用 Tool Use 強制結構化輸出 - response = await client.post( - "https://api.anthropic.com/v1/messages", - headers={ - "x-api-key": settings.CLAUDE_API_KEY, - "anthropic-version": "2023-06-01", - "content-type": "application/json", - }, - json={ - "model": get_model_registry().get_model("claude", "rca"), - "max_tokens": 2048, - "messages": [{"role": "user", "content": prompt}], - "tools": [{ - "name": "submit_analysis", - "description": "Submit the RCA analysis result in structured format", - "input_schema": { - "type": "object", - "properties": { - "action_title": {"type": "string"}, - "description": {"type": "string"}, - "suggested_action": {"type": "string", "enum": ["RESTART_DEPLOYMENT", "DELETE_POD", "SCALE_DEPLOYMENT", "APPLY_HPA", "TUNE_RESOURCES", "INVESTIGATE", "OBSERVE", "NO_ACTION"]}, - "kubectl_command": {"type": "string"}, - "target_resource": {"type": "string"}, - "namespace": {"type": "string"}, - "risk_level": {"type": "string", "enum": ["low", "medium", "critical"]}, - "blast_radius": { - "type": "object", - "properties": { - "affected_pods": {"type": "integer"}, - "estimated_downtime": {"type": "string"}, - "related_services": {"type": "array", "items": {"type": "string"}}, - "data_impact": {"type": "string", "enum": ["NONE", "READ_ONLY", "WRITE", "DESTRUCTIVE"]} - }, - "required": ["affected_pods", "estimated_downtime", "related_services", "data_impact"] - }, - "reasoning": {"type": "string"}, - "deviation_analysis": {"type": "string"}, - "confidence": {"type": "number"}, - "affected_services": {"type": "array", "items": {"type": "string"}} - }, - "required": ["action_title", "description", "suggested_action", "kubectl_command", "target_resource", "namespace", "risk_level", "blast_radius", "reasoning", "confidence"] - } - }], - "tool_choice": {"type": "tool", "name": "submit_analysis"}, - }, - timeout=30.0, + result = await provider.analyze(prompt, context=context) + return ( + result.raw_response or result.error or "claude_failed", + result.success, + result.tokens, + result.cost_usd, ) - response.raise_for_status() - data = response.json() - - # 從 Tool Use 回應中提取 JSON - for block in data.get("content", []): - if block.get("type") == "tool_use" and block.get("name") == "submit_analysis": - tool_input = block.get("input", {}) - logger.info("claude_tool_use_response", input_keys=list(tool_input.keys())) - return json.dumps(tool_input), True - - # Fallback: 嘗試從 text 內容提取 - for block in data.get("content", []): - if block.get("type") == "text": - return block.get("text", ""), True - - return "No valid response from Claude", False - - except Exception as e: - logger.warning("claude_call_failed", error=str(e)) - return str(e), False + finally: + await provider.close() # 2026-03-29 ogt: _call_nvidia 已移至 nvidia_provider.py (ARCHIVED) # 符合模組化規範 - 所有 NVIDIA API 呼叫統一由 NvidiaProvider / OpenClawNemoProvider 處理 @@ -954,6 +1334,7 @@ class OpenClawService: 2026-03-29 ogt: 加入 Token/Cost 追蹤 """ + alert_context = self._prepare_paid_cloud_route_context(alert_context) # 生成快取鍵 (基於 prompt + alert_context hash) # 2026-04-16 ogt + Claude Sonnet 4.6: 修復 — alertname 才是主要識別符 # 舊版用 alert_type:target_resource → 不同告警 (e.g. PostgreSQLDiskGrowth vs PodCrashLoop) @@ -968,21 +1349,43 @@ class OpenClawService: cached = await redis_client.get(cache_key) if cached: cached_data = json.loads(cached) + cached_provider = str(cached_data.get("provider") or "cached") + if _is_paid_cloud_provider_identity(cached_provider): + # Paid results are never returned from this legacy outer + # cache. Current disable/privacy/canary/cost policy must run + # on every cloud candidate. + logger.info( + "paid_cloud_outer_cache_bypassed", + cache_key=cache_key[:20], + provider=cached_provider, + reason="current_policy_recheck_required", + ) + raise ValueError("paid cloud outer cache is non-executable") + if cached_provider not in _OUTER_CACHE_LOCAL_PROVIDERS: + logger.info( + "untrusted_outer_cache_provider_bypassed", + cache_key=cache_key[:20], + provider=cached_provider, + ) + raise ValueError("outer cache provider provenance is not executable") logger.info( "llm_cache_hit", cache_key=cache_key[:20], - provider=cached_data.get("provider", "cached"), + provider=cached_provider, ) return ( cached_data["response"], - f"{cached_data['provider']}_cached", + f"{cached_provider}_cached", True, True, # from_cache 0, # tokens (cache hit, no new tokens) 0.0, # cost (cache hit, no cost) ) except Exception as e: - logger.warning("llm_cache_read_failed", error=str(e)) + logger.warning( + "llm_cache_read_failed", + error_type=type(e).__name__, + ) # 2. Cache Miss - 呼叫 LLM logger.info("llm_cache_miss", cache_key=cache_key[:20]) @@ -991,7 +1394,7 @@ class OpenClawService: ) # 3. 成功則寫入快取 - if success: + if success and not _is_paid_cloud_provider_identity(provider): try: redis_client = get_redis() cache_data = { @@ -1011,7 +1414,10 @@ class OpenClawService: ttl=cache_ttl, ) except Exception as e: - logger.warning("llm_cache_write_failed", error=str(e)) + logger.warning( + "llm_cache_write_failed", + error_type=type(e).__name__, + ) return response, provider, success, False, total_tokens, cost_usd # from_cache=False @@ -1070,6 +1476,7 @@ class OpenClawService: 2026-03-29 ogt: 加入 Token/Cost 追蹤 2026-04-02 ogt: Phase 24 ADR-052 絞殺者包裝 — USE_AI_ROUTER 新舊並存 """ + alert_context = self._prepare_paid_cloud_route_context(alert_context) alert_context = self._ensure_generation_correlation(alert_context, prompt) # ================================================================= # Phase 24 ADR-052: 絞殺者分支 (Strangler Fig) @@ -1247,12 +1654,19 @@ class OpenClawService: return _mock_json, "mock", True, 0, 0.0 # Phase 15.1 + 15.3: Langfuse 追蹤整合 + SignOz Deep Linking + alert_fingerprint = str((alert_context or {}).get("fingerprint", "")) + alert_fingerprint_receipt = _content_receipt_metadata(alert_fingerprint) with langfuse_trace( "openclaw_fallback_chain", metadata={ "prompt_length": len(prompt), "fallback_order": settings.AI_FALLBACK_ORDER, - "alert_fingerprint": (alert_context or {}).get("fingerprint", "unknown"), + "alert_fingerprint_length": alert_fingerprint_receipt[ + "content_length" + ], + "alert_fingerprint_sha256": alert_fingerprint_receipt[ + "content_sha256" + ], }, ) as trace: # Phase 15.3: SignOz → Langfuse 反向連結 @@ -1283,8 +1697,11 @@ class OpenClawService: task_type="alert_fast" if self._is_incident_alert_context(alert_context) else "diagnose", alert_context=alert_context, ) - if "gemini" in resolved_legacy_route: - legacy_provider_order.append("gemini") + legacy_provider_order.extend( + provider + for provider in ("claude", "gemini") + if provider in resolved_legacy_route + ) logger.info( "legacy_global_production_provider_order", provider_order=legacy_provider_order, @@ -1292,9 +1709,10 @@ class OpenClawService: ) for provider in legacy_provider_order: - # Rate Limit 檢查 (nvidia/gemini/claude 需檢查，ollama 不限) + # Legacy free providers retain this gate. Claude/Gemini own the + # atomic paid reservation inside their provider implementations. # 2026-03-30 ogt: 加入 nvidia (RPM=5 限制) - if provider in ("nvidia", "claude"): + if provider in ("nvidia",): allowed, reason = await rate_limiter.check_and_increment(provider) if not allowed: logger.warning( @@ -1312,6 +1730,19 @@ class OpenClawService: # 2026-03-29 ogt: Gemini 回傳 4 值 (含 token/cost)，其他 Provider 補 0 total_tokens = 0 cost_usd = 0.0 + provider_prompt = prompt + provider_context = alert_context + if provider in _PAID_CLOUD_PROVIDERS: + provider_prompt, provider_context, cloud_block_reason = ( + self._paid_cloud_execution_inputs(prompt, alert_context) + ) + if cloud_block_reason or provider_prompt is None: + logger.info( + "legacy_paid_cloud_context_blocked", + provider=provider, + reason=cloud_block_reason or "sanitized_prompt_missing", + ) + continue if provider == "ollama": response, success = await self._call_ollama( @@ -1320,8 +1751,8 @@ class OpenClawService: ) elif provider == "gemini": response, success, total_tokens, cost_usd = await self._call_gemini( - prompt, - context=alert_context, + provider_prompt, + context=provider_context, ) elif provider == "nvidia": # 2026-03-29 ogt: 使用 NvidiaProvider.chat() (模組化規範) @@ -1329,25 +1760,47 @@ class OpenClawService: nvidia_provider = get_nvidia_provider() response, success, total_tokens, cost_usd = await nvidia_provider.chat(prompt, use_json_mode=True) elif provider == "claude": - response, success = await self._call_claude(prompt) + response, success, total_tokens, cost_usd = await self._call_claude( + provider_prompt, + context=provider_context, + ) else: logger.warning("unknown_ai_provider", provider=provider) continue latency_ms = (time.time() - start_time) * 1000 - # Langfuse: 記錄每次 LLM 呼叫 + # Paid cloud content never enters Langfuse. Only hashes, + # lengths, and durable receipt IDs are exported. + paid_provider = provider in _PAID_CLOUD_PROVIDERS + prompt_receipt = _content_receipt_metadata(provider_prompt) + response_receipt = _content_receipt_metadata(response) + paid_receipt_id = None + if isinstance(provider_context, dict): + receipt = provider_context.get("cloud_sanitization_receipt") or {} + if isinstance(receipt, dict): + paid_receipt_id = receipt.get("receipt_id") trace.generation( name=f"{provider}_call", model=model_name, - input=prompt[:500], # 截斷避免過長 - output=response[:500] if success else f"ERROR: {response[:200]}", + input=None if paid_provider else provider_prompt[:500], + output=( + None + if paid_provider + else response[:500] if success else f"ERROR: {response[:200]}" + ), metadata={ "success": success, "latency_ms": round(latency_ms, 2), "provider": provider, "total_tokens": total_tokens, "cost_usd": cost_usd, + "content_redacted": paid_provider, + "prompt_length": prompt_receipt["content_length"], + "prompt_sha256": prompt_receipt["content_sha256"], + "response_length": response_receipt["content_length"], + "response_sha256": response_receipt["content_sha256"], + "sanitization_receipt_id": paid_receipt_id, }, ) @@ -1363,7 +1816,7 @@ class OpenClawService: trace.score(name="provider_success", value=1.0, comment=f"Success via {provider}") # 2026-03-29 ogt: 記錄累積成本 (Gemini/Claude) - if cost_usd > 0 and provider != "gemini": + if cost_usd > 0 and provider not in {"claude", "gemini"}: await rate_limiter.record_cost(provider, cost_usd) return response, provider, True, total_tokens, cost_usd @@ -1491,14 +1944,20 @@ class OpenClawService: logger.warning( "openclaw_deployment_hallucination_detected", - hallucinated=_deploy_guess, - inventory=sorted(_inventory_names), - original_kubectl_cmd=result.kubectl_command, - original_action=( - result.suggested_action.value - if hasattr(result.suggested_action, "value") - else str(result.suggested_action) - ), + hallucinated_length=len(_deploy_guess), + hallucinated_sha256=hashlib.sha256( + _deploy_guess.encode("utf-8") + ).hexdigest(), + inventory_count=len(_inventory_names), + inventory_sha256=hashlib.sha256( + ",".join(sorted(_inventory_names)).encode("utf-8") + ).hexdigest(), + **{ + f"command_{key}": value + for key, value in _content_receipt_metadata( + str(result.kubectl_command or "") + ).items() + }, namespace=k8s_ns, ) # 降級為安全調查動作,不執行破壞性操作 @@ -1545,7 +2004,7 @@ class OpenClawService: """ json_str = self._extract_json_from_response(raw_response) if not json_str: - logger.error("json_extraction_failed", raw_response=raw_response[:200]) + logger.error("json_extraction_failed", **_content_receipt_metadata(raw_response)) return None try: @@ -1608,9 +2067,8 @@ class OpenClawService: logger.info( "pydantic_validation_success", - action_title=decision.action_title, - risk_level=decision.risk_level.value, - blast_radius_pods=decision.blast_radius.affected_pods, + schema="OpenClawDecision", + required_fields_valid=True, ) return decision @@ -1618,8 +2076,8 @@ class OpenClawService: except Exception as e: logger.error( "pydantic_validation_failed", - error=str(e), - json_str=json_str[:300], + error_type=type(e).__name__, + **_content_receipt_metadata(json_str), ) return None @@ -1731,18 +2189,27 @@ Trace URL: {signoz_trace_url} self._validate_deployment_inventory(result, _k8s_inventory, _k8s_ns) if result: - logger.info( - "openclaw_analysis_complete", - action_title=result.action_title, - risk_level=result.risk_level, - confidence=result.confidence, - provider=provider, - signoz_integrated=signoz_metrics is not None, - ) + completion_metadata = { + "provider": provider, + "signoz_integrated": signoz_metrics is not None, + } + if provider.removesuffix("_cached") in _PAID_CLOUD_PROVIDERS: + completion_metadata.update( + content_redacted=True, + **_content_receipt_metadata(raw_response), + ) + else: + completion_metadata.update( + action_title=result.action_title, + risk_level=result.risk_level, + confidence=result.confidence, + ) + logger.info("openclaw_analysis_complete", **completion_metadata) else: logger.warning( "openclaw_analysis_parse_failed", - raw_response=raw_response[:300], + provider=provider, + **_content_receipt_metadata(raw_response), ) return result, provider, raw_response, signoz_metrics, signoz_trace_url, total_tokens, cost_usd @@ -1848,7 +2315,7 @@ provide diagnostic guidance rather than automated fixes. """ # Provider selection happens later in the strict - # GCP-A/GCP-B/host111/Gemini route. + # GCP-A/GCP-B/host111/Claude/Gemini route. base_prompt = OPENCLAW_SYSTEM_PROMPT proposal_prompt = f"""{base_prompt} @@ -1886,12 +2353,16 @@ Focus on: # Direct OpenClaw/Nemo delegation is superseded by the strict runtime # route. Incident analysis must enter the same ordered provider chain as - # alerts: GCP-A -> GCP-B -> host111 -> Gemini. + # alerts: GCP-A -> GCP-B -> host111 -> Claude -> Gemini. alert_context = { "incident_id": incident_id, "alert_type": signals[0].get("alert_name", "incident") if signals else "incident", "target_resource": target, "severity": severity, + "source": "incident_aggregator", + # Local proposal generation still uses the original rich prompt. + # This structured copy feeds only the allowlist cloud sanitizer. + "signals": signals, } # 2026-03-29 ogt: 修復 tuple unpacking (Token/Cost 追蹤) @@ -1923,14 +2394,22 @@ Focus on: self._validate_deployment_inventory(result, _k8s_inv, _k8s_ns_for_validate) if result: - logger.info( - "proposal_generation_complete", - incident_id=incident_id, - action_title=result.action_title, - risk_level=result.risk_level, - provider=provider, - from_cache=from_cache, - ) + completion_metadata = { + "incident_id": incident_id, + "provider": provider, + "from_cache": from_cache, + } + if provider.removesuffix("_cached") in _PAID_CLOUD_PROVIDERS: + completion_metadata.update( + content_redacted=True, + **_content_receipt_metadata(raw_response), + ) + else: + completion_metadata.update( + action_title=result.action_title, + risk_level=result.risk_level, + ) + logger.info("proposal_generation_complete", **completion_metadata) # 轉換為 proposal dict (optimization_suggestions 是 list[dict]) # 2026-03-29 ogt: 加入 ai_tokens/ai_cost 追蹤 @@ -1964,7 +2443,8 @@ Focus on: logger.warning( "proposal_parse_failed", incident_id=incident_id, - raw_response=raw_response[:300], + provider=provider, + **_content_receipt_metadata(raw_response), ) return None, provider, False diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index 9f88ac96b..35c01bd79 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -3602,21 +3602,22 @@ async def get_ai_route_status( health = _ai_route_health_map(route) selected_primary = _ai_route_runtime_endpoint_item(route.primary, priority=1) + policy_priority = { + str(item.get("provider_name")): int(item.get("priority") or 0) + for item in policy_order + } observed_fallback = ( { **_ai_route_runtime_endpoint_item( route.observed_fallback, - priority=( - 4 - if route.observed_fallback.provider_name == "gemini" - else 3 - if route.observed_fallback.provider_name == "ollama_local" - else 2 + priority=policy_priority.get( + route.observed_fallback.provider_name, + 0, ), ), "observation_only": True, "requires_generation_cost_guard": ( - route.observed_fallback.provider_name == "gemini" + route.observed_fallback.provider_name in {"claude", "gemini"} ), } if route.observed_fallback @@ -3645,6 +3646,7 @@ async def get_ai_route_status( "route_source": "ollama_failover_manager", "route_error": None, "health": health, + "claude_activation_evidence": None, "gemini_activation_evidence": None, "checked_at": checked_at, } @@ -3669,20 +3671,28 @@ def _validate_ai_route_workload(workload_type: str | None) -> OllamaWorkloadType def _ai_route_policy_order(workload: OllamaWorkloadType) -> list[dict[str, Any]]: - """Expose configured policy order: GCP-A -> GCP-B -> 111 -> Gemini.""" + """Expose fixed policy order: GCP-A -> GCP-B -> 111 -> Claude -> Gemini.""" items = [ _ai_route_policy_endpoint_item(endpoint, priority=index + 1) for index, endpoint in enumerate(resolve_ollama_order(workload)) ] - items.append({ + items.extend([{ "priority": len(items) + 1, + "provider_name": "claude", + "url": None, + "workload_type": workload, + "reason": "paid_cloud_fallback_after_all_ollama_endpoints", + "role": "paid_fallback", + "runtime": "cloud", + }, { + "priority": len(items) + 2, "provider_name": "gemini", "url": None, "workload_type": workload, - "reason": "final_cloud_fallback_after_all_ollama_endpoints", + "reason": "final_cloud_fallback_after_claude", "role": "final_fallback", "runtime": "cloud", - }) + }]) return items @@ -3700,13 +3710,19 @@ def _ai_route_fixed_runtime_chain( ) for index, endpoint in enumerate(resolve_ollama_order(workload)) ] - items.append({ + items.extend([{ "priority": len(items) + 1, + "provider_name": "claude", + "url": None, + "model": get_settings().CLAUDE_MODEL, + "runtime": "cloud", + }, { + "priority": len(items) + 2, "provider_name": "gemini", "url": None, "model": get_settings().GEMINI_MODEL, "runtime": "cloud", - }) + }]) return items @@ -3760,7 +3776,7 @@ async def _ai_route_lightweight_status_from_policy( if selected_index in {None, 0}: observed_fallback = ( { - **runtime_chain[-1], + **runtime_chain[len(endpoints)], "observation_only": True, "requires_generation_cost_guard": True, } @@ -3779,7 +3795,7 @@ async def _ai_route_lightweight_status_from_policy( else selected_primary["provider_name"] ) observation_reason = ( - "all Ollama endpoints offline; observed final Gemini policy candidate" + "all Ollama endpoints offline; observed Claude policy candidate" if selected_index is None else "selected primary connectivity healthy" if selected_index == 0 @@ -3804,6 +3820,7 @@ async def _ai_route_lightweight_status_from_policy( "route_source": "lightweight_connectivity_fallback", "route_error": None, "health": health_by_provider, + "claude_activation_evidence": None, "gemini_activation_evidence": None, "checked_at": checked_at, } @@ -3898,6 +3915,7 @@ def _ai_route_unavailable_status( "route_source": route_source, "route_error": route_error, "health": {}, + "claude_activation_evidence": None, "gemini_activation_evidence": None, "checked_at": checked_at, } @@ -4293,6 +4311,7 @@ def _ai_route_lane_state( policy_order: list[dict[str, Any]], selected_provider: str | None, health: dict[str, dict[str, Any]], + claude_activation_evidence: Mapping[str, Any] | None = None, gemini_activation_evidence: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Expose failover lane state separately from policy labels.""" @@ -4309,13 +4328,18 @@ def _ai_route_lane_state( if selected_index is not None else None ) - gemini_activation_unproven = bool( - selected_provider == "gemini" - and not _gemini_activation_evidence_is_durable( - gemini_activation_evidence, + paid_activation_evidence = ( + claude_activation_evidence + if selected_provider == "claude" + else gemini_activation_evidence + ) + paid_activation_unproven = bool( + selected_provider in {"claude", "gemini"} + and not _paid_provider_activation_evidence_is_durable( + paid_activation_evidence, ) ) - active_item = None if gemini_activation_unproven else selected_item + active_item = None if paid_activation_unproven else selected_item skipped_items = policy_order[:selected_index] if selected_index is not None else [] skipped_lanes = [ @@ -4324,12 +4348,12 @@ def _ai_route_lane_state( if item.get("runtime") == "ollama" ] - if gemini_activation_unproven: + if paid_activation_unproven: lane_mode = "unavailable" operator_action = { "human_required": False, "action": "inspect_ai_router", - "reason": "gemini_activation_evidence_missing", + "reason": f"{selected_provider}_activation_evidence_missing", } elif not selected_provider or active_item is None: lane_mode = "unavailable" @@ -4372,10 +4396,10 @@ def _ai_route_lane_state( } -def _gemini_activation_evidence_is_durable( +def _paid_provider_activation_evidence_is_durable( evidence: Mapping[str, Any] | None, ) -> bool: - """Require durable paid-call proof before Gemini can be an active lane.""" + """Require durable paid-call proof before a cloud lane can be active.""" payload = evidence or {} receipt_id = str(payload.get("receipt_id") or "").strip() receipt_status = str(payload.get("receipt_status") or "").strip() @@ -4393,6 +4417,13 @@ def _gemini_activation_evidence_is_durable( ) +def _gemini_activation_evidence_is_durable( + evidence: Mapping[str, Any] | None, +) -> bool: + """Backward-compatible alias for the existing Gemini proof contract.""" + return _paid_provider_activation_evidence_is_durable(evidence) + + def _ai_route_lane_item( item: dict[str, Any], health_item: dict[str, Any] | None, diff --git a/apps/api/src/services/runbook_generator.py b/apps/api/src/services/runbook_generator.py index d81abbd2f..23aad066b 100644 --- a/apps/api/src/services/runbook_generator.py +++ b/apps/api/src/services/runbook_generator.py @@ -19,7 +19,7 @@ Runbook Generator - Phase 25 P1 Knowledge Auto-Harvesting |------|------|--------|----------| | v1.0 | 2026-04-04 | Claude Code | 初始佔位（使用 generate() 但介面不存在） | | v1.1 | 2026-04-04 | ogt (首席架構師) | 接入舊版 cloud provider 介面；新增 Minimal fallback（已取代） | -| v2.0 | 2026-07-14 | Codex | 移除 direct NVIDIA；統一 GCP-A → GCP-B → 111 → Gemini 與原子費控 receipt | +| v2.1 | 2026-07-15 | Codex | 固定 GCP-A → GCP-B → 111 → Claude → Gemini；付費 lane 共用原子費控 receipt | """ from __future__ import annotations @@ -178,8 +178,8 @@ class NemotronRunbookGenerator: leWOOOgo 積木化： - 呼叫 KnowledgeService（不直接存 DB） - - 呼叫 AIRouterExecutor，唯一順序 GCP-A → GCP-B → host111 → Gemini - - Gemini 只能透過 provider 內的 durable disable-state 與原子費控 receipt + - 呼叫 AIRouterExecutor，唯一順序 GCP-A → GCP-B → host111 → Claude → Gemini + - Claude/Gemini 只能透過 provider 內的 durable disable-state 與原子費控 receipt """ _RUNBOOK_SYSTEM = ( diff --git a/apps/api/src/services/sre_k3s_controlled_automation_work_items.py b/apps/api/src/services/sre_k3s_controlled_automation_work_items.py index 6343ace36..e59ef2dbc 100644 --- a/apps/api/src/services/sre_k3s_controlled_automation_work_items.py +++ b/apps/api/src/services/sre_k3s_controlled_automation_work_items.py @@ -18,7 +18,7 @@ _PIPELINE = [ "Canonical Asset Normalize", "Typed Domain Router", "HolmesGPT Investigator", - "Ollama RCA / Gemini Critic", + "Ollama RCA / Claude Fallback / Gemini Critic", "Deterministic Policy", "Single Controlled Executor", "Independent Verifier", @@ -101,6 +101,8 @@ def load_sre_k3s_controlled_automation_work_items( raise ValueError(f"{path}: third provider hop must be host111 Ollama") if provider.get("gemini_paid_call_allowed") is not False: raise ValueError(f"{path}: Gemini paid calls require an explicit cost cap") + if provider.get("claude_paid_call_allowed") is not False: + raise ValueError(f"{path}: Claude paid calls require an explicit cost cap") if provider.get("production_provider_route_switch_allowed") is not False: raise ValueError(f"{path}: production provider switch must remain gated") @@ -210,6 +212,9 @@ def build_sre_k3s_program_projection(payload: dict[str, Any]) -> dict[str, Any]: "current_p0": payload["current_p0"], "rollups": payload["rollups"], "provider_order": payload["provider_policy"]["ordered_route"], + "claude_paid_call_allowed": payload["provider_policy"][ + "claude_paid_call_allowed" + ], "gemini_paid_call_allowed": payload["provider_policy"][ "gemini_paid_call_allowed" ], diff --git a/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py b/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py index 73142b343..67e43f2f4 100644 --- a/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py +++ b/apps/api/tests/integration/test_gemini_cost_guard_real_redis.py @@ -21,16 +21,54 @@ import redis.asyncio as redis_async from src.core import config as config_module from src.services.ai_rate_limiter import ( + DAILY_COST_MICRO_USD_KEY, DAILY_COST_RESERVED_MICRO_USD_KEY, DAILY_REQ_KEY, + DAILY_TOKEN_KEY, DAILY_TOKEN_RESERVED_KEY, GENERATION_RECEIPT_KEY, GENERATION_RUN_IDEMPOTENCY_KEY, + REDIS_KEY_PREFIX, + TOTAL_COST_KEY, TOTAL_COST_RESERVED_MICRO_USD_KEY, AIRateLimiter, ) +def _configure_paid_provider_limits( + monkeypatch: pytest.MonkeyPatch, + provider: str, + *, + rpm: int = 20, + daily_cost_usd: float = 5.0, + total_cost_usd: float = 5.0, + alert_threshold_usd: float = 4.0, +) -> None: + prefix = provider.upper() + monkeypatch.setattr(config_module.settings, f"{prefix}_RPM_LIMIT", rpm) + monkeypatch.setattr(config_module.settings, f"{prefix}_DAILY_QUOTA", 100) + monkeypatch.setattr( + config_module.settings, + f"{prefix}_DAILY_TOKEN_LIMIT", + 100_000, + ) + monkeypatch.setattr( + config_module.settings, + f"{prefix}_DAILY_COST_LIMIT_USD", + daily_cost_usd, + ) + monkeypatch.setattr( + config_module.settings, + f"{prefix}_TOTAL_COST_LIMIT_USD", + total_cost_usd, + ) + monkeypatch.setattr( + config_module.settings, + f"{prefix}_COST_ALERT_THRESHOLD_USD", + alert_threshold_usd, + ) + + def _free_local_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(("127.0.0.1", 0)) @@ -179,20 +217,24 @@ def ephemeral_redis_url(tmp_path_factory: pytest.TempPathFactory) -> Iterator[st @pytest.mark.integration @pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "model"), + [ + ("gemini", "gemini-2.5-flash-lite"), + ("claude", "claude-sonnet-5"), + ], +) async def test_lua_reservation_is_atomic_under_real_redis_concurrency( ephemeral_redis_url: str, monkeypatch: pytest.MonkeyPatch, + provider: str, + model: str, ) -> None: client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) await client.ping() await client.flushdb() - monkeypatch.setattr(config_module.settings, "GEMINI_RPM_LIMIT", 5) - monkeypatch.setattr(config_module.settings, "GEMINI_DAILY_QUOTA", 100) - monkeypatch.setattr(config_module.settings, "GEMINI_DAILY_TOKEN_LIMIT", 100_000) - monkeypatch.setattr(config_module.settings, "GEMINI_DAILY_COST_LIMIT_USD", 5.0) - monkeypatch.setattr(config_module.settings, "GEMINI_TOTAL_COST_LIMIT_USD", 5.0) - monkeypatch.setattr(config_module.settings, "GEMINI_COST_ALERT_THRESHOLD_USD", 4.0) + _configure_paid_provider_limits(monkeypatch, provider, rpm=5) limiter = AIRateLimiter() limiter._redis = client @@ -202,9 +244,9 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( reservations = await asyncio.gather( *( limiter.reserve_generation( - "gemini", + provider, f"isolated concurrency verifier {index}", - model="gemini-2.5-flash-lite", + model=model, max_output_tokens=1, context={ "trace_id": f"trace-{index}", @@ -225,7 +267,7 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( for reservation in reservations: receipt = await client.hgetall( GENERATION_RECEIPT_KEY.format( - provider="gemini", + provider=provider, receipt_id=reservation.receipt_id, ) ) @@ -238,7 +280,7 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( assert receipt[b"reserved_cost_micro_usd"] == b"0" request_count = await client.get( - DAILY_REQ_KEY.format(provider="gemini", date="2026-07-14") + DAILY_REQ_KEY.format(provider=provider, date="2026-07-14") ) assert int(request_count or 0) == 5 @@ -256,7 +298,7 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( assert int( await client.get( DAILY_TOKEN_RESERVED_KEY.format( - provider="gemini", + provider=provider, date="2026-07-14", ) ) @@ -265,7 +307,7 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( assert int( await client.get( DAILY_COST_RESERVED_MICRO_USD_KEY.format( - provider="gemini", + provider=provider, date="2026-07-14", ) ) @@ -273,7 +315,7 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( ) == 0 assert int( await client.get( - TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider="gemini") + TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider=provider) ) or 0 ) == 0 @@ -284,21 +326,29 @@ async def test_lua_reservation_is_atomic_under_real_redis_concurrency( @pytest.mark.integration @pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "model"), + [ + ("gemini", "gemini-2.5-flash-lite"), + ("claude", "claude-sonnet-5"), + ], +) async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( ephemeral_redis_url: str, monkeypatch: pytest.MonkeyPatch, + provider: str, + model: str, ) -> None: client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) await client.ping() await client.flushdb() - monkeypatch.setattr(config_module.settings, "GEMINI_RPM_LIMIT", 20) - monkeypatch.setattr(config_module.settings, "GEMINI_DAILY_QUOTA", 100) - monkeypatch.setattr(config_module.settings, "GEMINI_DAILY_TOKEN_LIMIT", 100_000) - monkeypatch.setattr(config_module.settings, "GEMINI_DAILY_COST_LIMIT_USD", 5.0) - monkeypatch.setattr(config_module.settings, "GEMINI_TOTAL_COST_LIMIT_USD", 5.0) - monkeypatch.setattr(config_module.settings, "GEMINI_COST_ALERT_THRESHOLD_USD", 4.0) - monkeypatch.setattr(config_module.settings, "GEMINI_FAILURE_COOLDOWN_SECONDS", 60) + _configure_paid_provider_limits(monkeypatch, provider) + monkeypatch.setattr( + config_module.settings, + f"{provider.upper()}_FAILURE_COOLDOWN_SECONDS", + 60, + ) limiter = AIRateLimiter() limiter._redis = client @@ -313,8 +363,9 @@ async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( reservations = await asyncio.gather( *( limiter.reserve_generation( - "gemini", + provider, "same-run concurrency verifier", + model=model, max_output_tokens=1, context={**identity, "trace_id": f"trace-same-run-{index}"}, ) @@ -329,7 +380,7 @@ async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( } assert len({reservation.receipt_id for reservation in reservations}) == 1 assert int( - await client.get(DAILY_REQ_KEY.format(provider="gemini", date="2026-07-14")) + await client.get(DAILY_REQ_KEY.format(provider=provider, date="2026-07-14")) or 0 ) == 1 @@ -339,8 +390,9 @@ async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( error_code="isolated_failure", ) cooldown_block = await limiter.reserve_generation( - "gemini", + provider, "retry after failure", + model=model, max_output_tokens=1, context={ "trace_id": "trace-retry", @@ -355,6 +407,238 @@ async def test_same_run_is_idempotent_and_failure_sets_work_item_cooldown( await client.aclose() +@pytest.mark.integration +@pytest.mark.asyncio +async def test_claude_daily_and_total_cost_caps_include_pending_reservations( + ephemeral_redis_url: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) + await client.ping() + await client.flushdb() + + # One-byte prompt + one output token at the conservative Sonnet 5 ceiling + # is exactly 18 micro-USD: (1 * $3 + 1 * $15) / 1M. + boundary_usd = 18 / 1_000_000 + _configure_paid_provider_limits( + monkeypatch, + "claude", + daily_cost_usd=boundary_usd, + total_cost_usd=5.0, + alert_threshold_usd=4.0, + ) + + limiter = AIRateLimiter() + limiter._redis = client + limiter._get_today = lambda: "2026-07-14" # type: ignore[method-assign] + limiter._seconds_until_tomorrow = lambda: 3600 # type: ignore[method-assign] + + first = await limiter.reserve_generation( + "claude", + "x", + model="claude-sonnet-5", + max_output_tokens=1, + context={ + "trace_id": "trace-claude-daily-1", + "run_id": "run-claude-daily-1", + "work_item_id": "work-claude-cost-boundary", + }, + ) + daily_block = await limiter.reserve_generation( + "claude", + "x", + model="claude-sonnet-5", + max_output_tokens=1, + context={ + "trace_id": "trace-claude-daily-2", + "run_id": "run-claude-daily-2", + "work_item_id": "work-claude-cost-boundary-2", + }, + ) + assert first.allowed is True + assert first.estimated_cost_usd == boundary_usd + assert daily_block.allowed is False + assert daily_block.reason == "daily_cost_limit" + assert int( + await client.get( + DAILY_COST_RESERVED_MICRO_USD_KEY.format( + provider="claude", + date="2026-07-14", + ) + ) + or 0 + ) == 18 + + assert await limiter.finalize_generation( + first, + success=True, + prompt_tokens=1, + completion_tokens=1, + ) + assert int( + await client.get( + DAILY_COST_MICRO_USD_KEY.format( + provider="claude", + date="2026-07-14", + ) + ) + or 0 + ) == 18 + + await client.flushdb() + _configure_paid_provider_limits( + monkeypatch, + "claude", + daily_cost_usd=5.0, + total_cost_usd=1.0, + alert_threshold_usd=1.0, + ) + await client.set(TOTAL_COST_KEY.format(provider="claude"), "0.999982") + # Keep this verifier hermetic: the warning path remains acknowledged in the + # isolated Redis DB, so the boundary test cannot contact Telegram. + await client.set(f"{REDIS_KEY_PREFIX}cost_warning_sent:claude", "1") + + total_first = await limiter.reserve_generation( + "claude", + "x", + model="claude-sonnet-5", + max_output_tokens=1, + context={ + "trace_id": "trace-claude-total-1", + "run_id": "run-claude-total-1", + "work_item_id": "work-claude-total-boundary", + }, + ) + total_block = await limiter.reserve_generation( + "claude", + "x", + model="claude-sonnet-5", + max_output_tokens=1, + context={ + "trace_id": "trace-claude-total-2", + "run_id": "run-claude-total-2", + "work_item_id": "work-claude-total-boundary-2", + }, + ) + assert total_first.allowed is True + assert total_block.allowed is False + assert total_block.reason == "total_cost_limit" + assert int( + await client.get( + TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider="claude") + ) + or 0 + ) == 18 + + assert await limiter.finalize_generation( + total_first, + success=True, + prompt_tokens=1, + completion_tokens=1, + ) + assert float(await client.get(TOTAL_COST_KEY.format(provider="claude")) or 0) == 1.0 + + await client.flushdb() + await client.aclose() + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_claude_finalize_is_atomic_and_idempotent_under_real_redis_race( + ephemeral_redis_url: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = redis_async.from_url(ephemeral_redis_url, decode_responses=False) + await client.ping() + await client.flushdb() + _configure_paid_provider_limits(monkeypatch, "claude") + + limiter = AIRateLimiter() + limiter._redis = client + limiter._get_today = lambda: "2026-07-14" # type: ignore[method-assign] + limiter._seconds_until_tomorrow = lambda: 3600 # type: ignore[method-assign] + + reservation = await limiter.reserve_generation( + "claude", + "finalize race", + model="claude-sonnet-5", + max_output_tokens=16, + context={ + "trace_id": "trace-claude-finalize-race", + "run_id": "run-claude-finalize-race", + "work_item_id": "work-claude-finalize-race", + }, + ) + assert reservation.allowed is True + + finalized = await asyncio.gather( + *( + limiter.finalize_generation( + reservation, + success=True, + prompt_tokens=7, + completion_tokens=3, + ) + for _ in range(20) + ) + ) + assert finalized == [True] * 20 + + receipt_key = GENERATION_RECEIPT_KEY.format( + provider="claude", + receipt_id=reservation.receipt_id, + ) + finalized_receipt = await client.hgetall(receipt_key) + assert finalized_receipt[b"status"] == b"succeeded" + assert finalized_receipt[b"actual_prompt_tokens"] == b"7" + assert finalized_receipt[b"actual_completion_tokens"] == b"3" + assert int( + await client.get( + DAILY_TOKEN_KEY.format(provider="claude", date="2026-07-14") + ) + or 0 + ) == 10 + assert int( + await client.get( + DAILY_COST_MICRO_USD_KEY.format( + provider="claude", + date="2026-07-14", + ) + ) + or 0 + ) == 66 + assert int( + await client.get( + TOTAL_COST_RESERVED_MICRO_USD_KEY.format(provider="claude") + ) + or 0 + ) == 0 + + adversarial = await asyncio.gather( + *( + limiter.finalize_generation( + reservation, + success=False, + prompt_tokens=999, + completion_tokens=999, + error_code="adversarial_replay", + ) + for _ in range(20) + ) + ) + assert adversarial == [True] * 20 + assert await client.hgetall(receipt_key) == finalized_receipt + assert int( + await client.get( + DAILY_TOKEN_KEY.format(provider="claude", date="2026-07-14") + ) + or 0 + ) == 10 + + await client.flushdb() + await client.aclose() + + @pytest.mark.integration @pytest.mark.asyncio async def test_finalized_receipt_and_run_claim_survive_configured_ttl_and_replay( diff --git a/apps/api/tests/test_ai_provider_route_matrix.py b/apps/api/tests/test_ai_provider_route_matrix.py index 318882f5a..0da018b83 100644 --- a/apps/api/tests/test_ai_provider_route_matrix.py +++ b/apps/api/tests/test_ai_provider_route_matrix.py @@ -23,7 +23,7 @@ def test_load_latest_ai_provider_route_matrix_reads_newest_file(tmp_path): assert loaded["generated_at"] == "2026-06-05T00:00:00+08:00" assert loaded["program_status"]["overall_completion_percent"] == 100 - assert loaded["rollups"]["total_routes"] == 6 + assert loaded["rollups"]["total_routes"] == 7 assert loaded["operation_boundaries"]["provider_switch_allowed"] is False @@ -52,6 +52,18 @@ def test_ai_provider_route_matrix_blocks_provider_and_paid_call_changes(tmp_path load_latest_ai_provider_route_matrix(tmp_path) +def test_ai_provider_route_matrix_blocks_direct_claude_call(tmp_path): + snapshot = _snapshot() + snapshot["operation_boundaries"]["claude_direct_call_allowed"] = True + (tmp_path / "ai_provider_route_matrix_2026-06-05.json").write_text( + json.dumps(snapshot), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="operation boundaries"): + load_latest_ai_provider_route_matrix(tmp_path) + + def test_ai_provider_route_matrix_requires_rollup_consistency(tmp_path): snapshot = _snapshot() snapshot["rollups"]["route_ids_requiring_action"] = [] @@ -165,6 +177,13 @@ def _snapshot( "blocked", "candidate_blocked", ), + _route( + "claude_paid_fallback_policy", + "Claude Paid Fallback", + "paid_cloud_fallback", + "verified", + "route_preserved", + ), _route( "gemini_final_fallback_policy", "Gemini Final Fallback", @@ -266,6 +285,7 @@ def _snapshot( "shadow_or_canary_allowed": False, "openclaw_replacement_allowed": False, "nemotron_shadow_allowed": False, + "claude_direct_call_allowed": False, "gemini_direct_call_allowed": False, "secret_read_allowed": False, "secret_plaintext_allowed": False, @@ -305,7 +325,13 @@ def _route( "route_gate": route_gate, "evidence_status": "committed_source", "current_policy": "只讀盤點 provider route。", - "provider_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], + "provider_order": [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ], "fallback_policy": "不切 provider。", "evidence_refs": ["apps/api/src/services/ai_router.py"], "next_action": "準備批准包，不改 runtime。", diff --git a/apps/api/tests/test_ai_provider_route_matrix_api.py b/apps/api/tests/test_ai_provider_route_matrix_api.py index e6541ba6a..9a1286863 100644 --- a/apps/api/tests/test_ai_provider_route_matrix_api.py +++ b/apps/api/tests/test_ai_provider_route_matrix_api.py @@ -19,7 +19,7 @@ def test_ai_provider_route_matrix_endpoint_returns_committed_snapshot(): assert data["program_status"]["current_task_id"] == "P1-004" assert data["program_status"]["next_task_id"] == "P1-005" assert data["program_status"]["read_only_mode"] is True - assert data["rollups"]["total_routes"] == len(data["provider_routes"]) == 7 + assert data["rollups"]["total_routes"] == len(data["provider_routes"]) == 8 assert data["rollups"]["provider_switch_allowed_count"] == 0 assert data["rollups"]["paid_api_call_allowed_count"] == 0 assert data["rollups"]["shadow_or_canary_allowed_count"] == 0 @@ -31,6 +31,7 @@ def test_ai_provider_route_matrix_endpoint_returns_committed_snapshot(): assert data["approval_boundaries"]["cost_change_approved"] is False assert any(route["route_id"] == "ollama_global_endpoint_order" for route in data["provider_routes"]) assert any(route["route_id"] == "alert_ai_ollama_first_lane" for route in data["provider_routes"]) + assert any(route["route_id"] == "claude_paid_fallback_policy" for route in data["provider_routes"]) nemotron = next( route for route in data["provider_routes"] if route["route_id"] == "nemotron_tool_calling_candidate" diff --git a/apps/api/tests/test_ai_provider_status_readback.py b/apps/api/tests/test_ai_provider_status_readback.py index c7c6990f4..009063cb9 100644 --- a/apps/api/tests/test_ai_provider_status_readback.py +++ b/apps/api/tests/test_ai_provider_status_readback.py @@ -15,13 +15,19 @@ PRODUCTION_PROVIDER_ORDER = [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] -def _usage(*, authenticated: bool | None = None) -> dict: +def _usage( + provider: str = "gemini", + *, + authenticated: bool | None = None, +) -> dict: + model = "claude-sonnet-5" if provider == "claude" else "gemini-2.5-flash-lite" return { - "provider": "gemini", + "provider": provider, "limited": True, "readback_status": "verified", "cost_exceeded": False, @@ -30,7 +36,7 @@ def _usage(*, authenticated: bool | None = None) -> dict: "status": "receipt_backed", }, "pricing_policy": { - "model": "gemini-2.5-flash-lite", + "model": model, "supported": True, }, "total_cost_usd": { @@ -40,7 +46,7 @@ def _usage(*, authenticated: bool | None = None) -> dict: "alert_threshold": 4.0, }, "authentication": { - "provider": "gemini", + "provider": provider, "configured": True, "authenticated": authenticated, "authentication_verified": authenticated is True, @@ -63,11 +69,14 @@ class _Limiter: self.error = error async def get_usage_stats(self, provider: str) -> dict: - assert provider == "gemini" if self.error: raise self.error - assert self.usage is not None - return dict(self.usage) + usage = self.usage or _usage(provider) + payload = dict(usage) if usage.get("provider") == provider else _usage( + provider, + authenticated=(usage.get("authentication") or {}).get("authenticated"), + ) + return payload def _configure( @@ -81,7 +90,15 @@ def _configure( "GEMINI_API_KEY", "configured-test-value", ) + monkeypatch.setattr( + config_module.settings, + "CLAUDE_API_KEY", + "configured-test-value", + ) + monkeypatch.setattr(config_module.settings, "CLAUDE_EXECUTION_MODE", "shadow") + monkeypatch.setattr(config_module.settings, "CLAUDE_CANARY_PERCENT", 0) monkeypatch.setenv("ENABLE_GEMINI", "true") + monkeypatch.setenv("ENABLE_CLAUDE", "true") monkeypatch.setattr(limiter_module, "get_ai_rate_limiter", lambda: limiter) monkeypatch.setattr( ai_control_module, @@ -104,6 +121,7 @@ async def test_ai_status_separates_configured_authenticated_and_cost_guard( "GCP-A", "GCP-B", "host111", + "Anthropic Claude API", "Gemini API", ] assert result["gemini_configured"] is True @@ -111,6 +129,8 @@ async def test_ai_status_separates_configured_authenticated_and_cost_guard( assert result["gemini_authenticated"] is None assert result["gemini_authentication_verified"] is False assert result["gemini_production_executable"] is False + assert result["claude_configured"] is True + assert result["claude_production_executable"] is False assert "configured-test-value" not in repr(result) @@ -129,7 +149,7 @@ async def test_disabled_gemini_remains_visible_but_non_executable( assert result["fallback_order"] == PRODUCTION_PROVIDER_ORDER assert result["gemini_enabled"] is False assert result["gemini_production_executable"] is False - assert result["production_route"]["hops"][3]["production_executable"] is False + assert result["production_route"]["hops"][4]["production_executable"] is False @pytest.mark.asyncio @@ -175,7 +195,7 @@ def test_cost_guard_static_readback_fails_closed_without_complete_nondegraded_ac assert route["gemini"]["cost_guard_ready"] is False assert route["gemini"]["production_executable"] is False - assert route["hops"][3]["production_executable"] is False + assert route["hops"][4]["production_executable"] is False @pytest.mark.parametrize("missing_field", ["pricing_policy", "cost_exceeded"]) @@ -212,8 +232,5 @@ async def test_status_endpoints_fail_closed_when_cost_readback_is_unavailable( assert status["production_route"]["gemini"]["cost_guard_ready"] is False assert health["gemini"]["readback_status"] == "unavailable" assert health["gemini_status"]["production_executable"] is False - assert health["claude"] == { - "provider": "claude", - "production_executable": False, - "mode": "non_executable_shadow_metadata_only", - } + assert health["claude"]["readback_status"] == "unavailable" + assert health["claude_status"]["production_executable"] is False diff --git a/apps/api/tests/test_ai_route_observed_fallback_readback.py b/apps/api/tests/test_ai_route_observed_fallback_readback.py index 07763a03b..113294d9d 100644 --- a/apps/api/tests/test_ai_route_observed_fallback_readback.py +++ b/apps/api/tests/test_ai_route_observed_fallback_readback.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import UTC, datetime from unittest.mock import AsyncMock, MagicMock import pytest @@ -15,7 +16,7 @@ from src.services.ollama_health_monitor import HealthReport, HealthStatus @pytest.mark.asyncio -async def test_all_ollama_offline_never_labels_gemini_selected_primary( +async def test_all_ollama_offline_never_labels_paid_fallback_selected_primary( monkeypatch: pytest.MonkeyPatch, ) -> None: gcp_a = OllamaEndpoint( @@ -33,6 +34,11 @@ async def test_all_ollama_offline_never_labels_gemini_selected_primary( provider_name="ollama_local", model="qwen3:14b", ) + claude = OllamaEndpoint( + url="", + provider_name="claude", + model="claude-sonnet-5", + ) gemini = OllamaEndpoint( url="", provider_name="gemini", @@ -41,14 +47,14 @@ async def test_all_ollama_offline_never_labels_gemini_selected_primary( offline = HealthReport(status=HealthStatus.OFFLINE, reason="offline") route = OllamaRoutingResult( primary=gcp_a, - fallback_chain=[gcp_b, local, gemini], + fallback_chain=[gcp_b, local, claude, gemini], routing_reason=( - "selected primary remains GCP-A; observed Gemini policy candidate" + "selected primary remains GCP-A; observed Claude policy candidate" ), health_gcp_a=offline, health_gcp_b=offline, health_local=offline, - observed_fallback=gemini, + observed_fallback=claude, ) manager = MagicMock() manager.select_provider = AsyncMock(return_value=route) @@ -71,9 +77,9 @@ async def test_all_ollama_offline_never_labels_gemini_selected_primary( assert response["selected_primary"]["provider_name"] == "ollama_gcp_a" assert response["observed_fallback"] == { "priority": 4, - "provider_name": "gemini", + "provider_name": "claude", "url": None, - "model": "gemini-2.5-flash-lite", + "model": "claude-sonnet-5", "runtime": "cloud", "observation_only": True, "requires_generation_cost_guard": True, @@ -81,8 +87,47 @@ async def test_all_ollama_offline_never_labels_gemini_selected_primary( assert [item["provider_name"] for item in response["fallback_chain"]] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] assert response["gemini_activation_evidence"] is None assert response["lane_mode"] == "unavailable" assert response["active_lane"] is None + + +@pytest.mark.asyncio +async def test_lightweight_all_offline_observes_claude_before_gemini( + monkeypatch: pytest.MonkeyPatch, +) -> None: + offline = HealthReport(status=HealthStatus.OFFLINE, reason="offline") + monkeypatch.setattr( + platform_operator_service, + "_ai_route_probe_connectivity", + AsyncMock(return_value=offline), + ) + monkeypatch.setattr( + platform_operator_service, + "_latest_ai_route_repair_evidence", + AsyncMock(return_value=None), + ) + policy = platform_operator_service._ai_route_policy_order("deep_rca") + + response = await platform_operator_service._ai_route_lightweight_status_from_policy( + workload="deep_rca", + policy_order=policy, + checked_at=datetime.now(UTC), + route_reason="route_check_timeout", + route_error="bounded timeout", + ) + + assert response["selected_provider"] == "ollama_gcp_a" + assert response["observed_fallback"]["provider_name"] == "claude" + assert response["observed_fallback"]["priority"] == 4 + assert [item["provider_name"] for item in response["fallback_chain"]] == [ + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ] + assert response["lane_mode"] == "unavailable" + assert response["active_lane"] is None diff --git a/apps/api/tests/test_ai_router_cache_provider_policy.py b/apps/api/tests/test_ai_router_cache_provider_policy.py index 967f086a4..bf9ee8e75 100644 --- a/apps/api/tests/test_ai_router_cache_provider_policy.py +++ b/apps/api/tests/test_ai_router_cache_provider_policy.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import json +from datetime import UTC, datetime, timedelta from typing import Any import pytest @@ -9,6 +10,7 @@ import pytest from src.services import ai_router as ai_router_module from src.services.ai_providers.interfaces import AIResult from src.services.ai_router import AIProviderRegistry, AIRouterExecutor +from src.services.openclaw import OpenClawService @pytest.fixture(autouse=True) @@ -19,14 +21,25 @@ def _explicitly_enable_gemini_environment( class _FakeRedis: - def __init__(self, cached_provider: str, *, gemini_disabled: bool = False) -> None: + def __init__( + self, + cached_provider: str, + *, + gemini_disabled: bool = False, + durable_receipts: dict[str, dict[str, Any]] | None = None, + ) -> None: self.cached_provider = cached_provider self.gemini_disabled = gemini_disabled + self.durable_receipts = durable_receipts or {} self.set_calls: list[tuple[str, str, int | None]] = [] async def get(self, key: str) -> str | None: if key.startswith("ai:control:disabled:"): return "1" if self.gemini_disabled else "0" + if key.startswith("ai:ollama:unavailable_receipt:"): + receipt_id = key.removeprefix("ai:ollama:unavailable_receipt:") + receipt = self.durable_receipts.get(receipt_id) + return json.dumps(receipt) if receipt else None return json.dumps({ "response": '{"provider":"stale"}', "provider": self.cached_provider, @@ -97,6 +110,40 @@ class _SlowLocalProvider(_FailingLocalProvider): ) +def _paid_alert_context(**overrides: Any) -> dict[str, Any]: + service = object.__new__(OpenClawService) + context = service._prepare_paid_cloud_route_context( + { + "alert_type": "HostHighCpuLoad", + "source": "test-alert-producer", + "target_resource": "node-110", + **overrides, + } + ) + assert context is not None + context.setdefault("run_id", "run-cache-policy-1") + return context + + +def _durable_unavailable_receipt(provider: str, run_id: str) -> dict[str, Any]: + observed_at = datetime.now(UTC) + return { + "schema_version": "ollama_unavailable_receipt_v1", + "receipt_id": f"ollama-unavailable:{run_id}:{provider}", + "source": "ollama_failover_manager", + "provider": provider, + "run_id": run_id, + "status": "bounded_unavailable", + "check_completed": True, + "reason": "endpoint_unreachable", + "observed_at": observed_at.isoformat(), + "expires_at": (observed_at + timedelta(seconds=120)).isoformat(), + "durable_write_ack": True, + "verifier_status": "verified", + "verified_by": "ollama_failover_manager.health_probe", + } + + @pytest.mark.asyncio async def test_executor_skips_cached_cloud_provider_when_ollama_lane_is_required( monkeypatch: pytest.MonkeyPatch, @@ -112,7 +159,11 @@ async def test_executor_skips_cached_cloud_provider_when_ollama_lane_is_required result = await AIRouterExecutor(registry).execute( prompt="diagnose alert", provider_order=["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], - context={"intent_hint": "diagnose", "alert_type": "HostHighCpuLoad"}, + context={ + "intent_hint": "diagnose", + "alert_type": "HostHighCpuLoad", + "data_classification": "sanitized", + }, ) assert result.provider == "ollama_gcp_a" @@ -136,7 +187,7 @@ async def test_executor_applies_gemini_runtime_disable_before_cache_lookup( result = await AIRouterExecutor(registry).execute( prompt="non-alert summary", provider_order=["ollama_gcp_a", "gemini"], - context={"intent_hint": "summary"}, + context={"intent_hint": "summary", "data_classification": "sanitized"}, ) assert result.provider == "ollama_gcp_a" @@ -160,7 +211,7 @@ async def test_executor_applies_gemini_environment_disable_before_cache_lookup( result = await AIRouterExecutor(registry).execute( prompt="non-alert summary", provider_order=["ollama_gcp_a", "gemini"], - context={"intent_hint": "summary"}, + context={"intent_hint": "summary", "data_classification": "sanitized"}, ) assert result.provider == "ollama_gcp_a" @@ -212,6 +263,7 @@ async def test_alert_cloud_backup_waits_for_real_ollama_local_attempt( "intent_hint": "diagnose", "alert_type": "HostHighCpuLoad", "alert_requires_ollama_before_cloud": True, + "data_classification": "sanitized", }, ) @@ -226,10 +278,12 @@ async def test_alert_cloud_backup_allowed_after_ollama_local_attempt( ) -> None: fake_redis = _FakeRedis(cached_provider="none") gcp_a = _FailingLocalProvider("ollama_gcp_a") + gcp_b = _FailingLocalProvider("ollama_gcp_b") local = _FailingLocalProvider("ollama_local") gemini = _CloudProvider() registry = AIProviderRegistry() registry.register(gcp_a) + registry.register(gcp_b) registry.register(local) registry.register(gemini) @@ -239,16 +293,16 @@ async def test_alert_cloud_backup_allowed_after_ollama_local_attempt( result = await AIRouterExecutor(registry).execute( prompt="diagnose alert", provider_order=["ollama_gcp_a", "ollama_local", "gemini"], - context={ - "intent_hint": "diagnose", - "alert_type": "HostHighCpuLoad", - "alert_requires_ollama_before_cloud": True, - }, + context=_paid_alert_context( + intent_hint="diagnose", + alert_requires_ollama_before_cloud=True, + ), ) assert result.success is True assert result.provider == "gemini" assert gcp_a.calls == 1 + assert gcp_b.calls == 1 assert local.calls == 1 assert gemini.calls == 1 @@ -257,7 +311,23 @@ async def test_alert_cloud_backup_allowed_after_ollama_local_attempt( async def test_alert_provider_timeout_allows_next_fallback( monkeypatch: pytest.MonkeyPatch, ) -> None: - fake_redis = _FakeRedis(cached_provider="none") + durable_receipts = { + receipt["receipt_id"]: receipt + for receipt in ( + _durable_unavailable_receipt( + "ollama_gcp_a", + "run-cache-policy-1", + ), + _durable_unavailable_receipt( + "ollama_gcp_b", + "run-cache-policy-1", + ), + ) + } + fake_redis = _FakeRedis( + cached_provider="none", + durable_receipts=durable_receipts, + ) local = _SlowLocalProvider("ollama_local") gemini = _CloudProvider() registry = AIProviderRegistry() @@ -270,12 +340,12 @@ async def test_alert_provider_timeout_allows_next_fallback( result = await AIRouterExecutor(registry).execute( prompt="diagnose alert", provider_order=["ollama_local", "gemini"], - context={ - "intent_hint": "alert_triage", - "alert_type": "HostHighCpuLoad", - "alert_requires_ollama_before_cloud": True, - "provider_timeout_seconds": 0.01, - }, + context=_paid_alert_context( + intent_hint="alert_triage", + alert_requires_ollama_before_cloud=True, + provider_timeout_seconds=0.01, + ollama_unavailable_receipts=list(durable_receipts.values()), + ), ) assert result.success is True diff --git a/apps/api/tests/test_ai_router_diagnose_fallback.py b/apps/api/tests/test_ai_router_diagnose_fallback.py index 22bc0dbd9..dca8d38df 100644 --- a/apps/api/tests/test_ai_router_diagnose_fallback.py +++ b/apps/api/tests/test_ai_router_diagnose_fallback.py @@ -29,6 +29,7 @@ PRODUCTION_ENUM_ORDER = [ AIProviderEnum.OLLAMA_GCP_A, AIProviderEnum.OLLAMA_GCP_B, AIProviderEnum.OLLAMA_LOCAL, + AIProviderEnum.CLAUDE, AIProviderEnum.GEMINI, ] PRODUCTION_NAME_ORDER = [provider.value for provider in PRODUCTION_ENUM_ORDER] @@ -42,6 +43,8 @@ class _NoCacheRedis: async def get(self, key: str) -> bytes | None: if key == "ai:control:disabled:gemini": return b"1" if self.gemini_disabled else b"0" + if key == "ai:control:disabled:claude": + return b"0" return None async def set(self, key: str, value: str, ex: int | None = None) -> None: @@ -56,7 +59,7 @@ class _Provider: self.name = name self.success = success self.calls = 0 - self.privacy_level = "cloud" if name == "gemini" else "local" + self.privacy_level = "cloud" if name in {"claude", "gemini"} else "local" async def analyze( self, @@ -75,6 +78,7 @@ class _Provider: @pytest.fixture(autouse=True) def _reset_router_singletons(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("ENABLE_GEMINI", "true") + monkeypatch.setenv("ENABLE_CLAUDE", "true") yield reset_ai_router() @@ -98,7 +102,7 @@ def _registry(*providers: _Provider) -> AIProviderRegistry: return registry -def test_legacy_providers_register_as_non_executable_shadow_metadata() -> None: +def test_claude_is_executable_but_legacy_providers_remain_shadow_metadata() -> None: registry = _registry( _Provider("claude", success=True), _Provider("nemotron", success=True), @@ -106,8 +110,8 @@ def test_legacy_providers_register_as_non_executable_shadow_metadata() -> None: ) assert set(registry.names()) == {"claude", "nemotron", "openclaw_nemo"} - assert registry.all_enabled() == [] - assert registry.get("claude") is None + assert registry.all_enabled() == [registry.get("claude")] + assert registry.get("claude") is not None assert registry.get("nemotron") is None assert registry.get("openclaw_nemo") is None @@ -135,7 +139,7 @@ async def test_health_selected_gemini_cannot_become_router_primary() -> None: decision = await router.route( "diagnose cascading failure", - context={"intent_hint": "diagnose"}, + context={"intent_hint": "diagnose", "data_classification": "sanitized"}, ) assert decision.selected_provider == AIProviderEnum.OLLAMA_GCP_A @@ -176,7 +180,7 @@ async def test_executor_rejects_direct_cloud_or_legacy_only_route( result = await executor.execute( prompt="bypass attempt", provider_order=["openclaw_nemo", "gemini", "claude"], - context={"intent_hint": "diagnose"}, + context={"intent_hint": "diagnose", "data_classification": "sanitized"}, ) assert result.success is False @@ -186,13 +190,14 @@ async def test_executor_rejects_direct_cloud_or_legacy_only_route( @pytest.mark.asyncio -async def test_executor_runs_exact_four_hops_in_order( +async def test_executor_runs_exact_five_hops_in_order( monkeypatch: pytest.MonkeyPatch, ) -> None: providers = [ _Provider("ollama_gcp_a", success=False), _Provider("ollama_gcp_b", success=False), _Provider("ollama_local", success=False), + _Provider("claude", success=False), _Provider("gemini", success=True), ] redis = _NoCacheRedis() @@ -202,12 +207,12 @@ async def test_executor_runs_exact_four_hops_in_order( result = await AIRouterExecutor(_registry(*providers)).execute( prompt="diagnose alert", provider_order=["ollama", "openclaw_nemo", "gemini", "claude"], - context={"intent_hint": "diagnose"}, + context={"intent_hint": "diagnose", "data_classification": "sanitized"}, ) assert result.success is True assert result.provider == "gemini" - assert [provider.calls for provider in providers] == [1, 1, 1, 1] + assert [provider.calls for provider in providers] == [1, 1, 1, 1, 1] @pytest.mark.asyncio @@ -218,6 +223,7 @@ async def test_disabled_gemini_is_never_readded_or_called( _Provider("ollama_gcp_a", success=False), _Provider("ollama_gcp_b", success=False), _Provider("ollama_local", success=False), + _Provider("claude", success=False), _Provider("gemini", success=True), ] redis = _NoCacheRedis(gemini_disabled=True) @@ -227,12 +233,12 @@ async def test_disabled_gemini_is_never_readded_or_called( result = await AIRouterExecutor(_registry(*providers)).execute( prompt="diagnose alert", provider_order=PRODUCTION_NAME_ORDER, - context={"intent_hint": "diagnose"}, + context={"intent_hint": "diagnose", "data_classification": "sanitized"}, ) assert result.success is False assert "gemini: disabled_by_runtime_control" in str(result.error) - assert [provider.calls for provider in providers] == [1, 1, 1, 0] + assert [provider.calls for provider in providers] == [1, 1, 1, 1, 0] @pytest.mark.asyncio @@ -243,6 +249,7 @@ async def test_code_review_local_gate_removes_paid_fallback( _Provider("ollama_gcp_a", success=False), _Provider("ollama_gcp_b", success=False), _Provider("ollama_local", success=True), + _Provider("claude", success=True), _Provider("gemini", success=True), ] redis = _NoCacheRedis() @@ -259,3 +266,40 @@ async def test_code_review_local_gate_removes_paid_fallback( assert result.success is True assert result.provider == "ollama_local" assert providers[-1].calls == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "privacy_context", + [ + {}, + {"data_classification": "private"}, + {"data_classification": "sanitized", "contains_secret": True}, + {"data_classification": "sanitized", "secret_value_exposed": True}, + {"data_classification": "sanitized", "raw_log": "unredacted"}, + ], +) +async def test_router_blocks_paid_cloud_before_cache_for_unclassified_or_sensitive_context( + monkeypatch: pytest.MonkeyPatch, + privacy_context: dict[str, object], +) -> None: + providers = [ + _Provider("ollama_gcp_a", success=False), + _Provider("ollama_gcp_b", success=False), + _Provider("ollama_local", success=False), + _Provider("claude", success=True), + _Provider("gemini", success=True), + ] + redis = _NoCacheRedis() + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + + result = await AIRouterExecutor(_registry(*providers)).execute( + prompt="opaque context", + provider_order=PRODUCTION_NAME_ORDER, + context={"intent_hint": "diagnose", **privacy_context}, + ) + + assert result.success is False + assert [provider.calls for provider in providers] == [1, 1, 1, 0, 0] + assert "cloud_context_" in str(result.error) diff --git a/apps/api/tests/test_ai_router_failover_integration.py b/apps/api/tests/test_ai_router_failover_integration.py index af5526345..24b8adfbf 100644 --- a/apps/api/tests/test_ai_router_failover_integration.py +++ b/apps/api/tests/test_ai_router_failover_integration.py @@ -16,6 +16,7 @@ from src.services.ai_router import AIProviderEnum, AIRouter, reset_ai_router EXPECTED_FALLBACK = [ AIProviderEnum.OLLAMA_GCP_B, AIProviderEnum.OLLAMA_LOCAL, + AIProviderEnum.CLAUDE, AIProviderEnum.GEMINI, ] @@ -48,7 +49,7 @@ def _router_with_observed_fallback(provider_name: str) -> tuple[AIRouter, MagicM @pytest.mark.asyncio @pytest.mark.parametrize( "observed_primary", - ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], + ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "claude", "gemini"], ) async def test_health_observation_never_reorders_execution_contract( observed_primary: str, diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index cf4bbab07..d3b2559bc 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -4125,15 +4125,17 @@ def test_timeline_sort_key_normalizes_datetime_and_iso_string() -> None: ] -def test_ai_route_policy_order_exposes_global_ollama_then_gemini() -> None: +def test_ai_route_policy_order_exposes_global_five_hop_chain() -> None: policy = _ai_route_policy_order("deep_rca") assert [item["provider_name"] for item in policy] == [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] + assert policy[-2]["role"] == "paid_fallback" assert policy[-1]["role"] == "final_fallback" assert policy[-1]["runtime"] == "cloud" @@ -4535,18 +4537,20 @@ async def test_ai_route_status_times_out_before_slow_provider_checks(monkeypatch assert [item["provider_name"] for item in response["fallback_chain"]] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] assert [item["provider_name"] for item in response["policy_order"]] == [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] @pytest.mark.asyncio -async def test_ai_route_status_lightweight_fallback_keeps_gemini_policy_only( +async def test_ai_route_status_lightweight_fallback_observes_claude_before_gemini( monkeypatch, ) -> None: class SlowFailoverManager: @@ -4589,7 +4593,7 @@ async def test_ai_route_status_lightweight_fallback_keeps_gemini_policy_only( assert response["selected_provider"] == "ollama_gcp_a" assert response["selected_primary"]["provider_name"] == "ollama_gcp_a" - assert response["observed_fallback"]["provider_name"] == "gemini" + assert response["observed_fallback"]["provider_name"] == "claude" assert response["observed_fallback"]["observation_only"] is True assert response["observed_fallback"]["requires_generation_cost_guard"] is True assert response["selected_model"] is not None @@ -4598,7 +4602,7 @@ async def test_ai_route_status_lightweight_fallback_keeps_gemini_policy_only( assert response["lane_mode"] == "unavailable" assert response["active_lane"] is None assert "selected primary remains ollama_gcp_a" in response["route_reason"] - assert "observed final Gemini policy candidate" in response["route_reason"] + assert "observed Claude policy candidate" in response["route_reason"] assert all(item["status"] == "offline" for item in response["health"].values()) diff --git a/apps/api/tests/test_claude_paid_fallback.py b/apps/api/tests/test_claude_paid_fallback.py new file mode 100644 index 000000000..a39b8d0e4 --- /dev/null +++ b/apps/api/tests/test_claude_paid_fallback.py @@ -0,0 +1,705 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from src.plugins.mcp.interfaces import MCPTool, MCPToolResult +from src.services import ai_control as ai_control_module +from src.services.ai_providers.claude import ClaudeProvider +from src.services.ai_providers.gemini import GeminiProvider +from src.services.ai_providers.interfaces import cloud_context_block_reason +from src.services.ai_rate_limiter import ( + AIGenerationReservation, + get_claude_pricing_policy, +) + + +def _settings(*, mode: str = "production", canary_percent: int = 0) -> SimpleNamespace: + return SimpleNamespace( + CLAUDE_API_KEY="configured-test-value", + CLAUDE_MODEL="claude-sonnet-5", + CLAUDE_MAX_OUTPUT_TOKENS=4096, + CLAUDE_EXECUTION_MODE=mode, + CLAUDE_CANARY_PERCENT=canary_percent, + ) + + +def _reservation( + *, + allowed: bool = True, + reason: str = "allowed", + receipt_id: str = "gen-claude-test-1", +) -> AIGenerationReservation: + pricing = get_claude_pricing_policy("claude-sonnet-5") + assert pricing is not None + return AIGenerationReservation( + provider="claude", + receipt_id=receipt_id, + allowed=allowed, + reason=reason, + estimated_prompt_tokens=20, + estimated_completion_tokens=4096, + estimated_cost_usd=pricing.cost_usd(20, 4096), + date="2026-07-15", + model=pricing.model, + pricing_source=pricing.source, + pricing_version=pricing.version, + pricing_checked_at=pricing.checked_at, + input_usd_per_million=pricing.input_usd_per_million, + output_usd_per_million=pricing.output_usd_per_million, + trace_id="trace-claude-1", + run_id="run-claude-1", + work_item_id="work-claude-1", + ) + + +def _context(**overrides: object) -> dict[str, object]: + return { + "trace_id": "trace-claude-1", + "run_id": "run-claude-1", + "work_item_id": "work-claude-1", + "data_classification": "sanitized", + **overrides, + } + + +class _ClaudeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return { + "stop_reason": "tool_use", + "usage": {"input_tokens": 12, "output_tokens": 8}, + "content": [ + { + "type": "tool_use", + "name": "submit_analysis", + "input": { + "action_title": "bounded action", + "suggested_action": "NO_ACTION", + }, + } + ], + } + + +class _ClaudeToolUseResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return { + "stop_reason": "tool_use", + "usage": {"input_tokens": 12, "output_tokens": 8}, + "content": [ + { + "type": "tool_use", + "id": "tool-use-1", + "name": "kubernetes__kubectl_get", + "input": {"resource": "pods"}, + } + ], + } + + +class _ClaudeTerminalResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return { + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 4}, + "content": [{"type": "text", "text": "bounded terminal result"}], + } + + +class _ClaudeRefusalResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return { + "stop_reason": "refusal", + "usage": {"input_tokens": 12, "output_tokens": 8}, + "content": [{"type": "text", "text": "not returned"}], + } + + +class _ClaudeMissingContentResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return { + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 8}, + "content": [], + } + + +def _mcp_tool() -> MCPTool: + return MCPTool( + name="kubectl_get", + description="read one Kubernetes resource", + input_schema={"type": "object", "properties": {}}, + server_name="kubernetes", + ) + + +def test_claude_sonnet_5_pricing_is_exact_and_conservative() -> None: + pricing = get_claude_pricing_policy("claude-sonnet-5") + + assert pricing is not None + assert pricing.input_usd_per_million == 3.0 + assert pricing.output_usd_per_million == 15.0 + assert pricing.version == "anthropic-standard-ceiling-2026-07-15" + assert get_claude_pricing_policy("claude-3-5-sonnet") is None + assert get_claude_pricing_policy("claude-sonnet-latest") is None + + +@pytest.mark.parametrize( + ("context", "reason"), + [ + ({}, "cloud_context_classification_missing"), + ({"data_classification": "private"}, "cloud_context_data_classification_blocked"), + ( + {"data_classification": "sanitized", "contains_secret": True}, + "cloud_context_sensitive_marker_blocked", + ), + ( + {"data_classification": "sanitized", "secret_value_exposed": True}, + "cloud_context_sensitive_marker_blocked", + ), + ( + {"data_classification": "sanitized", "raw_log_payload": "raw line"}, + "cloud_context_raw_log_blocked", + ), + ( + { + "dataClassification": "sanitized", + "headers": {"Authorization": "opaque-test-value"}, + }, + "cloud_context_sensitive_marker_blocked", + ), + ( + {"data_classification": "sanitized", "nested": {"password": "x"}}, + "cloud_context_sensitive_marker_blocked", + ), + ( + {"data_classification": "sanitized", "nested": {"Cookie": "x"}}, + "cloud_context_sensitive_marker_blocked", + ), + ( + { + "data_classification": "sanitized", + "nested": {"accessToken": "x"}, + }, + "cloud_context_sensitive_marker_blocked", + ), + ( + {"data_classification": "sanitized", "nested": {"apiKey": "x"}}, + "cloud_context_sensitive_marker_blocked", + ), + ( + {"data_classification": "sanitized", "nested": {"rawLog": "x"}}, + "cloud_context_raw_log_blocked", + ), + ( + {"data_classification": "sanitized", "require_local": True}, + "cloud_context_requires_local", + ), + ], +) +def test_cloud_context_gate_uses_structured_metadata_not_prompt_regex( + context: dict[str, object], + reason: str, +) -> None: + assert cloud_context_block_reason(context) == reason + + +@pytest.mark.asyncio +async def test_claude_shadow_mode_performs_zero_auth_reservation_or_http_calls() -> None: + provider = ClaudeProvider() + provider._get_client = AsyncMock() # type: ignore[method-assign] + disabled_readback = AsyncMock(return_value=False) + limiter_factory = MagicMock() + + with patch("src.services.ai_providers.claude.settings", _settings(mode="shadow")), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + disabled_readback, + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + limiter_factory, + ): + result = await provider.analyze("sanitized metadata", context=_context()) + health = await provider.health_check() + + assert result.success is False + assert result.error == "claude_shadow_metadata_only" + assert health is False + disabled_readback.assert_not_awaited() + limiter_factory.assert_not_called() + provider._get_client.assert_not_awaited() # type: ignore[attr-defined] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider_factory", + [ClaudeProvider, GeminiProvider], +) +async def test_paid_provider_sensitive_context_blocks_before_control_cost_or_http( + provider_factory, +) -> None: + provider = provider_factory() + provider._get_client = AsyncMock() # type: ignore[method-assign] + disabled_readback = AsyncMock(return_value=False) + limiter_factory = MagicMock() + provider_module = ( + "src.services.ai_providers.claude" + if provider.name == "claude" + else "src.services.ai_providers.gemini" + ) + provider_settings = ( + _settings(mode="production") + if provider.name == "claude" + else SimpleNamespace(GEMINI_API_KEY="configured-test-value") + ) + + with patch(f"{provider_module}.settings", provider_settings), patch( + f"{provider_module}.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + disabled_readback, + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + limiter_factory, + ): + result = await provider.analyze( + "opaque prompt not inspected by this gate", + context=_context(contains_secret=True), + ) + + assert result.success is False + assert result.error == "cloud_context_sensitive_marker_blocked" + disabled_readback.assert_not_awaited() + limiter_factory.assert_not_called() + provider._get_client.assert_not_awaited() # type: ignore[attr-defined] + + +@pytest.mark.asyncio +async def test_claude_cost_guard_blocks_without_http_call() -> None: + client = MagicMock() + client.is_closed = False + client.post = AsyncMock() + limiter = MagicMock() + limiter.reserve_generation = AsyncMock( + return_value=_reservation(allowed=False, reason="daily_cost_limit") + ) + provider = ClaudeProvider() + provider._http_client = client + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + AsyncMock(return_value=False), + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ): + result = await provider.analyze("sanitized metadata", context=_context()) + + assert result.success is False + assert result.error == "cost_guard_blocked:daily_cost_limit" + assert result.audit_metadata["generation_receipt_id"] == "gen-claude-test-1" + client.post.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_claude_success_uses_exact_model_and_no_manual_sampling_or_thinking() -> None: + client = MagicMock() + client.is_closed = False + client.post = AsyncMock(return_value=_ClaudeResponse()) + limiter = MagicMock() + limiter.reserve_generation = AsyncMock(return_value=_reservation()) + limiter.finalize_generation = AsyncMock(return_value=True) + provider = ClaudeProvider() + provider._http_client = client + context = _context() + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + AsyncMock(return_value=False), + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ): + result = await provider.analyze("sanitized metadata", context=context) + + assert result.success is True + assert json.loads(result.raw_response)["suggested_action"] == "NO_ACTION" + assert result.tokens == 20 + assert result.cost_usd == _reservation().actual_cost_usd(12, 8) + assert result.audit_metadata == { + "schema_version": "paid_provider_audit_v1", + "paid_provider": True, + "provider": "claude", + "fallback_position": 4, + "execution_mode": "production", + "generation_receipt_id": "gen-claude-test-1", + "generation_receipt_ids": [], + "pricing_version": "anthropic-standard-ceiling-2026-07-15", + "canary_bucket": None, + } + request = client.post.await_args.kwargs["json"] + assert request["model"] == "claude-sonnet-5" + assert request["max_tokens"] == 4096 + for forbidden in ("temperature", "top_p", "top_k", "thinking"): + assert forbidden not in request + limiter.reserve_generation.assert_awaited_once_with( + "claude", + "sanitized metadata", + model="claude-sonnet-5", + max_output_tokens=4096, + context=context, + ) + limiter.finalize_generation.assert_awaited_once_with( + _reservation(), + success=True, + prompt_tokens=12, + completion_tokens=8, + ) + + +@pytest.mark.asyncio +async def test_claude_safe_structured_tool_result_permits_next_iteration() -> None: + client = MagicMock() + client.is_closed = False + client.post = AsyncMock( + side_effect=[_ClaudeToolUseResponse(), _ClaudeTerminalResponse()] + ) + limiter = MagicMock() + limiter.reserve_generation = AsyncMock( + side_effect=[ + _reservation(receipt_id="gen-claude-tool-1"), + _reservation(receipt_id="gen-claude-tool-2"), + ] + ) + limiter.finalize_generation = AsyncMock(return_value=True) + tool_executor = AsyncMock( + return_value=MCPToolResult( + success=True, + execution_id="safe-tool-receipt", + output={ + "data_classification": "sanitized", + "status": "healthy", + }, + ) + ) + provider = ClaudeProvider() + provider._http_client = client + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + AsyncMock(return_value=False), + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ): + result = await provider.analyze_with_tools( + "sanitized investigation", + [_mcp_tool()], + tool_executor, + context=_context(), + ) + + assert result.success is True + assert result.raw_response == "bounded terminal result" + assert client.post.await_count == 2 + second_request = client.post.await_args_list[1].kwargs["json"] + serialized_messages = json.dumps(second_request["messages"]) + assert "safe-tool-receipt" in serialized_messages + assert "data_classification" in serialized_messages + assert limiter.finalize_generation.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("response", "expected_error", "receipt_error_code"), + [ + ( + _ClaudeRefusalResponse(), + "claude_provider_refusal", + "provider_refusal", + ), + ( + _ClaudeMissingContentResponse(), + "claude_response_missing_content", + "response_missing_content", + ), + ], +) +async def test_claude_invalid_terminal_response_finalizes_failure_once( + response: object, + expected_error: str, + receipt_error_code: str, +) -> None: + client = MagicMock() + client.is_closed = False + client.post = AsyncMock(return_value=response) + limiter = MagicMock() + reservation = _reservation(receipt_id="gen-claude-invalid-terminal") + limiter.reserve_generation = AsyncMock(return_value=reservation) + limiter.finalize_generation = AsyncMock(return_value=True) + provider = ClaudeProvider() + provider._http_client = client + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + AsyncMock(return_value=False), + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ): + result = await provider.analyze("sanitized metadata", context=_context()) + + assert result.success is False + assert result.error == expected_error + limiter.finalize_generation.assert_awaited_once_with( + reservation, + success=False, + prompt_tokens=12, + completion_tokens=8, + error_code=receipt_error_code, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("response", "expected_error", "receipt_error_code"), + [ + ( + _ClaudeRefusalResponse(), + "claude_provider_refusal", + "provider_refusal", + ), + ( + _ClaudeMissingContentResponse(), + "claude_agent_loop_response_missing_content", + "response_missing_content", + ), + ], +) +async def test_claude_agent_loop_invalid_response_finalizes_failure_once( + response: object, + expected_error: str, + receipt_error_code: str, +) -> None: + client = MagicMock() + client.is_closed = False + client.post = AsyncMock(return_value=response) + limiter = MagicMock() + reservation = _reservation(receipt_id="gen-claude-agent-invalid") + limiter.reserve_generation = AsyncMock(return_value=reservation) + limiter.finalize_generation = AsyncMock(return_value=True) + tool_executor = AsyncMock() + provider = ClaudeProvider() + provider._http_client = client + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + AsyncMock(return_value=False), + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ): + result = await provider.analyze_with_tools( + "sanitized investigation", + [_mcp_tool()], + tool_executor, + context=_context(), + ) + + assert result.success is False + assert result.error == expected_error + assert client.post.await_count == 1 + tool_executor.assert_not_awaited() + limiter.finalize_generation.assert_awaited_once_with( + reservation, + success=False, + prompt_tokens=12, + completion_tokens=8, + error_code=receipt_error_code, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tool_output", "reason", "leak_marker"), + [ + ( + {"status": "LEAK_UNCLASSIFIED_TOOL_RESULT"}, + "cloud_context_classification_missing", + "LEAK_UNCLASSIFIED_TOOL_RESULT", + ), + ( + { + "data_classification": "sanitized", + "headers": {"Authorization": "LEAK_AUTHORIZATION_TOOL_RESULT"}, + }, + "cloud_context_sensitive_marker_blocked", + "LEAK_AUTHORIZATION_TOOL_RESULT", + ), + ( + { + "data_classification": "sanitized", + "rawLog": "LEAK_RAW_LOG_TOOL_RESULT", + }, + "cloud_context_raw_log_blocked", + "LEAK_RAW_LOG_TOOL_RESULT", + ), + ], +) +async def test_claude_tool_result_privacy_block_stops_before_second_api_call( + tool_output: dict[str, object], + reason: str, + leak_marker: str, +) -> None: + client = MagicMock() + client.is_closed = False + client.post = AsyncMock(return_value=_ClaudeToolUseResponse()) + limiter = MagicMock() + reservation = _reservation(receipt_id="gen-claude-blocked-tool") + limiter.reserve_generation = AsyncMock(return_value=reservation) + limiter.finalize_generation = AsyncMock(return_value=True) + tool_executor = AsyncMock( + return_value=MCPToolResult( + success=True, + execution_id="blocked-tool-receipt", + output=tool_output, + ) + ) + provider = ClaudeProvider() + provider._http_client = client + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + AsyncMock(return_value=False), + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ), patch("src.services.ai_providers.claude.logger.warning") as warning: + result = await provider.analyze_with_tools( + "sanitized investigation", + [_mcp_tool()], + tool_executor, + context=_context(), + ) + + assert result.success is False + assert result.raw_response == "" + assert result.error == f"claude_tool_result_blocked:{reason}" + assert client.post.await_count == 1 + assert limiter.reserve_generation.await_count == 1 + privacy_receipt = result.audit_metadata["tool_result_privacy_receipt"] + assert privacy_receipt == { + "schema_version": "cloud_tool_result_privacy_v1", + "status": "blocked_no_forward", + "reason": reason, + "iteration": 1, + "provider_call_count": 1, + "paid_iteration_terminal": "failed", + "receipt_finalized": True, + "content_logged": False, + "content_forwarded": False, + } + limiter.finalize_generation.assert_awaited_once_with( + reservation, + success=False, + prompt_tokens=12, + completion_tokens=8, + error_code="tool_result_privacy_blocked", + ) + assert leak_marker not in json.dumps(result.to_dict()) + assert leak_marker not in str(warning.call_args) + assert leak_marker not in str(client.post.await_args_list) + + +@pytest.mark.asyncio +async def test_claude_tool_executor_failure_finalizes_once_without_error_leak() -> None: + leak_marker = "LEAK_TOOL_EXECUTOR_EXCEPTION_BODY" + client = MagicMock() + client.is_closed = False + client.post = AsyncMock(return_value=_ClaudeToolUseResponse()) + limiter = MagicMock() + reservation = _reservation(receipt_id="gen-claude-tool-executor-failed") + limiter.reserve_generation = AsyncMock(return_value=reservation) + limiter.finalize_generation = AsyncMock(return_value=True) + tool_executor = AsyncMock(side_effect=RuntimeError(leak_marker)) + provider = ClaudeProvider() + provider._http_client = client + + with patch("src.services.ai_providers.claude.settings", _settings()), patch( + "src.services.ai_providers.claude.is_provider_enabled_by_env", + return_value=True, + ), patch.object( + ai_control_module, + "is_provider_disabled", + AsyncMock(return_value=False), + ), patch( + "src.services.ai_rate_limiter.get_ai_rate_limiter", + return_value=limiter, + ), patch("src.services.ai_providers.claude.logger.warning") as warning: + result = await provider.analyze_with_tools( + "sanitized investigation", + [_mcp_tool()], + tool_executor, + context=_context(), + ) + + assert result.success is False + assert result.raw_response == "" + assert result.error == "claude_tool_executor_failed" + assert client.post.await_count == 1 + limiter.finalize_generation.assert_awaited_once_with( + reservation, + success=False, + prompt_tokens=12, + completion_tokens=8, + error_code="tool_executor_failed", + ) + assert leak_marker not in json.dumps(result.to_dict()) + assert leak_marker not in str(warning.call_args) diff --git a/apps/api/tests/test_gemini_cost_guard.py b/apps/api/tests/test_gemini_cost_guard.py index b5e2d990b..346370e1b 100644 --- a/apps/api/tests/test_gemini_cost_guard.py +++ b/apps/api/tests/test_gemini_cost_guard.py @@ -389,6 +389,7 @@ async def test_gemini_provider_does_not_call_api_when_cost_guard_blocks() -> Non "trace_id": "trace-cost-block", "run_id": "run-cost-block", "work_item_id": "work-cost-block", + "data_classification": "sanitized", }, ) @@ -427,6 +428,7 @@ async def test_gemini_provider_does_not_reserve_or_call_when_runtime_disabled( "trace_id": "trace-disabled", "run_id": "run-disabled", "work_item_id": "work-disabled", + "data_classification": "sanitized", }, ) @@ -469,6 +471,7 @@ async def test_gemini_provider_failure_finalizes_estimate_charged_receipt() -> N "trace_id": "trace-failure", "run_id": "run-failure", "work_item_id": "work-failure", + "data_classification": "sanitized", }, ) @@ -518,6 +521,7 @@ async def test_gemini_success_without_usage_metadata_is_not_zero_false_green() - "trace_id": "trace-no-usage", "run_id": "run-no-usage", "work_item_id": "work-no-usage", + "data_classification": "sanitized", }, ) diff --git a/apps/api/tests/test_gemini_provider_security.py b/apps/api/tests/test_gemini_provider_security.py index 919596240..68d34ea2f 100644 --- a/apps/api/tests/test_gemini_provider_security.py +++ b/apps/api/tests/test_gemini_provider_security.py @@ -91,6 +91,7 @@ async def test_gemini_provider_uses_header_auth_not_query_string(): "trace_id": "trace-security", "run_id": "run-security", "work_item_id": "work-security", + "data_classification": "sanitized", } result = await provider.analyze("hello", context=context) @@ -130,7 +131,7 @@ async def test_authentication_rejection_writes_bounded_receipt_code() -> None: ) response = httpx.Response(status_code=401, request=request) error = httpx.HTTPStatusError( - "authentication rejected", + "provider body SECRET_PROVIDER_PAYLOAD_MUST_NOT_BE_LOGGED", request=request, response=response, ) @@ -176,22 +177,30 @@ async def test_authentication_rejection_writes_bounded_receipt_code() -> None: ), patch( "src.services.ai_rate_limiter.get_ai_rate_limiter", return_value=limiter, - ): + ), patch("src.services.ai_providers.gemini.logger.warning") as warning: result = await provider.analyze( "test authentication", context={ "trace_id": "trace-auth", "run_id": "run-auth", "work_item_id": "work-auth", + "data_classification": "sanitized", }, ) assert result.success is False + assert result.error == "authentication_rejected" limiter.finalize_generation.assert_awaited_once_with( reservation, success=False, error_code="authentication_rejected", ) + assert "SECRET_PROVIDER_PAYLOAD_MUST_NOT_BE_LOGGED" not in str( + warning.call_args + ) + assert warning.call_args.kwargs["error_code"] == "authentication_rejected" + assert warning.call_args.kwargs["error_type"] == "HTTPStatusError" + assert "error" not in warning.call_args.kwargs @pytest.mark.asyncio diff --git a/apps/api/tests/test_gemini_route_manifest.py b/apps/api/tests/test_gemini_route_manifest.py index e403ff347..ebfb5bdae 100644 --- a/apps/api/tests/test_gemini_route_manifest.py +++ b/apps/api/tests/test_gemini_route_manifest.py @@ -20,11 +20,12 @@ PRODUCTION_PROVIDER_ORDER = [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] -def test_prod_and_dev_publish_exact_four_hop_fallback() -> None: +def test_prod_and_dev_publish_exact_five_hop_fallback() -> None: for path in ( "k8s/awoooi-prod/04-configmap.yaml", "k8s/awoooi-dev/02-configmap.yaml", @@ -33,6 +34,16 @@ def test_prod_and_dev_publish_exact_four_hop_fallback() -> None: assert json.loads(data["AI_FALLBACK_ORDER"]) == PRODUCTION_PROVIDER_ORDER assert data["ALERT_AI_ENFORCE_OLLAMA_FIRST"] == "true" assert data["ALERT_AI_STRICT_PROVIDER_CHAIN"] == "true" + assert data["CLAUDE_MODEL"] == "claude-sonnet-5" + assert data["ENABLE_CLAUDE"] == "false" + assert data["CLAUDE_EXECUTION_MODE"] == "shadow" + assert data["CLAUDE_CANARY_PERCENT"] == "0" + assert data["CLAUDE_DAILY_QUOTA"] == "50" + assert data["CLAUDE_RPM_LIMIT"] == "2" + assert data["CLAUDE_DAILY_TOKEN_LIMIT"] == "100000" + assert data["CLAUDE_DAILY_COST_LIMIT_USD"] == "1.0" + assert data["CLAUDE_TOTAL_COST_LIMIT_USD"] == "5.0" + assert data["CLAUDE_COST_ALERT_THRESHOLD_USD"] == "4.0" assert data["GEMINI_MODEL"] == "gemini-2.5-flash-lite" assert data["GEMINI_DAILY_QUOTA"] == "500" assert data["GEMINI_RPM_LIMIT"] == "10" @@ -72,11 +83,15 @@ def test_model_registry_publishes_only_supported_paid_fallback_model() -> None: assert models["fallback_order"] == PRODUCTION_PROVIDER_ORDER assert models["tool_calling_fallback_order"] == PRODUCTION_PROVIDER_ORDER assert models["production_route"]["provider_order"] == PRODUCTION_PROVIDER_ORDER - for provider in ("claude", "nvidia"): - assert models["providers"][provider]["production_executable"] is False - assert models["providers"][provider]["mode"] == ( - "non_executable_shadow_metadata_only" - ) + claude = models["providers"]["claude"] + assert set(claude["models"].values()) == {"claude-sonnet-5"} + assert claude["production_executable"] is True + assert claude["mode"] == "paid_fallback_shadow_by_default" + assert claude["auth"]["env_var"] == "CLAUDE_API_KEY" + assert models["providers"]["nvidia"]["production_executable"] is False + assert models["providers"]["nvidia"]["mode"] == ( + "non_executable_shadow_metadata_only" + ) def test_capabilities_publish_route_and_shadow_only_legacy_providers() -> None: @@ -91,7 +106,6 @@ def test_capabilities_publish_route_and_shadow_only_legacy_providers() -> None: assert { provider["name"] for provider in ai_providers["shadow_metadata_only"] } == { - "claude", "nemotron", "nvidia", "openclaw", @@ -117,6 +131,21 @@ def test_prod_runtime_endpoints_and_nemotron_override_match_strict_chain() -> No assert env["ENABLE_NEMOTRON_COLLABORATION"] == "false" +def test_worker_projects_only_the_canonical_claude_secret_name() -> None: + deployment = _load("k8s/awoooi-prod/08-deployment-worker.yaml") + env = { + item["name"]: item + for item in deployment["spec"]["template"]["spec"]["containers"][0]["env"] + } + + assert env["CLAUDE_API_KEY"]["valueFrom"]["secretKeyRef"] == { + "name": "awoooi-secrets", + "key": "CLAUDE_API_KEY", + "optional": True, + } + assert "ANTHROPIC_API_KEY" not in env + + def test_config_parser_rejects_route_reordering_or_provider_insertion() -> None: assert Settings.parse_ai_fallback(PRODUCTION_PROVIDER_ORDER) == ( PRODUCTION_PROVIDER_ORDER diff --git a/apps/api/tests/test_model_version_probe.py b/apps/api/tests/test_model_version_probe.py index dd5c1662d..f7b4dabb2 100644 --- a/apps/api/tests/test_model_version_probe.py +++ b/apps/api/tests/test_model_version_probe.py @@ -213,12 +213,13 @@ class TestProbeClaudeVersion: """CLAUDE_API_KEY 存在 → 回傳 claude provider info""" mock_settings = MagicMock() mock_settings.CLAUDE_API_KEY = "sk-fake" + mock_settings.CLAUDE_MODEL = "claude-sonnet-5" with patch("src.services.model_version_probe.settings", mock_settings): info = await probe_claude_version() assert info.provider == "claude" - assert "claude" in info.model + assert info.model == "claude-sonnet-5" assert info.version == info.model assert info.digest is None @@ -232,6 +233,17 @@ class TestProbeClaudeVersion: with pytest.raises(RuntimeError, match="CLAUDE_API_KEY"): await probe_claude_version() + @pytest.mark.asyncio + async def test_missing_model_raises(self): + """Configured key without an exact model cannot claim version truth.""" + mock_settings = MagicMock() + mock_settings.CLAUDE_API_KEY = "fake-key" + mock_settings.CLAUDE_MODEL = "" + + with patch("src.services.model_version_probe.settings", mock_settings): + with pytest.raises(RuntimeError, match="CLAUDE_MODEL"): + await probe_claude_version() + # ============================================================================= # probe_openclaw_nemo_version @@ -310,15 +322,15 @@ class TestProbeAllProviders: fake_results = [ ProviderVersionInfo(provider="ollama", model="qwen2.5:7b-instruct", version="v1"), ProviderVersionInfo(provider="ollama_local", model="qwen2.5:7b-instruct", version="v1"), + ProviderVersionInfo(provider="claude", model="claude-sonnet-5", version="claude-sonnet-5"), ProviderVersionInfo(provider="gemini", model="gemini-1.5-flash", version="gemini-1.5-flash"), - ProviderVersionInfo(provider="claude", model="claude-haiku-4-5-20251001", version="claude-haiku-4-5-20251001"), ProviderVersionInfo(provider="openclaw_nemo", model="deepseek-r1:14b", version="v1"), ] with patch("src.services.model_version_probe.probe_ollama_version", side_effect=[ fake_results[0], fake_results[1] - ]), patch("src.services.model_version_probe.probe_gemini_version", return_value=fake_results[2]), \ - patch("src.services.model_version_probe.probe_claude_version", return_value=fake_results[3]), \ + ]), patch("src.services.model_version_probe.probe_claude_version", return_value=fake_results[2]), \ + patch("src.services.model_version_probe.probe_gemini_version", return_value=fake_results[3]), \ patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=fake_results[4]): mock_settings = MagicMock() @@ -347,7 +359,7 @@ class TestProbeAllProviders: with patch("src.services.model_version_probe.probe_ollama_version", side_effect=_fail_ollama), \ patch("src.services.model_version_probe.probe_gemini_version", side_effect=_fail), \ patch("src.services.model_version_probe.probe_claude_version", return_value=ProviderVersionInfo( - provider="claude", model="claude-haiku-4-5-20251001", version="claude-haiku-4-5-20251001" + provider="claude", model="claude-sonnet-5", version="claude-sonnet-5" )), \ patch("src.services.model_version_probe.probe_openclaw_nemo_version", return_value=ProviderVersionInfo( provider="openclaw_nemo", model="deepseek-r1:14b", version="v1" @@ -361,7 +373,7 @@ class TestProbeAllProviders: with patch("src.services.model_version_probe.settings", mock_settings): results = await probe_all_providers() - # ollama(ok) + ollama_local(fail) + gemini(fail) + claude(ok) + openclaw_nemo(ok) → 3 + # ollama(ok) + ollama_local(fail) + claude(ok) + gemini(fail) + openclaw_nemo(ok) → 3 assert len(results) == 3 providers = {r.provider for r in results} assert "ollama" in providers diff --git a/apps/api/tests/test_ollama_failover_manager.py b/apps/api/tests/test_ollama_failover_manager.py index fde2bf9fb..f4409c980 100644 --- a/apps/api/tests/test_ollama_failover_manager.py +++ b/apps/api/tests/test_ollama_failover_manager.py @@ -174,10 +174,10 @@ class TestDecideRoute: ) assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" def test_gcp_a_degraded_keeps_exact_full_chain(self): - """三層不可用時 readback 仍公開不可重排的四跳鏈。""" + """三層不可用時 readback 仍公開不可重排的五跳鏈。""" manager = self._setup() h_gcp_a = _make_health(HealthStatus.DEGRADED, URL_GCP_A) h_gcp_b = _offline_health(URL_GCP_B) @@ -196,6 +196,7 @@ class TestDecideRoute: assert [item.provider_name for item in result.fallback_chain] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] @@ -220,7 +221,7 @@ class TestDecideRoute: ) assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" def test_gcp_a_offline_exact_chain_has_no_legacy_provider(self): """三層 OFFLINE 後仍無 Nemo/NVIDIA/Claude 插入。""" @@ -242,6 +243,7 @@ class TestDecideRoute: assert [item.provider_name for item in result.fallback_chain] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] @@ -264,6 +266,7 @@ class TestDecideRoute: assert [endpoint.provider_name for endpoint in result.fallback_chain] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] @@ -404,7 +407,7 @@ class TestSelectProvider: assert isinstance(result, OllamaRoutingResult) assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" @pytest.mark.asyncio async def test_audit_not_written_when_gcp_a_healthy(self): @@ -480,6 +483,11 @@ class TestOllamaRoutingResult: fb1 = OllamaEndpoint(url=URL_GCP_B, provider_name="ollama_gcp_b", model="m1") fb2 = OllamaEndpoint(url=URL_LOCAL, provider_name="ollama_local", model="m1") fb3 = OllamaEndpoint( + url="", + provider_name="claude", + model="claude-sonnet-5", + ) + fb4 = OllamaEndpoint( url="", provider_name="gemini", model="gemini-2.5-flash-lite", @@ -487,7 +495,7 @@ class TestOllamaRoutingResult: result = OllamaRoutingResult( primary=primary, - fallback_chain=[fb1, fb2, fb3], + fallback_chain=[fb1, fb2, fb3, fb4], routing_reason="test", health_gcp_a=_make_health(HealthStatus.HEALTHY), ) @@ -497,6 +505,7 @@ class TestOllamaRoutingResult: "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] @@ -634,8 +643,8 @@ class TestThreeLayerFailover: assert result.observed_fallback.provider_name == "ollama_local" @pytest.mark.asyncio - async def test_all_offline_observes_gemini_without_selecting_it(self): - """場景4：Gemini 是最後 observation，不是 selected primary。""" + async def test_all_offline_observes_claude_without_selecting_it(self): + """場景4：Claude 是第一個 paid observation，不是 selected primary。""" manager = self._make_manager_with_health( gcp_a=HealthStatus.OFFLINE, gcp_b=HealthStatus.OFFLINE, @@ -646,11 +655,11 @@ class TestThreeLayerFailover: result = await manager.select_provider() assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" @pytest.mark.asyncio async def test_all_offline_selection_does_not_consume_or_redirect_paid_quota(self): - """選路不消耗 quota；Gemini 仍只在四跳鏈尾端。""" + """選路不消耗 quota；Gemini 仍只在五跳鏈尾端。""" manager = self._make_manager_with_health( gcp_a=HealthStatus.OFFLINE, gcp_b=HealthStatus.OFFLINE, @@ -661,10 +670,11 @@ class TestThreeLayerFailover: result = await manager.select_provider() assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" assert [item.provider_name for item in result.fallback_chain] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] @@ -797,7 +807,7 @@ class TestGatherReturnExceptions: result = await manager.select_provider() assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" @pytest.mark.asyncio async def test_gcp_a_healthy_select_provider_primary_ollama(self): @@ -908,10 +918,11 @@ class TestGeminiQuota: result = await manager.select_provider() assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" assert [item.provider_name for item in result.fallback_chain] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] @@ -939,9 +950,10 @@ class TestGeminiQuota: result = await manager.select_provider() assert result.primary.provider_name == "ollama_gcp_a" - assert result.observed_fallback.provider_name == "gemini" + assert result.observed_fallback.provider_name == "claude" assert [item.provider_name for item in result.fallback_chain] == [ "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] diff --git a/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py b/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py index b79c30d16..fa6fe2c56 100644 --- a/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py +++ b/apps/api/tests/test_openclaw_alert_cloud_fallback_gate.py @@ -21,6 +21,10 @@ def _explicitly_enable_gemini_environment( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setenv("ENABLE_GEMINI", "true") + monkeypatch.setenv("ENABLE_CLAUDE", "false") + monkeypatch.setattr(openclaw_module.settings, "CLAUDE_API_KEY", "") + monkeypatch.setattr(openclaw_module.settings, "CLAUDE_EXECUTION_MODE", "shadow") + monkeypatch.setattr(openclaw_module.settings, "CLAUDE_CANARY_PERCENT", 0) @dataclass @@ -105,6 +109,23 @@ class _FakeExecutor: ) +def _prepared_paid_alert_context( + service: OpenClawService, + **overrides: Any, +) -> dict[str, Any]: + context = service._prepare_paid_cloud_route_context( + { + "incident_id": "INC-1", + "alertname": "HostHighCpuLoad", + "source": "test-alert-producer", + "target_resource": "node-110", + **overrides, + } + ) + assert context is not None + return context + + @pytest.mark.asyncio async def test_alert_context_uses_exact_ollama_lane_then_gemini( monkeypatch: pytest.MonkeyPatch, @@ -290,7 +311,7 @@ async def test_alert_context_sorts_ollama_lane_and_drops_cloud_providers( service = object.__new__(OpenClawService) provider_order = await service._resolve_alert_provider_order( task_type="diagnose", - alert_context={"incident_id": "INC-1", "alertname": "HostHighCpuLoad"}, + alert_context=_prepared_paid_alert_context(service), ) assert provider_order == ["ollama_gcp_a", "ollama_gcp_b", "ollama_local"] @@ -309,13 +330,46 @@ async def test_alert_context_drops_cloud_insertions_before_gemini_backup( service = object.__new__(OpenClawService) provider_order = await service._resolve_alert_provider_order( task_type="diagnose", - alert_context={"incident_id": "INC-1", "alertname": "HostHighCpuLoad"}, + alert_context=_prepared_paid_alert_context(service), cloud_provider_order=["claude", "openclaw_nemo", "gemini", "ollama"], ) assert provider_order == ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"] +@pytest.mark.asyncio +async def test_alert_context_appends_enabled_claude_before_gemini( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(openclaw_module.settings, "ALERT_AI_ALLOW_CLOUD_FALLBACK", True) + monkeypatch.setattr(openclaw_module.settings, "ALERT_AI_ENFORCE_OLLAMA_FIRST", True) + monkeypatch.setattr(openclaw_module.settings, "CLAUDE_API_KEY", "configured-test-value") + monkeypatch.setattr(openclaw_module.settings, "CLAUDE_EXECUTION_MODE", "production") + monkeypatch.setattr(openclaw_module.settings, "CLAUDE_CANARY_PERCENT", 0) + monkeypatch.setattr(openclaw_module.settings, "GEMINI_API_KEY", "configured-test-value") + monkeypatch.setenv("ENABLE_CLAUDE", "true") + monkeypatch.setattr(ai_control_module, "is_provider_disabled", AsyncMock(return_value=False)) + monkeypatch.setattr( + openclaw_module, + "get_ollama_failover_manager", + lambda: _UnorderedFailoverManager(), + ) + + service = object.__new__(OpenClawService) + provider_order = await service._resolve_alert_provider_order( + task_type="diagnose", + alert_context=_prepared_paid_alert_context(service), + ) + + assert provider_order == [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ] + + @pytest.mark.asyncio async def test_disabled_gemini_is_not_readded_to_alert_route( monkeypatch: pytest.MonkeyPatch, diff --git a/apps/api/tests/test_openclaw_paid_cloud_privacy_boundary.py b/apps/api/tests/test_openclaw_paid_cloud_privacy_boundary.py new file mode 100644 index 000000000..7dfd7dd8c --- /dev/null +++ b/apps/api/tests/test_openclaw_paid_cloud_privacy_boundary.py @@ -0,0 +1,670 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from typing import Any +from unittest.mock import AsyncMock + +import pytest +from structlog.testing import capture_logs + +from src.services import ai_router as ai_router_module +from src.services import openclaw as openclaw_module +from src.services.ai_providers.interfaces import AIResult +from src.services.ai_router import AIProviderRegistry, AIRouterExecutor +from src.services.openclaw import OpenClawService, build_paid_cloud_alert_context + + +class _Redis: + def __init__( + self, + *, + cached_provider: str | None = None, + paid_disabled: bool = False, + durable_receipts: dict[str, dict[str, Any]] | None = None, + ) -> None: + self.cached_provider = cached_provider + self.paid_disabled = paid_disabled + self.durable_receipts = durable_receipts or {} + self.set_calls: list[tuple[str, str, int | None]] = [] + + async def get(self, key: str) -> str | None: + if key.startswith("ai:control:disabled:"): + return "1" if self.paid_disabled else "0" + if key.startswith("ai:ollama:unavailable_receipt:"): + receipt_id = key.removeprefix("ai:ollama:unavailable_receipt:") + receipt = self.durable_receipts.get(receipt_id) + return json.dumps(receipt) if receipt else None + if self.cached_provider: + return json.dumps( + { + "provider": self.cached_provider, + "response": '{"cached":"must-not-run"}', + } + ) + return None + + async def set(self, key: str, value: str, ex: int | None = None) -> None: + self.set_calls.append((key, value, ex)) + + +class _Provider: + is_enabled = True + capabilities = {"rca", "chat"} + + def __init__( + self, + name: str, + *, + success: bool, + privacy_level: str, + attempt_order: list[str] | None = None, + failure_response: str = "", + failure_error: str = "bounded failure", + ) -> None: + self.name = name + self.success = success + self.privacy_level = privacy_level + self.attempt_order = attempt_order + self.failure_response = failure_response + self.failure_error = failure_error + self.calls: list[tuple[str, dict[str, Any] | None]] = [] + + async def analyze( + self, + prompt: str, + context: dict[str, Any] | None = None, + ) -> AIResult: + self.calls.append((prompt, context)) + if self.attempt_order is not None: + self.attempt_order.append(self.name) + return AIResult( + raw_response=( + '{"action_title":"safe","description":"safe",' + '"suggested_action":"NO_ACTION","kubectl_command":"",' + '"target_resource":"api","namespace":"awoooi-prod",' + '"risk_level":"low","blast_radius":{"affected_pods":0,' + '"estimated_downtime":"0","related_services":[],' + '"data_impact":"NONE"},"reasoning":"safe","confidence":0.9}' + if self.success + else self.failure_response + ), + success=self.success, + provider=self.name, + error=None if self.success else self.failure_error, + tokens=7, + audit_metadata={"generation_receipt_id": "paid-gen-1"}, + ) + + +class _Trace: + def __init__(self) -> None: + self.langfuse_trace_id = "trace-test" + self.generations: list[dict[str, Any]] = [] + self.root_metadata: dict[str, Any] = {} + + def __enter__(self) -> _Trace: + return self + + def __exit__(self, *_args: Any) -> None: + return None + + def generation(self, **kwargs: Any) -> None: + self.generations.append(kwargs) + + def score(self, **_kwargs: Any) -> None: + return None + + +def _trace_factory(trace: _Trace): + def _factory(*_args: Any, **kwargs: Any) -> _Trace: + trace.root_metadata = dict(kwargs.get("metadata") or {}) + return trace + + return _factory + + +def _raw_alert() -> dict[str, Any]: + return { + "alert_type": "DockerContainerUnhealthy", + "severity": "warning", + "source": "alertmanager", + "target_resource": "192.168.0.110", + "namespace": "awoooi-prod", + "fingerprint": "fp-1", + "message": "Authorization: Bearer raw-secret-must-never-reach-cloud", + "labels": {"token": "raw-label-secret"}, + "annotations": {"description": "raw annotation"}, + "raw_logs": "raw-log-secret", + "headers": {"authorization": "raw-header-secret"}, + } + + +def _prepared_context() -> dict[str, Any]: + service = object.__new__(OpenClawService) + prepared = service._prepare_paid_cloud_route_context(_raw_alert()) + assert prepared is not None + prepared.update( + trace_id="trace-1", + run_id="run-1", + work_item_id="work-1", + alert_requires_ollama_before_cloud=True, + ) + return prepared + + +def test_cloud_sanitizer_attests_only_allowlisted_structured_fields() -> None: + cloud_context, reason = build_paid_cloud_alert_context(_raw_alert()) + + assert reason == "verified_allowlisted_sanitization_receipt" + assert cloud_context is not None + serialized = json.dumps(cloud_context, ensure_ascii=False) + assert "raw-secret-must-never-reach-cloud" not in serialized + assert "raw-label-secret" not in serialized + assert "raw annotation" not in serialized + assert "raw-log-secret" not in serialized + assert "raw-header-secret" not in serialized + assert "192.168.0.110" not in serialized + assert "[PRIVATE_IP_REDACTED]" in serialized + receipt = cloud_context["cloud_sanitization_receipt"] + assert receipt["status"] == "verified" + assert receipt["raw_payload_forwarded"] is False + assert receipt["raw_log_payload_forwarded"] is False + assert receipt["secret_value_exposed"] is False + assert receipt["excluded_field_count"] == 5 + + +def test_cloud_sanitizer_redacts_bare_keys_and_all_private_network_ranges() -> None: + alert = { + "alert_type": "HostKeyLeak-sk-ant-api03-ABCDEFGHIJKLMNOPQRSTUV", + "source": "AIzaSyABCDEFGHIJKLMNOPQRSTUVWXY123456", + "target_resource": "10.21.4.8 and 172.20.1.4", + "namespace": "[fd00::1234] and fe80::1", + "fingerprint": "Authorization: Bearer naked-secret-value", + } + + cloud_context, reason = build_paid_cloud_alert_context(alert) + + assert reason == "verified_allowlisted_sanitization_receipt" + assert cloud_context is not None + serialized = json.dumps(cloud_context, ensure_ascii=False) + for forbidden in ( + "sk-ant-api03-ABCDEFGHIJKLMNOPQRSTUV", + "AIzaSyABCDEFGHIJKLMNOPQRSTUVWXY123456", + "10.21.4.8", + "172.20.1.4", + "fd00::1234", + "fe80::1", + "naked-secret-value", + ): + assert forbidden not in serialized + assert "[SECRET_REDACTED]" in serialized + assert serialized.count("[PRIVATE_IP_REDACTED]") >= 4 + receipt = cloud_context["cloud_sanitization_receipt"] + assert receipt["dlp_policy"] == "paid_cloud_allowlist_dlp_v1" + assert receipt["private_network_value_exposed"] is False + + +def test_prepare_rebuilds_and_rejects_caller_self_signed_cloud_payload() -> None: + alert = _raw_alert() + malicious_payload = { + "alert_type": "DockerContainerUnhealthy", + "target_resource": "sk-ant-api03-CALLERFORGEDSECRETVALUE", + } + canonical = json.dumps( + malicious_payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + forged_hash = openclaw_module.hashlib.sha256(canonical.encode()).hexdigest() + alert["paid_cloud_context"] = { + "data_classification": "sanitized", + "sanitized_payload": malicious_payload, + "cloud_sanitization_receipt": { + "schema_version": "openclaw_paid_cloud_sanitization_v1", + "status": "verified", + "receipt_id": f"openclaw-cloud-sanitize:{forged_hash[:24]}", + "sanitizer": "src.services.sanitization_service.sanitize", + "classifier": "allowlisted_structured_alert_fields_v1", + "dlp_policy": "paid_cloud_allowlist_dlp_v1", + "payload_sha256": forged_hash, + "raw_payload_forwarded": False, + "raw_log_payload_forwarded": False, + "secret_value_exposed": False, + "private_network_value_exposed": False, + }, + } + + prepared = object.__new__(OpenClawService)._prepare_paid_cloud_route_context(alert) + + assert prepared is not None + serialized = json.dumps(prepared["paid_cloud_context"], ensure_ascii=False) + assert "CALLERFORGEDSECRETVALUE" not in serialized + assert prepared["paid_cloud_context"]["sanitized_payload"]["target_resource"] == ( + "[PRIVATE_IP_REDACTED]" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cached_provider", + ["claude", "gemini", "claude_cached", "gemini:legacy", "claude-shadow"], +) +async def test_openclaw_outer_cache_never_returns_paid_result( + monkeypatch: pytest.MonkeyPatch, + cached_provider: str, +) -> None: + redis = _Redis(cached_provider=cached_provider) + service = object.__new__(OpenClawService) + fallback = AsyncMock(return_value=('{"fresh":"ollama"}', "ollama_gcp_a", True, 2, 0.0)) + monkeypatch.setattr(openclaw_module, "get_redis", lambda: redis) + monkeypatch.setattr(service, "_call_with_fallback", fallback) + + result = await service._call_with_cache("local raw prompt", _raw_alert()) + + assert result[0] == '{"fresh":"ollama"}' + assert result[1] == "ollama_gcp_a" + assert result[3] is False + fallback.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "provider", + ["claude", "gemini", "claude_cached", "gemini:legacy", "claude-shadow"], +) +async def test_openclaw_outer_cache_never_writes_paid_result( + monkeypatch: pytest.MonkeyPatch, + provider: str, +) -> None: + redis = _Redis() + service = object.__new__(OpenClawService) + fallback = AsyncMock(return_value=('{"paid":"response"}', provider, True, 2, 0.01)) + monkeypatch.setattr(openclaw_module, "get_redis", lambda: redis) + monkeypatch.setattr(service, "_call_with_fallback", fallback) + + result = await service._call_with_cache("local raw prompt", _raw_alert()) + + assert result[1] == provider + assert result[3] is False + assert redis.set_calls == [] + + +@pytest.mark.asyncio +async def test_actual_executor_uses_original_prompt_for_ollama_and_sanitized_prompt_for_paid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + trace = _Trace() + attempt_order: list[str] = [] + gcp_a = _Provider( + "ollama_gcp_a", + success=False, + privacy_level="local", + attempt_order=attempt_order, + ) + gcp_b = _Provider( + "ollama_gcp_b", + success=False, + privacy_level="local", + attempt_order=attempt_order, + ) + local = _Provider( + "ollama_local", + success=False, + privacy_level="local", + attempt_order=attempt_order, + ) + claude = _Provider( + "claude", + success=False, + privacy_level="cloud", + attempt_order=attempt_order, + failure_response="paid-claude-response-secret", + failure_error="paid-claude-error-secret", + ) + gemini = _Provider( + "gemini", + success=True, + privacy_level="cloud", + attempt_order=attempt_order, + ) + registry = AIProviderRegistry() + for provider in (gcp_a, gcp_b, local, claude, gemini): + registry.register(provider) + + monkeypatch.setenv("ENABLE_GEMINI", "true") + monkeypatch.setenv("ENABLE_CLAUDE", "true") + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + monkeypatch.setattr( + "src.services.langfuse_client.langfuse_trace", + _trace_factory(trace), + ) + original_prompt = "LOCAL ONLY raw-secret-must-never-reach-cloud" + + with capture_logs() as captured_logs: + result = await AIRouterExecutor(registry).execute( + prompt=original_prompt, + provider_order=[ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ], + context=_prepared_context(), + ) + + assert result.success is True + assert result.provider == "gemini" + assert attempt_order == [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini", + ] + assert [provider.calls[0][0] for provider in (gcp_a, gcp_b, local)] == [ + original_prompt, + original_prompt, + original_prompt, + ] + paid_prompt, paid_context = gemini.calls[0] + assert paid_prompt != original_prompt + assert "raw-secret-must-never-reach-cloud" not in paid_prompt + assert paid_context is not None + assert "message" not in paid_context + assert "labels" not in paid_context + assert "annotations" not in paid_context + assert "raw_logs" not in paid_context + assert "headers" not in paid_context + claude_prompt, claude_context = claude.calls[0] + assert claude_prompt == paid_prompt + assert claude_context == paid_context + paid_trace = next(item for item in trace.generations if item["name"] == "gemini_call") + assert paid_trace["input"] is None + assert paid_trace["output"] is None + assert paid_trace["metadata"]["content_redacted"] is True + assert paid_trace["metadata"]["sanitization_receipt_id"] + assert "raw-secret-must-never-reach-cloud" not in json.dumps(paid_trace) + assert "DockerContainerUnhealthy" not in json.dumps(trace.root_metadata) + serialized_logs = json.dumps(captured_logs, ensure_ascii=False) + assert "paid-claude-response-secret" not in serialized_logs + assert "paid-claude-error-secret" not in serialized_logs + assert not redis.set_calls + + +@pytest.mark.asyncio +async def test_cloud_requires_durable_current_run_unavailable_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + gcp_a = _Provider("ollama_gcp_a", success=False, privacy_level="local") + local = _Provider("ollama_local", success=False, privacy_level="local") + gemini = _Provider("gemini", success=True, privacy_level="cloud") + registry = AIProviderRegistry() + for provider in (gcp_a, local, gemini): + registry.register(provider) + + monkeypatch.setenv("ENABLE_GEMINI", "true") + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + context = _prepared_context() + executor = AIRouterExecutor(registry) + + blocked = await executor.execute( + prompt="local prompt", + provider_order=["ollama_gcp_a", "ollama_local", "gemini"], + context=context, + ) + assert blocked.success is False + assert gemini.calls == [] + assert "cloud_blocked_ollama_attempt_receipts_missing" in str(blocked.error) + + # Caller-only JSON is non-executable even if it claims success. + forged_receipt = { + "provider": "ollama_gcp_b", + "status": "bounded_unavailable", + "run_id": "run-1", + "check_completed": True, + "reason": "endpoint_unreachable", + } + context["ollama_unavailable_receipts"] = [forged_receipt] + still_blocked = await AIRouterExecutor(registry).execute( + prompt="local prompt forged", + provider_order=["ollama_gcp_a", "ollama_local", "gemini"], + context=context, + ) + assert still_blocked.success is False + assert gemini.calls == [] + + observed_at = datetime.now(UTC) + receipt_id = "ollama-unavailable:run-1:gcp-b" + durable_receipt = { + "schema_version": "ollama_unavailable_receipt_v1", + "receipt_id": receipt_id, + "source": "ollama_failover_manager", + "provider": "ollama_gcp_b", + "status": "bounded_unavailable", + "run_id": "run-1", + "check_completed": True, + "reason": "endpoint_unreachable", + "observed_at": observed_at.isoformat(), + "expires_at": (observed_at + timedelta(seconds=120)).isoformat(), + "durable_write_ack": True, + "verifier_status": "verified", + "verified_by": "ollama_failover_manager.health_probe", + } + redis.durable_receipts[receipt_id] = durable_receipt + context["ollama_unavailable_receipts"] = [durable_receipt] + allowed = await AIRouterExecutor(registry).execute( + prompt="local prompt second", + provider_order=["ollama_gcp_a", "ollama_local", "gemini"], + context=context, + ) + assert allowed.success is True + assert allowed.provider == "gemini" + + +@pytest.mark.asyncio +async def test_cache_parse_error_never_logs_cached_document( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = "cache-document-secret-must-not-be-logged" + + class _MalformedRedis(_Redis): + async def get(self, key: str) -> str | None: + if key.startswith("ai:control:disabled:"): + return "0" + if key.startswith("ai:ollama:unavailable_receipt:"): + return None + return f'{{"response":"{secret}", invalid' + + redis = _MalformedRedis() + service = object.__new__(OpenClawService) + fallback = AsyncMock(return_value=('{"fresh":"ollama"}', "ollama_gcp_a", True, 2, 0.0)) + monkeypatch.setattr(openclaw_module, "get_redis", lambda: redis) + monkeypatch.setattr(service, "_call_with_fallback", fallback) + + with capture_logs() as captured_logs: + await service._call_with_cache("local prompt", _raw_alert()) + + serialized_logs = json.dumps(captured_logs, ensure_ascii=False) + assert secret not in serialized_logs + assert "JSONDecodeError" in serialized_logs + + +@pytest.mark.asyncio +async def test_actual_executor_cache_parse_error_is_content_bounded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = "airouter-cache-document-secret-must-not-be-logged" + + class _MalformedRedis(_Redis): + async def get(self, key: str) -> str | None: + if key.startswith("ai:control:disabled:"): + return "0" + if key.startswith("ai:ollama:unavailable_receipt:"): + return None + return f'{{"response":"{secret}", invalid' + + provider = _Provider("ollama_gcp_a", success=True, privacy_level="local") + registry = AIProviderRegistry() + registry.register(provider) + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr( + "src.core.redis_client.get_redis", + lambda: _MalformedRedis(), + ) + + with capture_logs() as captured_logs: + result = await AIRouterExecutor(registry).execute( + prompt="local prompt", + provider_order=["ollama_gcp_a"], + context={"intent_hint": "diagnose"}, + ) + + assert result.success is True + serialized_logs = json.dumps(captured_logs, ensure_ascii=False) + assert secret not in serialized_logs + assert "JSONDecodeError" in serialized_logs + + +@pytest.mark.asyncio +async def test_actual_executor_rejects_self_signed_cloud_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + providers = [ + _Provider("ollama_gcp_a", success=False, privacy_level="local"), + _Provider("ollama_gcp_b", success=False, privacy_level="local"), + _Provider("ollama_local", success=False, privacy_level="local"), + _Provider("gemini", success=True, privacy_level="cloud"), + ] + registry = AIProviderRegistry() + for provider in providers: + registry.register(provider) + context = _prepared_context() + cloud_context = context["paid_cloud_context"] + cloud_context["sanitized_payload"]["target_resource"] = ( + "sk-ant-api03-CALLERFORGEDSECRETVALUE" + ) + canonical = json.dumps( + cloud_context["sanitized_payload"], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + forged_hash = openclaw_module.hashlib.sha256(canonical.encode()).hexdigest() + receipt = cloud_context["cloud_sanitization_receipt"] + receipt["payload_sha256"] = forged_hash + receipt["receipt_id"] = f"openclaw-cloud-sanitize:{forged_hash[:24]}" + + monkeypatch.setenv("ENABLE_GEMINI", "true") + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + result = await AIRouterExecutor(registry).execute( + prompt="local prompt", + provider_order=[ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "gemini", + ], + context=context, + ) + + assert result.success is False + assert providers[-1].calls == [] + assert "alert_cloud_sanitization_receipt_untrusted" in str(result.error) + + +@pytest.mark.asyncio +async def test_ollama_circuit_open_cannot_slide_to_paid_cloud( + monkeypatch: pytest.MonkeyPatch, +) -> None: + redis = _Redis() + providers = [ + _Provider("ollama_gcp_a", success=False, privacy_level="local"), + _Provider("ollama_gcp_b", success=False, privacy_level="local"), + _Provider("ollama_local", success=False, privacy_level="local"), + _Provider("gemini", success=True, privacy_level="cloud"), + ] + registry = AIProviderRegistry() + for provider in providers: + registry.register(provider) + executor = AIRouterExecutor(registry) + circuit = executor._get_circuit_breaker("ollama_gcp_a") + circuit._failure_count = circuit._failure_threshold + circuit._last_failure_time = 10**12 + + monkeypatch.setenv("ENABLE_GEMINI", "true") + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis) + result = await executor.execute( + prompt="local prompt", + provider_order=["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], + context=_prepared_context(), + ) + + assert result.success is False + assert providers[-1].calls == [] + assert "cloud_blocked_ollama_circuit_open" in str(result.error) + + +@pytest.mark.asyncio +async def test_legacy_paid_langfuse_generation_contains_only_hashes_and_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + trace = _Trace() + service = object.__new__(OpenClawService) + monkeypatch.setattr(openclaw_module.settings, "USE_AI_ROUTER", False) + monkeypatch.setattr(openclaw_module.settings, "MOCK_MODE", False) + monkeypatch.setattr( + openclaw_module, + "langfuse_trace", + _trace_factory(trace), + ) + monkeypatch.setattr( + service, + "_resolve_alert_provider_order", + AsyncMock( + return_value=["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"] + ), + ) + monkeypatch.setattr(service, "_call_ollama", AsyncMock(return_value=("local failed", False))) + monkeypatch.setattr( + service, + "_call_gemini", + AsyncMock(return_value=("paid-response-secret", True, 3, 0.01)), + ) + monkeypatch.setattr(service, "_get_model_name", lambda provider: provider) + + result = await service._call_with_fallback( + "LOCAL raw-secret-must-never-reach-cloud", + alert_context=_raw_alert(), + ) + + assert result[1] == "gemini" + paid_trace = next(item for item in trace.generations if item["name"] == "gemini_call") + assert paid_trace["input"] is None + assert paid_trace["output"] is None + serialized = json.dumps(paid_trace) + assert "raw-secret-must-never-reach-cloud" not in serialized + assert "paid-response-secret" not in serialized + assert paid_trace["metadata"]["prompt_sha256"] + assert paid_trace["metadata"]["response_sha256"] + assert "fp-1" not in json.dumps(trace.root_metadata) + + +def test_parse_failure_logs_hash_and_length_not_raw_response( + caplog: pytest.LogCaptureFixture, +) -> None: + service = object.__new__(OpenClawService) + secret = "raw-parse-secret-must-not-be-logged" + + assert service._parse_analysis_result(secret) is None + + assert secret not in caplog.text diff --git a/apps/api/tests/test_phase25_auto_harvesting.py b/apps/api/tests/test_phase25_auto_harvesting.py index 5d79deab4..2db67d19c 100644 --- a/apps/api/tests/test_phase25_auto_harvesting.py +++ b/apps/api/tests/test_phase25_auto_harvesting.py @@ -72,7 +72,7 @@ class TestRunbookGeneratorModule: assert "PUBLISHED" in source or "published" in source def test_runbook_uses_only_global_production_route(self): - """Runbook 不得繞過四跳路由直接呼叫舊 provider。""" + """Runbook 不得繞過五跳路由直接呼叫舊 provider。""" source = _RUNBOOK_GEN.read_text() assert "PRODUCTION_PROVIDER_ORDER" in source assert "get_ai_executor" in source diff --git a/apps/api/tests/test_runbook_ai_provider_route.py b/apps/api/tests/test_runbook_ai_provider_route.py index bafad49b4..5a2ae1568 100644 --- a/apps/api/tests/test_runbook_ai_provider_route.py +++ b/apps/api/tests/test_runbook_ai_provider_route.py @@ -65,6 +65,7 @@ async def test_runbook_generation_uses_exact_chain_and_receipt_correlations( "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] assert captured["context"] == { diff --git a/apps/api/tests/test_sentry_ai_provider_route.py b/apps/api/tests/test_sentry_ai_provider_route.py index 0621455d5..adde0e1d8 100644 --- a/apps/api/tests/test_sentry_ai_provider_route.py +++ b/apps/api/tests/test_sentry_ai_provider_route.py @@ -67,6 +67,7 @@ async def test_sentry_analysis_uses_global_provider_order_and_correlation_ids( "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] assert call["require_local"] is False diff --git a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py index 4961741e8..7da008d4d 100644 --- a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py +++ b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py @@ -33,7 +33,7 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "Canonical Asset Normalize", "Typed Domain Router", "HolmesGPT Investigator", - "Ollama RCA / Gemini Critic", + "Ollama RCA / Claude Fallback / Gemini Critic", "Deterministic Policy", "Single Controlled Executor", "Independent Verifier", @@ -43,11 +43,13 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini", ] + assert payload["provider_policy"]["claude_paid_call_allowed"] is False assert payload["provider_policy"]["gemini_paid_call_allowed"] is False assert payload["provider_policy"]["route_label"] == ( - "GCP-A -> GCP-B -> host111 Ollama -> Gemini API" + "GCP-A -> GCP-B -> host111 Ollama -> Anthropic Claude -> Gemini API" ) assert payload["provider_policy"]["hops"][2] == { "position": 3, @@ -86,6 +88,7 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None: assert projection["rollups"]["runtime_closed_items"] == 0 assert projection["rollups"]["program_completion_percent"] == 0 assert projection["domain_count"] == 7 + assert projection["claude_paid_call_allowed"] is False assert projection["gemini_paid_call_allowed"] is False assert "work_items" not in projection diff --git a/apps/api/tests/test_sre_typed_domain_router.py b/apps/api/tests/test_sre_typed_domain_router.py index 6a6a07013..c926ce5b5 100644 --- a/apps/api/tests/test_sre_typed_domain_router.py +++ b/apps/api/tests/test_sre_typed_domain_router.py @@ -189,6 +189,8 @@ def test_sentry_container_resolves_to_exact_host110_domain_route() -> None: "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", + "gemini", ] assert route["reasoning_contract"]["paid_call_allowed"] is False diff --git a/capabilities.json b/capabilities.json index b6994f814..c6a84d35e 100644 --- a/capabilities.json +++ b/capabilities.json @@ -134,8 +134,8 @@ "ai_providers": { "schema_version": "ai_provider_production_route_v1", - "fallback_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "gemini"], - "route_label": "GCP-A -> GCP-B -> host111 Ollama -> Gemini API", + "fallback_order": ["ollama_gcp_a", "ollama_gcp_b", "ollama_local", "claude", "gemini"], + "route_label": "GCP-A -> GCP-B -> host111 Ollama -> Anthropic Claude API -> Gemini API", "router_toggle": "USE_AI_ROUTER", "providers": [ { @@ -165,6 +165,18 @@ "production_executable": true, "description": "全域 production 第三跳" }, + { + "name": "claude", + "identity": "Anthropic Claude API", + "model": "claude-sonnet-5", + "input_cost_per_1m_tokens_usd": 3.0, + "output_cost_per_1m_tokens_usd": 15.0, + "timeout_seconds": 30, + "production_executable": true, + "default_execution_mode": "shadow", + "execution_gate": "configured_and_enabled_and_mode_selected_and_atomic_cost_guard_reserved", + "description": "Claude Sonnet 5 — 第四跳 SRE/架構付費備援，預設 shadow/no-call" + }, { "name": "gemini", "identity": "Gemini API", @@ -177,7 +189,6 @@ } ], "shadow_metadata_only": [ - {"name": "claude", "production_executable": false}, {"name": "nvidia", "production_executable": false}, {"name": "nemotron", "production_executable": false}, {"name": "openclaw_nemo", "production_executable": false}, diff --git a/docs/evaluations/ai_provider_route_matrix_2026-06-05.json b/docs/evaluations/ai_provider_route_matrix_2026-06-05.json index b529ed7b5..8bd701bbd 100644 --- a/docs/evaluations/ai_provider_route_matrix_2026-06-05.json +++ b/docs/evaluations/ai_provider_route_matrix_2026-06-05.json @@ -27,23 +27,23 @@ "docs/ai/AI_AGENT_AUTOMATION_WORKLIST_2026-06-04.md" ], "rollups": { - "total_routes": 7, + "total_routes": 8, "by_kind": { "ai_router_core": 1, "ollama_failover": 1, "alert_governance_lane": 1, "openclaw_nemo": 1, "nemotron_candidate": 1, - "paid_cloud_fallback": 1, + "paid_cloud_fallback": 2, "legacy_registry": 1 }, "by_status": { - "verified": 5, + "verified": 6, "action_required": 1, "blocked": 1 }, "by_route_gate": { - "route_preserved": 5, + "route_preserved": 6, "review_required": 1, "candidate_blocked": 1 }, @@ -75,11 +75,12 @@ "risk_level": "critical", "route_gate": "route_preserved", "evidence_status": "committed_source", - "current_policy": "AIRouter 的 intent、risk、complexity 只選 Ollama model；所有 production generation 共用 GCP-A → GCP-B → host111 → Gemini 四跳，不得直選 Claude / NVIDIA / Nemo / OpenClaw。", + "current_policy": "AIRouter 的 intent、risk、complexity 只選 Ollama model；所有 production generation 共用 GCP-A → GCP-B → host111 → Claude → Gemini 五跳，不得直選 NVIDIA / Nemo / OpenClaw。", "provider_order": [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini" ], "fallback_policy": "所有 risk / complexity / tool / background / code-review / runbook 路徑共用單一 production chain；Gemini disabled 時不得被重新加入。", @@ -96,11 +97,12 @@ "risk_level": "critical", "route_gate": "route_preserved", "evidence_status": "committed_manifest", - "current_policy": "生產 ConfigMap 與 deployment env 維持 OLLAMA_URL、OLLAMA_SECONDARY_URL、OLLAMA_FALLBACK_URL；policy order 是 GCP-A → GCP-B → 111，再到 Gemini final fallback。", + "current_policy": "生產 ConfigMap 與 deployment env 維持 OLLAMA_URL、OLLAMA_SECONDARY_URL、OLLAMA_FALLBACK_URL；policy order 是 GCP-A → GCP-B → 111，再到 Claude 與 Gemini paid fallback。", "provider_order": [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini" ], "fallback_policy": "OllamaFailoverManager 只在 primary 不健康時檢查後兩層；SLOW Ollama 仍優先於直接燒 Gemini quota。", @@ -120,11 +122,12 @@ "risk_level": "critical", "route_gate": "route_preserved", "evidence_status": "committed_source", - "current_policy": "告警與 AI governance lane 固定走 GCP-A / GCP-B / host111；cloud fallback 只允許 Gemini final hop。", + "current_policy": "告警與 AI governance lane 固定走 GCP-A / GCP-B / host111；cloud fallback 只允許受成本閘控制的 Claude 再 Gemini。", "provider_order": [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini" ], "fallback_policy": "ALERT_AI_ENFORCE_OLLAMA_FIRST=true；ALERT_AI_ALLOW_CLOUD_FALLBACK=true 只代表既有 final fallback，不代表新增付費呼叫批准。", @@ -148,9 +151,10 @@ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini" ], - "fallback_policy": "Production fallback 不含 OpenClaw/Nemo/NVIDIA/Claude；候選只可離線 replay。", + "fallback_policy": "Production fallback 不含 OpenClaw/Nemo/NVIDIA；候選只可離線 replay。", "evidence_refs": [ "apps/api/src/services/ai_providers/openclaw_nemo.py", "apps/api/src/services/ai_router.py", @@ -179,6 +183,31 @@ ], "next_action": "P3 才刷新 primary source evidence 與 5 筆 smoke；不得把 P1-004 matrix 當作 Nemotron 升級批准。" }, + { + "route_id": "claude_paid_fallback_policy", + "display_name": "Claude SRE / Architecture Paid Fallback", + "kind": "paid_cloud_fallback", + "status": "verified", + "risk_level": "critical", + "route_gate": "route_preserved", + "evidence_status": "committed_source", + "current_policy": "Claude Sonnet 5 位於三層 Ollama 之後、Gemini 之前；預設 shadow/no-call，只有 persistent enable、deterministic canary/production mode、auth readback 與原子成本 reservation 同時成立才可執行。", + "provider_order": [ + "ollama_gcp_a", + "ollama_gcp_b", + "ollama_local", + "claude", + "gemini" + ], + "fallback_policy": "Claude 使用獨立 RPM/request/token/daily/total cost cap、per-provider circuit breaker 與 durable receipt；定價未知、Redis 不可用、correlation 缺漏或 disable-state 不可證明時 no-write fail closed。", + "evidence_refs": [ + "apps/api/src/services/ai_providers/claude.py", + "apps/api/src/services/ai_rate_limiter.py", + "apps/api/src/services/ai_control.py", + "apps/api/models.json" + ], + "next_action": "完成 source/test 後仍須 secret reference readback、5 筆 sanitized replay、shadow scorecard、bounded canary 與 production receipt；不得從 source 綠燈宣稱 runtime closure。" + }, { "route_id": "gemini_final_fallback_policy", "display_name": "Gemini Final Fallback", @@ -187,11 +216,12 @@ "risk_level": "critical", "route_gate": "route_preserved", "evidence_status": "committed_source", - "current_policy": "Gemini 是三層 Ollama 之後唯一 final paid fallback；configured、enabled、authenticated、verified 與 cost guard 必須分開顯示。", + "current_policy": "Gemini 是三層 Ollama 與 Claude 之後的 final paid fallback；Claude/Gemini 的 configured、enabled、authenticated、verified 與 cost guard 必須分開顯示。", "provider_order": [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini" ], "fallback_policy": "Gemini 只能由原子 requests/tokens/cost reservation 放行；disabled 不重加，unknown model/pricing 必須 durable blocked receipt。", @@ -211,11 +241,12 @@ "risk_level": "medium", "route_gate": "route_preserved", "evidence_status": "committed_source", - "current_policy": "apps/api/models.json 已發布四跳 production route；Claude/NVIDIA/Nemo/OpenClaw 僅保留 non-executable shadow metadata。", + "current_policy": "apps/api/models.json 已發布五跳 production route；Claude 是 gated paid fallback，NVIDIA/Nemo/OpenClaw 僅保留 non-executable shadow metadata。", "provider_order": [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini" ], "fallback_policy": "Registry、ConfigMap、AI Router source 與 runtime readback 必須一致；source-only 仍不得宣稱 production closure。", @@ -271,7 +302,7 @@ "display_name": "AI Router 註解與現況 drift", "status": "resolved_in_source", "severity": "low", - "summary": "ai_router.py 現行說明已明確標示舊版路由 superseded，provider 固定四跳。", + "summary": "ai_router.py 現行說明已明確標示舊版路由 superseded，provider 固定五跳。", "evidence_refs": [ "apps/api/src/services/ai_router.py" ], @@ -282,7 +313,7 @@ "display_name": "models.json 與 runtime env 判讀差異", "status": "resolved_in_source", "severity": "low", - "summary": "models.json、ConfigMap 與 AI Router 已共用四跳 provider identity；endpoint URL 仍以 runtime env 與 production readback 為準。", + "summary": "models.json、ConfigMap 與 AI Router 已共用五跳 provider identity；endpoint URL 仍以 runtime env 與 production readback 為準。", "evidence_refs": [ "apps/api/models.json", "k8s/awoooi-prod/06-deployment-api.yaml" @@ -365,6 +396,7 @@ "shadow_or_canary_allowed": false, "openclaw_replacement_allowed": false, "nemotron_shadow_allowed": false, + "claude_direct_call_allowed": false, "gemini_direct_call_allowed": false, "secret_read_allowed": false, "secret_plaintext_allowed": false, diff --git a/docs/evaluations/service_health_gap_matrix_2026-06-05.json b/docs/evaluations/service_health_gap_matrix_2026-06-05.json index 1337590d7..52fa73577 100644 --- a/docs/evaluations/service_health_gap_matrix_2026-06-05.json +++ b/docs/evaluations/service_health_gap_matrix_2026-06-05.json @@ -121,7 +121,7 @@ "status": "action_required", "risk_level": "critical", "freshness_status": "source_mismatch", - "health_contract": "目前 AI provider route truth 是 GCP-A → GCP-B → 111；Gemini 只作 final fallback。", + "health_contract": "目前 AI provider route truth 是 GCP-A → GCP-B → 111 → Claude → Gemini；付費 provider 只能在三層 Ollama 後依序作受控 fallback。", "endpoint_contract": "`SERVICE-ENDPOINTS.md` 仍有 188 Ollama 參考，`scripts/health_check_session.sh` 又檢查 111 / 110；需把 endpoint truth 收斂為 owner-attested source，不可直接改路由。", "evidence_refs": [ "docs/reference/SERVICE-ENDPOINTS.md", diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index cbb4f9c05..672121bc3 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -17,7 +17,7 @@ "Canonical Asset Normalize", "Typed Domain Router", "HolmesGPT Investigator", - "Ollama RCA / Gemini Critic", + "Ollama RCA / Claude Fallback / Gemini Critic", "Deterministic Policy", "Single Controlled Executor", "Independent Verifier", @@ -37,9 +37,10 @@ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", + "claude", "gemini" ], - "route_label": "GCP-A -> GCP-B -> host111 Ollama -> Gemini API", + "route_label": "GCP-A -> GCP-B -> host111 Ollama -> Anthropic Claude -> Gemini API", "hops": [ { "position": 1, @@ -58,17 +59,26 @@ }, { "position": 4, + "provider": "claude", + "identity": "Anthropic Claude API" + }, + { + "position": 5, "provider": "gemini", "identity": "Gemini API" } ], "ollama_role": "primary_rca_and_action_planning", + "claude_role": "paid_architecture_rca_and_debug_fallback", "gemini_role": "final_fallback_and_optional_critic_only", + "claude_paid_call_allowed": false, "gemini_paid_call_allowed": false, - "gemini_cost_cap_status": "explicit_cost_cap_not_approved", + "claude_cost_cap_status": "source_configured_shadow_runtime_readback_pending", + "gemini_cost_cap_status": "source_configured_runtime_readback_pending", + "claude_credential_status": "runtime_secret_reference_verifier_pending_no_secret_value_read", "gemini_credential_status": "runtime_secret_reference_verifier_pending_no_secret_value_read", "production_provider_route_switch_allowed": false, - "required_before_gemini_enablement": [ + "required_before_paid_provider_enablement": [ "offline_replay_scorecard", "shadow_comparison", "bounded_canary", @@ -415,7 +425,7 @@ "order": 13, "priority": "P0", "phase": "phase_3", - "title": "Ollama RCA 與 Gemini Critic/cost gate", + "title": "Ollama RCA、Claude fallback 與 Gemini Critic/cost gate", "owner_lane": "ModelRouter", "risk": "critical", "status": "in_progress", @@ -424,13 +434,13 @@ ], "source_refs": [ "apps/api/src/services/ai_router.py", - "apps/api/src/services/ollama_failover.py" + "apps/api/src/services/ollama_failover_manager.py" ], "executor": "ordered model router", "verifier": "provider-attributed replay accuracy, latency and cost readback", - "rollback": "Gemini disabled; remain on ordered Ollama chain", - "exit_condition": "provider order is enforced and Gemini cannot run without explicit cost cap", - "next_action": "verify secret metadata presence without reading value, then build offline comparison" + "rollback": "Claude/Gemini disabled; remain on ordered Ollama chain", + "exit_condition": "provider order is enforced and paid providers cannot run without explicit cost cap", + "next_action": "deploy the disabled-by-default source route, verify secret metadata without reading values, then run replay, shadow comparison and bounded canary" }, { "id": "AIA-SRE-014", diff --git a/k8s/awoooi-dev/02-configmap.yaml b/k8s/awoooi-dev/02-configmap.yaml index bc30e3220..cf28afacd 100644 --- a/k8s/awoooi-dev/02-configmap.yaml +++ b/k8s/awoooi-dev/02-configmap.yaml @@ -33,7 +33,7 @@ data: LOG_LEVEL: "DEBUG" CORS_ORIGINS: '["http://localhost:3000","http://192.168.0.121:32344","http://192.168.0.125:32344"]' - AI_FALLBACK_ORDER: '["ollama_gcp_a","ollama_gcp_b","ollama_local","gemini"]' + AI_FALLBACK_ORDER: '["ollama_gcp_a","ollama_gcp_b","ollama_local","claude","gemini"]' GEMINI_MODEL: "gemini-2.5-flash-lite" GEMINI_DAILY_QUOTA: "500" GEMINI_RPM_LIMIT: "10" @@ -44,6 +44,21 @@ data: GEMINI_MAX_OUTPUT_TOKENS: "2048" # Historical key: auth-status receipt TTL only. Generation/run receipts are durable. GEMINI_USAGE_RECEIPT_TTL_SECONDS: "604800" + GEMINI_FAILURE_COOLDOWN_SECONDS: "60" + ENABLE_GEMINI: "false" + CLAUDE_MODEL: "claude-sonnet-5" + CLAUDE_DAILY_QUOTA: "50" + CLAUDE_RPM_LIMIT: "2" + CLAUDE_DAILY_TOKEN_LIMIT: "100000" + CLAUDE_DAILY_COST_LIMIT_USD: "1.0" + CLAUDE_TOTAL_COST_LIMIT_USD: "5.0" + CLAUDE_COST_ALERT_THRESHOLD_USD: "4.0" + CLAUDE_MAX_OUTPUT_TOKENS: "4096" + CLAUDE_USAGE_RECEIPT_TTL_SECONDS: "604800" + CLAUDE_FAILURE_COOLDOWN_SECONDS: "120" + CLAUDE_EXECUTION_MODE: "shadow" + CLAUDE_CANARY_PERCENT: "0" + ENABLE_CLAUDE: "false" AI_CACHE_TTL: "300" ENABLE_NEMOTRON_COLLABORATION: "true" diff --git a/k8s/awoooi-prod/03-secrets.example.yaml b/k8s/awoooi-prod/03-secrets.example.yaml index 42d920e69..847368088 100644 --- a/k8s/awoooi-prod/03-secrets.example.yaml +++ b/k8s/awoooi-prod/03-secrets.example.yaml @@ -28,17 +28,13 @@ stringData: REDIS_URL: "redis://192.168.0.188:6380/0" # ============================================================================ - # AI 服務 API Keys (ADR-006 備援順序: Ollama → Gemini → Claude) + # AI 服務 API Keys + # 固定路由: GCP-A → GCP-B → host111 Ollama → Anthropic Claude → Gemini + # Secret 僅由受控 runtime 注入；不得把真值提交到 Git。 # ============================================================================ GEMINI_API_KEY: "CHANGE_ME" CLAUDE_API_KEY: "CHANGE_ME" - # ============================================================================ - # Phase 9: Agent Teams (ADR-009) - # Claude Agent SDK 需要 ANTHROPIC_API_KEY - # ============================================================================ - ANTHROPIC_API_KEY: "CHANGE_ME" - # ============================================================================ # Phase 5.5: Telegram Gateway (OpenClaw 通知) # ============================================================================ diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index abf03a475..56c4cba1b 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -19,7 +19,7 @@ data: # 188 = CPU-only Ollama，推理極慢（>60s）；111 有 GPU，avg 10s # 2026-05-25 Codex: keep ADR-110 production policy visible and ordered. # High-volume callers use short endpoint cooldowns when GCP-A/B are down, - # but health/status must still show GCP-A -> GCP-B -> 111 before Gemini. + # but health/status must still show GCP-A -> GCP-B -> 111 -> Claude -> Gemini. OLLAMA_URL: "http://192.168.0.110:11435" OLLAMA_SECONDARY_URL: "http://192.168.0.110:11436" OLLAMA_FALLBACK_URL: "http://192.168.0.110:11437" @@ -51,9 +51,9 @@ data: DATABASE_POOL_TIMEOUT_SECONDS: "5" # AI 配置 (JSON array 格式 for pydantic-settings) - # Alert/incident runtime is strictly GCP-A -> GCP-B -> host111 -> Gemini. - # Gemini is the final paid fallback; NVIDIA/Nemo/Claude cannot enter this chain. - AI_FALLBACK_ORDER: '["ollama_gcp_a","ollama_gcp_b","ollama_local","gemini"]' + # Alert/incident runtime is strictly GCP-A -> GCP-B -> host111 -> Claude -> Gemini. + # Both paid lanes require persistent runtime enablement and atomic cost receipts. + AI_FALLBACK_ORDER: '["ollama_gcp_a","ollama_gcp_b","ollama_local","claude","gemini"]' AI_CACHE_TTL: "3600" # ============================================================================ @@ -83,6 +83,21 @@ data: GEMINI_USAGE_RECEIPT_TTL_SECONDS: "604800" GEMINI_FAILURE_COOLDOWN_SECONDS: "60" ENABLE_GEMINI: "false" + # Claude Sonnet 5 is source-integrated but starts shadow/no-call. Promotion is + # shadow -> deterministic canary -> production, with an independent budget. + CLAUDE_MODEL: "claude-sonnet-5" + CLAUDE_DAILY_QUOTA: "50" + CLAUDE_RPM_LIMIT: "2" + CLAUDE_DAILY_TOKEN_LIMIT: "100000" + CLAUDE_DAILY_COST_LIMIT_USD: "1.0" + CLAUDE_TOTAL_COST_LIMIT_USD: "5.0" + CLAUDE_COST_ALERT_THRESHOLD_USD: "4.0" + CLAUDE_MAX_OUTPUT_TOKENS: "4096" + CLAUDE_USAGE_RECEIPT_TTL_SECONDS: "604800" + CLAUDE_FAILURE_COOLDOWN_SECONDS: "120" + CLAUDE_EXECUTION_MODE: "shadow" + CLAUDE_CANARY_PERCENT: "0" + ENABLE_CLAUDE: "false" OPENCLAW_DEFAULT_MODEL: "qwen2.5:7b-instruct" OPENCLAW_TIMEOUT: "120" OLLAMA_DIAGNOSE_TIMEOUT_SECONDS: "300" diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 6b217c4bb..a4aa18e96 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -94,9 +94,6 @@ spec: - name: CLAUDE_API_KEY valueFrom: secretKeyRef: {name: awoooi-secrets, key: CLAUDE_API_KEY, optional: true} - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: {name: awoooi-secrets, key: ANTHROPIC_API_KEY, optional: true} - name: NVIDIA_API_KEY valueFrom: secretKeyRef: {name: awoooi-secrets, key: NVIDIA_API_KEY, optional: true} diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 75baccc43..24ebe6249 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -91,9 +91,6 @@ spec: - name: CLAUDE_API_KEY valueFrom: secretKeyRef: {name: awoooi-secrets, key: CLAUDE_API_KEY, optional: true} - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: {name: awoooi-secrets, key: ANTHROPIC_API_KEY, optional: true} - name: NVIDIA_API_KEY valueFrom: secretKeyRef: {name: awoooi-secrets, key: NVIDIA_API_KEY, optional: true}