diff --git a/apps/api/apps/api/src/repositories/__init__.py b/apps/api/apps/api/src/repositories/__init__.py new file mode 100644 index 000000000..22e441219 --- /dev/null +++ b/apps/api/apps/api/src/repositories/__init__.py @@ -0,0 +1,26 @@ +""" +Repository Layer - 資料存取抽象層 +================================== +Phase 16 R3: 抽取 Repository 層 + +設計原則: +- Protocol 介面定義 (DI 用) +- CRUD 操作封裝 +- Service 層不直接碰 DB/Redis + +版本: v1.0 +建立: 2026-03-26 (台北時區) +建立者: Claude Code (Phase 16 架構重構) +""" + +from src.repositories.interfaces import ( + IApprovalRepository, + IIncidentRepository, + ITimelineRepository, +) + +__all__ = [ + "IApprovalRepository", + "IIncidentRepository", + "ITimelineRepository", +] diff --git a/apps/api/apps/api/src/repositories/interfaces.py b/apps/api/apps/api/src/repositories/interfaces.py new file mode 100644 index 000000000..08a1d4fb1 --- /dev/null +++ b/apps/api/apps/api/src/repositories/interfaces.py @@ -0,0 +1,127 @@ +""" +Repository Interfaces - Protocol 定義 +====================================== +Phase 16 R3: Repository 層 Protocol 介面 + +設計原則: +- 使用 Python Protocol 實現 DI +- Service 層只依賴 Protocol，不依賴具體實作 +- 便於測試 (可注入 Mock Repository) + +版本: v1.0 +建立: 2026-03-26 (台北時區) +建立者: Claude Code (Phase 16 架構重構) +""" + +from typing import Protocol, runtime_checkable +from uuid import UUID + +from src.models.approval import ApprovalRequest, ApprovalRequestCreate, ApprovalStatus +from src.models.incident import Incident + + +@runtime_checkable +class IApprovalRepository(Protocol): + """ + Approval Repository Protocol + + 職責: ApprovalRecord CRUD 操作 + 實作: ApprovalDBRepository (PostgreSQL) + """ + + async def create(self, request: ApprovalRequestCreate) -> ApprovalRequest: + """建立新的 Approval""" + ... + + async def get_by_id(self, approval_id: UUID) -> ApprovalRequest | None: + """根據 ID 取得 Approval""" + ... + + async def get_pending(self) -> list[ApprovalRequest]: + """取得所有待審核的 Approval""" + ... + + async def update_status( + self, + approval_id: UUID, + status: ApprovalStatus, + actor: str | None = None, + ) -> ApprovalRequest | None: + """更新 Approval 狀態""" + ... + + async def add_signature( + self, + approval_id: UUID, + signer: str, + decision: str, + comment: str | None = None, + ) -> ApprovalRequest | None: + """新增簽核""" + ... + + +@runtime_checkable +class IIncidentRepository(Protocol): + """ + Incident Repository Protocol + + 職責: IncidentRecord CRUD 操作 + 實作: IncidentDBRepository (PostgreSQL) + """ + + async def create(self, incident: Incident) -> Incident: + """建立新的 Incident""" + ... + + async def get_by_id(self, incident_id: str) -> Incident | None: + """根據 ID 取得 Incident""" + ... + + async def get_active(self) -> list[Incident]: + """取得所有活躍的 Incident""" + ... + + async def update(self, incident: Incident) -> Incident | None: + """更新 Incident""" + ... + + async def upsert(self, incident: Incident) -> bool: + """Upsert Incident (存在則更新，不存在則建立)""" + ... + + +@runtime_checkable +class ITimelineRepository(Protocol): + """ + Timeline Repository Protocol + + 職責: TimelineEvent CRUD 操作 + 實作: TimelineDBRepository (PostgreSQL) + """ + + async def add_event( + self, + approval_id: UUID, + event_type: str, + actor: str, + details: dict | None = None, + ) -> bool: + """新增 Timeline 事件""" + ... + + async def get_events( + self, + approval_id: UUID, + limit: int = 100, + ) -> list[dict]: + """取得 Approval 的 Timeline 事件""" + ... + + async def get_recent_events( + self, + limit: int = 50, + hours: int = 24, + ) -> list[dict]: + """取得最近的 Timeline 事件""" + ... diff --git a/apps/api/migrations/phase18_audit_log_failure_loop.sql b/apps/api/migrations/phase18_audit_log_failure_loop.sql new file mode 100644 index 000000000..c5d892c25 --- /dev/null +++ b/apps/api/migrations/phase18_audit_log_failure_loop.sql @@ -0,0 +1,55 @@ +-- ============================================================================= +-- Phase 18: 失敗自動修復閉環 - AuditLog 表擴展 +-- ============================================================================= +-- 執行日期: 2026-03-26 +-- 執行者: 首席架構師 +-- ADR: ADR-023-failure-auto-repair-loop.md +-- ============================================================================= + +-- 新增授權來源追蹤欄位 +ALTER TABLE audit_logs +ADD COLUMN IF NOT EXISTS authorization_channel VARCHAR(20); + +COMMENT ON COLUMN audit_logs.authorization_channel IS 'Authorization source: web, telegram, auto'; + +-- 新增重試與修復追蹤欄位 +ALTER TABLE audit_logs +ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0 NOT NULL; + +COMMENT ON COLUMN audit_logs.retry_count IS 'Number of retry attempts'; + +ALTER TABLE audit_logs +ADD COLUMN IF NOT EXISTS failure_classification VARCHAR(50); + +COMMENT ON COLUMN audit_logs.failure_classification IS 'Failure type: TIMEOUT, K8S_ERROR, NETWORK_ERROR, PERMISSION_DENIED'; + +ALTER TABLE audit_logs +ADD COLUMN IF NOT EXISTS source_approval_id VARCHAR(36); + +COMMENT ON COLUMN audit_logs.source_approval_id IS 'Original approval ID if this is a repair attempt'; + +-- 新增自動修復狀態欄位 +ALTER TABLE audit_logs +ADD COLUMN IF NOT EXISTS auto_repair_attempted BOOLEAN DEFAULT FALSE NOT NULL; + +COMMENT ON COLUMN audit_logs.auto_repair_attempted IS 'Whether auto-repair was attempted'; + +ALTER TABLE audit_logs +ADD COLUMN IF NOT EXISTS auto_repair_result TEXT; + +COMMENT ON COLUMN audit_logs.auto_repair_result IS 'Auto-repair result: AI analysis and repair outcome'; + +-- 建立索引 +CREATE INDEX IF NOT EXISTS ix_audit_authorization_channel ON audit_logs(authorization_channel); +CREATE INDEX IF NOT EXISTS ix_audit_failure_classification ON audit_logs(failure_classification); +CREATE INDEX IF NOT EXISTS ix_audit_source_approval_id ON audit_logs(source_approval_id); + +-- ============================================================================= +-- 驗證 +-- ============================================================================= +-- 執行以下查詢確認欄位已建立: +-- SELECT column_name, data_type, is_nullable +-- FROM information_schema.columns +-- WHERE table_name = 'audit_logs' +-- AND column_name IN ('authorization_channel', 'retry_count', 'failure_classification', +-- 'source_approval_id', 'auto_repair_attempted', 'auto_repair_result'); diff --git a/apps/api/scripts/demo_multisig.py b/apps/api/scripts/demo_multisig.py index 3abfad647..63ee7e46e 100644 --- a/apps/api/scripts/demo_multisig.py +++ b/apps/api/scripts/demo_multisig.py @@ -5,7 +5,7 @@ CISO-101 Multi-Sig Demo Script 展示 CRITICAL 任務從發起到完成的完整信任鏈生命週期 流程: -1. ClawBot 發起 CRITICAL 操作 (DROP TABLE) +1. OpenClaw 發起 CRITICAL 操作 (DROP TABLE) 2. 第一位簽核者簽核 → 仍為 PENDING (1/2) 3. 第二位簽核者簽核 → 轉為 APPROVED → 觸發執行 @@ -67,7 +67,7 @@ def main(): print(""" This demo shows the complete CRITICAL approval lifecycle: - 1. ClawBot initiates a CRITICAL operation (DROP TABLE) + 1. OpenClaw initiates a CRITICAL operation (DROP TABLE) 2. First signer signs → Still PENDING (1/2) 3. Second signer signs → APPROVED → Execution triggered """) @@ -91,7 +91,7 @@ def main(): # ========================================================================== # Step 1: Create CRITICAL approval request # ========================================================================== - print_header("Step 1: ClawBot Initiates CRITICAL Operation") + print_header("Step 1: OpenClaw Initiates CRITICAL Operation") # Track approved requests approved_requests = [] @@ -124,14 +124,14 @@ def main(): DryRunCheck(name="Syntax Check", passed=True), DryRunCheck(name="Backup Available", passed=False, message="No recent backup!"), ], - requested_by="ClawBot", + requested_by="OpenClaw", expires_at=datetime.now(UTC) + timedelta(hours=1), ) approval = engine.create_approval(request) print(f""" - ClawBot 發起 CRITICAL 操作請求: + OpenClaw 發起 CRITICAL 操作請求: 動作: {request.action} 描述: {request.description} @@ -227,7 +227,7 @@ def main(): dry_run_checks=[ DryRunCheck(name="Resource Check", passed=True, message="5/20 pods"), ], - requested_by="ClawBot", + requested_by="OpenClaw", ) low_approval = engine.create_approval(low_request) diff --git a/apps/api/scripts/e2e_tool_call_verification.py b/apps/api/scripts/e2e_tool_call_verification.py index 753b2af31..f6801c869 100644 --- a/apps/api/scripts/e2e_tool_call_verification.py +++ b/apps/api/scripts/e2e_tool_call_verification.py @@ -40,7 +40,6 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "src")) import httpx - # ============================================================================= # Config # ============================================================================= @@ -318,7 +317,7 @@ class E2EVerification: found = any(a.get("id") == self.approval_id for a in approvals) if found: - print_success(f"Approval 在 pending 列表中") + print_success("Approval 在 pending 列表中") return True else: print_warn("Approval 不在 pending 列表 (可能已處理)") @@ -356,7 +355,7 @@ class E2EVerification: # Phase 18.2.2: 動態簽署 for i in range(min(remaining, len(SIGNER_POOL))): signer = SIGNER_POOL[i] - print_info(f"Signing with", signer["name"]) + print_info("Signing with", signer["name"]) async with httpx.AsyncClient(timeout=TIMEOUT) as client: response = await client.post( diff --git a/apps/api/scripts/fire_test_alert.py b/apps/api/scripts/fire_test_alert.py index ac4e2bf41..fde307844 100644 --- a/apps/api/scripts/fire_test_alert.py +++ b/apps/api/scripts/fire_test_alert.py @@ -2,10 +2,10 @@ """ 🚀 AWOOOI Phase 2 導彈腳本 - fire_test_alert.py =============================================== -向系統注入模擬告警，觸發 ClawBot AI 分析流程 +向系統注入模擬告警，觸發 OpenClaw AI 分析流程 用途: -- 驗證全鏈路 (Webhook → ClawBot → ApprovalCard) +- 驗證全鏈路 (Webhook → OpenClaw → ApprovalCard) - 測試戰情室前端是否即時彈出授權卡片 - 開發除錯用 (無需真實監控系統) diff --git a/apps/api/src/api/v1/audit_logs.py b/apps/api/src/api/v1/audit_logs.py index 7a2ead249..a2bf2057f 100644 --- a/apps/api/src/api/v1/audit_logs.py +++ b/apps/api/src/api/v1/audit_logs.py @@ -43,6 +43,13 @@ class AuditLogResponse(BaseModel): dry_run_passed: bool dry_run_message: str | None created_at: str + # Phase 18: 失敗自動修復閉環欄位 + authorization_channel: str | None = None + retry_count: int = 0 + failure_classification: str | None = None + source_approval_id: str | None = None + auto_repair_attempted: bool = False + auto_repair_result: str | None = None class AuditLogListResponse(BaseModel): @@ -86,6 +93,13 @@ def audit_log_to_response(log: AuditLog) -> AuditLogResponse: dry_run_passed=log.dry_run_passed, dry_run_message=log.dry_run_message, created_at=log.created_at.isoformat() if log.created_at else "", + # Phase 18: 失敗自動修復閉環欄位 + authorization_channel=log.authorization_channel, + retry_count=log.retry_count or 0, + failure_classification=log.failure_classification, + source_approval_id=log.source_approval_id, + auto_repair_attempted=log.auto_repair_attempted or False, + auto_repair_result=log.auto_repair_result, ) diff --git a/apps/api/src/api/v1/auto_repair.py b/apps/api/src/api/v1/auto_repair.py new file mode 100644 index 000000000..84a3aa6b1 --- /dev/null +++ b/apps/api/src/api/v1/auto_repair.py @@ -0,0 +1,199 @@ +""" +Auto Repair API Router - #8 自動升級決策 +======================================== +自動修復評估與執行 API 端點 + +Phase 8.2: API Router 實作 +建立時間: 2026-03-26 17:45 (台北時區) +建立者: Claude Code (#8 自動升級決策) + +遵循 leWOOOgo 積木化原則: +- Router 層只做 HTTP 轉發 +- 不直接存取 Redis/DB +- 業務邏輯委託給 Service 層 +""" + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from src.services.auto_repair_service import ( + get_auto_repair_service, +) +from src.services.incident_service import get_incident_service + +router = APIRouter(prefix="/auto-repair", tags=["Auto Repair"]) + + +# ============================================================================= +# Response Models +# ============================================================================= + + +class EvaluateResponse(BaseModel): + """評估自動修復回應""" + + can_auto_repair: bool = Field(description="是否可自動修復") + playbook_id: str | None = Field(None, description="匹配的 Playbook ID") + playbook_name: str | None = Field(None, description="Playbook 名稱") + reason: str = Field(description="決策原因") + risk_level: str = Field(default="MEDIUM", description="風險等級") + blocked_by: str | None = Field(None, description="阻擋原因代碼") + success_rate: float | None = Field(None, description="Playbook 成功率") + total_executions: int | None = Field(None, description="Playbook 執行次數") + + +class ExecuteRequest(BaseModel): + """執行自動修復請求""" + + incident_id: str = Field(description="Incident ID") + playbook_id: str = Field(description="Playbook ID") + force: bool = Field( + default=False, + description="強制執行 (跳過品質檢查，僅用於測試)", + ) + + +class ExecuteResponse(BaseModel): + """執行自動修復回應""" + + success: bool = Field(description="是否執行成功") + incident_id: str = Field(description="Incident ID") + playbook_id: str = Field(description="Playbook ID") + executed_steps: list[str] = Field(description="已執行的步驟") + error: str | None = Field(None, description="錯誤訊息") + execution_time_ms: int = Field(description="執行時間 (毫秒)") + + +# ============================================================================= +# API Endpoints +# ============================================================================= + + +@router.get("/evaluate/{incident_id}", response_model=EvaluateResponse) +async def evaluate_auto_repair(incident_id: str) -> EvaluateResponse: + """ + 評估 Incident 是否可自動修復 + + 決策條件: + 1. 有匹配的高品質 Playbook (成功率 >= 95%, 執行次數 >= 10) + 2. 動作風險等級 <= MEDIUM + 3. Incident 嚴重度 <= P2 + """ + # 取得 Incident + incident_service = get_incident_service() + incident = await incident_service.get_incident(incident_id) + + if not incident: + raise HTTPException( + status_code=404, + detail=f"Incident {incident_id} not found", + ) + + # 評估 + auto_repair_service = get_auto_repair_service() + decision = await auto_repair_service.evaluate_auto_repair(incident) + + return EvaluateResponse( + can_auto_repair=decision.can_auto_repair, + playbook_id=decision.playbook.playbook_id if decision.playbook else None, + playbook_name=decision.playbook.name if decision.playbook else None, + reason=decision.reason, + risk_level=decision.risk_level.value if decision.risk_level else "MEDIUM", + blocked_by=decision.blocked_by, + success_rate=decision.playbook.success_rate if decision.playbook else None, + total_executions=decision.playbook.total_executions if decision.playbook else None, + ) + + +@router.post("/execute", response_model=ExecuteResponse) +async def execute_auto_repair(request: ExecuteRequest) -> ExecuteResponse: + """ + 執行自動修復 + + 安全邊界: + - 僅執行 LOW/MEDIUM 風險動作 + - HIGH/CRITICAL 自動跳過 + """ + # 取得 Incident + incident_service = get_incident_service() + incident = await incident_service.get_incident(request.incident_id) + + if not incident: + raise HTTPException( + status_code=404, + detail=f"Incident {request.incident_id} not found", + ) + + # 取得 Playbook + from src.services.playbook_service import get_playbook_service + + playbook_service = get_playbook_service() + playbook = await playbook_service.get_by_id(request.playbook_id) + + if not playbook: + raise HTTPException( + status_code=404, + detail=f"Playbook {request.playbook_id} not found", + ) + + # 安全檢查 (除非 force=True) + if not request.force: + auto_repair_service = get_auto_repair_service() + decision = await auto_repair_service.evaluate_auto_repair(incident) + + if not decision.can_auto_repair: + raise HTTPException( + status_code=400, + detail=f"Auto repair not allowed: {decision.reason}", + ) + + if decision.playbook and decision.playbook.playbook_id != request.playbook_id: + raise HTTPException( + status_code=400, + detail=f"Playbook mismatch: expected {decision.playbook.playbook_id}", + ) + + # 執行 + auto_repair_service = get_auto_repair_service() + result = await auto_repair_service.execute_auto_repair(incident, playbook) + + return ExecuteResponse( + success=result.success, + incident_id=result.incident_id, + playbook_id=result.playbook_id, + executed_steps=result.executed_steps, + error=result.error, + execution_time_ms=result.execution_time_ms, + ) + + +@router.get("/stats") +async def get_auto_repair_stats() -> dict: + """ + 取得自動修復統計 + + 返回: + - 高品質 Playbook 數量 + - 自動修復執行次數 + - 成功率 + """ + from src.models.playbook import PlaybookStatus + from src.services.playbook_service import get_playbook_service + + playbook_service = get_playbook_service() + playbooks, total = await playbook_service.list_playbooks( + status=PlaybookStatus.APPROVED, + limit=100, + ) + + high_quality_count = sum(1 for p in playbooks if p.is_high_quality) + total_executions = sum(p.total_executions for p in playbooks) + total_success = sum(p.success_count for p in playbooks) + + return { + "approved_playbooks": len(playbooks), + "high_quality_playbooks": high_quality_count, + "total_executions": total_executions, + "overall_success_rate": total_success / total_executions if total_executions > 0 else 0.0, + "auto_repair_eligible": high_quality_count > 0, + } diff --git a/apps/api/src/api/v1/errors.py b/apps/api/src/api/v1/errors.py new file mode 100644 index 000000000..d4310bcad --- /dev/null +++ b/apps/api/src/api/v1/errors.py @@ -0,0 +1,410 @@ +""" +Errors BFF API - #40 Sentry 錯誤數據 API +======================================== +為前端提供 Sentry 錯誤追蹤數據 + +Phase 10: Sentry + OpenClaw + UI 整合 +建立時間: 2026-03-26 18:45 (台北時區) +建立者: Claude Code (#40 BFF API) +重構: 2026-03-26 21:20 (P2 架構改善 - SentryService 抽取) + +功能: +1. 列出近期錯誤 +2. 取得錯誤詳情 +3. 取得錯誤趨勢 +4. 觸發 AI 分析 + +遵循 leWOOOgo 積木化原則: +- Router 層只做 HTTP 轉發 +- 不直接存取 Redis/DB/外部 API +- 業務邏輯委託給 Service 層 +""" + +from fastapi import APIRouter, HTTPException, Query +from pydantic import BaseModel, Field + +from src.core.logging import get_logger +from src.services.sentry_service import get_sentry_service +from src.utils.timezone import now_taipei + +logger = get_logger(__name__) + +router = APIRouter(prefix="/errors", tags=["Errors"]) + + +# ============================================================================= +# Response Models +# ============================================================================= + + +class SentryIssue(BaseModel): + """Sentry Issue 簡化模型""" + + id: str = Field(description="Issue ID") + short_id: str = Field(description="短 ID (如 AWOOOI-123)") + title: str = Field(description="錯誤標題") + culprit: str | None = Field(None, description="錯誤來源 (函數/檔案)") + level: str = Field(description="嚴重度 (error, warning, info)") + status: str = Field(description="狀態 (unresolved, resolved, ignored)") + count: int = Field(description="發生次數") + user_count: int = Field(default=0, description="影響用戶數") + first_seen: str = Field(description="首次發生時間") + last_seen: str = Field(description="最後發生時間") + permalink: str | None = Field(None, description="Sentry 詳情連結") + + +class IssueListResponse(BaseModel): + """Issue 列表回應""" + + issues: list[SentryIssue] + total: int + has_more: bool + + +class IssueTrend(BaseModel): + """錯誤趨勢數據點""" + + timestamp: str + count: int + + +class TrendResponse(BaseModel): + """趨勢回應""" + + period: str # 24h, 7d, 30d + data: list[IssueTrend] + total_count: int + change_percent: float # 與前一週期相比 + + +class IssueStats(BaseModel): + """錯誤統計""" + + total_issues: int = 0 + unresolved_issues: int = 0 + error_count_24h: int = 0 + critical_count: int = 0 + projects: list[str] = [] + + +# ============================================================================= +# API Endpoints +# ============================================================================= + + +@router.get("/stats", response_model=IssueStats) +async def get_error_stats() -> IssueStats: + """ + 取得錯誤統計概覽 + + 返回: + - 總 Issue 數 + - 未解決 Issue 數 + - 24 小時內錯誤數 + - 重大錯誤數 + """ + sentry = get_sentry_service() + + # 取得組織專案列表 + projects = await sentry.list_projects() + + if not projects: + return IssueStats() + + project_slugs = [p.get("slug", "") for p in projects if isinstance(p, dict)] + + # 取得未解決 Issues + issues = await sentry.list_issues(query="is:unresolved", limit=100) + unresolved_count = len(issues) if issues else 0 + + # 統計 24h 內的錯誤 + issues_24h = await sentry.list_issues(query="is:unresolved age:-24h", limit=100) + error_24h = len(issues_24h) if issues_24h else 0 + + # 統計 critical/fatal + critical_issues = await sentry.list_issues(query="is:unresolved level:fatal", limit=100) + critical_count = len(critical_issues) if critical_issues else 0 + + return IssueStats( + total_issues=unresolved_count, + unresolved_issues=unresolved_count, + error_count_24h=error_24h, + critical_count=critical_count, + projects=project_slugs, + ) + + +@router.get("/issues", response_model=IssueListResponse) +async def list_issues( + project: str | None = Query(default=None, description="專案 slug"), + status: str = Query(default="unresolved", description="狀態過濾"), + level: str | None = Query(default=None, description="嚴重度過濾 (error, warning)"), + limit: int = Query(default=25, ge=1, le=100, description="每頁數量"), + cursor: str | None = Query(default=None, description="分頁游標"), +) -> IssueListResponse: + """ + 列出 Sentry Issues + + 支援過濾: + - status: unresolved, resolved, ignored + - level: fatal, error, warning, info + """ + sentry = get_sentry_service() + + query_parts = [f"is:{status}"] + if level: + query_parts.append(f"level:{level}") + + raw_issues = await sentry.list_issues( + project=project, + query=" ".join(query_parts), + limit=limit, + cursor=cursor, + ) + + if not raw_issues: + return IssueListResponse(issues=[], total=0, has_more=False) + + issues = [] + for issue in raw_issues: + if isinstance(issue, dict): + issues.append( + SentryIssue( + id=issue.get("id", ""), + short_id=issue.get("shortId", ""), + title=issue.get("title", "Unknown Error"), + culprit=issue.get("culprit"), + level=issue.get("level", "error"), + status=issue.get("status", "unresolved"), + count=issue.get("count", 0), + user_count=issue.get("userCount", 0), + first_seen=issue.get("firstSeen", ""), + last_seen=issue.get("lastSeen", ""), + permalink=issue.get("permalink"), + ) + ) + + return IssueListResponse( + issues=issues, + total=len(issues), + has_more=len(issues) >= limit, + ) + + +@router.get("/issues/{issue_id}") +async def get_issue_detail(issue_id: str) -> dict: + """ + 取得 Issue 詳情 + + 包含: + - 錯誤堆疊 + - 影響用戶 + - 錯誤上下文 + """ + sentry = get_sentry_service() + + issue = await sentry.get_issue(issue_id) + + if not issue: + raise HTTPException( + status_code=404, + detail=f"Issue {issue_id} not found", + ) + + # 取得最新事件 + events = await sentry.get_issue_events(issue_id, limit=1) + latest_event = events[0] if events and len(events) > 0 else None + + return { + "issue": issue, + "latest_event": latest_event, + "sentry_url": sentry.get_issue_url(issue_id), + } + + +@router.get("/trends", response_model=TrendResponse) +async def get_error_trends( + project: str | None = Query(default=None, description="專案 slug"), + period: str = Query(default="24h", description="時間範圍 (24h, 7d, 30d)"), +) -> TrendResponse: + """ + 取得錯誤趨勢 + + 返回指定時間範圍內的錯誤數量趨勢 + """ + sentry = get_sentry_service() + + # 計算時間範圍 + period_map = { + "24h": ("1h", 24), + "7d": ("1d", 7), + "30d": ("1d", 30), + } + + interval, points = period_map.get(period, ("1h", 24)) + + # Sentry Stats API + stats = await sentry.get_project_stats( + project=project, + stat="received", + resolution=interval, + ) + + if not stats: + return TrendResponse( + period=period, + data=[], + total_count=0, + change_percent=0.0, + ) + + # 轉換數據格式 + data = [] + total = 0 + + for point in stats[-points:] if len(stats) > points else stats: + if isinstance(point, list) and len(point) >= 2: + timestamp, count = point[0], point[1] + data.append(IssueTrend(timestamp=str(timestamp), count=count)) + total += count + + # 計算變化百分比 (與前一週期比較) + prev_total = sum(p[1] for p in stats[-points * 2 : -points] if isinstance(p, list) and len(p) >= 2) + change = ((total - prev_total) / prev_total * 100) if prev_total > 0 else 0.0 + + return TrendResponse( + period=period, + data=data, + total_count=total, + change_percent=round(change, 1), + ) + + +@router.post("/issues/{issue_id}/analyze") +async def trigger_ai_analysis( + issue_id: str, + project: str | None = Query(default=None, description="專案 slug"), +) -> dict: + """ + 觸發 AI 分析 + + 呼叫 OpenClaw 對指定 Issue 進行根因分析 + """ + sentry = get_sentry_service() + + # 取得 Issue 詳情 + issue = await sentry.get_issue(issue_id) + + if not issue: + raise HTTPException( + status_code=404, + detail=f"Issue {issue_id} not found", + ) + + # 取得最新事件 (含堆疊) + events = await sentry.get_issue_events(issue_id, limit=1, full=True) + + if not events: + raise HTTPException( + status_code=404, + detail=f"No events found for issue {issue_id}", + ) + + latest_event = events[0] + + # 提取分析所需資料 + title = issue.get("title", "Unknown Error") + level = issue.get("level", "error") + culprit = issue.get("culprit", "") + count = issue.get("count", 0) + stacktrace = _extract_stacktrace(latest_event) + context = _extract_context(latest_event) + + logger.info( + "error_analysis_triggered", + issue_id=issue_id, + title=title, + ) + + # 呼叫 Error Analyzer Service (#39) + from src.services.error_analyzer_service import get_error_analyzer_service + + analyzer = get_error_analyzer_service() + result, provider, success = await analyzer.analyze_error( + issue_id=issue_id, + title=title, + level=level, + culprit=culprit, + count=count, + stacktrace=stacktrace, + context=context, + ) + + sentry_url = sentry.get_issue_url(issue_id) + + if not success or result is None: + return { + "status": "failed", + "issue_id": issue_id, + "provider": provider, + "message": "AI analysis failed. Please try again later.", + "sentry_url": sentry_url, + } + + return { + "status": "completed", + "issue_id": issue_id, + "provider": provider, + "analysis": { + "root_cause": result.root_cause, + "category": result.category, + "severity": result.severity, + "impact_assessment": result.impact_assessment, + "fix_recommendation": result.fix_recommendation.model_dump(), + "prevention": [p.model_dump() for p in result.prevention], + "related_files": result.related_files, + "confidence": result.confidence, + "reasoning": result.reasoning, + }, + "analyzed_at": now_taipei().isoformat(), + "sentry_url": sentry_url, + } + + +# ============================================================================= +# Helper Functions (Event Parsing - Router 層合理) +# ============================================================================= + + +def _extract_stacktrace(event: dict) -> str: + """從事件中提取堆疊追蹤""" + entries = event.get("entries", []) + + for entry in entries: + if entry.get("type") == "exception": + values = entry.get("data", {}).get("values", []) + if values: + frames = values[0].get("stacktrace", {}).get("frames", []) + if frames: + # 取最後 10 個 frame + recent_frames = frames[-10:] + lines = [] + for f in recent_frames: + filename = f.get("filename", "") + lineno = f.get("lineNo", "") + function = f.get("function", "") + lines.append(f" {filename}:{lineno} in {function}") + return "\n".join(lines) + + return "No stacktrace available" + + +def _extract_context(event: dict) -> dict: + """從事件中提取上下文""" + return { + "browser": event.get("contexts", {}).get("browser", {}), + "os": event.get("contexts", {}).get("os", {}), + "device": event.get("contexts", {}).get("device", {}), + "tags": event.get("tags", []), + "user": event.get("user", {}), + } diff --git a/apps/api/src/api/v1/github_webhook.py b/apps/api/src/api/v1/github_webhook.py index e50ea7385..29e4e9675 100644 --- a/apps/api/src/api/v1/github_webhook.py +++ b/apps/api/src/api/v1/github_webhook.py @@ -1186,11 +1186,11 @@ async def send_ci_failure_telegram_alert( message_lines = [ f"{emoji} **CI 失敗診斷** | {repo}", - f"", + "", f"📋 **Workflow**: {workflow_name}", f"👤 **觸發者**: {sender}", f"🔗 [查看 Workflow]({workflow_url})", - f"", + "", ] if diagnosis: @@ -1200,7 +1200,7 @@ async def send_ci_failure_telegram_alert( f"**⚠️ 錯誤類型**: {diagnosis.error_type}", f"**🎯 風險等級**: {diagnosis.risk_level.upper()}", f"**🔧 修復決策**: {decision_text}", - f"", + "", ]) if diagnosis.suggestions: @@ -1210,7 +1210,7 @@ async def send_ci_failure_telegram_alert( # 顯示修復建議 (Phase 13.1 #78) if repair_decision and repair_decision.recommendations: - message_lines.extend([f"", f"**🔨 修復選項**:"]) + message_lines.extend(["", "**🔨 修復選項**:"]) for i, rec in enumerate(repair_decision.recommendations[:2], 1): confidence_pct = int(rec.confidence * 100) message_lines.append( @@ -1220,7 +1220,7 @@ async def send_ci_failure_telegram_alert( message_lines.append(f" `{rec.command[:50]}...`" if len(rec.command) > 50 else f" `{rec.command}`") message_lines.extend([ - f"", + "", f"🆔 `{diagnosis_id}`", ]) diff --git a/apps/api/src/api/v1/playbooks.py b/apps/api/src/api/v1/playbooks.py index 8fd04a070..1229e03dc 100644 --- a/apps/api/src/api/v1/playbooks.py +++ b/apps/api/src/api/v1/playbooks.py @@ -196,28 +196,22 @@ async def update_playbook( 更新 Playbook (人工編輯) 可更新名稱、描述、步驟、標籤等 + + Phase 8 P1 修復: 委託給 Service 層進行驗證 """ service = get_playbook_service() - playbook = await service.get_by_id(playbook_id) - if not playbook: - raise HTTPException( - status_code=404, - detail=f"Playbook {playbook_id} not found", - ) - - # 更新欄位 + # 委託給 Service 層進行驗證和更新 update_data = request.model_dump(exclude_unset=True) - for field, value in update_data.items(): - if value is not None: - setattr(playbook, field, value) - - updated = await service.update(playbook) + updated = await service.update_with_validation( + playbook_id=playbook_id, + update_data=update_data, + ) if not updated: raise HTTPException( - status_code=500, - detail="Failed to update playbook", + status_code=404, + detail=f"Playbook {playbook_id} not found or update not allowed", ) return PlaybookResponse.from_playbook(updated) diff --git a/apps/api/src/api/v1/stats.py b/apps/api/src/api/v1/stats.py index c99c79e52..2afea1ba1 100644 --- a/apps/api/src/api/v1/stats.py +++ b/apps/api/src/api/v1/stats.py @@ -23,7 +23,6 @@ from sqlalchemy.ext.asyncio import AsyncSession from src.core.logging import get_logger from src.db.base import get_db -from src.services.stats_service import get_stats_service from src.db.models import IncidentRecord from src.models.incident import IncidentStatus diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index e8bb81c25..400c8f4fb 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -46,10 +46,10 @@ from src.services.approval_db import get_approval_service # Phase 17 P0: Service 層 (消除 Router 直接存取 Redis) from src.services.incident_service import get_incident_service -from src.services.signal_producer import SignalData, get_signal_producer # Phase 5: OpenClaw AI Engine from src.services.openclaw import get_openclaw +from src.services.signal_producer import SignalData, get_signal_producer # Phase 5: Telegram Gateway (行動戰情室) from src.services.telegram_gateway import TelegramGatewayError, get_telegram_gateway diff --git a/apps/api/src/core/telemetry.py b/apps/api/src/core/telemetry.py index 830e237f7..41cc0ac0c 100644 --- a/apps/api/src/core/telemetry.py +++ b/apps/api/src/core/telemetry.py @@ -3,7 +3,7 @@ AWOOOI OpenTelemetry Configuration ================================== P0 基礎設施: 可觀測性鐵律 -Traces → SigNoz (192.168.0.188:4317) +Traces + Metrics → SigNoz (192.168.0.188:24317) 四主機架構強制校驗: | IP | 允許 OTEL? | @@ -16,16 +16,34 @@ Traces → SigNoz (192.168.0.188:4317) 優雅降級 (Graceful Degradation): - OTEL 連線失敗不會導致 API 崩潰 - 使用 BatchSpanProcessor 非同步傳輸 +- 使用 PeriodicExportingMetricReader 批量送出指標 - 連線超時後自動跳過追蹤 + +Phase 13.3 新增: +- OTEL Metrics (llm.tokens.*, llm.cost.*, llm.latency.*) +- Token Counter 整合 + +版本: v1.1 +最後修改: 2026-03-26 14:30 (台北時區) +修改者: Claude Code + +變更紀錄: +| 版本 | 日期 | 執行者 | 變更內容 | +|------|------|--------|----------| +| v1.0 | - | - | 初始 Traces 實作 | +| v1.1 | 2026-03-26 | Claude Code | Phase 13.3 新增 Metrics Provider | """ import logging -from opentelemetry import trace +from opentelemetry import metrics, trace +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.logging import LoggingInstrumentor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_VERSION, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -37,6 +55,7 @@ _logger = logging.getLogger("awoooi.telemetry") # Global state _tracer_provider: TracerProvider | None = None +_meter_provider: MeterProvider | None = None _initialized: bool = False @@ -84,7 +103,7 @@ def setup_telemetry(app) -> bool: - 如果 OTEL_ENABLED=false，跳過初始化 - 如果連線失敗，API 仍可正常運作 """ - global _tracer_provider, _initialized + global _tracer_provider, _meter_provider, _initialized # 檢查是否啟用 if settings.MOCK_MODE: @@ -114,12 +133,14 @@ def setup_telemetry(app) -> bool: "service.namespace": "awoooi", }) - # 建立 TracerProvider + # ===================================================================== + # Traces Provider + # ===================================================================== _tracer_provider = TracerProvider(resource=resource) # 建立 OTLP Exporter (gRPC) # 使用 BatchSpanProcessor 實現非同步傳輸 (優雅降級關鍵) - otlp_exporter = OTLPSpanExporter( + otlp_trace_exporter = OTLPSpanExporter( endpoint=settings.OTEL_EXPORTER_OTLP_ENDPOINT, insecure=True, # 內網使用，無需 TLS timeout=5, # 5 秒超時，避免阻塞 @@ -130,7 +151,7 @@ def setup_telemetry(app) -> bool: # 2. 連線失敗時自動丟棄 spans，不影響 API # 3. 記憶體保護: max_queue_size 限制 span_processor = BatchSpanProcessor( - otlp_exporter, + otlp_trace_exporter, max_queue_size=2048, # 最大佇列大小 max_export_batch_size=512, # 批量大小 schedule_delay_millis=5000, # 5 秒批量間隔 @@ -139,6 +160,32 @@ def setup_telemetry(app) -> bool: _tracer_provider.add_span_processor(span_processor) trace.set_tracer_provider(_tracer_provider) + # ===================================================================== + # Metrics Provider (Phase 13.3 #88) + # ===================================================================== + otlp_metric_exporter = OTLPMetricExporter( + endpoint=settings.OTEL_EXPORTER_OTLP_ENDPOINT, + insecure=True, + timeout=5, + ) + + # PeriodicExportingMetricReader: 每 30 秒批量送出指標 + metric_reader = PeriodicExportingMetricReader( + otlp_metric_exporter, + export_interval_millis=30000, # 30 秒 + ) + + _meter_provider = MeterProvider( + resource=resource, + metric_readers=[metric_reader], + ) + metrics.set_meter_provider(_meter_provider) + + _logger.info("OTEL Metrics Provider 初始化成功") + + # ===================================================================== + # Auto Instrumentation + # ===================================================================== # 自動埋入 FastAPI 追蹤 FastAPIInstrumentor.instrument_app( app, @@ -157,7 +204,7 @@ def setup_telemetry(app) -> bool: _initialized = True _logger.info( - f"OTEL 初始化成功: " + f"OTEL 初始化成功 (Traces + Metrics): " f"service={settings.OTEL_SERVICE_NAME}, " f"endpoint={settings.OTEL_EXPORTER_OTLP_ENDPOINT}" ) diff --git a/apps/api/src/db/models.py b/apps/api/src/db/models.py index e01609c31..eabb78bd8 100644 --- a/apps/api/src/db/models.py +++ b/apps/api/src/db/models.py @@ -278,6 +278,48 @@ class AuditLog(Base): ) dry_run_message: Mapped[str | None] = mapped_column(Text, nullable=True) + # ========================================================================== + # Phase 18: 失敗自動修復閉環欄位 (2026-03-26) + # ========================================================================== + + # 授權來源追蹤 + authorization_channel: Mapped[str | None] = mapped_column( + String(20), + nullable=True, + comment="Authorization source: web, telegram, auto", + ) + + # 重試與修復追蹤 + retry_count: Mapped[int] = mapped_column( + Integer, + default=0, + nullable=False, + comment="Number of retry attempts", + ) + failure_classification: Mapped[str | None] = mapped_column( + String(50), + nullable=True, + comment="Failure type: TIMEOUT, K8S_ERROR, NETWORK_ERROR, PERMISSION_DENIED", + ) + source_approval_id: Mapped[str | None] = mapped_column( + String(36), + nullable=True, + index=True, + comment="Original approval ID if this is a repair attempt", + ) + + # 自動修復狀態 + auto_repair_attempted: Mapped[bool] = mapped_column( + default=False, + nullable=False, + comment="Whether auto-repair was attempted", + ) + auto_repair_result: Mapped[str | None] = mapped_column( + Text, + nullable=True, + comment="Auto-repair result: AI analysis and repair outcome", + ) + # Timestamps created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), @@ -290,6 +332,8 @@ class AuditLog(Base): Index("ix_audit_operation_type", "operation_type"), Index("ix_audit_success", "success"), Index("ix_audit_created_at", "created_at"), + Index("ix_audit_authorization_channel", "authorization_channel"), # Phase 18 + Index("ix_audit_failure_classification", "failure_classification"), # Phase 18 ) diff --git a/apps/api/src/main.py b/apps/api/src/main.py index 42fe221ca..50d32af9a 100644 --- a/apps/api/src/main.py +++ b/apps/api/src/main.py @@ -34,7 +34,9 @@ from src.api.v1 import agents as agents_v1 # Phase 9.5: Agent Teams API from src.api.v1 import ai as ai_v1 from src.api.v1 import approvals as approvals_v1 from src.api.v1 import audit_logs as audit_logs_v1 +from src.api.v1 import auto_repair as auto_repair_v1 # #8: 自動升級決策 from src.api.v1 import dashboard as dashboard_v1 +from src.api.v1 import errors as errors_v1 # #40: Sentry 錯誤 BFF API from src.api.v1 import ( github_webhook as github_webhook_v1, # Phase 13.1: GitHub → OpenClaw ) @@ -394,6 +396,12 @@ app.include_router( app.include_router( playbooks_v1.router, prefix="/api/v1", tags=["Playbooks"] ) # #7: Playbook 萃取 +app.include_router( + auto_repair_v1.router, prefix="/api/v1", tags=["Auto Repair"] +) # #8: 自動升級決策 +app.include_router( + errors_v1.router, prefix="/api/v1", tags=["Errors"] +) # #40: Sentry 錯誤 BFF API app.include_router( proposals_router.router, tags=["Proposals (Legacy)"] ) # Phase 6.4g: lewooogo-brain (舊版) diff --git a/apps/api/src/models/playbook.py b/apps/api/src/models/playbook.py index c4ec01987..8cb611505 100644 --- a/apps/api/src/models/playbook.py +++ b/apps/api/src/models/playbook.py @@ -12,13 +12,15 @@ Phase 7.1: 資料模型定義 - 支援 PostgreSQL + Redis 雙層儲存 """ -from datetime import UTC, datetime +from datetime import datetime from enum import Enum from typing import Any from uuid import uuid4 from pydantic import BaseModel, ConfigDict, Field +from src.utils.timezone import now_taipei + # ============================================================================= # Enums # ============================================================================= @@ -123,8 +125,8 @@ class RepairStep(BaseModel): def generate_playbook_id() -> str: - """生成 Playbook ID""" - return f"PB-{datetime.now(UTC).strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" + """生成 Playbook ID (台北時區)""" + return f"PB-{now_taipei().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" class Playbook(BaseModel): @@ -195,8 +197,8 @@ class Playbook(BaseModel): notes: str | None = Field(None, description="人工補充說明") # === 時間軸 === - created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) - updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + created_at: datetime = Field(default_factory=now_taipei) + updated_at: datetime = Field(default_factory=now_taipei) model_config = ConfigDict(extra="ignore") diff --git a/apps/api/src/plugins/mcp/mcp_bridge.py b/apps/api/src/plugins/mcp/mcp_bridge.py index cc08ed00b..e2fa2a361 100644 --- a/apps/api/src/plugins/mcp/mcp_bridge.py +++ b/apps/api/src/plugins/mcp/mcp_bridge.py @@ -215,11 +215,11 @@ class MCPBridge: transport=MCPTransport.HTTP, endpoint="http://192.168.0.188:3301", # SignOz Query Service ) + # Phase 13.2 #82: Filesystem Provider (安全受限) self._servers["filesystem"] = MCPServer( name="filesystem", - transport=MCPTransport.STDIO, - endpoint="/usr/local/bin/mcp-filesystem", - args=["--root", "/tmp"], + transport=MCPTransport.HTTP, # 改用 HTTP，由 Provider 處理 + endpoint="internal://filesystem-provider", ) self._servers["database"] = MCPServer( name="database", diff --git a/apps/api/src/plugins/mcp/providers/filesystem_provider.py b/apps/api/src/plugins/mcp/providers/filesystem_provider.py new file mode 100644 index 000000000..f77cd5ad0 --- /dev/null +++ b/apps/api/src/plugins/mcp/providers/filesystem_provider.py @@ -0,0 +1,628 @@ +""" +Filesystem MCP Tool Provider - Phase 13.2 #82 +============================================== + +提供安全受限的文件讀取工具： +- read_file: 讀取文件內容 +- list_directory: 列出目錄 +- search_in_file: 搜尋文件內容 + +安全機制: +1. 目錄白名單 (只允許 /var/log, /tmp/awoooi 等) +2. 敏感文件黑名單 (.env, credentials, secrets) +3. 檔案大小限制 (預設 1MB) +4. 路徑遍歷防護 (禁止 ../) +5. Symlink 防護 (解析後必須在白名單內) + +透過 DI 注入配置，不直接讀取環境變數。 + +@see docs/adr/ADR-015-mcp-modular-architecture.md +@author Claude Code +@version 1.0 +@created 2026-03-26 (台北時區) +""" + +import os +import re +import uuid +from pathlib import Path +from typing import Any + +import structlog + +from src.plugins.mcp.interfaces import MCPTool, MCPToolProvider, MCPToolResult + +logger = structlog.get_logger(__name__) + +# ============================================================================= +# Security Configuration (可從 config.py 覆蓋) +# ============================================================================= + +# 目錄白名單 - 只允許讀取這些目錄 +# 注意: macOS 上 /var 是 symlink 到 /private/var，需同時包含兩者 +DEFAULT_ALLOWED_DIRECTORIES: list[str] = [ + "/var/log", # 系統日誌 (Linux) + "/private/var/log", # 系統日誌 (macOS symlink 解析後) + "/tmp/awoooi", # AWOOOI 暫存目錄 + "/private/tmp/awoooi", # macOS 暫存 + "/var/log/awoooi", # AWOOOI 日誌 + "/var/log/nginx", # Nginx 日誌 + "/var/log/kubernetes", # K8s 日誌 + "/var/log/containers", # K8s 容器日誌 + "/var/log/pods", # K8s Pod 日誌 +] + +# 敏感文件黑名單 - 禁止讀取這些 pattern +SENSITIVE_FILE_PATTERNS: list[str] = [ + r"\.env$", # 環境變數 + r"\.env\..+$", # .env.local, .env.production + r"credentials?\..*$", # credentials.json, credential.yaml + r"secrets?\..*$", # secret.yaml, secrets.json + r"\.pem$", # 憑證 + r"\.key$", # 私鑰 + r"\.p12$", # PKCS12 + r"\.pfx$", # 憑證 + r"id_rsa.*$", # SSH 私鑰 + r"\.ssh/.*$", # SSH 目錄 + r"password.*$", # 密碼檔案 + r"token.*$", # Token 檔案 + r"kubeconfig.*$", # K8s 配置 + r"\.kube/.*$", # K8s 目錄 + r"shadow$", # /etc/shadow + r"passwd$", # /etc/passwd (雖然不敏感但通常不需要) +] + +# 檔案大小限制 (bytes) +DEFAULT_MAX_FILE_SIZE: int = 1 * 1024 * 1024 # 1MB + +# 搜尋結果行數限制 +DEFAULT_MAX_SEARCH_RESULTS: int = 100 + + +class FilesystemSecurityError(Exception): + """檔案系統安全錯誤""" + pass + + +class FilesystemProvider(MCPToolProvider): + """ + Filesystem MCP Tool Provider + + 提供安全受限的文件讀取功能，所有操作都經過： + 1. 白名單驗證 (目錄) + 2. 黑名單驗證 (敏感文件) + 3. 路徑正規化 (防止遍歷攻擊) + 4. 大小限制 (防止 OOM) + + Security Notes: + - 此 Provider 只提供讀取功能，不提供寫入 + - 所有路徑在使用前都會被 resolve() 成絕對路徑 + - Symlink 會被解析，解析後的路徑必須仍在白名單內 + """ + + def __init__( + self, + allowed_directories: list[str] | None = None, + max_file_size: int = DEFAULT_MAX_FILE_SIZE, + max_search_results: int = DEFAULT_MAX_SEARCH_RESULTS, + ) -> None: + """ + 初始化 Provider + + Args: + allowed_directories: 允許讀取的目錄白名單 + max_file_size: 最大檔案大小 (bytes) + max_search_results: 搜尋結果最大行數 + """ + self._allowed_directories = allowed_directories or DEFAULT_ALLOWED_DIRECTORIES + self._max_file_size = max_file_size + self._max_search_results = max_search_results + + # 預編譯敏感文件 pattern + self._sensitive_patterns = [ + re.compile(pattern, re.IGNORECASE) + for pattern in SENSITIVE_FILE_PATTERNS + ] + + logger.info( + "filesystem_provider_initialized", + allowed_directories=self._allowed_directories, + max_file_size=self._max_file_size, + ) + + @property + def name(self) -> str: + return "filesystem" + + async def list_tools(self) -> list[MCPTool]: + return [ + MCPTool( + name="read_file", + description="Read file contents (max 1MB, restricted to whitelisted directories)", + input_schema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Absolute file path", + }, + "encoding": { + "type": "string", + "description": "File encoding (default: utf-8)", + "default": "utf-8", + }, + "tail_lines": { + "type": "integer", + "description": "Only read last N lines (for log files)", + }, + }, + "required": ["path"], + }, + server_name=self.name, + ), + MCPTool( + name="list_directory", + description="List directory contents (restricted to whitelisted directories)", + input_schema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Absolute directory path", + }, + "pattern": { + "type": "string", + "description": "Glob pattern filter (e.g., *.log)", + }, + "recursive": { + "type": "boolean", + "description": "Include subdirectories (default: false)", + "default": False, + }, + "limit": { + "type": "integer", + "description": "Max results (default: 100)", + "default": 100, + }, + }, + "required": ["path"], + }, + server_name=self.name, + ), + MCPTool( + name="search_in_file", + description="Search for pattern in file contents", + input_schema={ + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Absolute file path", + }, + "pattern": { + "type": "string", + "description": "Search pattern (regex supported)", + }, + "case_sensitive": { + "type": "boolean", + "description": "Case sensitive search (default: false)", + "default": False, + }, + "context_lines": { + "type": "integer", + "description": "Lines of context around matches (default: 2)", + "default": 2, + }, + }, + "required": ["path", "pattern"], + }, + server_name=self.name, + ), + ] + + async def execute( + self, + tool_name: str, + parameters: dict[str, Any], + ) -> MCPToolResult: + execution_id = str(uuid.uuid4())[:8] + + try: + if tool_name == "read_file": + output = await self._read_file(parameters) + elif tool_name == "list_directory": + output = await self._list_directory(parameters) + elif tool_name == "search_in_file": + output = await self._search_in_file(parameters) + else: + return MCPToolResult( + success=False, + execution_id=execution_id, + error=f"Unknown tool: {tool_name}", + ) + + logger.info( + "filesystem_tool_executed", + tool=tool_name, + execution_id=execution_id, + # 不記錄 path 以避免資訊洩漏 + ) + + return MCPToolResult( + success=True, + execution_id=execution_id, + output=output, + ) + + except FilesystemSecurityError as e: + logger.warning( + "filesystem_security_violation", + tool=tool_name, + execution_id=execution_id, + error=str(e), + ) + return MCPToolResult( + success=False, + execution_id=execution_id, + error=f"Security violation: {e}", + ) + except FileNotFoundError as e: + return MCPToolResult( + success=False, + execution_id=execution_id, + error=f"File not found: {e}", + ) + except PermissionError as e: + return MCPToolResult( + success=False, + execution_id=execution_id, + error=f"Permission denied: {e}", + ) + except Exception as e: + logger.exception( + "filesystem_provider_error", + tool=tool_name, + execution_id=execution_id, + error=str(e), + ) + return MCPToolResult( + success=False, + execution_id=execution_id, + error=str(e), + ) + + # ========================================================================= + # Security Methods + # ========================================================================= + + def _validate_path(self, path_str: str) -> Path: + """ + 驗證並正規化路徑 + + Security checks: + 1. 必須是絕對路徑 + 2. 解析 symlink 後必須在白名單目錄內 + 3. 不能包含 .. 遍歷 + 4. 不能匹配敏感文件 pattern + + Args: + path_str: 原始路徑字串 + + Returns: + Path: 正規化後的 Path 物件 + + Raises: + FilesystemSecurityError: 路徑違反安全規則 + """ + # 1. 檢查是否為絕對路徑 + if not path_str.startswith("/"): + raise FilesystemSecurityError(f"Path must be absolute: {path_str}") + + # 2. 檢查 .. 遍歷 (在 resolve 之前) + if ".." in path_str: + raise FilesystemSecurityError(f"Path traversal not allowed: {path_str}") + + # 3. 正規化路徑 (解析 symlink) + path = Path(path_str) + try: + resolved_path = path.resolve(strict=False) + except Exception as e: + raise FilesystemSecurityError(f"Invalid path: {e}") from e + + # 4. 檢查是否在白名單目錄內 + resolved_str = str(resolved_path) + in_whitelist = False + for allowed_dir in self._allowed_directories: + if resolved_str.startswith(allowed_dir): + in_whitelist = True + break + + if not in_whitelist: + raise FilesystemSecurityError( + f"Path not in allowed directories: {resolved_str}. " + f"Allowed: {self._allowed_directories}" + ) + + # 5. 檢查是否為敏感文件 + if self._is_sensitive_file(resolved_path): + raise FilesystemSecurityError( + f"Access to sensitive file denied: {resolved_path.name}" + ) + + return resolved_path + + def _is_sensitive_file(self, path: Path) -> bool: + """ + 檢查是否為敏感文件 + + Args: + path: 檔案路徑 + + Returns: + bool: True 如果是敏感文件 + """ + filename = path.name + full_path = str(path) + + for pattern in self._sensitive_patterns: + if pattern.search(filename) or pattern.search(full_path): + return True + + return False + + def _check_file_size(self, path: Path) -> int: + """ + 檢查檔案大小 + + Args: + path: 檔案路徑 + + Returns: + int: 檔案大小 (bytes) + + Raises: + FilesystemSecurityError: 檔案超過大小限制 + """ + try: + size = path.stat().st_size + except OSError as e: + raise FilesystemSecurityError(f"Cannot stat file: {e}") from e + + if size > self._max_file_size: + raise FilesystemSecurityError( + f"File too large: {size} bytes (max: {self._max_file_size})" + ) + + return size + + # ========================================================================= + # Tool Implementations + # ========================================================================= + + async def _read_file(self, parameters: dict) -> dict: + """ + 讀取文件內容 + + Args: + parameters: 包含 path, encoding, tail_lines + + Returns: + dict: {content, size, path, encoding} + """ + path_str = parameters.get("path", "") + encoding = parameters.get("encoding", "utf-8") + tail_lines = parameters.get("tail_lines") + + if not path_str: + return {"error": "Missing 'path' parameter"} + + # 安全驗證 + path = self._validate_path(path_str) + + if not path.is_file(): + return {"error": f"Not a file: {path_str}"} + + # 檢查檔案大小 + file_size = self._check_file_size(path) + + # 讀取內容 + try: + if tail_lines and tail_lines > 0: + # 只讀最後 N 行 (適合 log 文件) + content = self._read_last_lines(path, tail_lines, encoding) + else: + content = path.read_text(encoding=encoding) + except UnicodeDecodeError: + return {"error": f"Cannot decode file with encoding: {encoding}"} + + return { + "path": str(path), + "content": content, + "size": file_size, + "encoding": encoding, + "lines": content.count("\n") + 1, + } + + def _read_last_lines(self, path: Path, n: int, encoding: str = "utf-8") -> str: + """ + 讀取檔案最後 N 行 (適合大型 log 文件) + + 使用 seek 從檔案尾部讀取，效率更高 + """ + lines = [] + block_size = 8192 + + with open(path, "rb") as f: + # 移到檔案尾部 + f.seek(0, os.SEEK_END) + file_size = f.tell() + + # 從尾部逐塊讀取 + remaining = file_size + buffer = b"" + + while len(lines) <= n and remaining > 0: + read_size = min(block_size, remaining) + remaining -= read_size + f.seek(remaining) + block = f.read(read_size) + buffer = block + buffer + + # 計算行數 + lines = buffer.split(b"\n") + + # 取最後 N 行 + result_lines = lines[-n:] + return b"\n".join(result_lines).decode(encoding) + + async def _list_directory(self, parameters: dict) -> dict: + """ + 列出目錄內容 + + Args: + parameters: 包含 path, pattern, recursive, limit + + Returns: + dict: {path, entries: [{name, type, size, modified}]} + """ + path_str = parameters.get("path", "") + pattern = parameters.get("pattern", "*") + recursive = parameters.get("recursive", False) + limit = parameters.get("limit", 100) + + if not path_str: + return {"error": "Missing 'path' parameter"} + + # 安全驗證 + path = self._validate_path(path_str) + + if not path.is_dir(): + return {"error": f"Not a directory: {path_str}"} + + # 列出內容 + entries = [] + try: + if recursive: + items = list(path.rglob(pattern)) + else: + items = list(path.glob(pattern)) + + # 排序並限制數量 + items = sorted(items, key=lambda p: p.name)[:limit] + + for item in items: + # 跳過敏感文件 + if self._is_sensitive_file(item): + continue + + try: + stat = item.stat() + entries.append({ + "name": item.name, + "path": str(item), + "type": "directory" if item.is_dir() else "file", + "size": stat.st_size if item.is_file() else None, + "modified": stat.st_mtime, + }) + except (PermissionError, OSError): + # 無權限的項目跳過 + continue + + except PermissionError: + return {"error": f"Permission denied: {path_str}"} + + return { + "path": str(path), + "pattern": pattern, + "recursive": recursive, + "count": len(entries), + "entries": entries, + } + + async def _search_in_file(self, parameters: dict) -> dict: + """ + 在文件中搜尋 pattern + + Args: + parameters: 包含 path, pattern, case_sensitive, context_lines + + Returns: + dict: {path, pattern, matches: [{line_number, line, context}]} + """ + path_str = parameters.get("path", "") + search_pattern = parameters.get("pattern", "") + case_sensitive = parameters.get("case_sensitive", False) + context_lines = parameters.get("context_lines", 2) + + if not path_str: + return {"error": "Missing 'path' parameter"} + if not search_pattern: + return {"error": "Missing 'pattern' parameter"} + + # 安全驗證 + path = self._validate_path(path_str) + + if not path.is_file(): + return {"error": f"Not a file: {path_str}"} + + # 檢查檔案大小 + self._check_file_size(path) + + # 編譯搜尋 pattern + flags = 0 if case_sensitive else re.IGNORECASE + try: + regex = re.compile(search_pattern, flags) + except re.error as e: + return {"error": f"Invalid regex pattern: {e}"} + + # 搜尋內容 + matches = [] + try: + lines = path.read_text().splitlines() + + for i, line in enumerate(lines): + if regex.search(line): + # 取得前後 context + start = max(0, i - context_lines) + end = min(len(lines), i + context_lines + 1) + context = lines[start:end] + + matches.append({ + "line_number": i + 1, # 1-based + "line": line, + "context": context, + "context_start": start + 1, + }) + + # 限制結果數量 + if len(matches) >= self._max_search_results: + break + + except UnicodeDecodeError: + return {"error": "File is not valid UTF-8 text"} + + return { + "path": str(path), + "pattern": search_pattern, + "case_sensitive": case_sensitive, + "match_count": len(matches), + "truncated": len(matches) >= self._max_search_results, + "matches": matches, + } + + async def health_check(self) -> bool: + """ + 健康檢查 + + 驗證至少一個白名單目錄存在且可讀 + """ + for dir_path in self._allowed_directories: + path = Path(dir_path) + if path.exists() and path.is_dir(): + try: + # 嘗試列出目錄驗證權限 + list(path.iterdir()) + return True + except PermissionError: + continue + return False diff --git a/apps/api/src/repositories/audit_log_repository.py b/apps/api/src/repositories/audit_log_repository.py index e6cef3803..ed44ce75e 100644 --- a/apps/api/src/repositories/audit_log_repository.py +++ b/apps/api/src/repositories/audit_log_repository.py @@ -13,7 +13,6 @@ from typing import Any import structlog from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession from src.db.base import get_db_context from src.db.models import AuditLog diff --git a/apps/api/src/repositories/playbook_repository.py b/apps/api/src/repositories/playbook_repository.py index ee6dc04aa..9833ef00c 100644 --- a/apps/api/src/repositories/playbook_repository.py +++ b/apps/api/src/repositories/playbook_repository.py @@ -14,7 +14,6 @@ Phase 7.2: Repository 實作 """ import json -from datetime import UTC, datetime import structlog @@ -25,6 +24,7 @@ from src.models.playbook import ( SymptomPattern, ) from src.repositories.interfaces import IPlaybookRepository +from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) @@ -119,8 +119,8 @@ class PlaybookRepository: # 確保有建立時間 if not playbook.created_at: - playbook.created_at = datetime.now(UTC) - playbook.updated_at = datetime.now(UTC) + playbook.created_at = now_taipei() + playbook.updated_at = now_taipei() # 儲存 Playbook key = f"{PLAYBOOK_KEY_PREFIX}{playbook.playbook_id}" @@ -166,7 +166,7 @@ class PlaybookRepository: if not existing: return None - playbook.updated_at = datetime.now(UTC) + playbook.updated_at = now_taipei() redis_client = get_redis() key = f"{PLAYBOOK_KEY_PREFIX}{playbook.playbook_id}" @@ -202,7 +202,7 @@ class PlaybookRepository: return False playbook.status = PlaybookStatus.DEPRECATED - playbook.updated_at = datetime.now(UTC) + playbook.updated_at = now_taipei() await self.update(playbook) logger.info("playbook_deprecated", playbook_id=playbook_id) @@ -352,7 +352,7 @@ class PlaybookRepository: else: playbook.failure_count += 1 - playbook.last_used_at = datetime.now(UTC) + playbook.last_used_at = now_taipei() await self.update(playbook) logger.info( diff --git a/apps/api/src/services/__init__.py b/apps/api/src/services/__init__.py index 74ea559fc..4f95d57f2 100644 --- a/apps/api/src/services/__init__.py +++ b/apps/api/src/services/__init__.py @@ -44,6 +44,14 @@ from .graph_rag import ( create_mock_topology, topology_graph, ) +from .model_registry import ( + IModelRegistry, + ModelRegistry, + get_model, + get_model_by_complexity, + get_model_registry, + reset_model_registry, +) from .trust_engine import ( RiskAdjustment, RiskLevel, @@ -99,4 +107,11 @@ __all__ = [ "ConsensusResult", "AgentOpinion", "AgentType", + # Model Registry (Phase 12 P1) + "ModelRegistry", + "IModelRegistry", + "get_model_registry", + "get_model", + "get_model_by_complexity", + "reset_model_registry", ] diff --git a/apps/api/src/services/agent_service.py b/apps/api/src/services/agent_service.py index 8732fb462..05a05b128 100644 --- a/apps/api/src/services/agent_service.py +++ b/apps/api/src/services/agent_service.py @@ -14,7 +14,6 @@ Phase 17 R4: 從 agents.py Router 抽離 Redis 操作 """ import json -from datetime import UTC, datetime from enum import Enum from typing import Any, Protocol, runtime_checkable from uuid import uuid4 @@ -24,6 +23,7 @@ from src.core.redis_client import get_redis from src.core.sse import EventType, SSEEvent, get_publisher from src.models.incident import Incident, IncidentStatus, Severity, Signal from src.services.consensus_engine import get_consensus_engine +from src.utils.timezone import now_taipei, now_taipei_iso logger = get_logger("awoooi.agent_service") @@ -140,7 +140,7 @@ class AgentTaskRedisRepository: "agents_completed": 0, "total_agents": 4, "incident_id": incident_id, - "started_at": datetime.now(UTC).isoformat(), + "started_at": now_taipei_iso(), "trigger": trigger, } @@ -333,7 +333,7 @@ class AgentService: def generate_task_id(self) -> str: """產生新的 Task ID""" - return f"TASK-{datetime.now(UTC).strftime('%Y%m%d')}-{uuid4().hex[:8].upper()}" + return f"TASK-{now_taipei().strftime('%Y%m%d')}-{uuid4().hex[:8].upper()}" async def create_analysis_task( self, @@ -488,7 +488,7 @@ class AgentService: "final_reasoning": result.final_reasoning, "opinions": [op.to_dict() for op in result.opinions], "dissenting_opinions": result.dissenting_opinions, - "completed_at": datetime.now(UTC).isoformat(), + "completed_at": now_taipei_iso(), } await self._repository.save_task_result(task_id, task_data) @@ -526,7 +526,7 @@ class AgentService: "state": TaskState.FAILED.value, "progress": 0, "error": str(e), - "completed_at": datetime.now(UTC).isoformat(), + "completed_at": now_taipei_iso(), } await self._repository.save_task_result(task_id, task_data) @@ -638,7 +638,7 @@ class AgentService: alert_name=alert_name, severity=Severity(severity), source="manual", - fired_at=datetime.now(UTC), + fired_at=now_taipei(), )) return Incident( diff --git a/apps/api/src/services/auto_repair_service.py b/apps/api/src/services/auto_repair_service.py new file mode 100644 index 000000000..d4f979e1f --- /dev/null +++ b/apps/api/src/services/auto_repair_service.py @@ -0,0 +1,425 @@ +""" +Auto Repair Service - #8 自動升級決策 +===================================== +高品質 Playbook 自動修復執行 + +Phase 8: 自動化層實作 +建立時間: 2026-03-26 17:30 (台北時區) +建立者: Claude Code (#8 自動升級決策) + +遵循 leWOOOgo 積木化原則: +- Service 層只依賴 Repository/Service Interface +- 不直接存取 Redis/DB +- 封裝所有自動修復邏輯 + +觸發條件 (AND): +1. 有匹配的高品質 Playbook (is_high_quality = True) +2. Playbook 中的動作風險等級 <= MEDIUM +3. Incident 嚴重度 <= P2 + +安全邊界: +- HIGH/CRITICAL 風險動作永遠需要人工審核 +- P0/P1 嚴重度 Incident 需要人工確認 +""" + +from dataclasses import dataclass +from typing import Protocol + +import structlog + +from src.models.incident import Incident, Severity +from src.models.playbook import ( + ActionType, + Playbook, + RiskLevel, + SymptomPattern, +) +from src.services.executor import get_executor +from src.services.playbook_service import IPlaybookService, get_playbook_service + +logger = structlog.get_logger(__name__) + + +# ============================================================================= +# Types +# ============================================================================= + + +@dataclass +class AutoRepairDecision: + """自動修復決策結果""" + + can_auto_repair: bool + playbook: Playbook | None = None + reason: str = "" + risk_level: RiskLevel = RiskLevel.MEDIUM + blocked_by: str | None = None # 阻擋原因 (如 HIGH_RISK, P1_SEVERITY) + + +@dataclass +class AutoRepairResult: + """自動修復執行結果""" + + success: bool + playbook_id: str + incident_id: str + executed_steps: list[str] + error: str | None = None + execution_time_ms: int = 0 + + +# ============================================================================= +# Auto Repair Service Interface +# ============================================================================= + + +class IAutoRepairService(Protocol): + """自動修復服務介面""" + + async def evaluate_auto_repair( + self, + incident: Incident, + ) -> AutoRepairDecision: + """ + 評估是否可自動修復 + + Args: + incident: 待處理的 Incident + + Returns: + AutoRepairDecision: 決策結果 + """ + ... + + async def execute_auto_repair( + self, + incident: Incident, + playbook: Playbook, + ) -> AutoRepairResult: + """ + 執行自動修復 + + Args: + incident: 待處理的 Incident + playbook: 要執行的 Playbook + + Returns: + AutoRepairResult: 執行結果 + """ + ... + + +# ============================================================================= +# Auto Repair Service Implementation +# ============================================================================= + + +class AutoRepairService: + """ + 自動修復服務實作 + + 職責: + - 評估 Incident 是否可自動修復 + - 執行高品質 Playbook + - 更新執行統計 + """ + + # === 安全邊界常數 === + MAX_AUTO_REPAIR_RISK = RiskLevel.MEDIUM # 最高允許自動修復的風險等級 + MAX_AUTO_REPAIR_SEVERITY = Severity.P2 # 最高允許自動修復的嚴重度 + MIN_SIMILARITY_SCORE = 0.7 # 最低相似度門檻 + + def __init__( + self, + playbook_service: IPlaybookService | None = None, + ): + self._playbook_service = playbook_service or get_playbook_service() + + async def evaluate_auto_repair( + self, + incident: Incident, + ) -> AutoRepairDecision: + """ + 評估是否可自動修復 + + 決策流程: + 1. 檢查 Incident 嚴重度 (P0/P1 需人工) + 2. 從 Playbook 找匹配項 + 3. 檢查 Playbook 是否為高品質 + 4. 檢查動作風險等級 + """ + logger.info( + "auto_repair_evaluate_start", + incident_id=incident.incident_id, + severity=incident.severity.value if incident.severity else None, + ) + + # 1. 檢查 Incident 嚴重度 + if incident.severity and incident.severity.value in ["P0", "P1"]: + logger.info( + "auto_repair_blocked_severity", + incident_id=incident.incident_id, + severity=incident.severity.value, + ) + return AutoRepairDecision( + can_auto_repair=False, + reason=f"Incident 嚴重度 {incident.severity.value} 需要人工審核", + blocked_by="HIGH_SEVERITY", + ) + + # 2. 提取症狀模式 + symptoms = self._extract_symptoms(incident) + + # 3. 找匹配的 Playbook + recommendations = await self._playbook_service.get_recommendations( + symptoms=symptoms, + top_k=3, + ) + + if not recommendations: + logger.info( + "auto_repair_no_playbook_match", + incident_id=incident.incident_id, + ) + return AutoRepairDecision( + can_auto_repair=False, + reason="未找到匹配的 Playbook", + blocked_by="NO_MATCH", + ) + + # 4. 檢查最佳匹配 + best_match = recommendations[0] + + # 相似度檢查 + if best_match.similarity_score < self.MIN_SIMILARITY_SCORE: + return AutoRepairDecision( + can_auto_repair=False, + playbook=best_match.playbook, + reason=f"相似度 {best_match.similarity_score:.0%} 低於門檻 {self.MIN_SIMILARITY_SCORE:.0%}", + blocked_by="LOW_SIMILARITY", + ) + + # 高品質檢查 + if not best_match.playbook.is_high_quality: + return AutoRepairDecision( + can_auto_repair=False, + playbook=best_match.playbook, + reason=f"Playbook 尚未達到高品質標準 (成功率: {best_match.playbook.success_rate:.0%}, 執行次數: {best_match.playbook.total_executions})", + blocked_by="NOT_HIGH_QUALITY", + ) + + # 5. 檢查動作風險等級 + max_risk = self._get_max_risk_level(best_match.playbook) + + if self._risk_exceeds_threshold(max_risk): + return AutoRepairDecision( + can_auto_repair=False, + playbook=best_match.playbook, + reason=f"Playbook 包含 {max_risk.value} 風險動作，需要人工審核", + risk_level=max_risk, + blocked_by="HIGH_RISK", + ) + + # 6. 可以自動修復 + logger.info( + "auto_repair_approved", + incident_id=incident.incident_id, + playbook_id=best_match.playbook.playbook_id, + similarity=best_match.similarity_score, + success_rate=best_match.playbook.success_rate, + ) + + return AutoRepairDecision( + can_auto_repair=True, + playbook=best_match.playbook, + reason=f"匹配高品質 Playbook: {best_match.playbook.name} (成功率 {best_match.playbook.success_rate:.0%})", + risk_level=max_risk, + ) + + async def execute_auto_repair( + self, + incident: Incident, + playbook: Playbook, + ) -> AutoRepairResult: + """ + 執行自動修復 + + 流程: + 1. 依序執行 Playbook 中的 repair_steps + 2. 記錄執行結果 + 3. 更新 Playbook 統計 + """ + import time + + start_time = time.perf_counter() + executed_steps: list[str] = [] + + logger.info( + "auto_repair_execute_start", + incident_id=incident.incident_id, + playbook_id=playbook.playbook_id, + steps_count=len(playbook.repair_steps), + ) + + try: + # 執行每個步驟 + for step in playbook.repair_steps: + # 安全檢查: 跳過高風險步驟 + if self._risk_exceeds_threshold(step.risk_level): + logger.warning( + "auto_repair_skip_high_risk_step", + step_number=step.step_number, + risk_level=step.risk_level.value, + ) + continue + + # 執行步驟 + step_result = await self._execute_step(incident, step) + executed_steps.append( + f"Step {step.step_number}: {step.command[:50]}... -> {step_result}" + ) + + # 更新 Playbook 統計 + await self._playbook_service.record_execution( + playbook_id=playbook.playbook_id, + success=True, + ) + + execution_time = int((time.perf_counter() - start_time) * 1000) + + logger.info( + "auto_repair_execute_success", + incident_id=incident.incident_id, + playbook_id=playbook.playbook_id, + executed_steps=len(executed_steps), + execution_time_ms=execution_time, + ) + + return AutoRepairResult( + success=True, + playbook_id=playbook.playbook_id, + incident_id=incident.incident_id, + executed_steps=executed_steps, + execution_time_ms=execution_time, + ) + + except Exception as e: + # 更新失敗統計 + await self._playbook_service.record_execution( + playbook_id=playbook.playbook_id, + success=False, + ) + + execution_time = int((time.perf_counter() - start_time) * 1000) + + logger.error( + "auto_repair_execute_failed", + incident_id=incident.incident_id, + playbook_id=playbook.playbook_id, + error=str(e), + ) + + return AutoRepairResult( + success=False, + playbook_id=playbook.playbook_id, + incident_id=incident.incident_id, + executed_steps=executed_steps, + error=str(e), + execution_time_ms=execution_time, + ) + + # === Private Helpers === + + def _extract_symptoms(self, incident: Incident) -> SymptomPattern: + """從 Incident 提取症狀模式""" + alert_names = [] + keywords = [] + + if incident.signals: + for signal in incident.signals: + alert_names.append(signal.alert_name) + # 從 annotations 提取關鍵字 + if signal.annotations: + for value in signal.annotations.values(): + if isinstance(value, str) and len(value) < 50: + keywords.append(value) + + return SymptomPattern( + alert_names=alert_names, + affected_services=incident.affected_services or [], + severity_range=[incident.severity.value] if incident.severity else ["P2"], + keywords=keywords[:10], + ) + + def _get_max_risk_level(self, playbook: Playbook) -> RiskLevel: + """取得 Playbook 中最高的風險等級""" + risk_order = { + RiskLevel.LOW: 0, + RiskLevel.MEDIUM: 1, + RiskLevel.HIGH: 2, + RiskLevel.CRITICAL: 3, + } + + max_risk = RiskLevel.LOW + for step in playbook.repair_steps: + if risk_order.get(step.risk_level, 0) > risk_order.get(max_risk, 0): + max_risk = step.risk_level + + return max_risk + + def _risk_exceeds_threshold(self, risk: RiskLevel) -> bool: + """檢查風險是否超過自動修復門檻""" + high_risks = {RiskLevel.HIGH, RiskLevel.CRITICAL} + return risk in high_risks + + async def _execute_step(self, incident: Incident, step) -> str: + """ + 執行單一修復步驟 + + 目前整合: + - kubectl 命令: 透過 ActionExecutor + - script: 透過 subprocess + - manual: 跳過 (需人工) + """ + if step.action_type == ActionType.MANUAL: + return "SKIPPED (manual step)" + + if step.action_type == ActionType.KUBECTL: + # 整合 ActionExecutor + try: + executor = get_executor() + + # 替換 {target} 為實際目標 + command = step.command + if incident.affected_services: + command = command.replace("{target}", incident.affected_services[0]) + + result = await executor.execute_kubectl_command(command) + return "SUCCESS" if result.success else f"FAILED: {result.error}" + + except ImportError: + logger.warning("action_executor_not_available") + return "SKIPPED (executor not available)" + + return "UNKNOWN_ACTION_TYPE" + + +# ============================================================================= +# Singleton +# ============================================================================= + +_service: AutoRepairService | None = None + + +def get_auto_repair_service() -> IAutoRepairService: + """取得 AutoRepairService 單例""" + global _service + if _service is None: + _service = AutoRepairService() + return _service + + +def set_auto_repair_service(service: AutoRepairService | None) -> None: + """注入 AutoRepairService 實例 (用於 DI 或測試)""" + global _service + _service = service diff --git a/apps/api/src/services/ci_auto_repair.py b/apps/api/src/services/ci_auto_repair.py index 4d5594ac8..579a45313 100644 --- a/apps/api/src/services/ci_auto_repair.py +++ b/apps/api/src/services/ci_auto_repair.py @@ -21,14 +21,13 @@ CI 失敗自動修復服務，根據風險分級決定執行策略 from __future__ import annotations -import asyncio from dataclasses import dataclass from enum import Enum import structlog -from src.services.intent_classifier import IntentType, RiskLevel, get_intent_classifier from src.services.complexity_scorer import get_complexity_scorer +from src.services.intent_classifier import IntentType, RiskLevel, get_intent_classifier logger = structlog.get_logger(__name__) diff --git a/apps/api/src/services/error_analyzer_service.py b/apps/api/src/services/error_analyzer_service.py new file mode 100644 index 000000000..9cc35ff63 --- /dev/null +++ b/apps/api/src/services/error_analyzer_service.py @@ -0,0 +1,369 @@ +""" +Error Analyzer Service - #39 Sentry 錯誤 AI 分析 +================================================= +Phase 10: Sentry + OpenClaw + UI 整合 + +功能: +1. 接收 Sentry Issue + Stacktrace 數據 +2. 使用 OpenClaw LLM 進行根因分析 +3. 生成修復建議與預防措施 + +遵循 leWOOOgo 積木化原則: +- Service 層負責業務邏輯 +- 不直接存取 Redis/DB +- 使用 DI 支援測試 + +版本: v1.0 +建立: 2026-03-26 18:45 (台北時區) +建立者: Claude Code (#39 Error Analyzer Agent) +""" + +import json +from typing import Protocol, runtime_checkable + +from pydantic import BaseModel, Field + +from src.core.logging import get_logger +from src.utils.timezone import now_taipei_iso + +logger = get_logger("awoooi.error_analyzer") + + +# ============================================================================= +# Error Analysis Prompt +# ============================================================================= + +ERROR_ANALYZER_SYSTEM_PROMPT = """# OpenClaw Error Analyzer - AWOOOI 錯誤分析專家 + +You are a senior Software Engineer specialized in debugging and error analysis. + +## 🌐 Language Requirement (CRITICAL) +- You MUST respond in **Traditional Chinese (繁體中文/正體中文)** for all text fields +- FORBIDDEN: Simplified Chinese characters (简体字) +- Use Taiwan locale conventions (台灣用語) + +## 🎯 Your Mission +Analyze the given error from Sentry and provide: +1. **Root Cause Analysis** - Why did this error occur? +2. **Impact Assessment** - How serious is this error? +3. **Fix Recommendations** - How to fix this error? +4. **Prevention Suggestions** - How to prevent recurrence? + +## 📊 Analysis Categories +- **CODE_BUG**: Logic error, null pointer, type error +- **DEPENDENCY**: Third-party library issue, version conflict +- **CONFIGURATION**: Missing env var, wrong config +- **INFRASTRUCTURE**: Network, timeout, resource exhaustion +- **DATA_INTEGRITY**: Corrupt data, schema mismatch +- **EXTERNAL_SERVICE**: API failure, rate limit +- **UNKNOWN**: Cannot determine from available information + +## ⚠️ Output Rules +- Respond with ONLY valid JSON +- confidence MUST be between 0.0 and 1.0 +- severity MUST be one of: LOW, MEDIUM, HIGH, CRITICAL +- All text fields in Traditional Chinese + +## 📋 JSON Schema (REQUIRED) +```json +{ + "root_cause": "string - 根因分析 (繁體中文)", + "category": "CODE_BUG|DEPENDENCY|CONFIGURATION|INFRASTRUCTURE|DATA_INTEGRITY|EXTERNAL_SERVICE|UNKNOWN", + "severity": "LOW|MEDIUM|HIGH|CRITICAL", + "impact_assessment": "string - 影響評估 (繁體中文)", + "fix_recommendation": { + "summary": "string - 修復摘要", + "steps": ["array - 修復步驟"], + "code_suggestion": "string | null - 建議的代碼修改" + }, + "prevention": [ + { + "type": "CODE_REVIEW|UNIT_TEST|MONITORING|VALIDATION|ERROR_HANDLING", + "description": "string - 預防措施描述" + } + ], + "related_files": ["array - 可能相關的檔案路徑"], + "confidence": "number - 0.0 to 1.0", + "reasoning": "string - 分析推理過程 (繁體中文)" +} +``` + +Now analyze the following error: +""" + + +# ============================================================================= +# Response Models +# ============================================================================= + + +class FixRecommendation(BaseModel): + """修復建議""" + + summary: str = Field(description="修復摘要") + steps: list[str] = Field(default_factory=list, description="修復步驟") + code_suggestion: str | None = Field(None, description="建議的代碼修改") + + +class PreventionMeasure(BaseModel): + """預防措施""" + + type: str = Field(description="類型 (CODE_REVIEW, UNIT_TEST, etc.)") + description: str = Field(description="描述") + + +class ErrorAnalysisResult(BaseModel): + """錯誤分析結果""" + + root_cause: str = Field(description="根因分析") + category: str = Field(description="分類") + severity: str = Field(description="嚴重度") + impact_assessment: str = Field(description="影響評估") + fix_recommendation: FixRecommendation = Field(description="修復建議") + prevention: list[PreventionMeasure] = Field( + default_factory=list, description="預防措施" + ) + related_files: list[str] = Field(default_factory=list, description="相關檔案") + confidence: float = Field(description="信心度") + reasoning: str = Field(description="分析推理過程") + + +# ============================================================================= +# Protocol Interface +# ============================================================================= + + +@runtime_checkable +class ILLMProvider(Protocol): + """LLM Provider Protocol""" + + async def call(self, prompt: str) -> tuple[str, str, bool]: + """ + 呼叫 LLM + + Returns: + (response, provider_name, success) + """ + ... + + +# ============================================================================= +# Error Analyzer Service +# ============================================================================= + + +class ErrorAnalyzerService: + """ + Error Analyzer Service - Sentry 錯誤 AI 分析 + + 職責: + 1. 組裝分析 Prompt + 2. 呼叫 OpenClaw LLM + 3. 解析並驗證分析結果 + """ + + def __init__(self, llm_provider: ILLMProvider | None = None) -> None: + """ + 初始化 Error Analyzer Service + + Args: + llm_provider: LLM 提供者 (預設使用 OpenClaw) + """ + self._llm_provider = llm_provider + + async def _get_llm_provider(self) -> ILLMProvider: + """取得 LLM Provider (lazy init)""" + if self._llm_provider is None: + from src.services.openclaw import get_openclaw + + self._llm_provider = get_openclaw() + return self._llm_provider + + async def analyze_error( + self, + issue_id: str, + title: str, + level: str, + culprit: str | None, + count: int, + stacktrace: str, + context: dict | None = None, + ) -> tuple[ErrorAnalysisResult | None, str, bool]: + """ + 分析 Sentry 錯誤 + + Args: + issue_id: Sentry Issue ID + title: 錯誤標題 + level: 嚴重度 (error, warning, etc.) + culprit: 錯誤來源 (函數/檔案) + count: 發生次數 + stacktrace: 堆疊追蹤 + context: 額外上下文 (browser, os, tags, etc.) + + Returns: + (analysis_result, provider, success) + """ + # 組裝 Prompt + error_context = { + "issue_id": issue_id, + "title": title, + "level": level, + "culprit": culprit, + "occurrence_count": count, + "stacktrace": stacktrace, + "context": context or {}, + "analyzed_at": now_taipei_iso(), + } + + prompt = ERROR_ANALYZER_SYSTEM_PROMPT + "\n```json\n" + prompt += json.dumps(error_context, ensure_ascii=False, indent=2) + prompt += "\n```" + + logger.info( + "error_analysis_start", + issue_id=issue_id, + title=title, + level=level, + ) + + # 呼叫 LLM + try: + llm = await self._get_llm_provider() + response, provider, success = await llm.call(prompt) + + if not success: + logger.error( + "error_analysis_llm_failed", + issue_id=issue_id, + provider=provider, + ) + return None, provider, False + + logger.info( + "error_analysis_llm_response", + issue_id=issue_id, + provider=provider, + response_length=len(response), + ) + + # 解析結果 + result = self._parse_analysis_result(response) + + if result: + logger.info( + "error_analysis_complete", + issue_id=issue_id, + category=result.category, + severity=result.severity, + confidence=result.confidence, + ) + else: + logger.warning( + "error_analysis_parse_failed", + issue_id=issue_id, + raw_response=response[:300], + ) + + return result, provider, True + + except Exception as e: + logger.exception( + "error_analysis_failed", + issue_id=issue_id, + error=str(e), + ) + return None, "error", False + + def _parse_analysis_result(self, raw_response: str) -> ErrorAnalysisResult | None: + """ + 解析 LLM 回應為結構化結果 + + Args: + raw_response: LLM 原始回應 + + Returns: + 解析後的 ErrorAnalysisResult，解析失敗返回 None + """ + try: + # 嘗試找到 JSON 區塊 + json_str = raw_response + + # 處理可能的 markdown 包裝 + if "```json" in raw_response: + start = raw_response.find("```json") + 7 + end = raw_response.find("```", start) + if end > start: + json_str = raw_response[start:end] + elif "```" in raw_response: + start = raw_response.find("```") + 3 + end = raw_response.find("```", start) + if end > start: + json_str = raw_response[start:end] + + # 解析 JSON + data = json.loads(json_str.strip()) + + # 建立 FixRecommendation + fix_data = data.get("fix_recommendation", {}) + fix_recommendation = FixRecommendation( + summary=fix_data.get("summary", "無建議"), + steps=fix_data.get("steps", []), + code_suggestion=fix_data.get("code_suggestion"), + ) + + # 建立 PreventionMeasure 列表 + prevention = [] + for p in data.get("prevention", []): + prevention.append(PreventionMeasure( + type=p.get("type", "UNKNOWN"), + description=p.get("description", ""), + )) + + # 建立最終結果 + return ErrorAnalysisResult( + root_cause=data.get("root_cause", "無法判斷根因"), + category=data.get("category", "UNKNOWN"), + severity=data.get("severity", "MEDIUM"), + impact_assessment=data.get("impact_assessment", "影響評估中"), + fix_recommendation=fix_recommendation, + prevention=prevention, + related_files=data.get("related_files", []), + confidence=float(data.get("confidence", 0.5)), + reasoning=data.get("reasoning", ""), + ) + + except json.JSONDecodeError as e: + logger.warning( + "error_analysis_json_decode_failed", + error=str(e), + raw_response=raw_response[:200], + ) + return None + except Exception as e: + logger.warning( + "error_analysis_parse_error", + error=str(e), + ) + return None + + +# ============================================================================= +# Singleton +# ============================================================================= + +_error_analyzer_service: ErrorAnalyzerService | None = None + + +def get_error_analyzer_service() -> ErrorAnalyzerService: + """取得 Error Analyzer Service 實例 (Singleton)""" + global _error_analyzer_service + if _error_analyzer_service is None: + _error_analyzer_service = ErrorAnalyzerService() + return _error_analyzer_service + + +def set_error_analyzer_service(service: ErrorAnalyzerService) -> None: + """設定 Error Analyzer Service 實例 (for testing)""" + global _error_analyzer_service + _error_analyzer_service = service diff --git a/apps/api/src/services/playbook_service.py b/apps/api/src/services/playbook_service.py index de8874e37..e3fed15b4 100644 --- a/apps/api/src/services/playbook_service.py +++ b/apps/api/src/services/playbook_service.py @@ -13,7 +13,6 @@ Phase 7.3: Service 實作 - 封裝所有業務邏輯 """ -from datetime import UTC, datetime from typing import Protocol import structlog @@ -31,6 +30,7 @@ from src.models.playbook import ( ) from src.repositories.interfaces import IPlaybookRepository from src.repositories.playbook_repository import get_playbook_repository +from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) @@ -246,7 +246,7 @@ class PlaybookService: playbook.status = PlaybookStatus.APPROVED playbook.approved_by = approved_by - playbook.approved_at = datetime.now(UTC) + playbook.approved_at = now_taipei() if notes: playbook.notes = notes @@ -294,6 +294,83 @@ class PlaybookService: """更新 Playbook""" return await self._repository.update(playbook) + async def update_with_validation( + self, + playbook_id: str, + update_data: dict, + ) -> Playbook | None: + """ + 更新 Playbook (含驗證) + + Phase 8 P1 修復: 從 Router 層移至 Service 層進行驗證 + + 驗證規則: + - 禁止直接修改 playbook_id + - 禁止反向狀態轉換 (APPROVED → DRAFT) + - 統計欄位 (success_count, failure_count) 只能透過 record_execution 更新 + + Args: + playbook_id: Playbook ID + update_data: 要更新的欄位 (dict) + + Returns: + 更新後的 Playbook 或 None + """ + playbook = await self._repository.get_by_id(playbook_id) + if not playbook: + return None + + # 禁止修改的欄位 + forbidden_fields = { + "playbook_id", + "created_at", + "success_count", + "failure_count", + "last_used_at", + } + + for field in forbidden_fields: + if field in update_data: + logger.warning( + "playbook_update_forbidden_field", + playbook_id=playbook_id, + field=field, + ) + del update_data[field] + + # 狀態轉換驗證 + if "status" in update_data: + new_status = update_data["status"] + current_status = playbook.status + + # 允許的轉換: DRAFT → APPROVED, APPROVED → DEPRECATED + # 禁止: APPROVED → DRAFT, DEPRECATED → 任何 + if current_status == PlaybookStatus.DEPRECATED: + logger.warning( + "playbook_update_deprecated_status", + playbook_id=playbook_id, + ) + return None + + if ( + current_status == PlaybookStatus.APPROVED + and new_status == PlaybookStatus.DRAFT + ): + logger.warning( + "playbook_update_invalid_status_transition", + playbook_id=playbook_id, + from_status=current_status.value, + to_status=new_status, + ) + return None + + # 應用更新 + for field, value in update_data.items(): + if value is not None and hasattr(playbook, field): + setattr(playbook, field, value) + + return await self._repository.update(playbook) + async def delete(self, playbook_id: str) -> bool: """刪除 Playbook (軟刪除)""" return await self._repository.delete(playbook_id) diff --git a/apps/api/src/services/resource_resolver.py b/apps/api/src/services/resource_resolver.py index c4debe784..847a6ed73 100644 --- a/apps/api/src/services/resource_resolver.py +++ b/apps/api/src/services/resource_resolver.py @@ -26,7 +26,6 @@ from typing import Protocol, runtime_checkable import structlog from src.utils.k8s_naming import ( - NormalizeResult, ResourceType, extract_resource_hints, normalize_resource_name, diff --git a/apps/api/src/services/sentry_service.py b/apps/api/src/services/sentry_service.py new file mode 100644 index 000000000..93e993fd8 --- /dev/null +++ b/apps/api/src/services/sentry_service.py @@ -0,0 +1,218 @@ +""" +Sentry Service - Sentry API 封裝 +================================ +Phase 10: Sentry + OpenClaw + UI 整合 + +遵循 leWOOOgo 積木化原則: +- Service 層負責外部 API 呼叫 +- Router 層只做 HTTP 轉發 +- 單一職責: 只處理 Sentry API 互動 + +版本: v1.0 +建立: 2026-03-26 21:15 (台北時區) +建立者: Claude Code (P2 架構改善) +""" + +from typing import Any + +import httpx + +from src.core.config import settings +from src.core.logging import get_logger + +logger = get_logger("awoooi.sentry") + + +class SentryService: + """ + Sentry API Service + + 職責: + 1. 封裝 Sentry API 呼叫 + 2. 處理認證與錯誤 + 3. 提供類型化的資料存取 + """ + + def __init__( + self, + base_url: str | None = None, + org: str | None = None, + project: str | None = None, + auth_token: str | None = None, + timeout: float = 10.0, + ) -> None: + """ + 初始化 Sentry Service + + Args: + base_url: Sentry API URL (預設從 settings) + org: Sentry organization slug + project: Sentry project slug + auth_token: API auth token + timeout: 請求超時秒數 + """ + self.base_url = base_url or settings.SENTRY_SELF_HOSTED_URL + self.org = org or settings.SENTRY_ORG + self.project = project or settings.SENTRY_PROJECT + self.auth_token = auth_token or settings.SENTRY_AUTH_TOKEN + self.timeout = timeout + + async def _request( + self, + endpoint: str, + params: dict[str, Any] | None = None, + ) -> dict | list | None: + """ + 發送 Sentry API 請求 + + Args: + endpoint: API 端點 (不含 /api/0/ 前綴) + params: 查詢參數 + + Returns: + JSON 回應，失敗返回 None + """ + headers = {} + if self.auth_token: + headers["Authorization"] = f"Bearer {self.auth_token}" + + url = f"{self.base_url}/api/0/{endpoint}" + + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get(url, headers=headers, params=params) + + if response.status_code == 200: + return response.json() + elif response.status_code == 401: + logger.warning("sentry_api_unauthorized", endpoint=endpoint) + return None + else: + logger.warning( + "sentry_api_error", + status_code=response.status_code, + endpoint=endpoint, + ) + return None + + except httpx.TimeoutException: + logger.error("sentry_api_timeout", endpoint=endpoint) + return None + except Exception as e: + logger.error("sentry_api_failed", endpoint=endpoint, error=str(e)) + return None + + # ========================================================================= + # Organization APIs + # ========================================================================= + + async def list_projects(self) -> list[dict] | None: + """取得組織內所有專案""" + return await self._request(f"organizations/{self.org}/projects/") + + # ========================================================================= + # Project APIs + # ========================================================================= + + async def list_issues( + self, + project: str | None = None, + query: str = "is:unresolved", + limit: int = 25, + cursor: str | None = None, + ) -> list[dict] | None: + """ + 列出專案 Issues + + Args: + project: 專案 slug (預設使用設定值) + query: Sentry 搜尋語法 + limit: 每頁數量 + cursor: 分頁游標 + """ + project_slug = project or self.project + params: dict[str, Any] = {"query": query, "limit": limit} + if cursor: + params["cursor"] = cursor + + return await self._request( + f"projects/{self.org}/{project_slug}/issues/", + params=params, + ) + + async def get_project_stats( + self, + project: str | None = None, + stat: str = "received", + resolution: str = "1h", + ) -> list | None: + """ + 取得專案統計數據 + + Args: + project: 專案 slug + stat: 統計類型 (received, rejected, blacklisted) + resolution: 時間解析度 (1h, 1d, etc.) + """ + project_slug = project or self.project + return await self._request( + f"projects/{self.org}/{project_slug}/stats/", + params={"stat": stat, "resolution": resolution}, + ) + + # ========================================================================= + # Issue APIs + # ========================================================================= + + async def get_issue(self, issue_id: str) -> dict | None: + """取得 Issue 詳情""" + return await self._request(f"issues/{issue_id}/") + + async def get_issue_events( + self, + issue_id: str, + limit: int = 1, + full: bool = False, + ) -> list[dict] | None: + """ + 取得 Issue 事件 + + Args: + issue_id: Issue ID + limit: 事件數量 + full: 是否包含完整堆疊 + """ + params: dict[str, Any] = {"limit": limit} + if full: + params["full"] = "true" + + return await self._request(f"issues/{issue_id}/events/", params=params) + + # ========================================================================= + # Helper Methods + # ========================================================================= + + def get_issue_url(self, issue_id: str) -> str: + """取得 Issue 在 Sentry UI 的連結""" + return f"{self.base_url}/organizations/{self.org}/issues/{issue_id}/" + + +# ============================================================================= +# Singleton +# ============================================================================= + +_sentry_service: SentryService | None = None + + +def get_sentry_service() -> SentryService: + """取得 Sentry Service 實例 (Singleton)""" + global _sentry_service + if _sentry_service is None: + _sentry_service = SentryService() + return _sentry_service + + +def set_sentry_service(service: SentryService) -> None: + """設定 Sentry Service 實例 (for testing)""" + global _sentry_service + _sentry_service = service diff --git a/apps/api/src/services/stats_service.py b/apps/api/src/services/stats_service.py index bfcfaa76f..4d34c6848 100644 --- a/apps/api/src/services/stats_service.py +++ b/apps/api/src/services/stats_service.py @@ -13,7 +13,8 @@ Stats Service - Phase 17 P0 Router 層違規修復 """ import json -from typing import Any, Callable, Coroutine +from collections.abc import Callable, Coroutine +from typing import Any import structlog diff --git a/apps/api/src/services/token_counter.py b/apps/api/src/services/token_counter.py index 2cea3efab..1e2f4be29 100644 --- a/apps/api/src/services/token_counter.py +++ b/apps/api/src/services/token_counter.py @@ -32,7 +32,7 @@ SignOz 指標: import time from dataclasses import dataclass, field -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from typing import Protocol import structlog @@ -40,7 +40,6 @@ from opentelemetry import metrics from opentelemetry.metrics import Counter, Histogram, Meter from src.core.config import settings -from src.services.langfuse_client import get_langfuse logger = structlog.get_logger(__name__) @@ -450,7 +449,7 @@ class TokenCounter: # 建議 recommendation = "" if is_over_budget: - recommendation = f"建議切換到本地模型 (Ollama) 以節省成本" + recommendation = "建議切換到本地模型 (Ollama) 以節省成本" elif alert_triggered: recommendation = f"接近預算上限 ({max(daily_usage_percent, monthly_usage_percent):.1f}%)，考慮減少 {provider} 呼叫" diff --git a/apps/api/tests/llm_testing/__init__.py b/apps/api/tests/llm_testing/__init__.py index e959f196d..ae2524d17 100644 --- a/apps/api/tests/llm_testing/__init__.py +++ b/apps/api/tests/llm_testing/__init__.py @@ -13,10 +13,10 @@ LLM Testing Framework - ADR-018 Implementation from .golden_responses import GoldenResponseManager from .property_validators import ( - validate_kubectl_syntax, - validate_risk_level, validate_chinese_ratio, + validate_kubectl_syntax, validate_response_length, + validate_risk_level, ) from .schema_validators import LLMProposalOutput, validate_proposal_schema diff --git a/apps/api/tests/llm_testing/schema_validators.py b/apps/api/tests/llm_testing/schema_validators.py index fce2cfade..16c346fbb 100644 --- a/apps/api/tests/llm_testing/schema_validators.py +++ b/apps/api/tests/llm_testing/schema_validators.py @@ -9,7 +9,7 @@ Schema Validators - Tier 1 測試 import json import re -from typing import Any, Literal +from typing import Literal from pydantic import BaseModel, Field, ValidationError diff --git a/apps/api/tests/test_auto_repair_service.py b/apps/api/tests/test_auto_repair_service.py new file mode 100644 index 000000000..cf01e9f59 --- /dev/null +++ b/apps/api/tests/test_auto_repair_service.py @@ -0,0 +1,285 @@ +""" +Auto Repair Service Tests - #8 自動升級決策 +========================================== +測試自動修復服務層功能 + +版本: v1.0 +建立: 2026-03-26 (台北時區) +建立者: Claude Code (#8 自動升級決策) +""" + +import pytest + +from src.models.incident import Incident, IncidentStatus, Severity, Signal +from src.models.playbook import ( + ActionType, + Playbook, + PlaybookStatus, + RepairStep, + RiskLevel, + SymptomPattern, +) +from src.services.auto_repair_service import AutoRepairService +from src.utils.timezone import now_taipei + + +class MockPlaybookService: + """Mock playbook service for testing""" + + def __init__(self): + self._playbooks: dict[str, Playbook] = {} + self._recommendations: list = [] + + def add_playbook(self, playbook: Playbook): + self._playbooks[playbook.playbook_id] = playbook + + def set_recommendations(self, recommendations: list): + self._recommendations = recommendations + + async def get_recommendations(self, symptoms, top_k=3): + return self._recommendations + + async def get_by_id(self, playbook_id: str): + return self._playbooks.get(playbook_id) + + async def record_execution(self, playbook_id: str, success: bool): + playbook = self._playbooks.get(playbook_id) + if playbook: + if success: + playbook.success_count += 1 + else: + playbook.failure_count += 1 + return playbook is not None + + +def create_test_incident( + incident_id: str = "INC-TEST-001", + severity: Severity = Severity.P2, +) -> Incident: + """Create a test incident""" + now = now_taipei() + return Incident( + incident_id=incident_id, + status=IncidentStatus.OPEN, + severity=severity, + affected_services=["test-service"], + signals=[ + Signal( + alert_name="HighCPU", + severity=severity, + source="prometheus", + fired_at=now, + labels={"namespace": "prod"}, + ), + ], + ) + + +def create_high_quality_playbook( + playbook_id: str = "PB-TEST-001", + risk_level: RiskLevel = RiskLevel.MEDIUM, +) -> Playbook: + """Create a high quality playbook (success_rate >= 95%, count >= 10)""" + return Playbook( + playbook_id=playbook_id, + name="HighCPU - test-service 修復劇本", + description="High quality playbook for auto repair", + status=PlaybookStatus.APPROVED, + symptom_pattern=SymptomPattern( + alert_names=["HighCPU"], + affected_services=["test-service"], + severity_range=["P2"], + ), + repair_steps=[ + RepairStep( + step_number=1, + action_type=ActionType.KUBECTL, + command="kubectl rollout restart deployment/{target}", + risk_level=risk_level, + ), + ], + success_count=20, # >= 10 + failure_count=1, # success_rate = 95.2% + ai_confidence=0.9, + ) + + +class MockPlaybookRecommendation: + """Mock recommendation for testing""" + + def __init__(self, playbook: Playbook, similarity_score: float): + self.playbook = playbook + self.similarity_score = similarity_score + + +class TestAutoRepairService: + """Auto Repair Service unit tests""" + + @pytest.fixture + def mock_playbook_service(self): + return MockPlaybookService() + + @pytest.fixture + def service(self, mock_playbook_service): + return AutoRepairService(playbook_service=mock_playbook_service) + + @pytest.mark.asyncio + async def test_evaluate_blocks_p1_severity(self, service): + """Test that P1 severity incidents are blocked""" + incident = create_test_incident(severity=Severity.P1) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is False + assert decision.blocked_by == "HIGH_SEVERITY" + + @pytest.mark.asyncio + async def test_evaluate_blocks_p0_severity(self, service): + """Test that P0 severity incidents are blocked""" + incident = create_test_incident(severity=Severity.P0) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is False + assert decision.blocked_by == "HIGH_SEVERITY" + + @pytest.mark.asyncio + async def test_evaluate_no_playbook_match(self, service, mock_playbook_service): + """Test when no playbook matches""" + mock_playbook_service.set_recommendations([]) + incident = create_test_incident(severity=Severity.P2) + + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is False + assert decision.blocked_by == "NO_MATCH" + + @pytest.mark.asyncio + async def test_evaluate_low_similarity(self, service, mock_playbook_service): + """Test when similarity is too low""" + playbook = create_high_quality_playbook() + mock_playbook_service.add_playbook(playbook) + mock_playbook_service.set_recommendations([ + MockPlaybookRecommendation(playbook, similarity_score=0.5) # Below 0.7 + ]) + + incident = create_test_incident(severity=Severity.P2) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is False + assert decision.blocked_by == "LOW_SIMILARITY" + + @pytest.mark.asyncio + async def test_evaluate_not_high_quality(self, service, mock_playbook_service): + """Test when playbook is not high quality""" + playbook = Playbook( + playbook_id="PB-LOW-QUALITY", + name="Low quality playbook", + description="Not enough executions", + status=PlaybookStatus.APPROVED, + symptom_pattern=SymptomPattern( + alert_names=["HighCPU"], + affected_services=["test-service"], + ), + repair_steps=[], + success_count=5, # < 10 + failure_count=0, + ) + mock_playbook_service.add_playbook(playbook) + mock_playbook_service.set_recommendations([ + MockPlaybookRecommendation(playbook, similarity_score=0.9) + ]) + + incident = create_test_incident(severity=Severity.P2) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is False + assert decision.blocked_by == "NOT_HIGH_QUALITY" + + @pytest.mark.asyncio + async def test_evaluate_high_risk_blocked(self, service, mock_playbook_service): + """Test when playbook contains HIGH risk actions""" + playbook = create_high_quality_playbook(risk_level=RiskLevel.HIGH) + mock_playbook_service.add_playbook(playbook) + mock_playbook_service.set_recommendations([ + MockPlaybookRecommendation(playbook, similarity_score=0.9) + ]) + + incident = create_test_incident(severity=Severity.P2) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is False + assert decision.blocked_by == "HIGH_RISK" + + @pytest.mark.asyncio + async def test_evaluate_critical_risk_blocked(self, service, mock_playbook_service): + """Test when playbook contains CRITICAL risk actions""" + playbook = create_high_quality_playbook(risk_level=RiskLevel.CRITICAL) + mock_playbook_service.add_playbook(playbook) + mock_playbook_service.set_recommendations([ + MockPlaybookRecommendation(playbook, similarity_score=0.9) + ]) + + incident = create_test_incident(severity=Severity.P2) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is False + assert decision.blocked_by == "HIGH_RISK" + + @pytest.mark.asyncio + async def test_evaluate_success(self, service, mock_playbook_service): + """Test successful auto repair evaluation""" + playbook = create_high_quality_playbook(risk_level=RiskLevel.MEDIUM) + mock_playbook_service.add_playbook(playbook) + mock_playbook_service.set_recommendations([ + MockPlaybookRecommendation(playbook, similarity_score=0.85) + ]) + + incident = create_test_incident(severity=Severity.P2) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is True + assert decision.playbook is not None + assert decision.playbook.playbook_id == playbook.playbook_id + assert decision.blocked_by is None + + @pytest.mark.asyncio + async def test_evaluate_low_risk_allowed(self, service, mock_playbook_service): + """Test that LOW risk actions are allowed""" + playbook = create_high_quality_playbook(risk_level=RiskLevel.LOW) + mock_playbook_service.add_playbook(playbook) + mock_playbook_service.set_recommendations([ + MockPlaybookRecommendation(playbook, similarity_score=0.85) + ]) + + incident = create_test_incident(severity=Severity.P2) + decision = await service.evaluate_auto_repair(incident) + + assert decision.can_auto_repair is True + assert decision.risk_level == RiskLevel.LOW + + @pytest.mark.asyncio + async def test_is_high_quality_calculation(self): + """Test is_high_quality property""" + # High quality: APPROVED + 95%+ success rate + 10+ successes + playbook = create_high_quality_playbook() + assert playbook.is_high_quality is True + assert playbook.success_rate >= 0.95 + assert playbook.success_count >= 10 + + @pytest.mark.asyncio + async def test_not_high_quality_low_success_rate(self): + """Test playbook with low success rate is not high quality""" + playbook = Playbook( + playbook_id="PB-LOW-RATE", + name="Low success rate", + description="Too many failures", + status=PlaybookStatus.APPROVED, + symptom_pattern=SymptomPattern( + alert_names=["Test"], + affected_services=["test"], + ), + repair_steps=[], + success_count=15, + failure_count=5, # 75% success rate + ) + assert playbook.is_high_quality is False + assert playbook.success_rate < 0.95 diff --git a/apps/api/tests/test_llm_tier1_schema.py b/apps/api/tests/test_llm_tier1_schema.py index 2814c132b..42de5b461 100644 --- a/apps/api/tests/test_llm_tier1_schema.py +++ b/apps/api/tests/test_llm_tier1_schema.py @@ -22,14 +22,13 @@ from tests.llm_testing.property_validators import ( extract_kubectl_from_text, extract_risk_level_from_text, validate_kubectl_syntax, - validate_risk_level, validate_response_length, + validate_risk_level, ) from tests.llm_testing.schema_validators import ( validate_proposal_schema, ) - # ============================================================================= # Fixtures # ============================================================================= @@ -176,7 +175,7 @@ def test_tier1_summary(): total_golden = len(DEFAULT_GOLDEN_RESPONSES) valid_count = 0 - for name, golden in DEFAULT_GOLDEN_RESPONSES.items(): + for _name, golden in DEFAULT_GOLDEN_RESPONSES.items(): is_valid, _, _ = validate_proposal_schema(golden["response"]) if is_valid: valid_count += 1 diff --git a/apps/api/tests/test_model_regression.py b/apps/api/tests/test_model_regression.py index 34558eac4..f96c07c87 100644 --- a/apps/api/tests/test_model_regression.py +++ b/apps/api/tests/test_model_regression.py @@ -18,7 +18,7 @@ import pytest # Ollama 伺服器配置 OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.0.188:11434") DEFAULT_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:7b-instruct") -TIMEOUT = 120 # 秒 +TIMEOUT = 300 # 秒 (CPU 推理模式需 ~222-666 秒，見 2026-03-26 評估) # ============================================================================= @@ -68,7 +68,12 @@ REGRESSION_CASES = [ # ============================================================================= async def call_ollama(prompt: str, model: str = DEFAULT_MODEL) -> str | None: - """呼叫 Ollama API""" + """呼叫 Ollama API (含確定性參數) + + Phase 12.3 修正 (2026-03-26): + - 加入 temperature: 0.0 確保確定性輸出 + - 加入 seed: 42 確保可重現性 + """ try: async with httpx.AsyncClient(timeout=TIMEOUT) as client: response = await client.post( @@ -77,6 +82,10 @@ async def call_ollama(prompt: str, model: str = DEFAULT_MODEL) -> str | None: "model": model, "prompt": prompt, "stream": False, + "options": { + "temperature": 0.0, # 確定性輸出 + "seed": 42, # 可重現性 + }, }, ) response.raise_for_status() diff --git a/apps/api/tests/test_playbook_service.py b/apps/api/tests/test_playbook_service.py index 1608db8c9..8a0e5bc3a 100644 --- a/apps/api/tests/test_playbook_service.py +++ b/apps/api/tests/test_playbook_service.py @@ -8,8 +8,6 @@ Playbook Service Tests - #7 Playbook 萃取 建立者: Claude Code (Phase 7.5-7.6) """ -from datetime import UTC, datetime - import pytest from src.models.incident import ( @@ -28,6 +26,7 @@ from src.models.playbook import ( SymptomPattern, ) from src.services.playbook_service import PlaybookService +from src.utils.timezone import now_taipei class MockPlaybookRepository: @@ -123,7 +122,7 @@ def create_test_incident( """Create a test incident for extraction""" from src.models.incident import AIDecisionChain - now = datetime.now(UTC) + now = now_taipei() return Incident( incident_id=incident_id, status=status, diff --git a/apps/api/tests/test_prompt_validation.py b/apps/api/tests/test_prompt_validation.py index 3e46ba7c0..90c139467 100644 --- a/apps/api/tests/test_prompt_validation.py +++ b/apps/api/tests/test_prompt_validation.py @@ -15,29 +15,15 @@ from typing import Any import httpx import pytest +from src.core.prompts import OPENCLAW_TEST_PROMPT + # Ollama 配置 OLLAMA_URL = os.getenv("OLLAMA_URL", "http://192.168.0.188:11434") DEFAULT_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:7b-instruct") -TIMEOUT = 120 +TIMEOUT = 300 # 秒 (CPU 推理模式需 ~222-666 秒，見 2026-03-26 評估) - -# ============================================================================= -# System Prompts (與 OpenClaw 同步) -# ============================================================================= - -AWOOOI_SYSTEM_PROMPT = """你是 AWOOOI AIOps 平台的智慧助手 OpenClaw。 - -職責: -1. 分析告警並診斷根因 -2. 生成修復提案 (kubectl 命令) -3. 評估操作風險等級 (LOW/MEDIUM/HIGH/CRITICAL) - -規則: -- 只建議安全且可逆的操作 -- 高風險操作必須標記 CRITICAL -- 使用繁體中文回應 -- 回應簡潔，不超過 100 字 -""" +# 使用集中式 Prompt (Phase 17 P2 改進) +AWOOOI_SYSTEM_PROMPT = OPENCLAW_TEST_PROMPT # ============================================================================= @@ -98,7 +84,12 @@ async def call_with_system_prompt( user_prompt: str, model: str = DEFAULT_MODEL, ) -> str | None: - """使用 System Prompt 呼叫模型""" + """使用 System Prompt 呼叫模型 (含確定性參數) + + Phase 12.3 修正 (2026-03-26): + - 加入 temperature: 0.0 確保確定性輸出 + - 加入 seed: 42 確保可重現性 + """ try: async with httpx.AsyncClient(timeout=TIMEOUT) as client: # Ollama chat API @@ -111,6 +102,10 @@ async def call_with_system_prompt( {"role": "user", "content": user_prompt}, ], "stream": False, + "options": { + "temperature": 0.0, # 確定性輸出 + "seed": 42, # 可重現性 + }, }, ) response.raise_for_status()