diff --git a/.agents/skills/02-lewooogo-backend-core.md b/.agents/skills/02-lewooogo-backend-core.md index 1d7bd79f1..ef3371aea 100644 --- a/.agents/skills/02-lewooogo-backend-core.md +++ b/.agents/skills/02-lewooogo-backend-core.md @@ -6,7 +6,7 @@ --- -## 核心約束 (Four Iron Laws) +## 核心約束 (Six Iron Laws) ### 1. Async-First (非同步優先) @@ -62,7 +62,23 @@ print(f"Signal {signal.id} processed") import logging # 原生 logging ``` -### 5. ADR-013 Google Style Docstring +### 5. 台北時區鐵律 (禁止 UTC) + +```python +# ❌ 禁止 - UTC 時區 +from datetime import UTC +datetime.now(UTC) +datetime.utcnow() + +# ✅ 正確 - 台北時區工具 +from src.utils.timezone import now_taipei, now_taipei_iso +now_taipei() # datetime with +08:00 +now_taipei_iso() # '2026-03-25T02:08:04+08:00' +``` + +**Memory 參考**: `feedback_timezone_taipei.md` + +### 6. ADR-013 Google Style Docstring **強制場景**: 安全相關、複雜邏輯、外部依賴、危險操作 diff --git a/.agents/skills/04-awoooi-devops-commander.md b/.agents/skills/04-awoooi-devops-commander.md index 56755577e..ab3719750 100644 --- a/.agents/skills/04-awoooi-devops-commander.md +++ b/.agents/skills/04-awoooi-devops-commander.md @@ -22,6 +22,7 @@ ❌ 禁止在 .188 以外部署決策 AI (OpenClaw, Ollama) ❌ 禁止 K3s 直接訪問公網 (必須透過 .188 Nginx) ❌ 禁止繞過 NetworkPolicy 直連 Pod +❌ 禁止使用 UTC 時區 (必須台北 +8,參考 feedback_timezone_taipei.md) ``` --- @@ -400,9 +401,59 @@ curl -f https://api.awoooi.wooo.work/api/v1/health --- +## 🚨 Alertmanager 路由鐵律 (2026-03-25 災難) + +> **事故**: Claude 錯誤將 Alertmanager 指向 OpenClaw,導致 Telegram 發送**舊 AIOPS 格式** + +### 正確架構 + +``` +Alertmanager → AWOOOI API (K3s) → Telegram (AWOOOI 格式) +``` + +### 鐵律: Alertmanager 必須指向 AWOOOI API + +```yaml +# ✅ 正確配置 (/home/ollama/momo-pro/monitoring/alertmanager.yml) +receivers: + - name: awoooi + webhook_configs: + - url: 'http://192.168.0.120:32334/api/v1/webhooks/alertmanager' + send_resolved: true +``` + +### 禁止的配置 + +```yaml +# ❌ 禁止指向 OpenClaw (使用舊格式) +url: 'http://192.168.0.188:8088/api/v1/webhook/alertmanager' +``` + +### 職責分工 + +| 系統 | 職責 | Telegram 權限 | +|------|------|--------------| +| **AWOOOI API** (K3s 32334) | 告警處理 + 格式化 + 發送 | ✅ | +| **OpenClaw** (188:8088) | AI 大腦、LLM 分析 | ❌ 不處理告警發送 | + +### 驗證方式 + +```bash +# 檢查 Alertmanager 配置 +ssh ollama@192.168.0.188 "cat /home/ollama/momo-pro/monitoring/alertmanager.yml" +# url 必須包含 192.168.0.120:32334 + +# Telegram 格式驗證 +# AWOOOI 格式: ═══════════════ 🚨 CRITICAL | xxx ═══════════════ +# OpenClaw 舊格式: 🤝 [協同警報] xxx ← 禁止! +``` + +--- + ## 參考文檔 - `k8s/awoooi-prod/`: K8s Manifests - `.github/workflows/cd.yaml`: CD Pipeline - `docs/HARD_RULES.md`: 絕對禁止規則 - `reference_four_hosts.md`: 主機架構參考 +- `feedback_alertmanager_awoooi_flow.md`: **🔴 Alertmanager 正確流程** diff --git a/apps/api/src/api/v1/audit_logs.py b/apps/api/src/api/v1/audit_logs.py index 5392df545..32daeb6c4 100644 --- a/apps/api/src/api/v1/audit_logs.py +++ b/apps/api/src/api/v1/audit_logs.py @@ -11,9 +11,11 @@ Endpoints: 提供 K8s 操作執行的完整審計軌跡。 """ -from datetime import UTC, datetime +from datetime import timedelta from typing import Any +from src.utils.timezone import now_taipei + from fastapi import APIRouter, HTTPException, Query, status from pydantic import BaseModel from sqlalchemy import func, select @@ -233,7 +235,7 @@ async def get_audit_stats() -> AuditStatsResponse: by_namespace = {row[0]: row[1] for row in ns_result.fetchall()} # Last 24 hours - cutoff = datetime.now(UTC) - timedelta(hours=24) + cutoff = now_taipei() - timedelta(hours=24) last24_result = await db.execute( select(func.count(AuditLog.id)).where(AuditLog.created_at >= cutoff) ) diff --git a/apps/api/src/api/v1/incidents.py b/apps/api/src/api/v1/incidents.py index a8a311057..800041758 100644 --- a/apps/api/src/api/v1/incidents.py +++ b/apps/api/src/api/v1/incidents.py @@ -17,9 +17,10 @@ Phase 6.4 核心功能: - Proposal 必須關聯到 Incident """ -from datetime import UTC from typing import Any +from src.utils.timezone import now_taipei + from fastapi import APIRouter, HTTPException, status from pydantic import BaseModel, Field @@ -456,7 +457,7 @@ async def submit_feedback( if request.learning_notes is not None: incident.outcome.learning_notes = request.learning_notes incident.outcome.should_remember = request.should_remember - incident.updated_at = datetime.now(UTC) + incident.updated_at = now_taipei() # 3. 寫入 Redis try: @@ -488,7 +489,7 @@ async def submit_feedback( if record: record.outcome = incident.outcome.model_dump(mode="json") - record.updated_at = datetime.now(UTC) + record.updated_at = now_taipei() await db.commit() logger.info( "feedback_db_updated", @@ -569,7 +570,7 @@ async def debug_resolve_incident(incident_id: str) -> dict[str, Any]: if data: incident = Incident.model_validate_json(data) incident.status = IncidentStatus.RESOLVED - incident.updated_at = datetime.now(UTC) + incident.updated_at = now_taipei() await redis_client.set( f"incident:{incident_id}", incident.model_dump_json(), @@ -588,7 +589,7 @@ async def debug_resolve_incident(incident_id: str) -> dict[str, Any]: record = result.scalar_one_or_none() if record: record.status = "resolved" - record.updated_at = datetime.now(UTC) + record.updated_at = now_taipei() await db.commit() db_updated = True except Exception as e: @@ -729,7 +730,7 @@ async def sync_incidents_from_approvals() -> SyncResult: affected_services=[resource_name], proposal_ids=[UUID(approval_id)], created_at=approval.created_at, - updated_at=datetime.now(UTC), + updated_at=now_taipei(), ) key = f"incident:{incident.incident_id}" diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index a2f085097..2f8b8f007 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -8,10 +8,15 @@ AWOOOI API - Sentry Webhook Handler 2. 組裝錯誤上下文 3. 呼叫 OpenClaw Error Analyzer Agent 4. 結果回寫 Sentry Issue Comment +5. 發送 Telegram 告警 (含截圖) +6. 建立 Approval 供人工審核 + +🔴 HARD RULE: 時間顯示使用 Asia/Taipei (UTC+8) """ import logging -from datetime import datetime + +from src.utils.timezone import now_taipei, now_taipei_iso import httpx from fastapi import APIRouter, BackgroundTasks, HTTPException, Request @@ -227,7 +232,7 @@ async def post_sentry_comment( {analysis.prevention} --- -*分析信心度: {analysis.confidence:.0%} | 分析時間: {datetime.now().isoformat()}* +*分析信心度: {analysis.confidence:.0%} | 分析時間: {now_taipei_iso()}* """ try: diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 2410ed895..e6dd2c195 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -24,9 +24,10 @@ Endpoints: import hashlib import hmac -from datetime import UTC, datetime from typing import Literal +from src.utils.timezone import now_taipei + from fastapi import APIRouter, BackgroundTasks, Header, HTTPException, Request, status from pydantic import BaseModel, Field @@ -103,7 +104,7 @@ async def create_incident_for_approval( alert_name=alert_type, severity=severity, source=source, - fired_at=datetime.now(UTC), + fired_at=now_taipei(), labels={"namespace": namespace, "resource": target_resource}, annotations={"message": message}, ) @@ -552,7 +553,7 @@ async def produce_signal_to_stream(signal: SignalPayload) -> str: "message": signal.message, "labels": str(signal.labels or {}), "annotations": str(signal.annotations or {}), - "received_at": datetime.now(UTC).isoformat(), + "received_at": now_taipei().isoformat(), } # XADD 寫入 Stream @@ -803,7 +804,7 @@ async def receive_alert( detail=f"HMAC verification failed: {str(e)}", ) from e - alert_id = f"alert-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}" + alert_id = f"alert-{now_taipei().strftime('%Y%m%d%H%M%S')}" # ========================================================================== # 戰略 B Step 1: 生成告警指紋 @@ -1154,7 +1155,7 @@ async def alertmanager_webhook( ) alert = firing_alerts[0] - alert_id = f"alert-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}" + alert_id = f"alert-{now_taipei().strftime('%Y%m%d%H%M%S')}" # ========================================================================== # Alert Normalizer: 轉換 Alertmanager 格式 → AWOOOI AlertPayload diff --git a/apps/api/src/plugins/finops/cost_analyzer.py b/apps/api/src/plugins/finops/cost_analyzer.py index 5d1fd889b..3d207f565 100644 --- a/apps/api/src/plugins/finops/cost_analyzer.py +++ b/apps/api/src/plugins/finops/cost_analyzer.py @@ -14,7 +14,9 @@ Phase 3.3: 商業變現能力 - Day-1 ROI import logging from dataclasses import dataclass, field -from datetime import datetime, timedelta +from datetime import timedelta + +from src.utils.timezone import now_taipei from enum import Enum from typing import Literal @@ -232,7 +234,7 @@ class IdleResourceScanner: CostReport 包含所有浪費資源與建議動作 """ self._scan_counter += 1 - scan_id = f"scan-{self._scan_counter:04d}-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}" + scan_id = f"scan-{self._scan_counter:04d}-{now_taipei().strftime('%Y%m%d%H%M%S')}" logger.info(f"[FinOps] Starting full scan: {scan_id}") @@ -254,7 +256,7 @@ class IdleResourceScanner: report = CostReport( scan_id=scan_id, - scanned_at=datetime.utcnow(), + scanned_at=now_taipei(), cluster_name=cluster_name, total_wasted_usd=total_wasted, total_resources_scanned=self._get_mock_total_resources(), @@ -288,23 +290,23 @@ class IdleResourceScanner: "namespace": "database", "size_gb": 500, "storage_class": "gp3", - "created": datetime.utcnow() - timedelta(days=90), - "last_used": datetime.utcnow() - timedelta(days=60), + "created": now_taipei() - timedelta(days=90), + "last_used": now_taipei() - timedelta(days=60), }, { "name": "logs-elasticsearch-2023", "namespace": "logging", "size_gb": 200, "storage_class": "gp2", - "created": datetime.utcnow() - timedelta(days=180), - "last_used": datetime.utcnow() - timedelta(days=120), + "created": now_taipei() - timedelta(days=180), + "last_used": now_taipei() - timedelta(days=120), }, { "name": "cache-redis-temp", "namespace": "default", "size_gb": 50, "storage_class": "gp3", - "created": datetime.utcnow() - timedelta(days=30), + "created": now_taipei() - timedelta(days=30), "last_used": None, }, ] @@ -348,8 +350,8 @@ class IdleResourceScanner: "cpu_request": 2.0, # vCPU "mem_request_gb": 4.0, "avg_cpu_percent": 0.3, - "created": datetime.utcnow() - timedelta(days=120), - "last_active": datetime.utcnow() - timedelta(days=45), + "created": now_taipei() - timedelta(days=120), + "last_active": now_taipei() - timedelta(days=45), }, { "name": "test-worker-batch-xyz99", @@ -357,8 +359,8 @@ class IdleResourceScanner: "cpu_request": 1.0, "mem_request_gb": 2.0, "avg_cpu_percent": 0.1, - "created": datetime.utcnow() - timedelta(days=60), - "last_active": datetime.utcnow() - timedelta(days=30), + "created": now_taipei() - timedelta(days=60), + "last_active": now_taipei() - timedelta(days=30), }, { "name": "debug-shell-admin", @@ -366,8 +368,8 @@ class IdleResourceScanner: "cpu_request": 0.5, "mem_request_gb": 1.0, "avg_cpu_percent": 0.0, - "created": datetime.utcnow() - timedelta(days=14), - "last_active": datetime.utcnow() - timedelta(days=10), + "created": now_taipei() - timedelta(days=14), + "last_active": now_taipei() - timedelta(days=10), }, ] @@ -420,7 +422,7 @@ class IdleResourceScanner: "requested_mem_gb": 48.0, "actual_cpu": 2.0, "actual_mem_gb": 8.0, - "created": datetime.utcnow() - timedelta(days=200), + "created": now_taipei() - timedelta(days=200), }, { "name": "worker-gpu-unused", @@ -431,7 +433,7 @@ class IdleResourceScanner: "requested_mem_gb": 16.0, "actual_cpu": 0.5, "actual_mem_gb": 2.0, - "created": datetime.utcnow() - timedelta(days=90), + "created": now_taipei() - timedelta(days=90), }, ] @@ -468,7 +470,7 @@ class IdleResourceScanner: ), monthly_cost_usd=monthly_cost, created_at=node["created"], - last_used_at=datetime.utcnow(), + last_used_at=now_taipei(), spec={ "totalCpu": node["total_cpu"], "totalMemoryGb": node["total_mem_gb"], diff --git a/apps/api/src/plugins/mcp/mcp_bridge.py b/apps/api/src/plugins/mcp/mcp_bridge.py index 01cd1a255..21380cf7c 100644 --- a/apps/api/src/plugins/mcp/mcp_bridge.py +++ b/apps/api/src/plugins/mcp/mcp_bridge.py @@ -18,6 +18,8 @@ import re import uuid from dataclasses import dataclass, field from datetime import datetime + +from src.utils.timezone import now_taipei from enum import Enum from typing import Any @@ -53,7 +55,7 @@ class MCPToolResult: output: Any | None = None error: str | None = None duration: float = 0.0 - timestamp: datetime = field(default_factory=datetime.utcnow) + timestamp: datetime = field(default_factory=now_taipei) def to_dict(self) -> dict: return { @@ -390,7 +392,7 @@ class MCPBridge: MCPToolResult (符合 ActionResult 介面) """ execution_id = str(uuid.uuid4()) - start_time = datetime.utcnow() + start_time = now_taipei() try: # ======================================== @@ -494,7 +496,7 @@ class MCPBridge: def _calc_duration(self, start_time: datetime) -> float: """計算執行時間 (毫秒)""" - return (datetime.utcnow() - start_time).total_seconds() * 1000 + return (now_taipei() - start_time).total_seconds() * 1000 # ==================== ActionExecutor 介面對齊 ==================== diff --git a/apps/api/src/services/executor.py b/apps/api/src/services/executor.py index 677fb893d..702319075 100644 --- a/apps/api/src/services/executor.py +++ b/apps/api/src/services/executor.py @@ -21,7 +21,7 @@ Supported Operations: import asyncio import time from dataclasses import dataclass -from datetime import UTC, datetime +from src.utils.timezone import now_taipei, now_taipei_iso from enum import Enum from pathlib import Path from typing import Any @@ -376,7 +376,7 @@ class ActionExecutor: "template": { "metadata": { "annotations": { - "kubectl.kubernetes.io/restartedAt": datetime.now(UTC).isoformat() + "kubectl.kubernetes.io/restartedAt": now_taipei().isoformat() } } } @@ -1060,7 +1060,7 @@ async def execute_approved_proposal(approval_id: str) -> ExecutionResult: # Step 5: 更新狀態 new_status = "executed" if result.success else "failed" execution_log = { - "executed_at": datetime.now(UTC).isoformat(), + "executed_at": now_taipei().isoformat(), "success": result.success, "stdout": result.k8s_response.get("stdout", "") if result.k8s_response else "", "stderr": result.error or "", diff --git a/apps/api/src/services/notifications/base.py b/apps/api/src/services/notifications/base.py index 6baeffd33..70d9290da 100644 --- a/apps/api/src/services/notifications/base.py +++ b/apps/api/src/services/notifications/base.py @@ -11,7 +11,9 @@ Phase 6: leWOOOgo Output Plugins from abc import ABC, abstractmethod from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import datetime + +from src.utils.timezone import now_taipei from enum import Enum from typing import Any @@ -68,7 +70,7 @@ class NotificationMessage: confidence: float | None = None # 時間戳 - timestamp: datetime = field(default_factory=lambda: datetime.now(UTC)) + timestamp: datetime = field(default_factory=lambda: now_taipei()) @property def status_emoji(self) -> str: @@ -117,7 +119,7 @@ class NotificationResult: message: str response_data: dict[str, Any] | None = None error: str | None = None - timestamp: datetime = field(default_factory=lambda: datetime.now(UTC)) + timestamp: datetime = field(default_factory=lambda: now_taipei()) class NotificationProvider(ABC): diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index b5e55a5ac..1731cae5d 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -23,9 +23,9 @@ import json import random import re import time -from datetime import datetime - import httpx + +from src.utils.timezone import now_taipei_iso import structlog from src.core.config import settings @@ -789,7 +789,7 @@ class OpenClawService: cache_data = { "response": response, "provider": provider, - "cached_at": datetime.now().isoformat(), + "cached_at": now_taipei_iso(), } await redis_client.set( cache_key, diff --git a/apps/api/src/utils/timezone.py b/apps/api/src/utils/timezone.py new file mode 100644 index 000000000..971815ce4 --- /dev/null +++ b/apps/api/src/utils/timezone.py @@ -0,0 +1,70 @@ +""" +AWOOOI - 時區工具 +================== +統一使用 Asia/Taipei (UTC+8) 時區 + +🔴 HARD RULE: 全系統使用台北時區,禁止 UTC +""" + +from datetime import datetime, timezone, timedelta + +# 台北時區 (UTC+8) +TAIPEI_TZ = timezone(timedelta(hours=8)) + + +def now_taipei() -> datetime: + """ + 取得台北時區當前時間 + + Returns: + datetime: 帶有 +08:00 時區資訊的 datetime + + Example: + >>> now_taipei().isoformat() + '2026-03-25T02:08:04+08:00' + """ + return datetime.now(TAIPEI_TZ) + + +def to_taipei(dt: datetime) -> datetime: + """ + 將 datetime 轉換為台北時區 + + Args: + dt: 任何 datetime (可能是 UTC 或 naive) + + Returns: + datetime: 台北時區的 datetime + """ + if dt.tzinfo is None: + # naive datetime,假設是 UTC + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(TAIPEI_TZ) + + +def format_taipei(dt: datetime | None = None, fmt: str = "%Y-%m-%d %H:%M:%S") -> str: + """ + 格式化為台北時區字串 + + Args: + dt: datetime (預設為當前時間) + fmt: strftime 格式 + + Returns: + str: 格式化的時間字串 + """ + if dt is None: + dt = now_taipei() + else: + dt = to_taipei(dt) + return dt.strftime(fmt) + + +def now_taipei_iso() -> str: + """ + 取得 ISO 格式的台北時區時間 + + Returns: + str: ISO 格式,例如 '2026-03-25T02:08:04+08:00' + """ + return now_taipei().isoformat() diff --git a/apps/sensor/agent.py b/apps/sensor/agent.py index 5c3fb40b6..aecf4daf8 100644 --- a/apps/sensor/agent.py +++ b/apps/sensor/agent.py @@ -32,7 +32,10 @@ import random import socket import sys import time -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone + +# 台北時區 (UTC+8) - 系統統一時區 +TAIPEI_TZ = timezone(timedelta(hours=8)) from typing import Any from uuid import uuid4 @@ -160,7 +163,7 @@ class SensorAgent: return None # 建立標準 Signal 格式 - now = datetime.now(timezone.utc) + now = datetime.now(TAIPEI_TZ) signal = { "alert_name": alert.get("alert_name", "UnknownAlert"), "severity": alert.get("severity", "warning"), @@ -261,7 +264,7 @@ def main() -> int: print("=" * 70) print("AWOOOI Sensor Agent - Phase 6.5 神經末梢") print("=" * 70) - print(f"Time: {datetime.now().isoformat()}") + print(f"Time: {datetime.now(TAIPEI_TZ).isoformat()}") print(f"Host: {socket.gethostname()}") print() diff --git a/docs/HARD_RULES.md b/docs/HARD_RULES.md index df677de17..c42589c23 100644 --- a/docs/HARD_RULES.md +++ b/docs/HARD_RULES.md @@ -17,6 +17,9 @@ | **測試** | **Mock 測試** | **真實 DB/服務** | [→ No Mock Testing](#no-mock-testing) | | **API** | **單獨改路徑** | **前後端同步** | [→ API Path Naming](#api-path-naming) | | **部署** | **假設已部署** | **驗證 Pod 版本** | [→ Deployment Verification](#deployment-verification) | +| **Alertmanager** | **指向 OpenClaw** | **指向 AWOOOI API** | [→ Alertmanager Routing](#alertmanager-routing) | +| **簽核 UI** | **清空內容** | **保留原始內容** | [→ Approval Preserve Content](#approval-preserve-content) | +| **時區** | **UTC/utcnow** | **台北時區 +8** | [→ Timezone Taipei](#timezone-taipei) | --- @@ -224,6 +227,75 @@ curl -f https://api.awoooi.wooo.work/api/v1/health --- +## Alertmanager Routing + +**Memory:** `~/.claude/projects/-Users-ogt-awoooi/memory/feedback_alertmanager_awoooi_flow.md` + +```yaml +# ❌ 禁止 - 指向 OpenClaw (會使用舊 AIOPS 格式) +receivers: + - name: openclaw + webhook_configs: + - url: 'http://192.168.0.188:8088/api/v1/webhook/alertmanager' + +# ✅ 正確 - 指向 AWOOOI API (K3s) +receivers: + - name: awoooi + webhook_configs: + - url: 'http://192.168.0.120:32334/api/v1/webhooks/alertmanager' +``` + +**職責分工:** + +| 系統 | 職責 | Telegram 權限 | +|------|------|--------------| +| AWOOOI API (K3s 32334) | 告警處理 + 格式化 + 發送 | ✅ | +| OpenClaw (188:8088) | AI 大腦、LLM 分析 | ❌ | + +**原因:** 2026-03-25 災難:Claude 錯誤將 Alertmanager 指向 OpenClaw,導致 Telegram 發送舊 AIOPS 格式(🤝 [協同警報]),而非 AWOOOI 格式(═══ 🚨 CRITICAL ═══)。 + +--- + +## Approval Preserve Content + +**Memory:** `~/.claude/projects/-Users-ogt-awoooi/memory/feedback_approval_preserve_content.md` + +```tsx +// ❌ 禁止 - 簽核後清空內容 +if (approval.status === 'approved') { + return