""" Audit Sink with PII/Secret Redaction ====================================== AwoooP Phase 4.4: Audit log 寫入前的 sanitization pipeline(ADR-116) 2026-05-04 ogt + Claude Sonnet 4.6 設計原則: - audit log 不記錄 raw LLM input/output,只記 hash + schema validation result - PII / secret pattern 硬攔(不可被 caller 繞過) - 攔截清單:GCP IP、PostgreSQL password、Telegram token、SSH key、Bearer token 等 - redaction 後原值不可還原(替換為 [REDACTED:]) - 所有 audit 寫入透過此 sink(禁止其他 service 直接 INSERT audit_logs) 使用: from src.services.audit_sink import write_audit await write_audit( project_id="awoooi", action="run.completed", resource_type="run", resource_id=str(run_id), details={"trace_id": trace_id, "cost_usd": 0.012}, ) """ from __future__ import annotations import hashlib import json import re from typing import Any import structlog logger = structlog.get_logger(__name__) # ───────────────────────────────────────────────────────────────────────────── # Redaction patterns(ADR-116 P1-08) # ───────────────────────────────────────────────────────────────────────────── # 每個 pattern: (compiled_re, replacement_tag) _REDACTION_PATTERNS: list[tuple[re.Pattern[str], str]] = [ # Telegram bot token(數字:英數字母混合 32~64 字元) (re.compile(r"\d{8,12}:[A-Za-z0-9_-]{32,64}"), "TELEGRAM_TOKEN"), # PostgreSQL connection string (re.compile(r"postgresql(?:\+asyncpg)?://[^:]+:[^@]+@[^/\s]+"), "PG_DSN"), # Generic password in URL / config (re.compile(r"(?i)(?:password|passwd|pwd)\s*[:=]\s*\S+"), "PASSWORD"), # Bearer / Authorization header value (re.compile(r"(?i)(?:bearer|token)\s+[A-Za-z0-9\-._~+/]+=*"), "BEARER_TOKEN"), # AWS / GCP / NVIDIA API key patterns (re.compile(r"(?i)(?:api[_-]?key|apikey)\s*[:=]\s*[A-Za-z0-9\-._]{20,}"), "API_KEY"), # Private GCP internal IPs(ADR-116 禁止 GCP 內網 IP 進 log) (re.compile(r"\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"), (re.compile(r"\b172\.(?:1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"), (re.compile(r"\b192\.168\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"), # SSH private key (re.compile(r"-----BEGIN (?:RSA|EC|OPENSSH) PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+ PRIVATE KEY-----"), "SSH_PRIVATE_KEY"), # JWT(三段 base64 以 . 分隔) (re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+"), "JWT_TOKEN"), # Hex secret >= 32 位(可能是 HMAC key / session token) (re.compile(r"\b[0-9a-f]{64}\b"), "HEX_SECRET_64"), ] # 欄位名稱黑名單:這些 key 的 value 直接替換(不做 pattern 掃描) _BLOCKED_FIELD_NAMES = frozenset({ "password", "passwd", "pwd", "secret", "token", "api_key", "apikey", "private_key", "private_key_pem", "bot_token", "telegram_token", "hmac_key", "jwt", "authorization", "cookie", "session", }) # LLM raw input/output 欄位名稱(只記 hash) _LLM_RAW_FIELDS = frozenset({ "raw_input", "raw_output", "llm_input", "llm_output", "prompt", "completion", "system_prompt", }) # ───────────────────────────────────────────────────────────────────────────── # Sanitization pipeline # ───────────────────────────────────────────────────────────────────────────── def _redact_string(value: str) -> str: """對字串套用所有 redaction patterns""" for pattern, tag in _REDACTION_PATTERNS: value = pattern.sub(f"[REDACTED:{tag}]", value) return value def sanitize(details: dict[str, Any]) -> dict[str, Any]: """ 遞迴處理 details dict,套用所有 redaction 規則。 規則優先序: 1. key 在 _BLOCKED_FIELD_NAMES → value 替換為 [REDACTED:BLOCKED_FIELD] 2. key 在 _LLM_RAW_FIELDS → value 替換為 sha256(str(value))(只記 hash) 3. string value → pattern redaction 4. nested dict/list → 遞迴處理 """ return _sanitize_value(details, depth=0) def _sanitize_value(value: Any, depth: int = 0) -> Any: if depth > 10: return "[REDACTED:MAX_DEPTH]" if isinstance(value, dict): return {k: _sanitize_dict_entry(k, v, depth) for k, v in value.items()} if isinstance(value, list): return [_sanitize_value(item, depth + 1) for item in value] if isinstance(value, str): return _redact_string(value) return value def _sanitize_dict_entry(key: str, value: Any, depth: int) -> Any: key_lower = key.lower() if key_lower in _BLOCKED_FIELD_NAMES: return "[REDACTED:BLOCKED_FIELD]" if key_lower in _LLM_RAW_FIELDS: # 只記 sha256 hash,不記原始內容 raw_str = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value return f"[LLM_RAW_HASH:{hashlib.sha256(raw_str.encode()).hexdigest()[:16]}]" return _sanitize_value(value, depth + 1) # ───────────────────────────────────────────────────────────────────────────── # Audit write # ───────────────────────────────────────────────────────────────────────────── async def write_audit( *, project_id: str, action: str, resource_type: str, resource_id: str, details: dict[str, Any] | None = None, run_id: str | None = None, trace_id: str | None = None, ) -> None: """ 統一 audit log 寫入入口(Phase 4+ 所有 service 必須透過此方法)。 1. sanitize details(PII / secret redaction) 2. 附加 run_id / trace_id(可觀測性) 3. INSERT audit_logs(非阻擋 background task) """ import asyncio asyncio.create_task( _write_audit_impl( project_id=project_id, action=action, resource_type=resource_type, resource_id=resource_id, details=details, run_id=run_id, trace_id=trace_id, ), name="audit_sink_write", ) async def _write_audit_impl( *, project_id: str, action: str, resource_type: str, resource_id: str, details: dict[str, Any] | None, run_id: str | None, trace_id: str | None, ) -> None: try: from sqlalchemy import text as sa_text from src.db.base import get_db_context clean_details: dict[str, Any] = sanitize(details or {}) if run_id: clean_details["_run_id"] = run_id if trace_id: clean_details["_trace_id"] = trace_id async with get_db_context(project_id) as db: await db.execute( sa_text(""" INSERT INTO audit_logs (project_id, action, resource_type, resource_id, details) VALUES (:project_id, :action, :resource_type, :resource_id, :details::jsonb) """), { "project_id": project_id, "action": action, "resource_type": resource_type, "resource_id": resource_id, "details": json.dumps(clean_details), }, ) except Exception as exc: logger.warning( "audit_sink_write_failed", action=action, resource_id=resource_id, error=str(exc), ) # ───────────────────────────────────────────────────────────────────────────── # Convenience:可在測試中驗證 sanitization 結果 # ───────────────────────────────────────────────────────────────────────────── def sanitize_for_test(details: dict[str, Any]) -> dict[str, Any]: """同步 sanitize,供測試使用""" return sanitize(details)