""" Audit Sink with PII/Secret Redaction ====================================== AwoooP Phase 4.4: Audit log 寫入前的 sanitization pipeline(ADR-116) 2026-05-04 ogt + Claude Sonnet 4.6 設計原則: - audit log 不記錄 raw LLM input/output,只記 hash + schema validation result - PII / secret pattern 硬攔(不可被 caller 繞過) - 攔截清單:GCP IP、PostgreSQL password、Telegram token、SSH key、Bearer token 等 - redaction 後原值不可還原(替換為 [REDACTED:]) - 所有 generic audit 寫入透過此 sink(禁止直接假設 audit_logs 新版 schema) 使用: from src.services.audit_sink import write_audit await write_audit( project_id="awoooi", action="run.completed", resource_type="run", resource_id=str(run_id), details={"trace_id": trace_id, "cost_usd": 0.012}, ) """ from __future__ import annotations import hashlib import json import re from typing import Any import structlog logger = structlog.get_logger(__name__) AWOOOP_AUDIT_SCHEMA_VERSION = "awooop_sanitized_audit_v1" AWOOOP_AUDIT_REDACTION_VERSION = "audit_sink_v1" class AuditDurabilityError(RuntimeError): """Fail-closed error for a required immutable audit acknowledgement.""" def __init__(self, error_code: str) -> None: self.error_code = error_code super().__init__(error_code) def audit_idempotency_key_sha256(idempotency_key: str) -> str: """Return the non-secret stable identity stored in immutable context.""" return hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest() # ───────────────────────────────────────────────────────────────────────────── # Redaction patterns(ADR-116 P1-08) # ───────────────────────────────────────────────────────────────────────────── # 每個 pattern: (compiled_re, replacement_tag) _REDACTION_PATTERNS: list[tuple[re.Pattern[str], str]] = [ # Telegram bot token(數字:英數字母混合 32~64 字元) (re.compile(r"\d{8,12}:[A-Za-z0-9_-]{32,64}"), "TELEGRAM_TOKEN"), # PostgreSQL connection string (re.compile(r"postgresql(?:\+asyncpg)?://[^:]+:[^@]+@[^/\s]+"), "PG_DSN"), # Generic password in URL / config (re.compile(r"(?i)(?:password|passwd|pwd)\s*[:=]\s*\S+"), "PASSWORD"), # Bearer / Authorization header value (re.compile(r"(?i)(?:bearer|token)\s+[A-Za-z0-9\-._~+/]+=*"), "BEARER_TOKEN"), # AWS / GCP / NVIDIA API key patterns (re.compile(r"(?i)(?:api[_-]?key|apikey)\s*[:=]\s*[A-Za-z0-9\-._]{20,}"), "API_KEY"), # Private GCP internal IPs(ADR-116 禁止 GCP 內網 IP 進 log) (re.compile(r"\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"), (re.compile(r"\b172\.(?:1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"), (re.compile(r"\b192\.168\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"), # SSH private key (re.compile(r"-----BEGIN (?:RSA|EC|OPENSSH) PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+ PRIVATE KEY-----"), "SSH_PRIVATE_KEY"), # JWT(三段 base64 以 . 分隔) (re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+"), "JWT_TOKEN"), # Hex secret >= 32 位(可能是 HMAC key / session token) (re.compile(r"\b[0-9a-f]{64}\b"), "HEX_SECRET_64"), ] # 欄位名稱黑名單:這些 key 的 value 直接替換(不做 pattern 掃描) _BLOCKED_FIELD_NAMES = frozenset({ "password", "passwd", "pwd", "secret", "token", "api_key", "apikey", "private_key", "private_key_pem", "bot_token", "telegram_token", "hmac_key", "jwt", "authorization", "cookie", "session", }) # LLM raw input/output 欄位名稱(只記 hash) _LLM_RAW_FIELDS = frozenset({ "raw_input", "raw_output", "llm_input", "llm_output", "prompt", "completion", "system_prompt", }) # ───────────────────────────────────────────────────────────────────────────── # Sanitization pipeline # ───────────────────────────────────────────────────────────────────────────── def _redact_string(value: str) -> str: """對字串套用所有 redaction patterns""" for pattern, tag in _REDACTION_PATTERNS: value = pattern.sub(f"[REDACTED:{tag}]", value) return value def sanitize(details: dict[str, Any]) -> dict[str, Any]: """ 遞迴處理 details dict,套用所有 redaction 規則。 規則優先序: 1. key 在 _BLOCKED_FIELD_NAMES → value 替換為 [REDACTED:BLOCKED_FIELD] 2. key 在 _LLM_RAW_FIELDS → value 替換為 sha256(str(value))(只記 hash) 3. string value → pattern redaction 4. nested dict/list → 遞迴處理 """ return _sanitize_value(details, depth=0) def _sanitize_value(value: Any, depth: int = 0) -> Any: if depth > 10: return "[REDACTED:MAX_DEPTH]" if isinstance(value, dict): return {k: _sanitize_dict_entry(k, v, depth) for k, v in value.items()} if isinstance(value, list): return [_sanitize_value(item, depth + 1) for item in value] if isinstance(value, str): return _redact_string(value) return value def _sanitize_dict_entry(key: str, value: Any, depth: int) -> Any: key_lower = key.lower() if key_lower in _BLOCKED_FIELD_NAMES: return "[REDACTED:BLOCKED_FIELD]" if key_lower in _LLM_RAW_FIELDS: # 只記 sha256 hash,不記原始內容 raw_str = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value return f"[LLM_RAW_HASH:{hashlib.sha256(raw_str.encode()).hexdigest()[:16]}]" return _sanitize_value(value, depth + 1) # ───────────────────────────────────────────────────────────────────────────── # Audit write # ───────────────────────────────────────────────────────────────────────────── async def write_audit( *, project_id: str, action: str, resource_type: str, resource_id: str, details: dict[str, Any] | None = None, run_id: str | None = None, trace_id: str | None = None, require_durable: bool = False, idempotency_key: str | None = None, ) -> dict[str, Any] | None: """ 統一 audit log 寫入入口(Phase 4+ 所有 service 必須透過此方法)。 1. sanitize details(PII / secret redaction) 2. 附加 run_id / trace_id(可觀測性) 3. INSERT immutable alert_operation_log USER_ACTION/context event Critical authorization callers set ``require_durable=True`` and provide an idempotency key. That path is serialized with a transaction-scoped PostgreSQL advisory lock and returns only after the DB context commits. """ if require_durable and not str(idempotency_key or "").strip(): raise AuditDurabilityError("audit_idempotency_key_required") return await _write_audit_impl( project_id=project_id, action=action, resource_type=resource_type, resource_id=resource_id, details=details, run_id=run_id, trace_id=trace_id, require_durable=require_durable, idempotency_key=idempotency_key, ) async def _write_audit_impl( *, project_id: str, action: str, resource_type: str, resource_id: str, details: dict[str, Any] | None, run_id: str | None, trace_id: str | None, require_durable: bool = False, idempotency_key: str | None = None, ) -> dict[str, Any] | None: try: from sqlalchemy import text as sa_text from src.db.base import get_db_context clean_details: dict[str, Any] = sanitize(details or {}) clean_action = _redact_string(str(action or "unknown"))[:200] clean_resource_type = _redact_string(str(resource_type or "unknown"))[:80] clean_resource_id = _redact_string(str(resource_id or "unknown"))[:200] clean_run_id = _redact_string(str(run_id or ""))[:200] clean_trace_id = _redact_string(str(trace_id or ""))[:200] clean_project_id = _redact_string(str(project_id or ""))[:64] actor = _redact_string( str(clean_details.get("approver_id") or f"awooop:{clean_project_id}") )[:100] idempotency_key_value = str(idempotency_key or "").strip() idempotency_key_sha256 = ( audit_idempotency_key_sha256(idempotency_key_value) if idempotency_key_value else "" ) context = { "schema_version": AWOOOP_AUDIT_SCHEMA_VERSION, "redaction_version": AWOOOP_AUDIT_REDACTION_VERSION, "project_id": clean_project_id, "action": clean_action, "resource_type": clean_resource_type, "resource_id": clean_resource_id, "run_id": clean_run_id, "trace_id": clean_trace_id, "details": clean_details, "durable_write_required": require_durable, "idempotency_key_sha256": idempotency_key_sha256, } params = { "actor": actor, "action_detail": clean_action, "context": json.dumps(context, ensure_ascii=False), } async with get_db_context(project_id) as db: if idempotency_key_value: await db.execute( sa_text(""" SELECT pg_advisory_xact_lock( hashtextextended(:idempotency_key, 0) ) """), {"idempotency_key": idempotency_key_value}, ) existing_result = await db.execute( sa_text(""" SELECT COUNT(*) AS matching_count, MAX(created_at) AS created_at FROM alert_operation_log WHERE event_type = 'USER_ACTION' AND action_detail = :action_detail AND context ->> 'schema_version' = :schema_version AND context ->> 'project_id' = :project_id AND context ->> 'resource_type' = :resource_type AND context ->> 'resource_id' = :resource_id AND context ->> 'run_id' = :run_id AND context ->> 'idempotency_key_sha256' = :idempotency_key_sha256 """), { "action_detail": clean_action, "schema_version": AWOOOP_AUDIT_SCHEMA_VERSION, "project_id": clean_project_id, "resource_type": clean_resource_type, "resource_id": clean_resource_id, "run_id": clean_run_id, "idempotency_key_sha256": idempotency_key_sha256, }, ) existing = existing_result.mappings().one() existing_count = int(existing["matching_count"] or 0) if existing_count > 1: raise AuditDurabilityError( "audit_idempotency_duplicate_detected" ) if existing_count == 1: return { "status": "duplicate_existing", "durable_write_ack": True, "inserted": False, "created_at": existing["created_at"], } await db.execute( sa_text(""" INSERT INTO alert_operation_log (event_type, actor, action_detail, success, context) VALUES ( 'USER_ACTION', :actor, :action_detail, TRUE, CAST(:context AS JSONB) ) """), params, ) if idempotency_key_value: readback_result = await db.execute( sa_text(""" SELECT COUNT(*) AS matching_count, MAX(created_at) AS created_at FROM alert_operation_log WHERE event_type = 'USER_ACTION' AND action_detail = :action_detail AND context ->> 'schema_version' = :schema_version AND context ->> 'project_id' = :project_id AND context ->> 'resource_type' = :resource_type AND context ->> 'resource_id' = :resource_id AND context ->> 'run_id' = :run_id AND context ->> 'idempotency_key_sha256' = :idempotency_key_sha256 """), { "action_detail": clean_action, "schema_version": AWOOOP_AUDIT_SCHEMA_VERSION, "project_id": clean_project_id, "resource_type": clean_resource_type, "resource_id": clean_resource_id, "run_id": clean_run_id, "idempotency_key_sha256": idempotency_key_sha256, }, ) readback = readback_result.mappings().one() if int(readback["matching_count"] or 0) != 1: raise AuditDurabilityError( "audit_durable_readback_not_exactly_once" ) return { "status": "inserted_verified", "durable_write_ack": True, "inserted": True, "created_at": readback["created_at"], } return { "status": "inserted", "durable_write_ack": True, "inserted": True, "created_at": None, } except AuditDurabilityError: raise except Exception as exc: logger.warning( "audit_sink_write_failed", action=action, resource_id=resource_id, error_type=type(exc).__name__, ) if require_durable: raise AuditDurabilityError("audit_durable_write_failed") from exc return None # ───────────────────────────────────────────────────────────────────────────── # Convenience:可在測試中驗證 sanitization 結果 # ───────────────────────────────────────────────────────────────────────────── def sanitize_for_test(details: dict[str, Any]) -> dict[str, Any]: """同步 sanitize,供測試使用""" return sanitize(details)