dbed105066
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 6m36s
CD Pipeline / build-and-deploy (push) Successful in 15m16s
CD Pipeline / post-deploy-checks (push) Failing after 10m29s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 30s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Failing after 12m42s
379 lines
16 KiB
Python
379 lines
16 KiB
Python
"""
|
||
Audit Sink with PII/Secret Redaction
|
||
======================================
|
||
AwoooP Phase 4.4: Audit log 寫入前的 sanitization pipeline（ADR-116）
|
||
2026-05-04 ogt + Claude Sonnet 4.6
|
||
|
||
設計原則：
|
||
- audit log 不記錄 raw LLM input/output，只記 hash + schema validation result
|
||
- PII / secret pattern 硬攔（不可被 caller 繞過）
|
||
- 攔截清單：GCP IP、PostgreSQL password、Telegram token、SSH key、Bearer token 等
|
||
- redaction 後原值不可還原（替換為 [REDACTED:<type>]）
|
||
- 所有 generic audit 寫入透過此 sink（禁止直接假設 audit_logs 新版 schema）
|
||
|
||
使用：
|
||
from src.services.audit_sink import write_audit
|
||
|
||
await write_audit(
|
||
project_id="awoooi",
|
||
action="run.completed",
|
||
resource_type="run",
|
||
resource_id=str(run_id),
|
||
details={"trace_id": trace_id, "cost_usd": 0.012},
|
||
)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
from typing import Any
|
||
|
||
import structlog
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
AWOOOP_AUDIT_SCHEMA_VERSION = "awooop_sanitized_audit_v1"
|
||
AWOOOP_AUDIT_REDACTION_VERSION = "audit_sink_v1"
|
||
|
||
|
||
class AuditDurabilityError(RuntimeError):
|
||
"""Fail-closed error for a required immutable audit acknowledgement."""
|
||
|
||
def __init__(self, error_code: str) -> None:
|
||
self.error_code = error_code
|
||
super().__init__(error_code)
|
||
|
||
|
||
def audit_idempotency_key_sha256(idempotency_key: str) -> str:
|
||
"""Return the non-secret stable identity stored in immutable context."""
|
||
|
||
return hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest()
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Redaction patterns（ADR-116 P1-08）
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
# 每個 pattern: (compiled_re, replacement_tag)
|
||
_REDACTION_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||
# Telegram bot token（數字:英數字母混合 32~64 字元）
|
||
(re.compile(r"\d{8,12}:[A-Za-z0-9_-]{32,64}"), "TELEGRAM_TOKEN"),
|
||
|
||
# PostgreSQL connection string
|
||
(re.compile(r"postgresql(?:\+asyncpg)?://[^:]+:[^@]+@[^/\s]+"), "PG_DSN"),
|
||
|
||
# Generic password in URL / config
|
||
(re.compile(r"(?i)(?:password|passwd|pwd)\s*[:=]\s*\S+"), "PASSWORD"),
|
||
|
||
# Bearer / Authorization header value
|
||
(re.compile(r"(?i)(?:bearer|token)\s+[A-Za-z0-9\-._~+/]+=*"), "BEARER_TOKEN"),
|
||
|
||
# AWS / GCP / NVIDIA API key patterns
|
||
(re.compile(r"(?i)(?:api[_-]?key|apikey)\s*[:=]\s*[A-Za-z0-9\-._]{20,}"), "API_KEY"),
|
||
|
||
# Private GCP internal IPs（ADR-116 禁止 GCP 內網 IP 進 log）
|
||
(re.compile(r"\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"),
|
||
(re.compile(r"\b172\.(?:1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"),
|
||
(re.compile(r"\b192\.168\.\d{1,3}\.\d{1,3}\b"), "INTERNAL_IP"),
|
||
|
||
# SSH private key
|
||
(re.compile(r"-----BEGIN (?:RSA|EC|OPENSSH) PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+ PRIVATE KEY-----"), "SSH_PRIVATE_KEY"),
|
||
|
||
# JWT（三段 base64 以 . 分隔）
|
||
(re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+"), "JWT_TOKEN"),
|
||
|
||
# Hex secret >= 32 位（可能是 HMAC key / session token）
|
||
(re.compile(r"\b[0-9a-f]{64}\b"), "HEX_SECRET_64"),
|
||
]
|
||
|
||
# 欄位名稱黑名單：這些 key 的 value 直接替換（不做 pattern 掃描）
|
||
_BLOCKED_FIELD_NAMES = frozenset({
|
||
"password", "passwd", "pwd", "secret", "token", "api_key", "apikey",
|
||
"private_key", "private_key_pem", "bot_token", "telegram_token",
|
||
"hmac_key", "jwt", "authorization", "cookie", "session",
|
||
})
|
||
|
||
# LLM raw input/output 欄位名稱（只記 hash）
|
||
_LLM_RAW_FIELDS = frozenset({
|
||
"raw_input", "raw_output", "llm_input", "llm_output",
|
||
"prompt", "completion", "system_prompt",
|
||
})
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Sanitization pipeline
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
def _redact_string(value: str) -> str:
|
||
"""對字串套用所有 redaction patterns"""
|
||
for pattern, tag in _REDACTION_PATTERNS:
|
||
value = pattern.sub(f"[REDACTED:{tag}]", value)
|
||
return value
|
||
|
||
|
||
def sanitize(details: dict[str, Any]) -> dict[str, Any]:
|
||
"""
|
||
遞迴處理 details dict，套用所有 redaction 規則。
|
||
|
||
規則優先序：
|
||
1. key 在 _BLOCKED_FIELD_NAMES → value 替換為 [REDACTED:BLOCKED_FIELD]
|
||
2. key 在 _LLM_RAW_FIELDS → value 替換為 sha256(str(value))（只記 hash）
|
||
3. string value → pattern redaction
|
||
4. nested dict/list → 遞迴處理
|
||
"""
|
||
return _sanitize_value(details, depth=0)
|
||
|
||
|
||
def _sanitize_value(value: Any, depth: int = 0) -> Any:
|
||
if depth > 10:
|
||
return "[REDACTED:MAX_DEPTH]"
|
||
|
||
if isinstance(value, dict):
|
||
return {k: _sanitize_dict_entry(k, v, depth) for k, v in value.items()}
|
||
if isinstance(value, list):
|
||
return [_sanitize_value(item, depth + 1) for item in value]
|
||
if isinstance(value, str):
|
||
return _redact_string(value)
|
||
return value
|
||
|
||
|
||
def _sanitize_dict_entry(key: str, value: Any, depth: int) -> Any:
|
||
key_lower = key.lower()
|
||
|
||
if key_lower in _BLOCKED_FIELD_NAMES:
|
||
return "[REDACTED:BLOCKED_FIELD]"
|
||
|
||
if key_lower in _LLM_RAW_FIELDS:
|
||
# 只記 sha256 hash，不記原始內容
|
||
raw_str = json.dumps(value, ensure_ascii=False) if not isinstance(value, str) else value
|
||
return f"[LLM_RAW_HASH:{hashlib.sha256(raw_str.encode()).hexdigest()[:16]}]"
|
||
|
||
return _sanitize_value(value, depth + 1)
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Audit write
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
async def write_audit(
|
||
*,
|
||
project_id: str,
|
||
action: str,
|
||
resource_type: str,
|
||
resource_id: str,
|
||
details: dict[str, Any] | None = None,
|
||
run_id: str | None = None,
|
||
trace_id: str | None = None,
|
||
require_durable: bool = False,
|
||
idempotency_key: str | None = None,
|
||
) -> dict[str, Any] | None:
|
||
"""
|
||
統一 audit log 寫入入口（Phase 4+ 所有 service 必須透過此方法）。
|
||
|
||
1. sanitize details（PII / secret redaction）
|
||
2. 附加 run_id / trace_id（可觀測性）
|
||
3. INSERT immutable alert_operation_log USER_ACTION/context event
|
||
|
||
Critical authorization callers set ``require_durable=True`` and provide an
|
||
idempotency key. That path is serialized with a transaction-scoped
|
||
PostgreSQL advisory lock and returns only after the DB context commits.
|
||
"""
|
||
if require_durable and not str(idempotency_key or "").strip():
|
||
raise AuditDurabilityError("audit_idempotency_key_required")
|
||
|
||
return await _write_audit_impl(
|
||
project_id=project_id,
|
||
action=action,
|
||
resource_type=resource_type,
|
||
resource_id=resource_id,
|
||
details=details,
|
||
run_id=run_id,
|
||
trace_id=trace_id,
|
||
require_durable=require_durable,
|
||
idempotency_key=idempotency_key,
|
||
)
|
||
|
||
|
||
async def _write_audit_impl(
|
||
*,
|
||
project_id: str,
|
||
action: str,
|
||
resource_type: str,
|
||
resource_id: str,
|
||
details: dict[str, Any] | None,
|
||
run_id: str | None,
|
||
trace_id: str | None,
|
||
require_durable: bool = False,
|
||
idempotency_key: str | None = None,
|
||
) -> dict[str, Any] | None:
|
||
try:
|
||
from sqlalchemy import text as sa_text
|
||
|
||
from src.db.base import get_db_context
|
||
|
||
clean_details: dict[str, Any] = sanitize(details or {})
|
||
clean_action = _redact_string(str(action or "unknown"))[:200]
|
||
clean_resource_type = _redact_string(str(resource_type or "unknown"))[:80]
|
||
clean_resource_id = _redact_string(str(resource_id or "unknown"))[:200]
|
||
clean_run_id = _redact_string(str(run_id or ""))[:200]
|
||
clean_trace_id = _redact_string(str(trace_id or ""))[:200]
|
||
clean_project_id = _redact_string(str(project_id or ""))[:64]
|
||
actor = _redact_string(
|
||
str(clean_details.get("approver_id") or f"awooop:{clean_project_id}")
|
||
)[:100]
|
||
idempotency_key_value = str(idempotency_key or "").strip()
|
||
idempotency_key_sha256 = (
|
||
audit_idempotency_key_sha256(idempotency_key_value)
|
||
if idempotency_key_value
|
||
else ""
|
||
)
|
||
context = {
|
||
"schema_version": AWOOOP_AUDIT_SCHEMA_VERSION,
|
||
"redaction_version": AWOOOP_AUDIT_REDACTION_VERSION,
|
||
"project_id": clean_project_id,
|
||
"action": clean_action,
|
||
"resource_type": clean_resource_type,
|
||
"resource_id": clean_resource_id,
|
||
"run_id": clean_run_id,
|
||
"trace_id": clean_trace_id,
|
||
"details": clean_details,
|
||
"durable_write_required": require_durable,
|
||
"idempotency_key_sha256": idempotency_key_sha256,
|
||
}
|
||
params = {
|
||
"actor": actor,
|
||
"action_detail": clean_action,
|
||
"context": json.dumps(context, ensure_ascii=False),
|
||
}
|
||
|
||
async with get_db_context(project_id) as db:
|
||
if idempotency_key_value:
|
||
await db.execute(
|
||
sa_text("""
|
||
SELECT pg_advisory_xact_lock(
|
||
hashtextextended(:idempotency_key, 0)
|
||
)
|
||
"""),
|
||
{"idempotency_key": idempotency_key_value},
|
||
)
|
||
existing_result = await db.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*) AS matching_count,
|
||
MAX(created_at) AS created_at
|
||
FROM alert_operation_log
|
||
WHERE event_type = 'USER_ACTION'
|
||
AND action_detail = :action_detail
|
||
AND context ->> 'schema_version' = :schema_version
|
||
AND context ->> 'project_id' = :project_id
|
||
AND context ->> 'resource_type' = :resource_type
|
||
AND context ->> 'resource_id' = :resource_id
|
||
AND context ->> 'run_id' = :run_id
|
||
AND context ->> 'idempotency_key_sha256'
|
||
= :idempotency_key_sha256
|
||
"""),
|
||
{
|
||
"action_detail": clean_action,
|
||
"schema_version": AWOOOP_AUDIT_SCHEMA_VERSION,
|
||
"project_id": clean_project_id,
|
||
"resource_type": clean_resource_type,
|
||
"resource_id": clean_resource_id,
|
||
"run_id": clean_run_id,
|
||
"idempotency_key_sha256": idempotency_key_sha256,
|
||
},
|
||
)
|
||
existing = existing_result.mappings().one()
|
||
existing_count = int(existing["matching_count"] or 0)
|
||
if existing_count > 1:
|
||
raise AuditDurabilityError(
|
||
"audit_idempotency_duplicate_detected"
|
||
)
|
||
if existing_count == 1:
|
||
return {
|
||
"status": "duplicate_existing",
|
||
"durable_write_ack": True,
|
||
"inserted": False,
|
||
"created_at": existing["created_at"],
|
||
}
|
||
|
||
await db.execute(
|
||
sa_text("""
|
||
INSERT INTO alert_operation_log
|
||
(event_type, actor, action_detail, success, context)
|
||
VALUES
|
||
(
|
||
'USER_ACTION',
|
||
:actor,
|
||
:action_detail,
|
||
TRUE,
|
||
CAST(:context AS JSONB)
|
||
)
|
||
"""),
|
||
params,
|
||
)
|
||
if idempotency_key_value:
|
||
readback_result = await db.execute(
|
||
sa_text("""
|
||
SELECT COUNT(*) AS matching_count,
|
||
MAX(created_at) AS created_at
|
||
FROM alert_operation_log
|
||
WHERE event_type = 'USER_ACTION'
|
||
AND action_detail = :action_detail
|
||
AND context ->> 'schema_version' = :schema_version
|
||
AND context ->> 'project_id' = :project_id
|
||
AND context ->> 'resource_type' = :resource_type
|
||
AND context ->> 'resource_id' = :resource_id
|
||
AND context ->> 'run_id' = :run_id
|
||
AND context ->> 'idempotency_key_sha256'
|
||
= :idempotency_key_sha256
|
||
"""),
|
||
{
|
||
"action_detail": clean_action,
|
||
"schema_version": AWOOOP_AUDIT_SCHEMA_VERSION,
|
||
"project_id": clean_project_id,
|
||
"resource_type": clean_resource_type,
|
||
"resource_id": clean_resource_id,
|
||
"run_id": clean_run_id,
|
||
"idempotency_key_sha256": idempotency_key_sha256,
|
||
},
|
||
)
|
||
readback = readback_result.mappings().one()
|
||
if int(readback["matching_count"] or 0) != 1:
|
||
raise AuditDurabilityError(
|
||
"audit_durable_readback_not_exactly_once"
|
||
)
|
||
return {
|
||
"status": "inserted_verified",
|
||
"durable_write_ack": True,
|
||
"inserted": True,
|
||
"created_at": readback["created_at"],
|
||
}
|
||
return {
|
||
"status": "inserted",
|
||
"durable_write_ack": True,
|
||
"inserted": True,
|
||
"created_at": None,
|
||
}
|
||
except AuditDurabilityError:
|
||
raise
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"audit_sink_write_failed",
|
||
action=action,
|
||
resource_id=resource_id,
|
||
error_type=type(exc).__name__,
|
||
)
|
||
if require_durable:
|
||
raise AuditDurabilityError("audit_durable_write_failed") from exc
|
||
return None
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Convenience：可在測試中驗證 sanitization 結果
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
def sanitize_for_test(details: dict[str, Any]) -> dict[str, Any]:
|
||
"""同步 sanitize，供測試使用"""
|
||
return sanitize(details)
|