202 lines
7.3 KiB
Python
202 lines
7.3 KiB
Python
"""
|
|
NeMo/Nemotron Replay Request-Pack Sanitizer
|
|
==========================================
|
|
|
|
Builds an external-runner-safe request pack from internal fixtures. The goal is
|
|
to preserve incident semantics while removing sensitive-context markers such as
|
|
secret path names, htpasswd paths, and pgpass snippets before external replay.
|
|
|
|
This module is local and deterministic. It does not call external APIs, tools,
|
|
production systems, or LLMs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from src.services.agent_nemotron_replay_adapter import (
|
|
build_nemotron_replay_requests,
|
|
)
|
|
from src.services.agent_nemotron_replay_preflight import (
|
|
evaluate_nemotron_external_runner_preflight,
|
|
)
|
|
from src.services.agent_replay_input import (
|
|
build_candidate_inputs_from_fixtures,
|
|
)
|
|
from src.services.sanitization_service import sanitize
|
|
|
|
SANITIZE_REPORT_SCHEMA_VERSION = "agent_nemotron_request_pack_sanitize_report_v1"
|
|
SENSITIVE_CONTEXT_REDACTED = "[SENSITIVE_CONTEXT_REDACTED]"
|
|
|
|
_SENSITIVE_KEY_MARKERS = (
|
|
"authorization",
|
|
"bearer",
|
|
"password",
|
|
"passwd",
|
|
"pgpass",
|
|
"secret",
|
|
"token",
|
|
"api_key",
|
|
"apikey",
|
|
)
|
|
_SENSITIVE_CONTEXT_PATTERN = re.compile(
|
|
r"(?i)(?<![A-Za-z0-9_./-])"
|
|
r"[A-Za-z0-9_./:-]*(?:"
|
|
r"\.secrets?|secrets?|secret|htpasswd|pgpass|passwd|password|api[_-]?key|token"
|
|
r")[A-Za-z0-9_./:=:-]*"
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NemotronRequestPackSanitizeReport:
|
|
"""Sanitization summary for a NeMo request-pack rebuild."""
|
|
|
|
fixtures: int
|
|
candidate_inputs: int
|
|
requests: int
|
|
valid: bool
|
|
changed_fixture_records: int
|
|
sensitive_marker_records_before: int
|
|
sensitive_marker_records_after: int
|
|
preflight_valid: bool
|
|
failures: list[str] = field(default_factory=list)
|
|
marker_distribution_before: dict[str, int] = field(default_factory=dict)
|
|
marker_distribution_after: dict[str, int] = field(default_factory=dict)
|
|
preflight_failures: list[str] = field(default_factory=list)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": SANITIZE_REPORT_SCHEMA_VERSION,
|
|
"fixtures": self.fixtures,
|
|
"candidate_inputs": self.candidate_inputs,
|
|
"requests": self.requests,
|
|
"valid": self.valid,
|
|
"changed_fixture_records": self.changed_fixture_records,
|
|
"sensitive_marker_records_before": self.sensitive_marker_records_before,
|
|
"sensitive_marker_records_after": self.sensitive_marker_records_after,
|
|
"marker_distribution_before": dict(self.marker_distribution_before),
|
|
"marker_distribution_after": dict(self.marker_distribution_after),
|
|
"preflight_valid": self.preflight_valid,
|
|
"preflight_failures": list(self.preflight_failures),
|
|
"failures": list(self.failures),
|
|
}
|
|
|
|
|
|
def sanitize_nemotron_request_pack_from_fixtures(
|
|
fixtures: list[dict[str, Any]],
|
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], NemotronRequestPackSanitizeReport]:
|
|
"""Sanitize fixtures, rebuild candidate inputs, rebuild requests, and preflight."""
|
|
pre_before = evaluate_nemotron_external_runner_preflight(
|
|
fixtures=fixtures,
|
|
candidate_inputs=[
|
|
candidate_input.to_dict()
|
|
for candidate_input in build_candidate_inputs_from_fixtures(fixtures)
|
|
],
|
|
requests=[
|
|
request.to_dict()
|
|
for request in build_nemotron_replay_requests(
|
|
[
|
|
candidate_input.to_dict()
|
|
for candidate_input in build_candidate_inputs_from_fixtures(fixtures)
|
|
]
|
|
)
|
|
],
|
|
)
|
|
|
|
sanitized_fixtures = [_sanitize_fixture(fixture) for fixture in fixtures]
|
|
changed_records = sum(
|
|
1
|
|
for original, sanitized in zip(fixtures, sanitized_fixtures, strict=False)
|
|
if original.get("incident_context") != sanitized.get("incident_context")
|
|
)
|
|
candidate_inputs = [
|
|
candidate_input.to_dict()
|
|
for candidate_input in build_candidate_inputs_from_fixtures(sanitized_fixtures)
|
|
]
|
|
requests = [
|
|
request.to_dict()
|
|
for request in build_nemotron_replay_requests(candidate_inputs)
|
|
]
|
|
pre_after = evaluate_nemotron_external_runner_preflight(
|
|
fixtures=sanitized_fixtures,
|
|
candidate_inputs=candidate_inputs,
|
|
requests=requests,
|
|
)
|
|
|
|
report = NemotronRequestPackSanitizeReport(
|
|
fixtures=len(sanitized_fixtures),
|
|
candidate_inputs=len(candidate_inputs),
|
|
requests=len(requests),
|
|
valid=pre_after.valid,
|
|
changed_fixture_records=changed_records,
|
|
sensitive_marker_records_before=pre_before.sensitive_marker_records,
|
|
sensitive_marker_records_after=pre_after.sensitive_marker_records,
|
|
marker_distribution_before=pre_before.sensitive_marker_distribution,
|
|
marker_distribution_after=pre_after.sensitive_marker_distribution,
|
|
preflight_valid=pre_after.valid,
|
|
preflight_failures=list(pre_after.failures),
|
|
failures=[] if pre_after.valid else ["preflight_invalid_after_sanitize"],
|
|
)
|
|
return sanitized_fixtures, candidate_inputs, requests, report
|
|
|
|
|
|
def _sanitize_fixture(fixture: dict[str, Any]) -> dict[str, Any]:
|
|
sanitized = dict(fixture)
|
|
sanitized["incident_context"] = _sanitize_external_visible_value(
|
|
fixture.get("incident_context") or {}
|
|
)
|
|
sanitized["source_metadata"] = _sanitize_external_visible_value(
|
|
fixture.get("source_metadata") or {}
|
|
)
|
|
return sanitized
|
|
|
|
|
|
def _sanitize_external_visible_value(value: Any) -> Any:
|
|
if isinstance(value, dict):
|
|
sanitized: dict[str, Any] = {}
|
|
index = 0
|
|
for key, nested in value.items():
|
|
key_text = str(key)
|
|
if _is_sensitive_key(key_text):
|
|
safe_key = f"redacted_sensitive_field_{index}"
|
|
index += 1
|
|
sanitized[safe_key] = SENSITIVE_CONTEXT_REDACTED
|
|
else:
|
|
sanitized[key_text] = _sanitize_external_visible_value(nested)
|
|
return sanitized
|
|
if isinstance(value, list):
|
|
return [_sanitize_external_visible_value(item) for item in value]
|
|
if isinstance(value, tuple):
|
|
return [_sanitize_external_visible_value(item) for item in value]
|
|
if isinstance(value, str):
|
|
return _sanitize_external_visible_string(value)
|
|
return value
|
|
|
|
|
|
def _sanitize_external_visible_string(value: str) -> str:
|
|
text = sanitize(value, source_label="nemotron_replay_external_visible")
|
|
text = _SENSITIVE_CONTEXT_PATTERN.sub(SENSITIVE_CONTEXT_REDACTED, text)
|
|
return _collapse_repeated_redactions(text)
|
|
|
|
|
|
def _collapse_repeated_redactions(value: str) -> str:
|
|
serialized = value
|
|
repeated = f"{SENSITIVE_CONTEXT_REDACTED}{SENSITIVE_CONTEXT_REDACTED}"
|
|
while repeated in serialized:
|
|
serialized = serialized.replace(repeated, SENSITIVE_CONTEXT_REDACTED)
|
|
return serialized
|
|
|
|
|
|
def _is_sensitive_key(key: str) -> bool:
|
|
lowered = key.lower()
|
|
return any(marker in lowered for marker in _SENSITIVE_KEY_MARKERS)
|
|
|
|
|
|
def contains_sensitive_context_marker(payload: Any) -> bool:
|
|
"""Return true when payload still contains sensitive context marker text."""
|
|
serialized = json.dumps(payload, ensure_ascii=False, sort_keys=True).lower()
|
|
return any(marker in serialized for marker in _SENSITIVE_KEY_MARKERS)
|