959 lines
34 KiB
Python
959 lines
34 KiB
Python
"""Config Drift fingerprint FSM read model.
|
||
|
||
The drift scanner creates a new report_id on every scan, so the operator-facing
|
||
state must be keyed by the stable drift fingerprint instead of the volatile
|
||
report id. This service is intentionally conservative: it can record human
|
||
handoff and remediation breadcrumbs, but it does not adopt, merge, roll back,
|
||
or mutate incident state.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import datetime
|
||
from typing import Any, Literal
|
||
|
||
import httpx
|
||
import structlog
|
||
from sqlalchemy import text
|
||
|
||
from src.core.config import get_settings
|
||
from src.db.base import get_db_context
|
||
from src.models.drift import DriftReport
|
||
from src.repositories.drift_repository import get_drift_repository
|
||
from src.services.drift_repeat_state import build_drift_fingerprint
|
||
from src.utils.timezone import now_taipei
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
SCHEMA_VERSION = "drift_fingerprint_state_v1"
|
||
HANDOFF_SCHEMA_VERSION = "drift_fingerprint_handoff_history_v1"
|
||
REMEDIATION_SCHEMA_VERSION = "drift_fingerprint_remediation_history_v1"
|
||
SERVICE_ACTOR = "drift_fingerprint_state_service"
|
||
DEFAULT_NAMESPACE = "awoooi-prod"
|
||
|
||
DriftFingerprintHandoffKind = Literal[
|
||
"open_pr_review",
|
||
"manual_investigation",
|
||
"zero_diff_pr_cleanup",
|
||
]
|
||
|
||
DriftFingerprintRemediationKind = Literal[
|
||
"live_env_rollback",
|
||
"git_adopted",
|
||
"git_rollback",
|
||
"zero_diff_pr_cleanup",
|
||
"manual_noop",
|
||
]
|
||
|
||
DriftFingerprintRemediationStatus = Literal[
|
||
"executed_unverified",
|
||
"verified_no_drift",
|
||
"verification_failed",
|
||
]
|
||
|
||
|
||
class DriftFingerprintStateNotFoundError(Exception):
|
||
"""Raised when no drift report can be found for the requested selector."""
|
||
|
||
|
||
def _enum_value(value: Any) -> Any:
|
||
return getattr(value, "value", value)
|
||
|
||
|
||
def _iso(value: Any) -> str | None:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, datetime):
|
||
return value.isoformat()
|
||
return str(value)
|
||
|
||
|
||
def _report_summary(report: DriftReport) -> str:
|
||
return (
|
||
f"HIGH×{report.high_count}, "
|
||
f"MEDIUM×{report.medium_count}, "
|
||
f"INFO×{report.info_count}"
|
||
)
|
||
|
||
|
||
def _interpretation_summary(report: DriftReport) -> dict[str, Any] | None:
|
||
if report.interpretation is None:
|
||
return None
|
||
return {
|
||
"intent": _enum_value(report.interpretation.intent),
|
||
"risk": report.interpretation.risk,
|
||
"confidence": report.interpretation.confidence,
|
||
"explanation": report.interpretation.explanation,
|
||
}
|
||
|
||
|
||
def _pr_refs(pr: dict[str, Any] | None) -> list[str]:
|
||
if not pr:
|
||
return []
|
||
refs = [pr.get("html_url"), pr.get("url")]
|
||
number = pr.get("number")
|
||
if number is not None:
|
||
refs.append(f"pull/{number}")
|
||
return [str(ref) for ref in refs if ref]
|
||
|
||
|
||
def _derive_fsm_state(
|
||
report: DriftReport,
|
||
repeat_state: dict[str, Any],
|
||
open_pr: dict[str, Any] | None,
|
||
latest_handoff: dict[str, Any] | None,
|
||
latest_remediation: dict[str, Any] | None,
|
||
) -> str:
|
||
status = str(_enum_value(report.status))
|
||
remediation_status = (latest_remediation or {}).get("remediation_status")
|
||
if remediation_status == "verified_no_drift":
|
||
if report.high_count == 0 and report.medium_count == 0 and report.info_count == 0:
|
||
return "no_drift_verified"
|
||
return "remediated_verified"
|
||
if remediation_status == "executed_unverified":
|
||
return "remediation_executed_unverified"
|
||
if remediation_status == "verification_failed":
|
||
return "remediation_verification_failed"
|
||
|
||
if status == "adopted":
|
||
return "adopted_unverified"
|
||
if status == "rolled_back":
|
||
return "rolled_back"
|
||
if status in {"acknowledged", "ignored"}:
|
||
return status
|
||
|
||
if open_pr:
|
||
if open_pr.get("merged"):
|
||
return "pr_merged_unverified"
|
||
if open_pr.get("state") == "open" and open_pr.get("is_zero_diff"):
|
||
return "pr_open_zero_diff"
|
||
if open_pr.get("state") == "open":
|
||
return "pr_open_waiting_review"
|
||
|
||
if latest_handoff:
|
||
return "handoff_recorded"
|
||
|
||
if int(repeat_state.get("occurrences_12h") or 0) > 1:
|
||
return "pending_human_repeated"
|
||
return "pending_human"
|
||
|
||
|
||
def _next_step_for_state(state: str, open_pr: dict[str, Any] | None) -> str:
|
||
if state in {"no_drift_verified", "remediated_verified"}:
|
||
return "monitor_for_recurrence"
|
||
if state == "remediation_executed_unverified":
|
||
return "run_verification_scan_then_record_result"
|
||
if state == "remediation_verification_failed":
|
||
return "open_manual_investigation_with_failed_verification"
|
||
if state == "pr_open_zero_diff":
|
||
return "close_zero_diff_pr_and_prepare_real_yaml_patch"
|
||
if state == "pr_open_waiting_review":
|
||
return "review_pr_then_merge_or_reject"
|
||
if state == "pr_merged_unverified":
|
||
return "verify_git_baseline_then_mark_adopted"
|
||
if state == "handoff_recorded":
|
||
return "operator_review_handoff_and_execute_manual_plan"
|
||
if state == "adopted_unverified":
|
||
return "verify_k8s_matches_git_baseline"
|
||
if state == "rolled_back":
|
||
return "confirm_no_repeat_after_rollback"
|
||
if state in {"acknowledged", "ignored"}:
|
||
return "monitor_for_recurrence"
|
||
if open_pr and open_pr.get("lookup_error"):
|
||
return "retry_pr_lookup_then_review_drift"
|
||
return "manual_investigation_or_ansible_check_mode"
|
||
|
||
|
||
def build_drift_fingerprint_state(
|
||
report: DriftReport,
|
||
repeat_state: dict[str, Any],
|
||
*,
|
||
open_pr: dict[str, Any] | None = None,
|
||
latest_handoff: dict[str, Any] | None = None,
|
||
latest_remediation: dict[str, Any] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Build the operator-facing Config Drift fingerprint state payload."""
|
||
|
||
fingerprint = repeat_state.get("fingerprint") or build_drift_fingerprint(
|
||
report.namespace,
|
||
report.items,
|
||
)
|
||
fsm_state = _derive_fsm_state(
|
||
report,
|
||
repeat_state,
|
||
open_pr,
|
||
latest_handoff,
|
||
latest_remediation,
|
||
)
|
||
next_step = _next_step_for_state(fsm_state, open_pr)
|
||
|
||
return {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"namespace": report.namespace,
|
||
"fingerprint": fingerprint,
|
||
"latest_report_id": report.report_id,
|
||
"latest_status": str(_enum_value(report.status)),
|
||
"latest_scanned_at": _iso(report.scanned_at),
|
||
"latest_created_at": _iso(report.created_at),
|
||
"summary": _report_summary(report),
|
||
"high_count": report.high_count,
|
||
"medium_count": report.medium_count,
|
||
"info_count": report.info_count,
|
||
"interpretation": _interpretation_summary(report),
|
||
"repeat_state": repeat_state,
|
||
"occurrences_12h": repeat_state.get("occurrences_12h", 0),
|
||
"matching_strategy": repeat_state.get("matching_strategy"),
|
||
"operator_stage": fsm_state,
|
||
"fsm_state": fsm_state,
|
||
"next_step": next_step,
|
||
"open_pr": open_pr,
|
||
"latest_handoff": latest_handoff,
|
||
"latest_remediation": latest_remediation,
|
||
"strict_fingerprint": repeat_state.get("strict_fingerprint"),
|
||
"p0_escalation": {
|
||
"suppresses_repeated_p0": True,
|
||
"dedup_key_strategy": "semantic_drift_fingerprint",
|
||
"dedup_window_hours": 24,
|
||
},
|
||
"read_model_route": {
|
||
"agent_id": "openclaw",
|
||
"tool_name": "drift_fingerprint_state",
|
||
"required_scope": "read:drift read:gitea",
|
||
"flywheel_node": (
|
||
"drift_scanned>ai_analyzed>fingerprint_fsm>"
|
||
"operator_review"
|
||
),
|
||
},
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
"writes_drift_status": False,
|
||
"writes_ticket": False,
|
||
"writes_remediation_record": False,
|
||
"creates_external_ticket": False,
|
||
}
|
||
|
||
|
||
def _handoff_context(
|
||
state: dict[str, Any],
|
||
*,
|
||
handoff_kind: DriftFingerprintHandoffKind,
|
||
pr_url: str | None,
|
||
note: str | None,
|
||
) -> dict[str, Any]:
|
||
return {
|
||
"schema_version": HANDOFF_SCHEMA_VERSION,
|
||
"fingerprint": state.get("fingerprint"),
|
||
"namespace": state.get("namespace"),
|
||
"latest_report_id": state.get("latest_report_id"),
|
||
"handoff_kind": handoff_kind,
|
||
"handoff_status": "recorded",
|
||
"pr_url": pr_url,
|
||
"note": note,
|
||
"fsm_state": state.get("fsm_state"),
|
||
"next_step": state.get("next_step"),
|
||
"open_pr": state.get("open_pr"),
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
"writes_drift_status": False,
|
||
"writes_ticket": False,
|
||
"creates_external_ticket": False,
|
||
"recorded_at": now_taipei().isoformat(),
|
||
}
|
||
|
||
|
||
def _handoff_description(context: dict[str, Any]) -> str:
|
||
return (
|
||
f"fingerprint={context.get('fingerprint')} "
|
||
f"state={context.get('fsm_state')} "
|
||
f"kind={context.get('handoff_kind')} "
|
||
f"next={context.get('next_step')} "
|
||
f"pr={context.get('pr_url') or '--'}"
|
||
)[:500]
|
||
|
||
|
||
def _is_no_drift_report(report: DriftReport | None) -> bool:
|
||
if report is None:
|
||
return False
|
||
return report.high_count == 0 and report.medium_count == 0 and report.info_count == 0
|
||
|
||
|
||
def _remediation_context(
|
||
state: dict[str, Any],
|
||
*,
|
||
remediation_kind: DriftFingerprintRemediationKind,
|
||
remediation_status: DriftFingerprintRemediationStatus,
|
||
verification_report: DriftReport | None,
|
||
note: str | None,
|
||
commands_summary: list[str] | None,
|
||
) -> dict[str, Any]:
|
||
verification_summary = None
|
||
if verification_report is not None:
|
||
verification_summary = {
|
||
"report_id": verification_report.report_id,
|
||
"status": str(_enum_value(verification_report.status)),
|
||
"summary": _report_summary(verification_report),
|
||
"high_count": verification_report.high_count,
|
||
"medium_count": verification_report.medium_count,
|
||
"info_count": verification_report.info_count,
|
||
"scanned_at": _iso(verification_report.scanned_at),
|
||
"is_no_drift": _is_no_drift_report(verification_report),
|
||
}
|
||
|
||
return {
|
||
"schema_version": REMEDIATION_SCHEMA_VERSION,
|
||
"fingerprint": state.get("fingerprint"),
|
||
"namespace": state.get("namespace"),
|
||
"remediated_report_id": state.get("latest_report_id"),
|
||
"verification_report_id": getattr(verification_report, "report_id", None),
|
||
"verification_summary": verification_summary,
|
||
"remediation_kind": remediation_kind,
|
||
"remediation_status": remediation_status,
|
||
"note": note,
|
||
"commands_summary": [
|
||
str(command)[:180] for command in (commands_summary or [])[:12]
|
||
],
|
||
"fsm_state_before": state.get("fsm_state"),
|
||
"next_step_before": state.get("next_step"),
|
||
"writes_incident_state": False,
|
||
"writes_auto_repair_result": False,
|
||
"writes_drift_status": False,
|
||
"writes_ticket": False,
|
||
"writes_remediation_record": True,
|
||
"creates_external_ticket": False,
|
||
"recorded_at": now_taipei().isoformat(),
|
||
}
|
||
|
||
|
||
def _remediation_description(context: dict[str, Any]) -> str:
|
||
verification = context.get("verification_summary") or {}
|
||
return (
|
||
f"fingerprint={context.get('fingerprint')} "
|
||
f"kind={context.get('remediation_kind')} "
|
||
f"status={context.get('remediation_status')} "
|
||
f"remediated_report={context.get('remediated_report_id')} "
|
||
f"verification_report={context.get('verification_report_id') or '--'} "
|
||
f"verification={verification.get('summary') or '--'}"
|
||
)[:500]
|
||
|
||
|
||
class DriftFingerprintStateService:
|
||
"""Read and record the state of a stable Config Drift fingerprint."""
|
||
|
||
async def get_state(
|
||
self,
|
||
*,
|
||
report_id: str | None = None,
|
||
namespace: str | None = None,
|
||
) -> dict[str, Any]:
|
||
report = await self._load_report(report_id=report_id, namespace=namespace)
|
||
repo = get_drift_repository()
|
||
repeat_state = await repo.get_repeat_state(report, include_values=False)
|
||
fingerprint = repeat_state["fingerprint"]
|
||
open_pr = await self._lookup_open_pr(report)
|
||
latest_handoff = await self._fetch_latest_handoff(
|
||
fingerprint,
|
||
alternate_fingerprints=_repeat_state_fingerprint_aliases(repeat_state),
|
||
report_ids=_repeat_state_report_ids(repeat_state),
|
||
)
|
||
latest_remediation = await self._fetch_latest_remediation(
|
||
fingerprint,
|
||
alternate_fingerprints=_repeat_state_fingerprint_aliases(repeat_state),
|
||
report_ids={
|
||
report.report_id,
|
||
*_repeat_state_report_ids(repeat_state),
|
||
},
|
||
namespace=report.namespace,
|
||
allow_namespace_fallback=_is_no_drift_report(report),
|
||
)
|
||
return build_drift_fingerprint_state(
|
||
report,
|
||
repeat_state,
|
||
open_pr=open_pr,
|
||
latest_handoff=latest_handoff,
|
||
latest_remediation=latest_remediation,
|
||
)
|
||
|
||
async def record_handoff(
|
||
self,
|
||
*,
|
||
report_id: str | None = None,
|
||
namespace: str | None = None,
|
||
handoff_kind: DriftFingerprintHandoffKind = "open_pr_review",
|
||
pr_url: str | None = None,
|
||
note: str | None = None,
|
||
) -> dict[str, Any]:
|
||
state = await self.get_state(report_id=report_id, namespace=namespace)
|
||
context = _handoff_context(
|
||
state,
|
||
handoff_kind=handoff_kind,
|
||
pr_url=pr_url or _default_pr_url(state.get("open_pr")),
|
||
note=note,
|
||
)
|
||
|
||
history = {
|
||
"recorded": False,
|
||
"alert_operation_id": None,
|
||
"timeline_event_id": None,
|
||
"reason": None,
|
||
}
|
||
incident_id = str(state.get("latest_report_id") or "")
|
||
|
||
try:
|
||
from src.repositories.alert_operation_log_repository import (
|
||
get_alert_operation_log_repository,
|
||
)
|
||
|
||
record = await get_alert_operation_log_repository().append(
|
||
"ESCALATED",
|
||
incident_id=incident_id or None,
|
||
actor=SERVICE_ACTOR,
|
||
action_detail=f"drift_fingerprint_handoff:{handoff_kind}"[:200],
|
||
success=True,
|
||
context=context,
|
||
)
|
||
if record is not None:
|
||
history["alert_operation_id"] = getattr(record, "id", None)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"drift_fingerprint_handoff_aol_failed",
|
||
report_id=incident_id,
|
||
error=str(exc),
|
||
)
|
||
|
||
try:
|
||
from src.services.approval_db import get_timeline_service
|
||
|
||
event = await get_timeline_service().add_event(
|
||
event_type="human",
|
||
status="warning",
|
||
title="AwoooP drift fingerprint handoff",
|
||
description=_handoff_description(context),
|
||
actor=SERVICE_ACTOR,
|
||
actor_role=handoff_kind,
|
||
incident_id=incident_id or None,
|
||
)
|
||
if event:
|
||
history["timeline_event_id"] = event.get("id")
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"drift_fingerprint_handoff_timeline_failed",
|
||
report_id=incident_id,
|
||
error=str(exc),
|
||
)
|
||
|
||
history["recorded"] = bool(
|
||
history.get("alert_operation_id") or history.get("timeline_event_id")
|
||
)
|
||
if not history["recorded"]:
|
||
history["reason"] = "history_sink_unavailable"
|
||
|
||
updated_state = {
|
||
**state,
|
||
"latest_handoff": {
|
||
"handoff_kind": handoff_kind,
|
||
"handoff_status": "recorded" if history["recorded"] else "record_failed",
|
||
"pr_url": context.get("pr_url"),
|
||
"note": note,
|
||
"created_at": context.get("recorded_at"),
|
||
"source": "current_request",
|
||
},
|
||
}
|
||
updated_state["operator_stage"] = (
|
||
"handoff_recorded" if history["recorded"] else state["operator_stage"]
|
||
)
|
||
updated_state["fsm_state"] = updated_state["operator_stage"]
|
||
updated_state["next_step"] = _next_step_for_state(
|
||
updated_state["fsm_state"],
|
||
state.get("open_pr"),
|
||
)
|
||
|
||
return {
|
||
**updated_state,
|
||
"handoff_kind": handoff_kind,
|
||
"handoff_status": updated_state["latest_handoff"]["handoff_status"],
|
||
"history": history,
|
||
}
|
||
|
||
async def record_remediation(
|
||
self,
|
||
*,
|
||
report_id: str | None = None,
|
||
namespace: str | None = None,
|
||
remediation_kind: DriftFingerprintRemediationKind = "live_env_rollback",
|
||
remediation_status: DriftFingerprintRemediationStatus | None = None,
|
||
verification_report_id: str | None = None,
|
||
note: str | None = None,
|
||
commands_summary: list[str] | None = None,
|
||
) -> dict[str, Any]:
|
||
state = await self.get_state(report_id=report_id, namespace=namespace)
|
||
verification_report = await self._load_verification_report(
|
||
report_id=verification_report_id,
|
||
namespace=state.get("namespace") or namespace,
|
||
)
|
||
resolved_status: DriftFingerprintRemediationStatus = (
|
||
remediation_status
|
||
or (
|
||
"verified_no_drift"
|
||
if _is_no_drift_report(verification_report)
|
||
else "executed_unverified"
|
||
)
|
||
)
|
||
context = _remediation_context(
|
||
state,
|
||
remediation_kind=remediation_kind,
|
||
remediation_status=resolved_status,
|
||
verification_report=verification_report,
|
||
note=note,
|
||
commands_summary=commands_summary,
|
||
)
|
||
|
||
history = {
|
||
"recorded": False,
|
||
"alert_operation_id": None,
|
||
"timeline_event_id": None,
|
||
"reason": None,
|
||
}
|
||
incident_id = str(state.get("latest_report_id") or "")
|
||
success = resolved_status != "verification_failed"
|
||
|
||
try:
|
||
from src.repositories.alert_operation_log_repository import (
|
||
get_alert_operation_log_repository,
|
||
)
|
||
|
||
record = await get_alert_operation_log_repository().append(
|
||
"CHANGE_APPLIED",
|
||
incident_id=incident_id or None,
|
||
actor=SERVICE_ACTOR,
|
||
action_detail=(
|
||
f"drift_fingerprint_remediation:{remediation_kind}"
|
||
)[:200],
|
||
success=success,
|
||
context=context,
|
||
)
|
||
if record is not None:
|
||
history["alert_operation_id"] = getattr(record, "id", None)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"drift_fingerprint_remediation_aol_failed",
|
||
report_id=incident_id,
|
||
error=str(exc),
|
||
)
|
||
|
||
try:
|
||
from src.services.approval_db import get_timeline_service
|
||
|
||
event = await get_timeline_service().add_event(
|
||
event_type="exec",
|
||
status="success" if success else "error",
|
||
title="AwoooP drift remediation verified",
|
||
description=_remediation_description(context),
|
||
actor=SERVICE_ACTOR,
|
||
actor_role=remediation_kind,
|
||
incident_id=incident_id or None,
|
||
)
|
||
if event:
|
||
history["timeline_event_id"] = event.get("id")
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"drift_fingerprint_remediation_timeline_failed",
|
||
report_id=incident_id,
|
||
error=str(exc),
|
||
)
|
||
|
||
history["recorded"] = bool(
|
||
history.get("alert_operation_id") or history.get("timeline_event_id")
|
||
)
|
||
if not history["recorded"]:
|
||
history["reason"] = "history_sink_unavailable"
|
||
|
||
latest_remediation = {
|
||
"remediation_kind": remediation_kind,
|
||
"remediation_status": (
|
||
resolved_status if history["recorded"] else "record_failed"
|
||
),
|
||
"verification_report_id": context.get("verification_report_id"),
|
||
"verification_summary": context.get("verification_summary"),
|
||
"note": note,
|
||
"commands_summary": context.get("commands_summary"),
|
||
"created_at": context.get("recorded_at"),
|
||
"source": "current_request",
|
||
}
|
||
updated_state = build_drift_fingerprint_state(
|
||
await self._load_report(
|
||
report_id=str(state.get("latest_report_id") or ""),
|
||
namespace=state.get("namespace"),
|
||
),
|
||
{
|
||
**(state.get("repeat_state") or {}),
|
||
"fingerprint": state.get("fingerprint"),
|
||
},
|
||
open_pr=state.get("open_pr"),
|
||
latest_handoff=state.get("latest_handoff"),
|
||
latest_remediation=latest_remediation,
|
||
)
|
||
|
||
return {
|
||
**updated_state,
|
||
"remediation_kind": remediation_kind,
|
||
"remediation_status": latest_remediation["remediation_status"],
|
||
"history": history,
|
||
}
|
||
|
||
async def _load_report(
|
||
self,
|
||
*,
|
||
report_id: str | None,
|
||
namespace: str | None,
|
||
) -> DriftReport:
|
||
repo = get_drift_repository()
|
||
if report_id:
|
||
report = await repo.get(report_id)
|
||
if report is None:
|
||
raise DriftFingerprintStateNotFoundError(report_id)
|
||
return report
|
||
|
||
desired_namespace = namespace or DEFAULT_NAMESPACE
|
||
for report in await repo.list_recent(limit=50):
|
||
if report.namespace == desired_namespace:
|
||
return report
|
||
raise DriftFingerprintStateNotFoundError(desired_namespace)
|
||
|
||
async def _load_verification_report(
|
||
self,
|
||
*,
|
||
report_id: str | None,
|
||
namespace: str | None,
|
||
) -> DriftReport | None:
|
||
repo = get_drift_repository()
|
||
if report_id:
|
||
return await repo.get(report_id)
|
||
|
||
desired_namespace = namespace or DEFAULT_NAMESPACE
|
||
for report in await repo.list_recent(limit=50):
|
||
if report.namespace == desired_namespace and _is_no_drift_report(report):
|
||
return report
|
||
return None
|
||
|
||
async def _fetch_latest_handoff(
|
||
self,
|
||
fingerprint: str,
|
||
*,
|
||
alternate_fingerprints: set[str] | None = None,
|
||
report_ids: set[str] | None = None,
|
||
) -> dict[str, Any] | None:
|
||
fingerprints = {fingerprint, *(alternate_fingerprints or set())}
|
||
fingerprints = {value for value in fingerprints if value}
|
||
report_ids = {value for value in (report_ids or set()) if value}
|
||
try:
|
||
async with get_db_context() as db:
|
||
result = await db.execute(
|
||
text(
|
||
"""
|
||
SELECT id, action_detail, success, context, created_at
|
||
FROM alert_operation_log
|
||
WHERE actor = :actor
|
||
AND action_detail LIKE 'drift_fingerprint_handoff:%'
|
||
ORDER BY created_at DESC
|
||
LIMIT 100
|
||
"""
|
||
),
|
||
{"actor": SERVICE_ACTOR},
|
||
)
|
||
rows = result.mappings().all()
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"drift_fingerprint_handoff_lookup_failed",
|
||
fingerprint=fingerprint,
|
||
error=str(exc),
|
||
)
|
||
return {
|
||
"lookup_error": str(exc)[:160],
|
||
"handoff_status": "lookup_failed",
|
||
}
|
||
|
||
row = None
|
||
for candidate in rows:
|
||
context = candidate.get("context") or {}
|
||
if not isinstance(context, dict):
|
||
continue
|
||
if context.get("fingerprint") in fingerprints:
|
||
row = candidate
|
||
break
|
||
if context.get("latest_report_id") in report_ids:
|
||
row = candidate
|
||
break
|
||
|
||
if row is None:
|
||
return None
|
||
context = row.get("context") or {}
|
||
return {
|
||
"alert_operation_id": row.get("id"),
|
||
"action_detail": row.get("action_detail"),
|
||
"success": row.get("success"),
|
||
"created_at": _iso(row.get("created_at")),
|
||
"handoff_kind": context.get("handoff_kind"),
|
||
"handoff_status": context.get("handoff_status") or "recorded",
|
||
"pr_url": context.get("pr_url"),
|
||
"note": context.get("note"),
|
||
}
|
||
|
||
async def _fetch_latest_remediation(
|
||
self,
|
||
fingerprint: str,
|
||
*,
|
||
alternate_fingerprints: set[str] | None = None,
|
||
report_ids: set[str] | None = None,
|
||
namespace: str | None = None,
|
||
allow_namespace_fallback: bool = False,
|
||
) -> dict[str, Any] | None:
|
||
fingerprints = {fingerprint, *(alternate_fingerprints or set())}
|
||
fingerprints = {value for value in fingerprints if value}
|
||
report_ids = {value for value in (report_ids or set()) if value}
|
||
try:
|
||
async with get_db_context() as db:
|
||
result = await db.execute(
|
||
text(
|
||
"""
|
||
SELECT id, action_detail, success, context, created_at
|
||
FROM alert_operation_log
|
||
WHERE actor = :actor
|
||
AND action_detail LIKE 'drift_fingerprint_remediation:%'
|
||
ORDER BY created_at DESC
|
||
LIMIT 100
|
||
"""
|
||
),
|
||
{"actor": SERVICE_ACTOR},
|
||
)
|
||
rows = result.mappings().all()
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"drift_fingerprint_remediation_lookup_failed",
|
||
fingerprint=fingerprint,
|
||
error=str(exc),
|
||
)
|
||
return {
|
||
"lookup_error": str(exc)[:160],
|
||
"remediation_status": "lookup_failed",
|
||
}
|
||
|
||
namespace_fallback = None
|
||
row = None
|
||
for candidate in rows:
|
||
context = candidate.get("context") or {}
|
||
if not isinstance(context, dict):
|
||
continue
|
||
if context.get("fingerprint") in fingerprints:
|
||
row = candidate
|
||
break
|
||
if context.get("remediated_report_id") in report_ids:
|
||
row = candidate
|
||
break
|
||
if context.get("verification_report_id") in report_ids:
|
||
row = candidate
|
||
break
|
||
if (
|
||
allow_namespace_fallback
|
||
and namespace
|
||
and context.get("namespace") == namespace
|
||
and namespace_fallback is None
|
||
):
|
||
namespace_fallback = candidate
|
||
|
||
row = row or namespace_fallback
|
||
if row is None:
|
||
return None
|
||
context = row.get("context") or {}
|
||
return {
|
||
"alert_operation_id": row.get("id"),
|
||
"action_detail": row.get("action_detail"),
|
||
"success": row.get("success"),
|
||
"created_at": _iso(row.get("created_at")),
|
||
"remediation_kind": context.get("remediation_kind"),
|
||
"remediation_status": context.get("remediation_status") or "recorded",
|
||
"remediated_report_id": context.get("remediated_report_id"),
|
||
"verification_report_id": context.get("verification_report_id"),
|
||
"verification_summary": context.get("verification_summary"),
|
||
"note": context.get("note"),
|
||
"commands_summary": context.get("commands_summary"),
|
||
}
|
||
|
||
async def _lookup_open_pr(self, report: DriftReport) -> dict[str, Any] | None:
|
||
settings = get_settings()
|
||
api_url = settings.GITEA_API_URL.rstrip("/")
|
||
if not api_url:
|
||
return None
|
||
|
||
headers = {"Accept": "application/json"}
|
||
if settings.GITEA_API_TOKEN:
|
||
headers["Authorization"] = f"token {settings.GITEA_API_TOKEN}"
|
||
|
||
try:
|
||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||
pulls = await client.get(
|
||
(
|
||
f"{api_url}/api/v1/repos/"
|
||
f"{settings.GITEA_REPO_OWNER}/{settings.GITEA_REPO_NAME}/pulls"
|
||
),
|
||
params={"state": "open", "limit": 20},
|
||
headers=headers,
|
||
)
|
||
if pulls.status_code != 200:
|
||
return {
|
||
"lookup_status": "failed",
|
||
"lookup_error": f"pulls_http_{pulls.status_code}",
|
||
}
|
||
|
||
for pr in pulls.json() or []:
|
||
if not _matches_drift_pr(report, pr):
|
||
continue
|
||
return await _build_pr_state(
|
||
client=client,
|
||
api_url=api_url,
|
||
owner=settings.GITEA_REPO_OWNER,
|
||
repo=settings.GITEA_REPO_NAME,
|
||
headers=headers,
|
||
pr=pr,
|
||
)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"drift_fingerprint_pr_lookup_failed",
|
||
report_id=report.report_id,
|
||
error=str(exc),
|
||
)
|
||
return {
|
||
"lookup_status": "failed",
|
||
"lookup_error": str(exc)[:160],
|
||
}
|
||
|
||
return None
|
||
|
||
|
||
def _default_pr_url(open_pr: dict[str, Any] | None) -> str | None:
|
||
if not open_pr:
|
||
return None
|
||
return open_pr.get("html_url") or open_pr.get("url")
|
||
|
||
|
||
def _repeat_state_fingerprint_aliases(repeat_state: dict[str, Any]) -> set[str]:
|
||
aliases = {str(repeat_state.get("strict_fingerprint") or "")}
|
||
for report in repeat_state.get("reports") or []:
|
||
if isinstance(report, dict):
|
||
aliases.add(str(report.get("strict_fingerprint") or ""))
|
||
return {value for value in aliases if value}
|
||
|
||
|
||
def _repeat_state_report_ids(repeat_state: dict[str, Any]) -> set[str]:
|
||
report_ids = set()
|
||
for report in repeat_state.get("reports") or []:
|
||
if isinstance(report, dict):
|
||
report_ids.add(str(report.get("report_id") or ""))
|
||
return {value for value in report_ids if value}
|
||
|
||
|
||
def _matches_drift_pr(report: DriftReport, pr: dict[str, Any]) -> bool:
|
||
text_blob = "\n".join(
|
||
str(value or "")
|
||
for value in (pr.get("title"), pr.get("body"), pr.get("html_url"))
|
||
)
|
||
if report.report_id and report.report_id in text_blob:
|
||
return True
|
||
if report.namespace not in text_blob:
|
||
return False
|
||
expected_parts = [
|
||
f"HIGH×{report.high_count}",
|
||
f"MEDIUM×{report.medium_count}",
|
||
f"INFO×{report.info_count}",
|
||
]
|
||
return any(part in text_blob for part in expected_parts)
|
||
|
||
|
||
def _nested(mapping: dict[str, Any], *keys: str) -> Any:
|
||
current: Any = mapping
|
||
for key in keys:
|
||
if not isinstance(current, dict):
|
||
return None
|
||
current = current.get(key)
|
||
return current
|
||
|
||
|
||
async def _response_list_count(
|
||
client: httpx.AsyncClient,
|
||
url: str,
|
||
*,
|
||
headers: dict[str, str],
|
||
) -> tuple[int | None, str | None]:
|
||
try:
|
||
response = await client.get(url, headers=headers)
|
||
if response.status_code != 200:
|
||
return None, f"http_{response.status_code}"
|
||
payload = response.json()
|
||
if isinstance(payload, list):
|
||
return len(payload), None
|
||
if isinstance(payload, dict):
|
||
for key in ("items", "files", "commits"):
|
||
if isinstance(payload.get(key), list):
|
||
return len(payload[key]), None
|
||
return 0, None
|
||
except Exception as exc:
|
||
return None, str(exc)[:160]
|
||
|
||
|
||
async def _build_pr_state(
|
||
*,
|
||
client: httpx.AsyncClient,
|
||
api_url: str,
|
||
owner: str,
|
||
repo: str,
|
||
headers: dict[str, str],
|
||
pr: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
number = pr.get("number") or pr.get("index")
|
||
base_url = f"{api_url}/api/v1/repos/{owner}/{repo}/pulls/{number}"
|
||
files_count, files_error = await _response_list_count(
|
||
client,
|
||
f"{base_url}/files",
|
||
headers=headers,
|
||
)
|
||
commits_count, commits_error = await _response_list_count(
|
||
client,
|
||
f"{base_url}/commits",
|
||
headers=headers,
|
||
)
|
||
head_sha = _nested(pr, "head", "sha") or _nested(pr, "head", "ref")
|
||
base_sha = _nested(pr, "base", "sha") or _nested(pr, "base", "ref")
|
||
is_zero_diff = (
|
||
(files_count == 0 and commits_count == 0)
|
||
or (bool(head_sha) and head_sha == base_sha)
|
||
)
|
||
return {
|
||
"lookup_status": "ok",
|
||
"number": number,
|
||
"title": pr.get("title"),
|
||
"state": pr.get("state"),
|
||
"merged": bool(pr.get("merged")),
|
||
"mergeable": pr.get("mergeable"),
|
||
"html_url": pr.get("html_url"),
|
||
"url": pr.get("url"),
|
||
"head_ref": _nested(pr, "head", "ref"),
|
||
"base_ref": _nested(pr, "base", "ref"),
|
||
"head_sha": head_sha,
|
||
"base_sha": base_sha,
|
||
"file_count": files_count,
|
||
"commit_count": commits_count,
|
||
"files_lookup_error": files_error,
|
||
"commits_lookup_error": commits_error,
|
||
"is_zero_diff": is_zero_diff,
|
||
"refs": _pr_refs(pr),
|
||
}
|
||
|
||
|
||
_drift_fingerprint_state_service: DriftFingerprintStateService | None = None
|
||
|
||
|
||
def get_drift_fingerprint_state_service() -> DriftFingerprintStateService:
|
||
global _drift_fingerprint_state_service
|
||
if _drift_fingerprint_state_service is None:
|
||
_drift_fingerprint_state_service = DriftFingerprintStateService()
|
||
return _drift_fingerprint_state_service
|