fix(automation): enforce durable alert closure
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s

Restore D037 durable incident readback without weakening database failure semantics.

Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints.

Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls.

Keep SSH service refusal visible while separating it from broker network-policy reachability.
This commit is contained in:
ogt
2026-07-14 09:40:53 +08:00
parent 72a167a322
commit 6cf8429d17
56 changed files with 15100 additions and 227 deletions

View File

@@ -11,10 +11,12 @@ Phase 16 R3.3: Repository 層實作
建立者: Claude Code (Phase 16 架構重構)
"""
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
import structlog
from pydantic import ValidationError
from sqlalchemy import select
from src.db.base import get_db_context
@@ -22,6 +24,7 @@ from src.db.models import IncidentRecord
from src.models.incident import (
Incident,
IncidentFrequencyStats,
IncidentReadDegradation,
IncidentStatus,
Severity,
)
@@ -29,6 +32,37 @@ from src.repositories.interfaces import IIncidentRepository
logger = structlog.get_logger(__name__)
_ACTIVE_INCIDENT_STATUS_VALUES = frozenset({
IncidentStatus.INVESTIGATING.value,
IncidentStatus.MITIGATING.value,
})
@dataclass
class ActiveIncidentReadResult:
"""Valid active incidents plus public-safe per-row quarantine metadata."""
incidents: list[Incident] = field(default_factory=list)
degraded_records: list[IncidentReadDegradation] = field(default_factory=list)
@property
def degraded(self) -> bool:
return bool(self.degraded_records)
def _record_validation_reason(exc: ValidationError) -> str:
errors = exc.errors(include_input=False)
source_errors = [
error
for error in errors
if tuple(error.get("loc") or ())[-1:] == ("source",)
]
if source_errors and all(error.get("type") != "missing" for error in source_errors):
return "incident_signal_source_unsupported"
if any(tuple(error.get("loc") or ())[:1] == ("signals",) for error in errors):
return "incident_signal_schema_invalid"
return "incident_record_schema_invalid"
# =============================================================================
# Conversion Helpers
@@ -168,8 +202,13 @@ class IncidentDBRepository(IIncidentRepository):
record = result.scalar_one_or_none()
return _record_to_incident(record) if record else None
async def get_active(self, *, project_id: str | None = None) -> list[Incident]:
"""取得所有活躍的 Incident"""
async def get_active_readback(
self,
*,
project_id: str | None = None,
) -> ActiveIncidentReadResult:
"""Read valid active rows and quarantine only schema-invalid rows."""
active_statuses = [
IncidentStatus.INVESTIGATING,
IncidentStatus.MITIGATING,
@@ -181,7 +220,48 @@ class IncidentDBRepository(IIncidentRepository):
.order_by(IncidentRecord.created_at.desc())
)
records = result.scalars().all()
return [_record_to_incident(r) for r in records]
# SQL is authoritative, while the defensive filter prevents a
# terminal row from leaking into the active surface if an enum or
# test adapter returns a wider result set. In particular, verified
# terminal incident INC-20260711-D037E5 must never reappear here.
active_records = [
record
for record in records
if _normalize_status(record.status)
in _ACTIVE_INCIDENT_STATUS_VALUES
]
incidents: list[Incident] = []
degraded_records: list[IncidentReadDegradation] = []
for record in active_records:
try:
incidents.append(_record_to_incident(record))
except ValidationError as exc:
reason_code = _record_validation_reason(exc)
logger.warning(
"active_incident_row_quarantined",
incident_id=record.incident_id,
project_id=project_id,
reason_code=reason_code,
validation_error_count=len(exc.errors()),
redis_fallback_used=False,
)
degraded_records.append(
IncidentReadDegradation(
incident_id=record.incident_id,
reason_code=reason_code,
)
)
return ActiveIncidentReadResult(
incidents=incidents,
degraded_records=degraded_records,
)
async def get_active(self, *, project_id: str | None = None) -> list[Incident]:
"""取得所有可安全讀取的活躍 Incident。"""
return (
await self.get_active_readback(project_id=project_id)
).incidents
async def update(self, incident: Incident) -> Incident | None:
"""更新 Incident"""