fix(automation): expose durable history degradation
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m40s
CD Pipeline / build-and-deploy (push) Successful in 6m59s
CD Pipeline / post-deploy-checks (push) Successful in 1m51s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m40s
CD Pipeline / build-and-deploy (push) Successful in 6m59s
CD Pipeline / post-deploy-checks (push) Successful in 1m51s
This commit is contained in:
@@ -17,11 +17,13 @@ from fastapi import APIRouter, HTTPException, Query
|
|||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from src.core.csrf import CSRFToken # Phase 20: CSRF Protection
|
from src.core.csrf import CSRFToken # Phase 20: CSRF Protection
|
||||||
|
|
||||||
from src.services.auto_repair_service import (
|
from src.services.auto_repair_service import (
|
||||||
get_auto_repair_service,
|
get_auto_repair_service,
|
||||||
)
|
)
|
||||||
from src.services.incident_service import get_incident_service
|
from src.services.incident_service import (
|
||||||
|
IncidentDurableReadError,
|
||||||
|
get_incident_service,
|
||||||
|
)
|
||||||
|
|
||||||
router = APIRouter(prefix="/auto-repair", tags=["Auto Repair"])
|
router = APIRouter(prefix="/auto-repair", tags=["Auto Repair"])
|
||||||
|
|
||||||
@@ -251,6 +253,12 @@ class RepairHistoryResponse(BaseModel):
|
|||||||
@router.get("/history", response_model=RepairHistoryResponse)
|
@router.get("/history", response_model=RepairHistoryResponse)
|
||||||
async def get_repair_history(
|
async def get_repair_history(
|
||||||
limit: int = Query(20, ge=1, le=100),
|
limit: int = Query(20, ge=1, le=100),
|
||||||
|
project_id: str = Query(
|
||||||
|
"awoooi",
|
||||||
|
min_length=1,
|
||||||
|
max_length=64,
|
||||||
|
description="租戶 ID;legacy repair history 預設綁定 awoooi。",
|
||||||
|
),
|
||||||
) -> RepairHistoryResponse:
|
) -> RepairHistoryResponse:
|
||||||
"""
|
"""
|
||||||
取得修復歷史記錄。
|
取得修復歷史記錄。
|
||||||
@@ -259,7 +267,9 @@ async def get_repair_history(
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
incident_service = get_incident_service()
|
incident_service = get_incident_service()
|
||||||
all_incidents = await incident_service.get_active_incidents()
|
all_incidents = await incident_service.get_active_incidents(
|
||||||
|
project_id=project_id,
|
||||||
|
)
|
||||||
|
|
||||||
items: list[RepairHistoryItem] = []
|
items: list[RepairHistoryItem] = []
|
||||||
for incident in all_incidents:
|
for incident in all_incidents:
|
||||||
@@ -307,6 +317,23 @@ async def get_repair_history(
|
|||||||
|
|
||||||
return RepairHistoryResponse(count=len(items), items=items)
|
return RepairHistoryResponse(count=len(items), items=items)
|
||||||
|
|
||||||
except Exception:
|
except IncidentDurableReadError as exc:
|
||||||
# 任何錯誤都回傳空列表,不中斷前端
|
raise HTTPException(
|
||||||
return RepairHistoryResponse(count=0, items=[])
|
status_code=503,
|
||||||
|
detail={
|
||||||
|
"status": "degraded",
|
||||||
|
"reason_code": str(exc),
|
||||||
|
"source_of_truth": "postgresql",
|
||||||
|
"redis_fallback_used": False,
|
||||||
|
},
|
||||||
|
) from exc
|
||||||
|
except Exception as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail={
|
||||||
|
"status": "degraded",
|
||||||
|
"reason_code": "repair_history_readback_failed",
|
||||||
|
"source_of_truth": "postgresql",
|
||||||
|
"redis_fallback_used": False,
|
||||||
|
},
|
||||||
|
) from exc
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from unittest.mock import AsyncMock
|
|||||||
import pytest
|
import pytest
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from src.api.v1 import auto_repair as auto_repair_api
|
||||||
from src.api.v1 import incidents as incidents_api
|
from src.api.v1 import incidents as incidents_api
|
||||||
from src.models.incident import Incident, IncidentStatus, Severity
|
from src.models.incident import Incident, IncidentStatus, Severity
|
||||||
from src.repositories import incident_repository as incident_repository_module
|
from src.repositories import incident_repository as incident_repository_module
|
||||||
@@ -247,3 +248,52 @@ async def test_incident_detail_reports_durable_readback_degraded(monkeypatch):
|
|||||||
"source_of_truth": "postgresql",
|
"source_of_truth": "postgresql",
|
||||||
"redis_fallback_used": False,
|
"redis_fallback_used": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_repair_history_binds_project_to_durable_incident_read(monkeypatch):
|
||||||
|
durable_active = _incident(IncidentStatus.INVESTIGATING, minute=38)
|
||||||
|
|
||||||
|
class _Service:
|
||||||
|
get_active_incidents = AsyncMock(return_value=[durable_active])
|
||||||
|
|
||||||
|
service = _Service()
|
||||||
|
monkeypatch.setattr(auto_repair_api, "get_incident_service", lambda: service)
|
||||||
|
|
||||||
|
result = await auto_repair_api.get_repair_history(
|
||||||
|
limit=20,
|
||||||
|
project_id="awoooi",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.count == 0
|
||||||
|
service.get_active_incidents.assert_awaited_once_with(project_id="awoooi")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_repair_history_reports_durable_read_degraded(monkeypatch):
|
||||||
|
class _Service:
|
||||||
|
get_active_incidents = AsyncMock(
|
||||||
|
side_effect=IncidentDurableReadError(
|
||||||
|
"active_incidents_durable_read_unavailable"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
auto_repair_api,
|
||||||
|
"get_incident_service",
|
||||||
|
lambda: _Service(),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
await auto_repair_api.get_repair_history(
|
||||||
|
limit=20,
|
||||||
|
project_id="awoooi",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.status_code == 503
|
||||||
|
assert exc_info.value.detail == {
|
||||||
|
"status": "degraded",
|
||||||
|
"reason_code": "active_incidents_durable_read_unavailable",
|
||||||
|
"source_of_truth": "postgresql",
|
||||||
|
"redis_fallback_used": False,
|
||||||
|
}
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ export function NeuralCommandPanel() {
|
|||||||
const [statsRes, pbRes, histRes, approvalsRes, incidentsRes, dispRes] = await Promise.all([
|
const [statsRes, pbRes, histRes, approvalsRes, incidentsRes, dispRes] = await Promise.all([
|
||||||
fetch('/api/v1/auto-repair/stats'),
|
fetch('/api/v1/auto-repair/stats'),
|
||||||
fetch('/api/v1/playbooks/'),
|
fetch('/api/v1/playbooks/'),
|
||||||
fetch('/api/v1/auto-repair/history?limit=20'),
|
fetch('/api/v1/auto-repair/history?limit=20&project_id=awoooi'),
|
||||||
fetch('/api/v1/approvals/pending'),
|
fetch('/api/v1/approvals/pending'),
|
||||||
fetch('/api/v1/incidents?status=firing&limit=10'),
|
fetch('/api/v1/incidents?status=firing&limit=10'),
|
||||||
fetch('/api/v1/stats/disposition').catch(() => null),
|
fetch('/api/v1/stats/disposition').catch(() => null),
|
||||||
|
|||||||
Reference in New Issue
Block a user