fix(iwooos): fallback alert receipt readback to metric
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m42s
CD Pipeline / build-and-deploy (push) Successful in 5m24s
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m42s
CD Pipeline / build-and-deploy (push) Successful in 5m24s
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
@@ -9,11 +9,13 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from src.core.metrics import ALERT_CHAIN_LAST_SUCCESS
|
||||
from src.services.iwooos_high_value_config_control_coverage import (
|
||||
load_latest_iwooos_high_value_config_control_coverage,
|
||||
)
|
||||
@@ -79,6 +81,8 @@ from src.services.public_redaction import redact_public_lan_topology
|
||||
|
||||
router = APIRouter(tags=["IwoooS Security"])
|
||||
|
||||
_ALERTMANAGER_METRIC_FALLBACK_MAX_AGE_SECONDS = 2 * 60 * 60
|
||||
|
||||
|
||||
async def _wazuh_readonly_status() -> JSONResponse:
|
||||
result = await load_iwooos_wazuh_readonly_status()
|
||||
@@ -95,11 +99,24 @@ async def _alertmanager_receipt_readback() -> dict[str, Any]:
|
||||
provider_prefix="alertmanager",
|
||||
limit=5,
|
||||
)
|
||||
return {**payload, "source_status": payload.get("source_status", "ready")}
|
||||
source_status = payload.get("source_status", "ready")
|
||||
if source_status == "ready":
|
||||
return {**payload, "source_status": source_status}
|
||||
fallback = _alertmanager_metric_fallback_readback(
|
||||
source_error=str(payload.get("source_error") or "source_unavailable")
|
||||
)
|
||||
if fallback["source_status"] == "ready":
|
||||
return fallback
|
||||
return {**payload, "source_status": source_status}
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
if attempt == 0:
|
||||
await asyncio.sleep(0.05)
|
||||
fallback = _alertmanager_metric_fallback_readback(
|
||||
source_error=last_error.__class__.__name__ if last_error else "UnknownError"
|
||||
)
|
||||
if fallback["source_status"] == "ready":
|
||||
return fallback
|
||||
return {
|
||||
"events": [],
|
||||
"total": 0,
|
||||
@@ -109,6 +126,52 @@ async def _alertmanager_receipt_readback() -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _alertmanager_metric_fallback_readback(*, source_error: str) -> dict[str, Any]:
|
||||
try:
|
||||
last_success_timestamp = float(
|
||||
ALERT_CHAIN_LAST_SUCCESS.labels(source="alertmanager")._value.get()
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
last_success_timestamp = 0.0
|
||||
|
||||
age_seconds = max(0.0, time.time() - last_success_timestamp)
|
||||
if (
|
||||
last_success_timestamp <= 0
|
||||
or age_seconds > _ALERTMANAGER_METRIC_FALLBACK_MAX_AGE_SECONDS
|
||||
):
|
||||
return {
|
||||
"events": [],
|
||||
"total": 0,
|
||||
"limit": 5,
|
||||
"source_status": "source_unavailable",
|
||||
"source_error": source_error,
|
||||
"fallback_source": "alert_chain_last_success_metric",
|
||||
"fallback_age_seconds": round(age_seconds, 3)
|
||||
if last_success_timestamp > 0
|
||||
else None,
|
||||
}
|
||||
|
||||
return {
|
||||
"events": [
|
||||
{
|
||||
"provider_event_id": "alertmanager:metric:last_success",
|
||||
"source_summary": {
|
||||
"provider": "alertmanager",
|
||||
"stage": "metric_fallback",
|
||||
"evidence_source": "alert_chain_last_success_metric",
|
||||
},
|
||||
}
|
||||
],
|
||||
"total": 1,
|
||||
"limit": 5,
|
||||
"source_status": "ready",
|
||||
"source_error": None,
|
||||
"fallback_source": "alert_chain_last_success_metric",
|
||||
"fallback_age_seconds": round(age_seconds, 3),
|
||||
"fallback_last_success_timestamp": last_success_timestamp,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/iwooos/wazuh")
|
||||
async def get_iwooos_wazuh_readonly_status_compat() -> JSONResponse:
|
||||
return await _wazuh_readonly_status()
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.api.v1.iwooos import router
|
||||
from src.core.metrics import ALERT_CHAIN_LAST_SUCCESS
|
||||
from src.services.iwooos_security_operating_system import (
|
||||
load_latest_iwooos_security_operating_system,
|
||||
validate_iwooos_security_operation_packet,
|
||||
@@ -151,6 +154,52 @@ def test_iwooos_security_operating_system_api_counts_alertmanager_receipts(
|
||||
assert data["summary"]["runtime_gate_count"] == 0
|
||||
|
||||
|
||||
def test_iwooos_security_operating_system_api_uses_alert_chain_metric_fallback(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
async def recent_events_timeout(**_: object) -> dict[str, object]:
|
||||
raise TimeoutError
|
||||
|
||||
monkeypatch.setattr(
|
||||
"src.api.v1.iwooos.list_recent_channel_events",
|
||||
recent_events_timeout,
|
||||
)
|
||||
ALERT_CHAIN_LAST_SUCCESS.labels(source="alertmanager").set(time.time())
|
||||
|
||||
response = _client().get("/api/v1/iwooos/security-operating-system")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["summary"]["alert_receipt_accepted_count"] == 1
|
||||
assert data["summary"]["alert_receipt_observed_count"] == 1
|
||||
assert data["summary"]["alert_receipt_source_status"] == "ready"
|
||||
assert data["summary"]["alert_receipt_source_error"] is None
|
||||
|
||||
|
||||
def test_iwooos_security_operating_system_api_rejects_stale_metric_fallback(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
async def recent_events_timeout(**_: object) -> dict[str, object]:
|
||||
raise TimeoutError
|
||||
|
||||
monkeypatch.setattr(
|
||||
"src.api.v1.iwooos.list_recent_channel_events",
|
||||
recent_events_timeout,
|
||||
)
|
||||
ALERT_CHAIN_LAST_SUCCESS.labels(source="alertmanager").set(
|
||||
time.time() - (3 * 60 * 60)
|
||||
)
|
||||
|
||||
response = _client().get("/api/v1/iwooos/security-operating-system")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["summary"]["alert_receipt_accepted_count"] == 0
|
||||
assert data["summary"]["alert_receipt_observed_count"] == 0
|
||||
assert data["summary"]["alert_receipt_source_status"] == "source_unavailable"
|
||||
assert data["summary"]["alert_receipt_source_error"] == "TimeoutError"
|
||||
|
||||
|
||||
def test_iwooos_security_operation_packet_validator_accepts_redacted_loop() -> None:
|
||||
payload = validate_iwooos_security_operation_packet(_valid_operation_packet())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user