fix(reboot): overlay live stockplatform freshness in slo scorecard
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 59s
CD Pipeline / build-and-deploy (push) Successful in 4m37s
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 59s
CD Pipeline / build-and-deploy (push) Successful in 4m37s
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
@@ -448,6 +448,7 @@ from src.services.reboot_auto_recovery_drill_preflight import (
|
||||
load_latest_reboot_auto_recovery_drill_preflight,
|
||||
)
|
||||
from src.services.reboot_auto_recovery_slo_scorecard import (
|
||||
apply_stockplatform_runtime_readback,
|
||||
load_latest_reboot_auto_recovery_slo_scorecard,
|
||||
)
|
||||
from src.services.runtime_surface_inventory import (
|
||||
@@ -1482,6 +1483,10 @@ async def get_reboot_auto_recovery_slo_scorecard() -> dict[str, Any]:
|
||||
payload = await asyncio.to_thread(
|
||||
load_latest_reboot_auto_recovery_slo_scorecard
|
||||
)
|
||||
stockplatform_runtime = await asyncio.to_thread(
|
||||
load_latest_stockplatform_public_api_runtime_readback
|
||||
)
|
||||
apply_stockplatform_runtime_readback(payload, stockplatform_runtime)
|
||||
return redact_public_lan_topology(payload)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
|
||||
@@ -40,6 +40,135 @@ def load_latest_reboot_auto_recovery_slo_scorecard(
|
||||
return payload
|
||||
|
||||
|
||||
def apply_stockplatform_runtime_readback(
|
||||
payload: dict[str, Any],
|
||||
runtime_readback: dict[str, Any],
|
||||
) -> None:
|
||||
"""Overlay live StockPlatform public API data truth onto the scorecard."""
|
||||
readback = _dict(runtime_readback.get("readback"))
|
||||
freshness_status = str(readback.get("freshness_status") or "unknown")
|
||||
ingestion_status = str(readback.get("ingestion_status") or "unknown")
|
||||
freshness_blockers = _strings(readback.get("freshness_blockers"))
|
||||
ingestion_blockers = _strings(readback.get("ingestion_blockers"))
|
||||
latest_trading_date = str(readback.get("freshness_latest_trading_date") or "")
|
||||
freshness_ok = freshness_status == "ok"
|
||||
ingestion_ok = ingestion_status == "ok"
|
||||
stockplatform = _dict(payload.setdefault("stockplatform_data_freshness", {}))
|
||||
stockplatform.update(
|
||||
{
|
||||
"freshness_status": freshness_status,
|
||||
"ingestion_status": ingestion_status,
|
||||
"latest_trading_date": latest_trading_date,
|
||||
"freshness_blockers": freshness_blockers,
|
||||
"ingestion_blockers": ingestion_blockers,
|
||||
"freshness_sla_source_count": _int(
|
||||
readback.get("freshness_sla_source_count")
|
||||
),
|
||||
"freshness_source_count": _int(readback.get("freshness_source_count")),
|
||||
"freshness_non_ok_source_count": _int(
|
||||
readback.get("freshness_non_ok_source_count")
|
||||
),
|
||||
"live_runtime_overlay_applied": True,
|
||||
}
|
||||
)
|
||||
|
||||
required_checks = _dict(payload.setdefault("required_checks", {}))
|
||||
required_checks["stockplatform_freshness_ok"] = freshness_ok
|
||||
required_checks["stockplatform_ingestion_ok"] = ingestion_ok
|
||||
if not (freshness_ok and ingestion_ok):
|
||||
required_checks["product_data_green"] = False
|
||||
payload["product_data_green"] = False
|
||||
post_reboot = _dict(payload.setdefault("post_reboot_readiness", {}))
|
||||
post_reboot["product_data_green"] = False
|
||||
_append_live_stockplatform_blockers(
|
||||
payload=payload,
|
||||
freshness_status=freshness_status,
|
||||
ingestion_status=ingestion_status,
|
||||
freshness_blockers=freshness_blockers,
|
||||
ingestion_blockers=ingestion_blockers,
|
||||
)
|
||||
|
||||
completed_check_count = sum(1 for value in required_checks.values() if value)
|
||||
readiness_percent = _percent(
|
||||
completed_check_count / max(len(required_checks), 1) * 100
|
||||
)
|
||||
active_blockers = _strings(payload.get("active_blockers"))
|
||||
active_blocker_count = len(active_blockers)
|
||||
payload["active_blocker_count"] = active_blocker_count
|
||||
payload["readiness_percent"] = readiness_percent
|
||||
payload["stockplatform_freshness_status"] = freshness_status
|
||||
payload["stockplatform_ingestion_status"] = ingestion_status
|
||||
|
||||
readback_section = _dict(payload.setdefault("readback", {}))
|
||||
readback_section["active_blocker_count"] = active_blocker_count
|
||||
readback_section["readiness_percent"] = readiness_percent
|
||||
|
||||
rollups = _dict(payload.setdefault("rollups", {}))
|
||||
rollups["active_blocker_count"] = active_blocker_count
|
||||
rollups["completed_check_count"] = completed_check_count
|
||||
rollups["readiness_percent"] = readiness_percent
|
||||
rollups["product_data_green"] = payload.get("product_data_green") is True
|
||||
rollups["stockplatform_freshness_status"] = freshness_status
|
||||
rollups["stockplatform_ingestion_status"] = ingestion_status
|
||||
rollups["stockplatform_freshness_blocker_count"] = len(freshness_blockers)
|
||||
rollups["stockplatform_ingestion_blocker_count"] = len(ingestion_blockers)
|
||||
rollups["stockplatform_freshness_sla_source_count"] = _int(
|
||||
readback.get("freshness_sla_source_count")
|
||||
)
|
||||
rollups["stockplatform_freshness_source_count"] = _int(
|
||||
readback.get("freshness_source_count")
|
||||
)
|
||||
rollups["stockplatform_freshness_non_ok_source_count"] = _int(
|
||||
readback.get("freshness_non_ok_source_count")
|
||||
)
|
||||
|
||||
service_backup = _dict(
|
||||
payload.setdefault("controlled_service_data_backup_readback", {})
|
||||
)
|
||||
service_backup["product_data_green"] = payload.get("product_data_green") is True
|
||||
service_backup["stockplatform_freshness_status"] = freshness_status
|
||||
service_backup["stockplatform_ingestion_status"] = ingestion_status
|
||||
blocking_fields = _strings(service_backup.get("blocking_fields"))
|
||||
if not freshness_ok:
|
||||
blocking_fields.append("stockplatform_freshness_status")
|
||||
if not ingestion_ok:
|
||||
blocking_fields.append("stockplatform_ingestion_status")
|
||||
if payload.get("product_data_green") is not True:
|
||||
blocking_fields.append("product_data_green")
|
||||
service_backup["blocking_fields"] = _unique_strings(blocking_fields)
|
||||
service_backup["controlled_service_data_backup_blocker_count"] = len(
|
||||
service_backup["blocking_fields"]
|
||||
)
|
||||
rollups["controlled_service_data_backup_blocker_count"] = len(
|
||||
service_backup["blocking_fields"]
|
||||
)
|
||||
service_backup["status"] = "blocked_service_data_backup_readback_not_green"
|
||||
service_backup["can_clear_service_data_backup_blockers"] = False
|
||||
|
||||
|
||||
def _append_live_stockplatform_blockers(
|
||||
*,
|
||||
payload: dict[str, Any],
|
||||
freshness_status: str,
|
||||
ingestion_status: str,
|
||||
freshness_blockers: list[str],
|
||||
ingestion_blockers: list[str],
|
||||
) -> None:
|
||||
active_blockers = _strings(payload.get("active_blockers"))
|
||||
active_blockers.append("product_data_green_not_1")
|
||||
if freshness_status != "ok" and not freshness_blockers:
|
||||
active_blockers.append("stockplatform_freshness_status_not_ok")
|
||||
if ingestion_status != "ok" and not ingestion_blockers:
|
||||
active_blockers.append("stockplatform_ingestion_status_not_ok")
|
||||
active_blockers.extend(
|
||||
f"stockplatform_freshness_{blocker}" for blocker in freshness_blockers
|
||||
)
|
||||
active_blockers.extend(
|
||||
f"stockplatform_ingestion_{blocker}" for blocker in ingestion_blockers
|
||||
)
|
||||
payload["active_blockers"] = _unique_strings(active_blockers)
|
||||
|
||||
|
||||
def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
host_boot_detection = _dict(scorecard.get("host_boot_detection"))
|
||||
post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness"))
|
||||
@@ -764,5 +893,16 @@ def _strings(value: Any) -> list[str]:
|
||||
return [str(item) for item in value if item is not None]
|
||||
|
||||
|
||||
def _unique_strings(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
for value in values:
|
||||
if value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
unique.append(value)
|
||||
return unique
|
||||
|
||||
|
||||
def _taipei_now_iso() -> str:
|
||||
return datetime.now(ZoneInfo("Asia/Taipei")).isoformat(timespec="seconds")
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.api.v1 import agents
|
||||
from src.api.v1.agents import router
|
||||
from src.services.reboot_auto_recovery_drill_preflight import (
|
||||
load_latest_reboot_auto_recovery_drill_preflight,
|
||||
@@ -45,7 +46,12 @@ def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate():
|
||||
_assert_reboot_slo_payload(payload)
|
||||
|
||||
|
||||
def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback():
|
||||
def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
agents,
|
||||
"load_latest_stockplatform_public_api_runtime_readback",
|
||||
_stockplatform_runtime_ready,
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
client = TestClient(app)
|
||||
@@ -56,6 +62,59 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback():
|
||||
_assert_reboot_slo_payload(response.json())
|
||||
|
||||
|
||||
def test_reboot_auto_recovery_slo_scorecard_endpoint_overlays_live_stockplatform_blocked(
|
||||
monkeypatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
agents,
|
||||
"load_latest_stockplatform_public_api_runtime_readback",
|
||||
_stockplatform_runtime_blocked,
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/api/v1/agents/reboot-auto-recovery-slo-scorecard")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["stockplatform_freshness_status"] == "blocked"
|
||||
assert payload["stockplatform_ingestion_status"] == "blocked"
|
||||
assert payload["stockplatform_data_freshness"]["latest_trading_date"] == (
|
||||
"2026-07-02"
|
||||
)
|
||||
assert payload["stockplatform_data_freshness"]["freshness_sla_source_count"] == 9
|
||||
assert payload["stockplatform_data_freshness"]["freshness_blockers"] == [
|
||||
"core_margin_short_daily_missing",
|
||||
"ai_recommendations_stale",
|
||||
]
|
||||
assert payload["stockplatform_data_freshness"]["ingestion_blockers"] == [
|
||||
"core.margin_short_daily_incomplete"
|
||||
]
|
||||
assert payload["product_data_green"] is False
|
||||
assert payload["required_checks"]["stockplatform_freshness_ok"] is False
|
||||
assert payload["required_checks"]["stockplatform_ingestion_ok"] is False
|
||||
assert payload["required_checks"]["product_data_green"] is False
|
||||
assert payload["readiness_percent"] == 21
|
||||
assert payload["active_blocker_count"] == 15
|
||||
assert "product_data_green_not_1" in payload["active_blockers"]
|
||||
assert "stockplatform_freshness_core_margin_short_daily_missing" in payload[
|
||||
"active_blockers"
|
||||
]
|
||||
assert "stockplatform_ingestion_core.margin_short_daily_incomplete" in payload[
|
||||
"active_blockers"
|
||||
]
|
||||
assert payload["rollups"]["stockplatform_freshness_status"] == "blocked"
|
||||
assert payload["rollups"]["stockplatform_freshness_sla_source_count"] == 9
|
||||
assert payload["controlled_service_data_backup_readback"][
|
||||
"product_data_green"
|
||||
] is False
|
||||
assert "stockplatform_freshness_status" in payload[
|
||||
"controlled_service_data_backup_readback"
|
||||
]["blocking_fields"]
|
||||
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 7
|
||||
|
||||
|
||||
def test_reboot_auto_recovery_drill_preflight_loader_returns_break_glass_package():
|
||||
payload = load_latest_reboot_auto_recovery_drill_preflight()
|
||||
|
||||
@@ -439,3 +498,50 @@ def _assert_drill_preflight_payload(payload: dict):
|
||||
assert boundaries["github_api_used"] is False
|
||||
assert boundaries["runtime_write_allowed"] is False
|
||||
assert "host_reboot" in payload["forbidden_without_separate_break_glass"]
|
||||
|
||||
|
||||
def _stockplatform_runtime_ready() -> dict:
|
||||
return {
|
||||
"schema_version": "stockplatform_public_api_runtime_readback_v1",
|
||||
"status": "stockplatform_public_api_runtime_ready",
|
||||
"runtime_ready": True,
|
||||
"active_blockers": [],
|
||||
"readback": {
|
||||
"freshness_status": "ok",
|
||||
"ingestion_status": "ok",
|
||||
"freshness_latest_trading_date": "2026-07-02",
|
||||
"ingestion_latest_trading_date": "2026-07-02",
|
||||
"freshness_blockers": [],
|
||||
"ingestion_blockers": [],
|
||||
"freshness_sla_source_count": 9,
|
||||
"freshness_source_count": 9,
|
||||
"freshness_non_ok_source_count": 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _stockplatform_runtime_blocked() -> dict:
|
||||
return {
|
||||
"schema_version": "stockplatform_public_api_runtime_readback_v1",
|
||||
"status": "blocked_stockplatform_public_api_runtime_drift",
|
||||
"runtime_ready": False,
|
||||
"active_blockers": [
|
||||
"stockplatform_freshness_core_margin_short_daily_missing",
|
||||
"stockplatform_freshness_ai_recommendations_stale",
|
||||
"stockplatform_ingestion_core.margin_short_daily_incomplete",
|
||||
],
|
||||
"readback": {
|
||||
"freshness_status": "blocked",
|
||||
"ingestion_status": "blocked",
|
||||
"freshness_latest_trading_date": "2026-07-02",
|
||||
"ingestion_latest_trading_date": "2026-07-02",
|
||||
"freshness_blockers": [
|
||||
"core_margin_short_daily_missing",
|
||||
"ai_recommendations_stale",
|
||||
],
|
||||
"ingestion_blockers": ["core.margin_short_daily_incomplete"],
|
||||
"freshness_sla_source_count": 9,
|
||||
"freshness_source_count": 9,
|
||||
"freshness_non_ok_source_count": 3,
|
||||
},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user