fix(reboot): overlay live stockplatform freshness in slo scorecard
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 59s
CD Pipeline / build-and-deploy (push) Successful in 4m37s
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-02 19:25:19 +08:00
parent 7aa196ba5a
commit dc32550f99
3 changed files with 252 additions and 1 deletions

View File

@@ -448,6 +448,7 @@ from src.services.reboot_auto_recovery_drill_preflight import (
load_latest_reboot_auto_recovery_drill_preflight, load_latest_reboot_auto_recovery_drill_preflight,
) )
from src.services.reboot_auto_recovery_slo_scorecard import ( from src.services.reboot_auto_recovery_slo_scorecard import (
apply_stockplatform_runtime_readback,
load_latest_reboot_auto_recovery_slo_scorecard, load_latest_reboot_auto_recovery_slo_scorecard,
) )
from src.services.runtime_surface_inventory import ( from src.services.runtime_surface_inventory import (
@@ -1482,6 +1483,10 @@ async def get_reboot_auto_recovery_slo_scorecard() -> dict[str, Any]:
payload = await asyncio.to_thread( payload = await asyncio.to_thread(
load_latest_reboot_auto_recovery_slo_scorecard load_latest_reboot_auto_recovery_slo_scorecard
) )
stockplatform_runtime = await asyncio.to_thread(
load_latest_stockplatform_public_api_runtime_readback
)
apply_stockplatform_runtime_readback(payload, stockplatform_runtime)
return redact_public_lan_topology(payload) return redact_public_lan_topology(payload)
except FileNotFoundError as exc: except FileNotFoundError as exc:
raise HTTPException( raise HTTPException(

View File

@@ -40,6 +40,135 @@ def load_latest_reboot_auto_recovery_slo_scorecard(
return payload return payload
def apply_stockplatform_runtime_readback(
payload: dict[str, Any],
runtime_readback: dict[str, Any],
) -> None:
"""Overlay live StockPlatform public API data truth onto the scorecard."""
readback = _dict(runtime_readback.get("readback"))
freshness_status = str(readback.get("freshness_status") or "unknown")
ingestion_status = str(readback.get("ingestion_status") or "unknown")
freshness_blockers = _strings(readback.get("freshness_blockers"))
ingestion_blockers = _strings(readback.get("ingestion_blockers"))
latest_trading_date = str(readback.get("freshness_latest_trading_date") or "")
freshness_ok = freshness_status == "ok"
ingestion_ok = ingestion_status == "ok"
stockplatform = _dict(payload.setdefault("stockplatform_data_freshness", {}))
stockplatform.update(
{
"freshness_status": freshness_status,
"ingestion_status": ingestion_status,
"latest_trading_date": latest_trading_date,
"freshness_blockers": freshness_blockers,
"ingestion_blockers": ingestion_blockers,
"freshness_sla_source_count": _int(
readback.get("freshness_sla_source_count")
),
"freshness_source_count": _int(readback.get("freshness_source_count")),
"freshness_non_ok_source_count": _int(
readback.get("freshness_non_ok_source_count")
),
"live_runtime_overlay_applied": True,
}
)
required_checks = _dict(payload.setdefault("required_checks", {}))
required_checks["stockplatform_freshness_ok"] = freshness_ok
required_checks["stockplatform_ingestion_ok"] = ingestion_ok
if not (freshness_ok and ingestion_ok):
required_checks["product_data_green"] = False
payload["product_data_green"] = False
post_reboot = _dict(payload.setdefault("post_reboot_readiness", {}))
post_reboot["product_data_green"] = False
_append_live_stockplatform_blockers(
payload=payload,
freshness_status=freshness_status,
ingestion_status=ingestion_status,
freshness_blockers=freshness_blockers,
ingestion_blockers=ingestion_blockers,
)
completed_check_count = sum(1 for value in required_checks.values() if value)
readiness_percent = _percent(
completed_check_count / max(len(required_checks), 1) * 100
)
active_blockers = _strings(payload.get("active_blockers"))
active_blocker_count = len(active_blockers)
payload["active_blocker_count"] = active_blocker_count
payload["readiness_percent"] = readiness_percent
payload["stockplatform_freshness_status"] = freshness_status
payload["stockplatform_ingestion_status"] = ingestion_status
readback_section = _dict(payload.setdefault("readback", {}))
readback_section["active_blocker_count"] = active_blocker_count
readback_section["readiness_percent"] = readiness_percent
rollups = _dict(payload.setdefault("rollups", {}))
rollups["active_blocker_count"] = active_blocker_count
rollups["completed_check_count"] = completed_check_count
rollups["readiness_percent"] = readiness_percent
rollups["product_data_green"] = payload.get("product_data_green") is True
rollups["stockplatform_freshness_status"] = freshness_status
rollups["stockplatform_ingestion_status"] = ingestion_status
rollups["stockplatform_freshness_blocker_count"] = len(freshness_blockers)
rollups["stockplatform_ingestion_blocker_count"] = len(ingestion_blockers)
rollups["stockplatform_freshness_sla_source_count"] = _int(
readback.get("freshness_sla_source_count")
)
rollups["stockplatform_freshness_source_count"] = _int(
readback.get("freshness_source_count")
)
rollups["stockplatform_freshness_non_ok_source_count"] = _int(
readback.get("freshness_non_ok_source_count")
)
service_backup = _dict(
payload.setdefault("controlled_service_data_backup_readback", {})
)
service_backup["product_data_green"] = payload.get("product_data_green") is True
service_backup["stockplatform_freshness_status"] = freshness_status
service_backup["stockplatform_ingestion_status"] = ingestion_status
blocking_fields = _strings(service_backup.get("blocking_fields"))
if not freshness_ok:
blocking_fields.append("stockplatform_freshness_status")
if not ingestion_ok:
blocking_fields.append("stockplatform_ingestion_status")
if payload.get("product_data_green") is not True:
blocking_fields.append("product_data_green")
service_backup["blocking_fields"] = _unique_strings(blocking_fields)
service_backup["controlled_service_data_backup_blocker_count"] = len(
service_backup["blocking_fields"]
)
rollups["controlled_service_data_backup_blocker_count"] = len(
service_backup["blocking_fields"]
)
service_backup["status"] = "blocked_service_data_backup_readback_not_green"
service_backup["can_clear_service_data_backup_blockers"] = False
def _append_live_stockplatform_blockers(
*,
payload: dict[str, Any],
freshness_status: str,
ingestion_status: str,
freshness_blockers: list[str],
ingestion_blockers: list[str],
) -> None:
active_blockers = _strings(payload.get("active_blockers"))
active_blockers.append("product_data_green_not_1")
if freshness_status != "ok" and not freshness_blockers:
active_blockers.append("stockplatform_freshness_status_not_ok")
if ingestion_status != "ok" and not ingestion_blockers:
active_blockers.append("stockplatform_ingestion_status_not_ok")
active_blockers.extend(
f"stockplatform_freshness_{blocker}" for blocker in freshness_blockers
)
active_blockers.extend(
f"stockplatform_ingestion_{blocker}" for blocker in ingestion_blockers
)
payload["active_blockers"] = _unique_strings(active_blockers)
def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
host_boot_detection = _dict(scorecard.get("host_boot_detection")) host_boot_detection = _dict(scorecard.get("host_boot_detection"))
post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness")) post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness"))
@@ -764,5 +893,16 @@ def _strings(value: Any) -> list[str]:
return [str(item) for item in value if item is not None] return [str(item) for item in value if item is not None]
def _unique_strings(values: list[str]) -> list[str]:
seen: set[str] = set()
unique: list[str] = []
for value in values:
if value in seen:
continue
seen.add(value)
unique.append(value)
return unique
def _taipei_now_iso() -> str: def _taipei_now_iso() -> str:
return datetime.now(ZoneInfo("Asia/Taipei")).isoformat(timespec="seconds") return datetime.now(ZoneInfo("Asia/Taipei")).isoformat(timespec="seconds")

View File

@@ -3,6 +3,7 @@ from __future__ import annotations
from fastapi import FastAPI from fastapi import FastAPI
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from src.api.v1 import agents
from src.api.v1.agents import router from src.api.v1.agents import router
from src.services.reboot_auto_recovery_drill_preflight import ( from src.services.reboot_auto_recovery_drill_preflight import (
load_latest_reboot_auto_recovery_drill_preflight, load_latest_reboot_auto_recovery_drill_preflight,
@@ -45,7 +46,12 @@ def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate():
_assert_reboot_slo_payload(payload) _assert_reboot_slo_payload(payload)
def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(): def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch):
monkeypatch.setattr(
agents,
"load_latest_stockplatform_public_api_runtime_readback",
_stockplatform_runtime_ready,
)
app = FastAPI() app = FastAPI()
app.include_router(router, prefix="/api/v1") app.include_router(router, prefix="/api/v1")
client = TestClient(app) client = TestClient(app)
@@ -56,6 +62,59 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback():
_assert_reboot_slo_payload(response.json()) _assert_reboot_slo_payload(response.json())
def test_reboot_auto_recovery_slo_scorecard_endpoint_overlays_live_stockplatform_blocked(
monkeypatch,
):
monkeypatch.setattr(
agents,
"load_latest_stockplatform_public_api_runtime_readback",
_stockplatform_runtime_blocked,
)
app = FastAPI()
app.include_router(router, prefix="/api/v1")
client = TestClient(app)
response = client.get("/api/v1/agents/reboot-auto-recovery-slo-scorecard")
assert response.status_code == 200
payload = response.json()
assert payload["stockplatform_freshness_status"] == "blocked"
assert payload["stockplatform_ingestion_status"] == "blocked"
assert payload["stockplatform_data_freshness"]["latest_trading_date"] == (
"2026-07-02"
)
assert payload["stockplatform_data_freshness"]["freshness_sla_source_count"] == 9
assert payload["stockplatform_data_freshness"]["freshness_blockers"] == [
"core_margin_short_daily_missing",
"ai_recommendations_stale",
]
assert payload["stockplatform_data_freshness"]["ingestion_blockers"] == [
"core.margin_short_daily_incomplete"
]
assert payload["product_data_green"] is False
assert payload["required_checks"]["stockplatform_freshness_ok"] is False
assert payload["required_checks"]["stockplatform_ingestion_ok"] is False
assert payload["required_checks"]["product_data_green"] is False
assert payload["readiness_percent"] == 21
assert payload["active_blocker_count"] == 15
assert "product_data_green_not_1" in payload["active_blockers"]
assert "stockplatform_freshness_core_margin_short_daily_missing" in payload[
"active_blockers"
]
assert "stockplatform_ingestion_core.margin_short_daily_incomplete" in payload[
"active_blockers"
]
assert payload["rollups"]["stockplatform_freshness_status"] == "blocked"
assert payload["rollups"]["stockplatform_freshness_sla_source_count"] == 9
assert payload["controlled_service_data_backup_readback"][
"product_data_green"
] is False
assert "stockplatform_freshness_status" in payload[
"controlled_service_data_backup_readback"
]["blocking_fields"]
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 7
def test_reboot_auto_recovery_drill_preflight_loader_returns_break_glass_package(): def test_reboot_auto_recovery_drill_preflight_loader_returns_break_glass_package():
payload = load_latest_reboot_auto_recovery_drill_preflight() payload = load_latest_reboot_auto_recovery_drill_preflight()
@@ -439,3 +498,50 @@ def _assert_drill_preflight_payload(payload: dict):
assert boundaries["github_api_used"] is False assert boundaries["github_api_used"] is False
assert boundaries["runtime_write_allowed"] is False assert boundaries["runtime_write_allowed"] is False
assert "host_reboot" in payload["forbidden_without_separate_break_glass"] assert "host_reboot" in payload["forbidden_without_separate_break_glass"]
def _stockplatform_runtime_ready() -> dict:
return {
"schema_version": "stockplatform_public_api_runtime_readback_v1",
"status": "stockplatform_public_api_runtime_ready",
"runtime_ready": True,
"active_blockers": [],
"readback": {
"freshness_status": "ok",
"ingestion_status": "ok",
"freshness_latest_trading_date": "2026-07-02",
"ingestion_latest_trading_date": "2026-07-02",
"freshness_blockers": [],
"ingestion_blockers": [],
"freshness_sla_source_count": 9,
"freshness_source_count": 9,
"freshness_non_ok_source_count": 0,
},
}
def _stockplatform_runtime_blocked() -> dict:
return {
"schema_version": "stockplatform_public_api_runtime_readback_v1",
"status": "blocked_stockplatform_public_api_runtime_drift",
"runtime_ready": False,
"active_blockers": [
"stockplatform_freshness_core_margin_short_daily_missing",
"stockplatform_freshness_ai_recommendations_stale",
"stockplatform_ingestion_core.margin_short_daily_incomplete",
],
"readback": {
"freshness_status": "blocked",
"ingestion_status": "blocked",
"freshness_latest_trading_date": "2026-07-02",
"ingestion_latest_trading_date": "2026-07-02",
"freshness_blockers": [
"core_margin_short_daily_missing",
"ai_recommendations_stale",
],
"ingestion_blockers": ["core.margin_short_daily_incomplete"],
"freshness_sla_source_count": 9,
"freshness_source_count": 9,
"freshness_non_ok_source_count": 3,
},
}