wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(api): expose reboot SLO readback rollups
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 22s
Details
CD Pipeline / build-and-deploy (push) Successful in 4m10s
Details
CD Pipeline / post-deploy-checks (push) Successful in 1m1s
Details

Browse Source
This commit is contained in:
Your Name
2026-06-29 21:46:34 +08:00
parent 7d9df2bf30
commit f426522b59
3 changed files with 131 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py
View File

@@ -66,6 +66,68 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
can_claim_slo = (
scorecard.get("can_claim_all_services_recovered_within_target") is True
)
latest_verify_metric = _dict(scorecard.get("latest_verify_only_metric"))
active_blocker_count = len(active_blockers)
observed_host_count = len(_strings(host_boot_detection.get("observed_hosts")))
missing_host_count = len(_strings(host_boot_detection.get("missing_hosts")))
unreachable_host_count = len(_strings(host_boot_detection.get("unreachable_hosts")))
stale_host_count = len(_strings(host_boot_detection.get("stale_hosts")))
service_green = post_reboot_readiness.get("service_green") is True
product_data_green = post_reboot_readiness.get("product_data_green") is True
backup_core_green = post_reboot_readiness.get("backup_core_green") is True
host_188_service_green = post_reboot_readiness.get("host_188_service_green") is True
blocked_by_fresh_reboot_window_only = active_blockers == [
"host_boot_observation_older_than_target_window"
]
latest_verify_only_metric_present = bool(latest_verify_metric)
rollups = {
"active_blocker_count": active_blocker_count,
"readiness_percent": readiness_percent,
"completed_check_count": completed_check_count,
"required_check_count": len(required_checks),
"can_claim_all_services_recovered_within_target": can_claim_slo,
"observed_host_count": observed_host_count,
"missing_host_count": missing_host_count,
"unreachable_host_count": unreachable_host_count,
"stale_host_count": stale_host_count,
"post_start_blocked": _int(post_reboot_readiness.get("post_start_blocked")),
"service_green": service_green,
"product_data_green": product_data_green,
"backup_core_green": backup_core_green,
"host_188_service_green": host_188_service_green,
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
"latest_verify_only_metric_present": latest_verify_only_metric_present,
"latest_verify_only_metric_ready": _int(latest_verify_metric.get("ready")),
"latest_verify_only_metric_blocker_count": _int(
latest_verify_metric.get("blocker_count")
),
"latest_verify_only_metric_max_host_uptime_seconds": _int(
latest_verify_metric.get("max_host_uptime_seconds")
),
"latest_verify_only_metric_last_run_timestamp": _int(
latest_verify_metric.get("last_run_timestamp")
),
"stockplatform_freshness_status": str(
stockplatform.get("freshness_status") or "unknown"
),
"stockplatform_ingestion_status": str(
stockplatform.get("ingestion_status") or "unknown"
),
"stockplatform_freshness_blocker_count": len(
_strings(stockplatform.get("freshness_blockers"))
),
"stockplatform_ingestion_blocker_count": len(
_strings(stockplatform.get("ingestion_blockers"))
),
"stockplatform_final_retry_window_passed": _dict(
stockplatform.get("eod_window")
).get("final_retry_window_passed")
is True,
"stockplatform_controlled_recovery_gate_required": recovery_gate.get(
"required"
)
is True,
}
return {
"schema_version": _API_SCHEMA_VERSION,
"generated_at": str(scorecard.get("generated_at") or ""),
@@ -74,55 +136,47 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"status": str(scorecard.get("status") or "unknown"),
"safe_next_step": safe_next_step,
"can_claim_all_services_recovered_within_target": can_claim_slo,
"active_blocker_count": active_blocker_count,
"readiness_percent": readiness_percent,
"service_green": service_green,
"product_data_green": product_data_green,
"backup_core_green": backup_core_green,
"host_188_service_green": host_188_service_green,
"observed_host_count": observed_host_count,
"missing_host_count": missing_host_count,
"unreachable_host_count": unreachable_host_count,
"stale_host_count": stale_host_count,
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
"latest_verify_only_metric_present": latest_verify_only_metric_present,
"latest_verify_only_metric_ready": rollups["latest_verify_only_metric_ready"],
"latest_verify_only_metric_blocker_count": rollups[
"latest_verify_only_metric_blocker_count"
],
"latest_verify_only_metric_max_host_uptime_seconds": rollups[
"latest_verify_only_metric_max_host_uptime_seconds"
],
"latest_verify_only_metric_last_run_timestamp": rollups[
"latest_verify_only_metric_last_run_timestamp"
],
"stockplatform_freshness_status": rollups["stockplatform_freshness_status"],
"stockplatform_ingestion_status": rollups["stockplatform_ingestion_status"],
"readback": {
"workplan_id": "P0-006",
"workplan_title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO",
"source_scorecard_ref": f"docs/operations/{path.name}",
"target_minutes": _int(scorecard.get("target_minutes")),
"safe_next_step": safe_next_step,
"active_blocker_count": active_blocker_count,
"readiness_percent": readiness_percent,
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
"latest_verify_only_metric_present": latest_verify_only_metric_present,
},
"host_boot_detection": host_boot_detection,
"post_reboot_readiness": post_reboot_readiness,
"stockplatform_data_freshness": stockplatform,
"active_blockers": active_blockers,
"required_checks": required_checks,
"rollups": {
"active_blocker_count": len(active_blockers),
"readiness_percent": readiness_percent,
"completed_check_count": completed_check_count,
"required_check_count": len(required_checks),
"can_claim_all_services_recovered_within_target": can_claim_slo,
"observed_host_count": len(_strings(host_boot_detection.get("observed_hosts"))),
"missing_host_count": len(_strings(host_boot_detection.get("missing_hosts"))),
"unreachable_host_count": len(
_strings(host_boot_detection.get("unreachable_hosts"))
),
"stale_host_count": len(_strings(host_boot_detection.get("stale_hosts"))),
"post_start_blocked": _int(post_reboot_readiness.get("post_start_blocked")),
"service_green": post_reboot_readiness.get("service_green") is True,
"product_data_green": post_reboot_readiness.get("product_data_green") is True,
"backup_core_green": post_reboot_readiness.get("backup_core_green") is True,
"stockplatform_freshness_status": str(
stockplatform.get("freshness_status") or "unknown"
),
"stockplatform_ingestion_status": str(
stockplatform.get("ingestion_status") or "unknown"
),
"stockplatform_freshness_blocker_count": len(
_strings(stockplatform.get("freshness_blockers"))
),
"stockplatform_ingestion_blocker_count": len(
_strings(stockplatform.get("ingestion_blockers"))
),
"stockplatform_final_retry_window_passed": _dict(
stockplatform.get("eod_window")
).get("final_retry_window_passed")
is True,
"stockplatform_controlled_recovery_gate_required": recovery_gate.get(
"required"
)
is True,
},
"rollups": rollups,
"operation_boundaries": {
"read_only_api_allowed": True,
"host_reboot_performed": False,