fix(reboot): expose fixed recovery sop progress
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 51s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
AI 技術雷達監控 / ai-technology-watch (push) Successful in 38s

This commit is contained in:
Your Name
2026-07-02 13:54:34 +08:00
parent ef79edd872
commit f9469bcc21
9 changed files with 399 additions and 5 deletions

View File

@@ -459,9 +459,9 @@ _COMMANDER_INSERTED_REQUIREMENT_WORK_ITEMS: list[dict[str, Any]] = [
"lane": "reboot_runbook_fixed_order",
"request": "每次重啟排查都不一樣,也不知道多久恢復,不符合 SLA。",
"normalized_work_item": "固定化 reboot runbookfixed triage order、ETA、active blocker、remaining seconds、owner lane、next command。",
"current_state": "部分已有 scorecard / SOP仍需所有回報統一格式",
"current_state": "scorecard source / snapshot / API 已補 current_phase、eta_or_wait_reason、primary_blocker、next_safe_action仍需 production deploy readback 與下一次 fresh reboot/drill 證明",
"acceptance": "SLO scorecard 強制輸出 current_phase、eta_or_wait_reason、active_blockers、next_safe_action。",
"next_action": "將 reboot SLO scorecard 補齊固定輸出欄位",
"next_action": "推送後讀回 /api/v1/agents/reboot-auto-recovery-slo-scorecard確認固定欄位在 production 可見;再接 99/VMware 與 fresh reboot/drill",
"mapped_workplan_id": "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD",
},
{

View File

@@ -83,6 +83,12 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"host_boot_observation_older_than_target_window"
]
latest_verify_only_metric_present = bool(latest_verify_metric)
sop_progress = _build_reboot_sop_progress(
scorecard=scorecard,
active_blockers=active_blockers,
readiness_percent=readiness_percent,
can_claim_slo=can_claim_slo,
)
rollups = {
"active_blocker_count": active_blocker_count,
"readiness_percent": readiness_percent,
@@ -137,7 +143,11 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"priority": "P0-006",
"scope": "reboot_auto_recovery_slo_scorecard",
"status": str(scorecard.get("status") or "unknown"),
"current_phase": sop_progress["current_phase"],
"eta_or_wait_reason": sop_progress["eta_or_wait_reason"],
"safe_next_step": safe_next_step,
"next_safe_action": sop_progress["next_safe_action"],
"primary_blocker": sop_progress["primary_blocker"],
"can_claim_all_services_recovered_within_target": can_claim_slo,
"active_blocker_count": active_blocker_count,
"readiness_percent": readiness_percent,
@@ -168,12 +178,17 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"workplan_title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO",
"source_scorecard_ref": f"docs/operations/{path.name}",
"target_minutes": _int(scorecard.get("target_minutes")),
"current_phase": sop_progress["current_phase"],
"eta_or_wait_reason": sop_progress["eta_or_wait_reason"],
"safe_next_step": safe_next_step,
"next_safe_action": sop_progress["next_safe_action"],
"primary_blocker": sop_progress["primary_blocker"],
"active_blocker_count": active_blocker_count,
"readiness_percent": readiness_percent,
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
"latest_verify_only_metric_present": latest_verify_only_metric_present,
},
"reboot_sop_progress": sop_progress,
"host_boot_detection": host_boot_detection,
"post_reboot_readiness": post_reboot_readiness,
"stockplatform_data_freshness": stockplatform,
@@ -196,6 +211,140 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
}
def _build_reboot_sop_progress(
*,
scorecard: dict[str, Any],
active_blockers: list[str],
readiness_percent: int,
can_claim_slo: bool,
) -> dict[str, Any]:
current_phase = _reboot_sop_current_phase(active_blockers, can_claim_slo)
primary_blocker = _reboot_sop_primary_blocker(active_blockers)
eta_or_wait_reason = _reboot_sop_eta_or_wait_reason(
scorecard=scorecard,
active_blockers=active_blockers,
current_phase=current_phase,
primary_blocker=primary_blocker,
)
sla_eta = _dict(scorecard.get("sla_recovery_eta"))
return {
"current_phase": current_phase,
"eta_or_wait_reason": eta_or_wait_reason,
"primary_blocker": primary_blocker,
"active_blockers": active_blockers,
"active_blocker_count": len(active_blockers),
"readiness_percent": readiness_percent,
"next_safe_action": str(
scorecard.get("next_safe_action")
or scorecard.get("safe_next_step")
or ""
),
"fixed_triage_order": _strings(sla_eta.get("fixed_triage_order")),
}
def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) -> str:
if can_claim_slo and not active_blockers:
return "slo_ready"
if active_blockers == ["host_boot_observation_older_than_target_window"]:
return "awaiting_next_reboot_or_approved_drill"
host_boot_blockers = {
"all_host_reboot_detection_missing",
"stateful_reboot_event_detection_missing",
"host_boot_probe_missing_hosts",
"host_unreachable_after_reboot",
"host_boot_observation_older_than_target_window",
"host_uptime_unknown",
"reboot_event_missing_required_hosts",
"reboot_event_required_host_unreachable",
"fresh_all_host_reboot_event_missing",
"all_required_hosts_not_in_10_minute_reboot_window",
}
if any(blocker in host_boot_blockers for blocker in active_blockers):
return "host_boot_detection_blocked"
service_blockers = {
"post_reboot_summary_missing",
"post_start_blocked_not_zero",
"service_green_not_1",
"host_188_service_green_not_1",
"wazuh_dashboard_degraded",
}
if any(blocker in service_blockers for blocker in active_blockers):
return "post_reboot_service_readiness_blocked"
if any(
"stockplatform" in blocker or "product_data" in blocker
for blocker in active_blockers
):
return "product_data_freshness_blocked"
if any("backup" in blocker for blocker in active_blockers):
return "backup_readback_blocked"
if "local_disk_free_below_minimum" in active_blockers:
return "host_capacity_blocked"
if any(
blocker.startswith("host_") or blocker.startswith("awooop_")
for blocker in active_blockers
):
return "host_pressure_blocked"
return "slo_blocked"
def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
priority = [
"reboot_event_required_host_unreachable",
"host_unreachable_after_reboot",
"all_host_reboot_detection_missing",
"stateful_reboot_event_detection_missing",
"host_boot_probe_missing_hosts",
"fresh_all_host_reboot_event_missing",
"all_required_hosts_not_in_10_minute_reboot_window",
"host_boot_observation_older_than_target_window",
"host_uptime_unknown",
"post_start_blocked_not_zero",
"service_green_not_1",
"host_188_service_green_not_1",
"product_data_green_not_1",
"stockplatform_freshness_blocked",
"stockplatform_ingestion_blocked",
"backup_core_green_not_1",
"local_disk_free_below_minimum",
"wazuh_dashboard_degraded",
]
for blocker in priority:
if blocker in active_blockers:
return blocker
return active_blockers[0] if active_blockers else ""
def _reboot_sop_eta_or_wait_reason(
*,
scorecard: dict[str, Any],
active_blockers: list[str],
current_phase: str,
primary_blocker: str,
) -> str:
if current_phase == "slo_ready":
return "recovered_within_10_minute_slo"
sla_eta = _dict(scorecard.get("sla_recovery_eta"))
reboot_event = _dict(scorecard.get("reboot_event_detection"))
remaining = _int(sla_eta.get("target_seconds_remaining"))
deadline_status = str(sla_eta.get("deadline_status") or "unknown")
if remaining > 0:
return (
f"target_window_remaining_{remaining}s_but_blocked_by_"
f"{primary_blocker or 'unknown'}"
)
if reboot_event.get("readback_present") is not True:
return "reboot_event_readback_missing_eta_unavailable"
if deadline_status == "target_window_elapsed":
return (
"target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_"
"event_and_probe"
)
if active_blockers:
return f"eta_unavailable_until_primary_blocker_clears:{primary_blocker}"
return "eta_unavailable"
def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None:
boundaries = _dict(payload.get("operation_boundaries"))
forbidden_true = [