fix(reboot): expose fixed recovery sop progress
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 51s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
AI 技術雷達監控 / ai-technology-watch (push) Successful in 38s
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 51s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
AI 技術雷達監控 / ai-technology-watch (push) Successful in 38s
This commit is contained in:
@@ -459,9 +459,9 @@ _COMMANDER_INSERTED_REQUIREMENT_WORK_ITEMS: list[dict[str, Any]] = [
|
||||
"lane": "reboot_runbook_fixed_order",
|
||||
"request": "每次重啟排查都不一樣,也不知道多久恢復,不符合 SLA。",
|
||||
"normalized_work_item": "固定化 reboot runbook:fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command。",
|
||||
"current_state": "部分已有 scorecard / SOP;仍需所有回報統一格式。",
|
||||
"current_state": "scorecard source / snapshot / API 已補 current_phase、eta_or_wait_reason、primary_blocker、next_safe_action;仍需 production deploy readback 與下一次 fresh reboot/drill 證明。",
|
||||
"acceptance": "SLO scorecard 強制輸出 current_phase、eta_or_wait_reason、active_blockers、next_safe_action。",
|
||||
"next_action": "將 reboot SLO scorecard 補齊固定輸出欄位。",
|
||||
"next_action": "推送後讀回 /api/v1/agents/reboot-auto-recovery-slo-scorecard,確認固定欄位在 production 可見;再接 99/VMware 與 fresh reboot/drill。",
|
||||
"mapped_workplan_id": "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD",
|
||||
},
|
||||
{
|
||||
|
||||
@@ -83,6 +83,12 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"host_boot_observation_older_than_target_window"
|
||||
]
|
||||
latest_verify_only_metric_present = bool(latest_verify_metric)
|
||||
sop_progress = _build_reboot_sop_progress(
|
||||
scorecard=scorecard,
|
||||
active_blockers=active_blockers,
|
||||
readiness_percent=readiness_percent,
|
||||
can_claim_slo=can_claim_slo,
|
||||
)
|
||||
rollups = {
|
||||
"active_blocker_count": active_blocker_count,
|
||||
"readiness_percent": readiness_percent,
|
||||
@@ -137,7 +143,11 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"priority": "P0-006",
|
||||
"scope": "reboot_auto_recovery_slo_scorecard",
|
||||
"status": str(scorecard.get("status") or "unknown"),
|
||||
"current_phase": sop_progress["current_phase"],
|
||||
"eta_or_wait_reason": sop_progress["eta_or_wait_reason"],
|
||||
"safe_next_step": safe_next_step,
|
||||
"next_safe_action": sop_progress["next_safe_action"],
|
||||
"primary_blocker": sop_progress["primary_blocker"],
|
||||
"can_claim_all_services_recovered_within_target": can_claim_slo,
|
||||
"active_blocker_count": active_blocker_count,
|
||||
"readiness_percent": readiness_percent,
|
||||
@@ -168,12 +178,17 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"workplan_title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO",
|
||||
"source_scorecard_ref": f"docs/operations/{path.name}",
|
||||
"target_minutes": _int(scorecard.get("target_minutes")),
|
||||
"current_phase": sop_progress["current_phase"],
|
||||
"eta_or_wait_reason": sop_progress["eta_or_wait_reason"],
|
||||
"safe_next_step": safe_next_step,
|
||||
"next_safe_action": sop_progress["next_safe_action"],
|
||||
"primary_blocker": sop_progress["primary_blocker"],
|
||||
"active_blocker_count": active_blocker_count,
|
||||
"readiness_percent": readiness_percent,
|
||||
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
|
||||
"latest_verify_only_metric_present": latest_verify_only_metric_present,
|
||||
},
|
||||
"reboot_sop_progress": sop_progress,
|
||||
"host_boot_detection": host_boot_detection,
|
||||
"post_reboot_readiness": post_reboot_readiness,
|
||||
"stockplatform_data_freshness": stockplatform,
|
||||
@@ -196,6 +211,140 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _build_reboot_sop_progress(
|
||||
*,
|
||||
scorecard: dict[str, Any],
|
||||
active_blockers: list[str],
|
||||
readiness_percent: int,
|
||||
can_claim_slo: bool,
|
||||
) -> dict[str, Any]:
|
||||
current_phase = _reboot_sop_current_phase(active_blockers, can_claim_slo)
|
||||
primary_blocker = _reboot_sop_primary_blocker(active_blockers)
|
||||
eta_or_wait_reason = _reboot_sop_eta_or_wait_reason(
|
||||
scorecard=scorecard,
|
||||
active_blockers=active_blockers,
|
||||
current_phase=current_phase,
|
||||
primary_blocker=primary_blocker,
|
||||
)
|
||||
sla_eta = _dict(scorecard.get("sla_recovery_eta"))
|
||||
return {
|
||||
"current_phase": current_phase,
|
||||
"eta_or_wait_reason": eta_or_wait_reason,
|
||||
"primary_blocker": primary_blocker,
|
||||
"active_blockers": active_blockers,
|
||||
"active_blocker_count": len(active_blockers),
|
||||
"readiness_percent": readiness_percent,
|
||||
"next_safe_action": str(
|
||||
scorecard.get("next_safe_action")
|
||||
or scorecard.get("safe_next_step")
|
||||
or ""
|
||||
),
|
||||
"fixed_triage_order": _strings(sla_eta.get("fixed_triage_order")),
|
||||
}
|
||||
|
||||
|
||||
def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) -> str:
|
||||
if can_claim_slo and not active_blockers:
|
||||
return "slo_ready"
|
||||
if active_blockers == ["host_boot_observation_older_than_target_window"]:
|
||||
return "awaiting_next_reboot_or_approved_drill"
|
||||
host_boot_blockers = {
|
||||
"all_host_reboot_detection_missing",
|
||||
"stateful_reboot_event_detection_missing",
|
||||
"host_boot_probe_missing_hosts",
|
||||
"host_unreachable_after_reboot",
|
||||
"host_boot_observation_older_than_target_window",
|
||||
"host_uptime_unknown",
|
||||
"reboot_event_missing_required_hosts",
|
||||
"reboot_event_required_host_unreachable",
|
||||
"fresh_all_host_reboot_event_missing",
|
||||
"all_required_hosts_not_in_10_minute_reboot_window",
|
||||
}
|
||||
if any(blocker in host_boot_blockers for blocker in active_blockers):
|
||||
return "host_boot_detection_blocked"
|
||||
service_blockers = {
|
||||
"post_reboot_summary_missing",
|
||||
"post_start_blocked_not_zero",
|
||||
"service_green_not_1",
|
||||
"host_188_service_green_not_1",
|
||||
"wazuh_dashboard_degraded",
|
||||
}
|
||||
if any(blocker in service_blockers for blocker in active_blockers):
|
||||
return "post_reboot_service_readiness_blocked"
|
||||
if any(
|
||||
"stockplatform" in blocker or "product_data" in blocker
|
||||
for blocker in active_blockers
|
||||
):
|
||||
return "product_data_freshness_blocked"
|
||||
if any("backup" in blocker for blocker in active_blockers):
|
||||
return "backup_readback_blocked"
|
||||
if "local_disk_free_below_minimum" in active_blockers:
|
||||
return "host_capacity_blocked"
|
||||
if any(
|
||||
blocker.startswith("host_") or blocker.startswith("awooop_")
|
||||
for blocker in active_blockers
|
||||
):
|
||||
return "host_pressure_blocked"
|
||||
return "slo_blocked"
|
||||
|
||||
|
||||
def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
|
||||
priority = [
|
||||
"reboot_event_required_host_unreachable",
|
||||
"host_unreachable_after_reboot",
|
||||
"all_host_reboot_detection_missing",
|
||||
"stateful_reboot_event_detection_missing",
|
||||
"host_boot_probe_missing_hosts",
|
||||
"fresh_all_host_reboot_event_missing",
|
||||
"all_required_hosts_not_in_10_minute_reboot_window",
|
||||
"host_boot_observation_older_than_target_window",
|
||||
"host_uptime_unknown",
|
||||
"post_start_blocked_not_zero",
|
||||
"service_green_not_1",
|
||||
"host_188_service_green_not_1",
|
||||
"product_data_green_not_1",
|
||||
"stockplatform_freshness_blocked",
|
||||
"stockplatform_ingestion_blocked",
|
||||
"backup_core_green_not_1",
|
||||
"local_disk_free_below_minimum",
|
||||
"wazuh_dashboard_degraded",
|
||||
]
|
||||
for blocker in priority:
|
||||
if blocker in active_blockers:
|
||||
return blocker
|
||||
return active_blockers[0] if active_blockers else ""
|
||||
|
||||
|
||||
def _reboot_sop_eta_or_wait_reason(
|
||||
*,
|
||||
scorecard: dict[str, Any],
|
||||
active_blockers: list[str],
|
||||
current_phase: str,
|
||||
primary_blocker: str,
|
||||
) -> str:
|
||||
if current_phase == "slo_ready":
|
||||
return "recovered_within_10_minute_slo"
|
||||
sla_eta = _dict(scorecard.get("sla_recovery_eta"))
|
||||
reboot_event = _dict(scorecard.get("reboot_event_detection"))
|
||||
remaining = _int(sla_eta.get("target_seconds_remaining"))
|
||||
deadline_status = str(sla_eta.get("deadline_status") or "unknown")
|
||||
if remaining > 0:
|
||||
return (
|
||||
f"target_window_remaining_{remaining}s_but_blocked_by_"
|
||||
f"{primary_blocker or 'unknown'}"
|
||||
)
|
||||
if reboot_event.get("readback_present") is not True:
|
||||
return "reboot_event_readback_missing_eta_unavailable"
|
||||
if deadline_status == "target_window_elapsed":
|
||||
return (
|
||||
"target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_"
|
||||
"event_and_probe"
|
||||
)
|
||||
if active_blockers:
|
||||
return f"eta_unavailable_until_primary_blocker_clears:{primary_blocker}"
|
||||
return "eta_unavailable"
|
||||
|
||||
|
||||
def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None:
|
||||
boundaries = _dict(payload.get("operation_boundaries"))
|
||||
forbidden_true = [
|
||||
|
||||
Reference in New Issue
Block a user