feat(heartbeat): full K8s pod lifecycle state machine + regression tests
P0 #3 (徹底長期修系列) — 把 daily report 的 pod 健康判斷從「ready=False 一律告警」 升級到完整 K8s pod lifecycle state machine： | Phase | 行為 | |-------|------| | Succeeded / Completed | 跳過（CronJob/Job 跑完正常） | | Failed | 必告警 | | Unknown | 必告警 | | Pending <5min | 跳過（剛 schedule 合理） | | Pending >=5min | 告警「image pull / scheduling 卡住」| | Running ready=True | 健康，跳過 | | Running ready=False <2min | 跳過（剛起來 probe 還沒過）| | Running ready=False >=2min | 告警「readiness probe fail / 啟動異常」| | restarts >=3 | 必告警（無論 phase）| 實作： - PodInfo 加 start_time: Optional[str]（從 .status.startTime） - _get_pod_status kubectl custom-columns 加 STARTTIME - _build_warnings 完整 state machine + 閾值常數 regression test (test_heartbeat_pod_state_machine.py 13 個) 覆蓋每個 phase + 邊界條件，含 2026-05-02 統帥截圖鐵證重現（3 個 drift-scanner Succeeded pod 不該觸發「需關注 3 項」假警報）。 Tests: 13 passed (新增 test_heartbeat_pod_state_machine.py) 接續 a38d9112（單純 Succeeded skip），這次徹底處理 Pending/Failed/Unknown + 時間閾值 + 沒 start_time 的保守告警。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Your Name
@@ -72,6 +72,9 @@ class PodInfo:
|
||||
ready: bool
|
||||
status: str
|
||||
restarts: int = 0
|
||||
# 2026-05-03 Claude Opus 4.7 + 統帥 ogt：P0 #3 K8s pod state machine 完整化
|
||||
# 加 start_time 才能判斷 Pending/NotReady 是「剛起來合理」還是「卡住該告警」
|
||||
start_time: Optional[str] = None # ISO 8601 from .status.startTime
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -517,7 +520,9 @@ class HeartbeatReportService:
|
||||
return s
|
||||
|
||||
async def _get_pod_status(self) -> list[PodInfo]:
|
||||
"""查 awoooi-prod namespace 的所有 Pod 狀態"""
|
||||
"""查 awoooi-prod namespace 的所有 Pod 狀態
|
||||
2026-05-03 Claude Opus 4.7 + 統帥 ogt：P0 #3 加抓 STARTTIME 才能做 K8s state machine 判斷
|
||||
"""
|
||||
pods: list[PodInfo] = []
|
||||
try:
|
||||
import subprocess
|
||||
@@ -525,7 +530,8 @@ class HeartbeatReportService:
|
||||
["kubectl", "-n", "awoooi-prod", "get", "pods",
|
||||
"--no-headers", "-o",
|
||||
"custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[0].ready,"
|
||||
"STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount"],
|
||||
"STATUS:.status.phase,RESTARTS:.status.containerStatuses[0].restartCount,"
|
||||
"STARTTIME:.status.startTime"],
|
||||
capture_output=True, text=True, timeout=8,
|
||||
)
|
||||
for line in r.stdout.strip().splitlines():
|
||||
@@ -535,7 +541,11 @@ class HeartbeatReportService:
|
||||
ready = parts[1].lower() == "true"
|
||||
status = parts[2]
|
||||
restarts = int(parts[3]) if len(parts) >= 4 and parts[3].isdigit() else 0
|
||||
pods.append(PodInfo(name=name, ready=ready, status=status, restarts=restarts))
|
||||
start_time = parts[4] if len(parts) >= 5 and parts[4] != "<none>" else None
|
||||
pods.append(PodInfo(
|
||||
name=name, ready=ready, status=status,
|
||||
restarts=restarts, start_time=start_time,
|
||||
))
|
||||
except Exception as e:
|
||||
logger.debug("heartbeat_pod_status_failed", error=str(e))
|
||||
return pods
|
||||
@@ -700,18 +710,58 @@ class HeartbeatReportService:
|
||||
if report.alert_pipeline.pending_approval > 10:
|
||||
warnings.append(f"PENDING 積壓 {report.alert_pipeline.pending_approval} 筆，需人工處理")
|
||||
|
||||
# Pod 異常
|
||||
# 2026-05-02 Claude Opus 4.7 + 統帥 ogt：CronJob/Job 跑完的 Pod (Succeeded/Completed)
|
||||
# ready=False 是設計（容器已退出），不是異常。原本邏輯每天推「Pod drift-scanner-* 未就緒
|
||||
# (Succeeded)」3 條 false positive，讓統帥誤以為告警重複。
|
||||
# Pod 異常 — 2026-05-03 Claude Opus 4.7 + 統帥 ogt：P0 #3 完整 K8s pod state machine
|
||||
# K8s pod phases (https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/):
|
||||
# Pending — 已建立但容器還沒起（短暫 OK，>5min 異常 = image pull / scheduling 卡）
|
||||
# Running — 至少 1 容器跑中（ready=False 短暫 OK，>2min 異常 = probe fail）
|
||||
# Succeeded — 全容器成功結束（CronJob/Job 正常，不算未就緒）
|
||||
# Failed — 全容器結束，至少 1 fail（必告警）
|
||||
# Unknown — 狀態無法取得（必告警）
|
||||
from datetime import datetime, timezone
|
||||
_now = datetime.now(timezone.utc)
|
||||
_PENDING_THRESHOLD_MIN = 5
|
||||
_NOT_READY_THRESHOLD_MIN = 2
|
||||
|
||||
def _age_minutes(start_time: Optional[str]) -> Optional[float]:
|
||||
"""ISO 8601 startTime → 距今分鐘。None 或解析失敗返 None。"""
|
||||
if not start_time:
|
||||
return None
|
||||
try:
|
||||
# K8s startTime 格式：2026-05-03T12:34:56Z
|
||||
dt = datetime.fromisoformat(start_time.replace("Z", "+00:00"))
|
||||
return (_now - dt).total_seconds() / 60.0
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
for pod in report.pods:
|
||||
if pod.status in ("Succeeded", "Completed"):
|
||||
continue # CronJob/Job 跑完是成功，不算未就緒
|
||||
if not pod.ready:
|
||||
warnings.append(f"Pod {pod.name} 未就緒（{pod.status}）")
|
||||
phase = pod.status
|
||||
age_min = _age_minutes(pod.start_time)
|
||||
|
||||
# restart 次數高無論 phase 都告警（CrashLoop 中或跑完都值得追）
|
||||
# 放最前面，避免後面 continue 跳過
|
||||
if pod.restarts >= 3:
|
||||
warnings.append(f"Pod {pod.name} 重啟 {pod.restarts} 次")
|
||||
|
||||
if phase in ("Succeeded", "Completed"):
|
||||
# CronJob/Job 成功跑完，ready=False 是設計，phase 部分不算未就緒
|
||||
continue
|
||||
elif phase == "Failed":
|
||||
# 真正失敗 — 一定告警
|
||||
warnings.append(f"Pod {pod.name} Failed")
|
||||
elif phase == "Unknown":
|
||||
warnings.append(f"Pod {pod.name} 狀態 Unknown")
|
||||
elif phase == "Pending":
|
||||
# 短暫 Pending OK；持續 >5min 表示 image pull / scheduling 卡住
|
||||
if age_min is None or age_min >= _PENDING_THRESHOLD_MIN:
|
||||
age_str = f"{age_min:.0f}m" if age_min else "未知"
|
||||
warnings.append(f"Pod {pod.name} 持續 Pending {age_str}（image pull / scheduling 卡住）")
|
||||
elif phase == "Running" and not pod.ready:
|
||||
# Running 但 not ready：短暫 OK（剛起）；>2min 表示 probe fail / 啟動慢
|
||||
if age_min is None or age_min >= _NOT_READY_THRESHOLD_MIN:
|
||||
age_str = f"{age_min:.0f}m" if age_min else "未知"
|
||||
warnings.append(f"Pod {pod.name} NotReady {age_str}（readiness probe fail / 啟動異常）")
|
||||
# Running + ready=True 是健康狀態，跳過
|
||||
|
||||
return warnings
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user