fix(cd): retry alert chain api health smoke

scripts/alert_chain_smoke_test.py

@@ -41,6 +41,20 @@ from urllib.request import Request, urlopen
 # =============================================================================
 # 配置
 # =============================================================================
+def _env_int(name: str, default: int) -> int:
+    try:
+        return int(os.environ.get(name, str(default)))
+    except ValueError:
+        return default
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.environ.get(name, str(default)))
+    except ValueError:
+        return default
+
+
 DEFAULT_API_URL = "https://awoooi.wooo.work"
 SIGNOZ_URL = "http://192.168.0.188:3301"
 ALERTMANAGER_URL = "http://192.168.0.188:9093"
@@ -50,6 +64,12 @@ PROMETHEUS_URL = "http://192.168.0.110:9090"
 MAX_ALERT_CHAIN_SILENCE_SECONDS = 2 * 60 * 60
 
 TIMEOUT = 10  # 秒
+API_HEALTH_TIMEOUT = _env_int("ALERT_CHAIN_API_HEALTH_TIMEOUT", 20)
+API_HEALTH_ATTEMPTS = max(
+    1,
+    _env_int("ALERT_CHAIN_API_HEALTH_ATTEMPTS", 3),
+)
+API_HEALTH_RETRY_DELAY = _env_float("ALERT_CHAIN_API_HEALTH_RETRY_DELAY", 3.0)
 
 
 @dataclass(frozen=True)
@@ -117,6 +137,13 @@ def _http_error_message(error: Exception) -> str:
     return str(error)
 
 
+def _api_health_probe_summary(attempt: int) -> str:
+    return (
+        f"attempts={attempt}/{API_HEALTH_ATTEMPTS}, "
+        f"timeout={API_HEALTH_TIMEOUT}s"
+    )
+
+
 def _statuses_from_env(env_name: str) -> list[str] | None:
     """Return preflight pod statuses supplied by CI, or None to use kubectl."""
     if env_name not in os.environ:
@@ -216,12 +243,53 @@ class SmokeTestReport:
 # =============================================================================
 def check_api_health(api_url: str) -> CheckResult:
     """Check 1: API Health — core runtime must be up; provider degradation is warning evidence."""
-    try:
-        resp = http_get(f"{api_url}/api/v1/health", timeout=TIMEOUT)
-        data = resp.json()
+    last_error = "unknown"
+    resp: HttpGetResult | None = None
+    data: dict[str, Any] | None = None
+    used_attempt = 0
 
+    for attempt in range(1, API_HEALTH_ATTEMPTS + 1):
+        used_attempt = attempt
+        try:
+            resp = http_get(
+                f"{api_url}/api/v1/health",
+                timeout=API_HEALTH_TIMEOUT,
+            )
+            data = resp.json()
+        except (URLError, TimeoutError, OSError, json.JSONDecodeError) as e:
+            last_error = _http_error_message(e)
+            if attempt < API_HEALTH_ATTEMPTS:
+                time.sleep(API_HEALTH_RETRY_DELAY)
+                continue
+            return CheckResult(
+                "API Health",
+                False,
+                (
+                    f"無法連線: {last_error} "
+                    f"({_api_health_probe_summary(attempt)})"
+                ),
+            )
+
+        if resp.status_code >= 500 and attempt < API_HEALTH_ATTEMPTS:
+            last_error = f"HTTP {resp.status_code}"
+            time.sleep(API_HEALTH_RETRY_DELAY)
+            continue
+
+        break
+
+    if resp is None or data is None:
+        return CheckResult("API Health", False, f"無法連線: {last_error}")
+
+    try:
         if resp.status_code >= 400:
-            return CheckResult("API Health", False, f"HTTP {resp.status_code}")
+            return CheckResult(
+                "API Health",
+                False,
+                (
+                    f"HTTP {resp.status_code} "
+                    f"({_api_health_probe_summary(used_attempt)})"
+                ),
+            )
 
         components = data.get("components", {})
         core_components = ("api", "postgresql", "redis")
@@ -244,16 +312,30 @@ def check_api_health(api_url: str) -> CheckResult:
             return CheckResult(
                 "API Health",
                 True,
-                f"核心組件 UP；非阻塞降級: {', '.join(down_components)}",
+                (
+                    f"核心組件 UP；非阻塞降級: {', '.join(down_components)} "
+                    f"({_api_health_probe_summary(used_attempt)})"
+                ),
             )
 
         return CheckResult(
             "API Health",
             True,
-            f"所有 {len(components)} 個組件 UP ({data.get('environment', 'unknown')})",
+            (
+                f"所有 {len(components)} 個組件 UP "
+                f"({data.get('environment', 'unknown')}; "
+                f"{_api_health_probe_summary(used_attempt)})"
+            ),
         )
     except (URLError, TimeoutError, OSError, json.JSONDecodeError) as e:
-        return CheckResult("API Health", False, f"無法連線: {_http_error_message(e)}")
+        return CheckResult(
+            "API Health",
+            False,
+            (
+                f"無法連線: {_http_error_message(e)} "
+                f"({_api_health_probe_summary(used_attempt)})"
+            ),
+        )
 
 
 def _escape_prometheus_label_value(value: str) -> str: