fix(api): retry slow stockplatform freshness readback

apps/api/src/services/stockplatform_public_api_runtime_readback.py

@@ -25,6 +25,8 @@ _RECOVERY_RECEIPT_FILE = (
 )
 _DEFAULT_BASE_URL = "https://stock.wooo.work"
 _DEFAULT_TIMEOUT_SECONDS = 4.0
+_DEFAULT_DATA_ENDPOINT_TIMEOUT_SECONDS = 10.0
+_DEFAULT_PROBE_ATTEMPTS = 2
 
 Probe = Callable[[str, float], dict[str, Any]]
 
@@ -33,6 +35,8 @@ def load_latest_stockplatform_public_api_runtime_readback(
     *,
     base_url: str = _DEFAULT_BASE_URL,
     timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
+    data_endpoint_timeout_seconds: float = _DEFAULT_DATA_ENDPOINT_TIMEOUT_SECONDS,
+    probe_attempts: int = _DEFAULT_PROBE_ATTEMPTS,
     operations_dir: Path | None = None,
     probe: Probe | None = None,
 ) -> dict[str, Any]:
@@ -46,23 +50,42 @@ def load_latest_stockplatform_public_api_runtime_readback(
     recovery_control_receipt = _load_recovery_control_receipt(directory)
     committed_stock = _dict(committed_scorecard.get("stockplatform_data_freshness"))
     endpoints = {
-        "public_web_healthz": "/healthz",
-        "public_api_healthz": "/api/healthz",
-        "freshness": "/api/v1/system/freshness",
-        "ingestion": "/api/v1/system/ingestion",
+        "public_web_healthz": {
+            "path": "/healthz",
+            "parse_json": False,
+            "timeout_seconds": timeout_seconds,
+        },
+        "public_api_healthz": {
+            "path": "/api/healthz",
+            "parse_json": False,
+            "timeout_seconds": timeout_seconds,
+        },
+        "freshness": {
+            "path": "/api/v1/system/freshness",
+            "parse_json": True,
+            "timeout_seconds": data_endpoint_timeout_seconds,
+        },
+        "ingestion": {
+            "path": "/api/v1/system/ingestion",
+            "parse_json": True,
+            "timeout_seconds": data_endpoint_timeout_seconds,
+        },
     }
     probes = {
         name: _probe_endpoint(
             http_probe,
-            f"{normalized_base_url}{path}",
-            timeout_seconds,
-            parse_json=name in {"freshness", "ingestion"},
+            f"{normalized_base_url}{endpoint['path']}",
+            float(endpoint["timeout_seconds"]),
+            parse_json=bool(endpoint["parse_json"]),
+            attempts=probe_attempts,
         )
-        for name, path in endpoints.items()
+        for name, endpoint in endpoints.items()
     }
     return _build_payload(
         base_url=normalized_base_url,
         timeout_seconds=timeout_seconds,
+        data_endpoint_timeout_seconds=data_endpoint_timeout_seconds,
+        probe_attempts=probe_attempts,
         probes=probes,
         committed_stockplatform=committed_stock,
         recovery_control_receipt=recovery_control_receipt,
@@ -73,6 +96,8 @@ def _build_payload(
     *,
     base_url: str,
     timeout_seconds: float,
+    data_endpoint_timeout_seconds: float,
+    probe_attempts: int,
     probes: dict[str, dict[str, Any]],
     committed_stockplatform: dict[str, Any],
     recovery_control_receipt: dict[str, Any],
@@ -144,6 +169,8 @@ def _build_payload(
         "live_drift_from_committed_scorecard": live_drift_from_committed_scorecard,
         "base_url": base_url,
         "timeout_seconds": timeout_seconds,
+        "data_endpoint_timeout_seconds": data_endpoint_timeout_seconds,
+        "probe_attempts": probe_attempts,
         "checks": checks,
         "readback": {
             "web_health_http_status": web.get("http_status"),
@@ -379,23 +406,37 @@ def _probe_endpoint(
     timeout_seconds: float,
     *,
     parse_json: bool,
+    attempts: int,
 ) -> dict[str, Any]:
-    result = probe(url, timeout_seconds)
-    http_status = _int_or_none(result.get("http_status"))
-    body = str(result.get("body") or "")
-    payload: dict[str, Any] = {
+    bounded_attempts = max(1, int(attempts))
+    last_payload: dict[str, Any] = {
         "url": url,
-        "http_status": http_status,
-        "ok": http_status == 200,
-        "error": str(result.get("error") or ""),
+        "http_status": None,
+        "ok": False,
+        "error": "not_attempted",
+        "attempt_count": 0,
     }
-    if parse_json and http_status == 200:
-        try:
-            payload["json"] = json.loads(body)
-        except json.JSONDecodeError:
-            payload["json"] = {}
-            payload["error"] = "invalid_json"
-    return payload
+    for attempt in range(1, bounded_attempts + 1):
+        result = probe(url, timeout_seconds)
+        http_status = _int_or_none(result.get("http_status"))
+        body = str(result.get("body") or "")
+        payload: dict[str, Any] = {
+            "url": url,
+            "http_status": http_status,
+            "ok": http_status == 200,
+            "error": str(result.get("error") or ""),
+            "attempt_count": attempt,
+        }
+        if parse_json and http_status == 200:
+            try:
+                payload["json"] = json.loads(body)
+            except json.JSONDecodeError:
+                payload["json"] = {}
+                payload["error"] = "invalid_json"
+        if payload.get("ok") and (not parse_json or payload.get("json")):
+            return payload
+        last_payload = payload
+    return last_payload
 
 
 def _http_probe(url: str, timeout_seconds: float) -> dict[str, Any]: