fix(automation): bound asset reconciliation readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled

This commit is contained in:
ogt
2026-07-15 09:28:01 +08:00
parent 26996a288a
commit f8b6b88866
3 changed files with 143 additions and 2 deletions

View File

@@ -34,7 +34,12 @@ logger = structlog.get_logger(__name__)
_FIRST_DELAY_SECONDS = 90
_LOOP_BACKOFF_SECONDS = 30 * 60
_MAX_RECONCILIATION_CANDIDATES = 20_000
_RETRYABLE_RESULT_STATUSES = {"blocked_safe_no_write", "degraded_no_write"}
_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS = 30.0
_RETRYABLE_RESULT_STATUSES = {
"blocked_safe_no_write",
"degraded_cache_refresh",
"degraded_no_write",
}
_DAILY_TRIGGER_HOUR_TAIPEI = 3
_DAILY_TRIGGER_MINUTE_TAIPEI = 30
_TAIPEI = ZoneInfo("Asia/Taipei")
@@ -232,6 +237,7 @@ async def reconcile_once(
started = time.monotonic()
matrix = await build_asset_capability_matrix_with_live_readback(
project_id=project_id,
timeout_seconds=_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS,
include_live_only_assets=True,
)
if matrix.get("live_readback_status") != "ready":
@@ -264,6 +270,24 @@ async def reconcile_once(
candidates,
project_id=project_id,
)
public_matrix = await build_asset_capability_matrix_with_live_readback(
project_id=project_id,
timeout_seconds=_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS,
include_live_only_assets=False,
bypass_public_cache=True,
)
public_cache_refreshed = bool(
public_matrix.get("live_readback_status") == "ready"
and public_matrix.get("closed") is True
)
receipt["public_cache_refreshed"] = public_cache_refreshed
if not public_cache_refreshed:
receipt["status"] = "degraded_cache_refresh"
logger.warning(
"asset_capability_reconciliation_public_cache_refresh_degraded",
live_readback_status=public_matrix.get("live_readback_status"),
closed=public_matrix.get("closed"),
)
receipt["duration_ms"] = int((time.monotonic() - started) * 1000)
logger.info(
"asset_capability_reconciliation_completed",

View File

@@ -1314,6 +1314,7 @@ async def build_asset_capability_matrix_with_live_readback(
repo_root: Path | None = None,
timeout_seconds: float = LIVE_READBACK_TIMEOUT_SECONDS,
include_live_only_assets: bool = False,
bypass_public_cache: bool = False,
) -> dict[str, Any]:
"""Build a bounded matrix from live DB truth, failing closed to declared scope."""
@@ -1324,7 +1325,7 @@ async def build_asset_capability_matrix_with_live_readback(
bounded_timeout_seconds = max(0.001, float(timeout_seconds))
use_public_cache = repo_root is None and not include_live_only_assets
if use_public_cache:
if use_public_cache and not bypass_public_cache:
cached_payload = await _read_public_matrix_cache(
project_id,
namespace=_PUBLIC_MATRIX_FRESH_CACHE_NAMESPACE,