fix(automation): bound asset reconciliation readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
This commit is contained in:
@@ -34,7 +34,12 @@ logger = structlog.get_logger(__name__)
|
||||
_FIRST_DELAY_SECONDS = 90
|
||||
_LOOP_BACKOFF_SECONDS = 30 * 60
|
||||
_MAX_RECONCILIATION_CANDIDATES = 20_000
|
||||
_RETRYABLE_RESULT_STATUSES = {"blocked_safe_no_write", "degraded_no_write"}
|
||||
_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS = 30.0
|
||||
_RETRYABLE_RESULT_STATUSES = {
|
||||
"blocked_safe_no_write",
|
||||
"degraded_cache_refresh",
|
||||
"degraded_no_write",
|
||||
}
|
||||
_DAILY_TRIGGER_HOUR_TAIPEI = 3
|
||||
_DAILY_TRIGGER_MINUTE_TAIPEI = 30
|
||||
_TAIPEI = ZoneInfo("Asia/Taipei")
|
||||
@@ -232,6 +237,7 @@ async def reconcile_once(
|
||||
started = time.monotonic()
|
||||
matrix = await build_asset_capability_matrix_with_live_readback(
|
||||
project_id=project_id,
|
||||
timeout_seconds=_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS,
|
||||
include_live_only_assets=True,
|
||||
)
|
||||
if matrix.get("live_readback_status") != "ready":
|
||||
@@ -264,6 +270,24 @@ async def reconcile_once(
|
||||
candidates,
|
||||
project_id=project_id,
|
||||
)
|
||||
public_matrix = await build_asset_capability_matrix_with_live_readback(
|
||||
project_id=project_id,
|
||||
timeout_seconds=_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS,
|
||||
include_live_only_assets=False,
|
||||
bypass_public_cache=True,
|
||||
)
|
||||
public_cache_refreshed = bool(
|
||||
public_matrix.get("live_readback_status") == "ready"
|
||||
and public_matrix.get("closed") is True
|
||||
)
|
||||
receipt["public_cache_refreshed"] = public_cache_refreshed
|
||||
if not public_cache_refreshed:
|
||||
receipt["status"] = "degraded_cache_refresh"
|
||||
logger.warning(
|
||||
"asset_capability_reconciliation_public_cache_refresh_degraded",
|
||||
live_readback_status=public_matrix.get("live_readback_status"),
|
||||
closed=public_matrix.get("closed"),
|
||||
)
|
||||
receipt["duration_ms"] = int((time.monotonic() - started) * 1000)
|
||||
logger.info(
|
||||
"asset_capability_reconciliation_completed",
|
||||
|
||||
@@ -1314,6 +1314,7 @@ async def build_asset_capability_matrix_with_live_readback(
|
||||
repo_root: Path | None = None,
|
||||
timeout_seconds: float = LIVE_READBACK_TIMEOUT_SECONDS,
|
||||
include_live_only_assets: bool = False,
|
||||
bypass_public_cache: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a bounded matrix from live DB truth, failing closed to declared scope."""
|
||||
|
||||
@@ -1324,7 +1325,7 @@ async def build_asset_capability_matrix_with_live_readback(
|
||||
bounded_timeout_seconds = max(0.001, float(timeout_seconds))
|
||||
use_public_cache = repo_root is None and not include_live_only_assets
|
||||
|
||||
if use_public_cache:
|
||||
if use_public_cache and not bypass_public_cache:
|
||||
cached_payload = await _read_public_matrix_cache(
|
||||
project_id,
|
||||
namespace=_PUBLIC_MATRIX_FRESH_CACHE_NAMESPACE,
|
||||
|
||||
Reference in New Issue
Block a user