fix(api): ignore stale cd failure after production readback
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 38s
CD Pipeline / build-and-deploy (push) Successful in 6m11s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 38s
CD Pipeline / build-and-deploy (push) Successful in 6m11s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
This commit is contained in:
@@ -618,6 +618,44 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
)
|
||||
|
||||
state = _dict(payload.setdefault("mainline_execution_state", {}))
|
||||
current_head = _dict(payload.get("current_head"))
|
||||
production_readback_verified = bool(
|
||||
state.get("current_main_cd_run_status") == "production_readback_verified"
|
||||
and current_head.get("production_source_truth_available") is True
|
||||
and _is_sha(str(state.get("latest_successful_deployed_source_sha") or ""))
|
||||
)
|
||||
deploy_marker_resolved_by_production_readback = bool(
|
||||
production_readback_verified
|
||||
and (deploy_marker_readback_required or cd_failed_after_registry_ready)
|
||||
)
|
||||
active_deployment_closure_state = (
|
||||
"production_readback_verified"
|
||||
if deploy_marker_resolved_by_production_readback
|
||||
else deployment_closure_state
|
||||
)
|
||||
active_deploy_marker_readback_required = bool(
|
||||
deploy_marker_readback_required
|
||||
and not deploy_marker_resolved_by_production_readback
|
||||
)
|
||||
active_cd_failed_after_registry_ready = bool(
|
||||
cd_failed_after_registry_ready
|
||||
and not deploy_marker_resolved_by_production_readback
|
||||
)
|
||||
active_current_cd_run_id = (
|
||||
str(state.get("current_main_cd_run_id") or "")
|
||||
if deploy_marker_resolved_by_production_readback
|
||||
else current_cd_run_id
|
||||
)
|
||||
active_current_cd_run_status = (
|
||||
"production_readback_verified"
|
||||
if deploy_marker_resolved_by_production_readback
|
||||
else current_cd_run_status
|
||||
)
|
||||
active_current_cd_commit_sha = (
|
||||
str(state.get("latest_successful_deployed_source_sha") or "")
|
||||
if deploy_marker_resolved_by_production_readback
|
||||
else current_cd_commit_sha
|
||||
)
|
||||
state["active_p0_state"] = "blocked_ai_loop_current_blocker_execution_queue"
|
||||
state["next_executable_mainline_workplan_id"] = (
|
||||
"P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE"
|
||||
@@ -632,15 +670,38 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
registry_v2_status_classifier
|
||||
)
|
||||
state["ai_loop_current_blocker_deployment_closure_state"] = (
|
||||
deployment_closure_state
|
||||
active_deployment_closure_state
|
||||
)
|
||||
state["ai_loop_current_blocker_deploy_marker_readback_required"] = (
|
||||
active_deploy_marker_readback_required
|
||||
)
|
||||
state["ai_loop_current_blocker_current_cd_run_id"] = active_current_cd_run_id
|
||||
state["ai_loop_current_blocker_current_cd_run_status"] = (
|
||||
active_current_cd_run_status
|
||||
)
|
||||
state["ai_loop_current_blocker_current_cd_commit_sha"] = (
|
||||
active_current_cd_commit_sha
|
||||
)
|
||||
state["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
|
||||
active_cd_failed_after_registry_ready
|
||||
)
|
||||
state["ai_loop_current_blocker_deploy_marker_resolved_by_production_readback"] = (
|
||||
deploy_marker_resolved_by_production_readback
|
||||
)
|
||||
state["ai_loop_current_blocker_historical_deployment_closure_state"] = (
|
||||
deployment_closure_state
|
||||
)
|
||||
state["ai_loop_current_blocker_historical_deploy_marker_readback_required"] = (
|
||||
deploy_marker_readback_required
|
||||
)
|
||||
state["ai_loop_current_blocker_current_cd_run_id"] = current_cd_run_id
|
||||
state["ai_loop_current_blocker_current_cd_run_status"] = current_cd_run_status
|
||||
state["ai_loop_current_blocker_current_cd_commit_sha"] = current_cd_commit_sha
|
||||
state["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
|
||||
state["ai_loop_current_blocker_historical_current_cd_run_id"] = current_cd_run_id
|
||||
state["ai_loop_current_blocker_historical_current_cd_run_status"] = (
|
||||
current_cd_run_status
|
||||
)
|
||||
state["ai_loop_current_blocker_historical_current_cd_commit_sha"] = (
|
||||
current_cd_commit_sha
|
||||
)
|
||||
state["ai_loop_current_blocker_historical_cd_failed_after_registry_ready"] = (
|
||||
cd_failed_after_registry_ready
|
||||
)
|
||||
state["ai_loop_current_blocker_harbor_110_repair_run_id"] = (
|
||||
@@ -717,12 +778,12 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
+ ([pressure_blocker] if pressure_blocker else [])
|
||||
+ (
|
||||
["deploy_marker_readback_required_after_registry_ready"]
|
||||
if deploy_marker_readback_required
|
||||
if active_deploy_marker_readback_required
|
||||
else []
|
||||
)
|
||||
+ (
|
||||
["current_cd_failure_after_registry_ready"]
|
||||
if cd_failed_after_registry_ready
|
||||
if active_cd_failed_after_registry_ready
|
||||
else []
|
||||
)
|
||||
)
|
||||
@@ -749,19 +810,44 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
registry_v2_status_classifier
|
||||
)
|
||||
evidence["ai_loop_current_blocker_deployment_closure_state"] = (
|
||||
deployment_closure_state
|
||||
active_deployment_closure_state
|
||||
)
|
||||
evidence["ai_loop_current_blocker_deploy_marker_readback_required"] = (
|
||||
deploy_marker_readback_required
|
||||
active_deploy_marker_readback_required
|
||||
)
|
||||
evidence["ai_loop_current_blocker_current_cd_run_id"] = (
|
||||
active_current_cd_run_id
|
||||
)
|
||||
evidence["ai_loop_current_blocker_current_cd_run_id"] = current_cd_run_id
|
||||
evidence["ai_loop_current_blocker_current_cd_run_status"] = (
|
||||
current_cd_run_status
|
||||
active_current_cd_run_status
|
||||
)
|
||||
evidence["ai_loop_current_blocker_current_cd_commit_sha"] = (
|
||||
current_cd_commit_sha
|
||||
active_current_cd_commit_sha
|
||||
)
|
||||
evidence["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
|
||||
active_cd_failed_after_registry_ready
|
||||
)
|
||||
evidence[
|
||||
"ai_loop_current_blocker_deploy_marker_resolved_by_production_readback"
|
||||
] = deploy_marker_resolved_by_production_readback
|
||||
evidence["ai_loop_current_blocker_historical_deployment_closure_state"] = (
|
||||
deployment_closure_state
|
||||
)
|
||||
evidence[
|
||||
"ai_loop_current_blocker_historical_deploy_marker_readback_required"
|
||||
] = deploy_marker_readback_required
|
||||
evidence["ai_loop_current_blocker_historical_current_cd_run_id"] = (
|
||||
current_cd_run_id
|
||||
)
|
||||
evidence["ai_loop_current_blocker_historical_current_cd_run_status"] = (
|
||||
current_cd_run_status
|
||||
)
|
||||
evidence["ai_loop_current_blocker_historical_current_cd_commit_sha"] = (
|
||||
current_cd_commit_sha
|
||||
)
|
||||
evidence[
|
||||
"ai_loop_current_blocker_historical_cd_failed_after_registry_ready"
|
||||
] = (
|
||||
cd_failed_after_registry_ready
|
||||
)
|
||||
evidence["ai_loop_current_blocker_harbor_110_repair_run_id"] = (
|
||||
@@ -909,20 +995,21 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
"SSH/session control-path readback, ordered local-console phases, "
|
||||
"post-recovery queue readbacks, and metadata-only KM/RAG/MCP/"
|
||||
"PlayBook writeback."
|
||||
),
|
||||
(
|
||||
"P0-006-CD-DEPLOY-MARKER-READBACK: close the latest visible CD "
|
||||
f"{current_cd_run_id or 'unknown'} status "
|
||||
f"{current_cd_run_status or 'unknown'} and verify deploy marker / "
|
||||
"production image / priority API before claiming runtime closure."
|
||||
),
|
||||
(
|
||||
"P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: only "
|
||||
"rerun Harbor watchdog repair if registry /v2/ regresses below "
|
||||
"200/401; otherwise keep focus on 110 SSH control-path and "
|
||||
"deploy-marker closure."
|
||||
),
|
||||
)
|
||||
]
|
||||
if not deploy_marker_resolved_by_production_readback:
|
||||
payload["next_execution_order"].append(
|
||||
"P0-006-CD-DEPLOY-MARKER-READBACK: close the latest visible CD "
|
||||
f"{active_current_cd_run_id or 'unknown'} status "
|
||||
f"{active_current_cd_run_status or 'unknown'} and verify deploy marker / "
|
||||
"production image / priority API before claiming runtime closure."
|
||||
)
|
||||
payload["next_execution_order"].append(
|
||||
"P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: only "
|
||||
"rerun Harbor watchdog repair if registry /v2/ regresses below "
|
||||
"200/401; otherwise keep focus on 110 SSH control-path and "
|
||||
"deploy-marker closure."
|
||||
)
|
||||
else:
|
||||
payload["next_execution_order"] = [
|
||||
(
|
||||
@@ -938,14 +1025,15 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
"P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: after the "
|
||||
"AI Loop queue item verifies 110 control path, rerun Harbor "
|
||||
"watchdog check-mode / repair-once and registry /v2/ readback."
|
||||
),
|
||||
(
|
||||
)
|
||||
]
|
||||
if not deploy_marker_resolved_by_production_readback:
|
||||
payload["next_execution_order"].append(
|
||||
"P0-006-CD-DEPLOY-MARKER-READBACK: after registry /v2/ is "
|
||||
"200/401, let the next Gitea CD run build/push/deploy and then "
|
||||
"verify delivery-closure-workbench, priority work order, and "
|
||||
"production marker advance."
|
||||
),
|
||||
]
|
||||
)
|
||||
_refresh_rollups_after_stockplatform_overlay(payload, state)
|
||||
summary = _dict(payload.setdefault("summary", {}))
|
||||
summary["ai_loop_current_blocker_execution_queue_count"] = len(queue)
|
||||
@@ -955,14 +1043,22 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
registry_v2_status_classifier
|
||||
)
|
||||
summary["ai_loop_current_blocker_deployment_closure_state"] = (
|
||||
deployment_closure_state
|
||||
active_deployment_closure_state
|
||||
)
|
||||
summary["ai_loop_current_blocker_deploy_marker_readback_required"] = (
|
||||
deploy_marker_readback_required
|
||||
active_deploy_marker_readback_required
|
||||
)
|
||||
summary["ai_loop_current_blocker_current_cd_run_status"] = (
|
||||
active_current_cd_run_status
|
||||
)
|
||||
summary["ai_loop_current_blocker_current_cd_run_status"] = current_cd_run_status
|
||||
summary["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
|
||||
cd_failed_after_registry_ready
|
||||
active_cd_failed_after_registry_ready
|
||||
)
|
||||
summary["ai_loop_current_blocker_deploy_marker_resolved_by_production_readback"] = (
|
||||
deploy_marker_resolved_by_production_readback
|
||||
)
|
||||
summary["ai_loop_current_blocker_historical_current_cd_run_status"] = (
|
||||
current_cd_run_status
|
||||
)
|
||||
summary["ai_loop_current_blocker_harbor_110_repair_run_status"] = (
|
||||
harbor_110_repair_run_status
|
||||
|
||||
Reference in New Issue
Block a user