fix(api): ignore stale cd failure after production readback
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 38s
CD Pipeline / build-and-deploy (push) Successful in 6m11s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s

This commit is contained in:
ogt
2026-07-01 21:16:19 +08:00
parent 140d8142fd
commit 3c9f648963
3 changed files with 225 additions and 33 deletions

View File

@@ -618,6 +618,44 @@ def apply_ai_loop_current_blocker_execution_queue(
)
state = _dict(payload.setdefault("mainline_execution_state", {}))
current_head = _dict(payload.get("current_head"))
production_readback_verified = bool(
state.get("current_main_cd_run_status") == "production_readback_verified"
and current_head.get("production_source_truth_available") is True
and _is_sha(str(state.get("latest_successful_deployed_source_sha") or ""))
)
deploy_marker_resolved_by_production_readback = bool(
production_readback_verified
and (deploy_marker_readback_required or cd_failed_after_registry_ready)
)
active_deployment_closure_state = (
"production_readback_verified"
if deploy_marker_resolved_by_production_readback
else deployment_closure_state
)
active_deploy_marker_readback_required = bool(
deploy_marker_readback_required
and not deploy_marker_resolved_by_production_readback
)
active_cd_failed_after_registry_ready = bool(
cd_failed_after_registry_ready
and not deploy_marker_resolved_by_production_readback
)
active_current_cd_run_id = (
str(state.get("current_main_cd_run_id") or "")
if deploy_marker_resolved_by_production_readback
else current_cd_run_id
)
active_current_cd_run_status = (
"production_readback_verified"
if deploy_marker_resolved_by_production_readback
else current_cd_run_status
)
active_current_cd_commit_sha = (
str(state.get("latest_successful_deployed_source_sha") or "")
if deploy_marker_resolved_by_production_readback
else current_cd_commit_sha
)
state["active_p0_state"] = "blocked_ai_loop_current_blocker_execution_queue"
state["next_executable_mainline_workplan_id"] = (
"P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE"
@@ -632,15 +670,38 @@ def apply_ai_loop_current_blocker_execution_queue(
registry_v2_status_classifier
)
state["ai_loop_current_blocker_deployment_closure_state"] = (
deployment_closure_state
active_deployment_closure_state
)
state["ai_loop_current_blocker_deploy_marker_readback_required"] = (
active_deploy_marker_readback_required
)
state["ai_loop_current_blocker_current_cd_run_id"] = active_current_cd_run_id
state["ai_loop_current_blocker_current_cd_run_status"] = (
active_current_cd_run_status
)
state["ai_loop_current_blocker_current_cd_commit_sha"] = (
active_current_cd_commit_sha
)
state["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
active_cd_failed_after_registry_ready
)
state["ai_loop_current_blocker_deploy_marker_resolved_by_production_readback"] = (
deploy_marker_resolved_by_production_readback
)
state["ai_loop_current_blocker_historical_deployment_closure_state"] = (
deployment_closure_state
)
state["ai_loop_current_blocker_historical_deploy_marker_readback_required"] = (
deploy_marker_readback_required
)
state["ai_loop_current_blocker_current_cd_run_id"] = current_cd_run_id
state["ai_loop_current_blocker_current_cd_run_status"] = current_cd_run_status
state["ai_loop_current_blocker_current_cd_commit_sha"] = current_cd_commit_sha
state["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
state["ai_loop_current_blocker_historical_current_cd_run_id"] = current_cd_run_id
state["ai_loop_current_blocker_historical_current_cd_run_status"] = (
current_cd_run_status
)
state["ai_loop_current_blocker_historical_current_cd_commit_sha"] = (
current_cd_commit_sha
)
state["ai_loop_current_blocker_historical_cd_failed_after_registry_ready"] = (
cd_failed_after_registry_ready
)
state["ai_loop_current_blocker_harbor_110_repair_run_id"] = (
@@ -717,12 +778,12 @@ def apply_ai_loop_current_blocker_execution_queue(
+ ([pressure_blocker] if pressure_blocker else [])
+ (
["deploy_marker_readback_required_after_registry_ready"]
if deploy_marker_readback_required
if active_deploy_marker_readback_required
else []
)
+ (
["current_cd_failure_after_registry_ready"]
if cd_failed_after_registry_ready
if active_cd_failed_after_registry_ready
else []
)
)
@@ -749,19 +810,44 @@ def apply_ai_loop_current_blocker_execution_queue(
registry_v2_status_classifier
)
evidence["ai_loop_current_blocker_deployment_closure_state"] = (
deployment_closure_state
active_deployment_closure_state
)
evidence["ai_loop_current_blocker_deploy_marker_readback_required"] = (
deploy_marker_readback_required
active_deploy_marker_readback_required
)
evidence["ai_loop_current_blocker_current_cd_run_id"] = (
active_current_cd_run_id
)
evidence["ai_loop_current_blocker_current_cd_run_id"] = current_cd_run_id
evidence["ai_loop_current_blocker_current_cd_run_status"] = (
current_cd_run_status
active_current_cd_run_status
)
evidence["ai_loop_current_blocker_current_cd_commit_sha"] = (
current_cd_commit_sha
active_current_cd_commit_sha
)
evidence["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
active_cd_failed_after_registry_ready
)
evidence[
"ai_loop_current_blocker_deploy_marker_resolved_by_production_readback"
] = deploy_marker_resolved_by_production_readback
evidence["ai_loop_current_blocker_historical_deployment_closure_state"] = (
deployment_closure_state
)
evidence[
"ai_loop_current_blocker_historical_deploy_marker_readback_required"
] = deploy_marker_readback_required
evidence["ai_loop_current_blocker_historical_current_cd_run_id"] = (
current_cd_run_id
)
evidence["ai_loop_current_blocker_historical_current_cd_run_status"] = (
current_cd_run_status
)
evidence["ai_loop_current_blocker_historical_current_cd_commit_sha"] = (
current_cd_commit_sha
)
evidence[
"ai_loop_current_blocker_historical_cd_failed_after_registry_ready"
] = (
cd_failed_after_registry_ready
)
evidence["ai_loop_current_blocker_harbor_110_repair_run_id"] = (
@@ -909,20 +995,21 @@ def apply_ai_loop_current_blocker_execution_queue(
"SSH/session control-path readback, ordered local-console phases, "
"post-recovery queue readbacks, and metadata-only KM/RAG/MCP/"
"PlayBook writeback."
),
(
"P0-006-CD-DEPLOY-MARKER-READBACK: close the latest visible CD "
f"{current_cd_run_id or 'unknown'} status "
f"{current_cd_run_status or 'unknown'} and verify deploy marker / "
"production image / priority API before claiming runtime closure."
),
(
"P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: only "
"rerun Harbor watchdog repair if registry /v2/ regresses below "
"200/401; otherwise keep focus on 110 SSH control-path and "
"deploy-marker closure."
),
)
]
if not deploy_marker_resolved_by_production_readback:
payload["next_execution_order"].append(
"P0-006-CD-DEPLOY-MARKER-READBACK: close the latest visible CD "
f"{active_current_cd_run_id or 'unknown'} status "
f"{active_current_cd_run_status or 'unknown'} and verify deploy marker / "
"production image / priority API before claiming runtime closure."
)
payload["next_execution_order"].append(
"P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: only "
"rerun Harbor watchdog repair if registry /v2/ regresses below "
"200/401; otherwise keep focus on 110 SSH control-path and "
"deploy-marker closure."
)
else:
payload["next_execution_order"] = [
(
@@ -938,14 +1025,15 @@ def apply_ai_loop_current_blocker_execution_queue(
"P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: after the "
"AI Loop queue item verifies 110 control path, rerun Harbor "
"watchdog check-mode / repair-once and registry /v2/ readback."
),
(
)
]
if not deploy_marker_resolved_by_production_readback:
payload["next_execution_order"].append(
"P0-006-CD-DEPLOY-MARKER-READBACK: after registry /v2/ is "
"200/401, let the next Gitea CD run build/push/deploy and then "
"verify delivery-closure-workbench, priority work order, and "
"production marker advance."
),
]
)
_refresh_rollups_after_stockplatform_overlay(payload, state)
summary = _dict(payload.setdefault("summary", {}))
summary["ai_loop_current_blocker_execution_queue_count"] = len(queue)
@@ -955,14 +1043,22 @@ def apply_ai_loop_current_blocker_execution_queue(
registry_v2_status_classifier
)
summary["ai_loop_current_blocker_deployment_closure_state"] = (
deployment_closure_state
active_deployment_closure_state
)
summary["ai_loop_current_blocker_deploy_marker_readback_required"] = (
deploy_marker_readback_required
active_deploy_marker_readback_required
)
summary["ai_loop_current_blocker_current_cd_run_status"] = (
active_current_cd_run_status
)
summary["ai_loop_current_blocker_current_cd_run_status"] = current_cd_run_status
summary["ai_loop_current_blocker_cd_failed_after_registry_ready"] = (
cd_failed_after_registry_ready
active_cd_failed_after_registry_ready
)
summary["ai_loop_current_blocker_deploy_marker_resolved_by_production_readback"] = (
deploy_marker_resolved_by_production_readback
)
summary["ai_loop_current_blocker_historical_current_cd_run_status"] = (
current_cd_run_status
)
summary["ai_loop_current_blocker_harbor_110_repair_run_status"] = (
harbor_110_repair_run_status