fix(awooop): close stale ai loop blocker readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 45s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 45s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -694,6 +694,11 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
production_readback_verified
|
||||
and (deploy_marker_readback_required or cd_failed_after_registry_ready)
|
||||
)
|
||||
controlled_cd_lane_guardrails_resolved_by_production_readback = bool(
|
||||
production_readback_verified
|
||||
and blocker_id == "controlled_cd_lane_guardrails_blocked"
|
||||
and registry_v2_ready
|
||||
)
|
||||
queue_resolved_by_production_readback = bool(
|
||||
production_readback_verified
|
||||
and registry_v2_ready
|
||||
@@ -701,6 +706,7 @@ def apply_ai_loop_current_blocker_execution_queue(
|
||||
deploy_marker_readback_required
|
||||
or cd_failed_after_registry_ready
|
||||
or harbor_110_repair_failed_after_registry_ready
|
||||
or controlled_cd_lane_guardrails_resolved_by_production_readback
|
||||
)
|
||||
)
|
||||
if queue_resolved_by_production_readback:
|
||||
@@ -1408,8 +1414,24 @@ def _record_ai_loop_current_blocker_production_resolution(
|
||||
),
|
||||
}
|
||||
state.update(resolved_fields)
|
||||
resolved_blockers = {
|
||||
blocker_id,
|
||||
external_blocker,
|
||||
pressure_blocker,
|
||||
"ai_loop_current_blocker_execution_queue",
|
||||
"deploy_marker_readback_required_after_registry_ready",
|
||||
"current_cd_failure_after_registry_ready",
|
||||
}
|
||||
state["active_p0_live_active_blockers"] = [
|
||||
blocker
|
||||
for blocker in _strings(state.get("active_p0_live_active_blockers"))
|
||||
if blocker and blocker not in resolved_blockers
|
||||
]
|
||||
summary = _dict(payload.setdefault("summary", {}))
|
||||
summary.update(resolved_fields)
|
||||
summary["active_p0_live_active_blockers"] = list(
|
||||
state["active_p0_live_active_blockers"]
|
||||
)
|
||||
|
||||
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
|
||||
workplan = _dict(item)
|
||||
@@ -1417,6 +1439,11 @@ def _record_ai_loop_current_blocker_production_resolution(
|
||||
continue
|
||||
evidence = _dict(workplan.setdefault("evidence", {}))
|
||||
evidence.update(resolved_fields)
|
||||
evidence["active_blockers"] = [
|
||||
blocker
|
||||
for blocker in _strings(evidence.get("active_blockers"))
|
||||
if blocker and blocker not in resolved_blockers
|
||||
]
|
||||
break
|
||||
|
||||
|
||||
|
||||
@@ -929,6 +929,56 @@ def test_awoooi_priority_work_order_readback_does_not_reopen_stale_cd_failure_af
|
||||
)
|
||||
|
||||
|
||||
def test_awoooi_priority_work_order_readback_does_not_reopen_resolved_controlled_cd_lane_guardrails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
runtime_sha = "ec563465e7890f365f367c8faf6e99228bd19f91"
|
||||
runtime_short_sha = runtime_sha[:10]
|
||||
monkeypatch.setenv("AWOOOI_BUILD_COMMIT_SHA", runtime_sha)
|
||||
monkeypatch.setenv("AWOOOI_DESIRED_API_IMAGE_TAG", runtime_sha)
|
||||
|
||||
payload = load_latest_awoooi_priority_work_order_readback()
|
||||
apply_harbor_registry_controlled_recovery_preflight(
|
||||
payload,
|
||||
_harbor_registry_ready(),
|
||||
)
|
||||
executor = json.loads(
|
||||
json.dumps(load_latest_ai_agent_log_controlled_writeback_executor_readback())
|
||||
)
|
||||
|
||||
apply_ai_loop_current_blocker_execution_queue(payload, executor)
|
||||
|
||||
state = payload["mainline_execution_state"]
|
||||
evidence = payload["in_progress_or_blocked_in_priority_order"][0]["evidence"]
|
||||
blockers = state["active_p0_live_active_blockers"]
|
||||
assert payload["status"] == "p0_006_blocked_reboot_auto_recovery_slo_not_ready"
|
||||
assert state["active_p0_state"] == "blocked_reboot_auto_recovery_slo_not_ready"
|
||||
assert state["ai_loop_current_blocker_id"] == (
|
||||
"controlled_cd_lane_guardrails_blocked"
|
||||
)
|
||||
assert state["ai_loop_current_blocker_resolved_by_production_readback"] is True
|
||||
assert state["ai_loop_current_blocker_deployment_closure_state"] == (
|
||||
"production_readback_verified"
|
||||
)
|
||||
assert state["ai_loop_current_blocker_current_cd_run_id"] == (
|
||||
f"production_readback:{runtime_short_sha}"
|
||||
)
|
||||
assert state["ai_loop_current_blocker_current_cd_run_status"] == (
|
||||
"production_readback_verified"
|
||||
)
|
||||
assert state["ai_loop_current_blocker_safe_next_action_id"] == ""
|
||||
assert "ai_loop_current_blocker_execution_queue" not in blockers
|
||||
assert "controlled_cd_lane_guardrails_blocked" not in blockers
|
||||
assert evidence["ai_loop_current_blocker_resolved_by_production_readback"] is True
|
||||
assert evidence["ai_loop_current_blocker_deployment_closure_state"] == (
|
||||
"production_readback_verified"
|
||||
)
|
||||
assert payload["summary"][
|
||||
"ai_loop_current_blocker_resolved_by_production_readback"
|
||||
] is True
|
||||
assert payload["summary"]["ai_loop_current_blocker_safe_next_action_id"] == ""
|
||||
|
||||
|
||||
def test_awoooi_priority_work_order_readback_rejects_reordered_active_p0(tmp_path):
|
||||
operations_dir = tmp_path / "docs" / "operations"
|
||||
operations_dir.mkdir(parents=True)
|
||||
|
||||
@@ -9089,12 +9089,15 @@
|
||||
"receiptValue": "{inputs} inputs / {outputs} outputs",
|
||||
"currentCd": "Latest CD run",
|
||||
"currentCdValue": "#{run} · {status}",
|
||||
"currentCdDetail": "closure: {state}"
|
||||
"currentCdDetail": "closure: {state}",
|
||||
"resolvedByProductionReadback": "Closed by production readback",
|
||||
"noActionRequired": "Do not reopen this queue"
|
||||
},
|
||||
"rootCause": {
|
||||
"sessionTimeout": "Key accepted, session timeout",
|
||||
"offerTimeout": "Publickey offer timeout",
|
||||
"controlledCdLaneGuardrails": "SSH control path and Harbor v2 are readable; controlled CD lane guardrails are still blocked.",
|
||||
"productionReadbackResolved": "Latest production readback verified this queue item; it is no longer the current blocker.",
|
||||
"unknown": "Waiting for queue diagnosis"
|
||||
},
|
||||
"phases": {
|
||||
|
||||
@@ -9089,12 +9089,15 @@
|
||||
"receiptValue": "{inputs} inputs / {outputs} outputs",
|
||||
"currentCd": "最新 CD run",
|
||||
"currentCdValue": "#{run} · {status}",
|
||||
"currentCdDetail": "closure:{state}"
|
||||
"currentCdDetail": "closure:{state}",
|
||||
"resolvedByProductionReadback": "已由 production readback 關閉",
|
||||
"noActionRequired": "不需重開此 queue"
|
||||
},
|
||||
"rootCause": {
|
||||
"sessionTimeout": "Key accepted,session timeout",
|
||||
"offerTimeout": "Publickey offer timeout",
|
||||
"controlledCdLaneGuardrails": "SSH 控制通道與 Harbor v2 已可讀;目前卡在 controlled CD lane guardrails。",
|
||||
"productionReadbackResolved": "最新 production readback 已驗證此 queue item,不再當作目前卡點。",
|
||||
"unknown": "等待 queue 診斷"
|
||||
},
|
||||
"phases": {
|
||||
|
||||
@@ -1031,6 +1031,7 @@ type PriorityWorkOrderResponse = {
|
||||
ai_loop_current_blocker_id?: string | null;
|
||||
ai_loop_current_blocker_log_source_tag_count?: number | null;
|
||||
ai_loop_current_blocker_log_source_tag_keys?: string[] | null;
|
||||
ai_loop_current_blocker_resolved_by_production_readback?: boolean | null;
|
||||
ai_loop_current_blocker_harbor_recovery_receipt_input_count?: number | null;
|
||||
ai_loop_current_blocker_harbor_recovery_receipt_input_ids?: string[] | null;
|
||||
ai_loop_current_blocker_harbor_recovery_receipt_output_contract_count?: number | null;
|
||||
@@ -1069,6 +1070,7 @@ type PriorityWorkOrderResponse = {
|
||||
evidence?: {
|
||||
ai_loop_current_blocker_log_source_tags?: AiLoopLogSourceTag[] | null;
|
||||
ai_loop_log_source_tagging_contract?: AiLoopLogSourceContract[] | null;
|
||||
ai_loop_current_blocker_resolved_by_production_readback?: boolean | null;
|
||||
ai_loop_current_blocker_harbor_recovery_receipt_input_ids?: string[] | null;
|
||||
ai_loop_current_blocker_harbor_recovery_receipt_output_ids?: string[] | null;
|
||||
ai_loop_current_blocker_queue_readback_normalizer_field_ids?: string[] | null;
|
||||
@@ -7930,7 +7932,12 @@ function AiLoopLogSourceTagsPanel({
|
||||
evidence?.controlled_cd_lane_live_metric_blocker_count ??
|
||||
0;
|
||||
const currentBlockerId = summary?.ai_loop_current_blocker_id ?? "";
|
||||
const currentBlockerResolved =
|
||||
summary?.ai_loop_current_blocker_resolved_by_production_readback ??
|
||||
evidence?.ai_loop_current_blocker_resolved_by_production_readback ??
|
||||
false;
|
||||
const controlledCdLaneGuardrailsBlocked =
|
||||
!currentBlockerResolved &&
|
||||
currentBlockerId === "controlled_cd_lane_guardrails_blocked";
|
||||
const labelMap: Record<string, string> = {
|
||||
project_id: t("tagLabels.projectId"),
|
||||
@@ -8068,6 +8075,8 @@ function AiLoopLogSourceTagsPanel({
|
||||
: t("rootCause.unknown");
|
||||
const blockerDiagnosis = controlledCdLaneGuardrailsBlocked
|
||||
? t("rootCause.controlledCdLaneGuardrails")
|
||||
: currentBlockerResolved
|
||||
? t("rootCause.productionReadbackResolved")
|
||||
: rootCause;
|
||||
const controlledCdLanePhases = controlledCdLaneGuardrailsBlocked
|
||||
? [
|
||||
@@ -8096,8 +8105,12 @@ function AiLoopLogSourceTagsPanel({
|
||||
key: "blocker",
|
||||
icon: TriangleAlert,
|
||||
label: t("visual.blocker"),
|
||||
value: currentBlockerId || "--",
|
||||
tone: "border-[#f0c6a8] bg-[#fff8f1] text-[#9a4d16]",
|
||||
value: currentBlockerResolved
|
||||
? t("visual.resolvedByProductionReadback")
|
||||
: currentBlockerId || "--",
|
||||
tone: currentBlockerResolved
|
||||
? "border-[#b9d9c2] bg-[#f2fbf3] text-[#236332]"
|
||||
: "border-[#f0c6a8] bg-[#fff8f1] text-[#9a4d16]",
|
||||
},
|
||||
{
|
||||
key: "diagnosis",
|
||||
@@ -8142,7 +8155,9 @@ function AiLoopLogSourceTagsPanel({
|
||||
key: "safe-next-action",
|
||||
icon: ArrowRight,
|
||||
label: t("safeNextAction"),
|
||||
value: safeNextActionId || "--",
|
||||
value: currentBlockerResolved
|
||||
? t("visual.noActionRequired")
|
||||
: safeNextActionId || "--",
|
||||
detail: t("safeNextStage", {
|
||||
stage: safeNextActionStage || "--",
|
||||
local: String(Boolean(safeNextRequiresLocalConsole)),
|
||||
|
||||
Reference in New Issue
Block a user