diff --git a/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py b/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py index 1472e95c6..8a79f6772 100644 --- a/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py +++ b/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py @@ -134,6 +134,19 @@ def load_latest_ai_agent_log_controlled_writeback_executor_readback() -> dict[st item["harbor_recovery_receipt_output_contract_count"] for item in current_blocker_queue ), + "current_blocker_registry_v2_ready_count": sum( + 1 for item in current_blocker_queue if item["registry_v2_ready"] + ), + "current_blocker_deploy_marker_readback_required_count": sum( + 1 + for item in current_blocker_queue + if item["deploy_marker_readback_required"] + ), + "current_blocker_cd_failure_after_registry_ready_count": sum( + 1 + for item in current_blocker_queue + if item["cd_failed_after_registry_ready"] + ), "runtime_dispatch_performed": False, }, "active_blockers": active_blockers, @@ -366,6 +379,34 @@ def _current_blocker_queue_item(recovery: dict[str, Any]) -> dict[str, Any]: "node_load_classifier": node_load_classifier, "node_load_high": node_load_classifier == "high_load", "registry_v2_status": recovery.get("registry_v2_status"), + "registry_v2_ready": bool(recovery.get("registry_v2_ready") is True), + "registry_v2_status_classifier": str( + recovery.get("registry_v2_status_classifier") or "" + ), + "deployment_closure_state": str( + recovery.get("deployment_closure_state") or "" + ), + "deploy_marker_readback_required": bool( + recovery.get("deploy_marker_readback_required") is True + ), + "current_cd_run_id": str(recovery.get("current_cd_run_id") or ""), + "current_cd_run_status": str(recovery.get("current_cd_run_status") or ""), + "current_cd_commit_sha": str(recovery.get("current_cd_commit_sha") or ""), + "cd_failed_after_registry_ready": bool( + recovery.get("cd_failed_after_registry_ready") is True + ), + "harbor_110_repair_run_id": str( + recovery.get("harbor_110_repair_run_id") or "" + ), + "harbor_110_repair_run_status": str( + recovery.get("harbor_110_repair_run_status") or "" + ), + "harbor_110_repair_failed_after_registry_ready": bool( + recovery.get("harbor_110_repair_failed_after_registry_ready") is True + ), + "harbor_110_repair_failure_classifier": str( + recovery.get("harbor_110_repair_failure_classifier") or "" + ), "log_source_tags": log_source_tags, "log_source_tag_count": len(log_source_tags), "log_grouping_keys": list(_LOG_SOURCE_GROUPING_KEYS), @@ -796,6 +837,42 @@ def _queue_readback_normalizer_contract() -> list[dict[str, Any]]: ], "learning_targets": ["km", "rag", "playbook", "mcp", "verifier"], }, + { + "field_id": "current_cd_deploy_marker_closure", + "purpose": ( + "classify latest visible CD status, commit sha, and deploy-marker " + "closure after registry /v2/ has returned a ready 200/401 state" + ), + "writes_blockers": [ + "deploy_marker_readback_required_after_registry_ready", + "current_cd_failure_after_registry_ready", + ], + "learning_targets": ["km", "rag", "playbook", "mcp", "verifier", "ai_agent"], + }, + { + "field_id": "registry_v2_ready_state", + "purpose": ( + "distinguish registry /v2/ ready 200/401 evidence from stale " + "Harbor 502 repair blockers" + ), + "writes_blockers": [ + "public_registry_v2_verifier_not_green", + "internal_registry_v2_verifier_not_green", + ], + "learning_targets": ["km", "rag", "playbook", "mcp", "verifier", "ai_agent"], + }, + { + "field_id": "harbor_110_repair_failure_after_registry_ready", + "purpose": ( + "classify Harbor 110 repair failure as SSH/control-path closure " + "work when registry /v2/ is already ready" + ), + "writes_blockers": [ + "harbor_110_remote_ssh_publickey_auth_stalled", + "harbor_110_repair_failure_after_registry_ready", + ], + "learning_targets": ["km", "rag", "playbook", "mcp", "verifier", "ai_agent"], + }, { "field_id": "harbor_110_repair_visible_running_jobs_api_stale", "purpose": ( diff --git a/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py b/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py index 1232f6b29..ebd3ce2d3 100644 --- a/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py +++ b/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py @@ -204,6 +204,18 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None: if not blocker_id: return None source_sample_id = str(receipt.get("source_sample_id") or "") + registry_v2_status = classification.get("registry_v2_status") + registry_v2_ready = _registry_v2_ready(registry_v2_status) + current_cd_status = str(classification.get("current_cd_run_status") or "") + harbor_repair_status = str( + classification.get("harbor_110_repair_run_status") or "" + ) + cd_failed_after_registry_ready = bool( + registry_v2_ready and current_cd_status in {"Failure", "Blocked", "Canceled"} + ) + harbor_repair_failed_after_registry_ready = bool( + registry_v2_ready and harbor_repair_status in {"Failure", "Blocked"} + ) recovery = { "source_sample_id": source_sample_id, "project_id": str(receipt.get("project_id") or ""), @@ -220,7 +232,41 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None: classification.get("ssh_auth_classification") or "" ), "node_load_classifier": str(classification.get("node_load_classifier") or ""), - "registry_v2_status": classification.get("registry_v2_status"), + "registry_v2_status": registry_v2_status, + "registry_v2_ready": registry_v2_ready, + "registry_v2_status_classifier": _registry_v2_status_classifier( + registry_v2_status + ), + "deployment_closure_state": _deployment_closure_state( + registry_v2_ready=registry_v2_ready, + current_cd_status=current_cd_status, + harbor_repair_status=harbor_repair_status, + fallback=str(classification.get("deployment_closure_state") or ""), + ), + "deploy_marker_readback_required": bool( + registry_v2_ready + and ( + cd_failed_after_registry_ready + or harbor_repair_failed_after_registry_ready + or classification.get("deploy_marker_readback_required") is True + ) + ), + "current_cd_run_id": str(classification.get("current_cd_run_id") or ""), + "current_cd_run_status": current_cd_status, + "current_cd_commit_sha": str( + classification.get("current_cd_commit_sha") or "" + ), + "cd_failed_after_registry_ready": cd_failed_after_registry_ready, + "harbor_110_repair_run_id": str( + classification.get("harbor_110_repair_run_id") or "" + ), + "harbor_110_repair_run_status": harbor_repair_status, + "harbor_110_repair_failed_after_registry_ready": ( + harbor_repair_failed_after_registry_ready + ), + "harbor_110_repair_failure_classifier": str( + classification.get("harbor_110_repair_failure_classifier") or "" + ), "controlled_recovery_package": str( classification.get("controlled_recovery_package") or "" ), @@ -249,6 +295,43 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None: return recovery +def _registry_v2_ready(value: Any) -> bool: + try: + return int(str(value)) in {200, 401} + except (TypeError, ValueError): + return False + + +def _registry_v2_status_classifier(value: Any) -> str: + try: + status = int(str(value)) + except (TypeError, ValueError): + return "unknown" + if status == 200: + return "ready_http_200" + if status == 401: + return "ready_auth_required_http_401" + if status: + return f"blocked_http_{status}" + return "unknown" + + +def _deployment_closure_state( + *, + registry_v2_ready: bool, + current_cd_status: str, + harbor_repair_status: str, + fallback: str, +) -> str: + if registry_v2_ready and current_cd_status in {"Failure", "Blocked", "Canceled"}: + return "blocked_latest_visible_cd_failure_after_registry_ready" + if registry_v2_ready and harbor_repair_status in {"Failure", "Blocked"}: + return "blocked_harbor_110_repair_failure_after_registry_ready" + if registry_v2_ready: + return fallback or "registry_ready_waiting_deploy_marker_readback" + return fallback or "registry_v2_not_ready" + + def _list_of_dicts(value: Any) -> list[dict[str, Any]]: if not isinstance(value, list): return [] diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index ce7f0509b..2d4a4bbd7 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -513,6 +513,7 @@ def apply_ai_loop_current_blocker_execution_queue( if not _ai_loop_current_blocker_can_override( status=str(payload.get("status") or ""), blocker_id=blocker_id, + queue_item=first_item, ): return @@ -578,6 +579,30 @@ def apply_ai_loop_current_blocker_execution_queue( pressure_blocker = str(first_item.get("control_path_pressure_blocker") or "") node_load_classifier = str(first_item.get("node_load_classifier") or "") runtime_write_gate = str(first_item.get("runtime_write_gate") or "") + registry_v2_ready = bool(first_item.get("registry_v2_ready") is True) + registry_v2_status_classifier = str( + first_item.get("registry_v2_status_classifier") or "" + ) + deployment_closure_state = str(first_item.get("deployment_closure_state") or "") + deploy_marker_readback_required = bool( + first_item.get("deploy_marker_readback_required") is True + ) + current_cd_run_id = str(first_item.get("current_cd_run_id") or "") + current_cd_run_status = str(first_item.get("current_cd_run_status") or "") + current_cd_commit_sha = str(first_item.get("current_cd_commit_sha") or "") + cd_failed_after_registry_ready = bool( + first_item.get("cd_failed_after_registry_ready") is True + ) + harbor_110_repair_run_id = str(first_item.get("harbor_110_repair_run_id") or "") + harbor_110_repair_run_status = str( + first_item.get("harbor_110_repair_run_status") or "" + ) + harbor_110_repair_failure_classifier = str( + first_item.get("harbor_110_repair_failure_classifier") or "" + ) + harbor_110_repair_failed_after_registry_ready = bool( + first_item.get("harbor_110_repair_failed_after_registry_ready") is True + ) state = _dict(payload.setdefault("mainline_execution_state", {})) state["active_p0_state"] = "blocked_ai_loop_current_blocker_execution_queue" @@ -589,6 +614,34 @@ def apply_ai_loop_current_blocker_execution_queue( ) state["ai_loop_current_blocker_execution_queue_count"] = len(queue) state["ai_loop_current_blocker_id"] = blocker_id + state["ai_loop_current_blocker_registry_v2_ready"] = registry_v2_ready + state["ai_loop_current_blocker_registry_v2_status_classifier"] = ( + registry_v2_status_classifier + ) + state["ai_loop_current_blocker_deployment_closure_state"] = ( + deployment_closure_state + ) + state["ai_loop_current_blocker_deploy_marker_readback_required"] = ( + deploy_marker_readback_required + ) + state["ai_loop_current_blocker_current_cd_run_id"] = current_cd_run_id + state["ai_loop_current_blocker_current_cd_run_status"] = current_cd_run_status + state["ai_loop_current_blocker_current_cd_commit_sha"] = current_cd_commit_sha + state["ai_loop_current_blocker_cd_failed_after_registry_ready"] = ( + cd_failed_after_registry_ready + ) + state["ai_loop_current_blocker_harbor_110_repair_run_id"] = ( + harbor_110_repair_run_id + ) + state["ai_loop_current_blocker_harbor_110_repair_run_status"] = ( + harbor_110_repair_run_status + ) + state["ai_loop_current_blocker_harbor_110_repair_failure_classifier"] = ( + harbor_110_repair_failure_classifier + ) + state[ + "ai_loop_current_blocker_harbor_110_repair_failed_after_registry_ready" + ] = harbor_110_repair_failed_after_registry_ready state["ai_loop_current_blocker_control_path_blocker"] = external_blocker state["ai_loop_current_blocker_control_path_pressure_blocker"] = ( pressure_blocker @@ -639,6 +692,16 @@ def apply_ai_loop_current_blocker_execution_queue( + ([blocker_id] if blocker_id else []) + ([external_blocker] if external_blocker else []) + ([pressure_blocker] if pressure_blocker else []) + + ( + ["deploy_marker_readback_required_after_registry_ready"] + if deploy_marker_readback_required + else [] + ) + + ( + ["current_cd_failure_after_registry_ready"] + if cd_failed_after_registry_ready + else [] + ) ) for item in _list(payload.get("in_progress_or_blocked_in_priority_order")): @@ -658,6 +721,38 @@ def apply_ai_loop_current_blocker_execution_queue( evidence["ai_loop_current_blocker_registry_v2_status"] = first_item.get( "registry_v2_status" ) + evidence["ai_loop_current_blocker_registry_v2_ready"] = registry_v2_ready + evidence["ai_loop_current_blocker_registry_v2_status_classifier"] = ( + registry_v2_status_classifier + ) + evidence["ai_loop_current_blocker_deployment_closure_state"] = ( + deployment_closure_state + ) + evidence["ai_loop_current_blocker_deploy_marker_readback_required"] = ( + deploy_marker_readback_required + ) + evidence["ai_loop_current_blocker_current_cd_run_id"] = current_cd_run_id + evidence["ai_loop_current_blocker_current_cd_run_status"] = ( + current_cd_run_status + ) + evidence["ai_loop_current_blocker_current_cd_commit_sha"] = ( + current_cd_commit_sha + ) + evidence["ai_loop_current_blocker_cd_failed_after_registry_ready"] = ( + cd_failed_after_registry_ready + ) + evidence["ai_loop_current_blocker_harbor_110_repair_run_id"] = ( + harbor_110_repair_run_id + ) + evidence["ai_loop_current_blocker_harbor_110_repair_run_status"] = ( + harbor_110_repair_run_status + ) + evidence["ai_loop_current_blocker_harbor_110_repair_failure_classifier"] = ( + harbor_110_repair_failure_classifier + ) + evidence[ + "ai_loop_current_blocker_harbor_110_repair_failed_after_registry_ready" + ] = harbor_110_repair_failed_after_registry_ready evidence["ai_loop_current_blocker_controlled_recovery_package"] = ( controlled_recovery_package ) @@ -762,31 +857,77 @@ def apply_ai_loop_current_blocker_execution_queue( ) payload["status"] = "p0_006_blocked_ai_loop_current_blocker_execution_queue" - payload["next_execution_order"] = [ - ( - "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE: execute the " - f"{blocker_id} queue item through 110 control-path readback, " - f"{len(local_console_plan)} ordered local-console phases, post-recovery " - "readback commands, Harbor receipt inputs and outputs, and " - "metadata-only KM/RAG/MCP/PlayBook writeback from normalized queue " - "classifiers and control-path readiness classifiers." - ), - ( - "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: after the " - "AI Loop queue item verifies 110 control path, rerun Harbor " - "watchdog check-mode / repair-once and registry /v2/ readback." - ), - ( - "P0-006-CD-DEPLOY-MARKER-READBACK: after registry /v2/ is 200/401, " - "let the next Gitea CD run build/push/deploy and then verify " - "delivery-closure-workbench, priority work order, and production " - "marker advance." - ), - ] + if registry_v2_ready: + payload["next_execution_order"] = [ + ( + "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE: registry /v2/ " + f"is already {first_item.get('registry_v2_status')} " + f"({registry_v2_status_classifier}); do not reopen Harbor 502 " + f"as the main blocker. Execute {blocker_id} through 110 " + "SSH/session control-path readback, ordered local-console phases, " + "post-recovery queue readbacks, and metadata-only KM/RAG/MCP/" + "PlayBook writeback." + ), + ( + "P0-006-CD-DEPLOY-MARKER-READBACK: close the latest visible CD " + f"{current_cd_run_id or 'unknown'} status " + f"{current_cd_run_status or 'unknown'} and verify deploy marker / " + "production image / priority API before claiming runtime closure." + ), + ( + "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: only " + "rerun Harbor watchdog repair if registry /v2/ regresses below " + "200/401; otherwise keep focus on 110 SSH control-path and " + "deploy-marker closure." + ), + ] + else: + payload["next_execution_order"] = [ + ( + "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE: execute the " + f"{blocker_id} queue item through 110 control-path readback, " + f"{len(local_console_plan)} ordered local-console phases, " + "post-recovery readback commands, Harbor receipt inputs and " + "outputs, and metadata-only KM/RAG/MCP/PlayBook writeback from " + "normalized queue classifiers and control-path readiness " + "classifiers." + ), + ( + "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: after the " + "AI Loop queue item verifies 110 control path, rerun Harbor " + "watchdog check-mode / repair-once and registry /v2/ readback." + ), + ( + "P0-006-CD-DEPLOY-MARKER-READBACK: after registry /v2/ is " + "200/401, let the next Gitea CD run build/push/deploy and then " + "verify delivery-closure-workbench, priority work order, and " + "production marker advance." + ), + ] _refresh_rollups_after_stockplatform_overlay(payload, state) summary = _dict(payload.setdefault("summary", {})) summary["ai_loop_current_blocker_execution_queue_count"] = len(queue) summary["ai_loop_current_blocker_id"] = blocker_id + summary["ai_loop_current_blocker_registry_v2_ready"] = registry_v2_ready + summary["ai_loop_current_blocker_registry_v2_status_classifier"] = ( + registry_v2_status_classifier + ) + summary["ai_loop_current_blocker_deployment_closure_state"] = ( + deployment_closure_state + ) + summary["ai_loop_current_blocker_deploy_marker_readback_required"] = ( + deploy_marker_readback_required + ) + summary["ai_loop_current_blocker_current_cd_run_status"] = current_cd_run_status + summary["ai_loop_current_blocker_cd_failed_after_registry_ready"] = ( + cd_failed_after_registry_ready + ) + summary["ai_loop_current_blocker_harbor_110_repair_run_status"] = ( + harbor_110_repair_run_status + ) + summary["ai_loop_current_blocker_harbor_110_repair_failure_classifier"] = ( + harbor_110_repair_failure_classifier + ) summary["ai_loop_current_blocker_control_path_blocker"] = external_blocker summary["ai_loop_current_blocker_control_path_pressure_blocker"] = ( pressure_blocker @@ -836,12 +977,23 @@ def apply_ai_loop_current_blocker_execution_queue( ) -def _ai_loop_current_blocker_can_override(*, status: str, blocker_id: str) -> bool: +def _ai_loop_current_blocker_can_override( + *, + status: str, + blocker_id: str, + queue_item: dict[str, Any], +) -> bool: if status == "p0_006_blocked_harbor_registry_controlled_recovery_preflight": return True if status == "p0_006_blocked_stockplatform_public_api_runtime_drift": return blocker_id.startswith("harbor_110_") - return False + if not blocker_id.startswith("harbor_110_"): + return False + return bool( + queue_item.get("deploy_marker_readback_required") is True + or queue_item.get("cd_failed_after_registry_ready") is True + or queue_item.get("harbor_110_repair_failed_after_registry_ready") is True + ) def _harbor_registry_next_state(*, status: str, candidate_packaged: bool) -> str: diff --git a/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py b/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py index 904211f74..ee724526b 100644 --- a/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py +++ b/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py @@ -83,6 +83,15 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): ] == 8 ) + assert payload["rollups"]["current_blocker_registry_v2_ready_count"] == 1 + assert ( + payload["rollups"]["current_blocker_deploy_marker_readback_required_count"] + == 1 + ) + assert ( + payload["rollups"]["current_blocker_cd_failure_after_registry_ready_count"] + == 1 + ) assert payload["rollups"]["runtime_dispatch_performed"] is False batches = {batch["target"]: batch for batch in payload["execution_batches"]} @@ -150,7 +159,27 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert current_queue[0]["runner_label"] == "awoooi-non110-host" assert current_queue[0]["node_load_classifier"] == "load_not_high" assert current_queue[0]["node_load_high"] is False - assert current_queue[0]["registry_v2_status"] == 502 + assert current_queue[0]["registry_v2_status"] == 401 + assert current_queue[0]["registry_v2_ready"] is True + assert current_queue[0]["registry_v2_status_classifier"] == ( + "ready_auth_required_http_401" + ) + assert current_queue[0]["deployment_closure_state"] == ( + "blocked_latest_visible_cd_failure_after_registry_ready" + ) + assert current_queue[0]["deploy_marker_readback_required"] is True + assert current_queue[0]["current_cd_run_id"] == "4258" + assert current_queue[0]["current_cd_run_status"] == "Failure" + assert current_queue[0]["current_cd_commit_sha"] == ( + "06819ea96c058e7987811e853242390eaced7f91" + ) + assert current_queue[0]["cd_failed_after_registry_ready"] is True + assert current_queue[0]["harbor_110_repair_run_id"] == "4255" + assert current_queue[0]["harbor_110_repair_run_status"] == "Failure" + assert current_queue[0]["harbor_110_repair_failed_after_registry_ready"] is True + assert current_queue[0]["harbor_110_repair_failure_classifier"] == ( + "harbor_110_remote_ssh_publickey_auth_stalled" + ) assert current_queue[0]["log_source_tag_count"] == 10 source_tags = { item["tag_key"]: item for item in current_queue[0]["log_source_tags"] @@ -231,7 +260,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): ] for item in current_queue[0]["harbor_recovery_receipt_output_contract"] ) - assert current_queue[0]["queue_readback_normalizer_contract_count"] == 11 + assert current_queue[0]["queue_readback_normalizer_contract_count"] == 14 assert [ item["field_id"] for item in current_queue[0]["queue_readback_normalizer_contract"] @@ -244,6 +273,9 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): "latest_visible_harbor_110_repair_remote_ssh_publickey_offer_timeout", "latest_visible_harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout", "current_cd_workflow_runner_readiness", + "current_cd_deploy_marker_closure", + "registry_v2_ready_state", + "harbor_110_repair_failure_after_registry_ready", "harbor_110_repair_visible_running_jobs_api_stale", "current_cd_waiting_behind_harbor_110_repair_running", "controlled_profile_no_matching_runner_labels", diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index cacc6c34d..4cfd7d55e 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -112,19 +112,28 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot( assert response.status_code == 200 data = response.json() - assert data["status"] == "p0_006_blocked_reboot_auto_recovery_slo_not_ready" + assert data["status"] == "p0_006_blocked_ai_loop_current_blocker_execution_queue" assert data["mainline_execution_state"]["active_p0_workplan_id"] == "P0-006" assert data["mainline_execution_state"]["active_p0_state"] == ( - "blocked_reboot_auto_recovery_slo_not_ready" + "blocked_ai_loop_current_blocker_execution_queue" ) assert data["mainline_execution_state"][ "next_executable_mainline_workplan_id" - ] == "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD" + ] == "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE" + assert ( + data["mainline_execution_state"]["ai_loop_current_blocker_registry_v2_ready"] + is True + ) + assert data["mainline_execution_state"][ + "ai_loop_current_blocker_deploy_marker_readback_required" + ] is True assert data["mainline_execution_state"]["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready" assert data["mainline_execution_state"]["reboot_drill_preflight_runtime_readback_state"] == "blocked" assert data["rollups"]["stockplatform_public_api_runtime_ready"] is True - assert data["next_execution_order"][0].startswith("P0-006:") - assert "reboot SLO scorecard is blocked" in data["next_execution_order"][0] + assert data["next_execution_order"][0].startswith( + "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE" + ) + assert "do not reopen Harbor 502" in data["next_execution_order"][0] def test_awoooi_priority_work_order_readback_endpoint_redacts_ai_loop_queue( @@ -265,12 +274,61 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu ) assert state["ai_loop_current_blocker_control_path_pressure_blocker"] == "" assert state["ai_loop_current_blocker_node_load_classifier"] == "load_not_high" + assert state["ai_loop_current_blocker_registry_v2_ready"] is True + assert state["ai_loop_current_blocker_registry_v2_status_classifier"] == ( + "ready_auth_required_http_401" + ) + assert state["ai_loop_current_blocker_deployment_closure_state"] == ( + "blocked_latest_visible_cd_failure_after_registry_ready" + ) + assert state["ai_loop_current_blocker_deploy_marker_readback_required"] is True + assert state["ai_loop_current_blocker_current_cd_run_id"] == "4258" + assert state["ai_loop_current_blocker_current_cd_run_status"] == "Failure" + assert state["ai_loop_current_blocker_cd_failed_after_registry_ready"] is True + assert state["ai_loop_current_blocker_harbor_110_repair_run_id"] == "4255" + assert state["ai_loop_current_blocker_harbor_110_repair_run_status"] == ( + "Failure" + ) + assert state["ai_loop_current_blocker_harbor_110_repair_failure_classifier"] == ( + "harbor_110_remote_ssh_publickey_auth_stalled" + ) assert evidence["ai_loop_current_blocker_execution_queue_count"] == 1 assert evidence["ai_loop_current_blocker_id"] == ( "harbor_110_remote_ssh_publickey_auth_stalled" ) assert evidence["ai_loop_current_blocker_runner_label"] == "awoooi-non110-host" - assert evidence["ai_loop_current_blocker_registry_v2_status"] == 502 + assert evidence["ai_loop_current_blocker_registry_v2_status"] == 401 + assert evidence["ai_loop_current_blocker_registry_v2_ready"] is True + assert evidence["ai_loop_current_blocker_registry_v2_status_classifier"] == ( + "ready_auth_required_http_401" + ) + assert evidence["ai_loop_current_blocker_deployment_closure_state"] == ( + "blocked_latest_visible_cd_failure_after_registry_ready" + ) + assert ( + evidence["ai_loop_current_blocker_deploy_marker_readback_required"] + is True + ) + assert evidence["ai_loop_current_blocker_current_cd_run_id"] == "4258" + assert evidence["ai_loop_current_blocker_current_cd_run_status"] == "Failure" + assert ( + evidence["ai_loop_current_blocker_current_cd_commit_sha"] + == "06819ea96c058e7987811e853242390eaced7f91" + ) + assert evidence["ai_loop_current_blocker_cd_failed_after_registry_ready"] is True + assert evidence["ai_loop_current_blocker_harbor_110_repair_run_id"] == "4255" + assert evidence["ai_loop_current_blocker_harbor_110_repair_run_status"] == ( + "Failure" + ) + assert evidence["ai_loop_current_blocker_harbor_110_repair_failure_classifier"] == ( + "harbor_110_remote_ssh_publickey_auth_stalled" + ) + assert ( + evidence[ + "ai_loop_current_blocker_harbor_110_repair_failed_after_registry_ready" + ] + is True + ) assert evidence["ai_loop_current_blocker_controlled_recovery_package"] == ( "recover-110-control-path-and-harbor-local.sh --check" ) @@ -370,6 +428,9 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "latest_visible_harbor_110_repair_remote_ssh_publickey_offer_timeout", "latest_visible_harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout", "current_cd_workflow_runner_readiness", + "current_cd_deploy_marker_closure", + "registry_v2_ready_state", + "harbor_110_repair_failure_after_registry_ready", "harbor_110_repair_visible_running_jobs_api_stale", "current_cd_waiting_behind_harbor_110_repair_running", "controlled_profile_no_matching_runner_labels", @@ -433,6 +494,26 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu assert payload["summary"]["ai_loop_current_blocker_id"] == ( "harbor_110_remote_ssh_publickey_auth_stalled" ) + assert payload["summary"]["ai_loop_current_blocker_registry_v2_ready"] is True + assert payload["summary"]["ai_loop_current_blocker_deployment_closure_state"] == ( + "blocked_latest_visible_cd_failure_after_registry_ready" + ) + assert ( + payload["summary"][ + "ai_loop_current_blocker_deploy_marker_readback_required" + ] + is True + ) + assert payload["summary"]["ai_loop_current_blocker_current_cd_run_status"] == ( + "Failure" + ) + assert ( + payload["summary"]["ai_loop_current_blocker_cd_failed_after_registry_ready"] + is True + ) + assert payload["summary"][ + "ai_loop_current_blocker_harbor_110_repair_failure_classifier" + ] == "harbor_110_remote_ssh_publickey_auth_stalled" assert payload["summary"]["ai_loop_current_blocker_control_path_pressure_blocker"] == "" assert payload["summary"]["ai_loop_current_blocker_node_load_classifier"] == ( "load_not_high" @@ -466,7 +547,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu payload["summary"][ "ai_loop_current_blocker_queue_readback_normalizer_contract_count" ] - == 11 + == 14 ) assert payload["summary"][ "ai_loop_current_blocker_queue_readback_normalizer_field_ids" @@ -483,9 +564,9 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu assert payload["next_execution_order"][0].startswith( "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE" ) - assert "5 ordered local-console phases" in payload["next_execution_order"][0] - assert "Harbor receipt inputs" in payload["next_execution_order"][0] - assert "normalized queue classifiers" in payload["next_execution_order"][0] + assert "already 401" in payload["next_execution_order"][0] + assert "do not reopen Harbor 502" in payload["next_execution_order"][0] + assert "P0-006-CD-DEPLOY-MARKER-READBACK" in payload["next_execution_order"][1] def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift(): diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 2d0625d7f..c884119e3 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,27 @@ +## 2026-07-01 — 16:32 AI Loop registry-ready / deploy-marker blocker separation + +**照主線修正的問題**： +- 最新 live readback 顯示 `https://registry.wooo.work/v2/` 已回 `401`，代表 public registry route 不再是 502 主 blocker；但 Gitea CD `#4258` 仍 `Failure`，Harbor 110 repair `#4255` 仍 `Failure`，分類為 `harbor_110_remote_ssh_publickey_auth_stalled`。 +- `ai-agent-log-intelligence-runtime-sample-readback` 已更新 P0 runtime sample：`registry_v2_status=401`、`registry_v2_ready=true`、`current_cd_run_id=4258`、`current_cd_run_status=Failure`、`harbor_110_repair_run_id=4255`、`harbor_110_repair_run_status=Failure`、`deploy_marker_readback_required=true`。 +- `ai_agent_log_controlled_writeback_plan_readback` / `executor_readback` 新增 deployment closure 欄位：registry ready classifier、current CD status / commit、Harbor repair status、deploy marker readback required、CD failure after registry ready。 +- `awoooi_priority_work_order_readback` 現在把 AI Loop current blocker evidence 分成「registry 已 ready」與「110 SSH / CD deploy marker 未閉合」，`next_execution_order` 明確要求不要重新打開 Harbor 502 作為主線；下一步是 110 SSH/session control-path readback、CD failure closure、deploy marker / production image / priority API verifier。 +- Queue normalizer contract 新增 `current_cd_deploy_marker_closure`、`registry_v2_ready_state`、`harbor_110_repair_failure_after_registry_ready`，讓 KM / RAG / MCP / PlayBook / AI Agent writeback 能學到「401 ready 不是 502 blocker」。 + +**驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost:5432/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_ai_agent_log_controlled_writeback_plan_readback_api.py apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_ai_agent_autonomous_runtime_control.py -q`：`25 passed`。 +- `python3.11 -m ruff check ...`：通過。 +- `python3.11 -m py_compile ...`：通過。 +- `python3.11 -m json.tool docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json`、`git diff --check`：通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`：`GITEA_RUNNER_PRESSURE_GUARD_OK workflow_files=12 scheduled_workflows=4 auto_branch_events_on_110=0 generic_runner_labels=0`。 +- `pnpm -C apps/web typecheck`：通過；本機 `node_modules` 原缺 TypeScript binary，已用 `pnpm install --frozen-lockfile --prefer-offline` 修復，lockfile 未變。 +- i18n mirror check：`en_keys=14748`、`zh_keys=14748`、`missing_in_zh=0`、`missing_in_en=0`。 +- Playwright desktop / mobile smoke 未完成：本機 Chromium runtime 不存在，下載 `playwright install chromium` 時磁碟只剩約 `236MiB`，失敗於 `ENOSPC: no space left on device`；Playwright cache 未留下大檔。 + +**邊界**：未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未重啟主機 / Docker / Nginx / K3s / DB / firewall；本輪只改 API readback、metadata snapshot、測試與 LOGBOOK。 + +**下一步**： +- commit / push 後重新讀 Gitea queue、production deploy marker、Approvals / Runs / Work Items / Alerts，確認頁面不再把 Harbor 502 或人工處理當預設終局；若 production 尚未部署本 patch，繼續追 CD/deploy-marker readback，不宣稱 runtime closure。 + ## 2026-07-01 — 16:05 Gitea CD / ArgoCD post-reboot recovery **照主線修復的問題**： diff --git a/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json b/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json index 7c03b4356..8fbffd9ea 100644 --- a/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json +++ b/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json @@ -81,13 +81,16 @@ "tool": "read_public_gitea_queue_and_remote_control_classifier", "redaction_state": "metadata_only_no_raw_payload", "observed_events": [ - "harbor_registry_public_route_unavailable", + "harbor_registry_public_route_ready_http_401", "harbor_110_remote_control_channel_unavailable", "harbor_110_remote_ssh_reachable", "harbor_110_remote_ssh_publickey_auth_stalled", "harbor_110_remote_ssh_publickey_offer_timeout", + "harbor_110_remote_ssh_server_accepts_key_then_session_timeout", "harbor_110_remote_ssh_auth_permission_denied_false", "harbor_110_repair_jobs_payload_stale_or_cross_workflow", + "current_cd_failure_after_registry_ready", + "deploy_marker_readback_required_after_registry_ready", "bounded_ssh_timeout_seen", "controlled_profile_no_matching_runner_cleared", "node_load_not_high", @@ -99,6 +102,13 @@ "commit_sha", "runner_label", "registry_v2_status", + "registry_v2_ready", + "deployment_closure_state", + "current_cd_run_id", + "current_cd_run_status", + "current_cd_commit_sha", + "harbor_110_repair_run_id", + "harbor_110_repair_run_status", "ssh_auth_classification", "remote_control_channel", "remote_ssh_publickey_auth_stalled", @@ -124,12 +134,20 @@ "remote_ssh_reachable": true, "remote_ssh_publickey_auth_stalled": true, "remote_ssh_publickey_offer_timeout": true, - "remote_ssh_server_accepts_key_then_session_timeout": false, + "remote_ssh_server_accepts_key_then_session_timeout": true, "remote_ssh_auth_permission_denied": false, + "current_cd_run_id": "4258", + "current_cd_run_status": "Failure", + "current_cd_commit_sha": "06819ea96c058e7987811e853242390eaced7f91", + "harbor_110_repair_run_id": "4255", + "harbor_110_repair_run_status": "Failure", "harbor_110_repair_failure_classifier": "harbor_110_remote_ssh_publickey_auth_stalled", "harbor_110_repair_jobs_payload_stale_or_cross_workflow": true, "node_load_classifier": "load_not_high", - "registry_v2_status": 502, + "registry_v2_status": 401, + "registry_v2_ready": true, + "deployment_closure_state": "blocked_latest_visible_cd_failure_after_registry_ready", + "deploy_marker_readback_required": true, "controlled_recovery_package": "recover-110-control-path-and-harbor-local.sh --check", "post_apply_verifier": "check-awoooi-110-controlled-cd-lane-readiness.sh", "controlled_local_console_execution_plan": [ @@ -182,7 +200,7 @@ "docker_system_prune", "workflow_dispatch_from_this_endpoint" ], - "safe_next_step": "run_110_local_console_control_path_recovery_then_rerun_harbor_queue_and_registry_readback" + "safe_next_step": "run_110_ssh_session_control_path_recovery_then_verify_cd_and_deploy_marker_readback" }, "raw_log_payload_persisted": false }