diff --git a/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py b/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py index 5c3a9ae3a..e50c443c0 100644 --- a/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py +++ b/apps/api/src/services/ai_agent_log_controlled_writeback_executor_readback.py @@ -372,12 +372,17 @@ def _current_blocker_queue_item(recovery: dict[str, Any]) -> dict[str, Any]: recovery, control_path_pressure_blocker=control_path_pressure_blocker, ) + queue_status = "controlled_recovery_packaged_waiting_control_path_readback" + if blocker_id == "controlled_cd_lane_guardrails_blocked": + queue_status = ( + "controlled_recovery_packaged_waiting_controlled_cd_lane_guardrails_readback" + ) return { "queue_item_id": f"current-p0-blocker::{blocker_id}", "source_sample_id": source_sample_id, "incident_id": str(recovery.get("incident_id") or ""), "blocker_id": blocker_id, - "status": "controlled_recovery_packaged_waiting_control_path_readback", + "status": queue_status, "risk_tier": str(recovery.get("risk_tier") or "unknown"), "runner_label": str(recovery.get("runner_label") or ""), "node_load_classifier": node_load_classifier, @@ -506,6 +511,32 @@ def _current_blocker_safe_next_action( ], } + if blocker_id == "controlled_cd_lane_guardrails_blocked": + return { + "action_id": ( + "restore_controlled_cd_lane_guardrails_then_rerun_readiness_receipt" + ), + "stage": "controlled_cd_lane_guardrails_receipt_required", + "action": ( + "Verify the 110 controlled CD lane guardrails: config, ELF " + "binary, registration metadata, systemd limits, fail-closed " + "legacy runners, host pressure, and receipt writeback." + ), + "command": "check-awoooi-110-controlled-cd-lane-readiness.sh", + "post_verifier": ( + "harbor-registry-controlled-recovery-receipt validator && " + "priority work-order readback" + ), + "requires_local_console": False, + "blocker_fields": [ + "controlled_cd_lane_config_missing", + "controlled_cd_lane_binary_not_ready", + "controlled_cd_lane_registration_missing", + "controlled_cd_lane_service_not_active", + "controlled_cd_lane_pressure_guard", + ], + } + if control_path_pressure_blocker: return { "action_id": "wait_host_pressure_gate_then_rerun_110_control_path_readback", diff --git a/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py b/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py index ebd3ce2d3..ffbb3d6eb 100644 --- a/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py +++ b/apps/api/src/services/ai_agent_log_controlled_writeback_plan_readback.py @@ -213,9 +213,20 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None: cd_failed_after_registry_ready = bool( registry_v2_ready and current_cd_status in {"Failure", "Blocked", "Canceled"} ) - harbor_repair_failed_after_registry_ready = bool( + historical_harbor_repair_failed_after_registry_ready = bool( registry_v2_ready and harbor_repair_status in {"Failure", "Blocked"} ) + harbor_repair_failed_after_registry_ready = bool( + historical_harbor_repair_failed_after_registry_ready + and blocker_id != "controlled_cd_lane_guardrails_blocked" + ) + ssh_auth_classification = str( + classification.get("ssh_auth_classification") or "" + ) + external_control_path_blocker = "" + if ssh_auth_classification not in {"", "ssh_control_path_ready"}: + external_control_path_blocker = ssh_auth_classification + recovery = { "source_sample_id": source_sample_id, "project_id": str(receipt.get("project_id") or ""), @@ -240,7 +251,9 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None: "deployment_closure_state": _deployment_closure_state( registry_v2_ready=registry_v2_ready, current_cd_status=current_cd_status, - harbor_repair_status=harbor_repair_status, + harbor_repair_failed_after_registry_ready=( + harbor_repair_failed_after_registry_ready + ), fallback=str(classification.get("deployment_closure_state") or ""), ), "deploy_marker_readback_required": bool( @@ -278,9 +291,7 @@ def _current_blocker_recovery(receipt: dict[str, Any]) -> dict[str, Any] | None: "harbor_110_remote_control_channel_unavailable", "harbor_110_remote_ssh_publickey_auth_stalled", }, - "external_control_path_blocker": str( - classification.get("ssh_auth_classification") or "" - ), + "external_control_path_blocker": external_control_path_blocker, "metadata_only": True, } recovery["controlled_local_console_execution_plan"] = _list_of_dicts( @@ -320,12 +331,12 @@ def _deployment_closure_state( *, registry_v2_ready: bool, current_cd_status: str, - harbor_repair_status: str, + harbor_repair_failed_after_registry_ready: bool, fallback: str, ) -> str: if registry_v2_ready and current_cd_status in {"Failure", "Blocked", "Canceled"}: return "blocked_latest_visible_cd_failure_after_registry_ready" - if registry_v2_ready and harbor_repair_status in {"Failure", "Blocked"}: + if registry_v2_ready and harbor_repair_failed_after_registry_ready: return "blocked_harbor_110_repair_failure_after_registry_ready" if registry_v2_ready: return fallback or "registry_ready_waiting_deploy_marker_readback" diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index 9b104c73d..9d7e00649 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -697,9 +697,12 @@ def apply_ai_loop_current_blocker_execution_queue( state["next_executable_mainline_workplan_id"] = ( "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE" ) - state["next_executable_mainline_state"] = ( - "ai_loop_current_blocker_queue_waiting_110_control_path_readback" - ) + next_mainline_state = "ai_loop_current_blocker_queue_waiting_110_control_path_readback" + if blocker_id == "controlled_cd_lane_guardrails_blocked": + next_mainline_state = ( + "ai_loop_current_blocker_queue_waiting_controlled_cd_lane_guardrails_readback" + ) + state["next_executable_mainline_state"] = next_mainline_state state["ai_loop_current_blocker_execution_queue_count"] = len(queue) state["ai_loop_current_blocker_id"] = blocker_id state["ai_loop_current_blocker_registry_v2_ready"] = registry_v2_ready @@ -1002,16 +1005,16 @@ def apply_ai_loop_current_blocker_execution_queue( "The Harbor / 110 blocker is now visible as an AI Loop execution " "queue item with metadata-only KM/RAG/MCP/PlayBook/AI Agent " "writeback targets. The next action is not generic manual triage; " - "it is 110 control-path readback plus the bounded local recovery " - "package, ordered local-console phases, and post-apply verifier." - " Queue readback classifiers are normalized into explicit AI " - "blockers before KM/RAG/MCP/PlayBook writeback." + f"it is {blocker_id} through the bounded recovery package, ordered " + "verifier phases, and post-apply verifier. Queue readback " + "classifiers are normalized into explicit AI blockers before " + "KM/RAG/MCP/PlayBook writeback." ) professional_fix = _dict(workplan.setdefault("professional_fix", {})) professional_fix["action"] = ( "Run the AI Loop current blocker package in order: " f"{', '.join(local_console_phase_ids)}, then rerun " - "Harbor queue / registry /v2/ / deploy-marker readbacks and write " + "controlled CD lane / registry /v2/ / priority readbacks and write " "metadata-only KM/RAG/MCP/PlayBook receipts from normalized queue " "classifier fields. Do not restart Docker " "daemon, reboot hosts, drain nodes, switch registry provider, " @@ -1028,13 +1031,13 @@ def apply_ai_loop_current_blocker_execution_queue( "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE: registry /v2/ " f"is already {first_item.get('registry_v2_status')} " f"({registry_v2_status_classifier}); do not reopen Harbor 502 " - f"as the main blocker. Execute {blocker_id} through 110 " - "SSH/session control-path readback, ordered local-console phases, " - "post-recovery queue readbacks, and metadata-only KM/RAG/MCP/" + f"as the main blocker. Execute {blocker_id} through the " + "controlled CD lane guardrail verifier, registry readbacks, " + "priority work-order readback, and metadata-only KM/RAG/MCP/" "PlayBook writeback." ) ] - if not deploy_marker_resolved_by_production_readback: + if active_deploy_marker_readback_required or active_cd_failed_after_registry_ready: payload["next_execution_order"].append( "P0-006-CD-DEPLOY-MARKER-READBACK: close the latest visible CD " f"{active_current_cd_run_id or 'unknown'} status " @@ -1044,8 +1047,7 @@ def apply_ai_loop_current_blocker_execution_queue( payload["next_execution_order"].append( "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT: only " "rerun Harbor watchdog repair if registry /v2/ regresses below " - "200/401; otherwise keep focus on 110 SSH control-path and " - "deploy-marker closure." + "200/401; otherwise keep focus on controlled CD lane guardrails." ) else: payload["next_execution_order"] = [ @@ -1064,7 +1066,7 @@ def apply_ai_loop_current_blocker_execution_queue( "watchdog check-mode / repair-once and registry /v2/ readback." ) ] - if not deploy_marker_resolved_by_production_readback: + if active_deploy_marker_readback_required or active_cd_failed_after_registry_ready: payload["next_execution_order"].append( "P0-006-CD-DEPLOY-MARKER-READBACK: after registry /v2/ is " "200/401, let the next Gitea CD run build/push/deploy and then " @@ -1279,12 +1281,17 @@ def _ai_loop_current_blocker_can_override( blocker_id: str, queue_item: dict[str, Any], ) -> bool: + ai_loop_blocker_ids = { + "controlled_cd_lane_guardrails_blocked", + } if status == "p0_006_blocked_harbor_registry_controlled_recovery_preflight": return True if status == "p0_006_blocked_stockplatform_public_api_runtime_drift": - return blocker_id.startswith("harbor_110_") - if not blocker_id.startswith("harbor_110_"): + return blocker_id.startswith("harbor_110_") or blocker_id in ai_loop_blocker_ids + if not blocker_id.startswith("harbor_110_") and blocker_id not in ai_loop_blocker_ids: return False + if blocker_id in ai_loop_blocker_ids: + return True return bool( queue_item.get("deploy_marker_readback_required") is True or queue_item.get("cd_failed_after_registry_ready") is True diff --git a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py index fac62a80b..6269b54e0 100644 --- a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py +++ b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py @@ -50,7 +50,7 @@ def _assert_log_controlled_writeback_executor(payload: dict): assert payload["rollups"]["controlled_executor_dispatch_ready"] is True assert payload["rollups"]["controlled_apply_enabled_by_policy"] is True assert payload["rollups"]["current_blocker_execution_queue_count"] == 1 - assert payload["rollups"]["current_blocker_control_path_blocked_count"] == 1 + assert payload["rollups"]["current_blocker_control_path_blocked_count"] == 0 assert ( payload["rollups"]["current_blocker_control_path_pressure_blocked_count"] == 0 @@ -65,34 +65,30 @@ def _assert_log_controlled_writeback_executor(payload: dict): assert len(context["current_blocker_execution_queue"]) == 1 current_queue = context["current_blocker_execution_queue"][0] assert current_queue["queue_item_id"] == ( - "current-p0-blocker::harbor_110_remote_ssh_publickey_auth_stalled" + "current-p0-blocker::controlled_cd_lane_guardrails_blocked" ) assert current_queue["controlled_recovery_package"] == ( - "recover-110-control-path-and-harbor-local.sh --check" + "check-awoooi-110-controlled-cd-lane-readiness.sh" ) assert current_queue["post_apply_verifier"] == ( - "check-awoooi-110-controlled-cd-lane-readiness.sh" + "harbor-registry-controlled-recovery-receipt validator" ) assert [phase["phase_id"] for phase in current_queue[ "controlled_local_console_execution_plan" ]] == [ - "diagnose_remote_control_channel", - "preflight_control_path_and_harbor", - "repair_ssh_service_or_metadata_if_check_confirms_drift", - "repair_harbor_once_if_v2_still_502", - "verify_harbor_queue_and_controlled_cd_lane", + "verify_ssh_control_path_restored", + "verify_harbor_v2_ready", + "verify_controlled_cd_lane_guardrails", ] assert current_queue["post_recovery_readback_commands"] == [ - "read-public-gitea-actions-queue.py --json", + "check-awoooi-110-controlled-cd-lane-readiness.sh", "curl -k https://registry.wooo.work/v2/", "curl http://192.168.0.110:5000/v2/", ] assert "workflow_dispatch_from_this_endpoint" in current_queue[ "forbidden_runtime_actions" ] - assert current_queue["external_control_path_blocker"] == ( - "remote_ssh_publickey_offer_timeout" - ) + assert current_queue["external_control_path_blocker"] == "" assert current_queue["node_load_classifier"] == "load_not_high" assert current_queue["node_load_high"] is False assert current_queue["control_path_pressure_blocker"] == "" @@ -469,7 +465,7 @@ def test_ai_agent_autonomous_runtime_control_exposes_reports_and_executor_receip data["rollups"][ "live_log_controlled_writeback_current_blocker_control_path_blocked_count" ] - == 1 + == 0 ) assert ( data["rollups"][ diff --git a/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py b/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py index 8d4ede009..cbe845fb9 100644 --- a/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py +++ b/apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py @@ -67,7 +67,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert payload["rollups"]["controlled_executor_dispatch_ready"] is True assert payload["rollups"]["controlled_apply_enabled_by_policy"] is True assert payload["rollups"]["current_blocker_execution_queue_count"] == 1 - assert payload["rollups"]["current_blocker_control_path_blocked_count"] == 1 + assert payload["rollups"]["current_blocker_control_path_blocked_count"] == 0 assert ( payload["rollups"]["current_blocker_control_path_pressure_blocked_count"] == 0 @@ -86,7 +86,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert payload["rollups"]["current_blocker_registry_v2_ready_count"] == 1 assert ( payload["rollups"]["current_blocker_deploy_marker_readback_required_count"] - == 1 + == 0 ) assert ( payload["rollups"]["current_blocker_cd_failure_after_registry_ready_count"] @@ -118,7 +118,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): ) assert batch["current_blocker_recovery_count"] == 1 assert batch["current_blocker_recoveries"][0]["blocker_id"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked" ) assert batch["current_blocker_recoveries"][0]["runner_label"] == ( "awoooi-non110-host" @@ -151,10 +151,10 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): current_queue = context["current_blocker_execution_queue"] assert len(current_queue) == 1 assert current_queue[0]["queue_item_id"] == ( - "current-p0-blocker::harbor_110_remote_ssh_publickey_auth_stalled" + "current-p0-blocker::controlled_cd_lane_guardrails_blocked" ) assert current_queue[0]["status"] == ( - "controlled_recovery_packaged_waiting_control_path_readback" + "controlled_recovery_packaged_waiting_controlled_cd_lane_guardrails_readback" ) assert current_queue[0]["runner_label"] == "awoooi-non110-host" assert current_queue[0]["node_load_classifier"] == "load_not_high" @@ -165,20 +165,20 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): "ready_auth_required_http_401" ) assert current_queue[0]["deployment_closure_state"] == ( - "blocked_harbor_110_repair_failure_after_registry_ready" + "blocked_controlled_cd_lane_guardrails_after_ssh_and_harbor_ready" ) - assert current_queue[0]["deploy_marker_readback_required"] is True + assert current_queue[0]["deploy_marker_readback_required"] is False assert current_queue[0]["current_cd_run_id"] == "4293" - assert current_queue[0]["current_cd_run_status"] == "Success" + assert current_queue[0]["current_cd_run_status"] == "production_readback_verified" assert current_queue[0]["current_cd_commit_sha"] == ( "87512d32f5a73849b2d8090092677a745c85f9a8" ) assert current_queue[0]["cd_failed_after_registry_ready"] is False assert current_queue[0]["harbor_110_repair_run_id"] == "4255" assert current_queue[0]["harbor_110_repair_run_status"] == "Failure" - assert current_queue[0]["harbor_110_repair_failed_after_registry_ready"] is True + assert current_queue[0]["harbor_110_repair_failed_after_registry_ready"] is False assert current_queue[0]["harbor_110_repair_failure_classifier"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked_after_ssh_and_harbor_ready" ) assert current_queue[0]["log_source_tag_count"] == 10 source_tags = { @@ -195,7 +195,7 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): "gitea_queue_and_remote_control_classifier_readback" ) assert source_tags["signal_lane"]["tag_value"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked" ) assert all( item["metadata_only"] is True @@ -204,10 +204,10 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): for item in source_tags.values() ) assert current_queue[0]["controlled_recovery_package"] == ( - "recover-110-control-path-and-harbor-local.sh --check" + "check-awoooi-110-controlled-cd-lane-readiness.sh" ) assert current_queue[0]["post_apply_verifier"] == ( - "check-awoooi-110-controlled-cd-lane-readiness.sh" + "harbor-registry-controlled-recovery-receipt validator" ) assert current_queue[0]["harbor_recovery_receipt_endpoint"] == ( "/api/v1/agents/harbor-registry-controlled-recovery-receipt" @@ -234,14 +234,14 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): ] == "awoooi_production_deploy_readback_blocker_v1" assert current_queue[0]["harbor_recovery_receipt_output_contract_count"] == 8 assert current_queue[0]["safe_next_action_id"] == ( - "run_110_local_ssh_session_control_path_recovery_then_verify_cd_and_deploy_marker_readback" + "restore_controlled_cd_lane_guardrails_then_rerun_readiness_receipt" ) assert current_queue[0]["safe_next_action_stage"] == ( - "local_console_control_path_receipt_required" + "controlled_cd_lane_guardrails_receipt_required" ) - assert current_queue[0]["safe_next_action_requires_local_console"] is True + assert current_queue[0]["safe_next_action_requires_local_console"] is False assert current_queue[0]["safe_next_action_command"] == ( - "recover-110-control-path-and-harbor-local.sh --check" + "check-awoooi-110-controlled-cd-lane-readiness.sh" ) assert [ item["output_id"] @@ -303,20 +303,18 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert [phase["phase_id"] for phase in current_queue[0][ "controlled_local_console_execution_plan" ]] == [ - "diagnose_remote_control_channel", - "preflight_control_path_and_harbor", - "repair_ssh_service_or_metadata_if_check_confirms_drift", - "repair_harbor_once_if_v2_still_502", - "verify_harbor_queue_and_controlled_cd_lane", + "verify_ssh_control_path_restored", + "verify_harbor_v2_ready", + "verify_controlled_cd_lane_guardrails", ] assert current_queue[0]["controlled_local_console_execution_plan"][0][ "mode" ] == "read_only" - assert current_queue[0]["controlled_local_console_execution_plan"][3][ + assert current_queue[0]["controlled_local_console_execution_plan"][2][ "command" - ] == "recover-110-control-path-and-harbor-local.sh --repair-harbor-once" + ] == "check-awoooi-110-controlled-cd-lane-readiness.sh" assert current_queue[0]["post_recovery_readback_commands"] == [ - "read-public-gitea-actions-queue.py --json", + "check-awoooi-110-controlled-cd-lane-readiness.sh", "curl -k https://registry.wooo.work/v2/", _expected_local_registry_readback(public_endpoint=public_endpoint), ] @@ -326,12 +324,10 @@ def _assert_executor_readback(payload: dict, *, public_endpoint: bool = False): assert current_queue[0]["runtime_write_gate"] == ( "controlled_after_110_local_console_preflight" ) - assert current_queue[0]["runtime_apply_required_on_110_local_console"] is True - assert current_queue[0]["external_control_path_blocker"] == ( - "remote_ssh_publickey_offer_timeout" - ) + assert current_queue[0]["runtime_apply_required_on_110_local_console"] is False + assert current_queue[0]["external_control_path_blocker"] == "" assert current_queue[0]["control_path_pressure_blocker"] == "" - assert current_queue[0]["control_channel_readback_required"] is True + assert current_queue[0]["control_channel_readback_required"] is False assert set(current_queue[0]["learning_writeback_targets"]) == set(batches) assert len(current_queue[0]["target_batches"]) == 6 assert current_queue[0]["runtime_dispatch_performed"] is False diff --git a/apps/api/tests/test_ai_agent_log_controlled_writeback_plan_readback_api.py b/apps/api/tests/test_ai_agent_log_controlled_writeback_plan_readback_api.py index 90182d6a6..ea948272f 100644 --- a/apps/api/tests/test_ai_agent_log_controlled_writeback_plan_readback_api.py +++ b/apps/api/tests/test_ai_agent_log_controlled_writeback_plan_readback_api.py @@ -124,7 +124,7 @@ def _assert_controlled_writeback_plan(payload: dict, *, public_endpoint: bool = assert {plan["target"] for plan in p0_plans} == set(target_rollups) assert { plan["current_blocker_recovery"]["blocker_id"] for plan in p0_plans - } == {"harbor_110_remote_ssh_publickey_auth_stalled"} + } == {"controlled_cd_lane_guardrails_blocked"} assert { plan["current_blocker_recovery"]["project_id"] for plan in p0_plans } == {"awoooi"} @@ -140,11 +140,11 @@ def _assert_controlled_writeback_plan(payload: dict, *, public_endpoint: bool = assert { plan["current_blocker_recovery"]["controlled_recovery_package"] for plan in p0_plans - } == {"recover-110-control-path-and-harbor-local.sh --check"} + } == {"check-awoooi-110-controlled-cd-lane-readiness.sh"} assert { plan["current_blocker_recovery"]["post_apply_verifier"] for plan in p0_plans - } == {"check-awoooi-110-controlled-cd-lane-readiness.sh"} + } == {"harbor-registry-controlled-recovery-receipt validator"} assert all( [ phase["phase_id"] @@ -153,11 +153,9 @@ def _assert_controlled_writeback_plan(payload: dict, *, public_endpoint: bool = ] ] == [ - "diagnose_remote_control_channel", - "preflight_control_path_and_harbor", - "repair_ssh_service_or_metadata_if_check_confirms_drift", - "repair_harbor_once_if_v2_still_502", - "verify_harbor_queue_and_controlled_cd_lane", + "verify_ssh_control_path_restored", + "verify_harbor_v2_ready", + "verify_controlled_cd_lane_guardrails", ] for plan in p0_plans ) @@ -173,13 +171,13 @@ def _assert_controlled_writeback_plan(payload: dict, *, public_endpoint: bool = ) assert all( plan["current_blocker_recovery"]["runtime_apply_required_on_110_local_console"] - is True + is False for plan in p0_plans ) assert { plan["current_blocker_recovery"]["external_control_path_blocker"] for plan in p0_plans - } == {"remote_ssh_publickey_offer_timeout"} + } == {""} boundaries = payload["operation_boundaries"] assert boundaries["plan_readback_only"] is True @@ -203,7 +201,7 @@ def _expected_post_recovery_readback_commands( else "curl http://192.168.0.110:5000/v2/" ) return [ - "read-public-gitea-actions-queue.py --json", + "check-awoooi-110-controlled-cd-lane-readiness.sh", "curl -k https://registry.wooo.work/v2/", local_registry_readback, ] diff --git a/apps/api/tests/test_ai_agent_log_feedback_receipt_dry_run_api.py b/apps/api/tests/test_ai_agent_log_feedback_receipt_dry_run_api.py index 78b245b9b..ed58aa9c4 100644 --- a/apps/api/tests/test_ai_agent_log_feedback_receipt_dry_run_api.py +++ b/apps/api/tests/test_ai_agent_log_feedback_receipt_dry_run_api.py @@ -93,23 +93,23 @@ def _assert_feedback_dry_run_payload(payload: dict): } assert { receipt["classification"]["current_blocker"] for receipt in p0_receipts - } == {"harbor_110_remote_ssh_publickey_auth_stalled"} + } == {"controlled_cd_lane_guardrails_blocked"} assert { receipt["classification"]["controlled_recovery_package"] for receipt in p0_receipts - } == {"recover-110-control-path-and-harbor-local.sh --check"} + } == {"check-awoooi-110-controlled-cd-lane-readiness.sh"} assert { receipt["classification"]["post_apply_verifier"] for receipt in p0_receipts - } == {"check-awoooi-110-controlled-cd-lane-readiness.sh"} + } == {"harbor-registry-controlled-recovery-receipt validator"} assert all( - len(receipt["classification"]["controlled_local_console_execution_plan"]) == 5 + len(receipt["classification"]["controlled_local_console_execution_plan"]) == 3 for receipt in p0_receipts ) assert all( - receipt["classification"]["controlled_local_console_execution_plan"][3][ + receipt["classification"]["controlled_local_console_execution_plan"][2][ "command" ] - == "recover-110-control-path-and-harbor-local.sh --repair-harbor-once" + == "check-awoooi-110-controlled-cd-lane-readiness.sh" for receipt in p0_receipts ) assert all( diff --git a/apps/api/tests/test_ai_agent_log_intelligence_integration_readback_api.py b/apps/api/tests/test_ai_agent_log_intelligence_integration_readback_api.py index 4e98311a6..1d0d5d864 100644 --- a/apps/api/tests/test_ai_agent_log_intelligence_integration_readback_api.py +++ b/apps/api/tests/test_ai_agent_log_intelligence_integration_readback_api.py @@ -104,54 +104,52 @@ def _assert_log_intelligence_payload(payload: dict): p0_sample = samples["p0_110_harbor_runner_control_path_sample"] assert p0_sample["service"] == "awoooi-110-control-path" assert p0_sample["classification"]["current_blocker"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked" ) assert p0_sample["classification"]["ssh_auth_classification"] == ( - "remote_ssh_publickey_offer_timeout" + "ssh_control_path_ready" ) - assert p0_sample["classification"]["remote_control_channel"] == "unavailable" - assert p0_sample["classification"]["bounded_ssh_timeout_seen"] is True - assert p0_sample["classification"]["remote_ssh_publickey_auth_stalled"] is True + assert p0_sample["classification"]["remote_control_channel"] == "available" + assert p0_sample["classification"]["bounded_ssh_timeout_seen"] is False + assert p0_sample["classification"]["remote_ssh_publickey_auth_stalled"] is False assert ( p0_sample["classification"][ "remote_ssh_publickey_offer_timeout" ] - is True + is False ) assert ( p0_sample["classification"][ "remote_ssh_server_accepts_key_then_session_timeout" ] - is True + is False ) assert p0_sample["classification"]["remote_ssh_auth_permission_denied"] is False assert p0_sample["classification"]["remote_ssh_reachable"] is True assert p0_sample["classification"]["controlled_recovery_package"] == ( - "recover-110-control-path-and-harbor-local.sh --check" + "check-awoooi-110-controlled-cd-lane-readiness.sh" ) assert p0_sample["classification"]["post_apply_verifier"] == ( - "check-awoooi-110-controlled-cd-lane-readiness.sh" + "harbor-registry-controlled-recovery-receipt validator" ) recovery_plan = p0_sample["classification"][ "controlled_local_console_execution_plan" ] assert [phase["phase_id"] for phase in recovery_plan] == [ - "diagnose_remote_control_channel", - "preflight_control_path_and_harbor", - "repair_ssh_service_or_metadata_if_check_confirms_drift", - "repair_harbor_once_if_v2_still_502", - "verify_harbor_queue_and_controlled_cd_lane", + "verify_ssh_control_path_restored", + "verify_harbor_v2_ready", + "verify_controlled_cd_lane_guardrails", ] assert recovery_plan[0]["mode"] == "read_only" assert recovery_plan[0]["expected_result"] == ( - "remote_ssh_publickey_offer_timeout_classifier_without_secret_material" + "ssh_local_repair_control_channel_metadata_ready_and_no_secret_output" ) - assert recovery_plan[2]["mode"] == "controlled_apply" + assert recovery_plan[2]["mode"] == "post_apply_verifier" assert recovery_plan[2]["expected_result"] == ( - "ssh_authorized_keys_permissions_pam_or_account_lookup_path_repaired_without_key_material_read" + "config_binary_registration_service_pressure_guardrails_classified_without_runner_token_read" ) - assert recovery_plan[2]["runtime_write_allowed_only_on_110_local_console"] is True - assert "read-public-gitea-actions-queue.py --json" in p0_sample[ + assert recovery_plan[2]["runtime_write_allowed_only_on_110_local_console"] is False + assert "check-awoooi-110-controlled-cd-lane-readiness.sh" in p0_sample[ "classification" ]["post_recovery_readback_commands"] assert "read_runner_registration_token_or_runner_file" in p0_sample[ diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 469de8df7..254f813f8 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -126,7 +126,7 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot( ) assert data["mainline_execution_state"][ "ai_loop_current_blocker_deploy_marker_readback_required" - ] is True + ] is False assert data["mainline_execution_state"]["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready" assert data["mainline_execution_state"]["reboot_drill_preflight_runtime_readback_state"] == "blocked" assert data["rollups"]["stockplatform_public_api_runtime_ready"] is True @@ -173,13 +173,13 @@ def test_awoooi_priority_work_order_readback_endpoint_redacts_ai_loop_queue( "p0_006_blocked_ai_loop_current_blocker_execution_queue" ) assert evidence["ai_loop_current_blocker_post_recovery_readback_commands"] == [ - "read-public-gitea-actions-queue.py --json", + "check-awoooi-110-controlled-cd-lane-readiness.sh", "curl -k https://registry.wooo.work/v2/", "curl host:public-gateway/registry/v2/", ] assert len( evidence["ai_loop_current_blocker_local_console_execution_plan"] - ) == 5 + ) == 3 assert "read_runner_registration_token_or_runner_file" in evidence[ "ai_loop_current_blocker_forbidden_runtime_actions" ] @@ -267,11 +267,9 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu ) assert state["ai_loop_current_blocker_execution_queue_count"] == 1 assert state["ai_loop_current_blocker_id"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" - ) - assert state["ai_loop_current_blocker_control_path_blocker"] == ( - "remote_ssh_publickey_offer_timeout" + "controlled_cd_lane_guardrails_blocked" ) + assert state["ai_loop_current_blocker_control_path_blocker"] == "" assert state["ai_loop_current_blocker_control_path_pressure_blocker"] == "" assert state["ai_loop_current_blocker_node_load_classifier"] == "load_not_high" assert state["ai_loop_current_blocker_registry_v2_ready"] is True @@ -279,22 +277,25 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "ready_auth_required_http_401" ) assert state["ai_loop_current_blocker_deployment_closure_state"] == ( - "blocked_harbor_110_repair_failure_after_registry_ready" + "blocked_controlled_cd_lane_guardrails_after_ssh_and_harbor_ready" ) - assert state["ai_loop_current_blocker_deploy_marker_readback_required"] is True + assert state["ai_loop_current_blocker_deploy_marker_readback_required"] is False assert state["ai_loop_current_blocker_current_cd_run_id"] == "4293" - assert state["ai_loop_current_blocker_current_cd_run_status"] == "Success" + assert ( + state["ai_loop_current_blocker_current_cd_run_status"] + == "production_readback_verified" + ) assert state["ai_loop_current_blocker_cd_failed_after_registry_ready"] is False assert state["ai_loop_current_blocker_harbor_110_repair_run_id"] == "4255" assert state["ai_loop_current_blocker_harbor_110_repair_run_status"] == ( "Failure" ) assert state["ai_loop_current_blocker_harbor_110_repair_failure_classifier"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked_after_ssh_and_harbor_ready" ) assert evidence["ai_loop_current_blocker_execution_queue_count"] == 1 assert evidence["ai_loop_current_blocker_id"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked" ) assert evidence["ai_loop_current_blocker_runner_label"] == "awoooi-non110-host" assert evidence["ai_loop_current_blocker_registry_v2_status"] == 401 @@ -303,14 +304,17 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "ready_auth_required_http_401" ) assert evidence["ai_loop_current_blocker_deployment_closure_state"] == ( - "blocked_harbor_110_repair_failure_after_registry_ready" + "blocked_controlled_cd_lane_guardrails_after_ssh_and_harbor_ready" ) assert ( evidence["ai_loop_current_blocker_deploy_marker_readback_required"] - is True + is False ) assert evidence["ai_loop_current_blocker_current_cd_run_id"] == "4293" - assert evidence["ai_loop_current_blocker_current_cd_run_status"] == "Success" + assert ( + evidence["ai_loop_current_blocker_current_cd_run_status"] + == "production_readback_verified" + ) assert ( evidence["ai_loop_current_blocker_current_cd_commit_sha"] == "87512d32f5a73849b2d8090092677a745c85f9a8" @@ -321,26 +325,24 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "Failure" ) assert evidence["ai_loop_current_blocker_harbor_110_repair_failure_classifier"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked_after_ssh_and_harbor_ready" ) assert ( evidence[ "ai_loop_current_blocker_harbor_110_repair_failed_after_registry_ready" ] - is True + is False ) assert evidence["ai_loop_current_blocker_controlled_recovery_package"] == ( - "recover-110-control-path-and-harbor-local.sh --check" + "check-awoooi-110-controlled-cd-lane-readiness.sh" ) assert evidence["ai_loop_current_blocker_post_apply_verifier"] == ( - "check-awoooi-110-controlled-cd-lane-readiness.sh" + "harbor-registry-controlled-recovery-receipt validator" ) assert evidence["ai_loop_current_blocker_runtime_write_gate"] == ( "controlled_after_110_local_console_preflight" ) - assert evidence["ai_loop_current_blocker_control_path_blocker"] == ( - "remote_ssh_publickey_offer_timeout" - ) + assert evidence["ai_loop_current_blocker_control_path_blocker"] == "" assert evidence["ai_loop_current_blocker_control_path_pressure_blocker"] == "" assert evidence["ai_loop_current_blocker_node_load_classifier"] == "load_not_high" assert set(evidence["ai_loop_current_blocker_learning_writeback_targets"]) == { @@ -352,21 +354,19 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "ai_agent", } assert len(evidence["ai_loop_current_blocker_target_batches"]) == 6 - assert evidence["ai_loop_current_blocker_local_console_phase_count"] == 5 + assert evidence["ai_loop_current_blocker_local_console_phase_count"] == 3 assert evidence["ai_loop_current_blocker_local_console_phase_ids"] == [ - "diagnose_remote_control_channel", - "preflight_control_path_and_harbor", - "repair_ssh_service_or_metadata_if_check_confirms_drift", - "repair_harbor_once_if_v2_still_502", - "verify_harbor_queue_and_controlled_cd_lane", + "verify_ssh_control_path_restored", + "verify_harbor_v2_ready", + "verify_controlled_cd_lane_guardrails", ] assert evidence["ai_loop_current_blocker_local_console_execution_plan"][0][ "mode" ] == "read_only" - assert evidence["ai_loop_current_blocker_local_console_execution_plan"][3][ + assert evidence["ai_loop_current_blocker_local_console_execution_plan"][2][ "command" - ] == "recover-110-control-path-and-harbor-local.sh --repair-harbor-once" - assert "read-public-gitea-actions-queue.py --json" in evidence[ + ] == "check-awoooi-110-controlled-cd-lane-readiness.sh" + assert "check-awoooi-110-controlled-cd-lane-readiness.sh" in evidence[ "ai_loop_current_blocker_post_recovery_readback_commands" ] assert "restart_docker_daemon" in evidence[ @@ -376,7 +376,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu assert evidence["ai_loop_current_blocker_runtime_dispatch_performed"] is False assert evidence["ai_loop_current_blocker_raw_payload_required"] is False assert evidence["ai_loop_current_blocker_post_recovery_readback_commands"] == [ - "read-public-gitea-actions-queue.py --json", + "check-awoooi-110-controlled-cd-lane-readiness.sh", "curl -k https://registry.wooo.work/v2/", "curl http://192.168.0.110:5000/v2/", ] @@ -420,19 +420,19 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu 0 ]["writeback_targets"] == ["km", "rag", "playbook", "mcp", "verifier", "ai_agent"] assert evidence["ai_loop_current_blocker_safe_next_action_id"] == ( - "run_110_local_ssh_session_control_path_recovery_then_verify_cd_and_deploy_marker_readback" + "restore_controlled_cd_lane_guardrails_then_rerun_readiness_receipt" ) assert evidence["ai_loop_current_blocker_safe_next_action_stage"] == ( - "local_console_control_path_receipt_required" + "controlled_cd_lane_guardrails_receipt_required" ) assert ( evidence[ "ai_loop_current_blocker_safe_next_action_requires_local_console" ] - is True + is False ) assert evidence["ai_loop_current_blocker_safe_next_action_command"] == ( - "recover-110-control-path-and-harbor-local.sh --check" + "check-awoooi-110-controlled-cd-lane-readiness.sh" ) assert evidence["ai_loop_current_blocker_queue_readback_normalizer_field_ids"] == [ "cd_run_jobs_payload_classifier", @@ -480,7 +480,7 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu "read_public_gitea_queue_and_remote_control_classifier" ) assert source_tags["signal_lane"]["tag_value"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked" ) assert evidence["ai_loop_log_source_grouping_keys"] == [ "project_id", @@ -508,20 +508,20 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu assert "Do not restart Docker daemon" in in_progress["professional_fix"]["action"] assert payload["summary"]["ai_loop_current_blocker_execution_queue_count"] == 1 assert payload["summary"]["ai_loop_current_blocker_id"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked" ) assert payload["summary"]["ai_loop_current_blocker_registry_v2_ready"] is True assert payload["summary"]["ai_loop_current_blocker_deployment_closure_state"] == ( - "blocked_harbor_110_repair_failure_after_registry_ready" + "blocked_controlled_cd_lane_guardrails_after_ssh_and_harbor_ready" ) assert ( payload["summary"][ "ai_loop_current_blocker_deploy_marker_readback_required" ] - is True + is False ) assert payload["summary"]["ai_loop_current_blocker_current_cd_run_status"] == ( - "Success" + "production_readback_verified" ) assert ( payload["summary"]["ai_loop_current_blocker_cd_failed_after_registry_ready"] @@ -529,24 +529,24 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu ) assert payload["summary"][ "ai_loop_current_blocker_harbor_110_repair_failure_classifier" - ] == "harbor_110_remote_ssh_publickey_auth_stalled" + ] == "controlled_cd_lane_guardrails_blocked_after_ssh_and_harbor_ready" assert payload["summary"]["ai_loop_current_blocker_control_path_pressure_blocker"] == "" assert payload["summary"]["ai_loop_current_blocker_node_load_classifier"] == ( "load_not_high" ) - assert payload["summary"]["ai_loop_current_blocker_local_console_phase_count"] == 5 + assert payload["summary"]["ai_loop_current_blocker_local_console_phase_count"] == 3 assert payload["summary"]["ai_loop_current_blocker_safe_next_action_id"] == ( - "run_110_local_ssh_session_control_path_recovery_then_verify_cd_and_deploy_marker_readback" + "restore_controlled_cd_lane_guardrails_then_rerun_readiness_receipt" ) assert payload["summary"]["ai_loop_current_blocker_safe_next_action_command"] == ( - "recover-110-control-path-and-harbor-local.sh --check" + "check-awoooi-110-controlled-cd-lane-readiness.sh" ) - assert "read-public-gitea-actions-queue.py --json" in payload["summary"][ + assert "harbor-registry-controlled-recovery-receipt validator" in payload["summary"][ "ai_loop_current_blocker_safe_next_action_post_verifier" ] assert payload["summary"][ "ai_loop_current_blocker_safe_next_action_requires_local_console" - ] is True + ] is False assert payload["summary"][ "ai_loop_current_blocker_post_recovery_readback_command_count" ] == 3 @@ -594,7 +594,9 @@ def test_awoooi_priority_work_order_readback_overlays_ai_loop_current_blocker_qu ) assert "already 401" in payload["next_execution_order"][0] assert "do not reopen Harbor 502" in payload["next_execution_order"][0] - assert "P0-006-CD-DEPLOY-MARKER-READBACK" in payload["next_execution_order"][1] + assert "P0-006-HARBOR-REGISTRY-CONTROLLED-RECOVERY-PREFLIGHT" in payload[ + "next_execution_order" + ][1] def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift(): @@ -648,7 +650,7 @@ def test_awoooi_priority_work_order_readback_overlays_live_stockplatform_drift() ) -def test_awoooi_priority_work_order_ai_loop_overrides_stockplatform_drift_for_harbor_110_blocker(): +def test_awoooi_priority_work_order_ai_loop_overrides_stockplatform_drift_for_current_ai_loop_blocker(): payload = load_latest_awoooi_priority_work_order_readback() executor = load_latest_ai_agent_log_controlled_writeback_executor_readback() @@ -670,13 +672,11 @@ def test_awoooi_priority_work_order_ai_loop_overrides_stockplatform_drift_for_ha ) assert state["stockplatform_public_api_live_readback_state"] == "blocked" assert state["ai_loop_current_blocker_id"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" - ) - assert state["ai_loop_current_blocker_control_path_blocker"] == ( - "remote_ssh_publickey_offer_timeout" + "controlled_cd_lane_guardrails_blocked" ) + assert state["ai_loop_current_blocker_control_path_blocker"] == "" assert payload["summary"]["ai_loop_current_blocker_id"] == ( - "harbor_110_remote_ssh_publickey_auth_stalled" + "controlled_cd_lane_guardrails_blocked" ) assert payload["next_execution_order"][0].startswith( "P0-006-AI-LOOP-CURRENT-BLOCKER-EXECUTION-QUEUE" @@ -861,9 +861,7 @@ def test_awoooi_priority_work_order_readback_does_not_reopen_stale_cd_failure_af f"production_readback:{runtime_short_sha}" ) assert state["ai_loop_current_blocker_control_path_blocker"] == "" - assert state["ai_loop_current_blocker_historical_control_path_blocker"] == ( - "remote_ssh_publickey_offer_timeout" - ) + assert state["ai_loop_current_blocker_historical_control_path_blocker"] == "" assert state["ai_loop_current_blocker_historical_current_cd_run_id"] == "4258" assert state["ai_loop_current_blocker_historical_current_cd_run_status"] == ( "Failure" @@ -875,9 +873,7 @@ def test_awoooi_priority_work_order_readback_does_not_reopen_stale_cd_failure_af assert evidence["ai_loop_current_blocker_deploy_marker_readback_required"] is False assert evidence["ai_loop_current_blocker_cd_failed_after_registry_ready"] is False assert evidence["ai_loop_current_blocker_control_path_blocker"] == "" - assert evidence["ai_loop_current_blocker_historical_control_path_blocker"] == ( - "remote_ssh_publickey_offer_timeout" - ) + assert evidence["ai_loop_current_blocker_historical_control_path_blocker"] == "" assert evidence[ "ai_loop_current_blocker_historical_deploy_marker_readback_required" ] is True @@ -893,7 +889,7 @@ def test_awoooi_priority_work_order_readback_does_not_reopen_stale_cd_failure_af assert payload["summary"]["ai_loop_current_blocker_control_path_blocker"] == "" assert payload["summary"][ "ai_loop_current_blocker_historical_control_path_blocker" - ] == "remote_ssh_publickey_offer_timeout" + ] == "" assert all( "P0-006-CD-DEPLOY-MARKER-READBACK" not in item for item in payload["next_execution_order"] diff --git a/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json b/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json index b65425cdd..f8a72da74 100644 --- a/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json +++ b/docs/operations/ai-agent-log-intelligence-runtime-sample-readback.snapshot.json @@ -82,17 +82,20 @@ "redaction_state": "metadata_only_no_raw_payload", "observed_events": [ "harbor_registry_public_route_ready_http_401", - "harbor_110_remote_control_channel_unavailable", + "harbor_110_remote_control_channel_restored", "harbor_110_remote_ssh_reachable", - "harbor_110_remote_ssh_publickey_auth_stalled", - "harbor_110_remote_ssh_publickey_offer_timeout", - "harbor_110_remote_ssh_server_accepts_key_then_session_timeout", + "harbor_110_recovery_package_installed", + "harbor_watchdog_check_mode_packaged", + "harbor_local_v2_ready_http_401", "harbor_110_remote_ssh_auth_permission_denied_false", - "harbor_110_repair_jobs_payload_stale_or_cross_workflow", - "current_cd_failure_after_registry_ready", - "deploy_marker_readback_required_after_registry_ready", - "bounded_ssh_timeout_seen", - "controlled_profile_no_matching_runner_cleared", + "controlled_cd_lane_config_missing", + "controlled_cd_lane_binary_not_ready", + "controlled_cd_lane_registration_missing", + "controlled_cd_lane_service_masked_inactive", + "controlled_cd_lane_heavy_processes_present", + "current_cd_deploy_marker_resolved_by_production_readback", + "bounded_ssh_check_succeeded", + "controlled_profile_guardrails_blocked", "node_load_not_high", "controlled_lane_verifier_packaged", "local_recovery_package_packaged" @@ -114,6 +117,10 @@ "remote_ssh_publickey_auth_stalled", "remote_ssh_publickey_offer_timeout", "remote_ssh_auth_permission_denied", + "ssh_local_repair_control_channel_metadata_ready", + "watchdog_check_harbor_ready", + "controlled_cd_lane_blocker_count", + "controlled_cd_lane_safe_next_step", "harbor_110_repair_failure_classifier", "harbor_110_repair_jobs_payload_classifier", "bounded_ssh_timeout_seen", @@ -124,71 +131,63 @@ "rollback_ref" ], "classification": { - "incident_id": "p0-110-harbor-ssh-publickey-auth-stalled-20260701", + "incident_id": "p0-110-controlled-cd-lane-guardrails-20260701", "risk_tier": "high", "runner_label": "awoooi-non110-host", - "current_blocker": "harbor_110_remote_ssh_publickey_auth_stalled", - "ssh_auth_classification": "remote_ssh_publickey_offer_timeout", - "remote_control_channel": "unavailable", - "bounded_ssh_timeout_seen": true, + "current_blocker": "controlled_cd_lane_guardrails_blocked", + "ssh_auth_classification": "ssh_control_path_ready", + "remote_control_channel": "available", + "bounded_ssh_timeout_seen": false, "remote_ssh_reachable": true, - "remote_ssh_publickey_auth_stalled": true, - "remote_ssh_publickey_offer_timeout": true, - "remote_ssh_server_accepts_key_then_session_timeout": true, + "remote_ssh_publickey_auth_stalled": false, + "remote_ssh_publickey_offer_timeout": false, + "remote_ssh_server_accepts_key_then_session_timeout": false, "remote_ssh_auth_permission_denied": false, + "ssh_local_repair_control_channel_metadata_ready": true, + "ssh_local_repair_account_metadata_ready": true, + "watchdog_check_harbor_ready": true, + "controlled_cd_lane_ready": false, + "controlled_cd_lane_blocker_count": 8, + "controlled_cd_lane_safe_next_step": "restore_or_register_awoooi_cd_lane_drain_registration_without_printing_token_then_rerun_this_verifier", "current_cd_run_id": "4293", - "current_cd_run_status": "Success", + "current_cd_run_status": "production_readback_verified", "current_cd_commit_sha": "87512d32f5a73849b2d8090092677a745c85f9a8", "harbor_110_repair_run_id": "4255", "harbor_110_repair_run_status": "Failure", - "harbor_110_repair_failure_classifier": "harbor_110_remote_ssh_publickey_auth_stalled", + "harbor_110_repair_failure_classifier": "controlled_cd_lane_guardrails_blocked_after_ssh_and_harbor_ready", "harbor_110_repair_jobs_payload_stale_or_cross_workflow": true, "node_load_classifier": "load_not_high", "registry_v2_status": 401, "registry_v2_ready": true, - "deployment_closure_state": "blocked_harbor_110_repair_failure_after_registry_ready", - "deploy_marker_readback_required": true, - "controlled_recovery_package": "recover-110-control-path-and-harbor-local.sh --check", - "post_apply_verifier": "check-awoooi-110-controlled-cd-lane-readiness.sh", + "deployment_closure_state": "blocked_controlled_cd_lane_guardrails_after_ssh_and_harbor_ready", + "deploy_marker_readback_required": false, + "controlled_recovery_package": "check-awoooi-110-controlled-cd-lane-readiness.sh", + "post_apply_verifier": "harbor-registry-controlled-recovery-receipt validator", "controlled_local_console_execution_plan": [ { - "phase_id": "diagnose_remote_control_channel", + "phase_id": "verify_ssh_control_path_restored", "mode": "read_only", - "command": "read-public-gitea-actions-queue.py --json", - "expected_result": "remote_ssh_publickey_offer_timeout_classifier_without_secret_material", - "runtime_write_allowed_only_on_110_local_console": false - }, - { - "phase_id": "preflight_control_path_and_harbor", - "mode": "check", "command": "recover-110-control-path-and-harbor-local.sh --check", - "expected_result": "ssh_metadata_harbor_watchdog_and_cd_lane_readiness_readback", + "expected_result": "ssh_local_repair_control_channel_metadata_ready_and_no_secret_output", "runtime_write_allowed_only_on_110_local_console": false }, { - "phase_id": "repair_ssh_service_or_metadata_if_check_confirms_drift", - "mode": "controlled_apply", - "command": "recover-110-control-path-and-harbor-local.sh --apply-ssh-control-path", - "expected_result": "ssh_authorized_keys_permissions_pam_or_account_lookup_path_repaired_without_key_material_read", - "runtime_write_allowed_only_on_110_local_console": true + "phase_id": "verify_harbor_v2_ready", + "mode": "check", + "command": "harbor-watchdog.sh --check", + "expected_result": "public_and_internal_registry_v2_return_200_or_401", + "runtime_write_allowed_only_on_110_local_console": false }, { - "phase_id": "repair_harbor_once_if_v2_still_502", - "mode": "controlled_apply", - "command": "recover-110-control-path-and-harbor-local.sh --repair-harbor-once", - "expected_result": "harbor_local_v2_returns_200_or_401_after_single_bounded_repair", - "runtime_write_allowed_only_on_110_local_console": true - }, - { - "phase_id": "verify_harbor_queue_and_controlled_cd_lane", + "phase_id": "verify_controlled_cd_lane_guardrails", "mode": "post_apply_verifier", - "command": "read-public-gitea-actions-queue.py --json && check-awoooi-110-controlled-cd-lane-readiness.sh", - "expected_result": "remote_ssh_auth_stall_cleared_harbor_v2_ready_without_generic_labels_or_secret_reads", + "command": "check-awoooi-110-controlled-cd-lane-readiness.sh", + "expected_result": "config_binary_registration_service_pressure_guardrails_classified_without_runner_token_read", "runtime_write_allowed_only_on_110_local_console": false } ], "post_recovery_readback_commands": [ - "read-public-gitea-actions-queue.py --json", + "check-awoooi-110-controlled-cd-lane-readiness.sh", "curl -k https://registry.wooo.work/v2/", "curl http://192.168.0.110:5000/v2/" ], @@ -200,7 +199,7 @@ "docker_system_prune", "workflow_dispatch_from_this_endpoint" ], - "safe_next_step": "run_110_ssh_session_control_path_recovery_then_verify_cd_and_deploy_marker_readback" + "safe_next_step": "restore_or_register_awoooi_cd_lane_drain_registration_without_printing_token_then_rerun_this_verifier" }, "raw_log_payload_persisted": false }