#!/usr/bin/env python3 from __future__ import annotations import importlib.util import json import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "ops/runner/verify-awoooi-non110-cd-closure.py" def _load_module(): spec = importlib.util.spec_from_file_location( "verify_awoooi_non110_cd_closure", SCRIPT, ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _queue( *, no_matching: bool, harbor_110_no_matching: bool = False, harbor_110_remote_control_unavailable: bool = False, harbor_110_publickey_auth_stalled: bool = False, harbor_110_publickey_offer_timeout: bool = False, harbor_110_session_timeout: bool = False, harbor_110_command_path_ready: bool = False, harbor_110_running_jobs_api_stale: bool = False, current_cd_waiting_behind_harbor_110_running: bool = False, ) -> dict: status = ( "blocked_harbor_110_remote_ssh_publickey_auth_stalled" if harbor_110_publickey_auth_stalled else "blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback" if harbor_110_running_jobs_api_stale else "blocked_current_cd_waiting_behind_harbor_110_repair_running" if current_cd_waiting_behind_harbor_110_running else "blocked_harbor_110_remote_control_channel_unavailable" if harbor_110_remote_control_unavailable else "blocked_harbor_110_repair_no_matching_runner" if harbor_110_no_matching else "blocked_no_matching_online_runner" if no_matching else "no_matching_runner_not_visible" ) return { "schema_version": "awoooi_public_gitea_actions_queue_readback_v1", "status": status, "readback": { "no_matching_online_runner_visible": no_matching, "latest_visible_no_matching_runner_label": ( "awoooi-host" if harbor_110_no_matching else "awoooi-non110-ubuntu" if no_matching else "" ), "latest_visible_harbor_110_repair_no_matching_runner_label": ( "awoooi-host" if harbor_110_no_matching else "" ), "latest_visible_harbor_110_repair_remote_control_channel_unavailable": ( harbor_110_remote_control_unavailable or harbor_110_publickey_auth_stalled ), "latest_visible_harbor_110_repair_remote_ssh_publickey_auth_stalled": ( harbor_110_publickey_auth_stalled ), "latest_visible_harbor_110_repair_remote_ssh_command_path_ready": ( harbor_110_command_path_ready ), "latest_visible_harbor_110_repair_remote_ssh_publickey_reply_timeout_seen": ( harbor_110_publickey_auth_stalled ), "latest_visible_harbor_110_repair_remote_ssh_publickey_offer_timeout": ( harbor_110_publickey_offer_timeout ), "latest_visible_harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout": ( harbor_110_session_timeout ), "harbor_110_repair_visible_running_jobs_api_stale": ( harbor_110_running_jobs_api_stale ), "current_cd_waiting_behind_harbor_110_repair_running": ( current_cd_waiting_behind_harbor_110_running or harbor_110_running_jobs_api_stale ), }, "rollups": { "harbor_110_repair_remote_control_channel_unavailable": ( harbor_110_remote_control_unavailable or harbor_110_publickey_auth_stalled ), "harbor_110_repair_remote_ssh_publickey_auth_stalled": ( harbor_110_publickey_auth_stalled ), "harbor_110_repair_remote_ssh_command_path_ready": ( harbor_110_command_path_ready ), "harbor_110_repair_remote_ssh_publickey_reply_timeout_seen": ( harbor_110_publickey_auth_stalled ), "harbor_110_repair_remote_ssh_publickey_offer_timeout": ( harbor_110_publickey_offer_timeout ), "harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout": ( harbor_110_session_timeout ), "harbor_110_repair_visible_running_jobs_api_stale": ( harbor_110_running_jobs_api_stale ), "current_cd_waiting_behind_harbor_110_repair_running": ( current_cd_waiting_behind_harbor_110_running or harbor_110_running_jobs_api_stale ), }, "operation_boundaries": { "workflow_dispatch_performed": False, "secret_or_runner_token_read": False, "github_api_used": False, }, } def _workbench( *, image_current: bool, governance_ready: bool, runtime_source_current: bool | None = None, runtime_image_current: bool | None = None, ) -> dict: runtime_source_current = ( image_current if runtime_source_current is None else runtime_source_current ) runtime_image_current = ( image_current if runtime_image_current is None else runtime_image_current ) return { "schema_version": "delivery_closure_workbench_v1", "summary": { "source_count": 6, "production_deploy_image_tag_matches_main": image_current, "production_deploy_runtime_build_matches_committed_source_control_readback": ( runtime_source_current ), "production_deploy_runtime_build_matches_committed_production_image_tag": ( runtime_image_current ), "production_deploy_runtime_build_readback_status": ( "matches_committed_deploy_readback" if runtime_source_current and runtime_image_current else "runtime_build_diverges_from_committed_deploy_readback" ), "production_deploy_governance_fields_present": governance_ready, }, } def _deploy_snapshot() -> dict: return { "schema_version": "awoooi_production_deploy_readback_blocker_v1", "readback": { "non110_runner_ready": False, "non110_runner_ready_config_count": 1, "non110_runner_ready_binary_count": 1, "non110_runner_ready_registration_count": 0, "non110_runner_ready_service_count": 1, "non110_runner_ready_active_service_count": 0, "non110_runner_ready_autostart_path_count": 1, "non110_runner_remaining_blockers": [ "runner_registration_missing", "runner_service_not_active", ], "non110_runner_safe_next_step": "run_register_awoooi_non110_runner_script_without_printing_token_then_autostart_path_will_enable_service_and_rerun_this_verifier", }, } def _readiness(*, ready: bool) -> str: if ready: return "\n".join( [ "raw_runner_registration_read=false", "READY_CONFIG_COUNT=1", "READY_BINARY_COUNT=1", "READY_REGISTRATION_COUNT=1", "READY_SERVICE_COUNT=1", "READY_ACTIVE_SERVICE_COUNT=1", "READY_AUTOSTART_PATH_COUNT=1", "AWOOOI_NON110_RUNNER_READY=1", ] ) return "\n".join( [ "raw_runner_registration_read=false", "BLOCKER runner_registration_missing", "BLOCKER runner_service_not_active", "READY_CONFIG_COUNT=1", "READY_BINARY_COUNT=1", "READY_REGISTRATION_COUNT=0", "READY_SERVICE_COUNT=1", "READY_ACTIVE_SERVICE_COUNT=0", "READY_AUTOSTART_PATH_COUNT=1", "AWOOOI_NON110_RUNNER_READY=0", "safe_next_step=run_register_awoooi_non110_runner_script_without_printing_token_then_autostart_path_will_enable_service_and_rerun_this_verifier", ] ) def test_closure_verifier_blocks_runner_not_ready_without_secret_leak() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=False), queue=_queue(no_matching=True), production_workbench=_workbench(image_current=False, governance_ready=False), ) text = json.dumps(payload, sort_keys=True) assert payload["schema_version"] == module.SCHEMA_VERSION assert payload["status"] == "blocked_non110_runner_not_ready" assert payload["progress"]["ordered_step_count"] == 6 assert payload["progress"]["ordered_completed_prefix_count"] == 0 assert payload["progress"]["next_blocked_step_index"] == 1 assert payload["progress"]["next_blocked_step_id"] == ( "non110_runner_registration_metadata" ) assert payload["ordered_steps"][0]["status"] == "blocked" assert payload["ordered_steps"][0]["evidence_ready"] is False assert payload["ordered_steps"][1]["status"] == "pending" assert "runner_registration_missing" in payload["runner_readiness_blockers"] assert payload["readback"]["non110_runner_ready_registration_count"] == 0 assert payload["operation_boundaries"]["secret_or_runner_token_read"] is False assert payload["operation_boundaries"]["raw_runner_registration_read"] is False assert "secret-token-like-content" not in text def test_closure_verifier_blocks_queue_after_runner_ready() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue(no_matching=True), production_workbench=_workbench(image_current=False, governance_ready=False), ) assert payload["status"] == "blocked_no_matching_online_runner" assert payload["progress"]["ordered_completed_prefix_count"] == 2 assert payload["progress"]["next_blocked_step_index"] == 3 assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match" assert payload["ordered_steps"][0]["status"] == "complete" assert payload["ordered_steps"][1]["status"] == "complete" assert payload["ordered_steps"][2]["status"] == "blocked" assert "public_queue_still_has_no_matching_online_runner" in payload["blockers"] def test_closure_verifier_prioritizes_harbor_110_runner_label_blocker() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue(no_matching=True, harbor_110_no_matching=True), production_workbench=_workbench(image_current=False, governance_ready=False), ) assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner" assert "harbor_110_repair_no_matching_runner" in payload["blockers"] assert "public_queue_still_has_no_matching_online_runner" not in payload[ "blockers" ] assert payload["readback"]["harbor_110_repair_no_matching_runner"] is True assert ( payload["readback"]["harbor_110_repair_no_matching_runner_label"] == "awoooi-host" ) assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match" assert "check_awoooi_110_controlled_cd_lane_readiness" in payload[ "next_actions" ][0] assert "awoooi_host_runner_control_path" in payload["next_actions"][0] assert "check_awoooi_110_controlled_cd_lane_readiness" in payload["progress"][ "next_blocked_step_action" ] assert "awoooi_host_runner_control_path" in payload["progress"][ "next_blocked_step_action" ] def test_closure_verifier_blocks_harbor_110_publickey_auth_stalled() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue( no_matching=False, harbor_110_publickey_auth_stalled=True, harbor_110_publickey_offer_timeout=True, harbor_110_session_timeout=True, ), production_workbench=_workbench(image_current=True, governance_ready=True), ) assert payload["status"] == "blocked_harbor_110_remote_ssh_publickey_auth_stalled" assert "harbor_110_remote_ssh_publickey_auth_stalled" in payload["blockers"] assert payload["readback"]["harbor_110_remote_control_channel_unavailable"] is True assert payload["readback"]["harbor_110_remote_ssh_publickey_auth_stalled"] is True assert ( payload["readback"]["harbor_110_remote_ssh_publickey_reply_timeout_seen"] is True ) assert ( payload["readback"]["harbor_110_remote_ssh_publickey_offer_timeout"] is True ) assert ( payload["readback"][ "harbor_110_remote_ssh_server_accepts_key_then_session_timeout" ] is True ) assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match" assert payload["ordered_steps"][2]["status"] == "blocked" assert "repair_110_ssh_publickey_auth_local_check" in payload["next_actions"][0] assert "harbor_v2_readback" in payload["next_actions"][0] assert "repair_110_ssh_publickey_auth_local_check" in payload["progress"][ "next_blocked_step_action" ] def test_closure_verifier_command_path_ready_overrides_publickey_auth_stalled() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue( no_matching=False, harbor_110_publickey_auth_stalled=True, harbor_110_publickey_offer_timeout=True, harbor_110_session_timeout=True, harbor_110_command_path_ready=True, ), production_workbench=_workbench(image_current=True, governance_ready=True), ) assert payload["status"] == "closure_verified" assert payload["readback"]["harbor_110_remote_ssh_command_path_ready"] is True assert payload["readback"]["harbor_110_remote_control_channel_unavailable"] is False assert payload["readback"]["harbor_110_remote_ssh_publickey_auth_stalled"] is False assert payload["readback"]["harbor_110_remote_ssh_publickey_offer_timeout"] is False assert ( payload["readback"][ "harbor_110_remote_ssh_server_accepts_key_then_session_timeout" ] is False ) assert "harbor_110_remote_ssh_publickey_auth_stalled" not in payload[ "blockers" ] def test_closure_verifier_blocks_stale_harbor_running_readback() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue(no_matching=False, harbor_110_running_jobs_api_stale=True), production_workbench=_workbench(image_current=True, governance_ready=True), ) assert payload["status"] == ( "blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback" ) assert "harbor_110_repair_visible_running_jobs_api_stale" in payload["blockers"] assert ( payload["readback"]["harbor_110_repair_visible_running_jobs_api_stale"] is True ) assert ( payload["readback"][ "current_cd_waiting_behind_harbor_110_repair_running" ] is True ) assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match" assert "verify_harbor_110_repair_jobs_payload" in payload["next_actions"][0] assert "110_local_control_path" in payload["next_actions"][0] def test_closure_verifier_blocks_harbor_110_remote_control_unavailable() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue(no_matching=False, harbor_110_remote_control_unavailable=True), production_workbench=_workbench(image_current=True, governance_ready=True), ) assert payload["status"] == "blocked_harbor_110_remote_control_channel_unavailable" assert "harbor_110_remote_control_channel_unavailable" in payload["blockers"] assert payload["readback"]["harbor_110_remote_control_channel_unavailable"] is True assert payload["readback"]["harbor_110_remote_ssh_publickey_auth_stalled"] is False assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match" assert "repair_110_ssh_publickey_auth_local_check" in payload["next_actions"][0] def test_closure_verifier_uses_deploy_snapshot_when_readiness_file_missing() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text="", queue=_queue(no_matching=True), production_workbench=_workbench(image_current=False, governance_ready=False), production_deploy_snapshot=_deploy_snapshot(), ) assert payload["status"] == "blocked_non110_runner_not_ready" assert payload["readback"]["non110_runner_readiness_source"] == ( "committed_production_deploy_snapshot" ) assert payload["readback"]["non110_runner_ready_config_count"] == 1 assert payload["readback"]["non110_runner_ready_registration_count"] == 0 assert payload["runner_readiness_blockers"] == [ "runner_registration_missing", "runner_service_not_active", ] assert ( payload["operation_boundaries"][ "committed_production_deploy_snapshot_read" ] is True ) def test_closure_verifier_accepts_full_closure_evidence() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue(no_matching=False), production_workbench=_workbench(image_current=True, governance_ready=True), ) assert payload["status"] == "closure_verified" assert payload["progress"]["ordered_completed_prefix_count"] == 6 assert payload["progress"]["ordered_completion_percent"] == 100 assert payload["progress"]["next_blocked_step_id"] == "" assert all(step["status"] == "complete" for step in payload["ordered_steps"]) assert payload["blockers"] == [] assert payload["readback"]["production_deploy_image_tag_matches_main"] is True assert ( payload["readback"][ "production_deploy_runtime_build_matches_committed_source_control_readback" ] is True ) def test_closure_verifier_blocks_stale_runtime_even_when_desired_tag_matches() -> None: module = _load_module() payload = module.build_closure_verifier( readiness_text=_readiness(ready=True), queue=_queue(no_matching=False), production_workbench=_workbench( image_current=True, governance_ready=True, runtime_source_current=False, runtime_image_current=False, ), ) assert payload["status"] == "blocked_production_image_not_current" assert payload["progress"]["ordered_completed_prefix_count"] == 4 assert payload["progress"]["next_blocked_step_id"] == "production_image_tag_current" assert "production_image_tag_not_current" in payload["blockers"] assert payload["readback"]["production_deploy_image_tag_matches_main"] is False assert ( payload["readback"][ "production_deploy_runtime_build_matches_committed_source_control_readback" ] is False ) def test_cli_uses_fixture_files_without_live_dispatch(tmp_path: Path) -> None: readiness_path = tmp_path / "readiness.txt" queue_path = tmp_path / "queue.json" workbench_path = tmp_path / "workbench.json" readiness_path.write_text(_readiness(ready=False), encoding="utf-8") queue_path.write_text(json.dumps(_queue(no_matching=True)), encoding="utf-8") workbench_path.write_text( json.dumps(_workbench(image_current=False, governance_ready=False)), encoding="utf-8", ) result = subprocess.run( [ sys.executable, str(SCRIPT), "--readiness-file", str(readiness_path), "--queue-json-file", str(queue_path), "--production-workbench-json-file", str(workbench_path), "--json", ], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) assert result.returncode == 1 payload = json.loads(result.stdout) assert payload["status"] == "blocked_non110_runner_not_ready" assert payload["operation_boundaries"]["workflow_dispatch_performed"] is False assert payload["operation_boundaries"]["github_api_used"] is False def test_cli_uses_deploy_snapshot_without_readiness_file(tmp_path: Path) -> None: queue_path = tmp_path / "queue.json" snapshot_path = tmp_path / "deploy-snapshot.json" workbench_path = tmp_path / "workbench.json" queue_path.write_text(json.dumps(_queue(no_matching=True)), encoding="utf-8") snapshot_path.write_text(json.dumps(_deploy_snapshot()), encoding="utf-8") workbench_path.write_text( json.dumps(_workbench(image_current=False, governance_ready=False)), encoding="utf-8", ) result = subprocess.run( [ sys.executable, str(SCRIPT), "--production-deploy-snapshot-json-file", str(snapshot_path), "--queue-json-file", str(queue_path), "--production-workbench-json-file", str(workbench_path), "--json", ], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) assert result.returncode == 1 payload = json.loads(result.stdout) assert payload["status"] == "blocked_non110_runner_not_ready" assert payload["readback"]["non110_runner_readiness_source"] == ( "committed_production_deploy_snapshot" ) assert payload["operation_boundaries"]["raw_runner_registration_read"] is False