#!/usr/bin/env python3 from __future__ import annotations import argparse import json import subprocess import sys import urllib.request from pathlib import Path from typing import Any SCHEMA_VERSION = "awoooi_non110_cd_closure_verifier_v1" ROOT = Path(__file__).resolve().parents[2] PUBLIC_QUEUE_READER = ROOT / "ops/runner/read-public-gitea-actions-queue.py" DEFAULT_PRODUCTION_DEPLOY_SNAPSHOT = ( ROOT / "docs/operations/awoooi-production-deploy-readback-blocker.snapshot.json" ) DEFAULT_PRODUCTION_WORKBENCH_URL = ( "https://awoooi.wooo.work/api/v1/agents/delivery-closure-workbench" ) ORDERED_STEP_COUNT = 6 def _read_text(path: str | None) -> str: if not path: return "" return Path(path).read_text(encoding="utf-8") def _load_json_file(path: str | None) -> dict[str, Any]: if not path: return {} with Path(path).open(encoding="utf-8") as fh: payload = json.load(fh) return payload if isinstance(payload, dict) else {} def _load_url_json(url: str) -> dict[str, Any]: with urllib.request.urlopen(url, timeout=20) as response: payload = json.load(response) return payload if isinstance(payload, dict) else {} def _load_live_public_queue() -> dict[str, Any]: result = subprocess.run( [sys.executable, str(PUBLIC_QUEUE_READER), "--json"], check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) if result.returncode != 0: return { "schema_version": "missing_public_gitea_actions_queue_readback_v1", "status": "blocked_public_queue_readback_failed", "readback": {}, "rollups": {}, "operation_boundaries": { "workflow_dispatch_performed": False, "secret_or_runner_token_read": False, "github_api_used": False, }, "error": result.stderr.strip().splitlines()[-1:] or ["unknown_error"], } payload = json.loads(result.stdout) return payload if isinstance(payload, dict) else {} def _int(value: Any) -> int: try: return int(str(value)) except (TypeError, ValueError): return 0 def _strings(value: Any) -> list[str]: if not isinstance(value, list): return [] return [str(item) for item in value if str(item)] def parse_readiness_text(text: str) -> dict[str, Any]: key_values: dict[str, str] = {} blockers: list[str] = [] warnings: list[str] = [] for raw_line in text.splitlines(): line = raw_line.strip() if not line: continue if line.startswith("BLOCKER "): blockers.append(line.removeprefix("BLOCKER ").strip()) continue if line.startswith("WARNING "): warnings.append(line.removeprefix("WARNING ").strip()) continue if "=" in line and not line.startswith("RUNNER_REGISTRATION "): key, value = line.split("=", 1) if key.isidentifier() or key.isupper() or key in {"safe_next_step"}: key_values[key] = value return { "provided": bool(text.strip()), "source": "sanitized_readiness_text" if text.strip() else "", "ready": key_values.get("AWOOOI_NON110_RUNNER_READY") == "1", "blockers": blockers, "warnings": warnings, "ready_config_count": _int(key_values.get("READY_CONFIG_COUNT")), "ready_binary_count": _int(key_values.get("READY_BINARY_COUNT")), "ready_registration_count": _int( key_values.get("READY_REGISTRATION_COUNT") ), "ready_service_count": _int(key_values.get("READY_SERVICE_COUNT")), "ready_active_service_count": _int( key_values.get("READY_ACTIVE_SERVICE_COUNT") ), "ready_autostart_path_count": _int( key_values.get("READY_AUTOSTART_PATH_COUNT") ), "raw_runner_registration_read": key_values.get( "raw_runner_registration_read" ) == "true", "safe_next_step": key_values.get("safe_next_step", ""), } def readiness_from_production_deploy_snapshot(snapshot: dict[str, Any]) -> dict[str, Any]: readback = snapshot.get("readback") if isinstance(snapshot.get("readback"), dict) else {} if "non110_runner_ready" not in readback: return parse_readiness_text("") return { "provided": True, "source": "committed_production_deploy_snapshot", "ready": readback.get("non110_runner_ready") is True, "blockers": _strings(readback.get("non110_runner_remaining_blockers")), "warnings": [], "ready_config_count": _int(readback.get("non110_runner_ready_config_count")), "ready_binary_count": _int(readback.get("non110_runner_ready_binary_count")), "ready_registration_count": _int( readback.get("non110_runner_ready_registration_count") ), "ready_service_count": _int(readback.get("non110_runner_ready_service_count")), "ready_active_service_count": _int( readback.get("non110_runner_ready_active_service_count") ), "ready_autostart_path_count": _int( readback.get("non110_runner_ready_autostart_path_count") ), "raw_runner_registration_read": False, "safe_next_step": str(readback.get("non110_runner_safe_next_step") or ""), } def _production_summary(workbench: dict[str, Any]) -> dict[str, Any]: summary = workbench.get("summary") return summary if isinstance(summary, dict) else {} def _production_image_tag_current(production: dict[str, Any]) -> bool: return ( production.get("production_deploy_image_tag_matches_main") is True and production.get( "production_deploy_runtime_build_matches_committed_source_control_readback" ) is True and production.get( "production_deploy_runtime_build_matches_committed_production_image_tag" ) is True ) def _build_ordered_steps( *, readiness: dict[str, Any], public_queue_closure_ready: bool, queue_runner_match_next_action: str, production_workbench_present: bool, production_image_tag_matches_main: bool, production_governance_fields_present: bool, ) -> list[dict[str, Any]]: registration_ready = ( readiness["provided"] and readiness["ready_registration_count"] > 0 and not readiness["raw_runner_registration_read"] ) active_service_ready = ( readiness["provided"] and readiness["ready_active_service_count"] > 0 and not readiness["raw_runner_registration_read"] ) definitions = [ { "id": "non110_runner_registration_metadata", "title": "non-110 runner registration metadata present", "evidence_ready": registration_ready, "next_action": readiness["safe_next_step"] or "rerun_non110_runner_readiness_verifier", }, { "id": "non110_runner_active_service", "title": "non-110 runner service active", "evidence_ready": active_service_ready, "next_action": "wait_for_autostart_path_or_rerun_non110_runner_readiness_verifier", }, { "id": "public_queue_runner_match", "title": ( "public Gitea queue no longer shows controlled CD or Harbor " "repair blockers" ), "evidence_ready": public_queue_closure_ready, "next_action": queue_runner_match_next_action, }, { "id": "production_workbench_readback", "title": "production Delivery Workbench readback present", "evidence_ready": production_workbench_present, "next_action": "read_production_delivery_workbench_after_deploy", }, { "id": "production_image_tag_current", "title": "production image tag matches Gitea main", "evidence_ready": production_image_tag_matches_main, "next_action": "complete_authorized_cd_then_verify_image_tag_matches_main", }, { "id": "production_governance_fields_present", "title": "production governance fields present after deploy", "evidence_ready": production_governance_fields_present, "next_action": "verify_internal_governance_writeback_fields_after_deploy", }, ] first_blocked_index = next( ( index for index, step in enumerate(definitions) if step["evidence_ready"] is not True ), None, ) steps: list[dict[str, Any]] = [] for index, step in enumerate(definitions): if step["evidence_ready"] is True: status = "complete" elif index == first_blocked_index: status = "blocked" else: status = "pending" steps.append( { "index": index + 1, "id": step["id"], "title": step["title"], "status": status, "evidence_ready": step["evidence_ready"] is True, "next_action": step["next_action"] if status == "blocked" else "", } ) return steps def _progress_from_steps(steps: list[dict[str, Any]]) -> dict[str, Any]: evidence_completed = sum(1 for step in steps if step["evidence_ready"] is True) prefix_completed = 0 for step in steps: if step["status"] != "complete": break prefix_completed += 1 next_blocked = next((step for step in steps if step["status"] == "blocked"), {}) return { "ordered_step_count": len(steps), "ordered_completed_prefix_count": prefix_completed, "evidence_completed_step_count": evidence_completed, "ordered_completion_percent": round( (prefix_completed / max(len(steps), 1)) * 100 ), "evidence_completion_percent": round( (evidence_completed / max(len(steps), 1)) * 100 ), "next_blocked_step_index": _int(next_blocked.get("index")), "next_blocked_step_id": str(next_blocked.get("id") or ""), "next_blocked_step_action": str(next_blocked.get("next_action") or ""), } def build_closure_verifier( *, readiness_text: str, queue: dict[str, Any], production_workbench: dict[str, Any], production_deploy_snapshot: dict[str, Any] | None = None, ) -> dict[str, Any]: readiness = parse_readiness_text(readiness_text) if not readiness["provided"] and production_deploy_snapshot: readiness = readiness_from_production_deploy_snapshot( production_deploy_snapshot ) queue_readback = queue.get("readback") if isinstance(queue.get("readback"), dict) else {} queue_boundaries = ( queue.get("operation_boundaries") if isinstance(queue.get("operation_boundaries"), dict) else {} ) queue_status = str(queue.get("status") or "") queue_rollups = ( queue.get("rollups") if isinstance(queue.get("rollups"), dict) else {} ) production = _production_summary(production_workbench) no_matching_runner_visible = ( queue_readback.get("no_matching_online_runner_visible") is True ) harbor_110_repair_no_matching_runner_label = str( queue_readback.get("latest_visible_harbor_110_repair_no_matching_runner_label") or "" ) harbor_110_repair_no_matching_runner = ( queue_status == "blocked_harbor_110_repair_no_matching_runner" or bool(harbor_110_repair_no_matching_runner_label) ) harbor_110_remote_control_channel_unavailable = ( queue_status == "blocked_harbor_110_remote_control_channel_unavailable" or queue_readback.get( "latest_visible_harbor_110_repair_remote_control_channel_unavailable" ) is True or queue_rollups.get("harbor_110_repair_remote_control_channel_unavailable") is True ) harbor_110_remote_ssh_command_path_ready = ( queue_readback.get( "latest_visible_harbor_110_repair_remote_ssh_command_path_ready" ) is True or queue_rollups.get("harbor_110_repair_remote_ssh_command_path_ready") is True ) harbor_110_remote_ssh_publickey_auth_stalled = ( queue_status == "blocked_harbor_110_remote_ssh_publickey_auth_stalled" or queue_readback.get( "latest_visible_harbor_110_repair_remote_ssh_publickey_auth_stalled" ) is True or queue_rollups.get("harbor_110_repair_remote_ssh_publickey_auth_stalled") is True ) harbor_110_remote_ssh_publickey_reply_timeout_seen = ( queue_readback.get( "latest_visible_harbor_110_repair_remote_ssh_publickey_reply_timeout_seen" ) is True or queue_rollups.get( "harbor_110_repair_remote_ssh_publickey_reply_timeout_seen" ) is True ) harbor_110_remote_ssh_publickey_offer_timeout = ( queue_readback.get( "latest_visible_harbor_110_repair_remote_ssh_publickey_offer_timeout" ) is True or queue_rollups.get("harbor_110_repair_remote_ssh_publickey_offer_timeout") is True ) harbor_110_remote_ssh_server_accepts_key_then_session_timeout = ( queue_readback.get( "latest_visible_harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout" ) is True or queue_rollups.get( "harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout" ) is True ) if harbor_110_remote_ssh_command_path_ready: harbor_110_remote_control_channel_unavailable = False harbor_110_remote_ssh_publickey_auth_stalled = False harbor_110_remote_ssh_publickey_offer_timeout = False harbor_110_remote_ssh_server_accepts_key_then_session_timeout = False harbor_110_repair_visible_running_jobs_api_stale = ( queue_status == "blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback" or queue_readback.get("harbor_110_repair_visible_running_jobs_api_stale") is True or queue_rollups.get("harbor_110_repair_visible_running_jobs_api_stale") is True ) current_cd_waiting_behind_harbor_110_repair_running = ( queue_status in { "blocked_current_cd_waiting_behind_harbor_110_repair_running", "blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback", } or queue_readback.get( "current_cd_waiting_behind_harbor_110_repair_running" ) is True or queue_rollups.get("current_cd_waiting_behind_harbor_110_repair_running") is True ) queue_runner_match_next_action = ( "run_sudo_usr_local_bin_repair_110_ssh_publickey_auth_local_check_on_110_" "local_console_then_apply_if_metadata_only_then_rerun_public_queue_and_" "harbor_v2_readback" if harbor_110_remote_ssh_publickey_auth_stalled else "rerun_public_queue_readback_then_verify_harbor_110_repair_jobs_payload_" "and_110_local_control_path_before_unblocking_cd" if harbor_110_repair_visible_running_jobs_api_stale else "wait_for_harbor_110_repair_to_finish_then_rerun_public_queue_and_" "harbor_v2_readback" if current_cd_waiting_behind_harbor_110_repair_running else "run_sudo_usr_local_bin_repair_110_ssh_publickey_auth_local_check_on_110_" "local_console_then_rerun_public_queue_and_harbor_v2_readback" if harbor_110_remote_control_channel_unavailable else ( "run_ops_runner_check_awoooi_110_controlled_cd_lane_readiness_on_110_" "then_restore_awoooi_host_runner_control_path_without_legacy_or_generic_labels_" "then_rerun_harbor_110_repair_queue_readback" if harbor_110_repair_no_matching_runner else "rerun_public_queue_readback_until_no_matching_runner_is_absent" ) ) public_queue_closure_ready = not ( no_matching_runner_visible or harbor_110_repair_no_matching_runner or harbor_110_remote_control_channel_unavailable or harbor_110_remote_ssh_publickey_auth_stalled or harbor_110_repair_visible_running_jobs_api_stale or current_cd_waiting_behind_harbor_110_repair_running ) production_workbench_present = bool(production) production_image_tag_matches_main = _production_image_tag_current(production) production_governance_fields_present = ( production.get("production_deploy_governance_fields_present") is True ) blockers: list[str] = [] if not readiness["provided"]: blockers.append("missing_non110_readiness_readback") elif not readiness["ready"]: blockers.append("non110_runner_not_ready") if readiness["raw_runner_registration_read"]: blockers.append("raw_runner_registration_read_not_allowed") if harbor_110_repair_no_matching_runner: blockers.append("harbor_110_repair_no_matching_runner") elif harbor_110_remote_ssh_publickey_auth_stalled: blockers.append("harbor_110_remote_ssh_publickey_auth_stalled") elif harbor_110_repair_visible_running_jobs_api_stale: blockers.append("harbor_110_repair_visible_running_jobs_api_stale") elif current_cd_waiting_behind_harbor_110_repair_running: blockers.append("current_cd_waiting_behind_harbor_110_repair_running") elif harbor_110_remote_control_channel_unavailable: blockers.append("harbor_110_remote_control_channel_unavailable") elif no_matching_runner_visible: blockers.append("public_queue_still_has_no_matching_online_runner") if not production_workbench_present: blockers.append("production_workbench_readback_missing") elif not production_image_tag_matches_main: blockers.append("production_image_tag_not_current") if production_workbench_present and not production_governance_fields_present: blockers.append("production_governance_fields_missing") if "missing_non110_readiness_readback" in blockers: status = "blocked_missing_non110_readiness_readback" elif "non110_runner_not_ready" in blockers: status = "blocked_non110_runner_not_ready" elif "raw_runner_registration_read_not_allowed" in blockers: status = "blocked_secret_boundary_violation" elif "harbor_110_repair_no_matching_runner" in blockers: status = "blocked_harbor_110_repair_no_matching_runner" elif "harbor_110_remote_ssh_publickey_auth_stalled" in blockers: status = "blocked_harbor_110_remote_ssh_publickey_auth_stalled" elif "harbor_110_repair_visible_running_jobs_api_stale" in blockers: status = "blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback" elif "current_cd_waiting_behind_harbor_110_repair_running" in blockers: status = "blocked_current_cd_waiting_behind_harbor_110_repair_running" elif "harbor_110_remote_control_channel_unavailable" in blockers: status = "blocked_harbor_110_remote_control_channel_unavailable" elif "public_queue_still_has_no_matching_online_runner" in blockers: status = "blocked_no_matching_online_runner" elif "production_workbench_readback_missing" in blockers: status = "blocked_production_workbench_readback_missing" elif "production_image_tag_not_current" in blockers: status = "blocked_production_image_not_current" elif "production_governance_fields_missing" in blockers: status = "blocked_production_governance_fields_missing" else: status = "closure_verified" ordered_steps = _build_ordered_steps( readiness=readiness, public_queue_closure_ready=public_queue_closure_ready, queue_runner_match_next_action=queue_runner_match_next_action, production_workbench_present=production_workbench_present, production_image_tag_matches_main=production_image_tag_matches_main, production_governance_fields_present=production_governance_fields_present, ) progress = _progress_from_steps(ordered_steps) return { "schema_version": SCHEMA_VERSION, "status": status, "progress": progress, "readback": { "non110_runner_ready": readiness["ready"], "non110_runner_readiness_source": readiness["source"], "non110_runner_ready_config_count": readiness["ready_config_count"], "non110_runner_ready_binary_count": readiness["ready_binary_count"], "non110_runner_ready_registration_count": readiness[ "ready_registration_count" ], "non110_runner_ready_service_count": readiness["ready_service_count"], "non110_runner_ready_active_service_count": readiness[ "ready_active_service_count" ], "non110_runner_ready_autostart_path_count": readiness[ "ready_autostart_path_count" ], "public_actions_queue_status": str(queue.get("status") or ""), "public_actions_queue_schema_version": str( queue.get("schema_version") or "" ), "public_queue_no_matching_online_runner_visible": ( no_matching_runner_visible ), "public_queue_latest_no_matching_runner_label": str( queue_readback.get("latest_visible_no_matching_runner_label") or "" ), "harbor_110_repair_no_matching_runner": ( harbor_110_repair_no_matching_runner ), "harbor_110_repair_no_matching_runner_label": ( harbor_110_repair_no_matching_runner_label ), "harbor_110_remote_control_channel_unavailable": ( harbor_110_remote_control_channel_unavailable ), "harbor_110_remote_ssh_command_path_ready": ( harbor_110_remote_ssh_command_path_ready ), "harbor_110_remote_ssh_publickey_auth_stalled": ( harbor_110_remote_ssh_publickey_auth_stalled ), "harbor_110_remote_ssh_publickey_reply_timeout_seen": ( harbor_110_remote_ssh_publickey_reply_timeout_seen ), "harbor_110_remote_ssh_publickey_offer_timeout": ( harbor_110_remote_ssh_publickey_offer_timeout ), "harbor_110_remote_ssh_server_accepts_key_then_session_timeout": ( harbor_110_remote_ssh_server_accepts_key_then_session_timeout ), "harbor_110_repair_visible_running_jobs_api_stale": ( harbor_110_repair_visible_running_jobs_api_stale ), "current_cd_waiting_behind_harbor_110_repair_running": ( current_cd_waiting_behind_harbor_110_repair_running ), "production_workbench_present": production_workbench_present, "production_workbench_source_count": _int(production.get("source_count")), "production_deploy_image_tag_matches_main": ( production_image_tag_matches_main ), "production_deploy_runtime_build_matches_committed_source_control_readback": ( production.get( "production_deploy_runtime_build_matches_committed_source_control_readback" ) is True ), "production_deploy_runtime_build_matches_committed_production_image_tag": ( production.get( "production_deploy_runtime_build_matches_committed_production_image_tag" ) is True ), "production_deploy_runtime_build_readback_status": str( production.get("production_deploy_runtime_build_readback_status") or "" ), "production_deploy_governance_fields_present": ( production_governance_fields_present ), }, "blockers": blockers, "ordered_steps": ordered_steps, "runner_readiness_blockers": readiness["blockers"], "runner_readiness_warnings": readiness["warnings"], "next_actions": [ action for action in [ readiness["safe_next_step"] if not readiness["ready"] else "", queue_runner_match_next_action if not public_queue_closure_ready else "", "read_production_delivery_workbench_after_deploy" if not production_workbench_present else "", "complete_authorized_cd_then_verify_image_tag_matches_main" if production_workbench_present and not production_image_tag_matches_main else "", "verify_internal_governance_writeback_fields_after_deploy" if production_workbench_present and not production_governance_fields_present else "", ] if action ], "operation_boundaries": { "secret_or_runner_token_read": False, "raw_runner_registration_read": False, "committed_production_deploy_snapshot_read": ( readiness["source"] == "committed_production_deploy_snapshot" ), "workflow_dispatch_performed": False, "host_write_performed": False, "gitea_api_write_performed": False, "github_api_used": False, "public_queue_workflow_dispatch_performed": ( queue_boundaries.get("workflow_dispatch_performed") is True ), "public_queue_secret_or_runner_token_read": ( queue_boundaries.get("secret_or_runner_token_read") is True ), }, } def main() -> int: parser = argparse.ArgumentParser( description="Verify sanitized AWOOOI non-110 CD closure evidence." ) parser.add_argument("--readiness-file", default="") parser.add_argument( "--production-deploy-snapshot-json-file", default=str(DEFAULT_PRODUCTION_DEPLOY_SNAPSHOT), ) parser.add_argument("--queue-json-file", default="") parser.add_argument("--production-workbench-json-file", default="") parser.add_argument( "--production-workbench-url", default="", help="Read production workbench JSON from URL when no file is provided.", ) parser.add_argument("--json", action="store_true") args = parser.parse_args() readiness_text = _read_text(args.readiness_file) production_deploy_snapshot = _load_json_file( args.production_deploy_snapshot_json_file ) queue = ( _load_json_file(args.queue_json_file) if args.queue_json_file else _load_live_public_queue() ) if args.production_workbench_json_file: production_workbench = _load_json_file(args.production_workbench_json_file) elif args.production_workbench_url: production_workbench = _load_url_json(args.production_workbench_url) else: production_workbench = _load_url_json(DEFAULT_PRODUCTION_WORKBENCH_URL) payload = build_closure_verifier( readiness_text=readiness_text, queue=queue, production_workbench=production_workbench, production_deploy_snapshot=production_deploy_snapshot, ) if args.json: print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) else: print(f"status={payload['status']}") print(f"blocker_count={len(payload['blockers'])}") for blocker in payload["blockers"]: print(f"BLOCKER {blocker}") for action in payload["next_actions"]: print(f"safe_next_step={action}") return 0 if payload["status"] == "closure_verified" else 1 if __name__ == "__main__": raise SystemExit(main())