Files
awoooi/ops/runner/test_verify_awoooi_non110_cd_closure.py
ogt c90f1ced2f
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 36s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
fix(recovery): honor 110 ssh command-path readiness
2026-07-01 23:12:33 +08:00

549 lines
22 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import importlib.util
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "ops/runner/verify-awoooi-non110-cd-closure.py"
def _load_module():
spec = importlib.util.spec_from_file_location(
"verify_awoooi_non110_cd_closure",
SCRIPT,
)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _queue(
*,
no_matching: bool,
harbor_110_no_matching: bool = False,
harbor_110_remote_control_unavailable: bool = False,
harbor_110_publickey_auth_stalled: bool = False,
harbor_110_publickey_offer_timeout: bool = False,
harbor_110_session_timeout: bool = False,
harbor_110_command_path_ready: bool = False,
harbor_110_running_jobs_api_stale: bool = False,
current_cd_waiting_behind_harbor_110_running: bool = False,
) -> dict:
status = (
"blocked_harbor_110_remote_ssh_publickey_auth_stalled"
if harbor_110_publickey_auth_stalled
else "blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback"
if harbor_110_running_jobs_api_stale
else "blocked_current_cd_waiting_behind_harbor_110_repair_running"
if current_cd_waiting_behind_harbor_110_running
else "blocked_harbor_110_remote_control_channel_unavailable"
if harbor_110_remote_control_unavailable
else "blocked_harbor_110_repair_no_matching_runner"
if harbor_110_no_matching
else "blocked_no_matching_online_runner"
if no_matching
else "no_matching_runner_not_visible"
)
return {
"schema_version": "awoooi_public_gitea_actions_queue_readback_v1",
"status": status,
"readback": {
"no_matching_online_runner_visible": no_matching,
"latest_visible_no_matching_runner_label": (
"awoooi-host"
if harbor_110_no_matching
else "awoooi-non110-ubuntu"
if no_matching
else ""
),
"latest_visible_harbor_110_repair_no_matching_runner_label": (
"awoooi-host" if harbor_110_no_matching else ""
),
"latest_visible_harbor_110_repair_remote_control_channel_unavailable": (
harbor_110_remote_control_unavailable
or harbor_110_publickey_auth_stalled
),
"latest_visible_harbor_110_repair_remote_ssh_publickey_auth_stalled": (
harbor_110_publickey_auth_stalled
),
"latest_visible_harbor_110_repair_remote_ssh_command_path_ready": (
harbor_110_command_path_ready
),
"latest_visible_harbor_110_repair_remote_ssh_publickey_reply_timeout_seen": (
harbor_110_publickey_auth_stalled
),
"latest_visible_harbor_110_repair_remote_ssh_publickey_offer_timeout": (
harbor_110_publickey_offer_timeout
),
"latest_visible_harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout": (
harbor_110_session_timeout
),
"harbor_110_repair_visible_running_jobs_api_stale": (
harbor_110_running_jobs_api_stale
),
"current_cd_waiting_behind_harbor_110_repair_running": (
current_cd_waiting_behind_harbor_110_running
or harbor_110_running_jobs_api_stale
),
},
"rollups": {
"harbor_110_repair_remote_control_channel_unavailable": (
harbor_110_remote_control_unavailable
or harbor_110_publickey_auth_stalled
),
"harbor_110_repair_remote_ssh_publickey_auth_stalled": (
harbor_110_publickey_auth_stalled
),
"harbor_110_repair_remote_ssh_command_path_ready": (
harbor_110_command_path_ready
),
"harbor_110_repair_remote_ssh_publickey_reply_timeout_seen": (
harbor_110_publickey_auth_stalled
),
"harbor_110_repair_remote_ssh_publickey_offer_timeout": (
harbor_110_publickey_offer_timeout
),
"harbor_110_repair_remote_ssh_server_accepts_key_then_session_timeout": (
harbor_110_session_timeout
),
"harbor_110_repair_visible_running_jobs_api_stale": (
harbor_110_running_jobs_api_stale
),
"current_cd_waiting_behind_harbor_110_repair_running": (
current_cd_waiting_behind_harbor_110_running
or harbor_110_running_jobs_api_stale
),
},
"operation_boundaries": {
"workflow_dispatch_performed": False,
"secret_or_runner_token_read": False,
"github_api_used": False,
},
}
def _workbench(
*,
image_current: bool,
governance_ready: bool,
runtime_source_current: bool | None = None,
runtime_image_current: bool | None = None,
) -> dict:
runtime_source_current = (
image_current if runtime_source_current is None else runtime_source_current
)
runtime_image_current = (
image_current if runtime_image_current is None else runtime_image_current
)
return {
"schema_version": "delivery_closure_workbench_v1",
"summary": {
"source_count": 6,
"production_deploy_image_tag_matches_main": image_current,
"production_deploy_runtime_build_matches_committed_source_control_readback": (
runtime_source_current
),
"production_deploy_runtime_build_matches_committed_production_image_tag": (
runtime_image_current
),
"production_deploy_runtime_build_readback_status": (
"matches_committed_deploy_readback"
if runtime_source_current and runtime_image_current
else "runtime_build_diverges_from_committed_deploy_readback"
),
"production_deploy_governance_fields_present": governance_ready,
},
}
def _deploy_snapshot() -> dict:
return {
"schema_version": "awoooi_production_deploy_readback_blocker_v1",
"readback": {
"non110_runner_ready": False,
"non110_runner_ready_config_count": 1,
"non110_runner_ready_binary_count": 1,
"non110_runner_ready_registration_count": 0,
"non110_runner_ready_service_count": 1,
"non110_runner_ready_active_service_count": 0,
"non110_runner_ready_autostart_path_count": 1,
"non110_runner_remaining_blockers": [
"runner_registration_missing",
"runner_service_not_active",
],
"non110_runner_safe_next_step": "run_register_awoooi_non110_runner_script_without_printing_token_then_autostart_path_will_enable_service_and_rerun_this_verifier",
},
}
def _readiness(*, ready: bool) -> str:
if ready:
return "\n".join(
[
"raw_runner_registration_read=false",
"READY_CONFIG_COUNT=1",
"READY_BINARY_COUNT=1",
"READY_REGISTRATION_COUNT=1",
"READY_SERVICE_COUNT=1",
"READY_ACTIVE_SERVICE_COUNT=1",
"READY_AUTOSTART_PATH_COUNT=1",
"AWOOOI_NON110_RUNNER_READY=1",
]
)
return "\n".join(
[
"raw_runner_registration_read=false",
"BLOCKER runner_registration_missing",
"BLOCKER runner_service_not_active",
"READY_CONFIG_COUNT=1",
"READY_BINARY_COUNT=1",
"READY_REGISTRATION_COUNT=0",
"READY_SERVICE_COUNT=1",
"READY_ACTIVE_SERVICE_COUNT=0",
"READY_AUTOSTART_PATH_COUNT=1",
"AWOOOI_NON110_RUNNER_READY=0",
"safe_next_step=run_register_awoooi_non110_runner_script_without_printing_token_then_autostart_path_will_enable_service_and_rerun_this_verifier",
]
)
def test_closure_verifier_blocks_runner_not_ready_without_secret_leak() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=False),
queue=_queue(no_matching=True),
production_workbench=_workbench(image_current=False, governance_ready=False),
)
text = json.dumps(payload, sort_keys=True)
assert payload["schema_version"] == module.SCHEMA_VERSION
assert payload["status"] == "blocked_non110_runner_not_ready"
assert payload["progress"]["ordered_step_count"] == 6
assert payload["progress"]["ordered_completed_prefix_count"] == 0
assert payload["progress"]["next_blocked_step_index"] == 1
assert payload["progress"]["next_blocked_step_id"] == (
"non110_runner_registration_metadata"
)
assert payload["ordered_steps"][0]["status"] == "blocked"
assert payload["ordered_steps"][0]["evidence_ready"] is False
assert payload["ordered_steps"][1]["status"] == "pending"
assert "runner_registration_missing" in payload["runner_readiness_blockers"]
assert payload["readback"]["non110_runner_ready_registration_count"] == 0
assert payload["operation_boundaries"]["secret_or_runner_token_read"] is False
assert payload["operation_boundaries"]["raw_runner_registration_read"] is False
assert "secret-token-like-content" not in text
def test_closure_verifier_blocks_queue_after_runner_ready() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(no_matching=True),
production_workbench=_workbench(image_current=False, governance_ready=False),
)
assert payload["status"] == "blocked_no_matching_online_runner"
assert payload["progress"]["ordered_completed_prefix_count"] == 2
assert payload["progress"]["next_blocked_step_index"] == 3
assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match"
assert payload["ordered_steps"][0]["status"] == "complete"
assert payload["ordered_steps"][1]["status"] == "complete"
assert payload["ordered_steps"][2]["status"] == "blocked"
assert "public_queue_still_has_no_matching_online_runner" in payload["blockers"]
def test_closure_verifier_prioritizes_harbor_110_runner_label_blocker() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(no_matching=True, harbor_110_no_matching=True),
production_workbench=_workbench(image_current=False, governance_ready=False),
)
assert payload["status"] == "blocked_harbor_110_repair_no_matching_runner"
assert "harbor_110_repair_no_matching_runner" in payload["blockers"]
assert "public_queue_still_has_no_matching_online_runner" not in payload[
"blockers"
]
assert payload["readback"]["harbor_110_repair_no_matching_runner"] is True
assert (
payload["readback"]["harbor_110_repair_no_matching_runner_label"]
== "awoooi-host"
)
assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match"
assert "check_awoooi_110_controlled_cd_lane_readiness" in payload[
"next_actions"
][0]
assert "awoooi_host_runner_control_path" in payload["next_actions"][0]
assert "check_awoooi_110_controlled_cd_lane_readiness" in payload["progress"][
"next_blocked_step_action"
]
assert "awoooi_host_runner_control_path" in payload["progress"][
"next_blocked_step_action"
]
def test_closure_verifier_blocks_harbor_110_publickey_auth_stalled() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(
no_matching=False,
harbor_110_publickey_auth_stalled=True,
harbor_110_publickey_offer_timeout=True,
harbor_110_session_timeout=True,
),
production_workbench=_workbench(image_current=True, governance_ready=True),
)
assert payload["status"] == "blocked_harbor_110_remote_ssh_publickey_auth_stalled"
assert "harbor_110_remote_ssh_publickey_auth_stalled" in payload["blockers"]
assert payload["readback"]["harbor_110_remote_control_channel_unavailable"] is True
assert payload["readback"]["harbor_110_remote_ssh_publickey_auth_stalled"] is True
assert (
payload["readback"]["harbor_110_remote_ssh_publickey_reply_timeout_seen"]
is True
)
assert (
payload["readback"]["harbor_110_remote_ssh_publickey_offer_timeout"]
is True
)
assert (
payload["readback"][
"harbor_110_remote_ssh_server_accepts_key_then_session_timeout"
]
is True
)
assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match"
assert payload["ordered_steps"][2]["status"] == "blocked"
assert "repair_110_ssh_publickey_auth_local_check" in payload["next_actions"][0]
assert "harbor_v2_readback" in payload["next_actions"][0]
assert "repair_110_ssh_publickey_auth_local_check" in payload["progress"][
"next_blocked_step_action"
]
def test_closure_verifier_command_path_ready_overrides_publickey_auth_stalled() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(
no_matching=False,
harbor_110_publickey_auth_stalled=True,
harbor_110_publickey_offer_timeout=True,
harbor_110_session_timeout=True,
harbor_110_command_path_ready=True,
),
production_workbench=_workbench(image_current=True, governance_ready=True),
)
assert payload["status"] == "closure_verified"
assert payload["readback"]["harbor_110_remote_ssh_command_path_ready"] is True
assert payload["readback"]["harbor_110_remote_control_channel_unavailable"] is False
assert payload["readback"]["harbor_110_remote_ssh_publickey_auth_stalled"] is False
assert payload["readback"]["harbor_110_remote_ssh_publickey_offer_timeout"] is False
assert (
payload["readback"][
"harbor_110_remote_ssh_server_accepts_key_then_session_timeout"
]
is False
)
assert "harbor_110_remote_ssh_publickey_auth_stalled" not in payload[
"blockers"
]
def test_closure_verifier_blocks_stale_harbor_running_readback() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(no_matching=False, harbor_110_running_jobs_api_stale=True),
production_workbench=_workbench(image_current=True, governance_ready=True),
)
assert payload["status"] == (
"blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback"
)
assert "harbor_110_repair_visible_running_jobs_api_stale" in payload["blockers"]
assert (
payload["readback"]["harbor_110_repair_visible_running_jobs_api_stale"]
is True
)
assert (
payload["readback"][
"current_cd_waiting_behind_harbor_110_repair_running"
]
is True
)
assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match"
assert "verify_harbor_110_repair_jobs_payload" in payload["next_actions"][0]
assert "110_local_control_path" in payload["next_actions"][0]
def test_closure_verifier_blocks_harbor_110_remote_control_unavailable() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(no_matching=False, harbor_110_remote_control_unavailable=True),
production_workbench=_workbench(image_current=True, governance_ready=True),
)
assert payload["status"] == "blocked_harbor_110_remote_control_channel_unavailable"
assert "harbor_110_remote_control_channel_unavailable" in payload["blockers"]
assert payload["readback"]["harbor_110_remote_control_channel_unavailable"] is True
assert payload["readback"]["harbor_110_remote_ssh_publickey_auth_stalled"] is False
assert payload["progress"]["next_blocked_step_id"] == "public_queue_runner_match"
assert "repair_110_ssh_publickey_auth_local_check" in payload["next_actions"][0]
def test_closure_verifier_uses_deploy_snapshot_when_readiness_file_missing() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text="",
queue=_queue(no_matching=True),
production_workbench=_workbench(image_current=False, governance_ready=False),
production_deploy_snapshot=_deploy_snapshot(),
)
assert payload["status"] == "blocked_non110_runner_not_ready"
assert payload["readback"]["non110_runner_readiness_source"] == (
"committed_production_deploy_snapshot"
)
assert payload["readback"]["non110_runner_ready_config_count"] == 1
assert payload["readback"]["non110_runner_ready_registration_count"] == 0
assert payload["runner_readiness_blockers"] == [
"runner_registration_missing",
"runner_service_not_active",
]
assert (
payload["operation_boundaries"][
"committed_production_deploy_snapshot_read"
]
is True
)
def test_closure_verifier_accepts_full_closure_evidence() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(no_matching=False),
production_workbench=_workbench(image_current=True, governance_ready=True),
)
assert payload["status"] == "closure_verified"
assert payload["progress"]["ordered_completed_prefix_count"] == 6
assert payload["progress"]["ordered_completion_percent"] == 100
assert payload["progress"]["next_blocked_step_id"] == ""
assert all(step["status"] == "complete" for step in payload["ordered_steps"])
assert payload["blockers"] == []
assert payload["readback"]["production_deploy_image_tag_matches_main"] is True
assert (
payload["readback"][
"production_deploy_runtime_build_matches_committed_source_control_readback"
]
is True
)
def test_closure_verifier_blocks_stale_runtime_even_when_desired_tag_matches() -> None:
module = _load_module()
payload = module.build_closure_verifier(
readiness_text=_readiness(ready=True),
queue=_queue(no_matching=False),
production_workbench=_workbench(
image_current=True,
governance_ready=True,
runtime_source_current=False,
runtime_image_current=False,
),
)
assert payload["status"] == "blocked_production_image_not_current"
assert payload["progress"]["ordered_completed_prefix_count"] == 4
assert payload["progress"]["next_blocked_step_id"] == "production_image_tag_current"
assert "production_image_tag_not_current" in payload["blockers"]
assert payload["readback"]["production_deploy_image_tag_matches_main"] is False
assert (
payload["readback"][
"production_deploy_runtime_build_matches_committed_source_control_readback"
]
is False
)
def test_cli_uses_fixture_files_without_live_dispatch(tmp_path: Path) -> None:
readiness_path = tmp_path / "readiness.txt"
queue_path = tmp_path / "queue.json"
workbench_path = tmp_path / "workbench.json"
readiness_path.write_text(_readiness(ready=False), encoding="utf-8")
queue_path.write_text(json.dumps(_queue(no_matching=True)), encoding="utf-8")
workbench_path.write_text(
json.dumps(_workbench(image_current=False, governance_ready=False)),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--readiness-file",
str(readiness_path),
"--queue-json-file",
str(queue_path),
"--production-workbench-json-file",
str(workbench_path),
"--json",
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert result.returncode == 1
payload = json.loads(result.stdout)
assert payload["status"] == "blocked_non110_runner_not_ready"
assert payload["operation_boundaries"]["workflow_dispatch_performed"] is False
assert payload["operation_boundaries"]["github_api_used"] is False
def test_cli_uses_deploy_snapshot_without_readiness_file(tmp_path: Path) -> None:
queue_path = tmp_path / "queue.json"
snapshot_path = tmp_path / "deploy-snapshot.json"
workbench_path = tmp_path / "workbench.json"
queue_path.write_text(json.dumps(_queue(no_matching=True)), encoding="utf-8")
snapshot_path.write_text(json.dumps(_deploy_snapshot()), encoding="utf-8")
workbench_path.write_text(
json.dumps(_workbench(image_current=False, governance_ready=False)),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--production-deploy-snapshot-json-file",
str(snapshot_path),
"--queue-json-file",
str(queue_path),
"--production-workbench-json-file",
str(workbench_path),
"--json",
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert result.returncode == 1
payload = json.loads(result.stdout)
assert payload["status"] == "blocked_non110_runner_not_ready"
assert payload["readback"]["non110_runner_readiness_source"] == (
"committed_production_deploy_snapshot"
)
assert payload["operation_boundaries"]["raw_runner_registration_read"] is False