wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(recovery): close cold-start runtime blockers

Browse Source
This commit is contained in:
ogt
2026-07-11 15:08:27 +08:00
parent 7cea825d92
commit 276410fbb6
38 changed files with 1494 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py
View File

@@ -8,6 +8,8 @@ SCRIPT = ROOT / "scripts" / "reboot-recovery" / "188-host-hygiene-maintenance-ch
STARTUP_188 = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.sh"
STARTUP_188_SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.service"
DEPLOY_188 = ROOT / "scripts" / "reboot-recovery" / "deploy-to-188.sh"
HYGIENE_HELPER = ROOT / "ops" / "reboot-recovery" / "awoooi-host188-hygiene-helper.sh"
HYGIENE_BOOTSTRAP = ROOT / "scripts" / "reboot-recovery" / "bootstrap-host188-hygiene-executor.sh"
def test_188_and_110_default_to_reachable_runtime_identities() -> None:
@@ -46,6 +48,20 @@ def test_188_startup_keeps_clawbot_rebuild_bounded_and_opt_in() -> None:
assert "post-reboot SLO" in text
def test_188_startup_defaults_data_and_metadata_repairs_to_fail_closed() -> None:
text = STARTUP_188.read_text(encoding="utf-8")
assert 'CONTAINERD_METADATA_REPAIR_ALLOWED="${CONTAINERD_METADATA_REPAIR_ALLOWED:-0}"' in text
assert 'DOCKER_NETWORK_METADATA_REPAIR_ALLOWED="${DOCKER_NETWORK_METADATA_REPAIR_ALLOWED:-0}"' in text
assert 'KINE_MAINTENANCE_ALLOWED="${KINE_MAINTENANCE_ALLOWED:-0}"' in text
assert 'SIGNOZ_188_START_ALLOWED="${SIGNOZ_188_START_ALLOWED:-0}"' in text
assert 'if [ "$CONTAINERD_METADATA_REPAIR_ALLOWED" != "1" ]; then' in text
assert 'if [ "$DOCKER_NETWORK_METADATA_REPAIR_ALLOWED" != "1" ]; then' in text
assert 'if [ "$KINE_MAINTENANCE_ALLOWED" = "1" ]; then' in text
assert 'if [ "$SIGNOZ_188_START_ALLOWED" != "1" ]; then' in text
assert "docker network inspect aiops-network" in text
def test_188_startup_unit_timeout_matches_reboot_slo() -> None:
text = STARTUP_188_SERVICE.read_text(encoding="utf-8")
@@ -60,3 +76,24 @@ def test_188_deploy_helper_does_not_embed_sudo_password() -> None:
assert "PASSWORD" not in text
assert "sudo -n true" in text
assert "BLOCKED sudo_password_required_passwordless_sudo_or_tty_required" in text
def test_188_hygiene_executor_is_fixed_action_and_does_not_restart_products() -> None:
helper = HYGIENE_HELPER.read_text(encoding="utf-8")
bootstrap = HYGIENE_BOOTSTRAP.read_text(encoding="utf-8")
assert "check|reconcile|rollback-latest" not in helper
assert "reconcile)" in helper
assert "rollback-latest)" in helper
assert "POSTGRES_START_CONF" in helper
assert "manual" in helper
assert "systemctl daemon-reload" in helper
assert "systemctl reset-failed postgresql@14-main.service awoooi-startup.service" in helper
assert "PRODUCT_SERVICE_RESTARTS=0" in helper
assert "systemctl restart" not in helper
assert "systemctl start" not in helper
assert "--network=none" in bootstrap
assert "--userns=host" in bootstrap
assert "--pull=never" in bootstrap
assert "alpine@sha256:" in bootstrap
assert "visudo -cf" in bootstrap

20
scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py
View File

@@ -266,6 +266,26 @@ def test_runner_failclosed_enforcer_preserves_controlled_drain_staging_or_guarde
assert "CONTROLLED_DRAIN_PRESERVE_ALLOWED=" in text
def test_runner_failclosed_authority_is_singleton_and_idempotent() -> None:
text = FAILCLOSED_ENFORCER.read_text(encoding="utf-8")
assert "--ensure Check first and apply only when fail-closed state has drifted." in text
assert "failclosed_state_ok()" in text
assert 'elif [ "$MODE" = "ensure" ] && ! failclosed_state_ok; then' in text
assert (
"* * * * * root /usr/bin/flock -n "
"/run/lock/awoooi-runner-failclosed-authority.lock "
"/usr/bin/timeout 180 "
"/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh "
"--ensure"
) in text
assert (
"* * * * * root "
"/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh "
"--apply"
) not in text
def test_controlled_cd_lane_unit_source_has_required_accounting_guardrails() -> None:
text = CONTROLLED_CD_LANE_DRAIN_UNIT.read_text(encoding="utf-8")

5
scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py
View File

@@ -24,3 +24,8 @@ def test_unknown_backup_escrow_evidence_stays_blocked() -> None:
'[[ "$dr_escrow_blocked" == "1" ]] '
'&& next_required_gates+=("credential_escrow_evidence")'
) in text
assert 'POST_START_QUICK_CHECK_TIMEOUT_SECONDS="${POST_START_QUICK_CHECK_TIMEOUT_SECONDS:-90}"' in text
assert 'run_with_timeout "$POST_START_QUICK_CHECK_TIMEOUT_SECONDS"' in text
assert "POST_START_QUICK_CHECK_TIMEOUT=1" in text
assert "POST_START_TIMED_OUT=$post_start_timed_out" in text
assert 'next_required_gates+=("post_start_quick_check_timeout_recovery")' in text

3
scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py
View File

@@ -15,12 +15,15 @@ def test_post_start_self_checks_can_run_locally_on_110() -> None:
assert 'local_ip_list | grep' not in text
assert 'bash -lc "$command"' in text
assert 'SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-25}"' in text
assert 'SSH_TOTAL_TIMEOUT_SECONDS="${SSH_TOTAL_TIMEOUT_SECONDS:-$((SSH_CONNECT_TIMEOUT + SSH_COMMAND_TIMEOUT_SECONDS + 5))}"' in text
assert "printf -v quoted_command '%q' \"$command\"" in text
assert '-o ConnectionAttempts=1' in text
assert '-o ServerAliveInterval=5' in text
assert '-o ServerAliveCountMax=1' in text
assert '-o NumberOfPasswordPrompts=0' in text
assert "timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}" in text
assert 'run_with_timeout "$SSH_TOTAL_TIMEOUT_SECONDS" "${ssh_command[@]}"' in text
assert 'timeout --foreground --kill-after=3s "${timeout_seconds}s"' in text
def test_runner_pressure_is_capacity_evidence_after_routes_recover() -> None:

18
scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py
View File

@@ -8,6 +8,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
INSTALLER = ROOT / "scripts" / "reboot-recovery" / "install-reboot-auto-recovery-slo-110.sh"
SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-reboot-auto-recovery-slo.service"
TIMER = ROOT / "scripts" / "reboot-recovery" / "awoooi-reboot-auto-recovery-slo.timer"
EXPORTER = ROOT / "scripts" / "reboot-recovery" / "reboot-auto-recovery-slo-exporter.sh"
@@ -26,6 +27,7 @@ def test_installer_dry_run_exposes_apply_rollback_and_verify_contract() -> None:
assert "would_include_source_gates=" in result.stdout
assert "awooop_conversation_event_hot_path_indexes_2026-07-01.sql" in result.stdout
assert "ops/nginx" in result.stdout
assert "ops/reboot-recovery" in result.stdout
assert "would_enable_timer=awoooi-reboot-auto-recovery-slo.timer" in result.stdout
@@ -60,6 +62,7 @@ def test_installer_is_limited_to_verifier_timer_not_product_restarts() -> None:
assert 'payload="${payload_dir}/payload.tar.gz"' in text
assert "apps/api/migrations/awooop_conversation_event_hot_path_indexes_2026-07-01.sql" in text
assert "ops/nginx" in text
assert "ops/reboot-recovery" in text
assert "ops/maintenance" in text
assert "ops/monitoring" in text
assert "PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md" in text
@@ -83,6 +86,14 @@ def test_service_uses_deployed_source_root_and_bounded_oneshot() -> None:
assert "TimeoutStartSec=600" in text
def test_timer_waits_after_completion_instead_of_running_back_to_back() -> None:
text = TIMER.read_text(encoding="utf-8")
assert "OnBootSec=2min" in text
assert "OnUnitInactiveSec=5min" in text
assert "OnUnitActiveSec" not in text
def test_exporter_uses_user_writable_lock_after_creating_log_dir() -> None:
text = EXPORTER.read_text(encoding="utf-8")
@@ -108,9 +119,10 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
assert "windows99-vmware-verify.txt" in text
assert "collect-windows99-vmware-verify.sh\" --collect" in text
assert "WINDOWS99_MAX_AUTH_USERS=\"${WINDOWS99_MAX_AUTH_USERS:-5}\"" in text
assert "WINDOWS99_REMOTE_VERIFY_TIMEOUT=\"${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}\"" in text
assert "WINDOWS99_REQUIRED_VM_ALIASES=\"${WINDOWS99_REQUIRED_VM_ALIASES:-111 188 120 121 112}\"" in text
assert "WINDOWS99_SSH_USERS=\"${WINDOWS99_SSH_USERS:-Administrator}\"" in text
assert "WINDOWS99_MAX_AUTH_USERS=\"${WINDOWS99_MAX_AUTH_USERS:-1}\"" in text
assert "WINDOWS99_REMOTE_VERIFY_TIMEOUT=\"${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}\"" in text
assert "WINDOWS99_REQUIRED_VM_ALIASES=\"${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}\"" in text
assert "windows99_vmware_verify_alias_set_matches_current" in text
assert "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1" in text
assert "scorecard_args+=(--windows99-vmware-file" in text

60
scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
View File

@@ -77,8 +77,26 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
for host in ["99", "110", "111", "112", "120", "121", "188"]:
assert host in host_probe
assert 'run_with_timeout "${TCP_CONNECT_TIMEOUT_SECONDS:-2}"' in host_probe
assert 'WINDOWS99_SSH_TARGET="${WINDOWS99_SSH_TARGET:-Administrator@192.168.0.99}"' in host_probe
assert "probe_windows_ssh()" in host_probe
assert "Get-CimInstance Win32_OperatingSystem" in host_probe
assert "systemd_state=windows_ssh" in host_probe
assert '[[ "$boot_id" == windows_* && "$uptime_seconds" =~ ^[0-9]+$ ]]' in host_probe
assert '"111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy"' in host_probe
macos_readback = read("scripts/reboot-recovery/macos-host-boot-readback.sh")
macos_installer = read("scripts/reboot-recovery/install-macos111-host-boot-readback.sh")
assert "sysctl -n kern.boottime" in macos_readback
assert "s/^\\{ sec = ([0-9]+),.*/\\1/" in macos_readback
assert "boot_id=darwin_%s" in macos_readback
assert "127.0.0.1:11434/api/tags" in macos_readback
assert "restrict,command=" in macos_installer
assert "IdentitiesOnly=yes" in macos_installer
assert "CONTROL_PUBLIC_KEY_PATH" in macos_installer
assert "authorized_keys" in macos_installer
assert "AWOOOI-Start-VMware-VMs" in windows99
assert "NoAutoRebootWithLoggedOnUsers" in windows99
assert 'New-ItemProperty -Path $auPath -Name "AUPowerManagement" -Value 0' in windows99
assert "Host110Vmx" in windows99
assert "Host111Vmx" in windows99
assert "Host188Vmx" in windows99
@@ -88,10 +106,24 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
assert "EnableRecursiveDiscovery" in windows99
assert "D:\\Documents\\Virtual Machines" in windows99
assert "D:\\Downloads" in windows99
assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' in windows99
assert '[string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112")' in windows99
assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' not in windows99
assert '$SuppliedVmx["111"] -or $VmOrder -contains "111"' in windows99
agent99_update_policy = read("agent99-windows-update-policy.ps1")
assert 'New-ItemProperty -Path $auPolicy -Name "AUPowerManagement" -Value 0' in agent99_update_policy
assert "HOST111_VMWARE_TARGET=explicit_or_required" in windows99
assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99
assert "WINDOWS_UPDATE_POLICY" in windows99
exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh")
assert 'WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}"' in exporter
scorecard = read("scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py")
assert 'WINDOWS99_REQUIRED_VM_ALIASES = {"110", "112", "120", "121", "188"}' in scorecard
legacy_installer = read("scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1")
assert '[string[]]$VmNameAllowList = @("110", "188", "120", "121", "112")' in legacy_installer
assert "VM_POWER alias=$alias" in windows99
assert "Get-VmwareVmxProcessCommandLines" in windows99
assert "Get-VmxRunningSource" in windows99
@@ -186,7 +218,7 @@ def test_reboot_p0_contract_has_ai_log_triage_controlled_apply() -> None:
assert "HARBOR_REGISTRY_NOT_READY" in runner
assert "registry_public_ready" in runner
assert "registry_110_readback_missing" in runner
assert 'WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}"' in runner
assert 'WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}"' in runner
assert "WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS" in runner
assert "https://registry\\.wooo\\.work/v2/" in runner
assert "STOCK_UPSTREAM_NOT_READY" in runner
@@ -258,6 +290,7 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non
assert "192.168.0.125:32334" in ops_conf
assert "192.168.0.125:32335" in ops_conf
for marker in [
"location /api/sentry-tunnel {",
"location /api/ {",
"location ~ ^/api/v1/(dashboard/stream|agent/thinking) {",
"location /api/v1/ws {",
@@ -306,17 +339,32 @@ def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw(
def test_public_maintenance_edge_apply_script_is_guarded_and_reversible() -> None:
script = read("scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh")
helper = read("ops/reboot-recovery/awoooi-edge-maintenance-helper.sh")
bootstrap = read("scripts/reboot-recovery/bootstrap-edge-maintenance-executor.sh")
assert "MODE=\"check\"" in script
assert "MODE=\"apply\"" in script
assert "MODE=\"rollback\"" in script
assert "sudo -n true" in script
assert "sudo -n '$REMOTE_HELPER' check" in script
assert "sudo -n '$REMOTE_HELPER' apply" in script
assert "sudo -n '$REMOTE_HELPER' rollback-latest" in script
assert "edge_executor_payload_drift" in script
assert "BLOCKER privileged_edge_apply_channel_unavailable" in script
assert "nginx -t" in script
assert "systemctl reload nginx" in script
assert "BACKUP_CONF=" in script
assert "EDGE_FALLBACK_READY=1" in script
assert "EDGE_FALLBACK_READY=0" in script
assert "nginx -t" in helper
assert "systemctl reload nginx" in helper
assert "BACKUP_CONF=" in helper
assert "rollback-latest" in helper
assert "canonical_payload_not_root_owned" in helper
assert "canonical_payload_is_mutable" in helper
assert "grep -Ev '^[[:space:]]*#'" in helper
assert "docker run -i" in bootstrap
assert "--network=none" in bootstrap
assert "--userns=host" in bootstrap
assert "--pull=never" in bootstrap
assert "alpine@sha256:" in bootstrap
assert "visudo -cf" in bootstrap
assert "raw_secret" not in script.lower()
assert "docker restart" not in script
assert "reboot " not in script

21
scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py
View File

@@ -18,8 +18,8 @@ VALID_RED_VERIFY = """\
AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1
MODE=Verify
VMRUN_PRESENT=1
HOST111_VMWARE_TARGET=explicit_or_required
VMX alias=111 path=D:\\Documents\\Virtual Machines\\111\\111.vmx present=1
HOST111_VMWARE_TARGET=not_required_by_default_missing_no_vmx_found_on_99
VMX alias=110 path=D:\\Documents\\Virtual Machines\\110\\110.vmx present=1
VMX alias=188 path=D:\\Documents\\Virtual Machines\\188\\188.vmx present=1
VMX alias=120 path=D:\\Documents\\Virtual Machines\\120\\120.vmx present=1
VMX alias=121 path=D:\\Documents\\Virtual Machines\\121\\121.vmx present=1
@@ -28,7 +28,7 @@ VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Auto ok
VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Auto ok=1
VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1
WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1
VM_POWER alias=111 vmx_present=1 running=0
VM_POWER alias=110 vmx_present=1 running=0
VM_POWER alias=188 vmx_present=1 running=0
VM_POWER alias=120 vmx_present=1 running=0
VM_POWER alias=121 vmx_present=1 running=0
@@ -94,7 +94,8 @@ def test_valid_console_verify_red_artifact_is_accepted_and_normalized(
assert values["status"] == "valid_console_verify_artifact_vmware_not_ready"
assert values["missing_required_field_count"] == "0"
assert values["missing_vmx_aliases"] == ""
assert values["powered_off_aliases"] == "111,188,120,121,112"
assert values["required_aliases"] == "110,188,120,121,112"
assert values["powered_off_aliases"] == "110,188,120,121,112"
assert values["secret_value_read"] == "false"
assert normalized.read_text(encoding="utf-8").startswith(
"AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1\n"
@@ -152,11 +153,17 @@ def test_explicit_host111_artifact_can_be_validated_with_required_alias_override
raw = tmp_path / "console-111.txt"
raw.write_text(
VALID_RED_VERIFY.replace(
"HOST111_VMWARE_TARGET=explicit_or_required",
"HOST111_VMWARE_TARGET=not_required_by_default_missing_no_vmx_found_on_99",
"HOST111_VMWARE_TARGET=explicit_or_required",
)
.replace("VMX alias=111 path=D:\\Documents\\Virtual Machines\\111\\111.vmx present=1", "VMX alias=111 path= present=0")
.replace("VM_POWER alias=111 vmx_present=1 running=0", "VM_POWER alias=111 vmx_present=0 running=0"),
.replace(
"VMX alias=110 path=D:\\Documents\\Virtual Machines\\110\\110.vmx present=1",
"VMX alias=111 path= present=0",
)
.replace(
"VM_POWER alias=110 vmx_present=1 running=0",
"VM_POWER alias=111 vmx_present=0 running=0",
),
encoding="utf-8",
)

6
scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py
View File

@@ -49,6 +49,12 @@ def test_collector_contract_forbids_secret_and_runtime_actions() -> None:
assert "KbdInteractiveAuthentication=no" in text
assert "NumberOfPasswordPrompts=0" in text
assert "ConnectionAttempts=1" in text
assert "ServerAliveInterval=5" in text
assert "ServerAliveCountMax=2" in text
assert "StrictHostKeyChecking=accept-new" in text
assert "StrictHostKeyChecking=no" not in text
assert 'REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}"' in text
assert "SSH_USERS=(Administrator ogt wooo ooo administrator)" in text
assert "GSSAPIAuthentication=no" in text
assert "--via-host" in text
assert "via_host=" in text