from __future__ import annotations import importlib.util import json import subprocess import sys from pathlib import Path from types import ModuleType ROOT = Path(__file__).resolve().parents[3] def read(path: str) -> str: return (ROOT / path).read_text(encoding="utf-8") def load_public_maintenance_probe() -> ModuleType: path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py" spec = importlib.util.spec_from_file_location( "awoooi_public_maintenance_fallback_probe", path, ) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def location_block(conf: str, marker: str) -> str: start = conf.index(marker) brace = conf.index("{", start) depth = 0 for index in range(brace, len(conf)): char = conf[index] if char == "{": depth += 1 elif char == "}": depth -= 1 if depth == 0: return conf[start : index + 1] raise AssertionError(f"unterminated nginx location block: {marker}") def assert_proxy_location_has_l0_fallback(conf: str, marker: str) -> None: block = location_block(conf, marker) assert "proxy_pass " in block assert "proxy_intercept_errors on;" in block assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in block class FakeHttpResponse: def __init__(self, status: int, headers: dict[str, str], body: bytes) -> None: self.status = status self.headers = headers self.body = body def __enter__(self) -> FakeHttpResponse: return self def __exit__(self, *_args: object) -> None: return None def getcode(self) -> int: return self.status def read(self, _max_bytes: int) -> bytes: return self.body def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> None: host_probe = read("scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh") windows99 = read("scripts/reboot-recovery/windows99-vmware-autostart.ps1") for host in ["99", "110", "111", "112", "120", "121", "188"]: assert host in host_probe assert 'run_with_timeout "${TCP_CONNECT_TIMEOUT_SECONDS:-2}"' in host_probe assert "AWOOOI-Start-VMware-VMs" in windows99 assert "NoAutoRebootWithLoggedOnUsers" in windows99 assert "Host110Vmx" in windows99 assert "Host111Vmx" in windows99 assert "Host188Vmx" in windows99 assert "Host120Vmx" in windows99 assert "Host121Vmx" in windows99 assert "Host112Vmx" in windows99 assert "EnableRecursiveDiscovery" in windows99 assert "D:\\Documents\\Virtual Machines" in windows99 assert "D:\\Downloads" in windows99 assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' in windows99 assert "HOST111_VMWARE_TARGET=explicit_or_required" in windows99 assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99 assert "WINDOWS_UPDATE_POLICY" in windows99 assert "VM_POWER alias=$alias" in windows99 assert "Get-VmwareVmxProcessCommandLines" in windows99 assert "Get-VmxRunningSource" in windows99 assert "vmware_vmx_process" in windows99 assert "source=$runningSource" in windows99 assert "Win32_Process" in windows99 assert "VM_ALREADY_RUNNING" in windows99 assert "VMWARE_AUTOSTART_TASK_DETAIL" in windows99 assert "last_task_result" in windows99 assert "start_script_present" in windows99 assert "VMWARE_AUTOSTART_TASK_ACTION" in windows99 verify_exit = windows99.index('if ($Mode -eq "Verify")') apply_start = windows99.index("Write-StartupScript -VmMap $vmMap") assert verify_exit < apply_start assert 'if ($Mode -eq "Verify") {\n exit 0\n}' in windows99 assert "Restart-Computer" not in windows99 assert "Stop-Computer" not in windows99 def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: alerts = read("ops/monitoring/alerts-unified.yml") runbook = read("docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md") snippet = read("ops/maintenance/nginx-502-maintenance-snippet.conf") probe = read("scripts/reboot-recovery/public-maintenance-fallback-probe.py") exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") for alert_name in [ "HostRebootEventDetected", "RebootAutoRecoverySLOMissed", "RebootAutoRecoveryActiveBlocker", "RebootAutoRecoveryActiveBlockerMetricMissing", "PublicRouteServing5xxWithoutMaintenanceFallback", "BackupCoverageDomainStale", ]: assert alert_name in alerts assert "notification_type: TYPE-3" in alerts assert "proxy_intercept_errors on" in snippet assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in snippet assert "X-AWOOOI-Fallback" in snippet assert "502" in runbook assert "L0" in runbook assert "L1" in runbook assert "raw_5xx_without_fallback" in probe assert "status in {200, 502, 503, 504}" in probe assert "public-maintenance-fallback.json" in exporter assert "--public-maintenance-file" in exporter assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter assert "awoooi_reboot_auto_recovery_slo_active_blocker" in exporter assert "awoooi_windows99_no_secret_collector_readback_present" in exporter assert "awoooi_windows99_no_secret_collector_status" in exporter assert ( "awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready" in exporter ) assert "awoooi_windows99_no_secret_collector_port_22_open" in exporter assert "awoooi_windows99_no_secret_collector_remote_verify_attempted" in exporter assert 'blocker="{label(blocker)}"' in exporter assert "$labels.blocker" in alerts def test_reboot_p0_contract_has_ai_log_triage_controlled_apply() -> None: runner = read("scripts/reboot-recovery/ai-log-triage-auto-recovery.sh") exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") service = read("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service") full_host_orchestrator = read( "scripts/reboot-recovery/full-host-reboot-orchestrator.sh" ) full_host_service = read( "scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service" ) full_host_installer = read( "scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh" ) vmware_autostart = read( "scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1" ) windows_update = read( "scripts/reboot-recovery/windows-99/configure-windows-update-no-surprise-reboot.ps1" ) assert "MODE=\"check\"" in runner assert "--apply" in runner assert "TOO_MANY_OPEN_FILES" in runner assert "APPLY_FD_BUDGET_FOLLOWUP" in runner assert "LimitNOFILE=1048576" in runner assert "systemctl daemon-reload" in runner assert "RESTART_PERFORMED=false" in runner assert "SSH_COMMAND_SESSION_TIMEOUT" in runner assert "SSH_AUTH_FAILED" in runner assert "DOCKER_SCOPE_TIMEOUT" in runner assert "CONTAINERD_TASK_CLEANUP_STUCK" in runner assert "HARBOR_REGISTRY_NOT_READY" in runner assert "registry_public_ready" in runner assert "registry_110_readback_missing" in runner assert 'WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}"' in runner assert "WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS" in runner assert "https://registry\\.wooo\\.work/v2/" in runner assert "STOCK_UPSTREAM_NOT_READY" in runner assert "https://stock\\.wooo\\.work/api/v1/system/freshness" in runner assert "K3S_IMAGE_PULL_BLOCKED_BY_REGISTRY" in runner assert "^awoooi-[^[:space:]]+" in runner assert "EDGE_502_UPSTREAM_REFUSED" in runner assert "WINDOWS99_KALI_LOCK_IN_USE" in runner assert "harbor-watchdog.sh --repair-once" in runner assert "delete pod --wait=false" in runner assert "systemctl start" in runner assert "Forbidden even in --apply" in runner assert "node drain" in runner assert "Docker prune" in runner assert "ai-log-triage-auto-recovery.sh" in exporter assert "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" in exporter assert "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply" in service assert "run_ai_log_triage" in full_host_orchestrator assert "run_scorecard" in full_host_orchestrator assert "run_stock_upstream_gate_hint" in full_host_orchestrator assert "TELEGRAM_NOTIFY_SCRIPT" in full_host_orchestrator assert 'rc=$?' in full_host_orchestrator assert '"$rc" -ne 2' in full_host_orchestrator assert "AI_LOG_TRIAGE_REPORTED_FINDINGS" in full_host_orchestrator assert 'host=" $3 " evidence=" $4 " next_action=" $5' in full_host_orchestrator assert "/Users/ogt/" not in full_host_service assert "WorkingDirectory=/opt/awoooi/reboot-recovery" in full_host_service assert ( "/opt/awoooi/reboot-recovery/scripts/reboot-recovery/" "full-host-reboot-orchestrator.sh --apply" in full_host_service ) assert 'MODE="dry-run"' in full_host_installer assert 'UNIT_NAME="awoooi-full-host-reboot-orchestrator"' in full_host_installer assert "ENABLE_NOW=0" in full_host_installer assert "--enable-now" in full_host_installer assert "--verify-only" in full_host_installer assert "--rollback" in full_host_installer assert "DRY_RUN=1" in full_host_installer assert 'systemctl enable --now "${UNIT_NAME}.timer"' in full_host_installer assert "--delete" not in full_host_installer assert "awooooi-full-host-reboot-orchestrator" not in full_host_installer assert "awooi-full-host-reboot-orchestrator" not in full_host_installer assert "[switch]$WhatIfOnly" in vmware_autostart assert "if ($WhatIfOnly)" in vmware_autostart assert "[switch]$WhatIfOnly" in windows_update assert "WHATIF windows_update_policy" in windows_update def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> None: for path in [ "ops/nginx/awoooi.wooo.work.conf", "k8s/nginx/awoooi-prod.conf", ]: conf = read(path) assert "proxy_intercept_errors on;" in conf assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in conf assert "location = /__awoooi-maintenance.html" in conf assert "root /var/www/maintenance;" in conf assert "try_files /maintenance.html =503;" in conf assert 'add_header Retry-After "120" always;' in conf assert 'add_header X-AWOOOI-Fallback "local-maintenance" always;' in conf assert "/50x.html" not in conf ops_conf = read("ops/nginx/awoooi.wooo.work.conf") assert "192.168.0.125:32334" in ops_conf assert "192.168.0.125:32335" in ops_conf for marker in [ "location /api/ {", "location ~ ^/api/v1/(dashboard/stream|agent/thinking) {", "location /api/v1/ws {", "location / {\n proxy_pass http://awoooi_web;", "location /_next/static/ {", ]: assert_proxy_location_has_l0_fallback(ops_conf, marker) k8s_conf = read("k8s/nginx/awoooi-prod.conf") for marker in [ "location ~ ^/api/v1/(agent|dashboard)/stream {", "location /api/sentry-tunnel {", "location /api/ {", "location /api/health {", "location / {\n proxy_pass http://awoooi_prod_web;", "location ~* \\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2)$ {", ]: assert_proxy_location_has_l0_fallback(k8s_conf, marker) def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw( monkeypatch, ) -> None: probe = load_public_maintenance_probe() def fake_urlopen(_request, timeout: float) -> FakeHttpResponse: assert timeout == 8.0 return FakeHttpResponse( 502, {"X-AWOOOI-Fallback": "local-maintenance"}, b"HTTP 502 / 503 / 504 fallback page", ) monkeypatch.setattr(probe.urllib.request, "urlopen", fake_urlopen) route = probe.fetch_url( "https://awoooi.wooo.work/api/v1/health", timeout=8.0, max_bytes=65536, ) assert route["status"] == "maintenance_fallback_serving" assert route["maintenance_fallback_serving"] is True assert route["raw_5xx_without_fallback"] is False def test_public_maintenance_edge_apply_script_is_guarded_and_reversible() -> None: script = read("scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh") assert "MODE=\"check\"" in script assert "MODE=\"apply\"" in script assert "MODE=\"rollback\"" in script assert "sudo -n true" in script assert "BLOCKER privileged_edge_apply_channel_unavailable" in script assert "nginx -t" in script assert "systemctl reload nginx" in script assert "BACKUP_CONF=" in script assert "EDGE_FALLBACK_READY=1" in script assert "EDGE_FALLBACK_READY=0" in script assert "raw_secret" not in script.lower() assert "docker restart" not in script assert "reboot " not in script def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None: summary = tmp_path / "summary.txt" summary.write_text( "\n".join( [ "SERVICE_GREEN=1", "PRODUCT_DATA_GREEN=1", "BACKUP_CORE_GREEN=1", "POST_START_BLOCKED=0", "HOST_188_SERVICE_GREEN=1", ] ), encoding="utf-8", ) host_probe = tmp_path / "host-probe.txt" host_probe.write_text( "\n".join( f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120" for alias in ["99", "110", "111", "112", "120", "121", "188"] ), encoding="utf-8", ) reboot_event = tmp_path / "reboot-event.json" reboot_event.write_text( json.dumps( { "reboot_detected": True, "all_required_hosts_observed": True, "all_required_hosts_in_reboot_window": True, "fresh_boot_hosts": ["99", "110", "111", "112", "120", "121", "188"], "missing_hosts": [], "unreachable_hosts": [], "target_seconds_remaining": 480, "recovery_deadline_status": "inside_target_window", } ), encoding="utf-8", ) public_probe = tmp_path / "public-maintenance.json" public_probe.write_text( json.dumps( { "runtime_readback_present": True, "ready": False, "raw_5xx_without_fallback_count": 1, "route_unreachable_without_external_fallback_count": 0, "maintenance_fallback_route_count": 0, "raw_5xx_without_fallback_urls": ["https://awoooi.wooo.work/"], "blockers": ["public_route_raw_5xx_without_maintenance_fallback"], "routes": [ { "url": "https://awoooi.wooo.work/", "http_status": 502, "status": "raw_5xx_without_maintenance_fallback", "fallback_header": "", "maintenance_body_detected": False, "maintenance_fallback_serving": False, "raw_5xx_without_fallback": True, } ], } ), encoding="utf-8", ) windows99 = tmp_path / "windows99-vmware.txt" windows99.write_text( "\n".join( [ "VMRUN_PRESENT=1", "VMWARE_AUTOSTART_CONFIG_READY=1", "VMWARE_AUTOSTART_POWER_READY=1", "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1", "VMWARE_AUTOSTART_VERIFY_READY=1", "VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Automatic ok=1", "VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Automatic ok=1", "VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1", "WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1", *[ f"VMX alias={alias} path=D:\\Documents\\Virtual Machines\\{alias}\\{alias}.vmx present=1" for alias in ["111", "112", "120", "121", "188"] ], *[ f"VM_POWER alias={alias} vmx_present=1 running=1" for alias in ["111", "112", "120", "121", "188"] ], ] ), encoding="utf-8", ) result = subprocess.run( [ sys.executable, str(ROOT / "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"), "--summary-file", str(summary), "--host-probe-file", str(host_probe), "--reboot-event-file", str(reboot_event), "--public-maintenance-file", str(public_probe), "--windows99-vmware-file", str(windows99), ], text=True, capture_output=True, check=True, ) payload = json.loads(result.stdout) assert "public_route_raw_5xx_without_maintenance_fallback" in payload["active_blockers"] assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is False assert payload["current_phase"] == "public_maintenance_fallback_blocked" assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 1 def test_backup_exporter_emits_domain_level_backup_coverage() -> None: exporter = read("scripts/ops/backup-health-textfile-exporter.py") assert "awoooi_backup_coverage_domain_expected_info" in exporter assert "awoooi_backup_coverage_domain_fresh" in exporter assert "awoooi_backup_alert_receipt_expected_info" in exporter assert "awoooi_backup_alert_receipt_stage_fresh" in exporter assert "BACKUP_ALERT_RECEIPT_STAGES = (\"sent\", \"received\", \"dedup\", \"escalation\")" in exporter assert "AIOPS_BACKUP_ALERT_RECEIPT_DIR" in exporter assert "gitea_repo_mirror_from_110" in exporter assert "/home/ollama/backup/110/gitea" in exporter for domain in ["host", "database", "website", "service", "package", "tool", "log"]: assert f'"{domain}"' in exporter for alertname in [ "BackupJobStale", "BackupCoverageDomainStale", "GiteaPrivateBundleBackupIncomplete", "GiteaPrivateBundleRestoreDryRunMissing", "BackupRestoreDrillMissingOrFailed", "GiteaRestoreDrillMissingOrFailed", "BackupCredentialEscrowEvidenceMissing", ]: assert alertname in exporter def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: alerts = read("ops/monitoring/alerts-unified.yml") assert 'awoooi_backup_coverage_domain_expected_info{host="188"}' in alerts assert "BackupCoverageDomainMetricMissing188" in alerts assert "awoooi_backup_coverage_domain_fresh == 0" in alerts assert "GiteaPrivateBundleRestoreDryRunMissing" in alerts assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in alerts assert "BackupAlertReceiptMetricMissing" in alerts assert "BackupAlertReceiptStageMissing" in alerts assert "awoooi_backup_alert_receipt_expected_info" in alerts assert 'ALERTS{' in alerts assert 'alertname!="BackupAlertReceiptStageMissing"' in alerts assert "label_replace(" in alerts assert 'awoooi_backup_alert_receipt_fresh == 1' in alerts assert "$labels.alertname" in alerts def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: script = read("scripts/backup/gitea-repo-bundle-backup.sh") restore_script = read("scripts/backup/gitea-bundle-sample-restore-dry-run.sh") wrapper = read("scripts/backup/gitea-bundle-backup-sync-188.sh") exporter = read("scripts/ops/backup-health-textfile-exporter.py") playbook = read("infra/ansible/playbooks/110-devops.yml") assert "GIT_TERMINAL_PROMPT=0" in script assert "GIT_REMOTE_BASE" in script assert "--git-remote-base" in script assert "repo_remote_url" in script assert "bundle create" in script assert "bundle verify" in script assert ".sha256" in script assert "manifest.tsv" in script assert "latest-private-complete" in script assert "complete links unchanged" in script assert 'exit 1' in script assert "gitea dump" in script assert "does not replace `gitea dump`" in script assert "tokens or passwords" in script assert "git clone --mirror --quiet" in restore_script assert "fsck --connectivity-only" in restore_script assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in restore_script assert "never reads tokens" in restore_script assert "flock -n" in wrapper assert "latest-private-complete" in wrapper assert "--local-container" in wrapper assert "rsync -a --checksum" in wrapper assert "remote_sample_restore_fsck" in wrapper assert "refresh_remote_backup_health" in wrapper assert "rollback_remote_links" in wrapper assert "remote_manifest_checksum_ok" in wrapper assert "promoted=1" in wrapper assert '"gitea_bundle_sync_188"' in exporter assert 'job="gitea_private_bundle"' in exporter assert "gitea-bundle-backup-sync-188.sh" in playbook assert 'minute: "15"' in playbook assert 'hour: "0"' in playbook def test_prometheus_rule_guard_repairs_container_bind_mount_drift() -> None: guard = read("scripts/ops/prometheus-rule-drift-guard.sh") playbook = read("infra/ansible/playbooks/110-devops.yml") assert "runtime_matches_current" in guard assert "awoooi_prometheus_rule_drift_guard_runtime_matches_current" in guard assert "promtool check rules" in guard assert "--no-deps --force-recreate" in guard assert "failed to create active rules rollback copy" in guard assert "post-apply verifier failed; active rules rolled back" in guard assert "prometheus-rule-drift-guard.sh" in playbook assert 'minute: "*/5"' in playbook def test_velero_restore_test_recovers_a_missed_weekly_schedule() -> None: manifest = read("k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml") assert "startingDeadlineSeconds: 604800" in manifest assert 'schedule: "0 18 * * 6"' in manifest assert "concurrencyPolicy: Forbid" in manifest