from __future__ import annotations import importlib.util import json import subprocess import sys from pathlib import Path from types import ModuleType ROOT = Path(__file__).resolve().parents[3] def read(path: str) -> str: return (ROOT / path).read_text(encoding="utf-8") def load_public_maintenance_probe() -> ModuleType: path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py" spec = importlib.util.spec_from_file_location( "awoooi_public_maintenance_fallback_probe", path, ) assert spec is not None assert spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module def location_block(conf: str, marker: str) -> str: start = conf.index(marker) brace = conf.index("{", start) depth = 0 for index in range(brace, len(conf)): char = conf[index] if char == "{": depth += 1 elif char == "}": depth -= 1 if depth == 0: return conf[start : index + 1] raise AssertionError(f"unterminated nginx location block: {marker}") def assert_proxy_location_has_l0_fallback(conf: str, marker: str) -> None: block = location_block(conf, marker) assert "proxy_pass " in block assert "proxy_intercept_errors on;" in block assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in block class FakeHttpResponse: def __init__(self, status: int, headers: dict[str, str], body: bytes) -> None: self.status = status self.headers = headers self.body = body def __enter__(self) -> FakeHttpResponse: return self def __exit__(self, *_args: object) -> None: return None def getcode(self) -> int: return self.status def read(self, _max_bytes: int) -> bytes: return self.body def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> None: host_probe = read("scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh") windows99 = read("scripts/reboot-recovery/windows99-vmware-autostart.ps1") for host in ["99", "110", "111", "112", "120", "121", "188"]: assert host in host_probe assert 'run_with_timeout "${TCP_CONNECT_TIMEOUT_SECONDS:-2}"' in host_probe assert 'WINDOWS99_SSH_TARGET="${WINDOWS99_SSH_TARGET:-Administrator@192.168.0.99}"' in host_probe assert "probe_windows_ssh()" in host_probe assert "Get-CimInstance Win32_OperatingSystem" in host_probe assert "systemd_state=windows_ssh" in host_probe assert '[[ "$boot_id" == windows_* && "$uptime_seconds" =~ ^[0-9]+$ ]]' in host_probe assert '"111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy"' in host_probe assert '"112=kali@192.168.0.112:awoooi-host112-guest-recovery.timer"' in host_probe assert "probe_host112_guest()" in host_probe for field in ["console_ready", "services_ready", "recovery_timer_ready", "guest_ready"]: assert field in host_probe host112_readiness = read("scripts/reboot-recovery/host112-guest-readiness.sh") host112_installer = read("scripts/reboot-recovery/install-host112-guest-recovery.sh") host112_service = read( "scripts/reboot-recovery/awoooi-host112-guest-recovery.service" ) host112_timer = read("scripts/reboot-recovery/awoooi-host112-guest-recovery.timer") assert "systemctl set-default graphical.target" in host112_readiness assert "start_if_inactive lightdm.service" in host112_readiness assert "start_if_inactive open-vm-tools.service" in host112_readiness assert "enable_if_disabled ssh.service enable_ssh" in host112_readiness assert "start_if_inactive ssh.service start_ssh" in host112_readiness assert "ssh_port_listening=$ssh_port_listening" in host112_readiness assert "ssh_ready=$ssh_ready" in host112_readiness assert "INDEXER_RETRY_COOLDOWN_SECONDS" in host112_readiness assert "console_ready=$console_ready" in host112_readiness assert "guest_ready=$guest_ready" in host112_readiness assert "restrict,command=" in host112_installer assert "visudo -cf" in host112_installer assert "--rollback" in host112_installer assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service assert "TimeoutStartSec=480" in host112_service assert "OnBootSec=20s" in host112_timer assert "OnUnitInactiveSec=60s" in host112_timer macos_readback = read("scripts/reboot-recovery/macos-host-boot-readback.sh") macos_installer = read("scripts/reboot-recovery/install-macos111-host-boot-readback.sh") assert "sysctl -n kern.boottime" in macos_readback assert "s/^\\{ sec = ([0-9]+),.*/\\1/" in macos_readback assert "boot_id=darwin_%s" in macos_readback assert "127.0.0.1:11434/api/tags" in macos_readback assert "restrict,command=" in macos_installer assert "IdentitiesOnly=yes" in macos_installer assert "CONTROL_PUBLIC_KEY_PATH" in macos_installer assert "authorized_keys" in macos_installer assert "AWOOOI-Start-VMware-VMs" in windows99 assert "NoAutoRebootWithLoggedOnUsers" in windows99 assert 'New-ItemProperty -Path $auPath -Name "AUPowerManagement" -Value 0' in windows99 assert "Host110Vmx" in windows99 assert "Host111Vmx" in windows99 assert "Host188Vmx" in windows99 assert "Host120Vmx" in windows99 assert "Host121Vmx" in windows99 assert "Host112Vmx" in windows99 assert "EnableRecursiveDiscovery" in windows99 assert "D:\\Documents\\Virtual Machines" in windows99 assert "D:\\Downloads" in windows99 assert '[string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112")' in windows99 assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' not in windows99 assert '$SuppliedVmx["111"] -or $VmOrder -contains "111"' in windows99 agent99_update_policy = read("agent99-windows-update-policy.ps1") assert 'New-ItemProperty -Path $auPolicy -Name "AUPowerManagement" -Value 0' in agent99_update_policy assert "HOST111_VMWARE_TARGET=explicit_or_required" in windows99 assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99 assert "WINDOWS_UPDATE_POLICY" in windows99 exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") assert 'WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}"' in exporter scorecard = read("scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py") assert 'WINDOWS99_REQUIRED_VM_ALIASES = {"110", "112", "120", "121", "188"}' in scorecard legacy_installer = read("scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1") assert '[string[]]$VmNameAllowList = @("110", "188", "120", "121", "112")' in legacy_installer assert "VM_POWER alias=$alias" in windows99 assert "Get-VmwareVmxProcessCommandLines" in windows99 assert "Get-VmxRunningSource" in windows99 assert "vmware_vmx_process" in windows99 assert "source=$runningSource" in windows99 assert "Win32_Process" in windows99 assert "VM_ALREADY_RUNNING" in windows99 assert "VMWARE_AUTOSTART_TASK_DETAIL" in windows99 assert "last_task_result" in windows99 assert "start_script_present" in windows99 assert "VMWARE_AUTOSTART_TASK_ACTION" in windows99 assert "Get-VmwareVmxProcessReadback" in windows99 assert "VM_PROCESS alias=$alias" in windows99 assert "runtime_running_gui_session_isolated" in windows99 verify_exit = windows99.index('if ($Mode -eq "Verify")') apply_start = windows99.index("Write-StartupScript -VmMap $vmMap") assert verify_exit < apply_start assert 'if ($Mode -eq "Verify") {\n exit 0\n}' in windows99 assert "Restart-Computer" not in windows99 assert "Stop-Computer" not in windows99 def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: alerts = read("ops/monitoring/alerts-unified.yml") runbook = read("docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md") snippet = read("ops/maintenance/nginx-502-maintenance-snippet.conf") probe = read("scripts/reboot-recovery/public-maintenance-fallback-probe.py") exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") for alert_name in [ "HostRebootEventDetected", "RebootAutoRecoverySLOMissed", "RebootAutoRecoveryActiveBlocker", "RebootAutoRecoveryActiveBlockerMetricMissing", "PublicRouteServing5xxWithoutMaintenanceFallback", "BackupCoverageDomainStale", ]: assert alert_name in alerts assert "notification_type: TYPE-3" in alerts assert "proxy_intercept_errors on" in snippet assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in snippet assert "X-AWOOOI-Fallback" in snippet assert "502" in runbook assert "L0" in runbook assert "L1" in runbook assert "raw_5xx_without_fallback" in probe assert "status in {200, 502, 503, 504}" in probe assert "public-maintenance-fallback.json" in exporter assert "--public-maintenance-file" in exporter assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter assert "awoooi_reboot_auto_recovery_slo_active_blocker" in exporter assert "awoooi_windows99_no_secret_collector_readback_present" in exporter assert "awoooi_windows99_no_secret_collector_status" in exporter assert ( "awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready" in exporter ) assert "awoooi_windows99_no_secret_collector_port_22_open" in exporter assert "awoooi_windows99_no_secret_collector_remote_verify_attempted" in exporter assert 'blocker="{label(blocker)}"' in exporter assert "$labels.blocker" in alerts def test_reboot_p0_contract_has_ai_log_triage_controlled_apply() -> None: runner = read("scripts/reboot-recovery/ai-log-triage-auto-recovery.sh") exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") service = read("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service") full_host_orchestrator = read( "scripts/reboot-recovery/full-host-reboot-orchestrator.sh" ) full_host_service = read( "scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service" ) full_host_installer = read( "scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh" ) vmware_autostart = read( "scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1" ) windows_update = read( "scripts/reboot-recovery/windows-99/configure-windows-update-no-surprise-reboot.ps1" ) assert "MODE=\"check\"" in runner assert "--apply" in runner assert "TOO_MANY_OPEN_FILES" in runner assert "APPLY_FD_BUDGET_FOLLOWUP" in runner assert "LimitNOFILE=1048576" in runner assert "systemctl daemon-reload" in runner assert "RESTART_PERFORMED=false" in runner assert "SSH_COMMAND_SESSION_TIMEOUT" in runner assert "SSH_AUTH_FAILED" in runner assert "DOCKER_SCOPE_TIMEOUT" in runner assert "CONTAINERD_TASK_CLEANUP_STUCK" in runner assert "HARBOR_REGISTRY_NOT_READY" in runner assert "registry_public_ready" in runner assert "registry_110_readback_missing" in runner assert 'WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}"' in runner assert "WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS" in runner assert "https://registry\\.wooo\\.work/v2/" in runner assert "STOCK_UPSTREAM_NOT_READY" in runner assert "https://stock\\.wooo\\.work/api/v1/system/freshness" in runner assert "K3S_IMAGE_PULL_BLOCKED_BY_REGISTRY" in runner assert "^awoooi-[^[:space:]]+" in runner assert "EDGE_502_UPSTREAM_REFUSED" in runner assert "WINDOWS99_KALI_LOCK_IN_USE" in runner assert "harbor-watchdog.sh --repair-once" in runner assert "delete pod --wait=false" in runner assert "systemctl start" in runner assert "Forbidden even in --apply" in runner assert "node drain" in runner assert "Docker prune" in runner assert "ai-log-triage-auto-recovery.sh" in exporter assert "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" in exporter assert "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply" in service assert "run_ai_log_triage" in full_host_orchestrator assert "run_scorecard" in full_host_orchestrator assert "run_stock_upstream_gate_hint" in full_host_orchestrator assert "TELEGRAM_NOTIFY_SCRIPT" in full_host_orchestrator assert 'rc=$?' in full_host_orchestrator assert '"$rc" -ne 2' in full_host_orchestrator assert "AI_LOG_TRIAGE_REPORTED_FINDINGS" in full_host_orchestrator assert 'host=" $3 " evidence=" $4 " next_action=" $5' in full_host_orchestrator assert "/Users/ogt/" not in full_host_service assert "WorkingDirectory=/opt/awoooi/reboot-recovery" in full_host_service assert ( "/opt/awoooi/reboot-recovery/scripts/reboot-recovery/" "full-host-reboot-orchestrator.sh --apply" in full_host_service ) assert 'MODE="dry-run"' in full_host_installer assert 'UNIT_NAME="awoooi-full-host-reboot-orchestrator"' in full_host_installer assert "ENABLE_NOW=0" in full_host_installer assert "--enable-now" in full_host_installer assert "--verify-only" in full_host_installer assert "--rollback" in full_host_installer assert "DRY_RUN=1" in full_host_installer assert 'systemctl enable --now "${UNIT_NAME}.timer"' in full_host_installer assert "--delete" not in full_host_installer assert "awooooi-full-host-reboot-orchestrator" not in full_host_installer assert "awooi-full-host-reboot-orchestrator" not in full_host_installer assert "[switch]$WhatIfOnly" in vmware_autostart assert "if ($WhatIfOnly)" in vmware_autostart assert "[switch]$WhatIfOnly" in windows_update assert "WHATIF windows_update_policy" in windows_update def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> None: for path in [ "ops/nginx/awoooi.wooo.work.conf", "k8s/nginx/awoooi-prod.conf", ]: conf = read(path) assert "proxy_intercept_errors on;" in conf assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in conf assert "location = /__awoooi-maintenance.html" in conf assert "root /var/www/maintenance;" in conf assert "try_files /maintenance.html =503;" in conf assert 'add_header Retry-After "120" always;' in conf assert 'add_header X-AWOOOI-Fallback "local-maintenance" always;' in conf assert "/50x.html" not in conf ops_conf = read("ops/nginx/awoooi.wooo.work.conf") assert "192.168.0.125:32334" in ops_conf assert "192.168.0.125:32335" in ops_conf for marker in [ "location /api/sentry-tunnel {", "location /api/ {", "location ~ ^/api/v1/(dashboard/stream|agent/thinking) {", "location /api/v1/ws {", "location / {\n proxy_pass http://awoooi_web;", "location /_next/static/ {", ]: assert_proxy_location_has_l0_fallback(ops_conf, marker) k8s_conf = read("k8s/nginx/awoooi-prod.conf") for marker in [ "location ~ ^/api/v1/(agent|dashboard)/stream {", "location /api/sentry-tunnel {", "location /api/ {", "location /api/health {", "location / {\n proxy_pass http://awoooi_prod_web;", "location ~* \\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2)$ {", ]: assert_proxy_location_has_l0_fallback(k8s_conf, marker) def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw( monkeypatch, ) -> None: probe = load_public_maintenance_probe() def fake_urlopen(_request, timeout: float) -> FakeHttpResponse: assert timeout == 8.0 return FakeHttpResponse( 502, {"X-AWOOOI-Fallback": "local-maintenance"}, b"HTTP 502 / 503 / 504 fallback page", ) monkeypatch.setattr(probe.urllib.request, "urlopen", fake_urlopen) route = probe.fetch_url( "https://awoooi.wooo.work/api/v1/health", timeout=8.0, max_bytes=65536, ) assert route["status"] == "maintenance_fallback_serving" assert route["maintenance_fallback_serving"] is True assert route["raw_5xx_without_fallback"] is False def test_public_maintenance_edge_apply_script_is_guarded_and_reversible() -> None: script = read("scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh") helper = read("ops/reboot-recovery/awoooi-edge-maintenance-helper.sh") bootstrap = read("scripts/reboot-recovery/bootstrap-edge-maintenance-executor.sh") assert "MODE=\"check\"" in script assert "MODE=\"apply\"" in script assert "MODE=\"rollback\"" in script assert "sudo -n '$REMOTE_HELPER' check" in script assert "sudo -n '$REMOTE_HELPER' apply" in script assert "sudo -n '$REMOTE_HELPER' rollback-latest" in script assert "edge_executor_payload_drift" in script assert "BLOCKER privileged_edge_apply_channel_unavailable" in script assert "EDGE_FALLBACK_READY=1" in script assert "EDGE_FALLBACK_READY=0" in script assert "nginx -t" in helper assert "systemctl reload nginx" in helper assert "BACKUP_CONF=" in helper assert "rollback-latest" in helper assert "canonical_payload_not_root_owned" in helper assert "canonical_payload_is_mutable" in helper assert "grep -Ev '^[[:space:]]*#'" in helper assert "docker run -i" in bootstrap assert "--network=none" in bootstrap assert "--userns=host" in bootstrap assert "--pull=never" in bootstrap assert "alpine@sha256:" in bootstrap assert "visudo -cf" in bootstrap assert "raw_secret" not in script.lower() assert "docker restart" not in script assert "reboot " not in script def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None: summary = tmp_path / "summary.txt" summary.write_text( "\n".join( [ "SERVICE_GREEN=1", "PRODUCT_DATA_GREEN=1", "BACKUP_CORE_GREEN=1", "POST_START_BLOCKED=0", "HOST_188_SERVICE_GREEN=1", ] ), encoding="utf-8", ) host_probe = tmp_path / "host-probe.txt" host_probe.write_text( "\n".join( [ f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120" for alias in ["99", "110", "111", "120", "121", "188"] ] + [ "HOST_BOOT alias=112 host=192.168.0.112 reachable=1 uptime_seconds=120 " "systemd_state=running default_target=graphical.target graphical_target=active " "display_manager=active lightdm=active vmware_tools=active xorg=active " "wazuh_indexer=active wazuh_manager=active wazuh_dashboard=active filebeat=active " "console_ready=1 services_ready=1 recovery_timer_ready=1 guest_ready=1" ] ), encoding="utf-8", ) reboot_event = tmp_path / "reboot-event.json" reboot_event.write_text( json.dumps( { "reboot_detected": True, "all_required_hosts_observed": True, "all_required_hosts_in_reboot_window": True, "fresh_boot_hosts": ["99", "110", "111", "112", "120", "121", "188"], "missing_hosts": [], "unreachable_hosts": [], "target_seconds_remaining": 480, "recovery_deadline_status": "inside_target_window", } ), encoding="utf-8", ) public_probe = tmp_path / "public-maintenance.json" public_probe.write_text( json.dumps( { "runtime_readback_present": True, "ready": False, "raw_5xx_without_fallback_count": 1, "route_unreachable_without_external_fallback_count": 0, "maintenance_fallback_route_count": 0, "raw_5xx_without_fallback_urls": ["https://awoooi.wooo.work/"], "blockers": ["public_route_raw_5xx_without_maintenance_fallback"], "routes": [ { "url": "https://awoooi.wooo.work/", "http_status": 502, "status": "raw_5xx_without_maintenance_fallback", "fallback_header": "", "maintenance_body_detected": False, "maintenance_fallback_serving": False, "raw_5xx_without_fallback": True, } ], } ), encoding="utf-8", ) windows99 = tmp_path / "windows99-vmware.txt" windows99.write_text( "\n".join( [ "VMRUN_PRESENT=1", "VMWARE_AUTOSTART_CONFIG_READY=1", "VMWARE_AUTOSTART_POWER_READY=1", "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1", "VMWARE_AUTOSTART_VERIFY_READY=1", "VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Automatic ok=1", "VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Automatic ok=1", "VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1", "WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1", *[ f"VMX alias={alias} path=D:\\Documents\\Virtual Machines\\{alias}\\{alias}.vmx present=1" for alias in ["111", "112", "120", "121", "188"] ], *[ f"VM_POWER alias={alias} vmx_present=1 running=1" for alias in ["111", "112", "120", "121", "188"] ], ] ), encoding="utf-8", ) result = subprocess.run( [ sys.executable, str(ROOT / "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"), "--summary-file", str(summary), "--host-probe-file", str(host_probe), "--reboot-event-file", str(reboot_event), "--public-maintenance-file", str(public_probe), "--windows99-vmware-file", str(windows99), ], text=True, capture_output=True, check=True, ) payload = json.loads(result.stdout) assert "public_route_raw_5xx_without_maintenance_fallback" in payload["active_blockers"] assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is False assert payload["current_phase"] == "public_maintenance_fallback_blocked" assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 1 def test_backup_exporter_emits_domain_level_backup_coverage() -> None: exporter = read("scripts/ops/backup-health-textfile-exporter.py") assert "awoooi_backup_coverage_domain_expected_info" in exporter assert "awoooi_backup_coverage_domain_fresh" in exporter assert "awoooi_backup_alert_receipt_expected_info" in exporter assert "awoooi_backup_alert_receipt_stage_fresh" in exporter assert "BACKUP_ALERT_RECEIPT_STAGES = (\"sent\", \"received\", \"dedup\", \"escalation\")" in exporter assert "AIOPS_BACKUP_ALERT_RECEIPT_DIR" in exporter assert "gitea_repo_mirror_from_110" in exporter assert "/home/ollama/backup/110/gitea" in exporter for domain in ["host", "database", "website", "service", "package", "tool", "log"]: assert f'"{domain}"' in exporter for alertname in [ "BackupJobStale", "BackupCoverageDomainStale", "GiteaPrivateBundleBackupIncomplete", "GiteaPrivateBundleRestoreDryRunMissing", "BackupRestoreDrillMissingOrFailed", "GiteaRestoreDrillMissingOrFailed", "BackupCredentialEscrowEvidenceMissing", ]: assert alertname in exporter def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: alerts = read("ops/monitoring/alerts-unified.yml") assert 'awoooi_backup_coverage_domain_expected_info{host="188"}' in alerts assert "BackupCoverageDomainMetricMissing188" in alerts assert "awoooi_backup_coverage_domain_fresh == 0" in alerts assert "GiteaPrivateBundleRestoreDryRunMissing" in alerts assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in alerts assert "BackupAlertReceiptMetricMissing" in alerts assert "BackupAlertReceiptStageMissing" in alerts assert "awoooi_backup_alert_receipt_expected_info" in alerts assert 'ALERTS{' in alerts assert 'alertname!="BackupAlertReceiptStageMissing"' in alerts assert "label_replace(" in alerts assert 'awoooi_backup_alert_receipt_fresh == 1' in alerts assert "$labels.alertname" in alerts def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: script = read("scripts/backup/gitea-repo-bundle-backup.sh") restore_script = read("scripts/backup/gitea-bundle-sample-restore-dry-run.sh") wrapper = read("scripts/backup/gitea-bundle-backup-sync-188.sh") exporter = read("scripts/ops/backup-health-textfile-exporter.py") playbook = read("infra/ansible/playbooks/110-devops.yml") assert "GIT_TERMINAL_PROMPT=0" in script assert "GIT_REMOTE_BASE" in script assert "--git-remote-base" in script assert "repo_remote_url" in script assert "bundle create" in script assert "bundle verify" in script assert ".sha256" in script assert "manifest.tsv" in script assert "latest-private-complete" in script assert "complete links unchanged" in script assert 'exit 1' in script assert "gitea dump" in script assert "does not replace `gitea dump`" in script assert "tokens or passwords" in script assert "git clone --mirror --quiet" in restore_script assert "fsck --connectivity-only" in restore_script assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in restore_script assert "never reads tokens" in restore_script assert "flock -n" in wrapper assert "latest-private-complete" in wrapper assert "--local-container" in wrapper assert "rsync -a --checksum" in wrapper assert "remote_sample_restore_fsck" in wrapper assert "refresh_remote_backup_health" in wrapper assert "rollback_remote_links" in wrapper assert "remote_manifest_checksum_ok" in wrapper assert "promoted=1" in wrapper assert '"gitea_bundle_sync_188"' in exporter assert 'job="gitea_private_bundle"' in exporter assert "gitea-bundle-backup-sync-188.sh" in playbook assert 'minute: "15"' in playbook assert 'hour: "0"' in playbook def test_prometheus_rule_guard_repairs_container_bind_mount_drift() -> None: guard = read("scripts/ops/prometheus-rule-drift-guard.sh") playbook = read("infra/ansible/playbooks/110-devops.yml") assert "runtime_matches_current" in guard assert "awoooi_prometheus_rule_drift_guard_runtime_matches_current" in guard assert "promtool check rules" in guard assert "--no-deps --force-recreate" in guard assert "failed to create active rules rollback copy" in guard assert "post-apply verifier failed; active rules rolled back" in guard assert "prometheus-rule-drift-guard.sh" in playbook assert 'minute: "*/5"' in playbook def test_velero_restore_test_recovers_a_missed_weekly_schedule() -> None: manifest = read("k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml") assert "startingDeadlineSeconds: 604800" in manifest assert 'schedule: "0 18 * * 6"' in manifest assert "concurrencyPolicy: Forbid" in manifest