a7ffac9e1c
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 58s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
396 lines
15 KiB
Python
396 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
|
|
|
|
def read(path: str) -> str:
|
|
return (ROOT / path).read_text(encoding="utf-8")
|
|
|
|
|
|
def load_public_maintenance_probe() -> ModuleType:
|
|
path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py"
|
|
spec = importlib.util.spec_from_file_location(
|
|
"awoooi_public_maintenance_fallback_probe",
|
|
path,
|
|
)
|
|
assert spec is not None
|
|
assert spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def location_block(conf: str, marker: str) -> str:
|
|
start = conf.index(marker)
|
|
brace = conf.index("{", start)
|
|
depth = 0
|
|
for index in range(brace, len(conf)):
|
|
char = conf[index]
|
|
if char == "{":
|
|
depth += 1
|
|
elif char == "}":
|
|
depth -= 1
|
|
if depth == 0:
|
|
return conf[start : index + 1]
|
|
raise AssertionError(f"unterminated nginx location block: {marker}")
|
|
|
|
|
|
def assert_proxy_location_has_l0_fallback(conf: str, marker: str) -> None:
|
|
block = location_block(conf, marker)
|
|
|
|
assert "proxy_pass " in block
|
|
assert "proxy_intercept_errors on;" in block
|
|
assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in block
|
|
|
|
|
|
class FakeHttpResponse:
|
|
def __init__(self, status: int, headers: dict[str, str], body: bytes) -> None:
|
|
self.status = status
|
|
self.headers = headers
|
|
self.body = body
|
|
|
|
def __enter__(self) -> FakeHttpResponse:
|
|
return self
|
|
|
|
def __exit__(self, *_args: object) -> None:
|
|
return None
|
|
|
|
def getcode(self) -> int:
|
|
return self.status
|
|
|
|
def read(self, _max_bytes: int) -> bytes:
|
|
return self.body
|
|
|
|
|
|
def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> None:
|
|
host_probe = read("scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh")
|
|
windows99 = read("scripts/reboot-recovery/windows99-vmware-autostart.ps1")
|
|
|
|
for host in ["99", "110", "111", "112", "120", "121", "188"]:
|
|
assert host in host_probe
|
|
assert 'run_with_timeout "${TCP_CONNECT_TIMEOUT_SECONDS:-2}"' in host_probe
|
|
assert "AWOOOI-Start-VMware-VMs" in windows99
|
|
assert "NoAutoRebootWithLoggedOnUsers" in windows99
|
|
assert "Host110Vmx" in windows99
|
|
assert "Host111Vmx" in windows99
|
|
assert "Host188Vmx" in windows99
|
|
assert "Host120Vmx" in windows99
|
|
assert "Host121Vmx" in windows99
|
|
assert "Host112Vmx" in windows99
|
|
assert "D:\\Documents\\Virtual Machines" in windows99
|
|
assert "D:\\Downloads" in windows99
|
|
assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' in windows99
|
|
assert "HOST111_VMWARE_TARGET=explicit_or_required" in windows99
|
|
assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99
|
|
assert "WINDOWS_UPDATE_POLICY" in windows99
|
|
assert "VM_POWER alias=$alias" in windows99
|
|
assert "VM_ALREADY_RUNNING" in windows99
|
|
|
|
|
|
def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None:
|
|
alerts = read("ops/monitoring/alerts-unified.yml")
|
|
runbook = read("docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md")
|
|
snippet = read("ops/maintenance/nginx-502-maintenance-snippet.conf")
|
|
probe = read("scripts/reboot-recovery/public-maintenance-fallback-probe.py")
|
|
exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh")
|
|
|
|
for alert_name in [
|
|
"HostRebootEventDetected",
|
|
"RebootAutoRecoverySLOMissed",
|
|
"RebootAutoRecoveryActiveBlocker",
|
|
"RebootAutoRecoveryActiveBlockerMetricMissing",
|
|
"PublicRouteServing5xxWithoutMaintenanceFallback",
|
|
"BackupCoverageDomainStale",
|
|
]:
|
|
assert alert_name in alerts
|
|
assert "notification_type: TYPE-3" in alerts
|
|
assert "proxy_intercept_errors on" in snippet
|
|
assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in snippet
|
|
assert "X-AWOOOI-Fallback" in snippet
|
|
assert "502" in runbook
|
|
assert "L0" in runbook
|
|
assert "L1" in runbook
|
|
assert "raw_5xx_without_fallback" in probe
|
|
assert "status in {200, 502, 503, 504}" in probe
|
|
assert "public-maintenance-fallback.json" in exporter
|
|
assert "--public-maintenance-file" in exporter
|
|
assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter
|
|
assert "awoooi_reboot_auto_recovery_slo_active_blocker" in exporter
|
|
assert "awoooi_windows99_no_secret_collector_readback_present" in exporter
|
|
assert "awoooi_windows99_no_secret_collector_status" in exporter
|
|
assert (
|
|
"awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready" in exporter
|
|
)
|
|
assert "awoooi_windows99_no_secret_collector_port_22_open" in exporter
|
|
assert "awoooi_windows99_no_secret_collector_remote_verify_attempted" in exporter
|
|
assert 'blocker="{label(blocker)}"' in exporter
|
|
assert "$labels.blocker" in alerts
|
|
|
|
|
|
def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> None:
|
|
for path in [
|
|
"ops/nginx/awoooi.wooo.work.conf",
|
|
"k8s/nginx/awoooi-prod.conf",
|
|
]:
|
|
conf = read(path)
|
|
assert "proxy_intercept_errors on;" in conf
|
|
assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in conf
|
|
assert "location = /__awoooi-maintenance.html" in conf
|
|
assert "root /var/www/maintenance;" in conf
|
|
assert "try_files /maintenance.html =503;" in conf
|
|
assert 'add_header Retry-After "120" always;' in conf
|
|
assert 'add_header X-AWOOOI-Fallback "local-maintenance" always;' in conf
|
|
assert "/50x.html" not in conf
|
|
|
|
ops_conf = read("ops/nginx/awoooi.wooo.work.conf")
|
|
assert "192.168.0.125:32334" in ops_conf
|
|
assert "192.168.0.125:32335" in ops_conf
|
|
for marker in [
|
|
"location /api/ {",
|
|
"location ~ ^/api/v1/(dashboard/stream|agent/thinking) {",
|
|
"location /api/v1/ws {",
|
|
"location / {\n proxy_pass http://awoooi_web;",
|
|
"location /_next/static/ {",
|
|
]:
|
|
assert_proxy_location_has_l0_fallback(ops_conf, marker)
|
|
|
|
k8s_conf = read("k8s/nginx/awoooi-prod.conf")
|
|
for marker in [
|
|
"location ~ ^/api/v1/(agent|dashboard)/stream {",
|
|
"location /api/sentry-tunnel {",
|
|
"location /api/ {",
|
|
"location /api/health {",
|
|
"location / {\n proxy_pass http://awoooi_prod_web;",
|
|
"location ~* \\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2)$ {",
|
|
]:
|
|
assert_proxy_location_has_l0_fallback(k8s_conf, marker)
|
|
|
|
|
|
def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw(
|
|
monkeypatch,
|
|
) -> None:
|
|
probe = load_public_maintenance_probe()
|
|
|
|
def fake_urlopen(_request, timeout: float) -> FakeHttpResponse:
|
|
assert timeout == 8.0
|
|
return FakeHttpResponse(
|
|
502,
|
|
{"X-AWOOOI-Fallback": "local-maintenance"},
|
|
b"HTTP 502 / 503 / 504 fallback page",
|
|
)
|
|
|
|
monkeypatch.setattr(probe.urllib.request, "urlopen", fake_urlopen)
|
|
|
|
route = probe.fetch_url(
|
|
"https://awoooi.wooo.work/api/v1/health",
|
|
timeout=8.0,
|
|
max_bytes=65536,
|
|
)
|
|
|
|
assert route["status"] == "maintenance_fallback_serving"
|
|
assert route["maintenance_fallback_serving"] is True
|
|
assert route["raw_5xx_without_fallback"] is False
|
|
|
|
|
|
def test_public_maintenance_edge_apply_script_is_guarded_and_reversible() -> None:
|
|
script = read("scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh")
|
|
|
|
assert "MODE=\"check\"" in script
|
|
assert "MODE=\"apply\"" in script
|
|
assert "MODE=\"rollback\"" in script
|
|
assert "sudo -n true" in script
|
|
assert "BLOCKER privileged_edge_apply_channel_unavailable" in script
|
|
assert "nginx -t" in script
|
|
assert "systemctl reload nginx" in script
|
|
assert "BACKUP_CONF=" in script
|
|
assert "EDGE_FALLBACK_READY=1" in script
|
|
assert "EDGE_FALLBACK_READY=0" in script
|
|
assert "raw_secret" not in script.lower()
|
|
assert "docker restart" not in script
|
|
assert "reboot " not in script
|
|
|
|
|
|
def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None:
|
|
summary = tmp_path / "summary.txt"
|
|
summary.write_text(
|
|
"\n".join(
|
|
[
|
|
"SERVICE_GREEN=1",
|
|
"PRODUCT_DATA_GREEN=1",
|
|
"BACKUP_CORE_GREEN=1",
|
|
"POST_START_BLOCKED=0",
|
|
"HOST_188_SERVICE_GREEN=1",
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
host_probe = tmp_path / "host-probe.txt"
|
|
host_probe.write_text(
|
|
"\n".join(
|
|
f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120"
|
|
for alias in ["99", "110", "111", "112", "120", "121", "188"]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
reboot_event = tmp_path / "reboot-event.json"
|
|
reboot_event.write_text(
|
|
json.dumps(
|
|
{
|
|
"reboot_detected": True,
|
|
"all_required_hosts_observed": True,
|
|
"all_required_hosts_in_reboot_window": True,
|
|
"fresh_boot_hosts": ["99", "110", "111", "112", "120", "121", "188"],
|
|
"missing_hosts": [],
|
|
"unreachable_hosts": [],
|
|
"target_seconds_remaining": 480,
|
|
"recovery_deadline_status": "inside_target_window",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
public_probe = tmp_path / "public-maintenance.json"
|
|
public_probe.write_text(
|
|
json.dumps(
|
|
{
|
|
"runtime_readback_present": True,
|
|
"ready": False,
|
|
"raw_5xx_without_fallback_count": 1,
|
|
"route_unreachable_without_external_fallback_count": 0,
|
|
"maintenance_fallback_route_count": 0,
|
|
"raw_5xx_without_fallback_urls": ["https://awoooi.wooo.work/"],
|
|
"blockers": ["public_route_raw_5xx_without_maintenance_fallback"],
|
|
"routes": [
|
|
{
|
|
"url": "https://awoooi.wooo.work/",
|
|
"http_status": 502,
|
|
"status": "raw_5xx_without_maintenance_fallback",
|
|
"fallback_header": "",
|
|
"maintenance_body_detected": False,
|
|
"maintenance_fallback_serving": False,
|
|
"raw_5xx_without_fallback": True,
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
windows99 = tmp_path / "windows99-vmware.txt"
|
|
windows99.write_text(
|
|
"\n".join(
|
|
[
|
|
"VMRUN_PRESENT=1",
|
|
"VMWARE_AUTOSTART_CONFIG_READY=1",
|
|
"VMWARE_AUTOSTART_POWER_READY=1",
|
|
"WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1",
|
|
"VMWARE_AUTOSTART_VERIFY_READY=1",
|
|
"VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Automatic ok=1",
|
|
"VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Automatic ok=1",
|
|
"VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1",
|
|
"WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1",
|
|
*[
|
|
f"VMX alias={alias} path=D:\\Documents\\Virtual Machines\\{alias}\\{alias}.vmx present=1"
|
|
for alias in ["111", "112", "120", "121", "188"]
|
|
],
|
|
*[
|
|
f"VM_POWER alias={alias} vmx_present=1 running=1"
|
|
for alias in ["111", "112", "120", "121", "188"]
|
|
],
|
|
]
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(ROOT / "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"),
|
|
"--summary-file",
|
|
str(summary),
|
|
"--host-probe-file",
|
|
str(host_probe),
|
|
"--reboot-event-file",
|
|
str(reboot_event),
|
|
"--public-maintenance-file",
|
|
str(public_probe),
|
|
"--windows99-vmware-file",
|
|
str(windows99),
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
payload = json.loads(result.stdout)
|
|
|
|
assert "public_route_raw_5xx_without_maintenance_fallback" in payload["active_blockers"]
|
|
assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is False
|
|
assert payload["current_phase"] == "public_maintenance_fallback_blocked"
|
|
assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 1
|
|
|
|
|
|
def test_backup_exporter_emits_domain_level_backup_coverage() -> None:
|
|
exporter = read("scripts/ops/backup-health-textfile-exporter.py")
|
|
|
|
assert "awoooi_backup_coverage_domain_expected_info" in exporter
|
|
assert "awoooi_backup_coverage_domain_fresh" in exporter
|
|
assert "awoooi_backup_alert_receipt_expected_info" in exporter
|
|
assert "awoooi_backup_alert_receipt_stage_fresh" in exporter
|
|
assert "BACKUP_ALERT_RECEIPT_STAGES = (\"sent\", \"received\", \"dedup\", \"escalation\")" in exporter
|
|
assert "AIOPS_BACKUP_ALERT_RECEIPT_DIR" in exporter
|
|
assert "gitea_repo_mirror_from_110" in exporter
|
|
assert "/home/ollama/backup/110/gitea" in exporter
|
|
for domain in ["host", "database", "website", "service", "package", "tool", "log"]:
|
|
assert f'"{domain}"' in exporter
|
|
for alertname in [
|
|
"BackupJobStale",
|
|
"BackupCoverageDomainStale",
|
|
"GiteaPrivateBundleBackupIncomplete",
|
|
"GiteaPrivateBundleRestoreDryRunMissing",
|
|
"BackupRestoreDrillMissingOrFailed",
|
|
"BackupCredentialEscrowEvidenceMissing",
|
|
]:
|
|
assert alertname in exporter
|
|
|
|
|
|
def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None:
|
|
alerts = read("ops/monitoring/alerts-unified.yml")
|
|
|
|
assert 'awoooi_backup_coverage_domain_expected_info{host="188"}' in alerts
|
|
assert "BackupCoverageDomainMetricMissing188" in alerts
|
|
assert "awoooi_backup_coverage_domain_fresh == 0" in alerts
|
|
assert "GiteaPrivateBundleRestoreDryRunMissing" in alerts
|
|
assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in alerts
|
|
assert "BackupAlertReceiptMetricMissing" in alerts
|
|
assert "BackupAlertReceiptStageMissing" in alerts
|
|
assert "awoooi_backup_alert_receipt_expected_info" in alerts
|
|
assert "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0" in alerts
|
|
assert "$labels.receipt_channel" in alerts
|
|
assert "$value" in alerts
|
|
assert "required_alert/scope/stage/required_notification_type" in alerts
|
|
assert "telegram_awooop" in alerts
|
|
|
|
|
|
def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None:
|
|
script = read("scripts/backup/gitea-repo-bundle-backup.sh")
|
|
restore_script = read("scripts/backup/gitea-bundle-sample-restore-dry-run.sh")
|
|
|
|
assert "GIT_TERMINAL_PROMPT=0" in script
|
|
assert "bundle create" in script
|
|
assert "bundle verify" in script
|
|
assert ".sha256" in script
|
|
assert "manifest.tsv" in script
|
|
assert "gitea dump" in script
|
|
assert "does not replace `gitea dump`" in script
|
|
assert "tokens or passwords" in script
|
|
assert "git clone --mirror --quiet" in restore_script
|
|
assert "fsck --connectivity-only" in restore_script
|
|
assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in restore_script
|
|
assert "never reads tokens" in restore_script
|