fix(reboot): harden public maintenance fallback

scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+import importlib.util
 import json
 import subprocess
 import sys
 from pathlib import Path
+from types import ModuleType
 
 
 ROOT = Path(__file__).resolve().parents[3]
@@ -13,6 +15,61 @@ def read(path: str) -> str:
     return (ROOT / path).read_text(encoding="utf-8")
 
 
+def load_public_maintenance_probe() -> ModuleType:
+    path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py"
+    spec = importlib.util.spec_from_file_location(
+        "awoooi_public_maintenance_fallback_probe",
+        path,
+    )
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def location_block(conf: str, marker: str) -> str:
+    start = conf.index(marker)
+    brace = conf.index("{", start)
+    depth = 0
+    for index in range(brace, len(conf)):
+        char = conf[index]
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                return conf[start : index + 1]
+    raise AssertionError(f"unterminated nginx location block: {marker}")
+
+
+def assert_proxy_location_has_l0_fallback(conf: str, marker: str) -> None:
+    block = location_block(conf, marker)
+
+    assert "proxy_pass " in block
+    assert "proxy_intercept_errors on;" in block
+    assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in block
+
+
+class FakeHttpResponse:
+    def __init__(self, status: int, headers: dict[str, str], body: bytes) -> None:
+        self.status = status
+        self.headers = headers
+        self.body = body
+
+    def __enter__(self) -> FakeHttpResponse:
+        return self
+
+    def __exit__(self, *_args: object) -> None:
+        return None
+
+    def getcode(self) -> int:
+        return self.status
+
+    def read(self, _max_bytes: int) -> bytes:
+        return self.body
+
+
 def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> None:
     host_probe = read("scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh")
     windows99 = read("scripts/reboot-recovery/windows99-vmware-autostart.ps1")
@@ -56,11 +113,13 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None:
         assert alert_name in alerts
     assert "notification_type: TYPE-3" in alerts
     assert "proxy_intercept_errors on" in snippet
+    assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in snippet
     assert "X-AWOOOI-Fallback" in snippet
     assert "502" in runbook
     assert "L0" in runbook
     assert "L1" in runbook
     assert "raw_5xx_without_fallback" in probe
+    assert "status in {200, 502, 503, 504}" in probe
     assert "public-maintenance-fallback.json" in exporter
     assert "--public-maintenance-file" in exporter
     assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter
@@ -76,7 +135,7 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non
     ]:
         conf = read(path)
         assert "proxy_intercept_errors on;" in conf
-        assert "error_page 502 503 504 /__awoooi-maintenance.html;" in conf
+        assert "error_page 502 503 504 =503 /__awoooi-maintenance.html;" in conf
         assert "location = /__awoooi-maintenance.html" in conf
         assert "root /var/www/maintenance;" in conf
         assert "try_files /maintenance.html =503;" in conf
@@ -84,6 +143,53 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non
         assert 'add_header X-AWOOOI-Fallback "local-maintenance" always;' in conf
         assert "/50x.html" not in conf
 
+    ops_conf = read("ops/nginx/awoooi.wooo.work.conf")
+    for marker in [
+        "location /api/ {",
+        "location ~ ^/api/v1/(dashboard/stream|agent/thinking) {",
+        "location /api/v1/ws {",
+        "location / {\n        proxy_pass http://awoooi_web;",
+        "location /_next/static/ {",
+    ]:
+        assert_proxy_location_has_l0_fallback(ops_conf, marker)
+
+    k8s_conf = read("k8s/nginx/awoooi-prod.conf")
+    for marker in [
+        "location ~ ^/api/v1/(agent|dashboard)/stream {",
+        "location /api/sentry-tunnel {",
+        "location /api/ {",
+        "location /api/health {",
+        "location / {\n        proxy_pass http://awoooi_prod_web;",
+        "location ~* \\.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2)$ {",
+    ]:
+        assert_proxy_location_has_l0_fallback(k8s_conf, marker)
+
+
+def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw(
+    monkeypatch,
+) -> None:
+    probe = load_public_maintenance_probe()
+
+    def fake_urlopen(_request, timeout: float) -> FakeHttpResponse:
+        assert timeout == 8.0
+        return FakeHttpResponse(
+            502,
+            {"X-AWOOOI-Fallback": "local-maintenance"},
+            b"HTTP 502 / 503 / 504 fallback page",
+        )
+
+    monkeypatch.setattr(probe.urllib.request, "urlopen", fake_urlopen)
+
+    route = probe.fetch_url(
+        "https://awoooi.wooo.work/api/v1/health",
+        timeout=8.0,
+        max_bytes=65536,
+    )
+
+    assert route["status"] == "maintenance_fallback_serving"
+    assert route["maintenance_fallback_serving"] is True
+    assert route["raw_5xx_without_fallback"] is False
+
 
 def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> None:
     summary = tmp_path / "summary.txt"