From 5d5bc86fed7fe3a7dbc52b5b4d34f49812306d03 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 21:27:08 +0800 Subject: [PATCH] test(reboot): require maintenance fallback readback --- docs/LOGBOOK.md | 15 ++++ ...test_reboot_auto_recovery_slo_scorecard.py | 84 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 044420cf8..beadf06ca 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,18 @@ +## 2026-07-02 — 21:38 P0-006 public maintenance fallback test fixture closure + +**完成內容**： +- Gitea CD `#4450` 失敗點定位為 `test_green_summary_and_recent_all_host_probe_can_claim_slo`：新增 public maintenance runtime readback fail-closed gate 後，既有 green SLO fixture 未提供 `--public-maintenance-file`，因此仍用舊契約宣稱 ready。 +- 修正 `scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py`：新增 `PUBLIC_MAINTENANCE_GREEN` fixture，所有 green host / Stock / host-pressure helper 都帶入 public fallback runtime readback。 +- 新增 `test_missing_public_maintenance_runtime_readback_fails_closed`，明確鎖住缺 runtime probe 時 active blocker 必須包含 `public_maintenance_fallback_runtime_readback_missing`，不得回到 source-only ready。 + +**驗證**： +- `python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py -q -p no:cacheprovider`：`18 passed`。 +- `python3.11 -m py_compile scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py scripts/reboot-recovery/public-maintenance-fallback-probe.py`：通過。 +- `git diff --check`：通過。 + +**仍維持**： +- 未使用 GitHub / `gh` / GitHub API；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未 workflow_dispatch；未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall；未切 DNS / CDN / provider。 + ## 2026-07-02 — 21:24 P0-006 public maintenance fallback runtime gate **完成內容**： diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index 8f618ea1b..f2cf6e045 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -91,6 +91,45 @@ VMWARE_AUTOSTART_VERIFY_READY=1 """ +PUBLIC_MAINTENANCE_GREEN = { + "schema_version": "awoooi_public_maintenance_fallback_probe_v1", + "generated_at": "2026-06-29T14:30:00+08:00", + "runtime_readback_present": True, + "ready": True, + "target_url_count": 2, + "route_count": 2, + "raw_5xx_without_fallback_count": 0, + "route_unreachable_without_external_fallback_count": 0, + "maintenance_fallback_route_count": 0, + "raw_5xx_without_fallback_urls": [], + "route_unreachable_without_external_fallback_urls": [], + "maintenance_fallback_urls": [], + "blockers": [], + "routes": [ + { + "url": "https://awoooi.wooo.work/api/v1/health", + "http_status": 200, + "status": "normal_origin_serving", + "fallback_header": "", + "maintenance_body_detected": False, + "maintenance_fallback_serving": False, + "raw_5xx_without_fallback": False, + "route_unreachable_without_external_fallback": False, + }, + { + "url": "https://awoooi.wooo.work/", + "http_status": 200, + "status": "normal_origin_serving", + "fallback_header": "", + "maintenance_body_detected": False, + "maintenance_fallback_serving": False, + "raw_5xx_without_fallback": False, + "route_unreachable_without_external_fallback": False, + }, + ], +} + + WINDOWS99_MANAGEMENT_BLOCKED = { "schema_version": "windows99_management_channel_readback_v1", "generated_at": "2026-07-02T15:20:00+08:00", @@ -138,12 +177,14 @@ def run_scorecard( summary: str, probe: str = HOST_PROBE_GREEN, windows99: str = WINDOWS99_VMWARE_GREEN, + public_maintenance: dict | None = PUBLIC_MAINTENANCE_GREEN, windows99_management: str | None = None, ) -> dict: summary_path = tmp_path / "summary.txt" probe_path = tmp_path / "probe.txt" reboot_event_path = tmp_path / "reboot-event.json" windows99_path = tmp_path / "windows99-vmware.txt" + public_maintenance_path = tmp_path / "public-maintenance.json" summary_path.write_text(summary, encoding="utf-8") probe_path.write_text(probe, encoding="utf-8") reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8") @@ -163,6 +204,12 @@ def run_scorecard( "--generated-at", "2026-06-29T14:30:00+08:00", ] + if public_maintenance is not None: + public_maintenance_path.write_text( + json.dumps(public_maintenance), + encoding="utf-8", + ) + args.extend(["--public-maintenance-file", str(public_maintenance_path)]) if windows99_management is not None: windows99_management_path.write_text(windows99_management, encoding="utf-8") args.extend(["--windows99-management-file", str(windows99_management_path)]) @@ -185,11 +232,16 @@ def run_scorecard_with_host_pressure( reboot_event_path = tmp_path / "reboot-event.json" host_pressure_path = tmp_path / "host-pressure.json" windows99_path = tmp_path / "windows99-vmware.txt" + public_maintenance_path = tmp_path / "public-maintenance.json" summary_path.write_text(summary, encoding="utf-8") probe_path.write_text(HOST_PROBE_GREEN, encoding="utf-8") reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8") host_pressure_path.write_text(json.dumps(host_pressure), encoding="utf-8") windows99_path.write_text(WINDOWS99_VMWARE_GREEN, encoding="utf-8") + public_maintenance_path.write_text( + json.dumps(PUBLIC_MAINTENANCE_GREEN), + encoding="utf-8", + ) result = subprocess.run( [ sys.executable, @@ -202,6 +254,8 @@ def run_scorecard_with_host_pressure( str(reboot_event_path), "--host-pressure-file", str(host_pressure_path), + "--public-maintenance-file", + str(public_maintenance_path), "--windows99-vmware-file", str(windows99_path), "--generated-at", @@ -227,12 +281,17 @@ def run_scorecard_with_stock( freshness_path = tmp_path / "freshness.json" ingestion_path = tmp_path / "ingestion.json" windows99_path = tmp_path / "windows99-vmware.txt" + public_maintenance_path = tmp_path / "public-maintenance.json" summary_path.write_text(summary, encoding="utf-8") probe_path.write_text(HOST_PROBE_GREEN, encoding="utf-8") reboot_event_path.write_text(json.dumps(REBOOT_EVENT_GREEN), encoding="utf-8") freshness_path.write_text(json.dumps(freshness), encoding="utf-8") ingestion_path.write_text(json.dumps(ingestion), encoding="utf-8") windows99_path.write_text(WINDOWS99_VMWARE_GREEN, encoding="utf-8") + public_maintenance_path.write_text( + json.dumps(PUBLIC_MAINTENANCE_GREEN), + encoding="utf-8", + ) result = subprocess.run( [ sys.executable, @@ -247,6 +306,8 @@ def run_scorecard_with_stock( str(freshness_path), "--stock-ingestion-file", str(ingestion_path), + "--public-maintenance-file", + str(public_maintenance_path), "--windows99-vmware-file", str(windows99_path), "--generated-at", @@ -288,7 +349,11 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - is False ) assert payload["required_checks"]["windows99_management_channel_ready"] is True + assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is True assert payload["rollups"]["source_controls_present"] is True + assert payload["rollups"]["public_maintenance_runtime_readback_present"] is True + assert payload["rollups"]["public_maintenance_fallback_ready"] is True + assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 0 assert payload["rollups"]["windows99_vmware_verify_ready"] is True assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True assert payload["rollups"]["windows99_verify_collection_status"] == ( @@ -333,6 +398,25 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - ] is True +def test_missing_public_maintenance_runtime_readback_fails_closed(tmp_path: Path) -> None: + payload = run_scorecard( + tmp_path, + GREEN_SUMMARY, + public_maintenance=None, + ) + + assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert payload["can_claim_all_services_recovered_within_target"] is False + assert "public_maintenance_fallback_runtime_readback_missing" in payload[ + "active_blockers" + ] + assert ( + payload["required_checks"]["public_maintenance_fallback_runtime_ready"] + is False + ) + assert payload["current_phase"] == "public_maintenance_fallback_blocked" + + def test_missing_probe_fails_closed(tmp_path: Path) -> None: payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe="")