Merge remote-tracking branch 'origin/main' into codex/p0-obs-002-20260715

This commit is contained in:
ogt
2026-07-15 02:30:04 +08:00
17 changed files with 524 additions and 63 deletions

View File

@@ -20,6 +20,7 @@ RECOVERY_SCORECARD = (
ROOT / "scripts" / "reboot-recovery" / "full-stack-recovery-scorecard.sh"
)
VERIFY_DEPLOY = ROOT / "scripts" / "reboot-recovery" / "verify-cold-start-monitor-deploy.sh"
DEPLOY_ALERTS = ROOT / "scripts" / "ops" / "deploy-alerts.sh"
REPAIR_STARTUP_STUB = (
ROOT / "scripts" / "reboot-recovery" / "repair-110-startup-script-stub.sh"
)
@@ -72,6 +73,14 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
encoding="utf-8"
)
deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8")
alerts = (ROOT / "ops" / "monitoring" / "alerts-unified.yml").read_text(
encoding="utf-8"
)
recovery_scorecard = RECOVERY_SCORECARD.read_text(encoding="utf-8")
ansible_verifier = (
ROOT / "infra" / "ansible" / "roles" / "cold-start-monitor"
/ "tasks" / "main.yml"
).read_text(encoding="utf-8")
assert "Scope: 110 / 112 / 120 / 121 / 188." in text
assert "for host in 110 112 120 121 188; do" in text
@@ -86,6 +95,29 @@ def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
assert 'reason="host_unreachable",target="112"' in exporter
assert 'reason="guest_readiness_failed",target="112"' in exporter
assert 'scope="110_112_120_121_188"' in deploy_verifier
assert 'base_url + "/api/v1/rules"' in deploy_verifier
assert "COLD_START_RULE_SCOPE_OK" in deploy_verifier
assert 'stale_scope = \'scope="110_120_121_188"\'' in deploy_verifier
assert 'scope="110_112_120_121_188"' in alerts
assert 'scope="110_112_120_121_188"' in recovery_scorecard
assert 'scope="110_112_120_121_188"' in ansible_verifier
assert 'scope="110_120_121_188"' not in alerts
assert 'scope="110_120_121_188"' not in recovery_scorecard
assert 'scope="110_120_121_188"' not in ansible_verifier
def test_alert_rule_deploy_is_preflighted_verified_and_rollback_capable() -> None:
deploy = DEPLOY_ALERTS.read_text(encoding="utf-8")
assert "remote_promtool_preflight" in deploy
assert "promtool check rules" in deploy
assert "trap on_exit EXIT" in deploy
assert "rollback_deploy" in deploy
assert 'APPLY_STARTED=1' in deploy
assert 'DEPLOY_VERIFIED=1' in deploy
assert "PROMETHEUS_RUNTIME_RULES" in deploy
assert "verify-cold-start-monitor-deploy.sh" in deploy
assert "curl -fsS -X POST" in deploy
def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None:

View File

@@ -15,6 +15,13 @@ def read(path: str) -> str:
return (ROOT / path).read_text(encoding="utf-8")
def test_reboot_sop_never_embeds_passworded_sudo_startup() -> None:
sop = read("docs/runbooks/REBOOT-RECOVERY-SOP.md")
assert "sudo -n /usr/local/bin/awoooi-startup-110.sh" in sop
assert "| sudo -S /usr/local/bin/awoooi-startup-110.sh" not in sop
def load_public_maintenance_probe() -> ModuleType:
path = ROOT / "scripts/reboot-recovery/public-maintenance-fallback-probe.py"
spec = importlib.util.spec_from_file_location(