wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(ops): harden reboot recovery and backup alerts

Browse Source
This commit is contained in:
Your Name
2026-05-29 12:38:58 +08:00
parent 70637ec871
commit ae7b39d96a
14 changed files with 2354 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
scripts/ops/backup-alert-label-contract-check.py Executable file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env python3
"""
Validate the backup alert label contract.
Node exporter textfile metrics use labels such as job="backup_all" locally, but
Prometheus rewrites that metric label to exported_job because the scrape target
already has job="node-exporter-110". Backup alerts must therefore use
$labels.exported_job in user-facing text and exported_job="..." in expressions.
"""
from __future__ import annotations
import argparse
import json
import sys
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any
import yaml
DEFAULT_RULES = Path("ops/monitoring/alerts-unified.yml")
DEFAULT_BASELINE = Path("ops/reboot-recovery/full-stack-backup-baseline.yml")
class ContractError(RuntimeError):
pass
def _load_alerts(path: Path) -> dict[str, dict[str, Any]]:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
alerts: dict[str, dict[str, Any]] = {}
for group in data.get("groups") or []:
for rule in group.get("rules") or []:
name = rule.get("alert")
if name:
alerts[name] = rule
return alerts
def _annotation_text(rule: dict[str, Any]) -> str:
annotations = rule.get("annotations") or {}
return "\n".join(str(value) for value in annotations.values())
def _require_alert(alerts: dict[str, dict[str, Any]], name: str) -> dict[str, Any]:
if name not in alerts:
raise ContractError(f"missing alert: {name}")
return alerts[name]
def _require_contains(value: str, expected: str, label: str) -> None:
if expected not in value:
raise ContractError(f"{label} must contain {expected!r}")
def _require_not_contains(value: str, forbidden: str, label: str) -> None:
if forbidden in value:
raise ContractError(f"{label} must not contain {forbidden!r}")
def _expected_backup_alerts(path: Path) -> list[str]:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
alerts = data.get("monitoring_contract", {}).get("prometheus_alerts") or []
if not alerts:
raise ContractError(f"missing monitoring_contract.prometheus_alerts in {path}")
return [str(alert) for alert in alerts]
def static_check(path: Path, baseline_path: Path) -> list[str]:
alerts = _load_alerts(path)
lines: list[str] = []
missing = sorted(set(_expected_backup_alerts(baseline_path)) - set(alerts))
if missing:
raise ContractError(f"alerts-unified.yml missing baseline backup alerts: {missing}")
lines.append("OK alerts-unified.yml contains every baseline backup alert")
rule = _require_alert(alerts, "BackupExpectedJobMissing")
_require_contains(str(rule.get("expr", "")), "awoooi_backup_job_configured", "BackupExpectedJobMissing expr")
text = _annotation_text(rule)
_require_contains(text, "$labels.exported_job", "BackupExpectedJobMissing annotations")
_require_not_contains(text, "$labels.job", "BackupExpectedJobMissing annotations")
lines.append("OK BackupExpectedJobMissing uses exported_job label")
rule = _require_alert(alerts, "BackupJobStale")
_require_contains(str(rule.get("expr", "")), "awoooi_backup_job_fresh", "BackupJobStale expr")
text = _annotation_text(rule)
_require_contains(text, "$labels.exported_job", "BackupJobStale annotations")
_require_not_contains(text, "$labels.job", "BackupJobStale annotations")
for required_label in ["$labels.max_age_hours", "$labels.source", "$labels.target"]:
_require_contains(text, required_label, "BackupJobStale annotations")
lines.append("OK BackupJobStale uses exported_job/source/target labels")
rule = _require_alert(alerts, "BackupAggregateRunFailed")
_require_contains(
str(rule.get("expr", "")),
'awoooi_backup_last_run_failed_count{host="110",exported_job="backup_all"}',
"BackupAggregateRunFailed expr",
)
lines.append("OK BackupAggregateRunFailed filters exported_job=backup_all")
rule = _require_alert(alerts, "BackupConfigCapturePartial")
_require_contains(str(rule.get("expr", "")), "awoooi_backup_config_capture_ok", "BackupConfigCapturePartial expr")
text = _annotation_text(rule)
for required_label in ["$labels.target", "$labels.source"]:
_require_contains(text, required_label, "BackupConfigCapturePartial annotations")
lines.append("OK BackupConfigCapturePartial uses target/source labels")
rule = _require_alert(alerts, "BackupConfigCaptureStatusStale")
_require_contains(
str(rule.get("expr", "")),
"awoooi_backup_config_capture_status_timestamp",
"BackupConfigCaptureStatusStale expr",
)
lines.append("OK BackupConfigCaptureStatusStale checks config capture status timestamp")
rule = _require_alert(alerts, "BackupScriptMissing")
_require_contains(_annotation_text(rule), "$labels.script", "BackupScriptMissing annotations")
lines.append("OK BackupScriptMissing uses script label")
rule = _require_alert(alerts, "BackupCredentialEscrowEvidenceMissing")
_require_contains(_annotation_text(rule), "$labels.item", "BackupCredentialEscrowEvidenceMissing annotations")
lines.append("OK BackupCredentialEscrowEvidenceMissing uses item label")
return lines
def _prom_query(base_url: str, expr: str) -> list[dict[str, Any]]:
query = urllib.parse.urlencode({"query": expr})
url = f"{base_url.rstrip('/')}/api/v1/query?{query}"
with urllib.request.urlopen(url, timeout=8) as response:
payload = json.loads(response.read().decode("utf-8"))
if payload.get("status") != "success":
raise ContractError(f"Prometheus query failed for {expr}: {payload}")
return payload.get("data", {}).get("result") or []
def _prom_rules(base_url: str) -> list[dict[str, Any]]:
url = f"{base_url.rstrip('/')}/api/v1/rules"
with urllib.request.urlopen(url, timeout=8) as response:
payload = json.loads(response.read().decode("utf-8"))
if payload.get("status") != "success":
raise ContractError(f"Prometheus rules query failed: {payload}")
rules: list[dict[str, Any]] = []
for group in payload.get("data", {}).get("groups") or []:
for rule in group.get("rules") or []:
name = rule.get("name") or rule.get("alert")
if not name:
continue
rules.append(
{
"name": str(name),
"health": str(rule.get("health", "")),
"state": str(rule.get("state", "")),
"group": str(group.get("name", "")),
}
)
return rules
def _require_live_label(base_url: str, expr: str, labels: set[str]) -> str:
rows = _prom_query(base_url, expr)
if not rows:
raise ContractError(f"Prometheus query returned no series: {expr}")
metric = rows[0].get("metric") or {}
missing = sorted(label for label in labels if label not in metric)
if missing:
raise ContractError(f"{expr} missing labels {missing}; labels={sorted(metric)}")
return f"OK live {expr} exposes labels {','.join(sorted(labels))}"
def _require_live_rules(base_url: str, expected_alerts: list[str]) -> list[str]:
rules = _prom_rules(base_url)
by_name = {rule["name"]: rule for rule in rules}
missing = sorted(set(expected_alerts) - set(by_name))
if missing:
raise ContractError(f"Prometheus missing loaded backup alert rules: {missing}")
unhealthy = [
f"{rule['name']} health={rule['health']} group={rule['group']}"
for rule in by_name.values()
if rule["name"] in expected_alerts and rule["health"] not in {"", "ok"}
]
if unhealthy:
raise ContractError(f"Prometheus backup alert rule health is not ok: {unhealthy}")
state_counts: dict[str, int] = {}
for name in expected_alerts:
state = by_name[name]["state"] or "unknown"
state_counts[state] = state_counts.get(state, 0) + 1
state_summary = ",".join(f"{key}={state_counts[key]}" for key in sorted(state_counts))
return [
f"OK live Prometheus loaded {len(expected_alerts)} baseline backup alert rules",
f"OK live Prometheus backup alert rule states {state_summary}",
]
def live_check(base_url: str, baseline_path: Path) -> list[str]:
lines = [
_require_live_label(
base_url,
'awoooi_backup_job_configured{host="110"}',
{"exported_job", "host", "job"},
),
_require_live_label(
base_url,
'awoooi_backup_job_fresh{host="110"}',
{"exported_job", "host", "job", "source", "target", "max_age_hours"},
),
_require_live_label(
base_url,
'awoooi_backup_last_run_failed_count{host="110"}',
{"exported_job", "host", "job"},
),
_require_live_label(
base_url,
'awoooi_backup_dr_next_step_info{host="110"}',
{"host", "next_step"},
),
_require_live_label(
base_url,
'awoooi_backup_offsite_partial_fresh{host="110",provider="rclone"}',
{"host", "provider", "scope", "max_age_hours"},
),
_require_live_label(
base_url,
'awoooi_backup_config_capture_ok{host="110"}',
{"host", "target", "source", "critical"},
),
]
lines.extend(_require_live_rules(base_url, _expected_backup_alerts(baseline_path)))
return lines
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--rules", type=Path, default=DEFAULT_RULES)
parser.add_argument("--baseline", type=Path, default=DEFAULT_BASELINE)
parser.add_argument("--prometheus-url", default="")
args = parser.parse_args()
try:
for line in static_check(args.rules, args.baseline):
print(line)
if args.prometheus_url:
for line in live_check(args.prometheus_url, args.baseline):
print(line)
except (ContractError, OSError, yaml.YAMLError, json.JSONDecodeError) as exc:
print(f"BACKUP_ALERT_LABEL_CONTRACT_FAILED {exc}", file=sys.stderr)
return 1
print("BACKUP_ALERT_LABEL_CONTRACT_OK")
return 0
if __name__ == "__main__":
raise SystemExit(main())

242
scripts/ops/backup-alert-live-visibility-check.py Executable file
View File

@@ -0,0 +1,242 @@
#!/usr/bin/env python3
"""Verify live visibility for backup gap alerts.
This read-only check closes the gap between "metrics exist" and "alerts are
actually visible". If the offsite or credential-escrow gap metrics are present,
the corresponding Prometheus firing alerts must be visible. When Alertmanager is
provided, those same alerts must also be active there.
"""
from __future__ import annotations
import argparse
import json
import sys
import time
import urllib.parse
import urllib.request
from dataclasses import dataclass
from typing import Any
class VisibilityError(RuntimeError):
pass
@dataclass(frozen=True)
class RequiredAlert:
name: str
labels: dict[str, str]
COMMON_LABELS = {
"host": "110",
"auto_repair": "false",
"alert_category": "infrastructure",
"notification_type": "TYPE-1",
"severity": "warning",
}
def _json_get(url: str, timeout: int) -> Any:
with urllib.request.urlopen(url, timeout=timeout) as response:
return json.loads(response.read().decode("utf-8"))
def _prom_query(base_url: str, expr: str, timeout: int) -> list[dict[str, Any]]:
query = urllib.parse.urlencode({"query": expr})
url = f"{base_url.rstrip('/')}/api/v1/query?{query}"
payload = _json_get(url, timeout)
if payload.get("status") != "success":
raise VisibilityError(f"Prometheus query failed for {expr}: {payload}")
return payload.get("data", {}).get("result") or []
def _prom_alerts(base_url: str, timeout: int) -> list[dict[str, Any]]:
url = f"{base_url.rstrip('/')}/api/v1/alerts"
payload = _json_get(url, timeout)
if payload.get("status") != "success":
raise VisibilityError(f"Prometheus alerts query failed: {payload}")
return payload.get("data", {}).get("alerts") or []
def _alertmanager_alerts(base_url: str, timeout: int) -> list[dict[str, Any]]:
url = f"{base_url.rstrip('/')}/api/v2/alerts"
payload = _json_get(url, timeout)
if not isinstance(payload, list):
raise VisibilityError(f"Alertmanager alerts query returned unexpected payload: {payload}")
return payload
def _float_value(row: dict[str, Any], expr: str) -> float:
value = row.get("value")
if not isinstance(value, list) or len(value) < 2:
raise VisibilityError(f"Prometheus query returned unexpected value for {expr}: {row}")
try:
return float(value[1])
except (TypeError, ValueError) as exc:
raise VisibilityError(f"Prometheus query returned non-numeric value for {expr}: {row}") from exc
def _metric_labels(row: dict[str, Any]) -> dict[str, str]:
metric = row.get("metric") or {}
return {str(key): str(value) for key, value in metric.items()}
def _labels_match(actual: dict[str, str], expected: dict[str, str]) -> bool:
return all(actual.get(key) == value for key, value in expected.items())
def _find_prom_alert(alerts: list[dict[str, Any]], required: RequiredAlert) -> dict[str, Any] | None:
expected = {"alertname": required.name, **required.labels}
for alert in alerts:
if str(alert.get("state", "")) != "firing":
continue
labels = {str(key): str(value) for key, value in (alert.get("labels") or {}).items()}
if _labels_match(labels, expected):
return alert
return None
def _find_alertmanager_alert(alerts: list[dict[str, Any]], required: RequiredAlert) -> dict[str, Any] | None:
expected = {"alertname": required.name, **required.labels}
for alert in alerts:
status = alert.get("status") or {}
if str(status.get("state", "")) != "active":
continue
labels = {str(key): str(value) for key, value in (alert.get("labels") or {}).items()}
if _labels_match(labels, expected):
return alert
return None
def _require_prom_alert(alerts: list[dict[str, Any]], required: RequiredAlert) -> None:
if _find_prom_alert(alerts, required) is None:
raise VisibilityError(
f"missing Prometheus firing alert {required.name} with labels {required.labels}"
)
def _require_alertmanager_alert(alerts: list[dict[str, Any]], required: RequiredAlert) -> None:
if _find_alertmanager_alert(alerts, required) is None:
raise VisibilityError(
f"missing Alertmanager active alert {required.name} with labels {required.labels}"
)
def _sum_query_values(prometheus_url: str, expr: str, timeout: int) -> float:
return sum(_float_value(row, expr) for row in _prom_query(prometheus_url, expr, timeout))
def _max_query_value(prometheus_url: str, expr: str, timeout: int) -> float:
rows = _prom_query(prometheus_url, expr, timeout)
if not rows:
return 0
return max(_float_value(row, expr) for row in rows)
def _offsite_required_alerts(prometheus_url: str, host: str, timeout: int) -> tuple[list[RequiredAlert], str]:
expr = f'awoooi_backup_offsite_configured{{host="{host}"}}'
rows = _prom_query(prometheus_url, expr, timeout)
if not rows:
raise VisibilityError(f"Prometheus query returned no offsite configured series: {expr}")
configured_total = sum(_float_value(row, expr) for row in rows)
if configured_total == 0:
return (
[RequiredAlert("BackupOffsiteCopyNotConfigured", {**COMMON_LABELS, "host": host})],
"OK offsite gap metric requires BackupOffsiteCopyNotConfigured visibility",
)
fresh_expr = f'awoooi_backup_offsite_fresh{{host="{host}"}}'
if _sum_query_values(prometheus_url, fresh_expr, timeout) > 0:
return [], "OK offsite full marker is fresh; no offsite gap alert required"
enabled_expr = f'awoooi_backup_offsite_full_sync_enabled{{host="{host}"}}'
enabled_total = _sum_query_values(prometheus_url, enabled_expr, timeout)
if enabled_total > 0:
timestamp_expr = f'awoooi_backup_offsite_full_sync_enabled_timestamp{{host="{host}"}}'
enabled_timestamp = _max_query_value(prometheus_url, timestamp_expr, timeout)
enabled_age = int(time.time() - enabled_timestamp) if enabled_timestamp else 0
if enabled_timestamp and enabled_age <= 30 * 3600:
return (
[],
f"OK offsite full sync enabled within grace window; BackupOffsiteCopyStale not required yet age_seconds={enabled_age}",
)
return (
[RequiredAlert("BackupOffsiteCopyStale", {**COMMON_LABELS, "host": host})],
"OK offsite full marker gap requires BackupOffsiteCopyStale visibility",
)
def _escrow_required_alerts(prometheus_url: str, host: str, timeout: int) -> list[RequiredAlert]:
expr = f'awoooi_backup_credential_escrow_fresh{{host="{host}"}} == 0'
rows = _prom_query(prometheus_url, expr, timeout)
required: list[RequiredAlert] = []
for row in rows:
labels = _metric_labels(row)
item = labels.get("item")
if not item:
raise VisibilityError(f"Credential escrow gap metric missing item label: {row}")
required.append(
RequiredAlert(
"BackupCredentialEscrowEvidenceMissing",
{**COMMON_LABELS, "host": host, "item": item},
)
)
return sorted(required, key=lambda alert: alert.labels["item"])
def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: int) -> list[str]:
required_alerts: list[RequiredAlert] = []
lines: list[str] = []
offsite_alerts, offsite_line = _offsite_required_alerts(prometheus_url, host, timeout)
required_alerts.extend(offsite_alerts)
lines.append(offsite_line)
escrow_alerts = _escrow_required_alerts(prometheus_url, host, timeout)
required_alerts.extend(escrow_alerts)
if escrow_alerts:
escrow_items = ", ".join(alert.labels["item"] for alert in escrow_alerts)
lines.append(
f"OK credential escrow gap metrics require {len(escrow_alerts)} alert(s): {escrow_items}"
)
else:
lines.append("OK credential escrow markers are fresh; no escrow gap alert required")
prom_alerts = _prom_alerts(prometheus_url, timeout)
for required in required_alerts:
_require_prom_alert(prom_alerts, required)
lines.append(f"OK Prometheus exposes {len(required_alerts)} required backup gap firing alert(s)")
if alertmanager_url:
am_alerts = _alertmanager_alerts(alertmanager_url, timeout)
for required in required_alerts:
_require_alertmanager_alert(am_alerts, required)
lines.append(f"OK Alertmanager exposes {len(required_alerts)} required backup gap active alert(s)")
return lines
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--prometheus-url", required=True)
parser.add_argument("--alertmanager-url", default="")
parser.add_argument("--host", default="110")
parser.add_argument("--timeout", type=int, default=8)
args = parser.parse_args()
try:
for line in live_check(args.prometheus_url, args.alertmanager_url, args.host, args.timeout):
print(line)
except (VisibilityError, OSError, json.JSONDecodeError) as exc:
print(f"BACKUP_ALERT_LIVE_VISIBILITY_FAILED {exc}", file=sys.stderr)
return 1
print("BACKUP_ALERT_LIVE_VISIBILITY_OK")
return 0
if __name__ == "__main__":
raise SystemExit(main())

42
scripts/ops/prometheus-rule-drift-guard.sh
View File

@@ -1,9 +1,9 @@
#!/usr/bin/env bash
# Guard 110 Prometheus alert rules against stale deploys.
#
# The canonical file is the source of truth. The guard restores active
# alerts.yml only when the active file differs from canonical or when
# Prometheus is missing rule names declared by canonical.
# This script is intentionally narrow: it only restores the canonical alert
# rules file when required recovery/backup rules disappear from live Prometheus
# or when the active file differs from the canonical copy.
set -uo pipefail
@@ -14,6 +14,14 @@ CANONICAL_RULES="${CANONICAL_RULES:-/home/wooo/monitoring/alerts-unified.canonic
TEXTFILE="${TEXTFILE:-/home/wooo/node_exporter_textfiles/prometheus_rule_drift_guard.prom}"
LOG_FILE="${LOG_FILE:-/home/wooo/logs/prometheus-rule-drift-guard.log}"
REQUIRED_RULES=(
"BackupCredentialEscrowEvidenceMissing"
"BackupExpectedJobMissing"
"awoooi_recovery_core_ready"
"awoooi_recovery_dr_offsite_ready"
"ColdStartRecoveryBlocked"
)
log() {
mkdir -p "$(dirname "$LOG_FILE")" 2>/dev/null || true
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >>"$LOG_FILE"
@@ -34,7 +42,7 @@ awoooi_prometheus_rule_drift_guard_last_run_timestamp{host="${HOST_LABEL}",statu
# HELP awoooi_prometheus_rule_drift_guard_repaired Whether the guard restored canonical Prometheus rules on the last run.
# TYPE awoooi_prometheus_rule_drift_guard_repaired gauge
awoooi_prometheus_rule_drift_guard_repaired{host="${HOST_LABEL}"} ${repaired}
# HELP awoooi_prometheus_rule_drift_guard_missing_required_count Number of canonical live rules missing after the last check.
# HELP awoooi_prometheus_rule_drift_guard_missing_required_count Number of required live rules missing after the last check.
# TYPE awoooi_prometheus_rule_drift_guard_missing_required_count gauge
awoooi_prometheus_rule_drift_guard_missing_required_count{host="${HOST_LABEL}"} ${missing_count}
# HELP awoooi_prometheus_rule_drift_guard_current_matches_canonical Whether active alerts.yml matches canonical copy.
@@ -46,27 +54,13 @@ EOF
}
rules_missing_count() {
python3 - "$PROMETHEUS_URL" "$CANONICAL_RULES" <<'PY'
python3 - "$PROMETHEUS_URL" "${REQUIRED_RULES[@]}" <<'PY'
import json
import re
import sys
import urllib.request
base_url = sys.argv[1].rstrip("/")
canonical_path = sys.argv[2]
name_pattern = re.compile(r"^\s*-\s*(?:alert|record):\s*['\"]?([^'\"#]+?)['\"]?\s*(?:#.*)?$")
required: set[str] = set()
try:
with open(canonical_path, encoding="utf-8") as handle:
for line in handle:
match = name_pattern.match(line)
if match:
required.add(match.group(1).strip())
except Exception as exc:
print(f"CANONICAL_PARSE_FAILED:{exc}")
raise SystemExit(0)
required = set(sys.argv[2:])
try:
with urllib.request.urlopen(f"{base_url}/api/v1/rules", timeout=8) as response:
payload = json.loads(response.read().decode("utf-8"))
@@ -115,8 +109,8 @@ main() {
before_matches="$(matches_canonical)"
repaired=0
if [[ "$missing" == QUERY_FAILED:* || "$missing" == CANONICAL_PARSE_FAILED:* ]]; then
log "Prometheus/canonical query failed: ${missing}"
if [[ "$missing" == QUERY_FAILED:* ]]; then
log "Prometheus query failed: ${missing}"
write_textfile "query_failed" 0 999 "$before_matches"
return 1
fi
@@ -135,8 +129,8 @@ main() {
after_missing="$(rules_missing_count)"
after_matches="$(matches_canonical)"
if [[ "$after_missing" == QUERY_FAILED:* || "$after_missing" == CANONICAL_PARSE_FAILED:* ]]; then
log "post-restore Prometheus/canonical query failed: ${after_missing}"
if [[ "$after_missing" == QUERY_FAILED:* ]]; then
log "post-restore Prometheus query failed: ${after_missing}"
write_textfile "post_query_failed" "$repaired" 999 "$after_matches"
return 1
fi

148
scripts/ops/recovery-scorecard-contract-check.py Executable file
View File

@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""Validate recovery scorecard recording-rule contract."""
from __future__ import annotations
import argparse
import json
import sys
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any
import yaml
DEFAULT_RULES = Path("ops/monitoring/alerts-unified.yml")
DEFAULT_BASELINE = Path("ops/reboot-recovery/full-stack-backup-baseline.yml")
EXPECTED_CORE = 'awoooi_recovery_core_ready{host="110",scope="110_120_121_188"}'
EXPECTED_DR = 'awoooi_recovery_dr_offsite_ready{host="110"}'
class ContractError(RuntimeError):
pass
def _rules(path: Path) -> list[dict[str, Any]]:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
rules: list[dict[str, Any]] = []
for group in data.get("groups") or []:
rules.extend(group.get("rules") or [])
return rules
def _expected_recording_rules(path: Path) -> list[str]:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
rules = data.get("monitoring_contract", {}).get("prometheus_recording_rules") or []
if not rules:
raise ContractError(f"missing monitoring_contract.prometheus_recording_rules in {path}")
return [str(rule) for rule in rules]
def static_check(rules_path: Path, baseline_path: Path) -> list[str]:
rules = _rules(rules_path)
by_record = {str(rule.get("record")): rule for rule in rules if rule.get("record")}
expected = _expected_recording_rules(baseline_path)
missing = sorted(set(expected) - set(by_record))
if missing:
raise ContractError(f"alerts-unified.yml missing recovery recording rules: {missing}")
core_expr = str(by_record["awoooi_recovery_core_ready"].get("expr", ""))
for required in [
"awoooi_cold_start_last_result",
"awoooi_cold_start_warn_gates",
"awoooi_cold_start_blocked_gates",
"awoooi_cold_start_last_green_timestamp",
]:
if required not in core_expr:
raise ContractError(f"awoooi_recovery_core_ready expr missing {required}")
dr_expr = str(by_record["awoooi_recovery_dr_offsite_ready"].get("expr", ""))
for required in [
"awoooi_backup_offsite_configured",
"awoooi_backup_offsite_fresh",
"awoooi_backup_credential_escrow_fresh",
]:
if required not in dr_expr:
raise ContractError(f"awoooi_recovery_dr_offsite_ready expr missing {required}")
return [
"OK alerts-unified.yml contains every recovery scorecard recording rule",
"OK recovery core rule depends on cold-start green/warn/blocked/last-green metrics",
"OK recovery DR rule depends on provider-neutral offsite freshness and credential escrow freshness",
]
def _prom_query(base_url: str, expr: str) -> list[dict[str, Any]]:
url = f"{base_url.rstrip('/')}/api/v1/query?" + urllib.parse.urlencode({"query": expr})
with urllib.request.urlopen(url, timeout=8) as response:
payload = json.loads(response.read().decode("utf-8"))
if payload.get("status") != "success":
raise ContractError(f"Prometheus query failed for {expr}: {payload}")
return payload.get("data", {}).get("result") or []
def _single_value(base_url: str, expr: str) -> float:
rows = _prom_query(base_url, expr)
if len(rows) != 1:
raise ContractError(f"Prometheus query expected one series for {expr}, got {len(rows)}")
value = rows[0].get("value") or []
if len(value) < 2:
raise ContractError(f"Prometheus query returned malformed value for {expr}: {rows[0]}")
try:
number = float(value[1])
except (TypeError, ValueError) as exc:
raise ContractError(f"Prometheus query returned non-numeric value for {expr}: {rows[0]}") from exc
if number not in {0.0, 1.0}:
raise ContractError(f"Prometheus recovery scorecard metric must be 0 or 1: {expr}={number}")
return number
def live_check(
base_url: str,
expect_core_ready: bool = False,
expect_dr_ready: bool = False,
) -> list[str]:
core = _single_value(base_url, EXPECTED_CORE)
dr = _single_value(base_url, EXPECTED_DR)
lines = [
f"OK live {EXPECTED_CORE} value={int(core)}",
f"OK live {EXPECTED_DR} value={int(dr)}",
]
if expect_core_ready and core != 1.0:
raise ContractError(f"expected core recovery ready, got {core}")
if expect_dr_ready and dr != 1.0:
raise ContractError(f"expected DR offsite ready, got {dr}")
return lines
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--rules", type=Path, default=DEFAULT_RULES)
parser.add_argument("--baseline", type=Path, default=DEFAULT_BASELINE)
parser.add_argument("--prometheus-url", default="")
parser.add_argument("--expect-core-ready", action="store_true")
parser.add_argument("--expect-dr-ready", action="store_true")
args = parser.parse_args()
try:
for line in static_check(args.rules, args.baseline):
print(line)
if args.prometheus_url:
for line in live_check(
args.prometheus_url,
args.expect_core_ready,
args.expect_dr_ready,
):
print(line)
except (ContractError, OSError, yaml.YAMLError, json.JSONDecodeError) as exc:
print(f"RECOVERY_SCORECARD_CONTRACT_FAILED {exc}", file=sys.stderr)
return 1
print("RECOVERY_SCORECARD_CONTRACT_OK")
return 0
if __name__ == "__main__":
raise SystemExit(main())

52
scripts/reboot-recovery/cold-start-textfile-exporter.sh
View File

@@ -1,10 +1,8 @@
#!/usr/bin/env bash
# Export AWOOOI full-stack cold-start gate status as node-exporter textfile metrics.
#
# 2026-05-06 ogt + Codex: reboot recovery hardening.
# Intent: give Prometheus and the AI incident flow a durable, read-only signal
# for the 110/120/121/188 startup gates. This wrapper never sends the
# Alertmanager smoke event and never writes remote state.
# This wrapper is read-only: it never sends the Alertmanager smoke event and
# never mutates remote host/service state.
set -uo pipefail
@@ -13,6 +11,8 @@ TEXTFILE_DIR="${TEXTFILE_DIR:-${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_expo
OUTPUT_NAME="${OUTPUT_NAME:-cold_start_recovery.prom}"
LOG_DIR="${LOG_DIR:-/home/wooo/reboot-recovery}"
CHECK_TIMEOUT_SECONDS="${CHECK_TIMEOUT_SECONDS:-240}"
CHECK_WATCH_INTERVAL_SECONDS="${CHECK_WATCH_INTERVAL_SECONDS:-10}"
CHECK_WATCH_MAX_ATTEMPTS="${CHECK_WATCH_MAX_ATTEMPTS:-3}"
HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_120_121_188}"
LOCK_FILE="${LOCK_FILE:-/tmp/awoooi-cold-start-textfile-exporter.lock}"
@@ -35,6 +35,10 @@ write_metric_file() {
local blocked_state="${11}"
local check_failed="${12}"
local last_green="${13}"
local k3s_node_fs_blocker="${14}"
local public_route_tls_blocker="${15}"
local host_120_unreachable_blocker="${16}"
local backup_health_blocker="${17}"
local host scope
host=$(escape_label "$HOST_LABEL")
scope=$(escape_label "$SCOPE_LABEL")
@@ -70,10 +74,16 @@ awoooi_cold_start_last_result{host="$host",scope="$scope",result="green"} $green
awoooi_cold_start_last_result{host="$host",scope="$scope",result="degraded"} $degraded
awoooi_cold_start_last_result{host="$host",scope="$scope",result="blocked"} $blocked_state
awoooi_cold_start_last_result{host="$host",scope="$scope",result="check_failed"} $check_failed
# HELP awoooi_cold_start_blocker_reason Whether a known cold-start blocker reason was detected in the last log.
# TYPE awoooi_cold_start_blocker_reason gauge
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="k3s_node_filesystem_error",target="120"} $k3s_node_fs_blocker
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="public_route_tls_failure",target="public_https"} $public_route_tls_blocker
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="120"} $host_120_unreachable_blocker
awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="backup_health_blocked",target="110"} $backup_health_blocker
METRICS
}
if [ -n "${BASH_VERSION:-}" ] && command -v flock >/dev/null 2>&1; then
if command -v flock >/dev/null 2>&1; then
exec 9>"$LOCK_FILE"
if ! flock -n 9; then
exit 0
@@ -92,13 +102,19 @@ if [ ! -x "$CHECK_SCRIPT" ]; then
tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX")
last_green=$(cat "$state_file" 2>/dev/null || echo 0)
printf 'CHECK_SCRIPT not executable: %s\n' "$CHECK_SCRIPT" >"$log_file"
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green"
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0
chmod 0644 "$tmp_metric"
mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME"
exit 0
fi
timeout "$CHECK_TIMEOUT_SECONDS" bash "$CHECK_SCRIPT" --monitor-read-only --no-color >"$log_tmp" 2>&1
timeout "$CHECK_TIMEOUT_SECONDS" bash "$CHECK_SCRIPT" \
--monitor-read-only \
--no-color \
--watch \
--interval "$CHECK_WATCH_INTERVAL_SECONDS" \
--max-attempts "$CHECK_WATCH_MAX_ATTEMPTS" \
>"$log_tmp" 2>&1
exit_code=$?
mv "$log_tmp" "$log_file"
@@ -111,6 +127,10 @@ green=0
degraded=0
blocked_state=0
check_failed=0
k3s_node_fs_blocker=0
public_route_tls_blocker=0
host_120_unreachable_blocker=0
backup_health_blocker=0
if [ -n "$summary_line" ]; then
monitor_up=1
@@ -130,6 +150,22 @@ else
check_failed=1
fi
if grep -Eq 'NODE_FS_ERROR_EVENTS[[:space:]]+[1-9][0-9]*|K3s node filesystem error events present' "$log_file"; then
k3s_node_fs_blocker=1
fi
if grep -Eq 'PUBLIC_ROUTE_TLS .*(000|5[0-9][0-9])|public route .* TLS certificate verification failed' "$log_file"; then
public_route_tls_blocker=1
fi
if grep -Eq 'BLOCKED (ping 192\.168\.0\.120|ssh port 192\.168\.0\.120:22|ssh 120 k3s read-only check)' "$log_file"; then
host_120_unreachable_blocker=1
fi
if grep -Eq 'BLOCKED 110 backup health has stale expected jobs' "$log_file"; then
backup_health_blocker=1
fi
end_ts=$(date +%s)
if [ "$green" -eq 1 ]; then
printf '%s\n' "$end_ts" >"$state_file"
@@ -137,6 +173,6 @@ fi
last_green=$(cat "$state_file" 2>/dev/null || echo 0)
tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX")
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green"
write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker"
chmod 0644 "$tmp_metric"
mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME"

178
scripts/reboot-recovery/full-stack-cold-start-check.sh
View File

@@ -7,6 +7,7 @@ set -uo pipefail
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=6)
SEND_ALERT_TEST=0
MONITOR_READ_ONLY=0
NO_COLOR_FLAG=0
WATCH_MODE=0
WATCH_INTERVAL=60
WATCH_MAX_ATTEMPTS=30
@@ -30,15 +31,17 @@ USAGE
}
while [ "$#" -gt 0 ]; do
case "$1" in
arg="$1"
case "$arg" in
--send-alert-test)
SEND_ALERT_TEST=1
;;
--monitor-read-only)
MONITOR_READ_ONLY=1
SEND_ALERT_TEST=0
;;
--no-color)
NO_COLOR=1
NO_COLOR_FLAG=1
;;
--watch)
WATCH_MODE=1
@@ -64,7 +67,7 @@ while [ "$#" -gt 0 ]; do
exit 0
;;
*)
echo "Unknown argument: $1" >&2
echo "Unknown argument: $arg" >&2
usage >&2
exit 64
;;
@@ -72,7 +75,7 @@ while [ "$#" -gt 0 ]; do
shift
done
if [ -n "${NO_COLOR:-}" ]; then
if [ -n "${NO_COLOR:-}" ] || [ "$NO_COLOR_FLAG" -eq 1 ]; then
RED=""
GREEN=""
YELLOW=""
@@ -90,12 +93,6 @@ PASS=0
WARN=0
FAIL=0
reset_counters() {
PASS=0
WARN=0
FAIL=0
}
log_section() {
printf "\n%s=== %s ===%s\n" "$BLUE" "$1" "$NC"
}
@@ -198,6 +195,18 @@ probe_tcp() {
nc -G 3 -z "$host" "$port" >/dev/null 2>&1 || nc -w 3 -z "$host" "$port" >/dev/null 2>&1
}
print_neighbor_rows() {
if command -v arp >/dev/null 2>&1; then
arp -an | grep -E '192\.168\.0\.(110|120|121|188)'
return $?
fi
if command -v ip >/dev/null 2>&1; then
ip neigh show | grep -E '192\.168\.0\.(110|120|121|188)'
return $?
fi
return 1
}
print_header() {
echo "AWOOOI full-stack cold-start check"
date '+%Y-%m-%d %H:%M:%S %Z'
@@ -222,12 +231,12 @@ check_network() {
fi
done
if arp -an | grep -E '192\.168\.0\.(110|120|121|188)'; then
ok "ARP evidence printed"
if print_neighbor_rows; then
ok "neighbor evidence printed"
elif [ "$MONITOR_READ_ONLY" -eq 1 ]; then
ok "ARP evidence unavailable in monitor mode; ping and TCP gates passed"
ok "neighbor evidence unavailable in monitor mode; ping and TCP gates provide primary signal"
else
warn "no ARP rows printed for one or more hosts"
warn "no neighbor rows printed for one or more hosts"
fi
}
@@ -370,21 +379,34 @@ WEB_CODE $web_code"
check_public_routes() {
log_section "P2-PUBLIC-ROUTES"
local awoooi_api_code awoooi_web_code momo_code momo_health_code
awoooi_api_code=$(probe_http_code "https://awoooi.wooo.work/api/v1/health")
awoooi_web_code=$(probe_http_code "https://awoooi.wooo.work/")
momo_code=$(probe_http_code "https://mo.wooo.work/")
momo_health_code=$(probe_http_code "https://mo.wooo.work/health")
local item name url code tls_code
local routes=(
"awoooi_api|https://awoooi.wooo.work/api/v1/health"
"awoooi_web|https://awoooi.wooo.work/"
"momo_web|https://mo.wooo.work/"
"momo_health|https://mo.wooo.work/health"
"gitea|https://gitea.wooo.work/"
"harbor|https://harbor.wooo.work/"
"registry|https://registry.wooo.work/"
"sentry|https://sentry.wooo.work/"
"signoz|https://signoz.wooo.work/"
"stock|https://stock.wooo.work/"
"langfuse|https://langfuse.wooo.work/"
"bitan|https://bitan.wooo.work/"
"aiops|https://aiops.wooo.work/"
)
echo "AWOOOI_PUBLIC_API_CODE $awoooi_api_code"
echo "AWOOOI_PUBLIC_WEB_CODE $awoooi_web_code"
echo "MOMO_PUBLIC_CODE $momo_code"
echo "MOMO_PUBLIC_HEALTH_CODE $momo_health_code"
[[ "$awoooi_api_code" =~ ^[23] ]] && ok "AWOOOI public API reachable" || warn "AWOOOI public API not confirmed"
[[ "$awoooi_web_code" =~ ^[23] ]] && ok "AWOOOI public web reachable" || warn "AWOOOI public web not confirmed"
[[ "$momo_code" =~ ^[23] ]] && ok "momo public route reachable" || warn "momo public route not confirmed"
[[ "$momo_health_code" =~ ^[23] ]] && ok "momo public health reachable" || warn "momo public health not confirmed"
for item in "${routes[@]}"; do
name="${item%%|*}"
url="${item#*|}"
code=$(probe_http_code "$url")
echo "PUBLIC_ROUTE $name $code $url"
[[ "$code" =~ ^[23] ]] && ok "public route $name reachable" || warn "public route $name not confirmed"
tls_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 8 "$url" 2>/dev/null || true)
tls_code="${tls_code:-000}"
echo "PUBLIC_ROUTE_TLS $name $tls_code $url"
[[ "$tls_code" =~ ^[23] ]] && ok "public route $name TLS certificate verified" || fail "public route $name TLS certificate verification failed"
done
}
check_schedules() {
@@ -394,7 +416,7 @@ check_schedules() {
if out=$(host_cmd "ollama@192.168.0.188" '
now=$(date +%s)
echo "CRON_188 $(systemctl is-active cron 2>/dev/null || systemctl is-active crond 2>/dev/null || true)"
for f in /home/ollama/node_exporter_textfiles/backup.prom /home/ollama/node_exporter_textfiles/docker_restart_count.prom /home/ollama/node_exporter_textfiles/docker_stats.prom; do
for f in /home/ollama/node_exporter_textfiles/backup.prom /home/ollama/node_exporter_textfiles/backup_health.prom /home/ollama/node_exporter_textfiles/docker_restart_count.prom /home/ollama/node_exporter_textfiles/docker_stats.prom /home/ollama/node_exporter_textfiles/storage_health.prom; do
if [ -f "$f" ]; then
mt=$(stat -c %Y "$f")
echo "TEXTFILE_188 $(basename "$f") age=$((now - mt))"
@@ -405,17 +427,37 @@ done
if [ -f /home/ollama/node_exporter_textfiles/backup.prom ]; then
awk -v now="$now" "/^backup_110_last_success_timestamp / {printf \"BACKUP_110_AGE %d\\n\", now - int(\$2)}" /home/ollama/node_exporter_textfiles/backup.prom
fi
echo "SCHEDULER_STATE $(docker inspect -f "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}" momo-scheduler 2>/dev/null || true)"
echo "SCHEDULER_REGISTERED $(docker logs --since 6h momo-scheduler 2>&1 | grep -c "全部排程任務已註冊" || true)"
if [ -f /home/ollama/node_exporter_textfiles/backup_health.prom ]; then
awk "/^awoooi_backup_job_fresh/ {total++; if (int(\$2) == 0) stale++} /^awoooi_backup_job_configured/ {if (int(\$2) == 0) missing_cron++} /^awoooi_backup_script_present/ {if (int(\$2) == 0) missing_script++} END {printf \"BACKUP_HEALTH_188 total=%d stale=%d missing_cron=%d missing_script=%d\\n\", total+0, stale+0, missing_cron+0, missing_script+0}" /home/ollama/node_exporter_textfiles/backup_health.prom
fi
if [ -f /home/ollama/node_exporter_textfiles/storage_health.prom ]; then
awk "/^awoooi_host_storage_root_readonly/ {readonly=int(\$2)} /^awoooi_host_storage_current_boot_error_count/ {current=int(\$2)} END {printf \"STORAGE_HEALTH_188 root_readonly=%d current=%d\\n\", readonly+0, current+0}" /home/ollama/node_exporter_textfiles/storage_health.prom
fi
echo "SCHEDULER_CONTAINER_RUNNING $(docker inspect -f "{{.State.Running}}" momo-scheduler 2>/dev/null || true)"
echo "SCHEDULER_CONTAINER_HEALTH $(docker inspect -f "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}" momo-scheduler 2>/dev/null || true)"
echo "SCHEDULER_REGISTERED $(docker logs --tail 200 momo-scheduler 2>&1 | grep -c "全部排程任務已註冊" || true)"
echo "SCHEDULER_RECENT_ACTIVITY $(docker logs --since 2h momo-scheduler 2>&1 | grep -Ec "AutoImport|Meta-Analysis|Scheduler" || true)"
momo_sync=$(docker exec momo-db sh -c "psql -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -Atc \"WITH scope AS (SELECT min(snapshot_date::date) dmin, max(snapshot_date::date) dmax, count(*) sc FROM daily_sales_snapshot WHERE snapshot_date::date >= make_date(extract(year from current_date)::int, extract(month from current_date)::int, 1)), monthly AS (SELECT count(*) mc, min(\\\"日期\\\"::date) mmin, max(\\\"日期\\\"::date) mmax FROM realtime_sales_monthly, scope WHERE scope.sc > 0 AND \\\"日期\\\"::date BETWEEN scope.dmin AND scope.dmax) SELECT coalesce(scope.sc,0)::text || chr(124) || coalesce(monthly.mc,0)::text || chr(124) || coalesce(scope.dmin::text,chr(45)) || chr(124) || coalesce(scope.dmax::text,chr(45)) || chr(124) || coalesce(monthly.mmin::text,chr(45)) || chr(124) || coalesce(monthly.mmax::text,chr(45)) FROM scope, monthly;\"" 2>/dev/null || true)
echo "MOMO_MONTHLY_SYNC ${momo_sync:-unavailable}"
' 2>&1); then
echo "$out"
grep -q "CRON_188 active" <<<"$out" && ok "188 cron active" || warn "188 cron not confirmed"
awk '/TEXTFILE_188 backup.prom age=/ {split($3,a,"="); exit !(a[2] < 90000)}' <<<"$out" && ok "188 backup textfile fresh enough" || warn "188 backup textfile stale or missing"
awk '/TEXTFILE_188 backup_health.prom age=/ {split($3,a,"="); exit !(a[2] < 900)}' <<<"$out" && ok "188 backup health exporter fresh" || warn "188 backup health exporter stale"
awk '/TEXTFILE_188 docker_restart_count.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "188 docker restart exporter fresh" || warn "188 docker restart exporter stale"
awk '/TEXTFILE_188 docker_stats.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "188 docker stats exporter fresh" || warn "188 docker stats exporter stale"
awk '/TEXTFILE_188 storage_health.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "188 storage health exporter fresh" || warn "188 storage health exporter stale"
grep -q "STORAGE_HEALTH_188 root_readonly=0 current=0" <<<"$out" && ok "188 current boot storage health clean" || warn "188 storage health not clean"
awk '/BACKUP_110_AGE / {exit !($2 < 90000)}' <<<"$out" && ok "188 backup-from-110 success within 25h" || warn "188 backup-from-110 success not confirmed"
grep -q "SCHEDULER_STATE running healthy" <<<"$out" && ok "188 momo scheduler container healthy" || warn "188 momo scheduler health not confirmed"
awk '/SCHEDULER_REGISTERED / {exit !($2 > 0)}' <<<"$out" && ok "188 momo scheduler registered jobs within 6h" || warn "188 momo scheduler registration not confirmed within 6h"
grep -q "BACKUP_HEALTH_188 total=" <<<"$out" && awk '/BACKUP_HEALTH_188/ {split($3,a,"="); split($4,b,"="); split($5,c,"="); exit !((a[2]+b[2]+c[2]) == 0)}' <<<"$out" && ok "188 backup health has no stale expected jobs" || warn "188 backup health has stale expected jobs"
if grep -q "SCHEDULER_CONTAINER_HEALTH healthy" <<<"$out" && awk '/SCHEDULER_RECENT_ACTIVITY / {exit !($2 > 0)}' <<<"$out"; then
ok "188 momo scheduler healthy with recent task activity"
elif awk '/SCHEDULER_REGISTERED / {exit !($2 > 0)}' <<<"$out"; then
ok "188 momo scheduler registered jobs"
else
warn "188 momo scheduler registration/activity not confirmed"
fi
awk '/MOMO_MONTHLY_SYNC / {split($2,a,"|"); exit !(a[1] > 0 && a[1] == a[2] && a[3] == a[5] && a[4] == a[6])}' <<<"$out" && ok "188 momo current-month snapshot and realtime tables match" || warn "188 momo current-month snapshot/realtime sync not confirmed"
else
warn "188 schedule check unavailable"
echo "$out"
@@ -427,7 +469,7 @@ echo "CRON_110 $(systemctl is-active cron 2>/dev/null || systemctl is-active cro
echo "FAILED_UNITS_110 $(systemctl --failed --no-legend --plain 2>/dev/null | wc -l)"
echo "MOMO_STARTUP_ENABLED $(systemctl is-enabled momo-startup-complete.service 2>/dev/null || true)"
echo "STAGGERED_STARTUP_ENABLED $(systemctl is-enabled wooo-staggered-startup.service 2>/dev/null || true)"
for f in /home/wooo/node_exporter_textfiles/docker_stats.prom /home/wooo/node_exporter_textfiles/systemd_units.prom; do
for f in /home/wooo/node_exporter_textfiles/docker_stats.prom /home/wooo/node_exporter_textfiles/systemd_units.prom /home/wooo/node_exporter_textfiles/storage_health.prom /home/wooo/node_exporter_textfiles/backup_health.prom; do
if [ -f "$f" ]; then
mt=$(stat -c %Y "$f")
echo "TEXTFILE_110 $(basename "$f") age=$((now - mt))"
@@ -435,6 +477,12 @@ for f in /home/wooo/node_exporter_textfiles/docker_stats.prom /home/wooo/node_ex
echo "TEXTFILE_110 $(basename "$f") missing"
fi
done
if [ -f /home/wooo/node_exporter_textfiles/storage_health.prom ]; then
awk "/^awoooi_host_storage_root_readonly/ {readonly=int(\$2)} /^awoooi_host_storage_current_boot_error_count/ {current=int(\$2)} END {printf \"STORAGE_HEALTH_110 root_readonly=%d current=%d\\n\", readonly+0, current+0}" /home/wooo/node_exporter_textfiles/storage_health.prom
fi
if [ -f /home/wooo/node_exporter_textfiles/backup_health.prom ]; then
awk "/^awoooi_backup_job_fresh/ {total++; if (int(\$2) == 0) stale++} /^awoooi_backup_job_configured/ {if (int(\$2) == 0) missing_cron++} /^awoooi_backup_script_present/ {if (int(\$2) == 0) missing_script++} /^awoooi_backup_last_run_failed_count/ {if (\$0 ~ /(exported_job|job)=\"backup_all\"/) failed=int(\$2)} /^awoooi_backup_config_capture_critical_failed_count/ {config_failed=int(\$2)} /^awoooi_backup_integrity_fresh/ {integrity_total++; if (int(\$2) == 0) integrity_stale++} END {printf \"BACKUP_HEALTH_110 total=%d stale=%d missing_cron=%d missing_script=%d failed_count=%d config_failed=%d integrity_total=%d integrity_stale=%d\\n\", total+0, stale+0, missing_cron+0, missing_script+0, failed+0, config_failed+0, integrity_total+0, integrity_stale+0}" /home/wooo/node_exporter_textfiles/backup_health.prom
fi
' 2>&1); then
echo "$out"
grep -q "CRON_110 active" <<<"$out" && ok "110 cron active" || warn "110 cron not confirmed"
@@ -443,6 +491,11 @@ done
grep -q "STAGGERED_STARTUP_ENABLED disabled" <<<"$out" && ok "110 stale staggered startup unit disabled" || warn "110 stale staggered startup unit not disabled"
awk '/TEXTFILE_110 docker_stats.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "110 docker stats exporter fresh" || warn "110 docker stats exporter stale"
awk '/TEXTFILE_110 systemd_units.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "110 systemd units exporter fresh" || warn "110 systemd units exporter stale"
awk '/TEXTFILE_110 storage_health.prom age=/ {split($3,a,"="); exit !(a[2] < 300)}' <<<"$out" && ok "110 storage health exporter fresh" || warn "110 storage health exporter stale"
awk '/TEXTFILE_110 backup_health.prom age=/ {split($3,a,"="); exit !(a[2] < 900)}' <<<"$out" && ok "110 backup health exporter fresh" || warn "110 backup health exporter stale"
grep -q "STORAGE_HEALTH_110 root_readonly=0 current=0" <<<"$out" && ok "110 current boot storage health clean" || warn "110 storage health not clean"
grep -q "BACKUP_HEALTH_110 total=" <<<"$out" && awk '/BACKUP_HEALTH_110/ {split($3,a,"="); split($4,b,"="); split($5,c,"="); split($6,d,"="); split($7,e,"="); exit !((a[2]+b[2]+c[2]) == 0 && d[2] == 0 && e[2] == 0)}' <<<"$out" && ok "110 backup health has no stale expected jobs" || warn "110 latest aggregate/config backup had failed components; rerun backup-all after 120 recovers"
awk '/BACKUP_HEALTH_110/ {split($9,a,"="); exit !(a[2] == 0)}' <<<"$out" && ok "110 backup integrity and restore drill fresh" || warn "110 backup integrity or restore drill stale"
else
warn "110 schedule check unavailable"
echo "$out"
@@ -494,54 +547,41 @@ summary() {
echo "PASS=$PASS WARN=$WARN BLOCKED=$FAIL"
if [ "$FAIL" -gt 0 ]; then
echo "Result: BLOCKED. Fix the first blocked gate before releasing runner/CD/AI auto-remediation."
return 2
exit 2
fi
if [ "$WARN" -gt 0 ]; then
echo "Result: DEGRADED. Core gates passed but warnings remain."
return 1
exit 1
fi
echo "Result: GREEN. Full stack is ready for controlled runner/CD release."
return 0
}
run_once() {
reset_counters
print_header
check_network
check_188
check_110
check_k3s
check_workload_and_alertchain
check_public_routes
check_schedules
summary
}
if [ "$WATCH_MODE" -eq 1 ]; then
attempt=1
while :; do
if [ "$WATCH_MAX_ATTEMPTS" -eq 0 ]; then
printf "\nWatch attempt %s/unlimited\n" "$attempt"
else
printf "\nWatch attempt %s/%s\n" "$attempt" "$WATCH_MAX_ATTEMPTS"
fi
run_once
rc=2
while true; do
echo "WATCH_ATTEMPT=$attempt"
args=()
[ "$MONITOR_READ_ONLY" -eq 1 ] && args+=(--monitor-read-only)
[ "$NO_COLOR_FLAG" -eq 1 ] && args+=(--no-color)
[ "$SEND_ALERT_TEST" -eq 1 ] && args+=(--send-alert-test)
bash "$0" "${args[@]}"
rc=$?
if [ "$rc" -eq 0 ]; then
exit 0
fi
if [ "$WATCH_MAX_ATTEMPTS" -ne 0 ] && [ "$attempt" -ge "$WATCH_MAX_ATTEMPTS" ]; then
echo "Watch stopped before GREEN. Last result code: $rc"
[ "$rc" -eq 0 ] && exit 0
if [ "$WATCH_MAX_ATTEMPTS" -gt 0 ] && [ "$attempt" -ge "$WATCH_MAX_ATTEMPTS" ]; then
exit "$rc"
fi
echo "Waiting ${WATCH_INTERVAL}s before the next cold-start gate check..."
sleep "$WATCH_INTERVAL"
attempt=$((attempt + 1))
sleep "$WATCH_INTERVAL"
done
fi
run_once
exit $?
print_header
check_network
check_188
check_110
check_k3s
check_workload_and_alertchain
check_public_routes
check_schedules
summary