068c18e2f0
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 58s
CD Pipeline / build-and-deploy (push) Has started running
CD Pipeline / post-deploy-checks (push) Has been cancelled
620 lines
36 KiB
Bash
Executable File
620 lines
36 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Read-only audit for AWOOOI reboot-recovery readiness artifacts.
|
||
|
||
set -euo pipefail
|
||
|
||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
cd "$ROOT_DIR"
|
||
|
||
RUN_LIVE=0
|
||
NO_COLOR=0
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--live)
|
||
RUN_LIVE=1
|
||
;;
|
||
--no-color)
|
||
NO_COLOR=1
|
||
;;
|
||
-h|--help)
|
||
cat <<'USAGE'
|
||
Usage: bash scripts/reboot-recovery/reboot-recovery-readiness-audit.sh [--live] [--no-color]
|
||
|
||
Checks repo-side SOP/script/Ansible/alert/CI readiness. With --live, also runs
|
||
the read-only full-stack cold-start gate.
|
||
USAGE
|
||
exit 0
|
||
;;
|
||
*)
|
||
echo "Unknown argument: $arg" >&2
|
||
exit 2
|
||
;;
|
||
esac
|
||
done
|
||
|
||
if [ "$NO_COLOR" = "1" ]; then
|
||
green=""
|
||
yellow=""
|
||
red=""
|
||
reset=""
|
||
else
|
||
green="$(printf '\033[32m')"
|
||
yellow="$(printf '\033[33m')"
|
||
red="$(printf '\033[31m')"
|
||
reset="$(printf '\033[0m')"
|
||
fi
|
||
|
||
pass=0
|
||
warn=0
|
||
fail=0
|
||
|
||
ok() {
|
||
pass=$((pass + 1))
|
||
printf "%sOK%s %s\n" "$green" "$reset" "$*"
|
||
}
|
||
|
||
warning() {
|
||
warn=$((warn + 1))
|
||
printf "%sWARN%s %s\n" "$yellow" "$reset" "$*"
|
||
}
|
||
|
||
blocked() {
|
||
fail=$((fail + 1))
|
||
printf "%sBLOCKED%s %s\n" "$red" "$reset" "$*"
|
||
}
|
||
|
||
require_file() {
|
||
local path="$1"
|
||
local label="$2"
|
||
if [ -f "$path" ]; then
|
||
ok "$label exists: $path"
|
||
else
|
||
blocked "$label missing: $path"
|
||
fi
|
||
}
|
||
|
||
require_dir() {
|
||
local path="$1"
|
||
local label="$2"
|
||
if [ -d "$path" ]; then
|
||
ok "$label exists: $path"
|
||
else
|
||
blocked "$label missing: $path"
|
||
fi
|
||
}
|
||
|
||
pattern_present() {
|
||
local pattern="$1"
|
||
local path="$2"
|
||
if [ ! -f "$path" ]; then
|
||
return 1
|
||
fi
|
||
if command -v rg >/dev/null 2>&1; then
|
||
rg -q -- "$pattern" "$path"
|
||
return $?
|
||
fi
|
||
# Gitea runner images do not always provide ripgrep; keep this audit portable.
|
||
python3 - "$pattern" "$path" <<'PY'
|
||
import re
|
||
import sys
|
||
import warnings
|
||
from pathlib import Path
|
||
|
||
pattern = sys.argv[1]
|
||
path = Path(sys.argv[2])
|
||
try:
|
||
text = path.read_text(encoding="utf-8")
|
||
except UnicodeDecodeError:
|
||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||
|
||
warnings.filterwarnings("ignore", category=FutureWarning)
|
||
sys.exit(0 if re.search(pattern, text) else 1)
|
||
PY
|
||
}
|
||
|
||
require_pattern() {
|
||
local pattern="$1"
|
||
local path="$2"
|
||
local label="$3"
|
||
if pattern_present "$pattern" "$path"; then
|
||
ok "$label present in $path"
|
||
else
|
||
blocked "$label missing in $path"
|
||
fi
|
||
}
|
||
|
||
forbid_pattern() {
|
||
local pattern="$1"
|
||
local path="$2"
|
||
local label="$3"
|
||
if pattern_present "$pattern" "$path"; then
|
||
blocked "$label forbidden pattern present in $path"
|
||
else
|
||
ok "$label forbidden pattern absent in $path"
|
||
fi
|
||
}
|
||
|
||
run_with_retries() {
|
||
local attempts="$1"
|
||
local interval="$2"
|
||
shift 2
|
||
local attempt=1
|
||
while true; do
|
||
if "$@"; then
|
||
return 0
|
||
fi
|
||
if [ "$attempt" -ge "$attempts" ]; then
|
||
return 1
|
||
fi
|
||
sleep "$interval"
|
||
attempt=$((attempt + 1))
|
||
done
|
||
}
|
||
|
||
ensure_ansible_validation_path() {
|
||
local venv_dir="${ANSIBLE_VALIDATION_VENV:-/tmp/awoooi-ansible-venv}"
|
||
|
||
if python3 - <<'PY' >/dev/null 2>&1
|
||
import yaml # noqa: F401
|
||
PY
|
||
then
|
||
return 0
|
||
fi
|
||
|
||
if [ -x "$venv_dir/bin/python" ] && "$venv_dir/bin/python" - <<'PY' >/dev/null 2>&1; then
|
||
import yaml # noqa: F401
|
||
PY
|
||
export PATH="$venv_dir/bin:$PATH"
|
||
ok "Ansible validation venv reused for PyYAML: $venv_dir"
|
||
return 0
|
||
fi
|
||
|
||
warning "PyYAML unavailable in local python; bootstrapping Ansible validation venv"
|
||
if bash scripts/ops/bootstrap-ansible-validation-env.sh >/tmp/awoooi-ansible-validation-bootstrap.log 2>&1; then
|
||
export PATH="$venv_dir/bin:$PATH"
|
||
ok "Ansible validation venv bootstrapped: $venv_dir"
|
||
else
|
||
warning "Ansible validation venv bootstrap failed; see /tmp/awoooi-ansible-validation-bootstrap.log"
|
||
fi
|
||
}
|
||
|
||
echo "AWOOOI reboot-recovery readiness audit"
|
||
date
|
||
echo
|
||
|
||
echo "== SOP and baseline =="
|
||
require_file docs/runbooks/FULL-STACK-COLD-START-SOP.md "Full-stack cold-start SOP"
|
||
require_file docs/runbooks/REBOOT-RECOVERY-SOP.md "Legacy reboot recovery SOP"
|
||
require_file docs/runbooks/HOST-RESOURCE-BASELINE-110-188.md "110/188 resource baseline"
|
||
require_file docs/runbooks/ANSIBLE-OPERATING-MODEL.md "Ansible operating model"
|
||
require_file docs/runbooks/OFFSITE-BACKUP-ESCROW-RUNBOOK.md "Offsite backup and credential escrow runbook"
|
||
require_file ops/reboot-recovery/full-stack-cold-start-baseline.yml "Machine-readable baseline"
|
||
require_file ops/reboot-recovery/full-stack-backup-baseline.yml "Machine-readable backup baseline"
|
||
require_pattern "Plan B：降級運轉與回復路徑" docs/runbooks/FULL-STACK-COLD-START-SOP.md "SOP Plan B section"
|
||
require_pattern "B0_ABORTED_BEFORE_REBOOT" docs/runbooks/FULL-STACK-COLD-START-SOP.md "SOP Plan B B0 level"
|
||
require_pattern "B5_DR_COMPLETE" docs/runbooks/FULL-STACK-COLD-START-SOP.md "SOP Plan B B5 level"
|
||
require_pattern "RETURNED_TO_PLAN_A" docs/runbooks/FULL-STACK-COLD-START-SOP.md "SOP Plan B return-to-Plan-A closeout"
|
||
require_pattern "OPEN_INCIDENT_REQUIRED" docs/runbooks/FULL-STACK-COLD-START-SOP.md "SOP Plan B incident closeout"
|
||
require_pattern "P0-NETWORK" ops/reboot-recovery/full-stack-cold-start-baseline.yml "P0-NETWORK phase"
|
||
require_pattern "P2-SCHEDULES" ops/reboot-recovery/full-stack-cold-start-baseline.yml "P2-SCHEDULES phase"
|
||
require_pattern "P3-RUNNER-CD" ops/reboot-recovery/full-stack-cold-start-baseline.yml "P3-RUNNER-CD phase"
|
||
require_pattern "host_service_config_backup_success_under_48h" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Config backup gate"
|
||
require_pattern "plan_b:" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Machine-readable Plan B baseline"
|
||
require_pattern "B0_ABORTED_BEFORE_REBOOT" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B B0 level baseline"
|
||
require_pattern "B1_HOST_RECOVERY_ONLY" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B B1 level baseline"
|
||
require_pattern "B2_CORE_SERVICE_READY" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B B2 level baseline"
|
||
require_pattern "B3_SERVICE_AVAILABLE_DEGRADED" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B B3 level baseline"
|
||
require_pattern "B4_FULL_STACK_GREEN" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B B4 level baseline"
|
||
require_pattern "B5_DR_COMPLETE" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B B5 level baseline"
|
||
require_pattern "RETURNED_TO_PLAN_A" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B return-to-Plan-A closeout baseline"
|
||
require_pattern "SERVICE_AVAILABLE_DEGRADED" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B degraded closeout baseline"
|
||
require_pattern "OPEN_INCIDENT_REQUIRED" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B incident closeout baseline"
|
||
require_pattern "T\\+120" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B T+120 stop-line baseline"
|
||
require_pattern "do_not_call_route_200_full_stack_green" ops/reboot-recovery/full-stack-cold-start-baseline.yml "Plan B false-green blocker"
|
||
require_pattern "backup_domains" ops/reboot-recovery/full-stack-backup-baseline.yml "Backup domain inventory"
|
||
require_pattern "credential_escrow" ops/reboot-recovery/full-stack-backup-baseline.yml "Credential escrow backup domain"
|
||
require_pattern "external_dns_and_public_routes" ops/reboot-recovery/full-stack-backup-baseline.yml "External DNS/public route backup domain"
|
||
require_pattern "backup_repositories_and_integrity" ops/reboot-recovery/full-stack-backup-baseline.yml "Backup repository integrity domain"
|
||
require_pattern "source_of_truth_and_ops_memory" ops/reboot-recovery/full-stack-backup-baseline.yml "Source-of-truth and ops memory backup domain"
|
||
require_pattern "live_visibility_checks" ops/reboot-recovery/full-stack-backup-baseline.yml "Backup live alert visibility contract"
|
||
require_pattern "dr_offsite_scorecard" ops/reboot-recovery/full-stack-backup-baseline.yml "Strict DR offsite scorecard gate"
|
||
require_pattern "dr_offsite_operator_checklist" ops/reboot-recovery/full-stack-backup-baseline.yml "DR offsite operator checklist gate"
|
||
require_pattern "strict_dr_exit_conditions" ops/reboot-recovery/full-stack-backup-baseline.yml "Strict DR exit conditions"
|
||
|
||
echo
|
||
echo "== Scripts =="
|
||
require_file scripts/reboot-recovery/full-stack-cold-start-check.sh "Authoritative cold-start gate"
|
||
require_file scripts/reboot-recovery/full-stack-recovery-scorecard.sh "Full-stack recovery scorecard"
|
||
require_file scripts/reboot-recovery/120-fsck-maintenance-checklist.sh "120 filesystem maintenance checklist"
|
||
require_file scripts/reboot-recovery/dr-offsite-operator-checklist.sh "DR offsite operator checklist"
|
||
require_file scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite post-marker convergence waiter"
|
||
require_file scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start textfile exporter"
|
||
require_pattern "NODE_FS_ERROR_EVENTS" scripts/reboot-recovery/full-stack-cold-start-check.sh "K3s node filesystem event gate"
|
||
require_pattern "STALE_FAILED_JOBS" scripts/reboot-recovery/full-stack-cold-start-check.sh "K3s stale failed Job evidence counter"
|
||
require_pattern "ACTIVE_FAILED_JOBS" scripts/reboot-recovery/full-stack-cold-start-check.sh "K3s active failed Job blocker counter"
|
||
require_pattern "CHECK_WATCH_MAX_ATTEMPTS" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start textfile exporter retries transient route failures"
|
||
require_pattern "max-attempts" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start textfile exporter uses watch mode"
|
||
require_pattern "awoooi_cold_start_blocker_reason" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start blocker reason metric"
|
||
require_pattern "host_unreachable" scripts/reboot-recovery/cold-start-textfile-exporter.sh "Cold-start host unreachable blocker reason"
|
||
require_pattern "Do not run fsck against the mounted root filesystem" scripts/reboot-recovery/120-fsck-maintenance-checklist.sh "120 fsck checklist online fsck prohibition"
|
||
require_file scripts/reboot-recovery/install-cold-start-monitor-110.sh "110 monitor installer"
|
||
require_file scripts/reboot-recovery/reboot-recovery-readiness-audit.sh "Readiness audit"
|
||
require_file scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh "Cold-start monitor deploy parity check"
|
||
require_file scripts/reboot-recovery/p3-controlled-release-gate.sh "P3 controlled release gate"
|
||
require_file scripts/ops/ansible-validate.sh "Ansible validation script"
|
||
require_file scripts/ops/bootstrap-ansible-validation-env.sh "Ansible validation venv bootstrap script"
|
||
require_file scripts/ops/doc-secrets-sanity-check.py "Documentation secrets sanity check"
|
||
require_file scripts/ops/docker-stats-textfile-exporter.py "Docker stats textfile exporter"
|
||
require_file scripts/ops/systemd-units-textfile-exporter.py "Systemd units textfile exporter"
|
||
require_file scripts/ops/storage-health-textfile-exporter.py "Storage health textfile exporter"
|
||
require_file scripts/ops/backup-health-textfile-exporter.py "Backup health textfile exporter"
|
||
require_file scripts/ops/host-runaway-process-exporter.py "Host runaway process textfile exporter"
|
||
require_file scripts/ops/host-runaway-process-remediation.py "Host runaway process gated remediation helper"
|
||
require_file scripts/ops/backup-alert-label-contract-check.py "Backup alert label contract check"
|
||
require_file scripts/ops/backup-alert-live-visibility-check.py "Backup alert live visibility check"
|
||
require_file scripts/ops/recovery-scorecard-contract-check.py "Recovery scorecard contract check"
|
||
require_file scripts/backup/common.sh "Backup shared library"
|
||
require_file scripts/backup/backup-all.sh "Full backup orchestrator"
|
||
require_file scripts/backup/backup-status.sh "Daily backup Telegram heartbeat"
|
||
require_file scripts/backup/backup-gitea.sh "Gitea backup script"
|
||
require_file scripts/backup/backup-harbor.sh "Harbor backup script"
|
||
require_file scripts/backup/backup-momo.sh "110 momo restic backup script"
|
||
require_file scripts/backup/backup-awoooi.sh "AWOOOI daily DB backup"
|
||
require_file scripts/backup/backup-awoooi-frequent.sh "AWOOOI high-frequency DB backup"
|
||
require_file scripts/backup/backup-langfuse.sh "Langfuse backup script"
|
||
require_file scripts/backup/backup-monitoring.sh "Monitoring backup script"
|
||
require_file scripts/backup/backup-signoz.sh "SignOz backup script"
|
||
require_file scripts/backup/backup-open-webui.sh "Open-WebUI backup script"
|
||
require_file scripts/backup/backup-clawbot.sh "ClawBot backup script"
|
||
require_file scripts/backup/backup-configs.sh "Host/service config backup"
|
||
require_file scripts/backup/backup-momo-188-pg.sh "188 momo PostgreSQL backup script"
|
||
require_file scripts/backup/backup-sentry.sh "Sentry dedicated data backup"
|
||
require_file scripts/backup/backup-ai-artifacts.sh "AI artifacts and Ollama manifest backup"
|
||
require_file scripts/backup/backup-public-routes.sh "Public routes DNS/TLS evidence backup"
|
||
require_file scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive/rclone host-local config helper"
|
||
require_pattern "create-root-remote" scripts/backup/configure-offsite-rclone.sh "Offsite Google Drive root-scoped remote helper"
|
||
require_pattern "gdrive_awoooi_restic" docs/runbooks/OFFSITE-BACKUP-ESCROW-RUNBOOK.md "Offsite Google Drive root-scoped remote runbook"
|
||
require_file scripts/backup/configure-offsite-b2.sh "Offsite B2 legacy config helper"
|
||
require_file scripts/backup/sync-offsite-backups.sh "Offsite backup copy controller"
|
||
require_file scripts/backup/backup-offsite-readiness-gate.sh "Offsite backup readiness gate"
|
||
require_file scripts/backup/offsite-escrow-evidence-report.sh "Offsite and escrow evidence report"
|
||
require_file scripts/backup/verify-offsite-full-sync.sh "Offsite full sync remote verifier"
|
||
require_file scripts/backup/mark-credential-escrow-verified.sh "Credential escrow verification marker helper"
|
||
require_pattern "missing-commands" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow missing command template"
|
||
require_pattern "dry-run" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow marker dry-run validation"
|
||
require_pattern "placeholder" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow placeholder rejection"
|
||
require_pattern "BACKUP_COMMON_QUIET" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow command template stays quiet"
|
||
require_pattern "TEXTFILE_REFRESHED" scripts/backup/mark-credential-escrow-verified.sh "Credential escrow marker refreshes backup health textfile"
|
||
require_pattern "credential escrow missing command template" scripts/backup/offsite-escrow-evidence-report.sh "Credential escrow report command template"
|
||
require_pattern "dr-offsite-operator-checklist.sh --require-dr" scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite waiter final checklist gate"
|
||
require_pattern "backup-alert-live-visibility-check.py" scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite waiter alert visibility gate"
|
||
require_pattern "recovery-scorecard-contract-check.py" scripts/reboot-recovery/wait-dr-offsite-ready.sh "DR offsite waiter Prometheus recording-rule gate"
|
||
require_file scripts/backup/check-backup-integrity.sh "Backup integrity and restore drill script"
|
||
require_file scripts/backup/enforce-latest-only-retention.sh "Latest-only backup retention enforcer"
|
||
require_file scripts/ops/backup-from-110.sh "188 backup-from-110 script"
|
||
require_file scripts/cron_backup_restore_test.sh "Velero restore dry-run script"
|
||
forbid_pattern "^[[:space:]]*rclone[[:space:]]+sync" scripts/backup/backup-gitea.sh "Gitea backup direct rclone sync"
|
||
forbid_pattern "^[[:space:]]*rclone[[:space:]]+sync" scripts/backup/backup-harbor.sh "Harbor backup direct rclone sync"
|
||
forbid_pattern "^[[:space:]]*rclone[[:space:]]+sync" scripts/backup/backup-awoooi.sh "AWOOOI backup direct rclone sync"
|
||
|
||
echo
|
||
echo "== Ansible =="
|
||
require_dir infra/ansible/roles/cold-start-monitor "cold-start-monitor role"
|
||
require_dir infra/ansible/roles/runner-guardrails "runner-guardrails role"
|
||
require_dir infra/ansible/roles/host-textfile-exporters "host-textfile-exporters role"
|
||
require_file infra/ansible/roles/nginx/templates/188-internal-tools-https.conf.j2 "188 internal tools HTTPS template"
|
||
require_pattern "cold_start_monitor" infra/ansible/playbooks/110-devops.yml "110 cold-start monitor tag"
|
||
require_pattern "runner_guardrails" infra/ansible/playbooks/110-devops.yml "110 runner guardrails tag"
|
||
require_pattern "textfile_exporters" infra/ansible/playbooks/110-devops.yml "110 textfile exporters tag"
|
||
require_pattern "backup_jobs" infra/ansible/playbooks/110-devops.yml "110 backup jobs tag"
|
||
require_pattern "common.sh" infra/ansible/playbooks/110-devops.yml "110 backup shared library deploy"
|
||
require_pattern "backup-status.sh" infra/ansible/playbooks/110-devops.yml "110 daily backup Telegram heartbeat deploy"
|
||
require_pattern "AWOOOI daily backup Telegram heartbeat" infra/ansible/playbooks/110-devops.yml "110 daily backup Telegram heartbeat cron"
|
||
require_pattern "backup-gitea.sh" infra/ansible/playbooks/110-devops.yml "110 Gitea backup deploy"
|
||
require_pattern "backup-harbor.sh" infra/ansible/playbooks/110-devops.yml "110 Harbor backup deploy"
|
||
require_pattern "backup-momo.sh" infra/ansible/playbooks/110-devops.yml "110 momo backup deploy"
|
||
require_pattern "backup-awoooi.sh" infra/ansible/playbooks/110-devops.yml "110 AWOOOI backup deploy"
|
||
require_pattern "backup-configs.sh" infra/ansible/playbooks/110-devops.yml "110 config backup deploy"
|
||
require_pattern "offsite-escrow-evidence-report.sh" infra/ansible/playbooks/110-devops.yml "110 offsite evidence report deploy"
|
||
require_pattern "offsite-escrow-evidence-report.sh --no-color" infra/ansible/playbooks/110-devops.yml "110 offsite evidence report cron"
|
||
require_pattern "verify-offsite-full-sync.sh" infra/ansible/playbooks/110-devops.yml "110 offsite full sync verifier deploy"
|
||
require_pattern "verify-offsite-full-sync.sh --write-textfile" infra/ansible/playbooks/110-devops.yml "110 offsite full sync verifier cron"
|
||
require_pattern "offsite_escrow_evidence_report" scripts/ops/backup-health-textfile-exporter.py "110 offsite evidence report cron metric"
|
||
require_pattern "offsite_full_sync_verify" scripts/ops/backup-health-textfile-exporter.py "110 offsite full sync verifier cron metric"
|
||
require_pattern "awoooi_backup_dr_next_step_info" scripts/ops/backup-health-textfile-exporter.py "110 DR next-step textfile metric"
|
||
require_pattern "awoooi_backup_offsite_partial_fresh" scripts/ops/backup-health-textfile-exporter.py "110 partial offsite sync textfile metric"
|
||
require_pattern "awoooi_backup_offsite_full_sync_enabled" scripts/ops/backup-health-textfile-exporter.py "110 full offsite sync enable marker metric"
|
||
require_pattern "awoooi_backup_retention_latest_only" scripts/ops/backup-health-textfile-exporter.py "110 latest-only retention textfile metric"
|
||
require_pattern "awoooi_backup_cron_active_duplicate_count" scripts/ops/backup-health-textfile-exporter.py "110 backup cron duplicate textfile metric"
|
||
require_pattern "awoooi_backup_cron_singular_entry_ok" scripts/ops/backup-health-textfile-exporter.py "110 backup cron singular textfile metric"
|
||
require_pattern "awoooi_host_runaway_process_monitor_up" scripts/ops/host-runaway-process-exporter.py "110 runaway process monitor metric"
|
||
require_pattern "awoooi_host_runaway_process_remediation_authorized" scripts/ops/host-runaway-process-exporter.py "110 runaway process remediation authorization guard metric"
|
||
require_pattern "owner-approval-id" scripts/ops/host-runaway-process-remediation.py "Runaway process remediation owner approval gate"
|
||
require_pattern "maintenance-window-id" scripts/ops/host-runaway-process-remediation.py "Runaway process remediation maintenance window gate"
|
||
require_pattern "evidence-ref" scripts/ops/host-runaway-process-remediation.py "Runaway process remediation evidence gate"
|
||
require_pattern "textfile_exporters" infra/ansible/playbooks/188-ai-web.yml "188 textfile exporters tag"
|
||
require_pattern "backup-momo-188-pg.sh" infra/ansible/playbooks/188-ai-web.yml "188 momo PostgreSQL backup deploy"
|
||
require_pattern "/home/ollama/bin/momo-pg-backup.sh" infra/ansible/playbooks/188-ai-web.yml "188 host-owned momo backup entrypoint"
|
||
forbid_pattern "/home/ollama/momo-pro/scripts/pg_backup.sh" infra/ansible/playbooks/188-ai-web.yml "188 app-directory momo backup cron"
|
||
require_pattern "/home/ollama/bin/momo-pg-backup.sh" scripts/ops/backup-health-textfile-exporter.py "188 backup health executable entrypoint"
|
||
require_pattern "AWOOOI momo PostgreSQL daily backup" infra/ansible/playbooks/188-ai-web.yml "188 momo PostgreSQL backup cron"
|
||
require_pattern "188-internal-tools-https.conf.j2" infra/ansible/playbooks/nginx-sync.yml "188 HTTPS route sync"
|
||
|
||
echo
|
||
echo "== Monitoring and CI =="
|
||
require_pattern "awoooi_reboot_auto_recovery_slo_active_blocker" scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh "Reboot SLO active blocker textfile metric"
|
||
require_pattern "awoooi_reboot_auto_recovery_slo_primary_blocker" scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh "Reboot SLO primary blocker textfile metric"
|
||
require_pattern "RebootAutoRecoveryActiveBlocker" ops/monitoring/alerts-unified.yml "Reboot SLO active blocker alert"
|
||
require_pattern "RebootAutoRecoveryActiveBlockerMetricMissing" ops/monitoring/alerts-unified.yml "Reboot SLO active blocker metric missing alert"
|
||
require_pattern "cold_start_recovery_alerts" ops/monitoring/alerts-unified.yml "Cold-start alert group"
|
||
require_pattern "PrometheusRuleDriftGuardFailed" ops/monitoring/alerts-unified.yml "Prometheus rule drift guard failure alert"
|
||
require_pattern "PrometheusRuleDriftAutoRepaired" ops/monitoring/alerts-unified.yml "Prometheus rule drift repaired alert"
|
||
require_pattern "awoooi_prometheus_rule_drift_guard_missing_required_count" ops/monitoring/alerts-unified.yml "Prometheus rule drift guard missing-required metric alert"
|
||
require_pattern "ColdStartRecoveryBlocked" ops/monitoring/alerts-unified.yml "ColdStartRecoveryBlocked alert"
|
||
require_pattern "K3sNodeFilesystemErrorGateBlocked" ops/monitoring/alerts-unified.yml "K3s node filesystem blocker alert"
|
||
require_pattern "ColdStartHost120Unreachable" ops/monitoring/alerts-unified.yml "120 host unreachable cold-start alert"
|
||
require_pattern "awoooi_cold_start_blocker_reason" ops/monitoring/alerts-unified.yml "Cold-start blocker reason alert metric"
|
||
require_pattern "docker_container_cpu_cores" ops/monitoring/alerts-unified.yml "Docker CPU alert metric"
|
||
require_pattern "systemd_unit_watchdog_seconds" ops/monitoring/alerts-unified.yml "Systemd watchdog alert metric"
|
||
require_pattern "awoooi_host_storage_error_count" ops/monitoring/alerts-unified.yml "Storage health alert metric"
|
||
require_pattern "host_runaway_process_alerts" ops/monitoring/alerts-unified.yml "Host runaway process alert group"
|
||
require_pattern "HostOrphanBrowserSmokeHighCpu" ops/monitoring/alerts-unified.yml "Host orphan browser smoke alert"
|
||
require_pattern "HostCiRunnerLoadSaturation" ops/monitoring/alerts-unified.yml "Host CI runner load classification alert"
|
||
require_pattern "awoooi_host_runaway_process_remediation_authorized" ops/monitoring/alerts-unified.yml "Host runaway process remediation guard alert metric"
|
||
require_pattern "awoooi_backup_job_fresh" ops/monitoring/alerts-unified.yml "Backup freshness alert metric"
|
||
require_pattern "awoooi_backup_integrity_fresh" ops/monitoring/alerts-unified.yml "Backup integrity alert metric"
|
||
require_pattern "awoooi_backup_offsite_configured" ops/monitoring/alerts-unified.yml "Backup offsite alert metric"
|
||
require_pattern "awoooi_backup_credential_escrow_fresh" ops/monitoring/alerts-unified.yml "Backup credential escrow alert metric"
|
||
require_pattern "BackupRetentionPolicyNotLatestOnly" ops/monitoring/alerts-unified.yml "Backup latest-only retention alert"
|
||
require_pattern "BackupSnapshotRetentionExceeded" ops/monitoring/alerts-unified.yml "Backup snapshot count retention alert"
|
||
require_pattern "BackupScheduleDuplicateActiveEntries" ops/monitoring/alerts-unified.yml "Backup duplicate cron alert"
|
||
require_pattern "BackupScheduleSingletonMismatch" ops/monitoring/alerts-unified.yml "Backup singleton cron alert"
|
||
require_pattern "BackupOffsiteFullVerifyFailed" ops/monitoring/alerts-unified.yml "Backup offsite full verify alert"
|
||
require_pattern "BackupOffsiteRemoteSnapshotRetentionExceeded" ops/monitoring/alerts-unified.yml "Backup offsite remote snapshot retention alert"
|
||
require_pattern "BackupRestoreTestStale" ops/monitoring/alerts-unified.yml "Backup restore stale alert"
|
||
require_pattern "BackupOffsiteCopyNotConfigured" ops/monitoring/alerts-unified.yml "Backup offsite not configured alert"
|
||
require_pattern "BackupCredentialEscrowEvidenceMissing" ops/monitoring/alerts-unified.yml "Backup credential escrow evidence alert"
|
||
require_pattern "awoooi_recovery_core_ready" ops/monitoring/alerts-unified.yml "Recovery core ready recording rule"
|
||
require_pattern "awoooi_recovery_dr_offsite_ready" ops/monitoring/alerts-unified.yml "Recovery DR offsite ready recording rule"
|
||
require_pattern '\$labels\.exported_job' ops/monitoring/alerts-unified.yml "Backup alert exported_job annotation label"
|
||
require_pattern "ansible-validate.sh" .gitea/workflows/ansible-lint.yml "Gitea Ansible validation workflow"
|
||
require_pattern "bootstrap-ansible-validation-env.sh" .gitea/workflows/ansible-lint.yml "Gitea Ansible validation bootstrap workflow"
|
||
require_pattern "doc-secrets-sanity-check.py" .gitea/workflows/ansible-lint.yml "Gitea documentation secrets validation workflow"
|
||
require_pattern "backup-alert-label-contract-check.py" .gitea/workflows/ansible-lint.yml "Gitea backup alert label contract trigger"
|
||
require_pattern "backup-alert-live-visibility-check.py" .gitea/workflows/ansible-lint.yml "Gitea backup alert live visibility trigger"
|
||
require_pattern "recovery-scorecard-contract-check.py" .gitea/workflows/ansible-lint.yml "Gitea recovery scorecard contract trigger"
|
||
require_pattern "full-stack-recovery-scorecard.sh" .gitea/workflows/ansible-lint.yml "Gitea recovery scorecard trigger"
|
||
require_pattern "dr-offsite-operator-checklist.sh" .gitea/workflows/ansible-lint.yml "Gitea DR offsite checklist trigger"
|
||
require_pattern "scripts/reboot-recovery/\\*\\*" .gitea/workflows/ansible-lint.yml "Gitea reboot recovery scripts path contract"
|
||
require_pattern "verify-cold-start-monitor-deploy.sh" .gitea/workflows/ansible-lint.yml "Gitea cold-start deploy parity trigger"
|
||
require_pattern "docs/\\*\\*" .gitea/workflows/ansible-lint.yml "Gitea all-docs validation path contract"
|
||
require_pattern "\\.gitea/workflows/\\*\\*" .gitea/workflows/ansible-lint.yml "Gitea workflow self-validation path contract"
|
||
|
||
echo
|
||
echo "== Local validation =="
|
||
ensure_ansible_validation_path
|
||
|
||
if bash scripts/ops/ansible-validate.sh >/tmp/awoooi-ansible-validate.log 2>&1; then
|
||
ok "scripts/ops/ansible-validate.sh passed"
|
||
else
|
||
blocked "scripts/ops/ansible-validate.sh failed; see /tmp/awoooi-ansible-validate.log"
|
||
fi
|
||
|
||
if python3 scripts/ops/doc-secrets-sanity-check.py >/tmp/awoooi-doc-secrets-sanity.log 2>&1; then
|
||
ok "documentation secrets sanity check passed"
|
||
else
|
||
blocked "documentation secrets sanity check failed; see /tmp/awoooi-doc-secrets-sanity.log"
|
||
fi
|
||
|
||
if python3 scripts/ops/backup-alert-label-contract-check.py >/tmp/awoooi-backup-alert-label-contract.log 2>&1; then
|
||
ok "backup alert label contract check passed"
|
||
else
|
||
blocked "backup alert label contract check failed; see /tmp/awoooi-backup-alert-label-contract.log"
|
||
fi
|
||
|
||
if python3 scripts/ops/recovery-scorecard-contract-check.py >/tmp/awoooi-recovery-scorecard-contract.log 2>&1; then
|
||
ok "recovery scorecard contract check passed"
|
||
else
|
||
blocked "recovery scorecard contract check failed; see /tmp/awoooi-recovery-scorecard-contract.log"
|
||
fi
|
||
|
||
if command -v ansible-playbook >/dev/null 2>&1; then
|
||
ok "ansible-playbook available locally"
|
||
else
|
||
warning "ansible-playbook unavailable locally; CI/ops host must run syntax-check"
|
||
fi
|
||
|
||
if [ "$RUN_LIVE" = "1" ]; then
|
||
echo
|
||
echo "== Live read-only cold-start gate =="
|
||
if ssh wooo@192.168.0.110 "test -x /home/wooo/scripts/prometheus-rule-drift-guard.sh && test -f /home/wooo/monitoring/alerts-unified.canonical.yml && crontab -l 2>/dev/null | grep -q 'AWOOOI Prometheus rule drift guard'" >/tmp/awoooi-prometheus-rule-drift-guard-live.log 2>&1; then
|
||
ok "live 110 Prometheus rule drift guard installed"
|
||
else
|
||
blocked "live 110 Prometheus rule drift guard missing; run bash scripts/ops/deploy-alerts.sh"
|
||
fi
|
||
|
||
set +e
|
||
SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color --watch --interval 1 --max-attempts 1 >/tmp/awoooi-cold-start-live.log 2>&1
|
||
cold_start_rc=$?
|
||
set -e
|
||
cold_start_summary=$(grep -E '^PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' /tmp/awoooi-cold-start-live.log | tail -1 || true)
|
||
cold_start_blocked=$(awk -F'BLOCKED=' '/^PASS=/ {print $2}' <<<"$cold_start_summary")
|
||
if [ "$cold_start_rc" -eq 0 ]; then
|
||
ok "live read-only cold-start gate passed"
|
||
if python3 - <<'PY' >/tmp/awoooi-cold-start-alert-live-state.log 2>&1
|
||
import json
|
||
import sys
|
||
import urllib.parse
|
||
import urllib.request
|
||
|
||
expr = 'ALERTS{alertname=~"ColdStart.*",alertstate="firing"}'
|
||
url = "http://192.168.0.110:9090/api/v1/query?" + urllib.parse.urlencode({"query": expr})
|
||
payload = json.load(urllib.request.urlopen(url, timeout=8))
|
||
if payload.get("status") != "success":
|
||
print(f"Prometheus query failed: {payload}", file=sys.stderr)
|
||
sys.exit(1)
|
||
rows = payload.get("data", {}).get("result") or []
|
||
if rows:
|
||
names = sorted({(row.get("metric") or {}).get("alertname", "unknown") for row in rows})
|
||
print("Cold-start alerts still firing after GREEN gate: " + ", ".join(names), file=sys.stderr)
|
||
sys.exit(1)
|
||
print("COLD_START_ALERT_LIVE_STATE_OK")
|
||
PY
|
||
then
|
||
ok "live Prometheus cold-start alerts cleared after green gate"
|
||
else
|
||
blocked "live Prometheus cold-start alerts still firing after green gate; see /tmp/awoooi-cold-start-alert-live-state.log"
|
||
fi
|
||
elif [ "${cold_start_blocked:-1}" = "0" ]; then
|
||
warning "live read-only cold-start gate degraded but not blocked: ${cold_start_summary:-summary unavailable}"
|
||
else
|
||
blocked "live read-only cold-start gate failed: ${cold_start_summary:-summary unavailable}; see /tmp/awoooi-cold-start-live.log"
|
||
fi
|
||
|
||
if python3 scripts/ops/backup-alert-label-contract-check.py --prometheus-url http://192.168.0.110:9090 >/tmp/awoooi-backup-alert-label-contract-live.log 2>&1; then
|
||
ok "live Prometheus backup alert label contract passed"
|
||
else
|
||
blocked "live Prometheus backup alert label contract failed; run bash scripts/ops/deploy-alerts.sh, wait 1-2 evaluation cycles, then recheck /tmp/awoooi-backup-alert-label-contract-live.log"
|
||
fi
|
||
|
||
if run_with_retries 3 20 \
|
||
python3 scripts/ops/recovery-scorecard-contract-check.py \
|
||
--prometheus-url http://192.168.0.110:9090 \
|
||
--expect-core-ready \
|
||
>/tmp/awoooi-recovery-scorecard-contract-live.log 2>&1; then
|
||
ok "live Prometheus recovery scorecard contract passed"
|
||
elif [ "${cold_start_blocked:-1}" != "0" ]; then
|
||
blocked "live Prometheus recovery scorecard contract correctly cannot be core-ready while cold-start is blocked: ${cold_start_summary:-summary unavailable}; resolve the first cold-start blocker before expecting awoooi_recovery_core_ready=1"
|
||
else
|
||
blocked "live Prometheus recovery scorecard contract failed; run bash scripts/ops/deploy-alerts.sh, wait 1-2 evaluation cycles, then recheck /tmp/awoooi-recovery-scorecard-contract-live.log"
|
||
fi
|
||
|
||
if python3 scripts/ops/backup-alert-live-visibility-check.py \
|
||
--prometheus-url http://192.168.0.110:9090 \
|
||
--alertmanager-url http://192.168.0.110:9093 \
|
||
>/tmp/awoooi-backup-alert-live-visibility.log 2>&1; then
|
||
ok "live backup gap alert visibility passed"
|
||
else
|
||
blocked "live backup gap alert visibility failed; if gap metrics exist but alerts are missing, run bash scripts/ops/deploy-alerts.sh, wait for the 1m alert window, then recheck /tmp/awoooi-backup-alert-live-visibility.log"
|
||
fi
|
||
|
||
if bash scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh >/tmp/awoooi-cold-start-monitor-deploy-parity.log 2>&1; then
|
||
ok "live 110 cold-start monitor deploy parity passed"
|
||
elif [ "${cold_start_blocked:-1}" != "0" ]; then
|
||
blocked "live 110 cold-start monitor deploy parity cannot be green while cold-start is blocked: ${cold_start_summary:-summary unavailable}; see /tmp/awoooi-cold-start-monitor-deploy-parity.log"
|
||
else
|
||
blocked "live 110 cold-start monitor deploy parity failed; see /tmp/awoooi-cold-start-monitor-deploy-parity.log"
|
||
fi
|
||
|
||
if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
|
||
set -eu
|
||
test -x /backup/scripts/backup-status.sh
|
||
crontab -l | grep -q "/backup/scripts/backup-status.sh"
|
||
test -f /backup/state/backup-status-last-notified
|
||
' >/tmp/awoooi-backup-heartbeat-live.log 2>&1; then
|
||
ok "live 110 daily backup Telegram heartbeat installed and has notification marker"
|
||
else
|
||
blocked "live 110 daily backup Telegram heartbeat check failed; see /tmp/awoooi-backup-heartbeat-live.log"
|
||
fi
|
||
|
||
if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
|
||
set -eu
|
||
crontab -l | awk "
|
||
NF && \$0 !~ /^[[:space:]]*#/ {
|
||
count[\$0]++
|
||
}
|
||
END {
|
||
for (line in count) {
|
||
if (count[line] > 1) {
|
||
print count[line] \"x \" line
|
||
bad = 1
|
||
}
|
||
}
|
||
exit bad
|
||
}
|
||
"
|
||
' >/tmp/awoooi-110-cron-duplicate-live.log 2>&1; then
|
||
ok "live 110 crontab has no exact duplicate active entries"
|
||
else
|
||
blocked "live 110 crontab has duplicate active entries; see /tmp/awoooi-110-cron-duplicate-live.log"
|
||
fi
|
||
|
||
if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
|
||
set -eu
|
||
cron="$(crontab -l)"
|
||
bad=0
|
||
check_count() {
|
||
pattern="$1"
|
||
expected="$2"
|
||
count="$(printf "%s\n" "$cron" | awk -v p="$pattern" "index(\$0, p) && \$0 !~ /^[[:space:]]*#/ {c++} END {print c + 0}")"
|
||
if [ "$count" != "$expected" ]; then
|
||
printf "%s expected=%s actual=%s\n" "$pattern" "$expected" "$count"
|
||
bad=1
|
||
fi
|
||
}
|
||
check_count "/home/wooo/scripts/backup-health-textfile-exporter.py" 1
|
||
check_count "/backup/scripts/sync-offsite-backups.sh --mode status" 1
|
||
check_count "/backup/scripts/offsite-escrow-evidence-report.sh --no-color" 1
|
||
check_count "/backup/scripts/sync-offsite-backups.sh --mode sync" 1
|
||
check_count "/backup/scripts/verify-offsite-full-sync.sh --write-textfile" 1
|
||
exit "$bad"
|
||
' >/tmp/awoooi-110-backup-cron-count-live.log 2>&1; then
|
||
ok "live 110 backup/offsite cron entries are singular"
|
||
else
|
||
blocked "live 110 backup/offsite cron entry count mismatch; see /tmp/awoooi-110-backup-cron-count-live.log"
|
||
fi
|
||
|
||
if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
|
||
set -eu
|
||
ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 ollama@192.168.0.188 "
|
||
test -x /home/ollama/bin/momo-pg-backup.sh
|
||
crontab -l | grep -q /home/ollama/bin/momo-pg-backup.sh
|
||
! crontab -l | grep -q /home/ollama/momo-pro/scripts/pg_backup.sh
|
||
grep -q \"momo-pg-backup.sh\" /home/ollama/node_exporter_textfiles/backup_health.prom
|
||
"
|
||
' >/tmp/awoooi-188-momo-host-backup-live.log 2>&1; then
|
||
ok "live 188 momo backup uses host-owned executable entrypoint"
|
||
else
|
||
blocked "live 188 momo host-owned backup entrypoint check failed; see /tmp/awoooi-188-momo-host-backup-live.log"
|
||
fi
|
||
|
||
if ssh -o BatchMode=yes -o ConnectTimeout=8 wooo@192.168.0.110 '
|
||
set -eu
|
||
ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 ollama@192.168.0.188 "
|
||
policy=\$(docker inspect -f \"{{.HostConfig.RestartPolicy.Name}}\" momo-db)
|
||
health=\$(docker inspect -f \"{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}\" momo-db)
|
||
state=\$(docker inspect -f \"{{.State.Status}}\" momo-db)
|
||
printf \"momo-db state=%s health=%s restart=%s\n\" \"\$state\" \"\$health\" \"\$policy\"
|
||
test \"\$state\" = running
|
||
test \"\$health\" = healthy
|
||
case \"\$policy\" in
|
||
always|unless-stopped) exit 0 ;;
|
||
*) exit 1 ;;
|
||
esac
|
||
"
|
||
' >/tmp/awoooi-188-momo-db-restart-policy-live.log 2>&1; then
|
||
ok "live 188 momo-db restart policy and health passed"
|
||
else
|
||
blocked "live 188 momo-db restart policy or health failed; run docker update --restart unless-stopped momo-db and see /tmp/awoooi-188-momo-db-restart-policy-live.log"
|
||
fi
|
||
else
|
||
warning "live cold-start gate skipped; pass --live to verify runtime state"
|
||
fi
|
||
|
||
echo
|
||
echo "== Summary =="
|
||
echo "PASS=$pass WARN=$warn BLOCKED=$fail"
|
||
|
||
if [ "$fail" -gt 0 ]; then
|
||
echo "Result: NOT READY. Fix BLOCKED items before relying on reboot automation."
|
||
exit 1
|
||
fi
|
||
|
||
if [ "$warn" -gt 0 ]; then
|
||
echo "Result: READY WITH WARNINGS. Core SOP exists, but hardening remains."
|
||
exit 0
|
||
fi
|
||
|
||
echo "Result: READY. Reboot recovery SOP, scripts, monitoring, and CI gates are present."
|