awoooi/scripts/ops/backup-health-textfile-exporter.py

#!/usr/bin/env python3
"""
Backup health textfile exporter for full-stack reboot readiness.

2026-05-06 ogt + Codex: backup coverage follow-up after the reboot incident.
Why: a green service gate is not enough if the last restorable copy is stale.
This exporter is read-only; it checks cron/script presence and the latest
successful backup evidence, then writes node-exporter textfile metrics.
"""

from __future__ import annotations

import json
import os
import re
import shlex
import subprocess
import tempfile
import time
from datetime import datetime, timezone
from pathlib import Path


TEXTFILE_DIR = Path(os.environ.get("NODE_EXPORTER_TEXTFILE_DIR", "/var/lib/node_exporter/textfile_collector"))
OUTPUT_NAME = "backup_health.prom"
HOST_LABEL = os.environ.get("AIOPS_HOST_LABEL", os.uname().nodename)
LABEL_RE = re.compile(r'["\\\n]')
GITEA_BUNDLE_ROOT = Path(
    os.environ.get("AIOPS_GITEA_BUNDLE_ROOT", "/home/ollama/backup/110/gitea/git-bundles/latest-private-complete")
)
GITEA_BUNDLE_MAX_AGE_HOURS = float(os.environ.get("AIOPS_GITEA_BUNDLE_MAX_AGE_HOURS", "25"))
BACKUP_COMMON_SH = Path(os.environ.get("AIOPS_BACKUP_COMMON_SH", "/backup/scripts/common.sh"))
BACKUP_OFFSITE_ENV = Path(os.environ.get("AIOPS_BACKUP_OFFSITE_ENV", "/backup/scripts/offsite.env"))
OFFSITE_STATUS_DIR = Path(os.environ.get("AIOPS_OFFSITE_STATUS_DIR", "/backup/offsite"))
ESCROW_EVIDENCE_DIR = Path(os.environ.get("AIOPS_ESCROW_EVIDENCE_DIR", "/backup/escrow-evidence"))
CONFIG_CAPTURE_STATUS_FILE = Path(os.environ.get("AIOPS_CONFIG_CAPTURE_STATUS_FILE", "/backup/status/backup-configs-last-status.json"))
GITEA_RESTORE_DRILL_STATUS_FILE = Path(
    os.environ.get("AIOPS_GITEA_RESTORE_DRILL_STATUS_FILE", "/backup/integrity/gitea-restore-drill.status")
)
GITEA_RESTORE_DRILL_MAX_AGE_HOURS = float(os.environ.get("AIOPS_GITEA_RESTORE_DRILL_MAX_AGE_HOURS", "744"))
BACKUP_ALERT_RECEIPT_DIR = Path(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_DIR", "/backup/alert-receipts"))
BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS = float(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", "168"))
DEFAULT_GITEA_BUNDLE_EXPECTED_REPOS = [
    "wooo/awoooi",
    "wooo/ewoooc",
    "wooo/2026FIFAWorldCup",
    "wooo/agent-bounty-protocol",
    "wooo/AwoooGo",
    "wooo/stockplatform-v2",
    "wooo/vibework",
    "wooo/momo-pro-system",
    "wooo/tsenyang-website",
    "wooo/clawbot-v5",
    "wooo/bitan-pharmacy",
    "wooo/vtuber",
]
ESCROW_ITEMS = [
    "restic_repository_password",
    "offsite_provider_credentials",
    "break_glass_admin_credentials",
    "dns_registrar_recovery",
    "oauth_ai_provider_recovery",
]
BACKUP_ALERT_RECEIPT_STAGES = ("sent", "received", "dedup", "escalation")
BACKUP_ALERT_RECEIPT_REQUIREMENTS = [
    {"host": "110", "alertname": "BackupHealthMonitorMissing110", "scope": "monitor"},
    {"host": "188", "alertname": "BackupHealthMonitorMissing188", "scope": "monitor"},
    {"host": "110", "alertname": "BackupHealthMonitorStale", "scope": "monitor"},
    {"host": "110", "alertname": "BackupExpectedJobMissing", "scope": "schedule"},
    # 只宣告 requirement；沒有真實收據 marker 時 fresh 必須維持 0。
    {
        "host": "110",
        "alertname": "BackupScheduleSingletonMismatch",
        "scope": "schedule",
        "notification_type": "TYPE-1",
    },
    {"host": "110", "alertname": "BackupScriptMissing", "scope": "script"},
    {"host": "110", "alertname": "BackupJobStale", "scope": "freshness"},
    {"host": "110", "alertname": "BackupCoverageDomainStale", "scope": "coverage"},
    {"host": "188", "alertname": "GiteaPrivateBundleBackupIncomplete", "scope": "gitea_bundle"},
    {"host": "188", "alertname": "GiteaPrivateBundleRestoreDryRunMissing", "scope": "gitea_bundle_restore"},
    {"host": "110", "alertname": "BackupAggregateRunFailed", "scope": "aggregate"},
    {"host": "110", "alertname": "BackupConfigCapturePartial", "scope": "config_capture"},
    {"host": "110", "alertname": "BackupConfigCaptureStatusStale", "scope": "config_capture"},
    {"host": "110", "alertname": "BackupIntegrityCheckMissingOrFailed", "scope": "integrity"},
    {"host": "110", "alertname": "BackupRestoreDrillMissingOrFailed", "scope": "restore_drill"},
    {"host": "110", "alertname": "GiteaRestoreDrillMissingOrFailed", "scope": "gitea_restore_drill"},
    {"host": "110", "alertname": "BackupRestoreTestMissing", "scope": "velero_restore_test"},
    {"host": "110", "alertname": "BackupRestoreTestCronMissing", "scope": "velero_restore_test"},
    {"host": "110", "alertname": "BackupRestoreTestFailed", "scope": "velero_restore_test"},
    {"host": "110", "alertname": "BackupRestoreTestStale", "scope": "velero_restore_test"},
    {"host": "110", "alertname": "BackupOffsiteCopyNotConfigured", "scope": "offsite"},
    {"host": "110", "alertname": "BackupOffsiteCopyStale", "scope": "offsite"},
    {"host": "110", "alertname": "BackupRetentionPolicyNotLatestOnly", "scope": "retention"},
    {"host": "110", "alertname": "BackupSnapshotRetentionExceeded", "scope": "retention"},
    {"host": "110", "alertname": "BackupOffsiteFullVerifyFailed", "scope": "offsite_verify"},
    {"host": "110", "alertname": "BackupOffsiteRemoteSnapshotRetentionExceeded", "scope": "offsite_retention"},
    {"host": "110", "alertname": "BackupCredentialEscrowEvidenceMissing", "scope": "credential_escrow"},
]


def _escape_label(value: str) -> str:
    return LABEL_RE.sub(lambda m: {"\n": r"\n", "\\": r"\\", '"': r"\""}[m.group(0)], value)


def _run(command: list[str], timeout: int = 30) -> tuple[int, str, str]:
    try:
        result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False)
    except FileNotFoundError as exc:
        return 127, "", str(exc)
    except subprocess.TimeoutExpired as exc:
        stdout = exc.stdout if isinstance(exc.stdout, str) else ""
        stderr = exc.stderr if isinstance(exc.stderr, str) else "timeout"
        return 124, stdout, stderr
    return result.returncode, result.stdout, result.stderr


def _parse_time(value: str) -> int:
    if not value:
        return 0
    normalized = re.sub(r"\.(\d{6})\d+([+-]\d\d:\d\d|Z)$", r".\1\2", value)
    normalized = normalized.replace("Z", "+00:00")
    try:
        return int(datetime.fromisoformat(normalized).astimezone(timezone.utc).timestamp())
    except ValueError:
        return 0


def _parse_marker_timestamp(text: str) -> int:
    match = re.search(r"\b(\d{10})\b", text)
    if match:
        return int(match.group(1))
    for line in text.splitlines():
        parsed = _parse_time(line.strip())
        if parsed:
            return parsed
    return 0


def _marker_timestamp(paths: list[Path]) -> int:
    for path in paths:
        try:
            text = path.read_text(encoding="utf-8", errors="replace")
            parsed = _parse_marker_timestamp(text)
            return parsed or int(path.stat().st_mtime)
        except OSError:
            continue
    return 0


def _backup_alert_receipt_marker_timestamp(alertname: str, stage: str) -> int:
    return _marker_timestamp(
        [
            BACKUP_ALERT_RECEIPT_DIR / f"{alertname}.{stage}.last_success",
            BACKUP_ALERT_RECEIPT_DIR / alertname / f"{stage}.last_success",
            BACKUP_ALERT_RECEIPT_DIR / f"{alertname}.last_success",
            BACKUP_ALERT_RECEIPT_DIR / alertname / "last_success",
        ]
    )


def _backup_alert_receipt_metric_lines(host: str) -> list[str]:
    now = int(time.time())
    lines: list[str] = []
    for requirement in BACKUP_ALERT_RECEIPT_REQUIREMENTS:
        if requirement["host"] != host:
            continue
        alertname = requirement["alertname"]
        scope = requirement["scope"]
        notification_type = requirement.get("notification_type", "TYPE-3")
        base_labels = (
            f'host="{_escape_label(host)}",'
            f'required_alert="{_escape_label(alertname)}",'
            f'scope="{_escape_label(scope)}",'
            f'required_notification_type="{_escape_label(notification_type)}",'
            'receipt_channel="telegram_awooop",'
            f'max_age_hours="{BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS:g}"'
        )
        lines.append(f"awoooi_backup_alert_receipt_expected_info{{{base_labels}}} 1")

        stage_fresh_values: list[int] = []
        for stage in BACKUP_ALERT_RECEIPT_STAGES:
            timestamp = _backup_alert_receipt_marker_timestamp(alertname, stage)
            age = now - timestamp if timestamp else 0
            fresh = 1 if timestamp and age <= int(BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS * 3600) else 0
            stage_fresh_values.append(fresh)
            stage_labels = f'{base_labels},stage="{_escape_label(stage)}"'
            lines.extend(
                [
                    f"awoooi_backup_alert_receipt_stage_expected_info{{{stage_labels}}} 1",
                    f"awoooi_backup_alert_receipt_stage_last_success_timestamp{{{stage_labels}}} {timestamp}",
                    f"awoooi_backup_alert_receipt_stage_age_seconds{{{stage_labels}}} {age}",
                    f"awoooi_backup_alert_receipt_stage_fresh{{{stage_labels}}} {fresh}",
                ]
            )

        lines.append(f"awoooi_backup_alert_receipt_fresh{{{base_labels}}} {1 if all(stage_fresh_values) else 0}")
    return lines


def _shell_export_value(path: Path, key: str) -> str:
    try:
        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
    except OSError:
        return ""
    for line in lines:
        try:
            tokens = shlex.split(line, comments=True, posix=True)
        except ValueError:
            continue
        if tokens and tokens[0] == "export":
            tokens = tokens[1:]
        for token in tokens:
            if not token.startswith(f"{key}="):
                continue
            return token.split("=", 1)[1].strip()
    return ""


def _backup_config_value(key: str) -> str:
    for path in [BACKUP_OFFSITE_ENV, BACKUP_COMMON_SH]:
        value = _shell_export_value(path, key)
        if value:
            default_match = re.fullmatch(r"\$\{" + re.escape(key) + r":-([^}]+)\}", value)
            if default_match:
                return default_match.group(1)
            return value
    return ""


def _configured_secret(value: str) -> bool:
    return value.strip() not in {"", "CHANGE_ME", "CHANGEME", "TODO", "REDACTED"}


def _b2_configured() -> bool:
    return (
        _configured_secret(_backup_config_value("B2_ACCOUNT_ID"))
        and _configured_secret(_backup_config_value("B2_APPLICATION_KEY"))
        and _configured_secret(_backup_config_value("B2_BUCKET"))
    )


def _rclone_configured() -> bool:
    remote = _backup_config_value("OFFSITE_RCLONE_REMOTE") or os.environ.get("OFFSITE_RCLONE_REMOTE", "gdrive")
    rc, stdout, _ = _run(["rclone", "listremotes"], timeout=10)
    if rc == 0 and remote:
        return f"{remote}:" in {line.strip() for line in stdout.splitlines()}
    for path in [
        Path.home() / ".config/rclone/rclone.conf",
        Path("/home/wooo/.config/rclone/rclone.conf"),
        Path("/root/.config/rclone/rclone.conf"),
        Path("/etc/rclone.conf"),
    ]:
        try:
            if path.is_file() and path.stat().st_size > 0:
                return True
        except OSError:
            continue
    return False


def _cron_text() -> str:
    rc, stdout, _ = _run(["crontab", "-l"], timeout=10)
    return stdout if rc == 0 else ""


def _active_cron_lines(cron: str) -> list[str]:
    return [line.strip() for line in cron.splitlines() if line.strip() and not line.lstrip().startswith("#")]


def _cron_duplicate_metric_lines(host: str, cron: str) -> list[str]:
    lines: list[str] = []
    active_lines = _active_cron_lines(cron)
    duplicate_count = max(0, len(active_lines) - len(set(active_lines)))
    lines.append(f'awoooi_backup_cron_active_duplicate_count{{host="{_escape_label(host)}"}} {duplicate_count}')

    singular_patterns = {
        "backup_health_exporter": "/home/wooo/scripts/backup-health-textfile-exporter.py",
        "offsite_status": "/backup/scripts/sync-offsite-backups.sh --mode status",
        "offsite_escrow_evidence_report": "/backup/scripts/offsite-escrow-evidence-report.sh --no-color",
        "offsite_sync_gated": "/backup/scripts/sync-offsite-backups.sh --mode sync",
        "offsite_full_sync_verify": "/backup/scripts/verify-offsite-full-sync.sh --write-textfile",
    }
    for entry, pattern in singular_patterns.items():
        count = sum(1 for line in active_lines if pattern in line)
        labels = f'host="{_escape_label(host)}",entry="{_escape_label(entry)}"'
        lines.append(f"awoooi_backup_cron_singular_entry_count{{{labels}}} {count}")
        lines.append(f"awoooi_backup_cron_singular_entry_ok{{{labels}}} {1 if count == 1 else 0}")
    return lines


def _newest_file_timestamp(patterns: list[str]) -> int:
    newest = 0
    for pattern in patterns:
        for path in Path("/").glob(pattern.lstrip("/")):
            try:
                if path.is_file():
                    newest = max(newest, int(path.stat().st_mtime))
            except OSError:
                continue
    return newest


def _newest_tree_timestamp(root: Path, max_entries: int = 5000) -> tuple[int, int]:
    if not root.exists():
        return 0, 0
    newest = 0
    count = 0
    for path in root.rglob("*"):
        try:
            if not path.is_file():
                continue
            count += 1
            newest = max(newest, int(path.stat().st_mtime))
        except OSError:
            continue
        if count >= max_entries:
            break
    return newest, count


def _expected_gitea_bundle_repos() -> list[str]:
    configured = os.environ.get("AIOPS_GITEA_BUNDLE_EXPECTED_REPOS", "")
    repos = [item.strip() for item in configured.split(",") if item.strip()]
    return repos or DEFAULT_GITEA_BUNDLE_EXPECTED_REPOS


def _gitea_bundle_slug(repo: str) -> str:
    return repo.replace("/", "__")


def _read_gitea_bundle_manifest(root: Path) -> tuple[Path | None, dict[str, dict[str, str]]]:
    for manifest_path in [root / "manifest.remote.tsv", root / "manifest.tsv"]:
        try:
            rows = manifest_path.read_text(encoding="utf-8", errors="replace").splitlines()
        except OSError:
            continue
        if not rows:
            return manifest_path, {}
        header = rows[0].split("\t")
        parsed: dict[str, dict[str, str]] = {}
        for line in rows[1:]:
            if not line.strip():
                continue
            values = line.split("\t")
            row = {key: values[index] if index < len(values) else "" for index, key in enumerate(header)}
            repo = row.get("repo", "").strip()
            if repo:
                parsed[repo] = row
        return manifest_path, parsed
    return None, {}


def _resolve_bundle_path(root: Path, repo: str, manifest_value: str) -> Path:
    if manifest_value:
        candidate = Path(manifest_value)
        if candidate.exists():
            return candidate
        local_candidate = root / candidate.name
        if local_candidate.exists():
            return local_candidate
    slug_candidate = root / f"{_gitea_bundle_slug(repo)}.bundle"
    if slug_candidate.exists():
        return slug_candidate
    repo_name = repo.rsplit("/", 1)[-1]
    matches = sorted(root.glob(f"{repo_name}-local-*.bundle"), key=lambda path: path.stat().st_mtime, reverse=True)
    return matches[0] if matches else slug_candidate


def _resolve_checksum_path(root: Path, bundle_path: Path, manifest_value: str) -> Path:
    if manifest_value:
        candidate = Path(manifest_value)
        if candidate.exists():
            return candidate
        local_candidate = root / candidate.name
        if local_candidate.exists():
            return local_candidate
    return bundle_path.with_name(f"{bundle_path.name}.sha256")


def _checksum_digest_present(path: Path) -> int:
    try:
        first = path.read_text(encoding="utf-8", errors="replace").split()[0]
    except (IndexError, OSError):
        return 0
    return 1 if re.fullmatch(r"[0-9a-fA-F]{64}", first) else 0


def _gitea_bundle_metric_lines(host: str) -> tuple[list[str], int]:
    now = int(time.time())
    root = GITEA_BUNDLE_ROOT
    manifest_path, manifest_rows = _read_gitea_bundle_manifest(root)
    root_exists = int(root.exists())
    manifest_present = int(manifest_path is not None)
    newest_ts, file_count = _newest_tree_timestamp(root, max_entries=20000)
    age = now - newest_ts if newest_ts else 0
    bundle_fresh = 1 if newest_ts and age <= int(GITEA_BUNDLE_MAX_AGE_HOURS * 3600) else 0
    root_label = _escape_label(str(root))
    labels = f'host="{_escape_label(host)}",root="{root_label}",max_age_hours="{GITEA_BUNDLE_MAX_AGE_HOURS:g}"'
    lines = [
        f"awoooi_gitea_bundle_root_exists{{{labels}}} {root_exists}",
        f"awoooi_gitea_bundle_manifest_present{{{labels}}} {manifest_present}",
        f"awoooi_gitea_bundle_newest_timestamp{{{labels}}} {newest_ts}",
        f"awoooi_gitea_bundle_age_seconds{{{labels}}} {age}",
        f"awoooi_gitea_bundle_fresh{{{labels}}} {bundle_fresh}",
        f"awoooi_gitea_bundle_file_count{{{labels}}} {file_count}",
    ]

    missing_count = 0
    failed_count = 0
    checksum_missing_count = 0
    expected_repos = _expected_gitea_bundle_repos()
    for repo in expected_repos:
        row = manifest_rows.get(repo, {})
        status = row.get("status", "manifest_row_missing")
        bundle_path = _resolve_bundle_path(root, repo, row.get("bundle", ""))
        checksum_path = _resolve_checksum_path(root, bundle_path, row.get("checksum", ""))
        bundle_present = int(bundle_path.is_file())
        checksum_present = int(checksum_path.is_file())
        checksum_digest_present = _checksum_digest_present(checksum_path) if checksum_present else 0
        if status == "manifest_row_missing" and bundle_present and checksum_digest_present:
            status = "bundle_file_only"
        try:
            head_count = int(row.get("head_count", "0") or 0)
        except ValueError:
            head_count = 0
        repo_ok = int(
            bundle_present == 1
            and checksum_present == 1
            and checksum_digest_present == 1
            and status in {"ok", "bundle_file_only"}
        )
        if not bundle_present:
            missing_count += 1
        if not checksum_present or not checksum_digest_present:
            checksum_missing_count += 1
        if repo_ok == 0:
            failed_count += 1
        repo_labels = f'host="{_escape_label(host)}",repo="{_escape_label(repo)}"'
        status_labels = (
            f'{repo_labels},status="{_escape_label(status)}",'
            f'bundle="{_escape_label(bundle_path.name)}"'
        )
        lines.extend(
            [
                f"awoooi_gitea_bundle_expected_repo_info{{{repo_labels}}} 1",
                f"awoooi_gitea_bundle_repo_status_info{{{status_labels}}} 1",
                f"awoooi_gitea_bundle_repo_present{{{repo_labels}}} {bundle_present}",
                f"awoooi_gitea_bundle_repo_ok{{{repo_labels}}} {repo_ok}",
                f"awoooi_gitea_bundle_repo_head_count{{{repo_labels}}} {head_count}",
                f"awoooi_gitea_bundle_checksum_present{{{repo_labels}}} {checksum_present}",
                f"awoooi_gitea_bundle_checksum_digest_present{{{repo_labels}}} {checksum_digest_present}",
            ]
        )

    all_expected_ok = int(
        root_exists == 1
        and manifest_present == 1
        and bundle_fresh == 1
        and expected_repos
        and missing_count == 0
        and failed_count == 0
        and checksum_missing_count == 0
    )
    lines.extend(
        [
            f"awoooi_gitea_bundle_expected_repo_count{{{labels}}} {len(expected_repos)}",
            f"awoooi_gitea_bundle_expected_repo_missing_count{{{labels}}} {missing_count}",
            f"awoooi_gitea_bundle_failed_repo_count{{{labels}}} {failed_count}",
            f"awoooi_gitea_bundle_checksum_missing_count{{{labels}}} {checksum_missing_count}",
            f"awoooi_gitea_bundle_all_expected_ok{{{labels}}} {all_expected_ok}",
        ]
    )
    return lines, all_expected_ok


def _read_backup_110_timestamp() -> int:
    candidates = [
        Path("/home/ollama/node_exporter_textfiles/backup.prom"),
        Path("/home/ollama/backup/110/last_success"),
    ]
    for path in candidates:
        try:
            text = path.read_text(encoding="utf-8", errors="replace")
        except OSError:
            continue
        match = re.search(r"(?:backup_110_last_success_timestamp\s+)?(\d{10})", text)
        if match:
            return int(match.group(1))
    return 0


def _latest_restic_snapshot(repo: str) -> tuple[int, int]:
    password_file = os.environ.get("RESTIC_PASSWORD_FILE", "/backup/scripts/.restic-password")
    if not Path(repo).exists() or not Path(password_file).exists():
        return 0, 0
    rc, stdout, _ = _run(
        ["restic", "-r", repo, "snapshots", "--json", "--password-file", password_file],
        timeout=45,
    )
    if rc != 0:
        return 0, 0
    try:
        rows = json.loads(stdout)
    except json.JSONDecodeError:
        return 0, 0
    timestamps = [_parse_time(str(row.get("time", ""))) for row in rows]
    timestamps = [value for value in timestamps if value > 0]
    return (max(timestamps), len(timestamps)) if timestamps else (0, 0)


def _backup_all_failed_count_from_log(path: Path) -> tuple[int, int]:
    try:
        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
    except OSError:
        return 0, -1
    for line in reversed(lines):
        if "全服務備份完成" not in line:
            continue
        ts_match = re.match(r"\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\]", line)
        timestamp = 0
        if ts_match:
            timestamp = int(datetime.strptime(ts_match.group(1), "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc).timestamp()) - 8 * 3600
        failed_match = re.search(r"-\s+(\d+)\s+個失敗", line)
        if failed_match:
            return timestamp, int(failed_match.group(1))
        if "全部成功" in line:
            return timestamp, 0
    return 0, -1


def _latest_backup_all_failed_count() -> tuple[int, int]:
    candidates = [
        _backup_all_failed_count_from_log(Path("/backup/logs/cron.log")),
        _backup_all_failed_count_from_log(Path("/backup/logs/backup.log")),
    ]
    candidates = [row for row in candidates if row[0] > 0 and row[1] >= 0]
    if not candidates:
        return 0, -1
    return max(candidates, key=lambda row: row[0])


def _read_key_value_status(path: str) -> dict[str, int | str]:
    values: dict[str, int | str] = {}
    try:
        lines = Path(path).read_text(encoding="utf-8", errors="replace").splitlines()
    except OSError:
        return values
    for line in lines:
        if not line or line.startswith("#") or "=" not in line:
            continue
        key, value = line.split("=", 1)
        key = key.strip()
        value = value.strip()
        try:
            values[key] = int(float(value))
        except ValueError:
            values[key] = value
    return values


def _integrity_metric_lines(host: str) -> list[str]:
    now = int(time.time())
    specs = [
        ("restic_check", "/backup/integrity/check.status", 192),
        ("restore_drill", "/backup/integrity/restore-drill.status", 744),
    ]
    lines: list[str] = []
    for scope, path, max_age_hours in specs:
        values = _read_key_value_status(path)
        timestamp = int(values.get("timestamp", 0)) if "timestamp" in values else 0
        failed_count = int(values.get("failed_count", -1)) if "failed_count" in values else -1
        checked_count = int(values.get("checked_repo_count", 0)) if "checked_repo_count" in values else 0
        age = now - timestamp if timestamp else 0
        fresh = 1 if timestamp and age <= max_age_hours * 3600 and failed_count == 0 else 0
        labels = f'host="{_escape_label(host)}",scope="{scope}",max_age_hours="{max_age_hours}"'
        lines.extend(
            [
                f"awoooi_backup_integrity_last_success_timestamp{{{labels}}} {timestamp if failed_count == 0 else 0}",
                f"awoooi_backup_integrity_age_seconds{{{labels}}} {age}",
                f"awoooi_backup_integrity_fresh{{{labels}}} {fresh}",
                f"awoooi_backup_integrity_failed_repo_count{{{labels}}} {failed_count}",
                f"awoooi_backup_integrity_checked_repo_count{{{labels}}} {checked_count}",
            ]
        )
    return lines


def _gitea_restore_drill_metric_lines(host: str) -> list[str]:
    values = _read_key_value_status(str(GITEA_RESTORE_DRILL_STATUS_FILE))
    now = int(time.time())
    timestamp = int(values.get("timestamp", 0))
    success = int(values.get("success", 0))
    dump_size = int(values.get("dump_size_bytes", 0))
    backup_regular = int(values.get("backup_regular_repo_count", 0))
    backup_wiki = int(values.get("backup_wiki_repo_count", 0))
    live_regular = int(values.get("live_regular_repo_count", 0))
    live_wiki = int(values.get("live_wiki_repo_count", 0))
    database_present = int(values.get("database_dump_present", 0))
    config_present = int(values.get("config_present", 0))
    crc_ok = int(values.get("archive_crc_ok", 0))
    inventory_match = int(values.get("inventory_digest_match", 0))
    age = max(0, now - timestamp) if timestamp else 0
    count_match = int(backup_regular == live_regular and backup_wiki == live_wiki and backup_regular > 0)
    required_artifacts_ok = int(database_present > 0 and config_present > 0)
    verified = int(success == 1 and crc_ok == 1 and required_artifacts_ok == 1 and count_match == 1 and inventory_match == 1)
    fresh = int(timestamp > 0 and age <= int(GITEA_RESTORE_DRILL_MAX_AGE_HOURS * 3600) and verified == 1)
    reason = str(values.get("reason", "status_missing"))
    labels = f'host="{_escape_label(host)}",max_age_hours="{GITEA_RESTORE_DRILL_MAX_AGE_HOURS:g}"'

    return [
        f'awoooi_gitea_restore_drill_status_info{{{labels},reason="{_escape_label(reason)}"}} 1',
        f"awoooi_gitea_restore_drill_last_success_timestamp{{{labels}}} {timestamp if verified else 0}",
        f"awoooi_gitea_restore_drill_age_seconds{{{labels}}} {age}",
        f"awoooi_gitea_restore_drill_fresh{{{labels}}} {fresh}",
        f"awoooi_gitea_restore_drill_verified{{{labels}}} {verified}",
        f"awoooi_gitea_restore_drill_archive_crc_ok{{{labels}}} {crc_ok}",
        f"awoooi_gitea_restore_drill_required_artifacts_ok{{{labels}}} {required_artifacts_ok}",
        f"awoooi_gitea_restore_drill_repo_count_match{{{labels}}} {count_match}",
        f"awoooi_gitea_restore_drill_inventory_match{{{labels}}} {inventory_match}",
        f"awoooi_gitea_restore_drill_dump_size_bytes{{{labels}}} {dump_size}",
        f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="regular"}} {backup_regular}',
        f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="wiki"}} {backup_wiki}',
        f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="regular"}} {live_regular}',
        f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="wiki"}} {live_wiki}',
    ]


def _config_capture_metric_lines(host: str) -> list[str]:
    now = int(time.time())
    labels = f'host="{_escape_label(host)}"'
    try:
        document = json.loads(CONFIG_CAPTURE_STATUS_FILE.read_text(encoding="utf-8", errors="replace"))
    except (OSError, json.JSONDecodeError):
        return [
            f"awoooi_backup_config_capture_status_timestamp{{{labels}}} 0",
            f"awoooi_backup_config_capture_status_age_seconds{{{labels}}} 0",
            f"awoooi_backup_config_capture_critical_failed_count{{{labels}}} -1",
        ]

    timestamp = int(document.get("timestamp") or 0)
    critical_failed = int(document.get("critical_failed_count", -1))
    failed_count = int(document.get("failed_count", -1))
    snapshot_id = str(document.get("snapshot_id") or "unknown")
    duration = int(document.get("duration_seconds", 0) or 0)
    age = now - timestamp if timestamp else 0
    lines = [
        f"awoooi_backup_config_capture_status_timestamp{{{labels},snapshot_id=\"{_escape_label(snapshot_id)}\"}} {timestamp}",
        f"awoooi_backup_config_capture_status_age_seconds{{{labels}}} {age}",
        f"awoooi_backup_config_capture_critical_failed_count{{{labels}}} {critical_failed}",
        f"awoooi_backup_config_capture_failed_count{{{labels}}} {failed_count}",
        f"awoooi_backup_config_capture_duration_seconds{{{labels}}} {duration}",
    ]
    for item in document.get("items") or []:
        target = str(item.get("target") or "unknown")
        source = str(item.get("source") or "unknown")
        critical = "true" if item.get("critical") else "false"
        ok = 1 if item.get("ok") else 0
        item_labels = (
            f'host="{_escape_label(host)}",'
            f'target="{_escape_label(target)}",'
            f'source="{_escape_label(source)}",'
            f'critical="{critical}"'
        )
        lines.append(f"awoooi_backup_config_capture_ok{{{item_labels}}} {ok}")
    return lines


def _offsite_and_escrow_metric_lines(host: str) -> list[str]:
    now = int(time.time())
    lines: list[str] = []
    b2_configured = int(_b2_configured())
    rclone_configured = int(_rclone_configured())
    b2_full_timestamp = _marker_timestamp(
        [
            OFFSITE_STATUS_DIR / "b2-last-success",
            OFFSITE_STATUS_DIR / "b2.last_success",
            OFFSITE_STATUS_DIR / "last_success",
            Path("/backup/logs/offsite-b2.status"),
        ]
    )
    b2_partial_timestamp = _marker_timestamp(
        [
            OFFSITE_STATUS_DIR / "b2-partial-last-success",
            OFFSITE_STATUS_DIR / "b2.partial_last_success",
        ]
    )
    rclone_full_timestamp = _marker_timestamp(
        [
            OFFSITE_STATUS_DIR / "rclone-last-success",
            OFFSITE_STATUS_DIR / "rclone.last_success",
            OFFSITE_STATUS_DIR / "last_success",
            Path("/backup/logs/rclone-sync.status"),
        ]
    )
    rclone_partial_timestamp = _marker_timestamp(
        [
            OFFSITE_STATUS_DIR / "rclone-partial-last-success",
            OFFSITE_STATUS_DIR / "rclone.partial_last_success",
        ]
    )
    offsite_specs = [
        ("b2", b2_configured, b2_full_timestamp),
        ("rclone", rclone_configured, rclone_full_timestamp),
    ]
    for provider, configured, timestamp in offsite_specs:
        age = now - timestamp if timestamp else 0
        fresh = 1 if configured and timestamp and age <= 48 * 3600 else 0
        labels = f'host="{_escape_label(host)}",provider="{provider}",max_age_hours="48"'
        lines.extend(
            [
                f"awoooi_backup_offsite_configured{{{labels}}} {configured}",
                f"awoooi_backup_offsite_last_success_timestamp{{{labels}}} {timestamp}",
                f"awoooi_backup_offsite_age_seconds{{{labels}}} {age}",
                f"awoooi_backup_offsite_fresh{{{labels}}} {fresh}",
            ]
        )

    partial_fresh_by_provider: dict[str, int] = {}
    for provider, configured, timestamp in [
        ("b2", b2_configured, b2_partial_timestamp),
        ("rclone", rclone_configured, rclone_partial_timestamp),
    ]:
        partial_age = now - timestamp if timestamp else 0
        partial_fresh = 1 if configured and timestamp and partial_age <= 48 * 3600 else 0
        partial_fresh_by_provider[provider] = partial_fresh
        partial_labels = f'host="{_escape_label(host)}",provider="{provider}",scope="partial",max_age_hours="48"'
        lines.extend(
            [
                f"awoooi_backup_offsite_partial_last_success_timestamp{{{partial_labels}}} {timestamp}",
                f"awoooi_backup_offsite_partial_age_seconds{{{partial_labels}}} {partial_age}",
                f"awoooi_backup_offsite_partial_fresh{{{partial_labels}}} {partial_fresh}",
            ]
        )

    full_sync_enable_marker = OFFSITE_STATUS_DIR / "enable-rclone-sync"
    try:
        full_sync_enabled = 1 if full_sync_enable_marker.is_file() else 0
        full_sync_enabled_timestamp = int(full_sync_enable_marker.stat().st_mtime) if full_sync_enabled else 0
    except OSError:
        full_sync_enabled = 0
        full_sync_enabled_timestamp = 0
    full_sync_labels = f'host="{_escape_label(host)}",provider="rclone"'
    lines.extend(
        [
            f"awoooi_backup_offsite_full_sync_enabled{{{full_sync_labels}}} {full_sync_enabled}",
            f"awoooi_backup_offsite_full_sync_enabled_timestamp{{{full_sync_labels}}} {full_sync_enabled_timestamp}",
        ]
    )

    escrow_missing_count = 0
    for item in ESCROW_ITEMS:
        timestamp = _marker_timestamp(
            [
                ESCROW_EVIDENCE_DIR / f"{item}.last_verified",
                ESCROW_EVIDENCE_DIR / f"{item}.verified",
                ESCROW_EVIDENCE_DIR / item,
            ]
        )
        age = now - timestamp if timestamp else 0
        fresh = 1 if timestamp and age <= 744 * 3600 else 0
        escrow_missing_count += 0 if fresh else 1
        labels = f'host="{_escape_label(host)}",item="{item}",max_age_hours="744"'
        lines.extend(
            [
                f"awoooi_backup_credential_escrow_expected_info{{{labels}}} 1",
                f"awoooi_backup_credential_escrow_last_verified_timestamp{{{labels}}} {timestamp}",
                f"awoooi_backup_credential_escrow_age_seconds{{{labels}}} {age}",
                f"awoooi_backup_credential_escrow_fresh{{{labels}}} {fresh}",
            ]
        )
    offsite_configured = 1 if b2_configured or rclone_configured else 0
    any_partial_fresh = 1 if any(partial_fresh_by_provider.values()) else 0
    full_fresh = 1 if (
        (b2_configured and b2_full_timestamp and now - b2_full_timestamp <= 48 * 3600)
        or (rclone_configured and rclone_full_timestamp and now - rclone_full_timestamp <= 48 * 3600)
    ) else 0
    if not offsite_configured:
        next_step = "configure_google_drive_rclone_on_110_tty"
        phase = 1
    elif escrow_missing_count == 0 and full_fresh:
        next_step = "offsite_and_escrow_ready"
        phase = 5
    elif escrow_missing_count > 0 and full_fresh:
        next_step = "complete_credential_escrow_review"
        phase = 3
    elif not any_partial_fresh:
        next_step = "run_small_dry_run_then_partial_sync"
        phase = 2
    elif escrow_missing_count > 0:
        next_step = "complete_credential_escrow_review"
        phase = 3
    elif not full_fresh:
        next_step = "pre_full_sync_review"
        phase = 4
    else:
        next_step = "offsite_and_escrow_ready"
        phase = 5

    lines.extend(
        [
            f'awoooi_backup_dr_credential_escrow_missing_count{{host="{_escape_label(host)}"}} {escrow_missing_count}',
            f'awoooi_backup_dr_phase{{host="{_escape_label(host)}",next_step="{_escape_label(next_step)}"}} {phase}',
            f'awoooi_backup_dr_next_step_info{{host="{_escape_label(host)}",next_step="{_escape_label(next_step)}"}} 1',
        ]
    )
    return lines


def _retention_metric_lines(host: str) -> list[str]:
    mode = (_backup_config_value("BACKUP_RETENTION_MODE") or os.environ.get("BACKUP_RETENTION_MODE", "")).strip()
    keep_last = (_backup_config_value("KEEP_LAST") or os.environ.get("KEEP_LAST", "")).strip()
    offsite_delete_old = (
        _backup_config_value("OFFSITE_SYNC_DELETE_OLD") or os.environ.get("OFFSITE_SYNC_DELETE_OLD", "")
    ).strip()

    latest_only = 1 if mode == "latest" and keep_last == "1" else 0
    offsite_mirror = 1 if offsite_delete_old == "1" else 0
    labels = f'host="{_escape_label(host)}",scope="restic",mode="{_escape_label(mode or "unknown")}",keep_last="{_escape_label(keep_last or "unknown")}"'
    offsite_labels = (
        f'host="{_escape_label(host)}",scope="offsite",provider="rclone",'
        f'delete_old="{_escape_label(offsite_delete_old or "unknown")}"'
    )
    return [
        f"awoooi_backup_retention_latest_only{{{labels}}} {latest_only}",
        f"awoooi_backup_retention_offsite_delete_old_enabled{{{offsite_labels}}} {offsite_mirror}",
    ]


def _collect_velero_from_k8s() -> dict[str, int | str]:
    remote_script = r"""
python3 - <<'PY'
import datetime as dt
import json
import subprocess
import time


def kubectl(args):
    for prefix in (["sudo", "-n", "kubectl"], ["kubectl"]):
        result = subprocess.run(prefix + args, capture_output=True, text=True, timeout=20, check=False)
        if result.returncode == 0:
            return result.stdout
    return ""


def load_json(args):
    text = kubectl(args + ["-o", "json"])
    try:
        return json.loads(text) if text else {}
    except json.JSONDecodeError:
        return {}


def parse_ts(value):
    if not value:
        return 0
    try:
        return int(dt.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp())
    except ValueError:
        return 0


now = int(time.time())
schedules = load_json(["get", "schedules.velero.io", "-n", "velero"]).get("items") or []
backups = load_json(["get", "backups.velero.io", "-n", "velero"]).get("items") or []
cron = load_json(["get", "cronjob", "-n", "velero", "backup-restore-test"])
jobs = load_json(["get", "jobs", "-n", "velero", "-l", "component=backup-restore-test"]).get("items") or []

completed = []
for item in backups:
    if item.get("status", {}).get("phase") != "Completed":
        continue
    timestamp = parse_ts(item.get("status", {}).get("completionTimestamp") or item.get("metadata", {}).get("creationTimestamp"))
    if timestamp:
        completed.append(timestamp)

failed_jobs = 0
for job in jobs:
    conditions = job.get("status", {}).get("conditions") or []
    if any(row.get("type") == "Failed" and row.get("status") == "True" for row in conditions):
        failed_jobs += 1

last_success = parse_ts((cron.get("status") or {}).get("lastSuccessfulTime"))
latest_backup = max(completed) if completed else 0

print("monitor_up=1")
print(f"schedule_count={len(schedules)}")
print(f"schedule_paused_count={sum(1 for item in schedules if item.get('spec', {}).get('paused'))}")
print(f"latest_completed_backup_timestamp={latest_backup}")
print(f"latest_completed_backup_age_seconds={now - latest_backup if latest_backup else 0}")
print(f"latest_completed_backup_fresh={1 if latest_backup and now - latest_backup <= 90000 else 0}")
print(f"restore_test_cron_present={1 if cron.get('metadata', {}).get('name') == 'backup-restore-test' else 0}")
print(f"restore_test_last_success_timestamp={last_success}")
print(f"restore_test_last_success_age_seconds={now - last_success if last_success else 0}")
print(f"restore_test_last_success_fresh={1 if last_success and now - last_success <= 691200 else 0}")
print(f"restore_test_failed_jobs={failed_jobs}")
PY
"""
    hosts = os.environ.get("AIOPS_K8S_QUERY_HOSTS", "192.168.0.120 192.168.0.121 192.168.0.125").split()
    values: dict[str, int | str] = {"monitor_up": 0, "source": "unreachable"}
    for host in hosts:
        rc, stdout, _ = _run(
            [
                "ssh",
                "-o",
                "BatchMode=yes",
                "-o",
                "StrictHostKeyChecking=accept-new",
                "-o",
                "ConnectTimeout=8",
                f"wooo@{host}",
                remote_script,
            ],
            timeout=45,
        )
        if rc != 0:
            continue
        parsed: dict[str, int | str] = {"source": f"{host}-kubectl"}
        for line in stdout.splitlines():
            if "=" not in line:
                continue
            key, value = line.split("=", 1)
            try:
                parsed[key.strip()] = int(float(value.strip()))
            except ValueError:
                continue
        if int(parsed.get("monitor_up", 0)) == 1:
            return parsed
    return values


def _velero_metric_lines(host: str) -> list[str]:
    values = _collect_velero_from_k8s()
    labels = f'host="{_escape_label(host)}",source="{_escape_label(str(values.get("source", "unreachable")))}",namespace="velero"'
    return [
        f"awoooi_velero_monitor_up{{{labels}}} {values.get('monitor_up', 0)}",
        f"awoooi_velero_schedule_count{{{labels}}} {values.get('schedule_count', 0)}",
        f"awoooi_velero_schedule_paused_count{{{labels}}} {values.get('schedule_paused_count', 0)}",
        f"awoooi_velero_latest_completed_backup_timestamp{{{labels}}} {values.get('latest_completed_backup_timestamp', 0)}",
        f"awoooi_velero_latest_completed_backup_age_seconds{{{labels}}} {values.get('latest_completed_backup_age_seconds', 0)}",
        f"awoooi_velero_latest_completed_backup_fresh{{{labels},max_age_hours=\"25\"}} {values.get('latest_completed_backup_fresh', 0)}",
        f"awoooi_velero_restore_test_cron_present{{{labels},cronjob=\"backup-restore-test\"}} {values.get('restore_test_cron_present', 0)}",
        f"awoooi_velero_restore_test_last_success_timestamp{{{labels},cronjob=\"backup-restore-test\"}} {values.get('restore_test_last_success_timestamp', 0)}",
        f"awoooi_velero_restore_test_last_success_age_seconds{{{labels},cronjob=\"backup-restore-test\"}} {values.get('restore_test_last_success_age_seconds', 0)}",
        f"awoooi_velero_restore_test_last_success_fresh{{{labels},cronjob=\"backup-restore-test\",max_age_hours=\"192\"}} {values.get('restore_test_last_success_fresh', 0)}",
        f"awoooi_velero_restore_test_failed_jobs{{{labels},cronjob=\"backup-restore-test\"}} {values.get('restore_test_failed_jobs', 0)}",
    ]


def _metric_lines_for_job(
    *,
    host: str,
    job: str,
    source: str,
    target: str,
    backup_type: str,
    last_success: int,
    max_age_hours: float,
    sample_count: int = 0,
) -> list[str]:
    now = int(time.time())
    labels = (
        f'host="{_escape_label(host)}",'
        f'job="{_escape_label(job)}",'
        f'type="{_escape_label(backup_type)}",'
        f'source="{_escape_label(source)}",'
        f'target="{_escape_label(target)}",'
        f'max_age_hours="{max_age_hours:g}"'
    )
    age = now - last_success if last_success > 0 else 0
    fresh = 1 if last_success > 0 and age <= int(max_age_hours * 3600) else 0
    return [
        f"awoooi_backup_expected_job_info{{{labels}}} 1",
        f"awoooi_backup_job_last_success_timestamp{{{labels}}} {last_success}",
        f"awoooi_backup_job_age_seconds{{{labels}}} {age}",
        f"awoooi_backup_job_fresh{{{labels}}} {fresh}",
        f"awoooi_backup_job_snapshot_count{{{labels}}} {sample_count}",
    ]


def _base_lines(host: str) -> list[str]:
    now = int(time.time())
    return [
        "# HELP awoooi_backup_health_monitor_up Whether the backup health exporter completed.",
        "# TYPE awoooi_backup_health_monitor_up gauge",
        "# HELP awoooi_backup_health_last_run_timestamp Unix timestamp of the last backup health exporter run.",
        "# TYPE awoooi_backup_health_last_run_timestamp gauge",
        "# HELP awoooi_backup_expected_job_info Expected backup job inventory.",
        "# TYPE awoooi_backup_expected_job_info gauge",
        "# HELP awoooi_backup_coverage_domain_expected_info Expected backup coverage domains for host, DB, website, service, package, tool, and log recovery.",
        "# TYPE awoooi_backup_coverage_domain_expected_info gauge",
        "# HELP awoooi_backup_coverage_domain_fresh Whether every required evidence job for a backup coverage domain is fresh.",
        "# TYPE awoooi_backup_coverage_domain_fresh gauge",
        "# HELP awoooi_backup_alert_receipt_expected_info Backup alert receipt requirements that must have Telegram/AwoooP evidence.",
        "# TYPE awoooi_backup_alert_receipt_expected_info gauge",
        "# HELP awoooi_backup_alert_receipt_fresh Whether every required receipt stage for a backup alert is fresh.",
        "# TYPE awoooi_backup_alert_receipt_fresh gauge",
        "# HELP awoooi_backup_alert_receipt_stage_expected_info Required sent/received/dedup/escalation receipt stage for a backup alert.",
        "# TYPE awoooi_backup_alert_receipt_stage_expected_info gauge",
        "# HELP awoooi_backup_alert_receipt_stage_last_success_timestamp Unix timestamp of the backup alert receipt stage evidence marker.",
        "# TYPE awoooi_backup_alert_receipt_stage_last_success_timestamp gauge",
        "# HELP awoooi_backup_alert_receipt_stage_age_seconds Age of the backup alert receipt stage evidence marker.",
        "# TYPE awoooi_backup_alert_receipt_stage_age_seconds gauge",
        "# HELP awoooi_backup_alert_receipt_stage_fresh Whether a backup alert receipt stage marker is fresh.",
        "# TYPE awoooi_backup_alert_receipt_stage_fresh gauge",
        "# HELP awoooi_backup_job_configured Whether the expected backup cron/config is present.",
        "# TYPE awoooi_backup_job_configured gauge",
        "# HELP awoooi_backup_script_present Whether the backup script exists on this host.",
        "# TYPE awoooi_backup_script_present gauge",
        "# HELP awoooi_backup_job_last_success_timestamp Unix timestamp of the latest successful backup evidence.",
        "# TYPE awoooi_backup_job_last_success_timestamp gauge",
        "# HELP awoooi_backup_job_age_seconds Age of the latest successful backup evidence.",
        "# TYPE awoooi_backup_job_age_seconds gauge",
        "# HELP awoooi_backup_job_fresh Whether the latest successful backup evidence is within max_age_hours.",
        "# TYPE awoooi_backup_job_fresh gauge",
        "# HELP awoooi_backup_job_snapshot_count Number of snapshots or files considered for this job.",
        "# TYPE awoooi_backup_job_snapshot_count gauge",
        "# HELP awoooi_backup_last_run_failed_count Failed component count from the last aggregate backup run.",
        "# TYPE awoooi_backup_last_run_failed_count gauge",
        "# HELP awoooi_backup_integrity_last_success_timestamp Unix timestamp of latest successful backup integrity or restore drill run.",
        "# TYPE awoooi_backup_integrity_last_success_timestamp gauge",
        "# HELP awoooi_backup_integrity_age_seconds Age of backup integrity or restore drill status.",
        "# TYPE awoooi_backup_integrity_age_seconds gauge",
        "# HELP awoooi_backup_integrity_fresh Whether backup integrity or restore drill status is fresh and successful.",
        "# TYPE awoooi_backup_integrity_fresh gauge",
        "# HELP awoooi_backup_integrity_failed_repo_count Failed repository count from backup integrity or restore drill run.",
        "# TYPE awoooi_backup_integrity_failed_repo_count gauge",
        "# HELP awoooi_backup_integrity_checked_repo_count Checked repository count from backup integrity or restore drill run.",
        "# TYPE awoooi_backup_integrity_checked_repo_count gauge",
        "# HELP awoooi_gitea_restore_drill_status_info Bounded result reason from the isolated Gitea restore drill.",
        "# TYPE awoooi_gitea_restore_drill_status_info gauge",
        "# HELP awoooi_gitea_restore_drill_last_success_timestamp Unix timestamp of the latest fully verified isolated Gitea restore drill.",
        "# TYPE awoooi_gitea_restore_drill_last_success_timestamp gauge",
        "# HELP awoooi_gitea_restore_drill_age_seconds Age of the latest isolated Gitea restore drill evidence.",
        "# TYPE awoooi_gitea_restore_drill_age_seconds gauge",
        "# HELP awoooi_gitea_restore_drill_fresh Whether the isolated Gitea restore drill is verified and within max age.",
        "# TYPE awoooi_gitea_restore_drill_fresh gauge",
        "# HELP awoooi_gitea_restore_drill_verified Whether CRC, required assets, counts, and inventory all passed.",
        "# TYPE awoooi_gitea_restore_drill_verified gauge",
        "# HELP awoooi_gitea_restore_drill_archive_crc_ok Whether every Gitea dump ZIP member passed CRC verification.",
        "# TYPE awoooi_gitea_restore_drill_archive_crc_ok gauge",
        "# HELP awoooi_gitea_restore_drill_required_artifacts_ok Whether database dump and configuration are present.",
        "# TYPE awoooi_gitea_restore_drill_required_artifacts_ok gauge",
        "# HELP awoooi_gitea_restore_drill_repo_count_match Whether backup and live regular/wiki repo counts match.",
        "# TYPE awoooi_gitea_restore_drill_repo_count_match gauge",
        "# HELP awoooi_gitea_restore_drill_inventory_match Whether privacy-preserving backup and live repo inventories match.",
        "# TYPE awoooi_gitea_restore_drill_inventory_match gauge",
        "# HELP awoooi_gitea_restore_drill_dump_size_bytes Size of the verified Gitea dump archive.",
        "# TYPE awoooi_gitea_restore_drill_dump_size_bytes gauge",
        "# HELP awoooi_gitea_restore_drill_repo_count Regular or wiki repo count in backup or live inventory.",
        "# TYPE awoooi_gitea_restore_drill_repo_count gauge",
        "# HELP awoooi_backup_config_capture_status_timestamp Unix timestamp of the latest config-capture coverage status.",
        "# TYPE awoooi_backup_config_capture_status_timestamp gauge",
        "# HELP awoooi_backup_config_capture_status_age_seconds Age of the latest config-capture coverage status.",
        "# TYPE awoooi_backup_config_capture_status_age_seconds gauge",
        "# HELP awoooi_backup_config_capture_critical_failed_count Critical config-capture targets missing from the latest configs backup.",
        "# TYPE awoooi_backup_config_capture_critical_failed_count gauge",
        "# HELP awoooi_backup_config_capture_failed_count Total config-capture targets missing from the latest configs backup.",
        "# TYPE awoooi_backup_config_capture_failed_count gauge",
        "# HELP awoooi_backup_config_capture_duration_seconds Duration of the latest configs backup capture run.",
        "# TYPE awoooi_backup_config_capture_duration_seconds gauge",
        "# HELP awoooi_backup_config_capture_ok Whether the latest configs backup captured a specific target.",
        "# TYPE awoooi_backup_config_capture_ok gauge",
        "# HELP awoooi_backup_offsite_configured Whether an offsite backup provider appears configured without exposing credentials.",
        "# TYPE awoooi_backup_offsite_configured gauge",
        "# HELP awoooi_backup_offsite_last_success_timestamp Unix timestamp of latest offsite copy success marker.",
        "# TYPE awoooi_backup_offsite_last_success_timestamp gauge",
        "# HELP awoooi_backup_offsite_age_seconds Age of latest offsite copy success marker.",
        "# TYPE awoooi_backup_offsite_age_seconds gauge",
        "# HELP awoooi_backup_offsite_fresh Whether offsite copy success marker is fresh.",
        "# TYPE awoooi_backup_offsite_fresh gauge",
        "# HELP awoooi_backup_offsite_partial_last_success_timestamp Unix timestamp of latest partial offsite copy success marker.",
        "# TYPE awoooi_backup_offsite_partial_last_success_timestamp gauge",
        "# HELP awoooi_backup_offsite_partial_age_seconds Age of latest partial offsite copy success marker.",
        "# TYPE awoooi_backup_offsite_partial_age_seconds gauge",
        "# HELP awoooi_backup_offsite_partial_fresh Whether partial offsite copy success marker is fresh.",
        "# TYPE awoooi_backup_offsite_partial_fresh gauge",
        "# HELP awoooi_backup_offsite_full_sync_enabled Whether the gated full offsite sync enable marker exists.",
        "# TYPE awoooi_backup_offsite_full_sync_enabled gauge",
        "# HELP awoooi_backup_offsite_full_sync_enabled_timestamp Unix timestamp of the gated full offsite sync enable marker.",
        "# TYPE awoooi_backup_offsite_full_sync_enabled_timestamp gauge",
        "# HELP awoooi_backup_credential_escrow_expected_info Expected credential escrow evidence inventory.",
        "# TYPE awoooi_backup_credential_escrow_expected_info gauge",
        "# HELP awoooi_backup_credential_escrow_last_verified_timestamp Unix timestamp of credential escrow verification evidence.",
        "# TYPE awoooi_backup_credential_escrow_last_verified_timestamp gauge",
        "# HELP awoooi_backup_credential_escrow_age_seconds Age of credential escrow verification evidence.",
        "# TYPE awoooi_backup_credential_escrow_age_seconds gauge",
        "# HELP awoooi_backup_credential_escrow_fresh Whether credential escrow verification evidence is fresh.",
        "# TYPE awoooi_backup_credential_escrow_fresh gauge",
        "# HELP awoooi_backup_dr_credential_escrow_missing_count Number of credential escrow items that still need fresh human verification.",
        "# TYPE awoooi_backup_dr_credential_escrow_missing_count gauge",
        "# HELP awoooi_backup_dr_phase Numeric DR offsite completion phase for AI/operator triage.",
        "# TYPE awoooi_backup_dr_phase gauge",
        "# HELP awoooi_backup_dr_next_step_info Current human-safe next step for DR offsite completion.",
        "# TYPE awoooi_backup_dr_next_step_info gauge",
        "# HELP awoooi_backup_retention_latest_only Whether local restic backup retention is configured as latest-only keep-last=1.",
        "# TYPE awoooi_backup_retention_latest_only gauge",
        "# HELP awoooi_backup_retention_offsite_delete_old_enabled Whether offsite rclone sync is allowed to delete old remote backup files after successful mirror.",
        "# TYPE awoooi_backup_retention_offsite_delete_old_enabled gauge",
        "# HELP awoooi_backup_cron_active_duplicate_count Number of exact duplicate active crontab entries on the backup host.",
        "# TYPE awoooi_backup_cron_active_duplicate_count gauge",
        "# HELP awoooi_backup_cron_singular_entry_count Number of active crontab entries matching a backup/offsite singleton pattern.",
        "# TYPE awoooi_backup_cron_singular_entry_count gauge",
        "# HELP awoooi_backup_cron_singular_entry_ok Whether a backup/offsite singleton cron pattern has exactly one active entry.",
        "# TYPE awoooi_backup_cron_singular_entry_ok gauge",
        "# HELP awoooi_gitea_bundle_root_exists Whether the 188 Gitea private bundle backup root exists.",
        "# TYPE awoooi_gitea_bundle_root_exists gauge",
        "# HELP awoooi_gitea_bundle_manifest_present Whether the 188 Gitea private bundle manifest exists.",
        "# TYPE awoooi_gitea_bundle_manifest_present gauge",
        "# HELP awoooi_gitea_bundle_newest_timestamp Unix timestamp of newest file under the 188 Gitea private bundle backup root.",
        "# TYPE awoooi_gitea_bundle_newest_timestamp gauge",
        "# HELP awoooi_gitea_bundle_age_seconds Age of newest 188 Gitea private bundle backup evidence.",
        "# TYPE awoooi_gitea_bundle_age_seconds gauge",
        "# HELP awoooi_gitea_bundle_fresh Whether the 188 Gitea private bundle backup evidence is fresh.",
        "# TYPE awoooi_gitea_bundle_fresh gauge",
        "# HELP awoooi_gitea_bundle_file_count Number of files scanned under the 188 Gitea private bundle backup root.",
        "# TYPE awoooi_gitea_bundle_file_count gauge",
        "# HELP awoooi_gitea_bundle_expected_repo_info Expected Gitea repository that must have a private bundle backup.",
        "# TYPE awoooi_gitea_bundle_expected_repo_info gauge",
        "# HELP awoooi_gitea_bundle_repo_status_info Manifest or fallback status for an expected Gitea private bundle backup.",
        "# TYPE awoooi_gitea_bundle_repo_status_info gauge",
        "# HELP awoooi_gitea_bundle_repo_present Whether an expected Gitea repository bundle file exists.",
        "# TYPE awoooi_gitea_bundle_repo_present gauge",
        "# HELP awoooi_gitea_bundle_repo_ok Whether an expected Gitea repository has a present bundle and checksum evidence.",
        "# TYPE awoooi_gitea_bundle_repo_ok gauge",
        "# HELP awoooi_gitea_bundle_repo_head_count Head count recorded in the Gitea private bundle manifest.",
        "# TYPE awoooi_gitea_bundle_repo_head_count gauge",
        "# HELP awoooi_gitea_bundle_checksum_present Whether checksum evidence exists for an expected Gitea private bundle.",
        "# TYPE awoooi_gitea_bundle_checksum_present gauge",
        "# HELP awoooi_gitea_bundle_checksum_digest_present Whether checksum evidence contains a SHA-256 digest.",
        "# TYPE awoooi_gitea_bundle_checksum_digest_present gauge",
        "# HELP awoooi_gitea_bundle_expected_repo_count Number of expected Gitea private bundle repositories.",
        "# TYPE awoooi_gitea_bundle_expected_repo_count gauge",
        "# HELP awoooi_gitea_bundle_expected_repo_missing_count Number of expected Gitea private bundle repositories with no bundle file.",
        "# TYPE awoooi_gitea_bundle_expected_repo_missing_count gauge",
        "# HELP awoooi_gitea_bundle_failed_repo_count Number of expected Gitea private bundle repositories without complete bundle/checksum evidence.",
        "# TYPE awoooi_gitea_bundle_failed_repo_count gauge",
        "# HELP awoooi_gitea_bundle_checksum_missing_count Number of expected Gitea private bundle repositories without checksum evidence.",
        "# TYPE awoooi_gitea_bundle_checksum_missing_count gauge",
        "# HELP awoooi_gitea_bundle_all_expected_ok Whether all expected Gitea private bundle repositories are present and fresh.",
        "# TYPE awoooi_gitea_bundle_all_expected_ok gauge",
        "# HELP awoooi_velero_monitor_up Whether the backup health exporter can query Velero via a reachable K3s kubectl endpoint.",
        "# TYPE awoooi_velero_monitor_up gauge",
        "# HELP awoooi_velero_schedule_count Number of Velero schedules in the velero namespace.",
        "# TYPE awoooi_velero_schedule_count gauge",
        "# HELP awoooi_velero_schedule_paused_count Number of paused Velero schedules.",
        "# TYPE awoooi_velero_schedule_paused_count gauge",
        "# HELP awoooi_velero_latest_completed_backup_timestamp Unix timestamp of latest Completed Velero backup.",
        "# TYPE awoooi_velero_latest_completed_backup_timestamp gauge",
        "# HELP awoooi_velero_latest_completed_backup_age_seconds Age of latest Completed Velero backup.",
        "# TYPE awoooi_velero_latest_completed_backup_age_seconds gauge",
        "# HELP awoooi_velero_latest_completed_backup_fresh Whether latest Completed Velero backup is within max_age_hours.",
        "# TYPE awoooi_velero_latest_completed_backup_fresh gauge",
        "# HELP awoooi_velero_restore_test_cron_present Whether backup-restore-test CronJob exists.",
        "# TYPE awoooi_velero_restore_test_cron_present gauge",
        "# HELP awoooi_velero_restore_test_last_success_timestamp Unix timestamp of backup-restore-test lastSuccessfulTime.",
        "# TYPE awoooi_velero_restore_test_last_success_timestamp gauge",
        "# HELP awoooi_velero_restore_test_last_success_age_seconds Age of backup-restore-test lastSuccessfulTime.",
        "# TYPE awoooi_velero_restore_test_last_success_age_seconds gauge",
        "# HELP awoooi_velero_restore_test_last_success_fresh Whether backup-restore-test lastSuccessfulTime is within max_age_hours.",
        "# TYPE awoooi_velero_restore_test_last_success_fresh gauge",
        "# HELP awoooi_velero_restore_test_failed_jobs Failed backup-restore-test jobs retained in velero namespace.",
        "# TYPE awoooi_velero_restore_test_failed_jobs gauge",
        f'awoooi_backup_health_monitor_up{{host="{_escape_label(host)}"}} 1',
        f'awoooi_backup_health_last_run_timestamp{{host="{_escape_label(host)}"}} {now}',
    ]


def _collect_110(host: str) -> list[str]:
    cron = _cron_text()
    lines = _base_lines(host)
    expected_crons = {
        "backup_all": "/backup/scripts/backup-all.sh",
        "awoooi_frequent": "/backup/scripts/backup-awoooi-frequent.sh",
        "offsite_status": "/backup/scripts/sync-offsite-backups.sh --mode status",
        "offsite_sync_gated": "/backup/offsite/enable-rclone-sync",
        "offsite_escrow_evidence_report": "/backup/scripts/offsite-escrow-evidence-report.sh --no-color",
        "offsite_full_sync_verify": "/backup/scripts/verify-offsite-full-sync.sh --write-textfile",
        "backup_integrity_check": "/backup/scripts/check-backup-integrity.sh --mode check",
        "backup_restore_drill": "/backup/scripts/check-backup-integrity.sh --mode restore-drill",
        "gitea_bundle_sync_188": "/backup/scripts/gitea-bundle-backup-sync-188.sh",
    }
    for job, pattern in expected_crons.items():
        labels = f'host="{_escape_label(host)}",job="{_escape_label(job)}"'
        lines.append(f"awoooi_backup_job_configured{{{labels}}} {int(pattern in cron)}")
    for script in [
        "backup-all.sh",
        "backup-awoooi.sh",
        "backup-awoooi-frequent.sh",
        "backup-configs.sh",
        "backup-sentry.sh",
        "backup-ai-artifacts.sh",
        "backup-public-routes.sh",
        "configure-offsite-rclone.sh",
        "configure-offsite-b2.sh",
        "sync-offsite-backups.sh",
        "backup-offsite-readiness-gate.sh",
        "offsite-escrow-evidence-report.sh",
        "verify-offsite-full-sync.sh",
        "mark-credential-escrow-verified.sh",
        "check-backup-integrity.sh",
        "verify-gitea-backup.sh",
        "gitea-repo-bundle-backup.sh",
        "gitea-bundle-backup-sync-188.sh",
        "backup-gitea.sh",
        "backup-harbor.sh",
        "backup-momo.sh",
        "backup-langfuse.sh",
        "backup-monitoring.sh",
        "backup-signoz.sh",
        "backup-open-webui.sh",
        "backup-clawbot.sh",
    ]:
        labels = f'host="{_escape_label(host)}",script="{_escape_label(script)}"'
        lines.append(f"awoooi_backup_script_present{{{labels}}} {int(Path('/backup/scripts', script).exists())}")

    job_freshness: dict[str, int] = {}
    for job, repo, max_age in [
        ("awoooi_db", "/backup/awoooi", 7),
        ("configs", "/backup/configs", 48),
        ("sentry", "/backup/sentry", 48),
        ("gitea", "/backup/gitea", 48),
        ("harbor", "/backup/harbor", 48),
        ("momo", "/backup/momo", 48),
        ("langfuse", "/backup/langfuse", 48),
        ("monitoring", "/backup/monitoring", 48),
        ("signoz", "/backup/signoz", 48),
        ("open_webui", "/backup/open-webui", 48),
        ("clawbot", "/backup/clawbot", 48),
        ("ai_artifacts", "/backup/ai-artifacts", 48),
        ("public_routes", "/backup/public-routes", 168),
    ]:
        timestamp, count = _latest_restic_snapshot(repo)
        age = int(time.time()) - timestamp if timestamp else 0
        job_freshness[job] = 1 if timestamp and age <= max_age * 3600 else 0
        lines.extend(
            _metric_lines_for_job(
                host=host,
                job=job,
                source="110-restic",
                target=repo,
                backup_type="restic",
                last_success=timestamp,
                max_age_hours=max_age,
                sample_count=count,
            )
        )

    coverage_domains = {
        "host": ["configs"],
        "database": ["awoooi_db"],
        "website": ["public_routes", "momo"],
        "service": ["gitea", "harbor", "sentry", "monitoring", "signoz"],
        "package": ["configs", "ai_artifacts"],
        "tool": ["ai_artifacts", "open_webui", "clawbot", "langfuse"],
        "log": ["monitoring", "signoz", "sentry"],
    }
    for domain, jobs in coverage_domains.items():
        required = ",".join(jobs)
        fresh = 1 if all(job_freshness.get(job, 0) == 1 for job in jobs) else 0
        labels = (
            f'host="{_escape_label(host)}",'
            f'domain="{_escape_label(domain)}",'
            f'required_jobs="{_escape_label(required)}"'
        )
        lines.append(f"awoooi_backup_coverage_domain_expected_info{{{labels}}} 1")
        lines.append(f"awoooi_backup_coverage_domain_fresh{{{labels}}} {fresh}")

    backup_all_ts, failed_count = _latest_backup_all_failed_count()
    labels = f'host="{_escape_label(host)}",job="backup_all"'
    lines.append(f"awoooi_backup_last_run_failed_count{{{labels}}} {failed_count}")
    lines.append(f"awoooi_backup_job_last_success_timestamp{{{labels},type=\"aggregate\",source=\"110-cron-log\",target=\"/backup/logs/cron.log\",max_age_hours=\"48\"}} {backup_all_ts if failed_count == 0 else 0}")
    lines.extend(_integrity_metric_lines(host))
    lines.extend(_gitea_restore_drill_metric_lines(host))
    lines.extend(_config_capture_metric_lines(host))
    lines.extend(_offsite_and_escrow_metric_lines(host))
    lines.extend(_retention_metric_lines(host))
    lines.extend(_cron_duplicate_metric_lines(host, cron))
    lines.extend(_velero_metric_lines(host))
    lines.extend(_backup_alert_receipt_metric_lines(host))
    return lines


def _collect_188(host: str) -> list[str]:
    cron = _cron_text()
    lines = _base_lines(host)
    for job, pattern in {
        "backup_from_110": "/home/ollama/bin/backup-from-110.sh",
        "momo_pg_daily": "/home/ollama/bin/momo-pg-backup.sh",
    }.items():
        labels = f'host="{_escape_label(host)}",job="{_escape_label(job)}"'
        lines.append(f"awoooi_backup_job_configured{{{labels}}} {int(pattern in cron)}")

    for script in [
        "/home/ollama/bin/backup-from-110.sh",
        "/home/ollama/bin/momo-pg-backup.sh",
        "/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh",
        "/home/ollama/awoooi-ops/pg-backup.sh",
    ]:
        labels = f'host="{_escape_label(host)}",script="{_escape_label(Path(script).name)}"'
        lines.append(f"awoooi_backup_script_present{{{labels}}} {int(Path(script).exists() and os.access(script, os.X_OK))}")

    backup_from_110_ts = _read_backup_110_timestamp()
    lines.extend(
        _metric_lines_for_job(
            host=host,
            job="backup_from_110",
            source="188-rsync",
            target="/home/ollama/backup/110",
            backup_type="rsync",
            last_success=backup_from_110_ts,
            max_age_hours=25,
            sample_count=1,
        )
    )
    _, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea"))
    gitea_mirror_ts = backup_from_110_ts
    gitea_mirror_fresh = 1 if gitea_mirror_ts and int(time.time()) - gitea_mirror_ts <= 25 * 3600 else 0
    lines.extend(
        _metric_lines_for_job(
            host=host,
            job="gitea_repo_mirror_from_110",
            source="188-rsync-subtree",
            target="/home/ollama/backup/110/gitea",
            backup_type="rsync_subtree",
            last_success=gitea_mirror_ts,
            max_age_hours=25,
            sample_count=gitea_mirror_count,
        )
    )
    gitea_bundle_lines, gitea_bundle_ok = _gitea_bundle_metric_lines(host)
    lines.extend(gitea_bundle_lines)
    gitea_bundle_ts, gitea_bundle_count = _newest_tree_timestamp(GITEA_BUNDLE_ROOT, max_entries=20000)
    gitea_bundle_fresh = 1 if gitea_bundle_ts and int(time.time()) - gitea_bundle_ts <= 25 * 3600 else 0
    lines.extend(
        _metric_lines_for_job(
            host=host,
            job="gitea_private_bundle",
            source="188-git-bundle",
            target=str(GITEA_BUNDLE_ROOT),
            backup_type="git_bundle",
            last_success=gitea_bundle_ts,
            max_age_hours=25,
            sample_count=gitea_bundle_count,
        )
    )
    coverage_labels = (
        f'host="{_escape_label(host)}",'
        'domain="service",'
        'required_jobs="backup_from_110,gitea_repo_mirror_from_110,gitea_private_bundle_completeness"'
    )
    lines.append(f"awoooi_backup_coverage_domain_expected_info{{{coverage_labels}}} 1")
    lines.append(
        "awoooi_backup_coverage_domain_fresh"
        f"{{{coverage_labels}}} {1 if gitea_mirror_fresh and gitea_bundle_fresh and gitea_bundle_ok else 0}"
    )
    momo_ts = _newest_file_timestamp([
        "/home/ollama/momo_backups/*.sql.gz",
        "/home/ollama/momo-pro/backups/*.sql.gz",
        "/home/ollama/backups/momo_analytics_*.sql.gz",
    ])
    lines.extend(
        _metric_lines_for_job(
            host=host,
            job="momo_pg_daily",
            source="188-pg-dump",
            target="/home/ollama/momo_backups",
            backup_type="pg_dump",
            last_success=momo_ts,
            max_age_hours=30,
            sample_count=1 if momo_ts else 0,
        )
    )
    lines.extend(_backup_alert_receipt_metric_lines(host))
    return lines


def collect() -> str:
    host = HOST_LABEL
    if host == "110":
        lines = _collect_110(host)
    elif host == "188":
        lines = _collect_188(host)
    else:
        lines = _base_lines(host)
    return "\n".join(lines) + "\n"


def main() -> None:
    TEXTFILE_DIR.mkdir(parents=True, exist_ok=True)
    payload = collect()
    with tempfile.NamedTemporaryFile("w", dir=TEXTFILE_DIR, delete=False, encoding="utf-8") as tmp:
        tmp.write(payload)
        tmp_path = Path(tmp.name)
    output_path = TEXTFILE_DIR / OUTPUT_NAME
    tmp_path.replace(output_path)
    output_path.chmod(0o644)


if __name__ == "__main__":
    main()