fix(recovery): capture reboot lessons and gitea backup coverage
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 55s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
Your Name
2026-06-30 19:00:02 +08:00
parent 76fee33e1b
commit 4e909b2118
6 changed files with 174 additions and 6 deletions

View File

@@ -191,6 +191,24 @@ def _newest_file_timestamp(patterns: list[str]) -> int:
return newest
def _newest_tree_timestamp(root: Path, max_entries: int = 5000) -> tuple[int, int]:
if not root.exists():
return 0, 0
newest = 0
count = 0
for path in root.rglob("*"):
try:
if not path.is_file():
continue
count += 1
newest = max(newest, int(path.stat().st_mtime))
except OSError:
continue
if count >= max_entries:
break
return newest, count
def _read_backup_110_timestamp() -> int:
candidates = [
Path("/home/ollama/node_exporter_textfiles/backup.prom"),
@@ -907,6 +925,30 @@ def _collect_188(host: str) -> list[str]:
sample_count=1,
)
)
gitea_mirror_ts, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea"))
gitea_mirror_fresh = 1 if gitea_mirror_ts and int(time.time()) - gitea_mirror_ts <= 25 * 3600 else 0
lines.extend(
_metric_lines_for_job(
host=host,
job="gitea_repo_mirror_from_110",
source="188-rsync-subtree",
target="/home/ollama/backup/110/gitea",
backup_type="rsync_subtree",
last_success=gitea_mirror_ts,
max_age_hours=25,
sample_count=gitea_mirror_count,
)
)
coverage_labels = (
f'host="{_escape_label(host)}",'
'domain="service",'
'required_jobs="backup_from_110,gitea_repo_mirror_from_110"'
)
lines.append(f"awoooi_backup_coverage_domain_expected_info{{{coverage_labels}}} 1")
lines.append(
"awoooi_backup_coverage_domain_fresh"
f"{{{coverage_labels}}} {1 if gitea_mirror_fresh else 0}"
)
momo_ts = _newest_file_timestamp([
"/home/ollama/momo_backups/*.sql.gz",
"/home/ollama/momo-pro/backups/*.sql.gz",