fix(recovery): capture reboot lessons and gitea backup coverage
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 55s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 55s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -191,6 +191,24 @@ def _newest_file_timestamp(patterns: list[str]) -> int:
|
||||
return newest
|
||||
|
||||
|
||||
def _newest_tree_timestamp(root: Path, max_entries: int = 5000) -> tuple[int, int]:
|
||||
if not root.exists():
|
||||
return 0, 0
|
||||
newest = 0
|
||||
count = 0
|
||||
for path in root.rglob("*"):
|
||||
try:
|
||||
if not path.is_file():
|
||||
continue
|
||||
count += 1
|
||||
newest = max(newest, int(path.stat().st_mtime))
|
||||
except OSError:
|
||||
continue
|
||||
if count >= max_entries:
|
||||
break
|
||||
return newest, count
|
||||
|
||||
|
||||
def _read_backup_110_timestamp() -> int:
|
||||
candidates = [
|
||||
Path("/home/ollama/node_exporter_textfiles/backup.prom"),
|
||||
@@ -907,6 +925,30 @@ def _collect_188(host: str) -> list[str]:
|
||||
sample_count=1,
|
||||
)
|
||||
)
|
||||
gitea_mirror_ts, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea"))
|
||||
gitea_mirror_fresh = 1 if gitea_mirror_ts and int(time.time()) - gitea_mirror_ts <= 25 * 3600 else 0
|
||||
lines.extend(
|
||||
_metric_lines_for_job(
|
||||
host=host,
|
||||
job="gitea_repo_mirror_from_110",
|
||||
source="188-rsync-subtree",
|
||||
target="/home/ollama/backup/110/gitea",
|
||||
backup_type="rsync_subtree",
|
||||
last_success=gitea_mirror_ts,
|
||||
max_age_hours=25,
|
||||
sample_count=gitea_mirror_count,
|
||||
)
|
||||
)
|
||||
coverage_labels = (
|
||||
f'host="{_escape_label(host)}",'
|
||||
'domain="service",'
|
||||
'required_jobs="backup_from_110,gitea_repo_mirror_from_110"'
|
||||
)
|
||||
lines.append(f"awoooi_backup_coverage_domain_expected_info{{{coverage_labels}}} 1")
|
||||
lines.append(
|
||||
"awoooi_backup_coverage_domain_fresh"
|
||||
f"{{{coverage_labels}}} {1 if gitea_mirror_fresh else 0}"
|
||||
)
|
||||
momo_ts = _newest_file_timestamp([
|
||||
"/home/ollama/momo_backups/*.sql.gz",
|
||||
"/home/ollama/momo-pro/backups/*.sql.gz",
|
||||
|
||||
Reference in New Issue
Block a user