fix(ops): close 110 pressure and backup alert gaps

scripts/backup/backup-status.sh

@@ -116,6 +116,19 @@ metric_sum() {
     ' "${file}"
 }
 
+metric_sum_excluding_backup_all() {
+    local file="$1"
+    local metric="$2"
+    if [ ! -s "${file}" ]; then
+        echo 0
+        return 0
+    fi
+    awk -v metric="${metric}" '
+        $1 ~ ("^" metric "\\{") && $1 !~ /(exported_job|job)="backup_all"/ { sum += $2 }
+        END { print sum + 0 }
+    ' "${file}"
+}
+
 metric_first() {
     local file="$1"
     local metric="$2"
@@ -214,6 +227,8 @@ stale_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 0)"
 stale_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 0)"
 failed_total_110="$(metric_sum "${TEXTFILE_110}" "awoooi_backup_last_run_failed_count")"
 failed_total_188="$(metric_sum "${TEXTFILE_188_TMP}" "awoooi_backup_last_run_failed_count")"
+component_failed_110="$(metric_sum_excluding_backup_all "${TEXTFILE_110}" "awoooi_backup_last_run_failed_count")"
+component_failed_188="$(metric_sum_excluding_backup_all "${TEXTFILE_188_TMP}" "awoooi_backup_last_run_failed_count")"
 integrity_stale_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_integrity_fresh" 0)"
 offsite_configured="$(metric_sum "${TEXTFILE_110}" "awoooi_backup_offsite_configured")"
 offsite_fresh="$(metric_sum "${TEXTFILE_110}" "awoooi_backup_offsite_fresh")"
@@ -221,7 +236,7 @@ offsite_rclone_configured="$(awk '/^awoooi_backup_offsite_configured\{.*provider
 offsite_rclone_fresh="$(awk '/^awoooi_backup_offsite_fresh\{.*provider="rclone"/ { print $2; found=1; exit } END { if (!found) print 0 }' "${TEXTFILE_110}" 2>/dev/null || echo 0)"
 escrow_missing="$(metric_first "${TEXTFILE_110}" "awoooi_backup_dr_credential_escrow_missing_count")"
 
-core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + stale_110 + stale_188 + failed_total_110 + failed_total_188 + integrity_stale_110))
+core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + stale_110 + stale_188 + component_failed_110 + component_failed_188 + integrity_stale_110))
 dr_warnings=0
 if [ "${offsite_configured%.*}" -lt 1 ] 2>/dev/null; then
     dr_warnings=$((dr_warnings + 1))
@@ -250,7 +265,7 @@ missing_scripts_188="$(label_list_for_zero "${TEXTFILE_188_TMP}" "awoooi_backup_
 backup_all_ts="$(metric_value_for_label "${TEXTFILE_110}" "awoooi_backup_job_last_success_timestamp" "job" "backup_all")"
 last_backup_all="$(human_timestamp "${backup_all_ts}")"
 
-message="${headline}; 110備份=${fresh_total_110}/13 fresh failed=${failed_total_110}; 188備份=${fresh_total_188}/2 fresh failed=${failed_total_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}"
+message="${headline}; 110備份=${fresh_total_110}/13 fresh component_failed=${component_failed_110} aggregate_failed=${failed_total_110}; 188備份=${fresh_total_188}/2 fresh component_failed=${component_failed_188} aggregate_failed=${failed_total_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}"
 
 if [ "${core_blockers}" -gt 0 ]; then
     message="${message}; stale110=${stale_jobs_110:-none}; stale188=${stale_jobs_188:-none}; missing_script110=${missing_scripts_110:-none}; missing_script188=${missing_scripts_188:-none}"

scripts/backup/tests/test_backup_status_contract.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+BACKUP_STATUS = ROOT / "scripts" / "backup" / "backup-status.sh"
+
+
+def test_backup_status_keeps_aggregate_failure_out_of_core_blockers() -> None:
+    text = BACKUP_STATUS.read_text(encoding="utf-8")
+
+    assert "metric_sum_excluding_backup_all" in text
+    assert "component_failed_110" in text
+    assert "component_failed_188" in text
+    assert "aggregate_failed=${failed_total_110}" in text
+    core_line = next(line for line in text.splitlines() if line.startswith("core_blockers="))
+    assert "component_failed_110" in core_line
+    assert "component_failed_188" in core_line
+    assert "failed_total_110" not in core_line
+    assert "failed_total_188" not in core_line