fix(alerts): stop slow-query backup misrouting

apps/api/src/services/awooop_ansible_audit_service.py

@@ -285,6 +285,17 @@ _CATALOG: tuple[dict[str, Any], ...] = (
             "cron",
             "crontab",
         ],
+        "activation_keywords": [
+            "momo",
+            "momopostgresbackupfailed",
+            "backup",
+            "momo pg_backup",
+            "momo postgres backup",
+            "pg_backup",
+            "momo-pg-backup",
+            "cron",
+            "crontab",
+        ],
         "supports_check_mode": True,
         "auto_apply_enabled": True,
         "approval_required": False,
@@ -313,6 +324,16 @@ _CATALOG: tuple[dict[str, Any], ...] = (
             "openwebui",
             "docker-registry",
         ],
+        "activation_keywords": [
+            "dockercontainerunhealthy",
+            "signoz",
+            "minio",
+            "litellm",
+            "n8n",
+            "open-webui",
+            "openwebui",
+            "docker-registry",
+        ],
         "supports_check_mode": True,
         "auto_apply_enabled": True,
         "approval_required": False,
@@ -618,6 +639,11 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
     unmatched: list[str] = []
     for item in _CATALOG:
         matched = [keyword for keyword in item["keywords"] if keyword in haystack]
+        activation_keywords = item.get("activation_keywords") or []
+        matched_activation_keywords = [
+            keyword for keyword in activation_keywords if keyword in haystack
+        ]
+        semantic_match = not activation_keywords or bool(matched_activation_keywords)
         public_item = {
             key: value
             for key, value in item.items()
@@ -635,11 +661,13 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
                 "no_write_only",
             }
         }
-        if matched:
+        if matched and semantic_match:
             candidates.append({
                 **public_item,
                 "match_score": len(matched),
                 "matched_keywords": matched,
+                "semantic_match": True,
+                "matched_activation_keywords": matched_activation_keywords,
             })
         else:
             unmatched.append(item["catalog_id"])
@@ -792,6 +820,10 @@ def build_ansible_decision_audit_payload(
                 "risk_level": row["risk_level"],
                 "match_score": row["match_score"],
                 "matched_keywords": row["matched_keywords"],
+                "semantic_match": row["semantic_match"],
+                "matched_activation_keywords": row[
+                    "matched_activation_keywords"
+                ],
             }
             for row in candidates[:5]
         ],

apps/api/tests/test_awooop_truth_chain_service.py

@@ -1396,7 +1396,7 @@ def test_ansible_decision_audit_payload_is_dry_run_only() -> None:
     assert payload["dry_run_result"]["check_mode_executed"] is False
 
 
-def test_ansible_decision_audit_payload_marks_repair_candidate_queue_claimable() -> None:
+def test_ansible_decision_audit_rejects_host_number_only_catalog_match() -> None:
     incident = SimpleNamespace(
         incident_id="INC-NODE-188",
         project_id="awoooi",
@@ -1425,17 +1425,7 @@ def test_ansible_decision_audit_payload_marks_repair_candidate_queue_claimable()
         not_used_reason="repair candidate controlled queue ready",
     )
 
-    assert payload is not None
-    assert payload["operation_type"] == "ansible_candidate_matched"
-    assert payload["status"] == "dry_run"
-    assert payload["output"]["decision_effect"] == "check_mode_queue_ready"
-    assert payload["output"]["next_required_step"] == (
-        "awooop_ansible_check_mode_worker_claims_candidate"
-    )
-    assert payload["input"]["executor_candidates"]
-    assert payload["dry_run_result"]["check_mode_executed"] is False
-    assert payload["input"]["project_id"] == "awoooi"
-    assert payload["input"]["execution_priority"] == 0
+    assert payload is None
 
 
 def test_ansible_canary_candidate_preserves_approval_and_controlled_apply() -> None:

apps/api/tests/test_postgres_slow_query_automation_guard.py

@@ -0,0 +1,113 @@
+"""Regression coverage for PostgreSQL slow-query alert automation routing."""
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import yaml
+
+from src.services.awooop_ansible_audit_service import (
+    build_ansible_decision_audit_payload,
+)
+
+ROOT = Path(__file__).resolve().parents[3]
+RULE_FILES = (
+    ROOT / "ops" / "monitoring" / "alerts.yml",
+    ROOT / "ops" / "monitoring" / "alerts-unified.yml",
+)
+
+
+def _find_rule(path: Path, alertname: str) -> dict:
+    payload = yaml.safe_load(path.read_text(encoding="utf-8"))
+    for group in payload.get("groups", []):
+        for rule in group.get("rules", []):
+            if rule.get("alert") == alertname:
+                return rule
+    raise AssertionError(f"missing {alertname} in {path}")
+
+
+def _incident(alert_name: str, labels: dict[str, str]) -> SimpleNamespace:
+    return SimpleNamespace(
+        incident_id="INC-20260711-SEMANTIC",
+        project_id="awoooi",
+        alert_category="database_performance",
+        notification_type="TYPE-3",
+        severity=SimpleNamespace(value="P2"),
+        affected_services=["postgres"],
+        signals=[
+            SimpleNamespace(
+                alert_name=alert_name,
+                labels=labels,
+                annotations={"message": "PostgreSQL has a transaction over 60s"},
+            )
+        ],
+    )
+
+
+def test_backup_clients_are_excluded_from_slow_query_alert_mirrors() -> None:
+    expressions = []
+    for path in RULE_FILES:
+        rule = _find_rule(path, "PostgreSQLSlowQueries")
+        expressions.append(rule["expr"])
+        assert "application_name" in rule["expr"]
+        assert "pg_dump" in rule["expr"]
+        assert "pg_basebackup" in rule["expr"]
+
+    assert len(set(expressions)) == 1
+
+
+def test_slow_query_cannot_activate_momo_backup_or_ai_web_catalog() -> None:
+    incident = _incident(
+        "PostgreSQLSlowQueries",
+        {
+            "alertname": "PostgreSQLSlowQueries",
+            "application_name": "pg_dump",
+            "component": "postgres",
+            "host": "188",
+            "instance": "192.168.0.188:9187",
+            "datname": "awoooi_prod",
+        },
+    )
+
+    payload = build_ansible_decision_audit_payload(
+        incident=incident,
+        proposal_data={
+            "source": "truth_chain_candidate_backfill",
+            "risk_level": "medium",
+            "action": "enqueue_allowlisted_ansible_check_mode",
+        },
+        decision_path="repair_candidate_controlled_queue",
+        not_used_reason="candidate backfill",
+    )
+
+    assert payload is None
+
+
+def test_momo_backup_failure_still_activates_bounded_backup_catalog() -> None:
+    incident = _incident(
+        "MomoPostgresBackupFailed",
+        {
+            "alertname": "MomoPostgresBackupFailed",
+            "component": "postgres",
+            "host": "188",
+            "job": "momo-pg-backup",
+        },
+    )
+
+    payload = build_ansible_decision_audit_payload(
+        incident=incident,
+        proposal_data={
+            "source": "truth_chain_candidate_backfill",
+            "risk_level": "low",
+            "action": "enqueue_allowlisted_ansible_check_mode",
+        },
+        decision_path="repair_candidate_controlled_queue",
+        not_used_reason="candidate backfill",
+    )
+
+    assert payload is not None
+    candidate = payload["input"]["executor_candidates"][0]
+    assert candidate["catalog_id"] == "ansible:188-momo-backup-user"
+    assert candidate["semantic_match"] is True
+    assert "momopostgresbackupfailed" in candidate[
+        "matched_activation_keywords"
+    ]

ops/monitoring/alerts-unified.yml

@@ -607,7 +607,7 @@ groups:
     rules:
       # ---- PostgreSQL 詳細指標 ----
       - alert: PostgreSQLSlowQueries
-        expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod"} > 60
+        expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod",application_name!~"^(pg_dump|pg_basebackup)$"} > 60
         for: 5m
         labels:
           severity: warning
@@ -618,7 +618,7 @@ groups:
           auto_repair: "false"
         annotations:
           summary: "PostgreSQL 有慢查詢 (>60s)"
-          description: "awoooi_prod 資料庫最長事務超過 60 秒"
+          description: "awoooi_prod 非備份程序的最長事務超過 60 秒"
 
       - alert: PostgreSQLDeadlocks
         expr: increase(pg_stat_database_deadlocks{datname="awoooi_prod"}[5m]) > 0

ops/monitoring/alerts.yml

@@ -291,7 +291,7 @@ groups:
     rules:
       # ---- PostgreSQL 詳細指標 ----
       - alert: PostgreSQLSlowQueries
-        expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod"} > 60
+        expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod",application_name!~"^(pg_dump|pg_basebackup)$"} > 60
         for: 5m
         labels:
           severity: warning
@@ -302,7 +302,7 @@ groups:
           auto_repair: "false"
         annotations:
           summary: "PostgreSQL 有慢查詢 (>60s)"
-          description: "awoooi_prod 資料庫最長事務超過 60 秒"
+          description: "awoooi_prod 非備份程序的最長事務超過 60 秒"
 
       - alert: PostgreSQLDeadlocks
         expr: increase(pg_stat_database_deadlocks{datname="awoooi_prod"}[5m]) > 0