From b18a6c039e1837bdcbfc33e227103ff75eaa5574 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 20:29:08 +0800 Subject: [PATCH] fix(alerts): stop slow-query backup misrouting --- .../services/awooop_ansible_audit_service.py | 34 +++++- .../tests/test_awooop_truth_chain_service.py | 14 +-- ...st_postgres_slow_query_automation_guard.py | 113 ++++++++++++++++++ ops/monitoring/alerts-unified.yml | 4 +- ops/monitoring/alerts.yml | 4 +- 5 files changed, 152 insertions(+), 17 deletions(-) create mode 100644 apps/api/tests/test_postgres_slow_query_automation_guard.py diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 41336b060..4c68a9aa4 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -285,6 +285,17 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "cron", "crontab", ], + "activation_keywords": [ + "momo", + "momopostgresbackupfailed", + "backup", + "momo pg_backup", + "momo postgres backup", + "pg_backup", + "momo-pg-backup", + "cron", + "crontab", + ], "supports_check_mode": True, "auto_apply_enabled": True, "approval_required": False, @@ -313,6 +324,16 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "openwebui", "docker-registry", ], + "activation_keywords": [ + "dockercontainerunhealthy", + "signoz", + "minio", + "litellm", + "n8n", + "open-webui", + "openwebui", + "docker-registry", + ], "supports_check_mode": True, "auto_apply_enabled": True, "approval_required": False, @@ -618,6 +639,11 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None unmatched: list[str] = [] for item in _CATALOG: matched = [keyword for keyword in item["keywords"] if keyword in haystack] + activation_keywords = item.get("activation_keywords") or [] + matched_activation_keywords = [ + keyword for keyword in activation_keywords if keyword in haystack + ] + semantic_match = not activation_keywords or bool(matched_activation_keywords) public_item = { key: value for key, value in item.items() @@ -635,11 +661,13 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None "no_write_only", } } - if matched: + if matched and semantic_match: candidates.append({ **public_item, "match_score": len(matched), "matched_keywords": matched, + "semantic_match": True, + "matched_activation_keywords": matched_activation_keywords, }) else: unmatched.append(item["catalog_id"]) @@ -792,6 +820,10 @@ def build_ansible_decision_audit_payload( "risk_level": row["risk_level"], "match_score": row["match_score"], "matched_keywords": row["matched_keywords"], + "semantic_match": row["semantic_match"], + "matched_activation_keywords": row[ + "matched_activation_keywords" + ], } for row in candidates[:5] ], diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 5fdeac667..314cd0824 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -1396,7 +1396,7 @@ def test_ansible_decision_audit_payload_is_dry_run_only() -> None: assert payload["dry_run_result"]["check_mode_executed"] is False -def test_ansible_decision_audit_payload_marks_repair_candidate_queue_claimable() -> None: +def test_ansible_decision_audit_rejects_host_number_only_catalog_match() -> None: incident = SimpleNamespace( incident_id="INC-NODE-188", project_id="awoooi", @@ -1425,17 +1425,7 @@ def test_ansible_decision_audit_payload_marks_repair_candidate_queue_claimable() not_used_reason="repair candidate controlled queue ready", ) - assert payload is not None - assert payload["operation_type"] == "ansible_candidate_matched" - assert payload["status"] == "dry_run" - assert payload["output"]["decision_effect"] == "check_mode_queue_ready" - assert payload["output"]["next_required_step"] == ( - "awooop_ansible_check_mode_worker_claims_candidate" - ) - assert payload["input"]["executor_candidates"] - assert payload["dry_run_result"]["check_mode_executed"] is False - assert payload["input"]["project_id"] == "awoooi" - assert payload["input"]["execution_priority"] == 0 + assert payload is None def test_ansible_canary_candidate_preserves_approval_and_controlled_apply() -> None: diff --git a/apps/api/tests/test_postgres_slow_query_automation_guard.py b/apps/api/tests/test_postgres_slow_query_automation_guard.py new file mode 100644 index 000000000..f44bcbd3e --- /dev/null +++ b/apps/api/tests/test_postgres_slow_query_automation_guard.py @@ -0,0 +1,113 @@ +"""Regression coverage for PostgreSQL slow-query alert automation routing.""" + +from pathlib import Path +from types import SimpleNamespace + +import yaml + +from src.services.awooop_ansible_audit_service import ( + build_ansible_decision_audit_payload, +) + +ROOT = Path(__file__).resolve().parents[3] +RULE_FILES = ( + ROOT / "ops" / "monitoring" / "alerts.yml", + ROOT / "ops" / "monitoring" / "alerts-unified.yml", +) + + +def _find_rule(path: Path, alertname: str) -> dict: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + for group in payload.get("groups", []): + for rule in group.get("rules", []): + if rule.get("alert") == alertname: + return rule + raise AssertionError(f"missing {alertname} in {path}") + + +def _incident(alert_name: str, labels: dict[str, str]) -> SimpleNamespace: + return SimpleNamespace( + incident_id="INC-20260711-SEMANTIC", + project_id="awoooi", + alert_category="database_performance", + notification_type="TYPE-3", + severity=SimpleNamespace(value="P2"), + affected_services=["postgres"], + signals=[ + SimpleNamespace( + alert_name=alert_name, + labels=labels, + annotations={"message": "PostgreSQL has a transaction over 60s"}, + ) + ], + ) + + +def test_backup_clients_are_excluded_from_slow_query_alert_mirrors() -> None: + expressions = [] + for path in RULE_FILES: + rule = _find_rule(path, "PostgreSQLSlowQueries") + expressions.append(rule["expr"]) + assert "application_name" in rule["expr"] + assert "pg_dump" in rule["expr"] + assert "pg_basebackup" in rule["expr"] + + assert len(set(expressions)) == 1 + + +def test_slow_query_cannot_activate_momo_backup_or_ai_web_catalog() -> None: + incident = _incident( + "PostgreSQLSlowQueries", + { + "alertname": "PostgreSQLSlowQueries", + "application_name": "pg_dump", + "component": "postgres", + "host": "188", + "instance": "192.168.0.188:9187", + "datname": "awoooi_prod", + }, + ) + + payload = build_ansible_decision_audit_payload( + incident=incident, + proposal_data={ + "source": "truth_chain_candidate_backfill", + "risk_level": "medium", + "action": "enqueue_allowlisted_ansible_check_mode", + }, + decision_path="repair_candidate_controlled_queue", + not_used_reason="candidate backfill", + ) + + assert payload is None + + +def test_momo_backup_failure_still_activates_bounded_backup_catalog() -> None: + incident = _incident( + "MomoPostgresBackupFailed", + { + "alertname": "MomoPostgresBackupFailed", + "component": "postgres", + "host": "188", + "job": "momo-pg-backup", + }, + ) + + payload = build_ansible_decision_audit_payload( + incident=incident, + proposal_data={ + "source": "truth_chain_candidate_backfill", + "risk_level": "low", + "action": "enqueue_allowlisted_ansible_check_mode", + }, + decision_path="repair_candidate_controlled_queue", + not_used_reason="candidate backfill", + ) + + assert payload is not None + candidate = payload["input"]["executor_candidates"][0] + assert candidate["catalog_id"] == "ansible:188-momo-backup-user" + assert candidate["semantic_match"] is True + assert "momopostgresbackupfailed" in candidate[ + "matched_activation_keywords" + ] diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index b658d5d95..22b804a13 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -607,7 +607,7 @@ groups: rules: # ---- PostgreSQL 詳細指標 ---- - alert: PostgreSQLSlowQueries - expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod"} > 60 + expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod",application_name!~"^(pg_dump|pg_basebackup)$"} > 60 for: 5m labels: severity: warning @@ -618,7 +618,7 @@ groups: auto_repair: "false" annotations: summary: "PostgreSQL 有慢查詢 (>60s)" - description: "awoooi_prod 資料庫最長事務超過 60 秒" + description: "awoooi_prod 非備份程序的最長事務超過 60 秒" - alert: PostgreSQLDeadlocks expr: increase(pg_stat_database_deadlocks{datname="awoooi_prod"}[5m]) > 0 diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 04e011c4b..9dccc313f 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -291,7 +291,7 @@ groups: rules: # ---- PostgreSQL 詳細指標 ---- - alert: PostgreSQLSlowQueries - expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod"} > 60 + expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod",application_name!~"^(pg_dump|pg_basebackup)$"} > 60 for: 5m labels: severity: warning @@ -302,7 +302,7 @@ groups: auto_repair: "false" annotations: summary: "PostgreSQL 有慢查詢 (>60s)" - description: "awoooi_prod 資料庫最長事務超過 60 秒" + description: "awoooi_prod 非備份程序的最長事務超過 60 秒" - alert: PostgreSQLDeadlocks expr: increase(pg_stat_database_deadlocks{datname="awoooi_prod"}[5m]) > 0