From f493fe8dcb83fc85ac489b3688a6d8484d60e3cd Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Jul 2026 18:02:06 +0800 Subject: [PATCH] feat(alerting): add Telegram callback ingress sensor --- apps/api/src/core/metrics.py | 8 + apps/api/src/main.py | 15 + .../services/awooop_ansible_audit_service.py | 22 + .../services/awooop_ansible_post_verifier.py | 42 ++ apps/api/src/services/telegram_gateway.py | 9 + ...t_telegram_callback_monitoring_contract.py | 164 +++++++ .../110-telegram-callback-ingress-sensor.yml | 407 ++++++++++++++++++ k8s/monitoring/alert-chain-monitor.yaml | 21 + ops/monitoring/alerts-unified.yml | 21 + ops/monitoring/alerts.yml | 21 + 10 files changed, 730 insertions(+) create mode 100644 apps/api/tests/test_telegram_callback_monitoring_contract.py create mode 100644 infra/ansible/playbooks/110-telegram-callback-ingress-sensor.yml diff --git a/apps/api/src/core/metrics.py b/apps/api/src/core/metrics.py index 260adc670..d6eecdc2e 100644 --- a/apps/api/src/core/metrics.py +++ b/apps/api/src/core/metrics.py @@ -54,6 +54,14 @@ TELEGRAM_NOTIFICATIONS_TOTAL = Counter( ["source", "status"], # status: success/failed ) +TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP = Gauge( + "awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds", + ( + "Unix timestamp of the latest durable Telegram callback ingress receipt; " + "zero means no durable receipt has been observed by this API process" + ), +) + # ============================================================================= # Anomaly Counter Metrics (ADR-037) # ============================================================================= diff --git a/apps/api/src/main.py b/apps/api/src/main.py index c31af5d41..b4a760e8c 100644 --- a/apps/api/src/main.py +++ b/apps/api/src/main.py @@ -339,6 +339,11 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]: # OpenClaw (192.168.0.188) 是唯一的 Polling 實例 # AWOOOI API 只負責發送訊息，不接收 telegram_gw = get_telegram_gateway() + callback_receipt_loaded = await telegram_gw.refresh_callback_ingress_receipt() + logger.info( + "telegram_callback_ingress_receipt_bootstrap", + durable_receipt_loaded=callback_receipt_loaded, + ) if settings.TELEGRAM_ENABLE_POLLING: await telegram_gw.start_long_polling() logger.info("telegram_long_polling_started") @@ -1400,6 +1405,16 @@ app.include_router( @app.get("/metrics", include_in_schema=False) async def prometheus_metrics() -> Response: """Prometheus metrics endpoint for alerting""" + # Callback ingress receipts are shared in Redis while API metrics are + # process-local. Refresh before every scrape so a NodePort that alternates + # across replicas cannot expose stale/false-green transport evidence. + try: + await get_telegram_gateway().refresh_callback_ingress_receipt() + except Exception as exc: + logger.warning( + "prometheus_metrics_telegram_callback_receipt_error", + error_type=type(exc).__name__, + ) # 2026-05-19 Codex — T85 Alert Chain DB evidence refresh. # record_alert_chain_success() 是 process-local gauge；部署後第一個 scrape # 可能尚未收到新 webhook，導致 smoke test 誤判 metric 不存在。 diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index f5f6fa058..4d46c5430 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -636,6 +636,28 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "canonical_asset_id": "container:host_110:alertmanager", "catalog_revision": "2026-07-16-alertmanager-exact-container-v1", }, + { + "catalog_id": "ansible:110-telegram-callback-ingress-sensor", + "playbook_path": ( + "infra/ansible/playbooks/110-telegram-callback-ingress-sensor.yml" + ), + "check_mode_playbook_path": ( + "infra/ansible/playbooks/110-telegram-callback-ingress-sensor.yml" + ), + "inventory_hosts": ["host_110"], + "domains": ["monitoring", "prometheus", "alert_chain_health"], + "keywords": [ + "telegram callback ingress sensor", + "telegramcallbackingressunverified", + "awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds", + ], + "supports_check_mode": True, + "auto_apply_enabled": False, + "approval_required": False, + "risk_level": "medium", + "canonical_asset_id": "service:prometheus", + "catalog_revision": "2026-07-18-telegram-callback-sensor-v1", + }, { "catalog_id": "ansible:111-ollama-fallback", "playbook_path": "infra/ansible/playbooks/111-ollama-fallback.yml", diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index 5dcbdc597..659d56986 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -212,6 +212,48 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { 740, ), ), + "ansible:110-telegram-callback-ingress-sensor": ( + AssetPostcondition( + "host_110_telegram_callback_ingress_rule_healthy", + "metric", + "host_110", + "curl -fsS --max-time 5 http://127.0.0.1:9090/api/v1/rules | " + "python3 -c 'import json,sys; d=json.load(sys.stdin); " + "r=[x for g in d.get(\"data\",{}).get(\"groups\",[]) " + "for x in g.get(\"rules\",[]) " + "if x.get(\"name\")==\"TelegramCallbackIngressUnverified\"]; " + "assert len(r)==1 and r[0].get(\"health\")==\"ok\"; " + "q=\"\".join(str(r[0].get(\"query\") or \"\").split()); " + "assert \"awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds\" in q; " + "assert \">300\" in q and \"absent(\" in q; " + "labels=r[0].get(\"labels\") or {}; " + "assert labels.get(\"component\")==\"openclaw\"; " + "assert labels.get(\"canonical_asset_id\")==\"service:openclaw:host188\"; " + "assert labels.get(\"domain_router\")==\"docker_container\"'", + ), + AssetPostcondition( + "host_110_telegram_callback_ingress_metric_visible", + "metric", + "host_110", + "curl -fsS --max-time 5 --get " + "--data-urlencode " + "'query=awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds' " + "http://127.0.0.1:9090/api/v1/query | " + "python3 -c 'import json,sys; d=json.load(sys.stdin); " + "assert len(d.get(\"data\",{}).get(\"result\",[])) >= 1'", + ), + AssetPostcondition( + "host_110_telegram_callback_ingress_rule_hash_converged", + "metric", + "host_110", + "expected=$(sha256sum /home/wooo/monitoring/alerts.yml | awk '{print $1}'); " + "test -n \"$expected\" && " + "test \"$expected\" = \"$(sha256sum " + "/home/wooo/monitoring/alerts-unified.canonical.yml | awk '{print $1}')\" && " + "test \"$expected\" = \"$(docker exec prometheus sha256sum " + "/etc/prometheus/alerts.yml | awk '{print $1}')\"", + ), + ), "ansible:111-ollama-fallback": ( AssetPostcondition( "host_111_ollama_launchagent_allowlist_exact", diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 83caff89c..edb78f407 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -40,6 +40,9 @@ import structlog from opentelemetry import trace from src.core.config import settings +from src.core.metrics import ( + TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP, +) from src.core.redis_client import get_redis from src.services.agent99_controlled_dispatch_ledger import ( read_agent99_dispatch_receipt, @@ -5730,6 +5733,9 @@ class TelegramGateway: ) self._last_callback_ingress_receipt_durable = True self._callback_ingress_shared_read_error = "" + TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP.set( + received_at.timestamp() + ) except Exception as exc: self._callback_ingress_shared_read_error = type(exc).__name__ logger.error( @@ -5783,6 +5789,9 @@ class TelegramGateway: self._last_callback_ingress_source = source self._last_callback_ingress_receipt_durable = True self._callback_ingress_shared_read_error = "" + TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP.set( + received_at.timestamp() + ) if isinstance(update_id, int): self._last_update_id = max(int(self._last_update_id or 0), update_id) return True diff --git a/apps/api/tests/test_telegram_callback_monitoring_contract.py b/apps/api/tests/test_telegram_callback_monitoring_contract.py new file mode 100644 index 000000000..aa2474abe --- /dev/null +++ b/apps/api/tests/test_telegram_callback_monitoring_contract.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +from src.core.metrics import ( + TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP, +) +from src.services import telegram_gateway as telegram_gateway_module +from src.services.controlled_alert_target_router import resolve_typed_alert_target +from src.services.telegram_gateway import TelegramGateway + +ROOT = Path(__file__).resolve().parents[3] +CATALOG = ROOT / "apps" / "api" / "src" / "services" / "awooop_ansible_audit_service.py" +POST_VERIFIER = ( + ROOT / "apps" / "api" / "src" / "services" / "awooop_ansible_post_verifier.py" +) +PLAYBOOK = ( + ROOT + / "infra" + / "ansible" + / "playbooks" + / "110-telegram-callback-ingress-sensor.yml" +) +RULE_SOURCES = ( + ROOT / "ops" / "monitoring" / "alerts-unified.yml", + ROOT / "ops" / "monitoring" / "alerts.yml", + ROOT / "k8s" / "monitoring" / "alert-chain-monitor.yaml", +) + + +class _ConfiguredGateway(TelegramGateway): + @property + def bot_token(self) -> str: + return "configured" + + @property + def chat_id(self) -> str: + return "configured" + + +class _Redis: + def __init__(self) -> None: + self.values: dict[str, str] = {} + + async def setex(self, key: str, _ttl: int, value: str) -> None: + self.values[key] = value + + async def get(self, key: str) -> str | None: + return self.values.get(key) + + +def _rule(path: Path) -> dict: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + groups = payload["spec"]["groups"] if "spec" in payload else payload["groups"] + return next( + rule + for group in groups + for rule in group.get("rules", []) + if rule.get("alert") == "TelegramCallbackIngressUnverified" + ) + + +def test_callback_ingress_sensor_is_identical_and_exact_typed_in_all_sources() -> None: + rules = [_rule(path) for path in RULE_SOURCES] + expressions = {re.sub(r"\s+", "", str(rule["expr"])) for rule in rules} + + assert len(expressions) == 1 + expression = expressions.pop() + assert "awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds" in expression + assert ">300" in expression + assert "absent(" in expression + + for rule in rules: + assert rule["for"] == "2m" + assert rule["labels"] == { + "severity": "warning", + "layer": "docker-188", + "component": "openclaw", + "host": "188", + "team": "platform", + "auto_repair": "true", + "alert_category": "ai_agent", + "canonical_asset_id": "service:openclaw:host188", + "domain_router": "docker_container", + } + + route = resolve_typed_alert_target( + alertname="TelegramCallbackIngressUnverified", + target_resource=rule["labels"]["component"], + namespace="default", + labels=rule["labels"], + alert_category=rule["labels"]["alert_category"], + ) + assert route["canonical_asset_id"] == "service:openclaw:host188" + assert route["allowed_catalog_ids"] == [ + "ansible:188-openclaw-callback-forwarder" + ] + assert route["cross_domain_fallback_allowed"] is False + + +def test_sensor_has_exact_host110_catalog_rollback_and_independent_verifier() -> None: + catalog = CATALOG.read_text(encoding="utf-8") + verifier = POST_VERIFIER.read_text(encoding="utf-8") + playbook = PLAYBOOK.read_text(encoding="utf-8") + + assert '"catalog_id": "ansible:110-telegram-callback-ingress-sensor"' in catalog + assert '"inventory_hosts": ["host_110"]' in catalog + assert '"auto_apply_enabled": False' in catalog + assert '"canonical_asset_id": "service:prometheus"' in catalog + assert '"ansible:110-telegram-callback-ingress-sensor": (' in verifier + assert "host_110_telegram_callback_ingress_rule_healthy" in verifier + assert "host_110_telegram_callback_ingress_metric_visible" in verifier + assert "host_110_telegram_callback_ingress_rule_hash_converged" in verifier + assert "promtool" in playbook + assert "callback-sensor.bak" in playbook + assert "rescue:" in playbook + assert "Require exact rollback proof after any started apply" in playbook + assert "TelegramCallbackIngressUnverified" in playbook + + play = yaml.safe_load(playbook)[0] + assert play["hosts"] == "host_110" + assert "--force-recreate" in playbook + assert "prometheus_runtime_rules" in playbook + assert "sensor_rollback_hashes" in playbook + + +@pytest.mark.asyncio +async def test_durable_callback_receipt_updates_prometheus_timestamp( + monkeypatch, +) -> None: + redis = _Redis() + monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: redis) + TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP.set(0) + + gateway = _ConfiguredGateway() + await gateway.record_callback_ingress_receipt( + source="openclaw_188_hmac_forwarder", + update_id=99, + ) + + assert TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP._value.get() > 0 + + +@pytest.mark.asyncio +async def test_separate_scraped_replica_refreshes_shared_receipt_metric( + monkeypatch, +) -> None: + redis = _Redis() + monkeypatch.setattr(telegram_gateway_module, "get_redis", lambda: redis) + writer = _ConfiguredGateway() + scraped_replica = _ConfiguredGateway() + + await writer.record_callback_ingress_receipt( + source="openclaw_188_hmac_forwarder", + update_id=100, + ) + TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP.set(0) + + assert await scraped_replica.refresh_callback_ingress_receipt() is True + assert TELEGRAM_CALLBACK_INGRESS_LAST_RECEIPT_TIMESTAMP._value.get() > 0 diff --git a/infra/ansible/playbooks/110-telegram-callback-ingress-sensor.yml b/infra/ansible/playbooks/110-telegram-callback-ingress-sensor.yml new file mode 100644 index 000000000..e6b95556b --- /dev/null +++ b/infra/ansible/playbooks/110-telegram-callback-ingress-sensor.yml @@ -0,0 +1,407 @@ +--- +- name: Bounded host 110 Telegram callback ingress sensor deployment + hosts: host_110 + gather_facts: false + become: false + vars: + prometheus_container: prometheus + prometheus_compose_file: /home/wooo/monitoring/docker-compose.yml + prometheus_compose_service: prometheus + prometheus_rules_source: >- + {{ playbook_dir }}/../../../ops/monitoring/alerts-unified.yml + prometheus_rules_target: /home/wooo/monitoring/alerts.yml + prometheus_rules_canonical: >- + /home/wooo/monitoring/alerts-unified.canonical.yml + prometheus_runtime_rules: /etc/prometheus/alerts.yml + sensor_candidate_host: >- + /tmp/awoooi-telegram-callback-sensor.{{ sensor_deploy_stamp }}.yml + sensor_candidate_container: >- + /tmp/awoooi-telegram-callback-sensor.{{ sensor_deploy_stamp }}.yml + + tasks: + - name: Inspect exact Prometheus container running state + ansible.builtin.command: + argv: + - docker + - inspect + - --format + - "{{ '{{' }}.State.Running{{ '}}' }}" + - "{{ prometheus_container }}" + register: prometheus_running + check_mode: false + changed_when: false + failed_when: >- + prometheus_running.rc != 0 + or (prometheus_running.stdout | trim) != 'true' + + - name: Validate active Prometheus rules before bounded change + ansible.builtin.command: + argv: + - docker + - exec + - "{{ prometheus_container }}" + - promtool + - check + - rules + - "{{ prometheus_runtime_rules }}" + check_mode: false + changed_when: false + + - name: Inspect exact pre-apply rule hashes for rollback ownership + ansible.builtin.stat: + path: "{{ item }}" + checksum_algorithm: sha256 + get_checksum: true + loop: + - "{{ prometheus_rules_target }}" + - "{{ prometheus_rules_canonical }}" + register: prometheus_rule_files_before + changed_when: false + + - name: Require both exact pre-apply rule files + ansible.builtin.assert: + that: + - prometheus_rule_files_before.results[0].stat.exists + - prometheus_rule_files_before.results[1].stat.exists + fail_msg: telegram_callback_sensor_preapply_rule_file_missing + + - name: Preview exact runtime rule diff without writing + ansible.builtin.copy: + src: "{{ prometheus_rules_source }}" + dest: "{{ prometheus_rules_target }}" + owner: wooo + group: wooo + mode: "0644" + when: ansible_check_mode + + - name: Preview exact canonical rule diff without writing + ansible.builtin.copy: + src: "{{ prometheus_rules_source }}" + dest: "{{ prometheus_rules_canonical }}" + owner: wooo + group: wooo + mode: "0644" + when: ansible_check_mode + + - name: Initialize bounded apply ownership + ansible.builtin.set_fact: + sensor_apply_started: false + sensor_deploy_stamp: "{{ now(utc=true, fmt='%Y%m%d%H%M%S%f') }}" + changed_when: false + when: not ansible_check_mode + + - name: Deploy candidate with exact-container hash and rollback verification + when: not ansible_check_mode + block: + - name: Stage exact source candidate outside the bind mount + ansible.builtin.copy: + src: "{{ prometheus_rules_source }}" + dest: "{{ sensor_candidate_host }}" + owner: wooo + group: wooo + mode: "0644" + + - name: Stage candidate inside the exact Prometheus container + ansible.builtin.command: + argv: + - docker + - cp + - "{{ sensor_candidate_host }}" + - >- + {{ prometheus_container }}:{{ sensor_candidate_container }} + changed_when: true + + - name: Validate staged candidate with production promtool + ansible.builtin.command: + argv: + - docker + - exec + - "{{ prometheus_container }}" + - promtool + - check + - rules + - "{{ sensor_candidate_container }}" + changed_when: false + + - name: Capture exact candidate hash + ansible.builtin.command: + argv: + - sha256sum + - "{{ sensor_candidate_host }}" + register: sensor_candidate_hash + changed_when: false + + - name: Create exact runtime rollback copy + ansible.builtin.command: + argv: + - cp + - --preserve=mode,ownership,timestamps + - "{{ prometheus_rules_target }}" + - >- + {{ prometheus_rules_target }}.callback-sensor.bak.{{ sensor_deploy_stamp }} + register: sensor_runtime_backup + changed_when: true + + - name: Create exact canonical rollback copy + ansible.builtin.command: + argv: + - cp + - --preserve=mode,ownership,timestamps + - "{{ prometheus_rules_canonical }}" + - >- + {{ prometheus_rules_canonical }}.callback-sensor.bak.{{ sensor_deploy_stamp }} + register: sensor_canonical_backup + changed_when: true + + - name: Mark bounded apply ownership after both backups exist + ansible.builtin.set_fact: + sensor_apply_started: true + changed_when: false + + - name: Replace rule content while preserving the bind-mount inode + ansible.builtin.shell: | + set -euo pipefail + cat "{{ sensor_candidate_host }}" > "{{ prometheus_rules_target }}" + cat "{{ sensor_candidate_host }}" > "{{ prometheus_rules_canonical }}" + args: + executable: /bin/bash + changed_when: true + + - name: Verify both host rule hashes equal the candidate + ansible.builtin.shell: | + set -euo pipefail + expected="{{ sensor_candidate_hash.stdout.split()[0] }}" + test "$(sha256sum '{{ prometheus_rules_target }}' | awk '{print $1}')" = "${expected}" + test "$(sha256sum '{{ prometheus_rules_canonical }}' | awk '{print $1}')" = "${expected}" + args: + executable: /bin/bash + changed_when: false + + - name: Read runtime bind-mount hash before reload + ansible.builtin.command: + argv: + - docker + - exec + - "{{ prometheus_container }}" + - sha256sum + - "{{ prometheus_runtime_rules }}" + register: prometheus_runtime_hash_before_reload + changed_when: false + + - name: Record whether exact-container recreation is required + ansible.builtin.set_fact: + prometheus_recreate_required: >- + {{ prometheus_runtime_hash_before_reload.stdout.split()[0] + != sensor_candidate_hash.stdout.split()[0] }} + changed_when: false + + - name: Recreate only Prometheus when bind-mount inode is stale + ansible.builtin.command: + argv: + - docker + - compose + - -f + - "{{ prometheus_compose_file }}" + - up + - -d + - --no-deps + - --force-recreate + - "{{ prometheus_compose_service }}" + changed_when: true + when: prometheus_recreate_required | bool + + - name: Reload Prometheus when the bind-mount inode is current + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - -X + - POST + - http://127.0.0.1:9090/-/reload + changed_when: true + when: not (prometheus_recreate_required | bool) + + - name: Wait for exact Prometheus readiness + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - http://127.0.0.1:9090/-/ready + register: prometheus_ready_after_sensor_apply + retries: 12 + delay: 5 + until: prometheus_ready_after_sensor_apply.rc == 0 + changed_when: false + + - name: Verify exact runtime hash and callback sensor contract + ansible.builtin.shell: | + set -euo pipefail + expected="{{ sensor_candidate_hash.stdout.split()[0] }}" + runtime_hash="$(docker exec '{{ prometheus_container }}' sha256sum '{{ prometheus_runtime_rules }}' | awk '{print $1}')" + test "${runtime_hash}" = "${expected}" + curl -fsS --max-time 5 http://127.0.0.1:9090/api/v1/rules | + python3 -c ' + import json, sys + payload = json.load(sys.stdin) + rows = [ + rule + for group in payload.get("data", {}).get("groups", []) + for rule in group.get("rules", []) + if rule.get("name") == "TelegramCallbackIngressUnverified" + ] + assert len(rows) == 1, rows + rule = rows[0] + assert rule.get("health") == "ok", rule + query = "".join(str(rule.get("query") or "").split()) + assert "awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds" in query + assert ">300" in query and "absent(" in query + labels = rule.get("labels") or {} + assert labels.get("component") == "openclaw", labels + assert labels.get("canonical_asset_id") == "service:openclaw:host188", labels + assert labels.get("domain_router") == "docker_container", labels + ' + args: + executable: /bin/bash + changed_when: false + + - name: Remove staged host candidate after verified apply + ansible.builtin.file: + path: "{{ sensor_candidate_host }}" + state: absent + changed_when: true + + - name: Remove staged container candidate after verified apply + ansible.builtin.command: + argv: + - docker + - exec + - -u + - "0" + - "{{ prometheus_container }}" + - rm + - -f + - "{{ sensor_candidate_container }}" + changed_when: true + + rescue: + - name: Remove staged host candidate after failure + ansible.builtin.file: + path: "{{ sensor_candidate_host }}" + state: absent + changed_when: false + failed_when: false + + - name: Remove staged container candidate after failure + ansible.builtin.command: + argv: + - docker + - exec + - -u + - "0" + - "{{ prometheus_container }}" + - rm + - -f + - "{{ sensor_candidate_container }}" + register: sensor_candidate_container_cleanup + changed_when: false + failed_when: false + + - name: Restore exact pre-apply runtime rule content + ansible.builtin.shell: | + set -euo pipefail + cat "{{ prometheus_rules_target }}.callback-sensor.bak.{{ sensor_deploy_stamp }}" > "{{ prometheus_rules_target }}" + args: + executable: /bin/bash + register: sensor_runtime_restore + changed_when: true + failed_when: false + when: sensor_apply_started | default(false) | bool + + - name: Restore exact pre-apply canonical rule content + ansible.builtin.shell: | + set -euo pipefail + cat "{{ prometheus_rules_canonical }}.callback-sensor.bak.{{ sensor_deploy_stamp }}" > "{{ prometheus_rules_canonical }}" + args: + executable: /bin/bash + register: sensor_canonical_restore + changed_when: true + failed_when: false + when: sensor_apply_started | default(false) | bool + + - name: Recreate only Prometheus for exact rollback readback + ansible.builtin.command: + argv: + - docker + - compose + - -f + - "{{ prometheus_compose_file }}" + - up + - -d + - --no-deps + - --force-recreate + - "{{ prometheus_compose_service }}" + register: sensor_rollback_recreate + changed_when: true + failed_when: false + when: sensor_apply_started | default(false) | bool + + - name: Read exact rollback readiness + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - http://127.0.0.1:9090/-/ready + register: sensor_rollback_ready + retries: 12 + delay: 5 + until: sensor_rollback_ready.rc == 0 + changed_when: false + failed_when: false + when: sensor_apply_started | default(false) | bool + + - name: Read exact rollback host and runtime hashes + ansible.builtin.shell: | + set -euo pipefail + printf '%s\n' "$(sha256sum '{{ prometheus_rules_target }}' | awk '{print $1}')" + printf '%s\n' "$(sha256sum '{{ prometheus_rules_canonical }}' | awk '{print $1}')" + docker exec "{{ prometheus_container }}" sha256sum "{{ prometheus_runtime_rules }}" | awk '{print $1}' + args: + executable: /bin/bash + register: sensor_rollback_hashes + changed_when: false + failed_when: false + when: sensor_apply_started | default(false) | bool + + - name: Require exact rollback proof after any started apply + ansible.builtin.assert: + that: + - sensor_runtime_restore.rc == 0 + - sensor_canonical_restore.rc == 0 + - sensor_rollback_recreate.rc == 0 + - sensor_rollback_ready.rc == 0 + - sensor_rollback_hashes.rc == 0 + - sensor_rollback_hashes.stdout_lines | length == 3 + - >- + sensor_rollback_hashes.stdout_lines[0] + == prometheus_rule_files_before.results[0].stat.checksum + - >- + sensor_rollback_hashes.stdout_lines[1] + == prometheus_rule_files_before.results[1].stat.checksum + - >- + sensor_rollback_hashes.stdout_lines[2] + == prometheus_rule_files_before.results[0].stat.checksum + fail_msg: telegram_callback_sensor_rollback_verification_failed + when: sensor_apply_started | default(false) | bool + + - name: Fail closed with explicit rollback terminal + ansible.builtin.fail: + msg: >- + telegram_callback_ingress_sensor_deploy_failed; + apply_started={{ sensor_apply_started | default(false) }}; + rollback_verified={{ sensor_apply_started | default(false) }} diff --git a/k8s/monitoring/alert-chain-monitor.yaml b/k8s/monitoring/alert-chain-monitor.yaml index a2e391117..4f1e00d71 100644 --- a/k8s/monitoring/alert-chain-monitor.yaml +++ b/k8s/monitoring/alert-chain-monitor.yaml @@ -58,6 +58,27 @@ spec: description: "Alertmanager 自身的 webhook 單調 delivery counter，經唯一 webhook receiver source contract 標記為 awoooi-webhook；10 分鐘至少 5 次嘗試且失敗率持續超過 10%。Telegram 與 Agent99 poll 不納入分母。" runbook_url: "https://awoooi.internal/runbooks/alert-chain" + - alert: TelegramCallbackIngressUnverified + expr: | + ( + time() - awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds > 300 + ) + or absent(awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds) + for: 2m + labels: + severity: warning + layer: docker-188 + component: openclaw + host: "188" + team: platform + auto_repair: "true" + alert_category: ai_agent + canonical_asset_id: service:openclaw:host188 + domain_router: docker_container + annotations: + summary: "Telegram callback ingress 缺少 5 分鐘內 durable receipt" + description: "固定路由 service:openclaw:host188 -> ansible:188-openclaw-callback-forwarder；僅允許 exact Host188 Compose overlay 的 check-mode、bounded apply 與獨立 verifier，禁止 Kubernetes 或跨主機 fallback。" + # ----------------------------------------------------------------- # Sentry Webhook 鏈路異常 # ----------------------------------------------------------------- diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 593257bd2..863b59adf 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -903,6 +903,27 @@ groups: summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" description: "只使用經唯一 webhook receiver source contract 標記為 awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" + - alert: TelegramCallbackIngressUnverified + expr: | + ( + time() - awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds > 300 + ) + or absent(awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds) + for: 2m + labels: + severity: warning + layer: docker-188 + component: openclaw + host: "188" + team: platform + auto_repair: "true" + alert_category: ai_agent + canonical_asset_id: service:openclaw:host188 + domain_router: docker_container + annotations: + summary: "Telegram callback ingress 缺少 5 分鐘內 durable receipt" + description: "固定路由 service:openclaw:host188 -> ansible:188-openclaw-callback-forwarder；僅允許 exact Host188 Compose overlay 的 check-mode、bounded apply 與獨立 verifier，禁止 Kubernetes 或跨主機 fallback。" + - alert: AlertChainBroken_Sentry expr: | sum(rate(awoooi_webhook_requests_total{source="sentry",status!="success"}[5m])) diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 18cee8149..60b54a18d 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -584,6 +584,27 @@ groups: summary: "Alertmanager 實際 Webhook delivery 錯誤率 > 10%" description: "只使用經唯一 webhook receiver source contract 標記為 awoooi-webhook 的單調 delivery counter；Telegram 與 Agent99 poll 不納入分母。10 分鐘至少 5 次嘗試且失敗率持續超過 10%，進入 exact-host 受控診斷、修復與獨立 E2E verifier。" + - alert: TelegramCallbackIngressUnverified + expr: | + ( + time() - awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds > 300 + ) + or absent(awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds) + for: 2m + labels: + severity: warning + layer: docker-188 + component: openclaw + host: "188" + team: platform + auto_repair: "true" + alert_category: ai_agent + canonical_asset_id: service:openclaw:host188 + domain_router: docker_container + annotations: + summary: "Telegram callback ingress 缺少 5 分鐘內 durable receipt" + description: "固定路由 service:openclaw:host188 -> ansible:188-openclaw-callback-forwarder；僅允許 exact Host188 Compose overlay 的 check-mode、bounded apply 與獨立 verifier，禁止 Kubernetes 或跨主機 fallback。" + - alert: AlertChainBroken_Sentry expr: | sum(rate(awoooi_webhook_requests_total{source="sentry",status!="success"}[5m]))