diff --git a/.gitea/workflows/agent-market-watch.yaml b/.gitea/workflows/agent-market-watch.yaml index 3430b73dd..e59021f3f 100644 --- a/.gitea/workflows/agent-market-watch.yaml +++ b/.gitea/workflows/agent-market-watch.yaml @@ -14,7 +14,6 @@ on: env: GITEA_ACTIONS_URL: http://192.168.0.110:3001/wooo/awoooi/actions - SRE_GROUP_CHAT_ID: "-1003711974679" jobs: market-watch: @@ -502,7 +501,6 @@ jobs: - name: Summarize actionable change or failure if: always() env: - TG_CHAT_ID: ${{ env.SRE_GROUP_CHAT_ID }} JOB_STATUS: ${{ job.status }} CANDIDATE_COUNT: ${{ steps.watch.outputs.candidate_count }} SOURCE_COUNT: ${{ steps.watch.outputs.source_count }} diff --git a/.gitea/workflows/cd-dev.yaml b/.gitea/workflows/cd-dev.yaml index 4ed76b4e6..4402d3a27 100644 --- a/.gitea/workflows/cd-dev.yaml +++ b/.gitea/workflows/cd-dev.yaml @@ -19,7 +19,6 @@ concurrency: env: HARBOR: 192.168.0.110:5000 HARBOR_MIRROR: 192.168.0.110:5001 - SRE_GROUP_CHAT_ID: "-1003711974679" OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318 OTEL_SERVICE_NAME: awoooi-cd-dev OTEL_RESOURCE_ATTRIBUTES: service.version=${{ github.sha }},deployment.environment=dev diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 5e610631c..6205e9158 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -32,7 +32,6 @@ concurrency: env: HARBOR: registry.wooo.work - SRE_GROUP_CHAT_ID: "-1003711974679" # Harbor Proxy Cache (指向 DockerHub 的內部 Mirror，避免拉取限額) HARBOR_MIRROR: 192.168.0.110:5001 # OTEL CI/CD 監控 (2026-03-31 #46c - 遷移到 Gitea) diff --git a/.gitea/workflows/code-review.yaml b/.gitea/workflows/code-review.yaml index 43be1c441..42f9a3990 100644 --- a/.gitea/workflows/code-review.yaml +++ b/.gitea/workflows/code-review.yaml @@ -13,7 +13,6 @@ concurrency: env: REPORT_URL: https://mo.wooo.work/code-review/ GITEA_ACTIONS_URL: http://192.168.0.110:3001/wooo/awoooi/actions - SRE_GROUP_CHAT_ID: "-1003711974679" jobs: ai-code-review: @@ -110,12 +109,10 @@ jobs: - name: Notify Code Review Start if: steps.stale.outputs.skip != 'true' env: - SRE_GROUP_CHAT_ID: ${{ env.SRE_GROUP_CHAT_ID }} SHORT_SHA: ${{ steps.ctx.outputs.short_sha }} BRANCH: ${{ steps.ctx.outputs.branch }} COMMIT_MSG: ${{ steps.ctx.outputs.commit_msg }} FILES_DISPLAY: ${{ steps.ctx.outputs.files_display }} - TG_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} run: | set -euo pipefail html_escape() { sed 's/&/\&/g; s//\>/g'; } @@ -151,9 +148,7 @@ jobs: - name: Notify Code Review Completion if: always() && steps.stale.outputs.skip != 'true' env: - SRE_GROUP_CHAT_ID: ${{ env.SRE_GROUP_CHAT_ID }} SHORT_SHA: ${{ steps.ctx.outputs.short_sha }} - TG_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} run: | set -euo pipefail REPORT=/tmp/code-review-report.json diff --git a/.gitea/workflows/deploy-alerts.yaml b/.gitea/workflows/deploy-alerts.yaml index a9904cfe9..0d1e366e1 100644 --- a/.gitea/workflows/deploy-alerts.yaml +++ b/.gitea/workflows/deploy-alerts.yaml @@ -12,9 +12,6 @@ on: # push path filtering 不得在 runner 搬遷 / 硬限流前重開。 workflow_dispatch: -env: - SRE_GROUP_CHAT_ID: "-1003711974679" - jobs: deploy-alerts: name: "Deploy Prometheus Alert Rules" diff --git a/.gitea/workflows/e2e-health.yaml b/.gitea/workflows/e2e-health.yaml index abb8d101a..1515cb75a 100644 --- a/.gitea/workflows/e2e-health.yaml +++ b/.gitea/workflows/e2e-health.yaml @@ -19,7 +19,6 @@ env: OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318 OTEL_SERVICE_NAME: awoooi-e2e OTEL_RESOURCE_ATTRIBUTES: deployment.environment=production - SRE_GROUP_CHAT_ID: "-1003711974679" jobs: e2e-health: diff --git a/.gitea/workflows/run-migration.yml b/.gitea/workflows/run-migration.yml index 872bffc53..cc978cf55 100644 --- a/.gitea/workflows/run-migration.yml +++ b/.gitea/workflows/run-migration.yml @@ -17,9 +17,6 @@ on: # 不得由 push 自動觸發；恢復前必須完成 runner 搬遷 / 硬限流與 migration dry-run gate。 workflow_dispatch: -env: - SRE_GROUP_CHAT_ID: "-1003711974679" - jobs: migrate: runs-on: awoooi-non110-ubuntu diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index 25494fd6e..1f3ff1ad0 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -39,6 +39,9 @@ from src.models.approval import ( from src.services.anomaly_counter import get_anomaly_counter from src.services.approval_db import get_approval_service from src.services.channel_hub import record_external_alert_event +from src.services.notification_matrix import ( + get_trusted_alert_canonical_route_context, +) from src.services.openclaw_http_service import get_openclaw_http_service from src.services.sentry_service import get_sentry_service @@ -648,6 +651,25 @@ async def send_sentry_telegram_alert( title = error_context.get("title", "Unknown Error") culprit = error_context.get("culprit", "unknown") level = error_context.get("level", "error") + route_labels = { + key: value + for key in ( + "awoooi_product_id", + "awoooi_signal_family", + "awoooi_route_severity", + ) + if ( + value := _sentry_event_tag( + {"tags": error_context.get("tags", [])}, + key, + ) + ) + } + telegram_route_context = get_trusted_alert_canonical_route_context( + route_labels, + source="sentry_webhook", + source_severity=str(level), + ) # 發送 Sentry 告警卡片 (含 Y/n 按鈕) # TODO(2026-04-05): Sentry 路徑無 incident_id，待 Sentry→Incident 關聯後補傳 @@ -663,7 +685,10 @@ async def send_sentry_telegram_alert( anomaly_frequency=anomaly_frequency, # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=analysis.analyzed_by if analysis else "", - notification_type="TYPE-3", + notification_type="UNKNOWN", + product_id=(telegram_route_context or {}).get("product_id"), + signal_family=(telegram_route_context or {}).get("signal_family"), + route_severity=(telegram_route_context or {}).get("severity"), ) if _telegram_send_delivery_succeeded(delivery_result): diff --git a/apps/api/src/api/v1/signoz_webhook.py b/apps/api/src/api/v1/signoz_webhook.py index cb8394444..21d6cee4c 100644 --- a/apps/api/src/api/v1/signoz_webhook.py +++ b/apps/api/src/api/v1/signoz_webhook.py @@ -552,6 +552,10 @@ async def send_signoz_telegram( service_name = labels.get("service_name", labels.get("service", "unknown")) summary = annotations.get("summary", f"SignOz Alert: {alert_name}") description = annotations.get("description", "") + # This public SignOz receiver does not yet authenticate the monitoring + # source. It may persist/normalize the signal, but Telegram egress is + # fail-closed until an authenticated ingress supplies route identity. + telegram_route_context: dict[str, str] | None = None delivery_result = await telegram.send_approval_card( approval_id=approval_id, @@ -572,7 +576,10 @@ async def send_signoz_telegram( # 2026-04-02 ogt: 修復 ai_provider 未傳遞 → Telegram 顯示「AI 仲裁判定」而非具體模型名稱 ai_provider=ai_provider if ai_provider != "none" else "", incident_id=incident_id, - notification_type="TYPE-3", + notification_type="UNKNOWN", + product_id=(telegram_route_context or {}).get("product_id"), + signal_family=(telegram_route_context or {}).get("signal_family"), + route_severity=(telegram_route_context or {}).get("severity"), ) if _telegram_send_delivery_succeeded(delivery_result): diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 0c8bb69e8..3089895fe 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -90,6 +90,10 @@ from src.services.incident_service import ( get_incident_service, ) +from src.services.notification_matrix import ( + get_trusted_alert_canonical_route_context, +) + # Phase 5: OpenClaw AI Engine from src.services.openclaw import get_openclaw from src.services.playbook_match_resolver import resolve_playbook_id_for_alert @@ -994,6 +998,7 @@ async def _push_to_telegram_background( repair_candidate_promotion_summary: str = "", repair_candidate_work_item_href: str = "", repair_candidate_work_item_id: str = "", + telegram_route_context: dict[str, str] | None = None, ) -> None: """ 背景任務: 推送待簽核卡片到 Telegram (v7.0 含 SignOz 整合) @@ -1116,6 +1121,9 @@ async def _push_to_telegram_background( repair_candidate_promotion_summary=repair_candidate_promotion_summary, repair_candidate_work_item_href=repair_candidate_work_item_href, repair_candidate_work_item_id=repair_candidate_work_item_id, + product_id=(telegram_route_context or {}).get("product_id"), + signal_family=(telegram_route_context or {}).get("signal_family"), + route_severity=(telegram_route_context or {}).get("severity"), ) if not _telegram_send_delivery_succeeded(delivery_result): @@ -1934,6 +1942,11 @@ async def receive_alert( elif isinstance(first_suggestion, dict): auto_tuning_cmd = first_suggestion.get("kubectl_or_config", "") + telegram_route_context = get_trusted_alert_canonical_route_context( + alert.labels, + source="hmac_alert_webhook", + source_severity=alert.severity, + ) background_tasks.add_task( _push_to_telegram_background, approval_id=str(approval.id), @@ -1961,6 +1974,8 @@ async def receive_alert( ai_provider=ai_provider, # 2026-04-08 ogt: 補傳 incident_id 以啟用詳情/重診/歷史按鈕 incident_id=_cs1_incident_id, + notification_type="UNKNOWN", + telegram_route_context=telegram_route_context, # /alerts 路徑沒有 notification_type（非 Alertmanager 路徑），不需 TYPE-4D routing # ADR-075: alert_type → category 對應，啟用動態按鈕 alert_category={ @@ -2074,6 +2089,28 @@ def is_internal_ip(client_ip: str) -> bool: return False +def _trusted_internal_alertmanager_source(request: Request) -> tuple[bool, str]: + """Accept Alertmanager only when the peer and full proxy chain are private.""" + + peer_ip = request.client.host if request.client else "unknown" + if not is_internal_ip(peer_ip): + return False, peer_ip + + forwarded_chain = [ + value.strip() + for value in request.headers.get("X-Forwarded-For", "").split(",") + if value.strip() + ] + if not forwarded_chain: + return True, peer_ip + if not all(is_internal_ip(value) for value in forwarded_chain): + first_external = next( + value for value in forwarded_chain if not is_internal_ip(value) + ) + return False, first_external + return True, forwarded_chain[0] + + async def _process_new_alert_background( alert_context: dict, alert_id: str, @@ -2104,6 +2141,11 @@ async def _process_new_alert_background( "fingerprint": fingerprint, "alert_id": alert_id, } + telegram_route_context = get_trusted_alert_canonical_route_context( + alert_labels, + source="prometheus_alertmanager", + source_severity=severity, + ) rule_response = match_rule(alert_context) should_bypass_llm = _should_use_alertmanager_rule_first(rule_response, alert_category) @@ -2349,6 +2391,7 @@ async def _process_new_alert_background( notification_type=notification_type, alert_category=alert_category, fingerprint=fingerprint, + telegram_route_context=telegram_route_context, ) record_alert_chain_success("alertmanager") @@ -2599,6 +2642,7 @@ async def _process_new_alert_background( notification_type=notification_type, alert_category=alert_category, fingerprint=fingerprint, + telegram_route_context=telegram_route_context, ) record_alert_chain_success("alertmanager") @@ -2988,6 +3032,7 @@ async def _process_new_alert_background( if isinstance(_approval_metadata_cs4.get("repair_candidate_draft_package"), dict) else "" ), + telegram_route_context=telegram_route_context, ) except Exception as e: @@ -3042,12 +3087,10 @@ async def alertmanager_webhook( ) # 取得客戶端 IP - client_ip = request.client.host if request.client else "unknown" - forwarded_for = request.headers.get("X-Forwarded-For", "").split(",")[0].strip() - actual_ip = forwarded_for or client_ip + internal_source, actual_ip = _trusted_internal_alertmanager_source(request) # 內網檢查 - if not is_internal_ip(actual_ip): + if not internal_source: logger.warning( "alertmanager_external_rejected", client_ip=actual_ip, diff --git a/apps/api/src/services/notification_matrix.py b/apps/api/src/services/notification_matrix.py index 921e23db2..b6ca1455b 100644 --- a/apps/api/src/services/notification_matrix.py +++ b/apps/api/src/services/notification_matrix.py @@ -11,6 +11,7 @@ from __future__ import annotations import json import re +from collections.abc import Mapping from dataclasses import dataclass from enum import Enum from pathlib import Path @@ -89,6 +90,23 @@ _SHARED_SRE_SIGNAL_FAMILIES = frozenset( "incident_lifecycle", } ) +_TRUSTED_ALERT_ROUTE_SOURCES = frozenset( + { + "hmac_alert_webhook", + "prometheus_alertmanager", + "sentry_webhook", + } +) +_ROUTE_SEVERITY_BY_SOURCE_VALUE = { + "critical": "P0", + "fatal": "P0", + "error": "P1", + "high": "P1", + "warning": "P2", + "medium": "P2", + "info": "P3", + "low": "P3", +} _REQUIRED_ROUTE_FIELDS = frozenset( { "route_id", @@ -230,6 +248,44 @@ def get_legacy_canonical_route_context( } +def get_trusted_alert_canonical_route_context( + labels: Mapping[str, object] | None, + *, + source: str, + source_severity: str, +) -> dict[str, str] | None: + """Extract an explicit product route from controlled, namespaced labels. + + Product monitoring must never inherit the legacy TYPE-3 AWOOOI route. A + recognized ingress may opt into canonical routing only with the committed + ``awoooi_*`` label triplet. Missing or malformed labels return ``None`` so + the caller can emit a durable blocked/no-egress receipt. + """ + + if source not in _TRUSTED_ALERT_ROUTE_SOURCES or not isinstance(labels, Mapping): + return None + + product_id = str(labels.get("awoooi_product_id") or "").strip() + signal_family = str(labels.get("awoooi_signal_family") or "").strip() + if not product_id or not signal_family: + return None + + explicit_severity = str(labels.get("awoooi_route_severity") or "").strip().upper() + route_severity = explicit_severity or _ROUTE_SEVERITY_BY_SOURCE_VALUE.get( + str(source_severity or "").strip().lower(), + "", + ) + if route_severity not in _ALLOWED_SEVERITIES: + return None + + return { + "product_id": product_id, + "signal_family": signal_family, + "severity": route_severity, + "route_source": source, + } + + def resolve_chat_ids( notification_type: str, dm_chat_id: str, diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 25bc95342..4d3815a79 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -7163,6 +7163,9 @@ class TelegramGateway: repair_candidate_promotion_summary: str = "", repair_candidate_work_item_href: str = "", repair_candidate_work_item_id: str = "", + product_id: str | None = None, + signal_family: str | None = None, + route_severity: str | None = None, ) -> dict: """ 推送待簽核卡片到 Telegram (v7.0 含 SignOz 整合) @@ -7184,6 +7187,8 @@ class TelegramGateway: signoz_trace_url: 動態時間參數的 Trace URL auto_tuning_command: kubectl 調優指令 anomaly_frequency: 異常頻率統計 (ADR-037) + product_id/signal_family/route_severity: namespaced canonical + product route. When present it supersedes legacy TYPE-*. Returns: dict: Telegram API 回應 @@ -7306,13 +7311,20 @@ class TelegramGateway: # Unknown/missing notification_type is intentionally no-egress. The # central sender resolves only allowlisted legacy types via the # canonical registry and binds the actual runtime destination there. - payload = { + payload: dict[str, object] = { "text": text, "parse_mode": "HTML", "reply_markup": keyboard, "disable_web_page_preview": True, # 避免 SignOz URL 預覽 - _LEGACY_NOTIFICATION_TYPE_KEY: notification_type or "UNKNOWN", } + if any((product_id, signal_family, route_severity)): + payload[_CANONICAL_ROUTE_CONTEXT_KEY] = { + "product_id": product_id or "", + "signal_family": signal_family or "", + "severity": route_severity or "", + } + else: + payload[_LEGACY_NOTIFICATION_TYPE_KEY] = notification_type or "UNKNOWN" logger.info( "telegram_approval_card_sending", diff --git a/apps/api/tests/test_alertmanager_webhook_metrics.py b/apps/api/tests/test_alertmanager_webhook_metrics.py index 3da038a96..f9817361d 100644 --- a/apps/api/tests/test_alertmanager_webhook_metrics.py +++ b/apps/api/tests/test_alertmanager_webhook_metrics.py @@ -75,3 +75,38 @@ async def test_alertmanager_external_reject_records_error_metric() -> None: 'awoooi_webhook_requests_total{source="alertmanager",status="error"}' in _metrics_text() ) + + +@pytest.mark.asyncio +async def test_alertmanager_rejects_external_forwarded_source_behind_private_peer() -> None: + before = _webhook_count("error") + + with pytest.raises(HTTPException) as exc_info: + await alertmanager_webhook( + _request( + client_host="10.42.0.10", + headers=[(b"x-forwarded-for", b"8.8.8.8, 10.42.0.1")], + ), + AlertmanagerPayload(status="resolved", alerts=[]), + BackgroundTasks(), + ) + + assert exc_info.value.status_code == 403 + assert _webhook_count("error") == before + 1 + + +@pytest.mark.asyncio +async def test_alertmanager_accepts_private_forwarded_chain_from_private_peer() -> None: + before = _webhook_count("success") + + response = await alertmanager_webhook( + _request( + client_host="10.42.0.10", + headers=[(b"x-forwarded-for", b"192.168.0.110, 10.42.0.1")], + ), + AlertmanagerPayload(status="resolved", alerts=[]), + BackgroundTasks(), + ) + + assert response.success is True + assert _webhook_count("success") == before + 1 diff --git a/apps/api/tests/test_telegram_canonical_routing_registry.py b/apps/api/tests/test_telegram_canonical_routing_registry.py index 75f7e36c8..88649644b 100644 --- a/apps/api/tests/test_telegram_canonical_routing_registry.py +++ b/apps/api/tests/test_telegram_canonical_routing_registry.py @@ -2,11 +2,14 @@ from __future__ import annotations import copy import json +import re from pathlib import Path import pytest +import yaml from src.services.notification_matrix import ( + get_trusted_alert_canonical_route_context, load_canonical_telegram_routing_registry, resolve_canonical_telegram_route, ) @@ -24,6 +27,158 @@ EXPECTED_PRODUCTS = { "vibework", "vtuber", } +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def test_namespaced_product_ingress_route_is_explicit_and_policy_blocked() -> None: + context = get_trusted_alert_canonical_route_context( + { + "awoooi_product_id": "momo-pro", + "awoooi_signal_family": "crawler_and_data_freshness", + "awoooi_route_severity": "P2", + }, + source="prometheus_alertmanager", + source_severity="warning", + ) + + assert context == { + "product_id": "momo-pro", + "signal_family": "crawler_and_data_freshness", + "severity": "P2", + "route_source": "prometheus_alertmanager", + } + assert resolve_canonical_telegram_route(**{ + key: context[key] for key in ("product_id", "signal_family", "severity") + })["decision"] == "block" + + +@pytest.mark.parametrize( + ("labels", "source", "severity"), + [ + ({}, "prometheus_alertmanager", "critical"), + ( + { + "awoooi_product_id": "momo-pro", + "awoooi_signal_family": "crawler_and_data_freshness", + }, + "untrusted_ingress", + "critical", + ), + ( + { + "awoooi_product_id": "awoooi", + "awoooi_signal_family": "incident_lifecycle", + "awoooi_route_severity": "P1", + }, + "signoz_webhook", + "critical", + ), + ( + { + "awoooi_product_id": "momo-pro", + "awoooi_signal_family": "crawler_and_data_freshness", + "awoooi_route_severity": "SEV-1", + }, + "prometheus_alertmanager", + "critical", + ), + ], +) +def test_product_ingress_route_fails_closed_without_complete_trusted_context( + labels: dict[str, str], source: str, severity: str +) -> None: + assert get_trusted_alert_canonical_route_context( + labels, + source=source, + source_severity=severity, + ) is None + + +@pytest.mark.parametrize("rule_file", ["alerts-unified.yml", "alerts.yml"]) +def test_product_alert_rules_carry_canonical_route_labels(rule_file: str) -> None: + payload = yaml.safe_load((REPO_ROOT / "ops/monitoring" / rule_file).read_text()) + rules = { + rule["alert"]: rule["labels"] + for group in payload["groups"] + for rule in group.get("rules", []) + if "alert" in rule + } + expected = { + "OpenClawDown": ("clawbot-openclaw", "raw_monitoring_alert", "P1"), + "MoWoooWorkDown": ("momo-pro", "scheduler_and_sales_pipeline", "P1"), + "TsenyangWebsiteDown": ( + "tsenyang-website", + "public_site_and_agent_action_health", + "P1", + ), + "StockWoooWorkDown": ( + "stockplatform-v2", + "market_data_and_recommendation_freshness", + "P1", + ), + "BitanWoooWorkDown": ( + "bitan-pharmacy", + "container_and_product_health", + "P1", + ), + "MomoScraperSuccessLow": ( + "momo-pro", + "crawler_and_data_freshness", + "P2", + ), + } + for alertname, route in expected.items(): + labels = rules[alertname] + assert ( + labels["awoooi_product_id"], + labels["awoooi_signal_family"], + labels["awoooi_route_severity"], + ) == route + + +def test_specialized_openclaw_rule_cannot_fall_back_to_shared_sre() -> None: + payload = yaml.safe_load( + (REPO_ROOT / "k8s/monitoring/k3s-alerts-supplemental.yaml").read_text() + ) + rule = next( + rule + for group in payload["groups"] + for rule in group.get("rules", []) + if rule.get("alert") == "OpenClawDown" + ) + labels = rule["labels"] + assert { + key: labels[key] + for key in ( + "awoooi_product_id", + "awoooi_signal_family", + "awoooi_route_severity", + ) + } == { + "awoooi_product_id": "clawbot-openclaw", + "awoooi_signal_family": "raw_monitoring_alert", + "awoooi_route_severity": "P2", + } + + +def test_active_source_has_no_raw_numeric_telegram_destination_defaults() -> None: + candidate_paths = [ + REPO_ROOT / "docker-compose.yml", + REPO_ROOT / "k8s/awoooi-prod/04-configmap.yaml", + REPO_ROOT / "scripts/ops/deploy-docker-health-monitor.sh", + REPO_ROOT / "scripts/ops/docker-health-monitor.sh", + *(REPO_ROOT / ".gitea/workflows").glob("*.y*ml"), + ] + raw_numeric_identifier = re.compile(r"-?\d{8,}") + violations: list[str] = [] + for path in candidate_paths: + for line_number, line in enumerate(path.read_text().splitlines(), 1): + if ( + any(key in line for key in ("CHAT_ID", "USER_WHITELIST")) + and raw_numeric_identifier.search(line) + ): + violations.append(f"{path.relative_to(REPO_ROOT)}:{line_number}") + assert violations == [] def test_registry_covers_all_11_products_and_reconciles_rollups() -> None: diff --git a/apps/api/tests/test_telegram_canonical_sender_gate.py b/apps/api/tests/test_telegram_canonical_sender_gate.py index 02cc352f6..42658e58b 100644 --- a/apps/api/tests/test_telegram_canonical_sender_gate.py +++ b/apps/api/tests/test_telegram_canonical_sender_gate.py @@ -847,3 +847,30 @@ async def test_allowed_type3_approval_persists_runtime_chat_without_public_id( receipt = result["_awoooi_canonical_route_receipt"] assert receipt["destination_alias"] == "awoooi_sre_war_room" assert "chat_id" not in receipt + + +@pytest.mark.asyncio +async def test_explicit_blocked_product_route_supersedes_legacy_type3( + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, client = _prepared_gateway(monkeypatch) + _patch_approval_dependencies(monkeypatch, gateway) + + result = await gateway.send_approval_card( + approval_id="APR-PRODUCT", + risk_level="medium", + resource_name="momo-crawler", + root_cause="freshness degraded", + suggested_action="inspect evidence", + notification_type="TYPE-3", + product_id="momo-pro", + signal_family="crawler_and_data_freshness", + route_severity="P2", + ) + + assert result["_awooop_delivery_status"] == "blocked_no_egress" + assert result["_awooop_provider_send_performed"] is False + assert client.posts == [] + receipt = result["_awoooi_canonical_route_receipt"] + assert receipt["product_id"] == "momo-pro" + assert receipt["signal_family"] == "crawler_and_data_freshness" diff --git a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py index b92866146..ad6bfe267 100644 --- a/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py +++ b/apps/api/tests/test_telegram_delivery_truth_gateway_v2.py @@ -494,7 +494,7 @@ async def test_background_no_send_never_confirms_or_writes_sent( @pytest.mark.asyncio @pytest.mark.parametrize("module_name", ["sentry", "signoz"]) -async def test_monitoring_webhooks_use_type3_and_do_not_false_log_sent( +async def test_unclassified_monitoring_webhooks_fail_closed_and_do_not_false_log_sent( monkeypatch: pytest.MonkeyPatch, module_name: str, ) -> None: @@ -533,7 +533,10 @@ async def test_monitoring_webhooks_use_type3_and_do_not_false_log_sent( sent_event = "signoz_telegram_sent" no_send_event = "signoz_telegram_no_send" - assert captured[0]["notification_type"] == "TYPE-3" + assert captured[0]["notification_type"] == "UNKNOWN" + assert captured[0]["product_id"] is None + assert captured[0]["signal_family"] is None + assert captured[0]["route_severity"] is None assert ("warning", no_send_event) in logger.events assert ("info", sent_event) not in logger.events diff --git a/docker-compose.yml b/docker-compose.yml index b3b9aaf62..932261627 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,8 +76,8 @@ services: CORS_ORIGINS: '["http://localhost:3000","http://localhost:3001","http://localhost:3002","http://localhost:3003","http://web:3000"]' # Telegram Gateway (Phase 5.5) OPENCLAW_TG_BOT_TOKEN: "${OPENCLAW_TG_BOT_TOKEN:-}" - OPENCLAW_TG_CHAT_ID: "5619078117" - OPENCLAW_TG_USER_WHITELIST: "5619078117" + OPENCLAW_TG_CHAT_ID: "${OPENCLAW_TG_CHAT_ID:-}" + OPENCLAW_TG_USER_WHITELIST: "${OPENCLAW_TG_USER_WHITELIST:-}" # External Services (使用 host.docker.internal 存取宿主機服務) OLLAMA_URL: http://host.docker.internal:11434 OPENCLAW_URL: http://host.docker.internal:8088 diff --git a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md index 53b3a781b..22200319a 100644 --- a/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md +++ b/docs/security/TELEGRAM-CANONICAL-ROUTING-INVENTORY.md @@ -12,6 +12,40 @@ - 未知 product/signal/severity/destination 一律 no-egress；不再沿用 ADR-093 的 unknown-to-shared-group fallback。 - `AwoooI SRE 戰情室` 只收 shared infrastructure、security、DR 與 P0/P1 incident lifecycle。raw P2/P3、marketing、成功 heartbeat、未驗證 recommendation 和 bot 對話均不得進入。 +## 監控與告警來源總帳 + +| source truth | 數量 | 角色與目前判定 | +|---|---:|---| +| `ops/monitoring/alerts-unified.yml` | 23 groups / 145 alert declarations | production primary Prometheus rule source；產品訊號必須帶 namespaced canonical route labels | +| `ops/monitoring/alerts.yml` | 81 declarations | legacy duplicate source，不加入 production primary 總數；保留相同產品 labels 防止誤復用 | +| 11 個 specialized Prometheus / SignOz rule files | 112 declarations | K3s、DB、flywheel、MinIO/Kali、NVIDIA、Ollama、SLO、agent latency、SignOz 等專項訊號 | +| primary + specialized 合計 | 257 declarations / 226 unique alert names | 29 個 alert name 在兩個以上來源重複，extra declarations 31；重複不是多送授權，仍由 grouping/fingerprint/route policy 收斂 | +| AWOOOI API `alert_rules` | 29 matching/action rules | 用於分類與修復候選，不是新的 signal generator，不重複計入 257 | +| Telegram egress scanner | 805 files / 50 normalized gateway callsites | active direct Bot API 0；7 個 direct calls 全在 GitHub-frozen legacy source，禁止執行 | + +目前可明確歸戶的產品告警如下；未列出的 shared infrastructure/security/DR/K3s/DB/host 規則屬 `awoooi`，但只有 P0/P1 allowlisted lifecycle 能進 SRE 群： + +| product_id | 明確 signal / alert | canonical signal family | 現行 egress | +|---|---|---|---| +| `2026fifa` | repository 中沒有產品專屬 Prometheus rule | `data_freshness_and_recommendation` | blocked | +| `agent-bounty-protocol` | registry 有 A2A、traffic security、treasury/onboarding 3 lanes；沒有專屬 Prometheus rule | 對應 3 個 registry families | blocked，缺 durable receipt | +| `awooogo` | 無產品專屬 rule | `*` | disabled | +| `awoooi` | 145 primary + 112 specialized declarations 的 shared/core owner | shared infrastructure / security / DR / incident lifecycle | 僅 P0/P1 allowlisted route allowed | +| `bitan-pharmacy` | `BitanWoooWorkDown` | `container_and_product_health` | blocked | +| `clawbot-openclaw` | `OpenClawDown` | `raw_monitoring_alert` | blocked；不得借用 AWOOOI TYPE-3 | +| `momo-pro` | `MoWoooWorkDown`、`MomoScraperSuccessLow` | scheduler/sales、crawler/data freshness | blocked；configured destination 仍 `chat_not_found` | +| `stockplatform-v2` | `StockWoooWorkDown` | `market_data_and_recommendation_freshness` | blocked | +| `tsenyang-website` | `TsenyangWebsiteDown` | `public_site_and_agent_action_health` | blocked | +| `vibework` | 無專屬 rule；`VibeAIAgent` 不等於 standalone VibeWork | `*` | not implemented | +| `vtuber` | 無產品專屬 rule | `*` | disabled | + +## Source-side route corrections + +- Alertmanager 產品規則現在帶 `awoooi_product_id`、`awoooi_signal_family`、`awoooi_route_severity`；gateway 以這組明確 context 覆蓋 legacy `TYPE-3`，所以 blocked product 不會再落到 shared SRE。 +- SignOz public receiver 尚無 source authentication，因此即使 payload 自稱 canonical labels 也固定 no-egress；Sentry（signature verified）與 HMAC generic webhook 缺完整 namespaced context 時一律 `UNKNOWN / blocked_no_egress`。Alertmanager 另要求 direct peer 與完整 forwarded chain 都是 private IP，不再信任單一可偽造的 forwarded value。 +- `docker-compose`、production ConfigMap、docker-health scripts 與 7 個 Gitea workflows 的 11 個 raw destination defaults 已移除；runtime 只可由既有 secret/env alias 注入，缺值即 fail closed。白名單也不再內嵌 numeric default。 +- 本次只改 source policy，沒有讀 token、呼叫 Bot API、改 Telegram 群組、送測試訊息或偽造 delivery receipt。 + ## 11 產品路由清冊 | 產品 | 目前 proven route | 建議 canonical route | blocked unknown / egress status | source risk | receipt strength | diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index e18036039..20a940d70 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -107,7 +107,6 @@ data: # 2026-04-03 ogt: SRE 戰情室群組三頭政治 (Triumvirate ADR-053) # OPENCLAW_BOT_TOKEN / NEMOTRON_BOT_TOKEN 在 Secrets 中配置 # ============================================================================ - SRE_GROUP_CHAT_ID: "-1003711974679" # 2026-07-10 Codex: Agent99 direct SRE relay. Alertmanager still lands in # AWOOI API first; the sanitized bridge then POSTs to 99 so Agent99 can queue # allowlisted recovery and write evidence/KM. Optional relay token stays in diff --git a/k8s/monitoring/k3s-alerts-supplemental.yaml b/k8s/monitoring/k3s-alerts-supplemental.yaml index b621c6194..8c860e39b 100644 --- a/k8s/monitoring/k3s-alerts-supplemental.yaml +++ b/k8s/monitoring/k3s-alerts-supplemental.yaml @@ -39,6 +39,9 @@ groups: for: 2m labels: severity: warning + awoooi_product_id: clawbot-openclaw + awoooi_signal_family: raw_monitoring_alert + awoooi_route_severity: P2 team: ops component: openclaw annotations: diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 22b804a13..20b4b368c 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -703,6 +703,9 @@ groups: for: 2m labels: severity: critical + awoooi_product_id: clawbot-openclaw + awoooi_signal_family: raw_monitoring_alert + awoooi_route_severity: P1 layer: docker-188 component: openclaw host: "188" @@ -1133,6 +1136,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: momo-pro + awoooi_signal_family: scheduler_and_sales_pipeline + awoooi_route_severity: P1 layer: external component: momo-pro-system host: "188" @@ -1148,6 +1154,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: tsenyang-website + awoooi_signal_family: public_site_and_agent_action_health + awoooi_route_severity: P1 layer: external component: tsenyang-website host: "188" @@ -1162,6 +1171,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: stockplatform-v2 + awoooi_signal_family: market_data_and_recommendation_freshness + awoooi_route_severity: P1 layer: external component: stock-platform host: "110" @@ -1176,6 +1188,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: bitan-pharmacy + awoooi_signal_family: container_and_product_health + awoooi_route_severity: P1 layer: external component: bitan-app host: "188" @@ -1289,6 +1304,9 @@ groups: for: 10m labels: severity: warning + awoooi_product_id: momo-pro + awoooi_signal_family: crawler_and_data_freshness + awoooi_route_severity: P2 layer: docker-110 auto_repair: "false" alert_category: business diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 9dccc313f..3e21b64f2 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -387,6 +387,9 @@ groups: for: 2m labels: severity: critical + awoooi_product_id: clawbot-openclaw + awoooi_signal_family: raw_monitoring_alert + awoooi_route_severity: P1 layer: docker-188 component: openclaw host: "188" @@ -817,6 +820,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: momo-pro + awoooi_signal_family: scheduler_and_sales_pipeline + awoooi_route_severity: P1 layer: external component: momo-pro-system host: "188" @@ -832,6 +838,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: tsenyang-website + awoooi_signal_family: public_site_and_agent_action_health + awoooi_route_severity: P1 layer: external component: tsenyang-website host: "188" @@ -846,6 +855,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: stockplatform-v2 + awoooi_signal_family: market_data_and_recommendation_freshness + awoooi_route_severity: P1 layer: external component: stock-platform host: "110" @@ -860,6 +872,9 @@ groups: for: 3m labels: severity: critical + awoooi_product_id: bitan-pharmacy + awoooi_signal_family: container_and_product_health + awoooi_route_severity: P1 layer: external component: bitan-app host: "188" @@ -937,6 +952,9 @@ groups: for: 10m labels: severity: warning + awoooi_product_id: momo-pro + awoooi_signal_family: crawler_and_data_freshness + awoooi_route_severity: P2 layer: docker-110 auto_repair: "false" alert_category: business diff --git a/scripts/ops/deploy-docker-health-monitor.sh b/scripts/ops/deploy-docker-health-monitor.sh index 76c7df04e..e3c2fe902 100755 --- a/scripts/ops/deploy-docker-health-monitor.sh +++ b/scripts/ops/deploy-docker-health-monitor.sh @@ -86,7 +86,7 @@ deploy_to_host() { AWOOOI_API_URL=https://awoooi.wooo.work TELEGRAM_BOT_TOKEN=CHANGE_ME -SRE_GROUP_CHAT_ID=-1003711974679 +SRE_GROUP_CHAT_ID=CHANGE_ME SEND_COOLDOWN_SECONDS=300 ACTION_COOLDOWN_SECONDS=300 NOTIFY_COOLDOWN_SECONDS=1800 diff --git a/scripts/ops/docker-health-monitor.sh b/scripts/ops/docker-health-monitor.sh index db489d289..040712a9e 100755 --- a/scripts/ops/docker-health-monitor.sh +++ b/scripts/ops/docker-health-monitor.sh @@ -23,7 +23,7 @@ if [[ -f "$SECRETS_FILE" ]]; then fi : "${AWOOOI_API_URL:=https://awoooi.wooo.work}" -: "${SRE_GROUP_CHAT_ID:=-1003711974679}" +: "${SRE_GROUP_CHAT_ID:=}" : "${LOG_FILE:=/var/log/docker-health-monitor.log}" : "${SEND_COOLDOWN_SECONDS:=300}" : "${ACTION_COOLDOWN_SECONDS:=${SEND_COOLDOWN_SECONDS}}"