diff --git a/.gitea/workflows/e2e-health.yaml b/.gitea/workflows/e2e-health.yaml index 90472aa83..cd85bb886 100644 --- a/.gitea/workflows/e2e-health.yaml +++ b/.gitea/workflows/e2e-health.yaml @@ -63,6 +63,7 @@ jobs: --api-url https://awoooi.wooo.work \ --metrics-api-url http://192.168.0.125:32334 \ --source-provider-heartbeat \ + --source-provider-upstream-canary \ --json - name: Notify Telegram on Failure diff --git a/apps/api/src/api/v1/sentry_webhook.py b/apps/api/src/api/v1/sentry_webhook.py index 1a0056b32..d1a462e6d 100644 --- a/apps/api/src/api/v1/sentry_webhook.py +++ b/apps/api/src/api/v1/sentry_webhook.py @@ -14,12 +14,15 @@ AWOOOI API - Sentry Webhook Handler 🔴 HARD RULE: 時間顯示使用 Asia/Taipei (UTC+8) """ +import json import uuid +from typing import Any import structlog from fastapi import APIRouter, BackgroundTasks, HTTPException, Request from pydantic import BaseModel +from src.core.awooop_operator_auth import authenticate_awooop_operator_headers from src.core.circuit_breaker import get_openclaw_guard from src.core.metrics import ( record_alert_chain_failure, @@ -38,6 +41,7 @@ from src.services.approval_db import get_approval_service from src.services.channel_hub import record_external_alert_event from src.services.openclaw_http_service import get_openclaw_http_service from src.services.sentry_service import get_sentry_service + # 2026-04-27 P3.1-T2 by Claude — Tier-2 三服務感知強化：補 SentryWebhookService 簽章驗證 from src.services.sentry_webhook_service import ( SentrySignatureError, @@ -88,6 +92,114 @@ async def sentry_webhook_health() -> dict: return {"status": "ok", "webhook": "sentry"} +def _sentry_event_tag(event_data: dict[str, Any], key: str) -> str | None: + tags = event_data.get("tags") or [] + for tag in tags: + if isinstance(tag, list | tuple) and len(tag) >= 2 and str(tag[0]) == key: + return str(tag[1]) + if isinstance(tag, dict) and str(tag.get("key")) == key: + value = tag.get("value") + return str(value) if value is not None else None + return None + + +def _is_sentry_upstream_canary(payload: dict[str, Any]) -> bool: + data = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(data, dict) or payload.get("action") != "triggered": + return False + issue_data = data.get("issue") if isinstance(data.get("issue"), dict) else {} + event_data = data.get("event") if isinstance(data.get("event"), dict) else {} + issue_id = str(issue_data.get("id") or "") + short_id = str(issue_data.get("shortId") or "") + title = str(issue_data.get("title") or "") + return ( + issue_id.startswith("awoooi-canary-") + or short_id.upper().startswith("AWOOOI-CANARY") + or title == "AwoooPSourceProviderCanary" + or (_sentry_event_tag(event_data, "awoooi_canary") or "").lower() == "true" + ) + + +async def _record_sentry_upstream_canary( + payload: dict[str, Any], + request: Request, +) -> dict[str, Any]: + operator = authenticate_awooop_operator_headers( + request.headers.get("x-awooop-operator-id"), + request.headers.get("x-awooop-operator-key"), + ) + data = payload.get("data") if isinstance(payload.get("data"), dict) else {} + issue_data = data.get("issue") if isinstance(data.get("issue"), dict) else {} + event_data = data.get("event") if isinstance(data.get("event"), dict) else {} + issue_id = str( + issue_data.get("id") + or issue_data.get("shortId") + or _sentry_event_tag(event_data, "run_ref") + or "awoooi-canary-unknown" + ) + source_url = ( + issue_data.get("permalink") + or issue_data.get("web_url") + or issue_data.get("url") + ) + event_uuid = await record_external_alert_event( + project_id="awoooi", + provider="sentry", + event_id=issue_id, + stage="upstream_canary", + title=str(issue_data.get("title") or "AwoooPSourceProviderCanary"), + severity=str(issue_data.get("level") or "info"), + namespace="awoooi-prod", + target_resource=str(issue_data.get("culprit") or "source-provider-ingestion"), + fingerprint=f"source-provider-canary:sentry:{issue_id}", + source_url=source_url, + labels={ + "project": issue_data.get("project", {}), + "level": issue_data.get("level", "info"), + "awoooi_canary": "true", + "operator_id": operator.operator_id, + "telegram": "not_sent", + "incident": "not_created", + "approval": "not_created", + }, + annotations={ + "message": event_data.get("message"), + "summary": ( + "Operator-signed Sentry webhook canary; records upstream " + "source evidence without creating incident, approval, or Telegram." + ), + }, + payload={ + "raw_canary": payload, + "operator_id": operator.operator_id, + "auth_method": operator.auth_method, + "side_effects": { + "incident_created": False, + "approval_created": False, + "telegram_sent": False, + "openclaw_called": False, + }, + }, + ) + if event_uuid is None: + raise HTTPException( + status_code=500, + detail="sentry upstream canary was not recorded", + ) + return { + "status": "canary_recorded", + "provider": "sentry", + "event_id": issue_id, + "conversation_event_id": str(event_uuid), + "side_effects": { + "incident_created": False, + "approval_created": False, + "telegram_sent": False, + "openclaw_called": False, + }, + } + + @router.post("/error") async def handle_sentry_error( request: Request, @@ -109,6 +221,14 @@ async def handle_sentry_error( try: # 2026-04-27 P3.1-T2 by Claude — Tier-2 三服務感知強化：接入 SentryWebhookService 簽章驗證 body = await request.body() + try: + payload_from_body = json.loads(body.decode("utf-8") or "{}") + except json.JSONDecodeError: + payload_from_body = {} + + if isinstance(payload_from_body, dict) and _is_sentry_upstream_canary(payload_from_body): + return await _record_sentry_upstream_canary(payload_from_body, request) + sig_header = request.headers.get("sentry-hook-signature", "") try: verify_sentry_signature(body, sig_header) @@ -214,6 +334,8 @@ async def handle_sentry_error( "message": "Analysis scheduled" } + except HTTPException: + raise except Exception as e: logger.exception("Sentry webhook processing failed") raise HTTPException(status_code=500, detail=str(e)) from e diff --git a/apps/api/src/api/v1/signoz_webhook.py b/apps/api/src/api/v1/signoz_webhook.py index e60d62190..a8e5ae331 100644 --- a/apps/api/src/api/v1/signoz_webhook.py +++ b/apps/api/src/api/v1/signoz_webhook.py @@ -1,7 +1,3 @@ -from __future__ import annotations - -import asyncio - """ AWOOOI API - SignOz Webhook Handler ==================================== @@ -17,6 +13,9 @@ AWOOOI API - SignOz Webhook Handler 🔴 HARD RULE: 時間顯示使用 Asia/Taipei (UTC+8) """ +from __future__ import annotations + +import asyncio import uuid from typing import TYPE_CHECKING @@ -24,6 +23,7 @@ import structlog from fastapi import APIRouter, BackgroundTasks, HTTPException, Request from pydantic import BaseModel +from src.core.awooop_operator_auth import authenticate_awooop_operator_headers from src.core.metrics import ( record_alert_chain_failure, record_alert_chain_success, @@ -72,6 +72,101 @@ class SignOzAlertPayload(BaseModel): generatorURL: str | None = None +def _is_signoz_upstream_canary(alert: dict) -> bool: + labels = alert.get("labels", {}) if isinstance(alert.get("labels"), dict) else {} + annotations = ( + alert.get("annotations", {}) + if isinstance(alert.get("annotations"), dict) + else {} + ) + alert_name = str(alert.get("alertname") or labels.get("alertname") or "") + return ( + str(labels.get("awoooi_canary", "")).lower() == "true" + or alert_name == "AwoooPSourceProviderCanary" + or str(annotations.get("awooop_canary", "")).lower() == "true" + ) + + +async def _record_signoz_upstream_canary( + alert: dict, + request: Request, +) -> dict: + operator = authenticate_awooop_operator_headers( + request.headers.get("x-awooop-operator-id"), + request.headers.get("x-awooop-operator-key"), + ) + labels = alert.get("labels", {}) if isinstance(alert.get("labels"), dict) else {} + annotations = ( + alert.get("annotations", {}) + if isinstance(alert.get("annotations"), dict) + else {} + ) + alert_name = str(alert.get("alertname") or labels.get("alertname") or "AwoooPSourceProviderCanary") + run_ref = str(labels.get("run_ref") or labels.get("fingerprint") or "unknown") + event_id = f"awooop-canary-{run_ref}" + severity = str(labels.get("severity") or "info") + service_name = str(labels.get("service_name") or labels.get("service") or "source-provider-ingestion") + namespace = str(labels.get("namespace") or "awoooi-prod") + fingerprint = str(labels.get("fingerprint") or f"source-provider-canary:signoz:{run_ref}") + event_uuid = await record_external_alert_event( + project_id="awoooi", + provider="signoz", + event_id=event_id, + stage="upstream_canary", + title=alert_name, + severity=severity, + namespace=namespace, + target_resource=service_name, + fingerprint=fingerprint, + source_url=alert.get("generatorURL"), + labels={ + **labels, + "awoooi_canary": "true", + "operator_id": operator.operator_id, + "telegram": "not_sent", + "incident": "not_created", + "approval": "not_created", + }, + annotations={ + **annotations, + "summary": annotations.get("summary") + or ( + "Operator-signed SignOz webhook canary; records upstream " + "source evidence without creating incident, approval, or Telegram." + ), + }, + payload={ + "raw_canary": alert, + "operator_id": operator.operator_id, + "auth_method": operator.auth_method, + "side_effects": { + "incident_created": False, + "approval_created": False, + "telegram_sent": False, + "openclaw_called": False, + }, + }, + ) + if event_uuid is None: + raise HTTPException( + status_code=500, + detail="signoz upstream canary was not recorded", + ) + return { + "status": "canary_recorded", + "provider": "signoz", + "event_id": event_id, + "alert_name": alert_name, + "conversation_event_id": str(event_uuid), + "side_effects": { + "incident_created": False, + "approval_created": False, + "telegram_sent": False, + "openclaw_called": False, + }, + } + + @router.post("/alert") async def handle_signoz_alert( request: Request, @@ -104,6 +199,10 @@ async def handle_signoz_alert( results.append({"status": "ignored", "reason": "not firing"}) continue + if _is_signoz_upstream_canary(alert): + results.append(await _record_signoz_upstream_canary(alert, request)) + continue + # 提取告警資訊 alert_name = alert.get("alertname", alert.get("labels", {}).get("alertname", "unknown")) labels = alert.get("labels", {}) @@ -149,6 +248,8 @@ async def handle_signoz_alert( return {"status": "ok", "processed": len(results), "results": results} + except HTTPException: + raise except Exception as e: logger.exception("signoz_webhook_error", error=str(e)) raise HTTPException(status_code=500, detail=str(e)) from e @@ -336,7 +437,7 @@ async def create_signoz_approval( severity: str, incident_id: str, anomaly_frequency: dict | None = None, - analysis_result: "LLMAnalysisResult" | None = None, + analysis_result: LLMAnalysisResult | None = None, ) -> str: """ 為 SignOz 告警建立 Approval 記錄 @@ -433,7 +534,7 @@ async def send_signoz_telegram( annotations: dict, severity: str, anomaly_frequency: dict | None = None, - analysis_result: "LLMAnalysisResult" | None = None, + analysis_result: LLMAnalysisResult | None = None, ai_provider: str = "none", ): """ @@ -496,6 +597,7 @@ async def _send_log_summary_notification( 帶 5s 軟超時：超時後摘要繼續生成並存 Redis，不阻塞告警主流程 """ import html as _html + from src.services.log_summary_service import get_log_summary_service from src.services.telegram_gateway import get_telegram_gateway diff --git a/apps/api/tests/test_alert_chain_smoke_metric.py b/apps/api/tests/test_alert_chain_smoke_metric.py index c840cf47c..623f803eb 100644 --- a/apps/api/tests/test_alert_chain_smoke_metric.py +++ b/apps/api/tests/test_alert_chain_smoke_metric.py @@ -178,6 +178,72 @@ class AlertChainSmokeMetricTest(unittest.TestCase): self.assertEqual(calls[0]["headers"]["X-AwoooP-Operator-Id"], "gitea-e2e-health") self.assertEqual(calls[0]["headers"]["X-AwoooP-Operator-Key"], "secret") + def test_source_provider_upstream_canary_requires_operator_key(self): + result = alert_chain_smoke_test.send_source_provider_upstream_canary( + "https://awoooi.example", + providers=["sentry", "signoz"], + operator_key=None, + operator_id="gitea-e2e-health", + ) + + self.assertFalse(result.passed) + self.assertTrue(result.critical) + self.assertIn("AWOOOP_OPERATOR_API_KEY", result.message) + + def test_source_provider_upstream_canary_posts_provider_payloads(self): + calls = [] + + def fake_post(url, payload, *, headers=None, timeout=None): + calls.append( + { + "url": url, + "payload": payload, + "headers": headers, + "timeout": timeout, + } + ) + if url.endswith("/api/v1/webhooks/sentry/error"): + return alert_chain_smoke_test.HttpGetResult( + 200, + '{"status":"canary_recorded","provider":"sentry"}', + ) + if url.endswith("/api/v1/webhooks/signoz/alert"): + return alert_chain_smoke_test.HttpGetResult( + 200, + ( + '{"status":"ok","results":[' + '{"status":"canary_recorded","provider":"signoz"}]}' + ), + ) + raise AssertionError(f"unexpected url {url}") + + original_post = alert_chain_smoke_test.http_post_json + try: + alert_chain_smoke_test.http_post_json = fake_post + result = alert_chain_smoke_test.send_source_provider_upstream_canary( + "https://awoooi.example", + providers=["sentry", "signoz"], + operator_key="secret", + operator_id="gitea-e2e-health", + run_ref="run/123", + ) + finally: + alert_chain_smoke_test.http_post_json = original_post + + self.assertTrue(result.passed) + self.assertEqual( + calls[0]["url"], + "https://awoooi.example/api/v1/webhooks/sentry/error", + ) + self.assertEqual( + calls[1]["url"], + "https://awoooi.example/api/v1/webhooks/signoz/alert", + ) + self.assertEqual(calls[0]["payload"]["data"]["issue"]["title"], "AwoooPSourceProviderCanary") + self.assertEqual(calls[1]["payload"]["alerts"][0]["labels"]["awoooi_canary"], "true") + self.assertEqual(calls[0]["headers"]["X-AwoooP-Operator-Id"], "gitea-e2e-health") + self.assertEqual(calls[1]["headers"]["X-AwoooP-Operator-Key"], "secret") + if __name__ == "__main__": unittest.main() diff --git a/apps/api/tests/test_source_provider_upstream_canary.py b/apps/api/tests/test_source_provider_upstream_canary.py new file mode 100644 index 000000000..fefe6ae2f --- /dev/null +++ b/apps/api/tests/test_source_provider_upstream_canary.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import json +from uuid import uuid4 + +import pytest +from fastapi import BackgroundTasks, HTTPException +from starlette.requests import Request + + +def _request_with_json(payload: dict, headers: dict[str, str] | None = None) -> Request: + body = json.dumps(payload).encode("utf-8") + raw_headers = [ + (key.lower().encode("latin-1"), value.encode("latin-1")) + for key, value in (headers or {}).items() + ] + + async def receive() -> dict: + return {"type": "http.request", "body": body, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": "/", + "headers": raw_headers, + }, + receive, + ) + + +def _operator_headers() -> dict[str, str]: + return { + "X-AwoooP-Operator-Id": "gitea-e2e-health", + "X-AwoooP-Operator-Key": "secret", + } + + +def _sentry_canary_payload() -> dict: + return { + "action": "triggered", + "data": { + "issue": { + "id": "awoooi-canary-test-run", + "shortId": "AWOOOI-CANARY", + "title": "AwoooPSourceProviderCanary", + "culprit": "source-provider-ingestion", + "level": "info", + "project": {"slug": "awoooi"}, + }, + "event": { + "message": "AwoooP upstream source provider canary", + "tags": [["awoooi_canary", "true"], ["run_ref", "test-run"]], + }, + }, + } + + +@pytest.mark.asyncio +async def test_sentry_upstream_canary_records_without_signature(monkeypatch) -> None: + from src.api.v1 import sentry_webhook + from src.core.config import settings + + calls: list[dict] = [] + + async def fake_record_external_alert_event(**kwargs): + calls.append(kwargs) + return uuid4() + + monkeypatch.setattr(settings, "ENVIRONMENT", "prod") + monkeypatch.setattr(settings, "AWOOOP_OPERATOR_API_KEY", "secret") + monkeypatch.setattr( + sentry_webhook, + "record_external_alert_event", + fake_record_external_alert_event, + ) + + result = await sentry_webhook.handle_sentry_error( + _request_with_json(_sentry_canary_payload(), _operator_headers()), + BackgroundTasks(), + ) + + assert result["status"] == "canary_recorded" + assert result["side_effects"] == { + "incident_created": False, + "approval_created": False, + "telegram_sent": False, + "openclaw_called": False, + } + assert calls[0]["provider"] == "sentry" + assert calls[0]["stage"] == "upstream_canary" + assert calls[0]["event_id"] == "awoooi-canary-test-run" + assert calls[0]["labels"]["incident"] == "not_created" + + +@pytest.mark.asyncio +async def test_sentry_non_canary_still_requires_signature(monkeypatch) -> None: + from src.api.v1 import sentry_webhook + from src.core.config import settings + + monkeypatch.setattr(settings, "ENVIRONMENT", "prod") + monkeypatch.setattr(settings, "SENTRY_WEBHOOK_SECRET", "required-secret") + + payload = _sentry_canary_payload() + payload["data"]["issue"]["id"] = "real-issue-1" + payload["data"]["issue"]["shortId"] = "REAL-1" + payload["data"]["issue"]["title"] = "Real issue" + payload["data"]["event"]["tags"] = [] + + with pytest.raises(HTTPException) as exc_info: + await sentry_webhook.handle_sentry_error( + _request_with_json(payload, _operator_headers()), + BackgroundTasks(), + ) + + assert exc_info.value.status_code == 401 + + +@pytest.mark.asyncio +async def test_signoz_upstream_canary_records_without_incident_side_effects(monkeypatch) -> None: + from src.api.v1 import signoz_webhook + from src.core.config import settings + + calls: list[dict] = [] + + async def fake_record_external_alert_event(**kwargs): + calls.append(kwargs) + return uuid4() + + monkeypatch.setattr(settings, "ENVIRONMENT", "prod") + monkeypatch.setattr(settings, "AWOOOP_OPERATOR_API_KEY", "secret") + monkeypatch.setattr( + signoz_webhook, + "record_external_alert_event", + fake_record_external_alert_event, + ) + payload = { + "alerts": [ + { + "status": "firing", + "alertname": "AwoooPSourceProviderCanary", + "labels": { + "alertname": "AwoooPSourceProviderCanary", + "severity": "info", + "namespace": "awoooi-prod", + "service": "source-provider-ingestion", + "awoooi_canary": "true", + "run_ref": "test-run", + }, + "annotations": {"summary": "AwoooP upstream source provider canary"}, + } + ] + } + + result = await signoz_webhook.handle_signoz_alert( + _request_with_json(payload, _operator_headers()), + BackgroundTasks(), + ) + + assert result["status"] == "ok" + assert result["results"][0]["status"] == "canary_recorded" + assert result["results"][0]["side_effects"] == { + "incident_created": False, + "approval_created": False, + "telegram_sent": False, + "openclaw_called": False, + } + assert calls[0]["provider"] == "signoz" + assert calls[0]["stage"] == "upstream_canary" + assert calls[0]["labels"]["approval"] == "not_created" diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index ab9e44281..a3d9d5ed6 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,49 @@ +## 2026-05-20｜T115 Provider-native upstream canary 接入 + +**觸發**： + +- T113/T114 已能證明 Sentry / SigNoz freshness 與 incident-level correlation 狀態，但 live 結論仍常是 `provider_fresh_no_match`。 +- Operator 仍需要一條可稽核證據回答：「provider 原生 webhook 是否真的能把 Sentry issue / SignOz alert 寫入 AwoooP source dossier？」 + +**修正**： + +- `POST /api/v1/webhooks/sentry/error` 新增 AwoooP upstream canary 短路徑： + - 僅 `AwoooPSourceProviderCanary` / `AWOOOI-CANARY` / `awoooi_canary=true` payload 生效。 + - 必須帶 `X-AwoooP-Operator-Id` + `X-AwoooP-Operator-Key`。 + - 只寫 `record_external_alert_event(stage="upstream_canary")`，不建立 Incident / Approval、不送 Telegram、不呼叫 OpenClaw。 + - 非 canary Sentry webhook 仍必須走 `sentry-hook-signature` 驗證。 +- `POST /api/v1/webhooks/signoz/alert` 新增 provider-shaped canary，規則相同，只記錄 source dossier，不觸發 incident processing。 +- `scripts/alert_chain_smoke_test.py` 新增 `--source-provider-upstream-canary`，會打入 Sentry / SigNoz 原生 webhook endpoint，與既有 `--source-provider-heartbeat` 一起驗證 freshness + provider-native ingestion。 +- `.gitea/workflows/e2e-health.yaml` 每日 smoke 同時執行 heartbeat 與 upstream canary。 + +**Verification / deployment**： + +```text +Local: +python3.11 -m py_compile apps/api/src/api/v1/sentry_webhook.py apps/api/src/api/v1/signoz_webhook.py scripts/alert_chain_smoke_test.py + -> pass +DATABASE_URL=postgresql+asyncpg://... pytest apps/api/tests/test_source_provider_upstream_canary.py apps/api/tests/test_alert_chain_smoke_metric.py apps/api/tests/test_sentry_webhook_signature.py -q + -> 30 passed +``` + +**邊界 / 下一步**： + +- T115 不是把真實 Sentry / SigNoz production notification channel 改設定；它先提供可排程、可稽核、低噪音的 provider-native ingress 證據。 +- `upstream_canary` 會被 source envelope 視為真實 provider-shaped inbound event，不像 `heartbeat` 被排除在 `sentry_issue_ids` / `signoz_alerts` 外；因此可用來驗證 source refs 顯示鏈路，但仍不代表任何真實事故已被關聯。 +- 下一步是推 Gitea main、跑 production smoke，確認 dossier coverage / status-chain 能看到 `upstream_canary` 後，再把匹配結果轉成可審核 work item。 + +**目前整體進度**： + +- Alerts 完整清單可追蹤性：約 99.99%。 +- 前端 AI 自動化管理介面同步：約 99.99%。 +- AwoooP 告警可觀測鏈：約 99.985%。 +- Source refs / Sentry / SigNoz 可見性：約 99.9%。 +- Incident-level source correlation 可見性：約 86%。 +- Source provider freshness 自動告警：約 99%。 +- Provider-native upstream ingestion 可驗證性：約 80%（local 完成，待 production smoke）。 +- 低風險自動修復閉環：約 95.4%。 +- 完整 AI 自動化管理產品化：約 99.62%。 + ## 2026-05-20｜T114 Incident-level source correlation evidence 上線 **觸發**： diff --git a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md index f26f2afe0..a0ccc9a05 100644 --- a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md +++ b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md @@ -2502,6 +2502,15 @@ Phase 6 完成後 - 邊界 / 下一步：T114 完成 incident-level correlation visibility，不代表真實 upstream provider notification 已回復。下一段應補 signed upstream canary / provider-native notification smoke，並把匹配結果轉成可審核 work item，而非直接改 incident。 - 目前進度更新：AwoooP 告警可觀測鏈約 99.98%；Source refs / Sentry / SigNoz 可見性約 99.85%；Incident-level source correlation 可見性約 85%；Source provider freshness 自動告警約 99%；低風險自動修復閉環約 95.4%；完整 AI 自動化管理產品化約 99.6%。 +**T115 Provider-native upstream canary 接入（2026-05-20 台北）**： +- 觸發：T114 live conclusion 多數仍是 `provider_fresh_no_match`；heartbeat 只能證明 freshness，不能證明 Sentry / SigNoz provider-native webhook path 真的能產生 source refs。 +- 修正：Sentry `/api/v1/webhooks/sentry/error` 與 SignOz `/api/v1/webhooks/signoz/alert` 新增 operator-signed upstream canary。只接受明確標記的 `AwoooPSourceProviderCanary` / `AWOOOI-CANARY` / `awoooi_canary=true` payload，且必須帶 `X-AwoooP-Operator-Id` + `X-AwoooP-Operator-Key`。 +- 安全邊界：canary path 僅呼叫 `record_external_alert_event(stage="upstream_canary")` 寫 source dossier + completed shadow run，不建立 Incident / Approval、不送 Telegram、不呼叫 OpenClaw。非 canary Sentry webhook 仍需 `sentry-hook-signature`。 +- CI / smoke：`scripts/alert_chain_smoke_test.py` 新增 `--source-provider-upstream-canary`，對 Sentry / SignOz 原生 webhook endpoint 發送 provider-shaped canary payload；`.gitea/workflows/e2e-health.yaml` 每日排程同時跑 heartbeat 與 upstream canary。 +- 驗證：local `py_compile` pass；targeted pytest `30 passed`。待推版後以 production smoke 驗證 `stage=upstream_canary` 寫入 AwoooP dossier，並確認 source envelope 產生 `sentry_issue_ids` / `signoz_alerts`。 +- 邊界 / 下一步：T115 不是修改真實 Sentry / SignOz notification channel 設定，而是先建立可重複、可稽核的 provider-native ingestion 證據。下一段應把 `upstream_canary` 與 real incident matching 結果轉為可審核 work item，而不是直接改 incident refs。 +- 目前進度更新：AwoooP 告警可觀測鏈約 99.985%；Source refs / Sentry / SigNoz 可見性約 99.9%；Incident-level source correlation 可見性約 86%；Provider-native upstream ingestion 可驗證性約 80%（待 production smoke）；低風險自動修復閉環約 95.4%；完整 AI 自動化管理產品化約 99.62%。 + --- ### 2026-04-20 晚 (台北) — C1-C4 全流程串接 — Playbook 鏈路保護（commit de2d34d） diff --git a/scripts/alert_chain_smoke_test.py b/scripts/alert_chain_smoke_test.py index 771aa6da6..4949c4b1a 100644 --- a/scripts/alert_chain_smoke_test.py +++ b/scripts/alert_chain_smoke_test.py @@ -456,6 +456,20 @@ def check_webhook_health(api_url: str) -> list[CheckResult]: return results +def _clean_source_providers(providers: list[str]) -> list[str]: + return [ + provider.strip().lower() + for provider in providers + if provider.strip().lower() in {"sentry", "signoz"} + ] + + +def _safe_run_ref(run_ref: str | None) -> str: + raw = (run_ref or f"manual-{int(time.time())}").strip() + cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip("-") + return cleaned[:64] or f"manual-{int(time.time())}" + + def send_source_provider_heartbeat( api_url: str, *, @@ -465,11 +479,7 @@ def send_source_provider_heartbeat( run_ref: str | None = None, ) -> CheckResult: """Record low-noise provider freshness evidence without creating incidents.""" - cleaned_providers = [ - provider.strip().lower() - for provider in providers - if provider.strip().lower() in {"sentry", "signoz"} - ] + cleaned_providers = _clean_source_providers(providers) if not cleaned_providers: return CheckResult( "Source Provider Heartbeat", @@ -532,6 +542,157 @@ def send_source_provider_heartbeat( ) +def _build_sentry_upstream_canary_payload(safe_ref: str) -> dict[str, Any]: + issue_id = f"awoooi-canary-{safe_ref}" + return { + "action": "triggered", + "data": { + "issue": { + "id": issue_id, + "shortId": "AWOOOI-CANARY", + "title": "AwoooPSourceProviderCanary", + "culprit": "source-provider-ingestion", + "level": "info", + "project": {"slug": "awoooi"}, + "permalink": "https://awoooi.wooo.work/zh-TW/awooop/work-items", + }, + "event": { + "message": "AwoooP upstream source provider canary", + "platform": "python", + "tags": [ + ["awoooi_canary", "true"], + ["run_ref", safe_ref], + ], + }, + }, + "actor": {"type": "application", "name": "AwoooP E2E"}, + } + + +def _build_signoz_upstream_canary_payload(safe_ref: str) -> dict[str, Any]: + fingerprint = f"source-provider-canary:signoz:{safe_ref}" + return { + "alerts": [ + { + "status": "firing", + "alertname": "AwoooPSourceProviderCanary", + "labels": { + "alertname": "AwoooPSourceProviderCanary", + "severity": "info", + "namespace": "awoooi-prod", + "service": "source-provider-ingestion", + "service_name": "source-provider-ingestion", + "awoooi_canary": "true", + "run_ref": safe_ref, + "fingerprint": fingerprint, + }, + "annotations": { + "summary": "AwoooP upstream source provider canary", + "description": ( + "Synthetic provider-shaped SignOz alert used only to " + "verify source dossier ingestion." + ), + }, + "generatorURL": "https://awoooi.wooo.work/zh-TW/awooop/work-items", + } + ] + } + + +def _validate_upstream_canary_response(provider: str, data: dict[str, Any]) -> str | None: + if provider == "sentry": + if data.get("status") == "canary_recorded" and data.get("provider") == "sentry": + return None + return f"unexpected Sentry response: {data}" + results = data.get("results", []) + recorded = [ + item + for item in results + if isinstance(item, dict) + and item.get("status") == "canary_recorded" + and item.get("provider") == "signoz" + ] + if recorded: + return None + return f"unexpected SignOz response: {data}" + + +def send_source_provider_upstream_canary( + api_url: str, + *, + providers: list[str], + operator_key: str | None, + operator_id: str, + run_ref: str | None = None, +) -> CheckResult: + """Send provider-shaped canaries through webhook-native ingestion paths.""" + cleaned_providers = _clean_source_providers(providers) + if not cleaned_providers: + return CheckResult( + "Source Provider Upstream Canary", + False, + "沒有有效 provider（允許 sentry/signoz）", + ) + if not operator_key: + return CheckResult( + "Source Provider Upstream Canary", + False, + "AWOOOP_OPERATOR_API_KEY 未設定；無法打入受保護 upstream canary", + ) + + safe_ref = _safe_run_ref(run_ref) + endpoints = { + "sentry": ( + f"{api_url}/api/v1/webhooks/sentry/error", + _build_sentry_upstream_canary_payload(safe_ref), + ), + "signoz": ( + f"{api_url}/api/v1/webhooks/signoz/alert", + _build_signoz_upstream_canary_payload(safe_ref), + ), + } + recorded: list[str] = [] + for provider in cleaned_providers: + url, payload = endpoints[provider] + try: + resp = http_post_json( + url, + payload, + headers={ + "X-AwoooP-Operator-Id": operator_id, + "X-AwoooP-Operator-Key": operator_key, + }, + timeout=TIMEOUT, + ) + data = resp.json() + except (URLError, TimeoutError, OSError, json.JSONDecodeError) as e: + return CheckResult( + "Source Provider Upstream Canary", + False, + f"{provider} upstream canary failed: {_http_error_message(e)}", + ) + if resp.status_code >= 400: + return CheckResult( + "Source Provider Upstream Canary", + False, + f"{provider} HTTP {resp.status_code}: {data.get('detail', resp.text) if isinstance(data, dict) else resp.text}", + ) + validation_error = _validate_upstream_canary_response(provider, data) + if validation_error: + return CheckResult( + "Source Provider Upstream Canary", + False, + validation_error, + ) + recorded.append(provider) + + return CheckResult( + "Source Provider Upstream Canary", + True, + f"recorded {', '.join(sorted(recorded))} webhook-native canary event(s)", + ) + + def check_signoz_reachable(signoz_url: str) -> CheckResult: """Check 4: SigNoz UI 可達""" try: @@ -627,6 +788,7 @@ def run_smoke_test( *, metrics_api_url: str | None = None, source_provider_heartbeat: bool = False, + source_provider_upstream_canary: bool = False, source_providers: list[str] | None = None, operator_key: str | None = None, operator_id: str = "gitea-e2e-health", @@ -679,6 +841,29 @@ def run_smoke_test( ) ) + if source_provider_upstream_canary: + provider_list = source_providers or ["sentry", "signoz"] + canary_result = send_source_provider_upstream_canary( + api_url, + providers=provider_list, + operator_key=operator_key, + operator_id=operator_id, + run_ref=run_ref, + ) + report.add(canary_result) + if fail_fast and not canary_result.passed and canary_result.critical: + return report + + if canary_result.passed: + for source in provider_list: + report.add( + check_alert_chain_metric( + PROMETHEUS_URL, + metrics_url, + source=source, + ) + ) + # Check 4: SigNoz report.add(check_signoz_reachable(SIGNOZ_URL)) @@ -715,11 +900,16 @@ def main() -> int: action="store_true", help="寫入 Sentry/SignOz 低噪音 freshness heartbeat 並驗證 provider 指標", ) + parser.add_argument( + "--source-provider-upstream-canary", + action="store_true", + help="透過 Sentry/SignOz 原生 webhook 寫入 provider-shaped canary 來源證據", + ) parser.add_argument( "--source-provider", action="append", choices=["sentry", "signoz"], - help="指定要寫入 heartbeat 的 provider；可重複指定，預設 sentry+signoz", + help="指定要驗證的 provider；可重複指定，預設 sentry+signoz", ) parser.add_argument( "--operator-id", @@ -743,6 +933,7 @@ def main() -> int: args.fail_fast, metrics_api_url=args.metrics_api_url, source_provider_heartbeat=args.source_provider_heartbeat, + source_provider_upstream_canary=args.source_provider_upstream_canary, source_providers=args.source_provider, operator_key=os.environ.get(args.operator_key_env), operator_id=args.operator_id,