feat(awooop): add provider upstream canary

scripts/alert_chain_smoke_test.py

@@ -456,6 +456,20 @@ def check_webhook_health(api_url: str) -> list[CheckResult]:
     return results
 
 
+def _clean_source_providers(providers: list[str]) -> list[str]:
+    return [
+        provider.strip().lower()
+        for provider in providers
+        if provider.strip().lower() in {"sentry", "signoz"}
+    ]
+
+
+def _safe_run_ref(run_ref: str | None) -> str:
+    raw = (run_ref or f"manual-{int(time.time())}").strip()
+    cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "-", raw).strip("-")
+    return cleaned[:64] or f"manual-{int(time.time())}"
+
+
 def send_source_provider_heartbeat(
     api_url: str,
     *,
@@ -465,11 +479,7 @@ def send_source_provider_heartbeat(
     run_ref: str | None = None,
 ) -> CheckResult:
     """Record low-noise provider freshness evidence without creating incidents."""
-    cleaned_providers = [
-        provider.strip().lower()
-        for provider in providers
-        if provider.strip().lower() in {"sentry", "signoz"}
-    ]
+    cleaned_providers = _clean_source_providers(providers)
     if not cleaned_providers:
         return CheckResult(
             "Source Provider Heartbeat",
@@ -532,6 +542,157 @@ def send_source_provider_heartbeat(
         )
 
 
+def _build_sentry_upstream_canary_payload(safe_ref: str) -> dict[str, Any]:
+    issue_id = f"awoooi-canary-{safe_ref}"
+    return {
+        "action": "triggered",
+        "data": {
+            "issue": {
+                "id": issue_id,
+                "shortId": "AWOOOI-CANARY",
+                "title": "AwoooPSourceProviderCanary",
+                "culprit": "source-provider-ingestion",
+                "level": "info",
+                "project": {"slug": "awoooi"},
+                "permalink": "https://awoooi.wooo.work/zh-TW/awooop/work-items",
+            },
+            "event": {
+                "message": "AwoooP upstream source provider canary",
+                "platform": "python",
+                "tags": [
+                    ["awoooi_canary", "true"],
+                    ["run_ref", safe_ref],
+                ],
+            },
+        },
+        "actor": {"type": "application", "name": "AwoooP E2E"},
+    }
+
+
+def _build_signoz_upstream_canary_payload(safe_ref: str) -> dict[str, Any]:
+    fingerprint = f"source-provider-canary:signoz:{safe_ref}"
+    return {
+        "alerts": [
+            {
+                "status": "firing",
+                "alertname": "AwoooPSourceProviderCanary",
+                "labels": {
+                    "alertname": "AwoooPSourceProviderCanary",
+                    "severity": "info",
+                    "namespace": "awoooi-prod",
+                    "service": "source-provider-ingestion",
+                    "service_name": "source-provider-ingestion",
+                    "awoooi_canary": "true",
+                    "run_ref": safe_ref,
+                    "fingerprint": fingerprint,
+                },
+                "annotations": {
+                    "summary": "AwoooP upstream source provider canary",
+                    "description": (
+                        "Synthetic provider-shaped SignOz alert used only to "
+                        "verify source dossier ingestion."
+                    ),
+                },
+                "generatorURL": "https://awoooi.wooo.work/zh-TW/awooop/work-items",
+            }
+        ]
+    }
+
+
+def _validate_upstream_canary_response(provider: str, data: dict[str, Any]) -> str | None:
+    if provider == "sentry":
+        if data.get("status") == "canary_recorded" and data.get("provider") == "sentry":
+            return None
+        return f"unexpected Sentry response: {data}"
+    results = data.get("results", [])
+    recorded = [
+        item
+        for item in results
+        if isinstance(item, dict)
+        and item.get("status") == "canary_recorded"
+        and item.get("provider") == "signoz"
+    ]
+    if recorded:
+        return None
+    return f"unexpected SignOz response: {data}"
+
+
+def send_source_provider_upstream_canary(
+    api_url: str,
+    *,
+    providers: list[str],
+    operator_key: str | None,
+    operator_id: str,
+    run_ref: str | None = None,
+) -> CheckResult:
+    """Send provider-shaped canaries through webhook-native ingestion paths."""
+    cleaned_providers = _clean_source_providers(providers)
+    if not cleaned_providers:
+        return CheckResult(
+            "Source Provider Upstream Canary",
+            False,
+            "沒有有效 provider（允許 sentry/signoz）",
+        )
+    if not operator_key:
+        return CheckResult(
+            "Source Provider Upstream Canary",
+            False,
+            "AWOOOP_OPERATOR_API_KEY 未設定；無法打入受保護 upstream canary",
+        )
+
+    safe_ref = _safe_run_ref(run_ref)
+    endpoints = {
+        "sentry": (
+            f"{api_url}/api/v1/webhooks/sentry/error",
+            _build_sentry_upstream_canary_payload(safe_ref),
+        ),
+        "signoz": (
+            f"{api_url}/api/v1/webhooks/signoz/alert",
+            _build_signoz_upstream_canary_payload(safe_ref),
+        ),
+    }
+    recorded: list[str] = []
+    for provider in cleaned_providers:
+        url, payload = endpoints[provider]
+        try:
+            resp = http_post_json(
+                url,
+                payload,
+                headers={
+                    "X-AwoooP-Operator-Id": operator_id,
+                    "X-AwoooP-Operator-Key": operator_key,
+                },
+                timeout=TIMEOUT,
+            )
+            data = resp.json()
+        except (URLError, TimeoutError, OSError, json.JSONDecodeError) as e:
+            return CheckResult(
+                "Source Provider Upstream Canary",
+                False,
+                f"{provider} upstream canary failed: {_http_error_message(e)}",
+            )
+        if resp.status_code >= 400:
+            return CheckResult(
+                "Source Provider Upstream Canary",
+                False,
+                f"{provider} HTTP {resp.status_code}: {data.get('detail', resp.text) if isinstance(data, dict) else resp.text}",
+            )
+        validation_error = _validate_upstream_canary_response(provider, data)
+        if validation_error:
+            return CheckResult(
+                "Source Provider Upstream Canary",
+                False,
+                validation_error,
+            )
+        recorded.append(provider)
+
+    return CheckResult(
+        "Source Provider Upstream Canary",
+        True,
+        f"recorded {', '.join(sorted(recorded))} webhook-native canary event(s)",
+    )
+
+
 def check_signoz_reachable(signoz_url: str) -> CheckResult:
     """Check 4: SigNoz UI 可達"""
     try:
@@ -627,6 +788,7 @@ def run_smoke_test(
     *,
     metrics_api_url: str | None = None,
     source_provider_heartbeat: bool = False,
+    source_provider_upstream_canary: bool = False,
     source_providers: list[str] | None = None,
     operator_key: str | None = None,
     operator_id: str = "gitea-e2e-health",
@@ -679,6 +841,29 @@ def run_smoke_test(
                     )
                 )
 
+    if source_provider_upstream_canary:
+        provider_list = source_providers or ["sentry", "signoz"]
+        canary_result = send_source_provider_upstream_canary(
+            api_url,
+            providers=provider_list,
+            operator_key=operator_key,
+            operator_id=operator_id,
+            run_ref=run_ref,
+        )
+        report.add(canary_result)
+        if fail_fast and not canary_result.passed and canary_result.critical:
+            return report
+
+        if canary_result.passed:
+            for source in provider_list:
+                report.add(
+                    check_alert_chain_metric(
+                        PROMETHEUS_URL,
+                        metrics_url,
+                        source=source,
+                    )
+                )
+
     # Check 4: SigNoz
     report.add(check_signoz_reachable(SIGNOZ_URL))
 
@@ -715,11 +900,16 @@ def main() -> int:
         action="store_true",
         help="寫入 Sentry/SignOz 低噪音 freshness heartbeat 並驗證 provider 指標",
     )
+    parser.add_argument(
+        "--source-provider-upstream-canary",
+        action="store_true",
+        help="透過 Sentry/SignOz 原生 webhook 寫入 provider-shaped canary 來源證據",
+    )
     parser.add_argument(
         "--source-provider",
         action="append",
         choices=["sentry", "signoz"],
-        help="指定要寫入 heartbeat 的 provider；可重複指定，預設 sentry+signoz",
+        help="指定要驗證的 provider；可重複指定，預設 sentry+signoz",
     )
     parser.add_argument(
         "--operator-id",
@@ -743,6 +933,7 @@ def main() -> int:
         args.fail_fast,
         metrics_api_url=args.metrics_api_url,
         source_provider_heartbeat=args.source_provider_heartbeat,
+        source_provider_upstream_canary=args.source_provider_upstream_canary,
         source_providers=args.source_provider,
         operator_key=os.environ.get(args.operator_key_env),
         operator_id=args.operator_id,