From ad8ead254668324736446de2ff85dad06ab98649 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 12 May 2026 13:58:08 +0800 Subject: [PATCH] fix(awooop): route ci notifications through event mirror --- .gitea/workflows/cd-dev.yaml | 52 ++++++--- .gitea/workflows/cd.yaml | 88 +++++++++++---- .gitea/workflows/code-review.yaml | 57 +++++++--- .gitea/workflows/deploy-alerts.yaml | 17 ++- .gitea/workflows/e2e-health.yaml | 18 +++- .gitea/workflows/run-migration.yml | 13 ++- apps/api/src/api/v1/webhooks.py | 40 ++++++- apps/api/src/services/telegram_gateway.py | 9 +- .../tests/test_cicd_alertmanager_mapping.py | 62 +++++++++++ docs/LOGBOOK.md | 64 +++++++++++ scripts/ci/notify-awoooi-cicd.sh | 100 ++++++++++++++++++ 11 files changed, 461 insertions(+), 59 deletions(-) create mode 100644 apps/api/tests/test_cicd_alertmanager_mapping.py create mode 100755 scripts/ci/notify-awoooi-cicd.sh diff --git a/.gitea/workflows/cd-dev.yaml b/.gitea/workflows/cd-dev.yaml index c1ccf2cd6..525f1158b 100644 --- a/.gitea/workflows/cd-dev.yaml +++ b/.gitea/workflows/cd-dev.yaml @@ -43,10 +43,19 @@ jobs: ├ 📝 ${{ steps.commit.outputs.message }} ├ 🔖 ${{ steps.commit.outputs.short_sha }} └ 🌿 dev branch" - printf '%b' "$MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d "parse_mode=HTML" \ - --data-urlencode "text@-" + if AWOOI_CICD_STATUS=running \ + AWOOI_CICD_STAGE=dev-deploy \ + AWOOI_CICD_JOB_NAME="[DEV] 部署開始" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_SUMMARY="${{ steps.commit.outputs.message }}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "Dev deploy start notification mirrored through AWOOI API" + else + printf '%b' "$MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d "parse_mode=HTML" \ + --data-urlencode "text@-" + fi # API 測試 (同 prod CI，確保 dev 也通過) - name: Run API Tests @@ -184,10 +193,20 @@ jobs: ├ 🔖 ${{ steps.commit.outputs.short_sha }} ├ ⏱️ 耗時: ${MINUTES}m ${SECONDS}s └ 🩺 http://192.168.0.125:32344/api/v1/health" - printf '%b' "$MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d "parse_mode=HTML" \ - --data-urlencode "text@-" + if AWOOI_CICD_STATUS=success \ + AWOOI_CICD_STAGE=dev-deploy \ + AWOOI_CICD_JOB_NAME="[DEV] 部署完成" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_DURATION_SECONDS="${DURATION}" \ + AWOOI_CICD_SUMMARY="${{ steps.commit.outputs.message }}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "Dev deploy success notification mirrored through AWOOI API" + else + printf '%b' "$MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d "parse_mode=HTML" \ + --data-urlencode "text@-" + fi - name: Notify Dev Deploy Failure if: failure() @@ -196,7 +215,16 @@ jobs: ├ 📝 ${{ steps.commit.outputs.message }} ├ 🔖 ${{ steps.commit.outputs.short_sha }} └ 🔗 查看日誌" - printf '%b' "$MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d "parse_mode=HTML" \ - --data-urlencode "text@-" + if AWOOI_CICD_STATUS=failed \ + AWOOI_CICD_STAGE=dev-deploy \ + AWOOI_CICD_JOB_NAME="[DEV] 部署失敗" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_SUMMARY="${{ steps.commit.outputs.message }}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "Dev deploy failure notification mirrored through AWOOI API" + else + printf '%b' "$MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d "parse_mode=HTML" \ + --data-urlencode "text@-" + fi diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 753799c57..14a5a35e4 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -92,10 +92,20 @@ jobs: MSG=$(printf '🚀 AWOOOI 部署開始\n├ 📝 %s\n├ 🔖 %s\n└ 👤 %s' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}") # 2026-05-02 Claude Opus 4.7 + 統帥 ogt: notify 失敗不該擋整條 CI（鐵證: # curl 400 從 5/1 起連續炸 14 個 commit 的 build-and-deploy）— 對齊 line 922 既有 pattern - curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d "parse_mode=HTML" \ - --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + if AWOOI_CICD_STATUS=running \ + AWOOI_CICD_STAGE=tests \ + AWOOI_CICD_JOB_NAME="AWOOOI 部署開始" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \ + AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "✅ CI/CD start notification mirrored through AWOOI API" + else + curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d "parse_mode=HTML" \ + --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + fi @@ -255,10 +265,20 @@ jobs: ACTOR="${{ github.actor }}" COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&/g; s//\>/g') MSG=$(printf '❌ AWOOOI 部署失敗\n├ 📝 %s\n├ 🔖 %s\n├ 👤 %s\n├ 🧪 Stage: tests\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}") - curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d "parse_mode=HTML" \ - --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + if AWOOI_CICD_STATUS=failed \ + AWOOI_CICD_STAGE=tests \ + AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \ + AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "✅ CI/CD tests failure notification mirrored through AWOOI API" + else + curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d "parse_mode=HTML" \ + --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + fi build-and-deploy: # 2026-04-30 Codex: Docker builds run on the host runner. Long docker build @@ -861,10 +881,20 @@ jobs: ACTOR="${{ github.actor }}" COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&/g; s//\>/g') MSG=$(printf '❌ AWOOOI 部署失敗\n├ 📝 %s\n├ 🔖 %s\n├ 👤 %s\n├ 🏗️ Stage: build-and-deploy\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}") - curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d "parse_mode=HTML" \ - --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + if AWOOI_CICD_STATUS=failed \ + AWOOI_CICD_STAGE=build-and-deploy \ + AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \ + AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "✅ CI/CD build failure notification mirrored through AWOOI API" + else + curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d "parse_mode=HTML" \ + --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + fi post-deploy-checks: needs: build-and-deploy @@ -1024,9 +1054,19 @@ jobs: COMMIT_MSG="${{ steps.commit.outputs.message }}" SHORT_SHA="${{ steps.commit.outputs.short_sha }}" TG_MSG="✅ AWOOOI 部署完成\n├ 📝 ${COMMIT_MSG}\n├ 🔖 ${SHORT_SHA}\n├ ⏱️ 耗時: ${MINUTES}m ${SECONDS}s\n├ 📦 API: ✅ Web: ✅\n├ 🩺 Health: ✅\n├ 🔗 Alert Chain: ${ALERT_CHAIN_RESULT}\n├ 📊 Monitoring: ${MONITORING_RESULT}\n└ 🎭 Smoke: ${SMOKE_RESULT}" - printf '%b' "$TG_MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - --data-urlencode "text@-" || echo "TG notify warning (non-fatal)" + if AWOOI_CICD_STATUS=success \ + AWOOI_CICD_STAGE=post-deploy \ + AWOOI_CICD_JOB_NAME="AWOOOI 部署完成" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_DURATION_SECONDS="${DURATION}" \ + AWOOI_CICD_SUMMARY="API=✅; Web=✅; AlertChain=${ALERT_CHAIN_RESULT}; Monitoring=${MONITORING_RESULT}; Smoke=${SMOKE_RESULT}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "✅ CI/CD success notification mirrored through AWOOI API" + else + printf '%b' "$TG_MSG" | curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + --data-urlencode "text@-" || echo "TG notify warning (non-fatal)" + fi - name: Notify Pipeline Failure # 2026-04-16 ogt + Claude Sonnet 4.6: 改用 HTML 結構化格式 @@ -1037,7 +1077,17 @@ jobs: ACTOR="${{ github.actor }}" COMMIT_ESC=$(echo "$COMMIT_MSG" | sed 's/&/\&/g; s//\>/g') MSG=$(printf '❌ AWOOOI 部署失敗\n├ 📝 %s\n├ 🔖 %s\n├ 👤 %s\n├ 🩺 Stage: post-deploy-checks\n└ 🔗 http://192.168.0.110:3001/wooo/awoooi/actions' "${COMMIT_ESC}" "${SHORT_SHA}" "${ACTOR}") - curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d "parse_mode=HTML" \ - --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + if AWOOI_CICD_STATUS=failed \ + AWOOI_CICD_STAGE=post-deploy-checks \ + AWOOI_CICD_JOB_NAME="AWOOOI 部署失敗" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_TRIGGERED_BY="${ACTOR}" \ + AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "✅ CI/CD post-deploy failure notification mirrored through AWOOI API" + else + curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d "parse_mode=HTML" \ + --data-urlencode "text=${MSG}" || echo "TG notify failed (non-fatal): exit=$?" + fi diff --git a/.gitea/workflows/code-review.yaml b/.gitea/workflows/code-review.yaml index 54d090531..4ba1e2cad 100644 --- a/.gitea/workflows/code-review.yaml +++ b/.gitea/workflows/code-review.yaml @@ -110,18 +110,29 @@ jobs: FILES_DISPLAY: ${{ steps.ctx.outputs.files_display }} run: | set -euo pipefail - if [ -z "${TG_BOT_TOKEN:-}" ] || [ -z "${TG_CHAT_ID:-}" ]; then - echo "Telegram secret missing; skip start notification" - exit 0 - fi html_escape() { sed 's/&/\&/g; s//\>/g'; } COMMIT_ESC="$(printf '%s' "$COMMIT_MSG" | html_escape)" FILES_ESC="$(printf '%s\n' "$FILES_DISPLAY" | html_escape)" MSG="$(printf '🔍 Code Review 啟動\n──────────────────────\n📦 Commit %s 🌿 %s\n📝 %s\n📁 變更檔案：\n%s\n──────────────────────\n🤖 Hermes → OpenClaw → Elephant Alpha → NemoTron\n📊 即時進度：%s' "$SHORT_SHA" "$BRANCH" "$COMMIT_ESC" "$FILES_ESC" "$REPORT_URL" "$REPORT_URL")" - curl -fsS -X POST "https://api.telegram.org/bot${TG_BOT_TOKEN}/sendMessage" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --arg c "$TG_CHAT_ID" --arg t "$MSG" '{chat_id:$c,text:$t,parse_mode:"HTML",disable_web_page_preview:true}')" \ - >/dev/null + if AWOOI_CICD_STATUS=running \ + AWOOI_CICD_STAGE=code-review \ + AWOOI_CICD_JOB_NAME="Code Review 啟動" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_TRIGGERED_BY="${GITHUB_ACTOR:-CI}" \ + AWOOI_CICD_SUMMARY="${COMMIT_MSG}" \ + AWOOI_CICD_WORKFLOW_URL="${REPORT_URL}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "Code review start notification mirrored through AWOOI API" + else + if [ -z "${TG_BOT_TOKEN:-}" ] || [ -z "${TG_CHAT_ID:-}" ]; then + echo "Telegram secret missing and AWOOI API notify failed; skip start notification" + exit 0 + fi + curl -fsS -X POST "https://api.telegram.org/bot${TG_BOT_TOKEN}/sendMessage" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg c "$TG_CHAT_ID" --arg t "$MSG" '{chat_id:$c,text:$t,parse_mode:"HTML",disable_web_page_preview:true}')" \ + >/dev/null + fi - name: Run Deterministic Review if: steps.stale.outputs.skip != 'true' @@ -144,10 +155,6 @@ jobs: SHORT_SHA: ${{ steps.ctx.outputs.short_sha }} run: | set -euo pipefail - if [ -z "${TG_BOT_TOKEN:-}" ] || [ -z "${TG_CHAT_ID:-}" ]; then - echo "Telegram secret missing; skip completion notification" - exit 0 - fi REPORT=/tmp/code-review-report.json if [ ! -s "$REPORT" ]; then cat > "$REPORT" <<'JSON' @@ -180,7 +187,25 @@ jobs: TOP_ESC="$(printf '%s' "$TOP_ISSUE" | html_escape)" MSG="$(printf '%s Code Review 完成・%s\n──────────────────────\n🔴 CRITICAL %s 🟠 HIGH %s 🟡 MEDIUM %s 🟢 LOW %s\n──────────────────────\n⚠️ 主要問題\n%s\n\n🔍 整體風險等級\n%s：%s\n\n⚠️ 最高關注問題\n1. %s\n──────────────────────\n🤖 Elephant Alpha：%s ✅ %s\n📊 完整報告：%s' "$STATUS" "$SHORT_SHA" "$CRITICAL" "$HIGH" "$MEDIUM" "$LOW" "$ISSUE_LINE" "$RISK" "$SUMMARY_ESC" "$TOP_ESC" "$RISK" "$ACTION_ESC" "$REPORT_URL" "$REPORT_URL")" - curl -fsS -X POST "https://api.telegram.org/bot${TG_BOT_TOKEN}/sendMessage" \ - -H "Content-Type: application/json" \ - -d "$(jq -n --arg c "$TG_CHAT_ID" --arg t "$MSG" '{chat_id:$c,text:$t,parse_mode:"HTML",disable_web_page_preview:true}')" \ - >/dev/null + CICD_STATUS=success + if [ "$RISK" = "MEDIUM" ]; then CICD_STATUS=pending; fi + if [ "$RISK" = "HIGH" ] || [ "$RISK" = "CRITICAL" ]; then CICD_STATUS=failed; fi + if AWOOI_CICD_STATUS="${CICD_STATUS}" \ + AWOOI_CICD_STAGE=code-review \ + AWOOI_CICD_JOB_NAME="Code Review 完成・${RISK}" \ + AWOOI_CICD_COMMIT_SHA="${GITHUB_SHA}" \ + AWOOI_CICD_TRIGGERED_BY="${GITHUB_ACTOR:-CI}" \ + AWOOI_CICD_SUMMARY="CRITICAL=${CRITICAL}; HIGH=${HIGH}; MEDIUM=${MEDIUM}; LOW=${LOW}; ${SUMMARY}" \ + AWOOI_CICD_WORKFLOW_URL="${REPORT_URL}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "Code review completion notification mirrored through AWOOI API" + else + if [ -z "${TG_BOT_TOKEN:-}" ] || [ -z "${TG_CHAT_ID:-}" ]; then + echo "Telegram secret missing and AWOOI API notify failed; skip completion notification" + exit 0 + fi + curl -fsS -X POST "https://api.telegram.org/bot${TG_BOT_TOKEN}/sendMessage" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg c "$TG_CHAT_ID" --arg t "$MSG" '{chat_id:$c,text:$t,parse_mode:"HTML",disable_web_page_preview:true}')" \ + >/dev/null + fi diff --git a/.gitea/workflows/deploy-alerts.yaml b/.gitea/workflows/deploy-alerts.yaml index 30e9bfb79..11af3273d 100644 --- a/.gitea/workflows/deploy-alerts.yaml +++ b/.gitea/workflows/deploy-alerts.yaml @@ -50,6 +50,17 @@ jobs: SHORT_SHA="${{ github.sha }}" SHORT_SHA="${SHORT_SHA:0:7}" MSG="${EMOJI} Prometheus 告警規則部署 ${STATUS} (${SHORT_SHA})" - curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ - -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - --data-urlencode "text=${MSG}" || true + CICD_STATUS="success" + [ "$STATUS" != "success" ] && CICD_STATUS="failed" + if AWOOI_CICD_STATUS="${CICD_STATUS}" \ + AWOOI_CICD_STAGE=deploy-alerts \ + AWOOI_CICD_JOB_NAME="Prometheus 告警規則部署" \ + AWOOI_CICD_COMMIT_SHA="${{ github.sha }}" \ + AWOOI_CICD_SUMMARY="${MSG}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "Alert rule deploy notification mirrored through AWOOI API" + else + curl -fS -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ + -d "chat_id=${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + --data-urlencode "text=${MSG}" || true + fi diff --git a/.gitea/workflows/e2e-health.yaml b/.gitea/workflows/e2e-health.yaml index db05a5d03..c92859a34 100644 --- a/.gitea/workflows/e2e-health.yaml +++ b/.gitea/workflows/e2e-health.yaml @@ -54,7 +54,17 @@ jobs: - name: Notify Telegram on Failure if: failure() run: | - curl -s -X POST "https://api.telegram.org/bot${{ secrets.OPENCLAW_TG_BOT_TOKEN }}/sendMessage" \ - -d chat_id="${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ - -d parse_mode="HTML" \ - -d text="🔴 [E2E Health Check] 失敗%0A%0A📅 $(TZ=Asia/Taipei date '+%Y-%m-%d %H:%M')%0A🔗 API 健康檢查未通過%0A%0A請檢查 K3s 叢集狀態" + MSG="E2E Health Check 失敗；API 健康檢查未通過" + if AWOOI_CICD_STATUS=failed \ + AWOOI_CICD_STAGE=e2e-health \ + AWOOI_CICD_JOB_NAME="E2E Health Check" \ + AWOOI_CICD_COMMIT_SHA="${{ github.sha }}" \ + AWOOI_CICD_SUMMARY="${MSG}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "E2E failure notification mirrored through AWOOI API" + else + curl -s -X POST "https://api.telegram.org/bot${{ secrets.OPENCLAW_TG_BOT_TOKEN }}/sendMessage" \ + -d chat_id="${{ env.TELEGRAM_ALERT_CHAT_ID }}" \ + -d parse_mode="HTML" \ + -d text="🔴 [E2E Health Check] 失敗%0A%0A📅 $(TZ=Asia/Taipei date '+%Y-%m-%d %H:%M')%0A🔗 API 健康檢查未通過%0A%0A請檢查 K3s 叢集狀態" + fi diff --git a/.gitea/workflows/run-migration.yml b/.gitea/workflows/run-migration.yml index ba32b2ec3..38c9ba3f8 100644 --- a/.gitea/workflows/run-migration.yml +++ b/.gitea/workflows/run-migration.yml @@ -179,8 +179,19 @@ jobs: TG_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} TG_CHAT: ${{ env.TELEGRAM_ALERT_CHAT_ID }} run: | + STATUS="${{ job.status }}" + CICD_STATUS="success" + [ "$STATUS" != "success" ] && CICD_STATUS="failed" + if AWOOI_CICD_STATUS="${CICD_STATUS}" \ + AWOOI_CICD_STAGE=run-migration \ + AWOOI_CICD_JOB_NAME="Migration CI" \ + AWOOI_CICD_COMMIT_SHA="${{ github.sha }}" \ + AWOOI_CICD_SUMMARY="Migration CI: ${STATUS}" \ + scripts/ci/notify-awoooi-cicd.sh; then + echo "Migration notification mirrored through AWOOI API" + exit 0 + fi if [ -n "$TG_TOKEN" ] && [ -n "$TG_CHAT" ]; then - STATUS="${{ job.status }}" MSG="🗄️ Migration CI: \`${STATUS}\` — commit ${{ github.sha }}" curl -s -X POST "https://api.telegram.org/bot${TG_TOKEN}/sendMessage" \ -d chat_id="${TG_CHAT}" \ diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index b38bbc2b0..20a846e42 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -1439,6 +1439,39 @@ class AlertmanagerPayload(BaseModel): alerts: list[AlertmanagerAlert] +_CICD_JOB_STATUSES = frozenset({"running", "success", "failed", "pending"}) + + +def _cicd_job_status_from_alert(alert: AlertmanagerAlert) -> str: + """將 CI/CD Alertmanager label 轉成 TelegramGateway 支援的狀態。 + + 2026-05-12 Codex: Gitea workflow 先送進 AWOOI API，不能只靠 + severity=info 推 success，否則 failed/pending 事件進 AwoooP 後語義會失真。 + """ + labels = alert.labels or {} + for key in ("status", "job_status", "ci_status"): + value = str(labels.get(key) or "").strip().lower() + if value in _CICD_JOB_STATUSES: + return value + + severity = str(labels.get("severity") or "").strip().lower() + if severity == "info": + return "success" + if severity in {"critical", "error"}: + return "failed" + return "running" + + +def _cicd_duration_seconds_from_alert(alert: AlertmanagerAlert) -> int: + labels = alert.labels or {} + raw = labels.get("duration_seconds") or labels.get("duration") or 0 + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + return 0 + return max(value, 0) + + def is_internal_ip(client_ip: str) -> bool: """檢查是否為內網 IP""" import ipaddress @@ -2182,11 +2215,12 @@ async def alertmanager_webhook( telegram = get_telegram_gateway() # 解析 CI/CD 狀態 stage = alert.labels.get("stage", "") - job_status = "success" if alert.labels.get("severity") == "info" else "running" + job_status = _cicd_job_status_from_alert(alert) commit_sha = alert.labels.get("commit", "") triggered_by = alert.labels.get("triggered_by", "CI") workflow_url = alert.annotations.get("workflow_url", "") summary = alert.annotations.get("summary", alertname) + detail_message = alert.annotations.get("description", "") await telegram.send_cicd_progress( job_name=summary, @@ -2194,6 +2228,8 @@ async def alertmanager_webhook( stage=stage, commit_sha=commit_sha, triggered_by=triggered_by, + duration_seconds=_cicd_duration_seconds_from_alert(alert), + message=detail_message, workflow_url=workflow_url, ) @@ -2425,7 +2461,7 @@ async def alertmanager_webhook( record_alert_chain_success("alertmanager") return AlertResponse( success=True, - message=f"✅ TYPE-1 純資訊告警已通知 (no LLM)", + message="✅ TYPE-1 純資訊告警已通知 (no LLM)", alert_id=alert_id, approval_created=False, ) diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 0a1e619f7..1c64227d0 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -43,7 +43,6 @@ from src.services.security_interceptor import ( get_security_interceptor, ) from src.services.chat_manager import get_chat_manager -from src.services.notification_matrix import resolve_chat_ids # ADR-093 路由矩陣 # ============================================================================= # Snooze/Silence Redis Keys (2026-03-27 P1 優化) @@ -996,12 +995,18 @@ class CICDProgressMessage: safe_url = html.escape(self.workflow_url, quote=True) workflow_link = f"\n🔗 Workflow" + detail = "" + if self.message: + safe_message = html.escape(self.message[:240]) + detail = f"\n📝 {safe_message}" + # 簡潔訊息 stage_label = f" | {safe_stage}" if safe_stage else "" message = ( f"{status_emoji} [AWOOOI CI/CD]{stage_label}\n" f"📦 {safe_job}{duration_str}" f"{commit_info}" + f"{detail}" f"{workflow_link}" ) @@ -6168,7 +6173,7 @@ class TelegramGateway: s = await k8s.get_pod_status_summary(namespace="awoooi-prod") running, total = s.get("running", 0), s.get("total", 0) problems = s.get("problem_pods", []) - lines = [f"🖥 Cluster 狀態", f"• Pods: {running}/{total} Running"] + lines = ["🖥 Cluster 狀態", f"• Pods: {running}/{total} Running"] if problems: lines.append(f"• 異常: {len(problems)} 個") for p in problems[:5]: diff --git a/apps/api/tests/test_cicd_alertmanager_mapping.py b/apps/api/tests/test_cicd_alertmanager_mapping.py new file mode 100644 index 000000000..14a873930 --- /dev/null +++ b/apps/api/tests/test_cicd_alertmanager_mapping.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from src.api.v1.webhooks import ( + AlertmanagerAlert, + _cicd_duration_seconds_from_alert, + _cicd_job_status_from_alert, +) +from src.services.telegram_gateway import CICDProgressMessage + + +def test_cicd_alert_status_label_overrides_severity() -> None: + alert = AlertmanagerAlert( + status="firing", + labels={"status": "failed", "severity": "info"}, + annotations={}, + ) + + assert _cicd_job_status_from_alert(alert) == "failed" + + +def test_cicd_alert_legacy_info_severity_remains_success() -> None: + alert = AlertmanagerAlert( + status="firing", + labels={"severity": "info"}, + annotations={}, + ) + + assert _cicd_job_status_from_alert(alert) == "success" + + +def test_cicd_alert_duration_is_sanitized() -> None: + assert _cicd_duration_seconds_from_alert( + AlertmanagerAlert( + status="firing", + labels={"duration_seconds": "91"}, + annotations={}, + ) + ) == 91 + assert _cicd_duration_seconds_from_alert( + AlertmanagerAlert( + status="firing", + labels={"duration_seconds": "-5"}, + annotations={}, + ) + ) == 0 + + +def test_cicd_progress_message_includes_safe_detail() -> None: + message = CICDProgressMessage( + job_name="Code Review", + status="failed", + stage="review", + commit_sha="abcdef1234567890", + message="最高風險