diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 618d7fc..c8ba063 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -17,6 +17,8 @@ name: CD Pipeline +permissions: read-all + on: push: branches: [main] @@ -32,6 +34,7 @@ on: - 'services/**' - 'routes/**' - 'database/**' + - 'governance/**' - 'templates/**' - 'static/**' - 'monitoring/prometheus.yml' @@ -58,7 +61,7 @@ on: # 新 push 立即取消舊 job，只部署最新版本 concurrency: - group: cd-deploy-${{ github.ref }} + group: cd-deploy-${{ gitea.ref }} cancel-in-progress: true jobs: @@ -67,14 +70,29 @@ jobs: runs-on: ewoooc-host steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 + - name: 從目前 Gitea 取回原始碼 + env: + GITEA_TOKEN: ${{ gitea.token }} + GITEA_SERVER_URL: ${{ gitea.server_url }} + GITEA_REPOSITORY: ${{ gitea.repository }} + GITEA_SHA: ${{ gitea.sha }} + run: | + set -eu + AUTH_HEADER=$(printf 'x-access-token:%s' "$GITEA_TOKEN" | base64 | tr -d '\n') + git init . + git remote remove origin 2>/dev/null || true + git remote add origin "${GITEA_SERVER_URL}/${GITEA_REPOSITORY}.git" + git -c http.extraHeader="Authorization: Basic ${AUTH_HEADER}" \ + fetch --no-tags --depth=2 origin "$GITEA_SHA" + git checkout --force --detach FETCH_HEAD + git clean -fdx - name: 取得 Commit 資訊 id: commit + env: + GITEA_SHA: ${{ gitea.sha }} run: | - echo "short_sha=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + echo "short_sha=${GITEA_SHA::7}" >> $GITHUB_OUTPUT echo "message=$(git log -1 --pretty=%s | head -c 60)" >> $GITHUB_OUTPUT VERSION=$(sed -n "s/^SYSTEM_VERSION[[:space:]]*=[[:space:]]*[\"']\([^\"']*\)[\"'].*/\1/p" config.py | head -1) echo "version=$VERSION" >> $GITHUB_OUTPUT @@ -84,7 +102,7 @@ jobs: - name: 偵測部署類型 id: deploy_type run: | - if [[ "${{ github.event.inputs.force_rebuild }}" == "true" ]]; then + if [[ "${{ gitea.event.inputs.force_rebuild }}" == "true" ]]; then echo "type=rebuild" >> $GITHUB_OUTPUT echo "label=🔨 強制重建 Docker Image" >> $GITHUB_OUTPUT exit 0 @@ -110,6 +128,10 @@ jobs: echo "ℹ️ AI 觀測台 QA: skipped" fi + - name: 資訊安全與治理 release gate + run: | + python3 scripts/ops/report_security_governance_review.py --compact --strict + # 設定 SSH 金鑰 + 主機驗證（C2 fix: 移除 StrictHostKeyChecking no） - name: 設定 SSH 金鑰 env: @@ -139,7 +161,7 @@ jobs: env: COMMIT_MSG: ${{ steps.commit.outputs.message }} COMMIT_SHA: ${{ steps.commit.outputs.short_sha }} - COMMIT_ACTOR: ${{ github.actor }} + COMMIT_ACTOR: ${{ gitea.actor }} DEPLOY_LABEL: ${{ steps.deploy_type.outputs.label }} TG_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} TG_CHAT: ${{ secrets.TELEGRAM_CHAT_ID }} @@ -400,8 +422,8 @@ jobs: continue-on-error: true env: WEBHOOK_TOKEN: ${{ secrets.INTERNAL_WEBHOOK_TOKEN }} - COMMIT_SHA_FULL: ${{ github.sha }} - BRANCH_NAME: ${{ github.ref_name }} + COMMIT_SHA_FULL: ${{ gitea.sha }} + BRANCH_NAME: ${{ gitea.ref_name }} DEPLOY_TYPE: ${{ steps.deploy_type.outputs.type }} run: | CHANGED=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") diff --git a/CONSTITUTION.md b/CONSTITUTION.md index 55c3c72..fcb30ff 100644 --- a/CONSTITUTION.md +++ b/CONSTITUTION.md @@ -2,8 +2,9 @@ > 本文件定義專案開發的核心準則與不可違反的規範 > **建立日期**: 2026-01-12 -> **當前版本**: V10.64 (Keep only latest 50 AI product picks pending) -> **最後更新**: 2026-05-01 +> **當前版本**: V10.771 (Security governance automation review) +> **最後更新**: 2026-07-10 +> **治理基準**: `global_product_governance_v2` + ADR-038 --- @@ -155,10 +156,11 @@ - ❌ **禁止**: 使用 5888 或其他端口（已廢棄） - **配置位置**: `config.py` 的 `PUBLIC_URL` -### 第 16 條：資料庫路徑（絕對禁止違反） -- ✅ **正確**: `data/momo_database.db` -- ❌ **禁止**: 更改資料庫位置或名稱 -- **配置位置**: `config.py` 的 `DATABASE_PATH` +### 第 16 條：資料庫 runtime（絕對禁止違反） +- ✅ **正式環境**: PostgreSQL + pgvector，runtime identity 為 `momo-db`；以 production SQL receipt 與 migration readback 為真相。 +- ✅ **本機開發**: 可使用 `data/momo_database.db` 作為相容 fallback，但不得覆蓋 production truth。 +- ❌ **禁止**: 自動 restart / stop / recreate `momo-db`，或把 SQLite snapshot 誤報為正式資料庫完成證據。 +- **配置位置**: `config.py` 的 `DATABASE_PATH` / `DATABASE_TYPE` ### 第 17 條：時區設定（絕對禁止違反） - ✅ **正確**: 使用 `TAIPEI_TZ = pytz.timezone('Asia/Taipei')` @@ -195,14 +197,14 @@ - ❌ **禁止**: 修改功能但不更新版本號 ### 第 20 條：備份系統（強制要求） -- ✅ **正確**: 重大更新前必須執行 `python backup_system.py` -- ✅ **排除目錄**: `['backups', '__pycache__', '.git', '.idea', '.vscode', 'bin', 'bin 2']` -- ❌ **禁止**: 跳過備份直接上線 +- ✅ **正確**: 高風險資料變更前必須取得 fresh backup、checksum、offsite coverage 與可追溯 receipt。 +- ✅ **正確**: 備份完成不等於可恢復；必須定期在隔離環境執行 non-destructive restore drill 並量測 RPO/RTO。 +- ❌ **禁止**: 未驗證 restore receipt 就宣稱備援閉環完成。 ### 第 21 條：TODO 文件維護（強制要求） -- ✅ **正確**: 完成功能後更新 `TODO_NEXT_STEPS.txt` 標記 ✅ -- ✅ **正確**: 記錄修改的檔案與行數 -- ❌ **禁止**: 完成任務但不更新文件 +- ✅ **正確**: 工作優先序與狀態以 `docs/guides/ai_automation_mainline_work_items.md` 及機器可讀 governance review 為準。 +- ✅ **正確**: 完成狀態必須分開記錄 source/test、Gitea CD、production runtime、visible evidence。 +- ❌ **禁止**: 只更新 TODO、文件或欄位就把工作標為 completed。 --- @@ -283,9 +285,10 @@ ## 第十章：安全性規範 ### 第 31 條：敏感資訊保護（絕對禁止違反） -- ✅ **正確**: 所有 API Token、密碼存放於 `config.py` -- ❌ **禁止**: 硬編碼敏感資訊於程式碼中 -- ❌ **禁止**: 提交 `config.py` 至公開 Git 倉庫 +- ✅ **正確**: 程式碼與 `config.py` 只保存環境變數名稱、驗證與 fail-secure 邏輯；secret value 只存在受控 runtime secret store / Gitea Secret。 +- ✅ **正確**: webhook secret、API token、password、private key 必須使用 constant-time comparison、最小權限、輪替與失效機制。 +- ❌ **禁止**: 讀取、記錄、同步或提交明文 secret、raw session、authorization header、runtime secret volume。 +- ❌ **禁止**: 使用不安全預設值、query-string token 或 production login bypass。 ### 第 32 條：SQL 注入防護（絕對禁止違反） - ✅ **正確**: 使用 SQLAlchemy ORM 的參數化查詢 @@ -304,21 +307,30 @@ - ✅ **正確**: JavaScript 中使用 `textContent` 而非 `innerHTML` - ❌ **禁止**: 直接插入未驗證的使用者輸入 +### 第 33.1 條：存取控制與 Session（絕對禁止違反） +- ✅ **正確**: 除明確 public allowlist 外，業務資料、operator、admin、logs、import、CI/CD、crawler 與 notification surfaces 一律 deny-by-default。 +- ✅ **正確**: production cookie 必須 `Secure`、`HttpOnly`、適當 `SameSite`；敏感回應須 `no-store` 並具安全 response headers。 +- ✅ **正確**: internal/webhook endpoint 必須使用 signed transport 或 fail-secure token，且具 replay/duplicate guard。 +- ❌ **禁止**: 以 CSRF exemption 取代 authentication/authorization，或讓 shared password 自動等同永久 admin identity。 + +### 第 33.2 條：Software Supply Chain（絕對禁止違反） +- ✅ **正確**: Gitea CD 僅從目前 Gitea、受控內部 registry/mirror 或 repo-owned immutable artifact 取回 source/action/image。 +- ✅ **正確**: release gate 必須逐步具備 exact dependency lock、SAST、SCA、secret scan、SBOM、image digest、provenance 與部署版本 readback。 +- ❌ **禁止**: `uses: actions/*` 或任何 GitHub/raw/codeload/ghcr runtime dependency。 + --- ## 第十一章：部署與維運規範 ### 第 34 條：伺服器啟動（強制要求） -- ✅ **正確**: 使用 `nohup python3 app.py > /dev/null 2>&1 &` 背景執行 -- ✅ **正確**: 部署前檢查 Port 80 是否已被佔用 -- ❌ **禁止**: 直接執行 `python3 app.py`（關閉終端機會停止服務） +- ✅ **正確**: 正式服務由 188 Docker Compose + Gunicorn 管理，部署入口為 Gitea CD。 +- ✅ **正確**: image rebuild 必須 build 成功後才切換，並保留 internal/external `/health` readback。 +- ❌ **禁止**: 以裸 Python process、背景 shell 或未受控 SSH 指令作為正式部署方式。 ### 第 35 條：伺服器重啟（強制要求） -- ✅ **正確步驟**: - 1. `pkill -9 -f "python3 app.py"` (停止舊程序) - 2. `sleep 2` (等待端口釋放) - 3. `nohup python3 app.py > /dev/null 2>&1 &` (啟動新程序) - 4. `ps aux | grep "[p]ython3 app.py"` (確認運行) +- ✅ **正確**: sync 模式對 `momo-pro-system` 使用 Gunicorn HUP，scheduler/telegram-bot 才做 bounded restart。 +- ✅ **正確**: rebuild 只可 recreate `momo-app scheduler telegram-bot`，並以 post-verifier 確認版本與健康。 +- ❌ **禁止**: `docker compose --remove-orphans`、跨專案容器操作或任何 `momo-db` 生命週期操作。 ### 第 36 條：日誌輪替（建議執行） - ✅ **建議**: 定期清理過大的日誌檔案 (> 100MB) @@ -351,13 +363,14 @@ - **Hermes（採集層）**: Ollama 三主機級聯（GCP-A → GCP-B → 111），負責 embedding、去重、品質分數計算。成本 = $0 - **NemoTron（處理層）**: qwen3:14b Ollama-first，NVIDIA NIM Llama 3.1 8B 僅作備援，負責 tool calling 邏輯路由與 DB 寫入。NIM 限額 80 次/天 - **OpenClaw（應用層）**: Ollama-first；Gemini 僅作備援或 ADR-028 鎖定場景，負責最終 PPT 生成、洞察報告對外輸出。 -- **ElephantAlpha（編排層）**: 負責跨 Agent orchestration、HITL、AutoHeal bridge 與受控執行計畫，不可繞過安全入口 +- **ElephantAlpha（編排層）**: 負責跨 Agent orchestration、risk/policy decision、AutoHeal bridge 與受控執行計畫；low/medium/high 預設 controlled apply，critical 才走 break-glass。 - ❌ **禁止**：讓 OpenClaw 做 Hermes 層的苦力工作（高算力浪費） - ❌ **禁止**：讓 Hermes 直接生成對外報告（品質不足） - ❌ **禁止**：讓 ElephantAlpha 直接繞過 EventRouter / AutoHeal / ADR-011 執行高風險副作用 ### 第 41 條：AI 學習數據雙寫（絕對禁止違反） -- ✅ **正確**：所有 AI 產出（PPT 洞察、競品分析、對話記錄、Agent action、自癒紀錄）必須**雙寫** PostgreSQL `ai_insights` + pgvector embedding +- ✅ **正確**：通過 PromotionGate 且允許成為正式知識的 AI 產出，必須**雙寫** PostgreSQL `ai_insights` + pgvector embedding，並留下 durable acknowledgement。 +- ✅ **正確**：preview、blocked page、PixelRAG visual evidence、matcher candidate 與失敗 receipt 必須保持 evidence-only，不得為了追求雙寫而污染正式知識或價格表。 - ❌ **禁止**：只寫 DB 不寫 KM（RAG 無法語意搜尋） - ❌ **禁止**：只寫 KM 不寫 DB（精確 period/sku 查詢無法命中） - **理由**：DB 是精準命中，KM 是語意搜尋，兩者互補缺一不可（ADR-007） @@ -388,6 +401,8 @@ - ✅ **正確**：AutoHeal 是自癒副作用入口，失敗時必須安全降級為 alert / log / file queue - ✅ **正確**：L2 safe action 必須可審計、可回放、低副作用 - ✅ **正確**：AI 自動化觀測變更需同步 `/metrics`、Smoke dashboard 與 Grafana provisioning，避免告警閉環變成黑盒 +- ✅ **正確**：每個 side-effect run 必須以同一 `trace_id/run_id/work_item_id` 串接 sensor、identity、SOT diff、decision、risk、dry-run、bounded execution、independent verifier、rollback/retry 與 KM/RAG/PlayBook write acknowledgement。 +- ✅ **正確**：完成度必須同時公布 program、asset coverage、runtime closure；任一 closure 段缺失只能標示 partial/degraded。 - ❌ **禁止**：自動 restart / stop / recreate `momo-db` 或 `momo-postgres` - ❌ **禁止**：AI 分析失敗導致 Telegram 告警完全不送出 - **依據**：ADR-012、ADR-013、ADR-018 diff --git a/app.py b/app.py index 0c80c38..af48aae 100644 --- a/app.py +++ b/app.py @@ -170,9 +170,13 @@ app.config['SESSION_COOKIE_HTTPONLY'] = True # 防止 JavaScript 存取 cookie app.config['SESSION_COOKIE_SAMESITE'] = 'Lax' # 防止 CSRF 攻擊 app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=24) # Session 有效期 24 小時（延長避免長時間閒置斷線） -# 如果使用 HTTPS，啟用 SECURE cookie（本地開發時應設為 False） -# 注意：如果您的系統部署在 HTTPS 環境，請將 .env 中的 USE_HTTPS 設為 true -USE_HTTPS = os.getenv('USE_HTTPS', 'false').lower() == 'true' +# 正式入口為 HTTPS 時 secure cookie 預設開啟；本機 HTTP 開發可明確設 USE_HTTPS=false。 +_use_https_env = os.getenv('USE_HTTPS') +USE_HTTPS = ( + _use_https_env.lower() == 'true' + if _use_https_env is not None + else os.getenv('PUBLIC_URL', 'https://mo.wooo.work').lower().startswith('https://') +) if USE_HTTPS: app.config['SESSION_COOKIE_SECURE'] = True sys_log.info("[Security] ✅ HTTPS 模式已啟用，Session cookie 僅透過 HTTPS 傳輸") @@ -185,8 +189,8 @@ else: app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 sys_log.info("[Security] ✅ Flask 安全配置已載入") -sys_log.info(f"[Security] • Session 有效期: 2 小時") -sys_log.info(f"[Security] • 檔案上傳限制: 10 MB") +sys_log.info(f"[Security] • Session 有效期: 24 小時") +sys_log.info(f"[Security] • 檔案上傳限制: 100 MB") sys_log.info(f"[Security] • CSRF 防護: SameSite=Lax") sys_log.info(f"[Security] • XSS 防護: HttpOnly=True") @@ -636,6 +640,37 @@ def get_dashboard_stats(): # ================= 🛣️ 4. Flask 路由 ================= +# 敏感業務/維運端點由中央 policy 統一 fail-closed。 +from services.http_access_policy_service import enforce_http_access_policy # noqa: E402 +app.before_request(enforce_http_access_policy) + + +@app.after_request +def add_security_response_headers(response): + """全域瀏覽器安全基線；CSP 先以 report-only 收斂既有 inline 資產。""" + response.headers.setdefault('X-Content-Type-Options', 'nosniff') + response.headers.setdefault('X-Frame-Options', 'SAMEORIGIN') + response.headers.setdefault('Referrer-Policy', 'strict-origin-when-cross-origin') + response.headers.setdefault( + 'Permissions-Policy', + 'camera=(), microphone=(), geolocation=(), payment=()', + ) + response.headers.setdefault( + 'Strict-Transport-Security', + 'max-age=31536000; includeSubDomains', + ) + response.headers.setdefault( + 'Content-Security-Policy-Report-Only', + "default-src 'self'; object-src 'none'; base-uri 'self'; frame-ancestors 'self'; " + "img-src 'self' data: https:; font-src 'self' data: https:; " + "style-src 'self' 'unsafe-inline' https:; script-src 'self' 'unsafe-inline' https:; " + "connect-src 'self' https: wss:", + ) + if session.get('logged_in'): + response.headers.setdefault('Cache-Control', 'private, no-store') + return response + + # Session 自動續期機制 @app.before_request def refresh_session(): diff --git a/auth.py b/auth.py index 4fc2c5b..88d4c6f 100644 --- a/auth.py +++ b/auth.py @@ -1,6 +1,7 @@ +import hmac import os import time -from flask import render_template, redirect, url_for, request, session, flash +from flask import jsonify, render_template, redirect, url_for, request, session, flash from functools import wraps from werkzeug.security import check_password_hash from config import LOGIN_PASSWORD @@ -9,11 +10,17 @@ from datetime import datetime, timedelta # ========================================== # 🔓 登入功能開關 # ========================================== -# 設定環境變數 DISABLE_LOGIN=true 可關閉登入驗證 -DISABLE_LOGIN = os.getenv('DISABLE_LOGIN', 'false').lower() == 'true' +# 登入繞過只允許測試環境明確開啟，避免 production 因單一 env 誤設而全站失守。 +_DISABLE_LOGIN_REQUESTED = os.getenv('DISABLE_LOGIN', 'false').lower() == 'true' +_ALLOW_INSECURE_LOGIN_BYPASS = ( + os.getenv('MOMO_ALLOW_INSECURE_CONFIG_FOR_TESTS', 'false').lower() == 'true' +) +DISABLE_LOGIN = _DISABLE_LOGIN_REQUESTED and _ALLOW_INSECURE_LOGIN_BYPASS if DISABLE_LOGIN: print("⚠️ 警告：登入驗證已關閉 (DISABLE_LOGIN=true)") +elif _DISABLE_LOGIN_REQUESTED: + print("[Security] 已拒絕 DISABLE_LOGIN：僅測試環境可使用登入繞過") # ========================================== # 🔒 登入失敗追蹤與帳號鎖定機制 @@ -164,6 +171,41 @@ def login_required(f): return decorated_view +def require_authenticated_request(): + """供中央 HTTP policy 使用；API 回 401，頁面導向登入。""" + if DISABLE_LOGIN or session.get('logged_in'): + return None + + is_api_request = request.path.startswith('/api/') or '/api/' in request.path + if is_api_request: + return jsonify({ + 'success': False, + 'error': 'authentication_required', + 'message': '此端點需要登入。', + }), 401 + return redirect(url_for('login')) + + +def internal_key_required(f): + """Require the shared internal transport key for non-session service calls.""" + @wraps(f) + def decorated_view(*args, **kwargs): + expected = os.getenv('INTERNAL_API_KEY', '').strip() + provided = request.headers.get('X-Internal-Key', '').strip() + if not expected: + return jsonify({ + 'success': False, + 'error': 'internal_auth_not_configured', + }), 503 + if not provided or not hmac.compare_digest(provided, expected): + return jsonify({ + 'success': False, + 'error': 'invalid_internal_key', + }), 401 + return f(*args, **kwargs) + return decorated_view + + def get_current_user(): """ 取得目前登入的使用者資訊 @@ -176,10 +218,12 @@ def get_current_user(): return { 'logged_in': True, + 'user_id': session.get('user_id'), 'login_time': session.get('login_time'), 'client_ip': session.get('client_ip'), 'role': session.get('role', 'admin'), # 預設為 admin（向後兼容） - 'username': session.get('username', 'admin') + 'username': session.get('username', 'admin'), + 'auth_source': session.get('auth_source', 'legacy_shared_password'), } @@ -297,17 +341,21 @@ def init_auth_routes(app): is_password_valid = check_password_hash(LOGIN_PASSWORD, input_password) else: # 向後兼容：明文比對（僅用於過渡期） - is_password_valid = (input_password == LOGIN_PASSWORD) + is_password_valid = hmac.compare_digest(input_password, LOGIN_PASSWORD) if is_password_valid: print("⚠️ 警告：系統仍在使用明文密碼，請盡快執行密碼雜湊更新") # 4. 驗證結果處理 if is_password_valid: # 登入成功 + session.clear() session.permanent = True session['logged_in'] = True session['login_time'] = datetime.now().isoformat() session['client_ip'] = client_ip + session['role'] = 'admin' + session['username'] = 'legacy-admin' + session['auth_source'] = 'legacy_shared_password' # 清除失敗記錄 clear_login_attempts(client_ip) @@ -345,9 +393,7 @@ def init_auth_routes(app): 登出路由：清除 session 並導回登入頁。 """ client_ip = get_client_ip() - session.pop('logged_in', None) - session.pop('login_time', None) - session.pop('client_ip', None) + session.clear() print(f"👋 使用者已登出 | IP: {client_ip}") return redirect(url_for('login')) diff --git a/config.py b/config.py index 3fa492f..3e632f2 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.770" +SYSTEM_VERSION = "V10.771" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 9d2b355..865ed8a 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,20 @@ # PChome 業績成長自動化作戰系統 — AI 競價情報模組 Single Source of Truth > **最後更新**: 2026-07-10 (台北時間) -> **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯；PChome 後台業績匯入韌性已補強；產品定位正名為「PChome 業績成長自動化作戰系統」；外部市場來源正規化層、自動同步、作戰清單與價格參考表優先讀取、CSV 備援預檢、前台操作入口、高可見頁面繁中化守門、比價/作戰 UI 工作台化、AI 密集工作台文字密度守門、跨平台來源治理、商品身份 UI 契約、sitewide visual QA runtime readback、external MCP/RAG integration monitoring、PixelRAG RAG candidate replay monitoring、PixelRAG OCR/VLM replay contract monitoring、PixelRAG application portfolio、PChome auto-policy authorization guard monitoring、decision preflight machine evidence monitoring、decision closeout monitoring、authorization issuer gate monitoring、signing decision preflight monitoring、signing decision closeout monitoring、signing issuer guard monitoring、signing issuer closeout monitoring、signing execution preflight monitoring、signing execution closeout monitoring、signed receipt preflight monitoring、signed receipt closeout monitoring、signed receipt evidence intake monitoring、detached verification evidence validation monitoring、verifier receipt closeout monitoring、authorization evidence execution preflight monitoring、authorization evidence execution closeout monitoring、controlled apply final preflight monitoring、controlled dry-run package monitoring 與 controlled dry-run receipt closeout monitoring 已建立，GCP embedding 熔斷延後處理、110 proxy rescue 與 direct host health skip 已建立 -> **適用版本**: V10.770 +> **狀態**: 🟠 Partial。四 Agent、PixelRAG、MCP/RAG registry、PChome controlled-preview families 與多項 smoke/readback 已建立；但 access control、database identity/RBAC、full asset reconciliation、software supply chain、same-run controlled apply、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 +> **適用版本**: V10.771 + +--- + +## 零之負一、資訊安全與治理自動化基準（2026-07-10） + +- ADR-038 採 NIST CSF 2.0 `GOVERN / IDENTIFY / PROTECT / DETECT / RESPOND / RECOVER` 作為 program 結構，OWASP ASVS 5.0.0 L2 作為應用基準，NIST SSDF SP 800-218 v1.1 作為供應鏈基準。 +- `governance/ewoooc_asset_inventory.json` 是 full-asset seed；欄位完整不等於 runtime reconciliation 完成。 +- `services/http_access_policy_service.py` 統一保護 vendor、import、CI/CD、crawler、category、logs 與 operator surfaces；public exception 必須有明確 transport auth 或 public allowlist。 +- `services/security_governance_review_service.py`、`scripts/ops/report_security_governance_review.py` 與 `/api/ai-automation/security-governance-review` 提供 machine-readable checks、completion、release gate 與 ordered work items。 +- Gitea CD 必須執行 `report_security_governance_review.py --strict`，且不得解析 GitHub action/source/image。 +- production 安全與治理完成度必須分開顯示 program、asset coverage、runtime closure；目前結論是 partial，不是 complete。 +- 目前 ordered P0 以 `docs/guides/ai_automation_mainline_work_items.md` 為準：先關閉 access exposure，再做 identity/RBAC、webhook trust、supply chain、asset reconciliation、controlled-apply envelope、internal RAG canary 與 MCP/RAG runtime closure。 --- diff --git a/docs/adr/ADR-012-agent-action-ladder.md b/docs/adr/ADR-012-agent-action-ladder.md index e1a0cdb..ff04b33 100644 --- a/docs/adr/ADR-012-agent-action-ladder.md +++ b/docs/adr/ADR-012-agent-action-ladder.md @@ -5,6 +5,7 @@ - **Deciders**: 統帥 - **Related**: ADR-001（三 Agent 分工）, ADR-004（NemoTron Fallback）, ADR-007（AI Dual-Write）, ADR-011（跨專案隔離）, ADR-018（四 AI Agent 自動化控制面） - **Note**: [ADR-020](ADR-020-code-review-full-autoheal.md) 局部覆寫本 ADR 對「post-deploy code review pipeline」場景的 L3 HITL 規定 — 該場景改採全自動修復 + Git/CI/CD 回滾安全網。本 ADR 對 schema migration / 流量切換 / customer-facing 廣播 / AIOps prod SSH 等其他 L3 場景仍生效。 +- **2026-07-10 Sunset**: [ADR-038](ADR-038-security-governance-automation-control-plane.md) 覆寫 generic owner/manual/HITL terminal gate。low/medium/high 改採 risk-based controlled apply；critical break-glass 類型仍保留外部授權。 ## Context diff --git a/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md b/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md index 18f7a7e..0128e70 100644 --- a/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md +++ b/docs/adr/ADR-018-four-agent-ai-automation-control-plane.md @@ -4,6 +4,7 @@ - **Date**: 2026-04-29 - **Deciders**: 統帥 - **Related**: ADR-001, ADR-004, ADR-007, ADR-009, ADR-011, ADR-012, ADR-013 +- **2026-07-10 Update**: ADR-038 將 ElephantAlpha 的 generic HITL 改為 risk-based controlled apply；只有 critical break-glass 類型維持外部授權。 ## Context diff --git a/docs/adr/ADR-038-security-governance-automation-control-plane.md b/docs/adr/ADR-038-security-governance-automation-control-plane.md new file mode 100644 index 0000000..aef89d4 --- /dev/null +++ b/docs/adr/ADR-038-security-governance-automation-control-plane.md @@ -0,0 +1,101 @@ +# ADR-038: Security Governance Automation Control Plane + +- **Status**: Accepted +- **Date**: 2026-07-10 +- **Decider**: Product owner +- **Supersedes**: generic owner/manual/HITL terminal gates in ADR-012 and ADR-018 for low/medium/high risk work +- **Related**: ADR-011, ADR-012, ADR-013, ADR-018, ADR-020, ADR-031, ADR-032, ADR-033 + +## Context + +EwoooC has many AI automation receipts, smoke families and guarded preview stages, but the program still cannot prove a complete information-security and governance loop. Production review found four structural gaps: + +1. Sensitive routes evolved blueprint by blueprint instead of behind one deny-by-default policy. +2. Asset, identity, dependency, backup and AI runtime evidence are not reconciled by one control plane. +3. Preview/review stages do not share one `trace_id/run_id/work_item_id` through execution, verifier, rollback and learning. +4. Source/test/UI evidence has repeatedly been reported ahead of production runtime closure. + +## Decision + +### 1. Control model + +Use NIST CSF 2.0 as the program structure: + +- GOVERN: policy, ownership, risk, supply chain and measurable targets. +- IDENTIFY: canonical asset graph, dependency inventory and risk assessment. +- PROTECT: identity, authorization, secure defaults, data and platform controls. +- DETECT: fresh logs, metrics, traces, anomaly signals and control drift. +- RESPOND: correlated incident, bounded action, verifier and communication receipts. +- RECOVER: rollback/retry, backup/restore, RPO/RTO and learning writeback. + +OWASP ASVS 5.0.0 L2 is the application baseline, with selected L3 controls for operator, webhook, AI execution and admin surfaces. NIST SSDF SP 800-218 v1.1 is the software-supply-chain baseline. + +### 2. Source of truth + +Completion evidence is ordered: + +1. production runtime receipt, independent post-verifier and durable DB receipt; +2. Gitea main/deploy marker/CD run/production readback; +3. committed machine-readable inventory, policy and tests; +4. docs and snapshots; +5. conversation. + +Lower evidence cannot promote a higher incomplete layer to completed. + +### 3. Controlled apply + +Low, medium and high risk work defaults to controlled apply with blast-radius controls. Only critical break-glass classes remain externally gated: plaintext secrets, destructive database/storage operations, host/network cutovers, credentialed offensive scans, paid provider or core runtime replacement without replay/shadow/canary, destructive Git operations and raw secret-volume access. + +Every side-effect run must retain one `trace_id/run_id/work_item_id` across: + +`sensor -> normalize -> correlate -> decide -> risk/check -> bounded execute -> independent verify -> retry/rollback -> learn/writeback` + +Any missing stage produces `partial` or `degraded`, never completed. + +### 4. Machine-readable controls + +- `governance/ewoooc_asset_inventory.json` is the canonical asset seed and must be reconciled against runtime receipts. +- `services/http_access_policy_service.py` is the central policy for sensitive HTTP surfaces. +- `services/security_governance_review_service.py` produces NIST/OWASP-aligned findings, completion scores and ordered work items. +- `scripts/ops/report_security_governance_review.py --strict` is the source release gate. +- `/api/ai-automation/security-governance-review` is the protected production readback. + +### 5. Legacy policy sunset + +| Legacy policy | Owner | Expiry | Replacement | Verifier | +|---|---|---|---|---| +| Generic HITL/manual terminal for low/medium/high | product-governance | 2026-07-10 | risk-based controlled apply in ADR-038 | governance review work-item state | +| Shared password treated as complete RBAC | security | 2026-07-31 | database identity + least-privilege role migration | anonymous/role route matrix | +| `actions/checkout@v4` in Gitea CD | delivery | 2026-07-10 | Gitea job-token source checkout | Gitea-only supply-chain gate | +| Backup script treated as recovery completion | resilience | 2026-07-31 | checksum/offsite/isolated restore receipt | RPO/RTO restore verifier | +| Registry or preview readiness treated as runtime completion | AI platform | 2026-07-17 | MCP/RAG live health and internal RAG canary | production smoke/readback | + +## Alternatives Considered + +- Keep adding domain-specific receipt stages: rejected because it increases surface area without unifying assurance. +- Make every action human-approved: rejected for low/medium/high because it defeats the product's AI automation goal. +- Make every action fully autonomous without independent verification: rejected because it cannot bound production risk. +- Adopt an external governance platform as the source of truth: rejected until internal asset and receipt contracts are stable and exportable. + +## Consequences + +Positive: + +- Security and AI automation use the same measurable closure contract. +- Drift becomes a work item instead of an undocumented manual observation. +- Release gating can block critical source regressions without pretending all runtime debt is complete. +- Product, asset and runtime completion remain separate and honest. + +Costs and risks: + +- Existing routes and automation lanes require gradual migration. +- Shared identity, webhook secret activation, non-root containers and restore drills need canary rollout. +- The initial asset inventory has complete required fields but runtime reconciliation remains partial. + +## Verification + +1. `python3 scripts/ops/report_security_governance_review.py --strict` returns exit 0. +2. Anonymous requests to sensitive read/operator surfaces return 401 or login redirect. +3. Gitea workflow contains no external `uses:` action or forbidden GitHub source. +4. Production endpoint reports zero release-blocking source failures. +5. Program, asset and runtime closure percentages are reported separately. diff --git a/docs/adr/README.md b/docs/adr/README.md index e9f9337..9ae74f2 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -59,6 +59,7 @@ | [035](ADR-035-cross-platform-market-campaign-intelligence.md) | 跨平台市場活動情報系統 | Accepted | 2026-05-06 | | [036](ADR-036-fastapi-strangler-not-frontend-prerequisite.md) | FastAPI Strangler Migration，不作為前端 V3 前置條件 | Accepted | 2026-05-12 | | [037](ADR-037-webcrumbs-shared-ui-runtime.md) | Webcrumbs 共用 UI Runtime 接入 | Accepted | 2026-05-31 | +| [038](ADR-038-security-governance-automation-control-plane.md) | 資訊安全與治理自動化控制面（NIST CSF / OWASP ASVS / SSDF） | Accepted | 2026-07-10 | ## 規範 diff --git a/docs/guides/ai_automation_mainline_work_items.md b/docs/guides/ai_automation_mainline_work_items.md index 4ed8ca6..31a4ef4 100644 --- a/docs/guides/ai_automation_mainline_work_items.md +++ b/docs/guides/ai_automation_mainline_work_items.md @@ -1,50 +1,85 @@ # AI Automation Mainline Work Items > Updated: 2026-07-10 -> Scope: EwoooC / MOMO Pro System AI automation, PixelRAG, MCP/RAG, production UX. +> Governance: `global_product_governance_v2` + ADR-038 +> Current P0: `SEC-P0-001 deny-by-default access control` ## Source Of Truth -- Production readback stays authoritative after deploy: `/health`, container health, AI automation API readback, and smoke. -- Gitea is the only source-control truth. GitHub remains frozen and must not be used. -- Runtime truth must stay separate from source truth: a green test or registry does not mean MCP/RAG runtime is enabled. -- PixelRAG visual receipts are evidence, not formal price truth. They cannot write `competitor_prices`, `competitor_price_history`, or `ai_insights` without replay and promotion gates. +- Production runtime receipt/post-verifier/durable DB receipt is authoritative. +- Gitea main, deploy marker, CD run and production readback are the only source/deploy truth; GitHub remains frozen. +- Source/test/UI/CD green does not mean runtime closure. +- Completion must report program, asset coverage and runtime closure separately. +- PixelRAG visual evidence cannot write formal prices or `ai_insights` before identity, PromotionGate and internal RAG canary proof. -## P0 +## Ordered P0 -| Status | Work item | Evidence / next machine action | -|---|---|---| -| Completed | Multi-commerce PixelRAG visual evidence lane for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | API and CLI can emit expansion plans and marketplace manifests; PChome parser failures can queue manifests. | -| Completed | External MCP/RAG capability inventory absorbed into internal governance readback | `/api/ai-automation/external-mcp-rag-integration` and `scripts/ops/report_external_mcp_rag_integration.py` expose 9 capabilities, absorbed/unresolved counts, and runtime flags. | -| Completed | PixelRAG receipts to internal RAG candidate replay | `/api/ai-automation/pixelrag-rag-candidate-replay` and `scripts/ops/report_pixelrag_rag_candidate_replay.py` read visual capture receipts and platform probe worker receipts, split eligible / blocked / source-contract-fallback candidates, and expose combined OCR/VLM + source-contract next actions before identity matching and PromotionGate knowledge writes. | -| Completed | PixelRAG source-contract replay worker | `/api/ai-automation/pixelrag-source-contract-replay-worker` and `scripts/ops/run_pixelrag_source_contract_replay_worker.py` turn platform probe worker `source_contract_fallback` receipts into replayable source-contract artifact receipts with provenance, public-boundary, no-DB, and blocked-page-not-product-data guards. | -| Completed | PixelRAG marketplace source-contract adapter preflight | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-preflight` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_preflight.py` validate source-contract replay receipts for adapter readiness, canonical offer fields, provenance/rate-limit/public-boundary requirements, and no-write promotion gates. | -| Completed | PixelRAG marketplace adapter dry-run | `/api/ai-automation/pixelrag-marketplace-source-contract-adapter-dry-run` and `scripts/ops/run_pixelrag_marketplace_source_contract_adapter_dry_run.py` turn adapter preflight receipts into deterministic no-write field contracts and offer candidate contracts, with price writes blocked until identity matcher replay and PromotionGate. | -| Completed | PixelRAG marketplace identity matcher replay | `/api/ai-automation/pixelrag-marketplace-identity-matcher-replay` and `scripts/ops/run_pixelrag_marketplace_identity_matcher_replay.py` turn adapter dry-run receipts into deterministic identity candidate contracts and source provenance checks, with price writes blocked until PromotionGate replay. | -| Completed | PixelRAG marketplace PromotionGate replay | `/api/ai-automation/pixelrag-marketplace-promotion-gate-replay` and `scripts/ops/run_pixelrag_marketplace_promotion_gate_replay.py` turn identity matcher receipts into deterministic PromotionGate contracts and embedding-signature readiness, with price writes still blocked. | -| Completed | PixelRAG marketplace embedding signature guard replay | `/api/ai-automation/pixelrag-marketplace-embedding-signature-guard-replay` and `scripts/ops/run_pixelrag_marketplace_embedding_signature_guard_replay.py` turn PromotionGate receipts into deterministic embedding-signature guard contracts and candidate-knowledge readiness, with DB and price writes still blocked. | -| Completed | PixelRAG marketplace candidate knowledge replay | `/api/ai-automation/pixelrag-marketplace-candidate-knowledge-replay` and `scripts/ops/run_pixelrag_marketplace_candidate_knowledge_replay.py` turn embedding-signature guard receipts into deterministic internal RAG candidate preview contracts, with DB, `ai_insights`, and price writes still blocked until canary. | -| Completed | PixelRAG application portfolio and integration lanes | `/api/ai-automation/pixelrag-application-portfolio` and `scripts/ops/report_pixelrag_application_portfolio.py` expose commerce, RAG, UX, ops, marketing, and governance uses with priority, status, next machine action, and forbidden guardrails. | -| Completed | PixelRAG Ollama-first VLM replay worker | `/api/ai-automation/pixelrag-vlm-replay-worker` and `scripts/ops/run_pixelrag_vlm_replay_worker.py` dry-run or execute ready visual receipts against approved Ollama VLM routes, emit evidence-bound artifact receipts, and keep blocked pages out of product data. | -| Completed | PixelRAG VLM route readiness and auto model select | `/api/ai-automation/pixelrag-vlm-route-readiness` and `scripts/ops/report_pixelrag_vlm_route_readiness.py` read approved Ollama `/api/tags`, expose configured/candidate model readiness, and let execute mode avoid missing-model blind generate calls. | -| Completed | PixelRAG platform probe worker | `/api/ai-automation/pixelrag-platform-probe-worker` and `scripts/ops/run_pixelrag_platform_probe_worker.py` turn platform probe plans into dry-run/execute automation: Shopee interstitials go to public empty-context capture when runtime is available, missing Playwright auto-falls back to structured-source/backoff receipt, and Coupang 403/access denied goes to structured-source/backoff artifact receipts. | -| In progress | MCP/RAG runtime health in AI automation smoke | `/api/ai-automation/smoke` and `/api/ai-automation/scheduled-health-summary` include external MCP/RAG integration, PixelRAG RAG candidate replay, PixelRAG OCR/VLM replay contract, PixelRAG VLM route readiness, PixelRAG VLM replay worker, PixelRAG platform probe, PixelRAG platform probe worker, source-contract replay, adapter preflight, adapter dry-run, identity matcher replay, PromotionGate replay, embedding signature guard replay, and candidate knowledge replay families. | -| In progress | Formal production deploy/readback discipline | Every mainline change must update version, push Gitea main/dev, deploy to 188 without touching `momo-db`, and read back `/health` plus new endpoints. | +| Order | ID | Status | Work item | Exit evidence / next machine action | +|---:|---|---|---|---| +| 1 | `SEC-P0-001` | In progress | Deny-by-default route access control | Central policy now protects vendor/import/CI-CD/crawler/category/log/operator and sales-detail surfaces; every anonymous GET is explicitly allowlisted. Exit: anonymous production matrix passes and `/metrics` moves behind edge/internal auth. Next: `verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth`. | +| 2 | `SEC-P0-002` | Not started | Database identity + least-privilege RBAC | Replace shared admin password, default-admin fallback, process-local lockout and spoofable proxy identity. Exit: shadow auth, role matrix, durable audit/lockout, session revocation and canary cutover. | +| 3 | `SEC-P0-003` | In progress | Webhook trust and replay protection | Telegram secret-token verification code exists; production secret activation remains unproven. Exit: secret provisioned outside source, required mode enabled, invalid-secret 401 and authorized callback canary pass. | +| 4 | `SUPPLY-P0-001` | In progress | Gitea-only secure software supply chain | Gitea-native checkout and governance gate added. Exit: exact dependency lock, internal SAST/SCA/secret scan, SBOM, image digest/provenance, vulnerability SLA and production digest readback. | +| 5 | `GOV-P0-001` | In progress | Canonical full asset graph + runtime reconciliation | `governance/ewoooc_asset_inventory.json` seeds hosts, services, data, AI, routes, supply chain, observability and recovery. Exit: same-run probe receipt for every asset; drift auto-creates work items. | +| 6 | `GOV-P0-002` | Not started | Unified controlled-apply envelope | Introduce one `trace_id/run_id/work_item_id` across sensor, identity, SOT diff, decision, risk, dry-run, execution, verifier, rollback/retry and learning acknowledgement. Start with EventRouter + AutoHeal. | +| 7 | `RAG-P0-001` | Not started | Internal RAG candidate canary | V10.770 stops at candidate preview. Exit: bounded pgvector candidate canary, signature/readback/feedback receipt, no price write and no premature `ai_insights` promotion. Next: `run_internal_rag_candidate_canary`. | +| 8 | `MCP-P0-001` | In progress | MCP/RAG production runtime closure | Registry and smoke wiring exist, but runtime flags/ports must be live. Exit: MCP servers healthy, router enabled, RAG enabled, approved caller/tool boundary and production query canary pass. | +| 9 | `SEC-P0-004` | Not started | Security operations lifecycle and metrics | Add durable security incident state and publish MTTA, MTTR, recurrence, false positive, human intervention, verifier pass, rollback and freshness. Exit: detect-to-learn production receipt. | +| 10 | `REL-P0-001` | In progress | Formal deploy and visible proof discipline | Every functional change bumps version, passes Gitea CD, deploys only three app containers, leaves `momo-db` untouched, and reads back `/health`, new APIs and visible UI/security behavior. | ## P1 -| Status | Work item | Evidence / next machine action | +| Order | ID | Status | Work item | Exit evidence / next machine action | +|---:|---|---|---|---| +| 11 | `RES-P1-001` | Not started | Backup/offsite/restore automation | Fresh checksum and offsite coverage plus isolated non-destructive restore drill with measured RPO/RTO. | +| 12 | `PLAT-P1-001` | Not started | Non-root container and capability hardening | Shadow non-root image, writable-path inventory, capability drop/read-only filesystem canary, three-app rollout. | +| 13 | `APPSEC-P1-001` | In progress | CSP and DOM/XSS hardening | Security headers are present and CSP is report-only. Collect violations, remove high-risk `innerHTML`/inline sinks, then enforce CSP by canary. | +| 14 | `APPSEC-P1-002` | Not started | Unsafe shared-cache serialization removal | Replace writable pickle caches in dashboard/daily-sales/EDM/sales with constrained JSON or signed schema. | +| 15 | `ARCH-P1-001` | In progress | Split oversized policy/executor/verifier modules | Current top debts include 44k-line PChome mapping and 14k-line smoke service. Split by bounded family and independent tests. | +| 16 | `UX-P1-001` | In progress | Professional full-site UI/UX | Complete first-viewport cockpit, progressive disclosure, text-density reduction, Traditional Chinese product copy, accessibility, loading/error/empty/degraded states and desktop/mobile visual smoke. | +| 17 | `PIXELRAG-P1-001` | Not started | Ollama-first multimodal embedding benchmark | Verify approved visual embedding on GCP-A -> GCP-B -> 111 and design pgvector-compatible visual metadata; FAISS remains disallowed without ADR. | +| 18 | `MARKET-P1-001` | In progress | Marketplace source contracts | Finish stable, public-boundary, provenance and rate-limit contracts for Shopee, Coupang, Yahoo, ETMall, Friday and Rakuten; blocked pages remain non-product data. | +| 19 | `QA-P1-001` | In progress | Deterministic test and CI governance | Current baseline is 1,701 passed / 30 legacy failures / 9 skipped plus one missing report module at collection. Restore or retire the missing contract, remove wall-clock/copy drift, and split deterministic release tests from nightly integration and production canaries. | + +## P2 + +| Order | ID | Status | Work item | Exit evidence / next machine action | +|---:|---|---|---|---| +| 20 | `GOV-P2-001` | Not started | Continuous NIST/ASVS control trend | Persist governance snapshots, compare control/asset/runtime drift and create ordered work items automatically. | +| 21 | `DATA-P2-001` | Not started | Data classification and retention enforcement | Classify business, personal, operational and model data; automate retention, deletion eligibility and audit evidence without destructive default actions. | +| 22 | `CHAOS-P2-001` | Not started | Controlled resilience exercises | Run non-destructive model-host, MCP, queue, Telegram and app-container failure drills with rollback and learning receipts. | + +## Completed Foundations + +These are reusable foundations, not proof that the full program is complete. + +| Status | Capability | Current boundary | |---|---|---| -| Completed | OCR/VLM replay contract for visual fields | `/api/ai-automation/pixelrag-ocr-vlm-replay` and `scripts/ops/report_pixelrag_ocr_vlm_replay.py` turn saved tiles into no-write field contracts, output schemas, validation rules, and Ollama-first worker actions. | -| Completed | Ollama-first OCR/VLM extraction worker | `run_pixelrag_vlm_replay_worker.py --execute --write-receipt` executes the replay contract against approved Ollama multimodal routes and emits confidence/evidence artifact receipts without writing formal price truth. | -| Not started | Ollama-first multimodal embedding benchmark | Verify local Qwen3-VL or equivalent visual embedding on GCP-A -> GCP-B -> 111 before any visual vector retrieval. | -| Not started | pgvector-compatible visual evidence metadata | Design metadata-first retrieval without FAISS in production unless ADR approves a different store. | -| In progress | Coupang platform probe / structured API strategy | Treat 403 as platform barrier; `run_pixelrag_platform_probe_worker.py --execute --write-receipt` now emits structured-source/backoff receipts without network, DB writes, or product-data promotion. | -| In progress | Marketplace source contracts beyond MOMO/PChome | Source-contract replay receipts now enter marketplace adapter preflight as artifact receipts; next machine action is adapter dry-run for stable Shopee/Coupang/Yahoo/ETMall/friday/Rakuten contracts after provenance, rate-limit, public boundary, and replay evidence. | -| In progress | Professional product website UI/UX text reduction | Continue compact AI surfaces, visual QA, and user-facing copy guardrails; avoid engineering logs or work-session prose in the UI. | +| Completed | Multi-commerce PixelRAG visual evidence for momo, pchome, shopee_tw, coupang_tw, yahoo_shopping_tw, etmall_tw, friday_tw, rakuten_tw | Evidence-only; blocked pages are not product data. | +| Completed | External MCP/RAG capability inventory | Registry/integration readback exists; runtime enablement remains P0. | +| Completed | PixelRAG receipt -> RAG candidate replay | Candidate-only; no formal knowledge or price write. | +| Completed | Source-contract replay worker | Public-boundary artifact receipts only. | +| Completed | Marketplace adapter preflight and dry-run | Deterministic no-write contracts. | +| Completed | Marketplace identity matcher replay | Candidate identity only. | +| Completed | PromotionGate replay | No production write. | +| Completed | Embedding-signature guard replay | Signature readiness only. | +| Completed | Candidate knowledge replay | Internal RAG preview only; canary remains P0. | +| Completed | PixelRAG application portfolio | Commerce/RAG/UX/ops/marketing/governance inventory. | +| Completed | Ollama-first VLM route readiness and replay worker | Evidence-bound artifact output; no direct price write. | +| Completed | Platform probe worker | Shopee/Coupang barriers become structured fallback/backoff receipts. | -## Always Enforced +## Definition Of Done -- Manual review is exception-only for low-risk, machine-verifiable controlled apply. -- Break-glass still applies to secrets, destructive DB actions, force pushes, production provider switchovers, raw runtime volumes, and unverified core model replacement. -- AI automation smoke must expose health honestly: `MCP_ROUTER_ENABLED=false` or `RAG_ENABLED=false` is a runtime gap, not a source-code failure. +A work item can be `Completed` only when the same production run contains: + +1. sensor/source receipt; +2. normalized canonical asset identity; +3. source-of-truth diff; +4. AI decision and candidate action; +5. risk/policy decision; +6. check-mode/dry-run receipt; +7. idempotent bounded execution receipt; +8. independent post-verifier and rollback/no-write terminal; +9. incident/Telegram/KM/RAG/MCP/PlayBook durable acknowledgement. + +Missing any stage means `partial`, `degraded` or `blocked_with_safe_next_action`. diff --git a/governance/ewoooc_asset_inventory.json b/governance/ewoooc_asset_inventory.json new file mode 100644 index 0000000..2b7c5cc --- /dev/null +++ b/governance/ewoooc_asset_inventory.json @@ -0,0 +1,365 @@ +{ + "schema_version": "1.0", + "product_id": "ewoooc", + "governance_profile": "global_product_governance_v2", + "updated_at": "2026-07-10T18:30:00+08:00", + "required_fields": [ + "canonical_id", + "asset_type", + "owner_lane", + "source_truth", + "runtime_identity", + "signal_freshness", + "policy", + "executor", + "verifier", + "backup_restore", + "learning_targets" + ], + "assets": [ + { + "canonical_id": "product:ewoooc", + "asset_type": "product", + "owner_lane": "product-governance", + "source_truth": "Gitea main plus production health and protected product surfaces", + "runtime_identity": "https://mo.wooo.work", + "signal_freshness": {"source": "commit", "runtime_target": "/health", "max_age_minutes": 15}, + "policy": ["global_product_governance_v2", "CONSTITUTION.md"], + "executor": "Gitea CD on ewoooc-host", + "verifier": "production version truth plus HTTP and UI smoke", + "backup_restore": "source in Gitea; runtime data handled by database and artifact lanes", + "learning_targets": ["security governance review", "AI automation smoke", "KM/PlayBook"] + }, + { + "canonical_id": "repo:gitea:wooo/ewoooc", + "asset_type": "source_repository", + "owner_lane": "source-control", + "source_truth": "ssh://git@192.168.0.110:2222/wooo/ewoooc.git", + "runtime_identity": "Gitea repository wooo/ewoooc", + "signal_freshness": {"source": "git ls-remote", "runtime_target": "main/dev heads", "max_age_minutes": 30}, + "policy": ["GitHub freeze", "protected main", "Gitea-only actions"], + "executor": "git over Gitea SSH or Gitea job token", + "verifier": "remote head readback and deploy marker", + "backup_restore": "Gitea repository backup and offsite checksum lane", + "learning_targets": ["source deploy runtime truth"] + }, + { + "canonical_id": "pipeline:gitea-actions:ewoooc-cd", + "asset_type": "deployment_pipeline", + "owner_lane": "delivery", + "source_truth": ".gitea/workflows/cd.yaml", + "runtime_identity": "runner label ewoooc-host", + "signal_freshness": {"source": "workflow commit", "runtime_target": "latest Gitea Action run", "max_age_minutes": 60}, + "policy": ["Gitea-only", "momo-db protected", "three app containers only"], + "executor": "Gitea act runner", + "verifier": "CD health, version, smoke and post-deploy review", + "backup_restore": "rollback by known-good Gitea commit and image", + "learning_targets": ["CD failure classification", "deployment playbook"] + }, + { + "canonical_id": "host:lan:110", + "asset_type": "host_gateway", + "owner_lane": "platform", + "source_truth": "host and service runtime probes", + "runtime_identity": "192.168.0.110", + "signal_freshness": {"source": "SSH probe", "runtime_target": "gateway/Gitea/Nginx", "max_age_minutes": 15}, + "policy": ["gateway and Gitea only", "cross-project isolation"], + "executor": "bounded SSH via approved operator lane", + "verifier": "independent service and public route probes", + "backup_restore": "host configuration backup and Gitea dump lane", + "learning_targets": ["host health", "routing incidents"] + }, + { + "canonical_id": "host:lan:188", + "asset_type": "production_host", + "owner_lane": "runtime", + "source_truth": "Docker runtime receipts on 192.168.0.188", + "runtime_identity": "192.168.0.188", + "signal_freshness": {"source": "SSH/Docker probe", "runtime_target": "EwoooC containers", "max_age_minutes": 10}, + "policy": ["not an Ollama node", "momo-db protected", "cross-project isolation"], + "executor": "Gitea CD bounded compose actions", + "verifier": "container health plus external /health", + "backup_restore": "database backup and application rollback lanes", + "learning_targets": ["runtime incidents", "capacity history"] + }, + { + "canonical_id": "host:gcp:ollama-a", + "asset_type": "model_host", + "owner_lane": "ai-runtime", + "source_truth": "Ollama host health receipts", + "runtime_identity": "34.87.90.216:11434", + "signal_freshness": {"source": "approved model probe", "runtime_target": "/api/tags", "max_age_minutes": 20}, + "policy": ["Ollama primary", "no secret payload logging"], + "executor": "services.ollama_service", + "verifier": "host health and model availability readback", + "backup_restore": "fallback to GCP-B then 111", + "learning_targets": ["model latency", "fallback cause"] + }, + { + "canonical_id": "host:gcp:ollama-b", + "asset_type": "model_host", + "owner_lane": "ai-runtime", + "source_truth": "Ollama host health receipts", + "runtime_identity": "34.21.145.224:11434", + "signal_freshness": {"source": "approved model probe", "runtime_target": "/api/tags", "max_age_minutes": 20}, + "policy": ["Ollama secondary", "embedding fallback"], + "executor": "services.ollama_service", + "verifier": "host health and model availability readback", + "backup_restore": "fallback to 111 for approved non-batch traffic", + "learning_targets": ["model latency", "fallback cause"] + }, + { + "canonical_id": "host:lan:ollama-111", + "asset_type": "model_host", + "owner_lane": "ai-runtime", + "source_truth": "Ollama host health receipts", + "runtime_identity": "192.168.0.111:11434", + "signal_freshness": {"source": "approved model probe", "runtime_target": "/api/tags", "max_age_minutes": 20}, + "policy": ["third fallback", "no background embedding batches"], + "executor": "services.ollama_service", + "verifier": "host health and route policy readback", + "backup_restore": "no further model host fallback", + "learning_targets": ["fallback exhaustion"] + }, + { + "canonical_id": "network:public:mo.wooo.work", + "asset_type": "dns_tls_route", + "owner_lane": "edge", + "source_truth": "public DNS/TLS/HTTP readback", + "runtime_identity": "https://mo.wooo.work", + "signal_freshness": {"source": "Blackbox", "runtime_target": "/health", "max_age_minutes": 5}, + "policy": ["TLS only for user traffic", "health endpoint is lightweight"], + "executor": "Nginx on 110", + "verifier": "external TLS and HTTP probe", + "backup_restore": "versioned Nginx configuration and rollback", + "learning_targets": ["TLS expiry", "routing drift"] + }, + { + "canonical_id": "service:ewoooc:momo-app", + "asset_type": "application_service", + "owner_lane": "runtime", + "source_truth": "Docker inspect plus /health", + "runtime_identity": "momo-pro-system", + "signal_freshness": {"source": "Docker health", "runtime_target": "container port 80", "max_age_minutes": 5}, + "policy": ["Gunicorn gthread", "HUP sync reload", "no momo-db lifecycle action"], + "executor": "Gitea CD momo-app lane", + "verifier": "internal and external /health plus version truth", + "backup_restore": "known-good image and Gitea commit rollback", + "learning_targets": ["HTTP errors", "worker latency", "security events"] + }, + { + "canonical_id": "service:ewoooc:scheduler", + "asset_type": "scheduler_service", + "owner_lane": "automation-runtime", + "source_truth": "Docker health and scheduler receipts", + "runtime_identity": "momo-scheduler", + "signal_freshness": {"source": "job receipts", "runtime_target": "scheduler process", "max_age_minutes": 15}, + "policy": ["EventRouter", "bounded retry", "no momo-db restart"], + "executor": "run_scheduler.py", + "verifier": "scheduled health summary and job receipts", + "backup_restore": "restart scheduler only; replay durable queues", + "learning_targets": ["job failure recurrence", "MTTR"] + }, + { + "canonical_id": "service:ewoooc:telegram-bot", + "asset_type": "notification_service", + "owner_lane": "automation-runtime", + "source_truth": "Telegram API receipt plus Docker health", + "runtime_identity": "momo-telegram-bot", + "signal_freshness": {"source": "delivery receipt", "runtime_target": "bot process", "max_age_minutes": 15}, + "policy": ["authorized users/groups", "webhook secret target", "queue replay"], + "executor": "run_telegram_bot.py and OpenClaw webhook", + "verifier": "delivery receipt and callback readback", + "backup_restore": "file queue replay and bot restart only", + "learning_targets": ["delivery failure", "unauthorized attempt"] + }, + { + "canonical_id": "database:ewoooc:postgresql", + "asset_type": "database", + "owner_lane": "data", + "source_truth": "PostgreSQL durable receipts and migrations", + "runtime_identity": "momo-db", + "signal_freshness": {"source": "SQL health", "runtime_target": "PostgreSQL", "max_age_minutes": 5}, + "policy": ["protected resource", "pgvector primary", "no destructive automatic action"], + "executor": "SQLAlchemy and idempotent migrations", + "verifier": "independent readback queries and schema checks", + "backup_restore": "scheduled backup, checksum, offsite copy and restore drill", + "learning_targets": ["query health", "migration drift", "restore evidence"] + }, + { + "canonical_id": "data:ewoooc:pchome-sales", + "asset_type": "business_dataset", + "owner_lane": "sales-data", + "source_truth": "PostgreSQL sales tables plus import receipts", + "runtime_identity": "daily and monthly PChome sales records", + "signal_freshness": {"source": "import job receipt", "runtime_target": "latest business date", "max_age_minutes": 180}, + "policy": ["real data only", "Taipei time", "no mock values"], + "executor": "Google Drive/import services", + "verifier": "row count, business date and duplicate checks", + "backup_restore": "database backup and source file replay", + "learning_targets": ["import errors", "freshness drift"] + }, + { + "canonical_id": "data:ewoooc:market-intelligence", + "asset_type": "business_dataset", + "owner_lane": "market-intelligence", + "source_truth": "source receipts, external_offers and guarded candidate artifacts", + "runtime_identity": "market intel tables and receipt artifacts", + "signal_freshness": {"source": "source receipt", "runtime_target": "offer freshness", "max_age_minutes": 1440}, + "policy": ["provenance required", "PromotionGate", "no visual-only price write"], + "executor": "market intel controlled apply lanes", + "verifier": "identity matcher, PromotionGate and post-write readback", + "backup_restore": "database backup and replayable evidence artifacts", + "learning_targets": ["matcher quality", "false positive", "source barrier"] + }, + { + "canonical_id": "integration:ewoooc:google-drive-import", + "asset_type": "external_integration", + "owner_lane": "sales-data", + "source_truth": "Google Drive auth and import receipts", + "runtime_identity": "auto_import service", + "signal_freshness": {"source": "auth readiness", "runtime_target": "latest import job", "max_age_minutes": 180}, + "policy": ["authenticated operator surface", "no credential exposure"], + "executor": "services.google_drive_service", + "verifier": "job status and database readback", + "backup_restore": "replay source file without duplicate write", + "learning_targets": ["auth expiry", "import parser failure"] + }, + { + "canonical_id": "integration:ewoooc:telegram-webhook", + "asset_type": "webhook", + "owner_lane": "notification-security", + "source_truth": "Telegram webhook configuration and request receipts", + "runtime_identity": "/bot/telegram/webhook", + "signal_freshness": {"source": "webhook info", "runtime_target": "secret verification", "max_age_minutes": 60}, + "policy": ["secret token required target", "authorized user and group checks"], + "executor": "OpenClaw Telegram webhook", + "verifier": "invalid-secret rejection plus authorized callback canary", + "backup_restore": "restore prior webhook URL and replay no pending updates", + "learning_targets": ["auth failures", "duplicate updates"] + }, + { + "canonical_id": "ai:ewoooc:ollama-router", + "asset_type": "ai_runtime", + "owner_lane": "ai-platform", + "source_truth": "Ollama route and call receipts", + "runtime_identity": "GCP-A -> GCP-B -> 111", + "signal_freshness": {"source": "host probe", "runtime_target": "approved model route", "max_age_minutes": 20}, + "policy": ["Ollama-first", "Gemini hard-disabled default", "188 forbidden"], + "executor": "services.ollama_service", + "verifier": "model route readback and fallback receipt", + "backup_restore": "bounded fallback chain", + "learning_targets": ["latency", "model error", "fallback rate"] + }, + { + "canonical_id": "ai:ewoooc:mcp-router", + "asset_type": "mcp_control_plane", + "owner_lane": "ai-platform", + "source_truth": "MCP registry plus live server health", + "runtime_identity": "services.mcp_router and ports 3001-3004", + "signal_freshness": {"source": "MCP smoke", "runtime_target": "tool server health", "max_age_minutes": 15}, + "policy": ["known caller/tool allowlist", "Postgres/filesystem read-only"], + "executor": "services.mcp_router", + "verifier": "live health, call receipt and boundary tests", + "backup_restore": "disable router and fall back to approved static/Ollama path", + "learning_targets": ["tool success", "cache hit", "policy rejection"] + }, + { + "canonical_id": "ai:ewoooc:rag-service", + "asset_type": "rag_control_plane", + "owner_lane": "ai-platform", + "source_truth": "pgvector records and RAG query receipts", + "runtime_identity": "services.rag_service", + "signal_freshness": {"source": "RAG smoke", "runtime_target": "query/feedback logs", "max_age_minutes": 30}, + "policy": ["pgvector only", "BGE-M3 signature", "PromotionGate"], + "executor": "services.rag_service", + "verifier": "candidate canary, signature and retrieval quality checks", + "backup_restore": "disable retrieval and preserve structured DB truth", + "learning_targets": ["hit rate", "feedback", "recurrence"] + }, + { + "canonical_id": "ai:ewoooc:pixelrag", + "asset_type": "visual_evidence_pipeline", + "owner_lane": "market-intelligence", + "source_truth": "screenshot, tile and replay receipts", + "runtime_identity": "PixelRAG artifact lanes for 8 marketplaces", + "signal_freshness": {"source": "artifact receipt", "runtime_target": "capture/replay readback", "max_age_minutes": 1440}, + "policy": ["public evidence only", "blocked page is not product data", "no direct price write"], + "executor": "PixelRAG capture and Ollama VLM workers", + "verifier": "identity, PromotionGate, embedding signature and RAG canary", + "backup_restore": "immutable artifact receipts and no-write terminal", + "learning_targets": ["capture quality", "VLM confidence", "platform barrier"] + }, + { + "canonical_id": "observability:ewoooc:security-runtime", + "asset_type": "logs_metrics_traces_alerts", + "owner_lane": "observability", + "source_truth": "Prometheus, Grafana, logs, alerts and Telegram receipts", + "runtime_identity": "/metrics plus AI automation smoke", + "signal_freshness": {"source": "scrape and alert receipt", "runtime_target": "security/governance families", "max_age_minutes": 15}, + "policy": ["no public operational logs", "lifecycle receipts", "freshness SLO"], + "executor": "Prometheus/Blackbox/EventRouter", + "verifier": "alert-chain and dashboard readback", + "backup_restore": "metrics retention and durable incident receipts", + "learning_targets": ["MTTA", "MTTR", "false positive", "rollback rate"] + }, + { + "canonical_id": "backup:ewoooc:database-artifacts", + "asset_type": "backup_restore", + "owner_lane": "resilience", + "source_truth": "backup checksum and restore drill receipts", + "runtime_identity": "database and artifact backup lanes", + "signal_freshness": {"source": "backup receipt", "runtime_target": "latest successful restore drill", "max_age_minutes": 10080}, + "policy": ["checksum required", "offsite required", "restore drill required"], + "executor": "backup services and controlled restore workflow", + "verifier": "independent checksum plus isolated restore query", + "backup_restore": "this asset owns the recovery evidence itself", + "learning_targets": ["RPO", "RTO", "restore failure"] + }, + { + "canonical_id": "supply-chain:ewoooc:python-container", + "asset_type": "packages_images_sbom", + "owner_lane": "software-supply-chain", + "source_truth": "requirements, image digest, SBOM and vulnerability receipts", + "runtime_identity": "EwoooC Python 3.11 container image", + "signal_freshness": {"source": "CD security gate", "runtime_target": "deployed image digest", "max_age_minutes": 1440}, + "policy": ["Gitea/internal sources", "exact dependency lock target", "no GitHub action dependency"], + "executor": "Gitea CD build lane", + "verifier": "SAST, SCA, secret scan, SBOM and provenance checks", + "backup_restore": "known-good immutable image digest", + "learning_targets": ["CVE age", "dependency drift", "build provenance"] + }, + { + "canonical_id": "quality:ewoooc:test-control-plane", + "asset_type": "test_and_release_evidence", + "owner_lane": "quality-governance", + "source_truth": "pytest collection, deterministic release suite and runtime canary receipts", + "runtime_identity": "Gitea release gate plus scheduled nightly and production smoke lanes", + "signal_freshness": {"source": "governance/security_governance_review_baseline.json", "runtime_target": "latest Gitea and production verifier receipts", "max_age_minutes": 1440}, + "policy": ["release tests deterministic", "nightly integration separated", "production canary read-only first"], + "executor": "pytest and Gitea Actions", + "verifier": "collection gate, focused security suites and post-deploy smoke", + "backup_restore": "Gitea test source plus immutable test result receipt", + "learning_targets": ["flake rate", "failure age", "collection drift", "runtime escape rate"] + }, + { + "canonical_id": "surface:ewoooc:primary-ui-api", + "asset_type": "product_surface", + "owner_lane": "product-experience", + "source_truth": "Flask route map, templates and production visual readback", + "runtime_identity": "primary desktop and mobile user workflows", + "signal_freshness": {"source": "sitewide visual QA", "runtime_target": "desktop/mobile screenshots", "max_age_minutes": 1440}, + "policy": ["Traditional Chinese", "progressive disclosure", "authenticated operator surfaces", "no text walls"], + "executor": "Flask/Jinja frontend", + "verifier": "route, accessibility, responsive and visual smoke", + "backup_restore": "Gitea commit and frontend asset rollback", + "learning_targets": ["task completion", "error state", "text density"] + } + ], + "reconciliation": { + "status": "partial", + "source_inventory_complete": true, + "runtime_receipts_complete": false, + "next_machine_action": "probe_all_assets_and_write_same-run_reconciliation_receipt" + } +} diff --git a/governance/security_governance_review_baseline.json b/governance/security_governance_review_baseline.json new file mode 100644 index 0000000..79940ce --- /dev/null +++ b/governance/security_governance_review_baseline.json @@ -0,0 +1,40 @@ +{ + "schema_version": "1.0", + "product_id": "ewoooc", + "captured_at": "2026-07-10T20:40:00+08:00", + "source_base_sha": "c62ec7b504c7f74ce8a6a3b5312e30c1e6947513", + "review_branch": "codex/security-governance-review", + "collection": { + "status": "failed", + "collected_count": 1740, + "error_count": 1, + "missing_contract": "scripts/ops/report_pchome_mapping_backlog.py", + "consumer": "tests/test_pchome_mapping_backlog_report.py" + }, + "focused_suites": { + "ai_automation_smoke": {"passed": 46, "failed": 0}, + "security_and_governance": {"passed": 23, "failed": 0}, + "frontend_contract": {"passed": 29, "failed": 0} + }, + "full_suite_without_missing_contract": { + "status": "partial", + "command": ".venv/bin/python -m pytest -q --ignore=tests/test_pchome_mapping_backlog_report.py", + "passed": 1701, + "failed": 30, + "skipped": 9, + "duration_seconds": 361.66, + "failure_groups": [ + "legacy manual/HITL assertions conflict with current AI-controlled product copy", + "wall-clock dependent import fixture expired", + "modularization line-count snapshot drift", + "missing or stale source/test contract assertions", + "pre-existing service behavior and fixture mismatches outside this review diff" + ] + }, + "review_change_regression_evidence": { + "focused_security_suites_pass": true, + "failed_full_suite_targets_are_outside_security_review_source_diff": true, + "claim_full_suite_green": false + }, + "next_machine_action": "restore_or_retire_missing_pchome_report_contract_then_baseline_legacy_failures" +} diff --git a/routes/alert_routes.py b/routes/alert_routes.py index 265b625..baab0ba 100644 --- a/routes/alert_routes.py +++ b/routes/alert_routes.py @@ -6,6 +6,7 @@ 並執行自動排查和修復 """ +import hmac import os import json import subprocess @@ -78,6 +79,18 @@ ALLOWED_ISSUE_TYPES = frozenset({ 'cpu', 'container_cpu', }) +ALLOWED_AUTO_FIX_CONTAINERS = frozenset({ + 'momo-pro-system', + 'momo-scheduler', + 'momo-telegram-bot', +}) +PROTECTED_CONTAINERS = frozenset({ + 'momo-db', + 'momo-postgres', + 'momo-prometheus', + 'momo-grafana', +}) + def check_alert_auth(f): """告警 Webhook 認證裝飾器（fail-secure：密碼未設定時拒絕所有請求）""" @@ -89,7 +102,10 @@ def check_alert_auth(f): auth = request.authorization if not auth: return jsonify({'error': 'Missing authentication'}), 401 - if auth.username != ALERT_WEBHOOK_USER or auth.password != ALERT_WEBHOOK_PASSWORD: + if ( + not hmac.compare_digest(auth.username or '', ALERT_WEBHOOK_USER) + or not hmac.compare_digest(auth.password or '', ALERT_WEBHOOK_PASSWORD) + ): return jsonify({'error': 'Invalid credentials'}), 401 return f(*args, **kwargs) return decorated @@ -270,10 +286,12 @@ def auto_fix_container(container_name: str, issue_type: str) -> dict: try: # 排除關鍵容器 - protected_containers = ['momo-postgres', 'momo-prometheus', 'momo-grafana'] - if container_name in protected_containers: + if container_name in PROTECTED_CONTAINERS: result['message'] = "此為受保護的關鍵容器，不自動重啟" return result + if container_name not in ALLOWED_AUTO_FIX_CONTAINERS: + result['message'] = "容器不在 EwoooC 自動修復白名單" + return result # 根據問題類型決定修復動作 if issue_type in ['container_memory', 'memory']: @@ -454,6 +472,7 @@ def manual_fix(): @alert_bp.route('/api/alert/test', methods=['POST']) +@check_alert_auth def test_alert(): """測試告警通知""" message = """ @@ -473,6 +492,7 @@ def test_alert(): @alert_bp.route('/api/alert/status') +@check_alert_auth def alert_status(): """取得告警系統狀態""" return jsonify({ diff --git a/routes/misc_routes.py b/routes/misc_routes.py index 383c14c..cdce697 100644 --- a/routes/misc_routes.py +++ b/routes/misc_routes.py @@ -6,11 +6,18 @@ """ import requests from flask import Blueprint, jsonify, render_template, request +from urllib.parse import urljoin + +from auth import login_required +from services.logger_manager import SystemLogger +from utils.security import validate_public_http_url misc_bp = Blueprint('misc', __name__) +sys_log = SystemLogger('MiscRoutes').get_logger() @misc_bp.route('/api/test_url', methods=['POST']) +@login_required def test_url(): """API: 測試網址是否有效""" try: @@ -22,16 +29,39 @@ def test_url(): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" } - # 設定 10 秒超時，避免卡住 - response = requests.get(url, headers=headers, timeout=10) + current_url = validate_public_http_url(url) + response = None + for _ in range(4): + response = requests.get( + current_url, + headers=headers, + timeout=(3.05, 10), + allow_redirects=False, + stream=True, + ) + if response.status_code not in {301, 302, 303, 307, 308}: + break + location = response.headers.get('Location') + response.close() + if not location: + break + current_url = validate_public_http_url(urljoin(current_url, location)) - if response.status_code == 200: + if response is None: + raise ValueError('無法取得網址回應') + status_code = response.status_code + response.close() + + if status_code == 200: return jsonify({"status": "success", "message": f"✅ 連結有效 (Status: 200)"}) else: - return jsonify({"status": "warning", "message": f"⚠️ 連結回應異常 (Status: {response.status_code})"}) + return jsonify({"status": "warning", "message": f"⚠️ 連結回應異常 (Status: {status_code})"}) - except Exception as e: - return jsonify({"status": "error", "message": f"❌ 連線失敗: {str(e)}"}), 500 + except ValueError as exc: + return jsonify({"status": "error", "message": str(exc)}), 400 + except Exception as exc: + sys_log.warning(f"[Misc] [URL_CHECK] 連線失敗 | Error: {exc}") + return jsonify({"status": "error", "message": "連線失敗，請確認網址後再試。"}), 502 @misc_bp.route('/brand_assets') diff --git a/routes/notification_routes.py b/routes/notification_routes.py index c3baa9b..cf03e95 100644 --- a/routes/notification_routes.py +++ b/routes/notification_routes.py @@ -6,7 +6,7 @@ """ from flask import Blueprint, request, jsonify, render_template -from auth import login_required +from auth import internal_key_required, login_required from services.notification_service import NotificationTemplateService from services.logger_manager import SystemLogger from config import SYSTEM_VERSION @@ -87,6 +87,7 @@ def preview_template(code): @notification_bp.route('/api/notification/render', methods=['POST']) +@internal_key_required def render_for_n8n(): """ API: 為 n8n 渲染模板 diff --git a/routes/openclaw_bot_routes.py b/routes/openclaw_bot_routes.py index 2dc7dc9..372bc33 100644 --- a/routes/openclaw_bot_routes.py +++ b/routes/openclaw_bot_routes.py @@ -24,6 +24,7 @@ from __future__ import annotations import os import json +import hmac import re import threading import hashlib # Operation Ollama-First v5.0 P1: H6 PII fix — chat_id 進 meta 改 hash[:8] @@ -126,6 +127,11 @@ BOT_TOKEN = os.getenv('OPENCLAW_BOT_TOKEN') or os.getenv('TELEGRAM_BOT_TOK BOT_API_URL = f"https://api.telegram.org/bot{BOT_TOKEN}" ALLOWED_GROUP = int(os.getenv('OPENCLAW_GROUP_ID', '-1003940688311')) MOMO_BASE_URL = os.getenv('MOMO_BASE_URL', 'https://mo.wooo.work') +TELEGRAM_WEBHOOK_SECRET_TOKEN = os.getenv('TELEGRAM_WEBHOOK_SECRET_TOKEN', '').strip() +TELEGRAM_WEBHOOK_SECRET_REQUIRED = ( + os.getenv('TELEGRAM_WEBHOOK_SECRET_REQUIRED', 'false').strip().lower() + in {'1', 'true', 'yes', 'on'} +) NVIDIA_API_KEY = os.getenv('NVIDIA_API_KEY', '') NVIDIA_BASE_URL = 'https://integrate.api.nvidia.com/v1' CHAT_MODEL = 'deepseek-ai/deepseek-v3.2' @@ -8904,6 +8910,15 @@ def _identify_product_name_with_gemini_vision(img_b64: str, request_id: str) -> @openclaw_bot_bp.route('/bot/telegram/webhook', methods=['POST']) def telegram_webhook(): try: + provided_secret = request.headers.get('X-Telegram-Bot-Api-Secret-Token', '') + if TELEGRAM_WEBHOOK_SECRET_TOKEN: + if not hmac.compare_digest(provided_secret, TELEGRAM_WEBHOOK_SECRET_TOKEN): + sys_log.warning('[OpenClawBot] Telegram webhook secret 驗證失敗') + return jsonify({'ok': False, 'error': 'unauthorized'}), 401 + elif TELEGRAM_WEBHOOK_SECRET_REQUIRED: + sys_log.error('[OpenClawBot] Telegram webhook secret 必填但未設定') + return jsonify({'ok': False, 'error': 'webhook_secret_not_configured'}), 503 + # 每個 webhook request 先清空 ContextVar，避免同 worker thread 延用上一個 user。 _CURRENT_USER_ID_CTX.set(None) @@ -9398,11 +9413,14 @@ def telegram_webhook(): @openclaw_bot_bp.route('/bot/telegram/set_webhook', methods=['POST']) def set_webhook(): webhook_url = f"{MOMO_BASE_URL}/bot/telegram/webhook" - result = _tg('setWebhook', { + payload = { 'url': webhook_url, 'allowed_updates': ['message', 'callback_query'], 'drop_pending_updates': True, - }) + } + if TELEGRAM_WEBHOOK_SECRET_TOKEN: + payload['secret_token'] = TELEGRAM_WEBHOOK_SECRET_TOKEN + result = _tg('setWebhook', payload) register_commands() sys_log.info(f"[OpenClawBot] setWebhook → {result}") return jsonify({'ok': result.get('ok'), 'webhook_url': webhook_url}) diff --git a/routes/system_public_routes.py b/routes/system_public_routes.py index 7f1ab0e..402b2b9 100644 --- a/routes/system_public_routes.py +++ b/routes/system_public_routes.py @@ -1177,6 +1177,20 @@ def ai_automation_external_mcp_rag_integration_api(): )) +@system_public_bp.route('/api/ai-automation/security-governance-review') +@login_required +def ai_automation_security_governance_review_api(): + """Read-only NIST/OWASP-aligned security and governance review.""" + from services.security_governance_review_service import ( + build_security_governance_review, + ) + + detail_limit = request.args.get('detail_limit', 30, type=int) or 30 + return jsonify(build_security_governance_review( + detail_limit=max(5, min(detail_limit, 100)), + )) + + @system_public_bp.route('/api/ai-automation/smoke/history/export') @login_required def ai_automation_smoke_history_export(): diff --git a/scripts/ops/report_security_governance_review.py b/scripts/ops/report_security_governance_review.py new file mode 100644 index 0000000..fbe097a --- /dev/null +++ b/scripts/ops/report_security_governance_review.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Report the machine-readable EwoooC security/governance review.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from services.security_governance_review_service import ( # noqa: E402 + build_security_governance_review, +) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--detail-limit", type=int, default=30) + parser.add_argument("--compact", action="store_true") + parser.add_argument( + "--strict", + action="store_true", + help="Exit non-zero only for release-blocking source governance failures.", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + report = build_security_governance_review( + root=args.root, + detail_limit=args.detail_limit, + ) + if args.compact: + output = { + "policy": report["policy"], + "status": report["status"], + "completion": report["completion"], + "release_gate": report["release_gate"], + "next_machine_action": report["next_machine_action"], + } + else: + output = report + print(json.dumps(output, ensure_ascii=False, indent=2, sort_keys=True)) + if args.strict and report["release_gate"]["status"] != "pass": + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/ai_automation_smoke_service.py b/services/ai_automation_smoke_service.py index ff37fac..4f80e8c 100644 --- a/services/ai_automation_smoke_service.py +++ b/services/ai_automation_smoke_service.py @@ -662,6 +662,14 @@ def build_scheduled_automation_health_summary( if not pixelrag_platform_probe_worker or not pixelrag_platform_probe_worker_details: pixelrag_platform_probe_worker = _pixelrag_platform_probe_worker_check() pixelrag_platform_probe_worker_details = pixelrag_platform_probe_worker.get("details") or {} + security_governance = _find_check( + source_result, + "Security governance automation review", + ) + security_governance_details = security_governance.get("details") or {} + if not security_governance or not security_governance_details: + security_governance = _security_governance_review_check() + security_governance_details = security_governance.get("details") or {} smoke_status = source_result.get("status") or ("warning" if latest_history else "warning") freshness_family = _history_freshness_family( latest_history, @@ -690,6 +698,43 @@ def build_scheduled_automation_health_summary( "writes_database": False, }, }, + { + "key": "security_governance_review", + "label": "Security governance automation review", + "status": security_governance.get("status") or "warning", + "summary": ( + security_governance.get("summary") + or "Security governance review has no latest readback." + ), + "next_machine_action": security_governance_details.get( + "next_machine_action" + ) or "refresh_security_governance_review", + "details": { + "policy": security_governance_details.get("policy"), + "review_status": security_governance_details.get("review_status"), + "program_percent": security_governance_details.get("program_percent"), + "asset_field_coverage_percent": security_governance_details.get( + "asset_field_coverage_percent" + ), + "asset_runtime_reconciliation_complete": bool( + security_governance_details.get( + "asset_runtime_reconciliation_complete" + ) + ), + "runtime_closure_passed": int( + security_governance_details.get("runtime_closure_passed") or 0 + ), + "runtime_closure_total": int( + security_governance_details.get("runtime_closure_total") or 0 + ), + "blocking_failure_count": int( + security_governance_details.get("blocking_failure_count") or 0 + ), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + }, { "key": "pchome_controlled_apply_drift_monitor", "label": "PChome controlled apply drift monitor", @@ -5373,6 +5418,7 @@ def build_scheduled_automation_health_summary( "scheduled_outputs": { "health_summary_endpoint": "/api/ai-automation/scheduled-health-summary", "smoke_endpoint": "/api/ai-automation/smoke", + "security_governance_endpoint": "/api/ai-automation/security-governance-review", "daily_summary_send_endpoint": "/api/ai-automation/smoke/daily-summary/send", "telegram_send_in_preview": False, "writes_database": False, @@ -14119,6 +14165,60 @@ def _pixelrag_platform_probe_worker_check() -> Dict[str, Any]: ) +def _security_governance_review_check() -> Dict[str, Any]: + try: + from services.security_governance_review_service import ( + build_security_governance_review, + ) + + readback = build_security_governance_review(detail_limit=10) + completion = readback.get("completion") or {} + release_gate = readback.get("release_gate") or {} + blocking_count = int(release_gate.get("blocking_failure_count") or 0) + status = "critical" if blocking_count else ( + "ok" if readback.get("status") == "complete" else "warning" + ) + return _check( + "Security governance automation review", + status, + ( + f"Security governance program={completion.get('program_percent', 0)}%, " + f"asset_fields={completion.get('asset_field_coverage_percent', 0)}%, " + f"runtime_closure={completion.get('runtime_closure_passed', 0)}/" + f"{completion.get('runtime_closure_total', 0)}, blocking={blocking_count}" + ), + { + "policy": readback.get("policy"), + "review_status": readback.get("status"), + "program_percent": completion.get("program_percent"), + "asset_field_coverage_percent": completion.get( + "asset_field_coverage_percent" + ), + "asset_runtime_reconciliation_complete": completion.get( + "asset_runtime_reconciliation_complete" + ), + "runtime_closure_passed": completion.get("runtime_closure_passed"), + "runtime_closure_total": completion.get("runtime_closure_total"), + "blocking_failure_count": blocking_count, + "next_machine_action": readback.get("next_machine_action"), + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + except Exception as exc: + return _check( + "Security governance automation review", + "critical", + f"Security governance review failed: {exc}", + { + "writes_database": False, + "writes_database_count": 0, + "primary_human_gate_count": 0, + }, + ) + + def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: int = 20) -> Dict[str, Any]: checks: List[Dict[str, Any]] = [ _event_router_check(), @@ -14151,6 +14251,7 @@ def collect_ai_automation_smoke(*, record_history: bool = True, history_limit: i _ai_surface_html_readback_check(), _sitewide_ui_ux_agent_check(), _sitewide_visual_qa_check(), + _security_governance_review_check(), _external_mcp_rag_integration_check(), _pixelrag_rag_candidate_replay_check(), _pixelrag_source_contract_replay_worker_check(), diff --git a/services/code_review_pipeline_service.py b/services/code_review_pipeline_service.py index 131420e..80ebbe4 100644 --- a/services/code_review_pipeline_service.py +++ b/services/code_review_pipeline_service.py @@ -20,6 +20,7 @@ Pipeline： """ import json +import hmac import logging import os import re @@ -1268,4 +1269,4 @@ def verify_internal_token(request_token: str) -> bool: return True logger.error("[CodeReview] INTERNAL_WEBHOOK_TOKEN 未設定，拒絕 webhook") return False - return request_token == INTERNAL_TOKEN + return hmac.compare_digest(request_token or "", INTERNAL_TOKEN) diff --git a/services/http_access_policy_service.py b/services/http_access_policy_service.py new file mode 100644 index 0000000..d306ab1 --- /dev/null +++ b/services/http_access_policy_service.py @@ -0,0 +1,72 @@ +"""Central deny-by-default policy for sensitive HTTP surfaces.""" + +from __future__ import annotations + +POLICY = "ewoooc_sensitive_http_access_policy_v1" + +# Entire blueprints that expose business data or operator actions. +SESSION_REQUIRED_BLUEPRINTS = frozenset({ + "auto_import", + "category", + "cicd", + "crawler", + "crawler_management", + "vendor", +}) + +# Mixed public/private blueprints use endpoint-level protection. +SESSION_REQUIRED_ENDPOINTS = frozenset({ + "alert.alert_status", + "alert.test_alert", + "code_review.api_history", + "code_review.api_status", + "code_review.dashboard", + "misc.test_url", + "openclaw_bot.set_webhook", + "openclaw_bot.webhook_info", + "sales.abc_analysis_detail", + "system_public.get_logs_api", + "system_public.settings", + "system_public.show_logs", + "system_public.system_settings_page", +}) + +# Every anonymous GET must be intentional and small enough for public exposure. +INTENTIONALLY_PUBLIC_ENDPOINTS = frozenset({ + "bot_api.bot_api_status", + "elephant_alpha.status", + "misc.brand_assets", + "system_public.favicon", + "system_public.health_check", + "system_public.prometheus_metrics", + "system_public.webcrumbs_asset_proxy", + "user_bp.get_password_reqs", +}) + + +def is_session_required(endpoint: str | None, blueprint: str | None) -> bool: + return bool( + blueprint in SESSION_REQUIRED_BLUEPRINTS + or endpoint in SESSION_REQUIRED_ENDPOINTS + ) + + +def enforce_http_access_policy(): + """Protect sensitive routes before their route function executes.""" + from flask import request + + if not is_session_required(request.endpoint, request.blueprint): + return None + from auth import require_authenticated_request + + return require_authenticated_request() + + +__all__ = [ + "POLICY", + "SESSION_REQUIRED_BLUEPRINTS", + "SESSION_REQUIRED_ENDPOINTS", + "INTENTIONALLY_PUBLIC_ENDPOINTS", + "enforce_http_access_policy", + "is_session_required", +] diff --git a/services/security_governance_review_service.py b/services/security_governance_review_service.py new file mode 100644 index 0000000..9a29666 --- /dev/null +++ b/services/security_governance_review_service.py @@ -0,0 +1,670 @@ +"""Machine-readable security and governance review for EwoooC.""" + +from __future__ import annotations + +import ast +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from services.http_access_policy_service import ( + INTENTIONALLY_PUBLIC_ENDPOINTS, + SESSION_REQUIRED_BLUEPRINTS, + SESSION_REQUIRED_ENDPOINTS, +) + + +POLICY = "global_product_governance_v2_security_review_v1" +ROOT = Path(__file__).resolve().parents[1] +ASSET_INVENTORY_PATH = Path("governance/ewoooc_asset_inventory.json") +QUALITY_BASELINE_PATH = Path("governance/security_governance_review_baseline.json") + +MUTATING_METHODS = {"POST", "PUT", "PATCH", "DELETE"} +AUTH_DECORATOR_MARKERS = ( + "login_required", + "admin_required", + "role_required", + "permission_required", + "check_alert_auth", + "internal_key_required", + "require_api_token", + "verify_webhook", +) +INLINE_AUTH_CALLS = { + "_require_internal_key", + "verify_internal_token", + "verify_webhook_signature", + "compare_digest", +} +FORBIDDEN_SOURCE_MARKERS = ( + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + "ghcr.io", +) +ASSET_REQUIRED_FIELDS = { + "canonical_id", + "asset_type", + "owner_lane", + "source_truth", + "runtime_identity", + "signal_freshness", + "policy", + "executor", + "verifier", + "backup_restore", + "learning_targets", +} + + +def _read(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return "" + + +def _blueprint_names(tree: ast.Module) -> dict[str, str]: + names: dict[str, str] = {} + for node in tree.body: + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call): + continue + func = node.value.func + if not isinstance(func, ast.Name) or func.id != "Blueprint" or not node.value.args: + continue + first = node.value.args[0] + if not isinstance(first, ast.Constant) or not isinstance(first.value, str): + continue + for target in node.targets: + if isinstance(target, ast.Name): + names[target.id] = first.value + return names + + +def _route_metadata(decorator: ast.expr) -> tuple[str, set[str]] | None: + if not isinstance(decorator, ast.Call) or not isinstance(decorator.func, ast.Attribute): + return None + if decorator.func.attr != "route" or not decorator.args: + return None + first = decorator.args[0] + route = first.value if isinstance(first, ast.Constant) and isinstance(first.value, str) else "" + methods = {"GET"} + for keyword in decorator.keywords: + if keyword.arg != "methods" or not isinstance(keyword.value, (ast.List, ast.Tuple)): + continue + parsed = { + item.value.upper() + for item in keyword.value.elts + if isinstance(item, ast.Constant) and isinstance(item.value, str) + } + if parsed: + methods = parsed + return route, methods + + +def _called_names(node: ast.AST) -> set[str]: + names: set[str] = set() + for child in ast.walk(node): + if not isinstance(child, ast.Call): + continue + if isinstance(child.func, ast.Name): + names.add(child.func.id) + elif isinstance(child.func, ast.Attribute): + names.add(child.func.attr) + return names + + +def _scan_routes(root: Path) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + parse_errors: list[str] = [] + for path in sorted((root / "routes").glob("*.py")): + try: + tree = ast.parse(_read(path), filename=str(path)) + except SyntaxError: + parse_errors.append(path.relative_to(root).as_posix()) + continue + blueprints = _blueprint_names(tree) + for node in tree.body: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + decorator_text = [ast.unparse(item) for item in node.decorator_list] + explicit_guard = any( + marker in text + for marker in AUTH_DECORATOR_MARKERS + for text in decorator_text + ) + inline_guard = bool(_called_names(node) & INLINE_AUTH_CALLS) + for decorator in node.decorator_list: + metadata = _route_metadata(decorator) + if metadata is None: + continue + route, methods = metadata + blueprint_var = "" + if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Attribute): + if isinstance(decorator.func.value, ast.Name): + blueprint_var = decorator.func.value.id + blueprint = blueprints.get(blueprint_var, blueprint_var.removesuffix("_bp")) + endpoint = f"{blueprint}.{node.name}" if blueprint else node.name + central_guard = ( + blueprint in SESSION_REQUIRED_BLUEPRINTS + or endpoint in SESSION_REQUIRED_ENDPOINTS + ) + rows.append({ + "file": path.relative_to(root).as_posix(), + "line": node.lineno, + "endpoint": endpoint, + "route": route, + "methods": sorted(methods), + "mutating": bool(methods & MUTATING_METHODS), + "guarded": explicit_guard or inline_guard or central_guard, + "guard_source": ( + "decorator" if explicit_guard else + "inline" if inline_guard else + "central_policy" if central_guard else + "none" + ), + }) + + unguarded = [item for item in rows if not item["guarded"]] + mutating_unguarded = [item for item in unguarded if item["mutating"]] + unclassified_unguarded = [ + item for item in unguarded + if item["endpoint"] not in INTENTIONALLY_PUBLIC_ENDPOINTS + ] + return { + "route_count": len(rows), + "guarded_count": sum(1 for item in rows if item["guarded"]), + "unguarded_count": len(unguarded), + "unguarded": unguarded, + "unclassified_unguarded_count": len(unclassified_unguarded), + "unclassified_unguarded": unclassified_unguarded, + "mutating_unguarded_count": len(mutating_unguarded), + "mutating_unguarded": mutating_unguarded[:30], + "parse_errors": parse_errors, + } + + +def _asset_inventory(root: Path) -> dict[str, Any]: + path = root / ASSET_INVENTORY_PATH + try: + payload = json.loads(_read(path)) + except (TypeError, json.JSONDecodeError): + payload = {} + assets = payload.get("assets") if isinstance(payload, dict) else [] + if not isinstance(assets, list): + assets = [] + missing: list[dict[str, Any]] = [] + populated_fields = 0 + expected_fields = len(assets) * len(ASSET_REQUIRED_FIELDS) + for index, asset in enumerate(assets): + if not isinstance(asset, dict): + missing.append({"index": index, "missing_fields": sorted(ASSET_REQUIRED_FIELDS)}) + continue + missing_fields = sorted(field for field in ASSET_REQUIRED_FIELDS if not asset.get(field)) + populated_fields += len(ASSET_REQUIRED_FIELDS) - len(missing_fields) + if missing_fields: + missing.append({ + "index": index, + "canonical_id": asset.get("canonical_id"), + "missing_fields": missing_fields, + }) + field_coverage = round((populated_fields / expected_fields * 100), 1) if expected_fields else 0.0 + reconciliation = payload.get("reconciliation") if isinstance(payload, dict) else {} + return { + "path": ASSET_INVENTORY_PATH.as_posix(), + "asset_count": len(assets), + "field_coverage_percent": field_coverage, + "missing_field_assets": missing, + "schema_valid": bool(assets) and not missing, + "reconciliation_status": (reconciliation or {}).get("status", "missing"), + "runtime_receipts_complete": bool((reconciliation or {}).get("runtime_receipts_complete")), + } + + +def _quality_baseline(root: Path) -> dict[str, Any]: + path = root / QUALITY_BASELINE_PATH + try: + payload = json.loads(_read(path)) + except (TypeError, json.JSONDecodeError): + payload = {} + collection = payload.get("collection") if isinstance(payload, dict) else {} + full_suite = payload.get("full_suite_without_missing_contract") if isinstance(payload, dict) else {} + focused = payload.get("focused_suites") if isinstance(payload, dict) else {} + collection = collection if isinstance(collection, dict) else {} + full_suite = full_suite if isinstance(full_suite, dict) else {} + focused = focused if isinstance(focused, dict) else {} + return { + "path": QUALITY_BASELINE_PATH.as_posix(), + "captured_at": payload.get("captured_at") if isinstance(payload, dict) else None, + "collection_status": collection.get("status", "missing"), + "collection_error_count": int(collection.get("error_count") or 0), + "missing_contract": collection.get("missing_contract"), + "full_suite_passed": int(full_suite.get("passed") or 0), + "full_suite_failed": int(full_suite.get("failed") or 0), + "full_suite_skipped": int(full_suite.get("skipped") or 0), + "focused_suites": focused, + "next_machine_action": payload.get("next_machine_action") if isinstance(payload, dict) else None, + } + + +def _workflow_supply_chain(root: Path) -> dict[str, Any]: + files = sorted((root / ".gitea" / "workflows").glob("*.y*ml")) + forbidden: list[dict[str, Any]] = [] + external_actions: list[dict[str, Any]] = [] + for path in files: + for line_number, line in enumerate(_read(path).splitlines(), start=1): + lower = line.lower() + for marker in FORBIDDEN_SOURCE_MARKERS: + if marker in lower: + forbidden.append({ + "file": path.relative_to(root).as_posix(), + "line": line_number, + "marker": marker, + }) + stripped = line.strip() + if stripped.startswith("uses:") or " uses:" in stripped: + external_actions.append({ + "file": path.relative_to(root).as_posix(), + "line": line_number, + "reference": stripped[:180], + }) + return { + "workflow_count": len(files), + "forbidden_source_references": forbidden, + "external_action_references": external_actions, + "gitea_only": bool(files) and not forbidden and not external_actions, + } + + +def _dependency_posture(root: Path) -> dict[str, Any]: + requirements = [] + for raw in _read(root / "requirements.txt").splitlines(): + line = raw.split("#", 1)[0].strip() + if line and not line.startswith(("-", "--")): + requirements.append(line) + exact = [line for line in requirements if "==" in line or " @ " in line] + sbom_names = {"sbom.json", "bom.json", "cyclonedx.json", "spdx.json"} + sbom_candidates: list[str] = [] + for name in sorted(sbom_names): + candidate = root / name + if candidate.is_file(): + sbom_candidates.append(candidate.relative_to(root).as_posix()) + for folder in ("artifacts", "build", "dist", "governance"): + base = root / folder + if not base.is_dir(): + continue + sbom_candidates.extend( + path.relative_to(root).as_posix() + for path in base.rglob("*") + if path.is_file() and path.name.lower() in sbom_names + ) + return { + "direct_requirement_count": len(requirements), + "exact_requirement_count": len(exact), + "all_direct_requirements_exact": bool(requirements) and len(exact) == len(requirements), + "lock_exists": (root / "requirements.lock").exists(), + "sbom_files": sbom_candidates, + "automated_sca_configured": any( + marker in _read(path).lower() + for path in (root / ".gitea" / "workflows").glob("*.y*ml") + for marker in ("pip-audit", "trivy", "grype", "osv-scanner") + ), + } + + +def _large_python_files(root: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for folder in ("services", "routes"): + for path in (root / folder).rglob("*.py"): + try: + line_count = sum(1 for _ in path.open(encoding="utf-8", errors="ignore")) + except OSError: + continue + if line_count > 800: + rows.append({ + "file": path.relative_to(root).as_posix(), + "lines": line_count, + }) + return sorted(rows, key=lambda item: item["lines"], reverse=True) + + +def _check( + check_id: str, + function: str, + status: str, + severity: str, + summary: str, + evidence: dict[str, Any], + next_action: str, + *, + release_blocking: bool = False, +) -> dict[str, Any]: + return { + "id": check_id, + "function": function, + "status": status, + "severity": severity, + "summary": summary, + "evidence": evidence, + "next_machine_action": next_action, + "release_blocking": release_blocking, + } + + +def _work_items(checks: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + return [ + {"order": 1, "id": "SEC-P0-001", "priority": "P0", "status": "in_progress", "lane": "deny_by_default_access", "objective": "Close all sensitive and mutating routes behind central auth or signed internal transport.", "next_machine_action": "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth"}, + {"order": 2, "id": "SEC-P0-002", "priority": "P0", "status": "not_started", "lane": "identity_rbac", "objective": "Replace the shared admin password with database-backed identities, least-privilege RBAC, durable lockout and auditable sessions.", "next_machine_action": "inventory_active_users_then_shadow_database_auth_before_cutover"}, + {"order": 3, "id": "SEC-P0-003", "priority": "P0", "status": "in_progress", "lane": "webhook_trust", "objective": "Require Telegram and internal webhook transport secrets with replay protection.", "next_machine_action": "provision_telegram_webhook_secret_then_enable_required_mode_and_canary"}, + {"order": 4, "id": "SUPPLY-P0-001", "priority": "P0", "status": "in_progress", "lane": "software_supply_chain", "objective": "Use Gitea-only checkout, exact locks, SAST/SCA/secret scan, SBOM, image digest and provenance gates.", "next_machine_action": "add_internal_scanner_artifacts_and_exact_dependency_lock"}, + {"order": 5, "id": "GOV-P0-001", "priority": "P0", "status": "in_progress", "lane": "asset_graph", "objective": "Reconcile every host, service, data, AI, route, package and evidence asset against runtime truth.", "next_machine_action": "probe_all_assets_and_write_same_run_reconciliation_receipt"}, + {"order": 6, "id": "GOV-P0-002", "priority": "P0", "status": "not_started", "lane": "controlled_apply_contract", "objective": "Enforce one trace_id/run_id/work_item_id from sensor through verifier, rollback and learning acknowledgement.", "next_machine_action": "introduce_shared_governance_envelope_and_migrate_eventrouter_autoheal_first"}, + {"order": 7, "id": "RAG-P0-001", "priority": "P0", "status": "not_started", "lane": "internal_rag_canary", "objective": "Canary PixelRAG candidate knowledge into internal pgvector RAG without price or ai_insights promotion.", "next_machine_action": "run_internal_rag_candidate_canary"}, + {"order": 8, "id": "MCP-P0-001", "priority": "P0", "status": "in_progress", "lane": "mcp_rag_runtime", "objective": "Enable and verify MCP/RAG runtime health rather than registry-only readiness.", "next_machine_action": "close_mcp_ports_router_and_rag_runtime_readback"}, + {"order": 9, "id": "SEC-P0-004", "priority": "P0", "status": "not_started", "lane": "security_operations", "objective": "Publish security MTTA/MTTR, recurrence, false-positive, verifier and rollback metrics with incident closure receipts.", "next_machine_action": "add_security_event_metrics_and_incident_lifecycle_store"}, + {"order": 10, "id": "REL-P0-001", "priority": "P0", "status": "in_progress", "lane": "release_readback", "objective": "Separate source, test, CD, production runtime and visible proof for every release without touching momo-db.", "next_machine_action": "deploy_v10_771_then_read_back_security_matrix_and_container_identity"}, + {"order": 11, "id": "RES-P1-001", "priority": "P1", "status": "not_started", "lane": "resilience", "objective": "Automate checksum, offsite backup and isolated restore drills with measured RPO/RTO.", "next_machine_action": "build_non_destructive_restore_drill_receipt"}, + {"order": 12, "id": "PLAT-P1-001", "priority": "P1", "status": "not_started", "lane": "container_hardening", "objective": "Run application containers as non-root with minimal capabilities and read-only filesystems where compatible.", "next_machine_action": "shadow_nonroot_image_then_canary_three_app_containers"}, + {"order": 13, "id": "APPSEC-P1-001", "priority": "P1", "status": "in_progress", "lane": "browser_security", "objective": "Move CSP from report-only to enforced and remove unsafe DOM/inline sinks by measured rollout.", "next_machine_action": "collect_csp_violations_then_replace_high_risk_innerhtml_sinks"}, + {"order": 14, "id": "APPSEC-P1-002", "priority": "P1", "status": "not_started", "lane": "unsafe_cache_serialization", "objective": "Replace writable pickle caches with signed JSON or constrained serialization.", "next_machine_action": "migrate_dashboard_daily_sales_and_edm_shared_caches"}, + {"order": 15, "id": "ARCH-P1-001", "priority": "P1", "status": "in_progress", "lane": "modularization", "objective": "Split oversized services and routes so policy, executor and verifier can be independently tested.", "next_machine_action": "split_ai_automation_smoke_and_pchome_mapping_backlog_by_family"}, + {"order": 16, "id": "UX-P1-001", "priority": "P1", "status": "in_progress", "lane": "professional_product_ui", "objective": "Complete first-viewport cockpit, progressive disclosure, text-density reduction, accessibility and mobile visual smoke across primary pages.", "next_machine_action": "run_sitewide_visual_qa_then_fix_highest_traffic_text_walls"}, + {"order": 17, "id": "PIXELRAG-P1-001", "priority": "P1", "status": "not_started", "lane": "visual_embedding", "objective": "Benchmark Ollama-first multimodal embedding and pgvector-compatible visual metadata.", "next_machine_action": "benchmark_qwen3_vl_equivalent_on_gcp_a_gcp_b_111"}, + {"order": 18, "id": "MARKET-P1-001", "priority": "P1", "status": "in_progress", "lane": "marketplace_source_contracts", "objective": "Finish stable source contracts for Shopee, Coupang, Yahoo, ETMall, Friday and Rakuten with barrier classification.", "next_machine_action": "execute_provenance_rate_limit_and_public_boundary_replays"}, + {"order": 19, "id": "QA-P1-001", "priority": "P1", "status": "in_progress", "lane": "test_governance", "objective": "Restore clean collection, eliminate time/copy drift and split deterministic release tests from nightly integration and production canaries.", "next_machine_action": "restore_or_retire_missing_pchome_report_contract_then_baseline_legacy_failures"}, + {"order": 20, "id": "GOV-P2-001", "priority": "P2", "status": "not_started", "lane": "continuous_control_improvement", "objective": "Trend NIST CSF and OWASP ASVS control coverage and automatically create drift work items.", "next_machine_action": "persist_review_snapshots_and_diff_control_regressions"}, + {"order": 21, "id": "DATA-P2-001", "priority": "P2", "status": "not_started", "lane": "data_governance", "objective": "Classify business, personal, operational and model data and enforce retention with auditable non-destructive defaults.", "next_machine_action": "build_data_classification_and_retention_inventory"}, + {"order": 22, "id": "CHAOS-P2-001", "priority": "P2", "status": "not_started", "lane": "controlled_resilience", "objective": "Exercise model-host, MCP, queue, Telegram and app-container failures with bounded rollback and learning receipts.", "next_machine_action": "design_non_destructive_failure_drill_matrix"}, + ] + + +def build_security_governance_review( + *, + root: Path | str | None = None, + detail_limit: int = 30, +) -> dict[str, Any]: + """Review source and runtime flags without network, DB or secret access.""" + scan_root = Path(root).resolve() if root is not None else ROOT + detail_limit = max(5, min(int(detail_limit or 30), 100)) + + route_posture = _scan_routes(scan_root) + route_posture["unguarded"] = route_posture["unguarded"][:detail_limit] + route_posture["unclassified_unguarded"] = route_posture["unclassified_unguarded"][:detail_limit] + route_posture["mutating_unguarded"] = route_posture["mutating_unguarded"][:detail_limit] + assets = _asset_inventory(scan_root) + supply_chain = _workflow_supply_chain(scan_root) + dependencies = _dependency_posture(scan_root) + quality = _quality_baseline(scan_root) + large_files = _large_python_files(scan_root) + app_source = _read(scan_root / "app.py") + alert_source = _read(scan_root / "routes" / "alert_routes.py") + constitution = _read(scan_root / "CONSTITUTION.md") + dockerfile = _read(scan_root / "Dockerfile") + telegram_secret_configured = bool(os.getenv("TELEGRAM_WEBHOOK_SECRET_TOKEN")) + mcp_enabled = os.getenv("MCP_ROUTER_ENABLED", "false").lower() in {"1", "true", "yes", "on"} + rag_enabled = os.getenv("RAG_ENABLED", "false").lower() in {"1", "true", "yes", "on"} + + required_headers = { + "X-Content-Type-Options", + "X-Frame-Options", + "Referrer-Policy", + "Permissions-Policy", + "Strict-Transport-Security", + "Content-Security-Policy-Report-Only", + } + missing_headers = sorted(header for header in required_headers if header not in app_source) + session_secure_default = "PUBLIC_URL', 'https://mo.wooo.work'" in app_source + protected_resource_ready = all( + marker in alert_source + for marker in ("'momo-db'", "ALLOWED_AUTO_FIX_CONTAINERS", "PROTECTED_CONTAINERS") + ) + central_policy_registered = "enforce_http_access_policy" in app_source + + core_contract_source = "\n".join( + _read(scan_root / path) + for path in ( + "services/event_router.py", + "services/auto_heal_service.py", + "services/agent_actions.py", + "services/code_review_pipeline_service.py", + "services/elephant_alpha_autonomous_engine.py", + ) + ) + contract_fields = { + field: field in core_contract_source + for field in ("trace_id", "run_id", "work_item_id", "post_verifier", "rollback", "learning_recorded") + } + contract_coverage = sum(contract_fields.values()) + policy_drift_markers = [ + marker + for marker in ( + "所有 API Token、密碼存放於 `config.py`", + "nohup python3 app.py", + "pkill -9 -f \"python3 app.py\"", + ) + if marker in constitution + ] + + access_contract_ready = bool( + central_policy_registered + and route_posture["mutating_unguarded_count"] == 0 + and route_posture["unclassified_unguarded_count"] == 0 + ) + + checks = [ + _check( + "GV.AM-asset-inventory", + "GOVERN/IDENTIFY", + "partial" if assets["schema_valid"] and not assets["runtime_receipts_complete"] else "pass" if assets["schema_valid"] else "fail", + "high", + f"Asset inventory has {assets['asset_count']} assets and {assets['field_coverage_percent']}% required-field coverage; runtime reconciliation remains {assets['reconciliation_status']}.", + assets, + "probe_all_assets_and_write_same_run_reconciliation_receipt", + release_blocking=not assets["schema_valid"], + ), + _check( + "PR.AA-route-access-control", + "PROTECT", + "pass" if access_contract_ready else "fail", + "critical", + f"Route map: {route_posture['route_count']} total, {route_posture['unguarded_count']} intentionally public, {route_posture['unclassified_unguarded_count']} unclassified and {route_posture['mutating_unguarded_count']} mutating unguarded after central policy.", + route_posture, + "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth", + release_blocking=not access_contract_ready, + ), + _check( + "PR.DS-session-and-browser-baseline", + "PROTECT", + "pass" if not missing_headers and session_secure_default else "fail", + "high", + "Security headers and secure-cookie-by-public-URL baseline are present." if not missing_headers and session_secure_default else "Browser/session baseline is incomplete.", + {"missing_headers": missing_headers, "secure_cookie_default": session_secure_default}, + "enforce_csp_after_report_only_violation_cleanup", + release_blocking=bool(missing_headers) or not session_secure_default, + ), + _check( + "PR.IR-autoheal-resource-boundary", + "PROTECT/RESPOND", + "pass" if protected_resource_ready else "fail", + "critical", + "Auto-fix is deny-by-default for containers and explicitly protects momo-db." if protected_resource_ready else "Auto-fix container boundary is incomplete.", + {"protected_resource_ready": protected_resource_ready}, + "keep_container_allowlist_and_add_independent_post_verifier", + release_blocking=not protected_resource_ready, + ), + _check( + "GV.SC-gitea-only-supply-chain", + "GOVERN/PROTECT", + "pass" if supply_chain["gitea_only"] else "fail", + "critical", + "Gitea workflows have no external action or forbidden GitHub source reference." if supply_chain["gitea_only"] else "Workflow still depends on an external action/source.", + supply_chain, + "replace_external_action_resolution_with_gitea_job_token_checkout", + release_blocking=not supply_chain["gitea_only"], + ), + _check( + "ID.RA-dependency-vulnerability-management", + "IDENTIFY/PROTECT", + "pass" if dependencies["lock_exists"] and dependencies["sbom_files"] and dependencies["automated_sca_configured"] else "fail", + "high", + "Dependency lock, SBOM and automated SCA are not yet a complete release gate.", + dependencies, + "add_exact_lock_internal_sca_secret_scan_sbom_and_image_digest_receipts", + ), + _check( + "PR.PS-container-least-privilege", + "PROTECT", + "pass" if "\nUSER " in f"\n{dockerfile}" else "fail", + "high", + "Application image declares a non-root runtime user." if "\nUSER " in f"\n{dockerfile}" else "Application image currently defaults to root.", + {"dockerfile_user_declared": "\nUSER " in f"\n{dockerfile}"}, + "shadow_nonroot_image_then_canary_three_app_containers", + ), + _check( + "RS.MA-controlled-apply-envelope", + "RESPOND/RECOVER", + "pass" if contract_coverage == len(contract_fields) else "fail", + "critical", + f"Core automation contract contains {contract_coverage}/{len(contract_fields)} required closure fields.", + {"fields": contract_fields, "coverage_count": contract_coverage}, + "introduce_shared_governance_envelope_and_migrate_eventrouter_autoheal_first", + ), + _check( + "RC.RP-restore-drill", + "RECOVER", + "partial" if (scan_root / "backup_system.py").exists() else "fail", + "high", + "Backup code exists, but no durable isolated restore-drill receipt is committed or read back.", + {"backup_code_exists": (scan_root / "backup_system.py").exists(), "restore_receipt_exists": False}, + "build_non_destructive_restore_drill_receipt", + ), + _check( + "GV.PO-policy-alignment", + "GOVERN", + "pass" if not policy_drift_markers else "fail", + "high", + "Constitution is aligned to current production/governance doctrine." if not policy_drift_markers else "Legacy security/deployment doctrine still conflicts with production truth.", + {"legacy_markers": policy_drift_markers}, + "sunset_legacy_rules_with_owner_expiry_replacement_and_verifier", + release_blocking=bool(policy_drift_markers), + ), + _check( + "GV.OV-modularization", + "GOVERN", + "pass" if not large_files else "partial", + "medium", + f"{len(large_files)} service/route files exceed 800 lines; top file has {large_files[0]['lines'] if large_files else 0} lines.", + {"oversized_file_count": len(large_files), "top_files": large_files[:15]}, + "split_ai_automation_smoke_and_pchome_mapping_backlog_by_family", + ), + _check( + "GV.QA-test-governance", + "GOVERN/IDENTIFY", + "pass" if quality["collection_status"] == "passed" and quality["full_suite_failed"] == 0 else "partial", + "high", + f"Focused security suites are green, but full test governance has {quality['collection_error_count']} collection errors and {quality['full_suite_failed']} baseline failures.", + quality, + "restore_or_retire_missing_pchome_report_contract_then_baseline_legacy_failures", + ), + _check( + "PR.AA-telegram-webhook-transport", + "PROTECT", + "pass" if telegram_secret_configured else "partial", + "critical", + "Telegram webhook secret is configured." if telegram_secret_configured else "Code supports Telegram secret verification, but production secret configuration is not proven.", + {"secret_configured": telegram_secret_configured, "secret_value_read": False}, + "provision_telegram_webhook_secret_then_enable_required_mode_and_canary", + ), + _check( + "DE.CM-mcp-rag-runtime", + "DETECT", + "pass" if mcp_enabled and rag_enabled else "partial", + "high", + f"Runtime flags: MCP_ROUTER_ENABLED={mcp_enabled}, RAG_ENABLED={rag_enabled}.", + {"mcp_router_enabled": mcp_enabled, "rag_enabled": rag_enabled}, + "close_mcp_ports_router_and_rag_runtime_readback", + ), + _check( + "AI.RAG-internal-candidate-canary", + "DETECT/RESPOND", + "pass" if (scan_root / "services/internal_rag_candidate_canary_service.py").exists() else "fail", + "high", + "Internal RAG candidate canary exists." if (scan_root / "services/internal_rag_candidate_canary_service.py").exists() else "PixelRAG candidate knowledge stops before an internal RAG canary.", + {"service_exists": (scan_root / "services/internal_rag_candidate_canary_service.py").exists()}, + "run_internal_rag_candidate_canary", + ), + ] + checks_by_id = {item["id"]: item for item in checks} + blocking = [item for item in checks if item["release_blocking"] and item["status"] == "fail"] + status_counts = { + status: sum(1 for item in checks if item["status"] == status) + for status in ("pass", "partial", "fail") + } + score_weights = {"pass": 1.0, "partial": 0.5, "fail": 0.0} + program_percent = round( + sum(score_weights[item["status"]] for item in checks) / len(checks) * 100, + 1, + ) + runtime_checks = [ + checks_by_id["DE.CM-mcp-rag-runtime"], + checks_by_id["AI.RAG-internal-candidate-canary"], + checks_by_id["RC.RP-restore-drill"], + checks_by_id["RS.MA-controlled-apply-envelope"], + ] + runtime_passed = sum(1 for item in runtime_checks if item["status"] == "pass") + + return { + "success": True, + "policy": POLICY, + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": "complete" if status_counts["fail"] == 0 and status_counts["partial"] == 0 else "partial", + "answer_to_owner": "The security/governance target is not complete. The project has many local receipts, but access control, identity, asset reconciliation, supply-chain evidence, same-run controlled apply, restore proof and MCP/RAG runtime closure are not yet one production loop.", + "root_causes": [ + "Feature and receipt growth outran a single deny-by-default security policy.", + "Source/test/UI evidence was repeatedly counted ahead of production closure evidence.", + "AI lanes use many preview/review stages without one shared trace/run/work-item envelope.", + "Asset, dependency, identity, backup and runtime controls are not reconciled by one machine-readable control plane.", + "Large modules couple policy, execution and verification, making independent assurance expensive.", + ], + "benchmark_profile": { + "nist_csf_2_0": ["GOVERN", "IDENTIFY", "PROTECT", "DETECT", "RESPOND", "RECOVER"], + "owasp_asvs": "5.0.0", + "nist_ssdf": "SP 800-218 v1.1", + "application_level_target": "ASVS L2 with selected L3 controls for operator and automation surfaces", + }, + "completion": { + "program_percent": program_percent, + "asset_field_coverage_percent": assets["field_coverage_percent"], + "asset_runtime_reconciliation_complete": assets["runtime_receipts_complete"], + "runtime_closure_passed": runtime_passed, + "runtime_closure_total": len(runtime_checks), + "status_counts": status_counts, + }, + "checks": checks, + "quality_baseline": quality, + "release_gate": { + "status": "pass" if not blocking else "fail", + "blocking_failure_count": len(blocking), + "blocking_failures": [ + {"id": item["id"], "summary": item["summary"]} + for item in blocking + ], + "debt_does_not_equal_release_pass": True, + }, + "work_items": _work_items(checks_by_id), + "safety": { + "read_only": True, + "network_calls": False, + "database_reads": False, + "database_writes": False, + "secret_reads": False, + "external_side_effects": False, + }, + "next_machine_action": "verify_production_anonymous_matrix_and_move_metrics_behind_edge_auth", + } + + +__all__ = ["POLICY", "build_security_governance_review"] diff --git a/tests/test_ai_automation_smoke_service.py b/tests/test_ai_automation_smoke_service.py index a2eb71b..1dab687 100644 --- a/tests/test_ai_automation_smoke_service.py +++ b/tests/test_ai_automation_smoke_service.py @@ -1308,6 +1308,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) + monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check("security governance", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) @@ -1326,7 +1327,7 @@ def test_collect_ai_automation_smoke_uses_worst_status(monkeypatch): result = smoke.collect_ai_automation_smoke(record_history=False) assert result["status"] == "critical" - assert result["summary"] == {"ok": 42, "warning": 1, "critical": 1, "total": 44} + assert result["summary"] == {"ok": 43, "warning": 1, "critical": 1, "total": 45} def test_pchome_controlled_apply_drift_monitor_reports_verified_zero_drift(monkeypatch): @@ -3853,6 +3854,7 @@ def test_collect_ai_automation_smoke_persists_recent_history(tmp_path, monkeypat monkeypatch.setattr(smoke, "_ai_surface_html_readback_check", lambda: smoke._check("surface", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_ui_ux_agent_check", lambda: smoke._check("sitewide", "ok", "ok")) monkeypatch.setattr(smoke, "_sitewide_visual_qa_check", lambda: smoke._check("visual", "ok", "ok")) + monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check("security governance", "ok", "ok")) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check("external mcp rag", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_rag_candidate_replay_check", lambda: smoke._check("pixelrag replay", "ok", "ok")) monkeypatch.setattr(smoke, "_pixelrag_source_contract_replay_worker_check", lambda: smoke._check("pixelrag source contract", "ok", "ok")) @@ -3917,6 +3919,17 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( from datetime import datetime from services import ai_automation_smoke_service as smoke + monkeypatch.setattr( + smoke, + "_security_governance_review_check", + lambda: smoke._check( + "Security governance automation review", + "ok", + "security governance ok", + {"next_machine_action": "keep_monitoring"}, + ), + ) + history_path = tmp_path / "smoke_history.jsonl" history_path.write_text( json.dumps({ @@ -4307,7 +4320,7 @@ def test_scheduled_automation_health_summary_reads_history_without_side_effects( ) assert summary["policy"] == "read_only_ai_automation_scheduled_health_summary" assert summary["status"] == "ok" - assert summary["summary"]["total"] == 41 + assert summary["summary"]["total"] == 42 assert summary["summary"]["primary_human_gate_count"] == 0 assert summary["summary"]["writes_database_count"] == 0 assert pchome_family["status"] == "ok" @@ -6629,6 +6642,11 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): "visual ok", {"route_count": 12, "viewport_count": 3, "failed_count": 0}, )) + monkeypatch.setattr(smoke, "_security_governance_review_check", lambda: smoke._check( + "Security governance automation review", + "ok", + "security governance ok", + )) monkeypatch.setattr(smoke, "_external_mcp_rag_integration_check", lambda: smoke._check( "External MCP/RAG integration readback", "ok", @@ -6714,7 +6732,7 @@ def test_surface_html_readback_check_is_part_of_ai_smoke(monkeypatch): item for item in result["checks"] if item["name"] == "Sitewide visual QA readback" ) - assert result["summary"]["total"] == 44 + assert result["summary"]["total"] == 45 assert surface_check["status"] == "ok" assert surface_check["details"]["checked_surface_count"] == 10 assert sitewide_check["status"] == "ok" diff --git a/tests/test_frontend_v2_assets.py b/tests/test_frontend_v2_assets.py index 8b22400..0eeb620 100644 --- a/tests/test_frontend_v2_assets.py +++ b/tests/test_frontend_v2_assets.py @@ -393,7 +393,7 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "clear_competitor_intel_cache()" in route_source assert "_extract_match_diagnostic_reasons" in route_source assert "妝效質地不同" in route_source - assert "多款任選待確認" in route_source + assert "variant_selection_review" in route_source assert "MockRecord" not in route_source assert "{% for item in items %}" in dashboard assert "PChome 業績成長作戰台" in dashboard @@ -444,10 +444,10 @@ def test_dashboard_v2_is_production_default_and_uses_real_dashboard_data(): assert "否決候選" in dashboard assert "標記單位價" in dashboard assert "補搜尋" in dashboard - assert "覆核建議：" in dashboard + assert "AI 建議：" in dashboard assert "review.catalog_review_guidance.action_hint" in dashboard assert 'data-review-action="needs_research"' in dashboard - assert "人工閉環" in route_source + assert "AI 全自動閉環" in route_source assert "AI 挑品清單" in dashboard assert "比價覆核隊列" in dashboard assert "下一步" in dashboard @@ -534,7 +534,7 @@ def test_pchome_review_export_and_diagnostics_use_real_queue_data(): assert "商品 / MOMO" in dashboard assert "PChome 待確認商品" in dashboard assert "開 PChome 待確認商品" in dashboard - assert "確認採用這筆 PChome 商品為正式同款配對？" in dashboard + assert "執行 AI 決策：採用這筆 PChome 商品為正式同款配對？" in dashboard assert "PChome 候選" not in dashboard assert "覆核判讀" in dashboard assert "dashboard-review-workbench-row" in dashboard @@ -551,7 +551,7 @@ def test_pchome_review_export_and_diagnostics_use_real_queue_data(): assert 'title="決策追蹤"' in dashboard assert "優先 {{ envelope.severity" in dashboard assert "證據完整" in dashboard - assert "需人工" in dashboard + assert "AI 全自動閉環" in dashboard assert "review.decision_envelope" in dashboard assert "dashboard-review-envelope" in dashboard assert "review.diagnostic_reasons" in dashboard diff --git a/tests/test_security_governance_review_service.py b/tests/test_security_governance_review_service.py new file mode 100644 index 0000000..a6f9d2e --- /dev/null +++ b/tests/test_security_governance_review_service.py @@ -0,0 +1,164 @@ +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from flask import Blueprint, Flask, jsonify + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_security_governance_review_is_honest_and_release_gate_passes(): + from services.security_governance_review_service import ( + build_security_governance_review, + ) + + report = build_security_governance_review(root=ROOT, detail_limit=12) + + assert report["status"] == "partial" + assert report["release_gate"]["status"] == "pass" + assert report["release_gate"]["blocking_failure_count"] == 0 + assert report["completion"]["asset_field_coverage_percent"] == 100.0 + assert report["completion"]["asset_runtime_reconciliation_complete"] is False + assert report["completion"]["runtime_closure_passed"] < report["completion"]["runtime_closure_total"] + assert report["work_items"][0]["id"] == "SEC-P0-001" + assert len(report["work_items"]) == 22 + assert report["work_items"][9]["id"] == "REL-P0-001" + assert any(item["id"] == "QA-P1-001" for item in report["work_items"]) + assert report["work_items"][-1]["id"] == "CHAOS-P2-001" + checks = {item["id"]: item for item in report["checks"]} + access = checks["PR.AA-route-access-control"]["evidence"] + assert access["mutating_unguarded_count"] == 0 + assert access["unclassified_unguarded_count"] == 0 + assert access["unguarded_count"] == 8 + dependencies = checks["ID.RA-dependency-vulnerability-management"]["evidence"] + assert dependencies["sbom_files"] == [] + quality = checks["GV.QA-test-governance"] + assert quality["status"] == "partial" + assert quality["evidence"]["collection_error_count"] == 1 + assert quality["evidence"]["full_suite_passed"] == 1701 + assert report["safety"] == { + "read_only": True, + "network_calls": False, + "database_reads": False, + "database_writes": False, + "secret_reads": False, + "external_side_effects": False, + } + + +def test_security_governance_cli_strict_mode_passes_source_gate(): + completed = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "ops" / "report_security_governance_review.py"), + "--root", + str(ROOT), + "--compact", + "--strict", + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert completed.returncode == 0, completed.stderr or completed.stdout + payload = json.loads(completed.stdout) + assert payload["release_gate"]["status"] == "pass" + assert payload["status"] == "partial" + + +def test_sensitive_blueprint_policy_denies_anonymous_api(monkeypatch): + import auth + from services.http_access_policy_service import enforce_http_access_policy + + monkeypatch.setattr(auth, "DISABLE_LOGIN", False) + app = Flask(__name__) + app.secret_key = "test" + vendor = Blueprint("vendor", __name__) + + @vendor.route("/vendor-stockout/api/vendor/list") + def list_vendors(): + return jsonify({"success": True}) + + app.register_blueprint(vendor) + app.before_request(enforce_http_access_policy) + client = app.test_client() + + anonymous = client.get("/vendor-stockout/api/vendor/list") + assert anonymous.status_code == 401 + assert anonymous.get_json()["error"] == "authentication_required" + + with client.session_transaction() as session: + session["logged_in"] = True + authenticated = client.get("/vendor-stockout/api/vendor/list") + assert authenticated.status_code == 200 + + +def test_public_url_validator_blocks_ssrf_targets(monkeypatch): + from utils import security + + monkeypatch.setattr( + security.socket, + "getaddrinfo", + lambda *_args, **_kwargs: [(2, 1, 6, "", ("93.184.216.34", 443))], + ) + assert security.validate_public_http_url("https://example.com/path") == "https://example.com/path" + + monkeypatch.setattr( + security.socket, + "getaddrinfo", + lambda *_args, **_kwargs: [(2, 1, 6, "", ("127.0.0.1", 80))], + ) + with pytest.raises(ValueError, match="loopback"): + security.validate_public_http_url("http://example.com/") + + +def test_alert_auto_fix_protects_database_and_denies_unknown_container(monkeypatch): + from routes import alert_routes + + def fail_if_called(*_args, **_kwargs): + raise AssertionError("subprocess must not run for denied containers") + + monkeypatch.setattr(alert_routes.subprocess, "run", fail_if_called) + + protected = alert_routes.auto_fix_container("momo-db", "memory") + assert protected["success"] is False + assert "受保護" in protected["message"] + + unknown = alert_routes.auto_fix_container("stock-platform", "memory") + assert unknown["success"] is False + assert "白名單" in unknown["message"] + + +def test_security_governance_smoke_family_is_warning_for_nonblocking_debt(monkeypatch): + from services import ai_automation_smoke_service as smoke + from services import security_governance_review_service as review_service + + monkeypatch.setattr( + review_service, + "build_security_governance_review", + lambda **_kwargs: { + "policy": "test-policy", + "status": "partial", + "completion": { + "program_percent": 45.0, + "asset_field_coverage_percent": 100.0, + "asset_runtime_reconciliation_complete": False, + "runtime_closure_passed": 0, + "runtime_closure_total": 4, + }, + "release_gate": {"blocking_failure_count": 0}, + "next_machine_action": "close_access_control", + }, + ) + + check = smoke._security_governance_review_check() + + assert check["status"] == "warning" + assert check["details"]["blocking_failure_count"] == 0 + assert check["details"]["next_machine_action"] == "close_access_control" diff --git a/utils/security.py b/utils/security.py index ae6a792..9a0dbad 100644 --- a/utils/security.py +++ b/utils/security.py @@ -3,10 +3,13 @@ 從 app.py + utils/validators.py 整併的單一權威來源。純驗證邏輯，無 Flask 依賴。 舊的 utils/validators.py 已 deprecate，僅保留 re-export 不破壞既有 import。 """ +import ipaddress import os import re +import socket import unicodedata from pathlib import Path +from urllib.parse import urlparse import pandas as pd @@ -15,6 +18,41 @@ from utils.logger_manager import SystemLogger _log = SystemLogger("Security").get_logger() +def validate_public_http_url(value): + """Validate an outbound URL and reject SSRF targets before any request.""" + candidate = str(value or '').strip() + parsed = urlparse(candidate) + if parsed.scheme not in {'http', 'https'} or not parsed.hostname: + raise ValueError('只允許完整的 HTTP/HTTPS 網址') + if parsed.username or parsed.password: + raise ValueError('網址不可包含帳號或密碼') + if parsed.port not in {None, 80, 443}: + raise ValueError('只允許標準 HTTP/HTTPS 連接埠') + + try: + addresses = { + info[4][0] + for info in socket.getaddrinfo( + parsed.hostname, + parsed.port or (443 if parsed.scheme == 'https' else 80), + type=socket.SOCK_STREAM, + ) + } + except OSError as exc: + raise ValueError('網址主機無法解析') from exc + + if not addresses: + raise ValueError('網址主機沒有可用位址') + for address in addresses: + try: + ip = ipaddress.ip_address(address.split('%', 1)[0]) + except ValueError as exc: + raise ValueError('網址主機位址無效') from exc + if not ip.is_global: + raise ValueError('禁止連線至內網、loopback、link-local 或保留位址') + return candidate + + # ──────────────────────────────────────────────────────────────────────── # SQL Injection 防護 # ────────────────────────────────────────────────────────────────────────