From 2f3dcca22762ee043001cfae4267f5923e7dbc5c Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 3 Jul 2026 00:23:48 +0800 Subject: [PATCH] test(obs): add ai surface html readback guard --- docs/AI_INTELLIGENCE_MODULE_SOT.md | 1 + .../guides/external_professional_benchmark.md | 1 + .../pchome_ai_automation_priority_backlog.md | 14 +++-- tests/test_admin_observability_routes.py | 52 +++++++++++++++++++ 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 735a91d..14297eb 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -107,6 +107,7 @@ - 2026-07-02 起 `/observability/rag_queries` 必須採密集 AI 知識工作台：首屏使用 `data-density-guardrail="compact-knowledge-workbench"`，以「知識命中、省模閉環、回饋學習」短標籤承接 RAG 召回狀態；hero 說明、KPI 小註與頁尾重複標題不得佔用第一層視覺，畫面先保留查詢數、命中率、省模型、回饋分、查詢串流、使用情境品質與命中內容查閱。 - 2026-07-03 起 `/observability/promotion_review` 必須採密集 AI 晉升例外工作台：首屏使用 `data-density-guardrail="compact-promotion-workbench"`，以「AI 例外、去重守門、防污染」短標籤承接知識晉升例外狀態；hero 說明、KPI 小註與頁尾重複標題不得佔用第一層視覺，畫面先保留待決策、知識庫、30 日採用、污染攔截、例外佇列、策略權重與最近知識；不得把人工審核呈現為 primary flow。 - 2026-07-03 起 `/observability/ppt_audit_history` 必須採密集 AI 視覺 QA 工作台：首屏使用 `data-density-guardrail="compact-ppt-workbench"`，以「預覽就緒、視覺審核、修復閉環」短標籤承接 PPT 產線狀態；hero 說明、KPI 小註、runtime/env var、DB/table wording 與頁尾重複標題不得佔用第一層視覺，畫面先保留最新簡報、產線健康度、最近可預覽簡報、工作隊列、視覺問題追蹤、產出紀錄與修復流程。 +- 2026-07-03 起高可見 AI observability surfaces 必須有 route-level rendered HTML readback guard；`/observability/overview`、`/observability/ai_calls`、`/observability/quality_trend`、`/observability/budget`、`/observability/business_intel`、`/observability/host_health`、`/observability/rag_queries`、`/observability/agent_orchestration`、`/observability/promotion_review`、`/observability/ppt_audit_history` 必須在 Flask client render 後仍保留 compact density marker，且不得漏出 `PPT_VISION_ENABLED=`、`PPT_AUTO_GENERATION_ENABLED=`、`ppt_generation_runs`、DB write raw wording、manual/human gate primary wording、Artifact retention / Compact 回讀等產品面不該可見的 engineering copy。 - 2026-07-02 起 `/ai_intelligence` 商品明細與單品作戰詳情的四格價格證據必須可測：PChome 價格、MOMO 參考價、差距、可信度需以 `data-evidence` 固定，並以 `aria-label="價格證據"` 對應可掃描區塊；候選待確認或缺資料只能顯示「候選待確認 / 待補」，不得捏造價格或讓使用者打開 raw payload 才知道判斷依據。 - 2026-07-02 起 `/ai_intelligence` 必須是密集 AI 工作台，不得退回大段文字說明頁：首屏與明細可見內容只保留短狀態、數字、四格證據與下一步按鈕；KPI note、benchmark detail、alert 副句、策略說明、decision copy、來源長句與單品 reason list 不得佔用第一層視覺。`tests/test_ai_intelligence_text_density_guardrails.py` 必須鎖住 `data-density-guardrail="compact-ai-workbench"`、短任務文案、detail meta 與 hidden explanatory copy。 - 2026-07-02 起 `/observability/overview` 也必須採密集 AI 觀測工作台：首屏以 `data-density-guardrail="compact-observability-workbench"`、`AI 觀測 / 風險優先 / 下一步` 與 golden signals 先呈現狀態、數字與操作入口；hero lede、signal note、route desc、host meta 與資料來源長句不得佔用第一層視覺。`tests/test_observability_text_density_guardrails.py` 必須鎖住 compact marker 與 hidden explanatory copy。 diff --git a/docs/guides/external_professional_benchmark.md b/docs/guides/external_professional_benchmark.md index 4c65361..dd5cd53 100644 --- a/docs/guides/external_professional_benchmark.md +++ b/docs/guides/external_professional_benchmark.md @@ -81,6 +81,7 @@ Baymard 的商品頁與比較 UX 研究強調：使用者需要清楚的 product - AI knowledge surface rollout: `/observability/rag_queries` 已套用 compact AI knowledge workbench guardrails；`tests/test_rag_queries_text_density_guardrails.py` 鎖住首屏短標籤、知識命中/省模閉環/回饋學習核心訊號與 hidden explanatory copy。 - AI promotion exception surface rollout: `/observability/promotion_review` 已套用 compact AI promotion workbench guardrails；`tests/test_promotion_review_text_density_guardrails.py` 鎖住首屏短標籤、AI 例外/去重守門/防污染核心訊號與 non-manual wording。 - AI visual QA surface rollout: `/observability/ppt_audit_history` 已套用 compact AI visual QA workbench guardrails；`tests/test_ppt_audit_text_density_guardrails.py` 鎖住首屏短標籤、預覽/審核/修復核心訊號與 non-raw env/table wording。 +- Route HTML readback rollout: high-visibility AI observability surfaces 已新增 Flask route-level rendered HTML guard；`tests/test_admin_observability_routes.py::test_high_visibility_ai_surfaces_route_html_readback_keeps_compact_contract` 鎖住 10 個 route 的 compact density marker 與 non-raw engineering wording。 ## 下一步 TODO 候選 diff --git a/docs/guides/pchome_ai_automation_priority_backlog.md b/docs/guides/pchome_ai_automation_priority_backlog.md index ddc0fcd..cfc98b2 100644 --- a/docs/guides/pchome_ai_automation_priority_backlog.md +++ b/docs/guides/pchome_ai_automation_priority_backlog.md @@ -1,6 +1,6 @@ # PChome AI 自動化優先工作清單 -> 最後更新: 2026-07-02 Asia/Taipei +> 最後更新: 2026-07-03 Asia/Taipei > 範圍: PChome growth / MOMO mapping AI 自動化主線 > 真相來源: Gitea `dev` / `main`、正式環境 `/health`、正式 DB readback、machine-verifiable artifacts @@ -29,7 +29,7 @@ |---|---|---|---| | P0 | Runtime truth + AI controlled apply closure | 進行中，核心閉環已完成多段 | 先證明 AI 真的在正式環境可執行、可回讀、可驗證、可回滾 | | P1 | 產品可視化 + 專業網站體驗 | 進行中 | 使用者要看得到實作結果，AI 自動化不能只藏在 artifact / log | -| P2 | 外部主流 benchmark 落成 guardrails | 未開始 | 把主流專業做法變成可測規則，不只是參考口號 | +| P2 | 外部主流 benchmark 落成 guardrails | 已完成第一版，持續擴充 | 把主流專業做法變成可測規則，不只是參考口號 | | P3 | 擴大 AI 自動化覆蓋面 | 未開始 | 把目前 PChome retry lane 的閉環模式複製到更多安全場景 | | P4 | Source / deployment governance | 持續進行 | 保證版本、Gitea、正式環境、DB 安全紅線不再混亂 | @@ -101,7 +101,7 @@ ## P1 - Product Visibility And Professional Website Experience -狀態: 進行中，本輪已完成 `/ai_intelligence`、`/observability/overview`、`/ai_automation_smoke`、`/observability/agent_orchestration`、`/observability/ai_calls`、`/observability/quality_trend`、`/observability/budget`、`/observability/business_intel`、`/observability/host_health`、`/observability/rag_queries`、`/observability/promotion_review` 與 `/observability/ppt_audit_history` 密集 AI 工作台文字密度守門。 +狀態: 進行中，本輪已完成 `/ai_intelligence`、`/observability/overview`、`/ai_automation_smoke`、`/observability/agent_orchestration`、`/observability/ai_calls`、`/observability/quality_trend`、`/observability/budget`、`/observability/business_intel`、`/observability/host_health`、`/observability/rag_queries`、`/observability/promotion_review` 與 `/observability/ppt_audit_history` 密集 AI 工作台文字密度守門，並新增高可見 AI surfaces route-level HTML readback 防退化 guard。 目的: 讓 AI 自動化在產品裡可見，成為專業營運工作流，而不是只藏在後端。 @@ -168,6 +168,10 @@ - 首屏改成 `data-density-guardrail="compact-ppt-workbench"`，以「預覽就緒 / 視覺審核 / 修復閉環」短標籤承接 PPT 產線狀態。 - hero 說明、KPI 小註、runtime/env var、DB/table wording 與頁尾重複標題從可見 UI 退場，保留最新簡報、產線健康度、最近可預覽簡報、工作隊列、視覺問題追蹤與修復流程。 - `tests/test_ppt_audit_text_density_guardrails.py` 已鎖住 compact marker、第一層視覺 QA 訊號、non-raw wording 與產出紀錄語言。 +- 高可見 AI surfaces route-level HTML readback guard 已完成: + - Flask client 會實際 render `/observability/overview`、`/observability/ai_calls`、`/observability/quality_trend`、`/observability/budget`、`/observability/business_intel`、`/observability/host_health`、`/observability/rag_queries`、`/observability/agent_orchestration`、`/observability/promotion_review`、`/observability/ppt_audit_history`。 + - 每頁都必須保留 compact density marker，且不得回流 `PPT_VISION_ENABLED=`、`PPT_AUTO_GENERATION_ENABLED=`、`ppt_generation_runs`、DB write raw wording、manual/human gate primary wording、Artifact retention / Compact 回讀等產品面 raw engineering copy。 + - `tests/test_admin_observability_routes.py::test_high_visibility_ai_surfaces_route_html_readback_keeps_compact_contract` 已鎖住 route / HTML readback。 已完成 / 下一步，必須照順序: @@ -178,6 +182,7 @@ - 異動狀態 - 下一個機器動作 3. 增加 route / HTML readback，避免產品頁退回 raw engineering wording。 + - 已完成: 高可見 AI observability surfaces 已有 route-level rendered HTML guard。 完成標準: @@ -294,7 +299,8 @@ | P1.8 | AI calls compact workbench text-density guardrails | 已完成 | `/observability/ai_calls` first viewport hides explanatory copy; focused density guard test | Apply density guard to remaining high-visibility AI surfaces | | P1.9 | Quality trend compact workbench text-density guardrails | 已完成 | `/observability/quality_trend` first viewport hides explanatory copy; focused density guard test | Apply density guard to remaining high-visibility AI surfaces | | P1.10 | Promotion review compact AI exception workbench guardrails | 已完成 | `/observability/promotion_review` first viewport hides explanatory copy and avoids manual-primary wording; focused density guard test | Apply density guard to PPT audit surface | -| P1.11 | PPT audit compact AI visual QA workbench guardrails | 已完成 | `/observability/ppt_audit_history` first viewport hides explanatory copy and avoids env/table wording; focused density guard test | Continue P1 surface sweep and P3 automation expansion | +| P1.11 | PPT audit compact AI visual QA workbench guardrails | 已完成 | `/observability/ppt_audit_history` first viewport hides explanatory copy and avoids env/table wording; focused density guard test | P1.12 route-level HTML readback guard | +| P1.12 | High-visibility AI surface route HTML readback guard | 已完成 | 10 observability routes render compact markers and block raw engineering wording in Flask client readback | Continue P1 surface sweep and P3 automation expansion | | P2.1 | External benchmark encoded into requirements | 已完成 | benchmark guide + focused guard test + first-viewport status | P3.1 safe lane expansion | | P3.1 | Extend receipt / replay / drift pattern to more lanes | 已完成 | direct mapping candidate decision lane closeout route + focused tests | P3.2 scheduled automation health summaries | | P3.2 | Scheduled automation health summaries | 已完成 | `/api/ai-automation/scheduled-health-summary` + smoke service focused tests | P3.3 rollback evidence packages | diff --git a/tests/test_admin_observability_routes.py b/tests/test_admin_observability_routes.py index 57b3764..8b78323 100644 --- a/tests/test_admin_observability_routes.py +++ b/tests/test_admin_observability_routes.py @@ -1328,6 +1328,58 @@ def test_agent_orchestration_200(client, monkeypatch): assert r.status_code == 200 +def test_high_visibility_ai_surfaces_route_html_readback_keeps_compact_contract(client, monkeypatch): + """Route-level rendered HTML guard for professional AI workbench surfaces.""" + from routes import admin_observability_routes as mod + import requests as _r + import services.ollama_service as ollama_mod + + def fake_get(*_a, **_kw): + raise _r.exceptions.ConnectionError('mocked') + + monkeypatch.setattr(mod, 'get_session', lambda: _fake_session([])) + monkeypatch.setattr(_r, 'get', fake_get) + monkeypatch.setattr(ollama_mod, '_is_unhealthy', lambda _h: False, raising=False) + monkeypatch.setattr(ollama_mod, '_unhealthy_marks', {}, raising=False) + + surfaces = [ + ('/observability/overview', 'compact-observability-workbench'), + ('/observability/ai_calls', 'compact-ai-calls-workbench'), + ('/observability/quality_trend', 'compact-quality-workbench'), + ('/observability/budget', 'compact-budget-workbench'), + ('/observability/business_intel', 'compact-business-workbench'), + ('/observability/host_health', 'compact-runtime-workbench'), + ('/observability/rag_queries', 'compact-knowledge-workbench'), + ('/observability/agent_orchestration', 'compact-agent-workbench'), + ('/observability/promotion_review', 'compact-promotion-workbench'), + ('/observability/ppt_audit_history', 'compact-ppt-workbench'), + ] + forbidden_fragments = [ + 'PPT_VISION_ENABLED=', + 'PPT_AUTO_GENERATION_ENABLED=', + 'ppt_generation_runs', + 'DB 已寫入', + '待 DB 寫入', + 'DB 寫入失敗', + 'DB writes', + 'manual_required', + 'needs_human=true', + 'human gate', + 'manual review', + 'Artifact retention', + 'Compact 回讀', + ] + + for path, marker in surfaces: + response = client.get(path) + html = response.get_data(as_text=True) + + assert response.status_code == 200, path + assert f'data-density-guardrail="{marker}"' in html + for forbidden in forbidden_fragments: + assert forbidden not in html, f'{forbidden} leaked in {path}' + + def test_health_indicator_api_returns_json(client, monkeypatch): """/observability/api/health_indicator — Phase 52 topbar 健康指示燈 JSON API。""" from routes import admin_observability_routes as mod