#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ tests/test_openclaw_qa_golden_set.py OpenClaw Q&A 黃金集 A/B 對照框架 (Operation Ollama-First v5.0 — Phase 3, A7 fullstack-engineer) 目的: 在統帥盲測前，先建立 Ollama qwen3:14b vs Gemini 2.5 Flash 的「量化基線」。 10 題典型 momo 商業 Q&A，雙模型各跑一次，比對： - 簡體字污染數量（A2 黃燈警訊核心） - 回應長度 - 結構性指標（行數、列點數） - 拒答訊號 - 黃金關鍵字命中率（題目自帶 expect_keywords） 執行: RUN_GOLDEN_SET=1 pytest tests/test_openclaw_qa_golden_set.py -v -s # GCP 還沒拉 qwen3:14b 之前，預設 SKIP（避免 CI 紅燈） 紀律: - PII 紀律：題目/答案無真實 chat_id / username / 身份證 / 手機，全部去識別化 - 不對「正確性」做 hard assert；本框架專做「品質量化基線」收集 - 報告印到 stdout（pytest -s 顯示），人工檢視，不卡 CI """ import json import os import sys import time from typing import Dict, List, Optional import pytest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # ───────────────────────────────────────────────────────────────────────────── # 啟用條件：須三條件齊備才實跑 # 1. RUN_GOLDEN_SET=1 # 2. OllamaService 三主機級聯可解析出可達主機 # 3. GEMINI_API_KEY 已設 # 否則 SKIP。 # ───────────────────────────────────────────────────────────────────────────── def _ollama_reachable(host: str, timeout: float = 2.0) -> bool: try: import requests r = requests.get(f"{host.rstrip('/')}/api/version", timeout=timeout) return r.status_code == 200 except Exception: return False def _ollama_has_model(host: str, model: str, timeout: float = 3.0) -> bool: """檢查 Ollama 主機是否已 pull 指定模型。""" try: import requests r = requests.get(f"{host.rstrip('/')}/api/tags", timeout=timeout) if r.status_code != 200: return False tags = r.json().get('models', []) return any(m.get('name', '').startswith(model.split(':')[0]) for m in tags) except Exception: return False _RUN_GOLDEN = os.getenv('RUN_GOLDEN_SET', '0') == '1' _MODEL = os.getenv('OPENCLAW_QA_OLLAMA_MODEL', 'qwen3:14b') _HAS_GEMINI = bool(os.getenv('GEMINI_API_KEY')) def _resolved_ollama_host() -> str: from services.ollama_service import resolve_ollama_host return resolve_ollama_host() pytestmark = pytest.mark.skipif( not _RUN_GOLDEN, reason="黃金集需要 RUN_GOLDEN_SET=1 + GCP qwen3:14b ready + GEMINI_API_KEY；統帥盲測前才跑", ) # ───────────────────────────────────────────────────────────────────────────── # 黃金集（10 題；全部去 PII；情境取自 momo-pro 真實 Telegram 互動模式） # ───────────────────────────────────────────────────────────────────────────── GOLDEN_SET: List[Dict] = [ { "id": "g01_weekly_trend", "question": "本週 momo 業績趨勢如何？跟上週比？", "expect_keywords": ["業績", "週", "成長"], "category": "業績趨勢", }, { "id": "g02_competitor_threat", "question": "PChome 最近在 3C 類有發動補貼戰嗎？對我們影響？", "expect_keywords": ["PChome", "3C"], "category": "競品威脅", }, { "id": "g03_pricing_strategy", "question": "我有一支 SKU 比競品貴 8%，銷量持續下滑，該怎麼辦？", "expect_keywords": ["定價", "競品"], "category": "定價策略", }, { "id": "g04_seasonal", "question": "母親節檔期快到了，建議哪些品類加碼？", "expect_keywords": ["母親節", "品類"], "category": "季節機會", }, { "id": "g05_command_routing", "question": "我想看完整週報怎麼下指令？", "expect_keywords": ["weekly", "週報"], "category": "指令導引", }, { "id": "g06_top_threats", "question": "目前 TOP 5 最緊急的競價威脅是哪些？", "expect_keywords": ["威脅", "TOP"], "category": "威脅清單", }, { "id": "g07_inventory_signal", "question": "如何判斷某 SKU 該促銷出清？", "expect_keywords": ["促銷", "出清"], "category": "庫存決策", }, { "id": "g08_cross_category", "question": "家電 vs 生活雜貨，哪個品類本月成長動能比較強？", "expect_keywords": ["家電", "成長"], "category": "品類比較", }, { "id": "g09_data_unavailable", "question": "幫我看 2030 年的銷售預測。", "expect_keywords": ["資料", "無法"], # 期待模型誠實回應「資料不足」而非編造 "category": "資料邊界", }, { "id": "g10_action_item", "question": "綜合本週數據，給我 3 個 48 小時內必做行動。", "expect_keywords": ["行動", "建議"], "category": "行動清單", }, ] # ───────────────────────────────────────────────────────────────────────────── # Scoring helpers # ───────────────────────────────────────────────────────────────────────────── def _count_simplified(text: str) -> int: """重用 strategist service 的簡體字 hint 集合計數。""" from services.openclaw_strategist_service import _SIMPLIFIED_HINT_CHARS return sum(1 for c in (text or '') if c in _SIMPLIFIED_HINT_CHARS) def _count_keyword_hits(text: str, keywords: List[str]) -> int: if not text: return 0 return sum(1 for kw in keywords if kw in text) def _is_refusal(text: str) -> bool: from services.openclaw_strategist_service import _REFUSAL_PATTERNS return any(p in (text or '') for p in _REFUSAL_PATTERNS) def _structure_score(text: str) -> Dict[str, int]: """結構性量化指標。""" if not text: return {"lines": 0, "bullets": 0, "tables": 0} return { "lines": text.count('\n') + 1, # 條列符號粗略偵測（含中文「、」「，」開頭的列點） "bullets": sum(text.count(s) for s in ('- ', '• ', '* ', '1.', '2.', '3.')), "tables": text.count('|'), } def _score_response(qid: str, question: str, response: str, expect_kw: List[str]) -> Dict: structure = _structure_score(response) return { "qid": qid, "length": len(response or ''), "simplified_count": _count_simplified(response), "keyword_hits": _count_keyword_hits(response, expect_kw), "is_refusal": _is_refusal(response), "lines": structure["lines"], "bullets": structure["bullets"], "tables": structure["tables"], "preview": (response or '')[:120].replace('\n', ' / '), } # ───────────────────────────────────────────────────────────────────────────── # Caller wrappers (使用 service 的真實函式) # ───────────────────────────────────────────────────────────────────────────── def _call_ollama(question: str) -> Optional[str]: from services.openclaw_strategist_service import _call_qwen3_qa return _call_qwen3_qa(question, None, f"golden-{int(time.time())}") def _call_gemini_baseline(question: str) -> Optional[str]: from services.openclaw_strategist_service import _call_gemini system_prompt = ( "你是 MOMO Pro 電商情報策略師「OpenClaw」。以繁體中文（台灣用語）回覆使用者。" "嚴禁簡體字。回覆長度控制在 500 字內，可用 Markdown 條列。" ) return _call_gemini(system_prompt, question, temperature=0.5, caller="openclaw_qa_golden") # ───────────────────────────────────────────────────────────────────────────── # Tests # ───────────────────────────────────────────────────────────────────────────── def test_environment_ready(): """sanity check：跑黃金集前確認 Ollama 級聯 host + model + Gemini key 都 ready。""" host = _resolved_ollama_host() assert _ollama_reachable(host), f"Ollama 主機不可達：{host}" assert _ollama_has_model(host, _MODEL), ( f"Ollama 主機 {host} 尚未拉 {_MODEL}（請先完成 ollama pull）" ) assert _HAS_GEMINI, "GEMINI_API_KEY 未設" def test_golden_set_ab_comparison(capsys): """跑 10 題雙模型 A/B 對照，量化指標印到 stdout。 本測試不對「正確性」做 hard assert；目的是給統帥盲測前的「品質量化基線」。 僅 hard assert： - 雙模型至少都有回應（非全 None） - Gemini baseline 簡體字數量 == 0（baseline 不該污染） """ # 啟用 flag 讓 _call_qwen3_qa 走真實邏輯 os.environ['OPENCLAW_QA_OLLAMA_FIRST'] = 'true' rows = [] for item in GOLDEN_SET: qid = item['id'] question = item['question'] kws = item['expect_keywords'] ollama_resp = _call_ollama(question) gemini_resp = _call_gemini_baseline(question) rows.append({ 'qid': qid, 'category': item['category'], 'question': question, 'ollama': _score_response(qid, question, ollama_resp or '', kws), 'gemini': _score_response(qid, question, gemini_resp or '', kws), }) # 列印量化基線（pytest -s 才看得到） print("\n" + "=" * 100) print("OpenClaw QA 黃金集 A/B 量化基線（Ollama qwen3:14b vs Gemini 2.5 Flash）") print("=" * 100) for r in rows: print(f"\n[{r['qid']}] ({r['category']}) {r['question']}") for side in ('ollama', 'gemini'): s = r[side] print( f" {side:>7}: len={s['length']:>4} simp={s['simplified_count']:>2} " f"kw={s['keyword_hits']}/{len(GOLDEN_SET[0]['expect_keywords'])} " f"lines={s['lines']:>2} refusal={s['is_refusal']}" ) print(f" preview: {s['preview']}") # 匯出 JSON 給後續分析 out_path = os.path.join(os.path.dirname(__file__), 'logs', 'qa_golden_baseline.json') os.makedirs(os.path.dirname(out_path), exist_ok=True) with open(out_path, 'w', encoding='utf-8') as f: json.dump(rows, f, ensure_ascii=False, indent=2) print(f"\n基線已存：{out_path}") # Hard assertions（最少安全網） ollama_responded = sum(1 for r in rows if r['ollama']['length'] > 0) gemini_responded = sum(1 for r in rows if r['gemini']['length'] > 0) assert ollama_responded >= 8, f"Ollama 回應率過低：{ollama_responded}/10" assert gemini_responded >= 9, f"Gemini 回應率過低：{gemini_responded}/10" # Gemini baseline 不該有簡體污染（用以驗證測量本身正確） for r in rows: assert r['gemini']['simplified_count'] == 0, ( f"Gemini baseline 簡體污染（指標可能誤判）：{r['qid']} {r['gemini']['preview']}" )