From f99305e481e820d68c45809f4eb5d90c028f709c Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 16:12:48 +0800 Subject: [PATCH] feat(km): add full corpus asset taxonomy readback --- apps/api/src/api/v1/knowledge.py | 8 + apps/api/src/models/knowledge.py | 7 + apps/api/src/repositories/interfaces.py | 4 + .../src/repositories/knowledge_repository.py | 153 ++++++++++++++++++ apps/api/src/services/knowledge_service.py | 34 +++- .../test_knowledge_repository_read_model.py | 33 +++- ...est_knowledge_service_readback_degraded.py | 43 +++++ apps/web/messages/en.json | 2 +- apps/web/messages/zh-TW.json | 2 +- .../src/app/[locale]/knowledge-base/page.tsx | 43 +++-- 10 files changed, 312 insertions(+), 17 deletions(-) diff --git a/apps/api/src/api/v1/knowledge.py b/apps/api/src/api/v1/knowledge.py index dcd6a2b68..eb40ff098 100644 --- a/apps/api/src/api/v1/knowledge.py +++ b/apps/api/src/api/v1/knowledge.py @@ -20,6 +20,7 @@ from fastapi import APIRouter, HTTPException, Query from src.models.knowledge import ( EntryStatus, EntryType, + KnowledgeAssetTaxonomyCount, KnowledgeEntry, KnowledgeEntryCreate, KnowledgeEntryUpdate, @@ -105,6 +106,13 @@ async def get_categories() -> list[dict]: return [cat.model_dump() for cat in cats] +@router.get("/asset-taxonomy", response_model=list[KnowledgeAssetTaxonomyCount]) +async def get_asset_taxonomy() -> list[KnowledgeAssetTaxonomyCount]: + """取得 AI 自動化資產維度統計（只讀，不寫入）。""" + service = get_knowledge_service() + return await service.get_asset_taxonomy() + + @router.get("/{entry_id}", response_model=KnowledgeEntry) async def get_entry(entry_id: str) -> KnowledgeEntry: """取得單筆知識條目 (view_count +1)""" diff --git a/apps/api/src/models/knowledge.py b/apps/api/src/models/knowledge.py index bc6b1c7fd..5d3c760a2 100644 --- a/apps/api/src/models/knowledge.py +++ b/apps/api/src/models/knowledge.py @@ -122,11 +122,18 @@ class CategoryCount(BaseModel): count: int +class KnowledgeAssetTaxonomyCount(BaseModel): + """AI 自動化資產維度統計""" + key: str + count: int + + class KnowledgeListResponse(BaseModel): """列表回應""" items: list[KnowledgeEntry] total: int categories: list[CategoryCount] = Field(default_factory=list) + asset_taxonomy: list[KnowledgeAssetTaxonomyCount] = Field(default_factory=list) readback_status: str = "ready" operator_stage: str | None = None next_step: str | None = None diff --git a/apps/api/src/repositories/interfaces.py b/apps/api/src/repositories/interfaces.py index 2976b20ab..35362cfec 100644 --- a/apps/api/src/repositories/interfaces.py +++ b/apps/api/src/repositories/interfaces.py @@ -265,6 +265,10 @@ class IKnowledgeRepository(Protocol): """取得分類統計 [(category, count)]""" ... + async def get_asset_taxonomy_counts(self) -> list[tuple[str, int]]: + """取得 AI 自動化資產維度統計 [(taxonomy_key, count)]""" + ... + async def search(self, query: str, limit: int = 20) -> list[KnowledgeEntry]: """關鍵字搜尋 (title + content + tags)""" ... diff --git a/apps/api/src/repositories/knowledge_repository.py b/apps/api/src/repositories/knowledge_repository.py index a5336bbc2..bff9a8d35 100644 --- a/apps/api/src/repositories/knowledge_repository.py +++ b/apps/api/src/repositories/knowledge_repository.py @@ -29,6 +29,113 @@ logger = structlog.get_logger(__name__) _DEFAULT_CATEGORY = "general" +_ASSET_TAXONOMY_KEYS = ( + "project", + "product", + "website", + "service", + "package", + "tool", + "log", + "alert", + "playbook", + "rag", + "mcp", + "schedule", +) + +_ASSET_TAXONOMY_TERMS: dict[str, tuple[str, ...]] = { + "project": ( + "project", + "repo", + "repository", + "gitea", + "branch", + "workflow", + "source_control", + ), + "product": ( + "product", + "awoooi", + "awooop", + "iwooos", + "stockplatform", + "momo", + "awooogo", + "agent-bounty", + "agent_bounty", + "tsenyang", + "vibework", + "2026fifa", + ), + "website": ("website", "site", "nginx", "ssl", "domain", "route", "frontend"), + "service": ("service", "daemon", "api", "worker", "runtime", "container", "pod"), + "package": ( + "package", + "dependency", + "npm", + "pnpm", + "node", + "python", + "pip", + "prisma", + "next", + "library", + ), + "tool": ( + "tool", + "ansible", + "mcp", + "playbook", + "telegram", + "wazuh", + "kali", + "sentry", + "signoz", + "runner", + ), + "log": ( + "log", + "logs", + "event", + "timeline", + "trace", + "audit", + "callback", + "conversation_event", + "telemetry", + ), + "alert": ( + "alert", + "telegram", + "sentry", + "signoz", + "notification", + "warning", + "critical", + ), + "playbook": ("playbook", "runbook", "sop"), + "rag": ("rag", "vector", "embedding", "semantic", "retrieval"), + "mcp": ("mcp", "connector", "gateway", "tool integration", "tool-integration"), + "schedule": ("schedule", "cron", "job", "worker", "patrol", "recurrence", "cadence"), +} + +_ASSET_TAXONOMY_CATEGORY_HINTS: dict[str, tuple[str, ...]] = { + "website": ("external_site",), + "service": ( + "infrastructure", + "application", + "ai_system", + "database", + "host_resource", + "kubernetes", + "alert_handling", + ), + "tool": ("devops_tool", "AI自動化/Ansible受控修復"), + "alert": ("alert_handling",), + "playbook": ("auto_repair",), +} + def _enum_text(column): """Return a lowercase text view for enum/varchar columns across old KM schemas.""" @@ -85,6 +192,32 @@ def _knowledge_entry_columns(): ) +def _asset_taxonomy_condition(key: str): + """Build a broad-but-readable taxonomy matcher for AI automation surfaces.""" + category_expr = _normalized_category_expr() + category_text = func.lower(category_expr.cast(String)) + text_columns = ( + KnowledgeEntryRecord.title, + KnowledgeEntryRecord.content, + KnowledgeEntryRecord.tags.cast(String), + category_expr.cast(String), + ) + conditions = [] + category_hints = tuple( + hint.lower() for hint in _ASSET_TAXONOMY_CATEGORY_HINTS.get(key, ()) + ) + if category_hints: + conditions.append(category_text.in_(category_hints)) + for term in _ASSET_TAXONOMY_TERMS.get(key, ()): + pattern = f"%{term.lower()}%" + conditions.extend(func.lower(column).like(pattern) for column in text_columns) + if key == "playbook": + conditions.append(KnowledgeEntryRecord.related_playbook_id.is_not(None)) + if not conditions: + return category_text == key + return or_(*conditions) + + class KnowledgeDBRepository: """ Knowledge Repository - PostgreSQL 實作 @@ -290,6 +423,26 @@ class KnowledgeDBRepository: ) return [(_normalize_category(row.category), row.cnt) for row in result.all()] + async def get_asset_taxonomy_counts(self) -> list[tuple[str, int]]: + """取得 AI 自動化資產維度統計。 + + 這不是替代原始 category，而是讓 UI / Agent 可以用穩定 taxonomy + 將 KM 依專案、產品、網站、服務、套件、工具、Log、Alert、PlayBook、 + RAG、MCP 與排程分群。 + """ + rows: list[tuple[str, int]] = [] + for key in _ASSET_TAXONOMY_KEYS: + result = await self.db.execute( + select(func.count()) + .select_from(KnowledgeEntryRecord) + .where( + _active_status_filter(), + _asset_taxonomy_condition(key), + ) + ) + rows.append((key, int(result.scalar() or 0))) + return rows + async def search(self, query: str, limit: int = 20) -> list[KnowledgeEntry]: """關鍵字搜尋 (title + content + tags)""" like_q = f"%{query}%" diff --git a/apps/api/src/services/knowledge_service.py b/apps/api/src/services/knowledge_service.py index 8a2fdfdd5..530bee059 100644 --- a/apps/api/src/services/knowledge_service.py +++ b/apps/api/src/services/knowledge_service.py @@ -21,6 +21,7 @@ from src.models.knowledge import ( CategoryCount, EntryStatus, EntryType, + KnowledgeAssetTaxonomyCount, KnowledgeEntry, KnowledgeEntryCreate, KnowledgeEntryUpdate, @@ -49,6 +50,8 @@ _DEGRADED_CATEGORY_FALLBACKS = ( "general", ) +_ASSET_TAXONOMY_FALLBACKS = _DEGRADED_CATEGORY_FALLBACKS[:-1] + # ============================================================================= # Singleton # ============================================================================= @@ -65,6 +68,10 @@ def build_knowledge_list_readback_degraded_response(reason: str) -> KnowledgeLis CategoryCount(category=category, count=0) for category in _DEGRADED_CATEGORY_FALLBACKS ], + asset_taxonomy=[ + KnowledgeAssetTaxonomyCount(key=key, count=0) + for key in _ASSET_TAXONOMY_FALLBACKS + ], readback_status="degraded", operator_stage="knowledge_readback_degraded_ai_controlled_repair", next_step=( @@ -191,8 +198,16 @@ class KnowledgeService: categories = [ CategoryCount(category=cat, count=cnt) for cat, cnt in categories_raw ] + asset_taxonomy_raw = await repo.get_asset_taxonomy_counts() + asset_taxonomy = [ + KnowledgeAssetTaxonomyCount(key=key, count=count) + for key, count in asset_taxonomy_raw + ] return KnowledgeListResponse( - items=items, total=total, categories=categories + items=items, + total=total, + categories=categories, + asset_taxonomy=asset_taxonomy, ) except Exception as exc: # noqa: BLE001 - production readback must fail soft logger.warning( @@ -207,6 +222,23 @@ class KnowledgeService: ) return build_knowledge_list_readback_degraded_response(str(exc)) + async def get_asset_taxonomy(self) -> list[KnowledgeAssetTaxonomyCount]: + """取得 AI 自動化資產維度統計。""" + try: + async with get_db_context() as db: + repo: IKnowledgeRepository = KnowledgeDBRepository(db) + rows = await repo.get_asset_taxonomy_counts() + return [ + KnowledgeAssetTaxonomyCount(key=key, count=count) + for key, count in rows + ] + except Exception as exc: # noqa: BLE001 - taxonomy must not 500 the KM UI + logger.warning("knowledge_asset_taxonomy_readback_degraded", error=str(exc)) + return [ + KnowledgeAssetTaxonomyCount(key=key, count=0) + for key in _ASSET_TAXONOMY_FALLBACKS + ] + async def get_categories(self) -> list[CategoryCount]: """取得分類統計（直接呼叫 repo，不走 list_entries）""" try: diff --git a/apps/api/tests/test_knowledge_repository_read_model.py b/apps/api/tests/test_knowledge_repository_read_model.py index fad1cd15e..6ae75bde4 100644 --- a/apps/api/tests/test_knowledge_repository_read_model.py +++ b/apps/api/tests/test_knowledge_repository_read_model.py @@ -8,7 +8,11 @@ from sqlalchemy.dialects import postgresql from src.db.models import KnowledgeEntryRecord from src.models.knowledge import EntrySource, EntryStatus, EntryType -from src.repositories.knowledge_repository import KnowledgeDBRepository, _enum_text +from src.repositories.knowledge_repository import ( + KnowledgeDBRepository, + _ASSET_TAXONOMY_KEYS, + _enum_text, +) def _record(**overrides): @@ -177,6 +181,33 @@ async def test_get_categories_normalizes_null_or_blank_category() -> None: assert "coalesce(nullif(trim(knowledge_entries.category)" in compiled +@pytest.mark.asyncio +async def test_get_asset_taxonomy_counts_uses_stable_ai_automation_lenses() -> None: + class _TaxonomyDb: + def __init__(self): + self.statements = [] + self.values = list(range(10, 10 + len(_ASSET_TAXONOMY_KEYS))) + + async def execute(self, statement): + self.statements.append(statement) + return _ScalarResult(self.values.pop(0)) + + db = _TaxonomyDb() + repo = KnowledgeDBRepository(db=db) # type: ignore[arg-type] + + counts = await repo.get_asset_taxonomy_counts() + + assert [key for key, _ in counts] == list(_ASSET_TAXONOMY_KEYS) + assert [count for _, count in counts] == list(range(10, 10 + len(_ASSET_TAXONOMY_KEYS))) + compiled = "\n".join( + str(stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})) + for stmt in db.statements + ).lower() + assert "source_control" in compiled + assert "awoooi" in compiled + assert "related_playbook_id is not null" in compiled + + def test_enum_text_helper_casts_status_to_lowercase_text() -> None: compiled = str( (_enum_text(KnowledgeEntryRecord.status) == EntryStatus.ARCHIVED.value).compile( diff --git a/apps/api/tests/test_knowledge_service_readback_degraded.py b/apps/api/tests/test_knowledge_service_readback_degraded.py index a05aa3070..eba321a45 100644 --- a/apps/api/tests/test_knowledge_service_readback_degraded.py +++ b/apps/api/tests/test_knowledge_service_readback_degraded.py @@ -41,6 +41,21 @@ async def test_knowledge_list_entries_fails_soft_when_readback_breaks(monkeypatc "general", ] assert all(row.count == 0 for row in response.categories) + assert [row.key for row in response.asset_taxonomy] == [ + "project", + "product", + "website", + "service", + "package", + "tool", + "log", + "alert", + "playbook", + "rag", + "mcp", + "schedule", + ] + assert all(row.count == 0 for row in response.asset_taxonomy) assert response.readback_status == "degraded" assert response.operator_stage == "knowledge_readback_degraded_ai_controlled_repair" assert response.next_step == "queue_ai_controlled_km_readback_retry_tagging_and_connector_verifier" @@ -75,3 +90,31 @@ async def test_knowledge_categories_fails_soft_when_readback_breaks(monkeypatch) "general", ] assert all(row.count == 0 for row in categories) + + +@pytest.mark.asyncio +async def test_knowledge_asset_taxonomy_fails_soft_when_readback_breaks(monkeypatch) -> None: + monkeypatch.setattr( + knowledge_service_module, + "get_db_context", + lambda: _BrokenDbContext(), + ) + service = KnowledgeService.__new__(KnowledgeService) + + taxonomy = await service.get_asset_taxonomy() + + assert [row.key for row in taxonomy] == [ + "project", + "product", + "website", + "service", + "package", + "tool", + "log", + "alert", + "playbook", + "rag", + "mcp", + "schedule", + ] + assert all(row.count == 0 for row in taxonomy) diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index f43d90cab..5f10809a2 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -2187,7 +2187,7 @@ "clear": "清除", "matrixTitle": "全域分類 / 資產矩陣", "matrixSubtitle": "用目前 KM readback 與 RAG stats，把專案、產品、網站、服務、套件、工具、Log、Alert、PlayBook、RAG、MCP 與排程分群顯示。", - "autoUpdated": "API 驅動", + "autoUpdated": "Full-corpus readback", "global": "全域", "project": "專案", "product": "產品", diff --git a/apps/web/messages/zh-TW.json b/apps/web/messages/zh-TW.json index 7a6eff703..e1b2fa7c2 100644 --- a/apps/web/messages/zh-TW.json +++ b/apps/web/messages/zh-TW.json @@ -2187,7 +2187,7 @@ "clear": "清除", "matrixTitle": "全域分類 / 資產矩陣", "matrixSubtitle": "用目前 KM readback 與 RAG stats，把專案、產品、網站、服務、套件、工具、Log、Alert、PlayBook、RAG、MCP 與排程分群顯示。", - "autoUpdated": "API 驅動", + "autoUpdated": "全庫 readback", "global": "全域", "project": "專案", "product": "產品", diff --git a/apps/web/src/app/[locale]/knowledge-base/page.tsx b/apps/web/src/app/[locale]/knowledge-base/page.tsx index bb1e34ae9..e6dd82b38 100644 --- a/apps/web/src/app/[locale]/knowledge-base/page.tsx +++ b/apps/web/src/app/[locale]/knowledge-base/page.tsx @@ -53,10 +53,16 @@ interface CategoryCount { count: number } +interface AssetTaxonomyCount { + key: AssetLensKey + count: number +} + interface ListResponse { items: KnowledgeEntry[] total: number categories: CategoryCount[] + asset_taxonomy?: AssetTaxonomyCount[] readback_status?: 'ready' | 'degraded' | string operator_stage?: string | null next_step?: string | null @@ -264,6 +270,7 @@ const DEFAULT_CATEGORY_ORDER = [ const KNOWN_CATEGORY_KEYS = new Set([ 'AI自動化/Ansible受控修復', 'AI治理', + 'alert', 'alert_handling', 'application', 'ai_system', @@ -388,6 +395,7 @@ const CATEGORY_BAR_COLORS: Record = { } const CATEGORY_ORDER_RANK = new Map(DEFAULT_CATEGORY_ORDER.map((category, index) => [category, index])) +const ASSET_TAXONOMY_FALLBACK_COUNTS: AssetTaxonomyCount[] = ASSET_LENS_KEYS.map(key => ({ key, count: 0 })) const TAG_TONES: Record = { critical: 'border-status-critical/20 bg-status-critical/10 text-status-critical', @@ -485,6 +493,7 @@ export default function KnowledgeBasePage({ const [entries, setEntries] = useState([]) const [total, setTotal] = useState(0) const [categories, setCategories] = useState([]) + const [assetTaxonomy, setAssetTaxonomy] = useState(ASSET_TAXONOMY_FALLBACK_COUNTS) const [loading, setLoading] = useState(true) const [entryFetchError, setEntryFetchError] = useState(null) const [entryReadback, setEntryReadback] = useState | null>(null) @@ -529,6 +538,7 @@ export default function KnowledgeBasePage({ setEntries(data.items) setTotal(data.total) setCategories(data.categories) + setAssetTaxonomy(data.asset_taxonomy?.length ? data.asset_taxonomy : ASSET_TAXONOMY_FALLBACK_COUNTS) setEntryReadback({ readback_status: data.readback_status ?? 'ready', operator_stage: data.operator_stage ?? null, @@ -539,14 +549,16 @@ export default function KnowledgeBasePage({ const errorBody = await res.json().catch(() => null) as { detail?: string; message?: string } | null setEntries([]) setTotal(0) - setCategories([]) + setCategories(ASSET_TAXONOMY_FALLBACK_COUNTS.map(row => ({ category: row.key, count: 0 }))) + setAssetTaxonomy(ASSET_TAXONOMY_FALLBACK_COUNTS) setEntryReadback(null) setEntryFetchError(errorBody?.detail ?? errorBody?.message ?? `${res.status} ${res.statusText}`) } catch (err) { console.error('Failed to fetch knowledge entries', err) setEntries([]) setTotal(0) - setCategories([]) + setCategories(ASSET_TAXONOMY_FALLBACK_COUNTS.map(row => ({ category: row.key, count: 0 }))) + setAssetTaxonomy(ASSET_TAXONOMY_FALLBACK_COUNTS) setEntryReadback(null) setEntryFetchError(err instanceof Error ? err.message : String(err)) } finally { @@ -746,22 +758,27 @@ export default function KnowledgeBasePage({ }, [categories, totalCount]) const categoryNavigationRows = useMemo(() => { - if (categories.length === 0) return [] + const sourceRows = mergeCategoryCounts(categories) + const positiveRows = sourceRows.filter(row => row.count > 0) + const fallbackRows = ASSET_TAXONOMY_FALLBACK_COUNTS.map(row => ({ + category: row.key, + count: assetTaxonomy.find(item => item.key === row.key)?.count ?? row.count, + })) + const rows = positiveRows.length > 0 ? positiveRows : fallbackRows - const sourceRows = mergeCategoryCounts(categories).filter(row => row.count > 0) - - return [...sourceRows].sort((a, b) => { + return [...rows].sort((a, b) => { const aRank = CATEGORY_ORDER_RANK.get(a.category) ?? Number.MAX_SAFE_INTEGER const bRank = CATEGORY_ORDER_RANK.get(b.category) ?? Number.MAX_SAFE_INTEGER if (aRank !== bRank) return aRank - bRank return b.count - a.count }) - }, [categories]) + }, [assetTaxonomy, categories]) const assetLensRows = useMemo(() => { const ragChunks = governanceTelemetry.ragStats?.total_chunks ?? 0 + const taxonomyByKey = new Map(assetTaxonomy.map(row => [row.key, row.count])) const countWhere = (lens: AssetLensKey) => - baseDisplayedEntries.filter(entry => entryMatchesAssetLens(entry, lens)).length + taxonomyByKey.get(lens) ?? baseDisplayedEntries.filter(entry => entryMatchesAssetLens(entry, lens)).length const rows: Array<{ key: AssetLensKey icon: LucideIcon @@ -828,7 +845,7 @@ export default function KnowledgeBasePage({ icon: FileSearch, count: ragChunks || countWhere('rag'), tone: ragChunks > 0 ? 'text-status-healthy' : 'text-status-critical', - global: ragChunks > 0, + global: ragChunks > 0 || taxonomyByKey.has('rag'), }, { key: 'mcp', @@ -844,7 +861,7 @@ export default function KnowledgeBasePage({ }, ] return ASSET_LENS_KEYS.map(key => rows.find(row => row.key === key)).filter((row): row is NonNullable => Boolean(row)) - }, [baseDisplayedEntries, governanceTelemetry.ragStats]) + }, [assetTaxonomy, baseDisplayedEntries, governanceTelemetry.ragStats]) const qualityRows = useMemo(() => { const loaded = displayedEntries.length @@ -1254,7 +1271,7 @@ export default function KnowledgeBasePage({

- {(row.key === 'rag' ? governanceLoading : !knowledgeReadbackReady) ? '--' : formatCount(row.count)} + {(row.key === 'rag' ? governanceLoading && row.count === 0 : loading) ? '--' : formatCount(row.count)}

) @@ -1265,7 +1282,7 @@ export default function KnowledgeBasePage({
{t('rail.categoryTitle')} - {knowledgeReadbackReady ? categoryNavigationRows.length : '--'} + {loading ? '--' : categoryNavigationRows.length}
@@ -1578,7 +1595,7 @@ export default function KnowledgeBasePage({
{assetLensRows.map(row => { const Icon = row.icon - const value = (row.key === 'rag' ? governanceLoading : !knowledgeReadbackReady) + const value = (row.key === 'rag' ? governanceLoading && row.count === 0 : loading) ? '--' : formatCount(row.count) return (