diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 9b4819267..d94ea9111 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -72,9 +72,12 @@ "governance": "AI 治理", "awooop": "AwoooP", "awooopHome": "AwoooP 總覽", + "awooopWarRoom": "AI SRE 戰情室", "awooopWorkbench": "AwoooP 操作台", "codeReview": "程式碼審查", "knowledgeAutomation": "知識與自動化", + "observabilityCommandCenter": "全域可觀測", + "assetInventory": "資產與產品", "governanceSecurity": "治理與安全", "workItems": "工作鏈路", "runMonitor": "Run 監控", @@ -3063,6 +3066,7 @@ }, "governance": { "title": "AI 治理中樞", + "sectionVisible": "同頁可見", "complianceBadge": { "label": "AI 治理中樞", "loading": "載入中...", @@ -3076,7 +3080,7 @@ "agentMarket": "Agent 市場", "automationInventory": "自動化盤點" }, - "comingSoon": "本 Tab 即將上線", + "comingSoon": "本區塊即將上線", "slo": { "kpi": { "autonomy_rate": "自主化率", diff --git a/apps/web/messages/zh-TW.json b/apps/web/messages/zh-TW.json index 9b4819267..d94ea9111 100644 --- a/apps/web/messages/zh-TW.json +++ b/apps/web/messages/zh-TW.json @@ -72,9 +72,12 @@ "governance": "AI 治理", "awooop": "AwoooP", "awooopHome": "AwoooP 總覽", + "awooopWarRoom": "AI SRE 戰情室", "awooopWorkbench": "AwoooP 操作台", "codeReview": "程式碼審查", "knowledgeAutomation": "知識與自動化", + "observabilityCommandCenter": "全域可觀測", + "assetInventory": "資產與產品", "governanceSecurity": "治理與安全", "workItems": "工作鏈路", "runMonitor": "Run 監控", @@ -3063,6 +3066,7 @@ }, "governance": { "title": "AI 治理中樞", + "sectionVisible": "同頁可見", "complianceBadge": { "label": "AI 治理中樞", "loading": "載入中...", @@ -3076,7 +3080,7 @@ "agentMarket": "Agent 市場", "automationInventory": "自動化盤點" }, - "comingSoon": "本 Tab 即將上線", + "comingSoon": "本區塊即將上線", "slo": { "kpi": { "autonomy_rate": "自主化率", diff --git a/apps/web/src/app/[locale]/governance/page.tsx b/apps/web/src/app/[locale]/governance/page.tsx index 6706a8b24..4a07bdcd6 100644 --- a/apps/web/src/app/[locale]/governance/page.tsx +++ b/apps/web/src/app/[locale]/governance/page.tsx @@ -3,8 +3,8 @@ /** * GovernancePage — AI 治理中樞 (/governance) * ============================================ - * PR 2: 頁面骨架 + Sidebar 入口 + 三 Tab 空殼 - * PR 3-5 將填入 SLO / 治理事件 / AI 待辦 真實內容 + * D1K: 將原本隱藏在 tab query 裡的治理區塊改為同頁 sections。 + * 舊 ?tab= deep link 只作焦點排序,不再隱藏其他治理內容。 * * 設計方向: Nothing.tech × Anthropic Warmth * - ComplianceBadge 橫幅作為頁面頂部的治理狀態信號 @@ -49,6 +49,12 @@ export default function GovernancePage({ ] const requestedTab = Array.isArray(searchParams?.tab) ? searchParams?.tab[0] : searchParams?.tab const activeSection = governanceSections.find(section => section.id === requestedTab) ?? governanceSections[0] + const orderedSections = activeSection + ? [ + activeSection, + ...governanceSections.filter(section => section.id !== activeSection.id), + ] + : governanceSections return ( @@ -71,7 +77,7 @@ export default function GovernancePage({ return ( - {activeSection.content} + + {orderedSections.map((section) => { + const Icon = section.Icon + return ( + + + + + + + + {section.label} + {section.order} + + + + {t('sectionVisible')} + + + {section.content} + + ) + })} + ) } diff --git a/apps/web/src/components/layout/sidebar.tsx b/apps/web/src/components/layout/sidebar.tsx index 231068ea6..d5b7d8208 100644 --- a/apps/web/src/components/layout/sidebar.tsx +++ b/apps/web/src/components/layout/sidebar.tsx @@ -9,10 +9,10 @@ * - 無陰影 * - 單色圖示 * - * IA v2 (2026-06-04): - * - 頂層按照 operator 工作流排序,不按照技術模組排序。 - * - AwoooP / Runs / Work Items / Approvals / Alerts 直接成為全域入口。 - * - 頁內 tabs 只保留同一頁的視角切換,避免把任務入口藏在第二層。 + * IA D0 (2026-06-27): + * - 側欄只放高頻 primary 工作台。 + * - Runs / Approvals / Contracts 等流程細節保留深連結,不再攤平成主導航。 + * - 頁面內以同頁 sections 呈現關鍵狀態,減少二層分頁藏內容。 * * Phase 19: 使用 Z_INDEX.SIDEBAR (40) * @see lib/constants/z-index.ts @@ -125,7 +125,11 @@ export function Sidebar({ {/* 導航列表 - Operator workflow sections */} - {PRODUCT_NAV_SECTIONS.map(section => ( + {PRODUCT_NAV_SECTIONS.map(section => { + const visibleItems = section.items.filter(item => item.surface !== 'secondary') + if (visibleItems.length === 0) return null + + return ( {!collapsed && ( )} - {section.items.map(item => { + {visibleItems.map(item => { const active = isActive(item) const count = item.badge && mounted ? pendingCount : 0 return ( @@ -190,7 +194,8 @@ export function Sidebar({ ) })} - ))} + ) + })} {/* Separator */} diff --git a/apps/web/src/lib/navigation/product-ia.ts b/apps/web/src/lib/navigation/product-ia.ts index accbcc52c..b6065f4f2 100644 --- a/apps/web/src/lib/navigation/product-ia.ts +++ b/apps/web/src/lib/navigation/product-ia.ts @@ -29,6 +29,7 @@ export type ProductNavItem = { relatedPaths?: string[] exact?: boolean badge?: boolean + surface?: 'primary' | 'secondary' } export type ProductNavSection = { @@ -50,14 +51,20 @@ export const PRODUCT_NAV_SECTIONS: ProductNavSection[] = [ sectionKey: 'workspaces', items: [ { id: 'command-center', href: '/', labelKey: 'commandCenter', Icon: LayoutDashboard }, - { id: 'delivery', href: '/delivery', labelKey: 'delivery', Icon: Rocket }, { id: 'awooop-overview', href: '/awooop', - labelKey: 'awooopHome', + labelKey: 'awooopWarRoom', Icon: BrainCircuit, exact: true, }, + { + id: 'observability', + href: '/observability', + labelKey: 'observabilityCommandCenter', + Icon: Monitor, + aliases: ['/monitoring', '/apm', '/errors', '/apps', '/services', '/topology', '/alerts', '/alert-operation-logs', '/notifications'], + }, ], }, { @@ -69,11 +76,31 @@ export const PRODUCT_NAV_SECTIONS: ProductNavSection[] = [ labelKey: 'workItems', Icon: ClipboardList, }, + { + id: 'knowledge-automation', + href: '/knowledge-base', + labelKey: 'knowledgeAutomation', + Icon: BookOpen, + aliases: ['/knowledge', '/automation', '/auto-repair', '/drift', '/neural-command'], + }, + { + id: 'awooop-tenants', + href: '/awooop/tenants', + labelKey: 'assetInventory', + Icon: Building2, + }, + { + id: 'code-review', + href: '/code-review', + labelKey: 'codeReview', + Icon: GitPullRequest, + }, { id: 'awooop-runs', href: '/awooop/runs', labelKey: 'runMonitor', Icon: PlayCircle, + surface: 'secondary', }, { id: 'awooop-approvals', @@ -82,18 +109,14 @@ export const PRODUCT_NAV_SECTIONS: ProductNavSection[] = [ Icon: ShieldCheck, aliases: ['/authorizations'], badge: true, + surface: 'secondary', }, { id: 'awooop-contracts', href: '/awooop/contracts', labelKey: 'contracts', Icon: FileText, - }, - { - id: 'awooop-tenants', - href: '/awooop/tenants', - labelKey: 'tenants', - Icon: Building2, + surface: 'secondary', }, ], }, @@ -101,18 +124,10 @@ export const PRODUCT_NAV_SECTIONS: ProductNavSection[] = [ sectionKey: 'monitoring', items: [ { - id: 'observability', - href: '/observability', - labelKey: 'observability', - Icon: Monitor, - aliases: ['/monitoring', '/apm', '/errors', '/apps', '/services', '/topology'], - }, - { - id: 'alerts', - href: '/alerts', - labelKey: 'alerts', - Icon: Bell, - aliases: ['/alert-operation-logs', '/notifications'], + id: 'delivery', + href: '/delivery', + labelKey: 'delivery', + Icon: Rocket, }, { id: 'iwooos-security', @@ -120,47 +135,43 @@ export const PRODUCT_NAV_SECTIONS: ProductNavSection[] = [ labelKey: 'iwooos', Icon: ShieldCheck, aliases: ['/security-compliance'], - relatedPaths: ['/security', '/compliance'], - }, - { - id: 'code-review', - href: '/code-review', - labelKey: 'codeReview', - Icon: GitPullRequest, - }, - ], - }, - { - sectionKey: 'knowledge', - items: [ - { - id: 'knowledge-automation', - href: '/knowledge-base', - labelKey: 'knowledgeAutomation', - Icon: BookOpen, - aliases: ['/knowledge', '/automation', '/auto-repair', '/drift', '/neural-command'], - }, - { - id: 'reports', - href: '/reports', - labelKey: 'reports', - Icon: BarChart3, + relatedPaths: ['/security', '/compliance', '/governance'], }, { id: 'operations', href: '/operations', labelKey: 'operationsOverview', Icon: Package, - aliases: ['/deployments', '/tickets', '/cost', '/billing', '/action-logs'], + aliases: ['/deployments', '/tickets', '/cost', '/billing', '/action-logs', '/reports'], + }, + { + id: 'settings', + href: '/settings', + labelKey: 'settings', + Icon: Settings, }, ], }, { sectionKey: 'system', items: [ - { id: 'terminal', href: '/terminal', labelKey: 'terminal', Icon: Terminal }, - { id: 'settings', href: '/settings', labelKey: 'settings', Icon: Settings }, - { id: 'classic', href: '/classic', labelKey: 'classicAICenter', Icon: Radar }, + { + id: 'alerts', + href: '/alerts', + labelKey: 'alerts', + Icon: Bell, + aliases: ['/alert-operation-logs', '/notifications'], + surface: 'secondary', + }, + { + id: 'reports', + href: '/reports', + labelKey: 'reports', + Icon: BarChart3, + surface: 'secondary', + }, + { id: 'terminal', href: '/terminal', labelKey: 'terminal', Icon: Terminal, surface: 'secondary' }, + { id: 'classic', href: '/classic', labelKey: 'classicAICenter', Icon: Radar, surface: 'secondary' }, ], }, ] diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index bdb9e428c..a5de1087d 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -23,6 +23,32 @@ - controlled executor dispatch / live apply / Gateway queue write / Telegram send / Bot API / KM write / PlayBook trust write / production write / secret read / paid API / host write / kubectl / destructive operation:全部 `0`。 - 本段沒有正式下發 Ansible apply、沒有 Telegram live send、沒有主機寫入、沒有 secret value collection、沒有 destructive operation。 +## 2026-06-27|D1K 全站 IA / UIUX D0:主導航收斂與 Governance 去分頁化 + +**背景**:使用者明確指出網站整體流程、導航列、頁內分頁與大量文字不符合專業 AI 自動化產品期待;現況 sidebar 將 AwoooP overview / Work Items / Runs / Approvals / Contracts / Tenants、Observability、Alerts、IwoooS、Code Review、Knowledge、Reports、Operations、Terminal、Settings、Classic 等 19 個入口全部攤平,Governance 頁又以 `?tab=` 隱藏 5 個治理區塊,造成資訊架構混亂。 + +**本段完成內容**: +- `PRODUCT_NAV_SECTIONS` 新增 `surface=secondary`,sidebar 只顯示 primary 工作台;secondary 頁面仍保留 deep link 與 command palette,不再轟炸主導航。 +- Sidebar primary 收斂為 3 組、11 個主入口: + - 主工作區:指令中心、AI SRE 戰情室、全域可觀測。 + - 處理佇列:工作鏈路、知識與自動化、資產與產品、程式碼審查。 + - 監控與安全:交付閉環、IwoooS、營運總覽、設定。 +- Runs / Approvals / Contracts / Alerts / Reports / Terminal / Classic 改為 secondary;保留 URL、既有深連結、AwoooP 內部流程與 command palette 搜尋能力。 +- `/zh-TW/governance` 從單一 active tab 改為同頁可見 sections:SLO、治理事件、AI 待辦、Agent 市場、自動化盤點全部在同一頁呈現;舊 `?tab=` deep link 只用於排序焦點,不再隱藏其他區塊。 +- 新增繁中/i18n 導航詞:`AI SRE 戰情室`、`全域可觀測`、`資產與產品`、`同頁可見`。 + +**驗證**: +- `python3 -m json.tool apps/web/messages/zh-TW.json` / `apps/web/messages/en.json`:通過。 +- i18n key mirror:`14353 / 14353`,diff `0`。 +- `git diff --check`:通過。 +- `pnpm --filter @awoooi/web typecheck`:通過。 + +**完成度 / 邊界**: +- IA D0 主導航收斂:本地 `100%`。 +- Governance 去分頁化 D0:本地 `100%`。 +- 全站完整 UI / 圖表 / 拓樸 / 動畫 / 主流 AI 控制台化:仍在推進中,D0 只先修掉主導航過載與 Governance 分頁隱藏內容。 +- 本段不改 runtime gate、不改告警執行策略、不觸發主機動作。 + ## 2026-06-27|D1J 修復候選升級合約:受控自動化閉環可視化 **背景**:Telegram / AwoooP 告警卡已能指出 `repair_candidate_draft_ready_owner_review`,但 Work Items / Approvals 仍主要呈現「需人工」與長文字,操作員看不到 AI 已準備好的無寫入演練、Verifier 與 KM / PlayBook / Script 資產沉澱槽位,容易被誤判為 AI Agent 沒有任何自動化進展。 @@ -51,6 +77,25 @@ - 真實 executor / Ansible apply / SSH / Telegram 實發 / KM 寫入 / PlayBook trust 寫入:仍 `0 / false`,本段沒有開 runtime gate。 - 這段把「下一步要自動化什麼」從長文字變成可讀回的 contract 與 UI 卡片;下一段必須接 owner release / no-write rehearsal persistence / verifier result capture,不可再只新增靜態治理卡。 +## 2026-06-27|11:48 reboot SOP 現場恢復:MOMO 來源檔缺席與 110 CI/UX smoke 負載已分層 + +**時間與來源**: +- 2026-06-27 11:25-11:48 Asia/Taipei。 +- 來源:`scripts/reboot-recovery/post-reboot-readiness-summary.sh --no-color`、`scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh --host ollama@192.168.0.188 --freshness-max-days 2`、188 `ollama` crontab / textfile exporter / MOMO container readback、110 `backup-status`、120 `sudo -n k3s kubectl`、production route smoke、110 process / Docker stats readback。 + +**實際修復與判讀**: +- 188 `momo_pg_daily` crontab 在 11:25 再次漂移回舊 app-side path,導致 `configured_missing_188=1`。已在 188 做最小可逆 host 修復:備份 crontab 到 `/home/ollama/momo_backups/crontab-before-momo-pg-host-owned-20260627-112624.txt`,改回 `/home/ollama/bin/momo-pg-backup.sh`,刷新 exporter 後 `awoooi_backup_job_configured{host="188",job="momo_pg_daily"} 1`、`backup-status` 回 `core_blockers=0`。 +- 漂移來源已定位到受控開發工作區 `/Users/ogt/codex-workspaces/awoooi-dev` 的 Ansible playbook 仍保留舊 path;已同步修正 `188-momo-backup-user.yml` 與 `188-ai-web-readonly.yml`,並推到 `gitea-ssh` feature branch commit `27d16a81 fix(ops): align 188 momo backup path in dev workspace [skip ci]`,避免從 dev workspace 跑 Ansible 時再把 188 crontab 打回舊路徑。 +- MOMO 服務本身健康:`mo.wooo.work/health=200`、version `V10.719`、`momo-pro-system` / `momo-scheduler` / `momo-telegram-bot` healthy,scheduler `RestartCount=0`,Google token metadata `100000:100000:600` 符合 scheduler UID。 +- MOMO 資料 freshness blocked 的根因不是匯入器壞掉,而是 Drive 來源缺席。受控補跑 `import_service.auto_import_from_drive()` 回 `{"success": true, "message": "沒有找到待匯入的檔案", "file_count": 0}`;read-only Drive metadata 顯示 `DRIVE_INTAKE_COUNT 0`、archive 最新 `DRIVE_ARCHIVE_LATEST_MODIFIED 2026-06-25T04:21:47.000Z`、global latest 同為 `2026-06-25T04:21:47.000Z`。最新 import job `57` 已完成且 `sync_success=true`,資料範圍 `2026-06-01` 到 `2026-06-24`,DB parity `15383|15383|2026-06-01|2026-06-24|2026-06-01|2026-06-24`。 +- 110 CPU 高負載已分層:Gitea Actions / AWOOOI CD build-test、StockPlatform API/Postgres 查詢與外部 `stockplatform-product-ux-smoke` headless Chrome 疊加。已精準 SIGTERM 一組逾時 `stockplatform-product-ux-smoke` process group `104429`,`REMAINING_AFTER_TERM=0`;未重啟 Docker / systemd / Nginx / firewall / K3s / DB。 +- 120 / 121 K3s readback:兩節點 `Ready`,ArgoCD `awoooi-prod` 為 `Synced / Healthy`,revision `fccd8874fce5c5b6dc29d806c57577bdeef73403`,AWOOOI api/web/worker pods 均 `1/1 Running`。 + +**最新總結**: +- 11:51 summary artifact `/tmp/awoooi-post-reboot-readiness-20260627-115046/summary.txt`:`POST_START_RESULT=BLOCKED`、`SERVICE_GREEN=0`、`BACKUP_CORE_GREEN=1`、`HOST_188_HYGIENE_BLOCKED=0`、`STOCK_FRESHNESS_STATUS=ok`、`STOCK_LATEST_TRADING_DATE=2026-06-26`、`ESCROW_MISSING_COUNT=5`、`WAZUH_MANAGER_REGISTRY_ACCEPTED=0`。 +- 真正 runtime blocker:MOMO business data freshness blocked,因 Drive 沒有比 `2026-06-25T04:21:47Z` 更新的 `即時業績_當日` 來源檔;不可手寫 DB 或用舊 archive 重匯入假裝更新。 +- 仍不得宣稱:full-stack green、DR complete、Wazuh 全主機 registry accepted、runtime action authorized。 + ## 2026-06-27|00:58 reboot SOP 實際修復:188 MOMO backup core 假紅收斂 **時間與來源**: diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 46e91dcd9..4c2122fb7 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -14,7 +14,7 @@ v1.76 owner gate replay rule:同一輪 summary 產生後,owner packet 與 owner response preflight 必須優先使用 `--summary-file "$ARTIFACT_DIR/summary.txt"`,例如 `scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py --no-color --summary-file "$ARTIFACT_DIR/summary.txt" --output /tmp/awoooi-post-reboot-owner-packets.json` 與 `scripts/reboot-recovery/post-reboot-owner-response-preflight.py --no-color --summary-file "$ARTIFACT_DIR/summary.txt" --response-file `。只有在刻意要重新取 live evidence 時,才允許省略 `--summary-file`;否則 preflight 不得自己重跑 summary 造成同一輪狀態漂移。 -2026-06-27 02:42 最新 live revalidation:`scripts/reboot-recovery/post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260627-024151/summary.txt` 回傳 `POST_START_RESULT=FULL_STACK_GREEN_DR_ESCROW_BLOCKED`、`POST_START_PASS=38`、`POST_START_WARN=3`、`POST_START_BLOCKED=0`、`SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、`STOCK_FRESHNESS_STATUS=ok`、`STOCK_LATEST_TRADING_DATE=2026-06-26`、`STOCK_BLOCKERS=none`、`BACKUP_CORE_GREEN=1`、`HOST_188_HYGIENE_BLOCKED=0`、`WAZUH_MANAGER_REGISTRY_ACCEPTED=0`、`RUNTIME_ACTION_AUTHORIZED=0`。同輪 DR checklist 回 `CORE_COLD_START_GREEN=1`、`RECOVERY_STATE=CORE_READY_DR_OFFSITE_PENDING`、Prometheus contract `awoooi_recovery_core_ready=1`、`awoooi_recovery_dr_offsite_ready=0`。因此目前服務 / 資料 / 備份核心可宣稱恢復;DR complete 仍因 `ESCROW_MISSING_COUNT=5` 禁止宣稱,Wazuh 全主機納管仍因 manager registry accepted `0` 禁止宣稱。 +2026-06-27 11:51 最新 live revalidation:`scripts/reboot-recovery/post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260627-115046/summary.txt` 回傳 `POST_START_RESULT=BLOCKED`、`POST_START_PASS=37`、`POST_START_WARN=3`、`POST_START_BLOCKED=2`、`SERVICE_GREEN=0`、`PRODUCT_DATA_GREEN=1`、`STOCK_FRESHNESS_STATUS=ok`、`STOCK_LATEST_TRADING_DATE=2026-06-26`、`STOCK_BLOCKERS=none`、`BACKUP_CORE_GREEN=1`、`HOST_188_HYGIENE_BLOCKED=0`、`WAZUH_MANAGER_REGISTRY_ACCEPTED=0`、`RUNTIME_ACTION_AUTHORIZED=0`。本輪已再次修復 188 `momo_pg_daily` crontab configured drift,`backup-status` 回 `core_blockers=0`、`configured_missing_188=0`;K3s / ArgoCD live readback 顯示 120 / 121 皆 `Ready`,`awoooi-prod` 為 `Synced / Healthy`,api/web/worker pods 均 Running。現在 hard blocker 是 MOMO business data freshness:`daily_sales_snapshot` 最新仍為 `2026-06-24`,`DRIVE_INTAKE_COUNT=0`,Drive archive / global latest `即時業績_當日` 均為 `2026-06-25T04:21:47Z`,最新 import job `57` 已 clean completed 且 `sync_success=true`。因此可宣稱主機、K3s、public routes、backup core 與 Stock freshness 已恢復;不可宣稱 full-stack green,直到 MOMO 來源檔補齊並由正式 import pipeline 更新 DB。DR complete 仍因 `ESCROW_MISSING_COUNT=5` 禁止宣稱,Wazuh 全主機納管仍因 manager registry accepted `0` 禁止宣稱。 2026-06-27 00:58 最新 live summary:本輪先修復兩個實際 SOP blocker。第一,`scripts/ops/recovery-scorecard-contract-check.py` 已改成 PyYAML optional,標準 Python 環境也能驗證 recovery recording-rule contract,不會因 `ModuleNotFoundError: yaml` 中斷 DR/offsite checklist。第二,188 `ollama` crontab 已備份到 `/home/ollama/momo_backups/crontab-before-momo-pg-host-owned-20260627-001925.txt`,並把 `AWOOOI momo PostgreSQL daily backup` 從 app-side `/home/ollama/momo-pro/scripts/pg_backup.sh` 收斂回 host-owned `/home/ollama/bin/momo-pg-backup.sh`;刷新 188 textfile exporter 後 `awoooi_backup_job_configured{host="188",job="momo_pg_daily"} 1`。00:58 `scripts/reboot-recovery/post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260627-005728/summary.txt` 回傳 `POST_START_RESULT=FULL_STACK_GREEN_DR_ESCROW_BLOCKED`、`POST_START_PASS=38`、`POST_START_WARN=3`、`POST_START_BLOCKED=0`、`SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、`BACKUP_CORE_GREEN=1`、`ESCROW_MISSING_COUNT=5`、`HOST_188_HYGIENE_BLOCKED=0`、`WAZUH_MANAGER_REGISTRY_ACCEPTED=0`、`RUNTIME_ACTION_AUTHORIZED=0`。同輪 `backup-status` 回 `core_blockers=0`、`configured_missing_188=0`;Prometheus live contract 回 `awoooi_recovery_core_ready=1`、`awoooi_recovery_dr_offsite_ready=0`,表示主機 / K3s / public routes / product data / backup core 已恢復,DR 仍只因 credential escrow 缺 5 個 non-secret evidence marker blocked,Wazuh 全主機 registry accepted 仍為 0。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 22e830f17..b50640065 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -11,7 +11,7 @@ | Area | Status | Completion | Evidence | |------|--------|------------|----------| -| Overall recovery readiness | FULL_STACK_GREEN_DR_ESCROW_BLOCKED | 99% | 2026-06-27 02:42 live revalidation 覆蓋 00:16 暫時 blocked 判讀。`post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260627-024151/summary.txt` 回傳 `SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、`POST_START_RESULT=FULL_STACK_GREEN_DR_ESCROW_BLOCKED`、`POST_START_WARN=3`、`POST_START_BLOCKED=0`、`STOCK_FRESHNESS_STATUS=ok`、`STOCK_LATEST_TRADING_DATE=2026-06-26`、`STOCK_BLOCKERS=none`、`BACKUP_CORE_GREEN=1`、`ESCROW_MISSING_COUNT=5`、`WAZUH_MANAGER_REGISTRY_ACCEPTED=0`。00:16 的 blocker 是 188 `momo_pg_daily` configured drift:備份 fresh,但 exporter 因 crontab 仍指 app-side path 判 `configured_missing_188=1`;00:19 已備份 188 crontab 到 `/home/ollama/momo_backups/crontab-before-momo-pg-host-owned-20260627-001925.txt` 並收斂到 host-owned `/home/ollama/bin/momo-pg-backup.sh`,刷新 exporter 後 `awoooi_backup_job_configured{host="188",job="momo_pg_daily"} 1`,00:56 `backup-status` 回 `core_blockers=0`。02:41 DR checklist 回 `CORE_COLD_START_GREEN=1`、`RECOVERY_STATE=CORE_READY_DR_OFFSITE_PENDING`;Prometheus live contract 回 `awoooi_recovery_core_ready=1`、`awoooi_recovery_dr_offsite_ready=0`。主機 / K3s / public routes / AWOOOI / MOMO / Stock / backup core / 188 hygiene 已恢復。DR 仍因 credential escrow 缺 5 不能宣稱 complete;Wazuh registry 已有脫敏 manager readback,但尚未 Dashboard API / owner acceptance。 | +| Overall recovery readiness | SERVICE_BLOCKED_MOMO_SOURCE_ABSENCE | 96% | 2026-06-27 11:51 live revalidation 覆蓋 02:42 舊綠燈判讀。`post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260627-115046/summary.txt` 回傳 `SERVICE_GREEN=0`、`POST_START_RESULT=BLOCKED`、`POST_START_BLOCKED=2`、`BACKUP_CORE_GREEN=1`、`HOST_188_HYGIENE_BLOCKED=0`、`STOCK_FRESHNESS_STATUS=ok`、`STOCK_LATEST_TRADING_DATE=2026-06-26`、`ESCROW_MISSING_COUNT=5`、`WAZUH_MANAGER_REGISTRY_ACCEPTED=0`。本輪 188 `momo_pg_daily` configured drift 已再次修復;`backup-status` 回 `core_blockers=0`、`configured_missing_188=0`。K3s / ArgoCD readback:120 / 121 皆 `Ready`,`awoooi-prod Synced / Healthy`,api/web/worker pods Running。MOMO 服務健康但資料 freshness blocked:正式補跑 import 回 `file_count=0`,Drive metadata 回 `DRIVE_INTAKE_COUNT=0`、archive / global latest `2026-06-25T04:21:47Z`,latest import job `57` clean completed 且資料範圍只到 `2026-06-24`。不可手寫 DB 或用舊 archive 假裝更新;需等新來源檔進正式 intake 後再匯入。DR 仍因 credential escrow 缺 5 不能宣稱 complete;Wazuh manager registry accepted 仍為 0。 | | P0 host / K3s recovery | DONE | 100% | 120 booted after console fsck at `2026-06-12 15:13`; latest 2026-06-26 07:19 readback shows 120 and 121 reachable, K3s active, `mon` and `mon1` both `Ready control-plane`, AWOOOI API/Web replicas split across both nodes, ArgoCD `awoooi-prod Synced / Healthy` at revision `1fd5e2a8b0f18d24eed16aa2a44286bcbf230603`, and `km-vectorize` official 03:00 台北時間 run succeeded with `lastSuccess=2026-06-25T19:00:14Z`. | | P1 backup / alert / escrow | BLOCKED_DR_ESCROW | 98% | 2026-06-27 00:56 backup readback shows 110 `13/13 fresh failed=0`, 188 `2/2 fresh failed=0`, `core_blockers=0`, `integrity_stale=0`, `offsite_fresh=1`, `rclone_gdrive_fresh=1`, `configured_missing_188=0`, `escrow_missing=5`, last aggregate `2026-06-26 02:31:02`。188 MOMO backup crontab drift 已修復並保留 rollback crontab。DR remains blocked on real non-secret credential escrow evidence IDs; do not write placeholder markers or paste secret values. | | P2 service / data truth | DONE | 100% | Public routes 與 service health 為綠燈,MOMO health `V10.719`,current-month parity 為 `15383|15383|2026-06-01|2026-06-24|2026-06-01|2026-06-24`。StockPlatform `/api/v1/system/freshness` 為 `ok`,latest trading date `2026-06-26`,blockers `none`;先前 Stock EOD blocker 已由官方來源與正式 cron 自然收斂。 | diff --git a/scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh b/scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh index 178472843..55086914c 100755 --- a/scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh +++ b/scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh @@ -123,6 +123,78 @@ latest_source="$(find /home/ollama/momo-pro /backup -type f -name '即時業績_ emit LOCAL_EXACT_DAILY_SOURCE_COUNT "${source_count:-0}" emit LOCAL_EXACT_DAILY_SOURCE_LATEST "${latest_source:-none}" +docker exec -i momo-scheduler python - <<'PYDRIVE' 2>/dev/null || true +from services.google_drive_service import drive_service +from services.import_service import import_service + + +def emit(key, value): + print(f"{key} {value if value not in (None, '') else 'none'}") + + +try: + folder = import_service.get_config("gdrive_folder_path", "業績報表/當日業績") + pattern = import_service.get_config("gdrive_file_pattern", "即時業績_當日") + archive = import_service.get_config("gdrive_archive_folder", "已匯入") + failed = import_service.get_config("gdrive_failed_folder", "匯入失敗") +except Exception: + folder = "業績報表/當日業績" + pattern = "即時業績_當日" + archive = "已匯入" + failed = "匯入失敗" + +emit("DRIVE_INTAKE_FOLDER", folder) +emit("DRIVE_FILE_PATTERN", pattern) + +intake_files = drive_service.list_files_in_folder(folder, pattern) +emit("DRIVE_INTAKE_COUNT", len(intake_files)) +if intake_files: + latest = intake_files[0] + emit("DRIVE_INTAKE_LATEST_MODIFIED", latest.get("modifiedTime")) + emit("DRIVE_INTAKE_LATEST_SIZE", latest.get("size")) + +archive_files = drive_service.list_files_in_folder(archive, pattern) +emit("DRIVE_ARCHIVE_FOLDER", archive) +emit("DRIVE_ARCHIVE_COUNT", len(archive_files)) +if archive_files: + latest = archive_files[0] + emit("DRIVE_ARCHIVE_LATEST_MODIFIED", latest.get("modifiedTime")) + emit("DRIVE_ARCHIVE_LATEST_SIZE", latest.get("size")) + +failed_files = drive_service.list_files_in_folder(failed, pattern) +emit("DRIVE_FAILED_FOLDER", failed) +emit("DRIVE_FAILED_COUNT", len(failed_files)) +if failed_files: + latest = failed_files[0] + emit("DRIVE_FAILED_LATEST_MODIFIED", latest.get("modifiedTime")) + emit("DRIVE_FAILED_LATEST_SIZE", latest.get("size")) + +if not drive_service.service: + drive_service.authenticate() +if drive_service.service: + safe_pattern = drive_service._escape_query_value(pattern) + query = ( + f"name contains '{safe_pattern}' and trashed=false and " + "(mimeType='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' " + "or mimeType='application/vnd.ms-excel')" + ) + results = drive_service.service.files().list( + q=query, + spaces="drive", + fields="files(name,modifiedTime,size,mimeType)", + orderBy="modifiedTime desc", + pageSize=1, + ).execute() + global_files = results.get("files", []) + emit("DRIVE_GLOBAL_MATCH_COUNT_AT_LEAST", len(global_files)) + if global_files: + latest = global_files[0] + emit("DRIVE_GLOBAL_LATEST_MODIFIED", latest.get("modifiedTime")) + emit("DRIVE_GLOBAL_LATEST_SIZE", latest.get("size")) +else: + emit("DRIVE_GLOBAL_MATCH_COUNT_AT_LEAST", "unknown") +PYDRIVE + psql_query() { docker exec momo-db psql -h 127.0.0.1 -U momo -d momo_analytics -Atc "$1" 2>/dev/null || true } @@ -229,6 +301,27 @@ else ok "no exact local daily-sales source candidate found on 188 / backup paths" fi +drive_intake_count="$(num_for DRIVE_INTAKE_COUNT)" +drive_archive_count="$(num_for DRIVE_ARCHIVE_COUNT)" +drive_failed_count="$(num_for DRIVE_FAILED_COUNT)" +drive_archive_latest="$(value_for DRIVE_ARCHIVE_LATEST_MODIFIED)" +drive_global_latest="$(value_for DRIVE_GLOBAL_LATEST_MODIFIED)" +if [[ "$drive_intake_count" -gt 0 ]]; then + ok "Drive daily-sales intake has pending source files: count=$drive_intake_count" +else + warn "Drive daily-sales intake is empty; latest archive=${drive_archive_latest:-none} global_latest=${drive_global_latest:-none}" +fi +if [[ "$drive_archive_count" -gt 0 ]]; then + ok "Drive daily-sales archive metadata is visible: count=$drive_archive_count latest=${drive_archive_latest:-unknown}" +else + warn "Drive daily-sales archive metadata has no matching files" +fi +if [[ "$drive_failed_count" -gt 0 ]]; then + warn "Drive daily-sales failed-folder candidates exist: count=$drive_failed_count latest=$(value_for DRIVE_FAILED_LATEST_MODIFIED)" +else + ok "Drive daily-sales failed-folder has no matching candidates" +fi + import_config="$(value_for IMPORT_CONFIG)" [[ "$import_config" == *"當日業績匯入|即時業績_當日"* ]] && ok "Drive import config points to expected daily-sales intake" || blocked "Drive import config is unavailable or drifted: ${import_config:-missing}"
{section.label}
{section.order}