diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index b49112181..c8fac0d87 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -1556,15 +1556,39 @@ jobs: env: DOCKER_BUILDKIT: "1" run: | - docker build -f apps/api/Dockerfile \ + run_docker_step() { + local label="$1" + local timeout_seconds="$2" + shift 2 + echo "cd_docker_step_start=${label} timeout_seconds=${timeout_seconds}" + set +e + if command -v timeout >/dev/null 2>&1; then + timeout -k 30s "${timeout_seconds}s" "$@" + else + "$@" + fi + local rc=$? + set -e + echo "cd_docker_step_rc=${label}:${rc}" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + echo "BLOCKER cd_docker_step_timeout label=${label} timeout_seconds=${timeout_seconds}" + echo "NEXT_ACTION inspect_cd_runner_docker_build_push_or_registry_route_then_rerun_cd" + fi + return "$rc" + } + + run_docker_step api_build "${DOCKER_API_BUILD_TIMEOUT_SECONDS:-1200}" \ + docker build -f apps/api/Dockerfile \ --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from ${{ env.HARBOR }}/awoooi/api:latest \ --build-arg CACHE_BUST=${{ github.sha }} \ -t ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} \ -t ${{ env.HARBOR }}/awoooi/api:latest \ . - docker push ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} - docker push ${{ env.HARBOR }}/awoooi/api:latest + run_docker_step api_push_sha "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \ + docker push ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} + run_docker_step api_push_latest "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \ + docker push ${{ env.HARBOR }}/awoooi/api:latest # 2026-03-31 ogt: 移除中間通知，減少訊息雜訊 @@ -1578,7 +1602,29 @@ jobs: env: DOCKER_BUILDKIT: "1" run: | - docker build -f apps/web/Dockerfile \ + run_docker_step() { + local label="$1" + local timeout_seconds="$2" + shift 2 + echo "cd_docker_step_start=${label} timeout_seconds=${timeout_seconds}" + set +e + if command -v timeout >/dev/null 2>&1; then + timeout -k 30s "${timeout_seconds}s" "$@" + else + "$@" + fi + local rc=$? + set -e + echo "cd_docker_step_rc=${label}:${rc}" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + echo "BLOCKER cd_docker_step_timeout label=${label} timeout_seconds=${timeout_seconds}" + echo "NEXT_ACTION inspect_cd_runner_docker_build_push_or_registry_route_then_rerun_cd" + fi + return "$rc" + } + + run_docker_step web_build "${DOCKER_WEB_BUILD_TIMEOUT_SECONDS:-1500}" \ + docker build -f apps/web/Dockerfile \ --build-arg NEXT_PUBLIC_API_URL=https://awoooi.wooo.work \ --build-arg CACHE_BUST=${{ github.sha }} \ --build-arg BUILDKIT_INLINE_CACHE=1 \ @@ -1586,8 +1632,10 @@ jobs: -t ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} \ -t ${{ env.HARBOR }}/awoooi/web:latest \ . - docker push ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} - docker push ${{ env.HARBOR }}/awoooi/web:latest + run_docker_step web_push_sha "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \ + docker push ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} + run_docker_step web_push_latest "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \ + docker push ${{ env.HARBOR }}/awoooi/web:latest - name: Release Docker Build Lock if: always() diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 80ba36e05..242fb4dd5 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,18 @@ +## 2026-07-03 — 10:30 CD Docker build/push bounded timeout 與 queue classifier + +**完成內容**： +- Public Gitea queue 讀回最新 CD `#4556` 仍為 `Running`，尚無 deploy marker / production deploy readback；production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 仍是舊版行為：`active_blocker_count=8`、`runtime_metric_runtime_readback_added_blockers=[]`、`windows99_update_no_auto_reboot_ready=true`，不可宣稱最新 source 已上 production。 +- 讀公開 CD log 確認 build job 曾等待 `awoooi-cd-docker-build-lock`，302 秒後自清空鎖並取得 lock；API / Web image build/push 指令已進入，但 log 尚未出現 deploy marker、production readback matched 或 success notification。 +- 修正 `.gitea/workflows/cd.yaml`：`Build and Push API` / `Build and Push Web` 的 `docker build` 與 `docker push` 全部改由 `run_docker_step` 包住，預設 API build `1200s`、Web build `1500s`、push `600s`，timeout 時 fail-closed 輸出 `BLOCKER cd_docker_step_timeout label=... timeout_seconds=...` 與固定 `NEXT_ACTION inspect_cd_runner_docker_build_push_or_registry_route_then_rerun_cd`。 +- 修正 `ops/runner/read-public-gitea-actions-queue.py`：公開 queue readback 會解析 `cd_docker_step_timeout`，輸出 `latest_visible_cd_docker_step_timeout*`、`blocked_cd_docker_step_timeout`、rollup 與 safe-next-action，避免 CD 長時間 Running 時只能人工翻 log 猜卡點。 + +**已跑驗證**： +- `python3.11 -m py_compile ops/runner/read-public-gitea-actions-queue.py ops/runner/test_read_public_gitea_actions_queue.py ops/runner/test_cd_controlled_runtime_profile.py`：通過。 +- `python3.11 -m pytest ops/runner/test_cd_controlled_runtime_profile.py ops/runner/test_read_public_gitea_actions_queue.py -q -p no:cacheprovider`：`92 passed`。 + +**仍維持**： +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未啟動或關閉 VM；未重啟 host / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 10:58 Windows99 collected verifier 下一步上卷到 production API **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 0b85e01fb..9b4714d9c 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.110 +> Version: v1.111 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -52,6 +52,8 @@ v1.109 Windows99 no-secret collector candidate coverage rule：`collect-windows9 v1.110 Windows99 nested runtime blocker promotion rule：production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 不得只信 Prometheus `active_blocker` label 而忽略 Windows99 verifier 巢狀 evidence。若 `windows99_vmware_autostart.service_blockers`、`autostart_task.ok=false` 或 `policy_blockers` 存在，API overlay 必須把 `windows99_vmware_autostart_config_not_ready` 與 `windows99_update_no_auto_reboot_policy_not_ready` 重新提升到 `active_blockers`、`active_blocker_action_matrix`、readback 與 rollups，並列入 `runtime_metric_runtime_readback_added_blockers`。這些 blocker 只授權 no-secret verifier、source config / check-mode package 與 post-verifier；不得從 scorecard 直接重啟 Windows service、改 registry、套 Windows Update policy、啟動 / 關閉 VM 或重啟 host。 +v1.111 Gitea CD Docker step bounded-timeout rule：public queue 顯示 `cd.yaml` Running 不得視為 production 已更新或中性等待；必須同時讀 `latest_visible_cd_deploy_marker_*`、`latest_visible_cd_production_deploy_readback_matched_from_log`、`latest_visible_cd_failure_classifier`、`latest_visible_cd_inflight_classifier` 與 `latest_visible_cd_docker_step_timeout*`。`Build and Push API` / `Build and Push Web` 的 `docker build` 與 `docker push` 必須由 `run_docker_step` 包住，預設 API build `1200s`、Web build `1500s`、push `600s`，timeout 時 fail-closed 輸出 `BLOCKER cd_docker_step_timeout label=... timeout_seconds=...`。`read-public-gitea-actions-queue.py --json` 必須把此 blocker 上卷為 `blocked_cd_docker_step_timeout` 與固定 safe-next-action；下一步只能檢查 runner Docker build/push path、registry route 與 queue readback，不得用 workflow_dispatch、重啟 Docker / host / Nginx / K3s / DB、讀 secrets 或恢復 generic runner 掩蓋卡點。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 38bce3f12..831ed9432 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -13,7 +13,7 @@ 本段覆蓋舊的「單次重啟後人工排查」做法。所有後續狀態回報必須依此順序推進；噪音若會遮蔽 P0，就掛回同一列，不另開支線。 -### 2026-07-03 10:15 最新 P0 覆蓋排序 +### 2026-07-03 10:30 最新 P0 覆蓋排序 下表覆蓋 2026-06-30 初始事故列；舊表保留為歷史追蹤。所有新插入需求必須掛在本表，不得再分散成臨時支線。 @@ -21,11 +21,11 @@ |------|------|--------|----------|-------------------| | P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 08:59 production scorecard：`status=blocked_reboot_auto_recovery_slo_not_ready`、`active_blocker_count=8`、`readiness_percent=67`、`primary_blocker=reboot_event_required_host_unreachable`、`can_claim_all_services_recovered_within_target=false`。no-write artifact `/tmp/awoooi-reboot-continue-20260703-085205`：111 socket probe 仍 timeout / unreachable；99 RDP / VMConnect reachable 但 uptime unknown；188 socket probe reachable，前次 host probe 仍為 `systemd_state=degraded` / `startup_active=failed`。Production readback 同步顯示 `windows99_vmware_readback_present=true`、`windows99_vmware_config_ready=false`、`windows99_vmware_power_ready=false`、`windows99_missing_vmx_aliases=["111"]`、`windows99_powered_off_aliases=["111","112","120","121","188"]`。09:22 source/API contract 已把 unreliable console artifact 額外投影成 `windows99_console_clipboard_unreliable` 類 active blocker。 | 先收斂 99 / Windows99 / VMware 與 111：恢復 no-secret management channel 或取得 validator 可接受的完整 console Verify stdout，讀回 VMX / VM power / host uptime，再 rerun host probe + reboot-event detector；不得 reboot、不得 VM power change、不得讀 Windows 密碼，不得用 RDP clipboard 片段當完成證據。 | | P0-2 | BLOCKED_EDGE_PRIVILEGED_APPLY | Deploy / reboot 期間 public 502 維護頁與外部 fallback | Gitea CD `#4519` 已推 deploy marker `3aca484 -> a94ddd5`，但 marker 後 public probe 仍讀到 `https://awoooi.wooo.work/api/v1/health` raw `502`、fallback header/body 空；live 188 `/etc/nginx/sites-enabled/awoooi.wooo.work.conf` 缺 maintenance fallback，`/var/www/maintenance/maintenance.html` 缺失，`ollama@188` 無 passwordless sudo。 | 先跑 `scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --check` 留 drift receipt；具備 privileged channel 後執行 `--apply`，要求 backup、`nginx -t`、reload、public route probe 全綠。不得讀密碼、不得用 app restart 掩蓋 edge fallback drift。 | -| P0-3 | PARTIAL_GREEN_SOURCE_RUNTIME | 所有產品 / 網站版本與資料最新性 | Gitea `main=89d4d6112`；Production SLO readback 對齊 deploy marker `89d4d6112 chore(cd): deploy 17ba08c [skip ci]`；Stock freshness `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。AWOOOI health HTTP 200 但整體 `degraded`，SignOz / local Ollama 仍需列為 runtime degraded evidence。 | 將 source SHA / deploy marker / runtime endpoint / public route watch 固定進 scorecard；完成條件是每個 public product 都有 source、deploy、runtime、freshness 四層 readback，且 degraded components 有 owner lane。 | +| P0-3 | BLOCKED_DEPLOY_READBACK | 所有產品 / 網站版本與資料最新性 | Gitea `main=647d81163` 已含 Windows99 collected verifier / next safe action source；production SLO route 仍未讀回該 source：`active_blocker_count=8`、`runtime_metric_runtime_readback_added_blockers=[]`、`windows99_update_no_auto_reboot_ready=true`。Public Gitea CD `#4556` 仍為 `Running`，build log 顯示 Docker build lock 等待後自清空鎖並進入 API/Web build/push，但尚無 deploy marker / production deploy readback。10:30 source 已補 CD Docker build/push bounded timeout 與 queue classifier，避免下一輪只剩無 ETA 的 `Running`。Stock freshness 仍為 `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。 | 先讓 CD 完成 deploy marker / production readback；完成條件是 source SHA、deploy marker、runtime endpoint、public route watch、freshness 四層一致。CD `Running`、Gitea main 最新或 test green 都不得單獨宣稱 production 最新。 | | P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 10:15 source/API patch：Production Windows99 nested readback 仍為 `missing_vmx_aliases=["111"]`、`powered_off_aliases=["111","112","120","121","188"]`，且 nested `service_blockers=["VMAuthdService","VMnetDHCP"]`、`policy_blockers=["windows_update_policy_readback_missing"]` 不能再被壓扁成單一 guest power blocker；API overlay 已補 `windows99_vmware_autostart_config_not_ready` / `windows99_update_no_auto_reboot_policy_not_ready` runtime-readback-added blocker。09:53 no-secret management probe：99 reachable、RDP / Hyper-V VMConnect reachable，WinRM timeout，Mac vantage collector 仍為 `ssh_batchmode_auth_ready=0` / `blocked_ssh_publickey_auth_missing`；production vantage 的 no-secret collector readback 曾回 collected，但完成條件仍取決於 VMX / service / policy / power 全綠。 | 先修 111 VMX source 與 VMware service / policy readback，使 check-mode package 能解釋 powered-off aliases；不得從 scorecard 直接重啟 Windows service、改 registry、啟動 / 關閉 VM 或 host reboot。完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable，且 active blocker matrix 顯示 service / policy / VMX / power 分開收斂。 | | P0-5 | RUNTIME_READY_BACKUP_RECEIPT_GAP | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。2026-07-03 source / runtime 已部署 `awoooi_backup_alert_receipt_*` 指標與 Prometheus rules；110 exporter 讀回 88 個 stage requirement、188 讀回 12 個 stage requirement，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 已修成每 `host / receipt_channel` 聚合 pending：110 一條、188 一條。 | 補 `/backup/alert-receipts/*.last_success` 脫敏 marker；下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 | | P0-6 | RUNTIME_READY_ALERT_RECEIPT_GAP | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Reboot per-blocker alert 與 backup receipt alert rules 已 deploy/readback；backup receipt 缺段不再產生 100 條 stage 噪音，現在聚合成 110 / 188 兩條 host-level pending。scorecard 仍有 8 個 reboot active blockers，尚未完成 shutdown / reboot / backup alert 的 production 脫敏 delivery receipt 全矩陣。 | 補 alert receipt readback：host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale；完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | -| P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase=host_boot_detection_blocked`、`eta_or_wait_reason=reboot_event_readback_missing_eta_unavailable`、`primary_blocker=reboot_event_required_host_unreachable`、fixed triage order 與 next safe action；08:23 artifact 固定下一步為 no-secret Windows99 verify / host probe rerun。09:22 `active_blocker_action_matrix` 已能把 unreliable console artifact 指向 `windows99_console_or_no_secret_management_channel` 與固定 next safe action。10 分鐘內自動恢復仍未達標。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard；完成條件是重啟後自動判斷、主動告警、主動 rerun verifier，不再人工臨場猜流程。 | +| P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase=host_boot_detection_blocked`、`eta_or_wait_reason=reboot_event_readback_missing_eta_unavailable`、`primary_blocker=reboot_event_required_host_unreachable`、fixed triage order 與 next safe action；08:23 artifact 固定下一步為 no-secret Windows99 verify / host probe rerun。09:22 `active_blocker_action_matrix` 已能把 unreliable console artifact 指向 `windows99_console_or_no_secret_management_channel` 與固定 next safe action。10:30 `read-public-gitea-actions-queue.py` 已能把 `BLOCKER cd_docker_step_timeout` 上卷成 `blocked_cd_docker_step_timeout` / `latest_visible_cd_docker_step_timeout*` / fixed safe-next-action；10 分鐘內自動恢復仍未達標。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard；完成條件是重啟後自動判斷、主動告警、主動 rerun verifier，不再人工臨場猜流程。 | | P0-8 | PARTIAL_READY_POLICY | Windows99 禁止 Windows Update 無預警重啟 | 10:15 source/API patch：若 Windows99 nested verifier 回 `policy_blockers=["windows_update_policy_readback_missing"]`，production API 必須 fail-closed 重新提升 `windows99_update_no_auto_reboot_policy_not_ready`，不得只信舊 Prometheus `windows99_update_no_auto_reboot_ready=true`。 | 保留 no-secret verifier，補週期性 policy readback 與 Telegram drift alert；完成條件是 Windows Update policy drift / missing readback 會自動告警且不需讀 secret，不得從 scorecard 直接 apply registry。 | | 優先 | 狀態 | 工作項 | 2026-06-30 證據 | 下一步 / 完成條件 | diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 0b173ff51..be6bed899 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -65,11 +65,11 @@ | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；per-blocker reboot alerts 與 backup receipt rules 已 deploy/readback。Backup receipt 缺段已從 100 條 stage 噪音收斂為 110 / 188 兩條 host-level pending；仍需完整 shutdown/up E2E receipt | 補 Prometheus / Alertmanager active/resolved 與 outbound receipt；backup alert 先補 `/backup/alert-receipts/*.last_success` 脫敏 marker，不送測試 secret、不重啟主機 | | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram/AwoooP receipt | 已有 backup health exporter / alert rules / Gitea bundle restore dry-run；2026-07-03 runtime 讀回 110 有 88 個 receipt stage requirement、188 有 12 個，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 聚合 pending 110 / 188 各一條 | 補 `/backup/alert-receipts/*.last_success`；再補 Gitea full dump / DB / settings / issues / packages / LFS 與所有工具/log 全量備份監控 | | 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `67%`、active blockers `8`、primary `reboot_event_required_host_unreachable`；09:22 source/API contract 已把 unreliable console artifact 接到 `active_blocker_action_matrix`、owner lane 與 next safe action | 優先收斂 99 no-secret Verify / 111 reachability / 188 startup failed/degraded；不得用不同排查路徑繞過 scorecard | -| 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea main 已到 deploy marker `c68b74686`，production source readback `5d5bc86fed` verified；StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-02`，core price/chips/margin/AI recommendations 都是 `2026-07-02` | 建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | +| 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea `main=647d81163` 已含 Windows99 collected verifier next-step source，但 production SLO route 仍未讀回該 source：`active_blocker_count=8`、`runtime_metric_runtime_readback_added_blockers=[]`、`windows99_update_no_auto_reboot_ready=true`；Gitea CD `#4556` 仍 Running，尚無 deploy marker / production deploy readback。StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-02`。10:30 已補 CD Docker build/push bounded timeout 與 queue classifier source，避免下一輪無 ETA Running。 | 先讀回 AWOOOI deploy marker / production runtime SHA / scorecard 行為；再建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | | 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 2026-07-02 production `/api/v1/agents/gitea-repo-bundle-backup-readback` 已 ready：9 expected repos present/ok、missing=0、failed=0、checksum_missing=0、bundle_fresh=true、all_expected_ok=true、sample_restore_dry_run_ok=true；repo bundle / restore dry-run 層已關閉，不是 repo missing。 | 維持每日 bundle backup + restore dry-run monitoring；另補 Gitea full dump / DB / settings / issues / packages / LFS 備份 readback。禁止刪 repo / 改 visibility / 讀 token / restore 到 production | | 11 | CIR-P0-CPU-001 | P0 | 「110 / 188 CPU 負載持續過高，為什麼沒監控告警、沒主動修復」 | Sustained CPU pressure automation：Alertmanager → controller → evidence → service playbook → verifier → KM writeback | 110 已有 `Host110SustainedModeratePressure`、Gitea playbook、Stock/Postgres evidence；188 仍需同級 controller/alerts readback | 下一步接 `postgres_hot_query_or_backup_export_playbook`；並補 188 equivalent readback，不以單次下降結案 | | 12 | CIR-P0-CPU-002 | P0 | 「噪音會影響真問題，要整合一起做」 | Alert noise / real issue correlation：backup aggregate noise、CPU pressure、Gitea queue、Stock freshness 要分清主因與次因 | 部分已在 SOP 註記；仍需統一 correlation scorecard | 建立 incident correlation readback：primary_blocker、secondary_noise、ignored_noise_reason、evidence_ref | -| 13 | CIR-P0-CD-001 | P0 | 「所有專案都不能推版 / 要看到實作結果」 | Gitea-only CD baseline：每次 main push 要有 visible run、deploy marker、production readback；GitHub 不作解法 | AWOOOI 最新 main 可推，CD success/deploy marker 已多次證明；全產品未全綠 | 將 product governance matrix 接入各產品 Gitea CD readiness，不再只報 AWOOOI | +| 13 | CIR-P0-CD-001 | P0 | 「所有專案都不能推版 / 要看到實作結果」 | Gitea-only CD baseline：每次 main push 要有 visible run、deploy marker、production readback；GitHub 不作解法 | AWOOOI main 可推，但目前 latest CD `#4556` 仍 Running 且 production 尚未 readback 最新 source；source 已補 `cd_docker_step_timeout` bounded marker 與 queue classifier，避免 CD 卡住時無具名 blocker | 先推送 / 讀回 bounded CD classifier，接著讀 deploy marker / production runtime；再將 product governance matrix 接入各產品 Gitea CD readiness | | 14 | CIR-P1-AI-001 | P1 | 「AI 專業在哪？要能主動發現、主動修復」 | AI controlled repair loop：detect → classify → candidate → check-mode → controlled apply → post verifier → KM / PlayBook trust | CPU / Gitea / Telegram receipt 已部分落地；全域 AI loop 未全部接上 | 將每個 P0 runbook 補 `candidate_action`、`controlled_apply_allowed`、`post_verifier`、`trust_writeback` | | 15 | CIR-P1-KM-001 | P1 | 「修復過程、經驗完整沉澱進 SOP，整合到目前版本」 | 所有 P0 修復必須同步 LOGBOOK、SOP、PlayBook、workplan ledger；不能只留在對話 | 本台帳、LOGBOOK、SOP 已開始補；09:22 已把 Windows99 console clipboard 不可靠經驗寫入 SOP v1.108、P0 workplan 與 scorecard regression；仍需 API/UI read model | 把本台帳轉成 read-only API / governance UI row，並建立 `last_updated` / `evidence_count` | | 16 | CIR-P1-WORK-001 | P1 | 「所有已開始、進行中、已完成工作全部看清楚」 | 工作狀態盤點：Done / In Progress / Blocked / Deferred / Next Action + evidence | 本台帳已有初版 Done/In Progress/Blocked；需納入本節新 P0 | 更新下方 Done/In Progress/Blocked，把 reboot/backup/VMware/maintenance/CPU 全列入 | @@ -115,6 +115,7 @@ | Public maintenance fallback runtime readback | Gitea CD `#4459` / deploy marker `8d7a6faaf` 已讓 production scorecard 讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、P0 blockers `11`、readiness `47` | | Reboot SLO per-blocker 告警投影 | Source 已補 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}`、`RebootAutoRecoveryActiveBlocker`、`RebootAutoRecoveryActiveBlockerMetricMissing` 與契約測試 | | Backup alert receipt runtime contract | Source / runtime 已補 `awoooi_backup_alert_receipt_expected_info`、`awoooi_backup_alert_receipt_stage_fresh`、`BackupAlertReceiptMetricMissing*`、`BackupAlertReceiptStageMissing`、baseline contract、live visibility checker 與 focused tests；Prometheus rule 已部署，缺段 alert 已聚合成 110 / 188 host-level pending；不送 Telegram、不讀 token | +| CD Docker build/push timeout classifier source | `.gitea/workflows/cd.yaml` 已把 API/Web `docker build` / `docker push` 包進 `run_docker_step`，timeout 會輸出 `BLOCKER cd_docker_step_timeout`；`read-public-gitea-actions-queue.py` 已上卷 `blocked_cd_docker_step_timeout` / `latest_visible_cd_docker_step_timeout*` / fixed safe-next-action；本地 focused tests `92 passed` | ### In Progress @@ -134,6 +135,7 @@ | OpenClaw / Gather-style 持續動畫工作室 | route 已存在，已列為 P1 工作項 | 補 production desktop/mobile smoke、AwoooP 導流與截圖證據 | | AI 專業 UI / 非文字牆 cockpit | 已列為 P2 UX 驗收 | 將長文字區塊收斂成 first-viewport cockpit、cards、flow rows 與 expandable details | | 10 分鐘 reboot auto-recovery SLA | 2026-07-03 08:23 production scorecard 固定 `8` blockers / readiness `67%`，但 `can_claim_all_services_recovered_within_target=false`；artifact `/tmp/awoooi-reboot-verify-only-20260703-082310` 保留 host probe / reboot detector / scorecard | 收斂 99 no-secret Verify、111 reachability、188 startup failed/degraded，再 rerun host probe + reboot-event detector | +| AWOOOI latest source production deploy readback | Gitea `main=647d81163` 已前進，public CD `#4556` Running；production SLO route 仍是舊版 blocker 行為，尚未讀回 latest source；本輪 source 已補 CD timeout classifier，等待 push / CD / deploy marker / production readback | 推送 bounded classifier commit 後讀 `read-public-gitea-actions-queue.py --json`、deploy marker、production scorecard，未 matched 前不得宣稱最新版本已上 production | | 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 尚未收集；08:23 management probe 顯示 RDP / Hyper-V VMConnect reachable，但 SSH BatchMode / WinRM blocked，collector `blocked_ssh_publickey_auth_missing` | 收集 99 no-secret Verify output，確認 VM 111/188/120/121/112 running、scheduled task / services / Windows Update policy 全綠 | | Reboot SLO blocker 收斂 | production scorecard 已固定 `8` blockers / readiness `67%`，source / runtime 已補 per-blocker metric/alert；剩餘主 blocker 是 host boot detection + Windows99 VMware VMX / guest power | 依具名 blocker 收斂 99/111/188；不得用 route green 或 RDP 可見宣稱 SLA 完成 | | 全備份監控告警 coverage | exporter/rule 已有 host/DB/site/service/package/tool/log coverage 與 backup alert receipt requirement；runtime rules 已部署，缺段 alert 聚合成 110 / 188 host-level pending；production receipt markers / full Gitea dump / DB/settings/issues/packages/LFS 尚未全綠 | 補 `/backup/alert-receipts/*.last_success`、Gitea full dump 與 restore drill verifier | diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index 2f4694a54..6787e661b 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -78,6 +78,11 @@ _HARBOR_CONTROLLED_REPAIR_SKIP_RE = re.compile( _HARBOR_CONTROLLED_REPAIR_STATUS_RE = re.compile( r"harbor_controlled_repair_public_registry_v2_status=(?P\d{3})" ) +_CD_DOCKER_STEP_TIMEOUT_RE = re.compile( + r"BLOCKER cd_docker_step_timeout " + r"label=(?P