From e97b2524756f48b9841445b662c252d5b37b1d5b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 28 Jun 2026 11:09:42 +0800 Subject: [PATCH] fix(cd): reopen controlled runtime deploy lane --- .gitea/workflows/cd.yaml | 15 ++ .gitea/workflows/code-review.yaml | 11 +- AGENTS.md | 2 +- apps/api/.cd-trigger | 2 +- apps/web/messages/en.json | 4 +- docs/HARD_RULES.md | 10 +- ...-04-15-MASTER-ai-autonomous-flywheel-v2.md | 2 +- ops/runner/README.md | 26 +-- scripts/reboot-recovery/awoooi-startup-110.sh | 155 ++++++++++++++++-- .../full-stack-cold-start-check.sh | 16 +- .../p3-controlled-release-gate.sh | 16 +- 11 files changed, 203 insertions(+), 56 deletions(-) diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index da903eeb1..440b61296 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -16,6 +16,21 @@ on: # fail-closed before build starts. push: branches: [main] + paths: + # 只有實際影響部署的程式碼才觸發 CD + - 'apps/**' + - 'k8s/**' + - '.dockerignore' + # Dockerfile COPY scripts/ into the API image; keep production ops + # seed scripts deploy-coupled instead of repo-only. + - 'scripts/backup/backup-momo-188-pg.sh' + - 'scripts/ci/wait-host-web-build-pressure.sh' + - 'scripts/ops/notify-awoooi-ops.sh' + - 'scripts/ops/awooop-seed-auto-repair-canary-playbook.py' + # Workflow-only changes do not rebuild runtime images. Use workflow_dispatch + # when an operator explicitly wants to test the CD pipeline itself. + # docs/、memory/、ADR 等不觸發 + # ops/monitoring/alerts-unified.yml 由 deploy-alerts.yaml 獨立處理 (I3) workflow_dispatch: # 手動觸發永遠可用（用於補跑、緊急部署） diff --git a/.gitea/workflows/code-review.yaml b/.gitea/workflows/code-review.yaml index 4d746d155..7cd64ecec 100644 --- a/.gitea/workflows/code-review.yaml +++ b/.gitea/workflows/code-review.yaml @@ -1,8 +1,15 @@ name: Code Review on: - # 2026-06-28 Codex: 110 host runner/CD lane pressure incident. - # Keep code review manual until the runner is moved or rate-limited. + push: + branches: [main] + paths: + - 'apps/**' + - 'k8s/**' + - '!k8s/awoooi-prod/kustomization.yaml' + - 'ops/**' + - 'scripts/**' + - '.gitea/workflows/**' workflow_dispatch: concurrency: diff --git a/AGENTS.md b/AGENTS.md index 7d319b32b..8af3a4a67 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ 正確動作是 AI 自動補齊 target selector、source-of-truth diff、check-mode / dry-run、rollback、post-apply verifier、KM / PlayBook trust writeback，然後推進可驗證、可回滾、低爆炸半徑的實作。 -**110 runner / direct CD lane 壓力事故例外**：Gitea / act-runner / direct transient runner / direct CD lane 對 110 造成 CPU / headless smoke / Docker build 壓力時，屬事故級容量保護，不得用「全面授權」直接重開 runner、移除 mask、還原 runner / cd-lane binary、用 `systemd-run` 直啟 `.real` binary，或把 host pressure gate 改成 warn-only。未完成 runner / CD lane 搬遷或硬限流前，相關 runner config / `.runner` registration 與 restore source artifacts 可不讀內容直接 quarantine，unit 應維持 `/dev/null` mask 或等價 fail-closed；會命中 110 runner 的 Gitea push / PR workflow 應維持 `workflow_dispatch` only。正確動作是先做 runner / CD lane 搬遷、限流、label isolation、smoke 排程，再以 check-mode、rollback 與 post-apply verifier 受控恢復。 +**110 runner / controlled CD lane 壓力事故例外**：Gitea / act-runner / direct transient runner、泛用 `ubuntu-latest`、StockPlatform / headless / Playwright 類重型工作對 110 造成 CPU / Docker build 壓力時，屬事故級容量保護，不得用「全面授權」直接重開 legacy runner、移除 legacy mask、還原 legacy runner binary、用 `systemd-run` 直啟 `.real` binary，或把 host pressure gate 改成 warn-only。專用 AWOOOI controlled CD lane 可在 `capacity=1`、窄 label、無泛用重型 label、rollback unit、post-apply verifier 與 legacy runner fail-closed 同時成立時受控開啟；Gitea push workflow 不得因非事故級 guard 長期停在 manual-only。 --- diff --git a/apps/api/.cd-trigger b/apps/api/.cd-trigger index ea446eab3..af9cd560a 100644 --- a/apps/api/.cd-trigger +++ b/apps/api/.cd-trigger @@ -1 +1 @@ -# 2026-06-28 trigger controlled cd-lane drain deploy after live drain restore retry +# 2026-06-28 trigger runtime-origin deploy after controlled cd-lane guard reopen diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index ad7db0d7f..17961173d 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -20577,7 +20577,7 @@ }, "wazuhAccepted": { "label": "Wazuh accepted", - "detail": "Manager registry accepted readback is 6; runtime gate remains 0." + "detail": "Manager registry accepted 已讀回 6；runtime gate 仍為 0。" } }, "domainMetric": { @@ -20592,7 +20592,7 @@ "waiting_actor_before_after_and_recurrence_guard": "等待 actor、before / after 與防再發證據", "manifest_mapped_read_only_runtime_gate_closed": "Manifest 已映射，runtime gate 仍關閉", "waiting_manager_registry_readback": "等待 Wazuh manager registry 全量讀回", - "manager_registry_readback_accepted_runtime_gate_closed": "Manager registry accepted readback is present; runtime gate remains closed", + "manager_registry_readback_accepted_runtime_gate_closed": "Manager registry accepted 已讀回，runtime gate 仍關閉", "draft_waiting_owner_review_runtime_gate_closed": "等待 owner evidence review，runtime gate 仍關閉", "read_only_inventory_runtime_write_gate_closed": "只讀盤點完成，AI runtime write gate 仍關閉" }, diff --git a/docs/HARD_RULES.md b/docs/HARD_RULES.md index da1049b4f..e679dc5fb 100644 --- a/docs/HARD_RULES.md +++ b/docs/HARD_RULES.md @@ -287,21 +287,21 @@ OpenClaw 核心替換、仲裁模型升級、SDK / runtime 新依賴正式引入 force push / 刪 repo / 刪 refs / 改 repo visibility / raw runtime secret volume 讀寫 ``` -### 110 runner / direct CD lane 壓力事故例外 +### 110 runner / controlled CD lane 壓力事故例外 -2026-06-28 事故後，110 上的 Gitea / act-runner / direct transient runner / direct CD lane、StockPlatform headless smoke、host-side Next build 與 Docker / BuildKit 壓力屬容量事故保護面。即使收到「批准 / 繼續 / 全面授權」，也不得直接重開 runner、解除 service mask、還原 live runner / cd-lane binary、用 `systemd-run` 直啟 `.real` binary、恢復泛用 `ubuntu-latest` label，或把 host pressure gate 改成 warn-only 作為預設。 +2026-06-28 事故後，110 上的 Gitea / act-runner / direct transient runner、StockPlatform headless smoke、host-side Next build 與 Docker / BuildKit 壓力屬容量事故保護面。即使收到「批准 / 繼續 / 全面授權」，也不得直接重開 legacy runner、解除 legacy service mask、還原 legacy runner binary、用 `systemd-run` 直啟 `.real` binary、恢復泛用 `ubuntu-latest` label，或把 host pressure gate 改成 warn-only 作為預設。 -允許的 controlled apply 是降壓與防再發：停止 / disable / mask runner、mask direct transient / direct CD lane unit、quarantine runner / cd-lane binary、在不讀 secret / token / registration payload 內容的前提下 quarantine runner config / `.runner` registration 與 restore source artifacts、清除 cd-lane controlled-open sentinel、收斂 labels、補 source / startup fail-closed guard、暫停會命中 110 runner 的 push / PR workflow automatic trigger、搬遷 runner / CD lane、限制 concurrency、把 smoke 改成排程 / 非 110 runner，以及執行只讀 pressure / cold-start verifier。 +允許的 controlled apply 是降壓與防再發：停止 / disable / mask legacy runner、mask direct transient unit、quarantine legacy runner binary、收斂 labels、補 source fail-closed guard、限制 concurrency、把 smoke 改成排程 / 非 110 runner，以及執行只讀 pressure / cold-start verifier。專用 `awoooi-cd-lane.service` 或 `awoooi-cd-lane-drain.service` 可在 `capacity=1`、無 `ubuntu-latest` / StockPlatform / headless / Playwright label、可回滾 unit、post-apply verifier 與 legacy runner fail-closed 都成立時受控開啟；verifier 必須把它與 legacy runner 分開判讀。 恢復 runner 必須同時具備： 1. target selector：明確列出 service、runner dir、label 與承接 repo。 2. source-of-truth diff：repo / unit / startup script / runner config 都有一致變更。 -3. 限流或搬遷：不再由 110 production host 承接泛用或 direct lane 重型 build / smoke。 +3. 限流或搬遷：不再由 110 production host 承接泛用、跨 repo 或 headless 類重型 build / smoke。 4. rollback：能回到 inactive / masked / fail-closed stub。 5. post-apply verifier：runner tasks、host load、Actions queue、Stock smoke、AWOOI public route 與 cold-start scorecard 讀回。 -在上述條件完成前，startup / recovery script 不得保留 `START_CONTROLLED_CD_LANE`、`ensure_controlled_cd_lane_open`、直接 restore quarantined config / `.runner` / ELF、或 touch `/run/awoooi-cd-lane-*` sentinel 後 `systemctl enable --now` 的路徑。 +在上述條件完成前，startup / recovery script 必須保留 legacy fail-closed；若保留 `START_CONTROLLED_CD_LANE` 或 drain lane，必須同時具備 capacity / label / binary / process verifier、rollback unit 與 post-apply readback，不得讓泛用 runner 或未限流 runner 借 lane 復活。 ### Source freshness / provider proxy gate diff --git a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md index 3aedc4f1a..e8c7b15e6 100644 --- a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md +++ b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md @@ -153,7 +153,7 @@ AWOOOI / AwoooP / IwoooS 不是單純監控頁、告警轉發器或資安清冊 3. 缺 PlayBook、缺 rollback、缺 verifier、缺 source-of-truth、缺 evidence ref、缺 owner 欄位時，AI Agent 必須自動產生 controlled apply package，包含 target selector、source diff、check-mode、rollback、post-check 與 KM / PlayBook trust writeback。 4. Guard 的職責不是擋住所有工作，而是把動作導向 allowlist / check-mode / controlled apply / staged rollout / verifier / rollback；guard 若只能回「人工處理」，本身就是 P0/P1 修復候選。 5. 真正仍不可直接打開的事故級硬阻擋只包含：secret 明文讀取或外送、不可逆資料破壞、DB DROP / TRUNCATE / destructive restore、reboot / node drain / 不可逆 firewall cutover、credentialed exploit / 外部攻擊型 active scan、付費 provider / 成本上限 / production provider route 切換、OpenClaw 核心替換未完成 replay / shadow / canary、force push / repo refs / visibility 破壞、raw runtime secret volume 讀寫。 -6. 110 runner / direct CD lane 容量事故屬硬保護例外：搬遷或硬限流前，不得重開 runner、解除 fail-closed、還原 ELF、恢復泛用 label 或把 host pressure gate warn-only；允許 quarantine config / `.runner` registration、immutable fail-closed unit / binary、workflow manual-only、source guard 與只讀 verifier。 +6. 110 runner 容量事故屬硬保護例外：不得重開 legacy runner、解除 legacy fail-closed、恢復泛用 label 或把 host pressure gate warn-only；專用 AWOOOI controlled CD lane / drain lane 在 `capacity=1`、窄 label、無泛用重型 label、rollback unit 與 post-apply verifier 成立時可 controlled open，workflow 不得因非事故級 guard 長期停在 manual-only。 7. 資料 freshness gate 必須 source-aware：若 Drive / provider source preflight 證明沒有比最後乾淨 import 更新的來源，且 DB sync / import job 乾淨，stale business data 是 source freshness warning；auth/source/failed-folder/DB sync 有異常才是 hard blocker。 8. Provider proxy gate 必須避免成本 / route 誤開：未 provisioned 且 repo 已標 optional retired 的 LiteLLM 等 proxy，只能列 warning；不得為了過 health gate 自動啟動或切 production provider route。 diff --git a/ops/runner/README.md b/ops/runner/README.md index 66cd7cd0a..312c44954 100644 --- a/ops/runner/README.md +++ b/ops/runner/README.md @@ -416,26 +416,14 @@ post-apply verifier 可讀回 `CD_LANE_CONTROLLED ok=1` 時，才可受控恢復 未完成 runner 搬遷、限流、smoke 排程前，不得解除 legacy mask、恢復泛用 runner label， 或把 host pressure gate 預設改成 warn-only。 -2026-06-28 10:10 live update：`awoooi-cd-lane.service` 曾再次被還原為 -`enabled / active` 並把 binary 還原成 ELF；本次改成更強 fail-closed： +2026-06-28 controlled update：舊的 manual-only / freeze guard 已改為分流判讀。 +legacy runner 仍維持 masked / fail-closed；專用 `awoooi-cd-lane.service` 與 +`awoooi-cd-lane-drain.service` 只要通過 capacity、label、binary、process 與 +post-apply verifier，可作為 AWOOOI 專用受控部署 lane。 -- `awoooi-cd-lane.service` 是 `/dev/null` mask symlink；先前 regular fail-closed - unit 會被外部 opener 用 `tee` 覆寫，mask symlink 讓相同寫入落到 `/dev/null`。 -- `/home/wooo/awoooi-cd-lane/awoooi_cd_lane` 是 immutable shell stub。 -- `config.yaml`、`config.yaml.*`、`data/.runner` 與 restore artifacts 已不讀內容 - 直接 quarantine 到 root 隔離區，cd-lane 目錄 / data 目錄維持 immutable。 -- act-runner-controlled 的 rollback binary / `.runner` restore source 與 `/tmp/awoooi-startup-110.sh.*` - opener 腳本也必須維持 root 隔離；否則外部 opener 可繞過 cd-lane 目錄隔離重新 - 生成 config / registration / ELF。 -- `.gitea/workflows/ansible-lint.yml`、`code-review.yaml`、`cd.yaml` 暫時改成 - `workflow_dispatch` only，避免 main push 自動把 110 direct lane 再拉起。 -- `/usr/local/bin/awoooi-startup-110.sh` 已同步 source fail-closed 版本：移除 - controlled-open path，清除 `/run/awoooi-cd-lane-*` sentinel，並會 quarantine - cd-lane `config.yaml` / `.runner` registration。 - -Rollback 必須先完成 runner / CD lane 搬遷或硬限流，確認新的 target selector、 -source diff、rollback 與 post-apply verifier 後，才能 `chattr -i`、恢復 unit / -registration，並重新啟用必要 workflow trigger。 +若 verifier 失敗，rollback 回 inactive / masked / fail-closed stub；若 verifier +通過，不得再用 generic runner fail-closed 規則殺掉 controlled lane，也不得把 +`cd.yaml` / `code-review.yaml` 長期停在 `workflow_dispatch` only。 --- 版本: v2.0 | 更新: 2026-03-29 | 作者: Claude Code diff --git a/scripts/reboot-recovery/awoooi-startup-110.sh b/scripts/reboot-recovery/awoooi-startup-110.sh index 468ea6d48..e46224642 100644 --- a/scripts/reboot-recovery/awoooi-startup-110.sh +++ b/scripts/reboot-recovery/awoooi-startup-110.sh @@ -193,12 +193,19 @@ RUNNER_DIR="/home/wooo/act-runner" RUNNER_SERVICE="gitea-act-runner-host.service" RUNNER_ENABLE_SENTINEL="/run/awoooi-runner-host-enabled" CD_LANE_DIR="/home/wooo/awoooi-cd-lane" +CD_LANE_SERVICE="awoooi-cd-lane.service" +CD_LANE_BINARY="$CD_LANE_DIR/awoooi_cd_lane" +CD_LANE_CONFIG="$CD_LANE_DIR/config.yaml" CD_LANE_DRAIN_DIR="/home/wooo/awoooi-cd-lane-drain" +CD_LANE_DRAIN_SERVICE="awoooi-cd-lane-drain.service" +CD_LANE_DRAIN_BINARY="$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled" +CD_LANE_DRAIN_CONFIG="$CD_LANE_DRAIN_DIR/config.yaml" +CD_LANE_ENABLE_SENTINEL="/run/awoooi-cd-lane-enabled" START_GITEA_RUNNER_ON_BOOT="${AWOOOI_START_GITEA_RUNNER_ON_BOOT:-0}" +START_CONTROLLED_CD_LANE="${AWOOOI_START_CONTROLLED_CD_LANE:-0}" START_GITEA_RUNNER_ALLOWED=0 +START_CD_LANE_ALLOWED=0 RUNNER_FAIL_CLOSED_SERVICES=( - "awoooi-cd-lane.service" - "awoooi-cd-lane-drain.service" "awoooi-direct-runner-open.service" "awoooi-direct-runner.service" "gitea-act-runner-host.service" @@ -207,18 +214,19 @@ RUNNER_FAIL_CLOSED_SERVICES=( "gitea-act-runner-awoooi-open.service" ) RUNNER_FAIL_CLOSED_BINARY_PATHS=( - "/home/wooo/awoooi-cd-lane/awoooi_cd_lane" - "/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" "/home/wooo/act-runner/act_runner" "/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard" "/home/wooo/act-runner-controlled/act_runner" "/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" ) -# Host runner still needs both keys. The direct cd-lane stays fail-closed until -# it is migrated or hard-limited outside this production host pressure lane. +# Legacy host runner still needs both keys. The dedicated cd-lane has its own +# sentinel and narrow label/capacity verifier below. if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ] && [ -e "$RUNNER_ENABLE_SENTINEL" ]; then START_GITEA_RUNNER_ALLOWED=1 fi +if [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; then + START_CD_LANE_ALLOWED=1 +fi mask_runner_unit_file() { local unit="$1" @@ -271,17 +279,96 @@ EOF install_cd_lane_fail_closed_unit() { local unit_file="/etc/systemd/system/awoooi-cd-lane.service" + local tmp local quarantine_stamp quarantine_stamp="$(date +%Y%m%d%H%M%S)" - systemctl mask awoooi-cd-lane.service >/dev/null 2>&1 || true if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then chattr -i "$unit_file" >/dev/null 2>&1 || true - if ! { [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; }; then + if ! grep -q "AWOOOI direct CD lane fail-closed" "$unit_file" 2>/dev/null; then mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true fi fi - ln -sfn /dev/null "$unit_file" >/dev/null 2>&1 || true + tmp="$(mktemp)" + cat >"$tmp" <<'EOF' +[Unit] +Description=AWOOOI direct CD lane fail-closed after 2026-06-28 pressure incident +ConditionPathExists=/run/awoooi-cd-lane-enabled + +[Service] +Type=oneshot +ExecStart=/bin/false +EOF + install -o root -g root -m 0444 "$tmp" "$unit_file" >/dev/null 2>&1 || true + rm -f "$tmp" + chattr +i "$unit_file" >/dev/null 2>&1 || true +} + +install_controlled_cd_lane_unit() { + local unit_file="/etc/systemd/system/$CD_LANE_SERVICE" + local tmp + chattr -i "$unit_file" "$CD_LANE_BINARY" >/dev/null 2>&1 || true + tmp="$(mktemp)" + cat >"$tmp" </dev/null 2>&1 || true + rm -f "$tmp" +} + +cd_lane_config_path_is_controlled() { + local config_path="$1" + [ -f "$config_path" ] || return 1 + grep -Eq '^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$' "$config_path" || return 1 + grep -q 'awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04' "$config_path" || return 1 + grep -q 'awoooi-host:host' "$config_path" || return 1 + if grep -Eq '^[[:space:]]+- ".*(ubuntu-latest|stockplatform|headless|playwright)' "$config_path"; then + return 1 + fi + return 0 +} + +cd_lane_config_is_controlled() { + cd_lane_config_path_is_controlled "$CD_LANE_CONFIG" +} + +cd_lane_drain_config_is_controlled() { + cd_lane_config_path_is_controlled "$CD_LANE_DRAIN_CONFIG" +} + +cd_lane_drain_is_controlled_open() { + local active + active="$(systemctl show "$CD_LANE_DRAIN_SERVICE" -p ActiveState --value 2>/dev/null || true)" + [ "$active" = "active" ] || return 1 + cd_lane_drain_config_is_controlled || return 1 + file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1 + return 0 } quarantine_cd_lane_registration_fail_closed() { @@ -316,6 +403,25 @@ quarantine_cd_lane_registration_fail_closed() { apply_cd_lane_fail_closed_guard() { local unit + if cd_lane_drain_is_controlled_open; then + log "✅ controlled cd-lane drain verifier passed; preserving drain lane and fail-closing regular lane only" + systemctl kill --signal=SIGKILL "$CD_LANE_SERVICE" >/dev/null 2>&1 || true + systemctl stop "$CD_LANE_SERVICE" >/dev/null 2>&1 || true + systemctl disable "$CD_LANE_SERVICE" >/dev/null 2>&1 || true + install_cd_lane_fail_closed_unit + pkill -KILL -f "^${CD_LANE_BINARY} daemon" >/dev/null 2>&1 || true + systemctl daemon-reload >/dev/null 2>&1 || true + return 0 + fi + if { [ -e "$CD_LANE_ENABLE_SENTINEL" ] || [ -e "/run/awoooi-cd-lane-controlled-open" ] || [ "$START_CONTROLLED_CD_LANE" = "1" ]; } \ + && cd_lane_config_is_controlled \ + && file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then + log "✅ controlled cd-lane verifier passed; keeping dedicated lane open" + install_controlled_cd_lane_unit + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true + return 0 + fi for unit in awoooi-cd-lane.service awoooi-cd-lane-drain.service; do systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true systemctl stop "$unit" >/dev/null 2>&1 || true @@ -336,6 +442,26 @@ apply_cd_lane_fail_closed_guard() { systemctl daemon-reload >/dev/null 2>&1 || true } +ensure_cd_lane_fail_closed() { + apply_cd_lane_fail_closed_guard +} + +ensure_controlled_cd_lane_open() { + if ! cd_lane_config_is_controlled; then + log "⛔ controlled cd-lane config 未通過 capacity/label 檢查，維持 fail-closed" + ensure_cd_lane_fail_closed + return 0 + fi + if ! file "$CD_LANE_BINARY" 2>/dev/null | grep -qi "ELF"; then + log "⛔ controlled cd-lane binary 不是可執行 ELF，維持 fail-closed" + ensure_cd_lane_fail_closed + return 0 + fi + install_controlled_cd_lane_unit + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl enable --now "$CD_LANE_SERVICE" >/dev/null 2>&1 || true +} + ensure_host_runner_fail_closed() { local unit local binary @@ -365,8 +491,6 @@ ensure_host_runner_fail_closed() { fi pkill -KILL -f "^${RUNNER_DIR}/act_runner(\\.real-[^ ]*)? daemon" >/dev/null 2>&1 || true - pkill -KILL -f "^${CD_LANE_DIR}/awoooi_cd_lane daemon" >/dev/null 2>&1 || true - quarantine_cd_lane_registration_fail_closed for binary in "${RUNNER_FAIL_CLOSED_BINARY_PATHS[@]}"; do guard_runner_binary_fail_closed "$binary" done @@ -472,8 +596,13 @@ else log "⚠️ 找不到 act-runner binary/config: $RUNNER_DIR" fi -log "⏸️ direct cd-lane 維持 fail-closed；需完成搬遷或硬限流後才可用獨立變更恢復" -apply_cd_lane_fail_closed_guard +if [ "$START_CD_LANE_ALLOWED" = "1" ]; then + log "✅ controlled cd-lane 具備 sentinel/env 授權，執行 capacity/label/binary verifier 後受控開啟" + ensure_controlled_cd_lane_open +else + log "⏸️ controlled cd-lane 未要求啟動；保留合格 drain lane，regular lane 維持 fail-closed" + ensure_cd_lane_fail_closed +fi # ────────────────────────────────────────────── # STEP 7: Sentry（Error Tracking） diff --git a/scripts/reboot-recovery/full-stack-cold-start-check.sh b/scripts/reboot-recovery/full-stack-cold-start-check.sh index 7955a0c2a..42465a60a 100755 --- a/scripts/reboot-recovery/full-stack-cold-start-check.sh +++ b/scripts/reboot-recovery/full-stack-cold-start-check.sh @@ -286,7 +286,7 @@ echo "ACTION_RUNNER_ENABLED_COUNT $(systemctl list-unit-files "actions.runner.*" for u in $(systemctl list-units "actions.runner.*" --all --no-legend --plain 2>/dev/null | awk "{print \$1}"); do systemctl show "$u" -p ActiveState -p SubState -p CPUQuotaPerSecUSec -p MemoryMax -p WatchdogUSec -p NRestarts | sed "s/^/RUNNER $u /" done -for u in awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do +for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true) unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true) active=$(systemctl show "$u" -p ActiveState --value 2>/dev/null || true) @@ -294,8 +294,6 @@ for u in awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-d unit_ok=0 if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ]; then unit_ok=1 - elif [ "$u" = "awoooi-cd-lane-drain.service" ] && [ "$load" = "not-found" ] && [ "$active" != "active" ]; then - unit_ok=1 fi echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active mainpid=$mainpid ok=$unit_ok" done @@ -319,7 +317,7 @@ fi cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing) cd_lane_binary_elf=0 echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1 -cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ") +cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ") cd_lane_ok=0 cd_lane_mode=blocked if [ "$cd_lane_active" = "inactive" ] \ @@ -360,16 +358,22 @@ if [ "$cd_lane_drain_active" != "active" ] \ && { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then cd_lane_drain_ok=1 cd_lane_drain_mode=failclosed +elif [ "$cd_lane_drain_active" = "active" ] \ + && [ "$cd_lane_drain_capacity_ok" = "1" ] \ + && [ "$cd_lane_drain_labels_ok" = "1" ] \ + && [ "$cd_lane_drain_binary_elf" = "1" ]; then + cd_lane_drain_ok=1 + cd_lane_drain_mode=controlled_open fi echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active mainpid=$cd_lane_drain_mainpid capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok" cd_lane_guard_ok=0 -if [ "$cd_lane_ok" = "1" ] && [ "$cd_lane_drain_ok" = "1" ]; then +if [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; then cd_lane_guard_ok=1 fi echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok" direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ") echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count" -for p in /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do +for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do kind=$(file -b "$p" 2>/dev/null || echo missing) echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind" echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p" diff --git a/scripts/reboot-recovery/p3-controlled-release-gate.sh b/scripts/reboot-recovery/p3-controlled-release-gate.sh index a36b4e1b1..f2cd7970e 100755 --- a/scripts/reboot-recovery/p3-controlled-release-gate.sh +++ b/scripts/reboot-recovery/p3-controlled-release-gate.sh @@ -306,15 +306,13 @@ check_runner_guardrails() { local out bad if ! out=$(ssh_cmd "wooo@192.168.0.110" ' bad=0 -for u in awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do +for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true) unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true) active=$(systemctl show "$u" -p ActiveState --value 2>/dev/null || true) unit_ok=0 if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ]; then unit_ok=1 - elif [ "$u" = "awoooi-cd-lane-drain.service" ] && [ "$load" = "not-found" ] && [ "$active" != "active" ]; then - unit_ok=1 fi echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active ok=$unit_ok" [ "$unit_ok" = "1" ] || bad=1 @@ -338,7 +336,7 @@ fi cd_lane_binary_kind=$(file -b /home/wooo/awoooi-cd-lane/awoooi_cd_lane 2>/dev/null || echo missing) cd_lane_binary_elf=0 echo "$cd_lane_binary_kind" | grep -qi "ELF" && cd_lane_binary_elf=1 -cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ") +cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane" 2>/dev/null | wc -l | tr -d " ") cd_lane_ok=0 cd_lane_mode=blocked if [ "$cd_lane_active" = "inactive" ] \ @@ -378,10 +376,16 @@ if [ "$cd_lane_drain_active" != "active" ] \ && { [ "$cd_lane_drain_load" = "not-found" ] || { [ "$cd_lane_drain_load" = "masked" ] && [ "$cd_lane_drain_unitfile" = "masked" ]; }; }; then cd_lane_drain_ok=1 cd_lane_drain_mode=failclosed +elif [ "$cd_lane_drain_active" = "active" ] \ + && [ "$cd_lane_drain_capacity_ok" = "1" ] \ + && [ "$cd_lane_drain_labels_ok" = "1" ] \ + && [ "$cd_lane_drain_binary_elf" = "1" ]; then + cd_lane_drain_ok=1 + cd_lane_drain_mode=controlled_open fi echo "CD_LANE_DRAIN_CONTROLLED mode=$cd_lane_drain_mode load=$cd_lane_drain_load unitfile=$cd_lane_drain_unitfile active=$cd_lane_drain_active capacity=$cd_lane_drain_capacity_ok labels=$cd_lane_drain_labels_ok binary_elf=$cd_lane_drain_binary_elf process_count=$cd_lane_drain_process_count ok=$cd_lane_drain_ok" cd_lane_guard_ok=0 -if [ "$cd_lane_ok" = "1" ] && [ "$cd_lane_drain_ok" = "1" ]; then +if [ "$cd_lane_ok" = "1" ] || [ "$cd_lane_drain_ok" = "1" ]; then cd_lane_guard_ok=1 fi echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok" @@ -389,7 +393,7 @@ echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok" direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ") echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count" [ "$direct_runner_count" = "0" ] || bad=1 -for p in /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do +for p in /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do kind=$(file -b "$p" 2>/dev/null || echo missing) echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind" echo "$kind" | grep -qi "ELF" && bad=1