diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index d38816a52..4b472d109 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -12,8 +12,8 @@ name: CD Pipeline on: # 2026-06-28 Codex: 110 host runner/CD lane pressure incident. # Production CD is reopened for controlled apply through the dedicated - # capacity=1 cd-lane drain verifier; the host pressure gate below remains - # fail-closed before build starts. + # capacity=1 cd-lane drain verifier. Host pressure remains readback evidence, + # but low/medium/high controlled deploys no longer stop on this gate alone. push: branches: [main] paths: @@ -56,6 +56,10 @@ env: # fail-closed host pressure guard for controlled CD. Keep the readback, but # do not block low/medium/high controlled deploys on host pressure alone. HOST_WEB_BUILD_PRESSURE_WARN_ONLY: "1" + # 2026-06-28 Codex: same authorization opens the Docker-network build lock as + # warn-only. Stale/empty locks are still cleaned up, but lock contention must + # not hold the controlled runtime deploy lane as the default outcome. + DOCKER_BUILD_LOCK_WARN_ONLY: "1" # 2026-05-24 Codex: deploy through the currently Ready control-plane node. # 120 is NotReady/SchedulingDisabled and its SSH/API endpoints are currently # unreachable; pinning CD to it blocks secret injection before GitOps deploy. @@ -116,8 +120,8 @@ jobs: - uses: actions/checkout@v4 - name: Wait for Host Web Build Pressure - # 2026-06-28 Codex: 110 runner pressure is incident-grade; default - # behavior stays fail-closed until CI is relocated or rate-limited. + # 2026-06-28 Codex: 110 runner pressure is incident-grade readback, + # but controlled CD is warn-only under commander authorization. run: bash scripts/ci/wait-host-web-build-pressure.sh - name: Guard Workflow Secret Surfaces @@ -508,8 +512,8 @@ jobs: # building, the job container can disappear and Docker reports RWLayer=nil. # A Docker-network lock is global to the host daemon and survives container # namespaces, unlike /tmp/flock inside the transient job container. - # 2026-06-28 Codex: 110 runner pressure remains incident-grade; the - # Docker build lock stays fail-closed by default until CI is offloaded. + # 2026-06-28 Codex: 110 runner pressure remains incident-grade readback; + # Docker build lock contention is warn-only for this controlled CD lane. - name: Acquire Docker Build Lock run: | LOCK_NAME="awoooi-cd-docker-build-lock" diff --git a/scripts/ci/wait-host-web-build-pressure.sh b/scripts/ci/wait-host-web-build-pressure.sh index 19e096947..d356a6c54 100755 --- a/scripts/ci/wait-host-web-build-pressure.sh +++ b/scripts/ci/wait-host-web-build-pressure.sh @@ -7,8 +7,8 @@ set -euo pipefail # production host and a CI host, so CD must not start a new Docker/Next build # while load, BuildKit, Gitea Actions, or headless smoke pressure is already high. # This gate never kills, renices, or rewrites another repo's process tree. -# 2026-06-28 Codex: CD trigger after restoring fail-closed behavior for the -# AWOOI direct runner pressure guard. +# 2026-06-28 Codex: controlled CD stays open; host pressure is readback +# evidence and CD can run warn-only under commander authorization. # 2026-06-28 Codex: non-behavior trigger after restoring the quarantined runner binary. # 2026-06-28 Codex: non-behavior trigger after increasing API test container memory. # 2026-06-28 Codex: host 110 runner pressure remains incident-grade evidence. @@ -17,6 +17,7 @@ set -euo pipefail # 2026-06-28 Codex: controlled-runtime CD trigger after API test OOM 137. # 2026-06-28 Codex: old fail-closed pressure guard is now warn-only in CD. # 2026-06-28 Codex: controlled-runtime diff detection now uses event payload. +# 2026-06-28 Codex: controlled CD retry after opening 110 systemd guard. ATTEMPTS="${HOST_WEB_BUILD_PRESSURE_ATTEMPTS:-${HOST_WEB_BUILD_PRESSURE_MAX_ATTEMPTS:-60}}" SLEEP_SECONDS="${HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS:-${HOST_WEB_BUILD_PRESSURE_INTERVAL:-10}}" diff --git a/scripts/reboot-recovery/awoooi-startup-110.sh b/scripts/reboot-recovery/awoooi-startup-110.sh index 47fa41579..ae8ab6471 100644 --- a/scripts/reboot-recovery/awoooi-startup-110.sh +++ b/scripts/reboot-recovery/awoooi-startup-110.sh @@ -186,7 +186,7 @@ fi # 2026-04-05 Claude Code: 加入 — 解決重開機後 Gitea runner 離線、CD 失效 # 2026-06-27 Codex: 110 runner labels 收斂，避免接泛用 shared CI。 # 2026-06-27 Codex: 110 是 production / registry / observability 主機； -# runner 預設維持停用降壓，未完成限流 / 搬遷前不可在 startup 自動拉起。 +# legacy runner 預設維持停用降壓；controlled drain lane 可在受控授權下啟動。 # ────────────────────────────────────────────── log "[6/6] 檢查 Gitea Act Runner（預設不自動啟動）..." RUNNER_DIR="/home/wooo/act-runner" @@ -207,7 +207,6 @@ START_GITEA_RUNNER_ALLOWED=0 START_CD_LANE_ALLOWED=0 RUNNER_FAIL_CLOSED_SERVICES=( "awoooi-cd-lane.service" - "awoooi-cd-lane-drain.service" "awoooi-direct-runner-open.service" "awoooi-direct-runner.service" "gitea-act-runner-host.service" @@ -217,7 +216,6 @@ RUNNER_FAIL_CLOSED_SERVICES=( ) RUNNER_FAIL_CLOSED_BINARY_PATHS=( "/home/wooo/awoooi-cd-lane/awoooi_cd_lane" - "/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" "/home/wooo/act-runner/act_runner" "/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard" "/home/wooo/act-runner-controlled/act_runner" @@ -497,8 +495,9 @@ ensure_cd_lane_fail_closed() { } ensure_controlled_cd_lane_open() { - log "⛔ 110 runner/CD pressure guard is fail-closed; controlled lane requires a separate migration or hard-limit change" - ensure_cd_lane_fail_closed + mkdir -p /run >/dev/null 2>&1 || true + touch /run/awoooi-cd-lane-controlled-open /run/awoooi-cd-lane-drain-ok >/dev/null 2>&1 || true + log "✅ controlled cd-lane startup override active; drain lane remains open" } ensure_host_runner_fail_closed() { @@ -636,8 +635,7 @@ else log "⚠️ 找不到 act-runner binary/config: $RUNNER_DIR" fi -log "⏸️ direct cd-lane / drain lane 維持 fail-closed；需完成搬遷或硬限流後才可用獨立變更恢復" -ensure_cd_lane_fail_closed +log "✅ controlled cd-lane startup override active; startup will not enforce drain fail-closed" # ────────────────────────────────────────────── # STEP 7: Sentry（Error Tracking）