diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 742b7185e..909ec50b9 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -1015,9 +1015,12 @@ jobs: HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }} run: | HARBOR_REGISTRY="${{ env.HARBOR }}" - LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-12}" - LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-10}" + LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-3}" + LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-5}" WATCHDOG="/usr/local/bin/harbor-watchdog.sh" + HARBOR_110_NODE_EXPORTER_URL="${HARBOR_110_NODE_EXPORTER_URL:-http://192.168.0.110:9100/metrics}" + HARBOR_110_MAX_LOAD5_PER_CORE="${HARBOR_110_MAX_LOAD5_PER_CORE:-1.25}" + HARBOR_110_MAX_GITEA_CPU_CORES="${HARBOR_110_MAX_GITEA_CPU_CORES:-3.0}" probe_registry_v2() { curl --silent --show-error \ @@ -1040,6 +1043,35 @@ jobs: fi } + greater_than() { + awk -v left="$1" -v right="$2" 'BEGIN { exit !(left > right) }' + } + + preflight_110_capacity() { + metrics="$(curl --silent --show-error --max-time 5 "${HARBOR_110_NODE_EXPORTER_URL}" || true)" + if [ -z "${metrics}" ]; then + echo "harbor_110_capacity_metrics_unavailable=1" + return 0 + fi + load5_per_core="$(printf '%s\n' "${metrics}" | awk '$1 ~ /^awoooi_host_load5_per_core\\{/ && $0 ~ /host=\"110\"/ { value = $NF } END { print value }')" + gitea_cpu_cores="$(printf '%s\n' "${metrics}" | awk '$1 ~ /^docker_container_cpu_cores\\{/ && $0 ~ /container_name=\"gitea\"/ { value = $NF } END { print value }')" + if [ -n "${load5_per_core}" ]; then + echo "harbor_110_load5_per_core=${load5_per_core}" + if greater_than "${load5_per_core}" "${HARBOR_110_MAX_LOAD5_PER_CORE}"; then + echo "BLOCKER harbor_110_host_pressure_high load5_per_core=${load5_per_core} threshold=${HARBOR_110_MAX_LOAD5_PER_CORE}" + return 1 + fi + fi + if [ -n "${gitea_cpu_cores}" ]; then + echo "harbor_110_gitea_cpu_cores=${gitea_cpu_cores}" + if greater_than "${gitea_cpu_cores}" "${HARBOR_110_MAX_GITEA_CPU_CORES}"; then + echo "BLOCKER harbor_110_gitea_cpu_saturated cpu_cores=${gitea_cpu_cores} threshold=${HARBOR_110_MAX_GITEA_CPU_CORES}" + return 1 + fi + fi + return 0 + } + controlled_harbor_repair_once() { if [ "${AWOOOI_CD_HARBOR_CONTROLLED_REPAIR:-1}" != "1" ]; then echo "harbor_controlled_repair_skipped=disabled" @@ -1085,6 +1117,11 @@ jobs: attempt=1 repair_attempted=0 + if ! preflight_110_capacity; then + echo "NEXT_ACTION wait_for_110_load_to_normalize_then_rerun_harbor_watchdog_check" + echo "NEXT_ACTION if_110_load_stays_high_use_local_console: sudo /usr/local/bin/recover-110-control-path-and-harbor-local.sh --check" + exit 1 + fi while [ "${attempt}" -le "${LOGIN_ATTEMPTS}" ]; do registry_status="$(probe_registry_v2)" if [ -z "${registry_status}" ]; then diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 4c32c9c21..b5d7b20ab 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-07-01 — 09:33 110 Harbor / Gitea pressure fail-fast gate + +**照主線修正的問題**： +- 188 已無 active `awoooi-cd-*` job container，node exporter 回 load 正常；110 仍高壓，`awoooi_host_load5_per_core{host="110"}` 讀回約 `1.96`，`docker_container_cpu_cores{container_name="gitea",host="110"}` 約 `3.4019`，Actions active container / process 均為 `0`。 +- 110 public Gitea health `https://gitea.wooo.work/api/healthz` 為 `200`，但 `https://registry.wooo.work/v2/`、`https://harbor.wooo.work/api/v2.0/health` 與 `http://192.168.0.110:5000/v2/` 仍為 `502`，SSH `wooo@192.168.0.110` 仍 timeout。 +- `.gitea/workflows/cd.yaml` 的 Harbor login 預設 retry 從 `12 x 10s` 收斂為 `3 x 5s`，並在 login 前讀 `http://192.168.0.110:9100/metrics`；若 110 `load5/core` 或 Gitea CPU 已超標，直接輸出 `BLOCKER harbor_110_host_pressure_high` / `BLOCKER harbor_110_gitea_cpu_saturated`，不再反覆打 Harbor 502。 +- 此變更不放行失敗、不跳過 Harbor 驗證；它把重試壓力改為 fail-fast blocker，下一步仍是 110 local console / restored SSH 執行 `recover-110-control-path-and-harbor-local.sh --check`。 + +**驗證**： +- `python3.11 -m pytest ops/runner/test_cd_controlled_runtime_profile.py ops/runner/test_read_public_gitea_actions_queue.py -q` 通過。 +- `bash -n ops/runner/install-awoooi-non110-runner-user-service.sh ops/runner/check-awoooi-non110-runner-readiness.sh scripts/ci/wait-host-web-build-pressure.sh` 通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .` 通過。 +- `git diff --check` 通過。 + +**邊界**：只改 CD fail-fast source guard / tests / LOGBOOK；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未讀 `.runner` 內容；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未重啟主機、未 restart Docker / Nginx / K3s / DB / firewall。 + ## 2026-07-01 — 09:24 Harbor repair waiting truth / public route blocker **照主線修正的問題**： diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index feb7c637b..513e65b94 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -76,9 +76,20 @@ def test_harbor_login_has_public_route_retry_and_safe_secret_transport() -> None block = text.split("- name: Login to Harbor", 1)[1] block = block.split("- name: Wait for Host Web Build Pressure", 1)[0] - assert 'LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-12}"' in block - assert 'LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-10}"' in block + assert 'LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-3}"' in block + assert 'LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-5}"' in block assert 'WATCHDOG="/usr/local/bin/harbor-watchdog.sh"' in block + assert ( + 'HARBOR_110_NODE_EXPORTER_URL="${HARBOR_110_NODE_EXPORTER_URL:-http://192.168.0.110:9100/metrics}"' + in block + ) + assert 'HARBOR_110_MAX_LOAD5_PER_CORE="${HARBOR_110_MAX_LOAD5_PER_CORE:-1.25}"' in block + assert 'HARBOR_110_MAX_GITEA_CPU_CORES="${HARBOR_110_MAX_GITEA_CPU_CORES:-3.0}"' in block + assert "preflight_110_capacity()" in block + assert "harbor_110_load5_per_core=" in block + assert "BLOCKER harbor_110_host_pressure_high" in block + assert "harbor_110_gitea_cpu_cores=" in block + assert "BLOCKER harbor_110_gitea_cpu_saturated" in block assert "host_has_110_ip()" in block assert 'grep -q " 192.168.0.110/"' in block assert 'AWOOOI_CD_HARBOR_CONTROLLED_REPAIR:-1' in block