fix(ci): fail fast on 110 harbor pressure [skip ci]
This commit is contained in:
Your Name
@@ -1015,9 +1015,12 @@ jobs:
|
||||
HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }}
|
||||
run: |
|
||||
HARBOR_REGISTRY="${{ env.HARBOR }}"
|
||||
LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-12}"
|
||||
LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-10}"
|
||||
LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-3}"
|
||||
LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-5}"
|
||||
WATCHDOG="/usr/local/bin/harbor-watchdog.sh"
|
||||
HARBOR_110_NODE_EXPORTER_URL="${HARBOR_110_NODE_EXPORTER_URL:-http://192.168.0.110:9100/metrics}"
|
||||
HARBOR_110_MAX_LOAD5_PER_CORE="${HARBOR_110_MAX_LOAD5_PER_CORE:-1.25}"
|
||||
HARBOR_110_MAX_GITEA_CPU_CORES="${HARBOR_110_MAX_GITEA_CPU_CORES:-3.0}"
|
||||
|
||||
probe_registry_v2() {
|
||||
curl --silent --show-error \
|
||||
@@ -1040,6 +1043,35 @@ jobs:
|
||||
fi
|
||||
}
|
||||
|
||||
greater_than() {
|
||||
awk -v left="$1" -v right="$2" 'BEGIN { exit !(left > right) }'
|
||||
}
|
||||
|
||||
preflight_110_capacity() {
|
||||
metrics="$(curl --silent --show-error --max-time 5 "${HARBOR_110_NODE_EXPORTER_URL}" || true)"
|
||||
if [ -z "${metrics}" ]; then
|
||||
echo "harbor_110_capacity_metrics_unavailable=1"
|
||||
return 0
|
||||
fi
|
||||
load5_per_core="$(printf '%s\n' "${metrics}" | awk '$1 ~ /^awoooi_host_load5_per_core\\{/ && $0 ~ /host=\"110\"/ { value = $NF } END { print value }')"
|
||||
gitea_cpu_cores="$(printf '%s\n' "${metrics}" | awk '$1 ~ /^docker_container_cpu_cores\\{/ && $0 ~ /container_name=\"gitea\"/ { value = $NF } END { print value }')"
|
||||
if [ -n "${load5_per_core}" ]; then
|
||||
echo "harbor_110_load5_per_core=${load5_per_core}"
|
||||
if greater_than "${load5_per_core}" "${HARBOR_110_MAX_LOAD5_PER_CORE}"; then
|
||||
echo "BLOCKER harbor_110_host_pressure_high load5_per_core=${load5_per_core} threshold=${HARBOR_110_MAX_LOAD5_PER_CORE}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if [ -n "${gitea_cpu_cores}" ]; then
|
||||
echo "harbor_110_gitea_cpu_cores=${gitea_cpu_cores}"
|
||||
if greater_than "${gitea_cpu_cores}" "${HARBOR_110_MAX_GITEA_CPU_CORES}"; then
|
||||
echo "BLOCKER harbor_110_gitea_cpu_saturated cpu_cores=${gitea_cpu_cores} threshold=${HARBOR_110_MAX_GITEA_CPU_CORES}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
controlled_harbor_repair_once() {
|
||||
if [ "${AWOOOI_CD_HARBOR_CONTROLLED_REPAIR:-1}" != "1" ]; then
|
||||
echo "harbor_controlled_repair_skipped=disabled"
|
||||
@@ -1085,6 +1117,11 @@ jobs:
|
||||
|
||||
attempt=1
|
||||
repair_attempted=0
|
||||
if ! preflight_110_capacity; then
|
||||
echo "NEXT_ACTION wait_for_110_load_to_normalize_then_rerun_harbor_watchdog_check"
|
||||
echo "NEXT_ACTION if_110_load_stays_high_use_local_console: sudo /usr/local/bin/recover-110-control-path-and-harbor-local.sh --check"
|
||||
exit 1
|
||||
fi
|
||||
while [ "${attempt}" -le "${LOGIN_ATTEMPTS}" ]; do
|
||||
registry_status="$(probe_registry_v2)"
|
||||
if [ -z "${registry_status}" ]; then
|
||||
|
||||
@@ -1,3 +1,19 @@
|
||||
## 2026-07-01 — 09:33 110 Harbor / Gitea pressure fail-fast gate
|
||||
|
||||
**照主線修正的問題**：
|
||||
- 188 已無 active `awoooi-cd-*` job container，node exporter 回 load 正常；110 仍高壓，`awoooi_host_load5_per_core{host="110"}` 讀回約 `1.96`，`docker_container_cpu_cores{container_name="gitea",host="110"}` 約 `3.4019`，Actions active container / process 均為 `0`。
|
||||
- 110 public Gitea health `https://gitea.wooo.work/api/healthz` 為 `200`，但 `https://registry.wooo.work/v2/`、`https://harbor.wooo.work/api/v2.0/health` 與 `http://192.168.0.110:5000/v2/` 仍為 `502`，SSH `wooo@192.168.0.110` 仍 timeout。
|
||||
- `.gitea/workflows/cd.yaml` 的 Harbor login 預設 retry 從 `12 x 10s` 收斂為 `3 x 5s`，並在 login 前讀 `http://192.168.0.110:9100/metrics`；若 110 `load5/core` 或 Gitea CPU 已超標，直接輸出 `BLOCKER harbor_110_host_pressure_high` / `BLOCKER harbor_110_gitea_cpu_saturated`，不再反覆打 Harbor 502。
|
||||
- 此變更不放行失敗、不跳過 Harbor 驗證；它把重試壓力改為 fail-fast blocker，下一步仍是 110 local console / restored SSH 執行 `recover-110-control-path-and-harbor-local.sh --check`。
|
||||
|
||||
**驗證**：
|
||||
- `python3.11 -m pytest ops/runner/test_cd_controlled_runtime_profile.py ops/runner/test_read_public_gitea_actions_queue.py -q` 通過。
|
||||
- `bash -n ops/runner/install-awoooi-non110-runner-user-service.sh ops/runner/check-awoooi-non110-runner-readiness.sh scripts/ci/wait-host-web-build-pressure.sh` 通過。
|
||||
- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .` 通過。
|
||||
- `git diff --check` 通過。
|
||||
|
||||
**邊界**：只改 CD fail-fast source guard / tests / LOGBOOK；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未讀 `.runner` 內容；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未重啟主機、未 restart Docker / Nginx / K3s / DB / firewall。
|
||||
|
||||
## 2026-07-01 — 09:24 Harbor repair waiting truth / public route blocker
|
||||
|
||||
**照主線修正的問題**：
|
||||
|
||||
@@ -76,9 +76,20 @@ def test_harbor_login_has_public_route_retry_and_safe_secret_transport() -> None
|
||||
block = text.split("- name: Login to Harbor", 1)[1]
|
||||
block = block.split("- name: Wait for Host Web Build Pressure", 1)[0]
|
||||
|
||||
assert 'LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-12}"' in block
|
||||
assert 'LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-10}"' in block
|
||||
assert 'LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-3}"' in block
|
||||
assert 'LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-5}"' in block
|
||||
assert 'WATCHDOG="/usr/local/bin/harbor-watchdog.sh"' in block
|
||||
assert (
|
||||
'HARBOR_110_NODE_EXPORTER_URL="${HARBOR_110_NODE_EXPORTER_URL:-http://192.168.0.110:9100/metrics}"'
|
||||
in block
|
||||
)
|
||||
assert 'HARBOR_110_MAX_LOAD5_PER_CORE="${HARBOR_110_MAX_LOAD5_PER_CORE:-1.25}"' in block
|
||||
assert 'HARBOR_110_MAX_GITEA_CPU_CORES="${HARBOR_110_MAX_GITEA_CPU_CORES:-3.0}"' in block
|
||||
assert "preflight_110_capacity()" in block
|
||||
assert "harbor_110_load5_per_core=" in block
|
||||
assert "BLOCKER harbor_110_host_pressure_high" in block
|
||||
assert "harbor_110_gitea_cpu_cores=" in block
|
||||
assert "BLOCKER harbor_110_gitea_cpu_saturated" in block
|
||||
assert "host_has_110_ip()" in block
|
||||
assert 'grep -q " 192.168.0.110/"' in block
|
||||
assert 'AWOOOI_CD_HARBOR_CONTROLLED_REPAIR:-1' in block
|
||||
|
||||
Reference in New Issue
Block a user