diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index e42797805..346bedde1 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -955,16 +955,76 @@ jobs: HARBOR_REGISTRY="${{ env.HARBOR }}" LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-12}" LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-10}" + WATCHDOG="/usr/local/bin/harbor-watchdog.sh" + + probe_registry_v2() { + curl --silent --show-error \ + --output /dev/null \ + --write-out "%{http_code}" \ + --max-time 10 \ + "https://${HARBOR_REGISTRY}/v2/" || true + } + + host_has_110_ip() { + command -v ip >/dev/null 2>&1 || return 1 + ip -o -4 addr show 2>/dev/null | grep -q " 192.168.0.110/" + } + + run_watchdog() { + if [ "$(id -u)" = "0" ]; then + "${WATCHDOG}" "$@" + else + sudo -n "${WATCHDOG}" "$@" + fi + } + + controlled_harbor_repair_once() { + if [ "${AWOOOI_CD_HARBOR_CONTROLLED_REPAIR:-1}" != "1" ]; then + echo "harbor_controlled_repair_skipped=disabled" + return 1 + fi + if ! host_has_110_ip; then + echo "harbor_controlled_repair_skipped=not_110_host" + return 1 + fi + if [ ! -x "${WATCHDOG}" ]; then + echo "harbor_controlled_repair_skipped=watchdog_missing" + return 1 + fi + + echo "harbor_controlled_repair_check_start=1" + set +e + check_output="$(run_watchdog --check 2>&1)" + check_rc=$? + set -e + printf '%s\n' "${check_output}" | sed -n '1,80p' + echo "harbor_controlled_repair_check_rc=${check_rc}" + if [ "${check_rc}" -ne 0 ]; then + return 1 + fi + if ! printf '%s\n' "${check_output}" | grep -q "AWOOOI_HARBOR_WATCHDOG_CHECK"; then + echo "harbor_controlled_repair_blocked=missing_watchdog_check_marker" + return 1 + fi + if printf '%s\n' "${check_output}" | grep -q "harbor_ready=true"; then + echo "harbor_controlled_repair_skipped=already_ready" + return 0 + fi + + echo "harbor_controlled_repair_once_start=1" + set +e + repair_output="$(run_watchdog --repair-once 2>&1)" + repair_rc=$? + set -e + printf '%s\n' "${repair_output}" | sed -n '1,140p' + echo "harbor_controlled_repair_once_rc=${repair_rc}" + [ "${repair_rc}" -eq 0 ] + } attempt=1 + repair_attempted=0 while [ "${attempt}" -le "${LOGIN_ATTEMPTS}" ]; do - registry_status="$( - curl --silent --show-error \ - --output /dev/null \ - --write-out "%{http_code}" \ - --max-time 10 \ - "https://${HARBOR_REGISTRY}/v2/" || true - )" + registry_status="$(probe_registry_v2)" if [ -z "${registry_status}" ]; then registry_status="000" fi @@ -980,6 +1040,17 @@ jobs: echo "harbor_login_attempt=${attempt} docker_login_failed" else echo "harbor_login_attempt=${attempt} registry_v2_status=${registry_status}" + if [ "${repair_attempted}" = "0" ]; then + repair_attempted=1 + if controlled_harbor_repair_once; then + registry_status="$(probe_registry_v2)" + if [ -z "${registry_status}" ]; then + registry_status="000" + fi + echo "harbor_controlled_repair_public_registry_v2_status=${registry_status}" + continue + fi + fi fi if [ "${attempt}" -ge "${LOGIN_ATTEMPTS}" ]; then diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 118317054..66d8552f9 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-06-30 — 20:22 Harbor gate CD self-heal 接入 + +**照主線修正的問題**: +- Public Gitea CD `#4051` 已失敗,classifier=`harbor_registry_public_route_unavailable`、`registry_v2_status=502`、Harbor login attempt `12/12`;tests job 已成功,卡點是 registry runtime,不是 API 程式測試。 +- `.gitea/workflows/cd.yaml` 的 `Login to Harbor` gate 新增 controlled self-heal:當 public `/v2/` 非 `200/401` 時,只在 runner 本機有 `192.168.0.110` 且 `/usr/local/bin/harbor-watchdog.sh` 可執行時,先跑 `--check`,只有 check 證明不健康才跑一次 `--repair-once`,再重新 probe public registry `/v2/`。 +- 非 110 host、watchdog 不存在、非互動 sudo 不可用、check marker 不存在時一律 fail-closed,保留原本 `BLOCKER harbor_registry_public_route_unavailable` 與 110 local console / restored SSH next action。 +- `ops/runner/test_cd_controlled_runtime_profile.py` 鎖住 self-heal 邊界:110 IP target、`AWOOOI_CD_HARBOR_CONTROLLED_REPAIR`、`sudo -n`、watchdog check/repair-once、不得 `systemctl restart docker` 或 reboot。 + +**驗證**: +- `bash -n .gitea/workflows/cd.yaml`、`git diff --check` 通過。 +- `py_compile`、`ruff check` 通過。 +- focused pytest:`32 passed`。 +- `guard-gitea-runner-pressure.py`:`auto_branch_events_on_110=0`、`generic_runner_labels=0`;`check-gitea-step-env-secrets.js` 通過。 + +**邊界**:未讀 secret / token / `.env` / raw sessions / SQLite / auth;未使用 GitHub / `gh` / GitHub API;未 workflow_dispatch;未在目前 Codex session 直接 SSH / Docker / Nginx / K3s / DB / firewall 寫入;CD self-heal 僅允許 110-local watchdog bounded repair-once,禁止 Docker daemon restart、host reboot、node drain、provider switch 或 force push。 + ## 2026-06-30 — 20:21 Harbor registry controlled recovery receipt validator **照主線修正的問題**: diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 66932f111..144589cb9 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -74,6 +74,14 @@ def test_harbor_login_has_public_route_retry_and_safe_secret_transport() -> None assert 'LOGIN_ATTEMPTS="${HARBOR_LOGIN_ATTEMPTS:-12}"' in block assert 'LOGIN_SLEEP_SECONDS="${HARBOR_LOGIN_SLEEP_SECONDS:-10}"' in block + assert 'WATCHDOG="/usr/local/bin/harbor-watchdog.sh"' in block + assert "host_has_110_ip()" in block + assert 'grep -q " 192.168.0.110/"' in block + assert 'AWOOOI_CD_HARBOR_CONTROLLED_REPAIR:-1' in block + assert "harbor_controlled_repair_check_start=1" in block + assert "harbor_controlled_repair_once_start=1" in block + assert "harbor_controlled_repair_public_registry_v2_status=" in block + assert "sudo -n" in block assert '"https://${HARBOR_REGISTRY}/v2/"' in block assert 'registry_status="000"' in block assert '[ "${registry_status}" = "200" ] || [ "${registry_status}" = "401" ]' in block @@ -86,6 +94,8 @@ def test_harbor_login_has_public_route_retry_and_safe_secret_transport() -> None "sudo /usr/local/bin/recover-110-control-path-and-harbor-local.sh --apply-all" in block ) + assert "systemctl restart docker" not in block + assert "\nreboot" not in block assert "sleep \"${LOGIN_SLEEP_SECONDS}\"" in block assert "${HARBOR_PASSWORD}" in block assert "--password " not in block