From e1ac008bac35c4de95b81d6b63e123a517c23c42 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 17:09:02 +0800 Subject: [PATCH] fix(cd): retry transient harbor image pushes --- .gitea/workflows/cd.yaml | 66 +++++++++++++++++-- .../test_cd_controlled_runtime_profile.py | 16 +++-- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 4add81dda..a1e7b9572 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -1577,6 +1577,35 @@ jobs: return "$rc" } + run_docker_push_step() { + local label="$1" + local timeout_seconds="$2" + shift 2 + local max_attempts="${CD_DOCKER_PUSH_ATTEMPTS:-3}" + local sleep_seconds="${CD_DOCKER_PUSH_RETRY_SLEEP_SECONDS:-15}" + local attempt=1 + local rc=1 + while [ "${attempt}" -le "${max_attempts}" ]; do + echo "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}" + set +e + run_docker_step "${label}" "${timeout_seconds}" "$@" + rc=$? + set -e + if [ "${rc}" -eq 0 ]; then + return 0 + fi + if [ "${attempt}" -ge "${max_attempts}" ]; then + echo "BLOCKER cd_docker_push_failed label=${label} attempts=${max_attempts}" + echo "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd" + return "${rc}" + fi + echo "cd_docker_push_retry_sleep=${label}:${sleep_seconds}" + sleep "${sleep_seconds}" + attempt=$((attempt + 1)) + done + return "${rc}" + } + run_docker_step api_build "${DOCKER_API_BUILD_TIMEOUT_SECONDS:-1200}" \ docker build -f apps/api/Dockerfile \ --build-arg BUILDKIT_INLINE_CACHE=1 \ @@ -1585,9 +1614,9 @@ jobs: -t ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} \ -t ${{ env.HARBOR }}/awoooi/api:latest \ . - run_docker_step api_push_sha "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \ + run_docker_push_step api_push_sha "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \ docker push ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} - run_docker_step api_push_latest "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \ + run_docker_push_step api_push_latest "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \ docker push ${{ env.HARBOR }}/awoooi/api:latest # 2026-03-31 ogt: 移除中間通知，減少訊息雜訊 @@ -1623,6 +1652,35 @@ jobs: return "$rc" } + run_docker_push_step() { + local label="$1" + local timeout_seconds="$2" + shift 2 + local max_attempts="${CD_DOCKER_PUSH_ATTEMPTS:-3}" + local sleep_seconds="${CD_DOCKER_PUSH_RETRY_SLEEP_SECONDS:-15}" + local attempt=1 + local rc=1 + while [ "${attempt}" -le "${max_attempts}" ]; do + echo "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}" + set +e + run_docker_step "${label}" "${timeout_seconds}" "$@" + rc=$? + set -e + if [ "${rc}" -eq 0 ]; then + return 0 + fi + if [ "${attempt}" -ge "${max_attempts}" ]; then + echo "BLOCKER cd_docker_push_failed label=${label} attempts=${max_attempts}" + echo "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd" + return "${rc}" + fi + echo "cd_docker_push_retry_sleep=${label}:${sleep_seconds}" + sleep "${sleep_seconds}" + attempt=$((attempt + 1)) + done + return "${rc}" + } + # 2026-07-03 Codex: run #4610 proved the Harbor DockerHub proxy # cache path returned 401 for node:20-alpine. Keep the public tag # until proxy-cache pull auth has a deploy-log receipt. Pull it with @@ -1670,9 +1728,9 @@ jobs: -t ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} \ -t ${{ env.HARBOR }}/awoooi/web:latest \ . - run_docker_step web_push_sha "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \ + run_docker_push_step web_push_sha "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \ docker push ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} - run_docker_step web_push_latest "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \ + run_docker_push_step web_push_latest "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \ docker push ${{ env.HARBOR }}/awoooi/web:latest - name: Release Docker Build Lock diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index e8ab09eb5..54b06e3d6 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -4,7 +4,6 @@ from __future__ import annotations import re from pathlib import Path - ROOT = Path(__file__).resolve().parents[2] CD_WORKFLOW = ROOT / ".gitea" / "workflows" / "cd.yaml" HARBOR_110_REPAIR_WORKFLOW = ( @@ -409,15 +408,15 @@ def test_cd_docker_build_and_push_steps_are_bounded_and_classified() -> None: expected = { api_block: [ "run_docker_step api_build", - "run_docker_step api_push_sha", - "run_docker_step api_push_latest", + "run_docker_push_step api_push_sha", + "run_docker_push_step api_push_latest", "DOCKER_API_BUILD_TIMEOUT_SECONDS:-1200", "DOCKER_API_PUSH_TIMEOUT_SECONDS:-600", ], web_block: [ "run_docker_step web_build", - "run_docker_step web_push_sha", - "run_docker_step web_push_latest", + "run_docker_push_step web_push_sha", + "run_docker_push_step web_push_latest", "DOCKER_WEB_BUILD_TIMEOUT_SECONDS:-1500", "DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600", ], @@ -427,10 +426,17 @@ def test_cd_docker_build_and_push_steps_are_bounded_and_classified() -> None: assert "cd_docker_step_start=${label}" in block assert "cd_docker_step_rc=${label}:${rc}" in block assert "BLOCKER cd_docker_step_timeout label=${label}" in block + assert 'CD_DOCKER_PUSH_ATTEMPTS:-3' in block + assert "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}" in block + assert "BLOCKER cd_docker_push_failed label=${label}" in block assert ( "NEXT_ACTION inspect_cd_runner_docker_build_push_or_registry_route_then_rerun_cd" in block ) + assert ( + "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd" + in block + ) for marker in markers: assert marker in block