fix(cd): retry transient harbor image pushes

.gitea/workflows/cd.yaml

@@ -1577,6 +1577,35 @@ jobs:
             return "$rc"
           }
 
+          run_docker_push_step() {
+            local label="$1"
+            local timeout_seconds="$2"
+            shift 2
+            local max_attempts="${CD_DOCKER_PUSH_ATTEMPTS:-3}"
+            local sleep_seconds="${CD_DOCKER_PUSH_RETRY_SLEEP_SECONDS:-15}"
+            local attempt=1
+            local rc=1
+            while [ "${attempt}" -le "${max_attempts}" ]; do
+              echo "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}"
+              set +e
+              run_docker_step "${label}" "${timeout_seconds}" "$@"
+              rc=$?
+              set -e
+              if [ "${rc}" -eq 0 ]; then
+                return 0
+              fi
+              if [ "${attempt}" -ge "${max_attempts}" ]; then
+                echo "BLOCKER cd_docker_push_failed label=${label} attempts=${max_attempts}"
+                echo "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd"
+                return "${rc}"
+              fi
+              echo "cd_docker_push_retry_sleep=${label}:${sleep_seconds}"
+              sleep "${sleep_seconds}"
+              attempt=$((attempt + 1))
+            done
+            return "${rc}"
+          }
+
           run_docker_step api_build "${DOCKER_API_BUILD_TIMEOUT_SECONDS:-1200}" \
             docker build -f apps/api/Dockerfile \
             --build-arg BUILDKIT_INLINE_CACHE=1 \
@@ -1585,9 +1614,9 @@ jobs:
             -t ${{ env.HARBOR }}/awoooi/api:${{ github.sha }} \
             -t ${{ env.HARBOR }}/awoooi/api:latest \
             .
-          run_docker_step api_push_sha "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \
+          run_docker_push_step api_push_sha "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \
             docker push ${{ env.HARBOR }}/awoooi/api:${{ github.sha }}
-          run_docker_step api_push_latest "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \
+          run_docker_push_step api_push_latest "${DOCKER_API_PUSH_TIMEOUT_SECONDS:-600}" \
             docker push ${{ env.HARBOR }}/awoooi/api:latest
 
       # 2026-03-31 ogt: 移除中間通知，減少訊息雜訊
@@ -1623,6 +1652,35 @@ jobs:
             return "$rc"
           }
 
+          run_docker_push_step() {
+            local label="$1"
+            local timeout_seconds="$2"
+            shift 2
+            local max_attempts="${CD_DOCKER_PUSH_ATTEMPTS:-3}"
+            local sleep_seconds="${CD_DOCKER_PUSH_RETRY_SLEEP_SECONDS:-15}"
+            local attempt=1
+            local rc=1
+            while [ "${attempt}" -le "${max_attempts}" ]; do
+              echo "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}"
+              set +e
+              run_docker_step "${label}" "${timeout_seconds}" "$@"
+              rc=$?
+              set -e
+              if [ "${rc}" -eq 0 ]; then
+                return 0
+              fi
+              if [ "${attempt}" -ge "${max_attempts}" ]; then
+                echo "BLOCKER cd_docker_push_failed label=${label} attempts=${max_attempts}"
+                echo "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd"
+                return "${rc}"
+              fi
+              echo "cd_docker_push_retry_sleep=${label}:${sleep_seconds}"
+              sleep "${sleep_seconds}"
+              attempt=$((attempt + 1))
+            done
+            return "${rc}"
+          }
+
           # 2026-07-03 Codex: run #4610 proved the Harbor DockerHub proxy
           # cache path returned 401 for node:20-alpine. Keep the public tag
           # until proxy-cache pull auth has a deploy-log receipt. Pull it with
@@ -1670,9 +1728,9 @@ jobs:
             -t ${{ env.HARBOR }}/awoooi/web:${{ github.sha }} \
             -t ${{ env.HARBOR }}/awoooi/web:latest \
             .
-          run_docker_step web_push_sha "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \
+          run_docker_push_step web_push_sha "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \
             docker push ${{ env.HARBOR }}/awoooi/web:${{ github.sha }}
-          run_docker_step web_push_latest "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \
+          run_docker_push_step web_push_latest "${DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600}" \
             docker push ${{ env.HARBOR }}/awoooi/web:latest
 
       - name: Release Docker Build Lock

ops/runner/test_cd_controlled_runtime_profile.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 import re
 from pathlib import Path
 
-
 ROOT = Path(__file__).resolve().parents[2]
 CD_WORKFLOW = ROOT / ".gitea" / "workflows" / "cd.yaml"
 HARBOR_110_REPAIR_WORKFLOW = (
@@ -409,15 +408,15 @@ def test_cd_docker_build_and_push_steps_are_bounded_and_classified() -> None:
     expected = {
         api_block: [
             "run_docker_step api_build",
-            "run_docker_step api_push_sha",
-            "run_docker_step api_push_latest",
+            "run_docker_push_step api_push_sha",
+            "run_docker_push_step api_push_latest",
             "DOCKER_API_BUILD_TIMEOUT_SECONDS:-1200",
             "DOCKER_API_PUSH_TIMEOUT_SECONDS:-600",
         ],
         web_block: [
             "run_docker_step web_build",
-            "run_docker_step web_push_sha",
-            "run_docker_step web_push_latest",
+            "run_docker_push_step web_push_sha",
+            "run_docker_push_step web_push_latest",
             "DOCKER_WEB_BUILD_TIMEOUT_SECONDS:-1500",
             "DOCKER_WEB_PUSH_TIMEOUT_SECONDS:-600",
         ],
@@ -427,10 +426,17 @@ def test_cd_docker_build_and_push_steps_are_bounded_and_classified() -> None:
         assert "cd_docker_step_start=${label}" in block
         assert "cd_docker_step_rc=${label}:${rc}" in block
         assert "BLOCKER cd_docker_step_timeout label=${label}" in block
+        assert 'CD_DOCKER_PUSH_ATTEMPTS:-3' in block
+        assert "cd_docker_push_attempt=${label}:${attempt}/${max_attempts}" in block
+        assert "BLOCKER cd_docker_push_failed label=${label}" in block
         assert (
             "NEXT_ACTION inspect_cd_runner_docker_build_push_or_registry_route_then_rerun_cd"
             in block
         )
+        assert (
+            "NEXT_ACTION inspect_harbor_registry_push_500_or_storage_pressure_then_rerun_cd"
+            in block
+        )
         for marker in markers:
             assert marker in block