fix(recovery): close cold-start runtime blockers

This commit is contained in:
ogt
2026-07-11 15:08:27 +08:00
parent 7cea825d92
commit 276410fbb6
38 changed files with 1494 additions and 145 deletions

View File

@@ -8,6 +8,7 @@ set -uo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-6}"
SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-25}"
SSH_TOTAL_TIMEOUT_SECONDS="${SSH_TOTAL_TIMEOUT_SECONDS:-$((SSH_CONNECT_TIMEOUT + SSH_COMMAND_TIMEOUT_SECONDS + 5))}"
ROUTE_RETRY_ATTEMPTS="${ROUTE_RETRY_ATTEMPTS:-3}"
ROUTE_RETRY_DELAY_SECONDS="${ROUTE_RETRY_DELAY_SECONDS:-2}"
STOCK_FRESHNESS_RETRY_ATTEMPTS="${STOCK_FRESHNESS_RETRY_ATTEMPTS:-6}"
@@ -208,16 +209,28 @@ is_local_target() {
grep -Fxq "$host" <<<"$ips"
}
run_with_timeout() {
local timeout_seconds="$1"
shift
if command -v timeout >/dev/null 2>&1; then
timeout --foreground --kill-after=3s "${timeout_seconds}s" "$@"
elif command -v gtimeout >/dev/null 2>&1; then
gtimeout --foreground --kill-after=3s "${timeout_seconds}s" "$@"
else
"$@"
fi
}
ssh_read() {
local user_host="$1"
local command="$2"
local quoted_command=""
if is_local_target "$user_host"; then
bash -lc "$command"
run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" bash -lc "$command"
return $?
fi
printf -v quoted_command '%q' "$command"
ssh \
local ssh_command=(ssh \
-o BatchMode=yes \
-o ConnectTimeout="$SSH_CONNECT_TIMEOUT" \
-o ConnectionAttempts=1 \
@@ -226,7 +239,8 @@ ssh_read() {
-o PreferredAuthentications=publickey \
-o NumberOfPasswordPrompts=0 \
"$user_host" \
"if command -v timeout >/dev/null 2>&1; then timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}; else bash -lc ${quoted_command}; fi"
"if command -v timeout >/dev/null 2>&1; then timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}; else bash -lc ${quoted_command}; fi")
run_with_timeout "$SSH_TOTAL_TIMEOUT_SECONDS" "${ssh_command[@]}"
}
service_route_recovered() {