wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(ops): bound CD DB probes and restore SigNoz collector

Browse Source
This commit is contained in:
ogt
2026-07-15 02:29:51 +08:00
parent 27736add14
commit bfb2d2d040
11 changed files with 442 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
.gitea/workflows/cd.yaml
View File

@@ -3868,6 +3868,25 @@ jobs:
API_DESIRED_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \
-n awoooi-prod -o jsonpath='{.spec.replicas}')
API_HTTP_PROBE_SAFETY_RESERVE=4
API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE=$((
API_DB_AVAILABLE_CONNECTION_HEADROOM - API_HTTP_PROBE_SAFETY_RESERVE
))
if [ "$API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE" -lt 0 ]; then
API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE=0
fi
API_HTTP_CONCURRENCY_WORKERS=$API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE
if [ "$API_HTTP_CONCURRENCY_WORKERS" -gt 4 ]; then
API_HTTP_CONCURRENCY_WORKERS=4
fi
API_HTTP_CONCURRENCY_VERIFIED=false
API_HTTP_PROBE_SKIP_REASON=""
if [ "$CONNECTION_BUDGET_VERIFIED" != "true" ]; then
API_HTTP_PROBE_SKIP_REASON=connection_budget_not_verified
elif [ "$API_HTTP_CONCURRENCY_WORKERS" -lt 1 ]; then
API_HTTP_PROBE_SKIP_REASON=connection_headroom_reserved_for_runtime
else
export API_HTTP_CONCURRENCY_WORKERS
API_HTTP_CONCURRENCY_VERIFIED=$(python3 - <<'PY'
import concurrent.futures
import os
@@ -3904,11 +3923,14 @@ jobs:
"API_PUBLIC_READINESS_DELAY_SECONDS", 5, 0, 30
)
concurrency_attempts = bounded_int(
"API_HTTP_CONCURRENCY_ATTEMPTS", 4, 1, 8
"API_HTTP_CONCURRENCY_ATTEMPTS", 1, 1, 1
)
concurrency_delay_seconds = bounded_int(
"API_HTTP_CONCURRENCY_DELAY_SECONDS", 5, 0, 30
)
probe_worker_count = bounded_int(
"API_HTTP_CONCURRENCY_WORKERS", 4, 1, 4
)
def fetch_status(url: str, timeout: int) -> tuple[int | None, str]:
@@ -3955,18 +3977,18 @@ jobs:
def fetch(url: str) -> int:
status, error_type = fetch_status(url, timeout=20)
status, error_type = fetch_status(url, timeout=5)
if status is None:
raise RuntimeError(error_type)
return status
last_result = "unavailable"
concurrency_urls = (db_probe_url,) * 8
concurrency_urls = (db_probe_url,) * probe_worker_count
for attempt in range(1, concurrency_attempts + 1):
try:
with concurrent.futures.ThreadPoolExecutor(
max_workers=8
max_workers=probe_worker_count
) as executor:
statuses = list(executor.map(fetch, concurrency_urls))
except Exception as exc:
@@ -4000,6 +4022,10 @@ jobs:
print("false")
PY
)
fi
if [ -n "$API_HTTP_PROBE_SKIP_REASON" ]; then
echo "API database concurrency probe skipped: ${API_HTTP_PROBE_SKIP_REASON} headroom=${API_DB_AVAILABLE_CONNECTION_HEADROOM} reserve=${API_HTTP_PROBE_SAFETY_RESERVE}" >&2
fi
if [ "$API_HTTP_CONCURRENCY_VERIFIED" != "true" ]; then
CONNECTION_BUDGET_VERIFIED=false
fi
@@ -4100,6 +4126,9 @@ jobs:
--from-literal=broker_sequence_privilege_gap_count="$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \
--from-literal=representative_db_preflights_verified="$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" \
--from-literal=api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED" \
--from-literal=api_http_concurrency_probe_workers="$API_HTTP_CONCURRENCY_WORKERS" \
--from-literal=api_http_concurrency_probe_safety_reserve="$API_HTTP_PROBE_SAFETY_RESERVE" \
--from-literal=api_http_concurrency_probe_skip_reason="$API_HTTP_PROBE_SKIP_REASON" \
--from-literal=api_http_probe_restart_free="$API_PROBE_RESTART_FREE" \
--from-literal=api_restart_count_before="$API_RESTART_COUNT_BEFORE" \
--from-literal=api_restart_count_after="$API_RESTART_COUNT_AFTER" \