653 lines
24 KiB
Bash
Executable File
653 lines
24 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -uo pipefail
|
|
|
|
# One-entry read-only post-reboot check. This wrapper intentionally delegates
|
|
# deep checks to the existing recovery scripts and does not restart, patch,
|
|
# delete, import, reload, or write runtime state.
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-6}"
|
|
ROUTE_RETRY_ATTEMPTS="${ROUTE_RETRY_ATTEMPTS:-3}"
|
|
ROUTE_RETRY_DELAY_SECONDS="${ROUTE_RETRY_DELAY_SECONDS:-2}"
|
|
STOCK_FRESHNESS_RETRY_ATTEMPTS="${STOCK_FRESHNESS_RETRY_ATTEMPTS:-6}"
|
|
STOCK_FRESHNESS_RETRY_DELAY_SECONDS="${STOCK_FRESHNESS_RETRY_DELAY_SECONDS:-5}"
|
|
RUN_COLD_START=1
|
|
RUN_MOMO=1
|
|
RUN_STOCK=1
|
|
RUN_BACKUP=1
|
|
RUN_ROUTES=1
|
|
RUN_CPU=1
|
|
NO_COLOR_FLAG=0
|
|
COLD_START_PENDING_BLOCKERS=0
|
|
COLD_START_BLOCKED_SUMMARY=""
|
|
COLD_START_BLOCKED_LINES=""
|
|
ROUTE_SMOKE_BLOCKED=0
|
|
AWOOOI_API_ROUTE_OK=0
|
|
STOCK_EOD_WINDOW_PENDING=0
|
|
STOCK_EOD_CLASSIFICATION="not_evaluated"
|
|
STOCK_EOD_NEXT_ACTION="not_evaluated"
|
|
STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL="19:15"
|
|
|
|
PASS_COUNT=0
|
|
WARN_COUNT=0
|
|
BLOCKED_COUNT=0
|
|
SERVICE_WARN_COUNT=0
|
|
BOUNDARY_WARN_COUNT=0
|
|
EVIDENCE_WARN_COUNT=0
|
|
|
|
HOSTS=(
|
|
"192.168.0.110"
|
|
"192.168.0.120"
|
|
"192.168.0.121"
|
|
"192.168.0.188"
|
|
)
|
|
|
|
ROUTES=(
|
|
"https://awoooi.wooo.work/"
|
|
"https://awoooi.wooo.work/api/v1/health"
|
|
"https://awoooi.wooo.work/zh-TW/iwooos"
|
|
"https://vibework.wooo.work/"
|
|
"https://awooogo.wooo.work/"
|
|
"https://2026fifa.wooo.work/"
|
|
"https://agent.wooo.work/"
|
|
"https://mo.wooo.work/"
|
|
"https://mo.wooo.work/health"
|
|
"https://stock.wooo.work/"
|
|
"https://stock.wooo.work/healthz"
|
|
"https://stock.wooo.work/api/healthz"
|
|
"https://bitan.wooo.work/"
|
|
"https://tsenyang.com/"
|
|
"https://www.tsenyang.com/"
|
|
"https://vtuber.wooo.work/"
|
|
"https://gitea.wooo.work/"
|
|
"https://harbor.wooo.work/"
|
|
"https://registry.wooo.work/"
|
|
"https://sentry.wooo.work/"
|
|
"https://signoz.wooo.work/"
|
|
"https://langfuse.wooo.work/"
|
|
"https://aiops.wooo.work/"
|
|
)
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: post-start-quick-check.sh [options]
|
|
|
|
Read-only post-reboot quick check for 110 / 120 / 121 / 188.
|
|
|
|
Options:
|
|
--skip-cold-start Do not run full-stack-cold-start-check.sh.
|
|
--skip-momo Do not run momo-drive-token-source-recovery-preflight.sh.
|
|
--skip-stock Do not query StockPlatform data freshness.
|
|
--skip-backup Do not run /backup/scripts/backup-status.sh on 110.
|
|
--skip-routes Do not curl public route smoke targets.
|
|
--skip-cpu Do not read 110 CPU / process summary.
|
|
--no-color Disable ANSI color.
|
|
-h, --help Show this help.
|
|
|
|
Environment:
|
|
ROUTE_RETRY_ATTEMPTS Public route attempts before blocking. Default: 3.
|
|
ROUTE_RETRY_DELAY_SECONDS Delay between failed public route attempts. Default: 2.
|
|
STOCK_FRESHNESS_RETRY_ATTEMPTS Stock freshness attempts before blocking. Default: 6.
|
|
STOCK_FRESHNESS_RETRY_DELAY_SECONDS Delay between failed Stock freshness attempts. Default: 5.
|
|
|
|
Exit codes:
|
|
0 = no service blockers. Boundary / evidence warnings may still be present.
|
|
1 = service warnings only.
|
|
2 = service blockers observed.
|
|
|
|
This script never reads token content and never writes runtime state.
|
|
USAGE
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--skip-cold-start)
|
|
RUN_COLD_START=0
|
|
;;
|
|
--skip-momo)
|
|
RUN_MOMO=0
|
|
;;
|
|
--skip-stock)
|
|
RUN_STOCK=0
|
|
;;
|
|
--skip-backup)
|
|
RUN_BACKUP=0
|
|
;;
|
|
--skip-routes)
|
|
RUN_ROUTES=0
|
|
;;
|
|
--skip-cpu)
|
|
RUN_CPU=0
|
|
;;
|
|
--no-color)
|
|
NO_COLOR_FLAG=1
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
printf 'Unknown argument: %s\n' "$1" >&2
|
|
usage >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [[ -n "${NO_COLOR:-}" || "$NO_COLOR_FLAG" -eq 1 ]]; then
|
|
RED=""
|
|
GREEN=""
|
|
YELLOW=""
|
|
BLUE=""
|
|
NC=""
|
|
else
|
|
RED=$'\033[0;31m'
|
|
GREEN=$'\033[0;32m'
|
|
YELLOW=$'\033[1;33m'
|
|
BLUE=$'\033[0;34m'
|
|
NC=$'\033[0m'
|
|
fi
|
|
|
|
section() {
|
|
printf '\n%s=== %s ===%s\n' "$BLUE" "$1" "$NC"
|
|
}
|
|
|
|
ok() {
|
|
PASS_COUNT=$((PASS_COUNT + 1))
|
|
printf '%sOK%s %s\n' "$GREEN" "$NC" "$*"
|
|
}
|
|
|
|
warn() {
|
|
WARN_COUNT=$((WARN_COUNT + 1))
|
|
printf '%sWARN%s %s\n' "$YELLOW" "$NC" "$*"
|
|
}
|
|
|
|
service_warn() {
|
|
SERVICE_WARN_COUNT=$((SERVICE_WARN_COUNT + 1))
|
|
warn "$@"
|
|
}
|
|
|
|
boundary_warn() {
|
|
BOUNDARY_WARN_COUNT=$((BOUNDARY_WARN_COUNT + 1))
|
|
warn "$@"
|
|
}
|
|
|
|
evidence_warn() {
|
|
EVIDENCE_WARN_COUNT=$((EVIDENCE_WARN_COUNT + 1))
|
|
warn "$@"
|
|
}
|
|
|
|
blocked() {
|
|
BLOCKED_COUNT=$((BLOCKED_COUNT + 1))
|
|
printf '%sBLOCKED%s %s\n' "$RED" "$NC" "$*"
|
|
}
|
|
|
|
ssh_read() {
|
|
local user_host="$1"
|
|
local command="$2"
|
|
ssh -o BatchMode=yes -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" "$user_host" "$command"
|
|
}
|
|
|
|
run_and_capture() {
|
|
local label="$1"
|
|
shift
|
|
local tmp
|
|
tmp="$(mktemp -t post-start-quick-check.XXXXXX)"
|
|
if "$@" >"$tmp" 2>&1; then
|
|
ok "$label"
|
|
cat "$tmp"
|
|
rm -f "$tmp"
|
|
return 0
|
|
fi
|
|
local rc=$?
|
|
cat "$tmp"
|
|
rm -f "$tmp"
|
|
return "$rc"
|
|
}
|
|
|
|
section "主機 / SSH"
|
|
for host in "${HOSTS[@]}"; do
|
|
if ping -c 1 -W 1 "$host" >/dev/null 2>&1; then
|
|
ok "PING_OK $host"
|
|
else
|
|
blocked "PING_FAIL $host"
|
|
fi
|
|
|
|
if nc -z -w 2 "$host" 22 >/dev/null 2>&1; then
|
|
ok "SSH_PORT_OK $host"
|
|
else
|
|
blocked "SSH_PORT_FAIL $host"
|
|
fi
|
|
done
|
|
|
|
if [[ "$RUN_COLD_START" -eq 1 ]]; then
|
|
section "Cold-start scorecard"
|
|
cold_tmp="$(mktemp -t post-start-cold-start.XXXXXX)"
|
|
cold_rc=0
|
|
if bash "$ROOT_DIR/scripts/reboot-recovery/full-stack-cold-start-check.sh" --monitor-read-only --no-color --watch --interval 1 --max-attempts 1 >"$cold_tmp" 2>&1; then
|
|
cold_rc=0
|
|
else
|
|
cold_rc=$?
|
|
fi
|
|
cat "$cold_tmp"
|
|
cold_summary="$(grep -E 'PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$cold_tmp" | tail -n 1 || true)"
|
|
if [[ -n "$cold_summary" ]]; then
|
|
ok "cold-start summary: $cold_summary"
|
|
cold_warn=0
|
|
cold_blocked=0
|
|
if [[ "$cold_summary" =~ WARN=([0-9]+) ]]; then
|
|
cold_warn="${BASH_REMATCH[1]}"
|
|
fi
|
|
if [[ "$cold_summary" =~ BLOCKED=([0-9]+) ]]; then
|
|
cold_blocked="${BASH_REMATCH[1]}"
|
|
fi
|
|
if [[ "$cold_blocked" -gt 0 ]]; then
|
|
COLD_START_PENDING_BLOCKERS="$cold_blocked"
|
|
COLD_START_BLOCKED_SUMMARY="$cold_summary"
|
|
COLD_START_BLOCKED_LINES="$(grep -E '^BLOCKED ' "$cold_tmp" || true)"
|
|
evidence_warn "cold-start blockers pending wrapper retry classification: $cold_summary"
|
|
elif [[ "$cold_warn" -gt 0 ]]; then
|
|
service_warn "cold-start is warning-only, not blocked: $cold_summary"
|
|
elif [[ "$cold_rc" -eq 0 ]]; then
|
|
ok "cold-start command exited 0"
|
|
else
|
|
evidence_warn "cold-start exited $cold_rc but summary has no blockers: $cold_summary"
|
|
fi
|
|
else
|
|
if [[ "$cold_rc" -eq 0 ]]; then
|
|
service_warn "cold-start summary not found"
|
|
else
|
|
blocked "cold-start command returned $cold_rc without summary"
|
|
fi
|
|
fi
|
|
rm -f "$cold_tmp"
|
|
fi
|
|
|
|
if [[ "$RUN_MOMO" -eq 1 ]]; then
|
|
section "MOMO freshness"
|
|
momo_tmp="$(mktemp -t post-start-momo.XXXXXX)"
|
|
bash "$ROOT_DIR/scripts/reboot-recovery/momo-drive-token-source-recovery-preflight.sh" >"$momo_tmp" 2>&1
|
|
momo_rc=$?
|
|
cat "$momo_tmp"
|
|
momo_summary="$(grep -E 'MOMO_DRIVE_TOKEN_SOURCE_PREFLIGHT PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$momo_tmp" | tail -n 1 || true)"
|
|
case "$momo_rc" in
|
|
0)
|
|
ok "MOMO preflight clean"
|
|
;;
|
|
1)
|
|
if [[ "$momo_summary" =~ BLOCKED=0 ]]; then
|
|
evidence_warn "MOMO preflight has non-service warnings"
|
|
else
|
|
service_warn "MOMO preflight has warnings and no clean summary"
|
|
fi
|
|
;;
|
|
*)
|
|
blocked "MOMO preflight has blockers"
|
|
;;
|
|
esac
|
|
grep -E 'MOMO_DRIVE_TOKEN_SOURCE_PREFLIGHT|MOMO_HEALTH_VERSION|DB_MONTHLY_SYNC|DB_DAILY_FRESHNESS|DB_LATEST_DAILY_IMPORT_JOB' "$momo_tmp" || true
|
|
source_gate_output="$("$ROOT_DIR/scripts/reboot-recovery/momo-source-arrival-gate.py" --preflight-log "$momo_tmp" 2>&1)"
|
|
source_gate_rc=$?
|
|
printf '%s\n' "$source_gate_output"
|
|
if grep -q 'status=blocked_source_absent_fail_closed' <<<"$source_gate_output"; then
|
|
evidence_warn "MOMO source-arrival gate confirms source absent fail-closed"
|
|
elif grep -q 'status=source_arrived_ready_for_safe_import_preflight' <<<"$source_gate_output"; then
|
|
boundary_warn "MOMO source arrived; safe import preflight only, DB/Drive/runtime writes remain unauthorized"
|
|
elif grep -q 'status=freshness_already_green_recheck_cold_start' <<<"$source_gate_output"; then
|
|
ok "MOMO source-arrival gate reports freshness green; rerun post-reboot summary before updating declaration"
|
|
elif [[ "$source_gate_rc" -ne 0 ]]; then
|
|
service_warn "MOMO source-arrival gate did not produce a known state rc=$source_gate_rc"
|
|
else
|
|
evidence_warn "MOMO source-arrival gate produced a non-terminal state"
|
|
fi
|
|
rm -f "$momo_tmp"
|
|
fi
|
|
|
|
if [[ "$RUN_STOCK" -eq 1 ]]; then
|
|
section "StockPlatform freshness"
|
|
stock_tmp="$(mktemp -t post-start-stock.XXXXXX)"
|
|
stock_code=""
|
|
stock_attempt=1
|
|
while [[ "$stock_attempt" -le "$STOCK_FRESHNESS_RETRY_ATTEMPTS" ]]; do
|
|
stock_code="$(curl -k -sS -o "$stock_tmp" -w '%{http_code}' --max-time 12 "https://stock.wooo.work/api/v1/system/freshness" 2>/dev/null || true)"
|
|
if [[ "$stock_code" == 2* ]]; then
|
|
if [[ "$stock_attempt" -gt 1 ]]; then
|
|
evidence_warn "StockPlatform freshness recovered after attempt=$stock_attempt"
|
|
fi
|
|
break
|
|
fi
|
|
if [[ "$stock_attempt" -lt "$STOCK_FRESHNESS_RETRY_ATTEMPTS" ]]; then
|
|
sleep "$STOCK_FRESHNESS_RETRY_DELAY_SECONDS"
|
|
fi
|
|
stock_attempt=$((stock_attempt + 1))
|
|
done
|
|
if [[ "$stock_code" != 2* ]]; then
|
|
blocked "StockPlatform freshness endpoint returned ${stock_code:-curl_failed} attempts=$STOCK_FRESHNESS_RETRY_ATTEMPTS"
|
|
cat "$stock_tmp" || true
|
|
else
|
|
python3 - "$stock_tmp" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
path = sys.argv[1]
|
|
with open(path, "r", encoding="utf-8") as fh:
|
|
payload = json.load(fh)
|
|
|
|
print(f"STOCK_FRESHNESS_STATUS {payload.get('status')}")
|
|
print(f"STOCK_LATEST_TRADING_DATE {payload.get('latest_trading_date')}")
|
|
print("STOCK_BLOCKERS " + ",".join(payload.get("blockers") or []))
|
|
for source in payload.get("sources") or []:
|
|
print(
|
|
"STOCK_SOURCE "
|
|
f"{source.get('source')}|{source.get('status')}|"
|
|
f"{source.get('latest_date')}|{source.get('row_count')}"
|
|
)
|
|
PY
|
|
stock_status="$(python3 - "$stock_tmp" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], "r", encoding="utf-8") as fh:
|
|
print(json.load(fh).get("status") or "")
|
|
PY
|
|
)"
|
|
if [[ "$stock_status" == "ok" ]]; then
|
|
printf 'STOCK_EOD_WINDOW_PENDING 0\n'
|
|
printf 'STOCK_EOD_CLASSIFICATION ok\n'
|
|
printf 'STOCK_EOD_NEXT_ACTION none\n'
|
|
ok "StockPlatform freshness is ok"
|
|
else
|
|
stock_blockers="$(python3 - "$stock_tmp" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], "r", encoding="utf-8") as fh:
|
|
print(",".join(json.load(fh).get("blockers") or []))
|
|
PY
|
|
)"
|
|
stock_eod_context="$(python3 - "$stock_tmp" <<'PY'
|
|
import json
|
|
import sys
|
|
from datetime import datetime, time, timezone, timedelta
|
|
|
|
try:
|
|
from zoneinfo import ZoneInfo
|
|
except Exception: # pragma: no cover - old Python fallback
|
|
ZoneInfo = None
|
|
|
|
with open(sys.argv[1], "r", encoding="utf-8") as fh:
|
|
payload = json.load(fh)
|
|
|
|
tz = ZoneInfo("Asia/Taipei") if ZoneInfo else timezone(timedelta(hours=8))
|
|
now = datetime.now(tz)
|
|
today = now.date().isoformat()
|
|
latest = payload.get("latest_trading_date") or ""
|
|
status = payload.get("status") or "unknown"
|
|
blockers = payload.get("blockers") or []
|
|
first_full_window_end = time(19, 15)
|
|
final_retry_window_end = time(23, 35)
|
|
|
|
pending = 0
|
|
classification = "blocked_unknown"
|
|
next_action = "investigate_stockplatform_freshness"
|
|
|
|
if status == "ok":
|
|
classification = "ok"
|
|
next_action = "none"
|
|
elif latest == today and blockers and now.time() < first_full_window_end:
|
|
pending = 1
|
|
classification = "pending_first_eod_window"
|
|
next_action = "wait_for_18_20_19_10_cron_then_recheck"
|
|
elif latest == today and blockers and now.time() < final_retry_window_end:
|
|
classification = "after_first_eod_window_blocked"
|
|
next_action = "inspect_ingestion_logs_and_wait_retry_windows"
|
|
elif latest == today and blockers:
|
|
classification = "after_final_eod_window_blocked"
|
|
next_action = "open_stockplatform_data_recovery_gate"
|
|
elif blockers:
|
|
classification = "blocked_non_current_trading_day"
|
|
next_action = "inspect_trading_calendar_and_ingestion_logs"
|
|
|
|
print(f"STOCK_EOD_WINDOW_PENDING {pending}")
|
|
print(f"STOCK_EOD_CLASSIFICATION {classification}")
|
|
print(f"STOCK_EOD_NEXT_ACTION {next_action}")
|
|
print(f"STOCK_EOD_FIRST_FULL_WINDOW_END_LOCAL {first_full_window_end.strftime('%H:%M')}")
|
|
print(f"STOCK_EOD_FINAL_RETRY_WINDOW_END_LOCAL {final_retry_window_end.strftime('%H:%M')}")
|
|
print(f"STOCK_EOD_OBSERVED_AT_LOCAL {now.isoformat(timespec='seconds')}")
|
|
PY
|
|
)"
|
|
printf '%s\n' "$stock_eod_context"
|
|
STOCK_EOD_WINDOW_PENDING="$(awk '$1 == "STOCK_EOD_WINDOW_PENDING" {print $2}' <<<"$stock_eod_context" | tail -n 1)"
|
|
STOCK_EOD_CLASSIFICATION="$(awk '$1 == "STOCK_EOD_CLASSIFICATION" {print $2}' <<<"$stock_eod_context" | tail -n 1)"
|
|
STOCK_EOD_NEXT_ACTION="$(awk '$1 == "STOCK_EOD_NEXT_ACTION" {print $2}' <<<"$stock_eod_context" | tail -n 1)"
|
|
if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then
|
|
evidence_warn "StockPlatform freshness pending scheduled EOD window: ${STOCK_EOD_CLASSIFICATION:-unknown}; next=${STOCK_EOD_NEXT_ACTION:-unknown}; blockers=${stock_blockers:-no_blocker_list}"
|
|
else
|
|
blocked "StockPlatform freshness is ${stock_status:-unknown}: ${stock_blockers:-no_blocker_list}; classification=${STOCK_EOD_CLASSIFICATION:-unknown}"
|
|
fi
|
|
fi
|
|
fi
|
|
rm -f "$stock_tmp"
|
|
fi
|
|
|
|
if [[ "$RUN_BACKUP" -eq 1 ]]; then
|
|
section "Backup / offsite / escrow"
|
|
backup_tmp="$(mktemp -t post-start-backup.XXXXXX)"
|
|
if ssh_read "wooo@192.168.0.110" '/backup/scripts/backup-status.sh --no-notify --no-refresh' >"$backup_tmp" 2>&1; then
|
|
ok "backup-status readback succeeded"
|
|
else
|
|
blocked "backup-status readback failed"
|
|
fi
|
|
cat "$backup_tmp"
|
|
if grep -Eq 'core_blockers=0|CORE_BLOCKERS[ =]0' "$backup_tmp"; then
|
|
ok "backup core blockers are 0"
|
|
elif grep -Eq 'core_blockers=[1-9]|CORE_BLOCKERS[ =][1-9]' "$backup_tmp"; then
|
|
blocked "backup core blockers are non-zero"
|
|
else
|
|
service_warn "backup core blocker summary not confirmed"
|
|
fi
|
|
if grep -Eq 'escrow_missing=0|ESCROW_MISSING_COUNT[ =]0' "$backup_tmp"; then
|
|
ok "credential escrow missing is 0"
|
|
elif grep -Eq 'escrow_missing=[1-9]|ESCROW_MISSING_COUNT[ =][1-9]' "$backup_tmp"; then
|
|
boundary_warn "credential escrow still missing; DR_COMPLETE is forbidden"
|
|
escrow_status_tmp="$(mktemp -t post-start-escrow-status.XXXXXX)"
|
|
if ssh_read "wooo@192.168.0.110" '/backup/scripts/mark-credential-escrow-verified.sh --status' >"$escrow_status_tmp" 2>&1; then
|
|
evidence_warn "credential escrow missing items follow"
|
|
else
|
|
evidence_warn "credential escrow missing item readback failed"
|
|
fi
|
|
cat "$escrow_status_tmp"
|
|
rm -f "$escrow_status_tmp"
|
|
else
|
|
evidence_warn "credential escrow count not found"
|
|
fi
|
|
rm -f "$backup_tmp"
|
|
fi
|
|
|
|
if [[ "$RUN_ROUTES" -eq 1 ]]; then
|
|
section "Public routes"
|
|
for url in "${ROUTES[@]}"; do
|
|
code=""
|
|
attempt=1
|
|
while [[ "$attempt" -le "$ROUTE_RETRY_ATTEMPTS" ]]; do
|
|
code="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)"
|
|
case "$code" in
|
|
2*|3*)
|
|
break
|
|
;;
|
|
esac
|
|
if [[ "$attempt" -lt "$ROUTE_RETRY_ATTEMPTS" ]]; then
|
|
sleep "$ROUTE_RETRY_DELAY_SECONDS"
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
done
|
|
case "$code" in
|
|
2*|3*)
|
|
if [[ "$url" == "https://awoooi.wooo.work/api/v1/health" && "$code" == 2* ]]; then
|
|
AWOOOI_API_ROUTE_OK=1
|
|
fi
|
|
if [[ "$attempt" -gt 1 ]]; then
|
|
evidence_warn "$code $url recovered_after_attempt=$attempt"
|
|
else
|
|
ok "$code $url"
|
|
fi
|
|
;;
|
|
*)
|
|
ROUTE_SMOKE_BLOCKED=$((ROUTE_SMOKE_BLOCKED + 1))
|
|
blocked "${code:-curl_failed} $url attempts=$ROUTE_RETRY_ATTEMPTS"
|
|
;;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
if [[ "$COLD_START_PENDING_BLOCKERS" -gt 0 ]]; then
|
|
non_route_cold_blockers="$(printf '%s\n' "$COLD_START_BLOCKED_LINES" | grep -Ev '^BLOCKED public route ' || true)"
|
|
if [[ "$RUN_ROUTES" -eq 1 && "$ROUTE_SMOKE_BLOCKED" -eq 0 && "$AWOOOI_API_ROUTE_OK" -eq 1 ]]; then
|
|
non_route_cold_blockers="$(
|
|
printf '%s\n' "$non_route_cold_blockers" | grep -Ev '^BLOCKED AWOOOI API not reachable$|^BLOCKED AWOOI API not reachable$' || true
|
|
)"
|
|
fi
|
|
if [[ "$RUN_ROUTES" -eq 1 && "$ROUTE_SMOKE_BLOCKED" -eq 0 && -z "$non_route_cold_blockers" ]]; then
|
|
evidence_warn "cold-start route/API warmup blockers recovered under wrapper route retry: $COLD_START_BLOCKED_SUMMARY"
|
|
printf '%s\n' "$COLD_START_BLOCKED_LINES"
|
|
else
|
|
blocked "cold-start has blockers: $COLD_START_BLOCKED_SUMMARY"
|
|
printf '%s\n' "$COLD_START_BLOCKED_LINES"
|
|
fi
|
|
fi
|
|
|
|
if [[ "$RUN_CPU" -eq 1 ]]; then
|
|
section "110 CPU / process attribution"
|
|
cpu_tmp="$(mktemp -t post-start-cpu.XXXXXX)"
|
|
if ssh_read "wooo@192.168.0.110" 'uptime; vmstat 1 5; ps -eo pid,ppid,pgid,stat,pcpu,pmem,comm,args --sort=-pcpu | head -25' >"$cpu_tmp" 2>&1; then
|
|
ok "110 CPU/process readback succeeded"
|
|
else
|
|
evidence_warn "110 CPU/process readback failed"
|
|
fi
|
|
cat "$cpu_tmp"
|
|
if grep -Eiq 'chrome|chromium|playwright' "$cpu_tmp"; then
|
|
evidence_warn "browser/smoke process is visible; classify orphan vs active parent before action"
|
|
fi
|
|
if grep -Eiq 'gitea|actions|runner|npm|pnpm|pytest|pip-audit' "$cpu_tmp"; then
|
|
ok "active CI/build/test load is visible"
|
|
fi
|
|
rm -f "$cpu_tmp"
|
|
fi
|
|
|
|
section "110 runner fail-closed guard"
|
|
runner_tmp="$(mktemp -t post-start-runner.XXXXXX)"
|
|
if ssh_read "wooo@192.168.0.110" '
|
|
for u in awoooi-cd-lane.service awoooi-cd-lane-drain.service awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
|
|
load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true)
|
|
unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true)
|
|
active=$(systemctl show "$u" -p ActiveState --value 2>/dev/null || true)
|
|
mainpid=$(systemctl show "$u" -p MainPID --value 2>/dev/null || true)
|
|
unit_ok=0
|
|
unit_stub=0
|
|
if [ "$load" = "masked" ] && [ "$unitfile" = "masked" ] && [ "$active" = "inactive" ] && [ "${mainpid:-0}" = "0" ]; then
|
|
unit_ok=1
|
|
elif [ "$active" = "inactive" ] && [ "${mainpid:-0}" = "0" ] \
|
|
&& systemctl cat "$u" 2>/dev/null | grep -q "ConditionPathExists=/run/awoooi-runner-migrated-or-hard-limited"; then
|
|
unit_stub=1
|
|
unit_ok=1
|
|
fi
|
|
echo "RUNNER_FAILCLOSED_UNIT $u load=$load unitfile=$unitfile active=$active mainpid=$mainpid stub=$unit_stub ok=$unit_ok"
|
|
done
|
|
enforcer_timer_active=$(systemctl is-active awoooi-runner-failclosed-enforcer.timer 2>/dev/null || true)
|
|
enforcer_timer_enabled=$(systemctl is-enabled awoooi-runner-failclosed-enforcer.timer 2>/dev/null || true)
|
|
enforcer_service_result=$(systemctl show awoooi-runner-failclosed-enforcer.service -p Result --value 2>/dev/null || true)
|
|
echo "RUNNER_FAILCLOSED_ENFORCER timer_active=$enforcer_timer_active timer_enabled=$enforcer_timer_enabled service_result=$enforcer_service_result"
|
|
authority_timer_active=$(systemctl is-active awoooi-runner-failclosed-authority.timer 2>/dev/null || true)
|
|
authority_timer_enabled=$(systemctl is-enabled awoooi-runner-failclosed-authority.timer 2>/dev/null || true)
|
|
authority_service_result=$(systemctl show awoooi-runner-failclosed-authority.service -p Result --value 2>/dev/null || true)
|
|
echo "RUNNER_FAILCLOSED_AUTHORITY timer_active=$authority_timer_active timer_enabled=$authority_timer_enabled service_result=$authority_service_result"
|
|
cd_lane_process_count=$(pgrep -f "^/home/wooo/awoooi-cd-lane/awoooi_cd_lane|^/home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled" 2>/dev/null | wc -l | tr -d " ")
|
|
echo "CD_LANE_PROCESS_COUNT $cd_lane_process_count"
|
|
cd_lane_root_restore_left=unknown
|
|
if sudo -n true >/dev/null 2>&1; then
|
|
cd_lane_root_restore_left=$(sudo -n find /root -maxdepth 1 -type d \( -name "awoooi-runner-restore-sources-disabled*" -o -name "awoooi-cd-lane-disabled*" -o -name "awoooi-cd-lane-drain-disabled*" \) -print 2>/dev/null | wc -l | tr -d " ")
|
|
fi
|
|
echo "CD_LANE_ROOT_RESTORE_SOURCES left=$cd_lane_root_restore_left"
|
|
sentinel_left=0
|
|
for s in /run/awoooi-runner-host-enabled /run/awoooi-start-controlled-cd-lane /run/awoooi-start-controlled-cd-lane-drain /run/awoooi-start-cd-lane-allowed /run/awoooi-cd-lane-drain-ok /run/awoooi-cd-lane-ok /run/awoooi-cd-lane-enabled /run/awoooi-cd-lane-controlled-open; do
|
|
[ -e "$s" ] && sentinel_left=$((sentinel_left + 1))
|
|
done
|
|
echo "RUNNER_SENTINELS_LEFT $sentinel_left"
|
|
active_job_containers=$(docker ps --format "{{.Names}}" 2>/dev/null | grep -Ec "^(GITEA-ACTIONS-|awoooi-cd-)" || true)
|
|
echo "ACTIVE_JOB_CONTAINERS $active_job_containers"
|
|
cd_lane_guard_ok=0
|
|
if [ "$enforcer_timer_active" = "active" ] \
|
|
&& [ "$enforcer_timer_enabled" = "enabled" ] \
|
|
&& [ "$enforcer_service_result" = "success" ] \
|
|
&& [ "$authority_timer_active" = "active" ] \
|
|
&& [ "$authority_timer_enabled" = "enabled" ] \
|
|
&& [ "$authority_service_result" = "success" ] \
|
|
&& [ "$cd_lane_process_count" = "0" ] \
|
|
&& [ "$cd_lane_root_restore_left" = "0" ] \
|
|
&& [ "$sentinel_left" = "0" ] \
|
|
&& [ "$active_job_containers" = "0" ]; then
|
|
cd_lane_guard_ok=1
|
|
fi
|
|
echo "CD_LANE_GUARDRAILS_OK $cd_lane_guard_ok"
|
|
direct_runner_count=$(pgrep -f "^/home/wooo/act-runner/act_runner|^/home/wooo/act-runner-controlled/act_runner|^/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner" 2>/dev/null | wc -l | tr -d " ")
|
|
echo "RUNNER_DIRECT_PROCESS_COUNT $direct_runner_count"
|
|
for p in /home/wooo/awoooi-cd-lane/awoooi_cd_lane /home/wooo/awoooi-cd-lane-drain/awoooi_cd_lane_controlled /home/wooo/act-runner/act_runner /home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard /home/wooo/act-runner-controlled/act_runner /home/wooo/awoooi-controlled-runner/awoooi_controlled_runner; do
|
|
kind=$(file -b "$p" 2>/dev/null || echo missing)
|
|
echo "RUNNER_FAILCLOSED_BINARY $p kind=$kind"
|
|
echo "$kind" | grep -qi "ELF" && echo "RUNNER_FAILCLOSED_BINARY_ELF $p"
|
|
done
|
|
HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1 HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0 /usr/local/bin/awoooi-wait-host-web-build-pressure.sh
|
|
echo "RUNNER_PRESSURE_GATE_RC $?"
|
|
' >"$runner_tmp" 2>&1; then
|
|
ok "110 controlled runner readback succeeded"
|
|
else
|
|
blocked "110 controlled runner readback failed"
|
|
fi
|
|
cat "$runner_tmp"
|
|
if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then
|
|
ok "110 runner/CD lane units are fail-closed"
|
|
else
|
|
blocked "110 runner/CD lane units are not fail-closed"
|
|
fi
|
|
grep -q "RUNNER_FAILCLOSED_ENFORCER timer_active=active timer_enabled=enabled service_result=success" "$runner_tmp" && ok "110 fail-closed enforcer timer active and successful" || blocked "110 fail-closed enforcer timer not healthy"
|
|
grep -q "RUNNER_FAILCLOSED_AUTHORITY timer_active=active timer_enabled=enabled service_result=success" "$runner_tmp" && ok "110 fail-closed authority timer active and successful" || blocked "110 fail-closed authority timer not healthy"
|
|
grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 cd-lane/drain lane are fail-closed with enforcer" || blocked "110 cd-lane/drain lane fail-closed guardrails incomplete"
|
|
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || blocked "110 legacy direct runner process detected"
|
|
grep -q "ACTIVE_JOB_CONTAINERS 0" "$runner_tmp" && ok "110 Gitea/CD job container count is zero" || blocked "110 Gitea/CD job container still active"
|
|
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && blocked "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
|
|
grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || blocked "110 host pressure gate is blocking"
|
|
rm -f "$runner_tmp"
|
|
|
|
section "總結"
|
|
printf 'POST_START_QUICK_CHECK PASS=%s WARN=%s BLOCKED=%s\n' "$PASS_COUNT" "$WARN_COUNT" "$BLOCKED_COUNT"
|
|
printf 'POST_START_QUICK_CHECK_WARNINGS SERVICE=%s BOUNDARY=%s EVIDENCE=%s\n' "$SERVICE_WARN_COUNT" "$BOUNDARY_WARN_COUNT" "$EVIDENCE_WARN_COUNT"
|
|
|
|
if [[ "$BLOCKED_COUNT" -gt 0 ]]; then
|
|
printf 'RESULT=BLOCKED\n'
|
|
exit 2
|
|
fi
|
|
|
|
if [[ "$SERVICE_WARN_COUNT" -gt 0 ]]; then
|
|
printf 'RESULT=DEGRADED\n'
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$STOCK_EOD_WINDOW_PENDING" == "1" ]]; then
|
|
printf 'RESULT=PRODUCT_DATA_PENDING_EOD_WINDOW\n'
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$BOUNDARY_WARN_COUNT" -gt 0 ]]; then
|
|
printf 'RESULT=FULL_STACK_GREEN_DR_ESCROW_BLOCKED\n'
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$EVIDENCE_WARN_COUNT" -gt 0 ]]; then
|
|
printf 'RESULT=GREEN_WITH_EVIDENCE_WARNINGS\n'
|
|
exit 0
|
|
fi
|
|
|
|
printf 'RESULT=GREEN\n'
|
|
exit 0
|