fix(reboot): project runtime service green readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 1m27s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
Your Name
2026-07-03 03:16:38 +08:00
parent c36a65ecfb
commit 2bc73325a4
8 changed files with 211 additions and 53 deletions

View File

@@ -86,6 +86,9 @@ post_start_args=(--no-color)
if [[ "$NO_COLOR_FLAG" -eq 1 ]]; then
post_start_args=(--no-color)
fi
if [[ "${POST_REBOOT_SKIP_RUNNER_GUARD:-0}" == "1" ]]; then
post_start_args+=(--skip-runner-guard)
fi
post_start_rc=0
bash "$ROOT_DIR/scripts/reboot-recovery/post-start-quick-check.sh" "${post_start_args[@]}" >"$post_start_log" 2>&1 || post_start_rc=$?

View File

@@ -18,6 +18,7 @@ RUN_STOCK=1
RUN_BACKUP=1
RUN_ROUTES=1
RUN_CPU=1
RUN_RUNNER_GUARD=1
NO_COLOR_FLAG=0
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
COLD_START_PENDING_BLOCKERS=0
@@ -83,6 +84,7 @@ Options:
--skip-backup Do not run /backup/scripts/backup-status.sh on 110.
--skip-routes Do not curl public route smoke targets.
--skip-cpu Do not read 110 CPU / process summary.
--skip-runner-guard Do not run the 110 runner fail-closed guard section.
--no-color Disable ANSI color.
-h, --help Show this help.
@@ -122,6 +124,9 @@ while [[ $# -gt 0 ]]; do
--skip-cpu)
RUN_CPU=0
;;
--skip-runner-guard)
RUN_RUNNER_GUARD=0
;;
--no-color)
NO_COLOR_FLAG=1
;;
@@ -589,9 +594,10 @@ if [[ "$RUN_CPU" -eq 1 ]]; then
rm -f "$cpu_tmp"
fi
section "110 runner fail-closed guard"
runner_tmp="$(mktemp -t post-start-runner.XXXXXX)"
if ssh_read "wooo@192.168.0.110" '
if [[ "$RUN_RUNNER_GUARD" -eq 1 ]]; then
section "110 runner fail-closed guard"
runner_tmp="$(mktemp -t post-start-runner.XXXXXX)"
if ssh_read "wooo@192.168.0.110" '
for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true)
unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true)
@@ -705,21 +711,24 @@ done
HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1 HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0 /usr/local/bin/awoooi-wait-host-web-build-pressure.sh
echo "RUNNER_PRESSURE_GATE_RC $?"
' >"$runner_tmp" 2>&1; then
ok "110 controlled runner readback succeeded"
ok "110 controlled runner readback succeeded"
else
capacity_or_runner_issue "110 controlled runner readback failed"
fi
cat "$runner_tmp"
if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then
ok "110 legacy direct/Gitea runner units are fail-closed"
else
capacity_or_runner_issue "110 legacy direct/Gitea runner units are not fail-closed"
fi
grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || capacity_or_runner_issue "110 controlled cd-lane guardrails incomplete"
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || capacity_or_runner_issue "110 legacy direct runner process detected"
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && capacity_or_runner_issue "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || capacity_or_runner_issue "110 host pressure gate is blocking"
rm -f "$runner_tmp"
else
capacity_or_runner_issue "110 controlled runner readback failed"
evidence_warn "110 runner fail-closed guard skipped by caller; use guard-gitea-runner-pressure.py for runner lane verification"
fi
cat "$runner_tmp"
if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then
ok "110 legacy direct/Gitea runner units are fail-closed"
else
capacity_or_runner_issue "110 legacy direct/Gitea runner units are not fail-closed"
fi
grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || capacity_or_runner_issue "110 controlled cd-lane guardrails incomplete"
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || capacity_or_runner_issue "110 legacy direct runner process detected"
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && capacity_or_runner_issue "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || capacity_or_runner_issue "110 host pressure gate is blocking"
rm -f "$runner_tmp"
section "總結"
printf 'POST_START_QUICK_CHECK PASS=%s WARN=%s BLOCKED=%s\n' "$PASS_COUNT" "$WARN_COUNT" "$BLOCKED_COUNT"

View File

@@ -57,16 +57,19 @@ post_reboot_summary_command=(
)
if command -v timeout >/dev/null 2>&1; then
ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
timeout "${POST_REBOOT_READINESS_TIMEOUT_SECONDS}s" "${post_reboot_summary_command[@]}" \
>"$summary_file" 2>&1 \
|| printf 'POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1\n' >>"$summary_file"
elif command -v gtimeout >/dev/null 2>&1; then
ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
gtimeout "${POST_REBOOT_READINESS_TIMEOUT_SECONDS}s" "${post_reboot_summary_command[@]}" \
>"$summary_file" 2>&1 \
|| printf 'POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1\n' >>"$summary_file"
else
ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
"${post_reboot_summary_command[@]}" >"$summary_file" 2>&1 || true
fi

View File

@@ -10,6 +10,8 @@ SCRIPT = ROOT / "scripts" / "reboot-recovery" / "post-reboot-readiness-summary.s
def test_unknown_backup_escrow_evidence_stays_blocked() -> None:
text = SCRIPT.read_text(encoding="utf-8")
assert "POST_REBOOT_SKIP_RUNNER_GUARD" in text
assert 'post_start_args+=(--skip-runner-guard)' in text
assert "escrow_evidence_unknown=0" in text
assert 'elif [[ -z "$escrow_missing_count" ]]; then' in text
assert "escrow_evidence_unknown=1" in text

View File

@@ -26,6 +26,11 @@ def test_post_start_self_checks_can_run_locally_on_110() -> None:
def test_runner_pressure_is_capacity_evidence_after_routes_recover() -> None:
text = SCRIPT.read_text(encoding="utf-8")
assert "RUN_RUNNER_GUARD=1" in text
assert "--skip-runner-guard" in text
assert 'RUN_RUNNER_GUARD=0' in text
assert 'if [[ "$RUN_RUNNER_GUARD" -eq 1 ]]; then' in text
assert "110 runner fail-closed guard skipped by caller" in text
assert "service_route_recovered()" in text
assert "capacity_or_runner_issue()" in text
assert "cold-start non-route blockers retained as capacity/freshness evidence" in text

View File

@@ -113,6 +113,7 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
assert "windows99_vmware_verify_fallback_max_age_seconds" in text
assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text
assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text
assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
assert "awoooi_windows99_vmware_missing_vmx_alias" in text
assert "awoooi_windows99_vmware_powered_off_alias" in text