fix(reboot): project runtime service green readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 1m27s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 1m27s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -86,6 +86,9 @@ post_start_args=(--no-color)
|
||||
if [[ "$NO_COLOR_FLAG" -eq 1 ]]; then
|
||||
post_start_args=(--no-color)
|
||||
fi
|
||||
if [[ "${POST_REBOOT_SKIP_RUNNER_GUARD:-0}" == "1" ]]; then
|
||||
post_start_args+=(--skip-runner-guard)
|
||||
fi
|
||||
|
||||
post_start_rc=0
|
||||
bash "$ROOT_DIR/scripts/reboot-recovery/post-start-quick-check.sh" "${post_start_args[@]}" >"$post_start_log" 2>&1 || post_start_rc=$?
|
||||
|
||||
@@ -18,6 +18,7 @@ RUN_STOCK=1
|
||||
RUN_BACKUP=1
|
||||
RUN_ROUTES=1
|
||||
RUN_CPU=1
|
||||
RUN_RUNNER_GUARD=1
|
||||
NO_COLOR_FLAG=0
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
COLD_START_PENDING_BLOCKERS=0
|
||||
@@ -83,6 +84,7 @@ Options:
|
||||
--skip-backup Do not run /backup/scripts/backup-status.sh on 110.
|
||||
--skip-routes Do not curl public route smoke targets.
|
||||
--skip-cpu Do not read 110 CPU / process summary.
|
||||
--skip-runner-guard Do not run the 110 runner fail-closed guard section.
|
||||
--no-color Disable ANSI color.
|
||||
-h, --help Show this help.
|
||||
|
||||
@@ -122,6 +124,9 @@ while [[ $# -gt 0 ]]; do
|
||||
--skip-cpu)
|
||||
RUN_CPU=0
|
||||
;;
|
||||
--skip-runner-guard)
|
||||
RUN_RUNNER_GUARD=0
|
||||
;;
|
||||
--no-color)
|
||||
NO_COLOR_FLAG=1
|
||||
;;
|
||||
@@ -589,9 +594,10 @@ if [[ "$RUN_CPU" -eq 1 ]]; then
|
||||
rm -f "$cpu_tmp"
|
||||
fi
|
||||
|
||||
section "110 runner fail-closed guard"
|
||||
runner_tmp="$(mktemp -t post-start-runner.XXXXXX)"
|
||||
if ssh_read "wooo@192.168.0.110" '
|
||||
if [[ "$RUN_RUNNER_GUARD" -eq 1 ]]; then
|
||||
section "110 runner fail-closed guard"
|
||||
runner_tmp="$(mktemp -t post-start-runner.XXXXXX)"
|
||||
if ssh_read "wooo@192.168.0.110" '
|
||||
for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do
|
||||
load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true)
|
||||
unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true)
|
||||
@@ -705,21 +711,24 @@ done
|
||||
HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1 HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0 /usr/local/bin/awoooi-wait-host-web-build-pressure.sh
|
||||
echo "RUNNER_PRESSURE_GATE_RC $?"
|
||||
' >"$runner_tmp" 2>&1; then
|
||||
ok "110 controlled runner readback succeeded"
|
||||
ok "110 controlled runner readback succeeded"
|
||||
else
|
||||
capacity_or_runner_issue "110 controlled runner readback failed"
|
||||
fi
|
||||
cat "$runner_tmp"
|
||||
if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then
|
||||
ok "110 legacy direct/Gitea runner units are fail-closed"
|
||||
else
|
||||
capacity_or_runner_issue "110 legacy direct/Gitea runner units are not fail-closed"
|
||||
fi
|
||||
grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || capacity_or_runner_issue "110 controlled cd-lane guardrails incomplete"
|
||||
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || capacity_or_runner_issue "110 legacy direct runner process detected"
|
||||
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && capacity_or_runner_issue "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
|
||||
grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || capacity_or_runner_issue "110 host pressure gate is blocking"
|
||||
rm -f "$runner_tmp"
|
||||
else
|
||||
capacity_or_runner_issue "110 controlled runner readback failed"
|
||||
evidence_warn "110 runner fail-closed guard skipped by caller; use guard-gitea-runner-pressure.py for runner lane verification"
|
||||
fi
|
||||
cat "$runner_tmp"
|
||||
if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then
|
||||
ok "110 legacy direct/Gitea runner units are fail-closed"
|
||||
else
|
||||
capacity_or_runner_issue "110 legacy direct/Gitea runner units are not fail-closed"
|
||||
fi
|
||||
grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || capacity_or_runner_issue "110 controlled cd-lane guardrails incomplete"
|
||||
grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || capacity_or_runner_issue "110 legacy direct runner process detected"
|
||||
grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && capacity_or_runner_issue "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
|
||||
grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || capacity_or_runner_issue "110 host pressure gate is blocking"
|
||||
rm -f "$runner_tmp"
|
||||
|
||||
section "總結"
|
||||
printf 'POST_START_QUICK_CHECK PASS=%s WARN=%s BLOCKED=%s\n' "$PASS_COUNT" "$WARN_COUNT" "$BLOCKED_COUNT"
|
||||
|
||||
@@ -57,16 +57,19 @@ post_reboot_summary_command=(
|
||||
)
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \
|
||||
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
|
||||
timeout "${POST_REBOOT_READINESS_TIMEOUT_SECONDS}s" "${post_reboot_summary_command[@]}" \
|
||||
>"$summary_file" 2>&1 \
|
||||
|| printf 'POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1\n' >>"$summary_file"
|
||||
elif command -v gtimeout >/dev/null 2>&1; then
|
||||
ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \
|
||||
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
|
||||
gtimeout "${POST_REBOOT_READINESS_TIMEOUT_SECONDS}s" "${post_reboot_summary_command[@]}" \
|
||||
>"$summary_file" 2>&1 \
|
||||
|| printf 'POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1\n' >>"$summary_file"
|
||||
else
|
||||
ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \
|
||||
POST_REBOOT_SKIP_RUNNER_GUARD=1 \
|
||||
"${post_reboot_summary_command[@]}" >"$summary_file" 2>&1 || true
|
||||
fi
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ SCRIPT = ROOT / "scripts" / "reboot-recovery" / "post-reboot-readiness-summary.s
|
||||
def test_unknown_backup_escrow_evidence_stays_blocked() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "POST_REBOOT_SKIP_RUNNER_GUARD" in text
|
||||
assert 'post_start_args+=(--skip-runner-guard)' in text
|
||||
assert "escrow_evidence_unknown=0" in text
|
||||
assert 'elif [[ -z "$escrow_missing_count" ]]; then' in text
|
||||
assert "escrow_evidence_unknown=1" in text
|
||||
|
||||
@@ -26,6 +26,11 @@ def test_post_start_self_checks_can_run_locally_on_110() -> None:
|
||||
def test_runner_pressure_is_capacity_evidence_after_routes_recover() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "RUN_RUNNER_GUARD=1" in text
|
||||
assert "--skip-runner-guard" in text
|
||||
assert 'RUN_RUNNER_GUARD=0' in text
|
||||
assert 'if [[ "$RUN_RUNNER_GUARD" -eq 1 ]]; then' in text
|
||||
assert "110 runner fail-closed guard skipped by caller" in text
|
||||
assert "service_route_recovered()" in text
|
||||
assert "capacity_or_runner_issue()" in text
|
||||
assert "cold-start non-route blockers retained as capacity/freshness evidence" in text
|
||||
|
||||
@@ -113,6 +113,7 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
|
||||
assert "windows99_vmware_verify_fallback_max_age_seconds" in text
|
||||
assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text
|
||||
assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
|
||||
assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text
|
||||
assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
|
||||
assert "awoooi_windows99_vmware_missing_vmx_alias" in text
|
||||
assert "awoooi_windows99_vmware_powered_off_alias" in text
|
||||
|
||||
Reference in New Issue
Block a user