diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 5b1c0a550..f4d0086f5 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3009,70 +3009,99 @@ jobs: # 2026-05-02 ogt + Claude Sonnet 4.6: 加主機完整性檢查 # 根因：partial scan（如 110 timeout、其他成功）會讓 [-s file] 通過、 # 後續 patch 推進缺漏的 known_hosts → asyncssh 拒所有 SSH。 - # 修法：scan 完驗證所有 allowlisted 主機都在；缺任何一台就 abort， - # 不能覆蓋現有 secret，防止 production SSH 自動修復路徑癱瘓。 + # 修法：核心 executor 主機必須完整；Host111 是隔離的 provider lane， + # 首次 migration 掃不到時不得拖垮其他產品部署。既有 111 pin + # 只能保留，不能被 partial scan 覆蓋。 : > /tmp/known_hosts_repair : > /tmp/known_hosts_scan_err - EXPECTED_HOSTS=6 + CORE_HOSTS="192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188" + SCAN_HOSTS="192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188" + CORE_EXPECTED_HOSTS=5 + TARGET_HOSTS=6 + PRESENT_CORE=0 + HOST111_PRESENT=0 PRESENT=0 for scan_attempt in 1 2 3; do ssh-keyscan -T 5 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \ >> /tmp/known_hosts_repair 2>>/tmp/known_hosts_scan_err || true sort -u /tmp/known_hosts_repair -o /tmp/known_hosts_repair - PRESENT=0 - for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do + PRESENT_CORE=0 + for ip in \$CORE_HOSTS; do if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then - PRESENT=\$((PRESENT + 1)) + PRESENT_CORE=\$((PRESENT_CORE + 1)) fi done - if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then + HOST111_PRESENT=0 + if grep -qE '^192\.168\.0\.111[[:space:]]' /tmp/known_hosts_repair 2>/dev/null; then + HOST111_PRESENT=1 + fi + PRESENT=\$((PRESENT_CORE + HOST111_PRESENT)) + if [ "\$PRESENT" -eq "\$TARGET_HOSTS" ]; then break fi sleep "\$scan_attempt" done - for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do + for ip in \$SCAN_HOSTS; do if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then : else echo "⚠️ ssh-keyscan 缺主機 \${ip}" fi done - if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then + + EXISTING_KNOWN_HOSTS=/tmp/known_hosts_existing + EXISTING_CORE=0 + EXISTING_HOST111=0 + if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \ + -o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then + for ip in \$CORE_HOSTS; do + if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then + EXISTING_CORE=\$((EXISTING_CORE + 1)) + fi + done + if grep -qE '^192\.168\.0\.111[[:space:]]' "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then + EXISTING_HOST111=1 + fi + fi + EXISTING_PRESENT=\$((EXISTING_CORE + EXISTING_HOST111)) + + TRUST_SOURCE="" + TRUST_SELECTION="" + if [ "\$PRESENT_CORE" -eq "\$CORE_EXPECTED_HOSTS" ] && [ "\$HOST111_PRESENT" -eq 1 ]; then + TRUST_SOURCE=/tmp/known_hosts_repair + TRUST_SELECTION=fresh_6_of_6 + elif [ "\$EXISTING_CORE" -eq "\$CORE_EXPECTED_HOSTS" ] && [ "\$EXISTING_HOST111" -eq 1 ]; then + TRUST_SELECTION=preserve_existing_6_of_6 + elif [ "\$PRESENT_CORE" -eq "\$CORE_EXPECTED_HOSTS" ]; then + TRUST_SOURCE=/tmp/known_hosts_repair + TRUST_SELECTION=fresh_core_5_host111_trust_deferred + elif [ "\$EXISTING_CORE" -eq "\$CORE_EXPECTED_HOSTS" ]; then + TRUST_SELECTION=preserve_existing_core_5_host111_trust_deferred + else + echo "❌ host-key core trust incomplete; fresh=\${PRESENT_CORE}/\${CORE_EXPECTED_HOSTS} existing=\${EXISTING_CORE}/\${CORE_EXPECTED_HOSTS}" + rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err + exit 1 + fi + + if [ -n "\$TRUST_SOURCE" ]; then \$KUBECTL create secret generic awoooi-repair-known-hosts \ -n awoooi-prod \ - --from-file=known_hosts=/tmp/known_hosts_repair \ + --from-file=known_hosts="\$TRUST_SOURCE" \ --dry-run=client -o yaml | \$KUBECTL apply -f - \ && echo "✅ awoooi-repair-known-hosts Secret 已建立/更新" \ || echo "⚠️ awoooi-repair-known-hosts Secret 建立失敗 (非致命)" - KNOWN_HOSTS_B64=\$(base64 -w 0 /tmp/known_hosts_repair) + KNOWN_HOSTS_B64=\$(base64 -w 0 "\$TRUST_SOURCE") \$KUBECTL patch secret ssh-mcp-key -n awoooi-prod --type=merge \ -p="{\"data\":{\"known_hosts\":\"\${KNOWN_HOSTS_B64}\"}}" \ - && echo "✅ ssh-mcp-key known_hosts 已更新（6 台主機完整）" \ + && echo "✅ ssh-mcp-key known_hosts 已更新 selection=\${TRUST_SELECTION}" \ || { echo "❌ ssh-mcp-key known_hosts 更新失敗，停止部署"; exit 1; } - rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err else - # A transient guest SSH outage must not overwrite trust data, but it - # also must not blindly green-light the deploy. Decode only on the - # K3s controller, verify all six public host entries, then delete - # the temporary file without printing its contents. - EXISTING_KNOWN_HOSTS=/tmp/known_hosts_existing - EXISTING_PRESENT=0 - if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \ - -o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then - for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do - if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then - EXISTING_PRESENT=\$((EXISTING_PRESENT + 1)) - fi - done - fi - rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err - if [ "\$EXISTING_PRESENT" -eq "\$EXPECTED_HOSTS" ]; then - echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 6/6 and preserved" - else - echo "❌ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key coverage \${EXISTING_PRESENT}/\${EXPECTED_HOSTS}" - exit 1 - fi + echo "⚠️ ssh-mcp-key known_hosts preserved selection=\${TRUST_SELECTION} fresh=\${PRESENT}/\${TARGET_HOSTS} existing=\${EXISTING_PRESENT}/\${TARGET_HOSTS}" fi + if [ "\$HOST111_PRESENT" -eq 0 ] && [ "\$EXISTING_HOST111" -eq 0 ]; then + echo "⚠️ host111_trust_deferred isolated_lane=ansible:111-ollama-fallback" + fi + rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err echo "✅ 所有 Secrets 注入完成" SECRETS diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 7858229d6..85c4f3776 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -3894,6 +3894,12 @@ function Convert-AgentRecoveryReadback { if (-not $row -or -not $row.reachable) { $requiredHostFailures += $alias } } + $advisoryChecks = @( + [pscustomobject]@{ + name = "host_188_hygiene_green" + passed = [bool]((& $get "HOST_188_HYGIENE_BLOCKED" "1") -eq "0") + } + ) $checks = @( [pscustomobject]@{ name = "readback_present"; passed = (& $asBool "RECOVERY_READBACK") }, [pscustomobject]@{ name = "artifact_recent"; passed = $artifactRecent }, @@ -3904,7 +3910,6 @@ function Convert-AgentRecoveryReadback { [pscustomobject]@{ name = "stock_freshness_ok"; passed = [bool]((& $get "STOCK_FRESHNESS_STATUS") -eq "ok") }, [pscustomobject]@{ name = "backup_core_green"; passed = (& $asBool "BACKUP_CORE_GREEN") }, [pscustomobject]@{ name = "host_188_service_green"; passed = (& $asBool "HOST_188_SERVICE_GREEN") }, - [pscustomobject]@{ name = "host_188_hygiene_green"; passed = [bool]((& $get "HOST_188_HYGIENE_BLOCKED" "1") -eq "0") }, [pscustomobject]@{ name = "edge_fallback_ready"; passed = (& $asBool "EDGE_FALLBACK_READY") }, [pscustomobject]@{ name = "windows99_vmware_ready"; passed = (& $asBool "VMWARE_AUTOSTART_VERIFY_READY") }, [pscustomobject]@{ name = "windows99_update_policy_ready"; passed = (& $asBool "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY") }, @@ -3931,6 +3936,7 @@ function Convert-AgentRecoveryReadback { } $failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) + $advisoryFailedChecks = @($advisoryChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) $sloFailedChecks = @($sloChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) [pscustomobject]@{ schemaVersion = "agent99_recovery_coordinator_readback_v1" @@ -3953,6 +3959,8 @@ function Convert-AgentRecoveryReadback { hosts = $hosts checks = $checks failedChecks = $failedChecks + advisoryChecks = $advisoryChecks + advisoryFailedChecks = $advisoryFailedChecks sloChecks = $sloChecks sloFailedChecks = $sloFailedChecks } diff --git a/apps/api/tests/test_cd_known_hosts_recovery_gate.py b/apps/api/tests/test_cd_known_hosts_recovery_gate.py index a6859f2ce..5910601de 100644 --- a/apps/api/tests/test_cd_known_hosts_recovery_gate.py +++ b/apps/api/tests/test_cd_known_hosts_recovery_gate.py @@ -4,16 +4,21 @@ ROOT = Path(__file__).resolve().parents[3] WORKFLOW = ROOT / ".gitea/workflows/cd.yaml" -def test_cd_known_hosts_gate_retries_and_verifies_preserved_secret() -> None: +def test_cd_known_hosts_gate_keeps_core_fail_closed_and_host111_isolated() -> None: workflow = WORKFLOW.read_text(encoding="utf-8") gate = workflow[workflow.index(": > /tmp/known_hosts_repair") :] gate = gate[: gate.index('echo "✅ 所有 Secrets 注入完成"')] assert "for scan_attempt in 1 2 3" in gate assert 'sleep "\\$scan_attempt"' in gate - assert "existing ssh-mcp-key verified 6/6 and preserved" in gate - assert "EXISTING_PRESENT=\\$((EXISTING_PRESENT + 1))" in gate - assert 'if [ "\\$EXISTING_PRESENT" -eq "\\$EXPECTED_HOSTS" ]' in gate + assert "CORE_EXPECTED_HOSTS=5" in gate + assert "TARGET_HOSTS=6" in gate + assert "fresh_6_of_6" in gate + assert "preserve_existing_6_of_6" in gate + assert "fresh_core_5_host111_trust_deferred" in gate + assert "preserve_existing_core_5_host111_trust_deferred" in gate + assert "host-key core trust incomplete" in gate + assert "host111_trust_deferred isolated_lane=ansible:111-ollama-fallback" in gate assert 'rm -f "\\$EXISTING_KNOWN_HOSTS"' in gate assert "cat /tmp/known_hosts_scan_err" not in gate - assert "existing ssh-mcp-key coverage" in gate + assert "KNOWN_HOSTS_B64=\\$(base64 -w 0 \"\\$TRUST_SOURCE\")" in gate diff --git a/apps/api/tests/test_executor_network_policy_boundary.py b/apps/api/tests/test_executor_network_policy_boundary.py index dc30b3b13..8f647c00c 100644 --- a/apps/api/tests/test_executor_network_policy_boundary.py +++ b/apps/api/tests/test_executor_network_policy_boundary.py @@ -192,8 +192,12 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None: "192.168.0.121 192.168.0.188" ) assert f"ssh-keyscan -T 5 {expected_known_hosts}" in workflow - assert "EXPECTED_HOSTS=6" in workflow - assert workflow.count(expected_known_hosts) >= 4 + assert "CORE_EXPECTED_HOSTS=5" in workflow + assert "TARGET_HOSTS=6" in workflow + assert "fresh_core_5_host111_trust_deferred" in workflow + assert "preserve_existing_core_5_host111_trust_deferred" in workflow + assert "host111_trust_deferred isolated_lane=ansible:111-ollama-fallback" in workflow + assert workflow.count(expected_known_hosts) >= 2 assert workflow.count(expected_hosts) >= 2 assert "except ConnectionRefusedError:" in workflow assert "broker_ssh_refused_but_egress_permitted=" in workflow diff --git a/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service b/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service index 8d646eb7c..ab77660c3 100644 --- a/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service +++ b/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service @@ -15,10 +15,15 @@ Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles Environment=LOG_DIR=/home/wooo/reboot-recovery Environment=TARGET_MINUTES=10 Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply -Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420 -Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS=20 +Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=60 +Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS=5 +Environment=SSH_COMMAND_TIMEOUT_SECONDS=8 +Environment=POST_REBOOT_READINESS_TIMEOUT_SECONDS=60 +Environment=STOCK_READBACK_TIMEOUT_SECONDS=10 +Environment=PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS=5 +Environment=WINDOWS99_REMOTE_VERIFY_TIMEOUT=30 ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh -TimeoutStartSec=600 +TimeoutStartSec=330 StandardOutput=journal StandardError=journal diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index eeca69d53..7af8fd467 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -17,8 +17,8 @@ LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}" STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}" STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}" STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}" -POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}" -PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}" +POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}" +PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-5}" PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}" WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}" # Host 111 is a physical MacBook/Ollama node. It remains in whole-host probes, @@ -26,8 +26,8 @@ WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_S WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}" POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS="${POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS:-900}" AI_LOG_TRIAGE_AUTO_RECOVERY_MODE="${AI_LOG_TRIAGE_AUTO_RECOVERY_MODE:-check}" -AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-420}" -AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS:-20}" +AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-60}" +AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS:-5}" mkdir -p "$TEXTFILE_DIR" "$LOG_DIR" diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index ee901d502..bc1953d83 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -836,7 +836,7 @@ def source_controls() -> dict[str, bool]: "slo_systemd_service_ai_log_triage_apply_source_present": file_contains( source_file("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service"), "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply", - "AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420", + "AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=60", ), "full_host_reboot_orchestrator_source_present": file_contains( source_file("scripts/reboot-recovery/full-host-reboot-orchestrator.sh"), diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index 56c38e40b..33918b274 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -92,6 +92,19 @@ def test_coordinator_covers_all_hosts_and_full_recovery_evidence() -> None: assert field in control +def test_host188_hygiene_is_advisory_not_a_recovery_completion_gate() -> None: + control = read("agent99-control-plane.ps1") + conversion = control.split("function Convert-AgentRecoveryReadback", 1)[1] + conversion = conversion.split("function Invoke-AgentRecoveryCoordinatorReadback", 1)[0] + + advisory = conversion.split("$advisoryChecks = @(", 1)[1].split("$checks = @(", 1)[0] + mandatory = conversion.split("$checks = @(", 1)[1].split("$sloChecks = @()", 1)[0] + assert 'name = "host_188_hygiene_green"' in advisory + assert 'name = "host_188_hygiene_green"' not in mandatory + assert "advisoryFailedChecks = $advisoryFailedChecks" in conversion + assert "verified = [bool]($failedChecks.Count -eq 0)" in conversion + + def test_fresh_reboot_claim_requires_fresh_artifact_and_zero_blockers() -> None: control = read("agent99-control-plane.ps1") diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index 10b833973..f59a8d05f 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -87,7 +87,33 @@ def test_service_uses_deployed_source_root_and_bounded_oneshot() -> None: assert "Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles" in text assert "Environment=LOG_DIR=/home/wooo/reboot-recovery" in text assert "ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh" in text - assert "TimeoutStartSec=600" in text + assert "TimeoutStartSec=330" in text + + +def test_service_worst_case_probe_budget_finishes_before_agent99_freshness_window() -> None: + text = SERVICE.read_text(encoding="utf-8") + + def environment_seconds(name: str) -> int: + match = re.search(rf"^Environment={name}=([0-9]+)$", text, re.MULTILINE) + assert match is not None, name + return int(match.group(1)) + + service_timeout_match = re.search(r"^TimeoutStartSec=([0-9]+)$", text, re.MULTILINE) + assert service_timeout_match is not None + service_timeout = int(service_timeout_match.group(1)) + agent99_freshness_wait = 420 + worst_case_budget = ( + environment_seconds("AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS") + + environment_seconds("AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS") + + environment_seconds("SSH_COMMAND_TIMEOUT_SECONDS") * 6 + + environment_seconds("POST_REBOOT_READINESS_TIMEOUT_SECONDS") + + environment_seconds("STOCK_READBACK_TIMEOUT_SECONDS") * 2 + + environment_seconds("PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS") * 6 + + environment_seconds("WINDOWS99_REMOTE_VERIFY_TIMEOUT") + ) + + assert worst_case_budget <= service_timeout - 30 + assert service_timeout < agent99_freshness_wait def test_timer_waits_after_completion_instead_of_running_back_to_back() -> None: @@ -136,7 +162,7 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: assert "windows99_vmware_verify_fallback_max_age_seconds" in text assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text - assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"' in text + assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}"' in text assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text assert "POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1" in text