Merge remote-tracking branch 'gitea-ssh/main' into codex/p0-obs-002-credential-20260722

scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service

@@ -15,10 +15,15 @@ Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles
 Environment=LOG_DIR=/home/wooo/reboot-recovery
 Environment=TARGET_MINUTES=10
 Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply
-Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420
-Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS=20
+Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=60
+Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS=5
+Environment=SSH_COMMAND_TIMEOUT_SECONDS=8
+Environment=POST_REBOOT_READINESS_TIMEOUT_SECONDS=60
+Environment=STOCK_READBACK_TIMEOUT_SECONDS=10
+Environment=PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS=5
+Environment=WINDOWS99_REMOTE_VERIFY_TIMEOUT=30
 ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh
-TimeoutStartSec=600
+TimeoutStartSec=330
 StandardOutput=journal
 StandardError=journal
 

scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh

@@ -17,8 +17,8 @@ LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}"
 STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}"
 STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}"
 STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}"
-POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"
-PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}"
+POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}"
+PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-5}"
 PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}"
 WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}"
 # Host 111 is a physical MacBook/Ollama node. It remains in whole-host probes,
@@ -26,8 +26,8 @@ WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_S
 WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}"
 POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS="${POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS:-900}"
 AI_LOG_TRIAGE_AUTO_RECOVERY_MODE="${AI_LOG_TRIAGE_AUTO_RECOVERY_MODE:-check}"
-AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-420}"
-AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS:-20}"
+AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-60}"
+AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS:-5}"
 
 mkdir -p "$TEXTFILE_DIR" "$LOG_DIR"
 

scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py

@@ -836,7 +836,7 @@ def source_controls() -> dict[str, bool]:
         "slo_systemd_service_ai_log_triage_apply_source_present": file_contains(
             source_file("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service"),
             "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply",
-            "AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420",
+            "AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=60",
         ),
         "full_host_reboot_orchestrator_source_present": file_contains(
             source_file("scripts/reboot-recovery/full-host-reboot-orchestrator.sh"),

scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py

@@ -92,6 +92,19 @@ def test_coordinator_covers_all_hosts_and_full_recovery_evidence() -> None:
         assert field in control
 
 
+def test_host188_hygiene_is_advisory_not_a_recovery_completion_gate() -> None:
+    control = read("agent99-control-plane.ps1")
+    conversion = control.split("function Convert-AgentRecoveryReadback", 1)[1]
+    conversion = conversion.split("function Invoke-AgentRecoveryCoordinatorReadback", 1)[0]
+
+    advisory = conversion.split("$advisoryChecks = @(", 1)[1].split("$checks = @(", 1)[0]
+    mandatory = conversion.split("$checks = @(", 1)[1].split("$sloChecks = @()", 1)[0]
+    assert 'name = "host_188_hygiene_green"' in advisory
+    assert 'name = "host_188_hygiene_green"' not in mandatory
+    assert "advisoryFailedChecks = $advisoryFailedChecks" in conversion
+    assert "verified = [bool]($failedChecks.Count -eq 0)" in conversion
+
+
 def test_fresh_reboot_claim_requires_fresh_artifact_and_zero_blockers() -> None:
     control = read("agent99-control-plane.ps1")
 

scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py

@@ -87,7 +87,33 @@ def test_service_uses_deployed_source_root_and_bounded_oneshot() -> None:
     assert "Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles" in text
     assert "Environment=LOG_DIR=/home/wooo/reboot-recovery" in text
     assert "ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh" in text
-    assert "TimeoutStartSec=600" in text
+    assert "TimeoutStartSec=330" in text
+
+
+def test_service_worst_case_probe_budget_finishes_before_agent99_freshness_window() -> None:
+    text = SERVICE.read_text(encoding="utf-8")
+
+    def environment_seconds(name: str) -> int:
+        match = re.search(rf"^Environment={name}=([0-9]+)$", text, re.MULTILINE)
+        assert match is not None, name
+        return int(match.group(1))
+
+    service_timeout_match = re.search(r"^TimeoutStartSec=([0-9]+)$", text, re.MULTILINE)
+    assert service_timeout_match is not None
+    service_timeout = int(service_timeout_match.group(1))
+    agent99_freshness_wait = 420
+    worst_case_budget = (
+        environment_seconds("AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS")
+        + environment_seconds("AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS")
+        + environment_seconds("SSH_COMMAND_TIMEOUT_SECONDS") * 6
+        + environment_seconds("POST_REBOOT_READINESS_TIMEOUT_SECONDS")
+        + environment_seconds("STOCK_READBACK_TIMEOUT_SECONDS") * 2
+        + environment_seconds("PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS") * 6
+        + environment_seconds("WINDOWS99_REMOTE_VERIFY_TIMEOUT")
+    )
+
+    assert worst_case_budget <= service_timeout - 30
+    assert service_timeout < agent99_freshness_wait
 
 
 def test_timer_waits_after_completion_instead_of_running_back_to_back() -> None:
@@ -136,7 +162,7 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
     assert "windows99_vmware_verify_fallback_max_age_seconds" in text
     assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text
     assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
-    assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"' in text
+    assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}"' in text
     assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text
     assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
     assert "POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1" in text