Merge remote-tracking branch 'gitea-ssh/main' into codex/p0-obs-002-credential-20260722
All checks were successful
CD Pipeline / select-latest-carrier (push) Successful in 49s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 41s
CD Pipeline / build-and-deploy (push) Successful in 14m46s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 21s
CD Pipeline / revalidate-post-deploy-carrier (push) Successful in 39s
CD Pipeline / post-deploy-checks (push) Successful in 2m50s
All checks were successful
CD Pipeline / select-latest-carrier (push) Successful in 49s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m34s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 41s
CD Pipeline / build-and-deploy (push) Successful in 14m46s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 21s
CD Pipeline / revalidate-post-deploy-carrier (push) Successful in 39s
CD Pipeline / post-deploy-checks (push) Successful in 2m50s
This commit is contained in:
@@ -3009,70 +3009,99 @@ jobs:
|
||||
# 2026-05-02 ogt + Claude Sonnet 4.6: 加主機完整性檢查
|
||||
# 根因:partial scan(如 110 timeout、其他成功)會讓 [-s file] 通過、
|
||||
# 後續 patch 推進缺漏的 known_hosts → asyncssh 拒所有 SSH。
|
||||
# 修法:scan 完驗證所有 allowlisted 主機都在;缺任何一台就 abort,
|
||||
# 不能覆蓋現有 secret,防止 production SSH 自動修復路徑癱瘓。
|
||||
# 修法:核心 executor 主機必須完整;Host111 是隔離的 provider lane,
|
||||
# 首次 migration 掃不到時不得拖垮其他產品部署。既有 111 pin
|
||||
# 只能保留,不能被 partial scan 覆蓋。
|
||||
: > /tmp/known_hosts_repair
|
||||
: > /tmp/known_hosts_scan_err
|
||||
EXPECTED_HOSTS=6
|
||||
CORE_HOSTS="192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188"
|
||||
SCAN_HOSTS="192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188"
|
||||
CORE_EXPECTED_HOSTS=5
|
||||
TARGET_HOSTS=6
|
||||
PRESENT_CORE=0
|
||||
HOST111_PRESENT=0
|
||||
PRESENT=0
|
||||
for scan_attempt in 1 2 3; do
|
||||
ssh-keyscan -T 5 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \
|
||||
>> /tmp/known_hosts_repair 2>>/tmp/known_hosts_scan_err || true
|
||||
sort -u /tmp/known_hosts_repair -o /tmp/known_hosts_repair
|
||||
PRESENT=0
|
||||
for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
|
||||
PRESENT_CORE=0
|
||||
for ip in \$CORE_HOSTS; do
|
||||
if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
|
||||
PRESENT=\$((PRESENT + 1))
|
||||
PRESENT_CORE=\$((PRESENT_CORE + 1))
|
||||
fi
|
||||
done
|
||||
if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
|
||||
HOST111_PRESENT=0
|
||||
if grep -qE '^192\.168\.0\.111[[:space:]]' /tmp/known_hosts_repair 2>/dev/null; then
|
||||
HOST111_PRESENT=1
|
||||
fi
|
||||
PRESENT=\$((PRESENT_CORE + HOST111_PRESENT))
|
||||
if [ "\$PRESENT" -eq "\$TARGET_HOSTS" ]; then
|
||||
break
|
||||
fi
|
||||
sleep "\$scan_attempt"
|
||||
done
|
||||
for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
|
||||
for ip in \$SCAN_HOSTS; do
|
||||
if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
|
||||
:
|
||||
else
|
||||
echo "⚠️ ssh-keyscan 缺主機 \${ip}"
|
||||
fi
|
||||
done
|
||||
if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
|
||||
|
||||
EXISTING_KNOWN_HOSTS=/tmp/known_hosts_existing
|
||||
EXISTING_CORE=0
|
||||
EXISTING_HOST111=0
|
||||
if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \
|
||||
-o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then
|
||||
for ip in \$CORE_HOSTS; do
|
||||
if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
|
||||
EXISTING_CORE=\$((EXISTING_CORE + 1))
|
||||
fi
|
||||
done
|
||||
if grep -qE '^192\.168\.0\.111[[:space:]]' "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
|
||||
EXISTING_HOST111=1
|
||||
fi
|
||||
fi
|
||||
EXISTING_PRESENT=\$((EXISTING_CORE + EXISTING_HOST111))
|
||||
|
||||
TRUST_SOURCE=""
|
||||
TRUST_SELECTION=""
|
||||
if [ "\$PRESENT_CORE" -eq "\$CORE_EXPECTED_HOSTS" ] && [ "\$HOST111_PRESENT" -eq 1 ]; then
|
||||
TRUST_SOURCE=/tmp/known_hosts_repair
|
||||
TRUST_SELECTION=fresh_6_of_6
|
||||
elif [ "\$EXISTING_CORE" -eq "\$CORE_EXPECTED_HOSTS" ] && [ "\$EXISTING_HOST111" -eq 1 ]; then
|
||||
TRUST_SELECTION=preserve_existing_6_of_6
|
||||
elif [ "\$PRESENT_CORE" -eq "\$CORE_EXPECTED_HOSTS" ]; then
|
||||
TRUST_SOURCE=/tmp/known_hosts_repair
|
||||
TRUST_SELECTION=fresh_core_5_host111_trust_deferred
|
||||
elif [ "\$EXISTING_CORE" -eq "\$CORE_EXPECTED_HOSTS" ]; then
|
||||
TRUST_SELECTION=preserve_existing_core_5_host111_trust_deferred
|
||||
else
|
||||
echo "❌ host-key core trust incomplete; fresh=\${PRESENT_CORE}/\${CORE_EXPECTED_HOSTS} existing=\${EXISTING_CORE}/\${CORE_EXPECTED_HOSTS}"
|
||||
rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "\$TRUST_SOURCE" ]; then
|
||||
\$KUBECTL create secret generic awoooi-repair-known-hosts \
|
||||
-n awoooi-prod \
|
||||
--from-file=known_hosts=/tmp/known_hosts_repair \
|
||||
--from-file=known_hosts="\$TRUST_SOURCE" \
|
||||
--dry-run=client -o yaml | \$KUBECTL apply -f - \
|
||||
&& echo "✅ awoooi-repair-known-hosts Secret 已建立/更新" \
|
||||
|| echo "⚠️ awoooi-repair-known-hosts Secret 建立失敗 (非致命)"
|
||||
KNOWN_HOSTS_B64=\$(base64 -w 0 /tmp/known_hosts_repair)
|
||||
KNOWN_HOSTS_B64=\$(base64 -w 0 "\$TRUST_SOURCE")
|
||||
\$KUBECTL patch secret ssh-mcp-key -n awoooi-prod --type=merge \
|
||||
-p="{\"data\":{\"known_hosts\":\"\${KNOWN_HOSTS_B64}\"}}" \
|
||||
&& echo "✅ ssh-mcp-key known_hosts 已更新(6 台主機完整)" \
|
||||
&& echo "✅ ssh-mcp-key known_hosts 已更新 selection=\${TRUST_SELECTION}" \
|
||||
|| { echo "❌ ssh-mcp-key known_hosts 更新失敗,停止部署"; exit 1; }
|
||||
rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err
|
||||
else
|
||||
# A transient guest SSH outage must not overwrite trust data, but it
|
||||
# also must not blindly green-light the deploy. Decode only on the
|
||||
# K3s controller, verify all six public host entries, then delete
|
||||
# the temporary file without printing its contents.
|
||||
EXISTING_KNOWN_HOSTS=/tmp/known_hosts_existing
|
||||
EXISTING_PRESENT=0
|
||||
if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \
|
||||
-o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then
|
||||
for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
|
||||
if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
|
||||
EXISTING_PRESENT=\$((EXISTING_PRESENT + 1))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
|
||||
if [ "\$EXISTING_PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
|
||||
echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 6/6 and preserved"
|
||||
else
|
||||
echo "❌ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key coverage \${EXISTING_PRESENT}/\${EXPECTED_HOSTS}"
|
||||
exit 1
|
||||
fi
|
||||
echo "⚠️ ssh-mcp-key known_hosts preserved selection=\${TRUST_SELECTION} fresh=\${PRESENT}/\${TARGET_HOSTS} existing=\${EXISTING_PRESENT}/\${TARGET_HOSTS}"
|
||||
fi
|
||||
if [ "\$HOST111_PRESENT" -eq 0 ] && [ "\$EXISTING_HOST111" -eq 0 ]; then
|
||||
echo "⚠️ host111_trust_deferred isolated_lane=ansible:111-ollama-fallback"
|
||||
fi
|
||||
rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
|
||||
|
||||
echo "✅ 所有 Secrets 注入完成"
|
||||
SECRETS
|
||||
|
||||
@@ -3894,6 +3894,12 @@ function Convert-AgentRecoveryReadback {
|
||||
if (-not $row -or -not $row.reachable) { $requiredHostFailures += $alias }
|
||||
}
|
||||
|
||||
$advisoryChecks = @(
|
||||
[pscustomobject]@{
|
||||
name = "host_188_hygiene_green"
|
||||
passed = [bool]((& $get "HOST_188_HYGIENE_BLOCKED" "1") -eq "0")
|
||||
}
|
||||
)
|
||||
$checks = @(
|
||||
[pscustomobject]@{ name = "readback_present"; passed = (& $asBool "RECOVERY_READBACK") },
|
||||
[pscustomobject]@{ name = "artifact_recent"; passed = $artifactRecent },
|
||||
@@ -3904,7 +3910,6 @@ function Convert-AgentRecoveryReadback {
|
||||
[pscustomobject]@{ name = "stock_freshness_ok"; passed = [bool]((& $get "STOCK_FRESHNESS_STATUS") -eq "ok") },
|
||||
[pscustomobject]@{ name = "backup_core_green"; passed = (& $asBool "BACKUP_CORE_GREEN") },
|
||||
[pscustomobject]@{ name = "host_188_service_green"; passed = (& $asBool "HOST_188_SERVICE_GREEN") },
|
||||
[pscustomobject]@{ name = "host_188_hygiene_green"; passed = [bool]((& $get "HOST_188_HYGIENE_BLOCKED" "1") -eq "0") },
|
||||
[pscustomobject]@{ name = "edge_fallback_ready"; passed = (& $asBool "EDGE_FALLBACK_READY") },
|
||||
[pscustomobject]@{ name = "windows99_vmware_ready"; passed = (& $asBool "VMWARE_AUTOSTART_VERIFY_READY") },
|
||||
[pscustomobject]@{ name = "windows99_update_policy_ready"; passed = (& $asBool "WINDOWS_UPDATE_NO_AUTO_REBOOT_READY") },
|
||||
@@ -3931,6 +3936,7 @@ function Convert-AgentRecoveryReadback {
|
||||
}
|
||||
|
||||
$failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||||
$advisoryFailedChecks = @($advisoryChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||||
$sloFailedChecks = @($sloChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_recovery_coordinator_readback_v1"
|
||||
@@ -3953,6 +3959,8 @@ function Convert-AgentRecoveryReadback {
|
||||
hosts = $hosts
|
||||
checks = $checks
|
||||
failedChecks = $failedChecks
|
||||
advisoryChecks = $advisoryChecks
|
||||
advisoryFailedChecks = $advisoryFailedChecks
|
||||
sloChecks = $sloChecks
|
||||
sloFailedChecks = $sloFailedChecks
|
||||
}
|
||||
|
||||
@@ -4,16 +4,21 @@ ROOT = Path(__file__).resolve().parents[3]
|
||||
WORKFLOW = ROOT / ".gitea/workflows/cd.yaml"
|
||||
|
||||
|
||||
def test_cd_known_hosts_gate_retries_and_verifies_preserved_secret() -> None:
|
||||
def test_cd_known_hosts_gate_keeps_core_fail_closed_and_host111_isolated() -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
gate = workflow[workflow.index(": > /tmp/known_hosts_repair") :]
|
||||
gate = gate[: gate.index('echo "✅ 所有 Secrets 注入完成"')]
|
||||
|
||||
assert "for scan_attempt in 1 2 3" in gate
|
||||
assert 'sleep "\\$scan_attempt"' in gate
|
||||
assert "existing ssh-mcp-key verified 6/6 and preserved" in gate
|
||||
assert "EXISTING_PRESENT=\\$((EXISTING_PRESENT + 1))" in gate
|
||||
assert 'if [ "\\$EXISTING_PRESENT" -eq "\\$EXPECTED_HOSTS" ]' in gate
|
||||
assert "CORE_EXPECTED_HOSTS=5" in gate
|
||||
assert "TARGET_HOSTS=6" in gate
|
||||
assert "fresh_6_of_6" in gate
|
||||
assert "preserve_existing_6_of_6" in gate
|
||||
assert "fresh_core_5_host111_trust_deferred" in gate
|
||||
assert "preserve_existing_core_5_host111_trust_deferred" in gate
|
||||
assert "host-key core trust incomplete" in gate
|
||||
assert "host111_trust_deferred isolated_lane=ansible:111-ollama-fallback" in gate
|
||||
assert 'rm -f "\\$EXISTING_KNOWN_HOSTS"' in gate
|
||||
assert "cat /tmp/known_hosts_scan_err" not in gate
|
||||
assert "existing ssh-mcp-key coverage" in gate
|
||||
assert "KNOWN_HOSTS_B64=\\$(base64 -w 0 \"\\$TRUST_SOURCE\")" in gate
|
||||
|
||||
@@ -192,8 +192,12 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
|
||||
"192.168.0.121 192.168.0.188"
|
||||
)
|
||||
assert f"ssh-keyscan -T 5 {expected_known_hosts}" in workflow
|
||||
assert "EXPECTED_HOSTS=6" in workflow
|
||||
assert workflow.count(expected_known_hosts) >= 4
|
||||
assert "CORE_EXPECTED_HOSTS=5" in workflow
|
||||
assert "TARGET_HOSTS=6" in workflow
|
||||
assert "fresh_core_5_host111_trust_deferred" in workflow
|
||||
assert "preserve_existing_core_5_host111_trust_deferred" in workflow
|
||||
assert "host111_trust_deferred isolated_lane=ansible:111-ollama-fallback" in workflow
|
||||
assert workflow.count(expected_known_hosts) >= 2
|
||||
assert workflow.count(expected_hosts) >= 2
|
||||
assert "except ConnectionRefusedError:" in workflow
|
||||
assert "broker_ssh_refused_but_egress_permitted=" in workflow
|
||||
|
||||
@@ -15,10 +15,15 @@ Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles
|
||||
Environment=LOG_DIR=/home/wooo/reboot-recovery
|
||||
Environment=TARGET_MINUTES=10
|
||||
Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply
|
||||
Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420
|
||||
Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS=20
|
||||
Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=60
|
||||
Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS=5
|
||||
Environment=SSH_COMMAND_TIMEOUT_SECONDS=8
|
||||
Environment=POST_REBOOT_READINESS_TIMEOUT_SECONDS=60
|
||||
Environment=STOCK_READBACK_TIMEOUT_SECONDS=10
|
||||
Environment=PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS=5
|
||||
Environment=WINDOWS99_REMOTE_VERIFY_TIMEOUT=30
|
||||
ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh
|
||||
TimeoutStartSec=600
|
||||
TimeoutStartSec=330
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
|
||||
@@ -17,8 +17,8 @@ LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}"
|
||||
STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}"
|
||||
STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}"
|
||||
STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}"
|
||||
POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"
|
||||
PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}"
|
||||
POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}"
|
||||
PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-5}"
|
||||
PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}"
|
||||
WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}"
|
||||
# Host 111 is a physical MacBook/Ollama node. It remains in whole-host probes,
|
||||
@@ -26,8 +26,8 @@ WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_S
|
||||
WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}"
|
||||
POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS="${POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS:-900}"
|
||||
AI_LOG_TRIAGE_AUTO_RECOVERY_MODE="${AI_LOG_TRIAGE_AUTO_RECOVERY_MODE:-check}"
|
||||
AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-420}"
|
||||
AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS:-20}"
|
||||
AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-60}"
|
||||
AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS:-5}"
|
||||
|
||||
mkdir -p "$TEXTFILE_DIR" "$LOG_DIR"
|
||||
|
||||
|
||||
@@ -836,7 +836,7 @@ def source_controls() -> dict[str, bool]:
|
||||
"slo_systemd_service_ai_log_triage_apply_source_present": file_contains(
|
||||
source_file("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service"),
|
||||
"AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply",
|
||||
"AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420",
|
||||
"AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=60",
|
||||
),
|
||||
"full_host_reboot_orchestrator_source_present": file_contains(
|
||||
source_file("scripts/reboot-recovery/full-host-reboot-orchestrator.sh"),
|
||||
|
||||
@@ -92,6 +92,19 @@ def test_coordinator_covers_all_hosts_and_full_recovery_evidence() -> None:
|
||||
assert field in control
|
||||
|
||||
|
||||
def test_host188_hygiene_is_advisory_not_a_recovery_completion_gate() -> None:
|
||||
control = read("agent99-control-plane.ps1")
|
||||
conversion = control.split("function Convert-AgentRecoveryReadback", 1)[1]
|
||||
conversion = conversion.split("function Invoke-AgentRecoveryCoordinatorReadback", 1)[0]
|
||||
|
||||
advisory = conversion.split("$advisoryChecks = @(", 1)[1].split("$checks = @(", 1)[0]
|
||||
mandatory = conversion.split("$checks = @(", 1)[1].split("$sloChecks = @()", 1)[0]
|
||||
assert 'name = "host_188_hygiene_green"' in advisory
|
||||
assert 'name = "host_188_hygiene_green"' not in mandatory
|
||||
assert "advisoryFailedChecks = $advisoryFailedChecks" in conversion
|
||||
assert "verified = [bool]($failedChecks.Count -eq 0)" in conversion
|
||||
|
||||
|
||||
def test_fresh_reboot_claim_requires_fresh_artifact_and_zero_blockers() -> None:
|
||||
control = read("agent99-control-plane.ps1")
|
||||
|
||||
|
||||
@@ -87,7 +87,33 @@ def test_service_uses_deployed_source_root_and_bounded_oneshot() -> None:
|
||||
assert "Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles" in text
|
||||
assert "Environment=LOG_DIR=/home/wooo/reboot-recovery" in text
|
||||
assert "ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh" in text
|
||||
assert "TimeoutStartSec=600" in text
|
||||
assert "TimeoutStartSec=330" in text
|
||||
|
||||
|
||||
def test_service_worst_case_probe_budget_finishes_before_agent99_freshness_window() -> None:
|
||||
text = SERVICE.read_text(encoding="utf-8")
|
||||
|
||||
def environment_seconds(name: str) -> int:
|
||||
match = re.search(rf"^Environment={name}=([0-9]+)$", text, re.MULTILINE)
|
||||
assert match is not None, name
|
||||
return int(match.group(1))
|
||||
|
||||
service_timeout_match = re.search(r"^TimeoutStartSec=([0-9]+)$", text, re.MULTILINE)
|
||||
assert service_timeout_match is not None
|
||||
service_timeout = int(service_timeout_match.group(1))
|
||||
agent99_freshness_wait = 420
|
||||
worst_case_budget = (
|
||||
environment_seconds("AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS")
|
||||
+ environment_seconds("AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS")
|
||||
+ environment_seconds("SSH_COMMAND_TIMEOUT_SECONDS") * 6
|
||||
+ environment_seconds("POST_REBOOT_READINESS_TIMEOUT_SECONDS")
|
||||
+ environment_seconds("STOCK_READBACK_TIMEOUT_SECONDS") * 2
|
||||
+ environment_seconds("PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS") * 6
|
||||
+ environment_seconds("WINDOWS99_REMOTE_VERIFY_TIMEOUT")
|
||||
)
|
||||
|
||||
assert worst_case_budget <= service_timeout - 30
|
||||
assert service_timeout < agent99_freshness_wait
|
||||
|
||||
|
||||
def test_timer_waits_after_completion_instead_of_running_back_to_back() -> None:
|
||||
@@ -136,7 +162,7 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
|
||||
assert "windows99_vmware_verify_fallback_max_age_seconds" in text
|
||||
assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text
|
||||
assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
|
||||
assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"' in text
|
||||
assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-60}"' in text
|
||||
assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text
|
||||
assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
|
||||
assert "POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1" in text
|
||||
|
||||
Reference in New Issue
Block a user