fix(agent99): harden scorecard readback
This commit is contained in:
@@ -1629,7 +1629,7 @@ function Invoke-AgentRecoveryCoordinatorReadback {
|
||||
}
|
||||
|
||||
$readbackCommand = @'
|
||||
latest="$(find /home/wooo/reboot-recovery -maxdepth 1 -type d -name 'reboot-auto-recovery-slo-*' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-)"
|
||||
latest="$(find /home/wooo/reboot-recovery -mindepth 2 -maxdepth 2 -type f -name 'scorecard.json' -printf '%T@ %h\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-)"
|
||||
if [ -z "$latest" ]; then
|
||||
printf 'RECOVERY_READBACK=0\n'
|
||||
exit 0
|
||||
@@ -1641,7 +1641,7 @@ grep -E '^(POST_START_RESULT|POST_START_BLOCKED|SERVICE_GREEN|PRODUCT_DATA_GREEN
|
||||
grep -E '^HOST_BOOT ' "$latest/host-probe.txt" 2>/dev/null || true
|
||||
grep -E '^(EDGE_FALLBACK_READY|PRIVILEGED_EXECUTOR_READY|EDGE_EXECUTOR_PAYLOAD_PARITY)=' "$latest/public-maintenance-edge-fallback.txt" 2>/dev/null || true
|
||||
grep -E '^(VMWARE_AUTOSTART_CONFIG_READY|VMWARE_AUTOSTART_POWER_READY|WINDOWS_UPDATE_NO_AUTO_REBOOT_READY|VMWARE_AUTOSTART_VERIFY_READY)=' "$latest/windows99-vmware-verify.txt" 2>/dev/null || true
|
||||
python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); r=d.get("required_checks",{}); o=d.get("rollups",{}); b=lambda v:"1" if v else "0"; print("SCORECARD_STATUS="+str(d.get("status", ""))); print("SCORECARD_BLOCKER_COUNT="+str(o.get("active_blocker_count", -1))); print("SCORECARD_READINESS_PERCENT="+str(o.get("readiness_percent", -1))); print("SCORECARD_PRIMARY_BLOCKER="+str(d.get("primary_blocker", ""))); print("SCORECARD_CAN_CLAIM="+b(d.get("can_claim_all_services_recovered_within_target"))); print("SCORECARD_FRESH_REBOOT_WINDOW="+b(r.get("fresh_reboot_window_observed"))); print("SCORECARD_SOURCE_CONTROLS_READY="+b(r.get("source_controls_present"))); print("SCORECARD_PUBLIC_FALLBACK_READY="+b(r.get("public_maintenance_fallback_runtime_ready")))' "$latest/scorecard.json" 2>/dev/null || true
|
||||
printf %s aW1wb3J0IGpzb24sc3lzCmQ9anNvbi5sb2FkKG9wZW4oc3lzLmFyZ3ZbMV0pKQpyPWQuZ2V0KCJyZXF1aXJlZF9jaGVja3MiLHt9KQpvPWQuZ2V0KCJyb2xsdXBzIix7fSkKYj1sYW1iZGEgdjoiMSIgaWYgdiBlbHNlICIwIgpwcmludCgiU0NPUkVDQVJEX1NUQVRVUz0iK3N0cihkLmdldCgic3RhdHVzIiwiIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRVJfQ09VTlQ9IitzdHIoby5nZXQoImFjdGl2ZV9ibG9ja2VyX2NvdW50IiwtMSkpKQpwcmludCgiU0NPUkVDQVJEX1JFQURJTkVTU19QRVJDRU5UPSIrc3RyKG8uZ2V0KCJyZWFkaW5lc3NfcGVyY2VudCIsLTEpKSkKcHJpbnQoIlNDT1JFQ0FSRF9QUklNQVJZX0JMT0NLRVI9IitzdHIoZC5nZXQoInByaW1hcnlfYmxvY2tlciIsIiIpKSkKcHJpbnQoIlNDT1JFQ0FSRF9DQU5fQ0xBSU09IitiKGQuZ2V0KCJjYW5fY2xhaW1fYWxsX3NlcnZpY2VzX3JlY292ZXJlZF93aXRoaW5fdGFyZ2V0IikpKQpwcmludCgiU0NPUkVDQVJEX0ZSRVNIX1JFQk9PVF9XSU5ET1c9IitiKHIuZ2V0KCJmcmVzaF9yZWJvb3Rfd2luZG93X29ic2VydmVkIikpKQpwcmludCgiU0NPUkVDQVJEX1NPVVJDRV9DT05UUk9MU19SRUFEWT0iK2Ioci5nZXQoInNvdXJjZV9jb250cm9sc19wcmVzZW50IikpKQpwcmludCgiU0NPUkVDQVJEX1BVQkxJQ19GQUxMQkFDS19SRUFEWT0iK2Ioci5nZXQoInB1YmxpY19tYWludGVuYW5jZV9mYWxsYmFja19ydW50aW1lX3JlYWR5IikpKQ== | base64 -d | python3 - "$latest/scorecard.json" 2>/dev/null || true
|
||||
'@
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($(if ($RequireFreshRebootWindow) { $coordinator.freshWaitTimeoutSeconds } else { 1 }))
|
||||
|
||||
@@ -144,6 +144,17 @@ function Get-EvidenceReadback {
|
||||
})
|
||||
}
|
||||
} else { $null }
|
||||
recoveryCoordinator = if ($rawEvidence.recoveryCoordinator) {
|
||||
[pscustomobject]@{
|
||||
verified = [bool]$rawEvidence.recoveryCoordinator.verified
|
||||
requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow
|
||||
rebootSloClaimed = [bool]$rawEvidence.recoveryCoordinator.rebootSloClaimed
|
||||
scorecardStatus = [string]$rawEvidence.recoveryCoordinator.scorecardStatus
|
||||
scorecardBlockerCount = [int]$rawEvidence.recoveryCoordinator.scorecardBlockerCount
|
||||
readinessPercent = [int]$rawEvidence.recoveryCoordinator.readinessPercent
|
||||
failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks)
|
||||
}
|
||||
} else { $null }
|
||||
}
|
||||
} catch {
|
||||
$parseError = $_.Exception.GetType().Name
|
||||
@@ -242,6 +253,8 @@ $evidence = @(
|
||||
$operatingSystem = Get-CimInstance Win32_OperatingSystem
|
||||
$taskFailures = @($tasks | Where-Object { -not $_.healthy })
|
||||
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.fresh })
|
||||
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
|
||||
$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
|
||||
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
|
||||
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
|
||||
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
|
||||
@@ -252,6 +265,7 @@ if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blocking
|
||||
if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
||||
if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
|
||||
if ($requiredEvidenceFailures.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
|
||||
if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" }
|
||||
if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
|
||||
if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
|
||||
|
||||
@@ -275,6 +289,7 @@ $result = [pscustomobject]@{
|
||||
taskFailureCount = $taskFailures.Count
|
||||
freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count
|
||||
requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
|
||||
selfHealthSeverity = $selfHealthSeverity
|
||||
alertIncomingCount = [int]$alertIncoming.count
|
||||
alertRunningCount = [int]$alertRunning.count
|
||||
pendingCommandCount = [int]$commandQueue.count
|
||||
@@ -292,6 +307,7 @@ Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summar
|
||||
Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)"
|
||||
Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
|
||||
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
|
||||
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
|
||||
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
|
||||
Write-Output "SECRET_VALUE_READ=0"
|
||||
Write-Output "REMOTE_WRITE_PERFORMED=0"
|
||||
|
||||
@@ -75,6 +75,8 @@ def test_coordinator_reads_only_bounded_summary_artifacts() -> None:
|
||||
assert "public-maintenance-edge-fallback.txt" in coordinator
|
||||
assert "windows99-vmware-verify.txt" in coordinator
|
||||
assert "scorecard.json" in coordinator
|
||||
assert "base64 -d | python3 -" in coordinator
|
||||
assert "-type f -name 'scorecard.json'" in coordinator
|
||||
assert "*.log" not in coordinator
|
||||
assert ".env" not in coordinator
|
||||
assert "auth.json" not in coordinator
|
||||
@@ -88,6 +90,8 @@ def test_live_preflight_is_no_secret_and_no_remote_write() -> None:
|
||||
assert 'remoteWritePerformed = $false' in preflight
|
||||
assert 'Write-Output "SECRET_VALUE_READ=0"' in preflight
|
||||
assert 'Write-Output "REMOTE_WRITE_PERFORMED=0"' in preflight
|
||||
assert 'Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"' in preflight
|
||||
assert 'if ($selfHealthSeverity -ne "ok")' in preflight
|
||||
assert "runtime-manifest.json" in preflight
|
||||
assert "Get-ScheduledTask" in preflight
|
||||
assert "Get-NetTCPConnection -LocalPort 8787" in preflight
|
||||
|
||||
Reference in New Issue
Block a user