diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index ff2f49b09..2a4c701e0 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -212,6 +212,9 @@ Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" Add-Check "recovery:host111_wol_executor" ($control.Contains("Invoke-AgentExternalHostRecovery") -and $control.Contains("Send-AgentWakeOnLan") -and $control.Contains('Get-AgentObjectValue $externalHost "recoveryAction" "wake_on_lan"') -and $control.Contains('New-AgentRecoveryPhase "external_host_recovery"') -and $control.Contains('New-AgentOutcomeCheck "external_hosts_recovered"')) "Host111 has an allowlisted WOL executor with independent ping/TCP verification" Add-Check "recovery:startup_queue_closure" ($taskRegistration.Contains('$submitRequest') -and $taskRegistration.Contains('-Mode Recover -ControlledApply -RunNow') -and $taskRegistration.Contains('-Source agent99-startup-recovery')) "startup recovery runs through queue, outcome, callback, Telegram lifecycle, and KM/RAG" +Add-Check "backup:canonical_seven_host_truth" ($control.Contains("Get-AgentBackupCanonicalHostDefinitions") -and $control.Contains('assetId = "host:$($contract.alias)"') -and $control.Contains('backupEvidenceReadyCount') -and $control.Contains('host_backup_contract_not_registered')) "BackupCheck inventories 99/110/111/112/120/121/188 and separates runtime readback from verified backup evidence" +Add-Check "backup:bounded_google_drive_quota" ($control.Contains("Test-AgentGoogleDriveQuotaReadback") -and $control.Contains("timeout --signal=TERM --kill-after=5s") -and $control.Contains('canonicalRemoteCountedOnce = $true') -and $control.Contains('capacity_unknown_fail_closed') -and $control.Contains('rawOutputStored = $false')) "Google Drive quota uses one canonical remote, bounded read-only transport, sanitized evidence and fail-closed capacity truth" +Add-Check "backup:escrow_receipt_timestamp" ($control.Contains('"escrow-status:${mtime}:missing-0"')) "escrow receipt binds the exact metrics mtime without PowerShell scoped-variable interpolation" Add-Check "recovery:startup_controlled_identity" ($submitRequest.Contains("New-AgentControlledDispatchIdentity") -and $submitRequest.Contains("New-AgentUuidV5") -and $submitRequest.Contains("agent99_controlled_dispatch_identity_v1") -and $submitRequest.Contains("canonicalDigest") -and $submitRequest.Contains("singleFlightKey") -and $submitRequest.Contains("approvalId")) "local and startup controlled requests carry the canonical AWOOOI dispatch identity" Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, fixed WAL, attributed start, immutable receipt/readback and independent verifier" Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 0d4618ade..d7b77125e 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -4989,6 +4989,11 @@ function Get-AgentBackupHealthConfig { protectionReadTimeoutSeconds = if ($backupHealth -and $backupHealth.protectionReadTimeoutSeconds) { [int]$backupHealth.protectionReadTimeoutSeconds } else { 45 } protectionMaxAgeMinutes = if ($backupHealth -and $backupHealth.protectionMaxAgeMinutes) { [double]$backupHealth.protectionMaxAgeMinutes } else { 60 } protectionEscrowExpectedCount = if ($backupHealth -and $backupHealth.protectionEscrowExpectedCount) { [int]$backupHealth.protectionEscrowExpectedCount } else { 5 } + hostReadTimeoutSeconds = if ($backupHealth -and $backupHealth.PSObject.Properties["hostReadTimeoutSeconds"]) { [int]$backupHealth.hostReadTimeoutSeconds } else { 15 } + driveQuotaHost = if ($backupHealth -and $backupHealth.PSObject.Properties["driveQuotaHost"]) { [string]$backupHealth.driveQuotaHost } else { $backupHost } + driveQuotaRemote = if ($backupHealth -and $backupHealth.PSObject.Properties["driveQuotaRemote"]) { [string]$backupHealth.driveQuotaRemote } else { "gdrive_awoooi_restic:" } + driveQuotaCommandTimeoutSeconds = if ($backupHealth -and $backupHealth.PSObject.Properties["driveQuotaCommandTimeoutSeconds"]) { [int]$backupHealth.driveQuotaCommandTimeoutSeconds } else { 30 } + driveQuotaTransportTimeoutSeconds = if ($backupHealth -and $backupHealth.PSObject.Properties["driveQuotaTransportTimeoutSeconds"]) { [int]$backupHealth.driveQuotaTransportTimeoutSeconds } else { 45 } } } @@ -5133,7 +5138,7 @@ function Convert-AgentBackupProtectionReadback { escrow = [pscustomobject]@{ ok = $escrowOk missingCount = $escrowMissing - receiptId = if ($escrowOk) { "escrow-status:$mtime:missing-0" } else { $null } + receiptId = if ($escrowOk) { "escrow-status:${mtime}:missing-0" } else { $null } reason = if ($escrowOk) { "non_secret_metrics_verified" } elseif ($escrowMissing -lt 0) { "escrow_missing_metric_absent" } else { "escrow_evidence_incomplete" } freshMetricCount = $escrowFresh.Count expectedFreshMetricCount = $ExpectedEscrowFreshCount @@ -5182,6 +5187,127 @@ function Test-AgentBackupProtectionReceipts { Convert-AgentBackupProtectionReadback $readback $maxAgeMinutes ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds()) ([int]$BackupConfig.protectionEscrowExpectedCount) } +function Convert-AgentGoogleDriveQuotaReadback { + param( + [object]$Readback, + [string]$HostIp, + [string]$RemoteName, + [int]$CommandTimeoutSeconds + ) + + $output = [string](Get-AgentObjectValue $Readback "output" "") + $remoteExitMatch = [regex]::Match($output, "__AGENT99_QUOTA_EXIT__=(-?[0-9]+)") + $remoteExitCode = if ($remoteExitMatch.Success) { [int]$remoteExitMatch.Groups[1].Value } else { $null } + $jsonMatch = [regex]::Match( + $output, + "(?s)__AGENT99_QUOTA_JSON_BEGIN__\s*(?\{.*?\})\s*__AGENT99_QUOTA_JSON_END__" + ) + $quota = $null + $parseError = $false + if ($jsonMatch.Success) { + try { + $quota = $jsonMatch.Groups["json"].Value | ConvertFrom-Json -ErrorAction Stop + } catch { + $parseError = $true + } + } + + $totalBytes = if ($quota) { Get-AgentObjectValue $quota "total" $null } else { $null } + $freeBytes = if ($quota) { Get-AgentObjectValue $quota "free" $null } else { $null } + $usedBytes = if ($quota) { Get-AgentObjectValue $quota "used" $null } else { $null } + $trashedBytes = if ($quota) { Get-AgentObjectValue $quota "trashed" $null } else { $null } + $otherBytes = if ($quota) { Get-AgentObjectValue $quota "other" $null } else { $null } + $totalParsed = [int64]0 + $freeParsed = [int64]0 + $usedParsed = [int64]0 + $trashedParsed = [int64]0 + $otherParsed = [int64]0 + $totalValid = [int64]::TryParse([string]$totalBytes, [Globalization.NumberStyles]::Integer, [Globalization.CultureInfo]::InvariantCulture, [ref]$totalParsed) + $freeValid = [int64]::TryParse([string]$freeBytes, [Globalization.NumberStyles]::Integer, [Globalization.CultureInfo]::InvariantCulture, [ref]$freeParsed) + $usedValid = [int64]::TryParse([string]$usedBytes, [Globalization.NumberStyles]::Integer, [Globalization.CultureInfo]::InvariantCulture, [ref]$usedParsed) + $trashedValid = [bool]($null -ne $trashedBytes -and [int64]::TryParse([string]$trashedBytes, [Globalization.NumberStyles]::Integer, [Globalization.CultureInfo]::InvariantCulture, [ref]$trashedParsed)) + $otherValid = [bool]($null -ne $otherBytes -and [int64]::TryParse([string]$otherBytes, [Globalization.NumberStyles]::Integer, [Globalization.CultureInfo]::InvariantCulture, [ref]$otherParsed)) + $shapeValid = [bool]( + $totalValid -and $totalParsed -gt 0 -and + $freeValid -and $freeParsed -ge 0 -and + $usedValid -and $usedParsed -ge 0 + ) + $transportOk = [bool](Get-AgentObjectValue $Readback "ok" $false) + $ok = [bool]($transportOk -and $remoteExitCode -eq 0 -and $jsonMatch.Success -and -not $parseError -and $shapeValid) + $reason = if ($ok) { + "quota_readback_verified" + } elseif ($remoteExitCode -eq 124 -or $output -match "timeout_after_[0-9]+s") { + "quota_probe_timeout" + } elseif (-not $transportOk) { + "quota_transport_failed" + } elseif (-not $jsonMatch.Success -or $parseError) { + "quota_json_invalid" + } else { + "quota_shape_invalid" + } + + [pscustomobject]@{ + schemaVersion = "agent99_google_drive_quota_readback_v1" + ok = $ok + severity = if ($ok) { "ok" } else { "critical" } + reason = $reason + host = $HostIp + remote = $RemoteName + totalBytes = if ($shapeValid) { $totalParsed } else { $null } + freeBytes = if ($shapeValid) { $freeParsed } else { $null } + usedBytes = if ($shapeValid) { $usedParsed } else { $null } + trashedBytes = if ($shapeValid -and $trashedValid) { $trashedParsed } else { $null } + otherBytes = if ($shapeValid -and $otherValid) { $otherParsed } else { $null } + receiptId = if ($ok) { "gdrive-quota:${totalParsed}:${freeParsed}" } else { $null } + remoteExitCode = $remoteExitCode + transportExitCode = Get-AgentObjectValue $Readback "exitCode" $null + route = Get-AgentObjectValue $Readback "route" $null + elapsedMs = Get-AgentObjectValue $Readback "elapsedMs" $null + commandTimeoutSeconds = $CommandTimeoutSeconds + canonicalRemoteCountedOnce = $true + capacityDecision = if ($ok) { "quota_observed_capacity_model_required" } else { "capacity_unknown_fail_closed" } + rawOutputStored = $false + secretValueRead = $false + runtimeWritePerformed = $false + } +} + +function Test-AgentGoogleDriveQuotaReadback { + param([object]$BackupConfig) + + $hostIp = [string]$BackupConfig.driveQuotaHost + $remoteName = [string]$BackupConfig.driveQuotaRemote + $commandTimeoutSeconds = [math]::Min(60, [math]::Max(10, [int]$BackupConfig.driveQuotaCommandTimeoutSeconds)) + $transportTimeoutSeconds = [math]::Min(120, [math]::Max($commandTimeoutSeconds + 10, [int]$BackupConfig.driveQuotaTransportTimeoutSeconds)) + if ( + $hostIp -notmatch "^192\.168\.0\.(99|110|111|112|120|121|188)$" -or + $remoteName -notmatch "^[A-Za-z0-9_.-]+:$" + ) { + return [pscustomobject]@{ + schemaVersion = "agent99_google_drive_quota_readback_v1" + ok = $false + severity = "critical" + reason = "quota_target_contract_invalid" + host = $hostIp + remote = $remoteName + receiptId = $null + canonicalRemoteCountedOnce = $true + rawOutputStored = $false + secretValueRead = $false + runtimeWritePerformed = $false + } + } + + $quotedRemote = Quote-ShSingle $remoteName + $command = ( + 'set +e; printf "__AGENT99_QUOTA_JSON_BEGIN__\n"; ' + + "timeout --signal=TERM --kill-after=5s ${commandTimeoutSeconds}s rclone about $quotedRemote --json 2>/dev/null; " + + 'rc=$?; printf "\n__AGENT99_QUOTA_JSON_END__\n__AGENT99_QUOTA_EXIT__=%s\n" "$rc"; exit "$rc"' + ) + $readback = Invoke-HostSshText $hostIp $command $transportTimeoutSeconds 1 + Convert-AgentGoogleDriveQuotaReadback $readback $hostIp $remoteName $commandTimeoutSeconds +} + function Invoke-AgentBackupReadbackContractSelfTest { $fixture = [pscustomobject]@{ ok = $true @@ -5247,6 +5373,34 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 $remoteVerifyResult = Convert-AgentBackupProtectionReadback $remoteVerifyFixture 60 1783785660 $escrowMissingResult = Convert-AgentBackupProtectionReadback $escrowMissingFixture 60 1783785660 $escrowPartialResult = Convert-AgentBackupProtectionReadback $escrowPartialFixture 60 1783785660 + $quotaFixture = [pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + elapsedMs = 120 + output = @" +__AGENT99_QUOTA_JSON_BEGIN__ +{"total":5497558138880,"used":1160992511030,"trashed":0,"other":0,"free":4336565627850} +__AGENT99_QUOTA_JSON_END__ +__AGENT99_QUOTA_EXIT__=0 +"@ + } + $quotaResult = Convert-AgentGoogleDriveQuotaReadback $quotaFixture "192.168.0.110" "gdrive_awoooi_restic:" 30 + $quotaTimeoutResult = Convert-AgentGoogleDriveQuotaReadback ([pscustomobject]@{ + ok = $false + route = "selftest-read-only" + exitCode = -2 + elapsedMs = 45000 + output = "timeout_after_45s" + }) "192.168.0.110" "gdrive_awoooi_restic:" 30 + $quotaMalformedResult = Convert-AgentGoogleDriveQuotaReadback ([pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + elapsedMs = 50 + output = "__AGENT99_QUOTA_JSON_BEGIN__`nnot-json`n__AGENT99_QUOTA_JSON_END__`n__AGENT99_QUOTA_EXIT__=0" + }) "192.168.0.110" "gdrive_awoooi_restic:" 30 + $canonicalHosts = @(Get-AgentBackupCanonicalHostDefinitions) $terminalEligible = [bool]($result.offsiteVerify.ok -and $result.escrow.ok) $ok = [bool]( $terminalEligible -and @@ -5268,6 +5422,17 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 -not $remoteVerifyResult.offsiteVerify.ok -and -not $escrowMissingResult.escrow.ok -and -not $escrowPartialResult.escrow.ok -and + $quotaResult.ok -and + $quotaResult.totalBytes -eq 5497558138880 -and + $quotaResult.freeBytes -eq 4336565627850 -and + $quotaResult.receiptId -eq "gdrive-quota:5497558138880:4336565627850" -and + $quotaResult.canonicalRemoteCountedOnce -and + -not $quotaTimeoutResult.ok -and + $quotaTimeoutResult.reason -eq "quota_probe_timeout" -and + -not $quotaMalformedResult.ok -and + $quotaMalformedResult.reason -eq "quota_json_invalid" -and + $canonicalHosts.Count -eq 7 -and + @($canonicalHosts.assetId | Sort-Object -Unique).Count -eq 7 -and -not $result.backupRunPerformed -and -not $result.restoreRunPerformed -and -not $result.markerWritePerformed @@ -5284,8 +5449,14 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 configuredProviderRemoteVerifyFailedBlocked = [bool](-not $remoteVerifyResult.offsiteVerify.ok) escrowMissingBlocked = [bool](-not $escrowMissingResult.escrow.ok) escrowPartialFreshnessBlocked = [bool](-not $escrowPartialResult.escrow.ok) + boundedQuotaReadbackVerified = [bool]$quotaResult.ok + quotaTimeoutBlocked = [bool](-not $quotaTimeoutResult.ok) + malformedQuotaBlocked = [bool](-not $quotaMalformedResult.ok) + canonicalHostInventoryComplete = [bool]($canonicalHosts.Count -eq 7) } result = $result + quotaResult = $quotaResult + canonicalHosts = $canonicalHosts } } @@ -5530,6 +5701,143 @@ function Test-AgentBackupCronPatterns { $results } +function Get-AgentBackupCanonicalHostDefinitions { + $configuredHosts = if ($Config.PSObject.Properties["hosts"]) { @($Config.hosts | ForEach-Object { [string]$_ }) } else { @() } + $contracts = @( + [pscustomobject]@{ alias = "99"; role = "windows_execution_hub"; assets = @("windows_system", "vmware_inventory", "agent99_control_plane") }, + [pscustomobject]@{ alias = "110"; role = "backup_controller"; assets = @("gitea", "harbor", "backup_controller", "monitoring") }, + [pscustomobject]@{ alias = "111"; role = "workstation"; assets = @("workstation_safe_artifacts") }, + [pscustomobject]@{ alias = "112"; role = "security_node"; assets = @("wazuh_security_state") }, + [pscustomobject]@{ alias = "120"; role = "k3s_control_plane"; assets = @("k3s_etcd", "k3s_local_pv") }, + [pscustomobject]@{ alias = "121"; role = "k3s_worker"; assets = @("k3s_local_pv", "node_config") }, + [pscustomobject]@{ alias = "188"; role = "product_and_ai_host"; assets = @("product_datastores", "docker_volumes", "ai_assets") } + ) + foreach ($contract in $contracts) { + $hostIp = "192.168.0.$($contract.alias)" + [pscustomobject]@{ + assetId = "host:$($contract.alias)" + alias = [string]$contract.alias + host = $hostIp + role = [string]$contract.role + expectedBackupAssets = @($contract.assets) + required = $true + inventoryCovered = [bool]($contract.alias -eq "99" -or $hostIp -in $configuredHosts) + } + } +} + +function Test-AgentBackupHostRuntimeReadback { + param( + [object]$Definition, + [int]$TimeoutSeconds + ) + + if (-not $Definition.inventoryCovered) { + return [pscustomobject]@{ + assetId = $Definition.assetId + alias = $Definition.alias + host = $Definition.host + role = $Definition.role + expectedBackupAssets = @($Definition.expectedBackupAssets) + required = [bool]$Definition.required + inventoryCovered = $false + runtimeReadbackOk = $false + runtimeReason = "canonical_host_missing_from_agent_config" + hostName = $null + uptimeSeconds = $null + usedBytes = $null + route = "inventory" + exitCode = $null + elapsedMs = 0 + readOnly = $true + runtimeWritePerformed = $false + secretValueRead = $false + } + } + + if ($Definition.alias -eq "99") { + try { + $os = Get-CimInstance Win32_OperatingSystem -ErrorAction Stop + $fixedDrives = @(Get-CimInstance Win32_LogicalDisk -Filter "DriveType=3" -ErrorAction Stop) + $usedBytes = [int64]0 + foreach ($drive in $fixedDrives) { + if ($null -ne $drive.Size -and $null -ne $drive.FreeSpace) { + $usedBytes += ([int64]$drive.Size - [int64]$drive.FreeSpace) + } + } + return [pscustomobject]@{ + assetId = $Definition.assetId + alias = $Definition.alias + host = $Definition.host + role = $Definition.role + expectedBackupAssets = @($Definition.expectedBackupAssets) + required = [bool]$Definition.required + inventoryCovered = $true + runtimeReadbackOk = $true + runtimeReason = "windows_local_readback_ok" + hostName = [string]$env:COMPUTERNAME + uptimeSeconds = [math]::Max(0, [math]::Round(((Get-Date) - $os.LastBootUpTime).TotalSeconds, 0)) + usedBytes = $usedBytes + route = "windows99_local" + exitCode = 0 + elapsedMs = 0 + readOnly = $true + runtimeWritePerformed = $false + secretValueRead = $false + } + } catch { + return [pscustomobject]@{ + assetId = $Definition.assetId + alias = $Definition.alias + host = $Definition.host + role = $Definition.role + expectedBackupAssets = @($Definition.expectedBackupAssets) + required = [bool]$Definition.required + inventoryCovered = $true + runtimeReadbackOk = $false + runtimeReason = "windows_local_readback_failed" + hostName = $null + uptimeSeconds = $null + usedBytes = $null + route = "windows99_local" + exitCode = -1 + elapsedMs = 0 + readOnly = $true + runtimeWritePerformed = $false + secretValueRead = $false + } + } + } + + $boundedTimeout = [math]::Min(45, [math]::Max(5, $TimeoutSeconds)) + $command = "printf '__HOSTNAME__='; hostname; printf '__UPTIME_SECONDS__='; cut -d ' ' -f1 /proc/uptime; printf '__ROOT_USED_BYTES__='; df -B1 --output=used / 2>/dev/null | tail -n 1 | tr -d ' '" + $readback = Invoke-HostSshText $Definition.host $command $boundedTimeout 1 + $hostNameMatch = [regex]::Match([string]$readback.output, "__HOSTNAME__=([^\r\n]+)") + $uptimeMatch = [regex]::Match([string]$readback.output, "__UPTIME_SECONDS__=([0-9]+(?:\.[0-9]+)?)") + $usedMatch = [regex]::Match([string]$readback.output, "__ROOT_USED_BYTES__=([0-9]+)") + $runtimeReadbackOk = [bool]($readback.ok -and $hostNameMatch.Success -and $uptimeMatch.Success -and $usedMatch.Success) + [pscustomobject]@{ + assetId = $Definition.assetId + alias = $Definition.alias + host = $Definition.host + role = $Definition.role + expectedBackupAssets = @($Definition.expectedBackupAssets) + required = [bool]$Definition.required + inventoryCovered = $true + runtimeReadbackOk = $runtimeReadbackOk + runtimeReason = if ($runtimeReadbackOk) { "linux_metadata_readback_ok" } else { "linux_metadata_readback_failed" } + hostName = if ($hostNameMatch.Success) { $hostNameMatch.Groups[1].Value.Trim() } else { $null } + uptimeSeconds = if ($uptimeMatch.Success) { [double]::Parse($uptimeMatch.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture) } else { $null } + usedBytes = if ($usedMatch.Success) { [int64]$usedMatch.Groups[1].Value } else { $null } + route = Get-AgentObjectValue $readback "route" $null + exitCode = Get-AgentObjectValue $readback "exitCode" $null + elapsedMs = Get-AgentObjectValue $readback "elapsedMs" $null + readOnly = $true + runtimeWritePerformed = $false + secretValueRead = $false + } +} + function Test-AgentBackupHealth { $backupConfig = Get-AgentBackupHealthConfig $targets = @() @@ -5544,6 +5852,7 @@ function Test-AgentBackupHealth { $cron = Test-AgentBackupCronPatterns @($backupConfig.cronPatterns) $backupConfig.host $protection = Test-AgentBackupProtectionReceipts $backupConfig + $driveQuota = Test-AgentGoogleDriveQuotaReadback $backupConfig $targetCritical = @($targets | Where-Object { $_.severity -eq "critical" }).Count $targetWarning = @($targets | Where-Object { $_.severity -eq "warning" }).Count @@ -5552,7 +5861,55 @@ function Test-AgentBackupHealth { $cronCritical = @($cron | Where-Object { $_.severity -eq "critical" }).Count $cronWarning = @($cron | Where-Object { $_.severity -eq "warning" }).Count $protectionCritical = @(@($protection.offsiteVerify, $protection.escrow) | Where-Object { -not $_.ok }).Count - $critical = $targetCritical + $fileCritical + $cronCritical + $protectionCritical + $quotaCritical = if ($driveQuota.ok) { 0 } else { 1 } + + $controllerBackupReady = [bool]( + $targetCritical -eq 0 -and + $fileCritical -eq 0 -and + $cronCritical -eq 0 -and + $protectionCritical -eq 0 -and + $driveQuota.ok + ) + $hostCoverage = @() + foreach ($definition in @(Get-AgentBackupCanonicalHostDefinitions)) { + $runtime = Test-AgentBackupHostRuntimeReadback $definition ([int]$backupConfig.hostReadTimeoutSeconds) + $backupEvidenceReady = [bool]($definition.alias -eq "110" -and $controllerBackupReady) + $backupEvidenceReason = if ($definition.alias -eq "110") { + if ($controllerBackupReady) { "controller_backup_contract_verified" } else { "controller_backup_contract_degraded" } + } else { + "host_backup_contract_not_registered" + } + $rowSeverity = if ( + -not $runtime.inventoryCovered -or + -not $runtime.runtimeReadbackOk -or + -not $backupEvidenceReady + ) { "critical" } else { "ok" } + $hostCoverage += [pscustomobject]@{ + assetId = $runtime.assetId + alias = $runtime.alias + host = $runtime.host + role = $runtime.role + expectedBackupAssets = @($runtime.expectedBackupAssets) + required = [bool]$runtime.required + inventoryCovered = [bool]$runtime.inventoryCovered + runtimeReadbackOk = [bool]$runtime.runtimeReadbackOk + runtimeReason = $runtime.runtimeReason + backupEvidenceReady = $backupEvidenceReady + backupEvidenceReason = $backupEvidenceReason + severity = $rowSeverity + hostName = $runtime.hostName + uptimeSeconds = $runtime.uptimeSeconds + usedBytes = $runtime.usedBytes + route = $runtime.route + exitCode = $runtime.exitCode + elapsedMs = $runtime.elapsedMs + readOnly = $true + runtimeWritePerformed = $false + secretValueRead = $false + } + } + $hostCoverageCritical = @($hostCoverage | Where-Object { $_.required -and $_.severity -eq "critical" }).Count + $critical = $targetCritical + $fileCritical + $cronCritical + $protectionCritical + $quotaCritical + $hostCoverageCritical $warning = $targetWarning + $fileWarning + $cronWarning $severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } @@ -5564,12 +5921,18 @@ function Test-AgentBackupHealth { cronCritical = $cronCritical cronWarning = $cronWarning protectionCritical = $protectionCritical + quotaCritical = $quotaCritical + hostCoverageCritical = $hostCoverageCritical targetCount = @($targets).Count fileCheckCount = @($files).Count cronPatternCount = @($cron).Count + canonicalHostCount = @($hostCoverage).Count + inventoryCoveredCount = @($hostCoverage | Where-Object { $_.inventoryCovered }).Count + runtimeReadbackOkCount = @($hostCoverage | Where-Object { $_.runtimeReadbackOk }).Count + backupEvidenceReadyCount = @($hostCoverage | Where-Object { $_.backupEvidenceReady }).Count } if ($severity -in @("warning", "critical")) { - Record-AgentEvent "backup_health_degraded" $severity "severity=$severity targetCritical=$targetCritical fileCritical=$fileCritical cronCritical=$cronCritical" $summary -Alert + Record-AgentEvent "backup_health_degraded" $severity "severity=$severity hostClosure=$($summary.backupEvidenceReadyCount)/$($summary.canonicalHostCount) targetCritical=$targetCritical fileCritical=$fileCritical cronCritical=$cronCritical offsiteCritical=$protectionCritical quotaCritical=$quotaCritical" $summary -Alert } else { Write-AgentLog "backup_health ok targets=$($summary.targetCount) files=$($summary.fileCheckCount) cron=$($summary.cronPatternCount)" } @@ -5581,6 +5944,8 @@ function Test-AgentBackupHealth { targets = $targets files = $files cron = $cron + canonicalHosts = $hostCoverage + googleDriveQuota = $driveQuota offsiteVerify = $protection.offsiteVerify escrow = $protection.escrow protectionReadback = $protection diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py index 7f18c6a94..e909907bb 100644 --- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py +++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py @@ -167,7 +167,7 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None source = _CONTROL_PLANE.read_text(encoding="utf-8") readback = source[ source.index("function Test-AgentBackupProtectionReceipts") : - source.index("function Invoke-AgentBackupReadbackContractSelfTest") + source.index("function Convert-AgentGoogleDriveQuotaReadback") ] assert "[switch]$SelfTestBackupReadbackContract" in source @@ -220,3 +220,55 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None assert "-not $escrowPartialResult.escrow.ok" in source for mutation in ("rclone ", "rsync ", " rm ", "restore ", "Set-Content"): assert mutation not in readback + + +def test_agent99_backupcheck_covers_all_hosts_and_bounds_drive_quota() -> None: + source = _CONTROL_PLANE.read_text(encoding="utf-8") + host_contract = source[ + source.index("function Get-AgentBackupCanonicalHostDefinitions") : + source.index("function Test-AgentBackupHealth") + ] + quota_contract = source[ + source.index("function Convert-AgentGoogleDriveQuotaReadback") : + source.index("function Invoke-AgentBackupReadbackContractSelfTest") + ] + + for host_alias in ("99", "110", "111", "112", "120", "121", "188"): + assert f'alias = "{host_alias}"' in host_contract + for asset in ( + "windows_system", + "vmware_inventory", + "agent99_control_plane", + "gitea", + "harbor", + "workstation_safe_artifacts", + "wazuh_security_state", + "k3s_etcd", + "k3s_local_pv", + "product_datastores", + "docker_volumes", + "ai_assets", + ): + assert f'"{asset}"' in host_contract + + assert "runtimeReadbackOk" in source + assert "backupEvidenceReady" in source + assert '"host_backup_contract_not_registered"' in source + assert "backupEvidenceReadyCount" in source + assert 'schemaVersion = "agent99_google_drive_quota_readback_v1"' in quota_contract + assert "timeout --signal=TERM --kill-after=5s" in quota_contract + assert "rclone about" in quota_contract + assert 'remoteName -notmatch "^[A-Za-z0-9_.-]+:$"' in quota_contract + assert 'canonicalRemoteCountedOnce = $true' in quota_contract + assert 'capacityDecision = if ($ok)' in quota_contract + assert 'rawOutputStored = $false' in quota_contract + assert 'secretValueRead = $false' in quota_contract + for mutation in ("rclone copy", "rclone sync", "rclone delete", "rclone purge"): + assert mutation not in quota_contract + + +def test_agent99_backupcheck_escrow_receipt_uses_exact_mtime() -> None: + source = _CONTROL_PLANE.read_text(encoding="utf-8") + + assert '"escrow-status:${mtime}:missing-0"' in source + assert '"escrow-status:$mtime:missing-0"' not in source