diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 42de86857..69d52d61a 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -3468,6 +3468,7 @@ function Get-AgentBackupHealthConfig { fileChecks = if ($backupHealth -and $backupHealth.fileChecks) { @($backupHealth.fileChecks) } else { $fileChecks } cronPatterns = if ($backupHealth -and $backupHealth.cronPatterns) { @($backupHealth.cronPatterns) } else { $cronPatterns } protectionMetricsPath = if ($backupHealth -and $backupHealth.protectionMetricsPath) { [string]$backupHealth.protectionMetricsPath } else { "/home/wooo/node_exporter_textfiles/backup_health.prom" } + protectionVerifyMetricsPath = if ($backupHealth -and $backupHealth.protectionVerifyMetricsPath) { [string]$backupHealth.protectionVerifyMetricsPath } else { "/home/wooo/node_exporter_textfiles/offsite_full_sync_verify.prom" } protectionReadTimeoutSeconds = if ($backupHealth -and $backupHealth.protectionReadTimeoutSeconds) { [int]$backupHealth.protectionReadTimeoutSeconds } else { 45 } protectionMaxAgeMinutes = if ($backupHealth -and $backupHealth.protectionMaxAgeMinutes) { [double]$backupHealth.protectionMaxAgeMinutes } else { 60 } protectionEscrowExpectedCount = if ($backupHealth -and $backupHealth.protectionEscrowExpectedCount) { [int]$backupHealth.protectionEscrowExpectedCount } else { 5 } @@ -3527,8 +3528,12 @@ function Convert-AgentBackupProtectionReadback { $readback = $Readback $mtimeMatch = [regex]::Match([string]$readback.output, "__PROTECTION_MTIME__=([0-9]+)") $mtime = if ($mtimeMatch.Success) { $mtimeMatch.Groups[1].Value } else { "" } + $verifyMtimeMatch = [regex]::Match([string]$readback.output, "__OFFSITE_VERIFY_MTIME__=([0-9]+)") + $verifyMtime = if ($verifyMtimeMatch.Success) { $verifyMtimeMatch.Groups[1].Value } else { "" } $evidenceAgeMinutes = if ($mtime) { ($NowUnix - [double]$mtime) / 60 } else { $null } $evidenceFresh = [bool]($null -ne $evidenceAgeMinutes -and $evidenceAgeMinutes -ge -5 -and $evidenceAgeMinutes -le [math]::Max(1, $MaxAgeMinutes)) + $verifyEvidenceAgeMinutes = if ($verifyMtime) { ($NowUnix - [double]$verifyMtime) / 60 } else { $null } + $verifyEvidenceFresh = [bool]($null -ne $verifyEvidenceAgeMinutes -and $verifyEvidenceAgeMinutes -ge -5 -and $verifyEvidenceAgeMinutes -le [math]::Max(1, $MaxAgeMinutes)) $configured = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_configured") $offsiteFresh = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_fresh") $remoteVerify = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_remote_verify_ok") @@ -3580,6 +3585,7 @@ function Convert-AgentBackupProtectionReadback { } $offsiteOk = [bool]( $readback.ok -and $mtime -and $evidenceFresh -and + $verifyMtime -and $verifyEvidenceFresh -and $configuredShapeValid -and $configuredProviderNames.Count -gt 0 -and $verifiedProviderNames.Count -eq $configuredProviderNames.Count @@ -3593,8 +3599,8 @@ function Convert-AgentBackupProtectionReadback { [pscustomobject]@{ offsiteVerify = [pscustomobject]@{ ok = $offsiteOk - receiptId = if ($offsiteOk) { "offsite-verify:$mtime" } else { $null } - reason = if ($offsiteOk) { "read_only_metrics_verified" } elseif (-not $readback.ok) { "protection_metrics_readback_failed" } else { "offsite_verify_not_green" } + receiptId = if ($offsiteOk) { "offsite-verify:$verifyMtime" } else { $null } + reason = if ($offsiteOk) { "read_only_metrics_verified" } elseif (-not $readback.ok) { "protection_metrics_readback_failed" } elseif (-not $verifyEvidenceFresh) { "offsite_verify_evidence_stale_or_missing" } else { "offsite_verify_not_green" } configuredMetricCount = $configured.Count configuredProviderCount = $configuredProviderNames.Count verifiedProviderCount = $verifiedProviderNames.Count @@ -3603,6 +3609,8 @@ function Convert-AgentBackupProtectionReadback { remoteVerifyMetricCount = $remoteVerify.Count evidenceFresh = $evidenceFresh evidenceAgeMinutes = $evidenceAgeMinutes + verifyEvidenceFresh = $verifyEvidenceFresh + verifyEvidenceAgeMinutes = $verifyEvidenceAgeMinutes maxAgeMinutes = $MaxAgeMinutes } escrow = [pscustomobject]@{ @@ -3617,6 +3625,7 @@ function Convert-AgentBackupProtectionReadback { maxAgeMinutes = $MaxAgeMinutes } evidenceMtime = $mtime + verifyEvidenceMtime = $verifyMtime route = $readback.route exitCode = $readback.exitCode readOnly = $true @@ -3631,9 +3640,13 @@ function Test-AgentBackupProtectionReceipts { $hostIp = [string]$BackupConfig.host $path = [string]$BackupConfig.protectionMetricsPath + $verifyPath = [string]$BackupConfig.protectionVerifyMetricsPath $timeoutSeconds = [math]::Min(120, [math]::Max(5, [int]$BackupConfig.protectionReadTimeoutSeconds)) $maxAgeMinutes = [math]::Min(1440, [math]::Max(1, [double]$BackupConfig.protectionMaxAgeMinutes)) - if (-not $path -or $path -notmatch "^/[A-Za-z0-9_.\/-]+$" -or $path -match "\.\.") { + if ( + -not $path -or $path -notmatch "^/[A-Za-z0-9_.\/-]+$" -or $path -match "\.\." -or + -not $verifyPath -or $verifyPath -notmatch "^/[A-Za-z0-9_.\/-]+$" -or $verifyPath -match "\.\." + ) { return [pscustomobject]@{ offsiteVerify = [pscustomobject]@{ ok = $false; receiptId = $null; reason = "invalid_protection_metrics_path" } escrow = [pscustomobject]@{ ok = $false; missingCount = -1; receiptId = $null; reason = "invalid_protection_metrics_path" } @@ -3645,8 +3658,9 @@ function Test-AgentBackupProtectionReceipts { } $quotedPath = Quote-ShSingle $path + $quotedVerifyPath = Quote-ShSingle $verifyPath $metricPattern = "awoooi_backup_(offsite_configured|offsite_fresh|offsite_remote_verify_ok|credential_escrow_fresh|dr_credential_escrow_missing_count)" - $command = "if [ -r $quotedPath ]; then stat -c '__PROTECTION_MTIME__=%Y' $quotedPath; grep -E '^$metricPattern([ {])' $quotedPath || true; else echo __PROTECTION_MISSING__; fi" + $command = "if [ -r $quotedPath ]; then stat -c '__PROTECTION_MTIME__=%Y' $quotedPath; grep -E '^$metricPattern([ {])' $quotedPath || true; else echo __PROTECTION_MISSING__; fi; if [ -r $quotedVerifyPath ]; then stat -c '__OFFSITE_VERIFY_MTIME__=%Y' $quotedVerifyPath; grep -E '^awoooi_backup_offsite_remote_verify_ok([ {])' $quotedVerifyPath || true; else echo __OFFSITE_VERIFY_MISSING__; fi" $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1 Convert-AgentBackupProtectionReadback $readback $maxAgeMinutes ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds()) ([int]$BackupConfig.protectionEscrowExpectedCount) } @@ -3658,6 +3672,7 @@ function Invoke-AgentBackupReadbackContractSelfTest { exitCode = 0 output = @" __PROTECTION_MTIME__=1783785600 +__OFFSITE_VERIFY_MTIME__=1783785600 awoooi_backup_offsite_configured{host="110",provider="b2"} 0 awoooi_backup_offsite_configured{host="110",provider="rclone"} 1 awoooi_backup_offsite_fresh{host="110",provider="b2"} 0 @@ -3673,6 +3688,13 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 } $result = Convert-AgentBackupProtectionReadback $fixture 60 1783785660 $staleResult = Convert-AgentBackupProtectionReadback $fixture 60 1783792800 + $staleVerifyFixture = [pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + output = $fixture.output -replace '__OFFSITE_VERIFY_MTIME__=1783785600', '__OFFSITE_VERIFY_MTIME__=1783778400' + } + $staleVerifyResult = Convert-AgentBackupProtectionReadback $staleVerifyFixture 60 1783785660 $noProviderFixture = [pscustomobject]@{ ok = $true route = "selftest-read-only" @@ -3722,6 +3744,8 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 $result.readOnly -and -not $staleResult.offsiteVerify.ok -and -not $staleResult.escrow.ok -and + -not $staleVerifyResult.offsiteVerify.ok -and + $staleVerifyResult.escrow.ok -and -not $noProviderResult.offsiteVerify.ok -and -not $staleProviderResult.offsiteVerify.ok -and -not $remoteVerifyResult.offsiteVerify.ok -and @@ -3737,6 +3761,7 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 terminalEligible = $terminalEligible cases = [pscustomobject]@{ optionalB2ProviderTerminalEligible = $terminalEligible + staleOffsiteVerifyEvidenceBlocked = [bool](-not $staleVerifyResult.offsiteVerify.ok -and $staleVerifyResult.escrow.ok) allProvidersUnconfiguredBlocked = [bool](-not $noProviderResult.offsiteVerify.ok) configuredProviderStaleBlocked = [bool](-not $staleProviderResult.offsiteVerify.ok) configuredProviderRemoteVerifyFailedBlocked = [bool](-not $remoteVerifyResult.offsiteVerify.ok) diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py index fabffea4b..1c53cd96f 100644 --- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py +++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py @@ -131,6 +131,9 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None ): assert metric in source assert "stat -c '__PROTECTION_MTIME__=%Y'" in readback + assert "protectionVerifyMetricsPath" in source + assert "offsite_full_sync_verify.prom" in source + assert "stat -c '__OFFSITE_VERIFY_MTIME__=%Y'" in readback assert "grep -E" in readback assert "backupRunPerformed = $false" in source assert "restoreRunPerformed = $false" in source @@ -141,6 +144,8 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None assert "protectionEscrowExpectedCount" in source assert "$evidenceAgeMinutes -ge -5" in source assert "$evidenceAgeMinutes -le" in source + assert "$verifyEvidenceAgeMinutes -ge -5" in source + assert "$verifyEvidenceAgeMinutes -le" in source assert "function Get-AgentBackupProviderMetricRows" in source assert "$offsiteFresh.Count -eq $offsiteFreshRows.Count" in source assert "$remoteVerify.Count -eq $remoteVerifyRows.Count" in source @@ -151,6 +156,7 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None assert 'provider="b2"} 0' in source assert 'provider="rclone"} 1' in source assert "optionalB2ProviderTerminalEligible" in source + assert "staleOffsiteVerifyEvidenceBlocked" in source assert "allProvidersUnconfiguredBlocked" in source assert "configuredProviderStaleBlocked" in source assert "configuredProviderRemoteVerifyFailedBlocked" in source