ops(agent99): preserve callback recovery closure
This commit is contained in:
ogt
@@ -0,0 +1,274 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Check", "Apply")]
|
||||
[string]$Mode = "Check",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$TraceId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$RunId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$WorkItemId,
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$safeIdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+\-]{0,127}$"
|
||||
foreach ($identity in @($TraceId, $RunId, $WorkItemId)) {
|
||||
if ($identity -notmatch $safeIdentityPattern) {
|
||||
throw "controlled_identity_invalid"
|
||||
}
|
||||
}
|
||||
|
||||
$pendingDir = Join-Path $AgentRoot "state\completion-callbacks\pending"
|
||||
$processedCommandDir = Join-Path $AgentRoot "queue\processed"
|
||||
$evidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$migrationRoot = Join-Path $AgentRoot "state\completion-callbacks\migration-backups"
|
||||
$lockPath = Join-Path $AgentRoot "state\completion-callbacks\incident-id-migration.lock"
|
||||
$script:MigrationLock = $null
|
||||
|
||||
function Get-Sha256Text {
|
||||
param([string]$Value)
|
||||
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$bytes = [Text.Encoding]::UTF8.GetBytes($Value)
|
||||
return (($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") }) -join "")
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Sha256File {
|
||||
param([string]$Path)
|
||||
return (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Write-Utf8NoBom {
|
||||
param([string]$Path, [string]$Value)
|
||||
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
[IO.File]::WriteAllText($Path, $Value, $encoding)
|
||||
}
|
||||
|
||||
function Write-ImmutableJson {
|
||||
param([string]$Path, [object]$Value)
|
||||
|
||||
if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_exists" }
|
||||
$temporaryPath = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
$json = $Value | ConvertTo-Json -Depth 12
|
||||
$null = $json | ConvertFrom-Json
|
||||
Write-Utf8NoBom $temporaryPath $json
|
||||
[IO.File]::Move($temporaryPath, $Path)
|
||||
} finally {
|
||||
if (Test-Path -LiteralPath $temporaryPath) {
|
||||
Remove-Item -LiteralPath $temporaryPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-CallbackRows {
|
||||
$rows = @()
|
||||
$blockers = @()
|
||||
if (-not (Test-Path -LiteralPath $pendingDir -PathType Container)) {
|
||||
return [pscustomobject]@{ rows = @(); blockers = @() }
|
||||
}
|
||||
|
||||
foreach ($file in @(Get-ChildItem -LiteralPath $pendingDir -Filter "*.json" -File | Sort-Object Name)) {
|
||||
try {
|
||||
$payload = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json
|
||||
$runIdValue = [string]$payload.run_id
|
||||
$priorAlertId = [string]$payload.alert_id
|
||||
if (-not $runIdValue -or $runIdValue -notmatch "^[A-Za-z0-9._-]{1,128}$") {
|
||||
throw "pending_run_id_invalid"
|
||||
}
|
||||
if (-not $priorAlertId) { throw "pending_alert_id_missing" }
|
||||
|
||||
$processedPath = Join-Path $processedCommandDir "processed-$runIdValue.json"
|
||||
if (-not (Test-Path -LiteralPath $processedPath -PathType Leaf)) {
|
||||
throw "processed_result_missing"
|
||||
}
|
||||
$processed = Get-Content -LiteralPath $processedPath -Raw | ConvertFrom-Json
|
||||
$canonicalIncidentId = ""
|
||||
if ($processed.PSObject.Properties["identity"] -and $processed.identity) {
|
||||
$canonicalIncidentId = [string]$processed.identity.incident_id
|
||||
}
|
||||
if (-not $canonicalIncidentId -and $processed.PSObject.Properties["incidentId"]) {
|
||||
$canonicalIncidentId = [string]$processed.incidentId
|
||||
}
|
||||
if (
|
||||
-not $canonicalIncidentId -or
|
||||
$canonicalIncidentId.Length -gt 30 -or
|
||||
$canonicalIncidentId -notmatch "^INC-[A-Za-z0-9-]+$"
|
||||
) {
|
||||
throw "canonical_incident_id_invalid"
|
||||
}
|
||||
|
||||
$rows += [pscustomobject]@{
|
||||
file = $file.Name
|
||||
processedFile = Split-Path -Leaf $processedPath
|
||||
runId = $runIdValue
|
||||
priorAlertIdLength = $priorAlertId.Length
|
||||
priorAlertIdSha256 = Get-Sha256Text $priorAlertId
|
||||
canonicalIncidentId = $canonicalIncidentId
|
||||
needsMigration = [bool]($priorAlertId -ne $canonicalIncidentId)
|
||||
beforeFileSha256 = Get-Sha256File $file.FullName
|
||||
}
|
||||
} catch {
|
||||
$blockers += [pscustomobject]@{
|
||||
file = $file.Name
|
||||
reason = $_.Exception.Message
|
||||
}
|
||||
}
|
||||
}
|
||||
[pscustomobject]@{ rows = @($rows); blockers = @($blockers) }
|
||||
}
|
||||
|
||||
$scan = Get-CallbackRows
|
||||
$candidateRows = @($scan.rows | Where-Object { $_.needsMigration })
|
||||
$identityToken = (Get-Sha256Text "$TraceId|$RunId|$WorkItemId").Substring(0, 20)
|
||||
$receiptPath = Join-Path $evidenceDir "agent99-callback-incident-id-migration-$identityToken.json"
|
||||
$backupDir = Join-Path $migrationRoot "backup-$identityToken"
|
||||
$baseReceipt = [ordered]@{
|
||||
schemaVersion = "agent99_callback_incident_id_migration_v1"
|
||||
status = "pending"
|
||||
observedAt = (Get-Date -Format o)
|
||||
mode = $Mode
|
||||
traceId = $TraceId
|
||||
runId = $RunId
|
||||
workItemId = $WorkItemId
|
||||
pendingCount = @($scan.rows).Count + @($scan.blockers).Count
|
||||
candidateCount = $candidateRows.Count
|
||||
unchangedCount = @($scan.rows | Where-Object { -not $_.needsMigration }).Count
|
||||
blockerCount = @($scan.blockers).Count
|
||||
blockers = @($scan.blockers)
|
||||
rows = @($scan.rows)
|
||||
receiptPath = $receiptPath
|
||||
backupPath = if ($Mode -eq "Apply") { $backupDir } else { $null }
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
rawExternalAlertIdLogged = $false
|
||||
remoteWritePerformed = $false
|
||||
pendingPayloadWritePerformed = $false
|
||||
rollbackAttempted = $false
|
||||
rollbackVerified = $false
|
||||
modifiedCount = 0
|
||||
completedAt = $null
|
||||
failureType = $null
|
||||
}
|
||||
|
||||
if ($Mode -eq "Check") {
|
||||
$baseReceipt.status = if (@($scan.blockers).Count -gt 0) { "blocked_check" } else { "check_ready" }
|
||||
$baseReceipt | ConvertTo-Json -Depth 12 -Compress
|
||||
if (@($scan.blockers).Count -gt 0) { exit 2 }
|
||||
exit 0
|
||||
}
|
||||
|
||||
if (@($scan.blockers).Count -gt 0) {
|
||||
$baseReceipt.status = "blocked_before_apply"
|
||||
$baseReceipt | ConvertTo-Json -Depth 12 -Compress
|
||||
exit 2
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $evidenceDir, $migrationRoot | Out-Null
|
||||
try {
|
||||
$script:MigrationLock = [IO.File]::Open(
|
||||
$lockPath,
|
||||
[IO.FileMode]::OpenOrCreate,
|
||||
[IO.FileAccess]::ReadWrite,
|
||||
[IO.FileShare]::None
|
||||
)
|
||||
} catch {
|
||||
$baseReceipt.status = "blocked_single_flight_lock_unavailable"
|
||||
$baseReceipt | ConvertTo-Json -Depth 12 -Compress
|
||||
exit 75
|
||||
}
|
||||
|
||||
$modifiedRows = @()
|
||||
try {
|
||||
if (Test-Path -LiteralPath $receiptPath) { throw "immutable_receipt_exists" }
|
||||
if (Test-Path -LiteralPath $backupDir) { throw "immutable_backup_path_exists" }
|
||||
New-Item -ItemType Directory -Path $backupDir | Out-Null
|
||||
$baseReceipt.remoteWritePerformed = $true
|
||||
|
||||
foreach ($row in $candidateRows) {
|
||||
$pendingPath = Join-Path $pendingDir $row.file
|
||||
$backupPath = Join-Path $backupDir $row.file
|
||||
$replaceBackupPath = Join-Path $backupDir "$($row.file).replace-original"
|
||||
if ((Get-Sha256File $pendingPath) -ne $row.beforeFileSha256) {
|
||||
throw "pending_payload_changed_after_check"
|
||||
}
|
||||
Copy-Item -LiteralPath $pendingPath -Destination $backupPath
|
||||
if ((Get-Sha256File $backupPath) -ne $row.beforeFileSha256) {
|
||||
throw "pending_payload_backup_mismatch"
|
||||
}
|
||||
|
||||
$payload = Get-Content -LiteralPath $pendingPath -Raw | ConvertFrom-Json
|
||||
$payload.alert_id = $row.canonicalIncidentId
|
||||
$temporaryPath = "$pendingPath.migrate.$PID.$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
$json = $payload | ConvertTo-Json -Depth 12
|
||||
Write-Utf8NoBom $temporaryPath $json
|
||||
$candidate = Get-Content -LiteralPath $temporaryPath -Raw | ConvertFrom-Json
|
||||
if ([string]$candidate.alert_id -ne $row.canonicalIncidentId) {
|
||||
throw "candidate_post_condition_failed"
|
||||
}
|
||||
[IO.File]::Replace($temporaryPath, $pendingPath, $replaceBackupPath, $true)
|
||||
if ((Get-Sha256File $replaceBackupPath) -ne $row.beforeFileSha256) {
|
||||
throw "atomic_replace_backup_mismatch"
|
||||
}
|
||||
$modifiedRows += $row
|
||||
} finally {
|
||||
if (Test-Path -LiteralPath $temporaryPath) {
|
||||
Remove-Item -LiteralPath $temporaryPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
$baseReceipt.pendingPayloadWritePerformed = $true
|
||||
$postPayload = Get-Content -LiteralPath $pendingPath -Raw | ConvertFrom-Json
|
||||
if ([string]$postPayload.alert_id -ne $row.canonicalIncidentId) {
|
||||
throw "post_migration_readback_failed"
|
||||
}
|
||||
}
|
||||
|
||||
$baseReceipt.status = if ($candidateRows.Count -gt 0) { "migrated_verified" } else { "no_change_verified" }
|
||||
$baseReceipt.modifiedCount = $modifiedRows.Count
|
||||
$baseReceipt.completedAt = (Get-Date -Format o)
|
||||
Write-ImmutableJson $receiptPath $baseReceipt
|
||||
$baseReceipt | ConvertTo-Json -Depth 12 -Compress
|
||||
exit 0
|
||||
} catch {
|
||||
$failureType = $_.Exception.Message
|
||||
$baseReceipt.rollbackAttempted = [bool]($modifiedRows.Count -gt 0)
|
||||
$rollbackOk = $true
|
||||
foreach ($row in $modifiedRows) {
|
||||
try {
|
||||
$pendingPath = Join-Path $pendingDir $row.file
|
||||
$backupPath = Join-Path $backupDir $row.file
|
||||
$failedReplacementPath = Join-Path $backupDir "$($row.file).failed-migrated"
|
||||
$temporaryPath = "$pendingPath.rollback.$PID.$([Guid]::NewGuid().ToString('N'))"
|
||||
Copy-Item -LiteralPath $backupPath -Destination $temporaryPath
|
||||
[IO.File]::Replace($temporaryPath, $pendingPath, $failedReplacementPath, $true)
|
||||
if ((Get-Sha256File $pendingPath) -ne $row.beforeFileSha256) {
|
||||
$rollbackOk = $false
|
||||
}
|
||||
} catch {
|
||||
$rollbackOk = $false
|
||||
}
|
||||
}
|
||||
$baseReceipt.rollbackVerified = [bool]($baseReceipt.rollbackAttempted -and $rollbackOk)
|
||||
$baseReceipt.status = if ($baseReceipt.rollbackVerified) { "rolled_back_verified" } else { "migration_failed" }
|
||||
$baseReceipt.failureType = $failureType
|
||||
$baseReceipt.modifiedCount = $modifiedRows.Count
|
||||
$baseReceipt.completedAt = (Get-Date -Format o)
|
||||
if (-not (Test-Path -LiteralPath $receiptPath)) {
|
||||
Write-ImmutableJson $receiptPath $baseReceipt
|
||||
}
|
||||
$baseReceipt | ConvertTo-Json -Depth 12 -Compress
|
||||
exit 2
|
||||
} finally {
|
||||
if ($script:MigrationLock) {
|
||||
$script:MigrationLock.Dispose()
|
||||
$script:MigrationLock = $null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts/reboot-recovery/agent99-completion-callback-incident-id-migration.ps1"
|
||||
|
||||
|
||||
def test_migration_requires_controlled_identity_and_supports_check_apply() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert '[ValidateSet("Check", "Apply")]' in text
|
||||
assert "controlled_identity_invalid" in text
|
||||
assert '"check_ready"' in text
|
||||
assert '"migrated_verified"' in text
|
||||
assert '"rolled_back_verified"' in text
|
||||
|
||||
|
||||
def test_migration_uses_canonical_processed_identity_and_bounded_incident_id() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'processed.identity.incident_id' in text
|
||||
assert '$canonicalIncidentId.Length -gt 30' in text
|
||||
assert '"^INC-[A-Za-z0-9-]+$"' in text
|
||||
assert 'rawExternalAlertIdLogged = $false' in text
|
||||
assert 'priorAlertIdSha256 = Get-Sha256Text $priorAlertId' in text
|
||||
|
||||
|
||||
def test_migration_is_atomic_backed_up_and_fail_closed() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert '[IO.File]::Replace($temporaryPath, $pendingPath, $replaceBackupPath, $true)' in text
|
||||
assert "atomic_replace_backup_mismatch" in text
|
||||
assert 'pending_payload_backup_mismatch' in text
|
||||
assert 'pending_payload_changed_after_check' in text
|
||||
assert 'immutable_receipt_exists' in text
|
||||
assert 'rollbackVerified' in text
|
||||
assert 'secretValueRead = $false' in text
|
||||
Reference in New Issue
Block a user