43d2376d65
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 4m0s
CD Pipeline / build-and-deploy (push) Successful in 14m58s
CD Pipeline / post-deploy-checks (push) Successful in 8m50s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 10s
381 lines
16 KiB
PowerShell
381 lines
16 KiB
PowerShell
param(
|
|
[ValidateSet("Check", "Apply", "Verify")]
|
|
[string]$Mode = "Check",
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$TraceId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RunId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$WorkItemId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$SourceRevision,
|
|
[string]$AgentRoot = "C:\Wooo\Agent99"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$CommandId = "telegram_receipt_index_v2_apply_v1"
|
|
$CanonicalAsset = "public.awooop_outbound_message"
|
|
$MigrationSha256 = "ca36ef38097de2f4a88e950e136ae175fbe1c292ad4a2c30f3a9547c08be1dc8"
|
|
$NormalizedSqlSha256 = "a10e88b55b66eaac7c09933acf33b13f3c9ada73f89c4a099e7eccaefaf1d0eb"
|
|
$NormalizedReindexSqlSha256 = "1857625d45032fba6fea52ebfa6cd0961ffe170b8c09488c773c60a63c75b1b7"
|
|
$JumpHost = "192.168.0.110"
|
|
$ControlPlaneHost = "192.168.0.120"
|
|
$RemoteUser = "wooo"
|
|
$Kubectl = "/usr/local/bin/kubectl"
|
|
$Namespace = "awoooi-prod"
|
|
$PodSelector = "app=awoooi-api,environment=prod,system=awoooi"
|
|
$Container = "api"
|
|
$IdentityFile = Join-Path $AgentRoot "keys\agent99_ed25519"
|
|
$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$"
|
|
$ApiPodPattern = "^awoooi-api-[a-z0-9][a-z0-9-]{0,126}$"
|
|
$ExecutorPod = $null
|
|
$VerifierPod = $null
|
|
$TransientTransportOutputUsed = $false
|
|
$TransportCleanupSucceeded = $true
|
|
|
|
function Write-Agent99DbTerminal {
|
|
param(
|
|
[string]$Terminal,
|
|
[string]$ErrorCode,
|
|
[object]$CheckReceipt,
|
|
[object]$ApplyReceipt,
|
|
[object]$VerifierReceipt,
|
|
[object]$WritesPerformed
|
|
)
|
|
|
|
[pscustomobject]@{
|
|
schemaVersion = "agent99_db_bounded_executor_dispatch_v1"
|
|
commandId = $CommandId
|
|
canonicalAsset = $CanonicalAsset
|
|
domain = "database"
|
|
executor = "db_bounded_executor"
|
|
verifier = "db_independent_verifier"
|
|
executionHub = "host:192.168.0.99"
|
|
dispatchPath = "windows99_agent99_to_host110_to_host120_to_api_pod"
|
|
dispatchOnly = $true
|
|
windowsVmwareMutationPerformed = $false
|
|
kubectlPath = $Kubectl
|
|
namespace = $Namespace
|
|
podSelector = $PodSelector
|
|
executorPod = $ExecutorPod
|
|
verifierPod = $VerifierPod
|
|
traceId = $TraceId
|
|
runId = $RunId
|
|
workItemId = $WorkItemId
|
|
sourceRevision = $SourceRevision
|
|
migrationSha256 = $MigrationSha256
|
|
normalizedSqlSha256 = $NormalizedSqlSha256
|
|
normalizedReindexSqlSha256 = $NormalizedReindexSqlSha256
|
|
terminal = $Terminal
|
|
errorCode = if ($ErrorCode) { $ErrorCode } else { $null }
|
|
checkReceipt = $CheckReceipt
|
|
applyReceipt = $ApplyReceipt
|
|
verifierReceipt = $VerifierReceipt
|
|
independentVerifier = [bool](
|
|
$ApplyReceipt -and
|
|
$VerifierReceipt -and
|
|
[string]$ApplyReceipt.pod_name -and
|
|
[string]$VerifierReceipt.pod_name -and
|
|
[string]$ApplyReceipt.pod_name -ne [string]$VerifierReceipt.pod_name -and
|
|
[string]$ApplyReceipt.process_identity_sha256 -ne [string]$VerifierReceipt.process_identity_sha256
|
|
)
|
|
writesPerformed = $WritesPerformed
|
|
rawSqlAccepted = $false
|
|
arbitraryCommandAllowed = $false
|
|
crossDomainFallbackAllowed = $false
|
|
secretValueRead = $false
|
|
transientTransportOutputUsed = $TransientTransportOutputUsed
|
|
transientTransportOutputDeletedBeforeReceipt = [bool]($TransientTransportOutputUsed -and $TransportCleanupSucceeded)
|
|
rawTransportOutputPersisted = [bool]($TransientTransportOutputUsed -and -not $TransportCleanupSucceeded)
|
|
} | ConvertTo-Json -Compress -Depth 12
|
|
}
|
|
|
|
function Test-Agent99DbReceipt {
|
|
param([object]$Receipt, [string]$ExpectedStage, [string]$ExpectedPod)
|
|
|
|
if (-not $Receipt) { return $false }
|
|
$terminal = [string]$Receipt.terminal
|
|
$terminalAllowed = switch ($ExpectedStage) {
|
|
"check" { $terminal -in @("check_ready", "blocked", "failed") }
|
|
"apply" { $terminal -in @("apply_verified_local", "blocked", "verification_failed", "failed") }
|
|
"verify" { $terminal -in @("verified_healthy", "verification_failed", "failed") }
|
|
default { $false }
|
|
}
|
|
$writeContract = if ($ExpectedStage -eq "apply") {
|
|
[bool](
|
|
$Receipt.write_attempted -is [bool] -and
|
|
(
|
|
$Receipt.writes_performed -is [bool] -or
|
|
($terminal -eq "failed" -and $Receipt.write_attempted -eq $true -and $null -eq $Receipt.writes_performed)
|
|
)
|
|
)
|
|
} else {
|
|
[bool]($Receipt.write_attempted -eq $false -and $Receipt.writes_performed -eq $false)
|
|
}
|
|
$operation = [string]$Receipt.operation
|
|
$operationContract = if ($ExpectedStage -ne "apply") {
|
|
[bool]($operation -eq "none")
|
|
} elseif ($operation -eq "no_write") {
|
|
[bool]($Receipt.write_attempted -eq $false -and $Receipt.writes_performed -eq $false)
|
|
} elseif ($operation -in @("create_index_concurrently", "reindex_index_concurrently")) {
|
|
[bool]($Receipt.write_attempted -eq $true)
|
|
} else {
|
|
[bool]($operation -eq "none" -and $Receipt.write_attempted -eq $false)
|
|
}
|
|
$catalogContract = [bool](
|
|
($terminal -ne "check_ready" -or $Receipt.catalog.apply_ready -eq $true) -and
|
|
($terminal -notin @("apply_verified_local", "verified_healthy") -or (
|
|
$Receipt.catalog.index_shape_exact -eq $true -and
|
|
$Receipt.catalog.catalog_drift -eq $false
|
|
))
|
|
)
|
|
return [bool](
|
|
$terminalAllowed -and
|
|
$writeContract -and
|
|
$operationContract -and
|
|
$catalogContract -and
|
|
[string]$Receipt.schema_version -eq "awoooi_db_bounded_executor_receipt_v2" -and
|
|
[string]$Receipt.command_id -eq $CommandId -and
|
|
[string]$Receipt.canonical_asset -eq $CanonicalAsset -and
|
|
[string]$Receipt.domain -eq "database" -and
|
|
[string]$Receipt.executor -eq "db_bounded_executor" -and
|
|
[string]$Receipt.verifier -eq "db_independent_verifier" -and
|
|
[string]$Receipt.mode -eq $ExpectedStage -and
|
|
[string]$Receipt.trace_id -eq $TraceId -and
|
|
[string]$Receipt.run_id -eq $RunId -and
|
|
[string]$Receipt.work_item_id -eq $WorkItemId -and
|
|
[string]$Receipt.source_revision -eq $SourceRevision -and
|
|
[string]$Receipt.migration_sha256 -eq $MigrationSha256 -and
|
|
[string]$Receipt.normalized_sql_sha256 -eq $NormalizedSqlSha256 -and
|
|
[string]$Receipt.normalized_reindex_sql_sha256 -eq $NormalizedReindexSqlSha256 -and
|
|
[string]$Receipt.pod_name -eq $ExpectedPod -and
|
|
[string]$Receipt.pod_name -match $ApiPodPattern -and
|
|
[string]$Receipt.process_identity_sha256 -match "^[0-9a-f]{64}$" -and
|
|
($null -eq $Receipt.database_sqlstate -or [string]$Receipt.database_sqlstate -match "^[0-9A-Z]{5}$") -and
|
|
$Receipt.raw_sql_emitted -eq $false -and
|
|
$Receipt.secret_value_read -eq $false -and
|
|
$Receipt.arbitrary_command_allowed -eq $false -and
|
|
$Receipt.cross_domain_fallback_allowed -eq $false
|
|
)
|
|
}
|
|
|
|
function Invoke-Agent99ReadyApiPodReadback {
|
|
$remoteCommand = (
|
|
"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes " +
|
|
"-o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $RemoteUser " +
|
|
"$ControlPlaneHost sudo -n $Kubectl get pods " +
|
|
"--namespace $Namespace --selector '$PodSelector' " +
|
|
"--field-selector 'status.phase=Running' " +
|
|
"--output 'custom-columns=NAME:.metadata.name,PHASE:.status.phase,READY:.status.conditions[?(@.type==`"Ready`")].status,DELETING:.metadata.deletionTimestamp' " +
|
|
"--no-headers"
|
|
)
|
|
$arguments = @(
|
|
"-i", $IdentityFile,
|
|
"-o", "BatchMode=yes",
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "ConnectTimeout=8",
|
|
"-o", "ServerAliveInterval=20",
|
|
"-o", "ServerAliveCountMax=3",
|
|
"-l", $RemoteUser,
|
|
$JumpHost,
|
|
$remoteCommand
|
|
)
|
|
|
|
$token = [Guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "agent99-db-pods-$token.out"
|
|
$stderrPath = Join-Path $env:TEMP "agent99-db-pods-$token.err"
|
|
$script:TransientTransportOutputUsed = $true
|
|
try {
|
|
$process = Start-Process -FilePath "ssh.exe" -ArgumentList $arguments `
|
|
-NoNewWindow -RedirectStandardOutput $stdoutPath `
|
|
-RedirectStandardError $stderrPath -PassThru
|
|
if (-not $process.WaitForExit(120 * 1000)) {
|
|
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
|
|
return [pscustomobject]@{ ok = $false; errorCode = "pod_inventory_timeout"; pods = @() }
|
|
}
|
|
$process.WaitForExit() | Out-Null
|
|
if ($process.ExitCode -ne 0) {
|
|
return [pscustomobject]@{ ok = $false; errorCode = "pod_inventory_read_failed"; pods = @() }
|
|
}
|
|
|
|
$readyPods = @{}
|
|
foreach ($line in @(Get-Content -LiteralPath $stdoutPath -ErrorAction Stop)) {
|
|
$trimmed = [string]$line.Trim()
|
|
if (-not $trimmed) { continue }
|
|
$columns = @($trimmed -split "\s+")
|
|
if ($columns.Count -ne 4 -or [string]$columns[0] -notmatch $ApiPodPattern -or [string]$columns[1] -ne "Running") {
|
|
return [pscustomobject]@{ ok = $false; errorCode = "pod_inventory_contract_failed"; pods = @() }
|
|
}
|
|
if ([string]$columns[2] -eq "true" -and [string]$columns[3] -eq "<none>") {
|
|
$podName = [string]$columns[0]
|
|
$readyPods[$podName] = $true
|
|
}
|
|
}
|
|
$pods = @($readyPods.Keys | Sort-Object)
|
|
if ($pods.Count -lt 2) {
|
|
return [pscustomobject]@{ ok = $false; errorCode = "two_ready_api_pods_required"; pods = $pods }
|
|
}
|
|
return [pscustomobject]@{ ok = $true; errorCode = ""; pods = $pods }
|
|
} finally {
|
|
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue
|
|
if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) {
|
|
$script:TransportCleanupSucceeded = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99FixedDbStage {
|
|
param(
|
|
[ValidateSet("check", "apply", "verify")][string]$Stage,
|
|
[string]$PodName
|
|
)
|
|
|
|
if ($PodName -notmatch $ApiPodPattern) {
|
|
return [pscustomobject]@{ ok = $false; errorCode = "target_pod_contract_failed"; receipt = $null }
|
|
}
|
|
$remoteCommand = (
|
|
"ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes " +
|
|
"-o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $RemoteUser " +
|
|
"$ControlPlaneHost sudo -n $Kubectl exec " +
|
|
"--namespace $Namespace --container $Container $PodName -- " +
|
|
"python -B -m src.services.db_bounded_executor " +
|
|
"--mode $Stage --command-id $CommandId --canonical-asset $CanonicalAsset " +
|
|
"--migration-sha256 $MigrationSha256 --normalized-sql-sha256 $NormalizedSqlSha256 " +
|
|
"--normalized-reindex-sql-sha256 $NormalizedReindexSqlSha256 " +
|
|
"--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId --source-revision $SourceRevision"
|
|
)
|
|
$arguments = @(
|
|
"-i", $IdentityFile,
|
|
"-o", "BatchMode=yes",
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "ConnectTimeout=8",
|
|
"-o", "ServerAliveInterval=20",
|
|
"-o", "ServerAliveCountMax=3",
|
|
"-l", $RemoteUser,
|
|
$JumpHost,
|
|
$remoteCommand
|
|
)
|
|
|
|
$token = [Guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "agent99-db-$token.out"
|
|
$stderrPath = Join-Path $env:TEMP "agent99-db-$token.err"
|
|
$script:TransientTransportOutputUsed = $true
|
|
try {
|
|
$process = Start-Process -FilePath "ssh.exe" -ArgumentList $arguments `
|
|
-NoNewWindow -RedirectStandardOutput $stdoutPath `
|
|
-RedirectStandardError $stderrPath -PassThru
|
|
$timeoutSeconds = if ($Stage -eq "apply") { 900 } else { 120 }
|
|
if (-not $process.WaitForExit($timeoutSeconds * 1000)) {
|
|
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
|
|
return [pscustomobject]@{ ok = $false; errorCode = "dispatch_timeout"; receipt = $null }
|
|
}
|
|
$process.WaitForExit() | Out-Null
|
|
try {
|
|
$receipt = Get-Content -LiteralPath $stdoutPath -Raw | ConvertFrom-Json
|
|
} catch {
|
|
return [pscustomobject]@{ ok = $false; errorCode = "receipt_parse_failed"; receipt = $null }
|
|
}
|
|
if (-not (Test-Agent99DbReceipt $receipt $Stage $PodName)) {
|
|
return [pscustomobject]@{ ok = $false; errorCode = "receipt_contract_failed"; receipt = $null }
|
|
}
|
|
if ($process.ExitCode -ne 0) {
|
|
return [pscustomobject]@{ ok = $false; errorCode = "fixed_remote_stage_failed"; receipt = $receipt }
|
|
}
|
|
return [pscustomobject]@{ ok = $true; errorCode = ""; receipt = $receipt }
|
|
} finally {
|
|
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue
|
|
if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) {
|
|
$script:TransportCleanupSucceeded = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach ($identity in @($TraceId, $RunId, $WorkItemId)) {
|
|
if ($identity -notmatch $SafeIdPattern) {
|
|
Write-Agent99DbTerminal "blocked" "identity_missing_or_invalid" $null $null $null $false
|
|
exit 64
|
|
}
|
|
}
|
|
$SourceRevision = $SourceRevision.ToLowerInvariant()
|
|
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") {
|
|
Write-Agent99DbTerminal "blocked" "source_revision_missing_or_invalid" $null $null $null $false
|
|
exit 64
|
|
}
|
|
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) {
|
|
Write-Agent99DbTerminal "blocked" "dedicated_identity_unavailable" $null $null $null $false
|
|
exit 65
|
|
}
|
|
$podReadback = Invoke-Agent99ReadyApiPodReadback
|
|
if (-not $podReadback.ok) {
|
|
Write-Agent99DbTerminal "blocked" $podReadback.errorCode $null $null $null $false
|
|
exit 65
|
|
}
|
|
$ExecutorPod = [string]$podReadback.pods[0]
|
|
$VerifierPod = [string]$podReadback.pods[1]
|
|
if ($ExecutorPod -notmatch $ApiPodPattern -or $VerifierPod -notmatch $ApiPodPattern -or $ExecutorPod -eq $VerifierPod) {
|
|
Write-Agent99DbTerminal "blocked" "independent_verifier_pod_contract_failed" $null $null $null $false
|
|
exit 65
|
|
}
|
|
|
|
if ($Mode -eq "Check") {
|
|
$check = Invoke-Agent99FixedDbStage "check" $ExecutorPod
|
|
if (-not $check.ok -or [string]$check.receipt.terminal -ne "check_ready") {
|
|
Write-Agent99DbTerminal "blocked" $(if ($check.errorCode) { $check.errorCode } else { "precheck_failed" }) $check.receipt $null $null $false
|
|
exit 1
|
|
}
|
|
Write-Agent99DbTerminal "check_ready" "" $check.receipt $null $null $false
|
|
exit 0
|
|
}
|
|
|
|
if ($Mode -eq "Verify") {
|
|
$verify = Invoke-Agent99FixedDbStage "verify" $VerifierPod
|
|
if (-not $verify.ok -or [string]$verify.receipt.terminal -ne "verified_healthy") {
|
|
Write-Agent99DbTerminal "verification_failed" $(if ($verify.errorCode) { $verify.errorCode } else { "catalog_verification_failed" }) $null $null $verify.receipt $false
|
|
exit 1
|
|
}
|
|
Write-Agent99DbTerminal "verified_healthy" "" $null $null $verify.receipt $false
|
|
exit 0
|
|
}
|
|
|
|
# Apply is the only write path and always includes no-write precheck plus a
|
|
# second Ready API pod verifier under the same trace/run/work-item identity.
|
|
$check = Invoke-Agent99FixedDbStage "check" $ExecutorPod
|
|
if (-not $check.ok -or [string]$check.receipt.terminal -ne "check_ready") {
|
|
Write-Agent99DbTerminal "blocked" $(if ($check.errorCode) { $check.errorCode } else { "precheck_failed" }) $check.receipt $null $null $false
|
|
exit 1
|
|
}
|
|
$apply = Invoke-Agent99FixedDbStage "apply" $ExecutorPod
|
|
$applyWriteState = if ($apply.receipt -and $apply.receipt.writes_performed -eq $true) {
|
|
$true
|
|
} elseif ($apply.receipt -and $apply.receipt.write_attempted -eq $true -and $null -eq $apply.receipt.writes_performed) {
|
|
$null
|
|
} elseif (-not $apply.ok -and -not $apply.receipt) {
|
|
# A transport loss after dispatch cannot prove whether PostgreSQL accepted
|
|
# the concurrent index build. Preserve unknown instead of false-green.
|
|
$null
|
|
} else {
|
|
$false
|
|
}
|
|
if (-not $apply.ok -or [string]$apply.receipt.terminal -ne "apply_verified_local") {
|
|
Write-Agent99DbTerminal "apply_failed" $(if ($apply.errorCode) { $apply.errorCode } else { "bounded_apply_failed" }) $check.receipt $apply.receipt $null $applyWriteState
|
|
exit 1
|
|
}
|
|
$verify = Invoke-Agent99FixedDbStage "verify" $VerifierPod
|
|
$independent = [bool](
|
|
$verify.ok -and
|
|
[string]$verify.receipt.terminal -eq "verified_healthy" -and
|
|
[string]$apply.receipt.pod_name -ne [string]$verify.receipt.pod_name -and
|
|
[string]$apply.receipt.process_identity_sha256 -ne [string]$verify.receipt.process_identity_sha256
|
|
)
|
|
if (-not $independent) {
|
|
Write-Agent99DbTerminal "verification_failed" $(if ($verify.errorCode) { $verify.errorCode } else { "independent_verifier_failed" }) $check.receipt $apply.receipt $verify.receipt $applyWriteState
|
|
exit 1
|
|
}
|
|
Write-Agent99DbTerminal "verified_healthy" "" $check.receipt $apply.receipt $verify.receipt $applyWriteState
|
|
exit 0
|