299 lines
13 KiB
PowerShell
299 lines
13 KiB
PowerShell
param(
|
|
[string]$SourceRoot = (Split-Path -Parent $MyInvocation.MyCommand.Path),
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[string]$SourceRevision = "",
|
|
[switch]$RegisterTasks,
|
|
[switch]$ValidateOnly
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$BinDir = Join-Path $AgentRoot "bin"
|
|
$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json"
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
$StateDir = Join-Path $AgentRoot "state"
|
|
$DeployRoot = Join-Path $AgentRoot "deploy"
|
|
$StageDir = Join-Path $DeployRoot "staging-$stamp"
|
|
$BackupDir = Join-Path $DeployRoot "backup-$stamp"
|
|
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
|
|
$EvidencePath = Join-Path $EvidenceDir "agent99-deploy-$stamp.json"
|
|
$script:ConfigMigrations = @()
|
|
|
|
$runtimeFiles = @(
|
|
"agent99-bootstrap.ps1",
|
|
"agent99-contract-check.ps1",
|
|
"agent99-control-plane.ps1",
|
|
"agent99-deploy.ps1",
|
|
"agent99-register-tasks.ps1",
|
|
"agent99-sre-alert-inbox.ps1",
|
|
"agent99-sre-alert-relay.ps1",
|
|
"agent99-submit-request.ps1",
|
|
"agent99-synthetic-tests.ps1",
|
|
"agent99-telegram-inbox.ps1",
|
|
"agent99-windows-update-policy.ps1",
|
|
"host-reboot-scorecard.ps1",
|
|
"agent99.config.example.json",
|
|
"agent99.config.99.example.json"
|
|
)
|
|
|
|
function Ensure-Utf8Bom {
|
|
param([string]$Path)
|
|
$bytes = [IO.File]::ReadAllBytes($Path)
|
|
$text = [Text.Encoding]::UTF8.GetString($bytes)
|
|
if ($text.Length -gt 0 -and $text[0] -eq [char]0xFEFF) { $text = $text.Substring(1) }
|
|
[IO.File]::WriteAllText($Path, $text, (New-Object System.Text.UTF8Encoding($true)))
|
|
}
|
|
|
|
function Add-DefaultProperty {
|
|
param([object]$Object, [string]$Name, [object]$Value)
|
|
if (-not $Object.PSObject.Properties[$Name] -or $null -eq $Object.PSObject.Properties[$Name].Value) {
|
|
if ($Object.PSObject.Properties[$Name]) {
|
|
$Object.PSObject.Properties[$Name].Value = $Value
|
|
} else {
|
|
$Object | Add-Member -MemberType NoteProperty -Name $Name -Value $Value
|
|
}
|
|
}
|
|
}
|
|
|
|
function Set-AgentProperty {
|
|
param([object]$Object, [string]$Name, [object]$Value)
|
|
if ($Object.PSObject.Properties[$Name]) {
|
|
$Object.PSObject.Properties[$Name].Value = $Value
|
|
} else {
|
|
$Object | Add-Member -MemberType NoteProperty -Name $Name -Value $Value
|
|
}
|
|
}
|
|
|
|
function Write-DeployEvidence {
|
|
param([bool]$Ok, [string]$Status, [string]$ErrorText, [object[]]$Files, [object]$TaskResult)
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
|
|
[pscustomobject]@{
|
|
schemaVersion = "agent99_deploy_receipt_v2"
|
|
timestamp = (Get-Date -Format o)
|
|
ok = $Ok
|
|
status = $Status
|
|
sourceRoot = $SourceRoot
|
|
sourceRevision = $SourceRevision
|
|
agentRoot = $AgentRoot
|
|
stageDir = $StageDir
|
|
backupDir = $BackupDir
|
|
runtimeManifest = $ManifestPath
|
|
validateOnly = [bool]$ValidateOnly
|
|
registerTasks = [bool]$RegisterTasks
|
|
error = $ErrorText
|
|
configMigrations = @($script:ConfigMigrations)
|
|
files = $Files
|
|
taskRegistration = $TaskResult
|
|
} | ConvertTo-Json -Depth 10 | Set-Content -Path $EvidencePath -Encoding UTF8
|
|
}
|
|
|
|
function Restore-Agent99Deployment {
|
|
param([object[]]$Rows, [bool]$ConfigBackedUp, [bool]$ManifestBackedUp)
|
|
foreach ($item in $Rows) {
|
|
$backupPath = Join-Path $BackupDir $item.name
|
|
if ($item.existed -and (Test-Path $backupPath)) {
|
|
Copy-Item -Force $backupPath $item.livePath
|
|
} elseif (-not $item.existed -and (Test-Path $item.livePath)) {
|
|
Remove-Item -Force $item.livePath
|
|
}
|
|
}
|
|
if ($ConfigBackedUp) { Copy-Item -Force (Join-Path $BackupDir "agent99.config.json") $ConfigPath }
|
|
if ($ManifestBackedUp) {
|
|
Copy-Item -Force (Join-Path $BackupDir "runtime-manifest.json") $ManifestPath
|
|
} elseif (Test-Path $ManifestPath) {
|
|
Remove-Item -Force $ManifestPath
|
|
}
|
|
}
|
|
|
|
if (-not $SourceRevision) {
|
|
try {
|
|
$revision = & git -C $SourceRoot rev-parse HEAD 2>$null
|
|
if ($LASTEXITCODE -eq 0 -and $revision) { $SourceRevision = ([string]$revision).Trim() }
|
|
} catch {
|
|
$SourceRevision = ""
|
|
}
|
|
}
|
|
if (-not $SourceRevision) { $SourceRevision = "unknown" }
|
|
|
|
New-Item -ItemType Directory -Force -Path $StageDir, $EvidenceDir, $StateDir | Out-Null
|
|
foreach ($name in $runtimeFiles) {
|
|
$source = Join-Path $SourceRoot $name
|
|
if (-not (Test-Path $source)) { throw "Required deployment file missing: $source" }
|
|
$target = Join-Path $StageDir $name
|
|
Copy-Item -Force $source $target
|
|
if ($name.EndsWith(".ps1")) { Ensure-Utf8Bom $target }
|
|
}
|
|
|
|
& (Join-Path $StageDir "agent99-contract-check.ps1") -SourceRoot $StageDir | Out-Null
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-DeployEvidence $false "preflight_failed" "contract_check_failed" @() $null
|
|
throw "Agent99 staged contract check failed."
|
|
}
|
|
& (Join-Path $StageDir "agent99-synthetic-tests.ps1") -SourceRoot $StageDir -WorkRoot (Join-Path $StageDir "synthetic-work") | Out-Null
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-DeployEvidence $false "preflight_failed" "synthetic_tests_failed" @() $null
|
|
throw "Agent99 staged synthetic tests failed."
|
|
}
|
|
|
|
$stagedFiles = @($runtimeFiles | ForEach-Object {
|
|
$path = Join-Path $StageDir $_
|
|
[pscustomobject]@{ name = $_; sha256 = (Get-FileHash -Algorithm SHA256 -Path $path).Hash.ToLowerInvariant() }
|
|
})
|
|
|
|
if ($ValidateOnly) {
|
|
Write-DeployEvidence $true "validated" $null $stagedFiles $null
|
|
[pscustomobject]@{ ok = $true; status = "validated"; sourceRevision = $SourceRevision; evidence = $EvidencePath; files = $stagedFiles } | ConvertTo-Json -Depth 8
|
|
exit 0
|
|
}
|
|
|
|
New-Item -ItemType Directory -Force -Path $BinDir, $BackupDir | Out-Null
|
|
$promotionRows = @()
|
|
$configBackedUp = $false
|
|
$manifestBackedUp = $false
|
|
try {
|
|
foreach ($name in $runtimeFiles) {
|
|
$livePath = Join-Path $BinDir $name
|
|
$existed = Test-Path $livePath
|
|
if ($existed) { Copy-Item -Force $livePath (Join-Path $BackupDir $name) }
|
|
$promotionRows += [pscustomobject]@{ name = $name; livePath = $livePath; existed = [bool]$existed }
|
|
Copy-Item -Force (Join-Path $StageDir $name) $livePath
|
|
}
|
|
|
|
if (-not (Test-Path $ConfigPath)) { throw "Live Agent99 config not found: $ConfigPath" }
|
|
Copy-Item -Force $ConfigPath (Join-Path $BackupDir "agent99.config.json")
|
|
$configBackedUp = $true
|
|
if (Test-Path $ManifestPath) {
|
|
Copy-Item -Force $ManifestPath (Join-Path $BackupDir "runtime-manifest.json")
|
|
$manifestBackedUp = $true
|
|
}
|
|
|
|
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
|
Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
|
|
Add-DefaultProperty $config "sshTransport" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.sshTransport "serialized" $true
|
|
Add-DefaultProperty $config.sshTransport "lockTimeoutSeconds" 90
|
|
Add-DefaultProperty $config.sshTransport "lockPollMilliseconds" 200
|
|
Add-DefaultProperty $config "sshProcessGuard" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.sshProcessGuard "enabled" $true
|
|
Add-DefaultProperty $config.sshProcessGuard "staleMinutes" 15
|
|
Add-DefaultProperty $config.sshProcessGuard "hardStaleMinutes" 60
|
|
Add-DefaultProperty $config "k3s" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.k3s "jumpHost" "192.168.0.110"
|
|
Add-DefaultProperty $config.k3s "preferJumpHost" $true
|
|
Add-DefaultProperty $config.k3s "directHosts" @("192.168.0.110", "192.168.0.112")
|
|
Add-DefaultProperty $config "autoRecovery" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.autoRecovery "enabled" $true
|
|
Add-DefaultProperty $config.autoRecovery "triggerOnHostUnreachable" $true
|
|
Add-DefaultProperty $config.autoRecovery "minimumUnreachableHosts" 1
|
|
Add-DefaultProperty $config.autoRecovery "cooldownMinutes" 10
|
|
Add-DefaultProperty $config "recoverySlo" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.recoverySlo "targetMinutes" 10
|
|
Add-DefaultProperty $config.recoverySlo "hostTimeoutSeconds" 90
|
|
Add-DefaultProperty $config.recoverySlo "harborTimeoutSeconds" 120
|
|
Add-DefaultProperty $config.recoverySlo "workloadTimeoutSeconds" 180
|
|
Add-DefaultProperty $config "recoveryCoordinator" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.recoveryCoordinator "enabled" $true
|
|
Add-DefaultProperty $config.recoveryCoordinator "controllerHost" "192.168.0.110"
|
|
Add-DefaultProperty $config.recoveryCoordinator "readbackTimeoutSeconds" 15
|
|
Add-DefaultProperty $config.recoveryCoordinator "pollIntervalSeconds" 10
|
|
Add-DefaultProperty $config.recoveryCoordinator "freshWaitTimeoutSeconds" 420
|
|
Add-DefaultProperty $config.recoveryCoordinator "maxArtifactAgeSeconds" 600
|
|
Add-DefaultProperty $config.recoveryCoordinator "artifactClockSkewSeconds" 90
|
|
Add-DefaultProperty $config.recoveryCoordinator "requiredHostAliases" @("99", "110", "111", "112", "120", "121", "188")
|
|
Add-DefaultProperty $config "completionCallback" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.completionCallback "enabled" $true
|
|
Add-DefaultProperty $config.completionCallback "url" "https://awoooi.wooo.work/api/v1/agents/agent99/completion-callback"
|
|
Add-DefaultProperty $config.completionCallback "tokenEnv" "AGENT99_SRE_RELAY_TOKEN"
|
|
Add-DefaultProperty $config.completionCallback "timeoutSeconds" 15
|
|
Add-DefaultProperty $config.completionCallback "maxAttempts" 2
|
|
Add-DefaultProperty $config.completionCallback "batchLimit" 5
|
|
Add-DefaultProperty $config.completionCallback "retryAfterMinutes" 5
|
|
Add-DefaultProperty $config "telegram" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.telegram "repeatAfterMinutes" 120
|
|
Add-DefaultProperty $config "performance" ([pscustomobject]@{})
|
|
Add-DefaultProperty $config.performance "processCpuHostPercentWarning" 25
|
|
Add-DefaultProperty $config.performance "processCpuHostPercentCritical" 50
|
|
|
|
$policyConfig = Get-Content (Join-Path $StageDir "agent99.config.99.example.json") -Raw | ConvertFrom-Json
|
|
foreach ($sourceAction in @($policyConfig.performance.loadShedding.actions | Where-Object { $_.PSObject.Properties["policyVersion"] })) {
|
|
$liveAction = $config.performance.loadShedding.actions | Where-Object { $_.name -eq $sourceAction.name } | Select-Object -First 1
|
|
if (-not $liveAction) { continue }
|
|
$sourceVersion = [int]$sourceAction.policyVersion
|
|
$liveVersion = if ($liveAction.PSObject.Properties["policyVersion"]) { [int]$liveAction.policyVersion } else { 0 }
|
|
if ($sourceVersion -le $liveVersion) { continue }
|
|
foreach ($propertyName in @("policyVersion", "host", "command", "timeoutSeconds", "postVerifyCommand", "postVerifyMaxDiskUsedPercent", "postVerifyOkRegex", "rollback", "cooldownMinutes", "failureBackoffMinutes", "minSeverity", "reasonMatches", "enabled")) {
|
|
if ($sourceAction.PSObject.Properties[$propertyName]) {
|
|
Set-AgentProperty $liveAction $propertyName $sourceAction.PSObject.Properties[$propertyName].Value
|
|
}
|
|
}
|
|
$script:ConfigMigrations += [pscustomobject]@{
|
|
name = [string]$sourceAction.name
|
|
fromPolicyVersion = $liveVersion
|
|
toPolicyVersion = $sourceVersion
|
|
}
|
|
}
|
|
$config | ConvertTo-Json -Depth 20 | Set-Content -Path $ConfigPath -Encoding UTF8
|
|
|
|
$manifestRows = @()
|
|
foreach ($staged in $stagedFiles) {
|
|
$livePath = Join-Path $BinDir $staged.name
|
|
$runtimeSha = if (Test-Path $livePath) { (Get-FileHash -Algorithm SHA256 -Path $livePath).Hash.ToLowerInvariant() } else { "" }
|
|
$matched = [bool]($runtimeSha -and $runtimeSha -eq $staged.sha256)
|
|
$manifestRows += [pscustomobject]@{
|
|
name = $staged.name
|
|
sourceExists = $true
|
|
runtimeExists = [bool](Test-Path $livePath)
|
|
sourceSha256 = $staged.sha256
|
|
runtimeSha256 = $runtimeSha
|
|
contentMatched = $matched
|
|
matched = $matched
|
|
}
|
|
}
|
|
$runtimeMatched = [bool](@($manifestRows | Where-Object { -not $_.matched }).Count -eq 0)
|
|
[pscustomobject]@{
|
|
schemaVersion = "agent99_runtime_manifest_v1"
|
|
generatedAt = (Get-Date -Format o)
|
|
sourceRevision = $SourceRevision
|
|
runtimeMatched = $runtimeMatched
|
|
fileCount = $manifestRows.Count
|
|
mismatchCount = @($manifestRows | Where-Object { -not $_.matched }).Count
|
|
files = $manifestRows
|
|
} | ConvertTo-Json -Depth 10 | Set-Content -Path $ManifestPath -Encoding UTF8
|
|
if (-not $runtimeMatched) { throw "Promoted Agent99 runtime does not match staged source." }
|
|
|
|
& (Join-Path $BinDir "agent99-contract-check.ps1") -SourceRoot $BinDir | Out-Null
|
|
if ($LASTEXITCODE -ne 0) { throw "Post-promotion Agent99 contract check failed." }
|
|
& (Join-Path $BinDir "agent99-synthetic-tests.ps1") -SourceRoot $BinDir -WorkRoot (Join-Path $StageDir "live-synthetic-work") | Out-Null
|
|
if ($LASTEXITCODE -ne 0) { throw "Post-promotion Agent99 synthetic tests failed." }
|
|
} catch {
|
|
Restore-Agent99Deployment $promotionRows $configBackedUp $manifestBackedUp
|
|
Write-DeployEvidence $false "rolled_back" $_.Exception.Message $stagedFiles $null
|
|
throw
|
|
}
|
|
|
|
$taskResult = $null
|
|
if ($RegisterTasks) {
|
|
try {
|
|
& (Join-Path $BinDir "agent99-register-tasks.ps1") -AgentRoot $AgentRoot | Out-Null
|
|
$taskResult = [pscustomobject]@{ ok = $true; status = "registered" }
|
|
} catch {
|
|
$taskResult = [pscustomobject]@{ ok = $false; status = "registration_failed"; error = $_.Exception.Message }
|
|
}
|
|
}
|
|
|
|
$deployOk = [bool](-not $taskResult -or $taskResult.ok)
|
|
$deployStatus = if ($deployOk) { "deployed" } else { "deployed_task_registration_failed" }
|
|
Write-DeployEvidence $deployOk $deployStatus $(if ($taskResult -and -not $taskResult.ok) { $taskResult.error } else { $null }) $stagedFiles $taskResult
|
|
[pscustomobject]@{
|
|
ok = $deployOk
|
|
status = $deployStatus
|
|
sourceRevision = $SourceRevision
|
|
evidence = $EvidencePath
|
|
backupDir = $BackupDir
|
|
runtimeManifest = $ManifestPath
|
|
taskRegistration = $taskResult
|
|
files = $stagedFiles
|
|
} | ConvertTo-Json -Depth 10
|
|
if (-not $deployOk) { exit 2 }
|
|
exit 0
|