541a32a9cb
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
166 lines
7.1 KiB
PowerShell
166 lines
7.1 KiB
PowerShell
param(
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[string]$StagingDir = "",
|
|
[string]$RelayPrefix = "http://+:8787/agent99/sre-alert/",
|
|
[string]$TokenEnv = "AGENT99_SRE_RELAY_TOKEN",
|
|
[string]$AlertChainPollSourceUrl = "http://192.168.0.110:9093/api/v2/alerts",
|
|
[string]$AlertChainPollSourceHost = "192.168.0.110",
|
|
[int]$AlertChainPollIntervalSeconds = 60,
|
|
[string]$AlertChainIngressUrl = "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
if (-not $StagingDir) {
|
|
$StagingDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
}
|
|
|
|
$BinDir = Join-Path $AgentRoot "bin"
|
|
$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json"
|
|
$PlaybookDir = Join-Path $AgentRoot "playbooks"
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
|
|
New-Item -ItemType Directory -Force -Path $BinDir, $PlaybookDir, $EvidenceDir | Out-Null
|
|
|
|
foreach ($name in @(
|
|
"agent99-sre-alert-relay.ps1",
|
|
"agent99-alertmanager-alertchain-poll.ps1",
|
|
"agent99-register-tasks.ps1",
|
|
"agent99-bootstrap.ps1"
|
|
)) {
|
|
$src = Join-Path $StagingDir $name
|
|
if (-not (Test-Path $src)) {
|
|
throw "missing staged file: $name"
|
|
}
|
|
Copy-Item -Force $src (Join-Path $BinDir $name)
|
|
}
|
|
|
|
foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.json")) {
|
|
$src = Join-Path $StagingDir $name
|
|
if (Test-Path $src) {
|
|
Copy-Item -Force $src (Join-Path $PlaybookDir $name)
|
|
}
|
|
}
|
|
|
|
function Set-AgentProp {
|
|
param([object]$Object, [string]$Name, [object]$Value)
|
|
if ($Object.PSObject.Properties[$Name]) {
|
|
$Object.PSObject.Properties[$Name].Value = $Value
|
|
} else {
|
|
$Object | Add-Member -NotePropertyName $Name -NotePropertyValue $Value
|
|
}
|
|
}
|
|
|
|
if (-not (Test-Path $ConfigPath)) {
|
|
throw "missing config: $ConfigPath"
|
|
}
|
|
|
|
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
|
if (-not $config.PSObject.Properties["sreAlertRelay"]) {
|
|
$config | Add-Member -NotePropertyName "sreAlertRelay" -NotePropertyValue ([pscustomobject]@{})
|
|
}
|
|
Set-AgentProp $config.sreAlertRelay "enabled" $true
|
|
Set-AgentProp $config.sreAlertRelay "prefix" $RelayPrefix
|
|
Set-AgentProp $config.sreAlertRelay "tokenEnv" $TokenEnv
|
|
Set-AgentProp $config.sreAlertRelay "allowedRemotePrefixes" @("127.", "::1", "192.168.0.")
|
|
foreach ($legacyName in @("alertmanagerPrefix", "emergencyIngressUrlEnv")) {
|
|
if ($config.sreAlertRelay.PSObject.Properties[$legacyName]) {
|
|
$config.sreAlertRelay.PSObject.Properties.Remove($legacyName)
|
|
}
|
|
}
|
|
if (-not $config.sreAlertRelay.PSObject.Properties["alertChainPoll"]) {
|
|
$config.sreAlertRelay | Add-Member -NotePropertyName "alertChainPoll" -NotePropertyValue ([pscustomobject]@{})
|
|
}
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "enabled" $true
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "sourceUrl" $AlertChainPollSourceUrl
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "sourceHost" $AlertChainPollSourceHost
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "pollIntervalSeconds" $AlertChainPollIntervalSeconds
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "ingressUrl" $AlertChainIngressUrl
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "tokenEnv" $TokenEnv
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "alertname" "AlertChainBroken_Alertmanager"
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "agent99Role" "relay_coordination_only"
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "linuxExecutor" "host_ansible_executor"
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "catalogId" "ansible:110-alertmanager-delivery-recovery"
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "firstHopAuthenticationUsed" $false
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "firstHopSecretTransmitted" $false
|
|
Set-AgentProp $config.sreAlertRelay.alertChainPoll "rawPayloadStored" $false
|
|
|
|
if (-not $config.PSObject.Properties["selfHealth"]) {
|
|
$config | Add-Member -NotePropertyName "selfHealth" -NotePropertyValue ([pscustomobject]@{})
|
|
}
|
|
|
|
$tasks = @()
|
|
if ($config.selfHealth.PSObject.Properties["scheduledTasks"]) {
|
|
$tasks = @($config.selfHealth.scheduledTasks)
|
|
}
|
|
foreach ($taskName in @(
|
|
"Wooo-Agent99-Startup-Recovery",
|
|
"Wooo-Agent99-Heartbeat",
|
|
"Wooo-Agent99-Performance-Guard",
|
|
"Wooo-Agent99-Control-Loop",
|
|
"Wooo-Agent99-Telegram-Inbox",
|
|
"Wooo-Agent99-SRE-Alert-Inbox",
|
|
"Wooo-Agent99-SRE-Alert-Relay",
|
|
"Wooo-Agent99-Self-Health",
|
|
"Wooo-Agent99-Backup-Health"
|
|
)) {
|
|
if ($tasks -notcontains $taskName) {
|
|
$tasks += $taskName
|
|
}
|
|
}
|
|
Set-AgentProp $config.selfHealth "scheduledTasks" $tasks
|
|
|
|
$freshness = @()
|
|
if ($config.selfHealth.PSObject.Properties["evidenceFreshness"]) {
|
|
$freshness = @($config.selfHealth.evidenceFreshness)
|
|
}
|
|
if (-not ($freshness | Where-Object { $_.name -eq "sre_alert_relay" } | Select-Object -First 1)) {
|
|
$freshness += [pscustomobject]@{
|
|
name = "sre_alert_relay"
|
|
pattern = "agent99-SreAlertRelay-start-*.json"
|
|
maxAgeMinutes = 1440
|
|
required = $false
|
|
}
|
|
}
|
|
if (-not ($freshness | Where-Object { $_.name -eq "alertmanager_alertchain_poll" } | Select-Object -First 1)) {
|
|
$freshness += [pscustomobject]@{
|
|
name = "alertmanager_alertchain_poll"
|
|
pattern = "alertmanager-alertchain-poll\poll-*.json"
|
|
maxAgeMinutes = 5
|
|
required = $true
|
|
}
|
|
}
|
|
Set-AgentProp $config.selfHealth "evidenceFreshness" $freshness
|
|
$config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $ConfigPath
|
|
|
|
& (Join-Path $BinDir "agent99-register-tasks.ps1") -AgentRoot $AgentRoot | Out-Null
|
|
Start-ScheduledTask -TaskName "Wooo-Agent99-SRE-Alert-Relay"
|
|
Start-Sleep -Seconds 4
|
|
|
|
$task = Get-ScheduledTask -TaskName "Wooo-Agent99-SRE-Alert-Relay" -ErrorAction Stop
|
|
$info = Get-ScheduledTaskInfo -TaskName "Wooo-Agent99-SRE-Alert-Relay" -ErrorAction Stop
|
|
$listener = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue)
|
|
$latestEvidence = Get-ChildItem -Path $EvidenceDir -Filter "agent99-SreAlertRelay-start-*.json" -File -ErrorAction SilentlyContinue |
|
|
Sort-Object LastWriteTime -Descending |
|
|
Select-Object -First 1
|
|
|
|
[pscustomobject]@{
|
|
ok = $true
|
|
taskState = [string]$task.State
|
|
lastTaskResult = $info.LastTaskResult
|
|
listenerCount = $listener.Count
|
|
listenerLocal = @($listener | Select-Object -First 3 | ForEach-Object { "$($_.LocalAddress):$($_.LocalPort)" })
|
|
latestEvidence = if ($latestEvidence) { $latestEvidence.FullName } else { "" }
|
|
configRelayEnabled = [bool]$config.sreAlertRelay.enabled
|
|
configTokenEnv = [string]$config.sreAlertRelay.tokenEnv
|
|
alertChainPollMode = "in_process_nonblocking_timer"
|
|
configAlertChainPollSourceUrl = [string]$config.sreAlertRelay.alertChainPoll.sourceUrl
|
|
configAlertChainPollSourceHost = [string]$config.sreAlertRelay.alertChainPoll.sourceHost
|
|
configAlertChainPollIntervalSeconds = [int]$config.sreAlertRelay.alertChainPoll.pollIntervalSeconds
|
|
configAlertChainIngressUrl = [string]$config.sreAlertRelay.alertChainPoll.ingressUrl
|
|
configAgent99Role = [string]$config.sreAlertRelay.alertChainPoll.agent99Role
|
|
configLinuxExecutor = [string]$config.sreAlertRelay.alertChainPoll.linuxExecutor
|
|
configAlertChainCatalogId = [string]$config.sreAlertRelay.alertChainPoll.catalogId
|
|
scheduledTaskListed = @($config.selfHealth.scheduledTasks) -contains "Wooo-Agent99-SRE-Alert-Relay"
|
|
} | ConvertTo-Json -Depth 6 -Compress
|