|
|
|
|
@@ -145,15 +145,22 @@ function Format-AgentTelegramText {
|
|
|
|
|
$hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 }
|
|
|
|
|
$missing = if ($Data -and $Data.PSObject.Properties["missingFields"]) { @($Data.missingFields) } else { @() }
|
|
|
|
|
$candidatePath = if ($Data -and $Data.PSObject.Properties["candidatePath"]) { $Data.candidatePath } else { $null }
|
|
|
|
|
$lines += "Event: provider freshness triage"
|
|
|
|
|
$readback = if ($Data -and $Data.PSObject.Properties["verifierReadback"]) { $Data.verifierReadback } else { $null }
|
|
|
|
|
$status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" }
|
|
|
|
|
$lines += "Event: provider freshness auto triage"
|
|
|
|
|
$lines += "Target: $targetName"
|
|
|
|
|
$lines += "Candidate: $candidateId"
|
|
|
|
|
$lines += "Status: $status"
|
|
|
|
|
$lines += "Evidence hits: $hitCount"
|
|
|
|
|
if (@($missing).Count -gt 0) {
|
|
|
|
|
$lines += "Missing: $($missing -join ', ')"
|
|
|
|
|
if ($readback) {
|
|
|
|
|
$lines += "Direct readback: ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)"
|
|
|
|
|
}
|
|
|
|
|
$lines += "Action: require provider window, last seen, direct reference, and verifier readback before trusting dashboard green."
|
|
|
|
|
$lines += "Guard: no provider switch, no paid model call, no env change, no reload."
|
|
|
|
|
if (@($missing).Count -gt 0) {
|
|
|
|
|
$lines += "Still missing: $($missing -join ', ')"
|
|
|
|
|
} else {
|
|
|
|
|
$lines += "Missing: direct readback fields completed"
|
|
|
|
|
}
|
|
|
|
|
$lines += "Action: candidate and KM/RAG were written; duplicate same-state alerts are suppressed."
|
|
|
|
|
$lines += "Guard: no provider switch, paid model call, env change, or reload."
|
|
|
|
|
if ($candidatePath) {
|
|
|
|
|
$lines += "Evidence: $candidatePath"
|
|
|
|
|
}
|
|
|
|
|
@@ -186,22 +193,18 @@ function Format-AgentTelegramText {
|
|
|
|
|
$instruction = if ($Data -and $Data.PSObject.Properties["instruction"]) { $Data.instruction } else { $null }
|
|
|
|
|
$exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null }
|
|
|
|
|
$evidencePath = if ($Data -and $Data.PSObject.Properties["evidence"]) { $Data.evidence } else { $null }
|
|
|
|
|
$lines += "Event: operator command"
|
|
|
|
|
$lines += "Command: $commandId"
|
|
|
|
|
$lines += "Source: $source"
|
|
|
|
|
$lines += "Event: Agent99 automation result"
|
|
|
|
|
$lines += "Mode: $modeName"
|
|
|
|
|
if ($instruction) {
|
|
|
|
|
$lines += "Instruction: $instruction"
|
|
|
|
|
}
|
|
|
|
|
$lines += "Source: $source"
|
|
|
|
|
$lines += "Result: exitCode=$exitCode"
|
|
|
|
|
if ($Data -and $Data.PSObject.Properties["problem"]) {
|
|
|
|
|
$problem = $Data.problem
|
|
|
|
|
if ($problem -and $problem.PSObject.Properties["occurrences"]) {
|
|
|
|
|
$lines += "Problem count: occurrences=$($problem.occurrences), resolved=$($problem.resolved), failed=$($problem.failed)"
|
|
|
|
|
}
|
|
|
|
|
if ($problem -and $problem.PSObject.Properties["kmPath"]) {
|
|
|
|
|
$lines += "KM: $($problem.kmPath)"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($instruction -and $exitCode -ne 0) {
|
|
|
|
|
$lines += "Failure instruction: $instruction"
|
|
|
|
|
}
|
|
|
|
|
if ($evidencePath) {
|
|
|
|
|
$lines += "Evidence: $evidencePath"
|
|
|
|
|
@@ -354,19 +357,48 @@ function Record-AgentEvent {
|
|
|
|
|
$alertOperatorCommands = [bool]$Config.telegram.alertOperatorCommands
|
|
|
|
|
}
|
|
|
|
|
$operatorCommandAlert = [bool]($alertOperatorCommands -and $EventType -match "^operator_command_")
|
|
|
|
|
if ($operatorCommandAlert -and $Data) {
|
|
|
|
|
$operatorOk = $false
|
|
|
|
|
$operatorMode = ""
|
|
|
|
|
$operatorId = ""
|
|
|
|
|
$operatorOkValue = $null
|
|
|
|
|
$operatorExitCode = $null
|
|
|
|
|
if ($Data -is [System.Collections.IDictionary]) {
|
|
|
|
|
if ($Data.Contains("mode")) { $operatorMode = [string]$Data["mode"] }
|
|
|
|
|
if ($Data.Contains("id")) { $operatorId = [string]$Data["id"] }
|
|
|
|
|
if ($Data.Contains("ok")) { $operatorOkValue = $Data["ok"] }
|
|
|
|
|
if ($Data.Contains("exitCode")) { $operatorExitCode = $Data["exitCode"] }
|
|
|
|
|
} else {
|
|
|
|
|
$modeProp = $Data.PSObject.Properties.Match("mode") | Select-Object -First 1
|
|
|
|
|
$idProp = $Data.PSObject.Properties.Match("id") | Select-Object -First 1
|
|
|
|
|
$okProp = $Data.PSObject.Properties.Match("ok") | Select-Object -First 1
|
|
|
|
|
$exitCodeProp = $Data.PSObject.Properties.Match("exitCode") | Select-Object -First 1
|
|
|
|
|
if ($modeProp) { $operatorMode = [string]$modeProp.Value }
|
|
|
|
|
if ($idProp) { $operatorId = [string]$idProp.Value }
|
|
|
|
|
if ($okProp) { $operatorOkValue = $okProp.Value }
|
|
|
|
|
if ($exitCodeProp) { $operatorExitCode = $exitCodeProp.Value }
|
|
|
|
|
}
|
|
|
|
|
if ($null -ne $operatorOkValue -and [string]$operatorOkValue -match "^(?i:true|1)$") { $operatorOk = $true }
|
|
|
|
|
$parsedExitCode = -999
|
|
|
|
|
if ($null -ne $operatorExitCode -and [int]::TryParse([string]$operatorExitCode, [ref]$parsedExitCode) -and $parsedExitCode -eq 0) { $operatorOk = $true }
|
|
|
|
|
$alertSuccessfulOperatorCommands = $false
|
|
|
|
|
if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSuccessfulOperatorCommands"]) {
|
|
|
|
|
$alertSuccessfulOperatorCommands = [bool]$Config.telegram.alertSuccessfulOperatorCommands
|
|
|
|
|
}
|
|
|
|
|
if ($operatorOk -and -not $alertSuccessfulOperatorCommands) {
|
|
|
|
|
Write-AgentLog "telegram_operator_success_suppressed type=$EventType mode=$operatorMode id=$operatorId"
|
|
|
|
|
$operatorCommandAlert = $false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo))
|
|
|
|
|
if ($shouldAlert) {
|
|
|
|
|
if ($operatorCommandAlert) {
|
|
|
|
|
Send-AgentTelegram $Severity $EventType $Message $Data
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
$dedupeMinutes = 30
|
|
|
|
|
if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) {
|
|
|
|
|
$dedupeMinutes = [int]$Config.telegram.dedupeMinutes
|
|
|
|
|
}
|
|
|
|
|
$dedupeParts = @($EventType, $Severity)
|
|
|
|
|
if ($Data) {
|
|
|
|
|
foreach ($propName in @("host", "name", "path", "reason")) {
|
|
|
|
|
foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source")) {
|
|
|
|
|
if ($Data.PSObject.Properties[$propName]) {
|
|
|
|
|
$dedupeParts += "$propName=$($Data.PSObject.Properties[$propName].Value)"
|
|
|
|
|
}
|
|
|
|
|
@@ -374,10 +406,16 @@ function Record-AgentEvent {
|
|
|
|
|
if ($Data.PSObject.Properties["reasons"]) {
|
|
|
|
|
$dedupeParts += "reasons=$(@($Data.reasons) -join ',')"
|
|
|
|
|
}
|
|
|
|
|
if ($Data.PSObject.Properties["missingFields"]) {
|
|
|
|
|
$dedupeParts += "missing=$(@($Data.missingFields) -join ',')"
|
|
|
|
|
}
|
|
|
|
|
if ($Data.PSObject.Properties["problem"] -and $Data.problem -and $Data.problem.PSObject.Properties["key"]) {
|
|
|
|
|
$dedupeParts += "problem=$($Data.problem.key)"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($dedupeParts.Count -le 2) {
|
|
|
|
|
$normalizedMessage = $Message
|
|
|
|
|
foreach ($pattern in @("evidence=[^ ]+", "loadPerCore=[^ ]+", "memAvailPct=[^ ]+", "diskUsedPct=[^ ]+", "ageMinutes=[^ ]+")) {
|
|
|
|
|
foreach ($pattern in @("candidate=[^ ]+", "evidence=[^ ]+", "loadPerCore=[^ ]+", "memAvailPct=[^ ]+", "diskUsedPct=[^ ]+", "ageMinutes=[^ ]+")) {
|
|
|
|
|
$normalizedMessage = $normalizedMessage -replace $pattern, ($pattern.Split("=")[0] + "=<value>")
|
|
|
|
|
}
|
|
|
|
|
$dedupeParts += $normalizedMessage
|
|
|
|
|
@@ -1971,6 +2009,50 @@ function Invoke-AgentControlTick {
|
|
|
|
|
$result
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Invoke-AgentProviderFreshnessDirectReadback {
|
|
|
|
|
param(
|
|
|
|
|
[int]$LookbackMinutes
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$baseUrl = "https://awoooi.wooo.work/api/v1/platform/events/dossier/recurrence?project_id=awoooi&limit=20"
|
|
|
|
|
$startedAt = Get-Date
|
|
|
|
|
try {
|
|
|
|
|
$response = Invoke-RestMethod -Method Get -Uri $baseUrl -TimeoutSec 12
|
|
|
|
|
$summary = if ($response -and $response.PSObject.Properties["summary"]) { $response.summary } else { $null }
|
|
|
|
|
$latest = if ($summary -and $summary.PSObject.Properties["latest_received_at"]) { $summary.latest_received_at } else { $null }
|
|
|
|
|
$sourceTotal = if ($summary -and $summary.PSObject.Properties["source_event_total"]) { [int]$summary.source_event_total } else { 0 }
|
|
|
|
|
$itemsCount = if ($response -and $response.PSObject.Properties["items"]) { @($response.items).Count } else { 0 }
|
|
|
|
|
return [pscustomobject]@{
|
|
|
|
|
ok = $true
|
|
|
|
|
verifier = "awoooi-platform-events-recurrence"
|
|
|
|
|
directReference = $baseUrl
|
|
|
|
|
providerWindowMinutes = $LookbackMinutes
|
|
|
|
|
latestReceivedAt = $latest
|
|
|
|
|
sourceEventTotal = $sourceTotal
|
|
|
|
|
recurrenceGroupTotal = if ($summary -and $summary.PSObject.Properties["recurrence_group_total"]) { [int]$summary.recurrence_group_total } else { 0 }
|
|
|
|
|
itemsCount = $itemsCount
|
|
|
|
|
httpStatus = 200
|
|
|
|
|
durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2)
|
|
|
|
|
checkedAt = (Get-Date -Format o)
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
return [pscustomobject]@{
|
|
|
|
|
ok = $false
|
|
|
|
|
verifier = "awoooi-platform-events-recurrence"
|
|
|
|
|
directReference = $baseUrl
|
|
|
|
|
providerWindowMinutes = $LookbackMinutes
|
|
|
|
|
latestReceivedAt = $null
|
|
|
|
|
sourceEventTotal = 0
|
|
|
|
|
recurrenceGroupTotal = 0
|
|
|
|
|
itemsCount = 0
|
|
|
|
|
httpStatus = $null
|
|
|
|
|
durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2)
|
|
|
|
|
checkedAt = (Get-Date -Format o)
|
|
|
|
|
error = ($_ | Out-String).Trim()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Invoke-AgentProviderFreshnessTriage {
|
|
|
|
|
$providerConfig = if ($Config.PSObject.Properties["providerFreshness"]) { $Config.providerFreshness } else { $null }
|
|
|
|
|
$lookbackMinutes = if ($providerConfig -and $providerConfig.PSObject.Properties["lookbackMinutes"]) { [int]$providerConfig.lookbackMinutes } else { 120 }
|
|
|
|
|
@@ -2047,8 +2129,23 @@ function Invoke-AgentProviderFreshnessTriage {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$missingFields = @("providerWindow", "lastSeen", "directReference", "verifierReadback")
|
|
|
|
|
$status = if (@($hits).Count -gt 0) { "candidate_created_waiting_direct_readback" } else { "candidate_created_no_alert_body_seen" }
|
|
|
|
|
$verifierReadback = Invoke-AgentProviderFreshnessDirectReadback -LookbackMinutes $lookbackMinutes
|
|
|
|
|
$directReference = if ($verifierReadback -and $verifierReadback.PSObject.Properties["directReference"]) { $verifierReadback.directReference } else { $null }
|
|
|
|
|
$latestSeen = if ($verifierReadback -and $verifierReadback.PSObject.Properties["latestReceivedAt"]) { $verifierReadback.latestReceivedAt } else { $null }
|
|
|
|
|
$missingFields = @()
|
|
|
|
|
if (-not $lookbackMinutes) { $missingFields += "providerWindow" }
|
|
|
|
|
if (-not $latestSeen) { $missingFields += "lastSeen" }
|
|
|
|
|
if (-not $directReference) { $missingFields += "directReference" }
|
|
|
|
|
if (-not ($verifierReadback -and $verifierReadback.ok -eq $true)) { $missingFields += "verifierReadback" }
|
|
|
|
|
$status = if ($verifierReadback -and $verifierReadback.ok -eq $true -and $latestSeen) {
|
|
|
|
|
"direct_readback_ok"
|
|
|
|
|
} elseif ($verifierReadback -and $verifierReadback.ok -eq $true) {
|
|
|
|
|
"direct_readback_ok_no_source_events"
|
|
|
|
|
} elseif (@($hits).Count -gt 0) {
|
|
|
|
|
"candidate_created_waiting_direct_readback"
|
|
|
|
|
} else {
|
|
|
|
|
"candidate_created_no_alert_body_seen"
|
|
|
|
|
}
|
|
|
|
|
$candidate = [pscustomobject]@{
|
|
|
|
|
timestamp = (Get-Date -Format o)
|
|
|
|
|
candidateId = $candidateId
|
|
|
|
|
@@ -2060,15 +2157,20 @@ function Invoke-AgentProviderFreshnessTriage {
|
|
|
|
|
hits = @($hits | Sort-Object lastWrite -Descending | Select-Object -First 20)
|
|
|
|
|
targets = $targets
|
|
|
|
|
missingFields = $missingFields
|
|
|
|
|
providerWindowMinutes = $lookbackMinutes
|
|
|
|
|
lastSeen = $latestSeen
|
|
|
|
|
directReference = $directReference
|
|
|
|
|
verifierReadback = $verifierReadback
|
|
|
|
|
requiredVerifierReadback = @("provider_window", "last_seen", "direct_reference", "source_trace_or_sentry_signoz_ref")
|
|
|
|
|
noFalseGreenRisk = $true
|
|
|
|
|
allowedActions = @("create_candidate", "read_existing_evidence", "request_direct_reference", "write_km")
|
|
|
|
|
allowedActions = @("create_candidate", "read_existing_evidence", "provider_freshness_direct_readback", "request_direct_reference", "write_km")
|
|
|
|
|
prohibitedActions = @("provider_switch", "paid_model_call", "env_change", "reload")
|
|
|
|
|
candidatePath = $candidatePath
|
|
|
|
|
}
|
|
|
|
|
$candidate | ConvertTo-Json -Depth 10 | Set-Content -Path $candidatePath -Encoding UTF8
|
|
|
|
|
Write-AgentLog "provider_freshness candidate=$candidateId hits=$(@($hits).Count) status=$status path=$candidatePath"
|
|
|
|
|
Record-AgentEvent "provider_freshness_triage" "warning" "candidate=$candidateId hits=$(@($hits).Count) missing=$($missingFields -join ',') evidence=$candidatePath" $candidate -Alert
|
|
|
|
|
Write-AgentLog "provider_freshness candidate=$candidateId hits=$(@($hits).Count) status=$status missing=$($missingFields -join ',') path=$candidatePath"
|
|
|
|
|
$eventSeverity = if (@($missingFields).Count -gt 0) { "warning" } else { "info" }
|
|
|
|
|
Record-AgentEvent "provider_freshness_triage" $eventSeverity "target=provider_freshness_signal status=$status missing=$($missingFields -join ',')" $candidate -Alert
|
|
|
|
|
$candidate
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
