From 0a7a01155ed42020cf48e239353fbfc4e9734b23 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 10:07:44 +0800 Subject: [PATCH] fix(agent99): reduce alert noise and verify freshness --- agent99-control-plane.ps1 | 154 +++++++++++++++++++++++++++++++------- 1 file changed, 128 insertions(+), 26 deletions(-) diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 7f3bad331..8f7c856ea 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -145,15 +145,22 @@ function Format-AgentTelegramText { $hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 } $missing = if ($Data -and $Data.PSObject.Properties["missingFields"]) { @($Data.missingFields) } else { @() } $candidatePath = if ($Data -and $Data.PSObject.Properties["candidatePath"]) { $Data.candidatePath } else { $null } - $lines += "Event: provider freshness triage" + $readback = if ($Data -and $Data.PSObject.Properties["verifierReadback"]) { $Data.verifierReadback } else { $null } + $status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" } + $lines += "Event: provider freshness auto triage" $lines += "Target: $targetName" - $lines += "Candidate: $candidateId" + $lines += "Status: $status" $lines += "Evidence hits: $hitCount" - if (@($missing).Count -gt 0) { - $lines += "Missing: $($missing -join ', ')" + if ($readback) { + $lines += "Direct readback: ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)" } - $lines += "Action: require provider window, last seen, direct reference, and verifier readback before trusting dashboard green." - $lines += "Guard: no provider switch, no paid model call, no env change, no reload." + if (@($missing).Count -gt 0) { + $lines += "Still missing: $($missing -join ', ')" + } else { + $lines += "Missing: direct readback fields completed" + } + $lines += "Action: candidate and KM/RAG were written; duplicate same-state alerts are suppressed." + $lines += "Guard: no provider switch, paid model call, env change, or reload." if ($candidatePath) { $lines += "Evidence: $candidatePath" } @@ -186,22 +193,18 @@ function Format-AgentTelegramText { $instruction = if ($Data -and $Data.PSObject.Properties["instruction"]) { $Data.instruction } else { $null } $exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null } $evidencePath = if ($Data -and $Data.PSObject.Properties["evidence"]) { $Data.evidence } else { $null } - $lines += "Event: operator command" - $lines += "Command: $commandId" - $lines += "Source: $source" + $lines += "Event: Agent99 automation result" $lines += "Mode: $modeName" - if ($instruction) { - $lines += "Instruction: $instruction" - } + $lines += "Source: $source" $lines += "Result: exitCode=$exitCode" if ($Data -and $Data.PSObject.Properties["problem"]) { $problem = $Data.problem if ($problem -and $problem.PSObject.Properties["occurrences"]) { $lines += "Problem count: occurrences=$($problem.occurrences), resolved=$($problem.resolved), failed=$($problem.failed)" } - if ($problem -and $problem.PSObject.Properties["kmPath"]) { - $lines += "KM: $($problem.kmPath)" - } + } + if ($instruction -and $exitCode -ne 0) { + $lines += "Failure instruction: $instruction" } if ($evidencePath) { $lines += "Evidence: $evidencePath" @@ -354,19 +357,48 @@ function Record-AgentEvent { $alertOperatorCommands = [bool]$Config.telegram.alertOperatorCommands } $operatorCommandAlert = [bool]($alertOperatorCommands -and $EventType -match "^operator_command_") + if ($operatorCommandAlert -and $Data) { + $operatorOk = $false + $operatorMode = "" + $operatorId = "" + $operatorOkValue = $null + $operatorExitCode = $null + if ($Data -is [System.Collections.IDictionary]) { + if ($Data.Contains("mode")) { $operatorMode = [string]$Data["mode"] } + if ($Data.Contains("id")) { $operatorId = [string]$Data["id"] } + if ($Data.Contains("ok")) { $operatorOkValue = $Data["ok"] } + if ($Data.Contains("exitCode")) { $operatorExitCode = $Data["exitCode"] } + } else { + $modeProp = $Data.PSObject.Properties.Match("mode") | Select-Object -First 1 + $idProp = $Data.PSObject.Properties.Match("id") | Select-Object -First 1 + $okProp = $Data.PSObject.Properties.Match("ok") | Select-Object -First 1 + $exitCodeProp = $Data.PSObject.Properties.Match("exitCode") | Select-Object -First 1 + if ($modeProp) { $operatorMode = [string]$modeProp.Value } + if ($idProp) { $operatorId = [string]$idProp.Value } + if ($okProp) { $operatorOkValue = $okProp.Value } + if ($exitCodeProp) { $operatorExitCode = $exitCodeProp.Value } + } + if ($null -ne $operatorOkValue -and [string]$operatorOkValue -match "^(?i:true|1)$") { $operatorOk = $true } + $parsedExitCode = -999 + if ($null -ne $operatorExitCode -and [int]::TryParse([string]$operatorExitCode, [ref]$parsedExitCode) -and $parsedExitCode -eq 0) { $operatorOk = $true } + $alertSuccessfulOperatorCommands = $false + if ($Config.telegram -and $Config.telegram.PSObject.Properties["alertSuccessfulOperatorCommands"]) { + $alertSuccessfulOperatorCommands = [bool]$Config.telegram.alertSuccessfulOperatorCommands + } + if ($operatorOk -and -not $alertSuccessfulOperatorCommands) { + Write-AgentLog "telegram_operator_success_suppressed type=$EventType mode=$operatorMode id=$operatorId" + $operatorCommandAlert = $false + } + } $shouldAlert = ($Severity -in @("warning", "critical")) -or $operatorCommandAlert -or ($Alert -and $Severity -ne "info") -or ($alertAll -and ($Severity -ne "info" -or $alertInfo)) if ($shouldAlert) { - if ($operatorCommandAlert) { - Send-AgentTelegram $Severity $EventType $Message $Data - return - } $dedupeMinutes = 30 if ($Config.telegram -and $Config.telegram.PSObject.Properties["dedupeMinutes"]) { $dedupeMinutes = [int]$Config.telegram.dedupeMinutes } $dedupeParts = @($EventType, $Severity) if ($Data) { - foreach ($propName in @("host", "name", "path", "reason")) { + foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source")) { if ($Data.PSObject.Properties[$propName]) { $dedupeParts += "$propName=$($Data.PSObject.Properties[$propName].Value)" } @@ -374,10 +406,16 @@ function Record-AgentEvent { if ($Data.PSObject.Properties["reasons"]) { $dedupeParts += "reasons=$(@($Data.reasons) -join ',')" } + if ($Data.PSObject.Properties["missingFields"]) { + $dedupeParts += "missing=$(@($Data.missingFields) -join ',')" + } + if ($Data.PSObject.Properties["problem"] -and $Data.problem -and $Data.problem.PSObject.Properties["key"]) { + $dedupeParts += "problem=$($Data.problem.key)" + } } if ($dedupeParts.Count -le 2) { $normalizedMessage = $Message - foreach ($pattern in @("evidence=[^ ]+", "loadPerCore=[^ ]+", "memAvailPct=[^ ]+", "diskUsedPct=[^ ]+", "ageMinutes=[^ ]+")) { + foreach ($pattern in @("candidate=[^ ]+", "evidence=[^ ]+", "loadPerCore=[^ ]+", "memAvailPct=[^ ]+", "diskUsedPct=[^ ]+", "ageMinutes=[^ ]+")) { $normalizedMessage = $normalizedMessage -replace $pattern, ($pattern.Split("=")[0] + "=") } $dedupeParts += $normalizedMessage @@ -1971,6 +2009,50 @@ function Invoke-AgentControlTick { $result } +function Invoke-AgentProviderFreshnessDirectReadback { + param( + [int]$LookbackMinutes + ) + + $baseUrl = "https://awoooi.wooo.work/api/v1/platform/events/dossier/recurrence?project_id=awoooi&limit=20" + $startedAt = Get-Date + try { + $response = Invoke-RestMethod -Method Get -Uri $baseUrl -TimeoutSec 12 + $summary = if ($response -and $response.PSObject.Properties["summary"]) { $response.summary } else { $null } + $latest = if ($summary -and $summary.PSObject.Properties["latest_received_at"]) { $summary.latest_received_at } else { $null } + $sourceTotal = if ($summary -and $summary.PSObject.Properties["source_event_total"]) { [int]$summary.source_event_total } else { 0 } + $itemsCount = if ($response -and $response.PSObject.Properties["items"]) { @($response.items).Count } else { 0 } + return [pscustomobject]@{ + ok = $true + verifier = "awoooi-platform-events-recurrence" + directReference = $baseUrl + providerWindowMinutes = $LookbackMinutes + latestReceivedAt = $latest + sourceEventTotal = $sourceTotal + recurrenceGroupTotal = if ($summary -and $summary.PSObject.Properties["recurrence_group_total"]) { [int]$summary.recurrence_group_total } else { 0 } + itemsCount = $itemsCount + httpStatus = 200 + durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2) + checkedAt = (Get-Date -Format o) + } + } catch { + return [pscustomobject]@{ + ok = $false + verifier = "awoooi-platform-events-recurrence" + directReference = $baseUrl + providerWindowMinutes = $LookbackMinutes + latestReceivedAt = $null + sourceEventTotal = 0 + recurrenceGroupTotal = 0 + itemsCount = 0 + httpStatus = $null + durationSeconds = [math]::Round(((Get-Date) - $startedAt).TotalSeconds, 2) + checkedAt = (Get-Date -Format o) + error = ($_ | Out-String).Trim() + } + } +} + function Invoke-AgentProviderFreshnessTriage { $providerConfig = if ($Config.PSObject.Properties["providerFreshness"]) { $Config.providerFreshness } else { $null } $lookbackMinutes = if ($providerConfig -and $providerConfig.PSObject.Properties["lookbackMinutes"]) { [int]$providerConfig.lookbackMinutes } else { 120 } @@ -2047,8 +2129,23 @@ function Invoke-AgentProviderFreshnessTriage { } } - $missingFields = @("providerWindow", "lastSeen", "directReference", "verifierReadback") - $status = if (@($hits).Count -gt 0) { "candidate_created_waiting_direct_readback" } else { "candidate_created_no_alert_body_seen" } + $verifierReadback = Invoke-AgentProviderFreshnessDirectReadback -LookbackMinutes $lookbackMinutes + $directReference = if ($verifierReadback -and $verifierReadback.PSObject.Properties["directReference"]) { $verifierReadback.directReference } else { $null } + $latestSeen = if ($verifierReadback -and $verifierReadback.PSObject.Properties["latestReceivedAt"]) { $verifierReadback.latestReceivedAt } else { $null } + $missingFields = @() + if (-not $lookbackMinutes) { $missingFields += "providerWindow" } + if (-not $latestSeen) { $missingFields += "lastSeen" } + if (-not $directReference) { $missingFields += "directReference" } + if (-not ($verifierReadback -and $verifierReadback.ok -eq $true)) { $missingFields += "verifierReadback" } + $status = if ($verifierReadback -and $verifierReadback.ok -eq $true -and $latestSeen) { + "direct_readback_ok" + } elseif ($verifierReadback -and $verifierReadback.ok -eq $true) { + "direct_readback_ok_no_source_events" + } elseif (@($hits).Count -gt 0) { + "candidate_created_waiting_direct_readback" + } else { + "candidate_created_no_alert_body_seen" + } $candidate = [pscustomobject]@{ timestamp = (Get-Date -Format o) candidateId = $candidateId @@ -2060,15 +2157,20 @@ function Invoke-AgentProviderFreshnessTriage { hits = @($hits | Sort-Object lastWrite -Descending | Select-Object -First 20) targets = $targets missingFields = $missingFields + providerWindowMinutes = $lookbackMinutes + lastSeen = $latestSeen + directReference = $directReference + verifierReadback = $verifierReadback requiredVerifierReadback = @("provider_window", "last_seen", "direct_reference", "source_trace_or_sentry_signoz_ref") noFalseGreenRisk = $true - allowedActions = @("create_candidate", "read_existing_evidence", "request_direct_reference", "write_km") + allowedActions = @("create_candidate", "read_existing_evidence", "provider_freshness_direct_readback", "request_direct_reference", "write_km") prohibitedActions = @("provider_switch", "paid_model_call", "env_change", "reload") candidatePath = $candidatePath } $candidate | ConvertTo-Json -Depth 10 | Set-Content -Path $candidatePath -Encoding UTF8 - Write-AgentLog "provider_freshness candidate=$candidateId hits=$(@($hits).Count) status=$status path=$candidatePath" - Record-AgentEvent "provider_freshness_triage" "warning" "candidate=$candidateId hits=$(@($hits).Count) missing=$($missingFields -join ',') evidence=$candidatePath" $candidate -Alert + Write-AgentLog "provider_freshness candidate=$candidateId hits=$(@($hits).Count) status=$status missing=$($missingFields -join ',') path=$candidatePath" + $eventSeverity = if (@($missingFields).Count -gt 0) { "warning" } else { "info" } + Record-AgentEvent "provider_freshness_triage" $eventSeverity "target=provider_freshness_signal status=$status missing=$($missingFields -join ',')" $candidate -Alert $candidate }