diff --git a/AGENT99-AI-WORKSTATION-SOP.md b/AGENT99-AI-WORKSTATION-SOP.md index bbb4baba4..4c9b29b68 100644 --- a/AGENT99-AI-WORKSTATION-SOP.md +++ b/AGENT99-AI-WORKSTATION-SOP.md @@ -1,6 +1,6 @@ # Agent99 AI Workstation SOP -Last updated: 2026-07-10 +Last updated: 2026-07-11 Canonical enterprise scope and work order: @@ -26,6 +26,18 @@ Host 99 is not the only availability observer. An external watcher and maintenan - All `.ps1` runtime files are normalized to UTF-8 BOM by bootstrap so Windows PowerShell 5 cannot corrupt Traditional Chinese strings during parsing. - Staging verification on 99 must pass `-SelfTestOutcomeContract` and `runtimeMatched=true` before production Agent99 files are replaced. +## 2026-07-11 P0-002 Actionable Alert Dispatch Contract + +This source candidate is test-green but not yet deployed to the Windows 99 runtime. + +- Direct Alertmanager events, `ai_automation_alert_card_v1`, `ACTION REQUIRED` incident cards, and failed `AWOOOI CI/CD` cards use one sanitized Agent99 dispatcher. +- A cold-start or reboot gate is `kind=host_recovery`, `suggestedMode=Recover`; it must never become a fabricated Kubernetes deployment restart. +- A failed CI/CD card is `kind=cicd_failure`, `suggestedMode=Status`; Agent99 collects status/evidence while the source event remains open until a later successful CD/deploy receipt resolves it. +- Every dispatcher result writes a public-safe `agent99_dispatch_receipt` with alert id, kind, transport, HTTP status, `accepted`, and `inbox_triggered`. Raw response bodies, relay paths, tokens, chat ids, and workflow URLs are not stored. +- `HTTP 202` alone is not success. The bridge reports `dispatched` only when the relay body confirms `ok=true`; an unaccepted receipt releases the single-flight lock so the event can retry. +- Stateful verifier lanes may use `sourceEventResolutionPolicy=mode_verifier_can_resolve`. External-source lanes use `external_source_receipt_required` and cannot self-close from a local command exit. +- Source/test receipt: the focused post-rebase suite passed 249 tests, and the expanded all-Telegram plus Agent99/RepairCandidate suite passed 262 tests after integrating the Telegram single-owner mainline; YAML, shell/Python syntax, runner pressure, document secret, backup-alert, and recovery-scorecard checks passed. Windows 99 package deployment, runtime manifest reconciliation, and exact live card replay remain required before this P0 can close. + It must act as: - Secretary: reminders, status summaries, evidence packets, operator updates. @@ -717,7 +729,7 @@ Agent99 now uses a source-trusted deployment path, a dedicated bounded SSH trans - Source authority and deployment: - Gitea commit SHA is the source revision; a local scratch checkout or a Windows file timestamp is not deployment truth. - `agent99-deploy.ps1` validates PowerShell parse, Agent99 contracts, synthetic checks, staging content, backup, atomic promotion, task registration, and rollback. - - `C:\Wooo\Agent99\runtime-manifest.json.sourceRevision` must match the deployed Gitea SHA. A successful source test without this runtime readback is not a completed deployment. + - `C:\Wooo\Agent99\state\runtime-manifest.json.sourceRevision` must match the deployed Gitea SHA. A successful source test without this runtime readback is not a completed deployment. - Dedicated transport: - Private identity: `C:\Wooo\Agent99\keys\agent99_ed25519`; it remains on 99, is ACL-restricted, and is never copied into evidence, Telegram, logs, or the repository. - Public-key authorization is limited with `from="192.168.0.99",restrict` on 110 user `wooo` and 112 user `kali`. diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index 23221ed32..ddba2f27c 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -1,6 +1,6 @@ # Host Reboot AI Automation SOP -Last updated: 2026-07-10 +Last updated: 2026-07-11 Canonical enterprise scope and current work order: @@ -183,6 +183,15 @@ Producer-side Agent99 bridge: - Provider freshness bridge output must run direct readback, then require provider window, last seen, direct reference, and verifier readback before dashboard green is trusted. - Provider freshness no-false-green rule: route 200 or dashboard up is not enough. The Agent99 verifier must read `https://awoooi.wooo.work/api/v1/platform/events/dossier/recurrence?project_id=awoooi&limit=20` and record `providerWindowMinutes`, `lastSeen`, `directReference`, and `verifierReadback`. +2026-07-11 P0-002 source receipt, runtime deployment pending: + +- `ACTION REQUIRED` cards for `cold-start-gate` now enter the same Agent99 ingress as Alertmanager and route to `Recover` with the reboot/cold-start scorecard verifier. +- Failed `AWOOOI CI/CD` cards enter Agent99 as `cicd_failure` in `Status` mode. Agent99 records evidence but waits for the next successful Gitea CD/deploy/runtime receipt before source-event closure. +- The producer stores a sanitized relay acceptance receipt in the outbound source envelope before Telegram delivery; Telegram remains a human lifecycle surface, not the only queue or database. +- Cold-start candidates use the Agent99 execution route and never synthesize `kubectl rollout restart deployment/cold-start-gate`. +- A relay receipt with `accepted=false` releases the fingerprint single-flight lock and returns failed. This prevents a transport-level response from permanently suppressing an unprocessed incident. +- Verification completed in source: 249 focused post-rebase tests and 262 expanded all-Telegram plus Agent99/RepairCandidate tests passed after integrating the Telegram single-owner mainline; the repository static governance suite also passed. This does not prove Windows 99 is running this revision; runtime manifest and exact do-not-page replay are still mandatory. + Approved controlled actions for the 99 agent: - Start a missing VM. diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 37ac06a52..f8103809f 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -2665,7 +2665,8 @@ function Get-AgentOutcomeContract { [datetime]$StartedAt, [bool]$SourceEventResolutionRequired, [object]$SourceEventResolved = $null, - [string]$SourceEventEvidence = "" + [string]$SourceEventEvidence = "", + [string]$SourceEventResolutionPolicy = "external_source_receipt_required" ) $checks = @() @@ -2817,7 +2818,25 @@ function Get-AgentOutcomeContract { $basePassed = [bool]($transportOk -and $evidenceExists -and $evidenceParsed -and $evidenceFresh -and $modeMatches) $verifierPassed = [bool]($basePassed -and $modeVerifierPassed) - $sourceEventResolvedBool = if (-not $SourceEventResolutionRequired) { $true } else { [bool]($SourceEventResolved -eq $true) } + $sourceEventResolvedBool = if (-not $SourceEventResolutionRequired) { + $true + } elseif ($SourceEventResolved -eq $true) { + $true + } elseif ($SourceEventResolutionPolicy -eq "mode_verifier_can_resolve") { + [bool]$verifierPassed + } else { + $false + } + $effectiveSourceEventEvidence = if ($SourceEventEvidence) { + $SourceEventEvidence + } elseif ($SourceEventResolutionPolicy -eq "mode_verifier_can_resolve" -and $verifierPassed) { + [string]$LatestEvidence.path + } else { + "" + } + if ($SourceEventResolutionRequired) { + $checks += New-AgentOutcomeCheck "source_event_resolved" $sourceEventResolvedBool $true "policy=$SourceEventResolutionPolicy evidence=$effectiveSourceEventEvidence" + } $state = if (-not $transportOk) { "failed" } elseif (-not $basePassed) { @@ -2844,8 +2863,9 @@ function Get-AgentOutcomeContract { evidenceFresh = $evidenceFresh evidenceModeMatched = $modeMatches sourceEventResolutionRequired = $SourceEventResolutionRequired + sourceEventResolutionPolicy = $SourceEventResolutionPolicy sourceEventResolved = $sourceEventResolvedBool - sourceEventEvidence = if ($SourceEventEvidence) { $SourceEventEvidence } else { $null } + sourceEventEvidence = if ($effectiveSourceEventEvidence) { $effectiveSourceEventEvidence } else { $null } failedChecks = $failedChecks checks = $checks verifiedAt = (Get-Date -Format o) @@ -2902,6 +2922,8 @@ function Invoke-AgentOutcomeContractSelfTest { $operatorResolved = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $false $alertVerifying = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $true $false "selftest-source-event" + $alertResolvedByVerifier = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $true $false "" "mode_verifier_can_resolve" + $alertTransportFailedVerifier = Get-AgentOutcomeContract "PublicSmoke" 7 $publicOkEvidence $startedAt $true $false "" "mode_verifier_can_resolve" $failedPostCondition = Get-AgentOutcomeContract "PublicSmoke" 0 $publicFailedEvidence $startedAt $false $transportFailed = Get-AgentOutcomeContract "PublicSmoke" 7 $publicOkEvidence $startedAt $false $providerDegraded = Get-AgentOutcomeContract "ProviderFreshness" 0 $providerEvidence $startedAt $true $false @@ -2909,6 +2931,8 @@ function Invoke-AgentOutcomeContractSelfTest { $ok = [bool]( $operatorResolved.state -eq "resolved" -and $operatorResolved.verifierPassed -and $alertVerifying.state -eq "verifying" -and -not $alertVerifying.resolved -and + $alertResolvedByVerifier.state -eq "resolved" -and $alertResolvedByVerifier.sourceEventResolved -and + $alertTransportFailedVerifier.state -eq "failed" -and -not $alertTransportFailedVerifier.sourceEventResolved -and $failedPostCondition.state -eq "degraded" -and -not $failedPostCondition.verifierPassed -and $transportFailed.state -eq "failed" -and $providerDegraded.state -eq "degraded" -and @@ -2921,6 +2945,8 @@ function Invoke-AgentOutcomeContractSelfTest { cases = [pscustomobject]@{ operatorResolved = $operatorResolved alertVerifying = $alertVerifying + alertResolvedByVerifier = $alertResolvedByVerifier + alertTransportFailedVerifier = $alertTransportFailedVerifier failedPostCondition = $failedPostCondition transportFailed = $transportFailed providerDegraded = $providerDegraded @@ -3440,6 +3466,7 @@ function Invoke-AgentQueuedCommands { $alertService = if ($command.PSObject.Properties["alertService"]) { [string]$command.alertService } else { $null } $alertHost = if ($command.PSObject.Properties["alertHost"]) { [string]$command.alertHost } else { $null } $sourceEventResolutionRequired = if ($command.PSObject.Properties["sourceEventResolutionRequired"]) { Convert-AgentBool $command.sourceEventResolutionRequired } else { [bool]$alertId } + $sourceEventResolutionPolicy = if ($command.PSObject.Properties["sourceEventResolutionPolicy"]) { [string]$command.sourceEventResolutionPolicy } else { "external_source_receipt_required" } $sourceEventResolved = if ($command.PSObject.Properties["sourceEventResolved"]) { [object](Convert-AgentBool $command.sourceEventResolved) } else { $null } $sourceEventEvidence = if ($command.PSObject.Properties["sourceEventEvidence"]) { [string]$command.sourceEventEvidence } else { "" } $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" } @@ -3480,7 +3507,7 @@ function Invoke-AgentQueuedCommands { } else { -3 } - $outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence + $outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence -SourceEventResolutionPolicy $sourceEventResolutionPolicy $result = [pscustomobject]@{ id = $id mode = $modeName @@ -3500,6 +3527,7 @@ function Invoke-AgentQueuedCommands { outcomeState = $outcome.state verifierPassed = [bool]$outcome.verifierPassed sourceEventResolved = [bool]$outcome.sourceEventResolved + sourceEventResolutionPolicy = $sourceEventResolutionPolicy outcome = $outcome exitCode = $exitCode durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1) diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index 951d2c678..eeef17a9b 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -123,6 +123,7 @@ function Resolve-AgentAlertRoute { "alert_chain_health" = "Status" "application_ai_config_error" = "Status" "repository_missing" = "Status" + "cicd_failure" = "Status" } if ($kind -and $kindModes.ContainsKey($kind)) { return [pscustomobject]@{ mode = [string]$kindModes[$kind]; source = "kind"; matched = $kind } @@ -131,7 +132,7 @@ function Resolve-AgentAlertRoute { if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") { return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" } } - if ($text -match "reboot-auto-recovery|reboot_auto_recovery|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") { + if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") { return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" } } if ($text -match "backup|snapshot|restore drill|cron") { @@ -152,6 +153,9 @@ function Resolve-AgentAlertRoute { if ($text -match "harbor|registry") { return [pscustomobject]@{ mode = "HarborRepair"; source = "text_fallback"; matched = "harbor" } } + if ($text -match "ci/?cd|cicd|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") { + return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "cicd_failure" } + } if ($text -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return [pscustomobject]@{ mode = "AwoooRepair"; source = "text_fallback"; matched = "k3s" } } @@ -720,6 +724,9 @@ foreach ($file in $files) { $hostName = [string](Get-AgentField $alert "host" "") $replyChatId = [string](Get-AgentField $alert "replyChatId" "") $suppressTelegram = Test-AgentAlertSuppressTelegram $alert + $resolution = Get-AgentField $alert "resolution" $null + $sourceEventResolutionPolicy = [string](Get-AgentField $resolution "policy" "external_source_receipt_required") + $sourceEventResolutionRequired = Convert-AgentBool (Get-AgentField $resolution "required" $true) $requestPath = Join-Path $RequestDir "$queueId.json" $queuePath = Join-Path $QueueDir "$queueId.json" @@ -744,6 +751,8 @@ foreach ($file in $files) { alertHost = $hostName alertPath = $processedPath suppressTelegram = $suppressTelegram + sourceEventResolutionPolicy = $sourceEventResolutionPolicy + sourceEventResolutionRequired = $sourceEventResolutionRequired createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8 @@ -769,6 +778,8 @@ foreach ($file in $files) { alertPath = $processedPath replyChatId = $replyChatId suppressTelegram = $suppressTelegram + sourceEventResolutionPolicy = $sourceEventResolutionPolicy + sourceEventResolutionRequired = $sourceEventResolutionRequired createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 @@ -808,6 +819,8 @@ foreach ($file in $files) { dedupeDecision = $dedupeDecision controlledApply = $controlled suppressTelegram = $suppressTelegram + sourceEventResolutionPolicy = $sourceEventResolutionPolicy + sourceEventResolutionRequired = $sourceEventResolutionRequired requestPath = $requestPath queuePath = $queuePath alertPath = $processedPath diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 index db3bbcecc..dbdbedd69 100644 --- a/agent99-telegram-inbox.ps1 +++ b/agent99-telegram-inbox.ps1 @@ -257,7 +257,7 @@ function Test-AgentMonitoringAlertText { return $false } - if ($lower -match "alertmanager|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot-auto-recovery|postgres|postgresql|database|alertchain|alert_chain|monitoring pipeline|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|ai support|locale guard|missing_key|e_ai_missing_key") { + if ($lower -match "alertmanager|action required|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot-auto-recovery|cold[-_ ]start|\bci/?cd\b|\bcicd\b|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗|postgres|postgresql|database|alertchain|alert_chain|monitoring pipeline|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|ai support|locale guard|missing_key|e_ai_missing_key") { return $true } @@ -284,21 +284,22 @@ function Get-AgentTelegramAlertSeverity { $lower = $Text.ToLowerInvariant() if ($lower -match "resolved|recovered" -or (Test-AgentTextContainsCode $Text @(0x6062, 0x89E3))) { return "resolved" } if ($lower -match "critical|p0|emergency|fatal|down" -or (Test-AgentTextContainsCode $Text @(0x56B4, 0x91CD, 0x7DCA, 0x6025))) { return "critical" } + if ($lower -match "failed" -or $Text -match "失敗") { return "critical" } if ($lower -match "warning|warn|degraded|readback_failed|missing_key|locale guard|ai support" -or (Test-AgentTextContainsCode $Text @(0x544A, 0x8B66, 0x7570, 0x5E38))) { return "warning" } - if ($lower -match "failed") { return "critical" } return "warning" } function Get-AgentTelegramAlertKind { param([string]$Text) $lower = $Text.ToLowerInvariant() + if ($lower -match "\bci/?cd\b|\bcicd\b|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") { return "cicd_failure" } if ($lower -match "backup|snapshot|cron" -or (Test-AgentTextContainsCode $Text @(0x5099, 0x4EFD))) { return "backup_health" } if ($lower -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc") { return "wazuh_security_alert" } if ($lower -match "gitea|repo_missing|repository_missing|repo missing|repository missing") { return "repository_missing" } if ($lower -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") { return "provider_freshness_signal" } if ($lower -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag") { return "database_health" } if ($lower -match "alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") { return "alert_chain_health" } - if ($lower -match "reboot-auto-recovery|reboot_auto_recovery|all_required_hosts_not_in_10_minute_reboot_window|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot|restart|powered_off" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" } + if ($lower -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot|restart|powered_off" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" } if ($lower -match "502|site_502|public_route|website|route") { return "public_route_502" } if ($lower -match "missing_key|e_ai_missing_key|ai support|locale guard" -or (Test-AgentTextContainsCode $Text @(0x5BA2, 0x670D))) { return "application_ai_config_error" } if ($lower -match "cpu|load" -or (Test-AgentTextContainsCode $Text @(0x8CA0, 0x8F09))) { return "performance_cpu" } @@ -342,7 +343,7 @@ function Get-AgentTelegramAlertTarget { [string]$Service, [string]$HostName ) - $targetMatch = [regex]::Match($Text, "(?im)\bTarget\s*[:\uFF1A]\s*([^\r\n]+)") + $targetMatch = [regex]::Match($Text, "(?im)(?:\bTarget|資源)\s*[:\uFF1A]\s*([^\r\n]+)") if ($targetMatch.Success) { return $targetMatch.Groups[1].Value.Trim() } @@ -369,11 +370,30 @@ function Get-AgentTelegramSuggestedMode { "alert_chain_health" = "Status" "application_ai_config_error" = "Status" "repository_missing" = "Status" + "cicd_failure" = "Status" } if ($modeByKind.ContainsKey($Kind)) { return [string]$modeByKind[$Kind] } return "Status" } +function Get-AgentTelegramSourceEventResolutionPolicy { + param([string]$Kind) + if ($Kind -in @( + "provider_freshness_signal", + "backup_health", + "public_route_502", + "host_recovery", + "performance_cpu", + "performance_memory", + "performance_disk", + "harbor_registry", + "awoooi_k3s" + )) { + return "mode_verifier_can_resolve" + } + return "external_source_receipt_required" +} + function Convert-AgentTelegramRouteToken { param([string]$Value) $safe = [regex]::Replace($Value.ToLowerInvariant(), "[^a-z0-9_.-]+", "_").Trim([char[]]".-_") @@ -411,6 +431,7 @@ function New-AgentMonitoringAlertFromTelegram { $hostName = Get-AgentTelegramAlertHost $text $target = Get-AgentTelegramAlertTarget $text $kind $service $hostName $suggestedMode = Get-AgentTelegramSuggestedMode $kind + $sourceEventResolutionPolicy = Get-AgentTelegramSourceEventResolutionPolicy $kind $correlationKey = @( (Convert-AgentTelegramRouteToken $kind), (Convert-AgentTelegramRouteToken $target), @@ -446,6 +467,11 @@ function New-AgentMonitoringAlertFromTelegram { correlationKey = $correlationKey singleFlightWindowSeconds = 300 } + resolution = [pscustomobject]@{ + schemaVersion = "agent99_source_event_resolution_v1" + policy = $sourceEventResolutionPolicy + required = $true + } telegramUpdateId = $Update.update_id telegramMessageId = $Update.message_id telegramChatTitle = $Update.chat_title @@ -602,6 +628,24 @@ function Invoke-AgentTelegramInboxSelfTest { expectedKind = "public_route_502" expectedService = "awoooi" expectedHost = "" + }, + [pscustomobject]@{ + name = "action required cold start" + text = "ACTION REQUIRED INC-20260710-5700F7 Target: cold-start-gate" + expectedMonitoring = $true + expectedSeverity = "warning" + expectedKind = "host_recovery" + expectedService = "unknown" + expectedHost = "" + }, + [pscustomobject]@{ + name = "failed ci cd stays diagnostic" + text = "[AWOOOI CI/CD] build-and-deploy AWOOOI 部署失敗" + expectedMonitoring = $true + expectedSeverity = "critical" + expectedKind = "cicd_failure" + expectedService = "awoooi" + expectedHost = "" } ) diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index c6ccaf327..8484817e9 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -124,6 +124,7 @@ def resolve_agent99_alert_kind( return "alert_chain_health" if re.search( r"reboot-auto-recovery|reboot_auto_recovery|reboot recovery|" + r"cold[-_ ]start(?:[-_ ]gate)?|full[-_ ]stack[-_ ]cold[-_ ]start|" r"host_down|hostdown|host_reboot|vm_not_running|powered_off|" r"all_required_hosts_not_in_10_minute_reboot_window", text, @@ -135,6 +136,13 @@ def resolve_agent99_alert_kind( return "performance_pressure" if re.search(r"harbor|registry", text): return "harbor_registry" + if re.search( + r"\bci/?cd\b|\bcicd\b|build-and-deploy|build_and_deploy|" + r"deploy(?:ment)?[_ -](?:failed|failure)|build[_ -](?:failed|failure)|" + r"部署失敗|構建失敗", + text, + ): + return "cicd_failure" if re.search( r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text ): @@ -161,9 +169,25 @@ def _suggested_mode_for_kind(kind: str) -> str | None: "awoooi_k3s": "AwoooRepair", "host_recovery": "Recover", "application_ai_config_error": "Status", + "cicd_failure": "Status", + "monitoring_alert": "Status", }.get(kind) +def _resolution_policy_for_kind(kind: str) -> str: + if kind in { + "provider_freshness_signal", + "public_route_502", + "performance_pressure", + "backup_health", + "harbor_registry", + "awoooi_k3s", + "host_recovery", + }: + return "mode_verifier_can_resolve" + return "external_source_receipt_required" + + def _agent99_route_group_key( *, kind: str, @@ -343,6 +367,11 @@ def build_agent99_sre_alert( "sourceFingerprint": _safe_text(fingerprint, max_length=160), "singleFlightWindowSeconds": AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS, }, + "resolution": { + "schemaVersion": "agent99_source_event_resolution_v1", + "policy": _resolution_policy_for_kind(kind), + "required": True, + }, "awoooi": { "alertId": _safe_text(alert_id, max_length=160), "alertname": _safe_text(alertname, max_length=160), @@ -453,14 +482,60 @@ def post_agent99_sre_alert( def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str: + receipt = dispatch_agent99_sre_alert_with_receipt(payload) + return str(receipt["transport"]) + + +def dispatch_agent99_sre_alert_with_receipt( + payload: dict[str, Any], +) -> dict[str, Any]: + """Dispatch one alert and retain only public-safe acceptance evidence.""" + + alert_id = _safe_text(payload.get("id"), max_length=160) + kind = _safe_text(payload.get("kind"), max_length=120) if settings.AGENT99_SRE_ALERT_RELAY_URL: - post_agent99_sre_alert(payload) - return "relay" + raw_receipt = post_agent99_sre_alert(payload) or {} + response_payload: dict[str, Any] = {} + try: + parsed = json.loads(str(raw_receipt.get("body") or "")) + if isinstance(parsed, dict): + response_payload = parsed + except json.JSONDecodeError: + response_payload = {} + http_status = int(raw_receipt.get("status") or 0) + accepted = bool(200 <= http_status < 300 and response_payload.get("ok") is True) + inbox_triggered = bool(response_payload.get("inboxTriggered") is True) + return { + "schema_version": "agent99_sre_dispatch_receipt_v1", + "status": ( + "accepted_inbox_triggered" + if accepted and inbox_triggered + else "accepted_inbox_pending" + if accepted + else "relay_response_not_accepted" + ), + "transport": "relay", + "alert_id": alert_id, + "kind": kind, + "http_status": http_status, + "accepted": accepted, + "inbox_triggered": inbox_triggered, + "stores_raw_response": False, + } if settings.AGENT99_SRE_ALERT_INBOX_PATH: written = write_agent99_sre_alert(payload) if written is not None: - return "file" + return { + "schema_version": "agent99_sre_dispatch_receipt_v1", + "status": "file_written", + "transport": "file", + "alert_id": alert_id, + "kind": kind, + "accepted": True, + "inbox_triggered": False, + "stores_raw_response": False, + } logger.info( "agent99_sre_bridge_disabled", @@ -468,7 +543,16 @@ def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str: alert_id=payload.get("id"), kind=payload.get("kind"), ) - return "disabled" + return { + "schema_version": "agent99_sre_dispatch_receipt_v1", + "status": "bridge_disabled", + "transport": "disabled", + "alert_id": alert_id, + "kind": kind, + "accepted": False, + "inbox_triggered": False, + "stores_raw_response": False, + } async def bridge_alertmanager_to_agent99( @@ -527,19 +611,23 @@ async def bridge_alertmanager_to_agent99( "fingerprint": single_flight_fingerprint, "groupKey": routing.get("groupKey"), } - dispatch_result = await asyncio.to_thread(dispatch_agent99_sre_alert, payload) - if dispatch_result == "disabled": + dispatch_receipt = await asyncio.to_thread( + dispatch_agent99_sre_alert_with_receipt, payload + ) + if not dispatch_receipt.get("accepted"): await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id) single_flight_acquired = False return { "status": "failed", - "reason": "bridge_disabled", + "reason": dispatch_receipt.get("status") or "dispatch_not_accepted", + "dispatchReceipt": dispatch_receipt, "fingerprint": single_flight_fingerprint, "groupKey": routing.get("groupKey"), } return { "status": "dispatched", - "dispatch": dispatch_result, + "dispatch": dispatch_receipt.get("transport"), + "dispatchReceipt": dispatch_receipt, "fingerprint": single_flight_fingerprint, "groupKey": routing.get("groupKey"), "kind": payload.get("kind"), diff --git a/apps/api/src/services/repair_candidate_service.py b/apps/api/src/services/repair_candidate_service.py index 8e56b4dd0..459c634d0 100644 --- a/apps/api/src/services/repair_candidate_service.py +++ b/apps/api/src/services/repair_candidate_service.py @@ -314,6 +314,28 @@ class RepairCandidateService: if not isinstance(promotion_contract, dict): return + route_id = str(promotion_contract.get("route_id") or "") + if route_id.startswith("agent99_"): + suggested_mode = "Recover" if route_id.startswith("agent99_recover") else "Status" + handoff = { + "schema_version": "repair_candidate_controlled_executor_handoff_v1", + "status": "agent99_dispatch_required_on_actionable_card_send", + "operation_type": "agent99_actionable_dispatch", + "executor": "Agent99", + "route_id": route_id, + "suggested_mode": suggested_mode, + "dispatch_receipt_required": True, + "post_condition_verifier_required": True, + "check_mode_queued": False, + "runtime_execution_authorized": False, + } + result.metadata["controlled_executor_handoff"] = handoff + draft_package["controlled_executor_handoff"] = handoff + work_item = draft_package.get("awooop_work_item") + if isinstance(work_item, dict): + work_item["controlled_executor_handoff"] = handoff + return + proposal_data = { "source": "repair_candidate_controlled_queue", "risk_level": severity or getattr(getattr(incident, "severity", None), "value", ""), @@ -1194,13 +1216,23 @@ class RepairCandidateService: return False if "mcp_evidence_missing" in set(blockers): return False - if "playbook_command_not_safely_routable" in set(blockers): + route = str(template.get("suggested_route") or "").strip() + repair_template = str(template.get("repair_command_template") or "").strip() + safe_agent99_route = bool( + route in { + "agent99_recover_after_owner_review", + "agent99_status_after_owner_review", + } + and repair_template.startswith("agent99-mode ") + ) + if ( + "playbook_command_not_safely_routable" in set(blockers) + and not safe_agent99_route + ): return False if not coverage_gap.get("mcp_evidence_ready"): return False - repair_template = str(template.get("repair_command_template") or "").strip() - route = str(template.get("suggested_route") or "").strip() if not repair_template or not route: return False if "<" in repair_template: @@ -1241,7 +1273,35 @@ class RepairCandidateService: "publish_assets_to_runs_work_items_and_knowledge_base", ] - if target_kind == "k8s_workload": + if target_kind == "control_plane": + normalized_target = target.lower() + if any( + marker in normalized_target + for marker in ( + "cold-start", + "cold_start", + "reboot-auto-recovery", + "reboot_auto_recovery", + ) + ): + command_template = "agent99-mode Recover controlledApply=true" + rollback_template = "agent99-mode Status controlledApply=false" + route = "agent99_recover_after_owner_review" + verifier_plan.extend([ + "agent99 Recover outcome verifier must resolve", + "rerun reboot-auto-recovery-slo-scorecard", + "confirm required hosts, Harbor, K3s, and public routes are green", + ]) + else: + command_template = "agent99-mode Status controlledApply=false" + rollback_template = "no runtime write; retain previous production deployment" + route = "agent99_status_after_owner_review" + verifier_plan.extend([ + "compare failed commit with current production deploy marker", + "confirm public routes and current production services remain green", + "keep source CI failure open until a later successful run is observed", + ]) + elif target_kind == "k8s_workload": workload = target.replace("deployment/", "").replace("pod/", "").replace("svc/", "") workload_ref = workload or "" namespace_ref = ns or "" @@ -1442,6 +1502,25 @@ class RepairCandidateService: def _infer_target_kind(self, *, target_resource: str, namespace: str) -> str: target = (target_resource or "").lower() + control_plane_context = f"{target} {(namespace or '').lower()}" + if any( + marker in control_plane_context + for marker in ( + "cold-start", + "cold_start", + "reboot-auto-recovery", + "reboot_auto_recovery", + "provider_freshness", + "alert-chain", + "alert_chain", + "ci/cd", + "cicd", + "build-and-deploy", + "deployment failed", + "部署失敗", + ) + ): + return "control_plane" if any(marker in target for marker in ("postgres", "postgresql", "pgbouncer", "database", "db-")): return "database" if any(marker in target for marker in ("node-exporter", "host", "188", "111", "168")): diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 944d95a91..f282ec925 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -53,7 +53,7 @@ from src.services.approval_action_classifier import ( ) from src.services.agent99_sre_bridge import ( build_agent99_sre_alert, - dispatch_agent99_sre_alert, + dispatch_agent99_sre_alert_with_receipt, ) from src.services.chat_manager import get_chat_manager from src.services.operator_outcome import build_operator_outcome, normalize_operator_outcome @@ -163,6 +163,17 @@ _AI_SIGNAL_TARGET_LABEL_RE = re.compile( _ALERT_CARD_CODE_VALUE_RE = re.compile( r"(?P