feat(agent99): ship incident lifecycle cards
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -742,6 +742,7 @@ foreach ($file in $files) {
|
||||
$service = [string](Get-AgentField $alert "service" "unknown")
|
||||
$hostName = [string](Get-AgentField $alert "host" "")
|
||||
$replyChatId = [string](Get-AgentField $alert "replyChatId" "")
|
||||
$replyMessageId = [string](Get-AgentField $alert "telegramMessageId" "")
|
||||
$suppressTelegram = Test-AgentAlertSuppressTelegram $alert
|
||||
$resolution = Get-AgentField $alert "resolution" $null
|
||||
$sourceEventResolutionPolicy = [string](Get-AgentField $resolution "policy" "external_source_receipt_required")
|
||||
@@ -768,6 +769,7 @@ foreach ($file in $files) {
|
||||
alertSeverity = $severity
|
||||
alertService = $service
|
||||
alertHost = $hostName
|
||||
replyMessageId = $replyMessageId
|
||||
alertPath = $processedPath
|
||||
suppressTelegram = $suppressTelegram
|
||||
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
|
||||
@@ -796,6 +798,7 @@ foreach ($file in $files) {
|
||||
alertHost = $hostName
|
||||
alertPath = $processedPath
|
||||
replyChatId = $replyChatId
|
||||
replyMessageId = $replyMessageId
|
||||
suppressTelegram = $suppressTelegram
|
||||
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
|
||||
sourceEventResolutionRequired = $sourceEventResolutionRequired
|
||||
|
||||
@@ -40,9 +40,9 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None:
|
||||
"p0_count": 14,
|
||||
"p1_count": 8,
|
||||
"p2_count": 3,
|
||||
"in_progress_count": 9,
|
||||
"in_progress_count": 10,
|
||||
"in_progress_blocked_count": 0,
|
||||
"planned_count": 15,
|
||||
"planned_count": 14,
|
||||
"complete_count": 1,
|
||||
"current_p0_count": 1,
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ ROOT = Path(__file__).resolve().parents[3]
|
||||
CONTROL = ROOT / "agent99-control-plane.ps1"
|
||||
DEPLOY = ROOT / "agent99-deploy.ps1"
|
||||
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
|
||||
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
|
||||
|
||||
|
||||
def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None:
|
||||
@@ -47,13 +48,48 @@ def test_agent99_recovery_completion_has_human_lifecycle_receipt() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
|
||||
assert '$EventType -eq "recovery_slo_result"' in source
|
||||
assert '主機與服務恢復 / recovery lifecycle' in source
|
||||
assert '已恢復且符合目標 / recovered within target' in source
|
||||
assert '$lifecycleInfoAlert' in source
|
||||
assert '$EventType -in @("recovery_slo_result")' in source
|
||||
assert 'Agent99 恢復狀態 / Recovery Status' in source
|
||||
assert '恢復階段 / RECOVERY PHASES' in source
|
||||
assert 'Detect -> Apply -> Verify -> Notify receipt' in source
|
||||
assert 'schemaVersion = "agent99_incident_card_v5"' in source
|
||||
assert '主機與服務已通過恢復驗證' in source
|
||||
assert '本輪不是 fresh reboot drill,不宣稱全主機 SLA' in source
|
||||
assert 'Agent99 事故狀態 / INCIDENT STATUS' in source
|
||||
assert '使用者影響' in source
|
||||
assert 'Agent99 實際動作' in source
|
||||
assert '獨立驗證' in source
|
||||
|
||||
|
||||
def test_agent99_telegram_v5_hides_raw_evidence_and_has_operator_fields() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
formatter = source[source.index("function Format-AgentTelegramText {") :]
|
||||
formatter = formatter[: formatter.index("function Format-AgentTelegramCaption")]
|
||||
|
||||
assert '"資產:$($card.target)"' in formatter
|
||||
assert '"影響:$($card.impact)"' in formatter
|
||||
assert '"Agent99:$($card.action)"' in formatter
|
||||
assert '"驗證:$($card.verification)"' in formatter
|
||||
assert '"事件:$($card.incidentId)"' in formatter
|
||||
assert "evidencePath" not in formatter
|
||||
assert "candidatePath" not in formatter
|
||||
assert "kmPath" not in formatter
|
||||
assert "verifierName=" not in formatter
|
||||
assert 'messageFormat = "incident_card_zh_tw_v5"' in source
|
||||
|
||||
|
||||
def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
inbox = SRE_INBOX.read_text(encoding="utf-8")
|
||||
|
||||
assert 'Get-AgentField $alert "telegramMessageId"' in inbox
|
||||
assert "replyMessageId = $replyMessageId" in inbox
|
||||
assert 'PSObject.Properties["replyMessageId"]' in source
|
||||
assert 'PSObject.Properties["correlationKey"]' in source
|
||||
assert '"reply_parameters"' in source
|
||||
assert 'telegram_receipt_b64=' in source
|
||||
assert 'schemaVersion = "agent99_telegram_incident_state_v1"' in source
|
||||
assert 'rootMessageId = $rootMessageId' in source
|
||||
assert 'reason = "same_lifecycle_state"' in source
|
||||
assert 'reason = "dedupe_window"' in source
|
||||
|
||||
|
||||
def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None:
|
||||
@@ -65,6 +101,7 @@ def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None:
|
||||
assert 'suppressed = $true' in dedupe
|
||||
assert 'reason = "dedupe_window"' in dedupe
|
||||
assert 'dedupeAgeMinutes' in dedupe
|
||||
assert dedupe.index("Send-AgentTelegram") < len(dedupe)
|
||||
|
||||
|
||||
def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None:
|
||||
|
||||
@@ -655,7 +655,7 @@
|
||||
"id": "AG99-P1-003",
|
||||
"order": 17,
|
||||
"priority": "P1",
|
||||
"status": "planned",
|
||||
"status": "in_progress",
|
||||
"title": "Human-readable Traditional Chinese Telegram incident cards",
|
||||
"owner_lane": "telegram_incident_experience",
|
||||
"dependencies": ["AG99-P0-006"],
|
||||
@@ -666,8 +666,9 @@
|
||||
"prohibited_actions": ["send raw verifier dumps", "repeat the same text as an image", "expose internal paths or secrets", "declare resolved from send success", "send periodic same-state noise"],
|
||||
"verifier": "Message snapshot tests, duplicate suppression, state-change delivery and evidence-link checks.",
|
||||
"acceptance": "Every card explains severity, affected service or host, user impact, lifecycle state, AI diagnosis, controlled action, verifier result, duration, recurrence count and next update in Traditional Chinese. Charts or real screenshots are attached only when they improve understanding; one fingerprint produces one lifecycle thread and no same-state spam.",
|
||||
"implementation_progress": "The 2026-07-11 Agent99 incident-card screenshot was captured as a failing UX fixture and added to the P1 acceptance contract; runtime renderer replacement remains pending behind the current host 112 P0 closure.",
|
||||
"next_action": "Replace the raw automation-result renderer with detected, progress and recovered card templates; add fingerprint edit/thread dedup, redaction, chart-or-screenshot evidence rules and Telegram delivery snapshot tests."
|
||||
"implementation_progress": "The v5 source now normalizes every Agent99 alert into one Traditional Chinese incident model with deterministic incident ID/fingerprint, severity, lifecycle, affected asset, user impact, actual Agent99 action, independent verifier result, duration, recurrence and next update. Operator-command captions no longer expose Windows evidence paths, KM paths or raw verifier fields. Warning/critical and verified recovery events render 1200x760 evidence views for recovery phases, performance metrics, backup freshness and automation/freshness gates; external screenshots are accepted only when visualEvidenceVerified=true. Telegram source message IDs are preserved through the SRE inbox so Agent99 replies under the original alert, incident root/message state is persisted, same lifecycle state is suppressed until state changes, and failed sends do not create a dedupe marker. Windows 99 staging parser reports zero errors; the no-send self-test reports incident_card_zh_tw_v5, operator lifecycle recovered, visual generated, lifecycle state test passed, same_lifecycle_state suppression and root message preservation. Runtime atomic deployment and a production-principal lifecycle receipt remain open.",
|
||||
"evidence_refs": ["C:\\Wooo\\Agent99\\staging\\codex-alert-v5-20260711-202853\\evidence\\agent99-TelegramCardSelfTest-*.json", "Windows PowerShell 5.1 parser errors=0", "Agent99 tests 59 passed", "reboot-recovery tests 145 passed", "apps/api/tests/test_agent99_transport_recovery_deploy_contract.py", "scripts/reboot-recovery/agent99-recover-receipt-readback.ps1"],
|
||||
"next_action": "Create a local audited source commit, atomically deploy it to Windows 99, then run one production-principal Recover and require the v5 recovery lifecycle card, visual delivery or explicit same-state suppression, Telegram message receipt, persistent incident state and independent recovery verifier to pass. Do not send periodic test noise."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P1-004",
|
||||
@@ -790,9 +791,9 @@
|
||||
"p0_count": 14,
|
||||
"p1_count": 8,
|
||||
"p2_count": 3,
|
||||
"in_progress_count": 9,
|
||||
"in_progress_count": 10,
|
||||
"in_progress_blocked_count": 0,
|
||||
"planned_count": 15,
|
||||
"planned_count": 14,
|
||||
"complete_count": 1,
|
||||
"current_p0_count": 1
|
||||
},
|
||||
|
||||
158
scripts/reboot-recovery/agent99-recover-receipt-readback.ps1
Normal file
158
scripts/reboot-recovery/agent99-recover-receipt-readback.ps1
Normal file
@@ -0,0 +1,158 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99",
|
||||
[string]$NotBefore = ""
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$notBeforeTime = [datetime]::MinValue
|
||||
if ($NotBefore) {
|
||||
$notBeforeTime = [datetime]::Parse($NotBefore)
|
||||
}
|
||||
|
||||
$evidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$file = Get-ChildItem -Path $evidenceDir -Filter "agent99-Recover-*.json" -File -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.LastWriteTime -ge $notBeforeTime } |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -First 1
|
||||
if (-not $file) {
|
||||
throw "new_recover_evidence_missing"
|
||||
}
|
||||
|
||||
$raw = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
|
||||
$phases = @($raw.recoverySlo.phases | ForEach-Object {
|
||||
[pscustomobject]@{
|
||||
name = [string]$_.name
|
||||
status = [string]$_.status
|
||||
elapsedSeconds = [double]$_.elapsedSeconds
|
||||
}
|
||||
})
|
||||
$telegram = @($raw.telegram | ForEach-Object {
|
||||
$visualPath = if ($_.PSObject.Properties["visualPath"]) { [string]$_.visualPath } else { "" }
|
||||
[pscustomobject]@{
|
||||
eventType = [string]$_.eventType
|
||||
severity = [string]$_.severity
|
||||
sent = [bool]$_.sent
|
||||
suppressed = if ($_.PSObject.Properties["suppressed"]) { [bool]$_.suppressed } else { $false }
|
||||
reason = if ($_.PSObject.Properties["reason"]) { [string]$_.reason } else { "" }
|
||||
messageFormat = if ($_.PSObject.Properties["messageFormat"]) { [string]$_.messageFormat } else { "" }
|
||||
incidentId = if ($_.PSObject.Properties["incidentId"]) { [string]$_.incidentId } else { "" }
|
||||
fingerprint = if ($_.PSObject.Properties["fingerprint"]) { [string]$_.fingerprint } else { "" }
|
||||
lifecycle = if ($_.PSObject.Properties["lifecycle"]) { [string]$_.lifecycle } else { "" }
|
||||
stateKey = if ($_.PSObject.Properties["stateKey"]) { [string]$_.stateKey } else { "" }
|
||||
visualSent = if ($_.PSObject.Properties["visualSent"]) { [bool]$_.visualSent } else { $false }
|
||||
messageIdPresent = [bool]($_.PSObject.Properties["messageId"] -and $_.messageId)
|
||||
replyThreaded = [bool]($_.PSObject.Properties["replyToMessageId"] -and $_.replyToMessageId)
|
||||
incidentStateWritten = [bool]($_.PSObject.Properties["incidentStateWritten"] -and $_.incidentStateWritten)
|
||||
visualFile = if ($visualPath) { Split-Path -Leaf $visualPath } else { "" }
|
||||
visualExists = [bool]($visualPath -and (Test-Path -LiteralPath $visualPath))
|
||||
}
|
||||
})
|
||||
|
||||
$phaseFailures = @($phases | Where-Object { $_.status -ne "ok" })
|
||||
$sentCount = @($telegram | Where-Object { $_.sent }).Count
|
||||
$suppressedCount = @($telegram | Where-Object { $_.suppressed }).Count
|
||||
$lifecycleReceipts = @($telegram | Where-Object { $_.eventType -eq "recovery_slo_result" -and ($_.sent -or $_.suppressed) })
|
||||
$sentLifecycleReceipts = @($lifecycleReceipts | Where-Object { $_.sent })
|
||||
$notificationReceipted = [bool]($lifecycleReceipts.Count -gt 0)
|
||||
$humanReadableCardReady = [bool](
|
||||
$lifecycleReceipts.Count -gt 0 -and
|
||||
@($lifecycleReceipts | Where-Object {
|
||||
$_.messageFormat -ne "incident_card_zh_tw_v5" -or
|
||||
$_.incidentId -notmatch "^INC-[A-F0-9]{8}$" -or
|
||||
$_.lifecycle -notin @("remediating", "recovered", "blocked")
|
||||
}).Count -eq 0
|
||||
)
|
||||
$visualLifecycleReady = [bool](@($sentLifecycleReceipts | Where-Object { -not $_.visualSent }).Count -eq 0)
|
||||
$incidentStateReady = [bool](@($sentLifecycleReceipts | Where-Object { -not $_.messageIdPresent -or -not $_.incidentStateWritten }).Count -eq 0)
|
||||
$recoveryVerified = [bool](
|
||||
$raw.mode -eq "Recover" -and
|
||||
$raw.recoverySlo.completed -and
|
||||
$raw.recoverySlo.coordinatorVerified -and
|
||||
$phases.Count -gt 0 -and
|
||||
$phaseFailures.Count -eq 0
|
||||
)
|
||||
$recentReceipts = @(Get-ChildItem -Path $evidenceDir -Filter "agent99-Recover-*.json" -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -First 5 |
|
||||
ForEach-Object {
|
||||
$historyFile = $_
|
||||
try {
|
||||
$history = Get-Content -Path $historyFile.FullName -Raw | ConvertFrom-Json
|
||||
$historyTelegram = @($history.telegram)
|
||||
[pscustomobject]@{
|
||||
evidenceFile = $historyFile.Name
|
||||
evidenceLastWriteTime = $historyFile.LastWriteTime.ToString("o")
|
||||
runId = [string]$history.recoverySlo.runId
|
||||
completed = [bool]$history.recoverySlo.completed
|
||||
telegramAttemptCount = $historyTelegram.Count
|
||||
telegramSentCount = @($historyTelegram | Where-Object { $_.sent }).Count
|
||||
telegramSuppressedCount = @($historyTelegram | Where-Object { $_.PSObject.Properties["suppressed"] -and $_.suppressed }).Count
|
||||
visualSentCount = @($historyTelegram | Where-Object { $_.PSObject.Properties["visualSent"] -and $_.visualSent }).Count
|
||||
visualFiles = @($historyTelegram | Where-Object { $_.PSObject.Properties["visualPath"] -and $_.visualPath } | ForEach-Object { Split-Path -Leaf ([string]$_.visualPath) })
|
||||
}
|
||||
} catch {
|
||||
[pscustomobject]@{
|
||||
evidenceFile = $historyFile.Name
|
||||
evidenceLastWriteTime = $historyFile.LastWriteTime.ToString("o")
|
||||
parseError = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_recover_receipt_readback_v1"
|
||||
observedAt = (Get-Date).ToString("o")
|
||||
evidenceFile = $file.Name
|
||||
evidenceLastWriteTime = $file.LastWriteTime.ToString("o")
|
||||
secretValueRead = $false
|
||||
remoteWritePerformed = $false
|
||||
mode = [string]$raw.mode
|
||||
controlledApply = [bool]$raw.controlledApply
|
||||
vmCount = @($raw.vmResults).Count
|
||||
vmOkCount = @($raw.vmResults | Where-Object { $_.ok -or $_.skipped }).Count
|
||||
hostCount = @($raw.hosts).Count
|
||||
hostReachableCount = @($raw.hosts | Where-Object { $_.ping }).Count
|
||||
publicCount = @($raw.public).Count
|
||||
publicOkCount = @($raw.public | Where-Object { $_.ok }).Count
|
||||
recovery = [pscustomobject]@{
|
||||
runId = [string]$raw.recoverySlo.runId
|
||||
scope = [string]$raw.recoverySlo.scope
|
||||
completed = [bool]$raw.recoverySlo.completed
|
||||
withinSlo = [bool]$raw.recoverySlo.withinSlo
|
||||
elapsedSeconds = [double]$raw.recoverySlo.elapsedSeconds
|
||||
coordinatorVerified = [bool]$raw.recoverySlo.coordinatorVerified
|
||||
rebootSloClaimed = [bool]$raw.recoverySlo.rebootSloClaimed
|
||||
phases = $phases
|
||||
}
|
||||
telegramAttemptCount = $telegram.Count
|
||||
telegramSentCount = $sentCount
|
||||
telegramSuppressedCount = $suppressedCount
|
||||
lifecycleReceiptCount = $lifecycleReceipts.Count
|
||||
notificationReceipted = $notificationReceipted
|
||||
humanReadableCardReady = $humanReadableCardReady
|
||||
visualLifecycleReady = $visualLifecycleReady
|
||||
incidentStateReady = $incidentStateReady
|
||||
telegram = $telegram
|
||||
recentReceipts = $recentReceipts
|
||||
recoveryVerified = $recoveryVerified
|
||||
ok = [bool]($recoveryVerified -and $notificationReceipted -and $humanReadableCardReady -and $visualLifecycleReady -and $incidentStateReady)
|
||||
}
|
||||
|
||||
Write-Output "RECOVER_RECEIPT_OK=$([int]$result.ok)"
|
||||
Write-Output "RECOVERY_VERIFIED=$([int]$result.recoveryVerified)"
|
||||
Write-Output "NOTIFICATION_RECEIPTED=$([int]$result.notificationReceipted)"
|
||||
Write-Output "HUMAN_READABLE_CARD_READY=$([int]$result.humanReadableCardReady)"
|
||||
Write-Output "VISUAL_LIFECYCLE_READY=$([int]$result.visualLifecycleReady)"
|
||||
Write-Output "INCIDENT_STATE_READY=$([int]$result.incidentStateReady)"
|
||||
Write-Output "TELEGRAM_ATTEMPTS=$($result.telegramAttemptCount)"
|
||||
Write-Output "TELEGRAM_SENT=$($result.telegramSentCount)"
|
||||
Write-Output "TELEGRAM_SUPPRESSED=$($result.telegramSuppressedCount)"
|
||||
Write-Output "SECRET_VALUE_READ=0"
|
||||
Write-Output "REMOTE_WRITE_PERFORMED=0"
|
||||
Write-Output "JSON_BEGIN"
|
||||
$result | ConvertTo-Json -Depth 10 -Compress
|
||||
Write-Output "JSON_END"
|
||||
|
||||
if ($result.ok) { exit 0 }
|
||||
exit 2
|
||||
@@ -147,3 +147,23 @@ def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() ->
|
||||
assert safe_field in preflight
|
||||
for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"):
|
||||
assert forbidden not in preflight
|
||||
|
||||
|
||||
def test_recover_receipt_readback_requires_verifier_and_notification() -> None:
|
||||
readback = read("scripts/reboot-recovery/agent99-recover-receipt-readback.ps1")
|
||||
|
||||
assert 'schemaVersion = "agent99_recover_receipt_readback_v1"' in readback
|
||||
assert '$raw.recoverySlo.coordinatorVerified' in readback
|
||||
assert '$phaseFailures.Count -eq 0' in readback
|
||||
assert '$notificationReceipted' in readback
|
||||
assert '$humanReadableCardReady' in readback
|
||||
assert '$visualLifecycleReady' in readback
|
||||
assert '$incidentStateReady' in readback
|
||||
assert 'incident_card_zh_tw_v5' in readback
|
||||
assert '$recoveryVerified -and $notificationReceipted -and $humanReadableCardReady' in readback
|
||||
assert '$recentReceipts' in readback
|
||||
assert 'visualSentCount' in readback
|
||||
assert 'secretValueRead = $false' in readback
|
||||
assert 'remoteWritePerformed = $false' in readback
|
||||
for forbidden in ("botToken", "chatId", "messageText", "rawUpdate", ".env"):
|
||||
assert forbidden not in readback
|
||||
|
||||
Reference in New Issue
Block a user