feat(agent99): ship incident lifecycle cards

This commit is contained in:
ogt
2026-07-11 20:38:12 +08:00
parent 0a671bd46a
commit 83f7838131
7 changed files with 926 additions and 124 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -742,6 +742,7 @@ foreach ($file in $files) {
$service = [string](Get-AgentField $alert "service" "unknown")
$hostName = [string](Get-AgentField $alert "host" "")
$replyChatId = [string](Get-AgentField $alert "replyChatId" "")
$replyMessageId = [string](Get-AgentField $alert "telegramMessageId" "")
$suppressTelegram = Test-AgentAlertSuppressTelegram $alert
$resolution = Get-AgentField $alert "resolution" $null
$sourceEventResolutionPolicy = [string](Get-AgentField $resolution "policy" "external_source_receipt_required")
@@ -768,6 +769,7 @@ foreach ($file in $files) {
alertSeverity = $severity
alertService = $service
alertHost = $hostName
replyMessageId = $replyMessageId
alertPath = $processedPath
suppressTelegram = $suppressTelegram
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
@@ -796,6 +798,7 @@ foreach ($file in $files) {
alertHost = $hostName
alertPath = $processedPath
replyChatId = $replyChatId
replyMessageId = $replyMessageId
suppressTelegram = $suppressTelegram
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
sourceEventResolutionRequired = $sourceEventResolutionRequired

View File

@@ -40,9 +40,9 @@ def test_agent99_enterprise_work_items_loader_returns_complete_scope() -> None:
"p0_count": 14,
"p1_count": 8,
"p2_count": 3,
"in_progress_count": 9,
"in_progress_count": 10,
"in_progress_blocked_count": 0,
"planned_count": 15,
"planned_count": 14,
"complete_count": 1,
"current_p0_count": 1,
}

View File

@@ -8,6 +8,7 @@ ROOT = Path(__file__).resolve().parents[3]
CONTROL = ROOT / "agent99-control-plane.ps1"
DEPLOY = ROOT / "agent99-deploy.ps1"
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None:
@@ -47,13 +48,48 @@ def test_agent99_recovery_completion_has_human_lifecycle_receipt() -> None:
source = CONTROL.read_text(encoding="utf-8")
assert '$EventType -eq "recovery_slo_result"' in source
assert '主機與服務恢復 / recovery lifecycle' in source
assert '已恢復且符合目標 / recovered within target' in source
assert '$lifecycleInfoAlert' in source
assert '$EventType -in @("recovery_slo_result")' in source
assert 'Agent99 恢復狀態 / Recovery Status' in source
assert '恢復階段 / RECOVERY PHASES' in source
assert 'Detect -> Apply -> Verify -> Notify receipt' in source
assert 'schemaVersion = "agent99_incident_card_v5"' in source
assert '主機與服務已通過恢復驗證' in source
assert '本輪不是 fresh reboot drill不宣稱全主機 SLA' in source
assert 'Agent99 事故狀態 / INCIDENT STATUS' in source
assert '使用者影響' in source
assert 'Agent99 實際動作' in source
assert '獨立驗證' in source
def test_agent99_telegram_v5_hides_raw_evidence_and_has_operator_fields() -> None:
source = CONTROL.read_text(encoding="utf-8")
formatter = source[source.index("function Format-AgentTelegramText {") :]
formatter = formatter[: formatter.index("function Format-AgentTelegramCaption")]
assert '"資產:$($card.target)"' in formatter
assert '"影響:$($card.impact)"' in formatter
assert '"Agent99$($card.action)"' in formatter
assert '"驗證:$($card.verification)"' in formatter
assert '"事件:$($card.incidentId)"' in formatter
assert "evidencePath" not in formatter
assert "candidatePath" not in formatter
assert "kmPath" not in formatter
assert "verifierName=" not in formatter
assert 'messageFormat = "incident_card_zh_tw_v5"' in source
def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None:
source = CONTROL.read_text(encoding="utf-8")
inbox = SRE_INBOX.read_text(encoding="utf-8")
assert 'Get-AgentField $alert "telegramMessageId"' in inbox
assert "replyMessageId = $replyMessageId" in inbox
assert 'PSObject.Properties["replyMessageId"]' in source
assert 'PSObject.Properties["correlationKey"]' in source
assert '"reply_parameters"' in source
assert 'telegram_receipt_b64=' in source
assert 'schemaVersion = "agent99_telegram_incident_state_v1"' in source
assert 'rootMessageId = $rootMessageId' in source
assert 'reason = "same_lifecycle_state"' in source
assert 'reason = "dedupe_window"' in source
def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None:
@@ -65,6 +101,7 @@ def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None:
assert 'suppressed = $true' in dedupe
assert 'reason = "dedupe_window"' in dedupe
assert 'dedupeAgeMinutes' in dedupe
assert dedupe.index("Send-AgentTelegram") < len(dedupe)
def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None:

View File

@@ -655,7 +655,7 @@
"id": "AG99-P1-003",
"order": 17,
"priority": "P1",
"status": "planned",
"status": "in_progress",
"title": "Human-readable Traditional Chinese Telegram incident cards",
"owner_lane": "telegram_incident_experience",
"dependencies": ["AG99-P0-006"],
@@ -666,8 +666,9 @@
"prohibited_actions": ["send raw verifier dumps", "repeat the same text as an image", "expose internal paths or secrets", "declare resolved from send success", "send periodic same-state noise"],
"verifier": "Message snapshot tests, duplicate suppression, state-change delivery and evidence-link checks.",
"acceptance": "Every card explains severity, affected service or host, user impact, lifecycle state, AI diagnosis, controlled action, verifier result, duration, recurrence count and next update in Traditional Chinese. Charts or real screenshots are attached only when they improve understanding; one fingerprint produces one lifecycle thread and no same-state spam.",
"implementation_progress": "The 2026-07-11 Agent99 incident-card screenshot was captured as a failing UX fixture and added to the P1 acceptance contract; runtime renderer replacement remains pending behind the current host 112 P0 closure.",
"next_action": "Replace the raw automation-result renderer with detected, progress and recovered card templates; add fingerprint edit/thread dedup, redaction, chart-or-screenshot evidence rules and Telegram delivery snapshot tests."
"implementation_progress": "The v5 source now normalizes every Agent99 alert into one Traditional Chinese incident model with deterministic incident ID/fingerprint, severity, lifecycle, affected asset, user impact, actual Agent99 action, independent verifier result, duration, recurrence and next update. Operator-command captions no longer expose Windows evidence paths, KM paths or raw verifier fields. Warning/critical and verified recovery events render 1200x760 evidence views for recovery phases, performance metrics, backup freshness and automation/freshness gates; external screenshots are accepted only when visualEvidenceVerified=true. Telegram source message IDs are preserved through the SRE inbox so Agent99 replies under the original alert, incident root/message state is persisted, same lifecycle state is suppressed until state changes, and failed sends do not create a dedupe marker. Windows 99 staging parser reports zero errors; the no-send self-test reports incident_card_zh_tw_v5, operator lifecycle recovered, visual generated, lifecycle state test passed, same_lifecycle_state suppression and root message preservation. Runtime atomic deployment and a production-principal lifecycle receipt remain open.",
"evidence_refs": ["C:\\Wooo\\Agent99\\staging\\codex-alert-v5-20260711-202853\\evidence\\agent99-TelegramCardSelfTest-*.json", "Windows PowerShell 5.1 parser errors=0", "Agent99 tests 59 passed", "reboot-recovery tests 145 passed", "apps/api/tests/test_agent99_transport_recovery_deploy_contract.py", "scripts/reboot-recovery/agent99-recover-receipt-readback.ps1"],
"next_action": "Create a local audited source commit, atomically deploy it to Windows 99, then run one production-principal Recover and require the v5 recovery lifecycle card, visual delivery or explicit same-state suppression, Telegram message receipt, persistent incident state and independent recovery verifier to pass. Do not send periodic test noise."
},
{
"id": "AG99-P1-004",
@@ -790,9 +791,9 @@
"p0_count": 14,
"p1_count": 8,
"p2_count": 3,
"in_progress_count": 9,
"in_progress_count": 10,
"in_progress_blocked_count": 0,
"planned_count": 15,
"planned_count": 14,
"complete_count": 1,
"current_p0_count": 1
},

View File

@@ -0,0 +1,158 @@
[CmdletBinding()]
param(
[string]$AgentRoot = "C:\Wooo\Agent99",
[string]$NotBefore = ""
)
$ErrorActionPreference = "Stop"
$notBeforeTime = [datetime]::MinValue
if ($NotBefore) {
$notBeforeTime = [datetime]::Parse($NotBefore)
}
$evidenceDir = Join-Path $AgentRoot "evidence"
$file = Get-ChildItem -Path $evidenceDir -Filter "agent99-Recover-*.json" -File -ErrorAction SilentlyContinue |
Where-Object { $_.LastWriteTime -ge $notBeforeTime } |
Sort-Object LastWriteTime -Descending |
Select-Object -First 1
if (-not $file) {
throw "new_recover_evidence_missing"
}
$raw = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
$phases = @($raw.recoverySlo.phases | ForEach-Object {
[pscustomobject]@{
name = [string]$_.name
status = [string]$_.status
elapsedSeconds = [double]$_.elapsedSeconds
}
})
$telegram = @($raw.telegram | ForEach-Object {
$visualPath = if ($_.PSObject.Properties["visualPath"]) { [string]$_.visualPath } else { "" }
[pscustomobject]@{
eventType = [string]$_.eventType
severity = [string]$_.severity
sent = [bool]$_.sent
suppressed = if ($_.PSObject.Properties["suppressed"]) { [bool]$_.suppressed } else { $false }
reason = if ($_.PSObject.Properties["reason"]) { [string]$_.reason } else { "" }
messageFormat = if ($_.PSObject.Properties["messageFormat"]) { [string]$_.messageFormat } else { "" }
incidentId = if ($_.PSObject.Properties["incidentId"]) { [string]$_.incidentId } else { "" }
fingerprint = if ($_.PSObject.Properties["fingerprint"]) { [string]$_.fingerprint } else { "" }
lifecycle = if ($_.PSObject.Properties["lifecycle"]) { [string]$_.lifecycle } else { "" }
stateKey = if ($_.PSObject.Properties["stateKey"]) { [string]$_.stateKey } else { "" }
visualSent = if ($_.PSObject.Properties["visualSent"]) { [bool]$_.visualSent } else { $false }
messageIdPresent = [bool]($_.PSObject.Properties["messageId"] -and $_.messageId)
replyThreaded = [bool]($_.PSObject.Properties["replyToMessageId"] -and $_.replyToMessageId)
incidentStateWritten = [bool]($_.PSObject.Properties["incidentStateWritten"] -and $_.incidentStateWritten)
visualFile = if ($visualPath) { Split-Path -Leaf $visualPath } else { "" }
visualExists = [bool]($visualPath -and (Test-Path -LiteralPath $visualPath))
}
})
$phaseFailures = @($phases | Where-Object { $_.status -ne "ok" })
$sentCount = @($telegram | Where-Object { $_.sent }).Count
$suppressedCount = @($telegram | Where-Object { $_.suppressed }).Count
$lifecycleReceipts = @($telegram | Where-Object { $_.eventType -eq "recovery_slo_result" -and ($_.sent -or $_.suppressed) })
$sentLifecycleReceipts = @($lifecycleReceipts | Where-Object { $_.sent })
$notificationReceipted = [bool]($lifecycleReceipts.Count -gt 0)
$humanReadableCardReady = [bool](
$lifecycleReceipts.Count -gt 0 -and
@($lifecycleReceipts | Where-Object {
$_.messageFormat -ne "incident_card_zh_tw_v5" -or
$_.incidentId -notmatch "^INC-[A-F0-9]{8}$" -or
$_.lifecycle -notin @("remediating", "recovered", "blocked")
}).Count -eq 0
)
$visualLifecycleReady = [bool](@($sentLifecycleReceipts | Where-Object { -not $_.visualSent }).Count -eq 0)
$incidentStateReady = [bool](@($sentLifecycleReceipts | Where-Object { -not $_.messageIdPresent -or -not $_.incidentStateWritten }).Count -eq 0)
$recoveryVerified = [bool](
$raw.mode -eq "Recover" -and
$raw.recoverySlo.completed -and
$raw.recoverySlo.coordinatorVerified -and
$phases.Count -gt 0 -and
$phaseFailures.Count -eq 0
)
$recentReceipts = @(Get-ChildItem -Path $evidenceDir -Filter "agent99-Recover-*.json" -File -ErrorAction SilentlyContinue |
Sort-Object LastWriteTime -Descending |
Select-Object -First 5 |
ForEach-Object {
$historyFile = $_
try {
$history = Get-Content -Path $historyFile.FullName -Raw | ConvertFrom-Json
$historyTelegram = @($history.telegram)
[pscustomobject]@{
evidenceFile = $historyFile.Name
evidenceLastWriteTime = $historyFile.LastWriteTime.ToString("o")
runId = [string]$history.recoverySlo.runId
completed = [bool]$history.recoverySlo.completed
telegramAttemptCount = $historyTelegram.Count
telegramSentCount = @($historyTelegram | Where-Object { $_.sent }).Count
telegramSuppressedCount = @($historyTelegram | Where-Object { $_.PSObject.Properties["suppressed"] -and $_.suppressed }).Count
visualSentCount = @($historyTelegram | Where-Object { $_.PSObject.Properties["visualSent"] -and $_.visualSent }).Count
visualFiles = @($historyTelegram | Where-Object { $_.PSObject.Properties["visualPath"] -and $_.visualPath } | ForEach-Object { Split-Path -Leaf ([string]$_.visualPath) })
}
} catch {
[pscustomobject]@{
evidenceFile = $historyFile.Name
evidenceLastWriteTime = $historyFile.LastWriteTime.ToString("o")
parseError = $_.Exception.GetType().Name
}
}
})
$result = [pscustomobject]@{
schemaVersion = "agent99_recover_receipt_readback_v1"
observedAt = (Get-Date).ToString("o")
evidenceFile = $file.Name
evidenceLastWriteTime = $file.LastWriteTime.ToString("o")
secretValueRead = $false
remoteWritePerformed = $false
mode = [string]$raw.mode
controlledApply = [bool]$raw.controlledApply
vmCount = @($raw.vmResults).Count
vmOkCount = @($raw.vmResults | Where-Object { $_.ok -or $_.skipped }).Count
hostCount = @($raw.hosts).Count
hostReachableCount = @($raw.hosts | Where-Object { $_.ping }).Count
publicCount = @($raw.public).Count
publicOkCount = @($raw.public | Where-Object { $_.ok }).Count
recovery = [pscustomobject]@{
runId = [string]$raw.recoverySlo.runId
scope = [string]$raw.recoverySlo.scope
completed = [bool]$raw.recoverySlo.completed
withinSlo = [bool]$raw.recoverySlo.withinSlo
elapsedSeconds = [double]$raw.recoverySlo.elapsedSeconds
coordinatorVerified = [bool]$raw.recoverySlo.coordinatorVerified
rebootSloClaimed = [bool]$raw.recoverySlo.rebootSloClaimed
phases = $phases
}
telegramAttemptCount = $telegram.Count
telegramSentCount = $sentCount
telegramSuppressedCount = $suppressedCount
lifecycleReceiptCount = $lifecycleReceipts.Count
notificationReceipted = $notificationReceipted
humanReadableCardReady = $humanReadableCardReady
visualLifecycleReady = $visualLifecycleReady
incidentStateReady = $incidentStateReady
telegram = $telegram
recentReceipts = $recentReceipts
recoveryVerified = $recoveryVerified
ok = [bool]($recoveryVerified -and $notificationReceipted -and $humanReadableCardReady -and $visualLifecycleReady -and $incidentStateReady)
}
Write-Output "RECOVER_RECEIPT_OK=$([int]$result.ok)"
Write-Output "RECOVERY_VERIFIED=$([int]$result.recoveryVerified)"
Write-Output "NOTIFICATION_RECEIPTED=$([int]$result.notificationReceipted)"
Write-Output "HUMAN_READABLE_CARD_READY=$([int]$result.humanReadableCardReady)"
Write-Output "VISUAL_LIFECYCLE_READY=$([int]$result.visualLifecycleReady)"
Write-Output "INCIDENT_STATE_READY=$([int]$result.incidentStateReady)"
Write-Output "TELEGRAM_ATTEMPTS=$($result.telegramAttemptCount)"
Write-Output "TELEGRAM_SENT=$($result.telegramSentCount)"
Write-Output "TELEGRAM_SUPPRESSED=$($result.telegramSuppressedCount)"
Write-Output "SECRET_VALUE_READ=0"
Write-Output "REMOTE_WRITE_PERFORMED=0"
Write-Output "JSON_BEGIN"
$result | ConvertTo-Json -Depth 10 -Compress
Write-Output "JSON_END"
if ($result.ok) { exit 0 }
exit 2

View File

@@ -147,3 +147,23 @@ def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() ->
assert safe_field in preflight
for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"):
assert forbidden not in preflight
def test_recover_receipt_readback_requires_verifier_and_notification() -> None:
readback = read("scripts/reboot-recovery/agent99-recover-receipt-readback.ps1")
assert 'schemaVersion = "agent99_recover_receipt_readback_v1"' in readback
assert '$raw.recoverySlo.coordinatorVerified' in readback
assert '$phaseFailures.Count -eq 0' in readback
assert '$notificationReceipted' in readback
assert '$humanReadableCardReady' in readback
assert '$visualLifecycleReady' in readback
assert '$incidentStateReady' in readback
assert 'incident_card_zh_tw_v5' in readback
assert '$recoveryVerified -and $notificationReceipted -and $humanReadableCardReady' in readback
assert '$recentReceipts' in readback
assert 'visualSentCount' in readback
assert 'secretValueRead = $false' in readback
assert 'remoteWritePerformed = $false' in readback
for forbidden in ("botToken", "chatId", "messageText", "rawUpdate", ".env"):
assert forbidden not in readback