fix(agent99): defer canonical recovery during maintenance

2026-07-22 17:14:52 +08:00
parent b72e8a4de3
commit 56ac4c06f6
4 changed files with 314 additions and 30 deletions
--- a/scripts/reboot-recovery/agent99-live-preflight.ps1
+++ b/scripts/reboot-recovery/agent99-live-preflight.ps1
@@ -448,6 +448,146 @@ function Get-AgentControlLoopMaintenanceReadback {
  }
 }

+function Get-AgentPreflightSha256Hex {
+  param([string]$Text)
+
+  $sha = [Security.Cryptography.SHA256]::Create()
+  try {
+    $bytes = $sha.ComputeHash([Text.Encoding]::UTF8.GetBytes([string]$Text))
+    return [BitConverter]::ToString($bytes).Replace("-", "").ToLowerInvariant()
+  } finally {
+    $sha.Dispose()
+  }
+}
+
+function Convert-AgentPreflightGuidToNetworkBytes {
+  param([guid]$Guid)
+
+  [byte[]]$bytes = $Guid.ToByteArray()
+  return [byte[]]@(
+    $bytes[3], $bytes[2], $bytes[1], $bytes[0],
+    $bytes[5], $bytes[4], $bytes[7], $bytes[6],
+    $bytes[8], $bytes[9], $bytes[10], $bytes[11],
+    $bytes[12], $bytes[13], $bytes[14], $bytes[15]
+  )
+}
+
+function New-AgentPreflightUuidV5 {
+  param(
+    [guid]$Namespace,
+    [string]$Name
+  )
+
+  [byte[]]$namespaceBytes = Convert-AgentPreflightGuidToNetworkBytes $Namespace
+  [byte[]]$nameBytes = [Text.Encoding]::UTF8.GetBytes($Name)
+  $sha1 = [Security.Cryptography.SHA1]::Create()
+  try {
+    [byte[]]$hash = $sha1.ComputeHash($namespaceBytes + $nameBytes)
+  } finally {
+    $sha1.Dispose()
+  }
+  $hash[6] = [byte](($hash[6] -band 0x0f) -bor 0x50)
+  $hash[8] = [byte](($hash[8] -band 0x3f) -bor 0x80)
+  $hex = -join @($hash[0..15] | ForEach-Object { $_.ToString("x2") })
+  return [guid]::Parse((
+    "{0}-{1}-{2}-{3}-{4}" -f
+      $hex.Substring(0, 8),
+      $hex.Substring(8, 4),
+      $hex.Substring(12, 4),
+      $hex.Substring(16, 4),
+      $hex.Substring(20, 12)
+  ))
+}
+
+function Test-AgentCanonicalPendingCommandIdentity {
+  param([object]$Payload)
+
+  if (-not $Payload) { return $false }
+  try {
+    $projectId = ([string]$Payload.projectId).Trim()
+    $incidentId = ([string]$Payload.incidentId).Trim()
+    $sourceFingerprint = ([string]$Payload.sourceFingerprint).Trim()
+    $routeId = ([string]$Payload.dispatchRouteId).Trim()
+    $executionGeneration = ([string]$Payload.executionGeneration).Trim()
+    $approvalId = ([string]$Payload.approvalId).Trim()
+    $workItemId = ([string]$Payload.workItemId).Trim()
+    if (
+      -not $projectId -or
+      -not $incidentId -or
+      -not $sourceFingerprint -or
+      -not $routeId -or
+      -not $executionGeneration -or
+      -not $workItemId
+    ) { return $false }
+
+    $normalized = [ordered]@{
+      approval_id = $approvalId
+      execution_generation = $executionGeneration
+      incident_id = $incidentId
+      project_id = $projectId
+      route_id = $routeId
+      source_fingerprint = $sourceFingerprint
+      work_item_id = $workItemId
+    }
+    $canonical = $normalized | ConvertTo-Json -Compress -Depth 4
+    $digest = Get-AgentPreflightSha256Hex $canonical
+    $singleFlightCanonical = [ordered]@{
+      incident_id = $incidentId
+      project_id = $projectId
+      route_id = $routeId
+      source_fingerprint = $sourceFingerprint
+    } | ConvertTo-Json -Compress -Depth 4
+    $singleFlightDigest = Get-AgentPreflightSha256Hex $singleFlightCanonical
+    $runId = (New-AgentPreflightUuidV5 `
+      -Namespace ([guid]::Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) `
+      -Name "agent99-controlled-dispatch:$canonical").ToString()
+    $traceId = "00-$($digest.Substring(0, 32))-$($digest.Substring(32, 16))-01"
+    $idempotencyKey = "agent99-controlled:$digest"
+    $singleFlightKey = "agent99-controlled-flight:$singleFlightDigest"
+    $public = [ordered]@{
+      approval_id = $approvalId
+      execution_generation = $executionGeneration
+      idempotency_key = $idempotencyKey
+      incident_id = $incidentId
+      lock_owner = $runId
+      project_id = $projectId
+      route_id = $routeId
+      run_id = $runId
+      schema_version = "agent99_controlled_dispatch_identity_v1"
+      single_flight_key = $singleFlightKey
+      source_fingerprint = $sourceFingerprint
+      trace_id = $traceId
+      work_item_id = $workItemId
+    }
+    $expected = [ordered]@{
+      approvalId = $approvalId
+      automationRunId = $runId
+      canonicalDigest = Get-AgentPreflightSha256Hex (
+        $public | ConvertTo-Json -Compress -Depth 4
+      )
+      dispatchIdentitySchemaVersion = "agent99_controlled_dispatch_identity_v1"
+      dispatchRouteId = $routeId
+      executionGeneration = $executionGeneration
+      idempotencyKey = $idempotencyKey
+      incidentId = $incidentId
+      lockOwner = $runId
+      projectId = $projectId
+      singleFlightKey = $singleFlightKey
+      sourceFingerprint = $sourceFingerprint
+      traceId = $traceId
+      workItemId = $workItemId
+    }
+    foreach ($field in $expected.Keys) {
+      if ([string]$Payload.$field -cne [string]$expected[$field]) {
+        return $false
+      }
+    }
+    return $true
+  } catch {
+    return $false
+  }
+}
+
 function Get-AgentPendingCommandPromotionReadback {
  param(
    [object]$MaintenanceReadback,
@@ -493,7 +633,10 @@ function Get-AgentPendingCommandPromotionReadback {
    $result.reason = "maintenance_contract_invalid"
    return [pscustomobject]$result
  }
-  if ($Scope -eq "PromotionReserve" -and -not $MaintenanceReadback.valid) {
+  if (
+    $Scope -in @("PromotionReserve", "PostPromotionMaintenance") -and
+    -not $MaintenanceReadback.valid
+  ) {
    $result.reason = "maintenance_current_state_invalid"
    return [pscustomobject]$result
  }
@@ -578,6 +721,16 @@ function Get-AgentPendingCommandPromotionReadback {
      $payload -and
      @($nativeStringFields | Where-Object { $payload.$_ -isnot [string] }).Count -eq 0
    )
+    $allowedReason = [bool](
+      $payload -and
+      [string]$payload.reason -in @(
+        "host_reboot_detected",
+        "host_reboot_recovery_pending",
+        "host_unreachable_detected",
+        "host112_guest_not_ready",
+        "host188_edge_services_not_ready"
+      )
+    )
    $identityValid = [bool](
      -not $parseError -and
      $propertySetMatches -and
@@ -592,8 +745,8 @@ function Get-AgentPendingCommandPromotionReadback {
      [string]$payload.dispatchRouteId -eq "agent99_local_observer" -and
      [string]$payload.projectId -eq "awoooi" -and
      [string]$payload.requestedBy -eq "Agent99" -and
-      [string]$payload.reason -eq "host_unreachable_detected" -and
-      [string]$payload.externalId -eq "host_unreachable_detected" -and
+      $allowedReason -and
+      [string]$payload.externalId -eq [string]$payload.reason -and
      [string]$payload.approvalId -eq "" -and
      [string]$payload.automationRunId -match "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -and
      [string]$payload.lockOwner -eq [string]$payload.automationRunId -and
@@ -604,8 +757,9 @@ function Get-AgentPendingCommandPromotionReadback {
      [string]$payload.idempotencyKey -eq [string]$payload.correlationKey -and
      [string]$payload.singleFlightKey -match "^agent99-controlled-flight:[0-9a-f]{64}$" -and
      [string]$payload.incidentId -match "^agent99-auto-[0-9a-f]{20}$" -and
-      [string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):host_unreachable_detected" -and
+      [string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):$($payload.reason)" -and
      [string]$payload.traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$" -and
+      (Test-AgentCanonicalPendingCommandIdentity $payload) -and
      -not [string]::IsNullOrWhiteSpace([string]$payload.instruction) -and
      $createdAtValid -and
      $createdAt -ge $maintenanceTimestamp -and
@@ -634,10 +788,14 @@ function Get-AgentPendingCommandPromotionReadback {
  if ($allValid) {
    $result.eligibleMaintenanceAutoRecoverCount = $observedCount
    $result.contractValid = $true
-    if ($Scope -eq "PromotionReserve") {
+    if ($Scope -in @("PromotionReserve", "PostPromotionMaintenance")) {
      $result.deferredCount = $observedCount
      $result.blockingCount = 0
-      $result.reason = "bounded_maintenance_auto_recover_deferred_until_post_promotion"
+      $result.reason = if ($Scope -eq "PromotionReserve") {
+        "bounded_maintenance_auto_recover_deferred_until_post_promotion"
+      } else {
+        "bounded_maintenance_auto_recover_deferred_until_control_loop_restore"
+      }
    } else {
      $result.reason = "full_runtime_requires_maintenance_auto_recover_to_drain"
    }
@@ -1013,16 +1171,34 @@ $queues = @(
  (Get-DirectoryReadback "queue\failed")
 )
 $promotionEvidenceRequired = [bool]($ReadinessScope -ne "PromotionReserve")
+$postPromotionControlMaintenanceAccepted = [bool](
+  $ReadinessScope -eq "PostPromotionMaintenance" -and
+  @($tasks | Where-Object {
+    [string]$_.name -eq "Wooo-Agent99-Control-Loop" -and
+    [bool]$_.maintenanceFreezeAccepted
+  }).Count -eq 1
+)
+$controlTickEvidenceRequired = [bool](
+  $promotionEvidenceRequired -and
+  -not $postPromotionControlMaintenanceAccepted
+)
 $evidence = @(
  (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
  (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
-  (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
+  (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $controlTickEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
  (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
  (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
  (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
  (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
  (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0)
 )
+$controlTickEvidence = $evidence | Where-Object { $_.name -eq "control_tick" } | Select-Object -First 1
+if ($controlTickEvidence) {
+  $controlTickEvidence | Add-Member `
+    -NotePropertyName maintenanceFreezeAccepted `
+    -NotePropertyValue $postPromotionControlMaintenanceAccepted `
+    -Force
+}

 $operatingSystem = Get-CimInstance Win32_OperatingSystem
 $taskFailures = @($tasks | Where-Object { -not $_.healthy })
--- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1
+++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1
@@ -472,11 +472,24 @@ function Invoke-AgentLivePreflight {
    $ReadinessScope -eq "PromotionReserve" -or
    ($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched)
  )
+  $controlLoopMaintenanceRestorePending = [bool](
+    $ReadinessScope -eq "PostPromotionMaintenance" -and
+    $payload -and
+    @($payload.tasks | Where-Object {
+      [string]$_.name -eq "Wooo-Agent99-Control-Loop" -and
+      [bool]$_.maintenanceFreezeAccepted
+    }).Count -eq 1
+  )
  $pendingCommandsReadyForScope = [bool](
    $summary -and
    [int]$summary.pendingCommandBlockingCount -eq 0 -and
    (
      $ReadinessScope -eq "PromotionReserve" -or
+      (
+        $ReadinessScope -eq "PostPromotionMaintenance" -and
+        $controlLoopMaintenanceRestorePending -and
+        [int]$summary.pendingCommandCount -eq [int]$summary.promotionDeferredCommandCount
+      ) -or
      [int]$summary.pendingCommandCount -eq 0
    )
  )
@@ -500,14 +513,6 @@ function Invoke-AgentLivePreflight {
    $pendingCommandsReadyForScope -and
    $runtimePlaneBlockingReasons.Count -eq 0
  )
-  $controlLoopMaintenanceRestorePending = [bool](
-    $ReadinessScope -eq "PostPromotionMaintenance" -and
-    $payload -and
-    @($payload.tasks | Where-Object {
-      [string]$_.name -eq "Wooo-Agent99-Control-Loop" -and
-      [bool]$_.maintenanceFreezeAccepted
-    }).Count -eq 1
-  )
  return [pscustomobject]@{
    ok = $runtimePlaneReady
    readinessScope = $ReadinessScope
--- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py
+++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py
@@ -1,9 +1,11 @@
 from __future__ import annotations

 import json
+import hashlib
 import os
 import shutil
 import subprocess
+import uuid
 from copy import deepcopy
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -16,6 +18,72 @@ ROOT = Path(__file__).resolve().parents[3]
 PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1"


+def _canonical_dispatch_identity(
+    *, incident_id: str, source_fingerprint: str, work_item_id: str
+) -> dict[str, str]:
+    normalized = {
+        "approval_id": "",
+        "execution_generation": "1",
+        "incident_id": incident_id,
+        "project_id": "awoooi",
+        "route_id": "agent99_local_observer",
+        "source_fingerprint": source_fingerprint,
+        "work_item_id": work_item_id,
+    }
+    canonical = json.dumps(normalized, separators=(",", ":"))
+    digest = hashlib.sha256(canonical.encode()).hexdigest()
+    single_flight = json.dumps(
+        {
+            "incident_id": incident_id,
+            "project_id": "awoooi",
+            "route_id": "agent99_local_observer",
+            "source_fingerprint": source_fingerprint,
+        },
+        separators=(",", ":"),
+    )
+    single_flight_digest = hashlib.sha256(single_flight.encode()).hexdigest()
+    run_id = str(
+        uuid.uuid5(uuid.NAMESPACE_URL, f"agent99-controlled-dispatch:{canonical}")
+    )
+    trace_id = f"00-{digest[:32]}-{digest[32:48]}-01"
+    idempotency_key = f"agent99-controlled:{digest}"
+    single_flight_key = f"agent99-controlled-flight:{single_flight_digest}"
+    public = {
+        "approval_id": "",
+        "execution_generation": "1",
+        "idempotency_key": idempotency_key,
+        "incident_id": incident_id,
+        "lock_owner": run_id,
+        "project_id": "awoooi",
+        "route_id": "agent99_local_observer",
+        "run_id": run_id,
+        "schema_version": "agent99_controlled_dispatch_identity_v1",
+        "single_flight_key": single_flight_key,
+        "source_fingerprint": source_fingerprint,
+        "trace_id": trace_id,
+        "work_item_id": work_item_id,
+    }
+    return {
+        "approvalId": "",
+        "automationRunId": run_id,
+        "canonicalDigest": hashlib.sha256(
+            json.dumps(public, separators=(",", ":")).encode()
+        ).hexdigest(),
+        "correlationKey": idempotency_key,
+        "dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1",
+        "dispatchRouteId": "agent99_local_observer",
+        "executionGeneration": "1",
+        "idempotencyKey": idempotency_key,
+        "incidentId": incident_id,
+        "lockOwner": run_id,
+        "projectId": "awoooi",
+        "singleFlightKey": single_flight_key,
+        "sourceFingerprint": source_fingerprint,
+        "traceId": trace_id,
+        "workItemId": work_item_id,
+    }
+
+
 def _function_source() -> str:
    source = PREFLIGHT.read_text(encoding="utf-8")
    start = source.index("function Get-AgentLivePreflightDecision")
@@ -195,9 +263,18 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
    assert '$controlLoopMaintenance.valid' in source
    assert '$postPromotionReady = [bool]($runtimeHealthy -or $maintenanceFreezeAccepted)' in source
    assert '$task.healthy = $postPromotionReady' in source
+    assert "$controlTickEvidenceRequired" in source
+    assert "$postPromotionControlMaintenanceAccepted" in source
+    assert 'NotePropertyName maintenanceFreezeAccepted' in source
    assert "function Get-AgentPendingCommandPromotionReadback" in source
    assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source
+    assert '"bounded_maintenance_auto_recover_deferred_until_control_loop_restore"' in source
    assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
+    assert '"host_reboot_recovery_pending"' in source
+    assert '[string]$payload.externalId -eq [string]$payload.reason' in source
+    assert '"agent99-auto:$($payload.incidentId):$($payload.reason)"' in source
+    assert "function Test-AgentCanonicalPendingCommandIdentity" in source
+    assert "(Test-AgentCanonicalPendingCommandIdentity $payload) -and" in source
    assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source
    assert '$payload.observation -is [pscustomobject]' in source
    assert 'instructionContentReadback = $false' in source
@@ -543,34 +620,27 @@ def test_promotion_reserve_defers_only_exact_maintenance_auto_recover(
    queue.mkdir()
    now = datetime.now(timezone.utc)
    incident_id = "agent99-auto-" + ("a" * 20)
+    source_fingerprint = "e" * 64
    command_id = "auto-recover-20260719-152746-39ca4a72"
    command_path = queue / f"{command_id}.json"
+    work_item_id = f"agent99-auto:{incident_id}:host_unreachable_detected"
+    identity = _canonical_dispatch_identity(
+        incident_id=incident_id,
+        source_fingerprint=source_fingerprint,
+        work_item_id=work_item_id,
+    )
    payload = {
-        "approvalId": "",
-        "automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f",
-        "canonicalDigest": "b" * 64,
+        **identity,
        "controlledApply": True,
-        "correlationKey": "agent99-controlled:" + ("c" * 64),
        "createdAt": now.isoformat(),
-        "dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1",
-        "dispatchRouteId": "agent99_local_observer",
-        "executionGeneration": "1",
        "externalId": "host_unreachable_detected",
        "id": command_id,
-        "idempotencyKey": "agent99-controlled:" + ("c" * 64),
-        "incidentId": incident_id,
        "instruction": "recover exact observed unreachable host through controlled playbook",
-        "lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f",
        "mode": "Recover",
        "observation": {},
-        "projectId": "awoooi",
        "reason": "host_unreachable_detected",
        "requestedBy": "Agent99",
-        "singleFlightKey": "agent99-controlled-flight:" + ("d" * 64),
        "source": "agent99-recovery-observer",
-        "sourceFingerprint": "e" * 64,
-        "traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01",
-        "workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected",
    }

    function_source = _evidence_function_source()
@@ -622,6 +692,38 @@ Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scop
        "full_runtime_requires_maintenance_auto_recover_to_drain"
    )

+    post_maintenance = run("PostPromotionMaintenance")
+    assert post_maintenance["deferredCount"] == 1
+    assert post_maintenance["blockingCount"] == 0
+    assert post_maintenance["contractValid"] is True
+    assert post_maintenance["reason"] == (
+        "bounded_maintenance_auto_recover_deferred_until_control_loop_restore"
+    )
+
+    payload["externalId"] = "host_reboot_recovery_pending"
+    payload["reason"] = "host_reboot_recovery_pending"
+    payload["workItemId"] = (
+        f"agent99-auto:{incident_id}:host_reboot_recovery_pending"
+    )
+    command_path.write_text(json.dumps(payload), encoding="utf-8")
+    tampered_durable_boot_pending = run("PostPromotionMaintenance")
+    assert tampered_durable_boot_pending["deferredCount"] == 0
+    assert tampered_durable_boot_pending["blockingCount"] == 1
+    assert tampered_durable_boot_pending["contractValid"] is False
+
+    payload.update(
+        _canonical_dispatch_identity(
+            incident_id=incident_id,
+            source_fingerprint=source_fingerprint,
+            work_item_id=payload["workItemId"],
+        )
+    )
+    command_path.write_text(json.dumps(payload), encoding="utf-8")
+    durable_boot_pending = run("PostPromotionMaintenance")
+    assert durable_boot_pending["deferredCount"] == 1
+    assert durable_boot_pending["blockingCount"] == 0
+    assert durable_boot_pending["contractValid"] is True
+
    payload["mode"] = "Status"
    command_path.write_text(json.dumps(payload), encoding="utf-8")
    invalid = run("PromotionReserve")
--- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py
+++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py
@@ -429,6 +429,7 @@ def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() ->
    assert "postVerifyAttempts = $postVerifyAttempts" in source
    assert "postVerifyRetryCount = $postVerifyRetryCount" in source
    assert "controlLoopMaintenanceRestorePending = $controlLoopMaintenanceRestorePending" in source
+    assert '[int]$summary.pendingCommandCount -eq [int]$summary.promotionDeferredCommandCount' in source
    assert '"deployed_verified_maintenance_restore_pending"' in source
    assert "fullRuntimeVerifierPending = $maintenanceRestorePending" in source
    assert "completionClaim = [bool](-not $maintenanceRestorePending)" in source