diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 14d4b929b..a28b24ff4 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -1557,8 +1557,10 @@ function Invoke-LoadShedding { } $cooldownMinutes = if ($action.cooldownMinutes) { [int]$action.cooldownMinutes } else { 30 } + $failureBackoffMinutes = if ($action.PSObject.Properties["failureBackoffMinutes"]) { [int]$action.failureBackoffMinutes } else { 10 } $safeName = ($action.name -replace "[^A-Za-z0-9_.-]", "_") $markerPath = Join-Path $EvidenceDir "loadshedding-$safeName.marker" + $failureMarkerPath = Join-Path $EvidenceDir "loadshedding-$safeName.failure.marker" if (Test-Path $markerPath) { $ageMinutes = ((Get-Date) - (Get-Item $markerPath).LastWriteTime).TotalMinutes if ($ageMinutes -lt $cooldownMinutes) { @@ -1566,6 +1568,13 @@ function Invoke-LoadShedding { continue } } + if (Test-Path $failureMarkerPath) { + $failureAgeMinutes = ((Get-Date) - (Get-Item $failureMarkerPath).LastWriteTime).TotalMinutes + if ($failureAgeMinutes -lt $failureBackoffMinutes) { + $results += [pscustomobject]@{ name = $action.name; host = $action.host; skipped = $true; reason = "failure_backoff"; ageMinutes = [math]::Round($failureAgeMinutes, 1) } + continue + } + } Record-AgentEvent "load_shed_attempt" $hostSeverity "host=$($action.host) action=$($action.name) command=$($action.command)" $action -NoAlert $timeoutSeconds = if ($action.PSObject.Properties["timeoutSeconds"]) { [int]$action.timeoutSeconds } else { 60 } @@ -1597,14 +1606,21 @@ function Invoke-LoadShedding { } } - $effectiveOk = [bool]($run.ok -or $postVerifyOk) - Set-Content -Path $markerPath -Value (Get-Date -Format o) -Encoding UTF8 + $postVerifyRequired = [bool]($action.PSObject.Properties["postVerifyCommand"] -and $action.postVerifyCommand) + $effectiveOk = if ($postVerifyRequired) { [bool]$postVerifyOk } else { [bool]$run.ok } + if ($effectiveOk) { + Set-Content -Path $markerPath -Value (Get-Date -Format o) -Encoding UTF8 + Remove-Item -Force $failureMarkerPath -ErrorAction SilentlyContinue | Out-Null + } else { + Set-Content -Path $failureMarkerPath -Value (Get-Date -Format o) -Encoding UTF8 + } $result = [pscustomobject]@{ name = $action.name host = $action.host skipped = $false ok = $effectiveOk executionOk = [bool]$run.ok + postVerifyRequired = $postVerifyRequired postVerifyOk = [bool]$postVerifyOk postVerify = $postVerify route = $run.route diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 4296d3734..04259ec69 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -17,6 +17,7 @@ $StageDir = Join-Path $DeployRoot "staging-$stamp" $BackupDir = Join-Path $DeployRoot "backup-$stamp" $ManifestPath = Join-Path $StateDir "runtime-manifest.json" $EvidencePath = Join-Path $EvidenceDir "agent99-deploy-$stamp.json" +$script:ConfigMigrations = @() $runtimeFiles = @( "agent99-bootstrap.ps1", @@ -54,6 +55,15 @@ function Add-DefaultProperty { } } +function Set-AgentProperty { + param([object]$Object, [string]$Name, [object]$Value) + if ($Object.PSObject.Properties[$Name]) { + $Object.PSObject.Properties[$Name].Value = $Value + } else { + $Object | Add-Member -MemberType NoteProperty -Name $Name -Value $Value + } +} + function Write-DeployEvidence { param([bool]$Ok, [string]$Status, [string]$ErrorText, [object[]]$Files, [object]$TaskResult) New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null @@ -71,6 +81,7 @@ function Write-DeployEvidence { validateOnly = [bool]$ValidateOnly registerTasks = [bool]$RegisterTasks error = $ErrorText + configMigrations = @($script:ConfigMigrations) files = $Files taskRegistration = $TaskResult } | ConvertTo-Json -Depth 10 | Set-Content -Path $EvidencePath -Encoding UTF8 @@ -177,6 +188,25 @@ try { Add-DefaultProperty $config "performance" ([pscustomobject]@{}) Add-DefaultProperty $config.performance "processCpuHostPercentWarning" 25 Add-DefaultProperty $config.performance "processCpuHostPercentCritical" 50 + + $policyConfig = Get-Content (Join-Path $StageDir "agent99.config.99.example.json") -Raw | ConvertFrom-Json + foreach ($sourceAction in @($policyConfig.performance.loadShedding.actions | Where-Object { $_.PSObject.Properties["policyVersion"] })) { + $liveAction = $config.performance.loadShedding.actions | Where-Object { $_.name -eq $sourceAction.name } | Select-Object -First 1 + if (-not $liveAction) { continue } + $sourceVersion = [int]$sourceAction.policyVersion + $liveVersion = if ($liveAction.PSObject.Properties["policyVersion"]) { [int]$liveAction.policyVersion } else { 0 } + if ($sourceVersion -le $liveVersion) { continue } + foreach ($propertyName in @("policyVersion", "host", "command", "timeoutSeconds", "postVerifyCommand", "postVerifyMaxDiskUsedPercent", "postVerifyOkRegex", "rollback", "cooldownMinutes", "failureBackoffMinutes", "minSeverity", "reasonMatches", "enabled")) { + if ($sourceAction.PSObject.Properties[$propertyName]) { + Set-AgentProperty $liveAction $propertyName $sourceAction.PSObject.Properties[$propertyName].Value + } + } + $script:ConfigMigrations += [pscustomobject]@{ + name = [string]$sourceAction.name + fromPolicyVersion = $liveVersion + toPolicyVersion = $sourceVersion + } + } $config | ConvertTo-Json -Depth 20 | Set-Content -Path $ConfigPath -Encoding UTF8 $manifestRows = @() diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 317d02866..5c29501bb 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -156,13 +156,15 @@ "actions": [ { "name": "prune-188-docker-build-cache-disk-pressure", + "policyVersion": 2, "host": "192.168.0.188", - "command": "echo BEFORE_DF=$(df -P / | awk 'NR==2{print $5}'); docker system df; docker builder prune -af --filter until=24h; echo AFTER_DF=$(df -P / | awk 'NR==2{print $5}'); docker system df", + "command": "echo BEFORE_DF=$(df -P / | awk 'NR==2{print $5}'); docker system df; docker builder prune -af; echo AFTER_DF=$(df -P / | awk 'NR==2{print $5}'); docker system df", "timeoutSeconds": 240, "postVerifyCommand": "df -P / | awk 'NR==2{gsub(\"%\",\"\",$5); print \"__DISK_USED__=\"$5}'", "postVerifyMaxDiskUsedPercent": 89, "rollback": "echo rollback_not_applicable_for_build_cache_prune", "cooldownMinutes": 120, + "failureBackoffMinutes": 10, "minSeverity": "warning", "reasonMatches": [ "disk_used_warning", diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index d01fc3bba..3a7d8b8e6 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -68,3 +68,33 @@ def test_agent99_deployer_verifies_promoted_runtime_and_writes_revision() -> Non ): assert f'"{name}"' in deploy assert f'"{name}"' in bootstrap + + +def test_load_shedding_requires_configured_post_verifier() -> None: + source = CONTROL.read_text(encoding="utf-8") + function = source[source.index("function Invoke-LoadShedding") :] + function = function[: function.index("function Get-AgentBooleanSetting")] + + assert 'reason = "failure_backoff"' in function + assert 'postVerifyRequired = [bool]' in function + assert "$effectiveOk = if ($postVerifyRequired)" in function + assert "Set-Content -Path $failureMarkerPath" in function + assert "$run.ok -or $postVerifyOk" not in function + + +def test_disk_pressure_policy_is_versioned_and_migrated_by_deployer() -> None: + config = json.loads((ROOT / "agent99.config.99.example.json").read_text()) + deploy = DEPLOY.read_text(encoding="utf-8") + action = next( + item + for item in config["performance"]["loadShedding"]["actions"] + if item["name"] == "prune-188-docker-build-cache-disk-pressure" + ) + + assert action["policyVersion"] == 2 + assert "docker builder prune -af;" in action["command"] + assert "until=24h" not in action["command"] + assert action["failureBackoffMinutes"] == 10 + assert "$script:ConfigMigrations" in deploy + assert '"policyVersion"' in deploy + assert "toPolicyVersion = $sourceVersion" in deploy