79489635d8
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m36s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
275 lines
12 KiB
Python
275 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
_CONTROL_PLANE = _REPO_ROOT / "agent99-control-plane.ps1"
|
|
_BOOTSTRAP = _REPO_ROOT / "agent99-bootstrap.ps1"
|
|
_SRE_INBOX = _REPO_ROOT / "agent99-sre-alert-inbox.ps1"
|
|
|
|
|
|
def test_agent99_queue_uses_verified_outcome_not_exit_code() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
|
|
assert "function Get-AgentOutcomeContract" in source
|
|
assert 'schemaVersion = "agent99_outcome_contract_v1"' in source
|
|
assert "ok = [bool]$outcome.resolved" in source
|
|
assert "outcomeState = $outcome.state" in source
|
|
assert "verifierPassed = [bool]$outcome.verifierPassed" in source
|
|
assert "sourceEventResolutionRequired" in source
|
|
assert '"verifying"' in source
|
|
assert '"candidate_ready"' in source
|
|
assert "ok = [bool]($exitCode -eq 0)" not in source
|
|
assert 'New-AgentOutcomeCheck "host112_guest_ready" $host112Ok' in source
|
|
assert "$data.host112Recovery -and $data.host112Recovery.verified" in source
|
|
|
|
|
|
def test_agent99_problem_counts_separate_verified_resolution() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
|
|
assert 'schemaVersion = "agent99_problem_counts_v2"' in source
|
|
assert "totalVerifiedResolved" in source
|
|
assert "legacyResolvedCountMayIncludeTransportOnly" in source
|
|
assert "verifiedResolved" in source
|
|
assert "totalDegraded" in source
|
|
assert "totalBlocked" in source
|
|
assert "totalVerifying" in source
|
|
assert "totalCandidateReady" in source
|
|
|
|
|
|
def test_agent99_runtime_manifest_is_generated_and_self_checked() -> None:
|
|
control_source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
bootstrap_source = _BOOTSTRAP.read_text(encoding="utf-8")
|
|
|
|
assert "function Test-AgentRuntimeManifest" in control_source
|
|
assert 'reason = "runtime_manifest_missing"' in control_source
|
|
assert "runtimeManifest = Test-AgentRuntimeManifest" in control_source
|
|
assert 'schemaVersion = "agent99_runtime_manifest_v1"' in bootstrap_source
|
|
assert 'Join-Path $StateDir "runtime-manifest.json"' in bootstrap_source
|
|
assert "Get-FileHash -Algorithm SHA256" in bootstrap_source
|
|
assert "if (-not $runtimeMatched)" in bootstrap_source
|
|
|
|
|
|
def test_agent99_outcome_contract_has_runtime_self_test() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
|
|
assert "[switch]$SelfTestOutcomeContract" in source
|
|
assert "function Invoke-AgentOutcomeContractSelfTest" in source
|
|
assert '$operatorResolved.state -eq "resolved"' in source
|
|
assert '$alertVerifying.state -eq "verifying"' in source
|
|
assert '$alertResolvedByVerifier.state -eq "resolved"' in source
|
|
assert '$alertTransportFailedVerifier.state -eq "failed"' in source
|
|
assert "-not $alertTransportFailedVerifier.sourceEventResolved" in source
|
|
assert '"mode_verifier_can_resolve"' in source
|
|
assert "sourceEventResolutionPolicy" in source
|
|
assert '$failedPostCondition.state -eq "degraded"' in source
|
|
assert '$transportFailed.state -eq "failed"' in source
|
|
assert '$securityCandidate.state -eq "candidate_ready"' in source
|
|
|
|
|
|
def test_agent99_provider_freshness_is_provider_specific_and_fail_closed() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
converter = source[
|
|
source.index("function Convert-AgentProviderFreshnessCoverageReadback") :
|
|
source.index("function Invoke-AgentProviderFreshnessDirectReadback")
|
|
]
|
|
direct_readback = source[
|
|
source.index("function Invoke-AgentProviderFreshnessDirectReadback") :
|
|
source.index("function Invoke-AgentProviderFreshnessTriage")
|
|
]
|
|
synthetic = (
|
|
_REPO_ROOT / "agent99-synthetic-tests.ps1"
|
|
).read_text(encoding="utf-8")
|
|
|
|
assert '$requiredProviders = @("sentry", "signoz")' in converter
|
|
assert 'schemaVersion = "agent99_provider_freshness_readback_v2"' in converter
|
|
assert 'readbackScope = "provider_specific_coverage"' in converter
|
|
assert "genericRecurrenceAccepted = $false" in converter
|
|
assert "ageMinutes -le $LookbackMinutes" in converter
|
|
assert "allProvidersFresh = $allProvidersFresh" in converter
|
|
assert "/dossier/coverage?" in direct_readback
|
|
assert "provider=$provider" in direct_readback
|
|
assert "/dossier/recurrence" not in direct_readback
|
|
assert '$providerReadback.allProvidersFresh -eq $true' in source
|
|
assert 'provider_freshness_evidence_ref' in source
|
|
|
|
for case in (
|
|
"provider_freshness:mixed_fresh_stale_degraded",
|
|
"provider_freshness:both_fresh_resolved",
|
|
"provider_freshness:generic_recurrence_not_resolved",
|
|
):
|
|
assert case in synthetic
|
|
assert '$providerMixedDegraded.state -eq "degraded"' in source
|
|
assert '$providerFreshResolved.state -eq "resolved"' in source
|
|
assert '$providerGenericDegraded.state -eq "degraded"' in source
|
|
|
|
|
|
def test_agent99_posts_fenced_outcome_to_production_ingestion() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
inbox = _SRE_INBOX.read_text(encoding="utf-8")
|
|
|
|
assert "function Send-AgentOutcomeWriteback" in source
|
|
assert "/api/v1/webhooks/agent99/outcomes" in source
|
|
assert '"X-Agent99-Relay-Token" = $token' in source
|
|
assert "$attempt -le 3" in source
|
|
assert "agent99_outcome_receipt_id" in source
|
|
assert "post_verifier_evidence_ref" in source
|
|
assert "source_event_evidence_ref" in source
|
|
assert "function Get-AgentSafeHttpFailureReceipt" in source
|
|
assert "httpStatus = $httpStatus" in source
|
|
assert "errorCode = $detailCode" in source
|
|
assert "rawResponseStored = $false" in source
|
|
assert "secretValueLogged = $false" in source
|
|
assert "$lastFailure.retryable" in source
|
|
for field in (
|
|
"projectId",
|
|
"sourceFingerprint",
|
|
"dispatchRouteId",
|
|
"singleFlightKey",
|
|
"lockOwner",
|
|
"canonicalDigest",
|
|
"automationRunId",
|
|
"traceId",
|
|
"workItemId",
|
|
"idempotencyKey",
|
|
"executionGeneration",
|
|
"incidentId",
|
|
):
|
|
assert field in inbox
|
|
assert field in source
|
|
|
|
|
|
def test_agent99_completion_callback_uses_bounded_canonical_incident_id() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
|
|
assert "function Get-AgentCompletionCallbackIncidentId" in source
|
|
assert 'Get-AgentObjectValue $identity "incident_id" ""' in source
|
|
assert 'return "INC-AG99-' in source
|
|
assert "$callbackIncidentId = Get-AgentCompletionCallbackIncidentId" in source
|
|
assert "alert_id = $callbackIncidentId" in source
|
|
assert "alert_id = if ($alertId)" not in source
|
|
|
|
|
|
def test_agent99_completion_callback_preserves_canonical_execution_identity() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
|
|
assert '$identity = Get-AgentObjectValue $Result "identity" $null' in source
|
|
assert 'Get-AgentObjectValue $identity "project_id" ""' in source
|
|
assert 'Get-AgentObjectValue $identity "run_id" ""' in source
|
|
assert 'Get-AgentObjectValue $identity "trace_id" ""' in source
|
|
assert 'Get-AgentObjectValue $identity "work_item_id" ""' in source
|
|
assert "run_id = if ($canonicalRunId)" in source
|
|
assert "trace_id = if ($canonicalTraceId)" in source
|
|
assert "work_item_id = if ($canonicalWorkItemId)" in source
|
|
|
|
|
|
def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
readback = source[
|
|
source.index("function Test-AgentBackupProtectionReceipts") :
|
|
source.index("function Convert-AgentGoogleDriveQuotaReadback")
|
|
]
|
|
|
|
assert "[switch]$SelfTestBackupReadbackContract" in source
|
|
assert "function Convert-AgentBackupProtectionReadback" in source
|
|
for metric in (
|
|
"awoooi_backup_offsite_configured",
|
|
"awoooi_backup_offsite_fresh",
|
|
"awoooi_backup_offsite_remote_verify_ok",
|
|
"awoooi_backup_credential_escrow_fresh",
|
|
"awoooi_backup_dr_credential_escrow_missing_count",
|
|
):
|
|
assert metric in source
|
|
assert "stat -c '__PROTECTION_MTIME__=%Y'" in readback
|
|
assert "protectionVerifyMetricsPath" in source
|
|
assert "offsite_full_sync_verify.prom" in source
|
|
assert "stat -c '__OFFSITE_VERIFY_MTIME__=%Y'" in readback
|
|
assert "grep -E" in readback
|
|
assert "backupRunPerformed = $false" in source
|
|
assert "restoreRunPerformed = $false" in source
|
|
assert "markerWritePerformed = $false" in source
|
|
assert '"offsite_verify_evidence_ref"' in source
|
|
assert '"escrow_evidence_ref"' in source
|
|
assert "protectionMaxAgeMinutes" in source
|
|
assert "protectionEscrowExpectedCount" in source
|
|
assert "$evidenceAgeMinutes -ge -5" in source
|
|
assert "$evidenceAgeMinutes -le" in source
|
|
assert "$verifyEvidenceAgeMinutes -ge -5" in source
|
|
assert "$verifyEvidenceAgeMinutes -le" in source
|
|
assert "function Get-AgentBackupProviderMetricRows" in source
|
|
assert "$offsiteFresh.Count -eq $offsiteFreshRows.Count" in source
|
|
assert "$remoteVerify.Count -eq $remoteVerifyRows.Count" in source
|
|
assert "$configuredProviderNames.Count -gt 0" in source
|
|
assert "$verifiedProviderNames.Count -eq $configuredProviderNames.Count" in source
|
|
assert "$escrowFresh.Count -eq $ExpectedEscrowFreshCount" in source
|
|
assert "$escrowMissingValues.Count -eq 1" in source
|
|
assert 'provider="b2"} 0' in source
|
|
assert 'provider="rclone"} 1' in source
|
|
assert "optionalB2ProviderTerminalEligible" in source
|
|
assert "staleOffsiteVerifyEvidenceBlocked" in source
|
|
assert "allProvidersUnconfiguredBlocked" in source
|
|
assert "configuredProviderStaleBlocked" in source
|
|
assert "configuredProviderRemoteVerifyFailedBlocked" in source
|
|
assert "escrowMissingBlocked" in source
|
|
assert "escrowPartialFreshnessBlocked" in source
|
|
assert "-not $staleResult.offsiteVerify.ok" in source
|
|
assert "-not $noProviderResult.offsiteVerify.ok" in source
|
|
assert "-not $staleProviderResult.offsiteVerify.ok" in source
|
|
assert "-not $remoteVerifyResult.offsiteVerify.ok" in source
|
|
assert "-not $escrowMissingResult.escrow.ok" in source
|
|
assert "-not $escrowPartialResult.escrow.ok" in source
|
|
for mutation in ("rclone ", "rsync ", " rm ", "restore ", "Set-Content"):
|
|
assert mutation not in readback
|
|
|
|
|
|
def test_agent99_backupcheck_covers_all_hosts_and_bounds_drive_quota() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
host_contract = source[
|
|
source.index("function Get-AgentBackupCanonicalHostDefinitions") :
|
|
source.index("function Test-AgentBackupHealth")
|
|
]
|
|
quota_contract = source[
|
|
source.index("function Convert-AgentGoogleDriveQuotaReadback") :
|
|
source.index("function Invoke-AgentBackupReadbackContractSelfTest")
|
|
]
|
|
|
|
for host_alias in ("99", "110", "111", "112", "120", "121", "188"):
|
|
assert f'alias = "{host_alias}"' in host_contract
|
|
for asset in (
|
|
"windows_system",
|
|
"vmware_inventory",
|
|
"agent99_control_plane",
|
|
"gitea",
|
|
"harbor",
|
|
"workstation_safe_artifacts",
|
|
"wazuh_security_state",
|
|
"k3s_etcd",
|
|
"k3s_local_pv",
|
|
"product_datastores",
|
|
"docker_volumes",
|
|
"ai_assets",
|
|
):
|
|
assert f'"{asset}"' in host_contract
|
|
|
|
assert "runtimeReadbackOk" in source
|
|
assert "backupEvidenceReady" in source
|
|
assert '"host_backup_contract_not_registered"' in source
|
|
assert "backupEvidenceReadyCount" in source
|
|
assert 'schemaVersion = "agent99_google_drive_quota_readback_v1"' in quota_contract
|
|
assert "timeout --signal=TERM --kill-after=5s" in quota_contract
|
|
assert "rclone about" in quota_contract
|
|
assert 'remoteName -notmatch "^[A-Za-z0-9_.-]+:$"' in quota_contract
|
|
assert 'canonicalRemoteCountedOnce = $true' in quota_contract
|
|
assert 'capacityDecision = if ($ok)' in quota_contract
|
|
assert 'rawOutputStored = $false' in quota_contract
|
|
assert 'secretValueRead = $false' in quota_contract
|
|
for mutation in ("rclone copy", "rclone sync", "rclone delete", "rclone purge"):
|
|
assert mutation not in quota_contract
|
|
|
|
|
|
def test_agent99_backupcheck_escrow_receipt_uses_exact_mtime() -> None:
|
|
source = _CONTROL_PLANE.read_text(encoding="utf-8")
|
|
|
|
assert '"escrow-status:${mtime}:missing-0"' in source
|
|
assert '"escrow-status:$mtime:missing-0"' not in source
|