fix(aiops): verify provider freshness per source

This commit is contained in:
ogt
2026-07-16 23:17:13 +08:00
parent 6188aabd44
commit d9487cd55a
10 changed files with 547 additions and 76 deletions

View File

@@ -67,6 +67,43 @@ def test_agent99_outcome_contract_has_runtime_self_test() -> None:
assert '$securityCandidate.state -eq "candidate_ready"' in source
def test_agent99_provider_freshness_is_provider_specific_and_fail_closed() -> None:
source = _CONTROL_PLANE.read_text(encoding="utf-8")
converter = source[
source.index("function Convert-AgentProviderFreshnessCoverageReadback") :
source.index("function Invoke-AgentProviderFreshnessDirectReadback")
]
direct_readback = source[
source.index("function Invoke-AgentProviderFreshnessDirectReadback") :
source.index("function Invoke-AgentProviderFreshnessTriage")
]
synthetic = (
_REPO_ROOT / "agent99-synthetic-tests.ps1"
).read_text(encoding="utf-8")
assert '$requiredProviders = @("sentry", "signoz")' in converter
assert 'schemaVersion = "agent99_provider_freshness_readback_v2"' in converter
assert 'readbackScope = "provider_specific_coverage"' in converter
assert "genericRecurrenceAccepted = $false" in converter
assert "ageMinutes -le $LookbackMinutes" in converter
assert "allProvidersFresh = $allProvidersFresh" in converter
assert "/dossier/coverage?" in direct_readback
assert "provider=$provider" in direct_readback
assert "/dossier/recurrence" not in direct_readback
assert '$providerReadback.allProvidersFresh -eq $true' in source
assert 'provider_freshness_evidence_ref' in source
for case in (
"provider_freshness:mixed_fresh_stale_degraded",
"provider_freshness:both_fresh_resolved",
"provider_freshness:generic_recurrence_not_resolved",
):
assert case in synthetic
assert '$providerMixedDegraded.state -eq "degraded"' in source
assert '$providerFreshResolved.state -eq "resolved"' in source
assert '$providerGenericDegraded.state -eq "degraded"' in source
def test_agent99_posts_fenced_outcome_to_production_ingestion() -> None:
source = _CONTROL_PLANE.read_text(encoding="utf-8")
inbox = _SRE_INBOX.read_text(encoding="utf-8")

View File

@@ -124,6 +124,31 @@ def test_other_alert_chain_domains_do_not_fall_into_alertmanager_container() ->
assert route["cross_domain_fallback_allowed"] is False
def test_provider_freshness_routes_to_read_only_observability_lane() -> None:
route = resolve_typed_alert_target(
alertname="SourceProviderIngestionStale",
target_resource="source-ingestion",
namespace="monitoring",
labels={"component": "source-ingestion", "source": "signoz"},
alert_category="alertchain_provider_freshness",
)
assert route["resolution_status"] == "resolved"
assert route["target_kind"] == "observability_source_ingestion"
assert route["canonical_asset_id"] == "observability-provider:signoz"
assert route["executor"] == "provider_freshness_readonly_executor"
assert route["verifier"] == "provider_freshness_independent_verifier"
assert route["risk_class"] == "low"
assert route["controlled_apply_allowed"] is True
assert route["cross_domain_fallback_allowed"] is False
assert route["agent99_bridge"]["dispatch_allowed"] is False
contract = route["provider_freshness_contract"]
assert contract["execution_mode"] == "read_only"
assert contract["required_providers"] == ["sentry", "signoz"]
assert contract["mutating_actions_allowed"] is False
assert contract["generic_recurrence_can_resolve"] is False
def _sentry_runtime_incident() -> SimpleNamespace:
return SimpleNamespace(
incident_id="INC-20260714-471307",

View File

@@ -266,6 +266,16 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz
"""
card = format_aiops_signal_alert_card(raw_alert)
assert "已建立 RunAI check執行中" in card
assert "partial" in card
assert "receipt_pending" in card
assert "same_run_check_running_receipt_pending" in card
assert "same_run_verifier_receipt_pending" in card
assert "AI 自動處置佇列" in card
assert "兩者都 fresh 才能 resolved" in card
assert "generic recurrence 不得作為 resolved 證據" in card
assert "建議下一步" not in card
payload = _agent99_alert_from_ai_automation_card(
card,
source="unit-test",

View File

@@ -299,6 +299,13 @@ def test_aiops_signal_formatter_covers_non_host_alert_lanes(
assert "ai_automation_alert_card_v1" in result
assert event_type in result
assert lane in result
assert "已建立 RunAI check執行中" in result
assert "partial" in result
assert "receipt_pending" in result
assert "same_run_check_running_receipt_pending" in result
assert "same_run_verifier_receipt_pending" in result
assert "AI 自動處置佇列" in result
assert "建議下一步" not in result
assert "controlled_playbook_queue" in result
if event_type == "backup_restore_escrow_signal":
assert "controlled_playbook_queue=readback_only" in result
@@ -310,6 +317,23 @@ def test_aiops_signal_formatter_covers_non_host_alert_lanes(
assert "禁止事項" in result
def test_aiops_signal_terminal_claim_requires_complete_same_run_receipts() -> None:
result = format_aiops_signal_alert_card(
'alertname="SourceProviderIngestionStale" source="signoz" stale',
lifecycle={
"status": "resolved",
"terminal": True,
"execution_summary": "resolved without receipts",
},
)
assert "已建立 RunAI check執行中" in result
assert "resolved without receipts" not in result
assert "<code>partial</code> / <code>receipt_pending</code>" in result
assert "same_run_verifier_receipt_pending" in result
assert "建議下一步" not in result
@pytest.mark.asyncio
async def test_send_alert_notification_normalizes_host_resource_raw_dump(monkeypatch) -> None:
"""send_alert_notification 是最後出口,必須自動套用 AI 自動化事件包。"""