Merge remote-tracking branch 'origin/main' into codex/p0-obs-002-20260715
This commit is contained in:
@@ -117,6 +117,17 @@ def _evidence_refs() -> dict[str, str]:
|
||||
}
|
||||
|
||||
|
||||
def test_cold_start_source_queries_include_host112_scope() -> None:
|
||||
expected_scope = 'scope="110_112_120_121_188"'
|
||||
|
||||
assert reconcile.AGENT99_COLD_START_METRIC_SCOPE == "110_112_120_121_188"
|
||||
for name, query in reconcile._COLD_START_QUERIES.items():
|
||||
if name == "firing_blocking_alerts":
|
||||
continue
|
||||
assert expected_scope in query
|
||||
assert 'scope="110_120_121_188"' not in query
|
||||
|
||||
|
||||
def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
|
||||
@@ -2,16 +2,20 @@ import ast
|
||||
import asyncio
|
||||
import inspect
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.api.v1 import webhooks as webhooks_module
|
||||
from src.api.v1.webhooks import (
|
||||
_analyze_alertmanager_with_timeout,
|
||||
_process_new_alert_background,
|
||||
_should_bypass_alertmanager_llm,
|
||||
_should_use_alertmanager_rule_first,
|
||||
)
|
||||
from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES
|
||||
from src.services.alert_approval_guard import ApprovalActionGuardResult
|
||||
from src.services.alertmanager_llm_guard import (
|
||||
ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS,
|
||||
alertmanager_llm_inflight_key,
|
||||
@@ -175,6 +179,125 @@ async def test_alertmanager_analysis_error_returns_fallback():
|
||||
assert result == (None, "fallback_error", "", None, "", 0, 0.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent99_durable_route_dispatches_before_llm(monkeypatch):
|
||||
class ApprovalService:
|
||||
created_request = None
|
||||
|
||||
async def create_approval_with_fingerprint(self, *, request, fingerprint):
|
||||
self.created_request = request
|
||||
assert fingerprint == "fp-agent99-durable-route"
|
||||
return SimpleNamespace(id="approval-agent99-fast-path", incident_id=None)
|
||||
|
||||
async def update_incident_id(self, approval_id, incident_id):
|
||||
assert str(approval_id) == "approval-agent99-fast-path"
|
||||
assert incident_id == "INC-AGENT99-FAST-PATH"
|
||||
|
||||
service = ApprovalService()
|
||||
openclaw_factory = Mock(
|
||||
side_effect=AssertionError("durable Agent99 route must not create OpenClaw")
|
||||
)
|
||||
handoff = AsyncMock(
|
||||
return_value={
|
||||
"status": "agent99_dispatch_accepted_verifier_pending",
|
||||
"queued": True,
|
||||
}
|
||||
)
|
||||
telegram = AsyncMock()
|
||||
|
||||
monkeypatch.setattr(webhooks_module, "get_approval_service", lambda: service)
|
||||
monkeypatch.setattr(webhooks_module, "get_openclaw", openclaw_factory)
|
||||
monkeypatch.setattr(
|
||||
webhooks_module,
|
||||
"get_trusted_alert_canonical_route_context",
|
||||
lambda *_args, **_kwargs: {},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
webhooks_module,
|
||||
"match_rule",
|
||||
lambda _context: {
|
||||
"rule_id": "generic_fallback",
|
||||
"risk_level": "critical",
|
||||
"blast_radius": {},
|
||||
"kubectl_command": "",
|
||||
"description": "generic fallback must not block Agent99",
|
||||
"confidence": 0.0,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
webhooks_module,
|
||||
"resolve_playbook_id_for_alert",
|
||||
AsyncMock(return_value=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
webhooks_module,
|
||||
"guard_alert_approval_action",
|
||||
AsyncMock(
|
||||
side_effect=lambda **kwargs: ApprovalActionGuardResult(
|
||||
action=kwargs["action"]
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
webhooks_module,
|
||||
"get_auto_approve_policy",
|
||||
lambda: SimpleNamespace(
|
||||
evaluate=lambda _proposal: SimpleNamespace(
|
||||
should_auto_approve=False,
|
||||
reason=SimpleNamespace(value="bounded_durable_route"),
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
webhooks_module,
|
||||
"create_incident_for_approval",
|
||||
AsyncMock(return_value="INC-AGENT99-FAST-PATH"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
webhooks_module,
|
||||
"record_alertmanager_event",
|
||||
AsyncMock(),
|
||||
)
|
||||
monkeypatch.setattr(webhooks_module, "_try_auto_repair_background", handoff)
|
||||
monkeypatch.setattr(webhooks_module, "_push_to_telegram_background", telegram)
|
||||
monkeypatch.setattr(webhooks_module, "record_alert_chain_success", Mock())
|
||||
|
||||
await _process_new_alert_background(
|
||||
alert_context={
|
||||
"alertname": "RebootAutoRecoveryActiveBlocker",
|
||||
"annotations": {"summary": "cold-start gate blocked"},
|
||||
"source_url": "http://prometheus.invalid/graph",
|
||||
},
|
||||
alert_id="alert-agent99-fast-path",
|
||||
fingerprint="fp-agent99-durable-route",
|
||||
target_resource="reboot-auto-recovery-slo",
|
||||
namespace="default",
|
||||
alert_type="custom",
|
||||
message="cold-start-gate service recovery verifier",
|
||||
alertname="RebootAutoRecoveryActiveBlocker",
|
||||
severity="critical",
|
||||
alert_labels={"service": "cold-start-gate"},
|
||||
notification_type="TYPE-3",
|
||||
alert_category="general",
|
||||
can_auto_repair=True,
|
||||
)
|
||||
|
||||
openclaw_factory.assert_not_called()
|
||||
handoff.assert_awaited_once()
|
||||
assert handoff.await_args.kwargs["source_alertname"] == (
|
||||
"RebootAutoRecoveryActiveBlocker"
|
||||
)
|
||||
assert service.created_request.risk_level.value == "medium"
|
||||
assert service.created_request.action == (
|
||||
"AGENT99_CONTROLLED_ROUTE agent99:host_recovery:Recover"
|
||||
)
|
||||
assert service.created_request.metadata["source"] == "agent99_durable_route"
|
||||
telegram.assert_awaited_once()
|
||||
assert telegram.await_args.kwargs["automation_state"] == (
|
||||
"agent99_dispatch_accepted_verifier_pending"
|
||||
)
|
||||
|
||||
|
||||
def test_resolved_guard_stamp_without_timestamp_is_clean():
|
||||
assert _format_resolved_guard_stamp(None) == "✅ 此事件已解決"
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from src.services.awooop_ansible_check_mode_service import (
|
||||
_send_controlled_apply_telegram_receipt,
|
||||
claim_catalog_drift_failed_check_modes,
|
||||
claim_semantically_misrouted_verified_applies,
|
||||
claim_stale_pending_check_modes,
|
||||
run_pending_check_modes_once,
|
||||
)
|
||||
from src.services.awooop_ansible_post_verifier import postconditions_for_catalog
|
||||
@@ -100,6 +101,7 @@ def test_stale_generic_host188_row_is_semantically_remapped() -> None:
|
||||
{
|
||||
"op_id": "00000000-0000-0000-0000-000000000188",
|
||||
"parent_op_id": "00000000-0000-0000-0000-000000000100",
|
||||
"tags": ["ansible", "check_mode", "catalog_drift_replay"],
|
||||
"incident_id": "INC-DISK-188",
|
||||
"incident_alertname": "HostOutOfDiskSpace",
|
||||
"incident_alert_category": "host_resource",
|
||||
@@ -114,6 +116,15 @@ def test_stale_generic_host188_row_is_semantically_remapped() -> None:
|
||||
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
|
||||
"inventory_hosts": ["host_110"],
|
||||
"risk_level": "medium",
|
||||
"automation_run_id": (
|
||||
"00000000-0000-0000-0000-000000000188"
|
||||
),
|
||||
"catalog_drift_replay": True,
|
||||
"replay_of_check_mode_op_id": (
|
||||
"00000000-0000-0000-0000-000000000099"
|
||||
),
|
||||
"historical_projection_backfill": True,
|
||||
"telegram_provider_delivery": "shadow_only",
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -123,6 +134,37 @@ def test_stale_generic_host188_row_is_semantically_remapped() -> None:
|
||||
assert claim.inventory_hosts == ("host_188",)
|
||||
assert claim.input_payload["catalog_route_changed"] is True
|
||||
assert claim.input_payload["incident_semantic_route_rechecked"] is True
|
||||
assert claim.input_payload["catalog_drift_replay"] is True
|
||||
assert claim.input_payload["historical_projection_backfill"] is True
|
||||
assert claim.input_payload["telegram_provider_delivery"] == "shadow_only"
|
||||
|
||||
|
||||
def test_stale_catalog_replay_tag_recovers_missing_shadow_policy() -> None:
|
||||
claim = _claim_from_stale_check_mode_row(
|
||||
{
|
||||
"op_id": "00000000-0000-0000-0000-000000000198",
|
||||
"parent_op_id": "00000000-0000-0000-0000-000000000100",
|
||||
"tags": ["ansible", "check_mode", "catalog_drift_replay"],
|
||||
"incident_id": "INC-DISK-188-TAG-FALLBACK",
|
||||
"incident_alertname": "HostOutOfDiskSpace",
|
||||
"incident_alert_category": "host_resource",
|
||||
"incident_affected_services": ["node-exporter-188"],
|
||||
"input": {
|
||||
"source_candidate_op_id": (
|
||||
"00000000-0000-0000-0000-000000000100"
|
||||
),
|
||||
"incident_id": "INC-DISK-188-TAG-FALLBACK",
|
||||
"catalog_id": "ansible:110-devops",
|
||||
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
|
||||
"inventory_hosts": ["host_110"],
|
||||
"risk_level": "medium",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert claim is not None
|
||||
assert claim.input_payload["historical_projection_backfill"] is True
|
||||
assert claim.input_payload["telegram_provider_delivery"] == "shadow_only"
|
||||
|
||||
|
||||
def test_disk_catalogs_have_independent_threshold_verifiers() -> None:
|
||||
@@ -141,12 +183,14 @@ def test_disk_catalogs_have_independent_threshold_verifiers() -> None:
|
||||
|
||||
def test_historical_catalog_replay_is_per_failed_row_and_provider_silent() -> None:
|
||||
claim_source = inspect.getsource(claim_catalog_drift_failed_check_modes)
|
||||
stale_source = inspect.getsource(claim_stale_pending_check_modes)
|
||||
send_source = inspect.getsource(_send_controlled_apply_telegram_receipt)
|
||||
|
||||
assert "replay_of_check_mode_op_id" in claim_source
|
||||
assert "failed_check_mode_op_id" in claim_source
|
||||
assert '"historical_projection_backfill": True' in claim_source
|
||||
assert '"telegram_provider_delivery": "shadow_only"' in claim_source
|
||||
assert "check_mode.tags" in stale_source
|
||||
assert "telegram_provider_delivery" in send_source
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user