feat(sre): track commitments and correlate Agent99 cards
This commit is contained in:
@@ -106,3 +106,91 @@ async def test_telegram_unbound_card_fails_closed_without_receipt_read(
|
||||
assert projection["dispatch_performed"] is False
|
||||
assert projection["status"] == "receipt_unbound_fail_closed"
|
||||
assert reads == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_host_resource_card_projects_existing_incident_receipt(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
incident_id = "INC-20260719-A1B2C3"
|
||||
durable_run_id = "7cf8fdf7-0966-5ac6-95b2-09e32808b248"
|
||||
reads: list[tuple[str, str]] = []
|
||||
|
||||
async def fake_read(*, project_id: str, incident_id: str):
|
||||
reads.append((project_id, incident_id))
|
||||
return {
|
||||
"status": "dispatch_accepted_verifier_pending",
|
||||
"identity": {
|
||||
"incident_id": incident_id,
|
||||
"run_id": durable_run_id,
|
||||
"trace_id": "trace-safe",
|
||||
"work_item_id": f"agent99-dispatch:awoooi:{incident_id}:perf",
|
||||
"idempotency_key": "agent99-controlled:stable-perf",
|
||||
},
|
||||
"dispatch_receipt": {
|
||||
"transport": "relay",
|
||||
"accepted": True,
|
||||
"inbox_triggered": True,
|
||||
"queue_accepted": True,
|
||||
"dispatch_identity_matched": True,
|
||||
},
|
||||
"receipt_persisted": True,
|
||||
"runtime_closure_verified": False,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
telegram_gateway_module,
|
||||
"read_agent99_dispatch_receipt",
|
||||
fake_read,
|
||||
)
|
||||
card = telegram_gateway_module.format_host_resource_alert_card(
|
||||
"WARN host188 CPU 警告: used=91% load=8.3",
|
||||
lifecycle={"incident_id": incident_id},
|
||||
)
|
||||
|
||||
projection = await telegram_gateway_module._mirror_ai_automation_alert_card_to_agent99(
|
||||
card,
|
||||
source="unit-test",
|
||||
)
|
||||
|
||||
assert reads == [("awoooi", incident_id)]
|
||||
assert projection is not None
|
||||
assert projection["incident_id"] == incident_id
|
||||
assert projection["run_id"] == durable_run_id
|
||||
assert projection["receipt_persisted"] is True
|
||||
assert projection["dispatch_performed"] is False
|
||||
assert projection["runtime_closure_verified"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unbound_host_resource_card_does_not_claim_run_or_read_receipt(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
reads: list[str] = []
|
||||
|
||||
async def unexpected_read(*, project_id: str, incident_id: str):
|
||||
reads.append(f"{project_id}:{incident_id}")
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
telegram_gateway_module,
|
||||
"read_agent99_dispatch_receipt",
|
||||
unexpected_read,
|
||||
)
|
||||
card = telegram_gateway_module.format_host_resource_alert_card(
|
||||
"WARN host188 CPU 警告: used=91% load=8.3"
|
||||
)
|
||||
|
||||
projection = await telegram_gateway_module._mirror_ai_automation_alert_card_to_agent99(
|
||||
card,
|
||||
source="unit-test",
|
||||
)
|
||||
|
||||
assert "Incident:<code>unbound_fail_closed</code>" in card
|
||||
assert "Run:<code>not_created</code>" in card
|
||||
assert "已建立 Run" not in card
|
||||
assert "runtime_write_gate=0_until_incident" in card
|
||||
assert projection is not None
|
||||
assert projection["status"] == "receipt_unbound_fail_closed"
|
||||
assert projection["dispatch_performed"] is False
|
||||
assert reads == []
|
||||
|
||||
@@ -21,6 +21,25 @@ from src.services.sre_k3s_controlled_automation_work_items import (
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
OPERATIONS = ROOT / "docs" / "operations"
|
||||
SNAPSHOT = OPERATIONS / "sre-k3s-controlled-automation-work-items.snapshot.json"
|
||||
COMMITMENTS = (
|
||||
OPERATIONS / "sre-ai-agent-conversation-commitments.snapshot.json"
|
||||
)
|
||||
|
||||
|
||||
def _write_operations_fixture(
|
||||
tmp_path: Path,
|
||||
payload: dict[str, object],
|
||||
commitment_payload: dict[str, object] | None = None,
|
||||
) -> None:
|
||||
(tmp_path / SNAPSHOT.name).write_text(
|
||||
json.dumps(payload), encoding="utf-8"
|
||||
)
|
||||
(tmp_path / COMMITMENTS.name).write_text(
|
||||
json.dumps(commitment_payload)
|
||||
if commitment_payload is not None
|
||||
else COMMITMENTS.read_text(encoding="utf-8"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -> None:
|
||||
@@ -127,6 +146,29 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
|
||||
"blocked_cd_5384_failure_runtime_build_e614f061f7"
|
||||
),
|
||||
}
|
||||
commitment_registry = payload["conversation_commitment_registry"]
|
||||
assert commitment_registry["authoritative_for_execution_order"] is False
|
||||
assert commitment_registry["raw_conversation_embedded"] is False
|
||||
assert commitment_registry["secret_values_recorded"] is False
|
||||
assert commitment_registry["rollups"] == {
|
||||
"total_commitments": 70,
|
||||
"active_or_completed_commitments": 68,
|
||||
"by_status": {
|
||||
"analysis_or_governance_complete": 6,
|
||||
"source_implemented_runtime_pending": 23,
|
||||
"in_progress": 32,
|
||||
"not_started_or_no_current_evidence": 7,
|
||||
"superseded": 2,
|
||||
},
|
||||
"product_runtime_closed_commitments": 0,
|
||||
}
|
||||
commitments = {
|
||||
row["id"]: row for row in commitment_registry["commitments"]
|
||||
}
|
||||
assert commitments["AIA-CONV-012"]["title"].startswith("Claude API")
|
||||
assert commitments["AIA-CONV-013"]["title"].startswith("Gemini API")
|
||||
assert "Host112" in commitments["AIA-CONV-049"]["title"]
|
||||
assert commitments["AIA-CONV-060"]["status"] == "superseded"
|
||||
|
||||
|
||||
def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None:
|
||||
@@ -229,6 +271,12 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None:
|
||||
assert projection["immediate_execution_queue"][0]["work_item_id"] == (
|
||||
"AIA-SRE-017"
|
||||
)
|
||||
assert projection["conversation_commitments"]["total_commitments"] == 70
|
||||
assert (
|
||||
projection["conversation_commitments"]
|
||||
["product_runtime_closed_commitments"]
|
||||
== 0
|
||||
)
|
||||
assert "work_items" not in projection
|
||||
|
||||
|
||||
@@ -242,8 +290,7 @@ def test_loader_rejects_immediate_execution_queue_reordering(
|
||||
)
|
||||
for position, row in enumerate(payload["immediate_execution_queue"], start=1):
|
||||
row["position"] = position
|
||||
target = tmp_path / SNAPSHOT.name
|
||||
target.write_text(json.dumps(payload), encoding="utf-8")
|
||||
_write_operations_fixture(tmp_path, payload)
|
||||
|
||||
with pytest.raises(ValueError, match="safety order changed"):
|
||||
load_sre_k3s_controlled_automation_work_items(tmp_path)
|
||||
@@ -256,8 +303,7 @@ def test_loader_rejects_paid_enablement_without_cost_gate(
|
||||
payload["provider_policy"]["required_before_paid_provider_enablement"].remove(
|
||||
"production_cost_readback"
|
||||
)
|
||||
target = tmp_path / SNAPSHOT.name
|
||||
target.write_text(json.dumps(payload), encoding="utf-8")
|
||||
_write_operations_fixture(tmp_path, payload)
|
||||
|
||||
with pytest.raises(ValueError, match="explicit cost controls"):
|
||||
load_sre_k3s_controlled_automation_work_items(tmp_path)
|
||||
@@ -269,13 +315,26 @@ def test_loader_rejects_unknown_asset_executor(tmp_path: Path) -> None:
|
||||
row for row in payload["domain_routes"] if row["domain"] == "unknown"
|
||||
)
|
||||
unknown["executor"] = "generic_fallback"
|
||||
target = tmp_path / SNAPSHOT.name
|
||||
target.write_text(json.dumps(payload), encoding="utf-8")
|
||||
_write_operations_fixture(tmp_path, payload)
|
||||
|
||||
with pytest.raises(ValueError, match="unknown domain"):
|
||||
load_sre_k3s_controlled_automation_work_items(tmp_path)
|
||||
|
||||
|
||||
def test_loader_rejects_conversation_commitment_unknown_parent(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
payload = json.loads(SNAPSHOT.read_text(encoding="utf-8"))
|
||||
commitment_payload = json.loads(COMMITMENTS.read_text(encoding="utf-8"))
|
||||
commitment_payload["commitments"][0]["linked_work_items"] = [
|
||||
"AIA-SRE-999"
|
||||
]
|
||||
_write_operations_fixture(tmp_path, payload, commitment_payload)
|
||||
|
||||
with pytest.raises(ValueError, match="links unknown work items"):
|
||||
load_sre_k3s_controlled_automation_work_items(tmp_path)
|
||||
|
||||
|
||||
def test_endpoint_returns_full_work_ledger() -> None:
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
@@ -287,6 +346,7 @@ def test_endpoint_returns_full_work_ledger() -> None:
|
||||
payload = response.json()
|
||||
assert payload["program_id"] == "AIA-SRE-P0-20260715"
|
||||
assert len(payload["work_items"]) == 18
|
||||
assert len(payload["conversation_commitment_registry"]["commitments"]) == 70
|
||||
assert payload["rollups"]["runtime_closed_items"] == 0
|
||||
assert payload["independent_verifier_registry"][
|
||||
"source_coverage_complete"
|
||||
|
||||
@@ -9,6 +9,7 @@ from src.services.telegram_gateway import (
|
||||
_outbound_source_envelope,
|
||||
_sanitize_telegram_error,
|
||||
format_aiops_signal_alert_card,
|
||||
format_host_resource_alert_card,
|
||||
)
|
||||
|
||||
|
||||
@@ -299,6 +300,38 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz
|
||||
assert "abcdefghijklmnopqrstuvwxyz" not in str(payload)
|
||||
|
||||
|
||||
def test_host_resource_card_preserves_safe_incident_run_correlation() -> None:
|
||||
incident_id = "INC-20260719-A1B2C3"
|
||||
run_id = "7cf8fdf7-0966-5ac6-95b2-09e32808b248"
|
||||
raw_alert = (
|
||||
"WARN host188 CPU 警告: used=92% load=8.4\n"
|
||||
f"Incident: {incident_id}\n"
|
||||
f"Run: {run_id}\n"
|
||||
"root 259 184 0.9 28062752 604824 ? Sl 05:55 0:33 "
|
||||
"node /workspace/wooo/stockplatform-v2/node_modules/.bin/next build"
|
||||
)
|
||||
|
||||
card = format_host_resource_alert_card(raw_alert)
|
||||
metadata = _agent99_actionable_outbound_metadata(card)
|
||||
payload = _agent99_alert_from_ai_automation_card(card, source="unit-test")
|
||||
|
||||
assert "事件類型:<code>host_resource_pressure_signal</code>" in card
|
||||
assert "Target:<code>host188</code>" in card
|
||||
assert "Lane:<code>runner_build_resource_pressure</code>" in card
|
||||
assert f"Incident:<code>{incident_id}</code>" in card
|
||||
assert f"Run:<code>{run_id}</code>" in card
|
||||
assert "runtime_write_gate=controlled" in card
|
||||
assert "建議下一步" not in card
|
||||
assert metadata is not None
|
||||
assert metadata["incident_id"] == incident_id
|
||||
assert metadata["run_id"] == run_id
|
||||
assert payload is not None
|
||||
assert payload["kind"] == "performance_pressure"
|
||||
assert payload["suggestedMode"] == "Perf"
|
||||
assert payload["awoooi"]["incidentId"] == incident_id
|
||||
assert payload["awoooi"]["runId"] == run_id
|
||||
|
||||
|
||||
def test_action_required_cold_start_card_routes_to_agent99_recover() -> None:
|
||||
card = (
|
||||
"ℹ️ ACTION REQUIRED | <b>低風險</b>\n"
|
||||
|
||||
@@ -109,15 +109,22 @@ root 364 181 0.7 3491396 494608 ? Rl 05:56 0:18 /opt/hostedto
|
||||
assert "Load" in result
|
||||
assert "容器 root 進程" in result
|
||||
assert "AI 自動化判讀" in result
|
||||
assert "host_resource_pressure_signal" in result
|
||||
assert "Target:<code>h110-gitea</code>" in result
|
||||
assert "runner_build_resource_pressure" in result
|
||||
assert "controlled_playbook_queue" in result
|
||||
assert "runtime_write_gate=controlled" in result
|
||||
assert "runtime_write_gate=0_until_incident" in result
|
||||
assert "Incident:<code>unbound_fail_closed</code>" in result
|
||||
assert "Run:<code>not_created</code>" in result
|
||||
assert "incident_identity_unresolved" in result
|
||||
assert "不建立或宣稱 Run" in result
|
||||
assert "Top evidence" in result
|
||||
assert "PID 259" in result
|
||||
assert "Next.js build" in result
|
||||
assert "PID 364" in result
|
||||
assert "Next.js build worker" in result
|
||||
assert "建議下一步" in result
|
||||
assert "AI 自動處置佇列" in result
|
||||
assert "建議下一步" not in result
|
||||
assert "禁止事項" in result
|
||||
assert "allowlisted PlayBook" in result
|
||||
assert "root 259" not in result
|
||||
@@ -144,7 +151,7 @@ def test_orphan_browser_alert_becomes_runaway_process_event_packet() -> None:
|
||||
assert "check-mode" in result
|
||||
assert "受控 SIGTERM" in result
|
||||
assert "KM / PlayBook / Verifier" in result
|
||||
assert "runtime_write_gate=controlled" in result
|
||||
assert "runtime_write_gate=0_until_incident" in result
|
||||
assert "allowlisted PlayBook" in result
|
||||
assert "受控 apply" in result
|
||||
|
||||
@@ -164,7 +171,7 @@ def test_ci_runner_load_alert_becomes_capacity_event_packet() -> None:
|
||||
assert "Gitea Actions run" in result
|
||||
assert "合法 CI" in result
|
||||
assert "不做 process remediation" in result
|
||||
assert "runtime_write_gate=controlled" in result
|
||||
assert "runtime_write_gate=0_until_incident" in result
|
||||
assert "受控 cleanup" in result
|
||||
|
||||
|
||||
@@ -369,7 +376,9 @@ async def test_send_alert_notification_normalizes_host_resource_raw_dump(monkeyp
|
||||
assert receipt["schema_version"] == "telegram_alert_notification_receipt_v1"
|
||||
assert receipt["status"] == "alert_notification_sent"
|
||||
assert receipt["ai_automation_card_present"] is True
|
||||
assert receipt["controlled_apply_allowed"] is True
|
||||
assert receipt["controlled_apply_allowed"] is False
|
||||
assert "incident_identity_unresolved" in payload["text"]
|
||||
assert "Run:<code>not_created</code>" in payload["text"]
|
||||
assert receipt["manual_default_route_allowed"] is False
|
||||
assert receipt["low_medium_high_route"] == "ai_controlled_apply"
|
||||
|
||||
@@ -420,7 +429,7 @@ root 392 0.0 0.0 1096836 53400 ? Ssl 06:27 0:00 /opt/hostedto
|
||||
assert "Prisma generate" in result
|
||||
assert "容器 root 進程:<code>3</code>" in result
|
||||
assert "套件來源" in result
|
||||
assert "runtime_write_gate=controlled" in result
|
||||
assert "runtime_write_gate=0_until_incident" in result
|
||||
assert "root 365" not in result
|
||||
assert "checkpoint.prisma.io" not in result
|
||||
assert "node_modules" not in result
|
||||
@@ -514,7 +523,7 @@ def test_send_request_payload_normalizer_blocks_direct_host_raw_dump() -> None:
|
||||
assert result["parse_mode"] == "HTML"
|
||||
assert "P1 主機資源壓力|h110-gitea" in result["text"]
|
||||
assert "ai_automation_alert_card_v1" in result["text"]
|
||||
assert "runtime_write_gate=controlled" in result["text"]
|
||||
assert "runtime_write_gate=0_until_incident" in result["text"]
|
||||
assert "root 311" not in result["text"]
|
||||
assert "checkpoint.prisma.io" not in result["text"]
|
||||
assert "/workspace/wooo" not in result["text"]
|
||||
|
||||
Reference in New Issue
Block a user