feat(sre): track commitments and correlate Agent99 cards

This commit is contained in:
Your Name
2026-07-19 11:01:45 +08:00
parent a20de2cc62
commit 5382ce348f
15 changed files with 1120 additions and 19 deletions

View File

@@ -106,3 +106,91 @@ async def test_telegram_unbound_card_fails_closed_without_receipt_read(
assert projection["dispatch_performed"] is False
assert projection["status"] == "receipt_unbound_fail_closed"
assert reads == []
@pytest.mark.asyncio
async def test_host_resource_card_projects_existing_incident_receipt(
monkeypatch,
) -> None:
incident_id = "INC-20260719-A1B2C3"
durable_run_id = "7cf8fdf7-0966-5ac6-95b2-09e32808b248"
reads: list[tuple[str, str]] = []
async def fake_read(*, project_id: str, incident_id: str):
reads.append((project_id, incident_id))
return {
"status": "dispatch_accepted_verifier_pending",
"identity": {
"incident_id": incident_id,
"run_id": durable_run_id,
"trace_id": "trace-safe",
"work_item_id": f"agent99-dispatch:awoooi:{incident_id}:perf",
"idempotency_key": "agent99-controlled:stable-perf",
},
"dispatch_receipt": {
"transport": "relay",
"accepted": True,
"inbox_triggered": True,
"queue_accepted": True,
"dispatch_identity_matched": True,
},
"receipt_persisted": True,
"runtime_closure_verified": False,
}
monkeypatch.setattr(
telegram_gateway_module,
"read_agent99_dispatch_receipt",
fake_read,
)
card = telegram_gateway_module.format_host_resource_alert_card(
"WARN host188 CPU 警告: used=91% load=8.3",
lifecycle={"incident_id": incident_id},
)
projection = await telegram_gateway_module._mirror_ai_automation_alert_card_to_agent99(
card,
source="unit-test",
)
assert reads == [("awoooi", incident_id)]
assert projection is not None
assert projection["incident_id"] == incident_id
assert projection["run_id"] == durable_run_id
assert projection["receipt_persisted"] is True
assert projection["dispatch_performed"] is False
assert projection["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_unbound_host_resource_card_does_not_claim_run_or_read_receipt(
monkeypatch,
) -> None:
reads: list[str] = []
async def unexpected_read(*, project_id: str, incident_id: str):
reads.append(f"{project_id}:{incident_id}")
return None
monkeypatch.setattr(
telegram_gateway_module,
"read_agent99_dispatch_receipt",
unexpected_read,
)
card = telegram_gateway_module.format_host_resource_alert_card(
"WARN host188 CPU 警告: used=91% load=8.3"
)
projection = await telegram_gateway_module._mirror_ai_automation_alert_card_to_agent99(
card,
source="unit-test",
)
assert "Incident<code>unbound_fail_closed</code>" in card
assert "Run<code>not_created</code>" in card
assert "已建立 Run" not in card
assert "runtime_write_gate=0_until_incident" in card
assert projection is not None
assert projection["status"] == "receipt_unbound_fail_closed"
assert projection["dispatch_performed"] is False
assert reads == []

View File

@@ -21,6 +21,25 @@ from src.services.sre_k3s_controlled_automation_work_items import (
ROOT = Path(__file__).resolve().parents[3]
OPERATIONS = ROOT / "docs" / "operations"
SNAPSHOT = OPERATIONS / "sre-k3s-controlled-automation-work-items.snapshot.json"
COMMITMENTS = (
OPERATIONS / "sre-ai-agent-conversation-commitments.snapshot.json"
)
def _write_operations_fixture(
tmp_path: Path,
payload: dict[str, object],
commitment_payload: dict[str, object] | None = None,
) -> None:
(tmp_path / SNAPSHOT.name).write_text(
json.dumps(payload), encoding="utf-8"
)
(tmp_path / COMMITMENTS.name).write_text(
json.dumps(commitment_payload)
if commitment_payload is not None
else COMMITMENTS.read_text(encoding="utf-8"),
encoding="utf-8",
)
def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -> None:
@@ -127,6 +146,29 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
"blocked_cd_5384_failure_runtime_build_e614f061f7"
),
}
commitment_registry = payload["conversation_commitment_registry"]
assert commitment_registry["authoritative_for_execution_order"] is False
assert commitment_registry["raw_conversation_embedded"] is False
assert commitment_registry["secret_values_recorded"] is False
assert commitment_registry["rollups"] == {
"total_commitments": 70,
"active_or_completed_commitments": 68,
"by_status": {
"analysis_or_governance_complete": 6,
"source_implemented_runtime_pending": 23,
"in_progress": 32,
"not_started_or_no_current_evidence": 7,
"superseded": 2,
},
"product_runtime_closed_commitments": 0,
}
commitments = {
row["id"]: row for row in commitment_registry["commitments"]
}
assert commitments["AIA-CONV-012"]["title"].startswith("Claude API")
assert commitments["AIA-CONV-013"]["title"].startswith("Gemini API")
assert "Host112" in commitments["AIA-CONV-049"]["title"]
assert commitments["AIA-CONV-060"]["status"] == "superseded"
def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None:
@@ -229,6 +271,12 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None:
assert projection["immediate_execution_queue"][0]["work_item_id"] == (
"AIA-SRE-017"
)
assert projection["conversation_commitments"]["total_commitments"] == 70
assert (
projection["conversation_commitments"]
["product_runtime_closed_commitments"]
== 0
)
assert "work_items" not in projection
@@ -242,8 +290,7 @@ def test_loader_rejects_immediate_execution_queue_reordering(
)
for position, row in enumerate(payload["immediate_execution_queue"], start=1):
row["position"] = position
target = tmp_path / SNAPSHOT.name
target.write_text(json.dumps(payload), encoding="utf-8")
_write_operations_fixture(tmp_path, payload)
with pytest.raises(ValueError, match="safety order changed"):
load_sre_k3s_controlled_automation_work_items(tmp_path)
@@ -256,8 +303,7 @@ def test_loader_rejects_paid_enablement_without_cost_gate(
payload["provider_policy"]["required_before_paid_provider_enablement"].remove(
"production_cost_readback"
)
target = tmp_path / SNAPSHOT.name
target.write_text(json.dumps(payload), encoding="utf-8")
_write_operations_fixture(tmp_path, payload)
with pytest.raises(ValueError, match="explicit cost controls"):
load_sre_k3s_controlled_automation_work_items(tmp_path)
@@ -269,13 +315,26 @@ def test_loader_rejects_unknown_asset_executor(tmp_path: Path) -> None:
row for row in payload["domain_routes"] if row["domain"] == "unknown"
)
unknown["executor"] = "generic_fallback"
target = tmp_path / SNAPSHOT.name
target.write_text(json.dumps(payload), encoding="utf-8")
_write_operations_fixture(tmp_path, payload)
with pytest.raises(ValueError, match="unknown domain"):
load_sre_k3s_controlled_automation_work_items(tmp_path)
def test_loader_rejects_conversation_commitment_unknown_parent(
tmp_path: Path,
) -> None:
payload = json.loads(SNAPSHOT.read_text(encoding="utf-8"))
commitment_payload = json.loads(COMMITMENTS.read_text(encoding="utf-8"))
commitment_payload["commitments"][0]["linked_work_items"] = [
"AIA-SRE-999"
]
_write_operations_fixture(tmp_path, payload, commitment_payload)
with pytest.raises(ValueError, match="links unknown work items"):
load_sre_k3s_controlled_automation_work_items(tmp_path)
def test_endpoint_returns_full_work_ledger() -> None:
app = FastAPI()
app.include_router(router, prefix="/api/v1")
@@ -287,6 +346,7 @@ def test_endpoint_returns_full_work_ledger() -> None:
payload = response.json()
assert payload["program_id"] == "AIA-SRE-P0-20260715"
assert len(payload["work_items"]) == 18
assert len(payload["conversation_commitment_registry"]["commitments"]) == 70
assert payload["rollups"]["runtime_closed_items"] == 0
assert payload["independent_verifier_registry"][
"source_coverage_complete"

View File

@@ -9,6 +9,7 @@ from src.services.telegram_gateway import (
_outbound_source_envelope,
_sanitize_telegram_error,
format_aiops_signal_alert_card,
format_host_resource_alert_card,
)
@@ -299,6 +300,38 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz
assert "abcdefghijklmnopqrstuvwxyz" not in str(payload)
def test_host_resource_card_preserves_safe_incident_run_correlation() -> None:
incident_id = "INC-20260719-A1B2C3"
run_id = "7cf8fdf7-0966-5ac6-95b2-09e32808b248"
raw_alert = (
"WARN host188 CPU 警告: used=92% load=8.4\n"
f"Incident: {incident_id}\n"
f"Run: {run_id}\n"
"root 259 184 0.9 28062752 604824 ? Sl 05:55 0:33 "
"node /workspace/wooo/stockplatform-v2/node_modules/.bin/next build"
)
card = format_host_resource_alert_card(raw_alert)
metadata = _agent99_actionable_outbound_metadata(card)
payload = _agent99_alert_from_ai_automation_card(card, source="unit-test")
assert "事件類型:<code>host_resource_pressure_signal</code>" in card
assert "Target<code>host188</code>" in card
assert "Lane<code>runner_build_resource_pressure</code>" in card
assert f"Incident<code>{incident_id}</code>" in card
assert f"Run<code>{run_id}</code>" in card
assert "runtime_write_gate=controlled" in card
assert "建議下一步" not in card
assert metadata is not None
assert metadata["incident_id"] == incident_id
assert metadata["run_id"] == run_id
assert payload is not None
assert payload["kind"] == "performance_pressure"
assert payload["suggestedMode"] == "Perf"
assert payload["awoooi"]["incidentId"] == incident_id
assert payload["awoooi"]["runId"] == run_id
def test_action_required_cold_start_card_routes_to_agent99_recover() -> None:
card = (
" ACTION REQUIRED | <b>低風險</b>\n"

View File

@@ -109,15 +109,22 @@ root 364 181 0.7 3491396 494608 ? Rl 05:56 0:18 /opt/hostedto
assert "Load" in result
assert "容器 root 進程" in result
assert "AI 自動化判讀" in result
assert "host_resource_pressure_signal" in result
assert "Target<code>h110-gitea</code>" in result
assert "runner_build_resource_pressure" in result
assert "controlled_playbook_queue" in result
assert "runtime_write_gate=controlled" in result
assert "runtime_write_gate=0_until_incident" in result
assert "Incident<code>unbound_fail_closed</code>" in result
assert "Run<code>not_created</code>" in result
assert "incident_identity_unresolved" in result
assert "不建立或宣稱 Run" in result
assert "Top evidence" in result
assert "PID 259" in result
assert "Next.js build" in result
assert "PID 364" in result
assert "Next.js build worker" in result
assert "建議下一步" in result
assert "AI 自動處置佇列" in result
assert "建議下一步" not in result
assert "禁止事項" in result
assert "allowlisted PlayBook" in result
assert "root 259" not in result
@@ -144,7 +151,7 @@ def test_orphan_browser_alert_becomes_runaway_process_event_packet() -> None:
assert "check-mode" in result
assert "受控 SIGTERM" in result
assert "KM / PlayBook / Verifier" in result
assert "runtime_write_gate=controlled" in result
assert "runtime_write_gate=0_until_incident" in result
assert "allowlisted PlayBook" in result
assert "受控 apply" in result
@@ -164,7 +171,7 @@ def test_ci_runner_load_alert_becomes_capacity_event_packet() -> None:
assert "Gitea Actions run" in result
assert "合法 CI" in result
assert "不做 process remediation" in result
assert "runtime_write_gate=controlled" in result
assert "runtime_write_gate=0_until_incident" in result
assert "受控 cleanup" in result
@@ -369,7 +376,9 @@ async def test_send_alert_notification_normalizes_host_resource_raw_dump(monkeyp
assert receipt["schema_version"] == "telegram_alert_notification_receipt_v1"
assert receipt["status"] == "alert_notification_sent"
assert receipt["ai_automation_card_present"] is True
assert receipt["controlled_apply_allowed"] is True
assert receipt["controlled_apply_allowed"] is False
assert "incident_identity_unresolved" in payload["text"]
assert "Run<code>not_created</code>" in payload["text"]
assert receipt["manual_default_route_allowed"] is False
assert receipt["low_medium_high_route"] == "ai_controlled_apply"
@@ -420,7 +429,7 @@ root 392 0.0 0.0 1096836 53400 ? Ssl 06:27 0:00 /opt/hostedto
assert "Prisma generate" in result
assert "容器 root 進程:<code>3</code>" in result
assert "套件來源" in result
assert "runtime_write_gate=controlled" in result
assert "runtime_write_gate=0_until_incident" in result
assert "root 365" not in result
assert "checkpoint.prisma.io" not in result
assert "node_modules" not in result
@@ -514,7 +523,7 @@ def test_send_request_payload_normalizer_blocks_direct_host_raw_dump() -> None:
assert result["parse_mode"] == "HTML"
assert "P1 主機資源壓力h110-gitea" in result["text"]
assert "ai_automation_alert_card_v1" in result["text"]
assert "runtime_write_gate=controlled" in result["text"]
assert "runtime_write_gate=0_until_incident" in result["text"]
assert "root 311" not in result["text"]
assert "checkpoint.prisma.io" not in result["text"]
assert "/workspace/wooo" not in result["text"]