from __future__ import annotations import asyncio import inspect import json from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import src.jobs.asset_capability_reconciliation_job as reconciliation_job import src.services.ai_automation_asset_capability_matrix as matrix_service from src.jobs.asset_capability_reconciliation_job import ( _persist_reconciliation, build_reconciliation_summary_payload, build_reconciliation_work_item_input, ) from src.services.ai_automation_asset_capability_matrix import ( REQUIRED_CONTROL_IDS, STAGE_IDS, _load_live_rows, build_asset_capability_matrix, build_asset_capability_matrix_with_live_readback, build_reconciliation_candidates, ) from src.services.awoooi_priority_work_order_readback import ( apply_ai_automation_asset_capability_matrix, get_ai_automation_program_scope_classes, ) _REPO_ROOT = Path(__file__).resolve().parents[3] def _single_host_scope() -> list[dict[str, object]]: return [ { "id": "hosts", "label": "Hosts", "asset_count": 1, "asset_ids": ["99"], "inventory_sources": ["host-inventory.json"], } ] def _fully_covered_host(now: datetime) -> dict[str, object]: dimensions = ( "auto_monitoring", "auto_alerting", "auto_rule_creation", "auto_rule_matching", "auto_playbook", "auto_remediation", "auto_km_creation", ) return { "asset_id": 42, "asset_key": "host:99", "asset_type": "host", "environment": "prod", "namespace": "", "name": "99", "owner_team": "platform_sre", "source_repo": "wooo/awoooi", "lifecycle_state": "active", "last_seen_at": now, "coverage": { dimension: { "coverage_status": "green", "created_at": now, } for dimension in dimensions }, "compliance": { "backup_tested": { "status": "compliant", "detected_at": now, } }, "change_types": [], } def test_declared_catalog_expands_to_193_unique_assets() -> None: matrix = build_asset_capability_matrix( get_ai_automation_program_scope_classes(), repo_root=_REPO_ROOT, ) assert matrix["summary"]["declared_asset_count"] == 193 assert matrix["summary"]["matrix_asset_count"] == 193 assert len({asset["canonical_id"] for asset in matrix["assets"]}) == 193 assert { item["scope_id"]: item["declared_asset_count"] for item in matrix["scope_rollups"] } == { "hosts": 7, "services": 31, "products": 12, "websites": 31, "tools": 23, "packages": 61, "data_and_backup": 18, "observability_and_security": 10, } assert matrix["controls"]["full_scope_class_coverage"] is True assert matrix["operation_boundaries"]["raw_asset_key_exposed"] is False def test_public_projection_counts_live_only_assets_without_embedding_them() -> None: now = datetime(2026, 7, 15, 1, 0, tzinfo=UTC) live_assets = [ { "asset_id": index + 1, "asset_key": f"zxq{index:x}v", "asset_type": "host", "name": f"zxq{index:x}v", "last_seen_at": now, "coverage": {}, "compliance": {}, "change_types": [], } for index in range(1_000) ] public_matrix = build_asset_capability_matrix( _single_host_scope(), live_assets=live_assets, repo_root=_REPO_ROOT, generated_at=now, include_live_only_assets=False, ) reconciliation_matrix = build_asset_capability_matrix( _single_host_scope(), live_assets=live_assets, repo_root=_REPO_ROOT, generated_at=now, include_live_only_assets=True, ) assert len(public_matrix["assets"]) == 1 public_summary = public_matrix["summary"] assert public_summary["live_inventory_asset_count"] == 1_000 assert public_summary["matrix_asset_count"] == 1 assert public_summary["reconciled_asset_count"] == 1_001 assert public_summary["live_only_assets_embedded"] is False assert public_summary["embedded_live_only_asset_count"] == 0 assert public_summary["discovered_not_declared_count"] == 1_000 assert len(reconciliation_matrix["assets"]) == 1_001 assert reconciliation_matrix["summary"]["live_only_assets_embedded"] is True assert reconciliation_matrix["summary"]["embedded_live_only_asset_count"] == 1_000 assert ( public_matrix["matrix_fingerprint"] == (reconciliation_matrix["matrix_fingerprint"]) ) def test_indexed_matching_preserves_score_cap_and_original_live_order() -> None: now = datetime(2026, 7, 15, 1, 0, tzinfo=UTC) scope = [ { "id": "services", "label": "Services", "asset_count": 1, "asset_ids": ["one two three four five six seven"], "inventory_sources": ["service-inventory.json"], } ] live_assets = [ { "asset_id": 1, "asset_key": "candidate:first", "asset_type": "container", "name": "one two three four five six alpha", "last_seen_at": now, }, { "asset_id": 2, "asset_key": "candidate:second", "asset_type": "container", "name": "one two three four five six seven beta", "last_seen_at": now, }, ] matrix = build_asset_capability_matrix( scope, live_assets=live_assets, repo_root=_REPO_ROOT, generated_at=now, include_live_only_assets=False, ) assert matrix["assets"][0]["runtime_identity"]["asset_inventory_id"] == 1 def test_live_asset_has_per_stage_covered_status_and_closes_after_receipt() -> None: now = datetime(2026, 7, 11, 12, 0, tzinfo=UTC) initial = build_asset_capability_matrix( _single_host_scope(), live_assets=[_fully_covered_host(now)], latest_run={"run_id": "run-1", "status": "success", "ended_at": now}, repo_root=_REPO_ROOT, generated_at=now, ) receipt = { "op_id": "receipt-1", "status": "success", "created_at": now, "input": { "trace_id": "trace-1", "automation_run_id": "automation-run-1", "work_item_id": "AIA-P0-006", }, "output": { "matrix_fingerprint": initial["matrix_fingerprint"], "candidate_count": 0, "repository_readback_count": 0, "work_item_verifier_readback_count": 0, "repository_readback_verified": True, "closed": True, }, } matrix = build_asset_capability_matrix( _single_host_scope(), live_assets=[_fully_covered_host(now)], latest_run={"run_id": "run-1", "status": "success", "ended_at": now}, reconciliation_receipt=receipt, repo_root=_REPO_ROOT, generated_at=now, ) asset = matrix["assets"][0] assert set(asset["stages"]) == set(STAGE_IDS) assert {stage["status"] for stage in asset["stages"].values()} == {"covered"} assert all(matrix["controls"][control_id] for control_id in REQUIRED_CONTROL_IDS) assert matrix["closed"] is True assert matrix["summary"]["gap_asset_count"] == 0 def test_reconciliation_receipt_requires_closed_terminal() -> None: now = datetime(2026, 7, 11, 12, 0, tzinfo=UTC) initial = build_asset_capability_matrix( _single_host_scope(), live_assets=[_fully_covered_host(now)], latest_run={"run_id": "run-1", "status": "success", "ended_at": now}, repo_root=_REPO_ROOT, generated_at=now, ) matrix = build_asset_capability_matrix( _single_host_scope(), live_assets=[_fully_covered_host(now)], latest_run={"run_id": "run-1", "status": "success", "ended_at": now}, reconciliation_receipt={ "status": "success", "created_at": now, "output": { "matrix_fingerprint": initial["matrix_fingerprint"], "candidate_count": 0, "repository_readback_count": 0, "repository_readback_verified": True, "closed": False, }, }, repo_root=_REPO_ROOT, generated_at=now, ) assert matrix["controls"]["daily_reconciliation_receipt"] is False assert matrix["closed"] is False def test_outdated_live_receipts_are_classified_stale_per_stage() -> None: now = datetime(2026, 7, 11, 12, 0, tzinfo=UTC) old = now - timedelta(hours=40) live = _fully_covered_host(old) matrix = build_asset_capability_matrix( _single_host_scope(), live_assets=[live], latest_run={"run_id": "run-old", "status": "success", "ended_at": old}, repo_root=_REPO_ROOT, generated_at=now, ) asset = matrix["assets"][0] assert asset["stages"]["runtime_identity"]["status"] == "stale" assert asset["stages"]["sensor"]["status"] == "stale" assert asset["stages"]["policy"]["status"] == "stale" assert asset["stages"]["executor"]["status"] == "stale" assert asset["stages"]["verifier"]["status"] == "stale" assert asset["stages"]["backup_restore"]["status"] == "stale" assert asset["stages"]["learning_writeback"]["status"] == "stale" assert asset["stale_stage_ids"] == [ "runtime_identity", "sensor", "policy", "executor", "verifier", "backup_restore", "learning_writeback", ] def test_reconciliation_candidates_are_stable_and_public_safe() -> None: matrix = build_asset_capability_matrix( _single_host_scope(), repo_root=_REPO_ROOT, generated_at=datetime(2026, 7, 11, 12, 0, tzinfo=UTC), ) first = build_reconciliation_candidates(matrix) second = build_reconciliation_candidates(matrix) assert first == second assert len(first) == 1 assert first[0]["work_item_id"].startswith("AIA-P0-006-WI-") assert first[0]["candidate_kind"] == "capability_gap" assert first[0]["controlled_apply_allowed"] is True assert "asset_key" not in first[0] assert "host" not in first[0] def test_reconciliation_summary_requires_repository_readback() -> None: matrix = build_asset_capability_matrix( _single_host_scope(), latest_run={"run_id": "discovery-run-1", "status": "success"}, repo_root=_REPO_ROOT, ) candidates = build_reconciliation_candidates(matrix) failed = build_reconciliation_summary_payload( matrix, candidates, trace_id="trace-1", automation_run_id="run-1", work_item_id="AIA-P0-006", repository_readback_count=0, work_item_verifier_readback_count=0, created_count=1, existing_count=0, ) passed = build_reconciliation_summary_payload( matrix, candidates, trace_id="trace-1", automation_run_id="run-1", work_item_id="AIA-P0-006", repository_readback_count=1, work_item_verifier_readback_count=1, created_count=1, existing_count=0, ) assert failed["repository_readback_verified"] is False assert failed["closed"] is False assert passed["repository_readback_verified"] is True assert passed["closed"] is True def test_reconciliation_summary_requires_each_work_item_verifier() -> None: matrix = build_asset_capability_matrix( _single_host_scope(), latest_run={"run_id": "discovery-run-1", "status": "success"}, repo_root=_REPO_ROOT, ) candidates = build_reconciliation_candidates(matrix) payload = build_reconciliation_summary_payload( matrix, candidates, trace_id="trace-1", automation_run_id="run-1", work_item_id="AIA-P0-006", repository_readback_count=1, work_item_verifier_readback_count=0, created_count=1, existing_count=0, ) assert payload["repository_readback_verified"] is False assert payload["post_verifier"]["passed"] is False assert payload["closed"] is False def test_reconciliation_work_item_uses_same_trace_and_run_as_summary() -> None: matrix = build_asset_capability_matrix( _single_host_scope(), repo_root=_REPO_ROOT, ) candidate = build_reconciliation_candidates(matrix)[0] receipt_input = build_reconciliation_work_item_input( candidate, trace_id="trace-1", automation_run_id="reconciliation-run-1", program_work_item_id="AIA-P0-006", matrix_fingerprint=matrix["matrix_fingerprint"], ) assert receipt_input["trace_id"] == "trace-1" assert receipt_input["automation_run_id"] == "reconciliation-run-1" assert receipt_input["program_work_item_id"] == "AIA-P0-006" assert ( receipt_input["source_diff"]["matrix_fingerprint"] == (matrix["matrix_fingerprint"]) ) def test_reconciliation_persistence_batches_candidate_queries_and_inserts() -> None: source = inspect.getsource(_persist_reconciliation) loop_start = source.index("for candidate in candidate_rows:") loop_end = source.index("insert_work_item_sql =") assert "ANY(CAST(:candidate_fingerprints AS text[]))" in source assert "insert_parameters" in source assert "_WORK_ITEM_INSERT_BATCH_SIZE" in source assert "await db.execute" not in source[loop_start:loop_end] assert "AND output ->> 'closed' = 'true'" in source assert "work_item_verifier_readback_count" in source def test_reconcile_once_uses_bounded_worker_live_readback_timeout( monkeypatch: Any, ) -> None: captured: dict[str, object] = {} async def _fake_live_readback(**kwargs: object) -> dict[str, object]: captured.update(kwargs) return {"live_readback_status": "degraded"} monkeypatch.setattr( reconciliation_job, "build_asset_capability_matrix_with_live_readback", _fake_live_readback, ) result = asyncio.run( reconciliation_job.reconcile_once( triggered_by="controlled_replay", project_id="awoooi", ) ) assert captured == { "project_id": "awoooi", "timeout_seconds": 30.0, "include_live_only_assets": True, } assert result["status"] == "degraded_no_write" def test_reconcile_once_publishes_closed_public_matrix_cache( monkeypatch: Any, ) -> None: live_readback_calls: list[dict[str, object]] = [] async def _fake_live_readback(**kwargs: object) -> dict[str, object]: live_readback_calls.append(dict(kwargs)) if kwargs["include_live_only_assets"] is True: return {"live_readback_status": "ready"} return {"live_readback_status": "ready", "closed": True} async def _fake_persist( _matrix: object, _candidates: object, *, project_id: str, ) -> dict[str, object]: assert project_id == "awoooi" return {"closed": True, "candidate_count": 0} monkeypatch.setattr( reconciliation_job, "build_asset_capability_matrix_with_live_readback", _fake_live_readback, ) monkeypatch.setattr( reconciliation_job, "build_reconciliation_candidates", lambda _matrix: [], ) monkeypatch.setattr(reconciliation_job, "_persist_reconciliation", _fake_persist) result = asyncio.run( reconciliation_job.reconcile_once( triggered_by="controlled_replay", project_id="awoooi", ) ) assert live_readback_calls == [ { "project_id": "awoooi", "timeout_seconds": 30.0, "include_live_only_assets": True, }, { "project_id": "awoooi", "timeout_seconds": 30.0, "include_live_only_assets": False, "bypass_public_cache": True, }, ] assert result["public_cache_refreshed"] is True def test_reconciliation_loop_retries_degraded_result_before_poll( monkeypatch: Any, ) -> None: triggered_by_values: list[str] = [] sleep_values: list[int] = [] async def _fake_reconcile_once(*, triggered_by: str) -> dict[str, object]: triggered_by_values.append(triggered_by) if len(triggered_by_values) == 1: return {"status": "degraded_no_write", "reason": "transient_db_pressure"} return {"status": "success", "closed": True} async def _fake_sleep(seconds: int) -> None: sleep_values.append(seconds) if len(sleep_values) == 3: raise asyncio.CancelledError monkeypatch.setattr(reconciliation_job, "reconcile_once", _fake_reconcile_once) monkeypatch.setattr(reconciliation_job.asyncio, "sleep", _fake_sleep) monkeypatch.setattr(reconciliation_job, "_FIRST_DELAY_SECONDS", 90) monkeypatch.setattr(reconciliation_job, "_LOOP_BACKOFF_SECONDS", 1_800) monkeypatch.setattr(reconciliation_job, "_RECONCILIATION_POLL_SECONDS", 1) try: asyncio.run(reconciliation_job.run_asset_capability_reconciliation_loop()) except asyncio.CancelledError: pass else: raise AssertionError("loop should stop at the test cancellation point") assert triggered_by_values == ["startup", "retry"] assert sleep_values == [90, 1_800, 1] def test_priority_overlay_projects_aia_p0_006_runtime_controls() -> None: matrix = build_asset_capability_matrix( _single_host_scope(), repo_root=_REPO_ROOT, ) payload: dict[str, object] = {"summary": {}, "rollups": {}} apply_ai_automation_asset_capability_matrix(payload, matrix) assert payload["ai_automation_asset_capability_matrix"]["schema_version"] == ( "ai_automation_asset_capability_matrix_v1" ) assert payload["ai_automation_asset_capability_matrix"]["assets"] == [] assert payload["ai_automation_asset_capability_matrix"]["assets_embedded"] is False assert payload["ai_automation_asset_capability_matrix"]["assets_endpoint"] == ( "/api/v1/agents/ai-automation-asset-capability-matrix" ) assert ( payload["summary"]["ai_automation_asset_capability_declared_asset_count"] == 1 ) work_item = next( item for item in payload["ai_automation_program_ledger"]["work_items"] if item["id"] == "AIA-P0-006" ) assert work_item["status"] == "in_progress" assert work_item["runtime_progress"]["required_control_count"] == len( REQUIRED_CONTROL_IDS ) class _FakeMappings: def __init__(self, rows: list[dict[str, Any]]) -> None: self._rows = rows def first(self) -> dict[str, Any] | None: return self._rows[0] if self._rows else None def all(self) -> list[dict[str, Any]]: return self._rows class _FakeResult: def __init__(self, rows: list[dict[str, Any]]) -> None: self._rows = rows def mappings(self) -> _FakeMappings: return _FakeMappings(self._rows) class _FakeLiveReadbackDb: def __init__(self, now: datetime) -> None: self.now = now self.calls: list[tuple[str, dict[str, Any]]] = [] async def execute( self, statement: Any, params: dict[str, Any] | None = None, ) -> _FakeResult: sql = str(statement) bound = params or {} self.calls.append((sql, bound)) if "set_config('statement_timeout'" in sql: return _FakeResult([]) if "FROM asset_discovery_run" in sql: return _FakeResult( [ { "run_id": "00000000-0000-0000-0000-000000000001", "status": "success", "ended_at": self.now, } ] ) if "FROM asset_inventory" in sql: return _FakeResult( [ { "asset_id": 42, "asset_key": "host:99", "asset_type": "host", "environment": "prod", "namespace": "", "name": "99", "owner_team": "platform_sre", "source_repo": "wooo/awoooi", "source_commit_sha": "abc123", "lifecycle_state": "active", "last_seen_at": self.now, "dimension": "auto_monitoring", "coverage_status": "green", "coverage_created_at": self.now, } ] ) if "FROM asset_compliance_snapshot" in sql: return _FakeResult([]) if "FROM asset_change_event" in sql: return _FakeResult([{"asset_id": 42, "change_type": "asset_modified"}]) if "FROM automation_operation_log" in sql: return _FakeResult([]) raise AssertionError(f"unexpected SQL: {sql}") class _FakeDbContext: def __init__(self, db: _FakeLiveReadbackDb) -> None: self.db = db async def __aenter__(self) -> _FakeLiveReadbackDb: return self.db async def __aexit__(self, *_args: object) -> None: return None class _FakeReconciliationResult: def __init__( self, *, scalar: Any = None, rows: list[tuple[Any, ...]] | None = None, ) -> None: self._scalar = scalar self._rows = rows or [] def scalar_one_or_none(self) -> Any: return self._scalar def scalar_one(self) -> Any: return self._scalar def fetchall(self) -> list[tuple[Any, ...]]: return self._rows class _FakeReconciliationDb: def __init__(self) -> None: self.persisted_fingerprints: set[str] = set() self.verified_fingerprints: set[str] = set() self.summary_outputs: list[dict[str, Any]] = [] self.work_item_batch_sizes: list[int] = [] async def execute( self, statement: Any, params: dict[str, Any] | list[dict[str, Any]] | None = None, ) -> _FakeReconciliationResult: sql = str(statement) if "pg_advisory_xact_lock" in sql: return _FakeReconciliationResult() if "SELECT output" in sql: prior = self.summary_outputs[-1] if self.summary_outputs else None return _FakeReconciliationResult(scalar=prior) if "INSERT INTO automation_operation_log" in sql and isinstance(params, list): self.work_item_batch_sizes.append(len(params)) for row in params: payload = json.loads(str(row["input"])) self.persisted_fingerprints.add(payload["candidate_fingerprint"]) return _FakeReconciliationResult() if "UPDATE automation_operation_log" in sql: assert isinstance(params, dict) requested = set(params["candidate_fingerprints"]) self.verified_fingerprints.update( requested & self.persisted_fingerprints ) return _FakeReconciliationResult() if "SELECT DISTINCT" in sql and "candidate_fingerprint" in sql: assert isinstance(params, dict) requested = set(params["candidate_fingerprints"]) source = ( self.verified_fingerprints if "repository_readback_verified" in sql else self.persisted_fingerprints ) return _FakeReconciliationResult( rows=[(fingerprint,) for fingerprint in sorted(requested & source)] ) if "RETURNING op_id::text" in sql: assert isinstance(params, dict) self.summary_outputs.append( json.loads(str(params["output"])) ) return _FakeReconciliationResult(scalar="summary-op-1") raise AssertionError(f"unexpected SQL: {sql}") class _FakeReconciliationDbContext: def __init__(self, db: _FakeReconciliationDb) -> None: self.db = db async def __aenter__(self) -> _FakeReconciliationDb: return self.db async def __aexit__(self, *_args: object) -> None: return None def test_reconciliation_persists_bounded_batches_and_reuses_verified_summary( monkeypatch: Any, ) -> None: db = _FakeReconciliationDb() monkeypatch.setattr( "src.db.base.get_db_context", lambda _project_id: _FakeReconciliationDbContext(db), ) matrix = { "matrix_fingerprint": "matrix-fingerprint-1", "latest_discovery_run": { "run_id": "00000000-0000-0000-0000-000000000001" }, "controls": { control_id: True for control_id in REQUIRED_CONTROL_IDS[:6] }, } candidates = [ { "candidate_fingerprint": f"fingerprint-{index:04d}", "candidate_kind": "capability_gap", "canonical_asset_id": f"awoooi:services:asset-{index:04d}", "asset_inventory_id": index, "missing_stage_ids": ["sensor"], "stale_stage_ids": [], "change_types": [], } for index in range(301) ] first = asyncio.run( _persist_reconciliation(matrix, candidates, project_id="awoooi") ) second = asyncio.run( _persist_reconciliation(matrix, candidates, project_id="awoooi") ) assert db.work_item_batch_sizes == [250, 51] assert first["candidate_count"] == 301 assert first["repository_readback_count"] == 301 assert first["work_item_verifier_readback_count"] == 301 assert first["repository_readback_verified"] is True assert first["closed"] is True assert second["repository_readback_verified"] is True assert len(db.summary_outputs) == 1 def test_live_change_event_readback_is_scoped_to_selected_asset_ids( monkeypatch: Any, ) -> None: now = datetime(2026, 7, 15, 1, 0, tzinfo=UTC) db = _FakeLiveReadbackDb(now) monkeypatch.setattr( "src.db.base.get_db_context", lambda _project_id: _FakeDbContext(db), ) _latest_run, assets, _receipt = asyncio.run(_load_live_rows("awoooi")) timeout_sql, timeout_params = next( (sql, params) for sql, params in db.calls if "set_config('statement_timeout'" in sql ) assert "statement_timeout" in timeout_sql assert timeout_params == {"statement_timeout": "6000ms"} change_sql, change_params = next( (sql, params) for sql, params in db.calls if "FROM asset_change_event" in sql ) inventory_sql = next( sql for sql, _params in db.calls if "FROM asset_inventory" in sql ) assert "JOIN asset_coverage_snapshot" in inventory_sql assert "LEFT JOIN asset_coverage_snapshot" not in inventory_sql assert "asset_id = ANY(:asset_ids)" in change_sql assert change_params == {"asset_ids": [42]} assert assets[0]["change_types"] == ["asset_modified"] def test_live_readback_timeout_fails_closed_without_hanging( monkeypatch: Any, ) -> None: async def _slow_live_readback( _project_id: str, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: await asyncio.sleep(1) return {}, [], {} monkeypatch.setattr(matrix_service, "_load_live_rows", _slow_live_readback) payload = asyncio.run( build_asset_capability_matrix_with_live_readback( repo_root=_REPO_ROOT, timeout_seconds=0.01, ) ) assert payload["live_readback_status"] == "degraded" assert payload["live_readback_error_type"] == "TimeoutError" assert payload["live_readback_timeout_seconds"] == 0.01 assert payload["summary"]["declared_asset_count"] == 193 assert payload["operation_boundaries"]["host_write_performed"] is False def test_matrix_build_phase_is_inside_the_total_timeout(monkeypatch: Any) -> None: async def _fast_live_readback( _project_id: str, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: return {}, [], {} async def _slow_to_thread(*_args: Any, **_kwargs: Any) -> dict[str, Any]: await asyncio.sleep(1) return {} monkeypatch.setattr(matrix_service, "_load_live_rows", _fast_live_readback) monkeypatch.setattr(matrix_service.asyncio, "to_thread", _slow_to_thread) payload = asyncio.run( build_asset_capability_matrix_with_live_readback( repo_root=_REPO_ROOT, timeout_seconds=0.01, ) ) assert payload["live_readback_status"] == "degraded" assert payload["live_readback_error_type"] == "TimeoutError" assert payload["live_readback_timeout_seconds"] == 0.01 def test_public_matrix_returns_fresh_cache_without_live_db_read( monkeypatch: Any, ) -> None: async def _cached_read( _project_id: str, *, namespace: str, ttl_seconds: int, ) -> dict[str, Any] | None: assert namespace == matrix_service._PUBLIC_MATRIX_FRESH_CACHE_NAMESPACE assert ttl_seconds == matrix_service.PUBLIC_MATRIX_FRESH_CACHE_TTL_SECONDS return { "live_readback_status": "ready", "summary": {"live_observed_asset_count": 193}, "cache": {"status": "hit", "age_seconds": 2.5}, } async def _unexpected_live_read( _project_id: str, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: raise AssertionError("fresh cache must bypass the live DB read") monkeypatch.setattr(matrix_service, "_read_public_matrix_cache", _cached_read) monkeypatch.setattr(matrix_service, "_load_live_rows", _unexpected_live_read) payload = asyncio.run(build_asset_capability_matrix_with_live_readback()) assert payload["live_readback_status"] == "ready" assert payload["summary"]["live_observed_asset_count"] == 193 assert payload["live_readback_cache"]["mode"] == "fresh" assert payload["live_readback_cache"]["fallback_used"] is False def test_public_matrix_success_warms_cache_for_next_request( monkeypatch: Any, ) -> None: from src.services.operator_summary_cache import clear_operator_summary_cache live_read_count = 0 async def _live_readback( _project_id: str, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: nonlocal live_read_count live_read_count += 1 return {}, [], {} clear_operator_summary_cache() monkeypatch.setattr(matrix_service, "_load_live_rows", _live_readback) try: first = asyncio.run(build_asset_capability_matrix_with_live_readback()) second = asyncio.run(build_asset_capability_matrix_with_live_readback()) finally: clear_operator_summary_cache() assert first["live_readback_status"] == "ready" assert second["live_readback_status"] == "ready" assert second["live_readback_cache"]["mode"] == "fresh" assert live_read_count == 1 def test_public_matrix_cache_bypass_forces_shared_refresh(monkeypatch: Any) -> None: stored_payloads: list[dict[str, Any]] = [] async def _unexpected_cache_read(*_args: Any, **_kwargs: Any) -> dict[str, Any]: raise AssertionError("cache bypass must not read a prior projection") async def _live_readback( _project_id: str, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: return {}, [], {} async def _store_cache(_project_id: str, payload: dict[str, Any]) -> None: stored_payloads.append(payload) monkeypatch.setattr( matrix_service, "_read_public_matrix_cache", _unexpected_cache_read ) monkeypatch.setattr(matrix_service, "_load_live_rows", _live_readback) monkeypatch.setattr(matrix_service, "_store_public_matrix_caches", _store_cache) payload = asyncio.run( build_asset_capability_matrix_with_live_readback( bypass_public_cache=True, ) ) assert payload["live_readback_status"] == "ready" assert len(stored_payloads) == 1 assert stored_payloads[0]["matrix_fingerprint"] == payload["matrix_fingerprint"] def test_public_matrix_timeout_uses_last_ready_cache(monkeypatch: Any) -> None: async def _slow_live_readback( _project_id: str, ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: await asyncio.sleep(1) return {}, [], {} async def _cached_read( _project_id: str, *, namespace: str, ttl_seconds: int, ) -> dict[str, Any] | None: if namespace == matrix_service._PUBLIC_MATRIX_FRESH_CACHE_NAMESPACE: return None assert namespace == matrix_service._PUBLIC_MATRIX_FALLBACK_CACHE_NAMESPACE assert ttl_seconds == matrix_service.PUBLIC_MATRIX_FALLBACK_CACHE_TTL_SECONDS return { "live_readback_status": "ready", "summary": { "live_inventory_asset_count": 9595, "live_observed_asset_count": 193, "stage_coverage_percent": 18, }, "cache": {"status": "hit", "age_seconds": 51.2}, } monkeypatch.setattr(matrix_service, "_read_public_matrix_cache", _cached_read) monkeypatch.setattr(matrix_service, "_load_live_rows", _slow_live_readback) payload = asyncio.run( build_asset_capability_matrix_with_live_readback(timeout_seconds=0.01) ) assert payload["live_readback_status"] == "degraded_cached" assert payload["live_readback_error_type"] == "TimeoutError" assert payload["summary"]["live_inventory_asset_count"] == 9595 assert payload["summary"]["live_observed_asset_count"] == 193 assert payload["summary"]["stage_coverage_percent"] == 18 assert payload["live_readback_cache"]["mode"] == "stale_if_error" assert payload["live_readback_cache"]["fallback_used"] is True