diff --git a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py index 01b2cb7d9..49224ecc5 100644 --- a/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py +++ b/apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py @@ -9,6 +9,7 @@ the runtime apply by itself. from __future__ import annotations import asyncio +import json from collections.abc import Awaitable, Callable from types import SimpleNamespace from typing import Any @@ -104,22 +105,52 @@ def _build_backfill_proposal(incident: dict[str, Any]) -> dict[str, Any]: def _incident_for_evidence(incident: dict[str, Any]) -> SimpleNamespace: - signals = [ - SimpleNamespace( - alert_name=str(signal.get("alert_name") or ""), - labels=dict(signal.get("labels") or {}), - annotations=dict(signal.get("annotations") or {}), - source=str(signal.get("source") or ""), + raw_signals = incident.get("signals") or [] + if isinstance(raw_signals, str): + try: + raw_signals = json.loads(raw_signals) + except json.JSONDecodeError: + raw_signals = [] + if not isinstance(raw_signals, list): + raw_signals = [] + + signals: list[SimpleNamespace] = [] + affected_services: list[str] = [] + for value in raw_signals: + if not isinstance(value, dict): + continue + labels = value.get("labels") if isinstance(value.get("labels"), dict) else {} + annotations = ( + value.get("annotations") + if isinstance(value.get("annotations"), dict) + else {} ) - for signal in incident.get("signals") or [] - if isinstance(signal, dict) - ] + signals.append( + SimpleNamespace( + alert_name=str( + value.get("alert_name") + or labels.get("alertname") + or incident.get("alertname") + or "" + ), + labels=labels, + annotations=annotations, + source=str(value.get("source") or "alertmanager"), + ) + ) + for key in ("service", "service_name", "deployment", "job"): + service = str(labels.get(key) or "").strip() + if service and service not in affected_services: + affected_services.append(service) + + affected_services = list(incident.get("affected_services") or affected_services) return SimpleNamespace( incident_id=str(incident.get("incident_id") or ""), + project_id=str(incident.get("project_id") or "awoooi"), severity=incident.get("severity"), alertname=incident.get("alertname"), - affected_services=list(incident.get("affected_services") or []), signals=signals, + affected_services=affected_services, ) @@ -130,7 +161,7 @@ async def collect_ansible_candidate_pre_decision_evidence( automation_run_id: str, ) -> EvidenceSnapshot | None: return await get_pre_decision_investigator().investigate( - _incident_for_evidence(incident), + _incident_for_evidence({**incident, "project_id": project_id}), project_id=project_id, automation_run_id=automation_run_id, ) @@ -144,6 +175,7 @@ async def verify_ansible_candidate_pre_decision_evidence( if ( snapshot is None or snapshot.sensors_succeeded <= 0 + or not any(ready is True for ready in snapshot.mcp_health.values()) or not isinstance(snapshot.recent_logs, str) or not snapshot.recent_logs.strip() ): @@ -159,6 +191,11 @@ async def verify_ansible_candidate_pre_decision_evidence( AND post_execution_state IS NULL AND sensors_succeeded > 0 AND NULLIF(recent_logs, '') IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM jsonb_each(coalesce(mcp_health, '{}'::jsonb)) health + WHERE health.value = 'true'::jsonb + ) ) """), { diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index f48cf7989..f26c0470c 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -129,28 +129,31 @@ _AI_AUTOMATION_PROFESSIONAL_REVIEW_SOURCE = ( "docs/reviews/2026-07-10-security-governance-professional-review.snapshot.json" ) _AI_AUTOMATION_CODEBASE_REVIEW = { - "schema_version": "ai_automation_codebase_review_v1", - "reviewed_at": "2026-07-10T20:00:00+08:00", - "review_basis": "full baseline review through 542fbda1 plus targeted runtime delta review through 43ceb3b2", + "schema_version": "ai_automation_codebase_review_v2", + "reviewed_at": "2026-07-10T20:56:19+08:00", + "review_basis": "workspace-wide tracked source/config/test inventory at Gitea main plus current P0 tenant-evidence remediation delta", + "inventory_recompute_required_on_source_change": True, "scope": { - "reviewed_file_count": 977, - "reviewed_line_count": 402994, - "api_service_file_count": 227, - "api_route_file_count": 66, - "worker_job_file_count": 15, - "ops_script_file_count": 199, - "web_file_count": 69, - "k8s_file_count": 12, - "ansible_file_count": 23, - "test_file_count": 366, + "reviewed_file_count": 1971, + "reviewed_line_count": 742000, + "reviewed_line_count_semantics": "rounded_minimum_from_full_inventory_pass", + "api_service_file_count": 396, + "api_route_file_count": 43, + "worker_job_file_count": 24, + "ops_script_file_count": 375, + "web_file_count": 230, + "k8s_file_count": 63, + "ansible_file_count": 26, + "test_file_count": 538, }, "risk_metrics": { - "exception_handler_count": 1359, - "broad_exception_handler_count": 715, - "silent_exception_pass_count": 44, - "db_context_without_explicit_project_count": 125, - "fire_and_forget_task_count": 28, - "not_implemented_count": 7, + "exception_handler_count": 2231, + "broad_exception_handler_count": 1371, + "silent_exception_pass_count": 104, + "db_context_without_explicit_project_count": 248, + "fire_and_forget_task_count": 89, + "not_implemented_count": 8, + "todo_or_fixme_count": 52, }, "findings": [ { @@ -195,6 +198,23 @@ _AI_AUTOMATION_CODEBASE_REVIEW = { ], "mapped_work_item_ids": ["AIA-P0-004"], }, + { + "id": "ACR-P0-005", + "severity": "critical", + "title": "Evidence persistence loses tenant or incident identity and backfill can execute before usable MCP/log context", + "source_refs": [ + "apps/api/src/services/evidence_snapshot.py", + "apps/api/src/services/pre_decision_investigator.py", + "apps/api/src/services/signal_observation_service.py", + "apps/api/src/services/post_execution_verifier.py", + "apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py", + "apps/api/src/services/awooop_ansible_check_mode_service.py", + ], + "mapped_work_item_ids": ["AIA-P0-001"], + "source_remediation_state": "fixed_tests_passed_runtime_pending", + "source_test_evidence": "184 focused and 3955 non-integration tests passed", + "runtime_exit_condition": "production run persists same-tenant MCP and service-log evidence without Missing tenant context", + }, { "id": "ACR-P1-001", "severity": "high", @@ -218,6 +238,19 @@ _AI_AUTOMATION_CODEBASE_REVIEW = { ], "mapped_work_item_ids": ["AIA-P1-008"], }, + { + "id": "ACR-P1-003", + "severity": "medium", + "title": "Integration tests are not fully marked and can enter external DB or tenant-unaware endpoint paths during the default suite", + "source_refs": [ + "apps/api/pyproject.toml", + "apps/api/tests/integration/conftest.py", + "apps/api/tests/integration/test_drift_repository.py", + ], + "mapped_work_item_ids": ["AIA-P1-008"], + "source_remediation_state": "open", + "next_action": "mark the complete integration tree, provide isolated tenant-aware fixtures, and keep the default suite network-independent", + }, ], } _AI_AUTOMATION_PROGRAM_DEPENDENCIES = { @@ -506,7 +539,7 @@ _AI_AUTOMATION_PROGRAM_WORK_ITEMS: list[dict[str, Any]] = [ "metadata-only、source、test、UI、CD 與歷史 aggregate 不得關閉當前 run", "移除 hardcoded 100 completion 並公開缺少的 stage receipts", ], - "next_action": "先部署 service evidence、retry terminal 與 timeline same-run receipts，再完成 MCP、RAG 與 PlayBook trust 真實寫回。", + "next_action": "部署 tenant-aware EvidenceSnapshot、MCP timeline 與 cache identity 修正，觸發 production canary；並恢復至少一條 embedding provider 後驗證 RAG durable writeback。", }, { "id": "AIA-P0-011", @@ -7103,10 +7136,25 @@ def _apply_ai_automation_program_ledger(payload: dict[str, Any]) -> None: strict_runtime_closed = ( summary.get("ai_agent_autonomous_runtime_strict_closed") is True ) + runtime_work_item = next( + item for item in work_items if item["id"] == "AIA-P0-001" + ) + runtime_work_item["runtime_progress"] = { + "completion_percent": _int( + summary.get("ai_agent_autonomous_runtime_completion_percent") + ), + "present_stage_count": _int( + summary.get("ai_agent_autonomous_runtime_present_stage_count") + ), + "required_stage_count": _int( + summary.get("ai_agent_autonomous_runtime_required_stage_count") + ), + "missing_stage_ids": list( + summary.get("ai_agent_autonomous_runtime_missing_stage_ids") or [] + ), + "runtime_closed": strict_runtime_closed, + } if strict_runtime_closed: - runtime_work_item = next( - item for item in work_items if item["id"] == "AIA-P0-001" - ) runtime_work_item["status"] = "done" runtime_work_item["completion_evidence"] = { "source": "agent-autonomous-runtime-control", @@ -7529,6 +7577,9 @@ def apply_ai_automation_live_closure_readbacks( summary["ai_agent_autonomous_runtime_present_stage_count"] = _int( strict_runtime.get("present_stage_count") ) + summary["ai_agent_autonomous_runtime_missing_stage_ids"] = list( + strict_runtime.get("missing_stage_ids") or [] + ) summary["ai_agent_autonomous_runtime_same_run_correlation"] = ( strict_runtime.get("same_run_correlation") is True ) diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 253c7f0ae..8a00265e0 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -838,7 +838,10 @@ def build_ansible_context_runtime_stage_receipts( mcp_health = _json_loads(evidence.get("mcp_health")) sensors_attempted = max(0, _int_from_value(evidence.get("sensors_attempted"), default=0)) sensors_succeeded = max(0, _int_from_value(evidence.get("sensors_succeeded"), default=0)) - if any(value is True for value in mcp_health.values()) or sensors_succeeded > 0: + successful_tool_count = sum( + 1 for ready in mcp_health.values() if ready is True + ) + if sensors_succeeded > 0 and successful_tool_count > 0: receipts.append( _runtime_stage_receipt( claim, @@ -847,6 +850,7 @@ def build_ansible_context_runtime_stage_receipts( detail={ "evidence_id": evidence_id, "tool_status_count": len(mcp_health), + "successful_tool_count": successful_tool_count, "sensors_attempted": sensors_attempted, "sensors_succeeded": sensors_succeeded, "context_collected_before_candidate": True, diff --git a/apps/api/src/services/evidence_snapshot.py b/apps/api/src/services/evidence_snapshot.py index 71251a171..0086aea2d 100644 --- a/apps/api/src/services/evidence_snapshot.py +++ b/apps/api/src/services/evidence_snapshot.py @@ -105,6 +105,7 @@ class EvidenceSnapshot: # LLM 輸入摘要（由 Investigator 組裝） evidence_summary: str | None = None + persisted: bool = False # 執行前後 State pre_execution_state: dict[str, Any] | None = None @@ -234,11 +235,13 @@ class EvidenceSnapshot: ) db.add(record) await db.flush() + self.persisted = True logger.info( "evidence_snapshot_saved", snapshot_id=self.snapshot_id, incident_id=self.incident_id, + project_id=self.project_id, sensors_succeeded=self.sensors_succeeded, collection_ms=self.collection_duration_ms, ) @@ -436,6 +439,7 @@ async def get_latest_snapshot( post_execution_state=row.post_execution_state, verification_result=row.verification_result, matched_playbook_id=row.matched_playbook_id, + persisted=True, ) return snap diff --git a/apps/api/src/services/post_execution_verifier.py b/apps/api/src/services/post_execution_verifier.py index b22fea422..6c842e97b 100644 --- a/apps/api/src/services/post_execution_verifier.py +++ b/apps/api/src/services/post_execution_verifier.py @@ -694,7 +694,10 @@ async def _persist_fallback_snapshot( """ incident_id = _get_incident_id(incident) try: - snapshot = EvidenceSnapshot(incident_id=incident_id) + snapshot = EvidenceSnapshot( + incident_id=incident_id, + project_id=str(getattr(incident, "project_id", None) or "awoooi"), + ) snapshot.post_execution_state = post_state snapshot.verification_result = result snapshot.matched_playbook_id = _extract_playbook_id(action_taken) diff --git a/apps/api/src/services/pre_decision_investigator.py b/apps/api/src/services/pre_decision_investigator.py index d67e8c399..a879fccde 100644 --- a/apps/api/src/services/pre_decision_investigator.py +++ b/apps/api/src/services/pre_decision_investigator.py @@ -28,6 +28,7 @@ import asyncio import hashlib import json import time +import uuid from typing import TYPE_CHECKING, Any from uuid import UUID @@ -102,6 +103,7 @@ class PreDecisionInvestigator: """ start_ms = int(time.monotonic() * 1000) incident_id = incident.incident_id if hasattr(incident, "incident_id") else str(incident.id) + project_id = _get_project_id(incident, fallback=project_id) # 1. 計算 fingerprint 並查 cache fingerprint = _compute_fingerprint( @@ -112,7 +114,17 @@ class PreDecisionInvestigator: cached = await _get_cache(fingerprint, project_id=project_id) if cached is not None: logger.debug("investigator_cache_hit", incident_id=incident_id, fingerprint=fingerprint) - return cached + snapshot = _rebind_cached_snapshot( + cached, + incident=incident, + incident_id=incident_id, + project_id=project_id, + ) + try: + await snapshot.save() + except Exception: + logger.exception("investigator_cache_hit_save_failed", incident_id=incident_id) + return snapshot # 2. 取工具清單 alertname = _get_alertname(incident) @@ -130,19 +142,7 @@ class PreDecisionInvestigator: # 告警基礎資訊：sensors=0 時 AI 至少知道是什麼告警 # 2026-04-16 ogt + Claude Sonnet 4.6: 修復空 evidence → ABSTAIN 問題 - sigs = getattr(incident, "signals", []) or [] - sig0 = sigs[0] if sigs else None - snapshot.alert_info = { - "alert_name": alertname or getattr(incident, "alertname", "") or "", - "severity": str(getattr(incident, "severity", "")), - "affected_services": getattr(incident, "affected_services", []) or [], - "labels": labels, - "annotations": ( - ({k: v for k, v in (sig0.annotations or {}).items()} if sig0 else {}) - ), - "source": getattr(sig0, "source", "") if sig0 else "", - "incident_id": incident_id, - } + snapshot.alert_info = _build_alert_info(incident, incident_id) # 3. 並行蒐集（整體 INVESTIGATOR_TIMEOUT_SEC 保護） try: @@ -196,14 +196,17 @@ class PreDecisionInvestigator: snapshot.evidence_summary = snapshot.build_summary() # 6. 持久化（fire-and-await，Phase 3 學習閉環依賴此表） + persisted = False try: await snapshot.save() + persisted = True except Exception: logger.exception("investigator_save_failed", incident_id=incident_id) # 不 raise：snapshot 仍可用於決策，存儲失敗不阻塞主路徑 - # 7. 寫 cache - await _set_cache(fingerprint, snapshot) + # 7. 只快取已持久化的證據，避免把資料庫失敗掩蓋成可重用證據。 + if persisted: + await _set_cache(fingerprint, snapshot) logger.info( "investigator_done", @@ -415,7 +418,7 @@ class PreDecisionInvestigator: # 2026-04-18 ADR-090-D: MCP 呼叫入 timeline_events(MASTER §7.1 #4 KPI) try: _duration_ms = int((asyncio.get_event_loop().time() - _started) * 1000) - asyncio.create_task(_log_mcp_call_to_timeline( + await _log_mcp_call_to_timeline( snapshot_incident_id=getattr(snapshot, "incident_id", None), project_id=project_id, provider_name=reg.provider.name, @@ -423,7 +426,7 @@ class PreDecisionInvestigator: status=_mcp_status, error=_mcp_error, duration_ms=_duration_ms, - )) + ) except Exception: pass @@ -604,6 +607,35 @@ def _get_labels(incident: "Incident") -> dict[str, Any]: return {} +def _get_project_id(incident: "Incident", *, fallback: str = "awoooi") -> str: + """Resolve the tenant for background and request-driven incident paths.""" + explicit = getattr(incident, "project_id", None) + if explicit: + return str(explicit) + + try: + from src.core.context import get_current_project_id + + return str(get_current_project_id() or fallback) + except Exception: + return fallback + + +def _build_alert_info(incident: "Incident", incident_id: str) -> dict[str, Any]: + sigs = getattr(incident, "signals", []) or [] + sig0 = sigs[0] if sigs else None + annotations = getattr(sig0, "annotations", {}) if sig0 else {} + return { + "alert_name": _get_alertname(incident) or getattr(incident, "alertname", "") or "", + "severity": str(getattr(incident, "severity", "")), + "affected_services": getattr(incident, "affected_services", []) or [], + "labels": _get_labels(incident), + "annotations": {k: v for k, v in (annotations or {}).items()}, + "source": getattr(sig0, "source", "") if sig0 else "", + "incident_id": incident_id, + } + + _SHORT_HOST_MAP: dict[str, str] = { "110": "192.168.0.110", "120": "192.168.0.120", @@ -751,10 +783,18 @@ async def _get_cache( snapshot_id=data.get("snapshot_id", ""), ) snap.evidence_summary = data.get("evidence_summary", "") + snap.alert_info = data.get("alert_info") snap.k8s_state = data.get("k8s_state") snap.recent_logs = data.get("recent_logs") snap.metrics_snapshot = data.get("metrics_snapshot") + snap.recent_deployments = data.get("recent_deployments") + snap.business_metrics = data.get("business_metrics") + snap.historical_context = data.get("historical_context") + snap.peer_health = data.get("peer_health") + snap.dependency_topology = data.get("dependency_topology") + snap.anomaly_context = data.get("anomaly_context") snap.mcp_health = data.get("mcp_health", {}) + snap.collection_duration_ms = data.get("collection_duration_ms") snap.sensors_attempted = data.get("sensors_attempted", 0) snap.sensors_succeeded = data.get("sensors_succeeded", 0) return snap @@ -770,12 +810,21 @@ async def _set_cache(fingerprint: str, snapshot: EvidenceSnapshot) -> None: key = f"evidence:{fingerprint}" payload = { "incident_id": snapshot.incident_id, + "project_id": snapshot.project_id, "snapshot_id": snapshot.snapshot_id, "evidence_summary": snapshot.evidence_summary, + "alert_info": snapshot.alert_info, "k8s_state": snapshot.k8s_state, "recent_logs": snapshot.recent_logs, "metrics_snapshot": snapshot.metrics_snapshot, + "recent_deployments": snapshot.recent_deployments, + "business_metrics": snapshot.business_metrics, + "historical_context": snapshot.historical_context, + "peer_health": snapshot.peer_health, + "dependency_topology": snapshot.dependency_topology, + "anomaly_context": snapshot.anomaly_context, "mcp_health": snapshot.mcp_health, + "collection_duration_ms": snapshot.collection_duration_ms, "sensors_attempted": snapshot.sensors_attempted, "sensors_succeeded": snapshot.sensors_succeeded, } @@ -784,6 +833,37 @@ async def _set_cache(fingerprint: str, snapshot: EvidenceSnapshot) -> None: pass # cache 失敗不影響主路徑 +def _rebind_cached_snapshot( + cached: EvidenceSnapshot, + *, + incident: "Incident", + incident_id: str, + project_id: str, +) -> EvidenceSnapshot: + """Reuse sensor content while creating a durable identity for this incident.""" + snapshot = EvidenceSnapshot( + incident_id=incident_id, + project_id=project_id, + snapshot_id=str(uuid.uuid4()), + alert_info=_build_alert_info(incident, incident_id), + k8s_state=cached.k8s_state, + recent_logs=cached.recent_logs, + metrics_snapshot=cached.metrics_snapshot, + recent_deployments=cached.recent_deployments, + business_metrics=cached.business_metrics, + historical_context=cached.historical_context, + peer_health=cached.peer_health, + dependency_topology=cached.dependency_topology, + anomaly_context=cached.anomaly_context, + mcp_health=dict(cached.mcp_health), + collection_duration_ms=cached.collection_duration_ms, + sensors_attempted=cached.sensors_attempted, + sensors_succeeded=cached.sensors_succeeded, + ) + snapshot.evidence_summary = snapshot.build_summary() + return snapshot + + # ───────────────────────────────────────────────────────────────────────────── # Singleton # ───────────────────────────────────────────────────────────────────────────── diff --git a/apps/api/src/services/signal_observation_service.py b/apps/api/src/services/signal_observation_service.py index 12359f1c3..6794f2f59 100644 --- a/apps/api/src/services/signal_observation_service.py +++ b/apps/api/src/services/signal_observation_service.py @@ -376,7 +376,7 @@ async def _record_raw_signal_evidence_if_needed( signal_data.get("severity") or getattr(incident, "severity", None) ) annotations = _annotations(incident, signal_data) - snapshot = EvidenceSnapshot(incident_id=incident_id) + snapshot = EvidenceSnapshot(incident_id=incident_id, project_id=PROJECT_ID) snapshot.alert_info = { "alert_name": alertname, "severity": severity, diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 8eb7f5dd0..61e7c3637 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -962,12 +962,23 @@ def test_ai_automation_program_ledger_is_authoritative_and_runtime_strict(): ) assert program["professional_review"]["finding_count"] == 9 assert program["professional_review"]["critical_finding_count"] == 3 - assert program["codebase_review"]["scope"]["reviewed_file_count"] == 977 + assert program["codebase_review"]["scope"]["reviewed_file_count"] == 1971 assert program["codebase_review"]["risk_metrics"][ "broad_exception_handler_count" - ] == 715 - assert program_summary["codebase_review_finding_count"] == 6 - assert program_summary["codebase_review_critical_finding_count"] == 2 + ] == 1371 + assert program_summary["codebase_review_finding_count"] == 8 + assert program_summary["codebase_review_critical_finding_count"] == 3 + tenant_finding = next( + row + for row in program["codebase_review"]["findings"] + if row["id"] == "ACR-P0-005" + ) + assert tenant_finding["source_remediation_state"] == ( + "fixed_tests_passed_runtime_pending" + ) + assert work_item_by_id["AIA-P0-001"]["runtime_progress"][ + "runtime_closed" + ] is False assert work_item_by_id["AIA-P1-002"]["status"] == "in_progress" assert "CPU" in work_item_by_id["AIA-P1-002"]["title"] assert work_item_by_id["AIA-P1-004"]["schedule"] == { diff --git a/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py b/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py index 48b445f61..32d993825 100644 --- a/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py +++ b/apps/api/tests/test_awooop_ansible_candidate_backfill_job.py @@ -123,6 +123,43 @@ async def test_backfill_enqueues_catalog_matched_incident(monkeypatch: pytest.Mo assert recorded[0]["automation_run_id"] +@pytest.mark.asyncio +async def test_backfill_retries_automatically_when_predecision_evidence_is_incomplete( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from src.jobs import awooop_ansible_candidate_backfill_job as job + + fake_db = AsyncMock() + fake_db.execute = AsyncMock(return_value=_FakeResult([_candidate_incident()])) + + @asynccontextmanager + async def fake_db_context(project_id: str = "awoooi"): + yield fake_db + + async def fake_evidence_collector(**_kwargs): + return None + + async def fake_evidence_verifier(**kwargs): + assert kwargs["snapshot"] is None + return False + + recorder = AsyncMock(return_value=True) + monkeypatch.setattr(job, "get_db_context", fake_db_context) + monkeypatch.setattr(job.settings, "ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER", True) + + result = await job.enqueue_missing_ansible_candidates_once( + recorder=recorder, + evidence_collector=fake_evidence_collector, + evidence_verifier=fake_evidence_verifier, + receipt_backfiller=AsyncMock(return_value={"written": 0, "error": None}), + retry_replayer=AsyncMock(return_value={"error": None}), + ) + + assert result["queued"] == 0 + assert result["pre_decision_evidence_blocked"] == 1 + recorder.assert_not_awaited() + + @pytest.mark.asyncio async def test_backfill_skips_when_worker_disabled(monkeypatch: pytest.MonkeyPatch) -> None: from src.jobs import awooop_ansible_candidate_backfill_job as job diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index c278d142b..b405d002c 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -1969,6 +1969,33 @@ def test_ansible_context_receipts_reference_pre_decision_evidence_without_conten assert "evidence.post_execution_state IS NULL" in loader_source +def test_ansible_context_receipt_rejects_zero_success_mcp_attempts() -> None: + claim = AnsibleCheckModeClaim( + op_id="00000000-0000-0000-0000-000000000132", + source_candidate_op_id="00000000-0000-0000-0000-000000000131", + incident_id="INC-RUNTIME-CONTEXT-FAILED", + catalog_id="ansible:188-ai-web", + playbook_path="infra/ansible/playbooks/188-ai-web-readonly.yml", + apply_playbook_path="infra/ansible/playbooks/188-ai-web.yml", + inventory_hosts=("host_188",), + risk_level="medium", + input_payload={}, + ) + + receipts = build_ansible_context_runtime_stage_receipts( + claim, + { + "id": "evidence-failed", + "recent_logs": None, + "mcp_health": {"prometheus": False, "kubernetes": False}, + "sensors_attempted": 2, + "sensors_succeeded": 0, + }, + ) + + assert receipts == [] + + def test_ansible_timeline_projection_receipt_is_same_run_and_idempotent() -> None: claim = AnsibleCheckModeClaim( op_id="00000000-0000-0000-0000-000000000122", diff --git a/apps/api/tests/test_evidence_snapshot_tenant.py b/apps/api/tests/test_evidence_snapshot_tenant.py new file mode 100644 index 000000000..266ea8130 --- /dev/null +++ b/apps/api/tests/test_evidence_snapshot_tenant.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import pytest + +from src.services import evidence_snapshot as evidence_module +from src.services.evidence_snapshot import EvidenceSnapshot + + +class _FakeDb: + def __init__(self) -> None: + self.records: list[object] = [] + + def add(self, record: object) -> None: + self.records.append(record) + + async def flush(self) -> None: + return None + + +class _FakeDbContext: + def __init__(self, db: _FakeDb) -> None: + self.db = db + + async def __aenter__(self) -> _FakeDb: + return self.db + + async def __aexit__(self, *_args: object) -> None: + return None + + +@pytest.mark.asyncio +async def test_save_uses_snapshot_project_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + projects: list[str] = [] + db = _FakeDb() + + def fake_get_db_context(project_id: str) -> _FakeDbContext: + projects.append(project_id) + return _FakeDbContext(db) + + monkeypatch.setattr(evidence_module, "get_db_context", fake_get_db_context) + + snapshot = EvidenceSnapshot(incident_id="INC-TENANT", project_id="tenant-a") + await snapshot.save() + + assert projects == ["tenant-a"] + assert len(db.records) == 1 + assert snapshot.persisted is True diff --git a/apps/api/tests/test_pre_decision_investigator.py b/apps/api/tests/test_pre_decision_investigator.py index 22a44cf79..2e333eff1 100644 --- a/apps/api/tests/test_pre_decision_investigator.py +++ b/apps/api/tests/test_pre_decision_investigator.py @@ -224,6 +224,46 @@ class TestComputeFingerprint: assert all(c in "0123456789abcdef" for c in fp) +class TestInvestigateCacheIdentity: + @pytest.mark.asyncio + async def test_cache_hit_rebinds_identity_and_persists_current_incident( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + cached = EvidenceSnapshot( + incident_id="INC-OLD", + snapshot_id="snapshot-old", + recent_logs="shared log evidence", + sensors_attempted=2, + sensors_succeeded=2, + mcp_health={"kubectl_logs": True}, + ) + saved: list[EvidenceSnapshot] = [] + + async def fake_get_cache( + _fingerprint: str, + **_kwargs: object, + ) -> EvidenceSnapshot: + return cached + + async def fake_save(snapshot: EvidenceSnapshot) -> str: + saved.append(snapshot) + return snapshot.snapshot_id + + monkeypatch.setattr(pdi_module, "_get_cache", fake_get_cache) + monkeypatch.setattr(EvidenceSnapshot, "save", fake_save) + + incident = _stub_incident(alertname="CacheIdentity") + snapshot = await PreDecisionInvestigator().investigate(incident) + + assert snapshot.incident_id == incident.incident_id + assert snapshot.snapshot_id != cached.snapshot_id + assert snapshot.project_id == "awoooi" + assert snapshot.recent_logs == "shared log evidence" + assert snapshot.alert_info["incident_id"] == incident.incident_id + assert saved == [snapshot] + + # ───────────────────────────────────────────────────────────────────────────── # _build_tool_params # ─────────────────────────────────────────────────────────────────────────────