diff --git a/apps/api/src/main.py b/apps/api/src/main.py index 875e29313..f9ca12fce 100644 --- a/apps/api/src/main.py +++ b/apps/api/src/main.py @@ -64,8 +64,8 @@ from src.api.v1 import incidents as incidents_v1 # Phase 6.4: Decision Proposal from src.api.v1 import iwooos as iwooos_v1 # IwoooS security governance API from src.api.v1 import knowledge as knowledge_v1 # KB Phase 1: Knowledge Base from src.api.v1 import learning as learning_v1 # Phase D-G P0: Learning API -from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈) from src.api.v1 import mcp_control_plane as mcp_control_plane_v1 +from src.api.v1 import metrics as metrics_v1 # Phase 7: Gold Metrics (真實血脈) from src.api.v1 import monitoring as monitoring_v1 # 2026-04-03: 監控工具狀態 from src.api.v1 import notifications as notifications_v1 # 2026-04-10: 通知頻道狀態 from src.api.v1 import ( @@ -650,22 +650,24 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]: logger.warning("approval_timeout_resolver_schedule_failed", error=str(e)) # T73: 已有完成證據但仍卡在 INVESTIGATING 的舊 incident 小批次收斂。 - # 僅處理 auto-repair success / approval EXECUTION_SUCCESS / approval EXPIRED， - # 不自動關閉 manual_required 或單純 APPROVED 事件。 + # API production pool 只有 serving 預算；實際 loop 由單一 standalone + # signal worker 擁有，避免 coroutine 被 budget gate 關閉後仍假性記錄 + # scheduled。reconciler 本身只處理強證據或已不再 firing 的事件。 try: from src.jobs.incident_lifecycle_reconciler import ( INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, ) - from src.jobs.incident_lifecycle_reconciler import ( - run_incident_lifecycle_reconciler_loop, - ) - schedule_api_background_task(run_incident_lifecycle_reconciler_loop()) logger.info( - "incident_lifecycle_reconciler_scheduled", + "incident_lifecycle_reconciler_owner_delegated", interval_sec=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, + scheduled_in_api_process=False, + production_process_owner="awoooi-worker", ) except Exception as e: - logger.warning("incident_lifecycle_reconciler_schedule_failed", error=str(e)) + logger.warning( + "incident_lifecycle_reconciler_owner_resolution_failed", + error=str(e), + ) # AwoooP Ansible candidate backfill worker. # 把近期已命中 allowlisted PlayBook、但缺 durable candidate row 的事故補進 diff --git a/apps/api/src/services/telegram_alert_ai_automation_matrix.py b/apps/api/src/services/telegram_alert_ai_automation_matrix.py index 76a90e288..9e9c9fbd6 100644 --- a/apps/api/src/services/telegram_alert_ai_automation_matrix.py +++ b/apps/api/src/services/telegram_alert_ai_automation_matrix.py @@ -703,13 +703,14 @@ def _inspect_source_contract(repo_root: Path) -> dict[str, Any]: def _read_source_contract_text(repo_root: Path, relative_path: str) -> str: - candidates = [repo_root / relative_path] + source_path = repo_root / relative_path + candidates = [source_path] api_prefix = "apps/api/" if relative_path.startswith(api_prefix): candidates.append(repo_root / relative_path.removeprefix(api_prefix)) if ( relative_path == _AWOOOP_VERIFIED_CONTEXT_SURFACE_SOURCE - and not (repo_root / "apps").exists() + and not source_path.exists() ): candidates.append(repo_root / "src/services/telegram_alert_ai_automation_matrix.py") diff --git a/apps/api/src/workers/signal_worker.py b/apps/api/src/workers/signal_worker.py index 1a16c0a39..38c029333 100644 --- a/apps/api/src/workers/signal_worker.py +++ b/apps/api/src/workers/signal_worker.py @@ -623,7 +623,35 @@ async def _main() -> None: backup_restore_legacy_backfill_task: asyncio.Task[Any] | None = None security_maintenance_tasks: list[asyncio.Task[Any]] = [] agent99_controlled_dispatch_reconciler_task: asyncio.Task[Any] | None = None + incident_lifecycle_reconciler_task: asyncio.Task[Any] | None = None mcp_version_lifecycle_task: asyncio.Task[Any] | None = None + + # Production API pods reserve their DB pool for serving requests. This + # singleton worker therefore owns the existing bounded, fail-closed + # lifecycle reconciler. It resolves at most one configured batch per pass + # and independently reads current Prometheus firing state before treating + # inactive alerts as closure candidates. + from src.jobs.incident_lifecycle_reconciler import ( + BATCH_LIMIT as INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT, + ) + from src.jobs.incident_lifecycle_reconciler import ( + INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, + ) + from src.jobs.incident_lifecycle_reconciler import ( + run_incident_lifecycle_reconciler_loop, + ) + + incident_lifecycle_reconciler_task = asyncio.create_task( + run_incident_lifecycle_reconciler_loop(), + name="run_incident_lifecycle_reconciler_loop", + ) + logger.info( + "signal_worker_incident_lifecycle_reconciler_started", + owner="signal_worker", + interval_seconds=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL, + batch_limit=INCIDENT_LIFECYCLE_RECONCILER_BATCH_LIMIT, + direct_incident_closure_performed=False, + ) if settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER: from src.jobs.awooop_ansible_candidate_backfill_job import ( run_awooop_ansible_candidate_backfill_loop, @@ -794,6 +822,12 @@ async def _main() -> None: await agent99_controlled_dispatch_reconciler_task except asyncio.CancelledError: pass + if incident_lifecycle_reconciler_task is not None: + incident_lifecycle_reconciler_task.cancel() + try: + await incident_lifecycle_reconciler_task + except asyncio.CancelledError: + pass if mcp_version_lifecycle_task is not None: mcp_version_lifecycle_task.cancel() try: diff --git a/apps/api/tests/test_runtime_bootstrap_guards.py b/apps/api/tests/test_runtime_bootstrap_guards.py index f4b36be32..d42a70e7f 100644 --- a/apps/api/tests/test_runtime_bootstrap_guards.py +++ b/apps/api/tests/test_runtime_bootstrap_guards.py @@ -539,6 +539,40 @@ def test_signal_worker_gracefully_cancels_agent99_reconciler() -> None: assert source.index(await_task) < source.index("await worker.stop()") +def test_signal_worker_is_single_incident_lifecycle_reconciler_owner() -> None: + import inspect + + from src import main as api_main + from src.workers import signal_worker + + api_source = inspect.getsource(api_main.lifespan) + worker_source = inspect.getsource(signal_worker._main) + + assert "run_incident_lifecycle_reconciler_loop" not in api_source + assert "incident_lifecycle_reconciler_owner_delegated" in api_source + assert worker_source.count("run_incident_lifecycle_reconciler_loop()") == 1 + assert worker_source.index("if settings.DATABASE_BOOTSTRAP_DDL_ENABLED") < ( + worker_source.index("run_incident_lifecycle_reconciler_loop()") + ) + assert "signal_worker_incident_lifecycle_reconciler_started" in worker_source + assert 'owner="signal_worker"' in worker_source + + +def test_signal_worker_gracefully_cancels_incident_lifecycle_reconciler() -> None: + import inspect + + from src.workers import signal_worker + + source = inspect.getsource(signal_worker._main) + cancel = "incident_lifecycle_reconciler_task.cancel()" + await_task = "await incident_lifecycle_reconciler_task" + + assert cancel in source + assert await_task in source + assert source.index(cancel) < source.index(await_task) + assert source.index(await_task) < source.index("await worker.stop()") + + def test_production_signal_worker_is_single_agent99_reconciler_owner() -> None: import inspect diff --git a/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py b/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py index d4d2a5994..4ed760981 100644 --- a/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py +++ b/apps/api/tests/test_telegram_alert_ai_automation_matrix_api.py @@ -8,6 +8,7 @@ from fastapi.testclient import TestClient from src.api.v1.agents import router from src.services.telegram_alert_ai_automation_matrix import ( _inspect_source_contract, + _read_source_contract_text, load_latest_telegram_alert_ai_automation_matrix, ) @@ -210,6 +211,28 @@ def test_telegram_matrix_source_contract_supports_container_layout(tmp_path): assert proof["source_contract_missing_markers"] == [] +def test_telegram_matrix_source_contract_uses_embedded_surface_when_web_package_only( + tmp_path, +): + (tmp_path / "apps/web").mkdir(parents=True) + (tmp_path / "apps/web/package.json").write_text("{}\n", encoding="utf-8") + (tmp_path / "src/services").mkdir(parents=True) + embedded_source = ( + tmp_path / "src/services/telegram_alert_ai_automation_matrix.py" + ) + embedded_source.write_text( + 'data-testid="telegram-alert-post-apply-verifier-readback"\n', + encoding="utf-8", + ) + + source = _read_source_contract_text( + tmp_path, + "apps/web/src/components/awooop/autonomous-runtime-receipt-panel.tsx", + ) + + assert 'data-testid="telegram-alert-post-apply-verifier-readback"' in source + + def test_telegram_matrix_source_contract_reports_missing_web_surface_without_raising( tmp_path, ):