From af4cb2fd432e0f1098a072034637de90c6b6964f Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 03:54:47 +0800 Subject: [PATCH] fix(runtime): bind lifecycle tenant context --- .../src/jobs/incident_lifecycle_reconciler.py | 38 +++++++++++++------ .../test_incident_lifecycle_reconciler.py | 30 +++++++++++++++ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/apps/api/src/jobs/incident_lifecycle_reconciler.py b/apps/api/src/jobs/incident_lifecycle_reconciler.py index c90316c98..b85a31f59 100644 --- a/apps/api/src/jobs/incident_lifecycle_reconciler.py +++ b/apps/api/src/jobs/incident_lifecycle_reconciler.py @@ -23,6 +23,7 @@ import structlog from sqlalchemy import text from src.core.config import settings +from src.core.context import clear_project_context, set_project_context from src.db.base import get_db_context from src.utils.timezone import now_taipei @@ -31,6 +32,7 @@ logger = structlog.get_logger(__name__) BATCH_LIMIT = 100 INTERVAL_SECONDS = 1800 _PROMETHEUS_TIMEOUT_SECONDS = 5.0 +_PROJECT_ID = "awoooi" @dataclass(frozen=True) @@ -43,20 +45,32 @@ class LifecycleCandidate: async def run_incident_lifecycle_reconciler_loop() -> None: """每 30 分鐘收斂一小批已有完成證據的 stuck incident。""" - while True: - try: - resolved, errors = await reconcile_stuck_incidents() - if resolved > 0 or errors > 0: - logger.info( - "incident_lifecycle_reconciler_done", - resolved=resolved, - errors=errors, - batch_limit=BATCH_LIMIT, + context_tokens = set_project_context( + _PROJECT_ID, + source="incident_lifecycle_reconciler", + ) + try: + while True: + try: + resolved, errors = await reconcile_stuck_incidents() + if resolved > 0 or errors > 0: + logger.info( + "incident_lifecycle_reconciler_done", + project_id=_PROJECT_ID, + resolved=resolved, + errors=errors, + batch_limit=BATCH_LIMIT, + ) + except Exception as exc: + logger.warning( + "incident_lifecycle_reconciler_loop_failed", + project_id=_PROJECT_ID, + error=str(exc), ) - except Exception as exc: - logger.warning("incident_lifecycle_reconciler_loop_failed", error=str(exc)) - await asyncio.sleep(INTERVAL_SECONDS) + await asyncio.sleep(INTERVAL_SECONDS) + finally: + clear_project_context(context_tokens) async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int]: diff --git a/apps/api/tests/test_incident_lifecycle_reconciler.py b/apps/api/tests/test_incident_lifecycle_reconciler.py index 68aaa2b86..342d932a7 100644 --- a/apps/api/tests/test_incident_lifecycle_reconciler.py +++ b/apps/api/tests/test_incident_lifecycle_reconciler.py @@ -1,8 +1,11 @@ +import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock import pytest +from src.core.context import get_current_project_context +from src.jobs import incident_lifecycle_reconciler as reconciler from src.jobs.incident_lifecycle_reconciler import ( LifecycleCandidate, reconcile_stuck_incidents, @@ -50,3 +53,30 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch): "resolution_type": "timeout", "emit_postmortem": False, } + + +@pytest.mark.asyncio +async def test_reconciler_loop_binds_and_clears_project_context(monkeypatch): + observed_contexts: list[dict[str, str | None]] = [] + + async def fake_reconcile() -> tuple[int, int]: + observed_contexts.append(get_current_project_context()) + return 0, 0 + + async def cancel_after_first_pass(_seconds: int) -> None: + raise asyncio.CancelledError + + monkeypatch.setattr(reconciler, "reconcile_stuck_incidents", fake_reconcile) + monkeypatch.setattr(reconciler.asyncio, "sleep", cancel_after_first_pass) + + with pytest.raises(asyncio.CancelledError): + await reconciler.run_incident_lifecycle_reconciler_loop() + + assert observed_contexts == [ + { + "project_id": "awoooi", + "source": "incident_lifecycle_reconciler", + "request_id": None, + } + ] + assert get_current_project_context()["project_id"] is None