fix(runtime): bind lifecycle tenant context
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m25s
CD Pipeline / build-and-deploy (push) Successful in 14m30s
CD Pipeline / post-deploy-checks (push) Successful in 1m57s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 12s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m25s
CD Pipeline / build-and-deploy (push) Successful in 14m30s
CD Pipeline / post-deploy-checks (push) Successful in 1m57s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 12s
This commit is contained in:
@@ -23,6 +23,7 @@ import structlog
|
|||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
from src.core.config import settings
|
from src.core.config import settings
|
||||||
|
from src.core.context import clear_project_context, set_project_context
|
||||||
from src.db.base import get_db_context
|
from src.db.base import get_db_context
|
||||||
from src.utils.timezone import now_taipei
|
from src.utils.timezone import now_taipei
|
||||||
|
|
||||||
@@ -31,6 +32,7 @@ logger = structlog.get_logger(__name__)
|
|||||||
BATCH_LIMIT = 100
|
BATCH_LIMIT = 100
|
||||||
INTERVAL_SECONDS = 1800
|
INTERVAL_SECONDS = 1800
|
||||||
_PROMETHEUS_TIMEOUT_SECONDS = 5.0
|
_PROMETHEUS_TIMEOUT_SECONDS = 5.0
|
||||||
|
_PROJECT_ID = "awoooi"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -43,20 +45,32 @@ class LifecycleCandidate:
|
|||||||
|
|
||||||
async def run_incident_lifecycle_reconciler_loop() -> None:
|
async def run_incident_lifecycle_reconciler_loop() -> None:
|
||||||
"""每 30 分鐘收斂一小批已有完成證據的 stuck incident。"""
|
"""每 30 分鐘收斂一小批已有完成證據的 stuck incident。"""
|
||||||
|
context_tokens = set_project_context(
|
||||||
|
_PROJECT_ID,
|
||||||
|
source="incident_lifecycle_reconciler",
|
||||||
|
)
|
||||||
|
try:
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
resolved, errors = await reconcile_stuck_incidents()
|
resolved, errors = await reconcile_stuck_incidents()
|
||||||
if resolved > 0 or errors > 0:
|
if resolved > 0 or errors > 0:
|
||||||
logger.info(
|
logger.info(
|
||||||
"incident_lifecycle_reconciler_done",
|
"incident_lifecycle_reconciler_done",
|
||||||
|
project_id=_PROJECT_ID,
|
||||||
resolved=resolved,
|
resolved=resolved,
|
||||||
errors=errors,
|
errors=errors,
|
||||||
batch_limit=BATCH_LIMIT,
|
batch_limit=BATCH_LIMIT,
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning("incident_lifecycle_reconciler_loop_failed", error=str(exc))
|
logger.warning(
|
||||||
|
"incident_lifecycle_reconciler_loop_failed",
|
||||||
|
project_id=_PROJECT_ID,
|
||||||
|
error=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
await asyncio.sleep(INTERVAL_SECONDS)
|
await asyncio.sleep(INTERVAL_SECONDS)
|
||||||
|
finally:
|
||||||
|
clear_project_context(context_tokens)
|
||||||
|
|
||||||
|
|
||||||
async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int]:
|
async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int]:
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
|
import asyncio
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import AsyncMock
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from src.core.context import get_current_project_context
|
||||||
|
from src.jobs import incident_lifecycle_reconciler as reconciler
|
||||||
from src.jobs.incident_lifecycle_reconciler import (
|
from src.jobs.incident_lifecycle_reconciler import (
|
||||||
LifecycleCandidate,
|
LifecycleCandidate,
|
||||||
reconcile_stuck_incidents,
|
reconcile_stuck_incidents,
|
||||||
@@ -50,3 +53,30 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch):
|
|||||||
"resolution_type": "timeout",
|
"resolution_type": "timeout",
|
||||||
"emit_postmortem": False,
|
"emit_postmortem": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_reconciler_loop_binds_and_clears_project_context(monkeypatch):
|
||||||
|
observed_contexts: list[dict[str, str | None]] = []
|
||||||
|
|
||||||
|
async def fake_reconcile() -> tuple[int, int]:
|
||||||
|
observed_contexts.append(get_current_project_context())
|
||||||
|
return 0, 0
|
||||||
|
|
||||||
|
async def cancel_after_first_pass(_seconds: int) -> None:
|
||||||
|
raise asyncio.CancelledError
|
||||||
|
|
||||||
|
monkeypatch.setattr(reconciler, "reconcile_stuck_incidents", fake_reconcile)
|
||||||
|
monkeypatch.setattr(reconciler.asyncio, "sleep", cancel_after_first_pass)
|
||||||
|
|
||||||
|
with pytest.raises(asyncio.CancelledError):
|
||||||
|
await reconciler.run_incident_lifecycle_reconciler_loop()
|
||||||
|
|
||||||
|
assert observed_contexts == [
|
||||||
|
{
|
||||||
|
"project_id": "awoooi",
|
||||||
|
"source": "incident_lifecycle_reconciler",
|
||||||
|
"request_id": None,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
assert get_current_project_context()["project_id"] is None
|
||||||
|
|||||||
Reference in New Issue
Block a user