fix(runtime): bind lifecycle tenant context
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m25s
CD Pipeline / build-and-deploy (push) Successful in 14m30s
CD Pipeline / post-deploy-checks (push) Successful in 1m57s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 12s

This commit is contained in:
ogt
2026-07-15 03:54:47 +08:00
parent 11b676b6e6
commit af4cb2fd43
2 changed files with 56 additions and 12 deletions

View File

@@ -23,6 +23,7 @@ import structlog
from sqlalchemy import text
from src.core.config import settings
from src.core.context import clear_project_context, set_project_context
from src.db.base import get_db_context
from src.utils.timezone import now_taipei
@@ -31,6 +32,7 @@ logger = structlog.get_logger(__name__)
BATCH_LIMIT = 100
INTERVAL_SECONDS = 1800
_PROMETHEUS_TIMEOUT_SECONDS = 5.0
_PROJECT_ID = "awoooi"
@dataclass(frozen=True)
@@ -43,20 +45,32 @@ class LifecycleCandidate:
async def run_incident_lifecycle_reconciler_loop() -> None:
"""每 30 分鐘收斂一小批已有完成證據的 stuck incident。"""
while True:
try:
resolved, errors = await reconcile_stuck_incidents()
if resolved > 0 or errors > 0:
logger.info(
"incident_lifecycle_reconciler_done",
resolved=resolved,
errors=errors,
batch_limit=BATCH_LIMIT,
context_tokens = set_project_context(
_PROJECT_ID,
source="incident_lifecycle_reconciler",
)
try:
while True:
try:
resolved, errors = await reconcile_stuck_incidents()
if resolved > 0 or errors > 0:
logger.info(
"incident_lifecycle_reconciler_done",
project_id=_PROJECT_ID,
resolved=resolved,
errors=errors,
batch_limit=BATCH_LIMIT,
)
except Exception as exc:
logger.warning(
"incident_lifecycle_reconciler_loop_failed",
project_id=_PROJECT_ID,
error=str(exc),
)
except Exception as exc:
logger.warning("incident_lifecycle_reconciler_loop_failed", error=str(exc))
await asyncio.sleep(INTERVAL_SECONDS)
await asyncio.sleep(INTERVAL_SECONDS)
finally:
clear_project_context(context_tokens)
async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int]: