Merge remote-tracking branch 'origin/main' into codex/p0-obs-002-20260715
This commit is contained in:
@@ -23,6 +23,7 @@ import structlog
|
||||
from sqlalchemy import text
|
||||
|
||||
from src.core.config import settings
|
||||
from src.core.context import clear_project_context, set_project_context
|
||||
from src.db.base import get_db_context
|
||||
from src.utils.timezone import now_taipei
|
||||
|
||||
@@ -31,6 +32,7 @@ logger = structlog.get_logger(__name__)
|
||||
BATCH_LIMIT = 100
|
||||
INTERVAL_SECONDS = 1800
|
||||
_PROMETHEUS_TIMEOUT_SECONDS = 5.0
|
||||
_PROJECT_ID = "awoooi"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -43,20 +45,32 @@ class LifecycleCandidate:
|
||||
|
||||
async def run_incident_lifecycle_reconciler_loop() -> None:
|
||||
"""每 30 分鐘收斂一小批已有完成證據的 stuck incident。"""
|
||||
while True:
|
||||
try:
|
||||
resolved, errors = await reconcile_stuck_incidents()
|
||||
if resolved > 0 or errors > 0:
|
||||
logger.info(
|
||||
"incident_lifecycle_reconciler_done",
|
||||
resolved=resolved,
|
||||
errors=errors,
|
||||
batch_limit=BATCH_LIMIT,
|
||||
context_tokens = set_project_context(
|
||||
_PROJECT_ID,
|
||||
source="incident_lifecycle_reconciler",
|
||||
)
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
resolved, errors = await reconcile_stuck_incidents()
|
||||
if resolved > 0 or errors > 0:
|
||||
logger.info(
|
||||
"incident_lifecycle_reconciler_done",
|
||||
project_id=_PROJECT_ID,
|
||||
resolved=resolved,
|
||||
errors=errors,
|
||||
batch_limit=BATCH_LIMIT,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"incident_lifecycle_reconciler_loop_failed",
|
||||
project_id=_PROJECT_ID,
|
||||
error=str(exc),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("incident_lifecycle_reconciler_loop_failed", error=str(exc))
|
||||
|
||||
await asyncio.sleep(INTERVAL_SECONDS)
|
||||
await asyncio.sleep(INTERVAL_SECONDS)
|
||||
finally:
|
||||
clear_project_context(context_tokens)
|
||||
|
||||
|
||||
async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int]:
|
||||
|
||||
Reference in New Issue
Block a user