fix(automation): enforce durable alert closure
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s
Restore D037 durable incident readback without weakening database failure semantics. Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints. Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls. Keep SSH service refusal visible while separating it from broker network-policy reachability.
This commit is contained in:
@@ -620,6 +620,7 @@ async def _main() -> None:
|
||||
# Initialize and start worker
|
||||
worker = await init_signal_worker()
|
||||
ansible_candidate_backfill_task: asyncio.Task[Any] | None = None
|
||||
backup_restore_legacy_backfill_task: asyncio.Task[Any] | None = None
|
||||
if settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER:
|
||||
from src.jobs.awooop_ansible_candidate_backfill_job import (
|
||||
run_awooop_ansible_candidate_backfill_loop,
|
||||
@@ -634,6 +635,33 @@ async def _main() -> None:
|
||||
interval_seconds=settings.AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_INTERVAL_SECONDS,
|
||||
batch_limit=settings.AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_BATCH_LIMIT,
|
||||
)
|
||||
if settings.ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL:
|
||||
from src.jobs.backup_restore_legacy_backfill_job import (
|
||||
build_backup_restore_backfill_relay_preflight,
|
||||
run_backup_restore_legacy_backfill_loop,
|
||||
)
|
||||
|
||||
relay_preflight = build_backup_restore_backfill_relay_preflight()
|
||||
if relay_preflight["ready"] is True:
|
||||
backup_restore_legacy_backfill_task = asyncio.create_task(
|
||||
run_backup_restore_legacy_backfill_loop(),
|
||||
name="run_backup_restore_legacy_backfill_loop",
|
||||
)
|
||||
logger.info(
|
||||
"signal_worker_backup_restore_legacy_backfill_loop_started",
|
||||
batch_limit=settings.BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT,
|
||||
max_rows=settings.BACKUP_RESTORE_LEGACY_BACKFILL_MAX_ROWS,
|
||||
auth_mode=relay_preflight["auth_mode"],
|
||||
telegram_dispatch_performed=False,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"signal_worker_backup_restore_legacy_backfill_loop_skipped_fail_closed",
|
||||
status=relay_preflight["status"],
|
||||
auth_mode=relay_preflight["auth_mode"],
|
||||
endpoint_class=relay_preflight["endpoint_class"],
|
||||
telegram_dispatch_performed=False,
|
||||
)
|
||||
# Setup graceful shutdown
|
||||
shutdown_event = asyncio.Event()
|
||||
|
||||
@@ -665,6 +693,12 @@ async def _main() -> None:
|
||||
await ansible_candidate_backfill_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if backup_restore_legacy_backfill_task is not None:
|
||||
backup_restore_legacy_backfill_task.cancel()
|
||||
try:
|
||||
await backup_restore_legacy_backfill_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await worker.stop()
|
||||
await close_worker_redis_pool() # 關閉 Worker 專屬連線
|
||||
await close_redis_pool()
|
||||
|
||||
Reference in New Issue
Block a user