fix(automation): enforce durable alert closure
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s

Restore D037 durable incident readback without weakening database failure semantics.

Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints.

Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls.

Keep SSH service refusal visible while separating it from broker network-policy reachability.
This commit is contained in:
ogt
2026-07-14 09:40:53 +08:00
parent 72a167a322
commit 6cf8429d17
56 changed files with 15100 additions and 227 deletions

View File

@@ -620,6 +620,7 @@ async def _main() -> None:
# Initialize and start worker
worker = await init_signal_worker()
ansible_candidate_backfill_task: asyncio.Task[Any] | None = None
backup_restore_legacy_backfill_task: asyncio.Task[Any] | None = None
if settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER:
from src.jobs.awooop_ansible_candidate_backfill_job import (
run_awooop_ansible_candidate_backfill_loop,
@@ -634,6 +635,33 @@ async def _main() -> None:
interval_seconds=settings.AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_INTERVAL_SECONDS,
batch_limit=settings.AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_BATCH_LIMIT,
)
if settings.ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL:
from src.jobs.backup_restore_legacy_backfill_job import (
build_backup_restore_backfill_relay_preflight,
run_backup_restore_legacy_backfill_loop,
)
relay_preflight = build_backup_restore_backfill_relay_preflight()
if relay_preflight["ready"] is True:
backup_restore_legacy_backfill_task = asyncio.create_task(
run_backup_restore_legacy_backfill_loop(),
name="run_backup_restore_legacy_backfill_loop",
)
logger.info(
"signal_worker_backup_restore_legacy_backfill_loop_started",
batch_limit=settings.BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT,
max_rows=settings.BACKUP_RESTORE_LEGACY_BACKFILL_MAX_ROWS,
auth_mode=relay_preflight["auth_mode"],
telegram_dispatch_performed=False,
)
else:
logger.warning(
"signal_worker_backup_restore_legacy_backfill_loop_skipped_fail_closed",
status=relay_preflight["status"],
auth_mode=relay_preflight["auth_mode"],
endpoint_class=relay_preflight["endpoint_class"],
telegram_dispatch_performed=False,
)
# Setup graceful shutdown
shutdown_event = asyncio.Event()
@@ -665,6 +693,12 @@ async def _main() -> None:
await ansible_candidate_backfill_task
except asyncio.CancelledError:
pass
if backup_restore_legacy_backfill_task is not None:
backup_restore_legacy_backfill_task.cancel()
try:
await backup_restore_legacy_backfill_task
except asyncio.CancelledError:
pass
await worker.stop()
await close_worker_redis_pool() # 關閉 Worker 專屬連線
await close_redis_pool()