fix(automation): run asset reconciliation on worker
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled

This commit is contained in:
ogt
2026-07-15 08:18:17 +08:00
parent 618e499471
commit 47586431c0
6 changed files with 365 additions and 116 deletions

View File

@@ -71,6 +71,7 @@ GRACEFUL_SHUTDOWN_TIMEOUT_S = 75 # K8s terminationGracePeriodSeconds: 90
# Signal Worker
# =============================================================================
class SignalWorker:
"""
Redis Streams 訊號消費者
@@ -366,7 +367,9 @@ class SignalWorker:
span.set_attribute("messaging.destination", STREAM_KEY)
span.set_attribute("messaging.message_id", message_id)
span.set_attribute("signal.source", data.get("source", "unknown"))
span.set_attribute("signal.alert_name", data.get("alert_name", "unknown"))
span.set_attribute(
"signal.alert_name", data.get("alert_name", "unknown")
)
span.set_attribute("signal.severity", data.get("severity", "unknown"))
logger.info(
@@ -400,7 +403,9 @@ class SignalWorker:
severity=incident.severity.value,
signal_count=len(incident.signals),
affected_services=incident.affected_services,
persisted_to_pg=getattr(incident, "persisted_to_pg", False), # 2026-04-01 ogt: BrainIncident 無此欄位 (ADR-046 P2-01)
persisted_to_pg=getattr(
incident, "persisted_to_pg", False
), # 2026-04-01 ogt: BrainIncident 無此欄位 (ADR-046 P2-01)
)
try:
from src.services.signal_observation_service import (
@@ -505,6 +510,7 @@ def get_signal_worker() -> SignalWorker:
# Standalone Entry Point (for K8s Worker Deployment)
# =============================================================================
async def _write_health_files() -> None:
"""
Write health check files for K8s probes.
@@ -542,10 +548,7 @@ async def _heartbeat_loop(shutdown_event: asyncio.Event) -> None:
# 等待下次心跳或收到關閉信號
try:
await asyncio.wait_for(
shutdown_event.wait(),
timeout=HEARTBEAT_INTERVAL
)
await asyncio.wait_for(shutdown_event.wait(), timeout=HEARTBEAT_INTERVAL)
break # 收到關閉信號
except TimeoutError:
continue # 超時,繼續下次心跳
@@ -575,7 +578,9 @@ async def _main() -> None:
"signal_worker_standalone_starting",
environment=settings.ENVIRONMENT,
redis_url=settings.REDIS_URL.split("@")[-1] if settings.REDIS_URL else "N/A",
database_url=settings.DATABASE_URL.split("@")[-1] if settings.DATABASE_URL else "N/A",
database_url=settings.DATABASE_URL.split("@")[-1]
if settings.DATABASE_URL
else "N/A",
)
# Initialize Redis (API pool + Worker 專屬長連線池)
@@ -695,6 +700,9 @@ async def _main() -> None:
telegram_dispatch_performed=False,
)
if settings.ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER:
from src.jobs.asset_capability_reconciliation_job import (
run_asset_capability_reconciliation_loop,
)
from src.jobs.asset_change_tracker_job import run_asset_change_tracker_loop
from src.jobs.asset_scanner_job import run_asset_scanner_loop
from src.jobs.compliance_scanner_job import run_compliance_scanner_loop
@@ -707,6 +715,10 @@ async def _main() -> None:
("run_compliance_scanner_loop", run_compliance_scanner_loop),
("run_coverage_evaluator_loop", run_coverage_evaluator_loop),
("run_asset_change_tracker_loop", run_asset_change_tracker_loop),
(
"run_asset_capability_reconciliation_loop",
run_asset_capability_reconciliation_loop,
),
)
security_maintenance_tasks = [
asyncio.create_task(loop(), name=task_name)
@@ -791,12 +803,8 @@ async def _main() -> None:
"signal_worker_agent99_controlled_dispatch_reconciler_skipped_fail_closed",
owner="signal_worker",
reason="authenticated_relay_not_configured",
relay_url_configured=bool(
settings.AGENT99_SRE_ALERT_RELAY_URL.strip()
),
relay_token_configured=bool(
settings.AGENT99_SRE_ALERT_RELAY_TOKEN.strip()
),
relay_url_configured=bool(settings.AGENT99_SRE_ALERT_RELAY_URL.strip()),
relay_token_configured=bool(settings.AGENT99_SRE_ALERT_RELAY_TOKEN.strip()),
)
# Setup graceful shutdown
shutdown_event = asyncio.Event()
@@ -873,6 +881,7 @@ async def _main() -> None:
# Remove health files
from pathlib import Path
Path("/tmp/worker-healthy").unlink(missing_ok=True)
Path("/tmp/worker-ready").unlink(missing_ok=True)