Merge remote-tracking branch 'origin/main' into codex/aia-p0-006-security-control-plane-20260711

apps/api/src/jobs/asset_capability_reconciliation_job.py

@@ -13,6 +13,7 @@ import time
 import uuid
 from collections import Counter
 from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime
 from typing import Any
 
 import structlog
@@ -29,9 +30,12 @@ from src.services.ai_automation_asset_capability_matrix import (
 
 logger = structlog.get_logger(__name__)
 
-_FIRST_DELAY_SECONDS = 30
+_FIRST_DELAY_SECONDS = 7 * 60
 _LOOP_BACKOFF_SECONDS = 5 * 60
 _RECONCILIATION_POLL_SECONDS = 5 * 60
+_SOURCE_SETTLE_SECONDS = 5 * 60
+_SOURCE_STABILITY_DELAY_SECONDS = 5
+_SOURCE_STABILITY_RETRY_SECONDS = 60
 _RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS = 30.0
 _WORK_ITEM_INSERT_BATCH_SIZE = 250
 _MAX_RECONCILIATION_CANDIDATES = 20_000
@@ -39,6 +43,8 @@ _RETRYABLE_RESULT_STATUSES = {
     "blocked_safe_no_write",
     "degraded_cache_refresh",
     "degraded_no_write",
+    "source_snapshot_settling",
+    "source_snapshot_unstable",
 }
 
 
@@ -50,6 +56,24 @@ def _canonical_json(value: Any) -> str:
     return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
 
 
+def _latest_discovery_age_seconds(
+    matrix: Mapping[str, Any],
+    *,
+    now: datetime | None = None,
+) -> float | None:
+    ended_at = str(_dict(matrix.get("latest_discovery_run")).get("ended_at") or "")
+    if not ended_at:
+        return None
+    try:
+        parsed = datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=UTC)
+    current = (now or datetime.now(UTC)).astimezone(UTC)
+    return max(0.0, (current - parsed.astimezone(UTC)).total_seconds())
+
+
 def build_reconciliation_summary_payload(
     matrix: Mapping[str, Any],
     candidates: Sequence[Mapping[str, Any]],
@@ -177,11 +201,13 @@ def build_reconciliation_work_item_input(
 
 
 async def run_asset_capability_reconciliation_loop() -> None:
-    """Reconcile shortly after startup and retry each discovery fingerprint."""
+    """Reconcile settled source snapshots and retry each fingerprint."""
 
     logger.info(
         "asset_capability_reconciliation_loop_started",
+        first_delay_seconds=_FIRST_DELAY_SECONDS,
         poll_interval_seconds=_RECONCILIATION_POLL_SECONDS,
+        source_settle_seconds=_SOURCE_SETTLE_SECONDS,
     )
     await asyncio.sleep(_FIRST_DELAY_SECONDS)
     triggered_by = "startup"
@@ -197,13 +223,20 @@ async def run_asset_capability_reconciliation_loop() -> None:
             triggered_by = "retry"
             continue
         if str(result.get("status") or "") in _RETRYABLE_RESULT_STATUSES:
+            retry_seconds = max(
+                30,
+                min(
+                    _LOOP_BACKOFF_SECONDS,
+                    int(result.get("retry_seconds") or _LOOP_BACKOFF_SECONDS),
+                ),
+            )
             logger.warning(
                 "asset_capability_reconciliation_retry_scheduled",
                 status=result.get("status"),
                 reason=result.get("reason"),
-                retry_seconds=_LOOP_BACKOFF_SECONDS,
+                retry_seconds=retry_seconds,
             )
-            await asyncio.sleep(_LOOP_BACKOFF_SECONDS)
+            await asyncio.sleep(retry_seconds)
             triggered_by = "retry"
             continue
         await asyncio.sleep(_RECONCILIATION_POLL_SECONDS)
@@ -235,6 +268,70 @@ async def reconcile_once(
             "candidate_count": 0,
         }
 
+    discovery_age_seconds = _latest_discovery_age_seconds(matrix)
+    if discovery_age_seconds is None or discovery_age_seconds < _SOURCE_SETTLE_SECONDS:
+        remaining_seconds = (
+            _SOURCE_STABILITY_RETRY_SECONDS
+            if discovery_age_seconds is None
+            else max(
+                _SOURCE_STABILITY_RETRY_SECONDS,
+                int(_SOURCE_SETTLE_SECONDS - discovery_age_seconds) + 1,
+            )
+        )
+        logger.info(
+            "asset_capability_reconciliation_source_snapshot_settling",
+            discovery_age_seconds=discovery_age_seconds,
+            required_age_seconds=_SOURCE_SETTLE_SECONDS,
+            retry_seconds=remaining_seconds,
+        )
+        return {
+            "status": "source_snapshot_settling",
+            "reason": "latest_discovery_run_not_settled",
+            "discovery_age_seconds": discovery_age_seconds,
+            "required_age_seconds": _SOURCE_SETTLE_SECONDS,
+            "retry_seconds": remaining_seconds,
+            "external_runtime_write_performed": False,
+        }
+
+    await asyncio.sleep(_SOURCE_STABILITY_DELAY_SECONDS)
+    stable_matrix = await build_asset_capability_matrix_with_live_readback(
+        project_id=project_id,
+        timeout_seconds=_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS,
+        include_live_only_assets=True,
+    )
+    if stable_matrix.get("live_readback_status") != "ready":
+        return {
+            "status": "degraded_no_write",
+            "reason": "stability_verifier_live_readback_degraded",
+            "live_readback_status": stable_matrix.get("live_readback_status"),
+            "candidate_count": 0,
+            "retry_seconds": _SOURCE_STABILITY_RETRY_SECONDS,
+            "external_runtime_write_performed": False,
+        }
+
+    first_identity = (
+        str(matrix.get("matrix_fingerprint") or ""),
+        str(_dict(matrix.get("latest_discovery_run")).get("run_id") or ""),
+    )
+    stable_identity = (
+        str(stable_matrix.get("matrix_fingerprint") or ""),
+        str(_dict(stable_matrix.get("latest_discovery_run")).get("run_id") or ""),
+    )
+    if not all(stable_identity) or stable_identity != first_identity:
+        logger.warning(
+            "asset_capability_reconciliation_source_snapshot_unstable",
+            discovery_run_changed=first_identity[1] != stable_identity[1],
+            matrix_fingerprint_changed=first_identity[0] != stable_identity[0],
+            retry_seconds=_SOURCE_STABILITY_RETRY_SECONDS,
+        )
+        return {
+            "status": "source_snapshot_unstable",
+            "reason": "matrix_changed_during_stability_verifier",
+            "retry_seconds": _SOURCE_STABILITY_RETRY_SECONDS,
+            "external_runtime_write_performed": False,
+        }
+    matrix = stable_matrix
+
     candidates = build_reconciliation_candidates(matrix)
     if len(candidates) > _MAX_RECONCILIATION_CANDIDATES:
         logger.error(

apps/api/src/jobs/asset_change_tracker_job.py

@@ -29,7 +29,6 @@ from __future__ import annotations
 import asyncio
 import json as _json
 import time as _time
-from typing import Any
 
 import structlog
 
@@ -38,6 +37,7 @@ logger = structlog.get_logger(__name__)
 _TRACK_INTERVAL_SEC = 3600
 _FIRST_DELAY_SEC = 360
 _LOOP_BACKOFF_SEC = 600
+_PROJECT_ID = "awoooi"
 
 
 async def run_asset_change_tracker_loop() -> None:
@@ -92,9 +92,10 @@ async def track_once() -> dict[str, int]:
 async def _get_recent_runs(limit: int = 2) -> list[str]:
     """取最近 N 個 success 的 run_id (降序)."""
     from sqlalchemy import text as _sql
+
     from src.db.base import get_db_context
 
-    async with get_db_context() as db:
+    async with get_db_context(_PROJECT_ID) as db:
         rows = await db.execute(
             _sql("SELECT run_id FROM asset_discovery_run WHERE status='success' ORDER BY ended_at DESC LIMIT :lim"),
             {"lim": limit},
@@ -113,11 +114,12 @@ async def _diff_runs(newer_run: str, older_run: str) -> dict[str, int]:
       - newer ∩ older 且 metadata 變 = modified
     """
     from sqlalchemy import text as _sql
+
     from src.db.base import get_db_context
 
     stats = {"added": 0, "removed": 0, "modified": 0}
 
-    async with get_db_context() as db:
+    async with get_db_context(_PROJECT_ID) as db:
         # 1. Added: newer run 有但 older run 沒有
         result = await db.execute(
             _sql("""
@@ -216,10 +218,11 @@ async def _diff_runs(newer_run: str, older_run: str) -> dict[str, int]:
 async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -> None:
     try:
         from sqlalchemy import text as _sql
+
         from src.db.base import get_db_context
 
         aol_status = "failed" if error else "success"
-        async with get_db_context() as db:
+        async with get_db_context(_PROJECT_ID) as db:
             await db.execute(
                 _sql("""
                     INSERT INTO automation_operation_log (

apps/api/src/jobs/asset_scanner_job.py

@@ -206,7 +206,7 @@ async def run_asset_scanner_loop() -> None:
 
     while True:
         try:
-            await scan_once(triggered_by="cron")
+            await scan_once(triggered_by="cron", raise_on_failure=True)
         except Exception as e:
             logger.exception("asset_scanner_loop_error", error=str(e))
             await asyncio.sleep(_LOOP_BACKOFF_SEC)
@@ -227,6 +227,7 @@ async def scan_once(
         "domain_tls_inventory",
     ),
     scan_depth: str = "shallow",
+    raise_on_failure: bool = False,
 ) -> str:
     """
     執行一次資產盤點。
@@ -235,6 +236,7 @@ async def scan_once(
         triggered_by: cron / ai / human / incident
         scope:        本次掃描範圍標籤 (寫入 asset_discovery_run.scope)
         scan_depth:   shallow (僅 list) / deep (含 describe) / full
+        raise_on_failure: durable terminal 寫入後拋錯，供排程進入 bounded retry
 
     Returns:
         run_id (UUID 字串)。無法建立 durable header 時會拋出例外。
@@ -324,6 +326,8 @@ async def scan_once(
         disappeared=disappeared_count,
         duration_ms=duration_ms,
     )
+    if error_msg and raise_on_failure:
+        raise RuntimeError(f"asset_scan_failed:{run_id}:{error_msg}")
     return run_id
 
 

apps/api/src/jobs/rule_catalog_sync_job.py

@@ -48,6 +48,7 @@ _SYNC_INTERVAL_SEC = 3600       # 每 1 小時
 _FIRST_DELAY_SEC = 90           # 啟動後等 90s (等 Prometheus 就緒)
 _HTTP_TIMEOUT_SEC = 10
 _LOOP_BACKOFF_SEC = 300
+_PROJECT_ID = "awoooi"
 
 _PROM_RULES_ENDPOINT = "/api/v1/rules"
 
@@ -65,11 +66,19 @@ async def run_rule_catalog_sync_loop() -> None:
 
     while True:
         try:
-            await sync_once()
+            stats = await sync_once()
         except Exception as e:
             logger.exception("rule_catalog_sync_loop_error", error=str(e))
             await asyncio.sleep(_LOOP_BACKOFF_SEC)
             continue
+        if stats.get("failed", 0) > 0:
+            logger.warning(
+                "rule_catalog_sync_retry_scheduled",
+                failed=stats["failed"],
+                retry_seconds=_LOOP_BACKOFF_SEC,
+            )
+            await asyncio.sleep(_LOOP_BACKOFF_SEC)
+            continue
         await asyncio.sleep(_SYNC_INTERVAL_SEC)
 
 
@@ -78,11 +87,11 @@ async def sync_once() -> dict[str, int]:
     執行一次 Prometheus → alert_rule_catalog 同步.
 
     Returns:
-        {"total": N, "new": M, "updated": K, "unchanged": L}
-        失敗時所有值為 0,error 會寫 aol.
+        {"total": N, "new": M, "updated": K, "unchanged": L, "failed": F}
+        失敗時 failed > 0，error 會寫 AOL 並由排程 bounded retry.
     """
     started_ms = _time.time()
-    stats = {"total": 0, "new": 0, "updated": 0, "unchanged": 0}
+    stats = {"total": 0, "new": 0, "updated": 0, "unchanged": 0, "failed": 0}
     error_msg: str | None = None
 
     try:
@@ -98,6 +107,8 @@ async def sync_once() -> dict[str, int]:
                 stats["unchanged"] += 1
     except Exception as e:
         error_msg = f"{type(e).__name__}: {e}"[:1000]
+        processed = stats["new"] + stats["updated"] + stats["unchanged"]
+        stats["failed"] = max(1, stats["total"] - processed)
         logger.exception("rule_catalog_sync_once_failed", error=error_msg)
 
     duration_ms = int((_time.time() - started_ms) * 1000)
@@ -159,7 +170,7 @@ async def _fetch_prometheus_rules() -> list[dict[str, Any]]:
 
 def _parse_duration(d: Any) -> int:
     """Prometheus 的 duration 可能是 秒 (int/float) 或 '5m' 字串."""
-    if isinstance(d, (int, float)):
+    if isinstance(d, int | float):
         return int(d)
     if isinstance(d, str) and d:
         try:
@@ -189,10 +200,11 @@ async def _upsert_rule(rule: dict[str, Any]) -> str:
     比較 expr/severity/labels/annotations,有變化 → updated,否則 unchanged.
     """
     from sqlalchemy import text as _sql
+
     from src.db.base import get_db_context
 
     try:
-        async with get_db_context() as db:
+        async with get_db_context(_PROJECT_ID) as db:
             row = await db.execute(
                 _sql("""
                     INSERT INTO alert_rule_catalog (
@@ -237,7 +249,7 @@ async def _upsert_rule(rule: dict[str, Any]) -> str:
             return "new" if inserted else "updated"
     except Exception as e:
         logger.warning("rule_upsert_failed", rule_name=rule["rule_name"], error=str(e))
-        return "unchanged"
+        raise
 
 
 async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -> None:
@@ -256,9 +268,10 @@ async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -
 
     try:
         from sqlalchemy import text as _sql
+
         from src.db.base import get_db_context
 
-        async with get_db_context() as db:
+        async with get_db_context(_PROJECT_ID) as db:
             await db.execute(
                 _sql("""
                     INSERT INTO automation_operation_log (