From 990a1d2e5f32f16ed713069b24a1e3bdf0a89e29 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 04:31:19 +0800 Subject: [PATCH 01/45] chore(cd): deploy 6c1a12e [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index b0691e6c8..c11b109e7 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "af4cb2fd432e0f1098a072034637de90c6b6964f" + value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "af4cb2fd432e0f1098a072034637de90c6b6964f" + value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 50953fc60..37cf1e3b2 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "af4cb2fd432e0f1098a072034637de90c6b6964f" + value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 90643e6c4..7e28b2537 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "af4cb2fd432e0f1098a072034637de90c6b6964f" + value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 0b67f7c82..27fb2f9b5 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:171844f7b67fa1be9f477611ced536638b349f4163c5747264cefe72c8761896 +- digest: sha256:949b5c7bc1848db40e8f1437da1957ecab4aca33a30e3e772a74e66e89e7692f name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:c5e0142481d0d91f5b2033300c8824d84a2b11c5f5a62aee03db267e9c76f132 +- digest: sha256:494956f27c6f37d72a14badcb99d1ee6c20c71455324cccf1a55f9b546c304f7 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 8f741ce1006c49452cc8be34055c53059bdeadbc Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 04:32:15 +0800 Subject: [PATCH 02/45] fix(runtime): preserve lifecycle closure receipts --- .../src/jobs/incident_lifecycle_reconciler.py | 56 +++++++++------- .../test_incident_lifecycle_reconciler.py | 65 +++++++++++++++++++ 2 files changed, 97 insertions(+), 24 deletions(-) diff --git a/apps/api/src/jobs/incident_lifecycle_reconciler.py b/apps/api/src/jobs/incident_lifecycle_reconciler.py index b85a31f59..e1861b9a1 100644 --- a/apps/api/src/jobs/incident_lifecycle_reconciler.py +++ b/apps/api/src/jobs/incident_lifecycle_reconciler.py @@ -8,9 +8,14 @@ Incident Lifecycle Reconciler - auto_repair_executions.success = true - approval_records.status = EXECUTION_SUCCESS - approval_records.status = EXPIRED +- Prometheus 已無 firing alertname 的歷史 incident +- 同一 firing alertname 只保留最新一筆，舊重複案依統一 lifecycle 路徑收旂 不處理單純 APPROVED / NO_ACTION / manual_required，避免把仍需人工的事件 誤當作自動修復完成。 + +所有結案都必須經過 ``IncidentService.resolve_incident`` 並留下 +durable timeline receipt；禁止只改 incidents.status 製造假性終局。 """ from __future__ import annotations @@ -25,7 +30,6 @@ from sqlalchemy import text from src.core.config import settings from src.core.context import clear_project_context, set_project_context from src.db.base import get_db_context -from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) @@ -40,7 +44,6 @@ class LifecycleCandidate: incident_id: str resolution_type: str reason: str - direct_db_only: bool = False async def run_incident_lifecycle_reconciler_loop() -> None: @@ -96,31 +99,49 @@ async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int] if not candidates: return 0, 0 + from src.services.approval_db import get_timeline_service from src.services.incident_service import get_incident_service incident_service = get_incident_service() + timeline_service = get_timeline_service() resolved = 0 errors = 0 for candidate in candidates: try: - if candidate.direct_db_only: - result = await _resolve_db_only(candidate.incident_id) - else: - result = await incident_service.resolve_incident( - candidate.incident_id, - resolution_type=candidate.resolution_type, - emit_postmortem=False, - ) + result = await incident_service.resolve_incident( + candidate.incident_id, + resolution_type=candidate.resolution_type, + emit_postmortem=False, + ) if not result: continue + timeline_receipt = await timeline_service.add_event( + event_type="lifecycle_reconciled", + status="success", + title="Incident lifecycle reconciled from durable source truth", + description=( + f"reason={candidate.reason}; " + f"resolution_type={candidate.resolution_type}; " + "postmortem=suppressed_batch_reconcile; " + "telegram=not_sent" + ), + actor="incident_lifecycle_reconciler", + actor_role="ai_agent", + risk_level="low", + incident_id=candidate.incident_id, + project_id=_PROJECT_ID, + ) + if not timeline_receipt.get("id"): + raise RuntimeError("lifecycle_timeline_receipt_missing") resolved += 1 logger.info( "incident_lifecycle_reconciled", incident_id=candidate.incident_id, reason=candidate.reason, resolution_type=candidate.resolution_type, - direct_db_only=candidate.direct_db_only, + timeline_receipt_id=str(timeline_receipt["id"]), + telegram_sent=False, ) except Exception as exc: errors += 1 @@ -161,18 +182,6 @@ async def _fetch_active_alertnames() -> set[str] | None: return active_alertnames -async def _resolve_db_only(incident_id: str) -> bool: - from src.repositories.incident_repository import get_incident_repository - - now = now_taipei() - return await get_incident_repository().update_status( - incident_id=incident_id, - status="resolved", - updated_at=now, - resolved_at=now, - ) - - async def _fetch_candidates(limit: int) -> list[LifecycleCandidate]: async with get_db_context() as db: result = await db.execute( @@ -297,7 +306,6 @@ async def _fetch_inactive_or_duplicate_alert_candidates( incident_id=str(row["incident_id"]), resolution_type="timeout", reason=str(row["reason"]), - direct_db_only=True, ) for row in rows ] diff --git a/apps/api/tests/test_incident_lifecycle_reconciler.py b/apps/api/tests/test_incident_lifecycle_reconciler.py index 342d932a7..2b839fb84 100644 --- a/apps/api/tests/test_incident_lifecycle_reconciler.py +++ b/apps/api/tests/test_incident_lifecycle_reconciler.py @@ -15,6 +15,9 @@ from src.jobs.incident_lifecycle_reconciler import ( @pytest.mark.asyncio async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch): service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object())) + timeline = SimpleNamespace( + add_event=AsyncMock(side_effect=[{"id": "tl-1"}, {"id": "tl-2"}]) + ) monkeypatch.setattr( "src.jobs.incident_lifecycle_reconciler._fetch_candidates", @@ -37,6 +40,10 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch): "src.services.incident_service.get_incident_service", lambda: service, ) + monkeypatch.setattr( + "src.services.approval_db.get_timeline_service", + lambda: timeline, + ) resolved, errors = await reconcile_stuck_incidents(limit=2) @@ -53,6 +60,64 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch): "resolution_type": "timeout", "emit_postmortem": False, } + assert timeline.add_event.await_count == 2 + assert len(timeline.add_event.await_args_list[0].kwargs["event_type"]) <= 20 + assert timeline.add_event.await_args_list[0].kwargs == { + "event_type": "lifecycle_reconciled", + "status": "success", + "title": "Incident lifecycle reconciled from durable source truth", + "description": ( + "reason=approval_execution_success; resolution_type=auto_repair; " + "postmortem=suppressed_batch_reconcile; telegram=not_sent" + ), + "actor": "incident_lifecycle_reconciler", + "actor_role": "ai_agent", + "risk_level": "low", + "incident_id": "INC-EXEC-SUCCESS", + "project_id": "awoooi", + } + + +@pytest.mark.asyncio +async def test_inactive_alert_uses_service_and_writes_timeline(monkeypatch): + service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object())) + timeline = SimpleNamespace(add_event=AsyncMock(return_value={"id": "tl-inactive"})) + candidate = LifecycleCandidate( + incident_id="INC-INACTIVE", + resolution_type="timeout", + reason="inactive_alert_stale", + ) + + monkeypatch.setattr( + "src.jobs.incident_lifecycle_reconciler._fetch_candidates", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + "src.jobs.incident_lifecycle_reconciler._fetch_active_alertnames", + AsyncMock(return_value=set()), + ) + monkeypatch.setattr( + "src.jobs.incident_lifecycle_reconciler._fetch_inactive_or_duplicate_alert_candidates", + AsyncMock(return_value=[candidate]), + ) + monkeypatch.setattr( + "src.services.incident_service.get_incident_service", + lambda: service, + ) + monkeypatch.setattr( + "src.services.approval_db.get_timeline_service", + lambda: timeline, + ) + + resolved, errors = await reconcile_stuck_incidents(limit=1) + + assert (resolved, errors) == (1, 0) + service.resolve_incident.assert_awaited_once_with( + "INC-INACTIVE", + resolution_type="timeout", + emit_postmortem=False, + ) + timeline.add_event.assert_awaited_once() @pytest.mark.asyncio From 72a414ed2f9cc3e927ced2edaf257b42f0a40982 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 04:46:00 +0800 Subject: [PATCH 03/45] chore(cd): deploy e4da657 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index c11b109e7..ffb570d5f 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" + value: "e4da6570a645e504cc93647d396f45f167b3172a" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" + value: "e4da6570a645e504cc93647d396f45f167b3172a" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 37cf1e3b2..fc010b5a4 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" + value: "e4da6570a645e504cc93647d396f45f167b3172a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 7e28b2537..ffb594524 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6c1a12e388f7aa75c386d59c4c136763ecb51d74" + value: "e4da6570a645e504cc93647d396f45f167b3172a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 27fb2f9b5..86efddcaf 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:949b5c7bc1848db40e8f1437da1957ecab4aca33a30e3e772a74e66e89e7692f +- digest: sha256:5c44c63306a02b9375c006060b25297a76fdee42b3b57770a977d1c2fceac5df name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:494956f27c6f37d72a14badcb99d1ee6c20c71455324cccf1a55f9b546c304f7 +- digest: sha256:4139a54b724e5696c8dfcb079bbf8b4d84baec8bc1bfab03105eeff9842e8bc1 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 35d36c985da4d414cb0510dd7e34a4ac0439de5c Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 04:47:12 +0800 Subject: [PATCH 04/45] fix(automation): recover verified backup backfill failures --- .../backup_restore_legacy_backfill_job.py | 100 ++++++++++++++++++ ...test_backup_restore_legacy_backfill_job.py | 50 +++++++++ 2 files changed, 150 insertions(+) diff --git a/apps/api/src/jobs/backup_restore_legacy_backfill_job.py b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py index ceeebf62f..453b4cf0d 100644 --- a/apps/api/src/jobs/backup_restore_legacy_backfill_job.py +++ b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py @@ -43,6 +43,7 @@ BACKFILL_RESULT_SCHEMA_VERSION = "backup_restore_legacy_backfill_result_v1" BACKFILL_CURSOR_AGENT_ID = "backup_restore_legacy_backfill_cursor" BACKFILL_ITEM_AGENT_ID = "backup_restore_legacy_backfill" BACKFILL_IDEMPOTENCY_CHANNEL = "backup_restore_backfill" +BACKFILL_DB_CONTRACT_REPAIR_GENERATION = "cost_usd_state_bind_v1" BACKUP_RESTORE_EVENT_TYPE = "backup_restore_escrow_signal" BACKUP_RESTORE_LANE = "backup_restore_escrow_triage" PROJECTION_SCHEMA_VERSION = "telegram_agent99_dispatch_receipt_projection_v1" @@ -582,6 +583,58 @@ _ITEM_STATE_COUNTS_SQL = text( """ ) +_REQUEUE_REPAIRED_DBAPI_FAILURES_SQL = text( + """ + WITH repairable AS ( + SELECT item.run_id + FROM awooop_run_state item + WHERE item.project_id = :project_id + AND item.agent_id = :agent_id + AND item.state = 'failed' + AND item.error_code = 'E-BACKFILL-EXHAUSTED' + AND COALESCE(NULLIF(item.error_detail, ''), '{}')::jsonb #>> + '{last_result,reason}' = 'processor_error:DBAPIError' + AND COALESCE(NULLIF(item.error_detail, ''), '{}')::jsonb #>> + '{item,runtime_execution_authorized}' = 'false' + AND COALESCE( + COALESCE(NULLIF(item.error_detail, ''), '{}')::jsonb #>> + '{item,recovery_generation}', + '' + ) <> :repair_generation + AND item.worker_id IS NULL + AND item.lease_until IS NULL + ORDER BY item.created_at ASC, item.run_id ASC + FOR UPDATE SKIP LOCKED + LIMIT :limit + ) + UPDATE awooop_run_state item + SET state = 'waiting_tool', + attempt_count = 0, + max_attempts = GREATEST(item.max_attempts, 5), + worker_id = NULL, + lease_until = NULL, + heartbeat_at = NOW(), + completed_at = NULL, + error_code = NULL, + error_detail = JSONB_SET( + JSONB_SET( + COALESCE(NULLIF(item.error_detail, ''), '{}')::jsonb, + '{item,recovery_generation}', + TO_JSONB(CAST(:repair_generation AS TEXT)), + TRUE + ), + '{item,recovery_reason}', + TO_JSONB(CAST('verified_database_contract_repair' AS TEXT)), + TRUE + ) + FROM repairable + WHERE item.project_id = :project_id + AND item.run_id = repairable.run_id + AND item.agent_id = :agent_id + RETURNING item.run_id +""" +) + @dataclass(frozen=True) class LegacyBackupClaim: @@ -1187,6 +1240,44 @@ async def _claim_legacy_backup_items( return claims +async def _requeue_repaired_dbapi_failures( + *, + project_id: str, + limit: int, +) -> int: + """Requeue one bounded historical DB-contract failure generation. + + The original attempts predate the explicit ``cost_usd`` insert and typed + state binds. Only those exact, no-runtime-write failures are eligible, + and the recovery generation is persisted inside the stable item envelope + so the same source row cannot be reset repeatedly. + """ + + bounded_limit = max(1, min(int(limit), MAX_SCOPE_ROWS)) + async with get_db_context(project_id) as db: + result = await db.execute( + _REQUEUE_REPAIRED_DBAPI_FAILURES_SQL, + { + "project_id": project_id, + "agent_id": BACKFILL_ITEM_AGENT_ID, + "repair_generation": BACKFILL_DB_CONTRACT_REPAIR_GENERATION, + "limit": bounded_limit, + }, + ) + rows = result.mappings().all() + requeued = len(rows) + if requeued: + logger.info( + "backup_restore_legacy_backfill_db_contract_failures_requeued", + project_id=project_id, + repair_generation=BACKFILL_DB_CONTRACT_REPAIR_GENERATION, + requeued=requeued, + runtime_execution_authorized=False, + telegram_dispatch_performed=False, + ) + return requeued + + async def _load_legacy_backup_card( *, project_id: str, @@ -1876,6 +1967,8 @@ async def run_backup_restore_legacy_backfill_once( "failed": 0, "retried": 0, "lease_lost": 0, + "requeued_repaired_dbapi": 0, + "backfill_state_write_performed": False, "dispatch_total": 0, "source_total_at_start": None, "source_exhausted": False, @@ -1892,6 +1985,13 @@ async def run_backup_restore_legacy_backfill_once( "error": None, } try: + if processor is process_backup_restore_alertmanager_signal: + requeued = await _requeue_repaired_dbapi_failures( + project_id=normalized_project, + limit=max_rows, + ) + stats["requeued_repaired_dbapi"] = requeued + stats["backfill_state_write_performed"] = requeued > 0 reserved = await _reserve_legacy_backup_page( project_id=normalized_project, scan_limit=scan_limit, diff --git a/apps/api/tests/test_backup_restore_legacy_backfill_job.py b/apps/api/tests/test_backup_restore_legacy_backfill_job.py index 911819187..178c1e321 100644 --- a/apps/api/tests/test_backup_restore_legacy_backfill_job.py +++ b/apps/api/tests/test_backup_restore_legacy_backfill_job.py @@ -482,6 +482,51 @@ async def test_claim_sweeps_expired_max_attempt_before_selecting_retryable_work( assert statements == [job._EXHAUST_EXPIRED_CLAIMS_SQL, job._CLAIM_SQL] +@pytest.mark.asyncio +async def test_db_contract_repair_requeues_only_one_bounded_generation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + returned = [ + {"run_id": _claim_from_row(index).run_id} + for index in range(3) + ] + + class FakeDb: + async def execute( + self, + statement: Any, + params: dict[str, Any], + ) -> _Result: + assert statement is job._REQUEUE_REPAIRED_DBAPI_FAILURES_SQL + assert params == { + "project_id": "awoooi", + "agent_id": job.BACKFILL_ITEM_AGENT_ID, + "repair_generation": job.BACKFILL_DB_CONTRACT_REPAIR_GENERATION, + "limit": job.MAX_SCOPE_ROWS, + } + return _Result(rows=returned) + + monkeypatch.setattr( + job, + "get_db_context", + lambda _project_id: _DbContext(FakeDb()), + ) + + requeued = await job._requeue_repaired_dbapi_failures( + project_id="awoooi", + limit=job.MAX_SCOPE_ROWS + 500, + ) + + assert requeued == 3 + sql = str(job._REQUEUE_REPAIRED_DBAPI_FAILURES_SQL) + assert "E-BACKFILL-EXHAUSTED" in sql + assert "processor_error:DBAPIError" in sql + assert "runtime_execution_authorized" in sql + assert "recovery_generation" in sql + assert "FOR UPDATE SKIP LOCKED" in sql + assert "attempt_count = 0" in sql + + @pytest.mark.asyncio @pytest.mark.parametrize( ("item_total", "completed_total", "expected_status", "expected_cursor_state"), @@ -1356,6 +1401,8 @@ async def test_default_processor_accepts_repo_known_lan_without_token( update_cursor = AsyncMock( return_value={"status": "completed", "remaining_item_total": 0} ) + requeue = AsyncMock(return_value=0) + monkeypatch.setattr(job, "_requeue_repaired_dbapi_failures", requeue) monkeypatch.setattr(job, "_reserve_legacy_backup_page", reserve) monkeypatch.setattr(job, "_claim_legacy_backup_items", claim) monkeypatch.setattr(job, "_update_cursor_counters", update_cursor) @@ -1371,6 +1418,9 @@ async def test_default_processor_accepts_repo_known_lan_without_token( assert result["status"] == "completed" assert result["auth_mode"] == "lan_allowlist" assert result["relay_preflight"]["ready"] is True + assert result["requeued_repaired_dbapi"] == 0 + assert result["backfill_state_write_performed"] is False + requeue.assert_awaited_once_with(project_id="awoooi", limit=100) reserve.assert_awaited_once() claim.assert_awaited_once() From 956b55d66c9916a36d3d68311053ce32f0d7094e Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 04:24:02 +0800 Subject: [PATCH 05/45] feat(security): stage VPA recommender image --- .../runtime-image-mirror-staging-policy.json | 13 +++++++++++ .../test_runtime_image_mirror_controller.py | 23 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/config/security/runtime-image-mirror-staging-policy.json b/config/security/runtime-image-mirror-staging-policy.json index e0c9d2370..38d3c8708 100644 --- a/config/security/runtime-image-mirror-staging-policy.json +++ b/config/security/runtime-image-mirror-staging-policy.json @@ -41,6 +41,19 @@ "name": "argocd-redis", "container": "redis" } + }, + { + "asset_id": "runtime-image:vpa-recommender-1.6.0-canary", + "source_index_digest": "sha256:4af365370c187e4eb60302e937b18a188774698b049a1bfdd77d43e51b30abec", + "platform_digest": "sha256:f7da718d281d5e880e966d0182ffb2160b82517c7bc3a38f8002b49e7a4bd0a1", + "source_config_digest": "sha256:05526683ba80f70887ef5622c4ee7d78e1e0482d2a9e2f0e29ad92c32cb73819", + "push_ref": "registry.wooo.work/awoooi/runtime-mirror/vpa-recommender:1.6.0-linux-amd64", + "workload": { + "kind": "deployment", + "namespace": "kube-system", + "name": "vpa-recommender", + "container": "recommender" + } } ] } diff --git a/scripts/security/tests/test_runtime_image_mirror_controller.py b/scripts/security/tests/test_runtime_image_mirror_controller.py index d0b3f742e..fac8a7b40 100644 --- a/scripts/security/tests/test_runtime_image_mirror_controller.py +++ b/scripts/security/tests/test_runtime_image_mirror_controller.py @@ -98,13 +98,14 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): self.assertEqual(policy.work_item_id, "AIA-P0-006-02B") self.assertEqual(policy.risk_level, "high") - self.assertEqual(len(policy.images), 2) + self.assertEqual(len(policy.images), 3) by_asset = {image.asset_id: image for image in policy.images} self.assertEqual( set(by_asset), { "runtime-image:argocd-v3.3.6-canary", "runtime-image:redis-8.2.3-canary", + "runtime-image:vpa-recommender-1.6.0-canary", }, ) argocd = by_asset["runtime-image:argocd-v3.3.6-canary"] @@ -130,6 +131,26 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): self.assertEqual(redis.workload.name, "argocd-redis") self.assertEqual(redis.workload.container, "redis") self.assertEqual(redis.workload.container_kind, "container") + + vpa = by_asset["runtime-image:vpa-recommender-1.6.0-canary"] + self.assertEqual( + vpa.source_index_digest, + "sha256:4af365370c187e4eb60302e937b18a188774698b049a1bfdd77d43e51b30abec", + ) + self.assertEqual( + vpa.platform_digest, + "sha256:f7da718d281d5e880e966d0182ffb2160b82517c7bc3a38f8002b49e7a4bd0a1", + ) + self.assertEqual( + vpa.source_config_digest, + "sha256:05526683ba80f70887ef5622c4ee7d78e1e0482d2a9e2f0e29ad92c32cb73819", + ) + self.assertEqual(vpa.runtime_repository, "vpa-recommender") + self.assertEqual(vpa.workload.kind, "deployment") + self.assertEqual(vpa.workload.namespace, "kube-system") + self.assertEqual(vpa.workload.name, "vpa-recommender") + self.assertEqual(vpa.workload.container, "recommender") + self.assertEqual(vpa.workload.container_kind, "container") self.assertNotIn("github.com", raw) self.assertNotIn("ghcr.io", raw) self.assertIn('"external_pull_allowed": false', raw) From 098bdd81fe5f1d2c4d649f9c92dfef35fcc57bd8 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 04:59:24 +0800 Subject: [PATCH 06/45] fix(automation): reconcile new backup alert cards --- .../backup_restore_legacy_backfill_job.py | 330 +++++++++++++++++- ...test_backup_restore_legacy_backfill_job.py | 103 ++++++ 2 files changed, 428 insertions(+), 5 deletions(-) diff --git a/apps/api/src/jobs/backup_restore_legacy_backfill_job.py b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py index 453b4cf0d..04060260c 100644 --- a/apps/api/src/jobs/backup_restore_legacy_backfill_job.py +++ b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py @@ -44,6 +44,9 @@ BACKFILL_CURSOR_AGENT_ID = "backup_restore_legacy_backfill_cursor" BACKFILL_ITEM_AGENT_ID = "backup_restore_legacy_backfill" BACKFILL_IDEMPOTENCY_CHANNEL = "backup_restore_backfill" BACKFILL_DB_CONTRACT_REPAIR_GENERATION = "cost_usd_state_bind_v1" +LIVE_RECONCILE_SCHEMA_VERSION = "backup_restore_outbound_reconciler_v1" +LIVE_RECONCILE_AGENT_ID = "backup_restore_outbound_reconciler" +LIVE_RECONCILE_IDEMPOTENCY_CHANNEL = "backup_restore_outbound_reconcile" BACKUP_RESTORE_EVENT_TYPE = "backup_restore_escrow_signal" BACKUP_RESTORE_LANE = "backup_restore_escrow_triage" PROJECTION_SCHEMA_VERSION = "telegram_agent99_dispatch_receipt_projection_v1" @@ -252,6 +255,98 @@ _SOURCE_SNAPSHOT_SQL = text( """ ) +_LIVE_SOURCE_SCAN_SQL = text( + """ + SELECT + m.message_id, + m.queued_at, + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{source_refs,fingerprints,0}' AS source_fingerprint, + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{ai_automation_alert_card,event_type}' AS event_type, + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{ai_automation_alert_card,lane}' AS lane, + COALESCE( + NULLIF( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{ai_automation_alert_card,target}', + '' + ), + 'backup_restore' + ) AS target + FROM awooop_outbound_message m + WHERE m.project_id = :project_id + AND m.channel_type = 'telegram' + AND COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{ai_automation_alert_card,event_type}' = :event_type + AND COALESCE( + NULLIF( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{ai_automation_alert_card,target}', + '' + ), + 'backup_restore' + ) = 'backup_restore' + AND ( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,schema_version}' + <> :projection_schema + OR COALESCE( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,receipt_persisted}', + '' + ) <> 'true' + OR COALESCE( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,accepted}', + '' + ) <> 'true' + OR COALESCE( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,inbox_triggered}', + '' + ) <> 'true' + OR NULLIF( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,run_id}', + '' + ) IS NULL + OR NULLIF( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,trace_id}', + '' + ) IS NULL + OR NULLIF( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,work_item_id}', + '' + ) IS NULL + OR COALESCE( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,kind}', + '' + ) <> 'backup_health' + OR COALESCE( + COALESCE(m.source_envelope, '{}'::jsonb) #>> + '{agent99_dispatch_receipt,suggested_mode}', + '' + ) <> 'BackupCheck' + ) + AND NOT EXISTS ( + SELECT 1 + FROM awooop_run_state item + WHERE item.project_id = m.project_id + AND item.agent_id IN (:backfill_agent_id, :live_agent_id) + AND item.trigger_ref IN ( + ('legacy-backup:' || CAST(m.message_id AS TEXT)), + ('backup-signal:' || CAST(m.message_id AS TEXT)) + ) + ) + ORDER BY m.queued_at ASC, m.message_id ASC + LIMIT :limit +""" +) + _ITEM_RUN_INSERT_SQL = text( """ INSERT INTO awooop_run_state ( @@ -260,7 +355,7 @@ _ITEM_RUN_INSERT_SQL = text( is_shadow, input_sha256, cost_usd, step_count, error_detail, timeout_at ) VALUES ( :run_id, :project_id, :agent_id, 'waiting_tool', - 0, 5, 'backfill_replay', :trigger_ref, + 0, 5, :trigger_type, :trigger_ref, FALSE, :input_sha256, 0.0000, 1, :error_detail, NOW() + INTERVAL '7 days' ) @@ -643,6 +738,7 @@ class LegacyBackupClaim: attempt_count: int max_attempts: int item: dict[str, Any] + agent_id: str = BACKFILL_ITEM_AGENT_ID @dataclass(frozen=True) @@ -1122,6 +1218,7 @@ async def _reserve_legacy_backup_page( "run_id": item_run_id, "project_id": project_id, "agent_id": BACKFILL_ITEM_AGENT_ID, + "trigger_type": "backfill_replay", "trigger_ref": (f"legacy-backup:{item['source_message_id']}")[:256], "input_sha256": _sha256(_stable_json(item)), "error_detail": _stable_json({"item": item}), @@ -1187,10 +1284,72 @@ async def _reserve_legacy_backup_page( } +async def _reserve_live_backup_cards( + *, + project_id: str, + limit: int, +) -> dict[str, int]: + """Reserve newly emitted incomplete backup cards without reopening history.""" + + bounded_limit = max(1, min(int(limit), MAX_BATCH_LIMIT)) + async with get_db_context(project_id) as db: + result = await db.execute( + _LIVE_SOURCE_SCAN_SQL, + { + "project_id": project_id, + "event_type": BACKUP_RESTORE_EVENT_TYPE, + "projection_schema": PROJECTION_SCHEMA_VERSION, + "backfill_agent_id": BACKFILL_ITEM_AGENT_ID, + "live_agent_id": LIVE_RECONCILE_AGENT_ID, + "limit": bounded_limit, + }, + ) + rows = [_row_mapping(row) for row in result.mappings().all()] + reserved = 0 + deduplicated = 0 + for row in rows: + item = _item_payload_from_row(project_id=project_id, row=row) + item["reconciliation_mode"] = "continuous_outbound" + item_run_id = UUID(str(item["backfill_run_id"])) + await db.execute( + _ITEM_RUN_INSERT_SQL, + { + "run_id": item_run_id, + "project_id": project_id, + "agent_id": LIVE_RECONCILE_AGENT_ID, + "trigger_type": "outbound_reconcile", + "trigger_ref": ( + f"backup-signal:{item['source_message_id']}" + )[:256], + "input_sha256": _sha256(_stable_json(item)), + "error_detail": _stable_json({"item": item}), + }, + ) + idempotency = await db.execute( + _ITEM_IDEMPOTENCY_INSERT_SQL, + { + "project_id": project_id, + "channel_type": LIVE_RECONCILE_IDEMPOTENCY_CHANNEL, + "provider_event_id": item["occurrence_id"], + "run_id": item_run_id, + }, + ) + if idempotency.scalar_one_or_none() is not None: + reserved += 1 + else: + deduplicated += 1 + return { + "scanned": len(rows), + "reserved": reserved, + "deduplicated": deduplicated, + } + + async def _claim_legacy_backup_items( *, project_id: str, limit: int, + agent_id: str = BACKFILL_ITEM_AGENT_ID, ) -> list[LegacyBackupClaim]: bounded_limit = max(1, min(int(limit), MAX_BATCH_LIMIT)) claim_token = str(uuid4()) @@ -1199,7 +1358,7 @@ async def _claim_legacy_backup_items( _EXHAUST_EXPIRED_CLAIMS_SQL, { "project_id": project_id, - "agent_id": BACKFILL_ITEM_AGENT_ID, + "agent_id": agent_id, }, ) exhausted_run_ids = exhausted_result.mappings().all() @@ -1214,7 +1373,7 @@ async def _claim_legacy_backup_items( _CLAIM_SQL, { "project_id": project_id, - "agent_id": BACKFILL_ITEM_AGENT_ID, + "agent_id": agent_id, "limit": bounded_limit, "claim_token": claim_token, "lease_seconds": CLAIM_LEASE_SECONDS, @@ -1235,6 +1394,7 @@ async def _claim_legacy_backup_items( attempt_count=int(mapping.get("attempt_count") or 0), max_attempts=int(mapping.get("max_attempts") or 0), item=dict(item), + agent_id=agent_id, ) ) return claims @@ -1463,6 +1623,14 @@ async def _persist_legacy_dispatch_projection( ) -> str: backfill_receipt = { "schema_version": BACKFILL_SCHEMA_VERSION, + "reconciliation_schema_version": ( + LIVE_RECONCILE_SCHEMA_VERSION + if claim.agent_id == LIVE_RECONCILE_AGENT_ID + else BACKFILL_SCHEMA_VERSION + ), + "reconciliation_mode": str( + claim.item.get("reconciliation_mode") or "legacy_snapshot" + ), "status": "dispatch_receipt_projected", "source_message_id": card.message_id, "source_fingerprint_hash": occurrence["source_fingerprint_hash"], @@ -1490,7 +1658,7 @@ async def _persist_legacy_dispatch_projection( "source_lane": str(claim.item.get("lane") or BACKUP_RESTORE_LANE), "source_target": str(claim.item.get("target") or "backup_restore"), "claim_run_id": claim.run_id, - "agent_id": BACKFILL_ITEM_AGENT_ID, + "agent_id": claim.agent_id, "claim_token": claim.claim_token, } async with get_db_context(project_id) as db: @@ -1575,7 +1743,7 @@ async def _finish_backfill_claim( "error_detail": _stable_json(envelope), "project_id": project_id, "run_id": claim.run_id, - "agent_id": BACKFILL_ITEM_AGENT_ID, + "agent_id": claim.agent_id, "claim_token": claim.claim_token, }, ) @@ -1737,6 +1905,11 @@ async def _replay_backfill_claim( occurrence=occurrence, relay_auth_mode=relay_auth_mode, ) + projection["source"] = ( + LIVE_RECONCILE_SCHEMA_VERSION + if claim.agent_id == LIVE_RECONCILE_AGENT_ID + else BACKFILL_SCHEMA_VERSION + ) projection_status = await _persist_legacy_dispatch_projection( project_id=project_id, claim=claim, @@ -2064,6 +2237,151 @@ async def run_backup_restore_legacy_backfill_once( return stats +async def _live_reconcile_state_counts(*, project_id: str) -> dict[str, int]: + async with get_db_context(project_id) as db: + result = await db.execute( + _ITEM_STATE_COUNTS_SQL, + { + "project_id": project_id, + "agent_id": LIVE_RECONCILE_AGENT_ID, + }, + ) + row = result.mappings().one_or_none() + mapping = _row_mapping(row) if row is not None else {} + return { + "item_total": int(mapping.get("item_total") or 0), + "remaining_total": int(mapping.get("remaining_total") or 0), + "completed_total": int(mapping.get("completed_total") or 0), + "failed_total": int(mapping.get("failed_total") or 0), + } + + +async def run_backup_restore_outbound_reconciler_once( + *, + project_id: str = "awoooi", + limit: int = MAX_BATCH_LIMIT, + processor: Processor = process_backup_restore_alertmanager_signal, +) -> dict[str, Any]: + """Reconcile newly emitted backup cards in bounded continuous batches.""" + + normalized_project = str(project_id or "").strip() + if not normalized_project: + return { + "schema_version": LIVE_RECONCILE_SCHEMA_VERSION, + "status": "project_context_missing_fail_closed", + "runtime_execution_authorized": False, + "telegram_dispatch_performed": False, + } + relay_auth_mode = "injected_processor" + relay_preflight: dict[str, Any] | None = None + if processor is process_backup_restore_alertmanager_signal: + relay_preflight = build_backup_restore_backfill_relay_preflight() + if relay_preflight["ready"] is not True: + return { + "schema_version": LIVE_RECONCILE_SCHEMA_VERSION, + "status": "relay_preflight_failed_fail_closed", + "relay_preflight": relay_preflight, + "runtime_execution_authorized": False, + "telegram_dispatch_performed": False, + } + relay_auth_mode = str(relay_preflight["auth_mode"]) + stats: dict[str, Any] = { + "schema_version": LIVE_RECONCILE_SCHEMA_VERSION, + "project_id": normalized_project, + "status": "in_progress", + "auth_mode": relay_auth_mode, + "scanned": 0, + "reserved": 0, + "claimed": 0, + "completed": 0, + "deduplicated": 0, + "failed": 0, + "retried": 0, + "lease_lost": 0, + "dispatch_total": 0, + "runtime_execution_authorized": False, + "telegram_dispatch_performed": False, + "production_write_performed": False, + "prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS), + "error": None, + } + try: + reserved = await _reserve_live_backup_cards( + project_id=normalized_project, + limit=limit, + ) + stats.update(reserved) + claims = await _claim_legacy_backup_items( + project_id=normalized_project, + limit=limit, + agent_id=LIVE_RECONCILE_AGENT_ID, + ) + stats["claimed"] = len(claims) + for claim in claims: + outcome = await _replay_backfill_claim( + project_id=normalized_project, + claim=claim, + processor=processor, + relay_auth_mode=relay_auth_mode, + ) + if outcome == "completed": + stats["completed"] += 1 + stats["dispatch_total"] += 1 + elif outcome == "deduplicated": + stats["deduplicated"] += 1 + elif outcome == "retried": + stats["retried"] += 1 + elif outcome == "lease_lost": + stats["lease_lost"] += 1 + else: + stats["failed"] += 1 + counts = await _live_reconcile_state_counts(project_id=normalized_project) + stats.update(counts) + if counts["failed_total"] > 0: + stats["status"] = "blocked_failed_items_fail_closed" + elif counts["remaining_total"] > 0: + stats["status"] = "in_progress" + elif stats["scanned"] or stats["claimed"]: + stats["status"] = "reconciled" + else: + stats["status"] = "idle" + except Exception as exc: + stats["status"] = "database_or_worker_failure_fail_closed" + stats["error"] = type(exc).__name__ + stats["error_sqlstate"] = _database_sqlstate(exc) + logger.warning( + "backup_restore_outbound_reconciler_tick_failed", + project_id=normalized_project, + error_type=type(exc).__name__, + error_sqlstate=stats["error_sqlstate"], + runtime_execution_authorized=False, + telegram_dispatch_performed=False, + ) + logger.info("backup_restore_outbound_reconciler_tick", **stats) + return stats + + +async def run_backup_restore_outbound_reconciler_loop() -> None: + """Continuously reconcile new backup cards after the legacy cohort.""" + + while True: + sleep_seconds = float(settings.BACKUP_RESTORE_LEGACY_BACKFILL_INTERVAL_SECONDS) + try: + await run_backup_restore_outbound_reconciler_once( + project_id="awoooi", + limit=settings.BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT, + ) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning( + "backup_restore_outbound_reconciler_loop_failed", + error_type=type(exc).__name__, + ) + sleep_seconds = max(sleep_seconds, 60.0) + await asyncio.sleep(sleep_seconds) + + async def run_backup_restore_legacy_backfill_loop() -> None: """Drain the historical backlog in bounded, restart-safe batches.""" @@ -2103,6 +2421,7 @@ async def run_backup_restore_legacy_backfill_loop() -> None: source_total_at_start=result.get("source_total_at_start"), telegram_dispatch_performed=False, ) + await run_backup_restore_outbound_reconciler_loop() return if result.get("status") in { "blocked_scope_cap", @@ -2115,6 +2434,7 @@ async def run_backup_restore_legacy_backfill_loop() -> None: status=result.get("status"), telegram_dispatch_performed=False, ) + await run_backup_restore_outbound_reconciler_loop() return if result.get("error"): sleep_seconds = max(sleep_seconds, 60.0) diff --git a/apps/api/tests/test_backup_restore_legacy_backfill_job.py b/apps/api/tests/test_backup_restore_legacy_backfill_job.py index 178c1e321..88aee8c88 100644 --- a/apps/api/tests/test_backup_restore_legacy_backfill_job.py +++ b/apps/api/tests/test_backup_restore_legacy_backfill_job.py @@ -527,6 +527,102 @@ async def test_db_contract_repair_requeues_only_one_bounded_generation( assert "attempt_count = 0" in sql +@pytest.mark.asyncio +async def test_live_reconciler_reserves_only_unowned_incomplete_cards( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rows = [_source_row(201), _source_row(202)] + inserted_agents: list[str] = [] + trigger_refs: list[str] = [] + + class FakeDb: + async def execute( + self, + statement: Any, + params: dict[str, Any], + ) -> _Result: + if statement is job._LIVE_SOURCE_SCAN_SQL: + assert params["project_id"] == "awoooi" + assert params["backfill_agent_id"] == job.BACKFILL_ITEM_AGENT_ID + assert params["live_agent_id"] == job.LIVE_RECONCILE_AGENT_ID + assert params["limit"] == job.MAX_BATCH_LIMIT + return _Result(rows=rows) + if statement is job._ITEM_RUN_INSERT_SQL: + inserted_agents.append(str(params["agent_id"])) + trigger_refs.append(str(params["trigger_ref"])) + return _Result() + if statement is job._ITEM_IDEMPOTENCY_INSERT_SQL: + assert params["channel_type"] == job.LIVE_RECONCILE_IDEMPOTENCY_CHANNEL + return _Result(value=params["run_id"]) + raise AssertionError(f"unexpected statement: {statement}") + + monkeypatch.setattr( + job, + "get_db_context", + lambda _project_id: _DbContext(FakeDb()), + ) + + result = await job._reserve_live_backup_cards( + project_id="awoooi", + limit=job.MAX_BATCH_LIMIT + 10, + ) + + assert result == {"scanned": 2, "reserved": 2, "deduplicated": 0} + assert inserted_agents == [job.LIVE_RECONCILE_AGENT_ID] * 2 + assert all(value.startswith("backup-signal:") for value in trigger_refs) + sql = str(job._LIVE_SOURCE_SCAN_SQL) + assert "NOT EXISTS" in sql + assert "legacy-backup:" in sql + assert "backup-signal:" in sql + + +@pytest.mark.asyncio +async def test_live_reconciler_projects_bounded_agent99_receipt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + claim = job.LegacyBackupClaim( + **{ + **_claim_from_row(203).__dict__, + "agent_id": job.LIVE_RECONCILE_AGENT_ID, + } + ) + reserve = AsyncMock( + return_value={"scanned": 1, "reserved": 1, "deduplicated": 0} + ) + claims = AsyncMock(return_value=[claim]) + replay = AsyncMock(return_value="completed") + counts = AsyncMock( + return_value={ + "item_total": 1, + "remaining_total": 0, + "completed_total": 1, + "failed_total": 0, + } + ) + processor = AsyncMock() + monkeypatch.setattr(job, "_reserve_live_backup_cards", reserve) + monkeypatch.setattr(job, "_claim_legacy_backup_items", claims) + monkeypatch.setattr(job, "_replay_backfill_claim", replay) + monkeypatch.setattr(job, "_live_reconcile_state_counts", counts) + + result = await job.run_backup_restore_outbound_reconciler_once( + project_id="awoooi", + limit=7, + processor=processor, + ) + + assert result["status"] == "reconciled" + assert result["dispatch_total"] == 1 + assert result["runtime_execution_authorized"] is False + assert result["telegram_dispatch_performed"] is False + claims.assert_awaited_once_with( + project_id="awoooi", + limit=7, + agent_id=job.LIVE_RECONCILE_AGENT_ID, + ) + replay.assert_awaited_once() + + @pytest.mark.asyncio @pytest.mark.parametrize( ("item_total", "completed_total", "expected_status", "expected_cursor_state"), @@ -1437,6 +1533,7 @@ async def test_one_time_loop_exits_after_durable_snapshot_completes( } ) sleep = AsyncMock() + live_loop = AsyncMock() monkeypatch.setattr(job.settings, "ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL", True) monkeypatch.setattr( job.settings, @@ -1450,11 +1547,17 @@ async def test_one_time_loop_exits_after_durable_snapshot_completes( 0.0, ) monkeypatch.setattr(job, "run_backup_restore_legacy_backfill_once", tick) + monkeypatch.setattr( + job, + "run_backup_restore_outbound_reconciler_loop", + live_loop, + ) monkeypatch.setattr(job.asyncio, "sleep", sleep) await job.run_backup_restore_legacy_backfill_loop() tick.assert_awaited_once() + live_loop.assert_awaited_once() sleep.assert_awaited_once_with(0.0) From 096f61ed69987636b77f1dfe307171c1bed052e2 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 05:03:57 +0800 Subject: [PATCH 07/45] fix(agent99): rotate outcome reconciliation cohorts --- .../agent99_controlled_dispatch_ledger.py | 25 ++++++++++- .../test_agent99_controlled_dispatch_p1.py | 43 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/apps/api/src/services/agent99_controlled_dispatch_ledger.py b/apps/api/src/services/agent99_controlled_dispatch_ledger.py index f345d5e37..16c7f7dae 100644 --- a/apps/api/src/services/agent99_controlled_dispatch_ledger.py +++ b/apps/api/src/services/agent99_controlled_dispatch_ledger.py @@ -2215,14 +2215,37 @@ class PostgresAgent99DispatchLedger: ]), ) # Prioritize observed delivery/outcome lanes; pending crash - # recovery candidates are polled after them. + # recovery candidates are polled after them. Heartbeat is + # also the durable poll cursor: each selected cohort is + # touched below so unresolved old runs cannot starve newer + # outcomes forever. .order_by( (AwoooPRunState.state == "pending").asc(), + AwoooPRunState.heartbeat_at.asc().nullsfirst(), AwoooPRunState.created_at.asc(), ) .limit(max(1, min(int(limit), 100))) ) rows = result.all() + if rows: + await db.execute( + update(AwoooPRunState) + .where( + AwoooPRunState.project_id + == (project_id or "awoooi"), + AwoooPRunState.agent_id + == AGENT99_DISPATCH_AGENT_ID, + AwoooPRunState.run_id.in_([ + row.run_id for row in rows + ]), + AwoooPRunState.state.in_([ + "pending", + "running", + "waiting_tool", + ]), + ) + .values(heartbeat_at=_utc_now_naive()) + ) except Exception as exc: logger.warning( "agent99_dispatch_reconciliation_list_failed", diff --git a/apps/api/tests/test_agent99_controlled_dispatch_p1.py b/apps/api/tests/test_agent99_controlled_dispatch_p1.py index f7bfc4b73..eeea4cd7d 100644 --- a/apps/api/tests/test_agent99_controlled_dispatch_p1.py +++ b/apps/api/tests/test_agent99_controlled_dispatch_p1.py @@ -148,6 +148,49 @@ class _Context: return False +@pytest.mark.asyncio +async def test_reconciliation_poll_cohort_rotates_by_durable_heartbeat( + monkeypatch, +) -> None: + identity = _identity() + statements = [] + + class RowsResult: + def all(self): + return [ + SimpleNamespace( + run_id=identity.run_id, + state="waiting_tool", + error_detail=json.dumps({"identity": identity.public_dict()}), + ) + ] + + class ReconcileDb: + async def execute(self, statement): + statements.append(statement) + return RowsResult() if len(statements) == 1 else _ScalarResult() + + monkeypatch.setattr( + ledger_module, + "get_db_context", + lambda _project_id: _Context(ReconcileDb()), + ) + + items = await PostgresAgent99DispatchLedger().list_reconcilable( + project_id="awoooi", + limit=20, + ) + + assert len(items) == 1 + assert items[0]["identity"] == identity + assert len(statements) == 2 + select_sql = str(statements[0]) + update_sql = str(statements[1]) + assert "awooop_run_state.heartbeat_at ASC NULLS FIRST" in select_sql + assert "UPDATE awooop_run_state" in update_sql + assert "heartbeat_at" in update_sql + + @pytest.mark.asyncio async def test_reservation_claim_tokens_are_unique_and_running_is_reconcile_only( monkeypatch, From 323e8cc8e0a85eced6d55851765feaef607117c9 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 05:06:51 +0800 Subject: [PATCH 08/45] chore(cd): deploy 956b55d [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index ffb570d5f..dd1b790d6 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "e4da6570a645e504cc93647d396f45f167b3172a" + value: "956b55d66c9916a36d3d68311053ce32f0d7094e" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "e4da6570a645e504cc93647d396f45f167b3172a" + value: "956b55d66c9916a36d3d68311053ce32f0d7094e" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index fc010b5a4..209417ee9 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "e4da6570a645e504cc93647d396f45f167b3172a" + value: "956b55d66c9916a36d3d68311053ce32f0d7094e" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index ffb594524..1341ee393 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "e4da6570a645e504cc93647d396f45f167b3172a" + value: "956b55d66c9916a36d3d68311053ce32f0d7094e" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 86efddcaf..9c044cf89 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:5c44c63306a02b9375c006060b25297a76fdee42b3b57770a977d1c2fceac5df +- digest: sha256:9ab7979e8052f1361f9852114c3af1e73527875f14206de3bfe09b5dac143f13 name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:4139a54b724e5696c8dfcb079bbf8b4d84baec8bc1bfab03105eeff9842e8bc1 +- digest: sha256:6ae01c8a8202b84c060cdcaf2cc10626951950b1c79069a0063d4f1e02c29e0a name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 228d474534fad5202b7b7be1b5f8e2b60ba45e9a Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 01:23:37 +0800 Subject: [PATCH 09/45] fix(api): index asset capability projection --- .../asset_capability_reconciliation_job.py | 3 +- .../ai_automation_asset_capability_matrix.py | 237 +++++++++++------- ...t_ai_automation_asset_capability_matrix.py | 121 +++++++++ 3 files changed, 276 insertions(+), 85 deletions(-) diff --git a/apps/api/src/jobs/asset_capability_reconciliation_job.py b/apps/api/src/jobs/asset_capability_reconciliation_job.py index 5b2ecbac7..33625980f 100644 --- a/apps/api/src/jobs/asset_capability_reconciliation_job.py +++ b/apps/api/src/jobs/asset_capability_reconciliation_job.py @@ -216,7 +216,8 @@ async def reconcile_once( started = time.monotonic() matrix = await build_asset_capability_matrix_with_live_readback( - project_id=project_id + project_id=project_id, + include_live_only_assets=True, ) if matrix.get("live_readback_status") != "ready": logger.warning( diff --git a/apps/api/src/services/ai_automation_asset_capability_matrix.py b/apps/api/src/services/ai_automation_asset_capability_matrix.py index 6df7a1934..dda2f292b 100644 --- a/apps/api/src/services/ai_automation_asset_capability_matrix.py +++ b/apps/api/src/services/ai_automation_asset_capability_matrix.py @@ -12,7 +12,7 @@ import asyncio import hashlib import json import re -from collections import Counter +from collections import Counter, defaultdict from collections.abc import Iterable, Mapping, Sequence from datetime import UTC, datetime from pathlib import Path @@ -31,6 +31,7 @@ RECONCILIATION_WORK_ITEM_OPERATION = "asset_capability_reconciliation_work_item_ FRESHNESS_SLO_SECONDS = 2 * 60 * 60 RECONCILIATION_FRESHNESS_SLO_SECONDS = 36 * 60 * 60 LIVE_READBACK_TIMEOUT_SECONDS = 8.0 +LIVE_READBACK_DB_STATEMENT_TIMEOUT_MILLISECONDS = 6_000 STAGE_IDS = ( "source_truth", @@ -354,25 +355,67 @@ def build_declared_asset_catalog( return catalog -def _matching_score(declared: Mapping[str, Any], live: Mapping[str, Any]) -> int: - declared_values = ( - declared.get("declared_asset_id"), - declared.get("display_name"), - declared.get("canonical_id"), +def _identity_components(values: Iterable[Any]) -> tuple[set[str], set[str]]: + present_values = [value for value in values if value] + slugs = {_slug(value) for value in present_values} + tokens = set().union(*(_identity_tokens(value) for value in present_values)) + return slugs, tokens + + +def _build_live_identity_indexes( + live_rows: Sequence[Mapping[str, Any]], +) -> tuple[dict[str, list[int]], dict[str, list[int]]]: + slug_index: defaultdict[str, list[int]] = defaultdict(list) + token_index: defaultdict[str, list[int]] = defaultdict(list) + for index, live in enumerate(live_rows): + slugs, tokens = _identity_components( + (live.get("name"), live.get("asset_key"), live.get("source_repo")) + ) + for slug in slugs: + slug_index[slug].append(index) + for token in tokens: + token_index[token].append(index) + return dict(slug_index), dict(token_index) + + +def _best_live_match_index( + declared: Mapping[str, Any], + *, + slug_index: Mapping[str, Sequence[int]], + token_index: Mapping[str, Sequence[int]], + used_live_indexes: set[int], +) -> int | None: + declared_slugs, declared_tokens = _identity_components( + ( + declared.get("declared_asset_id"), + declared.get("display_name"), + declared.get("canonical_id"), + ) ) - live_values = (live.get("name"), live.get("asset_key"), live.get("source_repo")) - declared_slugs = {_slug(value) for value in declared_values if value} - live_slugs = {_slug(value) for value in live_values if value} - if declared_slugs & live_slugs: - return 100 - declared_tokens = set().union( - *(_identity_tokens(value) for value in declared_values) + exact_candidates = { + index + for slug in declared_slugs + for index in slug_index.get(slug, ()) + if index not in used_live_indexes + } + if exact_candidates: + return min(exact_candidates) + + overlap_counts: Counter[int] = Counter() + for token in declared_tokens: + overlap_counts.update( + index + for index in token_index.get(token, ()) + if index not in used_live_indexes + ) + if not overlap_counts: + return None + + # Scores cap at six overlapping tokens; original live order breaks ties. + return min( + overlap_counts, + key=lambda index: (-min(overlap_counts[index], 6), index), ) - live_tokens = set().union(*(_identity_tokens(value) for value in live_values)) - overlap = declared_tokens & live_tokens - if overlap: - return min(80, 20 + (len(overlap) * 10)) - return 0 def _is_fresh(value: Any, *, now: datetime, slo_seconds: int) -> bool: @@ -601,6 +644,21 @@ def _receipt_controls( } +def _matrix_fingerprint_row(asset: Mapping[str, Any]) -> dict[str, Any]: + return { + "canonical_id": asset["canonical_id"], + "declared": asset["declared"], + "runtime_identity": _dict(asset.get("runtime_identity")).get( + "asset_key_fingerprint" + ), + "stages": { + stage_id: _dict(_dict(asset.get("stages")).get(stage_id)).get("status") + for stage_id in STAGE_IDS + }, + "change_types": asset.get("change_types"), + } + + def build_asset_capability_matrix( scope_classes: Sequence[Mapping[str, Any]], *, @@ -610,6 +668,7 @@ def build_asset_capability_matrix( repo_root: Path | None = None, generated_at: datetime | None = None, live_readback_status: str = "ready", + include_live_only_assets: bool = True, ) -> dict[str, Any]: """Build a public-safe per-asset matrix from declared and live truth.""" @@ -626,19 +685,20 @@ def build_asset_capability_matrix( now=now, slo_seconds=FRESHNESS_SLO_SECONDS, ) - used_live_ids: set[int] = set() + slug_index, token_index = _build_live_identity_indexes(live_rows) + used_live_indexes: set[int] = set() assets: list[dict[str, Any]] = [] for declared in declared_assets: - candidates = [ - (live, _matching_score(declared, live)) - for live in live_rows - if int(live.get("asset_id") or 0) not in used_live_ids - ] - live, score = max(candidates, key=lambda item: item[1], default=(None, 0)) - matched_live = live if score >= 30 else None - if matched_live is not None: - used_live_ids.add(int(matched_live.get("asset_id") or 0)) + matched_index = _best_live_match_index( + declared, + slug_index=slug_index, + token_index=token_index, + used_live_indexes=used_live_indexes, + ) + matched_live = live_rows[matched_index] if matched_index is not None else None + if matched_index is not None: + used_live_indexes.add(matched_index) stages = _build_stages( declared, matched_live, @@ -669,52 +729,41 @@ def build_asset_capability_matrix( } ) - for live in live_rows: - live_id = int(live.get("asset_id") or 0) - if live_id in used_live_ids: - continue + fingerprint_rows = [_matrix_fingerprint_row(asset) for asset in assets] + unmatched_live_indexes = [ + index for index in range(len(live_rows)) if index not in used_live_indexes + ] + for live_index in unmatched_live_indexes: + live = live_rows[live_index] discovered = _live_only_asset(live, scope_labels) stages = _build_stages(discovered, live, now=now, run_fresh=run_fresh) - assets.append( - { - **discovered, - "runtime_identity": _public_runtime_identity( - str(discovered["canonical_id"]), - live, - ), - "stages": stages, - "missing_stage_ids": [ - stage_id - for stage_id, stage in stages.items() - if stage.get("status") == "missing" - ], - "stale_stage_ids": [ - stage_id - for stage_id, stage in stages.items() - if stage.get("status") == "stale" - ], - "change_types": list( - dict.fromkeys(["asset_added", *_strings(live.get("change_types"))]) - ), - } - ) + live_only_asset = { + **discovered, + "runtime_identity": _public_runtime_identity( + str(discovered["canonical_id"]), + live, + ), + "stages": stages, + "missing_stage_ids": [ + stage_id + for stage_id, stage in stages.items() + if stage.get("status") == "missing" + ], + "stale_stage_ids": [ + stage_id + for stage_id, stage in stages.items() + if stage.get("status") == "stale" + ], + "change_types": list( + dict.fromkeys(["asset_added", *_strings(live.get("change_types"))]) + ), + } + fingerprint_rows.append(_matrix_fingerprint_row(live_only_asset)) + if include_live_only_assets: + assets.append(live_only_asset) assets.sort(key=lambda row: (str(row["scope_id"]), str(row["canonical_id"]))) - fingerprint_rows = [ - { - "canonical_id": asset["canonical_id"], - "declared": asset["declared"], - "runtime_identity": _dict(asset.get("runtime_identity")).get( - "asset_key_fingerprint" - ), - "stages": { - stage_id: _dict(_dict(asset.get("stages")).get(stage_id)).get("status") - for stage_id in STAGE_IDS - }, - "change_types": asset.get("change_types"), - } - for asset in assets - ] + fingerprint_rows.sort(key=lambda row: str(row["canonical_id"])) matrix_fingerprint = _fingerprint(fingerprint_rows) canonical_ids = [str(asset.get("canonical_id") or "") for asset in assets] expected_scope_counts = { @@ -819,6 +868,12 @@ def build_asset_capability_matrix( "declared_asset_count": len(declared_assets), "live_inventory_asset_count": len(live_rows), "matrix_asset_count": len(assets), + "reconciled_asset_count": len(declared_assets) + + len(unmatched_live_indexes), + "live_only_assets_embedded": include_live_only_assets, + "embedded_live_only_asset_count": ( + len(unmatched_live_indexes) if include_live_only_assets else 0 + ), "live_observed_asset_count": live_observed_count, "declared_only_asset_count": len(declared_assets) - sum( @@ -828,9 +883,7 @@ def build_asset_capability_matrix( and _dict(asset.get("runtime_identity")).get("observation_status") == "live_observed" ), - "discovered_not_declared_count": sum( - 1 for asset in assets if asset.get("declared") is False - ), + "discovered_not_declared_count": len(unmatched_live_indexes), "gap_asset_count": len(gap_assets), "covered_stage_cell_count": stage_counts["covered"], "missing_stage_cell_count": stage_counts["missing"], @@ -972,6 +1025,14 @@ async def _load_live_rows( from src.db.base import get_db_context async with get_db_context(project_id) as db: + await db.execute( + text("SELECT set_config('statement_timeout', :statement_timeout, true)"), + { + "statement_timeout": ( + f"{LIVE_READBACK_DB_STATEMENT_TIMEOUT_MILLISECONDS}ms" + ) + }, + ) latest_result = await db.execute( text( """ @@ -1118,8 +1179,9 @@ async def build_asset_capability_matrix_with_live_readback( project_id: str = DEFAULT_PROJECT_ID, repo_root: Path | None = None, timeout_seconds: float = LIVE_READBACK_TIMEOUT_SECONDS, + include_live_only_assets: bool = False, ) -> dict[str, Any]: - """Build the matrix from live DB truth, failing closed to declared scope.""" + """Build a bounded matrix from live DB truth, failing closed to declared scope.""" from src.services.awoooi_priority_work_order_readback import ( get_ai_automation_program_scope_classes, @@ -1127,9 +1189,23 @@ async def build_asset_capability_matrix_with_live_readback( scope_classes = get_ai_automation_program_scope_classes() bounded_timeout_seconds = max(0.001, float(timeout_seconds)) + + async def _load_and_build() -> dict[str, Any]: + latest_run, live_assets, receipt = await _load_live_rows(project_id) + return await asyncio.to_thread( + build_asset_capability_matrix, + scope_classes, + live_assets=live_assets, + latest_run=latest_run, + reconciliation_receipt=receipt, + repo_root=repo_root, + live_readback_status="ready", + include_live_only_assets=include_live_only_assets, + ) + try: - latest_run, live_assets, receipt = await asyncio.wait_for( - _load_live_rows(project_id), + payload = await asyncio.wait_for( + _load_and_build(), timeout=bounded_timeout_seconds, ) except Exception as exc: @@ -1142,17 +1218,10 @@ async def build_asset_capability_matrix_with_live_readback( scope_classes, repo_root=repo_root, live_readback_status="degraded", + include_live_only_assets=include_live_only_assets, ) payload["live_readback_error_type"] = type(exc).__name__ payload["live_readback_timeout_seconds"] = bounded_timeout_seconds return payload - payload = build_asset_capability_matrix( - scope_classes, - live_assets=live_assets, - latest_run=latest_run, - reconciliation_receipt=receipt, - repo_root=repo_root, - live_readback_status="ready", - ) payload["live_readback_timeout_seconds"] = bounded_timeout_seconds return payload diff --git a/apps/api/tests/test_ai_automation_asset_capability_matrix.py b/apps/api/tests/test_ai_automation_asset_capability_matrix.py index cdb6acbbe..542701c8c 100644 --- a/apps/api/tests/test_ai_automation_asset_capability_matrix.py +++ b/apps/api/tests/test_ai_automation_asset_capability_matrix.py @@ -102,6 +102,93 @@ def test_declared_catalog_expands_to_193_unique_assets() -> None: assert matrix["operation_boundaries"]["raw_asset_key_exposed"] is False +def test_public_projection_counts_live_only_assets_without_embedding_them() -> None: + now = datetime(2026, 7, 15, 1, 0, tzinfo=UTC) + live_assets = [ + { + "asset_id": index + 1, + "asset_key": f"zxq{index:x}v", + "asset_type": "host", + "name": f"zxq{index:x}v", + "last_seen_at": now, + "coverage": {}, + "compliance": {}, + "change_types": [], + } + for index in range(1_000) + ] + + public_matrix = build_asset_capability_matrix( + _single_host_scope(), + live_assets=live_assets, + repo_root=_REPO_ROOT, + generated_at=now, + include_live_only_assets=False, + ) + reconciliation_matrix = build_asset_capability_matrix( + _single_host_scope(), + live_assets=live_assets, + repo_root=_REPO_ROOT, + generated_at=now, + include_live_only_assets=True, + ) + + assert len(public_matrix["assets"]) == 1 + public_summary = public_matrix["summary"] + assert public_summary["live_inventory_asset_count"] == 1_000 + assert public_summary["matrix_asset_count"] == 1 + assert public_summary["reconciled_asset_count"] == 1_001 + assert public_summary["live_only_assets_embedded"] is False + assert public_summary["embedded_live_only_asset_count"] == 0 + assert public_summary["discovered_not_declared_count"] == 1_000 + assert len(reconciliation_matrix["assets"]) == 1_001 + assert reconciliation_matrix["summary"]["live_only_assets_embedded"] is True + assert reconciliation_matrix["summary"]["embedded_live_only_asset_count"] == 1_000 + assert ( + public_matrix["matrix_fingerprint"] + == (reconciliation_matrix["matrix_fingerprint"]) + ) + + +def test_indexed_matching_preserves_score_cap_and_original_live_order() -> None: + now = datetime(2026, 7, 15, 1, 0, tzinfo=UTC) + scope = [ + { + "id": "services", + "label": "Services", + "asset_count": 1, + "asset_ids": ["one two three four five six seven"], + "inventory_sources": ["service-inventory.json"], + } + ] + live_assets = [ + { + "asset_id": 1, + "asset_key": "candidate:first", + "asset_type": "container", + "name": "one two three four five six alpha", + "last_seen_at": now, + }, + { + "asset_id": 2, + "asset_key": "candidate:second", + "asset_type": "container", + "name": "one two three four five six seven beta", + "last_seen_at": now, + }, + ] + + matrix = build_asset_capability_matrix( + scope, + live_assets=live_assets, + repo_root=_REPO_ROOT, + generated_at=now, + include_live_only_assets=False, + ) + + assert matrix["assets"][0]["runtime_identity"]["asset_inventory_id"] == 1 + + def test_live_asset_has_per_stage_covered_status_and_closes_after_receipt() -> None: now = datetime(2026, 7, 11, 12, 0, tzinfo=UTC) initial = build_asset_capability_matrix( @@ -318,6 +405,8 @@ class _FakeLiveReadbackDb: sql = str(statement) bound = params or {} self.calls.append((sql, bound)) + if "set_config('statement_timeout'" in sql: + return _FakeResult([]) if "FROM asset_discovery_run" in sql: return _FakeResult( [ @@ -382,6 +471,13 @@ def test_live_change_event_readback_is_scoped_to_selected_asset_ids( _latest_run, assets, _receipt = asyncio.run(_load_live_rows("awoooi")) + timeout_sql, timeout_params = next( + (sql, params) + for sql, params in db.calls + if "set_config('statement_timeout'" in sql + ) + assert "statement_timeout" in timeout_sql + assert timeout_params == {"statement_timeout": "6000ms"} change_sql, change_params = next( (sql, params) for sql, params in db.calls if "FROM asset_change_event" in sql ) @@ -413,3 +509,28 @@ def test_live_readback_timeout_fails_closed_without_hanging( assert payload["live_readback_timeout_seconds"] == 0.01 assert payload["summary"]["declared_asset_count"] == 193 assert payload["operation_boundaries"]["host_write_performed"] is False + + +def test_matrix_build_phase_is_inside_the_total_timeout(monkeypatch: Any) -> None: + async def _fast_live_readback( + _project_id: str, + ) -> tuple[dict[str, Any], list[dict[str, Any]], dict[str, Any]]: + return {}, [], {} + + async def _slow_to_thread(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + await asyncio.sleep(1) + return {} + + monkeypatch.setattr(matrix_service, "_load_live_rows", _fast_live_readback) + monkeypatch.setattr(matrix_service.asyncio, "to_thread", _slow_to_thread) + + payload = asyncio.run( + build_asset_capability_matrix_with_live_readback( + repo_root=_REPO_ROOT, + timeout_seconds=0.01, + ) + ) + + assert payload["live_readback_status"] == "degraded" + assert payload["live_readback_error_type"] == "TimeoutError" + assert payload["live_readback_timeout_seconds"] == 0.01 From 91f91813baff42cdf1095186fa8326ce164ae7cd Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 05:14:17 +0800 Subject: [PATCH 10/45] fix(agent99): bind offsite verifier freshness --- agent99-control-plane.ps1 | 33 ++++++++++++++++--- ..._agent99_control_plane_outcome_contract.py | 6 ++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 42de86857..69d52d61a 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -3468,6 +3468,7 @@ function Get-AgentBackupHealthConfig { fileChecks = if ($backupHealth -and $backupHealth.fileChecks) { @($backupHealth.fileChecks) } else { $fileChecks } cronPatterns = if ($backupHealth -and $backupHealth.cronPatterns) { @($backupHealth.cronPatterns) } else { $cronPatterns } protectionMetricsPath = if ($backupHealth -and $backupHealth.protectionMetricsPath) { [string]$backupHealth.protectionMetricsPath } else { "/home/wooo/node_exporter_textfiles/backup_health.prom" } + protectionVerifyMetricsPath = if ($backupHealth -and $backupHealth.protectionVerifyMetricsPath) { [string]$backupHealth.protectionVerifyMetricsPath } else { "/home/wooo/node_exporter_textfiles/offsite_full_sync_verify.prom" } protectionReadTimeoutSeconds = if ($backupHealth -and $backupHealth.protectionReadTimeoutSeconds) { [int]$backupHealth.protectionReadTimeoutSeconds } else { 45 } protectionMaxAgeMinutes = if ($backupHealth -and $backupHealth.protectionMaxAgeMinutes) { [double]$backupHealth.protectionMaxAgeMinutes } else { 60 } protectionEscrowExpectedCount = if ($backupHealth -and $backupHealth.protectionEscrowExpectedCount) { [int]$backupHealth.protectionEscrowExpectedCount } else { 5 } @@ -3527,8 +3528,12 @@ function Convert-AgentBackupProtectionReadback { $readback = $Readback $mtimeMatch = [regex]::Match([string]$readback.output, "__PROTECTION_MTIME__=([0-9]+)") $mtime = if ($mtimeMatch.Success) { $mtimeMatch.Groups[1].Value } else { "" } + $verifyMtimeMatch = [regex]::Match([string]$readback.output, "__OFFSITE_VERIFY_MTIME__=([0-9]+)") + $verifyMtime = if ($verifyMtimeMatch.Success) { $verifyMtimeMatch.Groups[1].Value } else { "" } $evidenceAgeMinutes = if ($mtime) { ($NowUnix - [double]$mtime) / 60 } else { $null } $evidenceFresh = [bool]($null -ne $evidenceAgeMinutes -and $evidenceAgeMinutes -ge -5 -and $evidenceAgeMinutes -le [math]::Max(1, $MaxAgeMinutes)) + $verifyEvidenceAgeMinutes = if ($verifyMtime) { ($NowUnix - [double]$verifyMtime) / 60 } else { $null } + $verifyEvidenceFresh = [bool]($null -ne $verifyEvidenceAgeMinutes -and $verifyEvidenceAgeMinutes -ge -5 -and $verifyEvidenceAgeMinutes -le [math]::Max(1, $MaxAgeMinutes)) $configured = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_configured") $offsiteFresh = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_fresh") $remoteVerify = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_remote_verify_ok") @@ -3580,6 +3585,7 @@ function Convert-AgentBackupProtectionReadback { } $offsiteOk = [bool]( $readback.ok -and $mtime -and $evidenceFresh -and + $verifyMtime -and $verifyEvidenceFresh -and $configuredShapeValid -and $configuredProviderNames.Count -gt 0 -and $verifiedProviderNames.Count -eq $configuredProviderNames.Count @@ -3593,8 +3599,8 @@ function Convert-AgentBackupProtectionReadback { [pscustomobject]@{ offsiteVerify = [pscustomobject]@{ ok = $offsiteOk - receiptId = if ($offsiteOk) { "offsite-verify:$mtime" } else { $null } - reason = if ($offsiteOk) { "read_only_metrics_verified" } elseif (-not $readback.ok) { "protection_metrics_readback_failed" } else { "offsite_verify_not_green" } + receiptId = if ($offsiteOk) { "offsite-verify:$verifyMtime" } else { $null } + reason = if ($offsiteOk) { "read_only_metrics_verified" } elseif (-not $readback.ok) { "protection_metrics_readback_failed" } elseif (-not $verifyEvidenceFresh) { "offsite_verify_evidence_stale_or_missing" } else { "offsite_verify_not_green" } configuredMetricCount = $configured.Count configuredProviderCount = $configuredProviderNames.Count verifiedProviderCount = $verifiedProviderNames.Count @@ -3603,6 +3609,8 @@ function Convert-AgentBackupProtectionReadback { remoteVerifyMetricCount = $remoteVerify.Count evidenceFresh = $evidenceFresh evidenceAgeMinutes = $evidenceAgeMinutes + verifyEvidenceFresh = $verifyEvidenceFresh + verifyEvidenceAgeMinutes = $verifyEvidenceAgeMinutes maxAgeMinutes = $MaxAgeMinutes } escrow = [pscustomobject]@{ @@ -3617,6 +3625,7 @@ function Convert-AgentBackupProtectionReadback { maxAgeMinutes = $MaxAgeMinutes } evidenceMtime = $mtime + verifyEvidenceMtime = $verifyMtime route = $readback.route exitCode = $readback.exitCode readOnly = $true @@ -3631,9 +3640,13 @@ function Test-AgentBackupProtectionReceipts { $hostIp = [string]$BackupConfig.host $path = [string]$BackupConfig.protectionMetricsPath + $verifyPath = [string]$BackupConfig.protectionVerifyMetricsPath $timeoutSeconds = [math]::Min(120, [math]::Max(5, [int]$BackupConfig.protectionReadTimeoutSeconds)) $maxAgeMinutes = [math]::Min(1440, [math]::Max(1, [double]$BackupConfig.protectionMaxAgeMinutes)) - if (-not $path -or $path -notmatch "^/[A-Za-z0-9_.\/-]+$" -or $path -match "\.\.") { + if ( + -not $path -or $path -notmatch "^/[A-Za-z0-9_.\/-]+$" -or $path -match "\.\." -or + -not $verifyPath -or $verifyPath -notmatch "^/[A-Za-z0-9_.\/-]+$" -or $verifyPath -match "\.\." + ) { return [pscustomobject]@{ offsiteVerify = [pscustomobject]@{ ok = $false; receiptId = $null; reason = "invalid_protection_metrics_path" } escrow = [pscustomobject]@{ ok = $false; missingCount = -1; receiptId = $null; reason = "invalid_protection_metrics_path" } @@ -3645,8 +3658,9 @@ function Test-AgentBackupProtectionReceipts { } $quotedPath = Quote-ShSingle $path + $quotedVerifyPath = Quote-ShSingle $verifyPath $metricPattern = "awoooi_backup_(offsite_configured|offsite_fresh|offsite_remote_verify_ok|credential_escrow_fresh|dr_credential_escrow_missing_count)" - $command = "if [ -r $quotedPath ]; then stat -c '__PROTECTION_MTIME__=%Y' $quotedPath; grep -E '^$metricPattern([ {])' $quotedPath || true; else echo __PROTECTION_MISSING__; fi" + $command = "if [ -r $quotedPath ]; then stat -c '__PROTECTION_MTIME__=%Y' $quotedPath; grep -E '^$metricPattern([ {])' $quotedPath || true; else echo __PROTECTION_MISSING__; fi; if [ -r $quotedVerifyPath ]; then stat -c '__OFFSITE_VERIFY_MTIME__=%Y' $quotedVerifyPath; grep -E '^awoooi_backup_offsite_remote_verify_ok([ {])' $quotedVerifyPath || true; else echo __OFFSITE_VERIFY_MISSING__; fi" $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1 Convert-AgentBackupProtectionReadback $readback $maxAgeMinutes ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds()) ([int]$BackupConfig.protectionEscrowExpectedCount) } @@ -3658,6 +3672,7 @@ function Invoke-AgentBackupReadbackContractSelfTest { exitCode = 0 output = @" __PROTECTION_MTIME__=1783785600 +__OFFSITE_VERIFY_MTIME__=1783785600 awoooi_backup_offsite_configured{host="110",provider="b2"} 0 awoooi_backup_offsite_configured{host="110",provider="rclone"} 1 awoooi_backup_offsite_fresh{host="110",provider="b2"} 0 @@ -3673,6 +3688,13 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 } $result = Convert-AgentBackupProtectionReadback $fixture 60 1783785660 $staleResult = Convert-AgentBackupProtectionReadback $fixture 60 1783792800 + $staleVerifyFixture = [pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + output = $fixture.output -replace '__OFFSITE_VERIFY_MTIME__=1783785600', '__OFFSITE_VERIFY_MTIME__=1783778400' + } + $staleVerifyResult = Convert-AgentBackupProtectionReadback $staleVerifyFixture 60 1783785660 $noProviderFixture = [pscustomobject]@{ ok = $true route = "selftest-read-only" @@ -3722,6 +3744,8 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 $result.readOnly -and -not $staleResult.offsiteVerify.ok -and -not $staleResult.escrow.ok -and + -not $staleVerifyResult.offsiteVerify.ok -and + $staleVerifyResult.escrow.ok -and -not $noProviderResult.offsiteVerify.ok -and -not $staleProviderResult.offsiteVerify.ok -and -not $remoteVerifyResult.offsiteVerify.ok -and @@ -3737,6 +3761,7 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 terminalEligible = $terminalEligible cases = [pscustomobject]@{ optionalB2ProviderTerminalEligible = $terminalEligible + staleOffsiteVerifyEvidenceBlocked = [bool](-not $staleVerifyResult.offsiteVerify.ok -and $staleVerifyResult.escrow.ok) allProvidersUnconfiguredBlocked = [bool](-not $noProviderResult.offsiteVerify.ok) configuredProviderStaleBlocked = [bool](-not $staleProviderResult.offsiteVerify.ok) configuredProviderRemoteVerifyFailedBlocked = [bool](-not $remoteVerifyResult.offsiteVerify.ok) diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py index fabffea4b..1c53cd96f 100644 --- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py +++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py @@ -131,6 +131,9 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None ): assert metric in source assert "stat -c '__PROTECTION_MTIME__=%Y'" in readback + assert "protectionVerifyMetricsPath" in source + assert "offsite_full_sync_verify.prom" in source + assert "stat -c '__OFFSITE_VERIFY_MTIME__=%Y'" in readback assert "grep -E" in readback assert "backupRunPerformed = $false" in source assert "restoreRunPerformed = $false" in source @@ -141,6 +144,8 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None assert "protectionEscrowExpectedCount" in source assert "$evidenceAgeMinutes -ge -5" in source assert "$evidenceAgeMinutes -le" in source + assert "$verifyEvidenceAgeMinutes -ge -5" in source + assert "$verifyEvidenceAgeMinutes -le" in source assert "function Get-AgentBackupProviderMetricRows" in source assert "$offsiteFresh.Count -eq $offsiteFreshRows.Count" in source assert "$remoteVerify.Count -eq $remoteVerifyRows.Count" in source @@ -151,6 +156,7 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None assert 'provider="b2"} 0' in source assert 'provider="rclone"} 1' in source assert "optionalB2ProviderTerminalEligible" in source + assert "staleOffsiteVerifyEvidenceBlocked" in source assert "allProvidersUnconfiguredBlocked" in source assert "configuredProviderStaleBlocked" in source assert "configuredProviderRemoteVerifyFailedBlocked" in source From 2bb6d8eede8ffcc9005e9d3f6369a6d5d59c4b9f Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 05:14:17 +0800 Subject: [PATCH 11/45] fix(backup): align daily database credential contract --- scripts/backup/backup-awoooi.sh | 158 ++++++++++++++++-- .../test_backup_awoooi_secret_contract.py | 20 ++- 2 files changed, 162 insertions(+), 16 deletions(-) diff --git a/scripts/backup/backup-awoooi.sh b/scripts/backup/backup-awoooi.sh index fc1799714..c5a7cd2c3 100755 --- a/scripts/backup/backup-awoooi.sh +++ b/scripts/backup/backup-awoooi.sh @@ -21,6 +21,70 @@ AWOOOI_DB_PORT="5432" K3S_DB_USER="postgres" LOCAL_REPO="${BACKUP_BASE}/awoooi" DUMP_DIR="/tmp/awoooi-backup-$$" +AWOOOI_K8S_HOST="${AWOOOI_K8S_HOST:-192.168.0.120}" +AWOOOI_K8S_HOSTS="${AWOOOI_K8S_HOSTS:-${AWOOOI_K8S_HOST} 192.168.0.121 192.168.0.125}" +AWOOOI_K8S_SECRET_NAME="${AWOOOI_K8S_SECRET_NAME:-awoooi-secrets}" +AWOOOI_K8S_NAMESPACE="${AWOOOI_K8S_NAMESPACE:-awoooi-prod}" +AWOOOI_K8S_DATABASE_URL_KEYS="${AWOOOI_K8S_DATABASE_URL_KEYS:-AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL}" +FORCE_RLS_RESTORE_SQL="" +FORCE_RLS_RESTORE_DB="" + +resolve_database_url() { + if [ -n "${AWOOOI_DATABASE_URL:-}" ]; then + printf '%s\n' "${AWOOOI_DATABASE_URL}" + return 0 + fi + if [ -n "${DATABASE_URL:-}" ]; then + printf '%s\n' "${DATABASE_URL}" + return 0 + fi + + # Resolve only inside the runtime backup process. The value is never + # written to logs, source, artifacts, or command output. + local k8s_host key encoded decoded + for k8s_host in ${AWOOOI_K8S_HOSTS}; do + for key in ${AWOOOI_K8S_DATABASE_URL_KEYS}; do + encoded="$(ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "wooo@${k8s_host}" \ + "sudo -n kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}' 2>/dev/null || kubectl get secret ${AWOOOI_K8S_SECRET_NAME} -n ${AWOOOI_K8S_NAMESPACE} -o jsonpath='{.data.${key}}'" \ + 2>/dev/null || true)" + decoded="$(printf '%s' "${encoded}" | base64 -d 2>/dev/null || true)" + if [ -n "${decoded}" ]; then + printf '%s\n' "${decoded}" + return 0 + fi + done + done + return 1 +} + +load_database_config() { + local database_url + database_url="$(resolve_database_url || true)" + if [ -z "${database_url}" ]; then + log_error "無法解析 AWOOOI DATABASE_URL；拒絕使用舊硬編密碼" + return 1 + fi + + eval "$( + python3 - 3<<< "${database_url}" <<'PY' +import shlex +from urllib.parse import unquote, urlparse + +with open(3) as source: + url = source.read().strip() +parsed = urlparse(url) + +values = { + "AWOOOI_DB_USER": unquote(parsed.username or "awoooi"), + "AWOOOI_DB_PASS": unquote(parsed.password or ""), + "AWOOOI_DB_HOST": parsed.hostname or "localhost", + "AWOOOI_DB_PORT": str(parsed.port or 5432), +} +for key, value in values.items(): + print(f"{key}={shlex.quote(value)}") +PY + )" +} quote_remote() { printf "%q" "$1" @@ -45,26 +109,95 @@ pgpass_line() { remote_pgpass_wrapper() { local command="$1" - printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGPASSFILE="$pgpass" %s' "${command}" + printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0" PGPASSFILE="$pgpass" %s' "${command}" } -run_remote_pg_dump() { +remote_psql_command() { + local database="$1" + printf "psql --no-password -U %s -h %s -p %s -d %s -v ON_ERROR_STOP=1" \ + "$(quote_remote "${AWOOOI_DB_USER}")" \ + "$(quote_remote "${AWOOOI_DB_HOST}")" \ + "$(quote_remote "${AWOOOI_DB_PORT}")" \ + "$(quote_remote "${database}")" +} + +run_remote_pgpass_command() { + local database="$1" + local command="$2" + pgpass_line "${database}" | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" +} + +collect_force_rls_sql() { + local database="$1" + local mode="$2" + local query + + query=" +select format('ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY;', n.nspname, c.relname) +from pg_class c +join pg_namespace n on n.oid = c.relnamespace +where c.relkind in ('r', 'p') + and c.relforcerowsecurity + and pg_get_userbyid(c.relowner) = current_user +order by 1; +" + run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -At -c $(quote_remote "${query}")" +} + +apply_remote_sql() { + local database="$1" + local sql="$2" + [ -n "${sql}" ] || return 0 + run_remote_pgpass_command "${database}" "$(remote_psql_command "${database}") -c $(quote_remote "${sql}")" >/dev/null +} + +restore_force_rls() { + if [ -n "${FORCE_RLS_RESTORE_DB}" ] && [ -n "${FORCE_RLS_RESTORE_SQL}" ]; then + if apply_remote_sql "${FORCE_RLS_RESTORE_DB}" "${FORCE_RLS_RESTORE_SQL}"; then + log_info "FORCE ROW LEVEL SECURITY 已恢復 (${FORCE_RLS_RESTORE_DB})" + else + log_error "FORCE ROW LEVEL SECURITY 恢復失敗 (${FORCE_RLS_RESTORE_DB})" + return 1 + fi + FORCE_RLS_RESTORE_DB="" + FORCE_RLS_RESTORE_SQL="" + fi +} + +trap restore_force_rls EXIT + +dump_database_with_rls_guard() { local database="$1" local output_file="$2" local stderr_file="${output_file}.stderr" - local command + local noforce_sql force_sql dump_rc - command="pg_dump --no-password -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") $(quote_remote "${database}")" - if pgpass_line "${database}" \ - | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" \ - >"${output_file}" 2>"${stderr_file}"; then + noforce_sql="$(collect_force_rls_sql "${database}" "NO FORCE")" + force_sql="$(printf '%s\n' "${noforce_sql}" | sed 's/NO FORCE/FORCE/')" + if [ -n "${noforce_sql}" ]; then + FORCE_RLS_RESTORE_DB="${database}" + FORCE_RLS_RESTORE_SQL="${force_sql}" + log_info "暫時解除 FORCE RLS 以完成完整 pg_dump (${database}, tables=$(printf '%s\n' "${noforce_sql}" | awk 'NF {count++} END {print count+0}'))" + apply_remote_sql "${database}" "${noforce_sql}" + fi + + set +e + run_remote_pgpass_command "${database}" "pg_dump --no-password \ + -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") \ + $(quote_remote "${database}")" >"${output_file}" 2>"${stderr_file}" + dump_rc=$? + set -e + + restore_force_rls + if [ "${dump_rc}" -eq 0 ]; then rm -f "${stderr_file}" return 0 fi + log_error "${database} dump 失敗，pg_dump stderr 尾端如下（已避免輸出 credential）：" tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true rm -f "${stderr_file}" - return 1 + return "${dump_rc}" } # 保留策略覆寫（比其他服務更長） @@ -76,8 +209,7 @@ main() { local start_time=$(date +%s) local failed=0 - if [ -z "${AWOOOI_DB_PASS}" ]; then - log_error "AWOOOI_DB_PASS is unavailable; refusing an unauthenticated or hardcoded fallback" + if ! load_database_config; then notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable" exit 1 fi @@ -89,7 +221,7 @@ main() { log_info "Dump awoooi_prod..." local timestamp=$(date "+%Y%m%d_%H%M%S") - if run_remote_pg_dump \ + if dump_database_with_rls_guard \ "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1) log_success "awoooi_prod dump 完成 (${size})" @@ -100,7 +232,7 @@ main() { # Step 2: awoooi_dev dump log_info "Dump awoooi_dev..." - if run_remote_pg_dump \ + if dump_database_with_rls_guard \ "awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1) log_success "awoooi_dev dump 完成 (${size})" @@ -110,7 +242,7 @@ main() { # Step 3: k3s_datastore dump（Kine 後端） log_info "Dump k3s_datastore..." - if run_remote_pg_dump \ + if dump_database_with_rls_guard \ "k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1) log_success "k3s_datastore dump 完成 (${size})" diff --git a/scripts/backup/tests/test_backup_awoooi_secret_contract.py b/scripts/backup/tests/test_backup_awoooi_secret_contract.py index e513cb955..a31b48bc6 100644 --- a/scripts/backup/tests/test_backup_awoooi_secret_contract.py +++ b/scripts/backup/tests/test_backup_awoooi_secret_contract.py @@ -18,8 +18,22 @@ def test_daily_backup_has_no_hardcoded_database_password() -> None: assert "pg_dump --no-password" in text -def test_daily_backup_fails_closed_without_runtime_credential() -> None: +def test_daily_backup_resolves_runtime_database_url_and_fails_closed() -> None: text = SCRIPT.read_text(encoding="utf-8") - assert 'if [ -z "${AWOOOI_DB_PASS}" ]; then' in text - assert "refusing an unauthenticated or hardcoded fallback" in text + assert "resolve_database_url()" in text + assert "load_database_config()" in text + assert "AWOOOI_BACKUP_DATABASE_URL BACKUP_DATABASE_URL DATABASE_URL" in text + assert "無法解析 AWOOOI DATABASE_URL；拒絕使用舊硬編密碼" in text + assert "base64 -d" in text + assert 'printf \'%s\\n\' "${decoded}"' in text + + +def test_daily_backup_uses_same_rls_safe_pg_dump_contract_as_frequent_lane() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert "dump_database_with_rls_guard()" in text + assert 'PGOPTIONS="-c statement_timeout=0 -c max_parallel_workers_per_gather=0"' in text + assert "ALTER TABLE %I.%I ${mode} ROW LEVEL SECURITY" in text + assert "trap restore_force_rls EXIT" in text + assert 'dump_database_with_rls_guard \\\n "awoooi_prod"' in text From b0dff51a06c471f330cbe153da39c2f3f8cc1154 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 05:27:29 +0800 Subject: [PATCH 12/45] chore(cd): deploy 228d474 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index dd1b790d6..dd4f747ca 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "956b55d66c9916a36d3d68311053ce32f0d7094e" + value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "956b55d66c9916a36d3d68311053ce32f0d7094e" + value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 209417ee9..31550d0f3 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "956b55d66c9916a36d3d68311053ce32f0d7094e" + value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 1341ee393..daffbd0c7 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "956b55d66c9916a36d3d68311053ce32f0d7094e" + value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 9c044cf89..b48371703 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:9ab7979e8052f1361f9852114c3af1e73527875f14206de3bfe09b5dac143f13 +- digest: sha256:ac7c44c87a188a0ba8f5accfb1fae818e4ee5a2edcad13ce0f471d34b2fad93f name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:6ae01c8a8202b84c060cdcaf2cc10626951950b1c79069a0063d4f1e02c29e0a +- digest: sha256:a4015063d883a74369c29425880696c7047a4ffe164ee4be05b680e9f50fb7da name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 7c6905a8f20d5008e713f3db70c7cfb332d7ad9b Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 05:13:50 +0800 Subject: [PATCH 13/45] feat(security): guard VPA image rollout --- .../security/runtime-image-mirror-policy.json | 19 ++ .../runtime_image_mirror_controller.py | 144 +++++++++++ .../test_runtime_image_mirror_controller.py | 241 +++++++++++++++++- 3 files changed, 402 insertions(+), 2 deletions(-) diff --git a/config/security/runtime-image-mirror-policy.json b/config/security/runtime-image-mirror-policy.json index b3a3cb5b7..26219f176 100644 --- a/config/security/runtime-image-mirror-policy.json +++ b/config/security/runtime-image-mirror-policy.json @@ -213,6 +213,25 @@ "name": "node-problem-detector", "container": "node-problem-detector" } + }, + { + "asset_id": "runtime-image:vpa-recommender", + "source_index_digest": "sha256:4af365370c187e4eb60302e937b18a188774698b049a1bfdd77d43e51b30abec", + "platform_digest": "sha256:f7da718d281d5e880e966d0182ffb2160b82517c7bc3a38f8002b49e7a4bd0a1", + "target_registry_digest": "sha256:6ae9b8c6ac256fe034f4e2741c2fef2c2ba2d64778c071d6d93c99cd86ba7c89", + "push_ref": "registry.wooo.work/awoooi/runtime-mirror/vpa-recommender:1.6.0-linux-amd64", + "runtime_ref": "192.168.0.110:5000/awoooi/runtime-mirror/vpa-recommender@sha256:6ae9b8c6ac256fe034f4e2741c2fef2c2ba2d64778c071d6d93c99cd86ba7c89", + "workload": { + "kind": "deployment", + "namespace": "kube-system", + "name": "vpa-recommender", + "container": "recommender" + }, + "availability_guard": { + "minimum_ready_replicas": 1, + "maximum_effective_unavailable": 0, + "minimum_effective_surge": 1 + } } ] } diff --git a/scripts/security/runtime_image_mirror_controller.py b/scripts/security/runtime_image_mirror_controller.py index 6438bf040..011c5ea77 100644 --- a/scripts/security/runtime_image_mirror_controller.py +++ b/scripts/security/runtime_image_mirror_controller.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import hashlib import json +import math import re import shlex import subprocess @@ -41,6 +42,13 @@ class Workload: container_kind: str +@dataclass(frozen=True) +class AvailabilityGuard: + minimum_ready_replicas: int + maximum_effective_unavailable: int + minimum_effective_surge: int + + @dataclass(frozen=True) class ImagePolicy: asset_id: str @@ -50,6 +58,7 @@ class ImagePolicy: push_ref: str runtime_ref: str workload: Workload + availability_guard: AvailabilityGuard | None @dataclass(frozen=True) @@ -112,6 +121,19 @@ def _require_name(value: Any, label: str) -> str: return name +def _require_nonnegative_int(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ControllerError(f"invalid_{label}") + return value + + +def _require_positive_int(value: Any, label: str) -> int: + number = _require_nonnegative_int(value, label) + if number == 0: + raise ControllerError(f"invalid_{label}") + return number + + def _policy_checksum(payload: dict[str, Any]) -> str: canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() return "sha256:" + hashlib.sha256(canonical).hexdigest() @@ -201,6 +223,27 @@ def load_policy(path: Path) -> Policy: if workload_key in workloads: raise ControllerError("duplicate_workload_target") workloads.add(workload_key) + availability_guard = None + guard_row = row.get("availability_guard") + if guard_row is not None: + if not isinstance(guard_row, dict): + raise ControllerError("availability_guard_invalid") + if workload.kind != "deployment": + raise ControllerError("availability_guard_requires_deployment") + availability_guard = AvailabilityGuard( + minimum_ready_replicas=_require_positive_int( + guard_row.get("minimum_ready_replicas"), + "minimum_ready_replicas", + ), + maximum_effective_unavailable=_require_nonnegative_int( + guard_row.get("maximum_effective_unavailable"), + "maximum_effective_unavailable", + ), + minimum_effective_surge=_require_positive_int( + guard_row.get("minimum_effective_surge"), + "minimum_effective_surge", + ), + ) images.append( ImagePolicy( asset_id=asset_id, @@ -210,6 +253,7 @@ def load_policy(path: Path) -> Policy: push_ref=push_ref, runtime_ref=runtime_ref, workload=workload, + availability_guard=availability_guard, ) ) @@ -372,6 +416,7 @@ class RuntimeCache: self, policy: Policy | StagingPolicy, ssh_key: Path, known_hosts: Path ) -> None: self.policy = policy + self._verified_sources: dict[tuple[str, str], str] = {} self.ssh = [ "ssh", "-i", @@ -403,6 +448,10 @@ class RuntimeCache: return sorted(set(matches))[0] def verify_source(self, image: ImagePolicy | StagingImage) -> str: + cache_key = (image.source_index_digest, image.platform_digest) + cached = self._verified_sources.get(cache_key) + if cached is not None: + return cached source_ref = self.source_ref(image.source_index_digest) check = _run(self._command("images", "check", f"name=={source_ref}")) if "complete" not in (check.stdout or ""): @@ -423,6 +472,7 @@ class RuntimeCache: ] if selected != [image.platform_digest]: raise ControllerError("runtime_cache_platform_digest_mismatch") + self._verified_sources[cache_key] = source_ref return source_ref def source_config_digest(self, platform_digest: str) -> str: @@ -840,6 +890,92 @@ class Kubernetes: raise ControllerError("workload_container_not_found") return values[0] + @staticmethod + def _effective_replicas( + value: Any, replicas: int, *, round_up: bool, label: str + ) -> int: + if isinstance(value, bool): + raise ControllerError(f"{label}_invalid") + if isinstance(value, int) and value >= 0: + return value + if isinstance(value, str) and value.endswith("%"): + try: + percent = int(value[:-1]) + except ValueError as exc: + raise ControllerError(f"{label}_invalid") from exc + if percent < 0 or percent > 100: + raise ControllerError(f"{label}_invalid") + scaled = replicas * percent / 100 + return math.ceil(scaled) if round_up else math.floor(scaled) + raise ControllerError(f"{label}_invalid") + + def verify_availability(self, item: ImagePolicy) -> None: + guard = item.availability_guard + if guard is None: + return + workload = self.workload(item) + metadata = workload.get("metadata", {}) + spec = workload.get("spec", {}) + status = workload.get("status", {}) + generation = metadata.get("generation") + observed_generation = status.get("observedGeneration") + replicas = spec.get("replicas", 1) + if ( + isinstance(replicas, bool) + or not isinstance(replicas, int) + or replicas < guard.minimum_ready_replicas + ): + raise ControllerError("availability_replicas_insufficient") + if ( + isinstance(generation, bool) + or not isinstance(generation, int) + or isinstance(observed_generation, bool) + or not isinstance(observed_generation, int) + or observed_generation < generation + ): + raise ControllerError("availability_generation_not_observed") + ready = status.get("readyReplicas", 0) + available = status.get("availableReplicas", 0) + unavailable = status.get("unavailableReplicas", 0) + if ( + isinstance(ready, bool) + or not isinstance(ready, int) + or isinstance(available, bool) + or not isinstance(available, int) + or ready < guard.minimum_ready_replicas + or available < guard.minimum_ready_replicas + ): + raise ControllerError("availability_ready_replicas_insufficient") + if ( + isinstance(unavailable, bool) + or not isinstance(unavailable, int) + or unavailable > guard.maximum_effective_unavailable + ): + raise ControllerError("availability_unavailable_replicas_exceeded") + + strategy = spec.get("strategy", {}) + if not isinstance(strategy, dict) or strategy.get("type") != "RollingUpdate": + raise ControllerError("availability_rolling_update_required") + rolling = strategy.get("rollingUpdate", {}) + if not isinstance(rolling, dict): + raise ControllerError("availability_rolling_update_invalid") + effective_unavailable = self._effective_replicas( + rolling.get("maxUnavailable", "25%"), + replicas, + round_up=False, + label="availability_max_unavailable", + ) + effective_surge = self._effective_replicas( + rolling.get("maxSurge", "25%"), + replicas, + round_up=True, + label="availability_max_surge", + ) + if effective_unavailable > guard.maximum_effective_unavailable: + raise ControllerError("availability_effective_unavailable_exceeded") + if effective_surge < guard.minimum_effective_surge: + raise ControllerError("availability_effective_surge_insufficient") + def patch_image(self, item: ImagePolicy, image_ref: str, *, dry_run: bool) -> None: container_field = self._spec_container_field(item) patch = json.dumps( @@ -994,6 +1130,7 @@ def apply_policy( changed: list[ImagePolicy] = [] for item in policy.images: previous[item.asset_id] = kubernetes.current_image(item) + kubernetes.verify_availability(item) kubernetes.patch_image(item, item.runtime_ref, dry_run=True) if apply: @@ -1005,6 +1142,7 @@ def apply_policy( changed.append(item) for item in policy.images: kubernetes.rollout(item) + kubernetes.verify_availability(item) except ControllerError: for item in reversed(changed): try: @@ -1038,6 +1176,12 @@ def apply_policy( "ai_candidate_action": "replace_forbidden_registry_with_internal_digest", "policy_decision": "controlled_apply_allowed", "server_dry_run_passed": True, + "availability_guard_count": sum( + 1 for item in policy.images if item.availability_guard is not None + ), + "availability_guard_verified_count": sum( + 1 for item in policy.images if item.availability_guard is not None + ), "execution_changed_count": len(changed), "independent_verifier_passed": verified, "verified_workload_count": compliant_count, diff --git a/scripts/security/tests/test_runtime_image_mirror_controller.py b/scripts/security/tests/test_runtime_image_mirror_controller.py index fac8a7b40..50bed6043 100644 --- a/scripts/security/tests/test_runtime_image_mirror_controller.py +++ b/scripts/security/tests/test_runtime_image_mirror_controller.py @@ -46,13 +46,24 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): policy = controller.load_staging_policy(STAGING_POLICY_PATH) return replace(policy, images=(policy.images[0],)) + def _image_with_availability_guard(self): + image = controller.load_policy(POLICY_PATH).images[0] + return replace( + image, + availability_guard=controller.AvailabilityGuard( + minimum_ready_replicas=1, + maximum_effective_unavailable=0, + minimum_effective_surge=1, + ), + ) + def test_policy_is_internal_digest_pinned_and_runtime_cache_only(self) -> None: policy = controller.load_policy(POLICY_PATH) raw = POLICY_PATH.read_text(encoding="utf-8") self.assertEqual(policy.work_item_id, "AIA-P0-006-02A") self.assertEqual(policy.risk_level, "high") - self.assertEqual(len(policy.images), 14) + self.assertEqual(len(policy.images), 15) self.assertNotIn("github.com", raw) self.assertNotIn("ghcr.io", raw) self.assertIn('"external_pull_allowed": false', raw) @@ -77,9 +88,30 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): ) ) container_kinds = [image.workload.container_kind for image in policy.images] - self.assertEqual(container_kinds.count("container"), 11) + self.assertEqual(container_kinds.count("container"), 12) self.assertEqual(container_kinds.count("init_container"), 3) + vpa = next( + image + for image in policy.images + if image.asset_id == "runtime-image:vpa-recommender" + ) + self.assertEqual( + vpa.target_registry_digest, + "sha256:6ae9b8c6ac256fe034f4e2741c2fef2c2ba2d64778c071d6d93c99cd86ba7c89", + ) + self.assertEqual(vpa.workload.namespace, "kube-system") + self.assertEqual(vpa.workload.name, "vpa-recommender") + self.assertEqual(vpa.workload.container, "recommender") + self.assertEqual( + vpa.availability_guard, + controller.AvailabilityGuard( + minimum_ready_replicas=1, + maximum_effective_unavailable=0, + minimum_effective_surge=1, + ), + ) + def test_policy_loads_and_validates_init_container_kind(self) -> None: image = self._image_with_container_kind("init_container") @@ -90,6 +122,25 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): ): self._image_with_container_kind("ephemeral_container") + def test_availability_guard_cannot_be_configured_as_noop(self) -> None: + payload = json.loads(POLICY_PATH.read_text(encoding="utf-8")) + vpa = next( + image + for image in payload["images"] + if image["asset_id"] == "runtime-image:vpa-recommender" + ) + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "policy.json" + for field in ("minimum_ready_replicas", "minimum_effective_surge"): + with self.subTest(field=field): + vpa["availability_guard"][field] = 0 + path.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex( + controller.ControllerError, f"invalid_{field}" + ): + controller.load_policy(path) + vpa["availability_guard"][field] = 1 + def test_staging_policy_is_runtime_cache_only_and_bounded_canary_scoped( self, ) -> None: @@ -176,6 +227,31 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): canary.runtime_ref.endswith("@" + canary.target_registry_digest) ) + def test_vpa_stable_policy_reuses_verified_staging_provenance(self) -> None: + policy = controller.load_policy(POLICY_PATH) + staging = next( + image + for image in controller.load_staging_policy(STAGING_POLICY_PATH).images + if image.asset_id == "runtime-image:vpa-recommender-1.6.0-canary" + ) + stable = next( + image + for image in policy.images + if image.asset_id == "runtime-image:vpa-recommender" + ) + + self.assertEqual(stable.source_index_digest, staging.source_index_digest) + self.assertEqual(stable.platform_digest, staging.platform_digest) + self.assertEqual(stable.push_ref, staging.push_ref) + self.assertEqual(stable.workload, staging.workload) + self.assertEqual( + stable.target_registry_digest, + "sha256:6ae9b8c6ac256fe034f4e2741c2fef2c2ba2d64778c071d6d93c99cd86ba7c89", + ) + self.assertTrue( + stable.runtime_ref.endswith("@" + stable.target_registry_digest) + ) + def test_argocd_controller_batch_reuses_verified_staging_artifact(self) -> None: policy = controller.load_policy(POLICY_PATH) staging = controller.load_staging_policy(STAGING_POLICY_PATH).images[0] @@ -459,6 +535,52 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): self.assertIn("--insecure", command) self.assertEqual(command[-1], controller._target_digest_ref(image)) + def test_runtime_cache_verifies_shared_source_digest_once(self) -> None: + policy = controller.load_policy(POLICY_PATH) + shared = [ + image + for image in policy.images + if image.source_index_digest + == "sha256:16b92ba472fbb9287459cc52e0ecff07288dff461209955098edb56ce866fe49" + ] + self.assertGreaterEqual(len(shared), 2) + source_ref = "quay.io/argoproj/argocd:v3.3.6" + source_list = ( + f"{source_ref} application/vnd.oci.image.index.v1+json " + f"{shared[0].source_index_digest}\n" + ) + source_index = json.dumps( + { + "manifests": [ + { + "digest": shared[0].platform_digest, + "platform": {"os": "linux", "architecture": "amd64"}, + } + ] + } + ) + cache = controller.RuntimeCache( + policy, + Path("/unused/key"), + Path("/unused/known-hosts"), + ) + completed = controller.subprocess.CompletedProcess + with patch.object( + controller, + "_run", + side_effect=[ + completed([], 0, stdout=source_list), + completed([], 0, stdout="complete"), + completed([], 0, stdout=source_index), + ], + ) as run: + first = cache.verify_source(shared[0]) + second = cache.verify_source(shared[1]) + + self.assertEqual(first, source_ref) + self.assertEqual(second, source_ref) + self.assertEqual(run.call_count, 3) + def test_source_manifests_match_internal_policy(self) -> None: policy = controller.load_policy(POLICY_PATH) by_asset = {image.asset_id: image for image in policy.images} @@ -784,6 +906,121 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): run.return_value.stdout = json.dumps(pods) kubernetes.verify_pods(image) + def test_single_replica_rolling_update_passes_availability_guard(self) -> None: + image = self._image_with_availability_guard() + workload = { + "metadata": {"generation": 4}, + "spec": { + "replicas": 1, + "strategy": { + "type": "RollingUpdate", + "rollingUpdate": { + "maxUnavailable": "25%", + "maxSurge": "25%", + }, + }, + }, + "status": { + "observedGeneration": 4, + "readyReplicas": 1, + "availableReplicas": 1, + "unavailableReplicas": 0, + }, + } + kubernetes = controller.Kubernetes( + use_sudo=False, + kubeconfig=None, + server=None, + ) + + with patch.object(kubernetes, "workload", return_value=workload): + kubernetes.verify_availability(image) + + self.assertEqual( + kubernetes._effective_replicas( + "25%", 1, round_up=False, label="unavailable" + ), + 0, + ) + self.assertEqual( + kubernetes._effective_replicas( + "25%", 1, round_up=True, label="surge" + ), + 1, + ) + + def test_availability_guard_fails_closed_on_unsafe_rollout_state(self) -> None: + image = self._image_with_availability_guard() + safe = { + "metadata": {"generation": 4}, + "spec": { + "replicas": 1, + "strategy": { + "type": "RollingUpdate", + "rollingUpdate": {"maxUnavailable": 0, "maxSurge": 1}, + }, + }, + "status": { + "observedGeneration": 4, + "readyReplicas": 1, + "availableReplicas": 1, + "unavailableReplicas": 0, + }, + } + cases = ( + ( + {**safe, "metadata": {"generation": 5}}, + "availability_generation_not_observed", + ), + ( + { + **safe, + "status": { + **safe["status"], + "readyReplicas": 0, + "availableReplicas": 0, + }, + }, + "availability_ready_replicas_insufficient", + ), + ( + { + **safe, + "spec": { + **safe["spec"], + "strategy": { + "type": "RollingUpdate", + "rollingUpdate": { + "maxUnavailable": 1, + "maxSurge": 0, + }, + }, + }, + }, + "availability_effective_unavailable_exceeded", + ), + ( + { + **safe, + "spec": {**safe["spec"], "strategy": {"type": "Recreate"}}, + }, + "availability_rolling_update_required", + ), + ) + kubernetes = controller.Kubernetes( + use_sudo=False, + kubeconfig=None, + server=None, + ) + + for workload, error_code in cases: + with ( + self.subTest(error_code=error_code), + patch.object(kubernetes, "workload", return_value=workload), + self.assertRaisesRegex(controller.ControllerError, error_code), + ): + kubernetes.verify_availability(image) + if __name__ == "__main__": unittest.main() From 8fec456bfecc5424575bd90017831940bed997ca Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 05:45:11 +0800 Subject: [PATCH 14/45] chore(cd): deploy c69eab8 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index dd4f747ca..1ead6ee9a 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" + value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" + value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 31550d0f3..2d5fc2d96 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" + value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index daffbd0c7..76fc771a8 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "228d474534fad5202b7b7be1b5f8e2b60ba45e9a" + value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index b48371703..ec6f99c73 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:ac7c44c87a188a0ba8f5accfb1fae818e4ee5a2edcad13ce0f471d34b2fad93f +- digest: sha256:8d7c86bf6a6ad0125733bed6eedd1428bdb95f94bdac288a01be5c7a35bbb501 name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:a4015063d883a74369c29425880696c7047a4ffe164ee4be05b680e9f50fb7da +- digest: sha256:d8b628409c4482bb72704b2a2a603f803262ca525dbeb7b25db24f216b466f51 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 3f9642cc285d6ccd3027589eb98b37b0cbd0327b Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 05:51:22 +0800 Subject: [PATCH 15/45] fix(backup): fit live reconcile idempotency key --- .../api/src/jobs/backup_restore_legacy_backfill_job.py | 5 ++++- .../tests/test_backup_restore_legacy_backfill_job.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/api/src/jobs/backup_restore_legacy_backfill_job.py b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py index 04060260c..4d37f9ff5 100644 --- a/apps/api/src/jobs/backup_restore_legacy_backfill_job.py +++ b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py @@ -46,7 +46,10 @@ BACKFILL_IDEMPOTENCY_CHANNEL = "backup_restore_backfill" BACKFILL_DB_CONTRACT_REPAIR_GENERATION = "cost_usd_state_bind_v1" LIVE_RECONCILE_SCHEMA_VERSION = "backup_restore_outbound_reconciler_v1" LIVE_RECONCILE_AGENT_ID = "backup_restore_outbound_reconciler" -LIVE_RECONCILE_IDEMPOTENCY_CHANNEL = "backup_restore_outbound_reconcile" +# ``awooop_run_idempotency.channel_type`` is VARCHAR(32). Keep this durable +# identity explicit and below that limit; the previous 33-character value +# caused PostgreSQL 22001 before a live backup card could be claimed. +LIVE_RECONCILE_IDEMPOTENCY_CHANNEL = "backup_restore_live_reconcile" BACKUP_RESTORE_EVENT_TYPE = "backup_restore_escrow_signal" BACKUP_RESTORE_LANE = "backup_restore_escrow_triage" PROJECTION_SCHEMA_VERSION = "telegram_agent99_dispatch_receipt_projection_v1" diff --git a/apps/api/tests/test_backup_restore_legacy_backfill_job.py b/apps/api/tests/test_backup_restore_legacy_backfill_job.py index 88aee8c88..6d9b59de2 100644 --- a/apps/api/tests/test_backup_restore_legacy_backfill_job.py +++ b/apps/api/tests/test_backup_restore_legacy_backfill_job.py @@ -576,6 +576,16 @@ async def test_live_reconciler_reserves_only_unowned_incomplete_cards( assert "backup-signal:" in sql +def test_live_reconciler_idempotency_channel_fits_durable_schema() -> None: + # awooop_run_idempotency.channel_type is VARCHAR(32). This invariant must + # be checked without relying on a mock database, which cannot reproduce a + # production PostgreSQL 22001 truncation error. + assert job.LIVE_RECONCILE_IDEMPOTENCY_CHANNEL == ( + "backup_restore_live_reconcile" + ) + assert len(job.LIVE_RECONCILE_IDEMPOTENCY_CHANNEL) <= 32 + + @pytest.mark.asyncio async def test_live_reconciler_projects_bounded_agent99_receipt( monkeypatch: pytest.MonkeyPatch, From 4b58dd41fc90c9e66e5148944b776fef3bdd8f6d Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 06:07:15 +0800 Subject: [PATCH 16/45] chore(cd): deploy c567067 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 1ead6ee9a..119cd2708 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" + value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" + value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 2d5fc2d96..f5f04cd3b 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" + value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 76fc771a8..867eee454 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "c69eab84f92b9fd1ff29c72002731e02224c4aaf" + value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index ec6f99c73..e1d87a86d 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:8d7c86bf6a6ad0125733bed6eedd1428bdb95f94bdac288a01be5c7a35bbb501 +- digest: sha256:732bf3d49cacdbb2294032493d79b1dbfb90e441d7a6698c764731cb4e937b9e name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:d8b628409c4482bb72704b2a2a603f803262ca525dbeb7b25db24f216b466f51 +- digest: sha256:821ddbd66d13603dfa60f5a8bab76586a3acdbfd25d71eb2c781fbf9b5ce45ff name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 511dd90159eae7c17e7f7aa1415d0a247454a508 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 06:03:28 +0800 Subject: [PATCH 17/45] feat(security): stage metrics server image --- .../runtime-image-mirror-staging-policy.json | 13 +++++++++++ .../test_runtime_image_mirror_controller.py | 23 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/config/security/runtime-image-mirror-staging-policy.json b/config/security/runtime-image-mirror-staging-policy.json index 38d3c8708..2ffbc5065 100644 --- a/config/security/runtime-image-mirror-staging-policy.json +++ b/config/security/runtime-image-mirror-staging-policy.json @@ -54,6 +54,19 @@ "name": "vpa-recommender", "container": "recommender" } + }, + { + "asset_id": "runtime-image:metrics-server-0.8.1-canary", + "source_index_digest": "sha256:b2d2efaf5ac3b366ed0f839d2412a2c4279d4fc2a2a733f12c52133faed36c41", + "platform_digest": "sha256:6231fb0a1ffab76c92ab880f51a0d11b290f688373647bcedff85af025dfd8a9", + "source_config_digest": "sha256:e76b3f3568b7f440dfd477c1d6de638d7769ba34c93eef999dee418eb72bc0e3", + "push_ref": "registry.wooo.work/awoooi/runtime-mirror/metrics-server:0.8.1-linux-amd64", + "workload": { + "kind": "deployment", + "namespace": "kube-system", + "name": "metrics-server", + "container": "metrics-server" + } } ] } diff --git a/scripts/security/tests/test_runtime_image_mirror_controller.py b/scripts/security/tests/test_runtime_image_mirror_controller.py index 50bed6043..966a63f41 100644 --- a/scripts/security/tests/test_runtime_image_mirror_controller.py +++ b/scripts/security/tests/test_runtime_image_mirror_controller.py @@ -149,12 +149,13 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): self.assertEqual(policy.work_item_id, "AIA-P0-006-02B") self.assertEqual(policy.risk_level, "high") - self.assertEqual(len(policy.images), 3) + self.assertEqual(len(policy.images), 4) by_asset = {image.asset_id: image for image in policy.images} self.assertEqual( set(by_asset), { "runtime-image:argocd-v3.3.6-canary", + "runtime-image:metrics-server-0.8.1-canary", "runtime-image:redis-8.2.3-canary", "runtime-image:vpa-recommender-1.6.0-canary", }, @@ -202,6 +203,26 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): self.assertEqual(vpa.workload.name, "vpa-recommender") self.assertEqual(vpa.workload.container, "recommender") self.assertEqual(vpa.workload.container_kind, "container") + + metrics = by_asset["runtime-image:metrics-server-0.8.1-canary"] + self.assertEqual( + metrics.source_index_digest, + "sha256:b2d2efaf5ac3b366ed0f839d2412a2c4279d4fc2a2a733f12c52133faed36c41", + ) + self.assertEqual( + metrics.platform_digest, + "sha256:6231fb0a1ffab76c92ab880f51a0d11b290f688373647bcedff85af025dfd8a9", + ) + self.assertEqual( + metrics.source_config_digest, + "sha256:e76b3f3568b7f440dfd477c1d6de638d7769ba34c93eef999dee418eb72bc0e3", + ) + self.assertEqual(metrics.runtime_repository, "metrics-server") + self.assertEqual(metrics.workload.kind, "deployment") + self.assertEqual(metrics.workload.namespace, "kube-system") + self.assertEqual(metrics.workload.name, "metrics-server") + self.assertEqual(metrics.workload.container, "metrics-server") + self.assertEqual(metrics.workload.container_kind, "container") self.assertNotIn("github.com", raw) self.assertNotIn("ghcr.io", raw) self.assertIn('"external_pull_allowed": false', raw) From c89869a414016703f4ab642f925a10420f845efd Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 06:28:30 +0800 Subject: [PATCH 18/45] chore(cd): deploy 511dd90 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 119cd2708..d8e4106ed 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" + value: "511dd90159eae7c17e7f7aa1415d0a247454a508" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" + value: "511dd90159eae7c17e7f7aa1415d0a247454a508" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index f5f04cd3b..993e40acc 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" + value: "511dd90159eae7c17e7f7aa1415d0a247454a508" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 867eee454..8222f0b71 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "c5670672323726caaa08af6b1ba88ed26d5d21cc" + value: "511dd90159eae7c17e7f7aa1415d0a247454a508" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index e1d87a86d..f1e14e2e5 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:732bf3d49cacdbb2294032493d79b1dbfb90e441d7a6698c764731cb4e937b9e +- digest: sha256:88b0eac786168f58213cbe53e9ac74de9a571f8611546ddb53df11fef7c69fbd name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:821ddbd66d13603dfa60f5a8bab76586a3acdbfd25d71eb2c781fbf9b5ce45ff +- digest: sha256:0917ca1aefcc50b00d746a5cecd5c3f2a6b9d313e76846b399d31ab6bec16794 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From a7c0b2c1e4dc3f24bb75f5d6f709b6816fb417f4 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 05:45:52 +0800 Subject: [PATCH 19/45] fix(web): render ready asset matrix metrics --- apps/web/src/app/[locale]/awooop/work-items/page.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/web/src/app/[locale]/awooop/work-items/page.tsx b/apps/web/src/app/[locale]/awooop/work-items/page.tsx index dee9dbd0b..e9b0b431f 100644 --- a/apps/web/src/app/[locale]/awooop/work-items/page.tsx +++ b/apps/web/src/app/[locale]/awooop/work-items/page.tsx @@ -12456,6 +12456,7 @@ function AssetCapabilityMatrixPanel({ }) { const t = useTranslations("awooop.workItems.assetCapabilityMatrix"); const matrix = priority?.ai_automation_asset_capability_matrix; + const isLoading = loading && !matrix; const summary = matrix?.summary; const assets = matrix?.assets ?? []; const scopeRollups = matrix?.scope_rollups ?? []; @@ -12529,7 +12530,7 @@ function AssetCapabilityMatrixPanel({ ) : (