From d8320ee7bbf176774155773ca1625eb8be9caeef Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Fri, 10 Jul 2026 16:36:35 +0800 Subject: [PATCH 1/2] chore(cd): deploy ea98503 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 72b7862d6..6bce90686 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -84,12 +84,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "5ea4405c2a75b51244da168a4834d27e24d74ee6" + value: "ea98503a9a8a30003a4314677a5c7f9638c927a9" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "5ea4405c2a75b51244da168a4834d27e24d74ee6" + value: "ea98503a9a8a30003a4314677a5c7f9638c927a9" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 671249d00..daf8aad89 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -41,7 +41,7 @@ resources: images: - name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api - newTag: 5ea4405c2a75b51244da168a4834d27e24d74ee6 + newTag: ea98503a9a8a30003a4314677a5c7f9638c927a9 - name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web - newTag: 5ea4405c2a75b51244da168a4834d27e24d74ee6 + newTag: ea98503a9a8a30003a4314677a5c7f9638c927a9 From 1a5a6a4fc121b3de0c67aaafad262874e6090707 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 16:43:41 +0800 Subject: [PATCH 2/2] fix(agent): reclaim stale check mode executions --- .../awooop_ansible_check_mode_service.py | 188 +++++++++++++++++- .../tests/test_awooop_truth_chain_service.py | 52 +++++ 2 files changed, 237 insertions(+), 3 deletions(-) diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 97f2bdd69..c1b66ba0d 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -585,6 +585,69 @@ def _claim_from_apply_operation_row(row: dict[str, Any]) -> tuple[AnsibleCheckMo return claim, result +def _claim_from_stale_check_mode_row( + row: dict[str, Any], +) -> AnsibleCheckModeClaim | None: + input_payload = _json_loads(row.get("input")) + source_candidate_op_id = str( + input_payload.get("source_candidate_op_id") + or row.get("parent_op_id") + or "" + ) + incident_id = str( + row.get("incident_id") + or input_payload.get("incident_id") + or "" + ) + catalog_id = str(input_payload.get("catalog_id") or "") + inventory_hosts = input_payload.get("inventory_hosts") + source_playbook_path = str( + input_payload.get("source_candidate_playbook_path") + or input_payload.get("catalog_playbook_path") + or input_payload.get("apply_playbook_path") + or input_payload.get("playbook_path") + or "" + ) + if ( + not source_candidate_op_id + or not incident_id + or not catalog_id + or not source_playbook_path + or not isinstance(inventory_hosts, list) + or not inventory_hosts + ): + return None + try: + claim_input = build_ansible_check_mode_claim_input( + source_candidate_op_id=source_candidate_op_id, + candidate_input={ + "incident_id": incident_id, + "executor": "ansible", + "executor_candidates": [ + { + "catalog_id": catalog_id, + "playbook_path": source_playbook_path, + "inventory_hosts": inventory_hosts, + "risk_level": input_payload.get("risk_level"), + } + ], + }, + ) + except ValueError: + return None + return AnsibleCheckModeClaim( + op_id=str(row.get("op_id") or ""), + source_candidate_op_id=source_candidate_op_id, + incident_id=incident_id, + catalog_id=str(claim_input["catalog_id"]), + playbook_path=str(claim_input["playbook_path"]), + apply_playbook_path=str(claim_input["apply_playbook_path"]), + inventory_hosts=tuple(str(host) for host in claim_input["inventory_hosts"]), + risk_level=str(claim_input.get("risk_level") or ""), + input_payload=claim_input, + ) + + async def _record_auto_repair_execution_receipt( claim: AnsibleCheckModeClaim, result: AnsibleRunResult, @@ -1227,6 +1290,94 @@ async def claim_pending_check_modes( return claims +async def claim_stale_pending_check_modes( + *, + project_id: str = "awoooi", + limit: int = 1, + stale_after_seconds: int, +) -> list[AnsibleCheckModeClaim]: + """Lease and revalidate stale pending rows before replaying check-mode.""" + + claims: list[AnsibleCheckModeClaim] = [] + async with get_db_context(project_id) as db: + result = await db.execute( + text(""" + SELECT + check_mode.op_id, + check_mode.parent_op_id, + coalesce( + check_mode.incident_id::text, + check_mode.input ->> 'incident_id' + ) AS incident_id, + check_mode.input + FROM automation_operation_log check_mode + WHERE check_mode.operation_type = 'ansible_check_mode_executed' + AND check_mode.status = 'pending' + AND coalesce( + nullif( + check_mode.dry_run_result ->> 'reclaimed_at', + '' + )::timestamptz, + check_mode.created_at + ) <= NOW() - (:stale_after_seconds * INTERVAL '1 second') + ORDER BY check_mode.created_at DESC + LIMIT :limit + FOR UPDATE SKIP LOCKED + """), + { + "limit": max(1, limit), + "stale_after_seconds": max(300, stale_after_seconds), + }, + ) + for row_value in result.mappings().all(): + row = dict(row_value) + claim = _claim_from_stale_check_mode_row(row) + if claim is None: + await db.execute( + text(""" + UPDATE automation_operation_log + SET status = 'failed', + error = 'stale_pending_reclaim_rejected_by_current_policy', + dry_run_result = coalesce( + dry_run_result, + '{}'::jsonb + ) || jsonb_build_object( + 'claim_state', 'stale_reclaim_rejected', + 'reclaimed_at', NOW(), + 'check_mode_executed', false + ) + WHERE op_id = CAST(:op_id AS uuid) + AND status = 'pending' + """), + {"op_id": str(row.get("op_id") or "")}, + ) + continue + await db.execute( + text(""" + UPDATE automation_operation_log + SET input = CAST(:input AS jsonb), + error = NULL, + dry_run_result = coalesce( + dry_run_result, + '{}'::jsonb + ) || jsonb_build_object( + 'claim_state', 'stale_reclaimed', + 'reclaimed_at', NOW(), + 'check_mode_executed', false, + 'apply_executed', false + ) + WHERE op_id = CAST(:op_id AS uuid) + AND status = 'pending' + """), + { + "input": json.dumps(claim.input_payload, ensure_ascii=False), + "op_id": claim.op_id, + }, + ) + claims.append(claim) + return claims + + async def recent_ansible_transport_blockers( *, project_id: str = "awoooi", @@ -1355,6 +1506,12 @@ async def finalize_check_mode_claim( text(""" UPDATE automation_operation_log SET status = :status, + input = jsonb_set( + coalesce(input, '{}'::jsonb), + '{automation_run_id}', + to_jsonb(CAST(:automation_run_id AS text)), + true + ), output = CAST(:output AS jsonb), dry_run_result = CAST(:dry_run_result AS jsonb), error = :error, @@ -1364,6 +1521,10 @@ async def finalize_check_mode_claim( """), { "status": status, + "automation_run_id": str( + claim.input_payload.get("automation_run_id") + or claim.source_candidate_op_id + ), "output": json.dumps(output, ensure_ascii=False), "dry_run_result": json.dumps(dry_run_result, ensure_ascii=False), "error": _tail(error or "", 2000) or None, @@ -1450,6 +1611,10 @@ async def run_controlled_apply_for_claim( "incident_db_id": _automation_operation_log_incident_id(claim.incident_id), "input": json.dumps({ **claim.input_payload, + "automation_run_id": str( + claim.input_payload.get("automation_run_id") + or claim.source_candidate_op_id + ), "execution_mode": "controlled_apply", "check_mode_op_id": claim.op_id, "playbook_path": claim.apply_playbook_path, @@ -1566,11 +1731,27 @@ async def run_pending_check_modes_once( logger.warning("ansible_check_mode_transport_blocked", blockers=transport_blockers) return {"claimed": 0, "completed": 0, "failed": 0, "blockers": transport_blockers} - claims = await claim_pending_check_modes( + effective_timeout_seconds = ( + timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS + ) + reclaimed_claims = await claim_stale_pending_check_modes( project_id=project_id, limit=limit, - candidate_max_age_hours=settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS, + stale_after_seconds=max(300, effective_timeout_seconds + 120), ) + remaining_limit = max(0, limit - len(reclaimed_claims)) + fresh_claims = ( + await claim_pending_check_modes( + project_id=project_id, + limit=remaining_limit, + candidate_max_age_hours=( + settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS + ), + ) + if remaining_limit + else [] + ) + claims = [*reclaimed_claims, *fresh_claims] completed = 0 failed = 0 apply_completed = 0 @@ -1579,7 +1760,7 @@ async def run_pending_check_modes_once( for claim in claims: result = await run_claimed_check_mode( claim, - timeout_seconds=timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS, + timeout_seconds=effective_timeout_seconds, project_id=project_id, ) completed += 1 @@ -1599,6 +1780,7 @@ async def run_pending_check_modes_once( apply_failed += 1 return { "claimed": len(claims), + "reclaimed": len(reclaimed_claims), "completed": completed, "failed": failed, "apply_completed": apply_completed, diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 95fd4f392..2fa3b3bfa 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -18,6 +18,7 @@ from src.services.awooop_ansible_check_mode_service import ( _automation_operation_log_incident_id, _build_auto_repair_execution_receipt, _claim_from_apply_operation_row, + _claim_from_stale_check_mode_row, _post_apply_km_path_type, _post_apply_verification_result, _record_auto_repair_execution_receipt, @@ -28,9 +29,12 @@ from src.services.awooop_ansible_check_mode_service import ( build_ansible_check_mode_claim_input, build_ansible_check_mode_command, claim_pending_check_modes, + claim_stale_pending_check_modes, detect_ansible_transport_blockers, + finalize_check_mode_claim, recent_ansible_transport_blockers, run_controlled_apply_for_claim, + run_pending_check_modes_once, ) from src.services.awooop_truth_chain_service import ( _ansible_playbook_roots, @@ -1639,6 +1643,46 @@ def test_ansible_apply_operation_row_reconstructs_from_input_without_physical_co assert result.returncode == 0 +def test_stale_check_mode_row_is_revalidated_with_canonical_run_id() -> None: + source_candidate_op_id = "00000000-0000-0000-0000-000000000091" + claim = _claim_from_stale_check_mode_row({ + "op_id": "00000000-0000-0000-0000-000000000092", + "parent_op_id": source_candidate_op_id, + "incident_id": "INC-20260709-46478E", + "input": { + "automation_run_id": "untrusted-old-run-id", + "incident_id": "INC-20260709-46478E", + "source_candidate_op_id": source_candidate_op_id, + "catalog_id": "ansible:188-momo-backup-user", + "catalog_playbook_path": ( + "infra/ansible/playbooks/188-momo-backup-user.yml" + ), + "inventory_hosts": ["host_188"], + "risk_level": "low", + }, + }) + + assert claim is not None + assert claim.op_id == "00000000-0000-0000-0000-000000000092" + assert claim.source_candidate_op_id == source_candidate_op_id + assert claim.input_payload["automation_run_id"] == source_candidate_op_id + assert claim.input_payload["controlled_apply_allowed"] is True + assert claim.playbook_path == "infra/ansible/playbooks/188-momo-backup-user.yml" + + +def test_stale_check_mode_reclaim_uses_lease_lock_and_current_policy() -> None: + source = inspect.getsource(claim_stale_pending_check_modes) + run_source = inspect.getsource(run_pending_check_modes_once) + + assert "FOR UPDATE SKIP LOCKED" in source + assert "reclaimed_at" in source + assert "ORDER BY check_mode.created_at DESC" in source + assert "stale_pending_reclaim_rejected_by_current_policy" in source + assert "_claim_from_stale_check_mode_row" in source + assert "claim_stale_pending_check_modes" in run_source + assert "effective_timeout_seconds + 120" in run_source + + def test_ansible_apply_receipt_backfill_queries_existing_apply_rows() -> None: source = inspect.getsource(backfill_missing_auto_repair_execution_receipts_once) @@ -1733,6 +1777,14 @@ def test_ansible_live_controlled_apply_sends_telegram_receipt_but_backfill_does_ assert inspect.iscoroutinefunction(_send_controlled_apply_telegram_receipt) +def test_ansible_check_and_apply_rows_persist_canonical_automation_run_id() -> None: + finalize_source = inspect.getsource(finalize_check_mode_claim) + apply_source = inspect.getsource(run_controlled_apply_for_claim) + + assert "'{automation_run_id}'" in finalize_source + assert '"automation_run_id": str(' in apply_source + + def test_ansible_post_apply_verifier_helpers_are_deterministic() -> None: assert _post_apply_km_path_type("03ca6836-1b76-4da2-8e3e-6d3b6df9254a") == ( "ansible_apply_receipt:03ca6836"