Merge remote-tracking branch 'origin/main' into codex/agent99-alert-control-20260710

apps/api/src/services/awooop_ansible_check_mode_service.py

@@ -585,6 +585,69 @@ def _claim_from_apply_operation_row(row: dict[str, Any]) -> tuple[AnsibleCheckMo
     return claim, result
 
 
+def _claim_from_stale_check_mode_row(
+    row: dict[str, Any],
+) -> AnsibleCheckModeClaim | None:
+    input_payload = _json_loads(row.get("input"))
+    source_candidate_op_id = str(
+        input_payload.get("source_candidate_op_id")
+        or row.get("parent_op_id")
+        or ""
+    )
+    incident_id = str(
+        row.get("incident_id")
+        or input_payload.get("incident_id")
+        or ""
+    )
+    catalog_id = str(input_payload.get("catalog_id") or "")
+    inventory_hosts = input_payload.get("inventory_hosts")
+    source_playbook_path = str(
+        input_payload.get("source_candidate_playbook_path")
+        or input_payload.get("catalog_playbook_path")
+        or input_payload.get("apply_playbook_path")
+        or input_payload.get("playbook_path")
+        or ""
+    )
+    if (
+        not source_candidate_op_id
+        or not incident_id
+        or not catalog_id
+        or not source_playbook_path
+        or not isinstance(inventory_hosts, list)
+        or not inventory_hosts
+    ):
+        return None
+    try:
+        claim_input = build_ansible_check_mode_claim_input(
+            source_candidate_op_id=source_candidate_op_id,
+            candidate_input={
+                "incident_id": incident_id,
+                "executor": "ansible",
+                "executor_candidates": [
+                    {
+                        "catalog_id": catalog_id,
+                        "playbook_path": source_playbook_path,
+                        "inventory_hosts": inventory_hosts,
+                        "risk_level": input_payload.get("risk_level"),
+                    }
+                ],
+            },
+        )
+    except ValueError:
+        return None
+    return AnsibleCheckModeClaim(
+        op_id=str(row.get("op_id") or ""),
+        source_candidate_op_id=source_candidate_op_id,
+        incident_id=incident_id,
+        catalog_id=str(claim_input["catalog_id"]),
+        playbook_path=str(claim_input["playbook_path"]),
+        apply_playbook_path=str(claim_input["apply_playbook_path"]),
+        inventory_hosts=tuple(str(host) for host in claim_input["inventory_hosts"]),
+        risk_level=str(claim_input.get("risk_level") or ""),
+        input_payload=claim_input,
+    )
+
+
 async def _record_auto_repair_execution_receipt(
     claim: AnsibleCheckModeClaim,
     result: AnsibleRunResult,
@@ -1227,6 +1290,94 @@ async def claim_pending_check_modes(
     return claims
 
 
+async def claim_stale_pending_check_modes(
+    *,
+    project_id: str = "awoooi",
+    limit: int = 1,
+    stale_after_seconds: int,
+) -> list[AnsibleCheckModeClaim]:
+    """Lease and revalidate stale pending rows before replaying check-mode."""
+
+    claims: list[AnsibleCheckModeClaim] = []
+    async with get_db_context(project_id) as db:
+        result = await db.execute(
+            text("""
+                SELECT
+                    check_mode.op_id,
+                    check_mode.parent_op_id,
+                    coalesce(
+                        check_mode.incident_id::text,
+                        check_mode.input ->> 'incident_id'
+                    ) AS incident_id,
+                    check_mode.input
+                FROM automation_operation_log check_mode
+                WHERE check_mode.operation_type = 'ansible_check_mode_executed'
+                  AND check_mode.status = 'pending'
+                  AND coalesce(
+                        nullif(
+                            check_mode.dry_run_result ->> 'reclaimed_at',
+                            ''
+                        )::timestamptz,
+                        check_mode.created_at
+                      ) <= NOW() - (:stale_after_seconds * INTERVAL '1 second')
+                ORDER BY check_mode.created_at DESC
+                LIMIT :limit
+                FOR UPDATE SKIP LOCKED
+            """),
+            {
+                "limit": max(1, limit),
+                "stale_after_seconds": max(300, stale_after_seconds),
+            },
+        )
+        for row_value in result.mappings().all():
+            row = dict(row_value)
+            claim = _claim_from_stale_check_mode_row(row)
+            if claim is None:
+                await db.execute(
+                    text("""
+                        UPDATE automation_operation_log
+                        SET status = 'failed',
+                            error = 'stale_pending_reclaim_rejected_by_current_policy',
+                            dry_run_result = coalesce(
+                                dry_run_result,
+                                '{}'::jsonb
+                            ) || jsonb_build_object(
+                                'claim_state', 'stale_reclaim_rejected',
+                                'reclaimed_at', NOW(),
+                                'check_mode_executed', false
+                            )
+                        WHERE op_id = CAST(:op_id AS uuid)
+                          AND status = 'pending'
+                    """),
+                    {"op_id": str(row.get("op_id") or "")},
+                )
+                continue
+            await db.execute(
+                text("""
+                    UPDATE automation_operation_log
+                    SET input = CAST(:input AS jsonb),
+                        error = NULL,
+                        dry_run_result = coalesce(
+                            dry_run_result,
+                            '{}'::jsonb
+                        ) || jsonb_build_object(
+                            'claim_state', 'stale_reclaimed',
+                            'reclaimed_at', NOW(),
+                            'check_mode_executed', false,
+                            'apply_executed', false
+                        )
+                    WHERE op_id = CAST(:op_id AS uuid)
+                      AND status = 'pending'
+                """),
+                {
+                    "input": json.dumps(claim.input_payload, ensure_ascii=False),
+                    "op_id": claim.op_id,
+                },
+            )
+            claims.append(claim)
+    return claims
+
+
 async def recent_ansible_transport_blockers(
     *,
     project_id: str = "awoooi",
@@ -1355,6 +1506,12 @@ async def finalize_check_mode_claim(
             text("""
                 UPDATE automation_operation_log
                 SET status = :status,
+                    input = jsonb_set(
+                        coalesce(input, '{}'::jsonb),
+                        '{automation_run_id}',
+                        to_jsonb(CAST(:automation_run_id AS text)),
+                        true
+                    ),
                     output = CAST(:output AS jsonb),
                     dry_run_result = CAST(:dry_run_result AS jsonb),
                     error = :error,
@@ -1364,6 +1521,10 @@ async def finalize_check_mode_claim(
             """),
             {
                 "status": status,
+                "automation_run_id": str(
+                    claim.input_payload.get("automation_run_id")
+                    or claim.source_candidate_op_id
+                ),
                 "output": json.dumps(output, ensure_ascii=False),
                 "dry_run_result": json.dumps(dry_run_result, ensure_ascii=False),
                 "error": _tail(error or "", 2000) or None,
@@ -1450,6 +1611,10 @@ async def run_controlled_apply_for_claim(
                 "incident_db_id": _automation_operation_log_incident_id(claim.incident_id),
                 "input": json.dumps({
                     **claim.input_payload,
+                    "automation_run_id": str(
+                        claim.input_payload.get("automation_run_id")
+                        or claim.source_candidate_op_id
+                    ),
                     "execution_mode": "controlled_apply",
                     "check_mode_op_id": claim.op_id,
                     "playbook_path": claim.apply_playbook_path,
@@ -1566,11 +1731,27 @@ async def run_pending_check_modes_once(
         logger.warning("ansible_check_mode_transport_blocked", blockers=transport_blockers)
         return {"claimed": 0, "completed": 0, "failed": 0, "blockers": transport_blockers}
 
-    claims = await claim_pending_check_modes(
+    effective_timeout_seconds = (
+        timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS
+    )
+    reclaimed_claims = await claim_stale_pending_check_modes(
         project_id=project_id,
         limit=limit,
-        candidate_max_age_hours=settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS,
+        stale_after_seconds=max(300, effective_timeout_seconds + 120),
     )
+    remaining_limit = max(0, limit - len(reclaimed_claims))
+    fresh_claims = (
+        await claim_pending_check_modes(
+            project_id=project_id,
+            limit=remaining_limit,
+            candidate_max_age_hours=(
+                settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS
+            ),
+        )
+        if remaining_limit
+        else []
+    )
+    claims = [*reclaimed_claims, *fresh_claims]
     completed = 0
     failed = 0
     apply_completed = 0
@@ -1579,7 +1760,7 @@ async def run_pending_check_modes_once(
     for claim in claims:
         result = await run_claimed_check_mode(
             claim,
-            timeout_seconds=timeout_seconds or settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS,
+            timeout_seconds=effective_timeout_seconds,
             project_id=project_id,
         )
         completed += 1
@@ -1599,6 +1780,7 @@ async def run_pending_check_modes_once(
                 apply_failed += 1
     return {
         "claimed": len(claims),
+        "reclaimed": len(reclaimed_claims),
         "completed": completed,
         "failed": failed,
         "apply_completed": apply_completed,

apps/api/tests/test_awooop_truth_chain_service.py

@@ -18,6 +18,7 @@ from src.services.awooop_ansible_check_mode_service import (
     _automation_operation_log_incident_id,
     _build_auto_repair_execution_receipt,
     _claim_from_apply_operation_row,
+    _claim_from_stale_check_mode_row,
     _post_apply_km_path_type,
     _post_apply_verification_result,
     _record_auto_repair_execution_receipt,
@@ -28,9 +29,12 @@ from src.services.awooop_ansible_check_mode_service import (
     build_ansible_check_mode_claim_input,
     build_ansible_check_mode_command,
     claim_pending_check_modes,
+    claim_stale_pending_check_modes,
     detect_ansible_transport_blockers,
+    finalize_check_mode_claim,
     recent_ansible_transport_blockers,
     run_controlled_apply_for_claim,
+    run_pending_check_modes_once,
 )
 from src.services.awooop_truth_chain_service import (
     _ansible_playbook_roots,
@@ -1639,6 +1643,46 @@ def test_ansible_apply_operation_row_reconstructs_from_input_without_physical_co
     assert result.returncode == 0
 
 
+def test_stale_check_mode_row_is_revalidated_with_canonical_run_id() -> None:
+    source_candidate_op_id = "00000000-0000-0000-0000-000000000091"
+    claim = _claim_from_stale_check_mode_row({
+        "op_id": "00000000-0000-0000-0000-000000000092",
+        "parent_op_id": source_candidate_op_id,
+        "incident_id": "INC-20260709-46478E",
+        "input": {
+            "automation_run_id": "untrusted-old-run-id",
+            "incident_id": "INC-20260709-46478E",
+            "source_candidate_op_id": source_candidate_op_id,
+            "catalog_id": "ansible:188-momo-backup-user",
+            "catalog_playbook_path": (
+                "infra/ansible/playbooks/188-momo-backup-user.yml"
+            ),
+            "inventory_hosts": ["host_188"],
+            "risk_level": "low",
+        },
+    })
+
+    assert claim is not None
+    assert claim.op_id == "00000000-0000-0000-0000-000000000092"
+    assert claim.source_candidate_op_id == source_candidate_op_id
+    assert claim.input_payload["automation_run_id"] == source_candidate_op_id
+    assert claim.input_payload["controlled_apply_allowed"] is True
+    assert claim.playbook_path == "infra/ansible/playbooks/188-momo-backup-user.yml"
+
+
+def test_stale_check_mode_reclaim_uses_lease_lock_and_current_policy() -> None:
+    source = inspect.getsource(claim_stale_pending_check_modes)
+    run_source = inspect.getsource(run_pending_check_modes_once)
+
+    assert "FOR UPDATE SKIP LOCKED" in source
+    assert "reclaimed_at" in source
+    assert "ORDER BY check_mode.created_at DESC" in source
+    assert "stale_pending_reclaim_rejected_by_current_policy" in source
+    assert "_claim_from_stale_check_mode_row" in source
+    assert "claim_stale_pending_check_modes" in run_source
+    assert "effective_timeout_seconds + 120" in run_source
+
+
 def test_ansible_apply_receipt_backfill_queries_existing_apply_rows() -> None:
     source = inspect.getsource(backfill_missing_auto_repair_execution_receipts_once)
 
@@ -1733,6 +1777,14 @@ def test_ansible_live_controlled_apply_sends_telegram_receipt_but_backfill_does_
     assert inspect.iscoroutinefunction(_send_controlled_apply_telegram_receipt)
 
 
+def test_ansible_check_and_apply_rows_persist_canonical_automation_run_id() -> None:
+    finalize_source = inspect.getsource(finalize_check_mode_claim)
+    apply_source = inspect.getsource(run_controlled_apply_for_claim)
+
+    assert "'{automation_run_id}'" in finalize_source
+    assert '"automation_run_id": str(' in apply_source
+
+
 def test_ansible_post_apply_verifier_helpers_are_deterministic() -> None:
     assert _post_apply_km_path_type("03ca6836-1b76-4da2-8e3e-6d3b6df9254a") == (
         "ansible_apply_receipt:03ca6836"

k8s/awoooi-prod/06-deployment-api.yaml

@@ -84,12 +84,12 @@ spec:
             - name: AWOOOI_BUILD_COMMIT_SHA
               # 2026-06-29 Codex: CD rewrites this to the deployed image tag so
               # production deploy readback does not rely on a stale static snapshot.
-              value: "5ea4405c2a75b51244da168a4834d27e24d74ee6"
+              value: "ea98503a9a8a30003a4314677a5c7f9638c927a9"
             - name: AWOOOI_DESIRED_API_IMAGE_TAG
               # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA.
               # Production readback compares runtime image truth against this
               # GitOps desired tag instead of doing a slow Gitea raw fetch.
-              value: "5ea4405c2a75b51244da168a4834d27e24d74ee6"
+              value: "ea98503a9a8a30003a4314677a5c7f9638c927a9"
             - name: DATABASE_POOL_SIZE
               # 2026-07-01 Codex: production role `awoooi` currently has a low
               # connection limit. Keep API pool conservative until DB role

k8s/awoooi-prod/kustomization.yaml

@@ -41,7 +41,7 @@ resources:
 images:
 - name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER
   newName: 192.168.0.110:5000/awoooi/api
-  newTag: 5ea4405c2a75b51244da168a4834d27e24d74ee6
+  newTag: ea98503a9a8a30003a4314677a5c7f9638c927a9
 - name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER
   newName: 192.168.0.110:5000/awoooi/web
-  newTag: 5ea4405c2a75b51244da168a4834d27e24d74ee6
+  newTag: ea98503a9a8a30003a4314677a5c7f9638c927a9