From 394e7e481141e52842b8a6a3112c686f28d51609 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 23:04:13 +0800 Subject: [PATCH 1/4] fix(agent): index failed apply retry selection --- apps/api/src/services/awooop_ansible_check_mode_service.py | 6 +++++- apps/api/tests/test_awooop_truth_chain_service.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 332debbfd..e48cf6c35 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -1784,7 +1784,11 @@ async def run_failed_apply_check_mode_replay_once( AND EXISTS ( SELECT 1 FROM incident_evidence verifier - WHERE verifier.post_execution_state ->> 'apply_op_id' + WHERE verifier.incident_id = coalesce( + apply.incident_id::text, + apply.input ->> 'incident_id' + ) + AND verifier.post_execution_state ->> 'apply_op_id' = apply.op_id::text AND verifier.verification_result = 'failed' ) diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 9d40a8506..1a0bd17e5 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -1935,6 +1935,8 @@ def test_failed_apply_retry_replay_is_no_write_and_idempotent() -> None: assert "FOR UPDATE SKIP LOCKED" in source assert "controlled_retry_check_mode_replay" in source assert "build_ansible_check_mode_command" in source + assert "verifier.incident_id = coalesce(" in source + assert "apply.input ->> 'incident_id'" in source assert "controlled_apply_allowed=True" in source assert '"approval_required_before_apply": False' in source assert '"owner_review_required": False' in source From fbc647c222fc77248506e346fc259da9df666ce7 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Fri, 10 Jul 2026 23:10:06 +0800 Subject: [PATCH 2/4] chore(cd): deploy 394e7e4 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index b381790a4..8927e4254 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -84,12 +84,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "682cbcf1ae93b21ad63bc3ac803b1cfa7e856a6a" + value: "394e7e481141e52842b8a6a3112c686f28d51609" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "682cbcf1ae93b21ad63bc3ac803b1cfa7e856a6a" + value: "394e7e481141e52842b8a6a3112c686f28d51609" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 6b6ad09ee..8a437357a 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -41,7 +41,7 @@ resources: images: - name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api - newTag: 682cbcf1ae93b21ad63bc3ac803b1cfa7e856a6a + newTag: 394e7e481141e52842b8a6a3112c686f28d51609 - name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web - newTag: 682cbcf1ae93b21ad63bc3ac803b1cfa7e856a6a + newTag: 394e7e481141e52842b8a6a3112c686f28d51609 From 4476389d5ae17c11c915a8298e44c8e9a87bf38e Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 23:19:51 +0800 Subject: [PATCH 3/4] fix(agent): anchor closure on terminal runs --- .../ai_agent_autonomous_runtime_control.py | 55 ++++++++++++++++++- ...est_ai_agent_autonomous_runtime_control.py | 42 ++++++++++++++ 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/apps/api/src/services/ai_agent_autonomous_runtime_control.py b/apps/api/src/services/ai_agent_autonomous_runtime_control.py index 51c596a09..4dee311fd 100644 --- a/apps/api/src/services/ai_agent_autonomous_runtime_control.py +++ b/apps/api/src/services/ai_agent_autonomous_runtime_control.py @@ -3277,9 +3277,17 @@ def _build_work_item_progress( def _first_operation( rows: Iterable[Mapping[str, Any]], operation_type: str, + *, + statuses: set[str] | None = None, ) -> dict[str, Any] | None: for row in rows: - if str(row.get("operation_type") or "") == operation_type: + if ( + str(row.get("operation_type") or "") == operation_type + and ( + statuses is None + or str(row.get("status") or "") in statuses + ) + ): return dict(row) return None @@ -3303,7 +3311,15 @@ def _missing_runtime_operation_chain_ref_ids( """Return exact apply/check/candidate receipt ids missing from a bounded read.""" operation_rows = [_row_mapping(row) for row in rows] - latest_apply = _first_operation(operation_rows, "ansible_apply_executed") + latest_observed_apply = _first_operation( + operation_rows, + "ansible_apply_executed", + ) + latest_apply = _first_operation( + operation_rows, + "ansible_apply_executed", + statuses={"success", "failed"}, + ) or latest_observed_apply if latest_apply is None: return [] @@ -3481,7 +3497,21 @@ def _autonomous_execution_loop_ledger( telegram_rows = [_row_mapping(row) for row in telegram_latest_rows] auto_repair_rows = [_row_mapping(row) for row in auto_repair_latest_rows] - latest_apply = _first_operation(operation_rows, "ansible_apply_executed") + latest_observed_apply = _first_operation( + operation_rows, + "ansible_apply_executed", + ) + latest_apply = _first_operation( + operation_rows, + "ansible_apply_executed", + statuses={"success", "failed"}, + ) or latest_observed_apply + latest_inflight_apply = ( + latest_observed_apply + if latest_observed_apply is not None + and latest_observed_apply.get("op_id") != (latest_apply or {}).get("op_id") + else None + ) latest_check = None latest_candidate = None if latest_apply is not None: @@ -3717,6 +3747,23 @@ def _autonomous_execution_loop_ledger( "schema_version": "ai_agent_autonomous_execution_loop_ledger_v1", "project_id": project_id, "operation_id": apply_op_id or check_mode_op_id or candidate_op_id or None, + "completion_anchor": ( + "latest_terminal_apply" + if latest_apply is not None + and str(latest_apply.get("status") or "") in {"success", "failed"} + else "latest_observed_operation" + ), + "latest_inflight_apply": ( + { + "op_id": latest_inflight_apply.get("op_id"), + "status": latest_inflight_apply.get("status"), + "automation_run_id": latest_inflight_apply.get("automation_run_id"), + "incident_id": latest_inflight_apply.get("incident_id"), + "created_at": latest_inflight_apply.get("created_at"), + } + if latest_inflight_apply is not None + else None + ), "automation_run_id": automation_run_id or None, "root_candidate_op_id": candidate_op_id or None, "check_mode_op_id": check_mode_op_id or None, @@ -6028,6 +6075,7 @@ _RUNTIME_OPERATION_LATEST_SQL = """ input ->> 'source_candidate_op_id' AS candidate_op_id FROM automation_operation_log WHERE operation_type = 'ansible_apply_executed' + AND status IN ('success', 'failed') ORDER BY created_at DESC LIMIT 1 ) @@ -6104,6 +6152,7 @@ _RUNTIME_OPERATION_LATEST_DIRECT_SQL = """ input ->> 'source_candidate_op_id' AS candidate_op_id FROM automation_operation_log WHERE operation_type = 'ansible_apply_executed' + AND status IN ('success', 'failed') ORDER BY created_at DESC LIMIT 1 ) diff --git a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py index 4ce2e79de..93dd8d46e 100644 --- a/apps/api/tests/test_ai_agent_autonomous_runtime_control.py +++ b/apps/api/tests/test_ai_agent_autonomous_runtime_control.py @@ -2448,11 +2448,53 @@ def test_runtime_telegram_receipt_queries_include_alert_notifications() -> None: def test_runtime_operation_latest_queries_prioritize_latest_apply_chain() -> None: for sql in (_RUNTIME_OPERATION_LATEST_SQL, _RUNTIME_OPERATION_LATEST_DIRECT_SQL): assert "WITH latest_apply_chain AS" in sql + assert "AND status IN ('success', 'failed')" in sql assert "operation_row.op_id::text = latest_apply_chain.apply_op_id" in sql assert "operation_row.op_id::text = latest_apply_chain.check_mode_op_id" in sql assert "operation_row.op_id::text = latest_apply_chain.candidate_op_id" in sql +def test_runtime_execution_loop_uses_terminal_apply_without_hiding_inflight_apply(): + ledger = runtime_control_module._autonomous_execution_loop_ledger( + project_id="awoooi", + operation_latest_rows=[ + { + "op_id": "pending-apply", + "operation_type": "ansible_apply_executed", + "status": "pending", + "automation_run_id": "pending-run", + "incident_id": "INC-PENDING", + "created_at": "2026-07-10T15:00:00Z", + }, + { + "op_id": "terminal-apply", + "operation_type": "ansible_apply_executed", + "status": "failed", + "automation_run_id": "terminal-run", + "incident_id": "INC-TERMINAL", + "created_at": "2026-07-10T14:59:00Z", + }, + ], + verifier_latest_rows=[], + km_latest_rows=[], + telegram_latest_rows=[], + auto_repair_latest_rows=[], + latest_flow_closure={}, + latest_failure_classification={}, + controlled_retry_package={}, + ) + + assert ledger["operation_id"] == "terminal-apply" + assert ledger["completion_anchor"] == "latest_terminal_apply" + assert ledger["latest_inflight_apply"] == { + "op_id": "pending-apply", + "status": "pending", + "automation_run_id": "pending-run", + "incident_id": "INC-PENDING", + "created_at": "2026-07-10T15:00:00Z", + } + + def test_runtime_execution_loop_ledger_does_not_mix_unrelated_check_mode_rows(): apply_op_id = "db3f12ce-08fc-4289-8c93-338305d5850c" readback = build_runtime_receipt_readback_from_rows( From 6a088c46762ebbc01c31533f93905fb22f9c38a6 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Fri, 10 Jul 2026 23:25:30 +0800 Subject: [PATCH 4/4] chore(cd): deploy 4476389 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 8927e4254..966c238f0 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -84,12 +84,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "394e7e481141e52842b8a6a3112c686f28d51609" + value: "4476389d5ae17c11c915a8298e44c8e9a87bf38e" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "394e7e481141e52842b8a6a3112c686f28d51609" + value: "4476389d5ae17c11c915a8298e44c8e9a87bf38e" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 8a437357a..48ad147c6 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -41,7 +41,7 @@ resources: images: - name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api - newTag: 394e7e481141e52842b8a6a3112c686f28d51609 + newTag: 4476389d5ae17c11c915a8298e44c8e9a87bf38e - name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web - newTag: 394e7e481141e52842b8a6a3112c686f28d51609 + newTag: 4476389d5ae17c11c915a8298e44c8e9a87bf38e