From 7c7784cf66a69bc065c5cd443d1926654f0f7e25 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 22:20:34 +0800 Subject: [PATCH 1/5] fix(security): pin Argo CD copyutil canary --- .../security/runtime-image-mirror-policy.json | 15 ++++++++++ .../test_runtime_image_mirror_controller.py | 29 ++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/config/security/runtime-image-mirror-policy.json b/config/security/runtime-image-mirror-policy.json index 8e34af8f9..98a7a732b 100644 --- a/config/security/runtime-image-mirror-policy.json +++ b/config/security/runtime-image-mirror-policy.json @@ -29,6 +29,21 @@ "container": "dex" } }, + { + "asset_id": "runtime-image:argocd-copyutil", + "source_index_digest": "sha256:16b92ba472fbb9287459cc52e0ecff07288dff461209955098edb56ce866fe49", + "platform_digest": "sha256:2f25a42949ea69c0dd33f4ce1918c6a01039d6c14a7ecc1d19088504a9d3e94f", + "target_registry_digest": "sha256:8b7e25d7e6036d6750c5bda8920cbd80d8902c25771666f6767dc97bd7fab8fc", + "push_ref": "registry.wooo.work/awoooi/runtime-mirror/argocd:v3.3.6-linux-amd64", + "runtime_ref": "192.168.0.110:5000/awoooi/runtime-mirror/argocd@sha256:8b7e25d7e6036d6750c5bda8920cbd80d8902c25771666f6767dc97bd7fab8fc", + "workload": { + "kind": "deployment", + "namespace": "argocd", + "name": "argocd-dex-server", + "container": "copyutil", + "container_kind": "init_container" + } + }, { "asset_id": "runtime-image:kured", "source_index_digest": "sha256:2c5d73bb4517a269def38a6cd54d34d82be81793bea9ff1bb35c6533515ad209", diff --git a/scripts/security/tests/test_runtime_image_mirror_controller.py b/scripts/security/tests/test_runtime_image_mirror_controller.py index 0a5e9b375..9fecee9da 100644 --- a/scripts/security/tests/test_runtime_image_mirror_controller.py +++ b/scripts/security/tests/test_runtime_image_mirror_controller.py @@ -47,7 +47,7 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): self.assertEqual(policy.work_item_id, "AIA-P0-006-02A") self.assertEqual(policy.risk_level, "high") - self.assertEqual(len(policy.images), 5) + self.assertEqual(len(policy.images), 6) self.assertNotIn("github.com", raw) self.assertNotIn("ghcr.io", raw) self.assertIn('"external_pull_allowed": false', raw) @@ -71,9 +71,9 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): for image in policy.images ) ) - self.assertTrue( - all(image.workload.container_kind == "container" for image in policy.images) - ) + container_kinds = [image.workload.container_kind for image in policy.images] + self.assertEqual(container_kinds.count("container"), 5) + self.assertEqual(container_kinds.count("init_container"), 1) def test_policy_loads_and_validates_init_container_kind(self) -> None: image = self._image_with_container_kind("init_container") @@ -103,6 +103,27 @@ class RuntimeImageMirrorControllerTest(unittest.TestCase): self.assertNotIn("ghcr.io", raw) self.assertIn('"external_pull_allowed": false', raw) + def test_argocd_copyutil_canary_matches_verified_staging_contract(self) -> None: + policy = controller.load_policy(POLICY_PATH) + staging = controller.load_staging_policy(STAGING_POLICY_PATH).images[0] + canary = next( + image + for image in policy.images + if image.asset_id == "runtime-image:argocd-copyutil" + ) + + self.assertEqual(canary.source_index_digest, staging.source_index_digest) + self.assertEqual(canary.platform_digest, staging.platform_digest) + self.assertEqual(canary.push_ref, staging.push_ref) + self.assertEqual(canary.workload, staging.workload) + self.assertEqual( + canary.target_registry_digest, + "sha256:8b7e25d7e6036d6750c5bda8920cbd80d8902c25771666f6767dc97bd7fab8fc", + ) + self.assertTrue( + canary.runtime_ref.endswith("@" + canary.target_registry_digest) + ) + def test_policy_rejects_external_pull_and_mutable_runtime_ref(self) -> None: payload = json.loads(POLICY_PATH.read_text(encoding="utf-8")) with tempfile.TemporaryDirectory() as temp: From 8939e6e92fe0994444a167a1ded2a2aea3021a6b Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Tue, 14 Jul 2026 22:37:04 +0800 Subject: [PATCH 2/5] chore(cd): deploy 7c7784c [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 3b44746f9..3e4cf593d 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "af03e863cb0737737542bf90b941eceed84d703d" + value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "af03e863cb0737737542bf90b941eceed84d703d" + value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index b020aa2fb..23e4f36e9 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "af03e863cb0737737542bf90b941eceed84d703d" + value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index d3daba937..5b16368d4 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "af03e863cb0737737542bf90b941eceed84d703d" + value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index fae68ff4f..49a3c5d35 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:5295557b830877fcf8e93506bc7790679b9b8ef347b3144fa5a3557e5f26d7f3 +- digest: sha256:005354b7665fcb10014df8e51c88f3a32e5c384618f0057203b4f85c6e9c91c0 name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:d367bfc1c7ab31fecf6fb147b0da572627f74d88267c206c0e9c99a519b956c9 +- digest: sha256:fc336fee71de11c657163e378fe98a52fccf5cab3d9410b80b126c9b5edcda20 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 1de58df5099960639537234b2799be507b86e18b Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 22:38:43 +0800 Subject: [PATCH 3/5] fix(agent99): reconcile accepted ignored dispatch safely --- agent99-sre-alert-relay.ps1 | 31 +++++++++++++++++-- .../tests/test_agent99_same_run_reconcile.py | 11 +++++++ .../deploy-agent99-via-windows99-ssh.sh | 6 +++- ...remote_atomic_deploy_transport_contract.py | 3 ++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index 283860c36..f158ea7f2 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -17,6 +17,7 @@ $BinDir = Join-Path $AgentRoot "bin" $QueueDir = Join-Path $AgentRoot "queue" $ProcessedDir = Join-Path $QueueDir "processed" $SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile" +$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored" $SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir | Out-Null @@ -381,6 +382,9 @@ function Convert-AgentStoredDispatchIdentity { $identity = Get-AgentField $Stored "identity" $null if ($identity) { return $identity } + $awoooi = Get-AgentField $Stored "awoooi" $null + $acceptedAlertIdentity = Get-AgentField $awoooi "agent99DispatchIdentity" $null + if ($acceptedAlertIdentity) { return $acceptedAlertIdentity } return [pscustomobject]@{ schema_version = "agent99_controlled_dispatch_identity_v1" project_id = [string](Get-AgentField $Stored "projectId" "") @@ -407,13 +411,32 @@ function Get-AgentExistingRecoverIdentity { $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "running-*.json" -File -ErrorAction SilentlyContinue) $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running|same-run-status)-" }) - foreach ($file in @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)) { + # The API's durable receipt predates the inbox's final classification. The + # exact legacy Recover envelope was accepted by the relay and then moved to + # ignored by an older inbox build. Admit only that immutable source envelope + # as identity evidence so the externally-resolved incident can run Status; + # never replay Recover or accept a generic ignored/reconcile payload. + $acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json" + $files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256) + $files += @(Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern -File -ErrorAction SilentlyContinue) + foreach ($file in $files) { try { $stored = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json if ([bool](Get-AgentField $stored "reconcileOnly" $false)) { continue } - if ([string](Get-AgentField $stored "mode" "") -ne "Recover") { continue } - if ([string](Get-AgentField $stored "automationRunId" "") -ne $RunId) { continue } + $storedMode = [string](Get-AgentField $stored "mode" (Get-AgentField $stored "suggestedMode" "")) + if ($storedMode -ne "Recover") { continue } $identity = Convert-AgentStoredDispatchIdentity $stored + if ([string](Get-AgentField $identity "run_id" "") -ne $RunId) { continue } + if ($file.DirectoryName -eq $AcceptedIgnoredDir) { + if ( + [string](Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId" -or + [string](Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager" -or + [string](Get-AgentField $stored "kind" "") -ne "host_recovery" -or + [string](Get-AgentField $stored "service" "") -ne "cold-start-gate" -or + (Get-AgentField $stored "controlledApply" $null) -isnot [bool] -or + -not [bool](Get-AgentField $stored "controlledApply" $false) + ) { continue } + } $check = Test-AgentSameRunDispatchIdentity $identity if ($check.ok) { return $identity } } catch { @@ -782,6 +805,8 @@ try { event = "same_run_status_reconcile" ok = [bool]$reconcile.ok status = [string]$reconcile.status + reason = [string](Get-AgentField $reconcile "reason" "") + httpStatus = [int]$reconcile.httpStatus remote = $remote runId = if ($reconcile.ok) { [string]$reconcile.automationRunId } else { "rejected" } reconcileOnly = $true diff --git a/apps/api/tests/test_agent99_same_run_reconcile.py b/apps/api/tests/test_agent99_same_run_reconcile.py index da80a582f..05063f0d3 100644 --- a/apps/api/tests/test_agent99_same_run_reconcile.py +++ b/apps/api/tests/test_agent99_same_run_reconcile.py @@ -734,6 +734,15 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> assert 'work_item_id" "") -ne "agent99-dispatch:awoooi:' in relay assert 'trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in relay assert "function Get-AgentExistingRecoverIdentity" in relay + assert '$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"' in relay + assert '$acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json"' in relay + assert '$files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)' in relay + assert 'Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern' in relay + assert 'Get-AgentField $awoooi "agent99DispatchIdentity"' in relay + assert 'Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId"' in relay + assert 'Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager"' in relay + assert 'Get-AgentField $stored "kind" "") -ne "host_recovery"' in relay + assert 'Get-AgentField $stored "service" "") -ne "cold-start-gate"' in relay assert "existing_dispatch_identity_not_found" in relay assert "existing_dispatch_identity_mismatch" in relay assert "function Test-AgentSameRunOutcomeEligible" in relay @@ -748,6 +757,8 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> assert 'reconcileOnly = $true' in relay assert 'suppressTelegram = $true' in relay assert 'sourceEventResolutionPolicy = "external_source_receipt_required"' in relay + assert 'reason = [string](Get-AgentField $reconcile "reason" "")' in relay + assert 'httpStatus = [int]$reconcile.httpStatus' in relay assert '"-ReconcileQueueOnly", "-ReconcileQueueId", $queueId' in relay assert '"-SuppressAlerts", "-SuppressReason"' in relay diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index 76b773ff0..12e557f03 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -426,7 +426,11 @@ if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p $c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''} try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name} $z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false} -[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status='check_ready';mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=14;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction='rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}|ConvertTo-Json -Compress -Depth 8''' +$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0) +$status=if($ready){'check_ready'}else{'check_failed_no_apply'} +$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'} +[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=14;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 +if(-not$ready){exit 65}''' script = ( script.replace("__SOURCE_REVISION__", source_revision) .replace("__MANIFEST_SHA256__", manifest_sha256) diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 522ec2a0b..15aa9709d 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -671,6 +671,9 @@ def test_check_mode_uses_bounded_control_command_without_ssh_stdin( assert "agent99-contract-check.ps1" in check_script assert "Get-FileHash" in check_script assert "runtimeContract" in check_script + assert "$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)" in check_script + assert "'check_failed_no_apply'" in check_script + assert "if(-not$ready){exit 65}" in check_script assert check_script.count("ProgressPreference") == 1 assert "remoteWritePerformed" in check_script assert "$z[$n]=$false" in check_script From 83180367d19768bde7008254a962d3a13d89209a Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 22:17:39 +0800 Subject: [PATCH 4/5] fix(agent): replay repaired failed applies --- .../services/awooop_ansible_audit_service.py | 2 +- .../awooop_ansible_check_mode_service.py | 256 ++++++++++++++++++ .../test_awooop_ansible_post_verifier.py | 2 +- ...t_host110_node_exporter_ansible_catalog.py | 104 ++++++- .../playbooks/110-node-exporter-repair.yml | 8 +- 5 files changed, 364 insertions(+), 8 deletions(-) diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index ea0311778..bbe4a6c3a 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -445,7 +445,7 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "auto_apply_enabled": True, "approval_required": False, "risk_level": "low", - "catalog_revision": "2026-07-14-node-exporter-bounded-v4", + "catalog_revision": "2026-07-14-node-exporter-bounded-v5", }, { "catalog_id": "ansible:110-devops-full-convergence", diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index e6f9cdbef..4363333e5 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -1119,6 +1119,107 @@ def _claim_from_stale_check_mode_row( ) +def _claim_from_failed_apply_catalog_repair_row( + row: dict[str, Any], +) -> AnsibleCheckModeClaim | None: + """Build a new bounded claim when a failed apply's catalog was repaired.""" + + input_payload = _json_loads(row.get("input")) + apply_op_id = str(row.get("op_id") or "") + source_candidate_op_id = str( + input_payload.get("source_candidate_op_id") or "" + ) + incident_id = str( + row.get("incident_id") + or input_payload.get("incident_id") + or "" + ) + catalog_id = str(input_payload.get("catalog_id") or "") + catalog_item = get_ansible_catalog_item(catalog_id) or {} + current_revision = str(catalog_item.get("catalog_revision") or "") + previous_revision = str(input_payload.get("catalog_revision") or "") + current_playbook_path = str(catalog_item.get("playbook_path") or "") + inventory_hosts = catalog_item.get("inventory_hosts") or [] + if not ( + apply_op_id + and source_candidate_op_id + and incident_id + and catalog_id + and current_revision + and current_revision != previous_revision + and current_playbook_path + and isinstance(inventory_hosts, list) + and inventory_hosts + ): + return None + + try: + claim_input = build_ansible_check_mode_claim_input( + source_candidate_op_id=source_candidate_op_id, + candidate_input={ + "incident_id": incident_id, + "approval_id": input_payload.get("approval_id"), + "proposal_risk_level": input_payload.get( + "proposal_risk_level" + ), + "decision_path": "failed_apply_catalog_repair_replay", + "router_source_sha": input_payload.get("router_source_sha"), + "target_selector": input_payload.get("target_selector") or {}, + "source_truth_diff": input_payload.get("source_truth_diff") + or {}, + "risk_policy_decision": input_payload.get( + "risk_policy_decision" + ) + or {}, + "executor_candidates": [ + { + "catalog_id": catalog_id, + "catalog_revision": current_revision, + "playbook_path": current_playbook_path, + "check_mode_playbook_path": str( + catalog_item.get("check_mode_playbook_path") + or current_playbook_path + ), + "inventory_hosts": inventory_hosts, + "risk_level": catalog_item.get("risk_level"), + } + ], + }, + ) + except ValueError: + return None + + automation_run_id = str( + input_payload.get("automation_run_id") + or source_candidate_op_id + ) + claim_input.update( + { + "automation_run_id": automation_run_id, + "catalog_repair_of_apply_op_id": apply_op_id, + "previous_catalog_revision": previous_revision or None, + "catalog_replay_revision": current_revision, + "apply_idempotency_key": ( + f"ansible-repair-apply:{apply_op_id}:{current_revision}" + ), + "bounded_execution_scope": "catalog_repair_single_apply", + } + ) + return AnsibleCheckModeClaim( + op_id="", + source_candidate_op_id=source_candidate_op_id, + incident_id=incident_id, + catalog_id=str(claim_input["catalog_id"]), + playbook_path=str(claim_input["playbook_path"]), + apply_playbook_path=str(claim_input["apply_playbook_path"]), + inventory_hosts=tuple( + str(host) for host in claim_input["inventory_hosts"] + ), + risk_level=str(claim_input.get("risk_level") or ""), + input_payload=claim_input, + ) + + async def _record_auto_repair_execution_receipt( claim: AnsibleCheckModeClaim, result: AnsibleRunResult, @@ -7715,6 +7816,129 @@ async def claim_catalog_drift_failed_check_modes( return claims +async def claim_catalog_drift_failed_applies( + *, + project_id: str = "awoooi", + limit: int = 1, + candidate_max_age_hours: int | None = None, +) -> list[AnsibleCheckModeClaim]: + """Re-enter controlled execution once for a repaired failed apply.""" + + claims: list[AnsibleCheckModeClaim] = [] + max_age_hours = ( + candidate_max_age_hours + or settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS + ) + async with get_db_context(project_id) as db: + result = await db.execute( + text(""" + SELECT + apply.op_id, + apply.parent_op_id, + coalesce( + apply.input ->> 'incident_id', + apply.incident_id::text + ) AS incident_id, + apply.input + FROM automation_operation_log apply + WHERE apply.operation_type = 'ansible_apply_executed' + AND apply.status = 'failed' + AND apply.created_at >= NOW() - ( + :candidate_max_age_hours * INTERVAL '1 hour' + ) + ORDER BY apply.created_at DESC + LIMIT :scan_limit + FOR UPDATE OF apply SKIP LOCKED + """), + { + "candidate_max_age_hours": max(1, max_age_hours), + "scan_limit": min(100, max(10, limit * 10)), + }, + ) + for row_value in result.mappings().all(): + canonical_claim = _claim_from_failed_apply_catalog_repair_row( + dict(row_value) + ) + if canonical_claim is None: + continue + replay_input = canonical_claim.input_payload + inserted = await db.execute( + text(""" + INSERT INTO automation_operation_log ( + operation_type, actor, status, incident_id, + input, output, dry_run_result, + parent_op_id, tags + ) SELECT + 'ansible_check_mode_executed', + 'ansible_check_mode_worker', + 'pending', + :incident_db_id, + CAST(:input AS jsonb), + '{}'::jsonb, + CAST(:dry_run_result AS jsonb), + CAST(:parent_op_id AS uuid), + :tags + WHERE NOT EXISTS ( + SELECT 1 + FROM automation_operation_log existing_replay + WHERE existing_replay.operation_type + = 'ansible_check_mode_executed' + AND existing_replay.input + ->> 'catalog_repair_of_apply_op_id' + = :failed_apply_op_id + AND existing_replay.input + ->> 'catalog_replay_revision' + = :catalog_replay_revision + ) + RETURNING op_id + """), + { + "incident_db_id": _automation_operation_log_incident_id( + canonical_claim.incident_id + ), + "input": json.dumps(replay_input, ensure_ascii=False), + "dry_run_result": json.dumps( + { + "check_mode_executed": False, + "apply_executed": False, + "claim_state": "failed_apply_catalog_repair", + "controlled_apply_allowed": bool( + replay_input.get("controlled_apply_allowed") + ), + "controlled_apply_blocker": replay_input.get( + "controlled_apply_blocker" + ), + }, + ensure_ascii=False, + ), + "parent_op_id": ( + canonical_claim.source_candidate_op_id + ), + "failed_apply_op_id": replay_input[ + "catalog_repair_of_apply_op_id" + ], + "catalog_replay_revision": replay_input[ + "catalog_replay_revision" + ], + "tags": [ + "ansible", + "check_mode", + "pending", + "failed_apply_catalog_repair", + ], + }, + ) + op_id_value = inserted.scalar_one_or_none() + if op_id_value is None: + continue + claims.append( + replace(canonical_claim, op_id=str(op_id_value)) + ) + if len(claims) >= max(1, limit): + break + return claims + + async def recent_ansible_transport_blockers( *, project_id: str = "awoooi", @@ -8492,6 +8716,31 @@ async def run_pending_check_modes_once( 0, remaining_limit - len(stdin_boundary_replay_claims), ) + failed_apply_catalog_replay_claims: list[AnsibleCheckModeClaim] = [] + failed_apply_catalog_replay_error: str | None = None + if remaining_limit: + try: + failed_apply_catalog_replay_claims = ( + await claim_catalog_drift_failed_applies( + project_id=project_id, + limit=remaining_limit, + candidate_max_age_hours=( + settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS + ), + ) + ) + except Exception as exc: + failed_apply_catalog_replay_error = type(exc).__name__ + logger.warning( + "ansible_failed_apply_catalog_repair_claim_failed", + project_id=project_id, + error_type=failed_apply_catalog_replay_error, + fresh_candidate_claim_continues=True, + ) + remaining_limit = max( + 0, + remaining_limit - len(failed_apply_catalog_replay_claims), + ) catalog_replay_claims: list[AnsibleCheckModeClaim] = [] catalog_replay_error: str | None = None if remaining_limit: @@ -8526,6 +8775,7 @@ async def run_pending_check_modes_once( claims = [ *reclaimed_claims, *stdin_boundary_replay_claims, + *failed_apply_catalog_replay_claims, *catalog_replay_claims, *fresh_claims, ] @@ -8674,6 +8924,12 @@ async def run_pending_check_modes_once( stdin_boundary_replay_terminal_written ), "stdin_boundary_replay_error": stdin_boundary_replay_error, + "failed_apply_catalog_replayed": len( + failed_apply_catalog_replay_claims + ), + "failed_apply_catalog_replay_error": ( + failed_apply_catalog_replay_error + ), "catalog_replayed": len(catalog_replay_claims), "completed": completed, "failed": failed, diff --git a/apps/api/tests/test_awooop_ansible_post_verifier.py b/apps/api/tests/test_awooop_ansible_post_verifier.py index a85ad6310..b7f89bd73 100644 --- a/apps/api/tests/test_awooop_ansible_post_verifier.py +++ b/apps/api/tests/test_awooop_ansible_post_verifier.py @@ -69,7 +69,7 @@ def test_host110_node_exporter_postconditions_are_bounded() -> None: assert catalog["risk_level"] == "low" assert catalog["auto_apply_enabled"] is True assert catalog["catalog_revision"] == ( - "2026-07-14-node-exporter-bounded-v4" + "2026-07-14-node-exporter-bounded-v5" ) diff --git a/apps/api/tests/test_host110_node_exporter_ansible_catalog.py b/apps/api/tests/test_host110_node_exporter_ansible_catalog.py index b4be166a8..3999364ce 100644 --- a/apps/api/tests/test_host110_node_exporter_ansible_catalog.py +++ b/apps/api/tests/test_host110_node_exporter_ansible_catalog.py @@ -1,5 +1,7 @@ from __future__ import annotations +import inspect +import subprocess from pathlib import Path import yaml @@ -9,7 +11,10 @@ from src.services.awooop_ansible_audit_service import ( get_ansible_catalog_item, ) from src.services.awooop_ansible_check_mode_service import ( + _claim_from_failed_apply_catalog_repair_row, _claim_from_stale_check_mode_row, + claim_catalog_drift_failed_applies, + run_pending_check_modes_once, ) ROOT = Path(__file__).resolve().parents[3] @@ -48,7 +53,7 @@ def test_node_exporter_playbook_only_reconciles_one_container() -> None: assert catalog is not None assert catalog["playbook_path"].endswith("110-node-exporter-repair.yml") assert catalog["catalog_revision"] == ( - "2026-07-14-node-exporter-bounded-v4" + "2026-07-14-node-exporter-bounded-v5" ) assert "docker\n - compose\n - up" in text assert "node-exporter" in text @@ -63,6 +68,27 @@ def test_node_exporter_playbook_only_reconciles_one_container() -> None: assert forbidden not in text.lower() +def test_node_exporter_post_apply_prometheus_probe_is_valid_bash() -> None: + payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) + verifier = next( + task + for task in payload[0]["tasks"] + if task["name"] + == "Verify Prometheus sees node-exporter after controlled apply" + ) + command = verifier["ansible.builtin.shell"]["cmd"] + + result = subprocess.run( + ["bash", "-n"], + input=command, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + def test_existing_110_devops_lineage_migrates_to_bounded_revision() -> None: claim = _claim_from_stale_check_mode_row( { @@ -90,6 +116,80 @@ def test_existing_110_devops_lineage_migrates_to_bounded_revision() -> None: assert claim.playbook_path.endswith("110-node-exporter-repair.yml") assert claim.apply_playbook_path.endswith("110-node-exporter-repair.yml") assert claim.input_payload["catalog_revision"] == ( - "2026-07-14-node-exporter-bounded-v4" + "2026-07-14-node-exporter-bounded-v5" ) assert claim.input_payload["controlled_apply_allowed"] is True + + +def test_failed_apply_catalog_repair_reenters_same_automation_run() -> None: + failed_apply_op_id = "00000000-0000-0000-0000-000000000201" + source_candidate_op_id = "00000000-0000-0000-0000-000000000110" + claim = _claim_from_failed_apply_catalog_repair_row( + { + "op_id": failed_apply_op_id, + "incident_id": "INC-NODE-EXPORTER-110", + "input": { + "automation_run_id": source_candidate_op_id, + "source_candidate_op_id": source_candidate_op_id, + "incident_id": "INC-NODE-EXPORTER-110", + "catalog_id": "ansible:110-devops", + "catalog_revision": ( + "2026-07-14-node-exporter-bounded-v4" + ), + "playbook_path": ( + "infra/ansible/playbooks/110-node-exporter-repair.yml" + ), + "inventory_hosts": ["host_110"], + "risk_level": "low", + }, + } + ) + + assert claim is not None + assert claim.source_candidate_op_id == source_candidate_op_id + assert claim.input_payload["automation_run_id"] == source_candidate_op_id + assert claim.input_payload["catalog_revision"] == ( + "2026-07-14-node-exporter-bounded-v5" + ) + assert claim.input_payload["catalog_repair_of_apply_op_id"] == ( + failed_apply_op_id + ) + assert claim.input_payload["controlled_apply_allowed"] is True + assert claim.input_payload["bounded_execution_scope"] == ( + "catalog_repair_single_apply" + ) + + +def test_failed_apply_current_revision_is_not_replayed() -> None: + claim = _claim_from_failed_apply_catalog_repair_row( + { + "op_id": "00000000-0000-0000-0000-000000000201", + "incident_id": "INC-NODE-EXPORTER-110", + "input": { + "automation_run_id": ( + "00000000-0000-0000-0000-000000000110" + ), + "source_candidate_op_id": ( + "00000000-0000-0000-0000-000000000110" + ), + "incident_id": "INC-NODE-EXPORTER-110", + "catalog_id": "ansible:110-devops", + "catalog_revision": ( + "2026-07-14-node-exporter-bounded-v5" + ), + }, + } + ) + + assert claim is None + + +def test_failed_apply_catalog_repair_is_deduped_and_prioritized() -> None: + claim_source = inspect.getsource(claim_catalog_drift_failed_applies) + worker_source = inspect.getsource(run_pending_check_modes_once) + + assert "catalog_repair_of_apply_op_id" in claim_source + assert "catalog_replay_revision" in claim_source + assert worker_source.index("claim_catalog_drift_failed_applies") < ( + worker_source.index("claim_catalog_drift_failed_check_modes") + ) diff --git a/infra/ansible/playbooks/110-node-exporter-repair.yml b/infra/ansible/playbooks/110-node-exporter-repair.yml index c4e607101..774e25497 100644 --- a/infra/ansible/playbooks/110-node-exporter-repair.yml +++ b/infra/ansible/playbooks/110-node-exporter-repair.yml @@ -88,11 +88,11 @@ - name: Verify Prometheus sees node-exporter after controlled apply ansible.builtin.shell: - cmd: >- + cmd: | for attempt in 1 2 3 4 5 6; do - curl -fsS --max-time 5 - 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' - | grep -Eq '"value":\[[^]]+,"1"\]' && exit 0; + if curl -fsS --max-time 5 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' | grep -Eq '"value":\[[^]]+,"1"\]'; then + exit 0 + fi sleep 5; done; exit 1 From 1a39034c933334d1a944fa0b06481be427c95486 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Tue, 14 Jul 2026 22:57:32 +0800 Subject: [PATCH 5/5] chore(cd): deploy 8318036 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 3e4cf593d..3c61da19a 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" + value: "83180367d19768bde7008254a962d3a13d89209a" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" + value: "83180367d19768bde7008254a962d3a13d89209a" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 23e4f36e9..779f9acdd 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" + value: "83180367d19768bde7008254a962d3a13d89209a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 5b16368d4..5e9cf2952 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "7c7784cf66a69bc065c5cd443d1926654f0f7e25" + value: "83180367d19768bde7008254a962d3a13d89209a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 49a3c5d35..7e1642e65 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:005354b7665fcb10014df8e51c88f3a32e5c384618f0057203b4f85c6e9c91c0 +- digest: sha256:34f4777782d4c32ae50b6adf7e365e51ecc76488cb07fadd0b1edde7fc183a11 name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:fc336fee71de11c657163e378fe98a52fccf5cab3d9410b80b126c9b5edcda20 +- digest: sha256:47b293f30796ddf995688e4cde66557b343c4a300e2166ccae3be02574fa7c4c name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web