From 2563b9c460e13f03973ff7f39f9f30f887efd2c8 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 11:25:00 +0800 Subject: [PATCH 1/5] chore(cd): deploy 6505e2c [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 6dca50685..d8032c803 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 683b639a9..729b79f92 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 33118d88e..a56468090 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -181,7 +181,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index dbf720196..c638d5532 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:a06af69531087ef123b91bcae5f4170f322035156a64e5d09ef8ebdc23ea1c7a +- digest: sha256:d8304318081a6c4267d78cc1bc6b43a88902a17bfccc7e1146989609282453ad name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:9e6ada8cef17243c83230d7890acc1da154add49569b407a590e633d5a5ea70a +- digest: sha256:7891f1fb9c2e6473d3ae102efc6c80ae1be48e1fda96a4de0a0c4d8f6cad4238 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From dd32cbf9cdc7e699f19cf91dcd7782945c4ed475 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 11:27:36 +0800 Subject: [PATCH 2/5] fix(ci): make MCP audit writeback repeatable --- .../mcp-external-artifact-mirror.yaml | 48 +-- .gitea/workflows/mcp-external-replay.yaml | 50 +-- docs/LOGBOOK.md | 9 +- ...tea_workflow_runner_health_2026-06-05.json | 3 + .../tests/test_write_gitea_audit_receipts.py | 290 ++++++++++++++++++ scripts/ci/write-gitea-audit-receipts.sh | 188 ++++++++++++ .../test_external_mcp_artifact_controller.py | 11 +- .../test_external_mcp_replay_controller.py | 9 + 8 files changed, 539 insertions(+), 69 deletions(-) create mode 100644 scripts/ci/tests/test_write_gitea_audit_receipts.py create mode 100755 scripts/ci/write-gitea-audit-receipts.sh diff --git a/.gitea/workflows/mcp-external-artifact-mirror.yaml b/.gitea/workflows/mcp-external-artifact-mirror.yaml index 1fa6be8da..607db7544 100644 --- a/.gitea/workflows/mcp-external-artifact-mirror.yaml +++ b/.gitea/workflows/mcp-external-artifact-mirror.yaml @@ -22,6 +22,8 @@ on: - scripts/security/verify_external_mcp_artifact_receipt.py - scripts/security/tests/test_external_mcp_artifact_controller.py - scripts/security/tests/test_verify_external_mcp_artifact_receipt.py + - scripts/ci/write-gitea-audit-receipts.sh + - scripts/ci/tests/test_write_gitea_audit_receipts.py - .gitea/workflows/mcp-external-artifact-mirror.yaml concurrency: @@ -68,9 +70,11 @@ jobs: scripts/security/external_mcp_artifact_controller.py \ scripts/security/verify_external_mcp_artifact_receipt.py \ scripts/security/tests/test_external_mcp_artifact_controller.py \ - scripts/security/tests/test_verify_external_mcp_artifact_receipt.py + scripts/security/tests/test_verify_external_mcp_artifact_receipt.py \ + scripts/ci/tests/test_write_gitea_audit_receipts.py python3 scripts/security/tests/test_external_mcp_artifact_controller.py python3 scripts/security/tests/test_verify_external_mcp_artifact_receipt.py + python3 scripts/ci/tests/test_write_gitea_audit_receipts.py python3 - <<'PY' import json from pathlib import Path @@ -194,14 +198,11 @@ jobs: set +x test -s "${CONTROLLER_RECEIPT_PATH}" test -s "${VERIFIER_RECEIPT_PATH}" + WRITEBACK_DIR="$(mktemp -d)" + cp "${CONTROLLER_RECEIPT_PATH}" "${WRITEBACK_DIR}/controller.json" + cp "${VERIFIER_RECEIPT_PATH}" "${WRITEBACK_DIR}/verifier.json" git config user.email "mcp-artifact-controller@awoooi.internal" git config user.name "AWOOOI MCP Artifact Controller" - git add "${CONTROLLER_RECEIPT_PATH}" "${VERIFIER_RECEIPT_PATH}" - git diff --cached --quiet && { - echo "receipt_writeback=no_change" - exit 0 - } - git commit -m "chore(mcp): record verified Playwright artifact mirror [skip ci] [metadata-only]" ASKPASS_PATH="$(mktemp)" export ASKPASS_PATH @@ -223,31 +224,16 @@ jobs: PY cleanup_push_auth() { rm -f "${ASKPASS_PATH}" + rm -rf "${WRITEBACK_DIR}" } trap cleanup_push_auth EXIT export GIT_ASKPASS="${ASKPASS_PATH}" export GIT_TERMINAL_PROMPT=0 - git remote remove gitea 2>/dev/null || true - git remote add gitea "${GITEA_SOURCE_URL}" - - for attempt in 1 2 3; do - if git ls-remote --exit-code --heads gitea \ - "refs/heads/${RECEIPT_BRANCH}" >/dev/null 2>&1; then - git fetch --no-tags --depth=100 gitea "${RECEIPT_BRANCH}" - if ! git merge --no-edit FETCH_HEAD; then - git merge --abort || true - echo "BLOCKER receipt_writeback_merge_conflict" - exit 1 - fi - fi - if git push gitea "HEAD:refs/heads/${RECEIPT_BRANCH}"; then - echo "receipt_writeback=verified_audit_branch_normal_push" - echo "receipt_branch=${RECEIPT_BRANCH}" - echo "receipt_commit_sha=$(git rev-parse HEAD)" - exit 0 - fi - echo "receipt_writeback_retry=${attempt}" - sleep 5 - done - echo "BLOCKER receipt_writeback_non_fast_forward_after_bounded_retry" - exit 1 + export GITEA_AUDIT_REMOTE_URL="${GITEA_SOURCE_URL}" + export GITEA_AUDIT_BRANCH="${RECEIPT_BRANCH}" + export GITEA_AUDIT_CONTROLLER_SOURCE="${WRITEBACK_DIR}/controller.json" + export GITEA_AUDIT_VERIFIER_SOURCE="${WRITEBACK_DIR}/verifier.json" + export GITEA_AUDIT_CONTROLLER_TARGET="${CONTROLLER_RECEIPT_PATH}" + export GITEA_AUDIT_VERIFIER_TARGET="${VERIFIER_RECEIPT_PATH}" + export GITEA_AUDIT_COMMIT_MESSAGE="chore(mcp): record verified Playwright artifact mirror [skip ci] [metadata-only]" + bash scripts/ci/write-gitea-audit-receipts.sh diff --git a/.gitea/workflows/mcp-external-replay.yaml b/.gitea/workflows/mcp-external-replay.yaml index 6328f0e84..bf1eff5a7 100644 --- a/.gitea/workflows/mcp-external-replay.yaml +++ b/.gitea/workflows/mcp-external-replay.yaml @@ -22,6 +22,8 @@ on: - scripts/security/verify_external_mcp_replay_receipt.py - scripts/security/tests/test_external_mcp_replay_controller.py - scripts/security/tests/test_verify_external_mcp_replay_receipt.py + - scripts/ci/write-gitea-audit-receipts.sh + - scripts/ci/tests/test_write_gitea_audit_receipts.py - .gitea/workflows/mcp-external-replay.yaml concurrency: @@ -76,9 +78,11 @@ jobs: scripts/security/external_mcp_replay_controller.py \ scripts/security/verify_external_mcp_replay_receipt.py \ scripts/security/tests/test_external_mcp_replay_controller.py \ - scripts/security/tests/test_verify_external_mcp_replay_receipt.py + scripts/security/tests/test_verify_external_mcp_replay_receipt.py \ + scripts/ci/tests/test_write_gitea_audit_receipts.py python3 scripts/security/tests/test_external_mcp_replay_controller.py python3 scripts/security/tests/test_verify_external_mcp_replay_receipt.py + python3 scripts/ci/tests/test_write_gitea_audit_receipts.py python3 - <<'PY' import json from pathlib import Path @@ -267,16 +271,11 @@ jobs: set +x test -s "${REPLAY_CONTROLLER_RECEIPT_PATH}" test -s "${REPLAY_VERIFIER_RECEIPT_PATH}" + WRITEBACK_DIR="$(mktemp -d)" + cp "${REPLAY_CONTROLLER_RECEIPT_PATH}" "${WRITEBACK_DIR}/controller.json" + cp "${REPLAY_VERIFIER_RECEIPT_PATH}" "${WRITEBACK_DIR}/verifier.json" git config user.email "mcp-replay-controller@awoooi.internal" git config user.name "AWOOOI MCP Replay Controller" - git add \ - "${REPLAY_CONTROLLER_RECEIPT_PATH}" \ - "${REPLAY_VERIFIER_RECEIPT_PATH}" - git diff --cached --quiet && { - echo "receipt_writeback=no_change" - exit 0 - } - git commit -m "chore(mcp): record verified Playwright protocol replay [skip ci] [metadata-only]" ASKPASS_PATH="$(mktemp)" export ASKPASS_PATH @@ -298,31 +297,16 @@ jobs: PY cleanup_push_auth() { rm -f "${ASKPASS_PATH}" + rm -rf "${WRITEBACK_DIR}" } trap cleanup_push_auth EXIT export GIT_ASKPASS="${ASKPASS_PATH}" export GIT_TERMINAL_PROMPT=0 - git remote remove gitea 2>/dev/null || true - git remote add gitea "${GITEA_SOURCE_URL}" - - for attempt in 1 2 3; do - if git ls-remote --exit-code --heads gitea \ - "refs/heads/${REPLAY_RECEIPT_BRANCH}" >/dev/null 2>&1; then - git fetch --no-tags --depth=100 gitea "${REPLAY_RECEIPT_BRANCH}" - if ! git merge --no-edit FETCH_HEAD; then - git merge --abort || true - echo "BLOCKER receipt_writeback_merge_conflict" - exit 1 - fi - fi - if git push gitea "HEAD:refs/heads/${REPLAY_RECEIPT_BRANCH}"; then - echo "receipt_writeback=verified_audit_branch_normal_push" - echo "receipt_branch=${REPLAY_RECEIPT_BRANCH}" - echo "receipt_commit_sha=$(git rev-parse HEAD)" - exit 0 - fi - echo "receipt_writeback_retry=${attempt}" - sleep 5 - done - echo "BLOCKER receipt_writeback_non_fast_forward_after_bounded_retry" - exit 1 + export GITEA_AUDIT_REMOTE_URL="${GITEA_SOURCE_URL}" + export GITEA_AUDIT_BRANCH="${REPLAY_RECEIPT_BRANCH}" + export GITEA_AUDIT_CONTROLLER_SOURCE="${WRITEBACK_DIR}/controller.json" + export GITEA_AUDIT_VERIFIER_SOURCE="${WRITEBACK_DIR}/verifier.json" + export GITEA_AUDIT_CONTROLLER_TARGET="${REPLAY_CONTROLLER_RECEIPT_PATH}" + export GITEA_AUDIT_VERIFIER_TARGET="${REPLAY_VERIFIER_RECEIPT_PATH}" + export GITEA_AUDIT_COMMIT_MESSAGE="chore(mcp): record verified Playwright protocol replay [skip ci] [metadata-only]" + bash scripts/ci/write-gitea-audit-receipts.sh diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 98bd573b1..04276b79f 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -10,16 +10,19 @@ **source/test evidence**: - controller/verifier 本機真實雙 replay 皆通過:MCP `Playwright@1.62.0-alpha-1783623505000`、protocol `2025-06-18`、24 tools、tool-name hash `12d23fea9d8d1a1de3f44863dc00cf393f0a2165323838cf93fed30a6a4cc237`、schema hash `c3737ced21147d0512b0400df5885c95a545bc915892b5c8f928cee78428ac99`,兩次均 clean exit/container removed。 -- 新 replay tests `37 passed`;artifact + replay supply-chain tests與 MCP control-plane/federation/version lifecycle/runner-health 聚焦回歸合計 `89 passed`。Ruff、py_compile、JSON/YAML parse、source-control owner guard與 `git diff --check` 通過。 +- artifact/replay controller + independent verifier `61 passed`、MCP control-plane/federation/version lifecycle/runner-health API `27 passed`、shared audit writeback `4 passed`,本輪聚焦回歸合計 `92 passed`。Ruff、py_compile、JSON/YAML parse、source-control owner guard與 `git diff --check` 通過。 +- Gitea replay `#5183` 已在 source `387f841...` 完成 controller + independent verifier,run `d3b8d15e-e5c6-43c0-b657-f9f23089677c`,audit commit `281d6e0058721dfbf84235d85c2f894a004ceea1`;24 tools、offline clean exit、container removal與所有 runtime/write boundary false 均有 durable receipt。 +- 後續 replay `#5186` 在 source `5eb46f9...` 的 controller/verifier 仍成功,但 final writeback 因 shallow main 與 audit branch `unrelated histories` 失敗;scheduled artifact `#5178` 有相同 terminal。根因不是 MCP replay 或 artifact verification,而是舊 workflow 把 audit branch merge 回 depth-1 main checkout。 +- 新增共用 fail-closed audit writeback:直接 checkout audit branch tip、只 stage兩個 allowlisted receipt、同 run identity與 branch/path/schema 綁定、最多 3 次 normal fast-forward push,無 merge、無 force。真實 temporary bare-Git functional tests涵蓋連續第二/第三次線性 push與拒絕 branch/path/identity mismatch,`4 passed`。 - global security mirror guard另抓到 latest main 既有 `apps/web/src/app/[locale]/iwooos/page.tsx` 的 `raw_blocked_waiting_state` 敏感字樣;本 work item未修改該檔,保留為獨立 drift blocker,不以 replay source 綠燈覆蓋。 **尚未完成 / 不誤報**: -- 目前是 source + 本機真實 replay證據;Gitea replay workflow、audit branch receipt、main CD/deploy marker與 production runtime/UI readback尚未產生,因此 production compatibility仍不得標 completed。 +- `#5183` 證明 protocol/schema replay本身與 durable receipt可成立,但最新 source `5eb46f9...` 的 repeatability writeback修正尚未 normal push / Gitea workflow terminal;artifact/replay recurring lanes仍要各取得一筆新的 success terminal,production catalog也尚未讀取該 receipt,因此不得把 compatibility或週期升級 loop標 completed。 - Browser binary/runtime immutable supply chain、public-origin egress proxy與 private-network denial、prompt-injection replay、accessibility/latency shadow、canary、Gateway adapter registration、rollback execution與正式 KM/RAG learning acknowledgement仍是 promotion blocker。 - 未讀 secret value、`.env`、raw session、SQLite/auth;未連線或下載 GitHub/Actions/hosted artifact,未使用 `npx -y`、`@latest`、`:latest`、force push、外部 RAG write或 production mutation。 **下一步**: -- normal push Gitea main並取得 replay workflow controller + standalone verifier audit receipts;再更新 MCP production read model/UI為 compatibility verified但 promotion disabled,之後才建立 immutable browser runtime與隔離 public-origin shadow/canary/rollback lane。 +- normal push shared writeback修正到 Gitea main,要求 artifact與 replay recurring workflow各自取得線性 audit commit;再更新 MCP production read model/UI為 compatibility verified但 promotion disabled,之後才建立 immutable browser runtime與隔離 public-origin shadow/canary/rollback lane。 ## 2026-07-15 — P0 MCP Playwright immutable mirror 與獨立供應鏈 verifier diff --git a/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json b/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json index 9656d8c3c..a3f3b4225 100644 --- a/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json +++ b/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json @@ -30,6 +30,7 @@ "docs/evaluations/ai_agent_version_lifecycle_update_proposal_2026-07-10.json", "scripts/ci/check-gitea-step-env-secrets.js", "scripts/ci/cleanup-host-runner-workspace.sh", + "scripts/ci/write-gitea-audit-receipts.sh", "scripts/ci/wait-host-web-build-pressure.sh", "scripts/ci/notify-awoooi-cicd.sh", "scripts/setup-runner-watchdog.sh", @@ -367,6 +368,7 @@ "config/mcp/playwright-mcp-artifact-policy.json", "scripts/security/external_mcp_artifact_controller.py", "scripts/security/verify_external_mcp_artifact_receipt.py", + "scripts/ci/write-gitea-audit-receipts.sh", "scripts/ci/wait-host-web-build-pressure.sh" ], "next_action": "維持 bounded capacity wait、獨立 verifier 與 mcp-artifact-receipts audit branch;不得啟動 artifact、寫 RAG、切 production route 或推回 deploy main。" @@ -397,6 +399,7 @@ "config/mcp/playwright-mcp-replay-policy.json", "scripts/security/external_mcp_replay_controller.py", "scripts/security/verify_external_mcp_replay_receipt.py", + "scripts/ci/write-gitea-audit-receipts.sh", "scripts/ci/wait-host-web-build-pressure.sh" ], "next_action": "先取得 mcp-replay-receipts 獨立 verifier receipt;browser runtime、public-origin egress proxy、shadow、canary、Gateway adapter registration、RAG 與 production route 仍保持停用。" diff --git a/scripts/ci/tests/test_write_gitea_audit_receipts.py b/scripts/ci/tests/test_write_gitea_audit_receipts.py new file mode 100644 index 000000000..2299b7a8b --- /dev/null +++ b/scripts/ci/tests/test_write_gitea_audit_receipts.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Functional tests for linear Gitea audit-branch receipt writeback.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts/ci/write-gitea-audit-receipts.sh" +CONTROLLER_TARGET = ( + "docs/operations/external-mcp-replays/fixture-controller.snapshot.json" +) +VERIFIER_TARGET = ( + "docs/operations/external-mcp-replays/fixture-verifier.snapshot.json" +) + + +def _run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> str: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if completed.returncode != 0: + raise AssertionError( + f"command failed: {command}\nstdout={completed.stdout}\nstderr={completed.stderr}" + ) + return completed.stdout.strip() + + +def _receipt(schema: str, run_id: str) -> dict: + return { + "schema_version": schema, + "run_id": run_id, + "trace_id": f"mcp-replay-{run_id}", + "work_item_id": "MCP-REPLAY-PLAYWRIGHT-001", + "status": "fixture_verified", + } + + +def _write_receipts(directory: Path, run_id: str) -> tuple[Path, Path]: + controller = directory / "controller.json" + verifier = directory / "verifier.json" + controller.write_text( + json.dumps( + _receipt("awoooi_external_mcp_replay_receipt_v1", run_id), + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + verifier.write_text( + json.dumps( + _receipt("awoooi_external_mcp_replay_verifier_receipt_v1", run_id), + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return controller, verifier + + +class AuditReceiptWritebackTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.remote = self.root / "remote.git" + self.seed = self.root / "seed" + _run(["git", "init", "--bare", str(self.remote)], cwd=self.root) + self.seed.mkdir() + _run(["git", "init"], cwd=self.seed) + _run(["git", "config", "user.email", "fixture@awoooi.internal"], cwd=self.seed) + _run(["git", "config", "user.name", "Fixture"], cwd=self.seed) + (self.seed / "base.txt").write_text("base\n", encoding="utf-8") + _run(["git", "add", "base.txt"], cwd=self.seed) + _run(["git", "commit", "-m", "base"], cwd=self.seed) + _run(["git", "branch", "-M", "main"], cwd=self.seed) + _run(["git", "remote", "add", "origin", f"file://{self.remote}"], cwd=self.seed) + _run(["git", "push", "origin", "main"], cwd=self.seed) + + _run(["git", "checkout", "-b", "mcp-replay-receipts"], cwd=self.seed) + initial_controller, initial_verifier = _write_receipts( + self.seed, "00000000-0000-4000-8000-000000000000" + ) + (self.seed / CONTROLLER_TARGET).parent.mkdir(parents=True) + (self.seed / CONTROLLER_TARGET).write_bytes(initial_controller.read_bytes()) + (self.seed / VERIFIER_TARGET).write_bytes(initial_verifier.read_bytes()) + _run(["git", "add", "docs"], cwd=self.seed) + _run(["git", "commit", "-m", "initial audit receipt"], cwd=self.seed) + _run(["git", "push", "origin", "mcp-replay-receipts"], cwd=self.seed) + self.initial_audit_sha = _run(["git", "rev-parse", "HEAD"], cwd=self.seed) + + _run(["git", "checkout", "main"], cwd=self.seed) + (self.seed / "main-only.txt").write_text("new main\n", encoding="utf-8") + _run(["git", "add", "main-only.txt"], cwd=self.seed) + _run(["git", "commit", "-m", "advance main"], cwd=self.seed) + _run(["git", "push", "origin", "main"], cwd=self.seed) + + def tearDown(self) -> None: + self.temp.cleanup() + + def _fresh_main_clone(self, name: str) -> Path: + clone = self.root / name + _run( + [ + "git", + "clone", + "--depth=1", + "--branch", + "main", + f"file://{self.remote}", + str(clone), + ], + cwd=self.root, + ) + _run(["git", "config", "user.email", "fixture@awoooi.internal"], cwd=clone) + _run(["git", "config", "user.name", "Fixture"], cwd=clone) + return clone + + def _apply(self, clone: Path, run_id: str) -> str: + source_dir = self.root / f"source-{run_id}" + source_dir.mkdir() + controller, verifier = _write_receipts(source_dir, run_id) + env = { + **os.environ, + "GIT_TERMINAL_PROMPT": "0", + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "mcp-replay-receipts", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "fixture audit receipt", + } + return _run(["bash", str(SCRIPT)], cwd=clone, env=env) + + def test_second_and_third_writebacks_are_linear_normal_pushes(self) -> None: + first_clone = self._fresh_main_clone("clone-one") + first_output = self._apply( + first_clone, "11111111-1111-4111-8111-111111111111" + ) + self.assertIn("receipt_writeback=verified_audit_branch_normal_push", first_output) + first_sha = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", "mcp-replay-receipts"], + cwd=self.root, + ) + first_parent = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", f"{first_sha}^"], + cwd=self.root, + ) + self.assertEqual(first_parent, self.initial_audit_sha) + + second_clone = self._fresh_main_clone("clone-two") + second_output = self._apply( + second_clone, "22222222-2222-4222-8222-222222222222" + ) + self.assertIn("receipt_writeback=verified_audit_branch_normal_push", second_output) + second_sha = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", "mcp-replay-receipts"], + cwd=self.root, + ) + second_parent = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", f"{second_sha}^"], + cwd=self.root, + ) + self.assertEqual(second_parent, first_sha) + latest = _run( + [ + "git", + "--git-dir", + str(self.remote), + "show", + f"mcp-replay-receipts:{CONTROLLER_TARGET}", + ], + cwd=self.root, + ) + self.assertEqual( + json.loads(latest)["run_id"], + "22222222-2222-4222-8222-222222222222", + ) + + def test_non_allowlisted_branch_is_rejected_without_push(self) -> None: + clone = self._fresh_main_clone("clone-rejected") + source_dir = self.root / "source-rejected" + source_dir.mkdir() + controller, verifier = _write_receipts( + source_dir, "33333333-3333-4333-8333-333333333333" + ) + env = { + **os.environ, + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "main", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "must not commit", + } + completed = subprocess.run( + ["bash", str(SCRIPT)], + cwd=clone, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("audit_writeback_branch_not_allowlisted", completed.stdout) + + def test_branch_and_receipt_directory_must_match(self) -> None: + clone = self._fresh_main_clone("clone-mismatched-target") + source_dir = self.root / "source-mismatched-target" + source_dir.mkdir() + controller, verifier = _write_receipts( + source_dir, "44444444-4444-4444-8444-444444444444" + ) + env = { + **os.environ, + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "mcp-artifact-receipts", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "must not commit", + } + completed = subprocess.run( + ["bash", str(SCRIPT)], + cwd=clone, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("audit_writeback_branch_target_mismatch", completed.stdout) + + def test_controller_and_verifier_identity_must_match(self) -> None: + clone = self._fresh_main_clone("clone-mismatched-identity") + source_dir = self.root / "source-mismatched-identity" + source_dir.mkdir() + controller, verifier = _write_receipts( + source_dir, "55555555-5555-4555-8555-555555555555" + ) + verifier_payload = json.loads(verifier.read_text(encoding="utf-8")) + verifier_payload["run_id"] = "66666666-6666-4666-8666-666666666666" + verifier.write_text( + json.dumps(verifier_payload, sort_keys=True) + "\n", encoding="utf-8" + ) + env = { + **os.environ, + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "mcp-replay-receipts", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "must not commit", + } + completed = subprocess.run( + ["bash", str(SCRIPT)], + cwd=clone, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("audit_writeback_receipt_identity_mismatch", completed.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/write-gitea-audit-receipts.sh b/scripts/ci/write-gitea-audit-receipts.sh new file mode 100755 index 000000000..08c29fd26 --- /dev/null +++ b/scripts/ci/write-gitea-audit-receipts.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +# Append two verified JSON receipts to a dedicated Gitea audit branch without +# merging the shallow workflow checkout. Authentication is supplied by the +# caller through GIT_ASKPASS; this script never accepts a token argument. + +required_env=( + GITEA_AUDIT_REMOTE_URL + GITEA_AUDIT_BRANCH + GITEA_AUDIT_CONTROLLER_SOURCE + GITEA_AUDIT_VERIFIER_SOURCE + GITEA_AUDIT_CONTROLLER_TARGET + GITEA_AUDIT_VERIFIER_TARGET + GITEA_AUDIT_COMMIT_MESSAGE +) +for name in "${required_env[@]}"; do + if [ -z "${!name:-}" ]; then + echo "BLOCKER audit_writeback_missing_${name}" + exit 2 + fi +done + +if [ "${GITEA_AUDIT_REMOTE_URL}" = "http://192.168.0.110:3001/wooo/awoooi.git" ]; then + if [ -z "${GIT_ASKPASS:-}" ] || [ ! -x "${GIT_ASKPASS}" ]; then + echo "BLOCKER audit_writeback_askpass_missing_or_not_executable" + exit 2 + fi +elif [[ "${GITEA_AUDIT_REMOTE_URL}" == file://* ]] && \ + [ "${GITEA_AUDIT_TEST_FILE_REMOTE:-0}" = "1" ]; then + : +else + echo "BLOCKER audit_writeback_remote_not_allowlisted" + exit 2 +fi +if ! [[ "${GITEA_AUDIT_BRANCH}" =~ ^mcp-(artifact|replay)-receipts$ ]]; then + echo "BLOCKER audit_writeback_branch_not_allowlisted" + exit 2 +fi +if ! [[ "${GITEA_AUDIT_CONTROLLER_TARGET}" =~ ^docs/operations/external-mcp-(artifacts|replays)/[A-Za-z0-9._-]+-controller\.snapshot\.json$ ]]; then + echo "BLOCKER audit_writeback_controller_target_not_allowlisted" + exit 2 +fi +if ! [[ "${GITEA_AUDIT_VERIFIER_TARGET}" =~ ^docs/operations/external-mcp-(artifacts|replays)/[A-Za-z0-9._-]+-verifier\.snapshot\.json$ ]]; then + echo "BLOCKER audit_writeback_verifier_target_not_allowlisted" + exit 2 +fi +if [ "$(dirname "${GITEA_AUDIT_CONTROLLER_TARGET}")" != "$(dirname "${GITEA_AUDIT_VERIFIER_TARGET}")" ]; then + echo "BLOCKER audit_writeback_target_directory_mismatch" + exit 2 +fi +case "${GITEA_AUDIT_BRANCH}" in + mcp-artifact-receipts) + expected_target_directory="docs/operations/external-mcp-artifacts" + expected_controller_schema="awoooi_external_mcp_artifact_receipt_v1" + expected_verifier_schema="awoooi_external_mcp_artifact_verifier_receipt_v1" + ;; + mcp-replay-receipts) + expected_target_directory="docs/operations/external-mcp-replays" + expected_controller_schema="awoooi_external_mcp_replay_receipt_v1" + expected_verifier_schema="awoooi_external_mcp_replay_verifier_receipt_v1" + ;; +esac +if [ "$(dirname "${GITEA_AUDIT_CONTROLLER_TARGET}")" != "${expected_target_directory}" ]; then + echo "BLOCKER audit_writeback_branch_target_mismatch" + exit 2 +fi +if [ "${#GITEA_AUDIT_COMMIT_MESSAGE}" -gt 200 ] || \ + [[ "${GITEA_AUDIT_COMMIT_MESSAGE}" == *$'\n'* ]] || \ + [[ "${GITEA_AUDIT_COMMIT_MESSAGE}" == *$'\r'* ]]; then + echo "BLOCKER audit_writeback_commit_message_invalid" + exit 2 +fi + +for source_path in \ + "${GITEA_AUDIT_CONTROLLER_SOURCE}" \ + "${GITEA_AUDIT_VERIFIER_SOURCE}"; do + if [ ! -f "${source_path}" ] || [ -L "${source_path}" ]; then + echo "BLOCKER audit_writeback_source_invalid" + exit 2 + fi + source_size="$(wc -c < "${source_path}" | tr -d ' ')" + if [ "${source_size}" -le 0 ] || [ "${source_size}" -gt 4194304 ]; then + echo "BLOCKER audit_writeback_source_size_invalid" + exit 2 + fi +done + +python3 - \ + "${GITEA_AUDIT_CONTROLLER_SOURCE}" \ + "${GITEA_AUDIT_VERIFIER_SOURCE}" \ + "${expected_controller_schema}" \ + "${expected_verifier_schema}" <<'PY' +import json +import sys +from pathlib import Path + +def fail(message: str) -> None: + print(message) + raise SystemExit(2) + + +try: + receipts = [ + json.loads(Path(value).read_text(encoding="utf-8")) for value in sys.argv[1:3] + ] +except (OSError, UnicodeError, json.JSONDecodeError): + fail("BLOCKER audit_writeback_receipt_json_invalid") + +expected_schemas = sys.argv[3:5] +identity_fields = ("run_id", "trace_id", "work_item_id") +for payload, expected_schema in zip(receipts, expected_schemas): + if not isinstance(payload, dict) or payload.get("schema_version") != expected_schema: + fail("BLOCKER audit_writeback_receipt_schema_invalid") + if any(not str(payload.get(field) or "") for field in identity_fields): + fail("BLOCKER audit_writeback_receipt_identity_missing") +if any(receipts[0][field] != receipts[1][field] for field in identity_fields): + fail("BLOCKER audit_writeback_receipt_identity_mismatch") +PY + +remote_name="gitea-audit" +git remote remove "${remote_name}" >/dev/null 2>&1 || true +git remote add "${remote_name}" "${GITEA_AUDIT_REMOTE_URL}" +rm -f \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}" + +for attempt in 1 2 3; do + if git ls-remote --exit-code --heads "${remote_name}" \ + "refs/heads/${GITEA_AUDIT_BRANCH}" >/dev/null 2>&1; then + git fetch --no-tags --depth=1 "${remote_name}" "${GITEA_AUDIT_BRANCH}" + git checkout --force --detach FETCH_HEAD + fi + + target_directory="$(dirname "${GITEA_AUDIT_CONTROLLER_TARGET}")" + for ancestor in docs docs/operations "${target_directory}"; do + if [ -L "${ancestor}" ]; then + echo "BLOCKER audit_writeback_target_symlink_rejected" + exit 2 + fi + done + mkdir -p "${target_directory}" + for target_path in \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}"; do + if [ -L "${target_path}" ]; then + echo "BLOCKER audit_writeback_target_symlink_rejected" + exit 2 + fi + done + install -m 0644 \ + "${GITEA_AUDIT_CONTROLLER_SOURCE}" \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" + install -m 0644 \ + "${GITEA_AUDIT_VERIFIER_SOURCE}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}" + git add \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}" + + while IFS= read -r staged_path; do + if [ "${staged_path}" != "${GITEA_AUDIT_CONTROLLER_TARGET}" ] && \ + [ "${staged_path}" != "${GITEA_AUDIT_VERIFIER_TARGET}" ]; then + echo "BLOCKER audit_writeback_unexpected_staged_path" + exit 2 + fi + done < <(git diff --cached --name-only) + + if git diff --cached --quiet; then + echo "receipt_writeback=no_change" + echo "receipt_branch=${GITEA_AUDIT_BRANCH}" + echo "receipt_commit_sha=$(git rev-parse HEAD)" + exit 0 + fi + git commit -m "${GITEA_AUDIT_COMMIT_MESSAGE}" + if git push "${remote_name}" "HEAD:refs/heads/${GITEA_AUDIT_BRANCH}"; then + echo "receipt_writeback=verified_audit_branch_normal_push" + echo "receipt_branch=${GITEA_AUDIT_BRANCH}" + echo "receipt_commit_sha=$(git rev-parse HEAD)" + exit 0 + fi + echo "receipt_writeback_retry=${attempt}" + sleep 5 +done + +echo "BLOCKER receipt_writeback_non_fast_forward_after_bounded_retry" +exit 1 diff --git a/scripts/security/tests/test_external_mcp_artifact_controller.py b/scripts/security/tests/test_external_mcp_artifact_controller.py index 273b68ef6..121f8ac37 100644 --- a/scripts/security/tests/test_external_mcp_artifact_controller.py +++ b/scripts/security/tests/test_external_mcp_artifact_controller.py @@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parents[3] CONTROLLER_PATH = ROOT / "scripts/security/external_mcp_artifact_controller.py" POLICY_PATH = ROOT / "config/mcp/playwright-mcp-artifact-policy.json" WORKFLOW_PATH = ROOT / ".gitea/workflows/mcp-external-artifact-mirror.yaml" +AUDIT_WRITEBACK_PATH = ROOT / "scripts/ci/write-gitea-audit-receipts.sh" RUN_ID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" TRACE_ID = f"mcp-artifact-{RUN_ID}" @@ -130,8 +131,14 @@ class ExternalMcpArtifactControllerTest(unittest.TestCase): self.assertIn("docker logout", raw) self.assertIn("verify_external_mcp_artifact_receipt.py", raw) self.assertIn("RECEIPT_BRANCH: mcp-artifact-receipts", raw) - self.assertIn('git merge --no-edit FETCH_HEAD', raw) - self.assertIn('HEAD:refs/heads/${RECEIPT_BRANCH}', raw) + self.assertIn("bash scripts/ci/write-gitea-audit-receipts.sh", raw) + writeback = AUDIT_WRITEBACK_PATH.read_text(encoding="utf-8") + self.assertIn("git checkout --force --detach FETCH_HEAD", writeback) + self.assertNotIn("git merge --no-edit FETCH_HEAD", writeback) + self.assertIn( + 'git push "${remote_name}" "HEAD:refs/heads/${GITEA_AUDIT_BRANCH}"', + writeback, + ) self.assertNotIn("git push gitea HEAD:main", raw) self.assertIn('cron: "13 2 * * 3"', raw) diff --git a/scripts/security/tests/test_external_mcp_replay_controller.py b/scripts/security/tests/test_external_mcp_replay_controller.py index c267a0550..c4d765a88 100644 --- a/scripts/security/tests/test_external_mcp_replay_controller.py +++ b/scripts/security/tests/test_external_mcp_replay_controller.py @@ -19,6 +19,7 @@ CONTROLLER_PATH = ROOT / "scripts/security/external_mcp_replay_controller.py" POLICY_PATH = ROOT / "config/mcp/playwright-mcp-replay-policy.json" ARTIFACT_POLICY_PATH = ROOT / "config/mcp/playwright-mcp-artifact-policy.json" WORKFLOW_PATH = ROOT / ".gitea/workflows/mcp-external-replay.yaml" +AUDIT_WRITEBACK_PATH = ROOT / "scripts/ci/write-gitea-audit-receipts.sh" SPEC = importlib.util.spec_from_file_location("external_mcp_replay_controller", CONTROLLER_PATH) assert SPEC and SPEC.loader @@ -389,6 +390,14 @@ class ReceiptAndWorkflowTests(unittest.TestCase): self.assertIn("wait-host-web-build-pressure.sh", raw) self.assertIn("mcp-replay-receipts", raw) self.assertIn("verify_external_mcp_replay_receipt.py", raw) + self.assertIn("bash scripts/ci/write-gitea-audit-receipts.sh", raw) + writeback = AUDIT_WRITEBACK_PATH.read_text(encoding="utf-8").lower() + self.assertIn("git checkout --force --detach fetch_head", writeback) + self.assertNotIn("git merge --no-edit fetch_head", writeback) + self.assertIn( + 'git push "${remote_name}" "head:refs/heads/${gitea_audit_branch}"', + writeback, + ) if __name__ == "__main__": From 849b0c6d23c77adf5132a38711d784f174d71595 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 11:36:38 +0800 Subject: [PATCH 3/5] fix(iwooos): correlate canonical ansible candidates --- .../iwooos_security_asset_control_plane.py | 20 ++++++++++++++++++- ...est_iwooos_security_asset_control_plane.py | 3 +++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/apps/api/src/services/iwooos_security_asset_control_plane.py b/apps/api/src/services/iwooos_security_asset_control_plane.py index 826b801de..68ed4315b 100644 --- a/apps/api/src/services/iwooos_security_asset_control_plane.py +++ b/apps/api/src/services/iwooos_security_asset_control_plane.py @@ -1959,7 +1959,25 @@ class IwoooSSecurityAssetControlPlaneService: NULLIF(route.incident_id::text, ''), NULLIF(route.input ->> 'incident_id', '') ) IS NOT NULL - AND NULLIF(route.input ->> 'catalog_id', '') IS NOT NULL) + AND ( + NULLIF(route.input ->> 'catalog_id', '') IS NOT NULL + OR EXISTS ( + SELECT 1 + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof( + route.input -> 'executor_candidates' + ) = 'array' + THEN route.input -> 'executor_candidates' + ELSE '[]'::jsonb + END + ) candidate(candidate_payload) + WHERE NULLIF( + candidate.candidate_payload ->> 'catalog_id', + '' + ) IS NOT NULL + ) + )) AS correlated_controlled_route_count_24h, (SELECT count(*) FROM incident_evidence verifier WHERE verifier.collected_at >= NOW() - INTERVAL '24 hours' diff --git a/apps/api/tests/test_iwooos_security_asset_control_plane.py b/apps/api/tests/test_iwooos_security_asset_control_plane.py index f121075f2..9e724ed12 100644 --- a/apps/api/tests/test_iwooos_security_asset_control_plane.py +++ b/apps/api/tests/test_iwooos_security_asset_control_plane.py @@ -744,6 +744,9 @@ def test_service_bounds_source_concurrency_for_database_budget(monkeypatch) -> N ) assert "correlated_controlled_route_count_24h" in siem_query assert "ansible_candidate_matched" in siem_query + assert "jsonb_array_elements" in siem_query + assert "executor_candidates" in siem_query + assert "candidate_payload ->> 'catalog_id'" in siem_query assert "independent_verifier_pass_count_24h" in siem_query assert "all_postconditions_passed" in siem_query assert "executor_returncode_trusted" in siem_query From bc66cecdb19c6d8e71f17e14df5584253b7bbf0a Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 11:47:32 +0800 Subject: [PATCH 4/5] chore(cd): deploy 7cd0f69 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index d8032c803..7f194d365 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 729b79f92..98d0a61f1 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index a56468090..6b3791fa7 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -181,7 +181,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index c638d5532..c5b647daf 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:d8304318081a6c4267d78cc1bc6b43a88902a17bfccc7e1146989609282453ad +- digest: sha256:6da3614f3925161eb9c6015ed4997543c89d3a1c83259935a6e28121f6587f9c name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:7891f1fb9c2e6473d3ae102efc6c80ae1be48e1fda96a4de0a0c4d8f6cad4238 +- digest: sha256:5c1cbb8c330c37349c3b7a60a533672f53d893e6eb565c53112be3af125be527 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 82759e7cd34c174f20a585eeb6c64bc5bb15a63b Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 11:51:30 +0800 Subject: [PATCH 5/5] fix(agent99): reduce SSH performance probe churn --- agent99-bootstrap.ps1 | 10 + agent99-control-plane.ps1 | 355 +++++++++++++++++- agent99-deploy.ps1 | 60 ++- agent99.config.99.example.json | 10 + agent99.config.example.json | 10 + ...nt99_node_exporter_performance_contract.py | 165 ++++++++ 6 files changed, 594 insertions(+), 16 deletions(-) create mode 100644 scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 1ad66b9e6..be020aeec 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -353,13 +353,23 @@ if (-not (Test-Path $configPath)) { "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [] } }, "backupHealth": { diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index e16a0a0f7..68bca2b88 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -634,6 +634,9 @@ function Format-AgentTelegramLegacyText { $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null $load = Get-AgentObjectValue $Data "loadPerCore" $null $mem = Get-AgentObjectValue $Data "memAvailablePercent" $null + $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null + $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null + $metricSource = Get-AgentObjectValue $Data "metricSource" "unknown" $lines += "事件 Event: 主機效能 / host performance" $lines += "主機 Host: $hostName" if ($reasons -contains "perf_readback_failed" -or ($null -eq $disk -and $null -eq $load -and $null -eq $mem)) { @@ -649,6 +652,12 @@ function Format-AgentTelegramLegacyText { } elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") { $lines += "原因 Reason: CPU load/core=$load。" $lines += "Agent99 動作 Action: 僅使用 allowlisted load reduction;不得任意停止服務。" + } elseif ($reasons -contains "kernel_percpu_memory_critical" -or $reasons -contains "kernel_sunreclaim_memory_critical") { + $lines += "原因 Reason: kernel memory pressure 已達 critical;Percpu=$kernelPercpu percent,SUnreclaim=$kernelSUnreclaim percent。" + $lines += "Agent99 動作 Action: 日常採 Node Exporter 無 SSH 監測;process 診斷限流,並保留受控 kernel remediation 與獨立 verifier。" + } elseif ($reasons -contains "kernel_percpu_memory_warning") { + $lines += "原因 Reason: kernel Percpu memory=$kernelPercpu percent,已達 warning threshold。" + $lines += "Agent99 動作 Action: 持續 Node Exporter readback,SSH 診斷依 cooldown 執行,避免監控本身增加 session churn。" } elseif ($reasons -contains "memory_available_low") { $lines += "原因 Reason: 可用記憶體為 $mem percent。" $lines += "Agent99 動作 Action: 先檢查 memory pressure,再進行 allowlisted remediation。" @@ -660,6 +669,9 @@ function Format-AgentTelegramLegacyText { if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" } if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" } if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" } + if ($null -ne $kernelPercpu -and [string]$kernelPercpu -ne "") { $metricParts += "kernel Percpu=$kernelPercpu percent" } + if ($null -ne $kernelSUnreclaim -and [string]$kernelSUnreclaim -ne "") { $metricParts += "kernel SUnreclaim=$kernelSUnreclaim percent" } + if ($metricSource -ne "unknown") { $metricParts += "source=$metricSource" } if ($metricParts.Count -gt 0) { $lines += "目前狀態 Current: $($metricParts -join ', ')." } else { @@ -964,22 +976,44 @@ function Get-AgentIncidentCardModel { $load = Get-AgentObjectValue $Data "loadPerCore" $null $memory = Get-AgentObjectValue $Data "memAvailablePercent" $null $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null - $title = if ($reasons -contains "perf_readback_failed") { "$hostName 效能讀回失敗" } else { "$hostName 主機效能異常" } + $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null + $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null + $metricSource = [string](Get-AgentObjectValue $Data "metricSource" "unknown") + $kernelPressure = [bool](@($reasons | Where-Object { $_ -match '^kernel_' }).Count -gt 0) + $title = if ($reasons -contains "perf_readback_failed") { + "$hostName 效能讀回失敗" + } elseif ($kernelPressure) { + "$hostName kernel memory pressure" + } else { + "$hostName 主機效能異常" + } $target = if ($hostName) { $hostName } else { "managed host" } if ($reasons -contains "perf_readback_failed" -or ($null -eq $load -and $null -eq $memory -and $null -eq $disk)) { $impact = "CPU、記憶體與磁碟數值目前不可信,不能把空白讀回當成主機故障。" $action = "先修復監控 transport/readback,再決定是否需要降載。" $verification = "等待完整三項主機指標重新讀回。" + } elseif ($kernelPressure) { + $impact = "Kernel Percpu 或不可回收記憶體持續偏高,可能壓縮服務可用記憶體;不能誤判為單一 process RSS。" + $action = "日常改用 Node Exporter 無 SSH 監測;process 診斷依 cooldown 限流,修復走受控 kernel remediation。" + $verification = "source=$metricSource;修復後須同時確認 Percpu、SUnreclaim、可用記憶體與 session churn 回落。" } else { $impact = "資源壓力可能降低服務回應速度或可用性。" $action = "只執行 allowlisted 降載或清理,禁止任意停服務或重啟主機。" - $verification = "修復後必須重新讀回 CPU、記憶體與磁碟才可關閉。" + $verification = "source=$metricSource;修復後必須重新讀回 CPU、記憶體與磁碟才可關閉。" + } + $metrics = if ($kernelPressure) { + @( + [pscustomobject]@{ name = "Kernel Percpu"; value = $kernelPercpu; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "Kernel SUnreclaim"; value = $kernelSUnreclaim; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" } + ) + } else { + @( + [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "磁碟使用"; value = $disk; scale = 100.0; suffix = "%" } + ) } - $metrics = @( - [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, - [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, - [pscustomobject]@{ name = "磁碟使用"; value = $disk; scale = 100.0; suffix = "%" } - ) $visualKind = "performance" } elseif ($EventType -match "^backup_") { $age = Get-AgentObjectValue $Data "ageMinutes" $null @@ -2730,12 +2764,214 @@ function Convert-AgentDouble { } } +function Get-AgentNodeExporterSource { + param([string]$TargetHost) + + $performance = $Config.performance + if ( + -not $performance -or + -not $performance.PSObject.Properties["nodeExporterUrls"] -or + -not $performance.nodeExporterUrls.PSObject.Properties[$TargetHost] + ) { + return [pscustomobject]@{ + configured = $false + valid = $false + url = $null + timeoutSeconds = 0 + reason = "node_exporter_not_configured" + } + } + + $rawUrl = [string]$performance.nodeExporterUrls.PSObject.Properties[$TargetHost].Value + $uri = $null + $validUri = [Uri]::TryCreate($rawUrl, [UriKind]::Absolute, [ref]$uri) + $valid = [bool]( + $validUri -and + $uri.Scheme -eq "http" -and + $uri.Host -eq $TargetHost -and + $uri.Port -eq 9100 -and + $uri.AbsolutePath -eq "/metrics" -and + -not $uri.UserInfo -and + -not $uri.Query -and + -not $uri.Fragment + ) + $timeoutSeconds = 8 + if ($performance.PSObject.Properties["nodeExporterTimeoutSeconds"]) { + $timeoutSeconds = [math]::Min(15, [math]::Max(2, [int]$performance.nodeExporterTimeoutSeconds)) + } + [pscustomobject]@{ + configured = $true + valid = $valid + url = if ($valid) { $uri.AbsoluteUri } else { $null } + timeoutSeconds = $timeoutSeconds + reason = if ($valid) { "ready" } else { "node_exporter_url_policy_rejected" } + } +} + +function Invoke-AgentNodeExporterPerformanceReadback { + param([string]$TargetHost) + + $source = Get-AgentNodeExporterSource $TargetHost + if (-not $source.configured -or -not $source.valid) { + return [pscustomobject]@{ + configured = [bool]$source.configured + ok = $false + route = "node_exporter_http" + reason = $source.reason + output = "" + elapsedMs = 0 + httpStatus = $null + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + try { + $response = Invoke-WebRequest -Method Get -Uri $source.url -UseBasicParsing -TimeoutSec $source.timeoutSeconds + $stopwatch.Stop() + $httpStatus = [int]$response.StatusCode + $content = [string]$response.Content + if ($httpStatus -ne 200 -or -not $content) { + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_http_not_ready" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $load1 = $null + $memAvailableBytes = $null + $memTotalBytes = $null + $kernelPercpuBytes = $null + $kernelSUnreclaimBytes = $null + $rootAvailableBytes = $null + $rootSizeBytes = $null + $cpuIds = @{} + foreach ($line in @($content -split "`r?`n")) { + if ($null -eq $load1 -and $line -match '^node_load1\s+([^\s]+)\s*$') { + $load1 = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $memAvailableBytes -and $line -match '^node_memory_MemAvailable_bytes\s+([^\s]+)\s*$') { + $memAvailableBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $memTotalBytes -and $line -match '^node_memory_MemTotal_bytes\s+([^\s]+)\s*$') { + $memTotalBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $kernelPercpuBytes -and $line -match '^node_memory_Percpu_bytes\s+([^\s]+)\s*$') { + $kernelPercpuBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $kernelSUnreclaimBytes -and $line -match '^node_memory_SUnreclaim_bytes\s+([^\s]+)\s*$') { + $kernelSUnreclaimBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($line -match '^node_cpu_seconds_total\{([^}]*)\}\s+[^\s]+\s*$') { + $labels = $Matches[1] + if ($labels -match '(^|,)mode="idle"(,|$)' -and $labels -match '(^|,)cpu="([^"]+)"(,|$)') { + $cpuIds[$Matches[2]] = $true + } + continue + } + if ($line -match '^node_filesystem_(avail|size)_bytes\{([^}]*)\}\s+([^\s]+)\s*$') { + $metricKind = $Matches[1] + $labels = $Matches[2] + $metricValue = Convert-AgentDouble $Matches[3] + if ($labels -match '(^|,)mountpoint="/"(,|$)') { + if ($metricKind -eq "avail" -and $null -eq $rootAvailableBytes) { $rootAvailableBytes = $metricValue } + if ($metricKind -eq "size" -and $null -eq $rootSizeBytes) { $rootSizeBytes = $metricValue } + } + } + } + + $cores = $cpuIds.Count + if ( + $cores -le 0 -or + $null -eq $load1 -or + $null -eq $memAvailableBytes -or + $null -eq $memTotalBytes -or + $memTotalBytes -le 0 -or + $null -eq $rootAvailableBytes -or + $null -eq $rootSizeBytes -or + $rootSizeBytes -le 0 + ) { + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_required_metrics_missing" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $memAvailablePercent = [math]::Round(($memAvailableBytes / $memTotalBytes) * 100, 2) + $kernelPercpuMemoryPercent = if ($null -ne $kernelPercpuBytes) { + [math]::Round(($kernelPercpuBytes / $memTotalBytes) * 100, 2) + } else { $null } + $kernelSUnreclaimMemoryPercent = if ($null -ne $kernelSUnreclaimBytes) { + [math]::Round(($kernelSUnreclaimBytes / $memTotalBytes) * 100, 2) + } else { $null } + $diskUsedPercent = [math]::Round((1 - ($rootAvailableBytes / $rootSizeBytes)) * 100, 0) + $diskUsedPercent = [math]::Min(100, [math]::Max(0, $diskUsedPercent)) + $canonical = "perf_schema=agent99_perf_readback_v1 os=linux cores=$cores load1=$load1 mem_available_percent=$memAvailablePercent disk_used_percent=$diskUsedPercent" + return [pscustomobject]@{ + configured = $true + ok = $true + exitCode = 0 + route = "node_exporter_http" + reason = "verified" + output = $canonical + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + kernelPercpuMemoryPercent = $kernelPercpuMemoryPercent + kernelSUnreclaimMemoryPercent = $kernelSUnreclaimMemoryPercent + kernelMemoryMetricsAvailable = [bool]($null -ne $kernelPercpuMemoryPercent -and $null -ne $kernelSUnreclaimMemoryPercent) + transportSerialized = $false + transportLockWaitMs = 0 + } + } catch { + $stopwatch.Stop() + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_transport_failed" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $null + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } +} + function Get-PerfThresholds { $perf = $Config.performance [pscustomobject]@{ loadPerCoreWarning = if ($perf.loadPerCoreWarning) { [double]$perf.loadPerCoreWarning } else { 0.9 } loadPerCoreCritical = if ($perf.loadPerCoreCritical) { [double]$perf.loadPerCoreCritical } else { 1.5 } memoryAvailablePercentCritical = if ($perf.memoryAvailablePercentCritical) { [double]$perf.memoryAvailablePercentCritical } else { 10 } + kernelPercpuMemoryPercentWarning = if ($perf.kernelPercpuMemoryPercentWarning) { [double]$perf.kernelPercpuMemoryPercentWarning } else { 10 } + kernelPercpuMemoryPercentCritical = if ($perf.kernelPercpuMemoryPercentCritical) { [double]$perf.kernelPercpuMemoryPercentCritical } else { 20 } + kernelSUnreclaimMemoryPercentCritical = if ($perf.kernelSUnreclaimMemoryPercentCritical) { [double]$perf.kernelSUnreclaimMemoryPercentCritical } else { 25 } diskUsedPercentWarning = if ($perf.diskUsedPercentWarning) { [double]$perf.diskUsedPercentWarning } else { 90 } diskUsedPercentCritical = if ($perf.diskUsedPercentCritical) { [double]$perf.diskUsedPercentCritical } else { 95 } } @@ -2754,6 +2990,17 @@ function Get-HostPerformance { $readRetries = [int]$Config.performance.readRetries } $results = @() + $previousPerfEvidence = Get-AgentLatestEvidence "agent99-Perf-*.json" + $previousPerfRows = if ( + $previousPerfEvidence.exists -and + -not $previousPerfEvidence.parseError -and + $previousPerfEvidence.data -and + $previousPerfEvidence.data.PSObject.Properties["performance"] + ) { @($previousPerfEvidence.data.performance) } else { @() } + $diagnosticCooldownMinutes = 15 + if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuDiagnosticCooldownMinutes"]) { + $diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5, [int]$Config.performance.topCpuDiagnosticCooldownMinutes)) + } $command = @' echo __AGENT99_PERF__ echo HOST=$(hostname) @@ -2772,7 +3019,16 @@ df -P / 2>/dev/null } $slowRetryUsed = $false $slowTimeoutSeconds = $null - $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries + $nodeExporterReadback = Invoke-AgentNodeExporterPerformanceReadback $hostIp + $metricsFallbackReason = $null + if ($nodeExporterReadback.ok) { + $readback = $nodeExporterReadback + } else { + if ($nodeExporterReadback.configured) { + $metricsFallbackReason = [string]$nodeExporterReadback.reason + } + $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries + } if (-not $readback.ok -and $readback.exitCode -eq -2) { $slowTimeoutSeconds = 90 if ($Config.performance -and $Config.performance.PSObject.Properties["slowReadTimeoutSeconds"]) { @@ -2795,6 +3051,8 @@ df -P / 2>/dev/null $load1 = $null $memAvailablePercent = $null $diskUsedPercent = $null + $kernelPercpuMemoryPercent = $null + $kernelSUnreclaimMemoryPercent = $null $canonicalPerfMatch = [regex]::Match( $text, "(?m)^perf_schema=agent99_perf_readback_v1\s+os=(?\S+)\s+cores=(?\d+)\s+load1=(?[0-9]+(?:[\.,][0-9]+)?)\s+mem_available_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s+disk_used_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s*$" @@ -2834,8 +3092,27 @@ df -P / 2>/dev/null if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } } } + if ($nodeExporterReadback.ok) { + if ($nodeExporterReadback.PSObject.Properties["kernelPercpuMemoryPercent"]) { + $kernelPercpuMemoryPercent = $nodeExporterReadback.kernelPercpuMemoryPercent + } + if ($nodeExporterReadback.PSObject.Properties["kernelSUnreclaimMemoryPercent"]) { + $kernelSUnreclaimMemoryPercent = $nodeExporterReadback.kernelSUnreclaimMemoryPercent + } + } $topCpu = @() + $diagnosticStatus = "not_required" + $diagnosticKind = "none" + $diagnosticLastAttemptAt = $null + $diagnosticCapturedAt = $null + $previousPerfRow = $previousPerfRows | Where-Object { [string]$_.host -eq $hostIp } | Select-Object -First 1 + if ($previousPerfRow -and $previousPerfRow.PSObject.Properties["diagnosticLastAttemptAt"]) { + $diagnosticLastAttemptAt = [string]$previousPerfRow.diagnosticLastAttemptAt + } + if ($previousPerfRow -and $previousPerfRow.PSObject.Properties["diagnosticCapturedAt"]) { + $diagnosticCapturedAt = [string]$previousPerfRow.diagnosticCapturedAt + } $loadPerCore = if ($null -ne $load1 -and $cores -gt 0) { [math]::Round($load1 / $cores, 2) } else { $null } $severity = "ok" @@ -2855,6 +3132,17 @@ df -P / 2>/dev/null $severity = "critical" $reasons += "memory_available_low" } + if ($null -ne $kernelPercpuMemoryPercent -and $kernelPercpuMemoryPercent -ge $thresholds.kernelPercpuMemoryPercentCritical) { + $severity = "critical" + $reasons += "kernel_percpu_memory_critical" + } elseif ($null -ne $kernelPercpuMemoryPercent -and $kernelPercpuMemoryPercent -ge $thresholds.kernelPercpuMemoryPercentWarning) { + if ($severity -ne "critical") { $severity = "warning" } + $reasons += "kernel_percpu_memory_warning" + } + if ($null -ne $kernelSUnreclaimMemoryPercent -and $kernelSUnreclaimMemoryPercent -ge $thresholds.kernelSUnreclaimMemoryPercentCritical) { + $severity = "critical" + $reasons += "kernel_sunreclaim_memory_critical" + } if ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentCritical) { $severity = "critical" $reasons += "disk_used_high" @@ -2871,13 +3159,38 @@ df -P / 2>/dev/null if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuTimeoutSeconds"]) { $topCpuTimeoutSeconds = [int]$Config.performance.topCpuTimeoutSeconds } - $shouldCaptureTopCpu = ($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning) + $processDiagnosticReason = [bool](@($reasons | Where-Object { $_ -match '^(load_per_core|memory_available|kernel_)' }).Count -gt 0) + $shouldCaptureTopCpu = $processDiagnosticReason -and (($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning)) if ($shouldCaptureTopCpu -and $readback.ok) { - $topReadback = Invoke-HostSshText $hostIp "ps -eo pid,ppid,comm,pcpu,pmem --sort=-pcpu" $topCpuTimeoutSeconds 1 - if ($topReadback.ok) { - $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) + $diagnosticDue = $true + if ($diagnosticLastAttemptAt) { + try { + $lastAttempt = [DateTimeOffset]::Parse($diagnosticLastAttemptAt) + $diagnosticDue = [bool](([DateTimeOffset]::Now - $lastAttempt).TotalMinutes -ge $diagnosticCooldownMinutes) + } catch { + $diagnosticDue = $true + } + } + if ($diagnosticDue) { + $diagnosticNow = [DateTimeOffset]::Now.ToString("o") + $diagnosticLastAttemptAt = $diagnosticNow + $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } + $diagnosticCommand = if ($diagnosticKind -eq "top_rss") { + "ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-rss" + } else { + "ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-pcpu" + } + $topReadback = Invoke-HostSshText $hostIp $diagnosticCommand $topCpuTimeoutSeconds 1 + if ($topReadback.ok) { + $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) + $diagnosticCapturedAt = $diagnosticNow + $diagnosticStatus = "captured" + } else { + $diagnosticStatus = "readback_failed_backoff_active" + } } else { - $topCpu = @("top_cpu_readback_failed: $($topReadback.output)") + $diagnosticStatus = "cooldown_suppressed" + $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } } } @@ -2892,7 +3205,19 @@ df -P / 2>/dev/null loadPerCore = $loadPerCore memAvailablePercent = $memAvailablePercent diskUsedPercent = $diskUsedPercent + kernelPercpuMemoryPercent = $kernelPercpuMemoryPercent + kernelSUnreclaimMemoryPercent = $kernelSUnreclaimMemoryPercent + kernelMemoryMetricsAvailable = [bool]($null -ne $kernelPercpuMemoryPercent -and $null -ne $kernelSUnreclaimMemoryPercent) topCpu = $topCpu + metricSource = if ($nodeExporterReadback.ok) { "node_exporter" } elseif ($nodeExporterReadback.configured) { "ssh_fallback" } else { "ssh" } + metricSourceUrl = if ($nodeExporterReadback.ok) { $nodeExporterReadback.sourceUrl } else { $null } + metricSourceHttpStatus = if ($nodeExporterReadback.ok) { $nodeExporterReadback.httpStatus } else { $null } + metricsFallbackReason = $metricsFallbackReason + diagnosticStatus = $diagnosticStatus + diagnosticKind = $diagnosticKind + diagnosticLastAttemptAt = $diagnosticLastAttemptAt + diagnosticCapturedAt = $diagnosticCapturedAt + diagnosticCooldownMinutes = $diagnosticCooldownMinutes output = $text readTimeoutSeconds = $hostTimeoutSeconds slowReadTimeoutSeconds = $slowTimeoutSeconds @@ -2901,9 +3226,9 @@ df -P / 2>/dev/null transportSerialized = if ($readback.PSObject.Properties["transportSerialized"]) { [bool]$readback.transportSerialized } else { $false } transportLockWaitMs = if ($readback.PSObject.Properties["transportLockWaitMs"]) { [long]$readback.transportLockWaitMs } else { $null } } - Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" + Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent kernelPercpuPct=$kernelPercpuMemoryPercent kernelSUnreclaimPct=$kernelSUnreclaimMemoryPercent route=$($readback.route) metricSource=$($result.metricSource) metricsFallback=$metricsFallbackReason diagnostic=$diagnosticStatus elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) { - Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert + Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent kernelPercpuPct=$kernelPercpuMemoryPercent kernelSUnreclaimPct=$kernelSUnreclaimMemoryPercent reasons=$($reasons -join ',')" $result -Alert } $results += $result } diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 3998fc0f4..5402d50b8 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -515,6 +515,44 @@ try { Add-DefaultProperty $config "performance" ([pscustomobject]@{}) Add-DefaultProperty $config.performance "processCpuHostPercentWarning" 25 Add-DefaultProperty $config.performance "processCpuHostPercentCritical" 50 + Add-DefaultProperty $config.performance "nodeExporterTimeoutSeconds" ([int]$policyConfig.performance.nodeExporterTimeoutSeconds) + Add-DefaultProperty $config.performance "nodeExporterUrls" ($policyConfig.performance.nodeExporterUrls | ConvertTo-Json -Depth 5 | ConvertFrom-Json) + Add-DefaultProperty $config.performance "topCpuDiagnosticCooldownMinutes" ([int]$policyConfig.performance.topCpuDiagnosticCooldownMinutes) + Add-DefaultProperty $config.performance "kernelPercpuMemoryPercentWarning" ([int]$policyConfig.performance.kernelPercpuMemoryPercentWarning) + Add-DefaultProperty $config.performance "kernelPercpuMemoryPercentCritical" ([int]$policyConfig.performance.kernelPercpuMemoryPercentCritical) + Add-DefaultProperty $config.performance "kernelSUnreclaimMemoryPercentCritical" ([int]$policyConfig.performance.kernelSUnreclaimMemoryPercentCritical) + foreach ($nodeExporterHost in @("192.168.0.110", "192.168.0.112", "192.168.0.188")) { + $expectedNodeExporterUrl = [string]$policyConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value + $currentNodeExporterUrl = if ($config.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost]) { + [string]$config.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value + } else { "" } + if ($currentNodeExporterUrl -ne $expectedNodeExporterUrl) { + Set-AgentProperty $config.performance.nodeExporterUrls $nodeExporterHost $expectedNodeExporterUrl + $script:ConfigMigrations += [pscustomobject]@{ + name = "node_exporter_performance_route_$($nodeExporterHost.Replace('.', '_'))" + from = $currentNodeExporterUrl + to = $expectedNodeExporterUrl + } + } + } + foreach ($performancePolicyName in @( + "nodeExporterTimeoutSeconds", + "topCpuDiagnosticCooldownMinutes", + "kernelPercpuMemoryPercentWarning", + "kernelPercpuMemoryPercentCritical", + "kernelSUnreclaimMemoryPercentCritical" + )) { + $expectedPerformanceValue = [int]$policyConfig.performance.PSObject.Properties[$performancePolicyName].Value + $currentPerformanceValue = [int]$config.performance.PSObject.Properties[$performancePolicyName].Value + if ($currentPerformanceValue -ne $expectedPerformanceValue) { + Set-AgentProperty $config.performance $performancePolicyName $expectedPerformanceValue + $script:ConfigMigrations += [pscustomobject]@{ + name = "performance_policy_$performancePolicyName" + from = $currentPerformanceValue + to = $expectedPerformanceValue + } + } + } foreach ($sourceAction in @($policyConfig.performance.loadShedding.actions | Where-Object { $_.PSObject.Properties["policyVersion"] })) { $liveAction = $config.performance.loadShedding.actions | Where-Object { $_.name -eq $sourceAction.name } | Select-Object -First 1 @@ -545,6 +583,25 @@ try { $persistedHosts = @($persistedConfig.hosts | ForEach-Object { [string]$_ }) $persistedSelfHealthHosts = @($persistedConfig.selfHealth.requiredHosts | ForEach-Object { [string]$_ }) $persistedPublicUrls = @($persistedConfig.publicUrls | ForEach-Object { [string]$_ }) + $nodeExporterRoutesReady = $true + foreach ($nodeExporterHost in @("192.168.0.110", "192.168.0.112", "192.168.0.188")) { + $expectedNodeExporterUrl = "http://${nodeExporterHost}:9100/metrics" + if ( + -not $persistedConfig.performance.nodeExporterUrls -or + -not $persistedConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost] -or + [string]$persistedConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value -ne $expectedNodeExporterUrl + ) { + $nodeExporterRoutesReady = $false + } + } + $nodeExporterPolicyReady = [bool]( + $nodeExporterRoutesReady -and + [int]$persistedConfig.performance.nodeExporterTimeoutSeconds -eq 8 -and + [int]$persistedConfig.performance.topCpuDiagnosticCooldownMinutes -eq 15 -and + [int]$persistedConfig.performance.kernelPercpuMemoryPercentWarning -eq 10 -and + [int]$persistedConfig.performance.kernelPercpuMemoryPercentCritical -eq 20 -and + [int]$persistedConfig.performance.kernelSUnreclaimMemoryPercentCritical -eq 25 + ) $persistedEdgeServices = if ($persistedConfig.PSObject.Properties["edgeServiceRecovery"]) { $persistedConfig.edgeServiceRecovery } else { $null } $persistedEdgeServicesReady = [bool]( $persistedEdgeServices -and @@ -571,6 +628,7 @@ try { "192.168.0.111" -notin $persistedSelfHealthHosts -or "https://n8n.wooo.work" -notin $persistedPublicUrls -or "https://ollama.wooo.work" -notin $persistedPublicUrls -or + -not $nodeExporterPolicyReady -or -not $persistedEdgeServicesReady -or -not $host111ExternalReady -or $persistedHost112Vm.Count -ne 1 -or @@ -579,7 +637,7 @@ try { $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Host188 edge recovery config post-verifier also failed." + throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Node-exporter performance routes or Host188 edge recovery config post-verifier also failed." } $manifestRows = @() diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 92980ff42..4533279a3 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -233,13 +233,23 @@ "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [ diff --git a/agent99.config.example.json b/agent99.config.example.json index 744ab7088..e8e664216 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -222,13 +222,23 @@ "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [] diff --git a/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py new file mode 100644 index 000000000..747518b37 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +CONTROL = ROOT / "agent99-control-plane.ps1" +DEPLOY = ROOT / "agent99-deploy.ps1" +BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" +CONFIGS = ( + ROOT / "agent99.config.example.json", + ROOT / "agent99.config.99.example.json", +) + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def function(source: str, name: str, next_name: str) -> str: + return source[ + source.index(f"function {name} {{") : source.index( + f"function {next_name} {{" + ) + ] + + +def test_node_exporter_routes_are_explicit_and_identical_in_policy_configs() -> None: + expected = { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics", + } + + for path in CONFIGS: + performance = json.loads(read(path))["performance"] + assert performance["nodeExporterUrls"] == expected + assert performance["nodeExporterTimeoutSeconds"] == 8 + assert performance["topCpuDiagnosticCooldownMinutes"] == 15 + assert performance["kernelPercpuMemoryPercentWarning"] == 10 + assert performance["kernelPercpuMemoryPercentCritical"] == 20 + assert performance["kernelSUnreclaimMemoryPercentCritical"] == 25 + + bootstrap = read(BOOTSTRAP) + for host, url in expected.items(): + assert f'"{host}": "{url}"' in bootstrap + + +def test_node_exporter_transport_is_fixed_to_host_port_path_and_no_credentials() -> None: + source = read(CONTROL) + policy = function( + source, + "Get-AgentNodeExporterSource", + "Invoke-AgentNodeExporterPerformanceReadback", + ) + transport = function( + source, + "Invoke-AgentNodeExporterPerformanceReadback", + "Get-PerfThresholds", + ) + + for requirement in ( + '$uri.Scheme -eq "http"', + '$uri.Host -eq $TargetHost', + '$uri.Port -eq 9100', + '$uri.AbsolutePath -eq "/metrics"', + '-not $uri.UserInfo', + '-not $uri.Query', + '-not $uri.Fragment', + 'node_exporter_url_policy_rejected', + ): + assert requirement in policy + assert "Invoke-WebRequest -Method Get" in transport + assert "-TimeoutSec $source.timeoutSeconds" in transport + assert 'reason = "node_exporter_transport_failed"' in transport + assert "password" not in transport.lower() + assert "token" not in transport.lower() + + +def test_node_exporter_parser_requires_complete_load_memory_cpu_and_root_disk() -> None: + source = read(CONTROL) + transport = function( + source, + "Invoke-AgentNodeExporterPerformanceReadback", + "Get-PerfThresholds", + ) + + for metric in ( + "node_load1", + "node_memory_MemAvailable_bytes", + "node_memory_MemTotal_bytes", + "node_memory_Percpu_bytes", + "node_memory_SUnreclaim_bytes", + "node_cpu_seconds_total", + "node_filesystem_(avail|size)_bytes", + ): + assert metric in transport + assert 'mountpoint="/"' in transport + assert 'reason = "node_exporter_required_metrics_missing"' in transport + assert "perf_schema=agent99_perf_readback_v1" in transport + assert 'route = "node_exporter_http"' in transport + assert "kernelPercpuMemoryPercent" in transport + assert "kernelSUnreclaimMemoryPercent" in transport + assert "kernelMemoryMetricsAvailable" in transport + + +def test_perf_uses_metrics_first_and_ssh_only_as_fallback_or_cooldown_diagnostic() -> None: + source = read(CONTROL) + perf = function(source, "Get-HostPerformance", "Record-PerformanceIssuesWithoutRemediation") + + metrics_at = perf.index("Invoke-AgentNodeExporterPerformanceReadback $hostIp") + fallback_at = perf.index("Invoke-HostSshText $hostIp $command") + assert metrics_at < fallback_at + assert "if ($nodeExporterReadback.ok)" in perf + assert "$metricsFallbackReason = [string]$nodeExporterReadback.reason" in perf + assert 'metricSource = if ($nodeExporterReadback.ok) { "node_exporter" }' in perf + assert 'diagnosticStatus = "cooldown_suppressed"' in perf + assert "topCpuDiagnosticCooldownMinutes" in perf + assert "$diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5" in perf + assert "memory_available_low" in perf + assert "kernel_percpu_memory_warning" in perf + assert "kernel_percpu_memory_critical" in perf + assert "kernel_sunreclaim_memory_critical" in perf + assert "kernelPercpuMemoryPercentWarning" in perf + assert "kernelPercpuMemoryPercentCritical" in perf + assert "kernelSUnreclaimMemoryPercentCritical" in perf + assert '"ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-rss"' in perf + assert "disk_used_warning" not in perf[ + perf.index("$processDiagnosticReason") : perf.index("$result =") + ] + + +def test_deploy_migrates_and_post_verifies_canonical_metric_routes() -> None: + deploy = read(DEPLOY) + + assert 'Add-DefaultProperty $config.performance "nodeExporterUrls"' in deploy + assert 'Set-AgentProperty $config.performance.nodeExporterUrls' in deploy + assert 'name = "node_exporter_performance_route_' in deploy + assert "$nodeExporterRoutesReady = $true" in deploy + assert "$nodeExporterPolicyReady = [bool](" in deploy + assert "-not $nodeExporterPolicyReady" in deploy + assert "node-exporter performance routes" in deploy.lower() + for policy_name in ( + "kernelPercpuMemoryPercentWarning", + "kernelPercpuMemoryPercentCritical", + "kernelSUnreclaimMemoryPercentCritical", + ): + assert f'Add-DefaultProperty $config.performance "{policy_name}"' in deploy + assert policy_name in deploy[deploy.index("$nodeExporterPolicyReady") :] + + +def test_performance_card_explains_kernel_memory_and_metric_source() -> None: + source = read(CONTROL) + legacy = function(source, "Format-AgentTelegramLegacyText", "Get-AgentSha256Hex") + model = function(source, "Get-AgentIncidentCardModel", "Format-AgentTelegramText") + + for formatter in (legacy, model): + assert "kernel memory pressure" in formatter + assert "Percpu" in formatter + assert "SUnreclaim" in formatter + assert 'Get-AgentObjectValue $Data "metricSource"' in formatter + assert "Node Exporter" in formatter + assert "$metrics = if ($kernelPressure)" in model + assert model.index('name = "Kernel Percpu"') < model.index('name = "CPU load/core"')