From 89cbc292da24e142d239712c3bca713915d26077 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Tue, 14 Jul 2026 20:40:04 +0800 Subject: [PATCH 01/10] chore(cd): deploy 1c3c62c [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 8caae219c..5eee89ec2 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "6960d39197facedc846ae04e1f2c5eb597f6d48f" + value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "6960d39197facedc846ae04e1f2c5eb597f6d48f" + value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index d13f082cd..b38d02858 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6960d39197facedc846ae04e1f2c5eb597f6d48f" + value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 046a4d0db..a5b542a98 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6960d39197facedc846ae04e1f2c5eb597f6d48f" + value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index f40ee79df..92a431fd0 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:539d0f7c43eed2dd5633f1154bea59be00f2a38f739414cc2a43fa940f31c633 +- digest: sha256:0ea8faa81f4ba625c46f1fa0d6a69ca8e3718782e09c59dd0dbcdca4a5f7bbd0 name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:e9f5604ff83dec1d652d2759e4e00f78603d0c018d2ec261f1257ed7ac91c872 +- digest: sha256:6dd391003d64930bf0badec26fbfa3f37b974af07acfd809352f6975d57685f9 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 0e28fbc1dc12f42d8122e37fbe2ecbedb875c8fe Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 20:13:12 +0800 Subject: [PATCH 02/10] fix(security): resolve TLS inventory in runtime image --- apps/api/src/jobs/asset_scanner_job.py | 22 +++++++++++++++++++--- apps/api/tests/test_asset_scanner_job.py | 12 ++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/apps/api/src/jobs/asset_scanner_job.py b/apps/api/src/jobs/asset_scanner_job.py index 939ebc83e..cef3d872f 100644 --- a/apps/api/src/jobs/asset_scanner_job.py +++ b/apps/api/src/jobs/asset_scanner_job.py @@ -1506,11 +1506,27 @@ def _build_domain_tls_inventory_assets( return [*domains.values(), *certificates.values()], relationships +def _domain_tls_inventory_candidates( + *, + cwd: Path | None = None, + source_path: Path | None = None, +) -> tuple[Path, ...]: + relative = Path("docs/security/domain-tls-certbot-inventory.snapshot.json") + source = (source_path or Path(__file__)).resolve() + roots = (cwd or Path.cwd(), *source.parents) + return tuple(dict.fromkeys(root / relative for root in roots)) + + def _load_domain_tls_inventory_payload(path: Path | None = None) -> dict[str, Any]: if path is None: - relative = Path("docs/security/domain-tls-certbot-inventory.snapshot.json") - candidates = (Path.cwd() / relative, Path(__file__).resolve().parents[4] / relative) - path = next((candidate for candidate in candidates if candidate.is_file()), None) + path = next( + ( + candidate + for candidate in _domain_tls_inventory_candidates() + if candidate.is_file() + ), + None, + ) if path is None or not path.is_file(): raise RuntimeError("domain_tls_inventory_snapshot_missing") if path.stat().st_size > 5_000_000: diff --git a/apps/api/tests/test_asset_scanner_job.py b/apps/api/tests/test_asset_scanner_job.py index ab5c03554..8ccb262d9 100644 --- a/apps/api/tests/test_asset_scanner_job.py +++ b/apps/api/tests/test_asset_scanner_job.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import json +from pathlib import Path from src.jobs import asset_scanner_job @@ -609,6 +610,17 @@ def test_domain_tls_inventory_projects_fingerprints_without_raw_paths() -> None: assert "privkey.pem" not in public_text +def test_domain_tls_inventory_candidates_support_shallow_runtime_paths() -> None: + candidates = asset_scanner_job._domain_tls_inventory_candidates( + cwd=Path("/runtime"), + source_path=Path("/app/src/jobs/asset_scanner_job.py"), + ) + + assert Path( + "/app/docs/security/domain-tls-certbot-inventory.snapshot.json" + ) in candidates + + def test_public_tls_expiry_classification_is_conservative() -> None: now_epoch = 1_700_000_000.0 From ec9d81ff6dc05d3cf1389fba9b57ab0dc71a1fa1 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 20:39:55 +0800 Subject: [PATCH 03/10] fix(wazuh): preserve degraded production readback --- apps/api/src/services/iwooos_wazuh_readonly_status.py | 8 ++++---- apps/api/tests/test_iwooos_wazuh_api.py | 4 ++-- apps/api/tests/test_iwooos_wazuh_prod_manifest.py | 10 +++++++--- apps/web/src/app/api/iwooos/wazuh/route.ts | 6 +++--- k8s/awoooi-prod/06-deployment-api.yaml | 9 +++++++++ 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/apps/api/src/services/iwooos_wazuh_readonly_status.py b/apps/api/src/services/iwooos_wazuh_readonly_status.py index c78a09d66..306199dfb 100644 --- a/apps/api/src/services/iwooos_wazuh_readonly_status.py +++ b/apps/api/src/services/iwooos_wazuh_readonly_status.py @@ -124,10 +124,10 @@ async def load_iwooos_wazuh_readonly_status() -> WazuhReadonlyStatus: base_url = https_url(env["base_url"]) if not base_url or not env["username"] or not env["password"]: - return boundary_status("misconfigured_missing_server_side_wazuh_env", 503) + return boundary_status("misconfigured_missing_server_side_wazuh_env") try: - auth_header = b64encode(f"{env['username']}:{env['password']}".encode("utf-8")).decode("ascii") + auth_header = b64encode(f"{env['username']}:{env['password']}".encode()).decode("ascii") async with httpx.AsyncClient(timeout=REQUEST_TIMEOUT_SECONDS) as client: auth = await fetch_json( client, @@ -136,7 +136,7 @@ async def load_iwooos_wazuh_readonly_status() -> WazuhReadonlyStatus: ) token = (auth.get("data") or {}).get("token") if not token: - return boundary_status("wazuh_auth_token_missing", 502) + return boundary_status("wazuh_auth_token_missing") bearer_headers = {"Authorization": f"Bearer {token}"} status_payload = await fetch_json( @@ -150,7 +150,7 @@ async def load_iwooos_wazuh_readonly_status() -> WazuhReadonlyStatus: bearer_headers, ) except (httpx.HTTPError, ValueError): - return boundary_status("wazuh_readonly_metadata_unavailable", 502) + return boundary_status("wazuh_readonly_metadata_unavailable") connection = ((status_payload.get("data") or {}).get("connection") or {}) affected_items = ((agents_payload.get("data") or {}).get("affected_items") or []) diff --git a/apps/api/tests/test_iwooos_wazuh_api.py b/apps/api/tests/test_iwooos_wazuh_api.py index 08a712183..a733d9b35 100644 --- a/apps/api/tests/test_iwooos_wazuh_api.py +++ b/apps/api/tests/test_iwooos_wazuh_api.py @@ -45,7 +45,7 @@ def test_iwooos_wazuh_v1_route_rejects_missing_server_side_env(monkeypatch: pyte response = _client().get("/api/v1/iwooos/wazuh") - assert response.status_code == 503 + assert response.status_code == 200 data = response.json() assert data["status"] == "misconfigured_missing_server_side_wazuh_env" assert data["configured"] is False @@ -61,7 +61,7 @@ def test_iwooos_wazuh_rejects_non_https_base_url(monkeypatch: pytest.MonkeyPatch response = _client().get("/api/iwooos/wazuh") - assert response.status_code == 503 + assert response.status_code == 200 data = response.json() assert data["status"] == "misconfigured_missing_server_side_wazuh_env" assert data["boundaries"]["secret_value_collection_allowed"] is False diff --git a/apps/api/tests/test_iwooos_wazuh_prod_manifest.py b/apps/api/tests/test_iwooos_wazuh_prod_manifest.py index 427447eb9..98bb31713 100644 --- a/apps/api/tests/test_iwooos_wazuh_prod_manifest.py +++ b/apps/api/tests/test_iwooos_wazuh_prod_manifest.py @@ -4,7 +4,6 @@ from pathlib import Path import yaml - REPO_ROOT = Path(__file__).resolve().parents[3] DEPLOYMENT_PATH = REPO_ROOT / "k8s" / "awoooi-prod" / "06-deployment-api.yaml" @@ -34,8 +33,13 @@ def test_iwooos_wazuh_readonly_live_metadata_enabled_by_gitops_flag_only() -> No assert env["IWOOOS_WAZUH_EXPECTED_MIN_AGENT_COUNT"]["value"] == "6" -def test_iwooos_wazuh_prod_manifest_does_not_inline_secret_values() -> None: +def test_iwooos_wazuh_prod_manifest_references_secret_values_without_inlining() -> None: env = _api_env() for name in ("WAZUH_API_BASE_URL", "WAZUH_API_USERNAME", "WAZUH_API_PASSWORD"): - assert name not in env + assert "value" not in env[name] + assert env[name]["valueFrom"]["secretKeyRef"] == { + "name": "awoooi-secrets", + "key": name, + "optional": True, + } diff --git a/apps/web/src/app/api/iwooos/wazuh/route.ts b/apps/web/src/app/api/iwooos/wazuh/route.ts index 243541b12..87b655b9d 100644 --- a/apps/web/src/app/api/iwooos/wazuh/route.ts +++ b/apps/web/src/app/api/iwooos/wazuh/route.ts @@ -128,7 +128,7 @@ export async function GET() { const baseUrl = requireHttpsBaseUrl(WAZUH_API_BASE_URL); if (!baseUrl || !WAZUH_API_USERNAME || !WAZUH_API_PASSWORD) { - return boundaryResponse('misconfigured_missing_server_side_wazuh_env', 503); + return boundaryResponse('misconfigured_missing_server_side_wazuh_env'); } try { @@ -143,7 +143,7 @@ export async function GET() { ); const token = auth.data?.token; if (!token) { - return boundaryResponse('wazuh_auth_token_missing', 502); + return boundaryResponse('wazuh_auth_token_missing'); } const [status, agents] = await Promise.all([ @@ -209,6 +209,6 @@ export async function GET() { }, }); } catch { - return boundaryResponse('wazuh_readonly_metadata_unavailable', 502); + return boundaryResponse('wazuh_readonly_metadata_unavailable'); } } diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 5eee89ec2..7a6a65311 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -192,6 +192,15 @@ spec: value: "true" - name: IWOOOS_WAZUH_EXPECTED_MIN_AGENT_COUNT value: "6" + - name: WAZUH_API_BASE_URL + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: WAZUH_API_BASE_URL, optional: true} + - name: WAZUH_API_USERNAME + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: WAZUH_API_USERNAME, optional: true} + - name: WAZUH_API_PASSWORD + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: WAZUH_API_PASSWORD, optional: true} - name: USE_AI_ROUTER value: "true" - name: ENABLE_NEMOTRON_COLLABORATION From fc19019ce4acfc252f2a2921b4cd572261b62a6c Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 19:10:56 +0800 Subject: [PATCH 04/10] fix(agent99): terminate remote stdin with sentinel --- .../deploy-agent99-via-windows99-ssh.sh | 29 +++++++++++++++++-- ...remote_atomic_deploy_transport_contract.py | 9 +++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index a8255ba77..03a40d40e 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -11,6 +11,7 @@ readonly GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git readonly RECEIVER_RELATIVE="scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" readonly PREFLIGHT_RELATIVE="scripts/reboot-recovery/agent99-live-preflight.ps1" readonly WRAPPER_RELATIVE="scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh" +readonly REMOTE_STDIN_SENTINEL="__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__" RUNTIME_FILES=( "agent99-bootstrap.ps1" @@ -303,7 +304,8 @@ PY python3 - \ "${BOOTSTRAP_PATH}" \ - "${BOOTSTRAP_PAYLOAD_PATH}" <<'PY' + "${BOOTSTRAP_PAYLOAD_PATH}" \ + "${REMOTE_STDIN_SENTINEL}" <<'PY' from __future__ import annotations import base64 @@ -312,13 +314,14 @@ import sys from pathlib import Path bootstrap_path, output_path = map(Path, sys.argv[1:3]) +sentinel = sys.argv[3] compressed = gzip.compress( bootstrap_path.read_bytes(), compresslevel=9, mtime=0, ) output_path.write_text( - base64.b64encode(compressed).decode("ascii"), + base64.b64encode(compressed).decode("ascii") + "\n" + sentinel + "\n", encoding="ascii", ) PY @@ -349,7 +352,27 @@ from __future__ import annotations import base64 -decoder = r'''$payload = [Console]::In.ReadToEnd() +decoder = r'''$sentinel = "__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__" +$lines = New-Object System.Collections.Generic.List[string] +$lineCount = 0 +$charCount = 0 +$terminated = $false +while (($line = [Console]::In.ReadLine()) -ne $null) { + if ($line -ceq $sentinel) { + $terminated = $true + break + } + $lineCount += 1 + $charCount += $line.Length + 2 + if ($lineCount -gt 8 -or $charCount -gt 1048576) { + throw "remote_bootstrap_input_limit_exceeded" + } + [void]$lines.Add($line) +} +if (-not $terminated) { + throw "remote_bootstrap_sentinel_missing" +} +$payload = [string]::Join("", $lines) $compressed = [Convert]::FromBase64String(($payload -replace '\s', '')) $inputStream = New-Object IO.MemoryStream(,$compressed) $gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress) diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index b1ceea467..d7908d8f3 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -148,7 +148,14 @@ def test_sender_builds_sha256_envelope_and_streams_it_without_remote_cli_data() assert "IO.Compression.GzipStream" in source assert 'decoder.encode("utf-16le")' in source assert "-EncodedCommand" in source - assert "[Console]::In.ReadToEnd()" in source + assert "[Console]::In.ReadToEnd()" not in source + assert "$reader.ReadToEnd()" in source + assert "__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__" in source + assert "[Console]::In.ReadLine()" in source + assert "remote_bootstrap_input_limit_exceeded" in source + assert "remote_bootstrap_sentinel_missing" in source + assert "$lineCount -gt 8" in source + assert "$charCount -gt 1048576" in source assert '<"${BOOTSTRAP_PAYLOAD_PATH}"' in source assert "rawEnvelope" not in source assert "sshpass" not in source From 66fb0c4f8d6c6e5737e6016f1cfcac774badafc2 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Tue, 14 Jul 2026 21:01:40 +0800 Subject: [PATCH 05/10] chore(cd): deploy fc19019 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 7a6a65311..9e47be90e 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" + value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" + value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index b38d02858..c8558c022 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" + value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index a5b542a98..0f672b92b 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "1c3c62c38db3c246f6248ee07b3f336abc119f7d" + value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 92a431fd0..3b70d8b41 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:0ea8faa81f4ba625c46f1fa0d6a69ca8e3718782e09c59dd0dbcdca4a5f7bbd0 +- digest: sha256:263313c98092def0e37e002b24add8619a88d24c5cbeadb357f6d119868967b9 name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:6dd391003d64930bf0badec26fbfa3f37b974af07acfd809352f6975d57685f9 +- digest: sha256:ff0b001767c7458d945302eb98177292d058046639333cf8e7557427e469edc8 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 4edd802497de6d00131fc74df79124f53c164221 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 21:07:29 +0800 Subject: [PATCH 06/10] fix(agent99): frame remote bootstrap input --- .../deploy-agent99-via-windows99-ssh.sh | 10 +++++++++- ..._agent99_remote_atomic_deploy_transport_contract.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index 03a40d40e..a00e59c9c 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -320,8 +320,16 @@ compressed = gzip.compress( compresslevel=9, mtime=0, ) +payload = base64.b64encode(compressed).decode("ascii") +line_width = 65536 +payload_lines = [ + payload[offset : offset + line_width] + for offset in range(0, len(payload), line_width) +] +if not payload_lines or len(payload_lines) > 8: + raise SystemExit("remote_bootstrap_payload_line_contract_failed") output_path.write_text( - base64.b64encode(compressed).decode("ascii") + "\n" + sentinel + "\n", + "\n".join(payload_lines) + "\n" + sentinel + "\n", encoding="ascii", ) PY diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index d7908d8f3..bb6975af5 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -154,6 +154,10 @@ def test_sender_builds_sha256_envelope_and_streams_it_without_remote_cli_data() assert "[Console]::In.ReadLine()" in source assert "remote_bootstrap_input_limit_exceeded" in source assert "remote_bootstrap_sentinel_missing" in source + assert "line_width = 65536" in source + assert "len(payload_lines) > 8" in source + assert '"\\n".join(payload_lines)' in source + assert "remote_bootstrap_payload_line_contract_failed" in source assert "$lineCount -gt 8" in source assert "$charCount -gt 1048576" in source assert '<"${BOOTSTRAP_PAYLOAD_PATH}"' in source From 3de0ba821e1414842fdfd3a3813c3591efc9684a Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 21:13:32 +0800 Subject: [PATCH 07/10] fix(agent99): tolerate fresh durable alerts during deploy --- .../agent99-live-preflight.ps1 | 8 ++++++- .../agent99-remote-atomic-deploy-receiver.ps1 | 3 ++- .../test_agent99_live_preflight_decision.py | 24 +++++++++++++++++++ ...remote_atomic_deploy_transport_contract.py | 5 ++-- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 20f8954e1..1860b99dd 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -21,6 +21,7 @@ function Get-AgentLivePreflightDecision { [int]$SelfHealthPerformanceIssueCount, [int]$StaleAlertExecutionCount, [int]$AlertIncomingCount, + [int]$StaleAlertIncomingCount, [int]$PendingCommandCount ) @@ -75,8 +76,10 @@ function Get-AgentLivePreflightDecision { } if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" } - if ($AlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) { + if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) { $blockingReasons += "pending_work_before_reboot" + } elseif ($AlertIncomingCount -gt 0) { + $warningReasons += "fresh_alert_pending_next_inbox_tick" } return [pscustomobject]@{ @@ -428,6 +431,7 @@ $decision = Get-AgentLivePreflightDecision ` -SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount ` -StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes ` -AlertIncomingCount $alertIncoming.count ` + -StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes ` -PendingCommandCount $commandQueue.count $blockingReasons = @($decision.blockingReasons) $warningReasons = @($decision.warningReasons) @@ -459,6 +463,7 @@ $result = [pscustomobject]@{ selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount alertIncomingCount = [int]$alertIncoming.count + alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes alertRunningCount = [int]$alertRunning.count pendingCommandCount = [int]$commandQueue.count relayListenerCount = $relayListeners.Count @@ -480,6 +485,7 @@ Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)" Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)" Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)" Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)" +Write-Output "ALERT_INCOMING_STALE_COUNT=$($result.summary.alertIncomingStaleCount)" Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)" Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)" Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')" diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index e5aff93fc..92583eedc 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -284,7 +284,7 @@ function Invoke-AgentLivePreflight { [int]$summary.requiredEvidenceParseFailureCount -eq 0 -and [int]$summary.requiredEvidenceContentFailureCount -eq 0 -and [int]$summary.relayListenerCount -gt 0 -and - [int]$summary.alertIncomingCount -eq 0 -and + [int]$summary.alertIncomingStaleCount -eq 0 -and [int]$summary.alertRunningCount -eq 0 -and [int]$summary.pendingCommandCount -eq 0 -and $runtimePlaneBlockingReasons.Count -eq 0 @@ -317,6 +317,7 @@ function Invoke-AgentLivePreflight { } | Where-Object { $_ } | Sort-Object -Unique) } else { @() } alertIncomingCount = if ($summary) { [int]$summary.alertIncomingCount } else { -1 } + alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 } alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 } pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 } preflightGreen = [bool]($summary -and $summary.preflightGreen) diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index cd0ead9d1..ef96b054e 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -41,6 +41,7 @@ def _base_case() -> dict[str, Any]: "selfHealthPerformanceIssueCount": 0, "staleAlertExecutionCount": 0, "alertIncomingCount": 0, + "staleAlertIncomingCount": 0, "pendingCommandCount": 0, } @@ -82,6 +83,7 @@ $parameters = @{{ SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount AlertIncomingCount = [int]$case.alertIncomingCount + StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount PendingCommandCount = [int]$case.pendingCommandCount }} Get-AgentLivePreflightDecision @parameters | @@ -106,6 +108,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '$warningReasons += "self_health_warning"' in decision assert '$warningReasons += "performance_warning"' in decision + assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision assert '"critical" { $blockingReasons += "self_health_critical" }' in decision assert 'default { $blockingReasons += "self_health_unclassified" }' in decision assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision @@ -172,6 +175,27 @@ def test_decision_behavior_when_powershell_is_available() -> None: ("sre_alert_relay_not_listening",), (), ), + ( + _with_overrides(alertIncomingCount=1), + True, + (), + ("fresh_alert_pending_next_inbox_tick",), + ), + ( + _with_overrides( + alertIncomingCount=1, + staleAlertIncomingCount=1, + ), + False, + ("pending_work_before_reboot",), + (), + ), + ( + _with_overrides(pendingCommandCount=1), + False, + ("pending_work_before_reboot",), + (), + ), ) for case, expected_eligible, expected_blockers, expected_warnings in cases: diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index bb6975af5..b008d6d8f 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -513,7 +513,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() - assert '$_ -ne "self_health_not_ok"' in preflight assert '[int]$summary.healthyTaskCount -eq [int]$summary.requiredTaskCount' in preflight assert '[int]$summary.requiredEvidenceFailureCount -eq 0' in preflight - assert '[int]$summary.alertIncomingCount -eq 0' in preflight + assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight + assert '[int]$summary.alertIncomingCount -eq 0' not in preflight assert '[int]$summary.alertRunningCount -eq 0' in preflight assert '[int]$summary.pendingCommandCount -eq 0' in preflight assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight @@ -651,4 +652,4 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport( assert "Administrator@192.168.0.99" in args assert str(identity) in args payload_bytes = int(bytes_path.read_text(encoding="utf-8")) - assert 50_000 < payload_bytes < 500_000 + assert 50_000 < payload_bytes < 1_048_576 From 604184bc97c60a1187e0f5219778717e8dcf4688 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Tue, 14 Jul 2026 21:25:27 +0800 Subject: [PATCH 08/10] chore(cd): deploy 3de0ba8 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 9e47be90e..24bac02c4 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" + value: "3de0ba821e1414842fdfd3a3813c3591efc9684a" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" + value: "3de0ba821e1414842fdfd3a3813c3591efc9684a" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index c8558c022..5f87e04a4 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" + value: "3de0ba821e1414842fdfd3a3813c3591efc9684a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 0f672b92b..5a256b340 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -173,7 +173,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "fc19019ce4acfc252f2a2921b4cd572261b62a6c" + value: "3de0ba821e1414842fdfd3a3813c3591efc9684a" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 3b70d8b41..26d44ee0c 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:263313c98092def0e37e002b24add8619a88d24c5cbeadb357f6d119868967b9 +- digest: sha256:8c45722e1162540ca88d705e25fe9da706c3feb6f89994f548cca47c04e460d1 name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:ff0b001767c7458d945302eb98177292d058046639333cf8e7557427e469edc8 +- digest: sha256:04a87a039743115f2db48951a78431bc446fcff1b7259789808cd8674af10c4e name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 5f4f1ab8651b68e2363a455529895adb21b2abc1 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 20:45:38 +0800 Subject: [PATCH 09/10] fix(agent): bound host110 monitoring repair --- .../services/awooop_ansible_audit_service.py | 50 +++++++-- .../awooop_ansible_check_mode_service.py | 12 ++- .../services/awooop_ansible_post_verifier.py | 25 +++++ .../test_awooop_ansible_post_verifier.py | 32 +++++- ...t_host110_node_exporter_ansible_catalog.py | 95 ++++++++++++++++ .../test_host_pressure_ansible_catalog.py | 18 +--- .../playbooks/110-node-exporter-repair.yml | 102 ++++++++++++++++++ scripts/backup/backup-awoooi.sh | 68 ++++++++++-- .../test_backup_awoooi_secret_contract.py | 25 +++++ 9 files changed, 387 insertions(+), 40 deletions(-) create mode 100644 apps/api/tests/test_host110_node_exporter_ansible_catalog.py create mode 100644 infra/ansible/playbooks/110-node-exporter-repair.yml create mode 100644 scripts/backup/tests/test_backup_awoooi_secret_contract.py diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 2a7de656a..ea0311778 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -426,14 +426,33 @@ _CATALOG: tuple[dict[str, Any], ...] = ( }, { "catalog_id": "ansible:110-devops", - "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml", + "check_mode_playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml", "inventory_hosts": ["host_110"], - "domains": ["swap", "harbor", "sentry", "gitea", "langfuse", "bitan", "runner", "keepalived", "nginx"], + "domains": ["monitoring", "prometheus", "node_exporter", "host_110"], "keywords": [ - "110", - "docker", - "container", - "dockercontainerunhealthy", + "node-exporter-110", + "node exporter 110", + "nodeexporterdown", + "node_exporter", + "node-exporter", + ], + "activation_keywords": [ + "node-exporter-110", + "node exporter 110", + ], + "supports_check_mode": True, + "auto_apply_enabled": True, + "approval_required": False, + "risk_level": "low", + "catalog_revision": "2026-07-14-node-exporter-bounded-v4", + }, + { + "catalog_id": "ansible:110-devops-full-convergence", + "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "check_mode_playbook_path": "infra/ansible/playbooks/110-devops.yml", + "inventory_hosts": ["host_110"], + "domains": [ "swap", "harbor", "sentry", @@ -441,14 +460,25 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "langfuse", "bitan", "runner", - "github-runner", "keepalived", + "nginx", + "backup", + ], + "keywords": [ + "110-devops-full-convergence", + "harbor", + "sentry", + "langfuse", + "bitan", + "keepalived", + "github-runner", ], "supports_check_mode": True, - "auto_apply_enabled": True, + "auto_apply_enabled": False, "approval_required": False, - "risk_level": "medium", - "catalog_revision": "2026-07-14-host110-harbor-route-v3", + "risk_level": "high", + "no_write_only": True, + "catalog_revision": "2026-07-14-full-convergence-quarantined-v1", }, { "catalog_id": "ansible:188-momo-backup-user", diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 5c27a9e0d..e6f9cdbef 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -1063,6 +1063,7 @@ def _claim_from_stale_check_mode_row( or "" ) catalog_id = str(input_payload.get("catalog_id") or "") + catalog_item = get_ansible_catalog_item(catalog_id) or {} inventory_hosts = input_payload.get("inventory_hosts") source_playbook_path = str( input_payload.get("source_candidate_playbook_path") @@ -1080,6 +1081,9 @@ def _claim_from_stale_check_mode_row( or not inventory_hosts ): return None + canonical_playbook_path = str(catalog_item.get("playbook_path") or "") + if not canonical_playbook_path: + return None try: claim_input = build_ansible_check_mode_claim_input( source_candidate_op_id=source_candidate_op_id, @@ -1089,9 +1093,13 @@ def _claim_from_stale_check_mode_row( "executor_candidates": [ { "catalog_id": catalog_id, - "playbook_path": source_playbook_path, + # A stale row is evidence, never execution authority. Path + # migrations must resolve through the current allowlisted + # catalog so a repaired bounded playbook can replace an + # unsafe or retired predecessor without trusting DB input. + "playbook_path": canonical_playbook_path, "inventory_hosts": inventory_hosts, - "risk_level": input_payload.get("risk_level"), + "risk_level": catalog_item.get("risk_level"), } ], }, diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index 6f273650c..be5806d5a 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -99,6 +99,31 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { ), ), "ansible:110-devops": ( + AssetPostcondition( + "host_110_node_exporter_container", + "service", + "host_110", + "test \"$(docker inspect --format '{{.State.Running}}' node-exporter 2>/dev/null)\" = true", + ), + AssetPostcondition( + "host_110_node_exporter_metrics", + "metric", + "host_110", + "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 " + "http://127.0.0.1:9100/metrics)\" = 200", + ), + AssetPostcondition( + "host_110_node_exporter_prometheus_scrape", + "metric", + "host_110", + "for attempt in 1 2 3 4 5 6; do " + "curl -fsS --max-time 5 " + "'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' " + "| grep -Eq '\"value\":\\[[^]]+,\"1\"\\]' && exit 0; " + "sleep 5; done; exit 1", + ), + ), + "ansible:110-devops-full-convergence": ( AssetPostcondition( "host_110_docker_runtime", "service", diff --git a/apps/api/tests/test_awooop_ansible_post_verifier.py b/apps/api/tests/test_awooop_ansible_post_verifier.py index 480eec23f..a85ad6310 100644 --- a/apps/api/tests/test_awooop_ansible_post_verifier.py +++ b/apps/api/tests/test_awooop_ansible_post_verifier.py @@ -52,15 +52,38 @@ def test_asset_postcondition_registry_covers_every_ansible_catalog() -> None: assert registered_catalog_ids() == catalog_ids -def test_host110_harbor_postcondition_matches_registry_backend() -> None: +def test_host110_node_exporter_postconditions_are_bounded() -> None: catalog = next( item for item in list_ansible_catalog() if item["catalog_id"] == "ansible:110-devops" ) + conditions = postconditions_for_catalog("ansible:110-devops") + + assert {item.condition_id for item in conditions} == { + "host_110_node_exporter_container", + "host_110_node_exporter_metrics", + "host_110_node_exporter_prometheus_scrape", + } + assert catalog["playbook_path"].endswith("110-node-exporter-repair.yml") + assert catalog["risk_level"] == "low" + assert catalog["auto_apply_enabled"] is True + assert catalog["catalog_revision"] == ( + "2026-07-14-node-exporter-bounded-v4" + ) + + +def test_host110_full_convergence_harbor_probe_is_quarantined() -> None: + catalog = next( + item + for item in list_ansible_catalog() + if item["catalog_id"] == "ansible:110-devops-full-convergence" + ) condition = next( item - for item in postconditions_for_catalog("ansible:110-devops") + for item in postconditions_for_catalog( + "ansible:110-devops-full-convergence" + ) if item.condition_id == "host_110_harbor_route" ) @@ -68,9 +91,8 @@ def test_host110_harbor_postcondition_matches_registry_backend() -> None: assert "200" in condition.probe assert "401" in condition.probe assert "sites-enabled/harbor.conf" not in condition.probe - assert catalog["catalog_revision"] == ( - "2026-07-14-host110-harbor-route-v3" - ) + assert catalog["auto_apply_enabled"] is False + assert catalog["no_write_only"] is True def test_read_only_probe_command_uses_structured_inventory(tmp_path: Path) -> None: diff --git a/apps/api/tests/test_host110_node_exporter_ansible_catalog.py b/apps/api/tests/test_host110_node_exporter_ansible_catalog.py new file mode 100644 index 000000000..b4be166a8 --- /dev/null +++ b/apps/api/tests/test_host110_node_exporter_ansible_catalog.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + +from src.services.awooop_ansible_audit_service import ( + _catalog_hints, + get_ansible_catalog_item, +) +from src.services.awooop_ansible_check_mode_service import ( + _claim_from_stale_check_mode_row, +) + +ROOT = Path(__file__).resolve().parents[3] +PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-node-exporter-repair.yml" + + +def test_node_exporter_alert_uses_bounded_controlled_apply_catalog() -> None: + hints = _catalog_hints( + { + "alertname": "NodeExporterDown", + "affected_services": ["node-exporter-110"], + "signals": [ + { + "alert_name": "NodeExporterDown", + "labels": {"job": "node-exporter-110", "host": "110"}, + } + ], + }, + None, + ) + + assert [row["catalog_id"] for row in hints["candidates"]] == [ + "ansible:110-devops" + ] + assert hints["candidates"][0]["auto_apply_enabled"] is True + assert hints["candidates"][0]["risk_level"] == "low" + + +def test_node_exporter_playbook_only_reconciles_one_container() -> None: + payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) + text = PLAYBOOK.read_text(encoding="utf-8") + catalog = get_ansible_catalog_item("ansible:110-devops") + + assert payload[0]["hosts"] == "host_110" + assert payload[0]["become"] is False + assert catalog is not None + assert catalog["playbook_path"].endswith("110-node-exporter-repair.yml") + assert catalog["catalog_revision"] == ( + "2026-07-14-node-exporter-bounded-v4" + ) + assert "docker\n - compose\n - up" in text + assert "node-exporter" in text + for forbidden in ( + "backup-awoooi.sh", + "swapon", + "keepalived", + "github-runner", + "docker system prune", + "reboot", + ): + assert forbidden not in text.lower() + + +def test_existing_110_devops_lineage_migrates_to_bounded_revision() -> None: + claim = _claim_from_stale_check_mode_row( + { + "op_id": "00000000-0000-0000-0000-000000000111", + "parent_op_id": "00000000-0000-0000-0000-000000000110", + "incident_id": "INC-NODE-EXPORTER-110", + "input": { + "source_candidate_op_id": ( + "00000000-0000-0000-0000-000000000110" + ), + "incident_id": "INC-NODE-EXPORTER-110", + "catalog_id": "ansible:110-devops", + "catalog_revision": "2026-07-14-host110-harbor-route-v3", + "catalog_playbook_path": ( + "infra/ansible/playbooks/110-devops.yml" + ), + "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "inventory_hosts": ["host_110"], + "risk_level": "medium", + }, + } + ) + + assert claim is not None + assert claim.playbook_path.endswith("110-node-exporter-repair.yml") + assert claim.apply_playbook_path.endswith("110-node-exporter-repair.yml") + assert claim.input_payload["catalog_revision"] == ( + "2026-07-14-node-exporter-bounded-v4" + ) + assert claim.input_payload["controlled_apply_allowed"] is True diff --git a/apps/api/tests/test_host_pressure_ansible_catalog.py b/apps/api/tests/test_host_pressure_ansible_catalog.py index 2f6c26f45..3405ce81b 100644 --- a/apps/api/tests/test_host_pressure_ansible_catalog.py +++ b/apps/api/tests/test_host_pressure_ansible_catalog.py @@ -37,13 +37,12 @@ def test_host_pressure_alert_prefers_dedicated_no_write_catalog() -> None: candidates = hints["candidates"] assert candidates[0]["catalog_id"] == "ansible:110-host-pressure-readonly" - generic = next(row for row in candidates if row["catalog_id"] == "ansible:110-devops") - assert candidates[0]["match_score"] > generic["match_score"] + assert all(row["catalog_id"] != "ansible:110-devops" for row in candidates) assert candidates[0]["auto_apply_enabled"] is False assert candidates[0]["no_write_only"] is True -def test_generic_110_fallback_remains_below_exact_host_pressure_route() -> None: +def test_generic_110_fallback_is_not_used_for_host_pressure() -> None: hints = _catalog_hints( { "alertname": "Host110SustainedModeratePressure", @@ -53,16 +52,9 @@ def test_generic_110_fallback_remains_below_exact_host_pressure_route() -> None: ) candidates = hints["candidates"] - dedicated = next( - row - for row in candidates - if row["catalog_id"] == "ansible:110-host-pressure-readonly" - ) - generic = next( - row for row in candidates if row["catalog_id"] == "ansible:110-devops" - ) - assert dedicated["match_score"] > generic["match_score"] - assert generic["matched_keywords"] == ["110"] + assert [row["catalog_id"] for row in candidates] == [ + "ansible:110-host-pressure-readonly" + ] def test_host_pressure_playbook_is_bounded_and_read_only() -> None: diff --git a/infra/ansible/playbooks/110-node-exporter-repair.yml b/infra/ansible/playbooks/110-node-exporter-repair.yml new file mode 100644 index 000000000..c4e607101 --- /dev/null +++ b/infra/ansible/playbooks/110-node-exporter-repair.yml @@ -0,0 +1,102 @@ +--- +- name: Bounded host 110 node-exporter repair + hosts: host_110 + gather_facts: false + become: false + vars: + node_exporter_container: node-exporter + node_exporter_compose_dir: /home/wooo/monitoring + + tasks: + - name: Inspect node-exporter container state + ansible.builtin.command: + argv: + - docker + - inspect + - --format + - "{{ '{{' }}.State.Running{{ '}}' }}" + - "{{ node_exporter_container }}" + register: node_exporter_container_probe + check_mode: false + changed_when: false + failed_when: false + + - name: Probe local node-exporter metrics + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - http://127.0.0.1:9100/metrics + register: node_exporter_metrics_probe + check_mode: false + changed_when: false + failed_when: false + + - name: Probe Prometheus scrape state + ansible.builtin.shell: + cmd: >- + curl -fsS --max-time 5 + 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' + | grep -Eq '"value":\[[^]]+,"1"\]' + executable: /bin/bash + register: node_exporter_prometheus_probe + check_mode: false + changed_when: false + failed_when: false + + - name: Decide whether bounded repair is required + ansible.builtin.set_fact: + node_exporter_repair_required: >- + {{ + node_exporter_container_probe.rc != 0 + or (node_exporter_container_probe.stdout | trim) != 'true' + or node_exporter_metrics_probe.rc != 0 + or node_exporter_prometheus_probe.rc != 0 + }} + + - name: Reconcile only the node-exporter service + ansible.builtin.command: + argv: + - docker + - compose + - up + - -d + - --force-recreate + - node-exporter + chdir: "{{ node_exporter_compose_dir }}" + when: node_exporter_repair_required | bool + changed_when: true + + - name: Verify local metrics after controlled apply + ansible.builtin.command: + argv: + - curl + - -fsS + - --retry + - "6" + - --retry-delay + - "5" + - --retry-all-errors + - --max-time + - "5" + - http://127.0.0.1:9100/metrics + check_mode: false + changed_when: false + when: not ansible_check_mode + + - name: Verify Prometheus sees node-exporter after controlled apply + ansible.builtin.shell: + cmd: >- + for attempt in 1 2 3 4 5 6; do + curl -fsS --max-time 5 + 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' + | grep -Eq '"value":\[[^]]+,"1"\]' && exit 0; + sleep 5; + done; + exit 1 + executable: /bin/bash + check_mode: false + changed_when: false + when: not ansible_check_mode diff --git a/scripts/backup/backup-awoooi.sh b/scripts/backup/backup-awoooi.sh index 8bb9d9486..fc1799714 100755 --- a/scripts/backup/backup-awoooi.sh +++ b/scripts/backup/backup-awoooi.sh @@ -15,13 +15,58 @@ source "$(dirname "$0")/common.sh" SERVICE="awoooi" AWOOOI_HOST="192.168.0.188" AWOOOI_DB_USER="awoooi" -AWOOOI_DB_PASS="awoooi_prod_2026" +AWOOOI_DB_PASS="${AWOOOI_DB_PASS:-}" AWOOOI_DB_HOST="localhost" AWOOOI_DB_PORT="5432" K3S_DB_USER="postgres" LOCAL_REPO="${BACKUP_BASE}/awoooi" DUMP_DIR="/tmp/awoooi-backup-$$" +quote_remote() { + printf "%q" "$1" +} + +pgpass_escape() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//:/\\:}" + printf '%s' "${value}" +} + +pgpass_line() { + local database="$1" + printf '%s:%s:%s:%s:%s\n' \ + "$(pgpass_escape "${AWOOOI_DB_HOST}")" \ + "$(pgpass_escape "${AWOOOI_DB_PORT}")" \ + "$(pgpass_escape "${database}")" \ + "$(pgpass_escape "${AWOOOI_DB_USER}")" \ + "$(pgpass_escape "${AWOOOI_DB_PASS}")" +} + +remote_pgpass_wrapper() { + local command="$1" + printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGPASSFILE="$pgpass" %s' "${command}" +} + +run_remote_pg_dump() { + local database="$1" + local output_file="$2" + local stderr_file="${output_file}.stderr" + local command + + command="pg_dump --no-password -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") $(quote_remote "${database}")" + if pgpass_line "${database}" \ + | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" \ + >"${output_file}" 2>"${stderr_file}"; then + rm -f "${stderr_file}" + return 0 + fi + + tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true + rm -f "${stderr_file}" + return 1 +} + # 保留策略覆寫(比其他服務更長) KEEP_DAILY=14 # 14 天每日 KEEP_WEEKLY=8 # 8 週每週 @@ -31,6 +76,12 @@ main() { local start_time=$(date +%s) local failed=0 + if [ -z "${AWOOOI_DB_PASS}" ]; then + log_error "AWOOOI_DB_PASS is unavailable; refusing an unauthenticated or hardcoded fallback" + notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable" + exit 1 + fi + log_info "========== 開始 AWOOOI 資料庫備份 ==========" mkdir -p "${DUMP_DIR}" @@ -38,9 +89,8 @@ main() { log_info "Dump awoooi_prod..." local timestamp=$(date "+%Y%m%d_%H%M%S") - if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \ - -U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \ - awoooi_prod" > "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" 2>&1; then + if run_remote_pg_dump \ + "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1) log_success "awoooi_prod dump 完成 (${size})" else @@ -50,9 +100,8 @@ main() { # Step 2: awoooi_dev dump log_info "Dump awoooi_dev..." - if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \ - -U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \ - awoooi_dev 2>/dev/null" > "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" 2>/dev/null; then + if run_remote_pg_dump \ + "awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1) log_success "awoooi_dev dump 完成 (${size})" else @@ -61,9 +110,8 @@ main() { # Step 3: k3s_datastore dump(Kine 後端) log_info "Dump k3s_datastore..." - if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \ - -U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \ - k3s_datastore 2>/dev/null" > "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" 2>/dev/null; then + if run_remote_pg_dump \ + "k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1) log_success "k3s_datastore dump 完成 (${size})" else diff --git a/scripts/backup/tests/test_backup_awoooi_secret_contract.py b/scripts/backup/tests/test_backup_awoooi_secret_contract.py new file mode 100644 index 000000000..e513cb955 --- /dev/null +++ b/scripts/backup/tests/test_backup_awoooi_secret_contract.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "backup" / "backup-awoooi.sh" + + +def test_daily_backup_has_no_hardcoded_database_password() -> None: + text = SCRIPT.read_text(encoding="utf-8") + assignment = re.search(r"^AWOOOI_DB_PASS=(.+)$", text, re.MULTILINE) + + assert assignment is not None + assert assignment.group(1) == '"${AWOOOI_DB_PASS:-}"' + assert "PGPASSWORD='${AWOOOI_DB_PASS}'" not in text + assert 'PGPASSFILE="$pgpass"' in text + assert "pg_dump --no-password" in text + + +def test_daily_backup_fails_closed_without_runtime_credential() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert 'if [ -z "${AWOOOI_DB_PASS}" ]; then' in text + assert "refusing an unauthenticated or hardcoded fallback" in text From df88473f8b5f947081e55203fc45037c246e686a Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 21:31:34 +0800 Subject: [PATCH 10/10] fix(agent99): use ephemeral apply transport --- .../agent99-remote-atomic-deploy-receiver.ps1 | 35 ++- .../deploy-agent99-via-windows99-ssh.sh | 211 +++++++++++++++++- ...remote_atomic_deploy_transport_contract.py | 113 ++++++++++ 3 files changed, 355 insertions(+), 4 deletions(-) diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index 92583eedc..9049046c3 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -1,7 +1,8 @@ [CmdletBinding()] param( [Parameter(Mandatory = $true)] - [string]$EnvelopeJson + [string]$EnvelopeJson, + [switch]$TransportPayloadDeletedBeforeReceiver ) $ErrorActionPreference = "Stop" @@ -48,6 +49,9 @@ $script:ReceiverRunId = "" $script:ReceiverWorkItemId = "" $script:ReceiverSourceRevision = "" $script:ReceiverManifestDigest = "" +$script:ReceiverTransportPayloadMode = "unknown" +$script:ReceiverTransientTransportPayloadStored = $false +$script:ReceiverTransportPayloadDeletedBeforeReceiver = $false function Get-AgentSha256Bytes { param([byte[]]$Bytes) @@ -593,6 +597,11 @@ function Write-AgentRemoteDeployReceipt { param([string]$Path, [object]$Receipt) if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" } + $Receipt | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force + $Receipt | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force + $Receipt | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force + $Receipt | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force + $Receipt | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force $temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))" try { $json = $Receipt | ConvertTo-Json -Depth 12 @@ -617,6 +626,11 @@ function Write-AgentRemoteDeployReceipt { function Write-AgentResultAndExit { param([object]$Result, [int]$ExitCode) + $Result | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force + $Result | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force + $Result | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force + $Result | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force + $Result | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force $Result | ConvertTo-Json -Compress -Depth 12 exit $ExitCode } @@ -629,6 +643,25 @@ try { if ([string]$envelope.transport -ne "110_to_99_ssh_publickey") { throw "invalid_transport" } $mode = [string]$envelope.mode if ($mode -notin @("check", "apply")) { throw "invalid_mode" } + $transportPayloadMode = [string]$envelope.transportPayloadMode + $transportPayloadContainsSecrets = [bool]$envelope.transportPayloadContainsSecrets + $expectedTransportPayloadMode = if ($mode -eq "apply") { + "ephemeral_file_delete_before_receiver" + } else { + "stdin_sentinel" + } + if ($transportPayloadMode -ne $expectedTransportPayloadMode) { throw "invalid_transport_payload_mode" } + if ($transportPayloadContainsSecrets) { throw "transport_payload_secret_contract_failed" } + $script:ReceiverTransportPayloadMode = $transportPayloadMode + $script:ReceiverTransientTransportPayloadStored = [bool]($mode -eq "apply") + if ($mode -eq "apply") { + if (-not $TransportPayloadDeletedBeforeReceiver) { + throw "transport_payload_delete_before_receiver_unverified" + } + $script:ReceiverTransportPayloadDeletedBeforeReceiver = $true + } elseif ($TransportPayloadDeletedBeforeReceiver) { + throw "unexpected_transport_payload_delete_receipt" + } $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant() if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" } if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" } diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index a00e59c9c..f72fbb61c 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -147,7 +147,7 @@ if [[ "${MODE}" == "apply" ]]; then [[ "${WORK_ITEM_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_work_item_id_missing_or_invalid" fi -for dependency in git python3 ssh mktemp hostname timeout; do +for dependency in git python3 ssh scp mktemp hostname timeout; do command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}" done @@ -260,6 +260,12 @@ envelope = { "sourceHostAlias": "110", "targetHostAlias": "99", "transport": "110_to_99_ssh_publickey", + "transportPayloadMode": ( + "ephemeral_file_delete_before_receiver" + if mode == "apply" + else "stdin_sentinel" + ), + "transportPayloadContainsSecrets": False, "mode": mode, "traceId": trace_id, "runId": run_id, @@ -287,17 +293,25 @@ python3 - \ from __future__ import annotations import base64 +import json import sys from pathlib import Path receiver_path, envelope_path, output_path = map(Path, sys.argv[1:4]) receiver_base64 = base64.b64encode(receiver_path.read_bytes()).decode("ascii") envelope_base64 = base64.b64encode(envelope_path.read_bytes()).decode("ascii") +envelope_mode = json.loads(envelope_path.read_text(encoding="utf-8"))["mode"] +transport_delete_switch = ( + " -TransportPayloadDeletedBeforeReceiver" + if envelope_mode == "apply" + else "" +) bootstrap = ( "$ErrorActionPreference = 'Stop'\r\n" f"$receiverText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{receiver_base64}'))\r\n" f"$envelopeText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{envelope_base64}'))\r\n" - "& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText\r\n" + "& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText" + f"{transport_delete_switch}\r\n" ) output_path.write_text(bootstrap, encoding="utf-8") PY @@ -334,6 +348,21 @@ output_path.write_text( ) PY +TRANSPORT_TOKEN="$(python3 - "${BOOTSTRAP_PAYLOAD_PATH}" <<'PY' +from __future__ import annotations + +import hashlib +import sys +from pathlib import Path + +print(hashlib.sha256(Path(sys.argv[1]).read_bytes()).hexdigest()[:20]) +PY +)" +readonly TRANSPORT_TOKEN +readonly REMOTE_TRANSPORT_DIR_WINDOWS='C:\Wooo\Agent99\deploy\transport-inbox' +readonly REMOTE_TRANSPORT_PATH_WINDOWS="${REMOTE_TRANSPORT_DIR_WINDOWS}\\agent99-bootstrap-${TRANSPORT_TOKEN}.b64" +readonly REMOTE_TRANSPORT_PATH_SCP="C:/Wooo/Agent99/deploy/transport-inbox/agent99-bootstrap-${TRANSPORT_TOKEN}.b64" + SSH_OPTIONS=( -T -o BatchMode=yes @@ -355,6 +384,27 @@ if [[ -n "${IDENTITY_FILE}" ]]; then SSH_OPTIONS+=( -i "${IDENTITY_FILE}" ) fi +SCP_OPTIONS=( + -q + -o BatchMode=yes + -o PreferredAuthentications=publickey + -o PubkeyAuthentication=yes + -o PasswordAuthentication=no + -o KbdInteractiveAuthentication=no + -o NumberOfPasswordPrompts=0 + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}" + -o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}" + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=2 + -o LogLevel=ERROR + -o IdentitiesOnly=yes +) +if [[ -n "${IDENTITY_FILE}" ]]; then + SCP_OPTIONS+=( -i "${IDENTITY_FILE}" ) +fi + readonly REMOTE_COMMAND="$(python3 - <<'PY' from __future__ import annotations @@ -400,9 +450,164 @@ print( ) PY )" + +readonly REMOTE_PREPARE_COMMAND="$(python3 - \ + "${REMOTE_TRANSPORT_DIR_WINDOWS}" \ + "${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY' +from __future__ import annotations + +import base64 +import sys + +directory = sys.argv[1].replace("'", "''") +path = sys.argv[2].replace("'", "''") +script = rf'''$ErrorActionPreference = "Stop" +$directory = '{directory}' +$payloadPath = '{path}' +New-Item -ItemType Directory -Force -Path $directory | Out-Null +if (Test-Path -LiteralPath $payloadPath) {{ + Remove-Item -LiteralPath $payloadPath -Force +}} +if (Test-Path -LiteralPath $payloadPath) {{ + throw "remote_transport_stale_payload_cleanup_failed" +}}''' +encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii") +print( + "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass " + f"-EncodedCommand {encoded}" +) +PY +)" + +readonly REMOTE_FILE_COMMAND="$(python3 - \ + "${REMOTE_TRANSPORT_PATH_WINDOWS}" \ + "${REMOTE_STDIN_SENTINEL}" <<'PY' +from __future__ import annotations + +import base64 +import sys + +path = sys.argv[1].replace("'", "''") +sentinel = sys.argv[2].replace("'", "''") +decoder = rf'''$ErrorActionPreference = "Stop" +$payloadPath = '{path}' +$sentinel = '{sentinel}' +$allLines = @() +try {{ + $payloadFile = Get-Item -LiteralPath $payloadPath -ErrorAction Stop + if ($payloadFile.Length -le 0 -or $payloadFile.Length -gt 1048576) {{ + throw "remote_bootstrap_file_size_invalid" + }} + $allLines = @([IO.File]::ReadAllLines($payloadPath, [Text.Encoding]::ASCII)) +}} finally {{ + if (Test-Path -LiteralPath $payloadPath) {{ + Remove-Item -LiteralPath $payloadPath -Force -ErrorAction SilentlyContinue + }} +}} +if (Test-Path -LiteralPath $payloadPath) {{ + throw "remote_bootstrap_file_delete_failed" +}} +if ($allLines.Count -lt 2 -or $allLines[-1] -cne $sentinel) {{ + throw "remote_bootstrap_sentinel_missing" +}} +$payloadLines = @($allLines[0..($allLines.Count - 2)]) +$charCount = [int](($payloadLines | Measure-Object -Property Length -Sum).Sum) + (2 * $payloadLines.Count) +if ($payloadLines.Count -gt 8 -or $charCount -gt 1048576) {{ + throw "remote_bootstrap_input_limit_exceeded" +}} +$payload = [string]::Join("", $payloadLines) +$compressed = [Convert]::FromBase64String(($payload -replace '\s', '')) +$inputStream = New-Object IO.MemoryStream(,$compressed) +$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress) +$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8) +try {{ + $inputText = $reader.ReadToEnd() +}} finally {{ + $reader.Dispose() + $gzip.Dispose() + $inputStream.Dispose() +}} +& ([ScriptBlock]::Create($inputText))''' +encoded = base64.b64encode(decoder.encode("utf-16le")).decode("ascii") +print( + "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass " + f"-EncodedCommand {encoded}" +) +PY +)" + +readonly REMOTE_CLEANUP_COMMAND="$(python3 - \ + "${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY' +from __future__ import annotations + +import base64 +import sys + +path = sys.argv[1].replace("'", "''") +script = rf'''$ErrorActionPreference = "Stop" +$payloadPath = '{path}' +if (Test-Path -LiteralPath $payloadPath) {{ + Remove-Item -LiteralPath $payloadPath -Force +}} +if (Test-Path -LiteralPath $payloadPath) {{ + throw "remote_transport_payload_cleanup_failed" +}}''' +encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii") +print( + "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass " + f"-EncodedCommand {encoded}" +) +PY +)" + +cleanup_remote_payload() { + timeout --signal=TERM --kill-after=5 30s \ + ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_CLEANUP_COMMAND}" /dev/null 2>&1 +} + +if [[ "${MODE}" == "check" ]]; then + set +e + timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \ + ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}" + status=$? + set -e + exit "${status}" +fi + +set +e +timeout --signal=TERM --kill-after=5 30s \ + ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_PREPARE_COMMAND}" &2 + exit "${prepare_status}" +fi + +set +e +timeout --signal=TERM --kill-after=10 120s \ + scp "${SCP_OPTIONS[@]}" "${BOOTSTRAP_PAYLOAD_PATH}" \ + "${TARGET}:${REMOTE_TRANSPORT_PATH_SCP}" +copy_status=$? +set -e +if [[ "${copy_status}" -ne 0 ]]; then + cleanup_remote_payload || true + printf 'agent99_transport_error=ephemeral_payload_copy_failed\n' >&2 + exit "${copy_status}" +fi + set +e timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \ - ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}" + ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_FILE_COMMAND}" &2 + if [[ "${status}" -eq 0 ]]; then + exit 74 + fi +fi exit "${status}" diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index b008d6d8f..ca5d896cf 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -89,6 +89,10 @@ def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None: assert "source_host_is_not_110" in source assert '"sourceHostAlias": "110"' in source assert '"transport": "110_to_99_ssh_publickey"' in source + assert '"ephemeral_file_delete_before_receiver"' in source + assert '"transportPayloadContainsSecrets": False' in source + assert 'scp "${SCP_OPTIONS[@]}"' in source + assert "REMOTE_CLEANUP_COMMAND" in source assert "-o BatchMode=yes" in source assert "-o PreferredAuthentications=publickey" in source assert "-o PubkeyAuthentication=yes" in source @@ -264,6 +268,11 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() assert "tokenValueRead = $false" in source assert "environmentSecretRead = $false" in source assert "rawEnvelopeStored = $false" in source + assert "transportPayloadMode" in source + assert "transientTransportPayloadStored" in source + assert "transientTransportPayloadDeletedBeforeReceiver" in source + assert "transportPayloadContainsSecrets" in source + assert "TransportPayloadDeletedBeforeReceiver" in source assert "$env:" not in source assert "GetEnvironmentVariable" not in source assert ( @@ -653,3 +662,107 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport( assert str(identity) in args payload_bytes = int(bytes_path.read_text(encoding="utf-8")) assert 50_000 < payload_bytes < 1_048_576 + + +def test_apply_uses_ephemeral_file_transport_and_cleans_up_without_ssh_stdin( + tmp_path: Path, +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + ssh_count_path = tmp_path / "ssh-count.txt" + ssh_stdin_path = tmp_path / "ssh-stdin-bytes.txt" + scp_args_path = tmp_path / "scp-args.txt" + known_hosts = tmp_path / "known_hosts" + identity = tmp_path / "existing_identity" + known_hosts.write_text("fixed-host-key-placeholder\n", encoding="utf-8") + identity.write_text("identity-content-is-not-read-by-wrapper\n", encoding="utf-8") + + fake_hostname = fake_bin / "hostname" + fake_hostname.write_text( + "#!/usr/bin/env bash\n" + "[[ \"${1:-}\" == '-I' ]] || exit 65\n" + "printf '%s\\n' '192.168.0.110 127.0.0.1'\n", + encoding="utf-8", + ) + fake_ssh = fake_bin / "ssh" + fake_ssh.write_text( + "#!/usr/bin/env bash\n" + f"count=$(cat {shlex.quote(str(ssh_count_path))} 2>/dev/null || printf '0')\n" + "count=$((count + 1))\n" + f"printf '%s' \"$count\" >{shlex.quote(str(ssh_count_path))}\n" + f"wc -c | tr -d ' ' >>{shlex.quote(str(ssh_stdin_path))}\n" + "if [[ \"$count\" -eq 2 ]]; then\n" + " printf '%s\\n' '{\"status\":\"deployed_verified\"}'\n" + "fi\n", + encoding="utf-8", + ) + fake_scp = fake_bin / "scp" + fake_scp.write_text( + "#!/usr/bin/env bash\n" + f"printf '%s\\n' \"$@\" >{shlex.quote(str(scp_args_path))}\n", + encoding="utf-8", + ) + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + "shift 3\n" + "exec \"$@\"\n", + encoding="utf-8", + ) + real_git = shutil.which("git") + assert real_git is not None + fake_git = fake_bin / "git" + fake_git.write_text( + "#!/usr/bin/env bash\n" + "case \" $* \" in\n" + " *\" remote get-url origin \"*) " + "printf '%s\\n' 'https://gitea.wooo.work/wooo/awoooi.git'; exit 0 ;;\n" + " *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n" + " *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) " + "printf '%040d\\n' 0; exit 0 ;;\n" + " *\" cat-file -e \"*) exit 0 ;;\n" + " *\" diff --quiet \"*) exit 0 ;;\n" + "esac\n" + f"exec {shlex.quote(real_git)} \"$@\"\n", + encoding="utf-8", + ) + for executable in ( + fake_hostname, + fake_ssh, + fake_scp, + fake_timeout, + fake_git, + ): + executable.chmod(0o700) + + environment = os.environ.copy() + environment["PATH"] = f"{fake_bin}:{environment['PATH']}" + result = subprocess.run( + [ + "bash", + str(SENDER), + "--apply", + "--trace-id", + "trace-fixture", + "--run-id", + "run-fixture", + "--work-item-id", + "AIA-P0-006", + "--known-hosts-file", + str(known_hosts), + "--identity-file", + str(identity), + ], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + assert result.returncode == 0, result.stderr + assert '"status":"deployed_verified"' in result.stdout + assert ssh_count_path.read_text(encoding="utf-8") == "3" + assert ssh_stdin_path.read_text(encoding="utf-8").splitlines() == ["0", "0", "0"] + scp_args = scp_args_path.read_text(encoding="utf-8") + assert "Administrator@192.168.0.99:C:/Wooo/Agent99/deploy/transport-inbox/" in scp_args + assert "agent99-bootstrap-" in scp_args