From 7cf98f642cb964e7eac4c8f9d576fb5ddcc136ce Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 19:22:07 +0800 Subject: [PATCH] fix(recovery): persist Host111 broker transport --- .gitea/workflows/cd.yaml | 14 ++--- .../services/awooop_ansible_post_verifier.py | 48 ++++++++++++-- .../test_executor_network_policy_boundary.py | 11 +++- .../test_host111_ollama_ansible_catalog.py | 29 +++++++-- infra/ansible/inventory/hosts.yml | 12 +++- .../ansible/playbooks/111-ollama-fallback.yml | 62 +++++++++++++++++-- 6 files changed, 150 insertions(+), 26 deletions(-) diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index ac6b210ae..ce7a1c805 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3013,14 +3013,14 @@ jobs: # 不能覆蓋現有 secret，防止 production SSH 自動修復路徑癱瘓。 : > /tmp/known_hosts_repair : > /tmp/known_hosts_scan_err - EXPECTED_HOSTS=5 + EXPECTED_HOSTS=6 PRESENT=0 for scan_attempt in 1 2 3; do - ssh-keyscan -T 5 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \ + ssh-keyscan -T 5 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \ >> /tmp/known_hosts_repair 2>>/tmp/known_hosts_scan_err || true sort -u /tmp/known_hosts_repair -o /tmp/known_hosts_repair PRESENT=0 - for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do + for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then PRESENT=\$((PRESENT + 1)) fi @@ -3030,7 +3030,7 @@ jobs: fi sleep "\$scan_attempt" done - for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do + for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then : else @@ -3047,7 +3047,7 @@ jobs: KNOWN_HOSTS_B64=\$(base64 -w 0 /tmp/known_hosts_repair) \$KUBECTL patch secret ssh-mcp-key -n awoooi-prod --type=merge \ -p="{\"data\":{\"known_hosts\":\"\${KNOWN_HOSTS_B64}\"}}" \ - && echo "✅ ssh-mcp-key known_hosts 已更新（5 台主機完整）" \ + && echo "✅ ssh-mcp-key known_hosts 已更新（6 台主機完整）" \ || { echo "❌ ssh-mcp-key known_hosts 更新失敗，停止部署"; exit 1; } rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err else @@ -3059,7 +3059,7 @@ jobs: EXISTING_PRESENT=0 if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \ -o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then - for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do + for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then EXISTING_PRESENT=\$((EXISTING_PRESENT + 1)) fi @@ -3067,7 +3067,7 @@ jobs: fi rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err if [ "\$EXISTING_PRESENT" -eq "\$EXPECTED_HOSTS" ]; then - echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 5/5 and preserved" + echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 6/6 and preserved" else echo "❌ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key coverage \${EXISTING_PRESENT}/\${EXPECTED_HOSTS}" exit 1 diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index 6b3d15202..f1c0b85ad 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -6,6 +6,7 @@ import asyncio import hashlib import os import re +import shlex import time from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass @@ -24,9 +25,11 @@ VERIFIER_ID = "asset_specific_read_only_host_postconditions" INDEPENDENT_SOURCE = "broker_ssh_host_runtime_readback" _SAFE_HOST_ALIAS_RE = re.compile(r"^[A-Za-z0-9_.-]+$") _SAFE_REMOTE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") -_FIXED_PROXY_JUMP_BY_HOST = { +_FIXED_PROXY_TARGET_BY_HOST = { "host_111": "wooo@192.168.0.110", } +_INVENTORY_PROXY_KEY_PATH = Path("/run/secrets/ssh_mcp_key") +_INVENTORY_PROXY_KNOWN_HOSTS_PATH = Path("/etc/ssh-mcp/known_hosts") _RUNTIME_CLOSURE_RECEIPT_CATALOGS = frozenset( {"ansible:188-openclaw-callback-forwarder"} ) @@ -699,6 +702,23 @@ def _load_inventory_hosts(inventory_path: Path) -> dict[str, dict[str, str]]: return hosts +def _pinned_proxy_command( + *, + proxy_target: str, + ssh_key_path: Path, + known_hosts_path: Path, +) -> str: + if not re.fullmatch(r"[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+", proxy_target): + raise ValueError("postcondition_proxy_target_invalid") + return ( + f"ssh -i {shlex.quote(str(ssh_key_path))} " + f"-o UserKnownHostsFile={shlex.quote(str(known_hosts_path))} " + "-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes " + "-o BatchMode=yes -o ConnectTimeout=10 " + f"-W %h:%p {proxy_target}" + ) + + def build_read_only_probe_command( condition: AssetPostcondition, *, @@ -729,14 +749,30 @@ def build_read_only_probe_command( "-o", "ConnectTimeout=10", ] - proxy_jump = _FIXED_PROXY_JUMP_BY_HOST.get(condition.inventory_host) - if proxy_jump is not None: + proxy_target = _FIXED_PROXY_TARGET_BY_HOST.get(condition.inventory_host) + if proxy_target is not None: + inventory_proxy_command = _pinned_proxy_command( + proxy_target=proxy_target, + ssh_key_path=_INVENTORY_PROXY_KEY_PATH, + known_hosts_path=_INVENTORY_PROXY_KNOWN_HOSTS_PATH, + ) expected_common_args = ( - f"-o ProxyJump={proxy_jump} -o StrictHostKeyChecking=yes" + f'-o ProxyCommand="{inventory_proxy_command}" ' + "-o StrictHostKeyChecking=yes" ) if target.get("ansible_ssh_common_args") != expected_common_args: - raise ValueError("postcondition_proxy_jump_policy_mismatch") - command.extend(["-o", f"ProxyJump={proxy_jump}"]) + raise ValueError("postcondition_proxy_command_policy_mismatch") + command.extend( + [ + "-o", + "ProxyCommand=" + + _pinned_proxy_command( + proxy_target=proxy_target, + ssh_key_path=ssh_key_path, + known_hosts_path=known_hosts_path, + ), + ] + ) command.extend([f"{user}@{host}", condition.probe]) return command diff --git a/apps/api/tests/test_executor_network_policy_boundary.py b/apps/api/tests/test_executor_network_policy_boundary.py index f7f35714e..dc30b3b13 100644 --- a/apps/api/tests/test_executor_network_policy_boundary.py +++ b/apps/api/tests/test_executor_network_policy_boundary.py @@ -187,9 +187,14 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None: "192.168.0.110 192.168.0.112 192.168.0.120 " "192.168.0.121 192.168.0.188" ) - assert f"ssh-keyscan -T 5 {expected_hosts}" in workflow - assert "EXPECTED_HOSTS=5" in workflow - assert workflow.count(expected_hosts) >= 4 + expected_known_hosts = ( + "192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 " + "192.168.0.121 192.168.0.188" + ) + assert f"ssh-keyscan -T 5 {expected_known_hosts}" in workflow + assert "EXPECTED_HOSTS=6" in workflow + assert workflow.count(expected_known_hosts) >= 4 + assert workflow.count(expected_hosts) >= 2 assert "except ConnectionRefusedError:" in workflow assert "broker_ssh_refused_but_egress_permitted=" in workflow assert "broker_ssh_allowlisted_endpoint_unavailable=" in workflow diff --git a/apps/api/tests/test_host111_ollama_ansible_catalog.py b/apps/api/tests/test_host111_ollama_ansible_catalog.py index d9afc3ae1..c2f239200 100644 --- a/apps/api/tests/test_host111_ollama_ansible_catalog.py +++ b/apps/api/tests/test_host111_ollama_ansible_catalog.py @@ -30,6 +30,13 @@ EXPECTED_ALLOWLIST = ( "192.168.0.120/32,192.168.0.121/32" ) EXPECTED_PROXY_JUMP = "wooo@192.168.0.110" +EXPECTED_PROXY_COMMAND = ( + "ssh -i /run/secrets/ssh_mcp_key " + "-o UserKnownHostsFile=/etc/ssh-mcp/known_hosts " + "-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes " + "-o BatchMode=yes -o ConnectTimeout=10 " + f"-W %h:%p {EXPECTED_PROXY_JUMP}" +) def _flatten_tasks(tasks: list[dict]) -> list[dict]: @@ -150,7 +157,12 @@ def test_host111_playbook_is_launchagent_bounded_and_check_mode_safe() -> None: assert play["vars"]["proxy_label"] == "com.momo.ollama111-allow-proxy" assert play["vars"]["allowed_cidrs"] == EXPECTED_ALLOWLIST assert "192.168.0.110/32" not in play["vars"]["allowed_cidrs"].split(",") - assert "launchctl bootstrap" in source + assert source.count("launchctl bootout") == 2 + assert source.count("launchctl bootstrap") == 2 + assert source.count("launchctl kickstart -k") == 2 + assert source.count('test "$stopped" -eq 1') == 2 + assert source.count('test "$loaded" -eq 1') == 2 + assert source.count('test "$running" -eq 1') == 2 assert "systemctl" not in source assert "docker restart" not in source @@ -203,9 +215,10 @@ def test_host111_broker_transport_preserves_fixed_proxyjump_and_strict_safety( host111 = inventory["all"]["children"]["ollama_fallback"]["hosts"][ "host_111" ] - assert host111["ansible_ssh_common_args"] == ( - f"-o ProxyJump={EXPECTED_PROXY_JUMP} -o StrictHostKeyChecking=yes" + assert f'-o ProxyCommand="{EXPECTED_PROXY_COMMAND}"' in ( + host111["ansible_ssh_common_args"] ) + assert "ProxyJump" not in host111["ansible_ssh_common_args"] assert "accept-new" not in host111["ansible_ssh_common_args"] key = tmp_path / "ssh_mcp_key" @@ -227,7 +240,7 @@ def test_host111_broker_transport_preserves_fixed_proxyjump_and_strict_safety( assert "ProxyJump" not in spec.env["ANSIBLE_SSH_ARGS"] -def test_host111_independent_verifier_uses_same_fixed_proxyjump( +def test_host111_independent_verifier_uses_same_pinned_proxy_command( tmp_path: Path, ) -> None: condition = next( @@ -245,7 +258,13 @@ def test_host111_independent_verifier_uses_same_fixed_proxyjump( known_hosts_path=known_hosts, ) - assert f"ProxyJump={EXPECTED_PROXY_JUMP}" in command + expected_verifier_proxy_command = ( + f"ssh -i {key} -o UserKnownHostsFile={known_hosts} " + "-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes " + "-o BatchMode=yes -o ConnectTimeout=10 " + f"-W %h:%p {EXPECTED_PROXY_JUMP}" + ) + assert f"ProxyCommand={expected_verifier_proxy_command}" in command assert "StrictHostKeyChecking=yes" in command assert "BatchMode=yes" in command assert "ooo@192.168.0.111" in command diff --git a/infra/ansible/inventory/hosts.yml b/infra/ansible/inventory/hosts.yml index e52a3c952..5477f4f18 100644 --- a/infra/ansible/inventory/hosts.yml +++ b/infra/ansible/inventory/hosts.yml @@ -26,7 +26,17 @@ all: ansible_host: 192.168.0.111 ansible_user: ooo ansible_ssh_private_key_file: "~/.ssh/id_rsa" - ansible_ssh_common_args: "-o ProxyJump=wooo@192.168.0.110 -o StrictHostKeyChecking=yes" + # ProxyJump starts a second ssh process and does not inherit the + # executor's pinned key/known_hosts options. Keep both hops pinned. + ansible_ssh_common_args: >- + -o ProxyCommand="ssh -i /run/secrets/ssh_mcp_key + -o UserKnownHostsFile=/etc/ssh-mcp/known_hosts + -o StrictHostKeyChecking=yes + -o IdentitiesOnly=yes + -o BatchMode=yes + -o ConnectTimeout=10 + -W %h:%p wooo@192.168.0.110" + -o StrictHostKeyChecking=yes k3s_masters: hosts: diff --git a/infra/ansible/playbooks/111-ollama-fallback.yml b/infra/ansible/playbooks/111-ollama-fallback.yml index 618d2d5af..8e89510d1 100644 --- a/infra/ansible/playbooks/111-ollama-fallback.yml +++ b/infra/ansible/playbooks/111-ollama-fallback.yml @@ -145,10 +145,37 @@ - name: "Ollama111 | rollback 重新啟動 allow proxy" ansible.builtin.shell: cmd: | - set -e + set -eu launchctl enable gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true launchctl bootout gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true - launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }} + stopped=0 + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if ! launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} >/dev/null 2>&1; then + stopped=1 + break + fi + sleep 1 + done + test "$stopped" -eq 1 + loaded=0 + for attempt in 1 2 3 4 5; do + if launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }} >/dev/null 2>&1; then + loaded=1 + break + fi + sleep 1 + done + test "$loaded" -eq 1 + launchctl kickstart -k gui/{{ proxy_user_uid }}/{{ proxy_label }} + running=0 + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null | grep -q 'state = running'; then + running=1 + break + fi + sleep 1 + done + test "$running" -eq 1 when: not ansible_check_mode changed_when: true tags: ["111", "ollama-fallback", "rollback"] @@ -195,8 +222,35 @@ - name: Restart ollama111 allow proxy ansible.builtin.shell: cmd: | - set -e + set -eu launchctl enable gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true launchctl bootout gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true - launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }} + stopped=0 + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if ! launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} >/dev/null 2>&1; then + stopped=1 + break + fi + sleep 1 + done + test "$stopped" -eq 1 + loaded=0 + for attempt in 1 2 3 4 5; do + if launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }} >/dev/null 2>&1; then + loaded=1 + break + fi + sleep 1 + done + test "$loaded" -eq 1 + launchctl kickstart -k gui/{{ proxy_user_uid }}/{{ proxy_label }} + running=0 + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null | grep -q 'state = running'; then + running=1 + break + fi + sleep 1 + done + test "$running" -eq 1 changed_when: true