fix(recovery): persist Host111 broker transport

.gitea/workflows/cd.yaml

@@ -3013,14 +3013,14 @@ jobs:
           #       不能覆蓋現有 secret，防止 production SSH 自動修復路徑癱瘓。
           : > /tmp/known_hosts_repair
           : > /tmp/known_hosts_scan_err
-          EXPECTED_HOSTS=5
+          EXPECTED_HOSTS=6
           PRESENT=0
           for scan_attempt in 1 2 3; do
-            ssh-keyscan -T 5 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \
+            ssh-keyscan -T 5 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \
               >> /tmp/known_hosts_repair 2>>/tmp/known_hosts_scan_err || true
             sort -u /tmp/known_hosts_repair -o /tmp/known_hosts_repair
             PRESENT=0
-            for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
+            for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
               if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
                 PRESENT=\$((PRESENT + 1))
               fi
@@ -3030,7 +3030,7 @@ jobs:
             fi
             sleep "\$scan_attempt"
           done
-          for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
+          for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
             if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
               :
             else
@@ -3047,7 +3047,7 @@ jobs:
             KNOWN_HOSTS_B64=\$(base64 -w 0 /tmp/known_hosts_repair)
             \$KUBECTL patch secret ssh-mcp-key -n awoooi-prod --type=merge \
               -p="{\"data\":{\"known_hosts\":\"\${KNOWN_HOSTS_B64}\"}}" \
-              && echo "✅ ssh-mcp-key known_hosts 已更新（5 台主機完整）" \
+              && echo "✅ ssh-mcp-key known_hosts 已更新（6 台主機完整）" \
               || { echo "❌ ssh-mcp-key known_hosts 更新失敗，停止部署"; exit 1; }
             rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err
           else
@@ -3059,7 +3059,7 @@ jobs:
             EXISTING_PRESENT=0
             if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \
                 -o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then
-              for ip in 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
+              for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
                 if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
                   EXISTING_PRESENT=\$((EXISTING_PRESENT + 1))
                 fi
@@ -3067,7 +3067,7 @@ jobs:
             fi
             rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
             if [ "\$EXISTING_PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
-              echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 5/5 and preserved"
+              echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 6/6 and preserved"
             else
               echo "❌ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key coverage \${EXISTING_PRESENT}/\${EXPECTED_HOSTS}"
               exit 1

apps/api/src/services/awooop_ansible_post_verifier.py

@@ -6,6 +6,7 @@ import asyncio
 import hashlib
 import os
 import re
+import shlex
 import time
 from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
@@ -24,9 +25,11 @@ VERIFIER_ID = "asset_specific_read_only_host_postconditions"
 INDEPENDENT_SOURCE = "broker_ssh_host_runtime_readback"
 _SAFE_HOST_ALIAS_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
 _SAFE_REMOTE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
-_FIXED_PROXY_JUMP_BY_HOST = {
+_FIXED_PROXY_TARGET_BY_HOST = {
     "host_111": "wooo@192.168.0.110",
 }
+_INVENTORY_PROXY_KEY_PATH = Path("/run/secrets/ssh_mcp_key")
+_INVENTORY_PROXY_KNOWN_HOSTS_PATH = Path("/etc/ssh-mcp/known_hosts")
 _RUNTIME_CLOSURE_RECEIPT_CATALOGS = frozenset(
     {"ansible:188-openclaw-callback-forwarder"}
 )
@@ -699,6 +702,23 @@ def _load_inventory_hosts(inventory_path: Path) -> dict[str, dict[str, str]]:
     return hosts
 
 
+def _pinned_proxy_command(
+    *,
+    proxy_target: str,
+    ssh_key_path: Path,
+    known_hosts_path: Path,
+) -> str:
+    if not re.fullmatch(r"[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+", proxy_target):
+        raise ValueError("postcondition_proxy_target_invalid")
+    return (
+        f"ssh -i {shlex.quote(str(ssh_key_path))} "
+        f"-o UserKnownHostsFile={shlex.quote(str(known_hosts_path))} "
+        "-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes "
+        "-o BatchMode=yes -o ConnectTimeout=10 "
+        f"-W %h:%p {proxy_target}"
+    )
+
+
 def build_read_only_probe_command(
     condition: AssetPostcondition,
     *,
@@ -729,14 +749,30 @@ def build_read_only_probe_command(
         "-o",
         "ConnectTimeout=10",
     ]
-    proxy_jump = _FIXED_PROXY_JUMP_BY_HOST.get(condition.inventory_host)
+    proxy_target = _FIXED_PROXY_TARGET_BY_HOST.get(condition.inventory_host)
-    if proxy_jump is not None:
+    if proxy_target is not None:
+        inventory_proxy_command = _pinned_proxy_command(
+            proxy_target=proxy_target,
+            ssh_key_path=_INVENTORY_PROXY_KEY_PATH,
+            known_hosts_path=_INVENTORY_PROXY_KNOWN_HOSTS_PATH,
+        )
         expected_common_args = (
-            f"-o ProxyJump={proxy_jump} -o StrictHostKeyChecking=yes"
+            f'-o ProxyCommand="{inventory_proxy_command}" '
+            "-o StrictHostKeyChecking=yes"
         )
         if target.get("ansible_ssh_common_args") != expected_common_args:
-            raise ValueError("postcondition_proxy_jump_policy_mismatch")
+            raise ValueError("postcondition_proxy_command_policy_mismatch")
-        command.extend(["-o", f"ProxyJump={proxy_jump}"])
+        command.extend(
+            [
+                "-o",
+                "ProxyCommand="
+                + _pinned_proxy_command(
+                    proxy_target=proxy_target,
+                    ssh_key_path=ssh_key_path,
+                    known_hosts_path=known_hosts_path,
+                ),
+            ]
+        )
     command.extend([f"{user}@{host}", condition.probe])
     return command
 

apps/api/tests/test_executor_network_policy_boundary.py

@@ -187,9 +187,14 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
         "192.168.0.110 192.168.0.112 192.168.0.120 "
         "192.168.0.121 192.168.0.188"
     )
-    assert f"ssh-keyscan -T 5 {expected_hosts}" in workflow
+    expected_known_hosts = (
-    assert "EXPECTED_HOSTS=5" in workflow
+        "192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 "
-    assert workflow.count(expected_hosts) >= 4
+        "192.168.0.121 192.168.0.188"
+    )
+    assert f"ssh-keyscan -T 5 {expected_known_hosts}" in workflow
+    assert "EXPECTED_HOSTS=6" in workflow
+    assert workflow.count(expected_known_hosts) >= 4
+    assert workflow.count(expected_hosts) >= 2
     assert "except ConnectionRefusedError:" in workflow
     assert "broker_ssh_refused_but_egress_permitted=" in workflow
     assert "broker_ssh_allowlisted_endpoint_unavailable=" in workflow

apps/api/tests/test_host111_ollama_ansible_catalog.py

@@ -30,6 +30,13 @@ EXPECTED_ALLOWLIST = (
     "192.168.0.120/32,192.168.0.121/32"
 )
 EXPECTED_PROXY_JUMP = "wooo@192.168.0.110"
+EXPECTED_PROXY_COMMAND = (
+    "ssh -i /run/secrets/ssh_mcp_key "
+    "-o UserKnownHostsFile=/etc/ssh-mcp/known_hosts "
+    "-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes "
+    "-o BatchMode=yes -o ConnectTimeout=10 "
+    f"-W %h:%p {EXPECTED_PROXY_JUMP}"
+)
 
 
 def _flatten_tasks(tasks: list[dict]) -> list[dict]:
@@ -150,7 +157,12 @@ def test_host111_playbook_is_launchagent_bounded_and_check_mode_safe() -> None:
     assert play["vars"]["proxy_label"] == "com.momo.ollama111-allow-proxy"
     assert play["vars"]["allowed_cidrs"] == EXPECTED_ALLOWLIST
     assert "192.168.0.110/32" not in play["vars"]["allowed_cidrs"].split(",")
-    assert "launchctl bootstrap" in source
+    assert source.count("launchctl bootout") == 2
+    assert source.count("launchctl bootstrap") == 2
+    assert source.count("launchctl kickstart -k") == 2
+    assert source.count('test "$stopped" -eq 1') == 2
+    assert source.count('test "$loaded" -eq 1') == 2
+    assert source.count('test "$running" -eq 1') == 2
     assert "systemctl" not in source
     assert "docker restart" not in source
 
@@ -203,9 +215,10 @@ def test_host111_broker_transport_preserves_fixed_proxyjump_and_strict_safety(
     host111 = inventory["all"]["children"]["ollama_fallback"]["hosts"][
         "host_111"
     ]
-    assert host111["ansible_ssh_common_args"] == (
+    assert f'-o ProxyCommand="{EXPECTED_PROXY_COMMAND}"' in (
-        f"-o ProxyJump={EXPECTED_PROXY_JUMP} -o StrictHostKeyChecking=yes"
+        host111["ansible_ssh_common_args"]
     )
+    assert "ProxyJump" not in host111["ansible_ssh_common_args"]
     assert "accept-new" not in host111["ansible_ssh_common_args"]
 
     key = tmp_path / "ssh_mcp_key"
@@ -227,7 +240,7 @@ def test_host111_broker_transport_preserves_fixed_proxyjump_and_strict_safety(
         assert "ProxyJump" not in spec.env["ANSIBLE_SSH_ARGS"]
 
 
-def test_host111_independent_verifier_uses_same_fixed_proxyjump(
+def test_host111_independent_verifier_uses_same_pinned_proxy_command(
     tmp_path: Path,
 ) -> None:
     condition = next(
@@ -245,7 +258,13 @@ def test_host111_independent_verifier_uses_same_fixed_proxyjump(
         known_hosts_path=known_hosts,
     )
 
-    assert f"ProxyJump={EXPECTED_PROXY_JUMP}" in command
+    expected_verifier_proxy_command = (
+        f"ssh -i {key} -o UserKnownHostsFile={known_hosts} "
+        "-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes "
+        "-o BatchMode=yes -o ConnectTimeout=10 "
+        f"-W %h:%p {EXPECTED_PROXY_JUMP}"
+    )
+    assert f"ProxyCommand={expected_verifier_proxy_command}" in command
     assert "StrictHostKeyChecking=yes" in command
     assert "BatchMode=yes" in command
     assert "ooo@192.168.0.111" in command

infra/ansible/inventory/hosts.yml

@@ -26,7 +26,17 @@ all:
           ansible_host: 192.168.0.111
           ansible_user: ooo
           ansible_ssh_private_key_file: "~/.ssh/id_rsa"
-          ansible_ssh_common_args: "-o ProxyJump=wooo@192.168.0.110 -o StrictHostKeyChecking=yes"
+          # ProxyJump starts a second ssh process and does not inherit the
+          # executor's pinned key/known_hosts options. Keep both hops pinned.
+          ansible_ssh_common_args: >-
+            -o ProxyCommand="ssh -i /run/secrets/ssh_mcp_key
+            -o UserKnownHostsFile=/etc/ssh-mcp/known_hosts
+            -o StrictHostKeyChecking=yes
+            -o IdentitiesOnly=yes
+            -o BatchMode=yes
+            -o ConnectTimeout=10
+            -W %h:%p wooo@192.168.0.110"
+            -o StrictHostKeyChecking=yes
 
     k3s_masters:
       hosts:

infra/ansible/playbooks/111-ollama-fallback.yml

@@ -145,10 +145,37 @@
         - name: "Ollama111 | rollback 重新啟動 allow proxy"
           ansible.builtin.shell:
             cmd: |
-              set -e
+              set -eu
               launchctl enable gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
               launchctl bootout gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
-              launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }}
+              stopped=0
+              for attempt in 1 2 3 4 5 6 7 8 9 10; do
+                if ! launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} >/dev/null 2>&1; then
+                  stopped=1
+                  break
+                fi
+                sleep 1
+              done
+              test "$stopped" -eq 1
+              loaded=0
+              for attempt in 1 2 3 4 5; do
+                if launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }} >/dev/null 2>&1; then
+                  loaded=1
+                  break
+                fi
+                sleep 1
+              done
+              test "$loaded" -eq 1
+              launchctl kickstart -k gui/{{ proxy_user_uid }}/{{ proxy_label }}
+              running=0
+              for attempt in 1 2 3 4 5 6 7 8 9 10; do
+                if launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null | grep -q 'state = running'; then
+                  running=1
+                  break
+                fi
+                sleep 1
+              done
+              test "$running" -eq 1
           when: not ansible_check_mode
           changed_when: true
           tags: ["111", "ollama-fallback", "rollback"]
@@ -195,8 +222,35 @@
     - name: Restart ollama111 allow proxy
       ansible.builtin.shell:
         cmd: |
-          set -e
+          set -eu
           launchctl enable gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
           launchctl bootout gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
-          launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }}
+          stopped=0
+          for attempt in 1 2 3 4 5 6 7 8 9 10; do
+            if ! launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} >/dev/null 2>&1; then
+              stopped=1
+              break
+            fi
+            sleep 1
+          done
+          test "$stopped" -eq 1
+          loaded=0
+          for attempt in 1 2 3 4 5; do
+            if launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }} >/dev/null 2>&1; then
+              loaded=1
+              break
+            fi
+            sleep 1
+          done
+          test "$loaded" -eq 1
+          launchctl kickstart -k gui/{{ proxy_user_uid }}/{{ proxy_label }}
+          running=0
+          for attempt in 1 2 3 4 5 6 7 8 9 10; do
+            if launchctl print gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null | grep -q 'state = running'; then
+              running=1
+              break
+            fi
+            sleep 1
+          done
+          test "$running" -eq 1
       changed_when: true