wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(cd): isolate Host111 trust migration

Browse Source
This commit is contained in:
Your Name
2026-07-22 19:31:16 +08:00
parent 18a7c1603e
commit 2bcd8f6a66
3 changed files with 79 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
.gitea/workflows/cd.yaml
View File

@@ -3009,70 +3009,99 @@ jobs:
# 2026-05-02 ogt + Claude Sonnet 4.6: 加主機完整性檢查
# 根因：partial scan（如 110 timeout、其他成功）會讓 [-s file] 通過、
# 後續 patch 推進缺漏的 known_hosts → asyncssh 拒所有 SSH。
# 修法：scan 完驗證所有 allowlisted 主機都在；缺任何一台就 abort，
# 不能覆蓋現有 secret，防止 production SSH 自動修復路徑癱瘓。
# 修法：核心 executor 主機必須完整；Host111 是隔離的 provider lane，
# 首次 migration 掃不到時不得拖垮其他產品部署。既有 111 pin
# 只能保留，不能被 partial scan 覆蓋。
: > /tmp/known_hosts_repair
: > /tmp/known_hosts_scan_err
EXPECTED_HOSTS=6
CORE_HOSTS="192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188"
SCAN_HOSTS="192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188"
CORE_EXPECTED_HOSTS=5
TARGET_HOSTS=6
PRESENT_CORE=0
HOST111_PRESENT=0
PRESENT=0
for scan_attempt in 1 2 3; do
ssh-keyscan -T 5 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \
>> /tmp/known_hosts_repair 2>>/tmp/known_hosts_scan_err || true
sort -u /tmp/known_hosts_repair -o /tmp/known_hosts_repair
PRESENT=0
for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
PRESENT_CORE=0
for ip in \$CORE_HOSTS; do
if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
PRESENT=\$((PRESENT + 1))
PRESENT_CORE=\$((PRESENT_CORE + 1))
fi
done
if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
HOST111_PRESENT=0
if grep -qE '^192\.168\.0\.111[[:space:]]' /tmp/known_hosts_repair 2>/dev/null; then
HOST111_PRESENT=1
fi
PRESENT=\$((PRESENT_CORE + HOST111_PRESENT))
if [ "\$PRESENT" -eq "\$TARGET_HOSTS" ]; then
break
fi
sleep "\$scan_attempt"
done
for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
for ip in \$SCAN_HOSTS; do
if grep -qE "^\${ip}[[:space:]]" /tmp/known_hosts_repair 2>/dev/null; then
:
else
echo "⚠️ ssh-keyscan 缺主機 \${ip}"
fi
done
if [ "\$PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
EXISTING_KNOWN_HOSTS=/tmp/known_hosts_existing
EXISTING_CORE=0
EXISTING_HOST111=0
if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \
-o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then
for ip in \$CORE_HOSTS; do
if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
EXISTING_CORE=\$((EXISTING_CORE + 1))
fi
done
if grep -qE '^192\.168\.0\.111[[:space:]]' "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
EXISTING_HOST111=1
fi
fi
EXISTING_PRESENT=\$((EXISTING_CORE + EXISTING_HOST111))
TRUST_SOURCE=""
TRUST_SELECTION=""
if [ "\$PRESENT_CORE" -eq "\$CORE_EXPECTED_HOSTS" ] && [ "\$HOST111_PRESENT" -eq 1 ]; then
TRUST_SOURCE=/tmp/known_hosts_repair
TRUST_SELECTION=fresh_6_of_6
elif [ "\$EXISTING_CORE" -eq "\$CORE_EXPECTED_HOSTS" ] && [ "\$EXISTING_HOST111" -eq 1 ]; then
TRUST_SELECTION=preserve_existing_6_of_6
elif [ "\$PRESENT_CORE" -eq "\$CORE_EXPECTED_HOSTS" ]; then
TRUST_SOURCE=/tmp/known_hosts_repair
TRUST_SELECTION=fresh_core_5_host111_trust_deferred
elif [ "\$EXISTING_CORE" -eq "\$CORE_EXPECTED_HOSTS" ]; then
TRUST_SELECTION=preserve_existing_core_5_host111_trust_deferred
else
echo "❌ host-key core trust incomplete; fresh=\${PRESENT_CORE}/\${CORE_EXPECTED_HOSTS} existing=\${EXISTING_CORE}/\${CORE_EXPECTED_HOSTS}"
rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
exit 1
fi
if [ -n "\$TRUST_SOURCE" ]; then
\$KUBECTL create secret generic awoooi-repair-known-hosts \
-n awoooi-prod \
--from-file=known_hosts=/tmp/known_hosts_repair \
--from-file=known_hosts="\$TRUST_SOURCE" \
--dry-run=client -o yaml | \$KUBECTL apply -f - \
&& echo "✅ awoooi-repair-known-hosts Secret 已建立/更新" \
|| echo "⚠️ awoooi-repair-known-hosts Secret 建立失敗 (非致命)"
KNOWN_HOSTS_B64=\$(base64 -w 0 /tmp/known_hosts_repair)
KNOWN_HOSTS_B64=\$(base64 -w 0 "\$TRUST_SOURCE")
\$KUBECTL patch secret ssh-mcp-key -n awoooi-prod --type=merge \
-p="{\"data\":{\"known_hosts\":\"\${KNOWN_HOSTS_B64}\"}}" \
&& echo "✅ ssh-mcp-key known_hosts 已更新（6 台主機完整）" \
&& echo "✅ ssh-mcp-key known_hosts 已更新 selection=\${TRUST_SELECTION}" \
|| { echo "❌ ssh-mcp-key known_hosts 更新失敗，停止部署"; exit 1; }
rm -f /tmp/known_hosts_repair /tmp/known_hosts_scan_err
else
# A transient guest SSH outage must not overwrite trust data, but it
# also must not blindly green-light the deploy. Decode only on the
# K3s controller, verify all six public host entries, then delete
# the temporary file without printing its contents.
EXISTING_KNOWN_HOSTS=/tmp/known_hosts_existing
EXISTING_PRESENT=0
if \$KUBECTL get secret ssh-mcp-key -n awoooi-prod \
-o jsonpath='{.data.known_hosts}' | base64 -d > "\$EXISTING_KNOWN_HOSTS"; then
for ip in 192.168.0.110 192.168.0.111 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188; do
if grep -qE "^\${ip}[[:space:]]" "\$EXISTING_KNOWN_HOSTS" 2>/dev/null; then
EXISTING_PRESENT=\$((EXISTING_PRESENT + 1))
fi
done
fi
rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
if [ "\$EXISTING_PRESENT" -eq "\$EXPECTED_HOSTS" ]; then
echo "⚠️ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key verified 6/6 and preserved"
else
echo "❌ fresh host-key scan \${PRESENT}/\${EXPECTED_HOSTS}; existing ssh-mcp-key coverage \${EXISTING_PRESENT}/\${EXPECTED_HOSTS}"
exit 1
fi
echo "⚠️ ssh-mcp-key known_hosts preserved selection=\${TRUST_SELECTION} fresh=\${PRESENT}/\${TARGET_HOSTS} existing=\${EXISTING_PRESENT}/\${TARGET_HOSTS}"
fi
if [ "\$HOST111_PRESENT" -eq 0 ] && [ "\$EXISTING_HOST111" -eq 0 ]; then
echo "⚠️ host111_trust_deferred isolated_lane=ansible:111-ollama-fallback"
fi
rm -f "\$EXISTING_KNOWN_HOSTS" /tmp/known_hosts_repair /tmp/known_hosts_scan_err
echo "✅ 所有 Secrets 注入完成"
SECRETS

15
apps/api/tests/test_cd_known_hosts_recovery_gate.py
View File

@@ -4,16 +4,21 @@ ROOT = Path(__file__).resolve().parents[3]
WORKFLOW = ROOT / ".gitea/workflows/cd.yaml"
def test_cd_known_hosts_gate_retries_and_verifies_preserved_secret() -> None:
def test_cd_known_hosts_gate_keeps_core_fail_closed_and_host111_isolated() -> None:
workflow = WORKFLOW.read_text(encoding="utf-8")
gate = workflow[workflow.index(": > /tmp/known_hosts_repair") :]
gate = gate[: gate.index('echo "✅ 所有 Secrets 注入完成"')]
assert "for scan_attempt in 1 2 3" in gate
assert 'sleep "\\$scan_attempt"' in gate
assert "existing ssh-mcp-key verified 6/6 and preserved" in gate
assert "EXISTING_PRESENT=\\$((EXISTING_PRESENT + 1))" in gate
assert 'if [ "\\$EXISTING_PRESENT" -eq "\\$EXPECTED_HOSTS" ]' in gate
assert "CORE_EXPECTED_HOSTS=5" in gate
assert "TARGET_HOSTS=6" in gate
assert "fresh_6_of_6" in gate
assert "preserve_existing_6_of_6" in gate
assert "fresh_core_5_host111_trust_deferred" in gate
assert "preserve_existing_core_5_host111_trust_deferred" in gate
assert "host-key core trust incomplete" in gate
assert "host111_trust_deferred isolated_lane=ansible:111-ollama-fallback" in gate
assert 'rm -f "\\$EXISTING_KNOWN_HOSTS"' in gate
assert "cat /tmp/known_hosts_scan_err" not in gate
assert "existing ssh-mcp-key coverage" in gate
assert "KNOWN_HOSTS_B64=\\$(base64 -w 0 \"\\$TRUST_SOURCE\")" in gate

8
apps/api/tests/test_executor_network_policy_boundary.py
View File

@@ -192,8 +192,12 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
"192.168.0.121 192.168.0.188"
)
assert f"ssh-keyscan -T 5 {expected_known_hosts}" in workflow
assert "EXPECTED_HOSTS=6" in workflow
assert workflow.count(expected_known_hosts) >= 4
assert "CORE_EXPECTED_HOSTS=5" in workflow
assert "TARGET_HOSTS=6" in workflow
assert "fresh_core_5_host111_trust_deferred" in workflow
assert "preserve_existing_core_5_host111_trust_deferred" in workflow
assert "host111_trust_deferred isolated_lane=ansible:111-ollama-fallback" in workflow
assert workflow.count(expected_known_hosts) >= 2
assert workflow.count(expected_hosts) >= 2
assert "except ConnectionRefusedError:" in workflow
assert "broker_ssh_refused_but_egress_permitted=" in workflow