fix(sre): close bounded runtime contract gaps
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 4m41s
CD Pipeline / build-and-deploy (push) Successful in 15m36s
CD Pipeline / post-deploy-checks (push) Successful in 4m19s

This commit is contained in:
ogt
2026-07-16 21:21:43 +08:00
parent c7d33572a9
commit 08f3a70ce6
32 changed files with 3041 additions and 157 deletions

View File

@@ -26,7 +26,7 @@ all:
ansible_host: 192.168.0.111
ansible_user: ooo
ansible_ssh_private_key_file: "~/.ssh/id_rsa"
ansible_ssh_common_args: "-o ProxyJump=192.168.0.110 -o StrictHostKeyChecking=accept-new"
ansible_ssh_common_args: "-o ProxyJump=wooo@192.168.0.110 -o StrictHostKeyChecking=yes"
k3s_masters:
hosts:

View File

@@ -20,96 +20,176 @@
changed_when: false
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 收斂 K3s direct provider allowlist"
ansible.builtin.command:
cmd: "/usr/libexec/PlistBuddy -c 'Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ allowed_cidrs }}' {{ proxy_plist }}"
when: current_allowlist.stdout != allowed_cidrs
changed_when: true
notify: Restart ollama111 allow proxy
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 受控 apply 與 verifier失敗即 rollback"
block:
- name: "Ollama111 | 收斂 K3s direct provider allowlist"
ansible.builtin.command:
argv:
- /usr/libexec/PlistBuddy
- -c
- "Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ allowed_cidrs }}"
- "{{ proxy_plist }}"
when:
- not ansible_check_mode
- current_allowlist.stdout != allowed_cidrs
changed_when: true
notify: Restart ollama111 allow proxy
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 套用 allowlist handler 後再驗證"
ansible.builtin.meta: flush_handlers
- name: "Ollama111 | 套用 allowlist handler 後再驗證"
ansible.builtin.meta: flush_handlers
when: not ansible_check_mode
- name: "Ollama111 | 讀回生效 allowlist"
ansible.builtin.command:
cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}"
register: effective_allowlist
check_mode: false
changed_when: false
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 讀回生效 allowlist"
ansible.builtin.command:
cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}"
register: effective_allowlist
check_mode: false
changed_when: false
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 驗證 K3s origins 並排除 retired host110"
ansible.builtin.assert:
that:
- effective_allowlist.stdout == allowed_cidrs
- "'192.168.0.120/32' in effective_allowlist.stdout.split(',')"
- "'192.168.0.121/32' in effective_allowlist.stdout.split(',')"
fail_msg: ollama111_allowlist_runtime_identity_mismatch
changed_when: false
when: not ansible_check_mode
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 驗證 K3s origins 並排除 retired host110"
ansible.builtin.assert:
that:
- effective_allowlist.stdout == allowed_cidrs
- "'192.168.0.120/32' in effective_allowlist.stdout.split(',')"
- "'192.168.0.121/32' in effective_allowlist.stdout.split(',')"
fail_msg: ollama111_allowlist_runtime_identity_mismatch
changed_when: false
when: not ansible_check_mode
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | check-mode 發布精確 source diff"
ansible.builtin.debug:
msg:
schema_version: ollama111_allowlist_check_v1
current_allowlist_sha_only: true
would_change: "{{ current_allowlist.stdout != allowed_cidrs }}"
expected_k3s_origins:
- host120
- host121
retired_host110_allowed_after_apply: false
persistent_writes_performed: 0
when: ansible_check_mode
changed_when: current_allowlist.stdout != allowed_cidrs
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | check-mode 發布精確 source diff"
ansible.builtin.debug:
msg:
schema_version: ollama111_allowlist_check_v1
current_allowlist_sha_only: true
would_change: "{{ current_allowlist.stdout != allowed_cidrs }}"
expected_k3s_origins:
- host120
- host121
retired_host110_allowed_after_apply: false
persistent_writes_performed: 0
when: ansible_check_mode
changed_when: current_allowlist.stdout != allowed_cidrs
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 驗證本機 Ollama tag API"
ansible.builtin.uri:
url: http://127.0.0.1:11434/api/tags
method: GET
return_content: false
status_code: 200
check_mode: false
changed_when: false
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 驗證本機 Ollama tag API"
ansible.builtin.uri:
url: http://127.0.0.1:11434/api/tags
method: GET
return_content: false
status_code: 200
check_mode: false
changed_when: false
tags: ["111", "ollama-fallback"]
- name: "Ollama111 | 從每個 K3s node 驗證 direct provider path"
ansible.builtin.uri:
url: http://192.168.0.111:11434/api/tags
method: GET
return_content: false
status_code: 200
timeout: 10
delegate_to: "{{ item }}"
loop:
- host_120
- host_121
register: k3s_direct_provider_readback
check_mode: false
changed_when: false
when: >-
not ansible_check_mode
or current_allowlist.stdout == allowed_cidrs
tags: ["111", "ollama-fallback", "independent-verifier"]
- name: "Ollama111 | 從每個 K3s node 驗證 direct provider path"
ansible.builtin.uri:
url: http://192.168.0.111:11434/api/tags
method: GET
return_content: false
status_code: 200
timeout: 10
delegate_to: "{{ item }}"
loop:
- host_120
- host_121
register: k3s_direct_provider_readback
check_mode: false
changed_when: false
when: >-
not ansible_check_mode
or current_allowlist.stdout == allowed_cidrs
tags: ["111", "ollama-fallback", "independent-verifier"]
- name: "Ollama111 | 發布 K3s path verifier receipt"
ansible.builtin.debug:
msg:
schema_version: ollama111_k3s_path_verifier_v1
canonical_asset_id: ai-provider:ollama_local
probe_origins:
- host120
- host121
http_statuses: >-
{{ k3s_direct_provider_readback.results
| map(attribute='status') | list }}
retired_host110_allowed: false
runtime_verified: true
changed_when: false
when: not ansible_check_mode
tags: ["111", "ollama-fallback", "independent-verifier"]
- name: "Ollama111 | 發布 K3s path verifier receipt"
ansible.builtin.debug:
msg:
schema_version: ollama111_k3s_path_verifier_v1
canonical_asset_id: ai-provider:ollama_local
probe_origins:
- host120
- host121
http_statuses: >-
{{ k3s_direct_provider_readback.results
| map(attribute='status') | list }}
retired_host110_allowed: false
runtime_verified: true
changed_when: false
when: not ansible_check_mode
tags: ["111", "ollama-fallback", "independent-verifier"]
rescue:
- name: "Ollama111 | check-mode failure 維持 no-write"
ansible.builtin.debug:
msg:
schema_version: ollama111_check_failure_no_write_v1
failed_task: "{{ ansible_failed_task.name | default('unknown') }}"
rollback_performed: false
persistent_writes_performed: 0
when: ansible_check_mode
changed_when: false
- name: "Ollama111 | rollback 恢復原 allowlist"
ansible.builtin.command:
argv:
- /usr/libexec/PlistBuddy
- -c
- "Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ current_allowlist.stdout }}"
- "{{ proxy_plist }}"
when: not ansible_check_mode
changed_when: true
tags: ["111", "ollama-fallback", "rollback"]
- name: "Ollama111 | rollback 重新啟動 allow proxy"
ansible.builtin.shell:
cmd: |
set -e
launchctl enable gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
launchctl bootout gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }}
when: not ansible_check_mode
changed_when: true
tags: ["111", "ollama-fallback", "rollback"]
- name: "Ollama111 | rollback 讀回原 allowlist"
ansible.builtin.command:
cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}"
register: rollback_allowlist
when: not ansible_check_mode
check_mode: false
changed_when: false
tags: ["111", "ollama-fallback", "rollback"]
- name: "Ollama111 | rollback 驗證原 allowlist 已恢復"
ansible.builtin.assert:
that:
- rollback_allowlist.stdout == current_allowlist.stdout
fail_msg: ollama111_allowlist_rollback_readback_mismatch
when: not ansible_check_mode
changed_when: false
tags: ["111", "ollama-fallback", "rollback"]
- name: "Ollama111 | 發布 rollback receipt"
ansible.builtin.debug:
msg:
schema_version: ollama111_allowlist_rollback_v1
canonical_asset_id: ai-provider:ollama_local
failed_task: "{{ ansible_failed_task.name | default('unknown') }}"
original_allowlist_readback_verified: true
proxy_restarted: true
rollback_performed: true
when: not ansible_check_mode
changed_when: false
tags: ["111", "ollama-fallback", "rollback"]
- name: "Ollama111 | apply 或 verifier 失敗後保持 failed terminal"
ansible.builtin.fail:
msg: >-
ollama111_controlled_lane_failed
task={{ ansible_failed_task.name | default('unknown') }}
rollback_performed={{ not ansible_check_mode }}
handlers:
- name: Restart ollama111 allow proxy