fix(sre): close bounded runtime contract gaps
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 4m41s
CD Pipeline / build-and-deploy (push) Successful in 15m36s
CD Pipeline / post-deploy-checks (push) Successful in 4m19s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 4m41s
CD Pipeline / build-and-deploy (push) Successful in 15m36s
CD Pipeline / post-deploy-checks (push) Successful in 4m19s
This commit is contained in:
@@ -26,7 +26,7 @@ all:
|
||||
ansible_host: 192.168.0.111
|
||||
ansible_user: ooo
|
||||
ansible_ssh_private_key_file: "~/.ssh/id_rsa"
|
||||
ansible_ssh_common_args: "-o ProxyJump=192.168.0.110 -o StrictHostKeyChecking=accept-new"
|
||||
ansible_ssh_common_args: "-o ProxyJump=wooo@192.168.0.110 -o StrictHostKeyChecking=yes"
|
||||
|
||||
k3s_masters:
|
||||
hosts:
|
||||
|
||||
@@ -20,96 +20,176 @@
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback"]
|
||||
|
||||
- name: "Ollama111 | 收斂 K3s direct provider allowlist"
|
||||
ansible.builtin.command:
|
||||
cmd: "/usr/libexec/PlistBuddy -c 'Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ allowed_cidrs }}' {{ proxy_plist }}"
|
||||
when: current_allowlist.stdout != allowed_cidrs
|
||||
changed_when: true
|
||||
notify: Restart ollama111 allow proxy
|
||||
tags: ["111", "ollama-fallback"]
|
||||
- name: "Ollama111 | 受控 apply 與 verifier,失敗即 rollback"
|
||||
block:
|
||||
- name: "Ollama111 | 收斂 K3s direct provider allowlist"
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/libexec/PlistBuddy
|
||||
- -c
|
||||
- "Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ allowed_cidrs }}"
|
||||
- "{{ proxy_plist }}"
|
||||
when:
|
||||
- not ansible_check_mode
|
||||
- current_allowlist.stdout != allowed_cidrs
|
||||
changed_when: true
|
||||
notify: Restart ollama111 allow proxy
|
||||
tags: ["111", "ollama-fallback"]
|
||||
|
||||
- name: "Ollama111 | 套用 allowlist handler 後再驗證"
|
||||
ansible.builtin.meta: flush_handlers
|
||||
- name: "Ollama111 | 套用 allowlist handler 後再驗證"
|
||||
ansible.builtin.meta: flush_handlers
|
||||
when: not ansible_check_mode
|
||||
|
||||
- name: "Ollama111 | 讀回生效 allowlist"
|
||||
ansible.builtin.command:
|
||||
cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}"
|
||||
register: effective_allowlist
|
||||
check_mode: false
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback"]
|
||||
- name: "Ollama111 | 讀回生效 allowlist"
|
||||
ansible.builtin.command:
|
||||
cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}"
|
||||
register: effective_allowlist
|
||||
check_mode: false
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback"]
|
||||
|
||||
- name: "Ollama111 | 驗證 K3s origins 並排除 retired host110"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- effective_allowlist.stdout == allowed_cidrs
|
||||
- "'192.168.0.120/32' in effective_allowlist.stdout.split(',')"
|
||||
- "'192.168.0.121/32' in effective_allowlist.stdout.split(',')"
|
||||
fail_msg: ollama111_allowlist_runtime_identity_mismatch
|
||||
changed_when: false
|
||||
when: not ansible_check_mode
|
||||
tags: ["111", "ollama-fallback"]
|
||||
- name: "Ollama111 | 驗證 K3s origins 並排除 retired host110"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- effective_allowlist.stdout == allowed_cidrs
|
||||
- "'192.168.0.120/32' in effective_allowlist.stdout.split(',')"
|
||||
- "'192.168.0.121/32' in effective_allowlist.stdout.split(',')"
|
||||
fail_msg: ollama111_allowlist_runtime_identity_mismatch
|
||||
changed_when: false
|
||||
when: not ansible_check_mode
|
||||
tags: ["111", "ollama-fallback"]
|
||||
|
||||
- name: "Ollama111 | check-mode 發布精確 source diff"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
schema_version: ollama111_allowlist_check_v1
|
||||
current_allowlist_sha_only: true
|
||||
would_change: "{{ current_allowlist.stdout != allowed_cidrs }}"
|
||||
expected_k3s_origins:
|
||||
- host120
|
||||
- host121
|
||||
retired_host110_allowed_after_apply: false
|
||||
persistent_writes_performed: 0
|
||||
when: ansible_check_mode
|
||||
changed_when: current_allowlist.stdout != allowed_cidrs
|
||||
tags: ["111", "ollama-fallback"]
|
||||
- name: "Ollama111 | check-mode 發布精確 source diff"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
schema_version: ollama111_allowlist_check_v1
|
||||
current_allowlist_sha_only: true
|
||||
would_change: "{{ current_allowlist.stdout != allowed_cidrs }}"
|
||||
expected_k3s_origins:
|
||||
- host120
|
||||
- host121
|
||||
retired_host110_allowed_after_apply: false
|
||||
persistent_writes_performed: 0
|
||||
when: ansible_check_mode
|
||||
changed_when: current_allowlist.stdout != allowed_cidrs
|
||||
tags: ["111", "ollama-fallback"]
|
||||
|
||||
- name: "Ollama111 | 驗證本機 Ollama tag API"
|
||||
ansible.builtin.uri:
|
||||
url: http://127.0.0.1:11434/api/tags
|
||||
method: GET
|
||||
return_content: false
|
||||
status_code: 200
|
||||
check_mode: false
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback"]
|
||||
- name: "Ollama111 | 驗證本機 Ollama tag API"
|
||||
ansible.builtin.uri:
|
||||
url: http://127.0.0.1:11434/api/tags
|
||||
method: GET
|
||||
return_content: false
|
||||
status_code: 200
|
||||
check_mode: false
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback"]
|
||||
|
||||
- name: "Ollama111 | 從每個 K3s node 驗證 direct provider path"
|
||||
ansible.builtin.uri:
|
||||
url: http://192.168.0.111:11434/api/tags
|
||||
method: GET
|
||||
return_content: false
|
||||
status_code: 200
|
||||
timeout: 10
|
||||
delegate_to: "{{ item }}"
|
||||
loop:
|
||||
- host_120
|
||||
- host_121
|
||||
register: k3s_direct_provider_readback
|
||||
check_mode: false
|
||||
changed_when: false
|
||||
when: >-
|
||||
not ansible_check_mode
|
||||
or current_allowlist.stdout == allowed_cidrs
|
||||
tags: ["111", "ollama-fallback", "independent-verifier"]
|
||||
- name: "Ollama111 | 從每個 K3s node 驗證 direct provider path"
|
||||
ansible.builtin.uri:
|
||||
url: http://192.168.0.111:11434/api/tags
|
||||
method: GET
|
||||
return_content: false
|
||||
status_code: 200
|
||||
timeout: 10
|
||||
delegate_to: "{{ item }}"
|
||||
loop:
|
||||
- host_120
|
||||
- host_121
|
||||
register: k3s_direct_provider_readback
|
||||
check_mode: false
|
||||
changed_when: false
|
||||
when: >-
|
||||
not ansible_check_mode
|
||||
or current_allowlist.stdout == allowed_cidrs
|
||||
tags: ["111", "ollama-fallback", "independent-verifier"]
|
||||
|
||||
- name: "Ollama111 | 發布 K3s path verifier receipt"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
schema_version: ollama111_k3s_path_verifier_v1
|
||||
canonical_asset_id: ai-provider:ollama_local
|
||||
probe_origins:
|
||||
- host120
|
||||
- host121
|
||||
http_statuses: >-
|
||||
{{ k3s_direct_provider_readback.results
|
||||
| map(attribute='status') | list }}
|
||||
retired_host110_allowed: false
|
||||
runtime_verified: true
|
||||
changed_when: false
|
||||
when: not ansible_check_mode
|
||||
tags: ["111", "ollama-fallback", "independent-verifier"]
|
||||
- name: "Ollama111 | 發布 K3s path verifier receipt"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
schema_version: ollama111_k3s_path_verifier_v1
|
||||
canonical_asset_id: ai-provider:ollama_local
|
||||
probe_origins:
|
||||
- host120
|
||||
- host121
|
||||
http_statuses: >-
|
||||
{{ k3s_direct_provider_readback.results
|
||||
| map(attribute='status') | list }}
|
||||
retired_host110_allowed: false
|
||||
runtime_verified: true
|
||||
changed_when: false
|
||||
when: not ansible_check_mode
|
||||
tags: ["111", "ollama-fallback", "independent-verifier"]
|
||||
|
||||
rescue:
|
||||
- name: "Ollama111 | check-mode failure 維持 no-write"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
schema_version: ollama111_check_failure_no_write_v1
|
||||
failed_task: "{{ ansible_failed_task.name | default('unknown') }}"
|
||||
rollback_performed: false
|
||||
persistent_writes_performed: 0
|
||||
when: ansible_check_mode
|
||||
changed_when: false
|
||||
|
||||
- name: "Ollama111 | rollback 恢復原 allowlist"
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- /usr/libexec/PlistBuddy
|
||||
- -c
|
||||
- "Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ current_allowlist.stdout }}"
|
||||
- "{{ proxy_plist }}"
|
||||
when: not ansible_check_mode
|
||||
changed_when: true
|
||||
tags: ["111", "ollama-fallback", "rollback"]
|
||||
|
||||
- name: "Ollama111 | rollback 重新啟動 allow proxy"
|
||||
ansible.builtin.shell:
|
||||
cmd: |
|
||||
set -e
|
||||
launchctl enable gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
|
||||
launchctl bootout gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true
|
||||
launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }}
|
||||
when: not ansible_check_mode
|
||||
changed_when: true
|
||||
tags: ["111", "ollama-fallback", "rollback"]
|
||||
|
||||
- name: "Ollama111 | rollback 讀回原 allowlist"
|
||||
ansible.builtin.command:
|
||||
cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}"
|
||||
register: rollback_allowlist
|
||||
when: not ansible_check_mode
|
||||
check_mode: false
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback", "rollback"]
|
||||
|
||||
- name: "Ollama111 | rollback 驗證原 allowlist 已恢復"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- rollback_allowlist.stdout == current_allowlist.stdout
|
||||
fail_msg: ollama111_allowlist_rollback_readback_mismatch
|
||||
when: not ansible_check_mode
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback", "rollback"]
|
||||
|
||||
- name: "Ollama111 | 發布 rollback receipt"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
schema_version: ollama111_allowlist_rollback_v1
|
||||
canonical_asset_id: ai-provider:ollama_local
|
||||
failed_task: "{{ ansible_failed_task.name | default('unknown') }}"
|
||||
original_allowlist_readback_verified: true
|
||||
proxy_restarted: true
|
||||
rollback_performed: true
|
||||
when: not ansible_check_mode
|
||||
changed_when: false
|
||||
tags: ["111", "ollama-fallback", "rollback"]
|
||||
|
||||
- name: "Ollama111 | apply 或 verifier 失敗後保持 failed terminal"
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
ollama111_controlled_lane_failed
|
||||
task={{ ansible_failed_task.name | default('unknown') }}
|
||||
rollback_performed={{ not ansible_check_mode }}
|
||||
|
||||
handlers:
|
||||
- name: Restart ollama111 allow proxy
|
||||
|
||||
Reference in New Issue
Block a user