diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 2a7de656a..ea0311778 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -426,14 +426,33 @@ _CATALOG: tuple[dict[str, Any], ...] = ( }, { "catalog_id": "ansible:110-devops", - "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml", + "check_mode_playbook_path": "infra/ansible/playbooks/110-node-exporter-repair.yml", "inventory_hosts": ["host_110"], - "domains": ["swap", "harbor", "sentry", "gitea", "langfuse", "bitan", "runner", "keepalived", "nginx"], + "domains": ["monitoring", "prometheus", "node_exporter", "host_110"], "keywords": [ - "110", - "docker", - "container", - "dockercontainerunhealthy", + "node-exporter-110", + "node exporter 110", + "nodeexporterdown", + "node_exporter", + "node-exporter", + ], + "activation_keywords": [ + "node-exporter-110", + "node exporter 110", + ], + "supports_check_mode": True, + "auto_apply_enabled": True, + "approval_required": False, + "risk_level": "low", + "catalog_revision": "2026-07-14-node-exporter-bounded-v4", + }, + { + "catalog_id": "ansible:110-devops-full-convergence", + "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "check_mode_playbook_path": "infra/ansible/playbooks/110-devops.yml", + "inventory_hosts": ["host_110"], + "domains": [ "swap", "harbor", "sentry", @@ -441,14 +460,25 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "langfuse", "bitan", "runner", - "github-runner", "keepalived", + "nginx", + "backup", + ], + "keywords": [ + "110-devops-full-convergence", + "harbor", + "sentry", + "langfuse", + "bitan", + "keepalived", + "github-runner", ], "supports_check_mode": True, - "auto_apply_enabled": True, + "auto_apply_enabled": False, "approval_required": False, - "risk_level": "medium", - "catalog_revision": "2026-07-14-host110-harbor-route-v3", + "risk_level": "high", + "no_write_only": True, + "catalog_revision": "2026-07-14-full-convergence-quarantined-v1", }, { "catalog_id": "ansible:188-momo-backup-user", diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 5c27a9e0d..e6f9cdbef 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -1063,6 +1063,7 @@ def _claim_from_stale_check_mode_row( or "" ) catalog_id = str(input_payload.get("catalog_id") or "") + catalog_item = get_ansible_catalog_item(catalog_id) or {} inventory_hosts = input_payload.get("inventory_hosts") source_playbook_path = str( input_payload.get("source_candidate_playbook_path") @@ -1080,6 +1081,9 @@ def _claim_from_stale_check_mode_row( or not inventory_hosts ): return None + canonical_playbook_path = str(catalog_item.get("playbook_path") or "") + if not canonical_playbook_path: + return None try: claim_input = build_ansible_check_mode_claim_input( source_candidate_op_id=source_candidate_op_id, @@ -1089,9 +1093,13 @@ def _claim_from_stale_check_mode_row( "executor_candidates": [ { "catalog_id": catalog_id, - "playbook_path": source_playbook_path, + # A stale row is evidence, never execution authority. Path + # migrations must resolve through the current allowlisted + # catalog so a repaired bounded playbook can replace an + # unsafe or retired predecessor without trusting DB input. + "playbook_path": canonical_playbook_path, "inventory_hosts": inventory_hosts, - "risk_level": input_payload.get("risk_level"), + "risk_level": catalog_item.get("risk_level"), } ], }, diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index 6f273650c..be5806d5a 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -99,6 +99,31 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { ), ), "ansible:110-devops": ( + AssetPostcondition( + "host_110_node_exporter_container", + "service", + "host_110", + "test \"$(docker inspect --format '{{.State.Running}}' node-exporter 2>/dev/null)\" = true", + ), + AssetPostcondition( + "host_110_node_exporter_metrics", + "metric", + "host_110", + "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 " + "http://127.0.0.1:9100/metrics)\" = 200", + ), + AssetPostcondition( + "host_110_node_exporter_prometheus_scrape", + "metric", + "host_110", + "for attempt in 1 2 3 4 5 6; do " + "curl -fsS --max-time 5 " + "'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' " + "| grep -Eq '\"value\":\\[[^]]+,\"1\"\\]' && exit 0; " + "sleep 5; done; exit 1", + ), + ), + "ansible:110-devops-full-convergence": ( AssetPostcondition( "host_110_docker_runtime", "service", diff --git a/apps/api/tests/test_awooop_ansible_post_verifier.py b/apps/api/tests/test_awooop_ansible_post_verifier.py index 480eec23f..a85ad6310 100644 --- a/apps/api/tests/test_awooop_ansible_post_verifier.py +++ b/apps/api/tests/test_awooop_ansible_post_verifier.py @@ -52,15 +52,38 @@ def test_asset_postcondition_registry_covers_every_ansible_catalog() -> None: assert registered_catalog_ids() == catalog_ids -def test_host110_harbor_postcondition_matches_registry_backend() -> None: +def test_host110_node_exporter_postconditions_are_bounded() -> None: catalog = next( item for item in list_ansible_catalog() if item["catalog_id"] == "ansible:110-devops" ) + conditions = postconditions_for_catalog("ansible:110-devops") + + assert {item.condition_id for item in conditions} == { + "host_110_node_exporter_container", + "host_110_node_exporter_metrics", + "host_110_node_exporter_prometheus_scrape", + } + assert catalog["playbook_path"].endswith("110-node-exporter-repair.yml") + assert catalog["risk_level"] == "low" + assert catalog["auto_apply_enabled"] is True + assert catalog["catalog_revision"] == ( + "2026-07-14-node-exporter-bounded-v4" + ) + + +def test_host110_full_convergence_harbor_probe_is_quarantined() -> None: + catalog = next( + item + for item in list_ansible_catalog() + if item["catalog_id"] == "ansible:110-devops-full-convergence" + ) condition = next( item - for item in postconditions_for_catalog("ansible:110-devops") + for item in postconditions_for_catalog( + "ansible:110-devops-full-convergence" + ) if item.condition_id == "host_110_harbor_route" ) @@ -68,9 +91,8 @@ def test_host110_harbor_postcondition_matches_registry_backend() -> None: assert "200" in condition.probe assert "401" in condition.probe assert "sites-enabled/harbor.conf" not in condition.probe - assert catalog["catalog_revision"] == ( - "2026-07-14-host110-harbor-route-v3" - ) + assert catalog["auto_apply_enabled"] is False + assert catalog["no_write_only"] is True def test_read_only_probe_command_uses_structured_inventory(tmp_path: Path) -> None: diff --git a/apps/api/tests/test_host110_node_exporter_ansible_catalog.py b/apps/api/tests/test_host110_node_exporter_ansible_catalog.py new file mode 100644 index 000000000..b4be166a8 --- /dev/null +++ b/apps/api/tests/test_host110_node_exporter_ansible_catalog.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + +from src.services.awooop_ansible_audit_service import ( + _catalog_hints, + get_ansible_catalog_item, +) +from src.services.awooop_ansible_check_mode_service import ( + _claim_from_stale_check_mode_row, +) + +ROOT = Path(__file__).resolve().parents[3] +PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-node-exporter-repair.yml" + + +def test_node_exporter_alert_uses_bounded_controlled_apply_catalog() -> None: + hints = _catalog_hints( + { + "alertname": "NodeExporterDown", + "affected_services": ["node-exporter-110"], + "signals": [ + { + "alert_name": "NodeExporterDown", + "labels": {"job": "node-exporter-110", "host": "110"}, + } + ], + }, + None, + ) + + assert [row["catalog_id"] for row in hints["candidates"]] == [ + "ansible:110-devops" + ] + assert hints["candidates"][0]["auto_apply_enabled"] is True + assert hints["candidates"][0]["risk_level"] == "low" + + +def test_node_exporter_playbook_only_reconciles_one_container() -> None: + payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) + text = PLAYBOOK.read_text(encoding="utf-8") + catalog = get_ansible_catalog_item("ansible:110-devops") + + assert payload[0]["hosts"] == "host_110" + assert payload[0]["become"] is False + assert catalog is not None + assert catalog["playbook_path"].endswith("110-node-exporter-repair.yml") + assert catalog["catalog_revision"] == ( + "2026-07-14-node-exporter-bounded-v4" + ) + assert "docker\n - compose\n - up" in text + assert "node-exporter" in text + for forbidden in ( + "backup-awoooi.sh", + "swapon", + "keepalived", + "github-runner", + "docker system prune", + "reboot", + ): + assert forbidden not in text.lower() + + +def test_existing_110_devops_lineage_migrates_to_bounded_revision() -> None: + claim = _claim_from_stale_check_mode_row( + { + "op_id": "00000000-0000-0000-0000-000000000111", + "parent_op_id": "00000000-0000-0000-0000-000000000110", + "incident_id": "INC-NODE-EXPORTER-110", + "input": { + "source_candidate_op_id": ( + "00000000-0000-0000-0000-000000000110" + ), + "incident_id": "INC-NODE-EXPORTER-110", + "catalog_id": "ansible:110-devops", + "catalog_revision": "2026-07-14-host110-harbor-route-v3", + "catalog_playbook_path": ( + "infra/ansible/playbooks/110-devops.yml" + ), + "playbook_path": "infra/ansible/playbooks/110-devops.yml", + "inventory_hosts": ["host_110"], + "risk_level": "medium", + }, + } + ) + + assert claim is not None + assert claim.playbook_path.endswith("110-node-exporter-repair.yml") + assert claim.apply_playbook_path.endswith("110-node-exporter-repair.yml") + assert claim.input_payload["catalog_revision"] == ( + "2026-07-14-node-exporter-bounded-v4" + ) + assert claim.input_payload["controlled_apply_allowed"] is True diff --git a/apps/api/tests/test_host_pressure_ansible_catalog.py b/apps/api/tests/test_host_pressure_ansible_catalog.py index 2f6c26f45..3405ce81b 100644 --- a/apps/api/tests/test_host_pressure_ansible_catalog.py +++ b/apps/api/tests/test_host_pressure_ansible_catalog.py @@ -37,13 +37,12 @@ def test_host_pressure_alert_prefers_dedicated_no_write_catalog() -> None: candidates = hints["candidates"] assert candidates[0]["catalog_id"] == "ansible:110-host-pressure-readonly" - generic = next(row for row in candidates if row["catalog_id"] == "ansible:110-devops") - assert candidates[0]["match_score"] > generic["match_score"] + assert all(row["catalog_id"] != "ansible:110-devops" for row in candidates) assert candidates[0]["auto_apply_enabled"] is False assert candidates[0]["no_write_only"] is True -def test_generic_110_fallback_remains_below_exact_host_pressure_route() -> None: +def test_generic_110_fallback_is_not_used_for_host_pressure() -> None: hints = _catalog_hints( { "alertname": "Host110SustainedModeratePressure", @@ -53,16 +52,9 @@ def test_generic_110_fallback_remains_below_exact_host_pressure_route() -> None: ) candidates = hints["candidates"] - dedicated = next( - row - for row in candidates - if row["catalog_id"] == "ansible:110-host-pressure-readonly" - ) - generic = next( - row for row in candidates if row["catalog_id"] == "ansible:110-devops" - ) - assert dedicated["match_score"] > generic["match_score"] - assert generic["matched_keywords"] == ["110"] + assert [row["catalog_id"] for row in candidates] == [ + "ansible:110-host-pressure-readonly" + ] def test_host_pressure_playbook_is_bounded_and_read_only() -> None: diff --git a/infra/ansible/playbooks/110-node-exporter-repair.yml b/infra/ansible/playbooks/110-node-exporter-repair.yml new file mode 100644 index 000000000..c4e607101 --- /dev/null +++ b/infra/ansible/playbooks/110-node-exporter-repair.yml @@ -0,0 +1,102 @@ +--- +- name: Bounded host 110 node-exporter repair + hosts: host_110 + gather_facts: false + become: false + vars: + node_exporter_container: node-exporter + node_exporter_compose_dir: /home/wooo/monitoring + + tasks: + - name: Inspect node-exporter container state + ansible.builtin.command: + argv: + - docker + - inspect + - --format + - "{{ '{{' }}.State.Running{{ '}}' }}" + - "{{ node_exporter_container }}" + register: node_exporter_container_probe + check_mode: false + changed_when: false + failed_when: false + + - name: Probe local node-exporter metrics + ansible.builtin.command: + argv: + - curl + - -fsS + - --max-time + - "5" + - http://127.0.0.1:9100/metrics + register: node_exporter_metrics_probe + check_mode: false + changed_when: false + failed_when: false + + - name: Probe Prometheus scrape state + ansible.builtin.shell: + cmd: >- + curl -fsS --max-time 5 + 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' + | grep -Eq '"value":\[[^]]+,"1"\]' + executable: /bin/bash + register: node_exporter_prometheus_probe + check_mode: false + changed_when: false + failed_when: false + + - name: Decide whether bounded repair is required + ansible.builtin.set_fact: + node_exporter_repair_required: >- + {{ + node_exporter_container_probe.rc != 0 + or (node_exporter_container_probe.stdout | trim) != 'true' + or node_exporter_metrics_probe.rc != 0 + or node_exporter_prometheus_probe.rc != 0 + }} + + - name: Reconcile only the node-exporter service + ansible.builtin.command: + argv: + - docker + - compose + - up + - -d + - --force-recreate + - node-exporter + chdir: "{{ node_exporter_compose_dir }}" + when: node_exporter_repair_required | bool + changed_when: true + + - name: Verify local metrics after controlled apply + ansible.builtin.command: + argv: + - curl + - -fsS + - --retry + - "6" + - --retry-delay + - "5" + - --retry-all-errors + - --max-time + - "5" + - http://127.0.0.1:9100/metrics + check_mode: false + changed_when: false + when: not ansible_check_mode + + - name: Verify Prometheus sees node-exporter after controlled apply + ansible.builtin.shell: + cmd: >- + for attempt in 1 2 3 4 5 6; do + curl -fsS --max-time 5 + 'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' + | grep -Eq '"value":\[[^]]+,"1"\]' && exit 0; + sleep 5; + done; + exit 1 + executable: /bin/bash + check_mode: false + changed_when: false + when: not ansible_check_mode diff --git a/scripts/backup/backup-awoooi.sh b/scripts/backup/backup-awoooi.sh index 8bb9d9486..fc1799714 100755 --- a/scripts/backup/backup-awoooi.sh +++ b/scripts/backup/backup-awoooi.sh @@ -15,13 +15,58 @@ source "$(dirname "$0")/common.sh" SERVICE="awoooi" AWOOOI_HOST="192.168.0.188" AWOOOI_DB_USER="awoooi" -AWOOOI_DB_PASS="awoooi_prod_2026" +AWOOOI_DB_PASS="${AWOOOI_DB_PASS:-}" AWOOOI_DB_HOST="localhost" AWOOOI_DB_PORT="5432" K3S_DB_USER="postgres" LOCAL_REPO="${BACKUP_BASE}/awoooi" DUMP_DIR="/tmp/awoooi-backup-$$" +quote_remote() { + printf "%q" "$1" +} + +pgpass_escape() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//:/\\:}" + printf '%s' "${value}" +} + +pgpass_line() { + local database="$1" + printf '%s:%s:%s:%s:%s\n' \ + "$(pgpass_escape "${AWOOOI_DB_HOST}")" \ + "$(pgpass_escape "${AWOOOI_DB_PORT}")" \ + "$(pgpass_escape "${database}")" \ + "$(pgpass_escape "${AWOOOI_DB_USER}")" \ + "$(pgpass_escape "${AWOOOI_DB_PASS}")" +} + +remote_pgpass_wrapper() { + local command="$1" + printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGPASSFILE="$pgpass" %s' "${command}" +} + +run_remote_pg_dump() { + local database="$1" + local output_file="$2" + local stderr_file="${output_file}.stderr" + local command + + command="pg_dump --no-password -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") $(quote_remote "${database}")" + if pgpass_line "${database}" \ + | ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" \ + >"${output_file}" 2>"${stderr_file}"; then + rm -f "${stderr_file}" + return 0 + fi + + tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true + rm -f "${stderr_file}" + return 1 +} + # 保留策略覆寫(比其他服務更長) KEEP_DAILY=14 # 14 天每日 KEEP_WEEKLY=8 # 8 週每週 @@ -31,6 +76,12 @@ main() { local start_time=$(date +%s) local failed=0 + if [ -z "${AWOOOI_DB_PASS}" ]; then + log_error "AWOOOI_DB_PASS is unavailable; refusing an unauthenticated or hardcoded fallback" + notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable" + exit 1 + fi + log_info "========== 開始 AWOOOI 資料庫備份 ==========" mkdir -p "${DUMP_DIR}" @@ -38,9 +89,8 @@ main() { log_info "Dump awoooi_prod..." local timestamp=$(date "+%Y%m%d_%H%M%S") - if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \ - -U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \ - awoooi_prod" > "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" 2>&1; then + if run_remote_pg_dump \ + "awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1) log_success "awoooi_prod dump 完成 (${size})" else @@ -50,9 +100,8 @@ main() { # Step 2: awoooi_dev dump log_info "Dump awoooi_dev..." - if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \ - -U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \ - awoooi_dev 2>/dev/null" > "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" 2>/dev/null; then + if run_remote_pg_dump \ + "awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1) log_success "awoooi_dev dump 完成 (${size})" else @@ -61,9 +110,8 @@ main() { # Step 3: k3s_datastore dump(Kine 後端) log_info "Dump k3s_datastore..." - if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \ - -U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \ - k3s_datastore 2>/dev/null" > "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" 2>/dev/null; then + if run_remote_pg_dump \ + "k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1) log_success "k3s_datastore dump 完成 (${size})" else diff --git a/scripts/backup/tests/test_backup_awoooi_secret_contract.py b/scripts/backup/tests/test_backup_awoooi_secret_contract.py new file mode 100644 index 000000000..e513cb955 --- /dev/null +++ b/scripts/backup/tests/test_backup_awoooi_secret_contract.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "backup" / "backup-awoooi.sh" + + +def test_daily_backup_has_no_hardcoded_database_password() -> None: + text = SCRIPT.read_text(encoding="utf-8") + assignment = re.search(r"^AWOOOI_DB_PASS=(.+)$", text, re.MULTILINE) + + assert assignment is not None + assert assignment.group(1) == '"${AWOOOI_DB_PASS:-}"' + assert "PGPASSWORD='${AWOOOI_DB_PASS}'" not in text + assert 'PGPASSFILE="$pgpass"' in text + assert "pg_dump --no-password" in text + + +def test_daily_backup_fails_closed_without_runtime_credential() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert 'if [ -z "${AWOOOI_DB_PASS}" ]; then' in text + assert "refusing an unauthenticated or hardcoded fallback" in text