#!/usr/bin/env bash # Render and deploy ops/alertmanager/alertmanager.yml to the 110 Docker Alertmanager. # # This script keeps the direct-Telegram route aligned: # - inject Telegram bot token and SRE group chat id from K8s secret or env # - validate with amtool before touching the live config # - create a 0600 rollback copy without a world-readable intermediate # - keep the bind-mounted live file inode and protect it as runtime-UID 0400 # - verify container readability before reload; fail closed with secure rollback # - reload Alertmanager with SIGHUP # # Usage: # bash scripts/ops/deploy-alertmanager-config.sh [--dry-run] # # Optional env: # TARGET_HOST=192.168.0.110 # TARGET_PATH=/home/wooo/monitoring/alertmanager.yml # K8S_HOST=192.168.0.120 # K8S_NAMESPACE=awoooi-prod # K8S_SECRET=awoooi-secrets # TELEGRAM_BOT_TOKEN=... # SRE_GROUP_CHAT_ID=... set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" CONFIG_TEMPLATE="${REPO_ROOT}/ops/alertmanager/alertmanager.yml" TARGET_HOST="${TARGET_HOST:-192.168.0.110}" TARGET_USER="${TARGET_USER:-wooo}" TARGET_PATH="${TARGET_PATH:-/home/wooo/monitoring/alertmanager.yml}" K8S_HOST="${K8S_HOST:-192.168.0.120}" K8S_USER="${K8S_USER:-wooo}" K8S_NAMESPACE="${K8S_NAMESPACE:-awoooi-prod}" K8S_SECRET="${K8S_SECRET:-awoooi-secrets}" DRY_RUN="${1:-}" log() { printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*"; } die() { echo "ERROR: $*" >&2 exit 1 } decode_b64() { python3 -c 'import base64,sys; print(base64.b64decode(sys.stdin.read()).decode().strip())' } secret_key_b64() { local key="$1" ssh -o BatchMode=yes -o ConnectTimeout=8 "${K8S_USER}@${K8S_HOST}" \ "sudo -n kubectl -n '${K8S_NAMESPACE}' get secret '${K8S_SECRET}' -o jsonpath='{.data.${key}}'" 2>/dev/null } read_secret_first_available() { local env_value="$1" shift if [[ -n "$env_value" ]]; then printf '%s' "$env_value" return 0 fi local key raw for key in "$@"; do raw="$(secret_key_b64 "$key" || true)" if [[ -n "$raw" ]]; then printf '%s' "$raw" | decode_b64 return 0 fi done return 1 } [[ -f "$CONFIG_TEMPLATE" ]] || die "template not found: ${CONFIG_TEMPLATE}" TELEGRAM_BOT_TOKEN="$( read_secret_first_available \ "${TELEGRAM_BOT_TOKEN:-}" \ OPENCLAW_TG_BOT_TOKEN \ OPENCLAW_BOT_TOKEN \ TELEGRAM_BOT_TOKEN \ TG_BOT_TOKEN )" || die "missing Telegram bot token; set TELEGRAM_BOT_TOKEN or add one of the known keys to ${K8S_SECRET}" SRE_GROUP_CHAT_ID="$( read_secret_first_available \ "${SRE_GROUP_CHAT_ID:-}" \ SRE_GROUP_CHAT_ID )" || die "missing SRE_GROUP_CHAT_ID" [[ "$SRE_GROUP_CHAT_ID" =~ ^-?[0-9]+$ ]] || die "SRE_GROUP_CHAT_ID must be a Telegram numeric chat id" export TELEGRAM_BOT_TOKEN SRE_GROUP_CHAT_ID tmp_rendered="$(mktemp)" trap 'rm -f "$tmp_rendered"' EXIT chmod 600 "$tmp_rendered" python3 - "$CONFIG_TEMPLATE" "$tmp_rendered" <<'PY' from pathlib import Path import os import sys template = Path(sys.argv[1]) target = Path(sys.argv[2]) text = template.read_text() text = text.replace("TELEGRAM_BOT_TOKEN_PLACEHOLDER", os.environ["TELEGRAM_BOT_TOKEN"]) text = text.replace("SRE_GROUP_CHAT_ID_PLACEHOLDER", os.environ["SRE_GROUP_CHAT_ID"]) if any( placeholder in text for placeholder in ( "TELEGRAM_BOT_TOKEN_PLACEHOLDER", "SRE_GROUP_CHAT_ID_PLACEHOLDER", ) ): raise SystemExit("unreplaced secret placeholder remains in rendered config") target.write_text(text) PY log "Validating rendered config with live Alertmanager amtool on ${TARGET_HOST}" ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \ "docker exec -i alertmanager amtool check-config /dev/stdin" \ < "$tmp_rendered" if [[ "$DRY_RUN" == "--dry-run" ]]; then log "DRY RUN: rendered config validated; not deploying" exit 0 fi log "Uploading rendered config to ${TARGET_HOST}:${TARGET_PATH}" ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \ "umask 077 && cat > /tmp/alertmanager.yml.new" < "$tmp_rendered" ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" "bash -s" <&2 exit 1 } [[ -f "\$staged" && ! -L "\$staged" ]] || { echo 'ERROR: staged Alertmanager config must be a regular non-symlink file' >&2 exit 1 } sudo -n true # Read the effective host UID/GID from the container process itself. This also # works when Docker user namespaces remap container identities. container_pid="\$(docker inspect --format '{{.State.Pid}}' "\$container")" [[ "\$container_pid" =~ ^[1-9][0-9]*$ ]] || { echo 'ERROR: Alertmanager container host PID is unavailable' >&2 exit 1 } status_path="/proc/\${container_pid}/status" [[ -r "\$status_path" ]] || { echo 'ERROR: Alertmanager runtime identity cannot be verified' >&2 exit 1 } runtime_uid="\$(awk '/^Uid:/{print \$2; exit}' "\$status_path")" runtime_gid="\$(awk '/^Gid:/{print \$2; exit}' "\$status_path")" [[ "\$runtime_uid" =~ ^[0-9]+$ && "\$runtime_gid" =~ ^[0-9]+$ ]] || { echo 'ERROR: Alertmanager runtime UID/GID is invalid' >&2 exit 1 } backup="\${target}.bak.\$(date +%Y%m%d%H%M%S)" operator_uid="\$(id -u)" operator_gid="\$(id -g)" umask 077 # install applies 0600 while creating the copy; cp followed by chmod would leave # a secret-bearing backup readable during the gap. sudo -n install -m 0600 -o "\$operator_uid" -g "\$operator_gid" "\$target" "\$backup" [[ "\$(stat -c '%a' "\$backup")" == '600' ]] || { echo 'ERROR: secure Alertmanager rollback copy was not created' >&2 exit 1 } # Harden legacy copies produced by older versions of this script without # reading or printing their contents. target_dir="\$(dirname "\$target")" target_base="\$(basename "\$target")" while IFS= read -r -d '' legacy_backup; do sudo -n chmod 0600 "\$legacy_backup" done < <(find "\$target_dir" -maxdepth 1 -type f -name "\${target_base}.bak.*" -print0) reload_attempted=false rollback_secure() { local rc="\${1:-1}" trap - ERR set +e if [[ -f "\$backup" ]]; then # Remove access before restoring bytes so rollback cannot briefly recreate # the legacy world-readable state. sudo -n chmod 0000 "\$target" sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target" sudo -n chmod 0400 "\$target" sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$backup" "\$target" rollback_verified=false if docker exec "\$container" sh -ec \ 'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' \ >/dev/null 2>&1; then rollback_verified=true fi if [[ "\$reload_attempted" == 'true' && "\$rollback_verified" == 'true' ]]; then docker kill -s HUP "\$container" >/dev/null 2>&1 fi fi rm -f "\$staged" echo 'ERROR: Alertmanager deploy failed; secure rollback attempted' >&2 exit "\$rc" } trap 'rollback_secure \$?' ERR # Alertmanager bind-mounts this single file. Preserve its inode and remove all # access before changing ownership, so no secret bytes are ever written while # legacy group/other read bits remain. Validate the permission model against the # running container before replacing the old config bytes. sudo -n chmod 0000 "\$target" sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target" sudo -n chmod 0400 "\$target" [[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]] [[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]] [[ "\$(stat -c '%a' "\$target")" == '400' ]] docker exec "\$container" sh -ec \ 'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$staged" "\$target" rm -f "\$staged" [[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]] [[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]] [[ "\$(stat -c '%a' "\$target")" == '400' ]] docker exec "\$container" sh -ec \ 'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' reload_attempted=true docker kill -s HUP "\$container" >/dev/null sleep 2 docker inspect "\$container" --format 'status={{.State.Status}} started={{.State.StartedAt}}' [[ "\$(docker inspect "\$container" --format '{{.State.Status}}')" == 'running' ]] trap - ERR echo "backup=\$backup config_mode=0400 runtime_owner=\${runtime_uid}:\${runtime_gid}" REMOTE log "Alertmanager config deployed and reloaded"