541a32a9cb
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
249 lines
8.4 KiB
Bash
Executable File
249 lines
8.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Render and deploy ops/alertmanager/alertmanager.yml to the 110 Docker Alertmanager.
|
|
#
|
|
# This script keeps the direct-Telegram route aligned:
|
|
# - inject Telegram bot token and SRE group chat id from K8s secret or env
|
|
# - validate with amtool before touching the live config
|
|
# - create a 0600 rollback copy without a world-readable intermediate
|
|
# - keep the bind-mounted live file inode and protect it as runtime-UID 0400
|
|
# - verify container readability before reload; fail closed with secure rollback
|
|
# - reload Alertmanager with SIGHUP
|
|
#
|
|
# Usage:
|
|
# bash scripts/ops/deploy-alertmanager-config.sh [--dry-run]
|
|
#
|
|
# Optional env:
|
|
# TARGET_HOST=192.168.0.110
|
|
# TARGET_PATH=/home/wooo/monitoring/alertmanager.yml
|
|
# K8S_HOST=192.168.0.120
|
|
# K8S_NAMESPACE=awoooi-prod
|
|
# K8S_SECRET=awoooi-secrets
|
|
# TELEGRAM_BOT_TOKEN=...
|
|
# SRE_GROUP_CHAT_ID=...
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
CONFIG_TEMPLATE="${REPO_ROOT}/ops/alertmanager/alertmanager.yml"
|
|
|
|
TARGET_HOST="${TARGET_HOST:-192.168.0.110}"
|
|
TARGET_USER="${TARGET_USER:-wooo}"
|
|
TARGET_PATH="${TARGET_PATH:-/home/wooo/monitoring/alertmanager.yml}"
|
|
K8S_HOST="${K8S_HOST:-192.168.0.120}"
|
|
K8S_USER="${K8S_USER:-wooo}"
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-awoooi-prod}"
|
|
K8S_SECRET="${K8S_SECRET:-awoooi-secrets}"
|
|
DRY_RUN="${1:-}"
|
|
|
|
log() { printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*"; }
|
|
|
|
die() {
|
|
echo "ERROR: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
decode_b64() {
|
|
python3 -c 'import base64,sys; print(base64.b64decode(sys.stdin.read()).decode().strip())'
|
|
}
|
|
|
|
secret_key_b64() {
|
|
local key="$1"
|
|
ssh -o BatchMode=yes -o ConnectTimeout=8 "${K8S_USER}@${K8S_HOST}" \
|
|
"sudo -n kubectl -n '${K8S_NAMESPACE}' get secret '${K8S_SECRET}' -o jsonpath='{.data.${key}}'" 2>/dev/null
|
|
}
|
|
|
|
read_secret_first_available() {
|
|
local env_value="$1"
|
|
shift
|
|
if [[ -n "$env_value" ]]; then
|
|
printf '%s' "$env_value"
|
|
return 0
|
|
fi
|
|
|
|
local key raw
|
|
for key in "$@"; do
|
|
raw="$(secret_key_b64 "$key" || true)"
|
|
if [[ -n "$raw" ]]; then
|
|
printf '%s' "$raw" | decode_b64
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
[[ -f "$CONFIG_TEMPLATE" ]] || die "template not found: ${CONFIG_TEMPLATE}"
|
|
|
|
TELEGRAM_BOT_TOKEN="$(
|
|
read_secret_first_available \
|
|
"${TELEGRAM_BOT_TOKEN:-}" \
|
|
OPENCLAW_TG_BOT_TOKEN \
|
|
OPENCLAW_BOT_TOKEN \
|
|
TELEGRAM_BOT_TOKEN \
|
|
TG_BOT_TOKEN
|
|
)" || die "missing Telegram bot token; set TELEGRAM_BOT_TOKEN or add one of the known keys to ${K8S_SECRET}"
|
|
|
|
SRE_GROUP_CHAT_ID="$(
|
|
read_secret_first_available \
|
|
"${SRE_GROUP_CHAT_ID:-}" \
|
|
SRE_GROUP_CHAT_ID
|
|
)" || die "missing SRE_GROUP_CHAT_ID"
|
|
|
|
[[ "$SRE_GROUP_CHAT_ID" =~ ^-?[0-9]+$ ]] || die "SRE_GROUP_CHAT_ID must be a Telegram numeric chat id"
|
|
export TELEGRAM_BOT_TOKEN SRE_GROUP_CHAT_ID
|
|
|
|
tmp_rendered="$(mktemp)"
|
|
trap 'rm -f "$tmp_rendered"' EXIT
|
|
chmod 600 "$tmp_rendered"
|
|
|
|
python3 - "$CONFIG_TEMPLATE" "$tmp_rendered" <<'PY'
|
|
from pathlib import Path
|
|
import os
|
|
import sys
|
|
|
|
template = Path(sys.argv[1])
|
|
target = Path(sys.argv[2])
|
|
text = template.read_text()
|
|
text = text.replace("TELEGRAM_BOT_TOKEN_PLACEHOLDER", os.environ["TELEGRAM_BOT_TOKEN"])
|
|
text = text.replace("SRE_GROUP_CHAT_ID_PLACEHOLDER", os.environ["SRE_GROUP_CHAT_ID"])
|
|
if any(
|
|
placeholder in text
|
|
for placeholder in (
|
|
"TELEGRAM_BOT_TOKEN_PLACEHOLDER",
|
|
"SRE_GROUP_CHAT_ID_PLACEHOLDER",
|
|
)
|
|
):
|
|
raise SystemExit("unreplaced secret placeholder remains in rendered config")
|
|
target.write_text(text)
|
|
PY
|
|
|
|
log "Validating rendered config with live Alertmanager amtool on ${TARGET_HOST}"
|
|
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
|
|
"docker exec -i alertmanager amtool check-config /dev/stdin" \
|
|
< "$tmp_rendered"
|
|
|
|
if [[ "$DRY_RUN" == "--dry-run" ]]; then
|
|
log "DRY RUN: rendered config validated; not deploying"
|
|
exit 0
|
|
fi
|
|
|
|
log "Uploading rendered config to ${TARGET_HOST}:${TARGET_PATH}"
|
|
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
|
|
"umask 077 && cat > /tmp/alertmanager.yml.new" < "$tmp_rendered"
|
|
|
|
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" "bash -s" <<REMOTE
|
|
set -euo pipefail
|
|
target='${TARGET_PATH}'
|
|
staged='/tmp/alertmanager.yml.new'
|
|
container='alertmanager'
|
|
trap 'rm -f "\$staged"' EXIT
|
|
|
|
[[ -f "\$target" && ! -L "\$target" ]] || {
|
|
echo 'ERROR: live Alertmanager config must be a regular non-symlink file' >&2
|
|
exit 1
|
|
}
|
|
[[ -f "\$staged" && ! -L "\$staged" ]] || {
|
|
echo 'ERROR: staged Alertmanager config must be a regular non-symlink file' >&2
|
|
exit 1
|
|
}
|
|
sudo -n true
|
|
|
|
# Read the effective host UID/GID from the container process itself. This also
|
|
# works when Docker user namespaces remap container identities.
|
|
container_pid="\$(docker inspect --format '{{.State.Pid}}' "\$container")"
|
|
[[ "\$container_pid" =~ ^[1-9][0-9]*$ ]] || {
|
|
echo 'ERROR: Alertmanager container host PID is unavailable' >&2
|
|
exit 1
|
|
}
|
|
status_path="/proc/\${container_pid}/status"
|
|
[[ -r "\$status_path" ]] || {
|
|
echo 'ERROR: Alertmanager runtime identity cannot be verified' >&2
|
|
exit 1
|
|
}
|
|
runtime_uid="\$(awk '/^Uid:/{print \$2; exit}' "\$status_path")"
|
|
runtime_gid="\$(awk '/^Gid:/{print \$2; exit}' "\$status_path")"
|
|
[[ "\$runtime_uid" =~ ^[0-9]+$ && "\$runtime_gid" =~ ^[0-9]+$ ]] || {
|
|
echo 'ERROR: Alertmanager runtime UID/GID is invalid' >&2
|
|
exit 1
|
|
}
|
|
|
|
backup="\${target}.bak.\$(date +%Y%m%d%H%M%S)"
|
|
operator_uid="\$(id -u)"
|
|
operator_gid="\$(id -g)"
|
|
umask 077
|
|
# install applies 0600 while creating the copy; cp followed by chmod would leave
|
|
# a secret-bearing backup readable during the gap.
|
|
sudo -n install -m 0600 -o "\$operator_uid" -g "\$operator_gid" "\$target" "\$backup"
|
|
[[ "\$(stat -c '%a' "\$backup")" == '600' ]] || {
|
|
echo 'ERROR: secure Alertmanager rollback copy was not created' >&2
|
|
exit 1
|
|
}
|
|
|
|
# Harden legacy copies produced by older versions of this script without
|
|
# reading or printing their contents.
|
|
target_dir="\$(dirname "\$target")"
|
|
target_base="\$(basename "\$target")"
|
|
while IFS= read -r -d '' legacy_backup; do
|
|
sudo -n chmod 0600 "\$legacy_backup"
|
|
done < <(find "\$target_dir" -maxdepth 1 -type f -name "\${target_base}.bak.*" -print0)
|
|
|
|
reload_attempted=false
|
|
rollback_secure() {
|
|
local rc="\${1:-1}"
|
|
trap - ERR
|
|
set +e
|
|
if [[ -f "\$backup" ]]; then
|
|
# Remove access before restoring bytes so rollback cannot briefly recreate
|
|
# the legacy world-readable state.
|
|
sudo -n chmod 0000 "\$target"
|
|
sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
|
|
sudo -n chmod 0400 "\$target"
|
|
sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$backup" "\$target"
|
|
rollback_verified=false
|
|
if docker exec "\$container" sh -ec \
|
|
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' \
|
|
>/dev/null 2>&1; then
|
|
rollback_verified=true
|
|
fi
|
|
if [[ "\$reload_attempted" == 'true' && "\$rollback_verified" == 'true' ]]; then
|
|
docker kill -s HUP "\$container" >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
rm -f "\$staged"
|
|
echo 'ERROR: Alertmanager deploy failed; secure rollback attempted' >&2
|
|
exit "\$rc"
|
|
}
|
|
trap 'rollback_secure \$?' ERR
|
|
|
|
# Alertmanager bind-mounts this single file. Preserve its inode and remove all
|
|
# access before changing ownership, so no secret bytes are ever written while
|
|
# legacy group/other read bits remain. Validate the permission model against the
|
|
# running container before replacing the old config bytes.
|
|
sudo -n chmod 0000 "\$target"
|
|
sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
|
|
sudo -n chmod 0400 "\$target"
|
|
[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
|
|
[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
|
|
[[ "\$(stat -c '%a' "\$target")" == '400' ]]
|
|
docker exec "\$container" sh -ec \
|
|
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
|
|
|
|
sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$staged" "\$target"
|
|
rm -f "\$staged"
|
|
[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
|
|
[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
|
|
[[ "\$(stat -c '%a' "\$target")" == '400' ]]
|
|
docker exec "\$container" sh -ec \
|
|
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
|
|
|
|
reload_attempted=true
|
|
docker kill -s HUP "\$container" >/dev/null
|
|
sleep 2
|
|
docker inspect "\$container" --format 'status={{.State.Status}} started={{.State.StartedAt}}'
|
|
[[ "\$(docker inspect "\$container" --format '{{.State.Status}}')" == 'running' ]]
|
|
trap - ERR
|
|
echo "backup=\$backup config_mode=0400 runtime_owner=\${runtime_uid}:\${runtime_gid}"
|
|
REMOTE
|
|
|
|
log "Alertmanager config deployed and reloaded"
|