feat(sre): enforce typed controlled automation
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
ogt
2026-07-16 17:32:01 +08:00
parent dfec0c9f7f
commit 541a32a9cb
130 changed files with 14849 additions and 1738 deletions

View File

@@ -45,7 +45,41 @@ EXPECTED_JOBS = {
"prometheus": "Prometheus self-scrape",
"blackbox-http": "Blackbox HTTP probe",
"blackbox-tcp": "Blackbox TCP probe",
"github-actions": "GitHub Actions exporter",
"gitea-native": "Gitea native metrics",
}
# Jobs alone are not sufficient evidence for multi-target scrape jobs. A
# healthy website target can keep ``blackbox-http`` green while both AI
# provider probes are absent. Keep the exact cloud-edge identities here so
# source/runtime reconciliation fails closed on missing, down, or duplicated
# provider targets.
EXPECTED_TARGETS = {
"ollama_gcp_a_cloud_edge": {
"description": "GCP-A Ollama cloud-edge /api/tags from host110",
"labels": {
"job": "blackbox-http",
"provider": "ollama_gcp_a",
"probe_origin": "host110",
"boundary": "cloud_edge_availability_only",
"instance": "http://34.143.170.20:11434/api/tags",
},
},
"ollama_gcp_b_cloud_edge": {
"description": "GCP-B Ollama cloud-edge /api/tags from host110",
"labels": {
"job": "blackbox-http",
"provider": "ollama_gcp_b",
"probe_origin": "host110",
"boundary": "cloud_edge_availability_only",
"instance": "http://34.21.145.224:11434/api/tags",
},
},
}
# GitHub is globally frozen. A legacy exporter still appearing UP must be
# reported as policy drift, not counted as harmless surplus coverage.
FORBIDDEN_JOBS = {
"github-actions": "retired GitHub exporter must be removed from production",
}
# 允許 down 的 target (已知問題,不影響覆蓋率計算)
@@ -116,6 +150,42 @@ def analyze_targets(targets_data: dict) -> dict:
return jobs
def analyze_expected_targets(targets_data: dict) -> dict:
"""Resolve exact multi-target identities without allowing job-level false green."""
active = targets_data.get("activeTargets", [])
target_status: dict[str, dict] = {}
for target_id, expected in EXPECTED_TARGETS.items():
expected_labels = expected["labels"]
matches = [
target
for target in active
if all(
target.get("labels", {}).get(label) == value
for label, value in expected_labels.items()
)
]
health_values = [str(target.get("health", "unknown")) for target in matches]
if not matches:
state = "missing"
elif len(matches) != 1:
state = "ambiguous"
elif health_values == ["up"]:
state = "up"
else:
state = "down"
target_status[target_id] = {
"description": expected["description"],
"expected_labels": expected_labels,
"match_count": len(matches),
"health": health_values,
"state": state,
}
return target_status
def build_report(jobs: dict) -> dict:
"""建立覆蓋率報告"""
total_jobs = len(jobs)
@@ -132,6 +202,7 @@ def build_report(jobs: dict) -> dict:
expected_covered = sum(1 for j in EXPECTED_JOBS if j in jobs and jobs[j]["up"])
coverage_pct = round(expected_covered / len(EXPECTED_JOBS) * 100, 1)
forbidden_jobs_present = [job for job in FORBIDDEN_JOBS if job in jobs]
return {
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
@@ -143,23 +214,65 @@ def build_report(jobs: dict) -> dict:
"down_jobs": down_jobs,
"real_down_jobs": len(real_down_jobs),
"expected_coverage_pct": coverage_pct,
"forbidden_jobs_present": len(forbidden_jobs_present),
},
"jobs": jobs,
"expected_jobs": EXPECTED_JOBS,
"known_down": KNOWN_DOWN_TARGETS,
"real_down_jobs": list(real_down_jobs.keys()),
"missing_expected": [j for j in EXPECTED_JOBS if j not in jobs],
"forbidden_jobs": FORBIDDEN_JOBS,
"forbidden_jobs_present": forbidden_jobs_present,
}
def build_report_from_targets(targets_data: dict) -> dict:
"""從 Prometheus targets API payload 建立覆蓋率報告"""
return build_report(analyze_targets(targets_data))
report = build_report(analyze_targets(targets_data))
expected_targets = analyze_expected_targets(targets_data)
missing_expected_targets = [
target_id
for target_id, target in expected_targets.items()
if target["state"] == "missing"
]
down_expected_targets = [
target_id
for target_id, target in expected_targets.items()
if target["state"] == "down"
]
ambiguous_expected_targets = [
target_id
for target_id, target in expected_targets.items()
if target["state"] == "ambiguous"
]
report["expected_targets"] = expected_targets
report["missing_expected_targets"] = missing_expected_targets
report["down_expected_targets"] = down_expected_targets
report["ambiguous_expected_targets"] = ambiguous_expected_targets
report["summary"].update(
{
"expected_target_count": len(expected_targets),
"expected_targets_up": sum(
target["state"] == "up" for target in expected_targets.values()
),
"expected_targets_missing": len(missing_expected_targets),
"expected_targets_down": len(down_expected_targets),
"expected_targets_ambiguous": len(ambiguous_expected_targets),
}
)
return report
def report_needs_stabilization(report: dict) -> bool:
"""是否需要重查,避免 post-deploy 瞬間 scrape 狀態造成 false red."""
return bool(report["real_down_jobs"] or report["missing_expected"])
return bool(
report["real_down_jobs"]
or report["missing_expected"]
or report["missing_expected_targets"]
or report["down_expected_targets"]
or report["ambiguous_expected_targets"]
or report["forbidden_jobs_present"]
)
def stabilization_reason(report: dict) -> str:
@@ -168,6 +281,24 @@ def stabilization_reason(report: dict) -> str:
parts.append(f"real_down={','.join(report['real_down_jobs'])}")
if report["missing_expected"]:
parts.append(f"missing_expected={','.join(report['missing_expected'])}")
if report["missing_expected_targets"]:
parts.append(
"missing_expected_targets="
f"{','.join(report['missing_expected_targets'])}"
)
if report["down_expected_targets"]:
parts.append(
f"down_expected_targets={','.join(report['down_expected_targets'])}"
)
if report["ambiguous_expected_targets"]:
parts.append(
"ambiguous_expected_targets="
f"{','.join(report['ambiguous_expected_targets'])}"
)
if report["forbidden_jobs_present"]:
parts.append(
f"forbidden_jobs_present={','.join(report['forbidden_jobs_present'])}"
)
return "; ".join(parts) if parts else "stable"
@@ -226,18 +357,26 @@ def print_human_report(report: dict) -> None:
"""輸出人可讀格式報告"""
s = report["summary"]
print(f"\n{'='*60}")
print(f" AWOOOI 監控覆蓋率報告")
print(" AWOOOI 監控覆蓋率報告")
print(f" 生成時間: {report['generated_at']}")
print(f"{'='*60}")
print(f"\n📊 總覽")
print("\n📊 總覽")
print(f" Jobs 總數: {s['total_jobs']}")
print(f" 全部 UP: {s['up_jobs']}")
print(f" 部分 UP: {s['partial_jobs']}")
print(f" 全部 DOWN: {s['down_jobs']}")
print(f" 真實問題 (非已知): {s['real_down_jobs']}")
print(f" 預期覆蓋率: {s['expected_coverage_pct']}% ({COVERAGE_THRESHOLD}% 門檻)")
print(
" 精確 Target: "
f"{s['expected_targets_up']}/{s['expected_target_count']} UP "
f"(missing={s['expected_targets_missing']}, "
f"down={s['expected_targets_down']}, "
f"ambiguous={s['expected_targets_ambiguous']})"
)
print(f" 禁止 Job 漂移: {s['forbidden_jobs_present']}")
print(f"\n✅ 預期服務狀態")
print("\n✅ 預期服務狀態")
for job, desc in report["expected_jobs"].items():
jobs = report["jobs"]
if job not in jobs:
@@ -256,24 +395,42 @@ def print_human_report(report: dict) -> None:
if job in report["jobs"] and report["jobs"][job]["down"]
]
if known_down_present:
print(f"\n⚠️ 已知 DOWN (不影響覆蓋率)")
print("\n⚠️ 已知 DOWN (不影響覆蓋率)")
for job, reason in known_down_present:
print(f" {job:<30} {reason}")
if report["real_down_jobs"]:
print(f"\n🔴 需處理的 DOWN targets")
print("\n🔴 需處理的 DOWN targets")
for job in report["real_down_jobs"]:
instances = report["jobs"][job].get("down", [])
print(f" {job}: {', '.join(instances)}")
if report["missing_expected"]:
print(f"\n🔴 缺少預期服務監控")
print("\n🔴 缺少預期服務監控")
for job in report["missing_expected"]:
print(f" {job}: {report['expected_jobs'][job]}")
if report["forbidden_jobs_present"]:
print("\n🔴 禁止的 legacy monitoring job 仍存在")
for job in report["forbidden_jobs_present"]:
print(f" {job}: {report['forbidden_jobs'][job]}")
if (
report["missing_expected_targets"]
or report["down_expected_targets"]
or report["ambiguous_expected_targets"]
):
print("\n🔴 精確 Target identity/readback 未通過")
for target_id, target in report["expected_targets"].items():
if target["state"] != "up":
print(
f" {target_id}: {target['state']} "
f"matches={target['match_count']} {target['description']}"
)
stabilization = report.get("stabilization")
if stabilization and stabilization["attempt"] > 1:
print(f"\n⏱️ Prometheus target 穩定化")
print("\n⏱️ Prometheus target 穩定化")
print(
" "
f"{stabilization['status']} after "
@@ -282,7 +439,19 @@ def print_human_report(report: dict) -> None:
pct = s["expected_coverage_pct"]
threshold = COVERAGE_THRESHOLD
if pct >= threshold and not report["real_down_jobs"]:
exact_targets_healthy = not (
report["missing_expected_targets"]
or report["down_expected_targets"]
or report["ambiguous_expected_targets"]
)
monitoring_contract_healthy = (
exact_targets_healthy and not report["forbidden_jobs_present"]
)
if (
pct >= threshold
and not report["real_down_jobs"]
and monitoring_contract_healthy
):
print(f"\n✅ 監控健康: 覆蓋率 {pct}% >= {threshold}%,無真實問題\n")
elif pct >= threshold:
print(f"\n⚠️ 覆蓋率達標 ({pct}%),但有 {s['real_down_jobs']} 個真實 DOWN 需處理\n")
@@ -335,7 +504,21 @@ def main() -> None:
if args.check:
pct = report["summary"]["expected_coverage_pct"]
real_down = report["summary"]["real_down_jobs"]
if pct < COVERAGE_THRESHOLD or real_down > 0:
exact_target_failures = sum(
report["summary"][key]
for key in (
"expected_targets_missing",
"expected_targets_down",
"expected_targets_ambiguous",
)
)
forbidden_jobs = report["summary"]["forbidden_jobs_present"]
if (
pct < COVERAGE_THRESHOLD
or real_down > 0
or exact_target_failures > 0
or forbidden_jobs > 0
):
sys.exit(1)

View File

@@ -1,11 +1,12 @@
#!/usr/bin/env bash
# Render and deploy ops/alertmanager/alertmanager.yml to the 110 Docker Alertmanager.
#
# This script keeps the live direct-Telegram emergency route aligned with Git:
# This script keeps the direct-Telegram route aligned:
# - inject Telegram bot token and SRE group chat id from K8s secret or env
# - validate with amtool before touching the live config
# - back up the live file
# - keep the bind-mounted live file inode and readable permissions intact
# - create a 0600 rollback copy without a world-readable intermediate
# - keep the bind-mounted live file inode and protect it as runtime-UID 0400
# - verify container readability before reload; fail closed with secure rollback
# - reload Alertmanager with SIGHUP
#
# Usage:
@@ -105,14 +106,20 @@ target = Path(sys.argv[2])
text = template.read_text()
text = text.replace("TELEGRAM_BOT_TOKEN_PLACEHOLDER", os.environ["TELEGRAM_BOT_TOKEN"])
text = text.replace("SRE_GROUP_CHAT_ID_PLACEHOLDER", os.environ["SRE_GROUP_CHAT_ID"])
if "TELEGRAM_BOT_TOKEN_PLACEHOLDER" in text or "SRE_GROUP_CHAT_ID_PLACEHOLDER" in text:
if any(
placeholder in text
for placeholder in (
"TELEGRAM_BOT_TOKEN_PLACEHOLDER",
"SRE_GROUP_CHAT_ID_PLACEHOLDER",
)
):
raise SystemExit("unreplaced secret placeholder remains in rendered config")
target.write_text(text)
PY
log "Validating rendered config with live Alertmanager amtool on ${TARGET_HOST}"
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
"docker exec -i alertmanager sh -c 'cat >/tmp/alertmanager-rendered.yml && amtool check-config /tmp/alertmanager-rendered.yml'" \
"docker exec -i alertmanager amtool check-config /dev/stdin" \
< "$tmp_rendered"
if [[ "$DRY_RUN" == "--dry-run" ]]; then
@@ -127,18 +134,115 @@ ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" "bash -s" <<REMOTE
set -euo pipefail
target='${TARGET_PATH}'
staged='/tmp/alertmanager.yml.new'
container='alertmanager'
trap 'rm -f "\$staged"' EXIT
[[ -f "\$target" && ! -L "\$target" ]] || {
echo 'ERROR: live Alertmanager config must be a regular non-symlink file' >&2
exit 1
}
[[ -f "\$staged" && ! -L "\$staged" ]] || {
echo 'ERROR: staged Alertmanager config must be a regular non-symlink file' >&2
exit 1
}
sudo -n true
# Read the effective host UID/GID from the container process itself. This also
# works when Docker user namespaces remap container identities.
container_pid="\$(docker inspect --format '{{.State.Pid}}' "\$container")"
[[ "\$container_pid" =~ ^[1-9][0-9]*$ ]] || {
echo 'ERROR: Alertmanager container host PID is unavailable' >&2
exit 1
}
status_path="/proc/\${container_pid}/status"
[[ -r "\$status_path" ]] || {
echo 'ERROR: Alertmanager runtime identity cannot be verified' >&2
exit 1
}
runtime_uid="\$(awk '/^Uid:/{print \$2; exit}' "\$status_path")"
runtime_gid="\$(awk '/^Gid:/{print \$2; exit}' "\$status_path")"
[[ "\$runtime_uid" =~ ^[0-9]+$ && "\$runtime_gid" =~ ^[0-9]+$ ]] || {
echo 'ERROR: Alertmanager runtime UID/GID is invalid' >&2
exit 1
}
backup="\${target}.bak.\$(date +%Y%m%d%H%M%S)"
cp "\$target" "\$backup"
# Alertmanager bind-mounts a single file. Keep the existing inode instead of mv'ing
# a replacement over it, then restore readable permissions for the container user.
cat /tmp/alertmanager.yml.new > "\$target"
chmod 0644 "\$target"
rm -f /tmp/alertmanager.yml.new
docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml
docker kill -s HUP alertmanager >/dev/null
operator_uid="\$(id -u)"
operator_gid="\$(id -g)"
umask 077
# install applies 0600 while creating the copy; cp followed by chmod would leave
# a secret-bearing backup readable during the gap.
sudo -n install -m 0600 -o "\$operator_uid" -g "\$operator_gid" "\$target" "\$backup"
[[ "\$(stat -c '%a' "\$backup")" == '600' ]] || {
echo 'ERROR: secure Alertmanager rollback copy was not created' >&2
exit 1
}
# Harden legacy copies produced by older versions of this script without
# reading or printing their contents.
target_dir="\$(dirname "\$target")"
target_base="\$(basename "\$target")"
while IFS= read -r -d '' legacy_backup; do
sudo -n chmod 0600 "\$legacy_backup"
done < <(find "\$target_dir" -maxdepth 1 -type f -name "\${target_base}.bak.*" -print0)
reload_attempted=false
rollback_secure() {
local rc="\${1:-1}"
trap - ERR
set +e
if [[ -f "\$backup" ]]; then
# Remove access before restoring bytes so rollback cannot briefly recreate
# the legacy world-readable state.
sudo -n chmod 0000 "\$target"
sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
sudo -n chmod 0400 "\$target"
sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$backup" "\$target"
rollback_verified=false
if docker exec "\$container" sh -ec \
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' \
>/dev/null 2>&1; then
rollback_verified=true
fi
if [[ "\$reload_attempted" == 'true' && "\$rollback_verified" == 'true' ]]; then
docker kill -s HUP "\$container" >/dev/null 2>&1
fi
fi
rm -f "\$staged"
echo 'ERROR: Alertmanager deploy failed; secure rollback attempted' >&2
exit "\$rc"
}
trap 'rollback_secure \$?' ERR
# Alertmanager bind-mounts this single file. Preserve its inode and remove all
# access before changing ownership, so no secret bytes are ever written while
# legacy group/other read bits remain. Validate the permission model against the
# running container before replacing the old config bytes.
sudo -n chmod 0000 "\$target"
sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
sudo -n chmod 0400 "\$target"
[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
[[ "\$(stat -c '%a' "\$target")" == '400' ]]
docker exec "\$container" sh -ec \
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$staged" "\$target"
rm -f "\$staged"
[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
[[ "\$(stat -c '%a' "\$target")" == '400' ]]
docker exec "\$container" sh -ec \
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
reload_attempted=true
docker kill -s HUP "\$container" >/dev/null
sleep 2
docker inspect alertmanager --format 'status={{.State.Status}} started={{.State.StartedAt}}'
echo "backup=\$backup"
docker inspect "\$container" --format 'status={{.State.Status}} started={{.State.StartedAt}}'
[[ "\$(docker inspect "\$container" --format '{{.State.Status}}')" == 'running' ]]
trap - ERR
echo "backup=\$backup config_mode=0400 runtime_owner=\${runtime_uid}:\${runtime_gid}"
REMOTE
log "Alertmanager config deployed and reloaded"

View File

@@ -0,0 +1,413 @@
#!/usr/bin/env bash
# Bounded Prometheus scrape-source convergence for Alertmanager delivery truth.
set -euo pipefail
TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
RUN_ID="${RUN_ID:?RUN_ID is required}"
WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
ALERTMANAGER_METRICS_URL="${ALERTMANAGER_METRICS_URL:-http://127.0.0.1:9093/metrics}"
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
MODE="${1:---dry-run}"
case "${MODE}" in
apply|--dry-run) ;;
*) printf 'usage: %s [--dry-run|apply]\n' "$0" >&2; exit 64 ;;
esac
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
case "${value}" in
*[!A-Za-z0-9._-]*|'')
printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
exit 64
;;
esac
done
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
"${PROMETHEUS_URL}" "${ALERTMANAGER_METRICS_URL}" \
"${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
set -euo pipefail
TRACE_ID="$1"
RUN_ID="$2"
WORK_ITEM_ID="$3"
MODE="$4"
PROMETHEUS_URL="$5"
ALERTMANAGER_METRICS_URL="$6"
PROMETHEUS_CONTAINER="$7"
REMOTE_CONFIG="$8"
REMOTE_CANDIDATE="/tmp/awoooi-prometheus-alertmanager-runtime.${RUN_ID}.yml"
CONTAINER_CANDIDATE="/etc/prometheus/.awoooi-alertmanager-runtime.${RUN_ID}.yml"
BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
APPLY_STARTED=0
DEPLOY_VERIFIED=0
SOURCE_SHA=""
CANDIDATE_SHA=""
exec 9>/tmp/awoooi-prometheus-config.lock
if ! flock -n 9; then
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=prometheus_config_apply_in_progress\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2
exit 75
fi
cleanup() {
rm -f "${REMOTE_CANDIDATE}"
docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
rm -f "${CONTAINER_CANDIDATE}" >/dev/null 2>&1 || true
}
rollback() {
local rollback_rc=0
local current_sha backup_sha restored_sha
current_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
backup_sha="$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" || rollback_rc=1
if [ "${rollback_rc}" -eq 0 ] && [ "${current_sha}" = "${SOURCE_SHA}" ]; then
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=no_write_source_intact source_sha=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}"
return 0
fi
if [ "${rollback_rc}" -ne 0 ] \
|| [ "${current_sha}" != "${CANDIDATE_SHA}" ] \
|| [ "${backup_sha}" != "${SOURCE_SHA}" ]; then
rollback_rc=1
else
cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" || rollback_rc=1
restored_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
[ "${restored_sha}" = "${SOURCE_SHA}" ] || rollback_rc=1
fi
if [ "${rollback_rc}" -eq 0 ]; then
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null \
|| rollback_rc=1
fi
if [ "${rollback_rc}" -eq 0 ]; then
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null || rollback_rc=1
fi
if [ "${rollback_rc}" -eq 0 ]; then
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
return 0
fi
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed ready=false backup=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" >&2
return 1
}
on_exit() {
rc=$?
trap - EXIT
set +e
cleanup
rollback_rc=0
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
&& [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
rollback || rollback_rc=$?
fi
if [ "${rollback_rc}" -ne 0 ]; then
exit 70
fi
exit "${rc}"
}
trap on_exit EXIT
test -f "${REMOTE_CONFIG}"
test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")"
grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
from pathlib import Path
import re
import sys
source = Path(sys.argv[1])
candidate = Path(sys.argv[2])
text = source.read_text(encoding="utf-8")
begin = " # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
end = " # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
block = """ # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
- job_name: 'alertmanager-runtime'
scrape_interval: 30s
metrics_path: /metrics
static_configs:
- targets: ['alertmanager:9093']
labels:
host: '110'
service: 'alertmanager'
env: 'prod'
# END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
"""
if text.count(begin) != text.count(end) or text.count(begin) > 1:
raise SystemExit("alertmanager runtime managed block is ambiguous")
if begin in text:
pattern = re.escape(begin) + r".*?" + re.escape(end) + r"\n?"
text, count = re.subn(pattern, block, text, count=1, flags=re.S)
if count != 1:
raise SystemExit("managed block replacement failed")
else:
legacy_pattern = (
r"(?ms)^ - job_name: ['\"]alertmanager-runtime['\"]\n"
r".*?(?=^ - job_name:|\Z)"
)
legacy_matches = list(re.finditer(legacy_pattern, text))
if len(legacy_matches) == 1:
text = re.sub(legacy_pattern, block, text, count=1)
elif legacy_matches:
raise SystemExit("legacy alertmanager runtime job is ambiguous")
else:
anchors = (
" # === AWOOOI API Metrics",
" - job_name: 'awoooi-api'\n",
)
anchor = next((value for value in anchors if value in text), None)
if anchor is None or text.count(anchor) != 1:
raise SystemExit("awoooi-api insertion anchor missing or ambiguous")
text = text.replace(anchor, block + anchor, 1)
provider_begin = " # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES\n"
provider_end = " # END AWOOOI OLLAMA CLOUD EDGE PROBES\n"
provider_block = """ # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES
- targets:
- http://34.143.170.20:11434/api/tags
labels:
service: ollama-cloud-edge
provider: ollama_gcp_a
probe_origin: host110
boundary: cloud_edge_availability_only
- targets:
- http://34.21.145.224:11434/api/tags
labels:
service: ollama-cloud-edge
provider: ollama_gcp_b
probe_origin: host110
boundary: cloud_edge_availability_only
# END AWOOOI OLLAMA CLOUD EDGE PROBES
"""
if (
text.count(provider_begin) != text.count(provider_end)
or text.count(provider_begin) > 1
):
raise SystemExit("Ollama cloud edge managed block is ambiguous")
if provider_begin in text:
provider_pattern = (
re.escape(provider_begin)
+ r".*?"
+ re.escape(provider_end)
)
text, count = re.subn(
provider_pattern,
provider_block,
text,
count=1,
flags=re.S,
)
if count != 1:
raise SystemExit("Ollama cloud edge managed block replacement failed")
else:
blackbox_match = re.search(
r"(?ms)^ - job_name: ['\"]blackbox-http['\"]\n"
r".*?(?=^ - job_name:|\Z)",
text,
)
if blackbox_match is None:
raise SystemExit("blackbox-http job missing")
blackbox_job = blackbox_match.group(0)
if "provider: ollama_gcp_" in blackbox_job:
raise SystemExit("unmanaged Ollama cloud edge targets are ambiguous")
relabel_anchor = " relabel_configs:\n"
if blackbox_job.count(relabel_anchor) != 1:
raise SystemExit("blackbox-http relabel anchor missing or ambiguous")
patched_job = blackbox_job.replace(
relabel_anchor,
provider_block + relabel_anchor,
1,
)
text = text[: blackbox_match.start()] + patched_job + text[blackbox_match.end() :]
if text.count("job_name: 'alertmanager-runtime'") != 1:
raise SystemExit("alertmanager runtime job must exist exactly once")
if "targets: ['alertmanager:9093']" not in text:
raise SystemExit("canonical Alertmanager target missing")
if text.count("provider: ollama_gcp_a") != 1:
raise SystemExit("canonical GCP-A cloud edge target missing or ambiguous")
if text.count("provider: ollama_gcp_b") != 1:
raise SystemExit("canonical GCP-B cloud edge target missing or ambiguous")
if "192.168.0.111:11434" in text:
raise SystemExit("host111 must not be probed from retired host110 boundary")
candidate.write_text(text, encoding="utf-8")
PY
docker cp "${REMOTE_CANDIDATE}" \
"${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE}" >/dev/null
docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${CONTAINER_CANDIDATE}"
CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass persistent_writes_performed=0 temporary_validation_artifact=true\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
"${SOURCE_SHA}" "${CANDIDATE_SHA}"
if [ "${MODE}" = "--dry-run" ]; then
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_persistent_write\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
exit 0
fi
cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
test "$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
APPLY_STARTED=1
cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=alertmanager_runtime_scrape_converged active_sha=%s backup=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${CANDIDATE_SHA}" \
"${BACKUP_CONFIG}"
target_readback() {
curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
import json
import sys
targets = json.load(sys.stdin)["data"]["activeTargets"]
matches = [
target for target in targets
if target.get("labels", {}).get("job") == "alertmanager-runtime"
]
if len(matches) != 1:
print(f"{len(matches)}|missing|never")
else:
target = matches[0]
print("{}|{}|{}".format(
len(matches),
target.get("health", "unknown"),
target.get("lastScrape", "never"),
))
'
}
provider_target_readback() {
curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
import json
import sys
targets = json.load(sys.stdin)["data"]["activeTargets"]
providers = ("ollama_gcp_a", "ollama_gcp_b")
matches = {
provider: [
target
for target in targets
if target.get("labels", {}).get("job") == "blackbox-http"
and target.get("labels", {}).get("provider") == provider
and target.get("labels", {}).get("probe_origin") == "host110"
]
for provider in providers
}
ready = all(
len(matches[provider]) == 1
and matches[provider][0].get("lastScrape", "never") != "never"
for provider in providers
)
values = []
for provider in providers:
target = matches[provider][0] if len(matches[provider]) == 1 else {}
values.extend([
target.get("lastScrape", "never"),
target.get("health", "missing"),
])
print("|".join(["1" if ready else "0", *values]))
'
}
FIRST_SCRAPE=""
for attempt in $(seq 1 9); do
IFS='|' read -r count health last_scrape <<<"$(target_readback)"
printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
"${last_scrape}"
if [ "${count}" = 1 ] && [ "${health}" = up ] \
&& [ "${last_scrape}" != never ]; then
FIRST_SCRAPE="${last_scrape}"
break
fi
sleep 10
done
test -n "${FIRST_SCRAPE}"
FIRST_GCP_A_SCRAPE=""
FIRST_GCP_B_SCRAPE=""
for attempt in $(seq 1 9); do
IFS='|' read -r providers_ready gcp_a_scrape gcp_a_health \
gcp_b_scrape gcp_b_health <<<"$(provider_target_readback)"
printf 'provider_target_probe trace_id=%s run_id=%s attempt=%s ready=%s gcp_a_health=%s gcp_b_health=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${attempt}" "${providers_ready}" \
"${gcp_a_health}" "${gcp_b_health}"
if [ "${providers_ready}" = 1 ]; then
FIRST_GCP_A_SCRAPE="${gcp_a_scrape}"
FIRST_GCP_B_SCRAPE="${gcp_b_scrape}"
break
fi
sleep 10
done
test -n "${FIRST_GCP_A_SCRAPE}"
test -n "${FIRST_GCP_B_SCRAPE}"
sleep 31
IFS='|' read -r count health SECOND_SCRAPE <<<"$(target_readback)"
test "${count}" = 1
test "${health}" = up
test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
IFS='|' read -r providers_ready SECOND_GCP_A_SCRAPE gcp_a_health \
SECOND_GCP_B_SCRAPE gcp_b_health <<<"$(provider_target_readback)"
test "${providers_ready}" = 1
test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}"
test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}"
SERIES_COUNT="$(curl -fsS --get \
--data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
import json
import sys
print(len(json.load(sys.stdin)["data"]["result"]))
')"
test "${SERIES_COUNT}" = 1
FAILED_SERIES_COUNT="$(curl -fsS --get \
--data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
import json
import sys
print(len(json.load(sys.stdin)["data"]["result"]))
')"
test "${FAILED_SERIES_COUNT}" = 1
PROVIDER_SERIES_COUNT="$(curl -fsS --get \
--data-urlencode 'query=probe_success{job="blackbox-http",provider=~"ollama_gcp_a|ollama_gcp_b",probe_origin="host110"}' \
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
import json
import sys
print(len(json.load(sys.stdin)["data"]["result"]))
')"
test "${PROVIDER_SERIES_COUNT}" = 2
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
DEPLOY_VERIFIED=1
printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=two_scrapes_up first=%s second=%s delivery_series_count=%s failed_series_count=%s provider_series_count=%s gcp_a_health=%s gcp_b_health=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
"${SECOND_SCRAPE}" "${SERIES_COUNT}" "${FAILED_SERIES_COUNT}" \
"${PROVIDER_SERIES_COUNT}" "${gcp_a_health}" "${gcp_b_health}"
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed backup=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
REMOTE

View File

@@ -185,6 +185,16 @@ if [[ "$NO_ALERTS_QUERY" != *'source="alertmanager"'* ]]; then
fi
log "✅ NoAlertsReceived2Hours query 已限制 alertmanager 主鏈路"
ALERT_CHAIN_BROKEN_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'AlertChainBroken_Alertmanager'), ''))")
if [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_failed_total'* ]] \
|| [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_total'* ]] \
|| [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver="awoooi-webhook"'* ]] \
|| [[ "$ALERT_CHAIN_BROKEN_QUERY" == *'awoooi_webhook_requests_total'* ]]; then
echo "ERROR: AlertChainBroken_Alertmanager 未使用 primary awoooi-webhook monotonic delivery truth: ${ALERT_CHAIN_BROKEN_QUERY}"
exit 1
fi
log "✅ AlertChainBroken_Alertmanager 只使用 primary awoooi-webhook monotonic delivery truth"
SOURCE_PROVIDER_STALE_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'SourceProviderIngestionStale'), ''))")
if [[ "$SOURCE_PROVIDER_STALE_QUERY" != *'source=~"sentry|signoz"'* ]]; then
echo "ERROR: SourceProviderIngestionStale query 未限制 Sentry/SignOz provider freshness: ${SOURCE_PROVIDER_STALE_QUERY}"

99
scripts/ops/ollama111-fallback-proxy-diagnose.sh Executable file → Normal file
View File

@@ -1,78 +1,35 @@
#!/usr/bin/env bash
# Compatibility entrypoint retained for operators; the host110 proxy lane is
# retired. All probes use the canonical direct provider identities.
set -euo pipefail
# Read-only diagnosis for ADR-110 local fallback:
# API Pod -> 110:11437 Nginx proxy -> 111:11434 Ollama allowlist proxy.
GCP_A_URL="${GCP_A_URL:-http://34.143.170.20:11434}"
GCP_B_URL="${GCP_B_URL:-http://34.21.145.224:11434}"
HOST111_URL="${HOST111_URL:-http://192.168.0.111:11434}"
PUBLIC_STATUS_URL="${PUBLIC_STATUS_URL:-https://awoooi.wooo.work/api/v1/ai/status}"
CURL_TIMEOUT="${CURL_TIMEOUT:-8}"
NAMESPACE="${NAMESPACE:-awoooi-prod}"
DEPLOYMENT="${DEPLOYMENT:-awoooi-api}"
PROXY_HOST="${PROXY_HOST:-wooo@192.168.0.110}"
LOCAL_HOST="${LOCAL_HOST:-ollama-111-gpu}"
LOCAL_IP="${LOCAL_IP:-192.168.0.111}"
PROXY_PORT="${PROXY_PORT:-11437}"
LOCAL_PORT="${LOCAL_PORT:-11434}"
SSH_TIMEOUT="${SSH_TIMEOUT:-8}"
CURL_TIMEOUT="${CURL_TIMEOUT:-5}"
section() {
printf '\n== %s ==\n' "$1"
probe() {
local provider="$1"
local url="$2"
local code
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \
"${url}/api/tags" 2>/dev/null || true)"
case "${code}" in
200) printf 'provider=%s boundary=%s status=healthy http_code=200\n' \
"${provider}" "$3" ;;
*) printf 'provider=%s boundary=%s status=unavailable http_code=%s\n' \
"${provider}" "$3" "${code:-000}" ;;
esac
}
run_local() {
set +e
"$@"
local rc=$?
set -e
printf 'exit_code=%s\n' "$rc"
}
printf 'schema_version=ollama_direct_route_diagnosis_v1 writes_performed=0\n'
probe ollama_gcp_a "${GCP_A_URL}" cloud
probe ollama_gcp_b "${GCP_B_URL}" cloud
probe ollama_local "${HOST111_URL}" local
section "Production health provider chain"
run_local curl -sS -m 12 https://awoooi.wooo.work/api/v1/health
section "API pod view"
set +e
kubectl -n "${NAMESPACE}" exec -i "deploy/${DEPLOYMENT}" -- sh -lc "
set +e
echo OLLAMA_URL=\$OLLAMA_URL
echo OLLAMA_SECONDARY_URL=\$OLLAMA_SECONDARY_URL
echo OLLAMA_FALLBACK_URL=\$OLLAMA_FALLBACK_URL
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://192.168.0.110:${PROXY_PORT}/api/tags | head -60
"
printf 'exit_code=%s\n' "$?"
set -e
section "110 proxy upstream reachability"
run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${PROXY_HOST}" "
set +e
echo route:
ip route get ${LOCAL_IP}
echo neigh:
ip neigh show ${LOCAL_IP}
echo ping:
ping -c 3 -W 2 ${LOCAL_IP}
echo direct_111:
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://${LOCAL_IP}:${LOCAL_PORT}/api/tags | head -60
echo proxy_11437:
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${PROXY_PORT}/api/tags | head -60
echo nginx_recent_errors:
tail -20 /var/log/nginx/ollama-local-error.log 2>/dev/null || true
"
section "111 host direct SSH view"
run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${LOCAL_HOST}" "
set +e
hostname
date
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${LOCAL_PORT}/api/tags | head -60
launchctl print gui/501/com.momo.ollama111-allow-proxy 2>/dev/null | head -60 || true
pmset -g custom 2>/dev/null | sed -n '1,80p' || true
"
cat <<'EOF'
Interpretation:
- If 110 shows "ip neigh INCOMPLETE", "Destination Host Unreachable", or "No route to host",
the 111 host or LAN path is down. Do not restart API or change provider order.
- If 110 can reach 111 directly but 11437 returns 502, inspect/reload the 110 Nginx proxy.
- If 111 direct SSH works but 127.0.0.1:11434 fails, recover the 111 Ollama/allowlist LaunchAgent.
EOF
status_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \
"${PUBLIC_STATUS_URL}" 2>/dev/null || true)"
printf 'production_route_readback_http=%s\n' "${status_code:-000}"
printf '%s\n' \
'terminal=read_only provider_order=ollama_gcp_a,ollama_gcp_b,ollama_local,claude,gemini host110_transport_selected=false'

View File

@@ -0,0 +1,64 @@
#!/usr/bin/env python3
"""Execute the bounded five-lane SRE comparison and paid-provider canary."""
from __future__ import annotations
import argparse
import asyncio
import json
import re
import sys
from pathlib import Path
_REPO_ROOT = Path(__file__).resolve().parents[2]
_API_ROOT = _REPO_ROOT / "apps" / "api"
if str(_API_ROOT) not in sys.path:
sys.path.insert(0, str(_API_ROOT))
async def _run_validation(**kwargs):
from src.services.paid_provider_canary_validation import (
run_paid_provider_canary_validation,
)
return await run_paid_provider_canary_validation(**kwargs)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--run-ref", required=True)
parser.add_argument("--operator-id", default="codex-controlled-canary")
parser.add_argument("--authorization-ref", required=True)
args = parser.parse_args()
try:
receipt = asyncio.run(
_run_validation(
run_ref=args.run_ref,
operator_id=args.operator_id,
authorization_ref=args.authorization_ref,
)
)
except BaseException as exc:
error_code = str(exc)
if not re.fullmatch(r"[a-z][a-z0-9_]{0,119}", error_code):
error_code = f"canary_failed_{type(exc).__name__}"
print(
json.dumps(
{
"schema_version": "paid_provider_sre_canary_error_v1",
"status": "blocked_with_safe_next_action",
"error_code": error_code[:120],
"raw_prompt_persisted": False,
"raw_response_persisted": False,
"secret_value_returned": False,
},
sort_keys=True,
)
)
return 2
print(json.dumps(receipt, ensure_ascii=False, sort_keys=True))
return 0 if receipt["independent_verifier"]["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,38 @@
from __future__ import annotations
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-runtime-scrape.sh"
def test_controlled_scrape_apply_has_check_rollback_and_independent_verifier() -> None:
text = SCRIPT.read_text(encoding="utf-8")
assert 'TRACE_ID="${TRACE_ID:?TRACE_ID is required}"' in text
assert 'RUN_ID="${RUN_ID:?RUN_ID is required}"' in text
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"' in text
assert "promtool check config" in text
assert "writes_performed=0" in text
assert "rollback_receipt" in text
assert "terminal=rollback_failed ready=false" in text
assert "flock -n 9" in text
assert '!= "${CANDIDATE_SHA}"' in text
assert '!= "${SOURCE_SHA}"' in text
assert "alertmanager_notification_requests_total" in text
assert "alertmanager_notification_requests_failed_total" in text
assert 'receiver="awoooi-webhook"' in text
assert text.count('receiver="awoooi-webhook"') >= 4
assert "two_scrapes_up" in text
assert "delivery_series_count" in text
def test_controlled_scrape_apply_never_replaces_the_full_live_config() -> None:
text = SCRIPT.read_text(encoding="utf-8")
assert "BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH" in text
assert "job_name: 'alertmanager-runtime'" in text
assert "targets: ['alertmanager:9093']" in text
assert "legacy_pattern" in text
assert "k8s/monitoring/prometheus.yml" not in text

View File

@@ -7,6 +7,7 @@ import yaml
ROOT = Path(__file__).resolve().parents[3]
ALERTMANAGER_CONFIG = ROOT / "ops" / "alertmanager" / "alertmanager.yml"
DEPLOY_SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-config.sh"
def _config() -> dict:
@@ -37,3 +38,70 @@ def test_emergency_direct_telegram_stays_limited_to_alert_chain() -> None:
route_text = str(_config()["route"])
assert "AWOOOIApiDown" in route_text
assert 'severity="critical"' in route_text
def test_emergency_card_is_bounded_and_has_honest_automation_attribution() -> None:
receiver = _receiver("telegram-direct")
telegram = receiver["telegram_configs"][0]
message = telegram["message"]
assert "SRE BYPASS告警主鏈異常" in message
assert "Prometheus deterministic rule未呼叫 LLM" in message
assert "Executornone" in message
assert "未宣稱已修復" in message
assert "Verifiersource-specific delivery metric" in message
assert "故障主機" not in message
telegram_route = next(
route
for route in _config()["route"]["routes"]
if route["receiver"] == "telegram-direct"
)
assert telegram_route["repeat_interval"] == "2h"
def test_alertmanager_has_no_agent99_inbound_receiver_or_secret_first_hop() -> None:
config = _config()
assert all(
item.get("receiver") != "agent99-alert-chain-relay"
for item in config["route"]["routes"]
)
assert all(
receiver.get("name") != "agent99-alert-chain-relay"
for receiver in config["receivers"]
)
source = ALERTMANAGER_CONFIG.read_text(encoding="utf-8")
assert "192.168.0.99:8787" not in source
assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source
assert "authorization:" not in source
def test_alertmanager_deploy_does_not_inject_agent99_relay_secret() -> None:
source = DEPLOY_SCRIPT.read_text(encoding="utf-8")
assert "AGENT99_SRE_ALERT_RELAY_TOKEN" not in source
assert 'os.environ["AGENT99_SRE_RELAY_TOKEN"]' not in source
assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source
assert "unreplaced secret placeholder remains" in source
assert "set -x" not in source
def test_secret_bearing_alertmanager_config_and_backups_are_not_world_readable() -> None:
source = DEPLOY_SCRIPT.read_text(encoding="utf-8")
assert "install -m 0600" in source
assert 'chmod 0600 "\\$legacy_backup"' in source
assert 'chmod 0400 "\\$target"' in source
assert 'chmod 0000 "\\$target"' in source
assert "chmod 0644" not in source
assert "/proc/\\${container_pid}/status" in source
assert "runtime_uid" in source
assert "runtime_gid" in source
assert "test -r /etc/alertmanager/alertmanager.yml" in source
assert "rollback_secure" in source
assert "secure rollback attempted" in source
assert "amtool check-config /etc/alertmanager/alertmanager.yml" in source
assert "amtool check-config /dev/stdin" in source
assert "/tmp/alertmanager-rendered.yml" not in source
assert "trap 'rm -f \"\\$staged\"' EXIT" in source
assert source.index('chmod 0000 "\\$target"') < source.index(
"sudo -n sh -c 'cat \"\\$1\" > \"\\$2\"'"
)

View File

@@ -456,7 +456,7 @@ $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-
$decision = Get-AgentLivePreflightDecision `
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
-Manifest $manifest `
-ExpectedRuntimeFileCount 14 `
-ExpectedRuntimeFileCount 15 `
-TaskFailureCount $taskFailures.Count `
-RelayListenerCount $relayListeners.Count `
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `

View File

@@ -24,6 +24,7 @@ $FixedRuntimeFiles = @(
"agent99-control-plane.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
"agent99-sre-alert-inbox.ps1",
"agent99-sre-alert-relay.ps1",
"agent99-submit-request.ps1",
@@ -716,7 +717,7 @@ try {
}
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" }
if ([int]$envelope.expectedRuntimeFileCount -ne 15) { throw "invalid_expected_runtime_file_count" }
$traceId = [string]$envelope.traceId
$runId = [string]$envelope.runId
@@ -751,7 +752,7 @@ try {
$sourceManifest = [pscustomobject]@{
schemaVersion = "agent99_remote_source_manifest_v1"
sourceRevision = $sourceRevision
fileCount = 14
fileCount = 15
manifestSha256 = $manifestDigest
files = @($FixedRuntimeFiles | ForEach-Object {
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
@@ -788,7 +789,7 @@ try {
status = "check_ready"
mode = "check"
sourceRevision = $sourceRevision
expectedRuntimeFileCount = 14
expectedRuntimeFileCount = 15
manifestSha256 = $manifestDigest
plannedStagingPath = $stagePath
runtimeManifest = $currentManifest
@@ -875,7 +876,7 @@ try {
if (
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
[int]$existingSourceManifest.fileCount -ne 14 -or
[int]$existingSourceManifest.fileCount -ne 15 -or
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
) { throw "existing_staging_manifest_mismatch" }
} catch {
@@ -899,7 +900,7 @@ try {
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
$live.runtimeMatched -and
$live.sourceRevision -eq $sourceRevision -and
$live.fileCount -eq 14 -and
$live.fileCount -eq 15 -and
$live.mismatchCount -eq 0 -and
$liveLauncherContract.ok -and
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
@@ -1135,7 +1136,7 @@ try {
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 14 -and
$runtimeManifest.fileCount -eq 15 -and
$runtimeManifest.mismatchCount -eq 0 -and
$launcherContract.ok -and
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and

View File

@@ -19,6 +19,7 @@ RUNTIME_FILES=(
"agent99-control-plane.ps1"
"agent99-deploy.ps1"
"agent99-register-tasks.ps1"
"agent99-alertmanager-alertchain-poll.ps1"
"agent99-sre-alert-inbox.ps1"
"agent99-sre-alert-relay.ps1"
"agent99-submit-request.ps1"
@@ -234,7 +235,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
output_path = Path(sys.argv[7])
runtime_names = sys.argv[8:]
if len(runtime_names) != 14 or len(set(runtime_names)) != 14:
if len(runtime_names) != 15 or len(set(runtime_names)) != 15:
raise SystemExit("fixed_runtime_file_contract_failed")
files: list[dict[str, str]] = []
@@ -273,7 +274,7 @@ envelope = {
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": source_revision,
"expectedRuntimeFileCount": 14,
"expectedRuntimeFileCount": 15,
"manifestSha256": manifest_sha256,
"files": files,
"livePreflight": {
@@ -425,14 +426,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 14-and$m.mismatchCount-eq 0-and$rows.Count-eq 14-and$seen.Count-eq 14-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 15-and$m.mismatchCount-eq 0-and$rows.Count-eq 15-and$seen.Count-eq 15-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=14;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=15;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
if(-not$ready){exit 65}'''
script = (
script.replace("__SOURCE_REVISION__", source_revision)

View File

@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
"exists": True,
"sourceRevision": "a" * 40,
"runtimeMatched": True,
"fileCount": 14,
"fileCount": 15,
"mismatchCount": 0,
"parseError": "",
},
@@ -81,7 +81,7 @@ $case = @'
$parameters = @{{
AgentRootPresent = [bool]$case.agentRootPresent
Manifest = $case.manifest
ExpectedRuntimeFileCount = 14
ExpectedRuntimeFileCount = 15
TaskFailureCount = [int]$case.taskFailureCount
RelayListenerCount = [int]$case.relayListenerCount
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount

View File

@@ -25,6 +25,7 @@ EXPECTED_RUNTIME_FILES = (
"agent99-control-plane.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
"agent99-sre-alert-inbox.ps1",
"agent99-sre-alert-relay.ps1",
"agent99-submit-request.ps1",
@@ -77,7 +78,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None
assert sender_files == EXPECTED_RUNTIME_FILES
assert receiver_files == EXPECTED_RUNTIME_FILES
assert deployer_files == EXPECTED_RUNTIME_FILES
assert "expectedRuntimeFileCount\": 14" in sender
assert "expectedRuntimeFileCount\": 15" in sender
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
assert "required_runtime_file_missing" in receiver
assert "duplicate_runtime_file" in receiver
@@ -230,7 +231,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
assert '$runtimeManifest.fileCount -eq 14' in source
assert '$runtimeManifest.fileCount -eq 15' in source
assert '$runtimeManifest.mismatchCount -eq 0' in source
assert 'status = "deployed_verified"' in source
assert '$failurePhase = "deploy"' in source
@@ -699,8 +700,9 @@ def test_check_mode_uses_bounded_control_command_without_ssh_stdin(
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
"printf '%040d\\n' 0; exit 0 ;;\n"
" *\" cat-file -e \"*) exit 0 ;;\n"
" *\" diff --quiet \"*) exit 0 ;;\n"
" *\" cat-file -e \"*) exit 0 ;;\n"
" *\" ls-files --error-unmatch \"*) exit 0 ;;\n"
" *\" diff --quiet \"*) exit 0 ;;\n"
"esac\n"
f"exec {shlex.quote(real_git)} \"$@\"\n",
encoding="utf-8",
@@ -854,8 +856,9 @@ def test_apply_uses_ephemeral_file_transport_and_cleans_up_without_ssh_stdin(
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
"printf '%040d\\n' 0; exit 0 ;;\n"
" *\" cat-file -e \"*) exit 0 ;;\n"
" *\" diff --quiet \"*) exit 0 ;;\n"
" *\" cat-file -e \"*) exit 0 ;;\n"
" *\" ls-files --error-unmatch \"*) exit 0 ;;\n"
" *\" diff --quiet \"*) exit 0 ;;\n"
"esac\n"
f"exec {shlex.quote(real_git)} \"$@\"\n",
encoding="utf-8",

View File

@@ -189,19 +189,20 @@ SURFACES = [
"requires_live_evidence": True,
},
{
"surface_id": "ollama_proxy_gateway",
"label": "Ollama proxy gateway / local fallback boundary",
"expected_scope": "proxy route、local fallback、gateway health",
"config_kind": "proxy_gateway",
"surface_id": "ollama_provider_transport_boundary",
"label": "Ollama direct route / mesh transport / local fallback boundary",
"expected_scope": "GCP direct/mesh transport、host111 local boundary、provider health",
"config_kind": "provider_transport",
"source_refs": [
"infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2",
"ops/monitoring/service-registry.yaml",
"k8s/awoooi-prod/04-configmap.yaml",
"infra/ansible/playbooks/111-ollama-fallback.yml",
"apps/api/src/services/ollama_endpoint_resolver.py",
"apps/api/src/services/ollama_failover_manager.py",
"apps/api/src/services/provider_proxy.py",
],
"write_capable_surface": True,
"paid_provider_related": False,
"data_egress_related": False,
"data_egress_related": True,
"requires_live_evidence": True,
},
{

View File

@@ -337,7 +337,8 @@ CATEGORIES = [
"apps/api/src/services/ai_providers/**",
"apps/api/src/services/**/*model*",
"apps/api/src/services/**/*provider*",
"infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2",
"ops/monitoring/service-registry.yaml",
"k8s/awoooi-prod/04-configmap.yaml",
"docs/ai/**",
"docs/**/*Ollama*",
),

View File

@@ -53,15 +53,6 @@ SOURCES = [
control_tier="C0",
owner_gate="public_tools_owner_response_required",
),
NginxSource(
config_id="host110_ollama_proxy",
host="192.168.0.110",
role="ollama_proxy_gateway",
source_path="infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2",
live_path="/etc/nginx/sites-enabled/110-ollama-proxy.conf",
control_tier="C1",
owner_gate="ai_provider_proxy_owner_response_required",
),
]