feat(sre): enforce typed controlled automation
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -45,7 +45,41 @@ EXPECTED_JOBS = {
|
||||
"prometheus": "Prometheus self-scrape",
|
||||
"blackbox-http": "Blackbox HTTP probe",
|
||||
"blackbox-tcp": "Blackbox TCP probe",
|
||||
"github-actions": "GitHub Actions exporter",
|
||||
"gitea-native": "Gitea native metrics",
|
||||
}
|
||||
|
||||
# Jobs alone are not sufficient evidence for multi-target scrape jobs. A
|
||||
# healthy website target can keep ``blackbox-http`` green while both AI
|
||||
# provider probes are absent. Keep the exact cloud-edge identities here so
|
||||
# source/runtime reconciliation fails closed on missing, down, or duplicated
|
||||
# provider targets.
|
||||
EXPECTED_TARGETS = {
|
||||
"ollama_gcp_a_cloud_edge": {
|
||||
"description": "GCP-A Ollama cloud-edge /api/tags from host110",
|
||||
"labels": {
|
||||
"job": "blackbox-http",
|
||||
"provider": "ollama_gcp_a",
|
||||
"probe_origin": "host110",
|
||||
"boundary": "cloud_edge_availability_only",
|
||||
"instance": "http://34.143.170.20:11434/api/tags",
|
||||
},
|
||||
},
|
||||
"ollama_gcp_b_cloud_edge": {
|
||||
"description": "GCP-B Ollama cloud-edge /api/tags from host110",
|
||||
"labels": {
|
||||
"job": "blackbox-http",
|
||||
"provider": "ollama_gcp_b",
|
||||
"probe_origin": "host110",
|
||||
"boundary": "cloud_edge_availability_only",
|
||||
"instance": "http://34.21.145.224:11434/api/tags",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# GitHub is globally frozen. A legacy exporter still appearing UP must be
|
||||
# reported as policy drift, not counted as harmless surplus coverage.
|
||||
FORBIDDEN_JOBS = {
|
||||
"github-actions": "retired GitHub exporter must be removed from production",
|
||||
}
|
||||
|
||||
# 允許 down 的 target (已知問題,不影響覆蓋率計算)
|
||||
@@ -116,6 +150,42 @@ def analyze_targets(targets_data: dict) -> dict:
|
||||
return jobs
|
||||
|
||||
|
||||
def analyze_expected_targets(targets_data: dict) -> dict:
|
||||
"""Resolve exact multi-target identities without allowing job-level false green."""
|
||||
active = targets_data.get("activeTargets", [])
|
||||
target_status: dict[str, dict] = {}
|
||||
|
||||
for target_id, expected in EXPECTED_TARGETS.items():
|
||||
expected_labels = expected["labels"]
|
||||
matches = [
|
||||
target
|
||||
for target in active
|
||||
if all(
|
||||
target.get("labels", {}).get(label) == value
|
||||
for label, value in expected_labels.items()
|
||||
)
|
||||
]
|
||||
health_values = [str(target.get("health", "unknown")) for target in matches]
|
||||
if not matches:
|
||||
state = "missing"
|
||||
elif len(matches) != 1:
|
||||
state = "ambiguous"
|
||||
elif health_values == ["up"]:
|
||||
state = "up"
|
||||
else:
|
||||
state = "down"
|
||||
|
||||
target_status[target_id] = {
|
||||
"description": expected["description"],
|
||||
"expected_labels": expected_labels,
|
||||
"match_count": len(matches),
|
||||
"health": health_values,
|
||||
"state": state,
|
||||
}
|
||||
|
||||
return target_status
|
||||
|
||||
|
||||
def build_report(jobs: dict) -> dict:
|
||||
"""建立覆蓋率報告"""
|
||||
total_jobs = len(jobs)
|
||||
@@ -132,6 +202,7 @@ def build_report(jobs: dict) -> dict:
|
||||
|
||||
expected_covered = sum(1 for j in EXPECTED_JOBS if j in jobs and jobs[j]["up"])
|
||||
coverage_pct = round(expected_covered / len(EXPECTED_JOBS) * 100, 1)
|
||||
forbidden_jobs_present = [job for job in FORBIDDEN_JOBS if job in jobs]
|
||||
|
||||
return {
|
||||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
@@ -143,23 +214,65 @@ def build_report(jobs: dict) -> dict:
|
||||
"down_jobs": down_jobs,
|
||||
"real_down_jobs": len(real_down_jobs),
|
||||
"expected_coverage_pct": coverage_pct,
|
||||
"forbidden_jobs_present": len(forbidden_jobs_present),
|
||||
},
|
||||
"jobs": jobs,
|
||||
"expected_jobs": EXPECTED_JOBS,
|
||||
"known_down": KNOWN_DOWN_TARGETS,
|
||||
"real_down_jobs": list(real_down_jobs.keys()),
|
||||
"missing_expected": [j for j in EXPECTED_JOBS if j not in jobs],
|
||||
"forbidden_jobs": FORBIDDEN_JOBS,
|
||||
"forbidden_jobs_present": forbidden_jobs_present,
|
||||
}
|
||||
|
||||
|
||||
def build_report_from_targets(targets_data: dict) -> dict:
|
||||
"""從 Prometheus targets API payload 建立覆蓋率報告"""
|
||||
return build_report(analyze_targets(targets_data))
|
||||
report = build_report(analyze_targets(targets_data))
|
||||
expected_targets = analyze_expected_targets(targets_data)
|
||||
missing_expected_targets = [
|
||||
target_id
|
||||
for target_id, target in expected_targets.items()
|
||||
if target["state"] == "missing"
|
||||
]
|
||||
down_expected_targets = [
|
||||
target_id
|
||||
for target_id, target in expected_targets.items()
|
||||
if target["state"] == "down"
|
||||
]
|
||||
ambiguous_expected_targets = [
|
||||
target_id
|
||||
for target_id, target in expected_targets.items()
|
||||
if target["state"] == "ambiguous"
|
||||
]
|
||||
report["expected_targets"] = expected_targets
|
||||
report["missing_expected_targets"] = missing_expected_targets
|
||||
report["down_expected_targets"] = down_expected_targets
|
||||
report["ambiguous_expected_targets"] = ambiguous_expected_targets
|
||||
report["summary"].update(
|
||||
{
|
||||
"expected_target_count": len(expected_targets),
|
||||
"expected_targets_up": sum(
|
||||
target["state"] == "up" for target in expected_targets.values()
|
||||
),
|
||||
"expected_targets_missing": len(missing_expected_targets),
|
||||
"expected_targets_down": len(down_expected_targets),
|
||||
"expected_targets_ambiguous": len(ambiguous_expected_targets),
|
||||
}
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def report_needs_stabilization(report: dict) -> bool:
|
||||
"""是否需要重查,避免 post-deploy 瞬間 scrape 狀態造成 false red."""
|
||||
return bool(report["real_down_jobs"] or report["missing_expected"])
|
||||
return bool(
|
||||
report["real_down_jobs"]
|
||||
or report["missing_expected"]
|
||||
or report["missing_expected_targets"]
|
||||
or report["down_expected_targets"]
|
||||
or report["ambiguous_expected_targets"]
|
||||
or report["forbidden_jobs_present"]
|
||||
)
|
||||
|
||||
|
||||
def stabilization_reason(report: dict) -> str:
|
||||
@@ -168,6 +281,24 @@ def stabilization_reason(report: dict) -> str:
|
||||
parts.append(f"real_down={','.join(report['real_down_jobs'])}")
|
||||
if report["missing_expected"]:
|
||||
parts.append(f"missing_expected={','.join(report['missing_expected'])}")
|
||||
if report["missing_expected_targets"]:
|
||||
parts.append(
|
||||
"missing_expected_targets="
|
||||
f"{','.join(report['missing_expected_targets'])}"
|
||||
)
|
||||
if report["down_expected_targets"]:
|
||||
parts.append(
|
||||
f"down_expected_targets={','.join(report['down_expected_targets'])}"
|
||||
)
|
||||
if report["ambiguous_expected_targets"]:
|
||||
parts.append(
|
||||
"ambiguous_expected_targets="
|
||||
f"{','.join(report['ambiguous_expected_targets'])}"
|
||||
)
|
||||
if report["forbidden_jobs_present"]:
|
||||
parts.append(
|
||||
f"forbidden_jobs_present={','.join(report['forbidden_jobs_present'])}"
|
||||
)
|
||||
return "; ".join(parts) if parts else "stable"
|
||||
|
||||
|
||||
@@ -226,18 +357,26 @@ def print_human_report(report: dict) -> None:
|
||||
"""輸出人可讀格式報告"""
|
||||
s = report["summary"]
|
||||
print(f"\n{'='*60}")
|
||||
print(f" AWOOOI 監控覆蓋率報告")
|
||||
print(" AWOOOI 監控覆蓋率報告")
|
||||
print(f" 生成時間: {report['generated_at']}")
|
||||
print(f"{'='*60}")
|
||||
print(f"\n📊 總覽")
|
||||
print("\n📊 總覽")
|
||||
print(f" Jobs 總數: {s['total_jobs']}")
|
||||
print(f" 全部 UP: {s['up_jobs']}")
|
||||
print(f" 部分 UP: {s['partial_jobs']}")
|
||||
print(f" 全部 DOWN: {s['down_jobs']}")
|
||||
print(f" 真實問題 (非已知): {s['real_down_jobs']}")
|
||||
print(f" 預期覆蓋率: {s['expected_coverage_pct']}% ({COVERAGE_THRESHOLD}% 門檻)")
|
||||
print(
|
||||
" 精確 Target: "
|
||||
f"{s['expected_targets_up']}/{s['expected_target_count']} UP "
|
||||
f"(missing={s['expected_targets_missing']}, "
|
||||
f"down={s['expected_targets_down']}, "
|
||||
f"ambiguous={s['expected_targets_ambiguous']})"
|
||||
)
|
||||
print(f" 禁止 Job 漂移: {s['forbidden_jobs_present']}")
|
||||
|
||||
print(f"\n✅ 預期服務狀態")
|
||||
print("\n✅ 預期服務狀態")
|
||||
for job, desc in report["expected_jobs"].items():
|
||||
jobs = report["jobs"]
|
||||
if job not in jobs:
|
||||
@@ -256,24 +395,42 @@ def print_human_report(report: dict) -> None:
|
||||
if job in report["jobs"] and report["jobs"][job]["down"]
|
||||
]
|
||||
if known_down_present:
|
||||
print(f"\n⚠️ 已知 DOWN (不影響覆蓋率)")
|
||||
print("\n⚠️ 已知 DOWN (不影響覆蓋率)")
|
||||
for job, reason in known_down_present:
|
||||
print(f" {job:<30} {reason}")
|
||||
|
||||
if report["real_down_jobs"]:
|
||||
print(f"\n🔴 需處理的 DOWN targets")
|
||||
print("\n🔴 需處理的 DOWN targets")
|
||||
for job in report["real_down_jobs"]:
|
||||
instances = report["jobs"][job].get("down", [])
|
||||
print(f" {job}: {', '.join(instances)}")
|
||||
|
||||
if report["missing_expected"]:
|
||||
print(f"\n🔴 缺少預期服務監控")
|
||||
print("\n🔴 缺少預期服務監控")
|
||||
for job in report["missing_expected"]:
|
||||
print(f" {job}: {report['expected_jobs'][job]}")
|
||||
|
||||
if report["forbidden_jobs_present"]:
|
||||
print("\n🔴 禁止的 legacy monitoring job 仍存在")
|
||||
for job in report["forbidden_jobs_present"]:
|
||||
print(f" {job}: {report['forbidden_jobs'][job]}")
|
||||
|
||||
if (
|
||||
report["missing_expected_targets"]
|
||||
or report["down_expected_targets"]
|
||||
or report["ambiguous_expected_targets"]
|
||||
):
|
||||
print("\n🔴 精確 Target identity/readback 未通過")
|
||||
for target_id, target in report["expected_targets"].items():
|
||||
if target["state"] != "up":
|
||||
print(
|
||||
f" {target_id}: {target['state']} "
|
||||
f"matches={target['match_count']} {target['description']}"
|
||||
)
|
||||
|
||||
stabilization = report.get("stabilization")
|
||||
if stabilization and stabilization["attempt"] > 1:
|
||||
print(f"\n⏱️ Prometheus target 穩定化")
|
||||
print("\n⏱️ Prometheus target 穩定化")
|
||||
print(
|
||||
" "
|
||||
f"{stabilization['status']} after "
|
||||
@@ -282,7 +439,19 @@ def print_human_report(report: dict) -> None:
|
||||
|
||||
pct = s["expected_coverage_pct"]
|
||||
threshold = COVERAGE_THRESHOLD
|
||||
if pct >= threshold and not report["real_down_jobs"]:
|
||||
exact_targets_healthy = not (
|
||||
report["missing_expected_targets"]
|
||||
or report["down_expected_targets"]
|
||||
or report["ambiguous_expected_targets"]
|
||||
)
|
||||
monitoring_contract_healthy = (
|
||||
exact_targets_healthy and not report["forbidden_jobs_present"]
|
||||
)
|
||||
if (
|
||||
pct >= threshold
|
||||
and not report["real_down_jobs"]
|
||||
and monitoring_contract_healthy
|
||||
):
|
||||
print(f"\n✅ 監控健康: 覆蓋率 {pct}% >= {threshold}%,無真實問題\n")
|
||||
elif pct >= threshold:
|
||||
print(f"\n⚠️ 覆蓋率達標 ({pct}%),但有 {s['real_down_jobs']} 個真實 DOWN 需處理\n")
|
||||
@@ -335,7 +504,21 @@ def main() -> None:
|
||||
if args.check:
|
||||
pct = report["summary"]["expected_coverage_pct"]
|
||||
real_down = report["summary"]["real_down_jobs"]
|
||||
if pct < COVERAGE_THRESHOLD or real_down > 0:
|
||||
exact_target_failures = sum(
|
||||
report["summary"][key]
|
||||
for key in (
|
||||
"expected_targets_missing",
|
||||
"expected_targets_down",
|
||||
"expected_targets_ambiguous",
|
||||
)
|
||||
)
|
||||
forbidden_jobs = report["summary"]["forbidden_jobs_present"]
|
||||
if (
|
||||
pct < COVERAGE_THRESHOLD
|
||||
or real_down > 0
|
||||
or exact_target_failures > 0
|
||||
or forbidden_jobs > 0
|
||||
):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Render and deploy ops/alertmanager/alertmanager.yml to the 110 Docker Alertmanager.
|
||||
#
|
||||
# This script keeps the live direct-Telegram emergency route aligned with Git:
|
||||
# This script keeps the direct-Telegram route aligned:
|
||||
# - inject Telegram bot token and SRE group chat id from K8s secret or env
|
||||
# - validate with amtool before touching the live config
|
||||
# - back up the live file
|
||||
# - keep the bind-mounted live file inode and readable permissions intact
|
||||
# - create a 0600 rollback copy without a world-readable intermediate
|
||||
# - keep the bind-mounted live file inode and protect it as runtime-UID 0400
|
||||
# - verify container readability before reload; fail closed with secure rollback
|
||||
# - reload Alertmanager with SIGHUP
|
||||
#
|
||||
# Usage:
|
||||
@@ -105,14 +106,20 @@ target = Path(sys.argv[2])
|
||||
text = template.read_text()
|
||||
text = text.replace("TELEGRAM_BOT_TOKEN_PLACEHOLDER", os.environ["TELEGRAM_BOT_TOKEN"])
|
||||
text = text.replace("SRE_GROUP_CHAT_ID_PLACEHOLDER", os.environ["SRE_GROUP_CHAT_ID"])
|
||||
if "TELEGRAM_BOT_TOKEN_PLACEHOLDER" in text or "SRE_GROUP_CHAT_ID_PLACEHOLDER" in text:
|
||||
if any(
|
||||
placeholder in text
|
||||
for placeholder in (
|
||||
"TELEGRAM_BOT_TOKEN_PLACEHOLDER",
|
||||
"SRE_GROUP_CHAT_ID_PLACEHOLDER",
|
||||
)
|
||||
):
|
||||
raise SystemExit("unreplaced secret placeholder remains in rendered config")
|
||||
target.write_text(text)
|
||||
PY
|
||||
|
||||
log "Validating rendered config with live Alertmanager amtool on ${TARGET_HOST}"
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
|
||||
"docker exec -i alertmanager sh -c 'cat >/tmp/alertmanager-rendered.yml && amtool check-config /tmp/alertmanager-rendered.yml'" \
|
||||
"docker exec -i alertmanager amtool check-config /dev/stdin" \
|
||||
< "$tmp_rendered"
|
||||
|
||||
if [[ "$DRY_RUN" == "--dry-run" ]]; then
|
||||
@@ -127,18 +134,115 @@ ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" \
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_USER}@${TARGET_HOST}" "bash -s" <<REMOTE
|
||||
set -euo pipefail
|
||||
target='${TARGET_PATH}'
|
||||
staged='/tmp/alertmanager.yml.new'
|
||||
container='alertmanager'
|
||||
trap 'rm -f "\$staged"' EXIT
|
||||
|
||||
[[ -f "\$target" && ! -L "\$target" ]] || {
|
||||
echo 'ERROR: live Alertmanager config must be a regular non-symlink file' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -f "\$staged" && ! -L "\$staged" ]] || {
|
||||
echo 'ERROR: staged Alertmanager config must be a regular non-symlink file' >&2
|
||||
exit 1
|
||||
}
|
||||
sudo -n true
|
||||
|
||||
# Read the effective host UID/GID from the container process itself. This also
|
||||
# works when Docker user namespaces remap container identities.
|
||||
container_pid="\$(docker inspect --format '{{.State.Pid}}' "\$container")"
|
||||
[[ "\$container_pid" =~ ^[1-9][0-9]*$ ]] || {
|
||||
echo 'ERROR: Alertmanager container host PID is unavailable' >&2
|
||||
exit 1
|
||||
}
|
||||
status_path="/proc/\${container_pid}/status"
|
||||
[[ -r "\$status_path" ]] || {
|
||||
echo 'ERROR: Alertmanager runtime identity cannot be verified' >&2
|
||||
exit 1
|
||||
}
|
||||
runtime_uid="\$(awk '/^Uid:/{print \$2; exit}' "\$status_path")"
|
||||
runtime_gid="\$(awk '/^Gid:/{print \$2; exit}' "\$status_path")"
|
||||
[[ "\$runtime_uid" =~ ^[0-9]+$ && "\$runtime_gid" =~ ^[0-9]+$ ]] || {
|
||||
echo 'ERROR: Alertmanager runtime UID/GID is invalid' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
backup="\${target}.bak.\$(date +%Y%m%d%H%M%S)"
|
||||
cp "\$target" "\$backup"
|
||||
# Alertmanager bind-mounts a single file. Keep the existing inode instead of mv'ing
|
||||
# a replacement over it, then restore readable permissions for the container user.
|
||||
cat /tmp/alertmanager.yml.new > "\$target"
|
||||
chmod 0644 "\$target"
|
||||
rm -f /tmp/alertmanager.yml.new
|
||||
docker exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml
|
||||
docker kill -s HUP alertmanager >/dev/null
|
||||
operator_uid="\$(id -u)"
|
||||
operator_gid="\$(id -g)"
|
||||
umask 077
|
||||
# install applies 0600 while creating the copy; cp followed by chmod would leave
|
||||
# a secret-bearing backup readable during the gap.
|
||||
sudo -n install -m 0600 -o "\$operator_uid" -g "\$operator_gid" "\$target" "\$backup"
|
||||
[[ "\$(stat -c '%a' "\$backup")" == '600' ]] || {
|
||||
echo 'ERROR: secure Alertmanager rollback copy was not created' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Harden legacy copies produced by older versions of this script without
|
||||
# reading or printing their contents.
|
||||
target_dir="\$(dirname "\$target")"
|
||||
target_base="\$(basename "\$target")"
|
||||
while IFS= read -r -d '' legacy_backup; do
|
||||
sudo -n chmod 0600 "\$legacy_backup"
|
||||
done < <(find "\$target_dir" -maxdepth 1 -type f -name "\${target_base}.bak.*" -print0)
|
||||
|
||||
reload_attempted=false
|
||||
rollback_secure() {
|
||||
local rc="\${1:-1}"
|
||||
trap - ERR
|
||||
set +e
|
||||
if [[ -f "\$backup" ]]; then
|
||||
# Remove access before restoring bytes so rollback cannot briefly recreate
|
||||
# the legacy world-readable state.
|
||||
sudo -n chmod 0000 "\$target"
|
||||
sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
|
||||
sudo -n chmod 0400 "\$target"
|
||||
sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$backup" "\$target"
|
||||
rollback_verified=false
|
||||
if docker exec "\$container" sh -ec \
|
||||
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml' \
|
||||
>/dev/null 2>&1; then
|
||||
rollback_verified=true
|
||||
fi
|
||||
if [[ "\$reload_attempted" == 'true' && "\$rollback_verified" == 'true' ]]; then
|
||||
docker kill -s HUP "\$container" >/dev/null 2>&1
|
||||
fi
|
||||
fi
|
||||
rm -f "\$staged"
|
||||
echo 'ERROR: Alertmanager deploy failed; secure rollback attempted' >&2
|
||||
exit "\$rc"
|
||||
}
|
||||
trap 'rollback_secure \$?' ERR
|
||||
|
||||
# Alertmanager bind-mounts this single file. Preserve its inode and remove all
|
||||
# access before changing ownership, so no secret bytes are ever written while
|
||||
# legacy group/other read bits remain. Validate the permission model against the
|
||||
# running container before replacing the old config bytes.
|
||||
sudo -n chmod 0000 "\$target"
|
||||
sudo -n chown "\${runtime_uid}:\${runtime_gid}" "\$target"
|
||||
sudo -n chmod 0400 "\$target"
|
||||
[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
|
||||
[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
|
||||
[[ "\$(stat -c '%a' "\$target")" == '400' ]]
|
||||
docker exec "\$container" sh -ec \
|
||||
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
|
||||
|
||||
sudo -n sh -c 'cat "\$1" > "\$2"' _ "\$staged" "\$target"
|
||||
rm -f "\$staged"
|
||||
[[ "\$(stat -c '%u' "\$target")" == "\$runtime_uid" ]]
|
||||
[[ "\$(stat -c '%g' "\$target")" == "\$runtime_gid" ]]
|
||||
[[ "\$(stat -c '%a' "\$target")" == '400' ]]
|
||||
docker exec "\$container" sh -ec \
|
||||
'test -r /etc/alertmanager/alertmanager.yml && amtool check-config /etc/alertmanager/alertmanager.yml'
|
||||
|
||||
reload_attempted=true
|
||||
docker kill -s HUP "\$container" >/dev/null
|
||||
sleep 2
|
||||
docker inspect alertmanager --format 'status={{.State.Status}} started={{.State.StartedAt}}'
|
||||
echo "backup=\$backup"
|
||||
docker inspect "\$container" --format 'status={{.State.Status}} started={{.State.StartedAt}}'
|
||||
[[ "\$(docker inspect "\$container" --format '{{.State.Status}}')" == 'running' ]]
|
||||
trap - ERR
|
||||
echo "backup=\$backup config_mode=0400 runtime_owner=\${runtime_uid}:\${runtime_gid}"
|
||||
REMOTE
|
||||
|
||||
log "Alertmanager config deployed and reloaded"
|
||||
|
||||
413
scripts/ops/deploy-alertmanager-runtime-scrape.sh
Executable file
413
scripts/ops/deploy-alertmanager-runtime-scrape.sh
Executable file
@@ -0,0 +1,413 @@
|
||||
#!/usr/bin/env bash
|
||||
# Bounded Prometheus scrape-source convergence for Alertmanager delivery truth.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
|
||||
RUN_ID="${RUN_ID:?RUN_ID is required}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
|
||||
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
|
||||
PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
|
||||
ALERTMANAGER_METRICS_URL="${ALERTMANAGER_METRICS_URL:-http://127.0.0.1:9093/metrics}"
|
||||
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
|
||||
REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
|
||||
MODE="${1:---dry-run}"
|
||||
|
||||
case "${MODE}" in
|
||||
apply|--dry-run) ;;
|
||||
*) printf 'usage: %s [--dry-run|apply]\n' "$0" >&2; exit 64 ;;
|
||||
esac
|
||||
|
||||
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
||||
case "${value}" in
|
||||
*[!A-Za-z0-9._-]*|'')
|
||||
printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
|
||||
bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
||||
"${PROMETHEUS_URL}" "${ALERTMANAGER_METRICS_URL}" \
|
||||
"${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
MODE="$4"
|
||||
PROMETHEUS_URL="$5"
|
||||
ALERTMANAGER_METRICS_URL="$6"
|
||||
PROMETHEUS_CONTAINER="$7"
|
||||
REMOTE_CONFIG="$8"
|
||||
REMOTE_CANDIDATE="/tmp/awoooi-prometheus-alertmanager-runtime.${RUN_ID}.yml"
|
||||
CONTAINER_CANDIDATE="/etc/prometheus/.awoooi-alertmanager-runtime.${RUN_ID}.yml"
|
||||
BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
|
||||
APPLY_STARTED=0
|
||||
DEPLOY_VERIFIED=0
|
||||
SOURCE_SHA=""
|
||||
CANDIDATE_SHA=""
|
||||
|
||||
exec 9>/tmp/awoooi-prometheus-config.lock
|
||||
if ! flock -n 9; then
|
||||
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=prometheus_config_apply_in_progress\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2
|
||||
exit 75
|
||||
fi
|
||||
|
||||
cleanup() {
|
||||
rm -f "${REMOTE_CANDIDATE}"
|
||||
docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
|
||||
rm -f "${CONTAINER_CANDIDATE}" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
rollback() {
|
||||
local rollback_rc=0
|
||||
local current_sha backup_sha restored_sha
|
||||
current_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
|
||||
backup_sha="$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" || rollback_rc=1
|
||||
if [ "${rollback_rc}" -eq 0 ] && [ "${current_sha}" = "${SOURCE_SHA}" ]; then
|
||||
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=no_write_source_intact source_sha=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}"
|
||||
return 0
|
||||
fi
|
||||
if [ "${rollback_rc}" -ne 0 ] \
|
||||
|| [ "${current_sha}" != "${CANDIDATE_SHA}" ] \
|
||||
|| [ "${backup_sha}" != "${SOURCE_SHA}" ]; then
|
||||
rollback_rc=1
|
||||
else
|
||||
cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" || rollback_rc=1
|
||||
restored_sha="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" || rollback_rc=1
|
||||
[ "${restored_sha}" = "${SOURCE_SHA}" ] || rollback_rc=1
|
||||
fi
|
||||
if [ "${rollback_rc}" -eq 0 ]; then
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null \
|
||||
|| rollback_rc=1
|
||||
fi
|
||||
if [ "${rollback_rc}" -eq 0 ]; then
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null || rollback_rc=1
|
||||
fi
|
||||
if [ "${rollback_rc}" -eq 0 ]; then
|
||||
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
|
||||
return 0
|
||||
fi
|
||||
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed ready=false backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
rc=$?
|
||||
trap - EXIT
|
||||
set +e
|
||||
cleanup
|
||||
rollback_rc=0
|
||||
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
|
||||
&& [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
||||
rollback || rollback_rc=$?
|
||||
fi
|
||||
if [ "${rollback_rc}" -ne 0 ]; then
|
||||
exit 70
|
||||
fi
|
||||
exit "${rc}"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
test -f "${REMOTE_CONFIG}"
|
||||
test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
METRICS_SNAPSHOT="$(curl -fsS "${ALERTMANAGER_METRICS_URL}")"
|
||||
grep -Eq '^alertmanager_notification_requests_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
|
||||
grep -Eq '^alertmanager_notification_requests_failed_total\{[^}]*receiver="awoooi-webhook"' <<<"${METRICS_SNAPSHOT}"
|
||||
SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
||||
|
||||
python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
source = Path(sys.argv[1])
|
||||
candidate = Path(sys.argv[2])
|
||||
text = source.read_text(encoding="utf-8")
|
||||
|
||||
begin = " # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
|
||||
end = " # END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH\n"
|
||||
block = """ # BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
|
||||
- job_name: 'alertmanager-runtime'
|
||||
scrape_interval: 30s
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ['alertmanager:9093']
|
||||
labels:
|
||||
host: '110'
|
||||
service: 'alertmanager'
|
||||
env: 'prod'
|
||||
# END AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH
|
||||
|
||||
"""
|
||||
|
||||
if text.count(begin) != text.count(end) or text.count(begin) > 1:
|
||||
raise SystemExit("alertmanager runtime managed block is ambiguous")
|
||||
if begin in text:
|
||||
pattern = re.escape(begin) + r".*?" + re.escape(end) + r"\n?"
|
||||
text, count = re.subn(pattern, block, text, count=1, flags=re.S)
|
||||
if count != 1:
|
||||
raise SystemExit("managed block replacement failed")
|
||||
else:
|
||||
legacy_pattern = (
|
||||
r"(?ms)^ - job_name: ['\"]alertmanager-runtime['\"]\n"
|
||||
r".*?(?=^ - job_name:|\Z)"
|
||||
)
|
||||
legacy_matches = list(re.finditer(legacy_pattern, text))
|
||||
if len(legacy_matches) == 1:
|
||||
text = re.sub(legacy_pattern, block, text, count=1)
|
||||
elif legacy_matches:
|
||||
raise SystemExit("legacy alertmanager runtime job is ambiguous")
|
||||
else:
|
||||
anchors = (
|
||||
" # === AWOOOI API Metrics",
|
||||
" - job_name: 'awoooi-api'\n",
|
||||
)
|
||||
anchor = next((value for value in anchors if value in text), None)
|
||||
if anchor is None or text.count(anchor) != 1:
|
||||
raise SystemExit("awoooi-api insertion anchor missing or ambiguous")
|
||||
text = text.replace(anchor, block + anchor, 1)
|
||||
|
||||
provider_begin = " # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES\n"
|
||||
provider_end = " # END AWOOOI OLLAMA CLOUD EDGE PROBES\n"
|
||||
provider_block = """ # BEGIN AWOOOI OLLAMA CLOUD EDGE PROBES
|
||||
- targets:
|
||||
- http://34.143.170.20:11434/api/tags
|
||||
labels:
|
||||
service: ollama-cloud-edge
|
||||
provider: ollama_gcp_a
|
||||
probe_origin: host110
|
||||
boundary: cloud_edge_availability_only
|
||||
- targets:
|
||||
- http://34.21.145.224:11434/api/tags
|
||||
labels:
|
||||
service: ollama-cloud-edge
|
||||
provider: ollama_gcp_b
|
||||
probe_origin: host110
|
||||
boundary: cloud_edge_availability_only
|
||||
# END AWOOOI OLLAMA CLOUD EDGE PROBES
|
||||
"""
|
||||
if (
|
||||
text.count(provider_begin) != text.count(provider_end)
|
||||
or text.count(provider_begin) > 1
|
||||
):
|
||||
raise SystemExit("Ollama cloud edge managed block is ambiguous")
|
||||
if provider_begin in text:
|
||||
provider_pattern = (
|
||||
re.escape(provider_begin)
|
||||
+ r".*?"
|
||||
+ re.escape(provider_end)
|
||||
)
|
||||
text, count = re.subn(
|
||||
provider_pattern,
|
||||
provider_block,
|
||||
text,
|
||||
count=1,
|
||||
flags=re.S,
|
||||
)
|
||||
if count != 1:
|
||||
raise SystemExit("Ollama cloud edge managed block replacement failed")
|
||||
else:
|
||||
blackbox_match = re.search(
|
||||
r"(?ms)^ - job_name: ['\"]blackbox-http['\"]\n"
|
||||
r".*?(?=^ - job_name:|\Z)",
|
||||
text,
|
||||
)
|
||||
if blackbox_match is None:
|
||||
raise SystemExit("blackbox-http job missing")
|
||||
blackbox_job = blackbox_match.group(0)
|
||||
if "provider: ollama_gcp_" in blackbox_job:
|
||||
raise SystemExit("unmanaged Ollama cloud edge targets are ambiguous")
|
||||
relabel_anchor = " relabel_configs:\n"
|
||||
if blackbox_job.count(relabel_anchor) != 1:
|
||||
raise SystemExit("blackbox-http relabel anchor missing or ambiguous")
|
||||
patched_job = blackbox_job.replace(
|
||||
relabel_anchor,
|
||||
provider_block + relabel_anchor,
|
||||
1,
|
||||
)
|
||||
text = text[: blackbox_match.start()] + patched_job + text[blackbox_match.end() :]
|
||||
|
||||
if text.count("job_name: 'alertmanager-runtime'") != 1:
|
||||
raise SystemExit("alertmanager runtime job must exist exactly once")
|
||||
if "targets: ['alertmanager:9093']" not in text:
|
||||
raise SystemExit("canonical Alertmanager target missing")
|
||||
if text.count("provider: ollama_gcp_a") != 1:
|
||||
raise SystemExit("canonical GCP-A cloud edge target missing or ambiguous")
|
||||
if text.count("provider: ollama_gcp_b") != 1:
|
||||
raise SystemExit("canonical GCP-B cloud edge target missing or ambiguous")
|
||||
if "192.168.0.111:11434" in text:
|
||||
raise SystemExit("host111 must not be probed from retired host110 boundary")
|
||||
candidate.write_text(text, encoding="utf-8")
|
||||
PY
|
||||
|
||||
docker cp "${REMOTE_CANDIDATE}" \
|
||||
"${PROMETHEUS_CONTAINER}:${CONTAINER_CANDIDATE}" >/dev/null
|
||||
docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${CONTAINER_CANDIDATE}"
|
||||
|
||||
CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
|
||||
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
|
||||
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass persistent_writes_performed=0 temporary_validation_artifact=true\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
||||
"${SOURCE_SHA}" "${CANDIDATE_SHA}"
|
||||
|
||||
if [ "${MODE}" = "--dry-run" ]; then
|
||||
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_persistent_write\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
|
||||
test "$(sha256sum "${BACKUP_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
|
||||
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${SOURCE_SHA}"
|
||||
APPLY_STARTED=1
|
||||
cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
|
||||
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=alertmanager_runtime_scrape_converged active_sha=%s backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${CANDIDATE_SHA}" \
|
||||
"${BACKUP_CONFIG}"
|
||||
|
||||
target_readback() {
|
||||
curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
targets = json.load(sys.stdin)["data"]["activeTargets"]
|
||||
matches = [
|
||||
target for target in targets
|
||||
if target.get("labels", {}).get("job") == "alertmanager-runtime"
|
||||
]
|
||||
if len(matches) != 1:
|
||||
print(f"{len(matches)}|missing|never")
|
||||
else:
|
||||
target = matches[0]
|
||||
print("{}|{}|{}".format(
|
||||
len(matches),
|
||||
target.get("health", "unknown"),
|
||||
target.get("lastScrape", "never"),
|
||||
))
|
||||
'
|
||||
}
|
||||
|
||||
provider_target_readback() {
|
||||
curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
targets = json.load(sys.stdin)["data"]["activeTargets"]
|
||||
providers = ("ollama_gcp_a", "ollama_gcp_b")
|
||||
matches = {
|
||||
provider: [
|
||||
target
|
||||
for target in targets
|
||||
if target.get("labels", {}).get("job") == "blackbox-http"
|
||||
and target.get("labels", {}).get("provider") == provider
|
||||
and target.get("labels", {}).get("probe_origin") == "host110"
|
||||
]
|
||||
for provider in providers
|
||||
}
|
||||
ready = all(
|
||||
len(matches[provider]) == 1
|
||||
and matches[provider][0].get("lastScrape", "never") != "never"
|
||||
for provider in providers
|
||||
)
|
||||
values = []
|
||||
for provider in providers:
|
||||
target = matches[provider][0] if len(matches[provider]) == 1 else {}
|
||||
values.extend([
|
||||
target.get("lastScrape", "never"),
|
||||
target.get("health", "missing"),
|
||||
])
|
||||
print("|".join(["1" if ready else "0", *values]))
|
||||
'
|
||||
}
|
||||
|
||||
FIRST_SCRAPE=""
|
||||
for attempt in $(seq 1 9); do
|
||||
IFS='|' read -r count health last_scrape <<<"$(target_readback)"
|
||||
printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
|
||||
"${last_scrape}"
|
||||
if [ "${count}" = 1 ] && [ "${health}" = up ] \
|
||||
&& [ "${last_scrape}" != never ]; then
|
||||
FIRST_SCRAPE="${last_scrape}"
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
test -n "${FIRST_SCRAPE}"
|
||||
|
||||
FIRST_GCP_A_SCRAPE=""
|
||||
FIRST_GCP_B_SCRAPE=""
|
||||
for attempt in $(seq 1 9); do
|
||||
IFS='|' read -r providers_ready gcp_a_scrape gcp_a_health \
|
||||
gcp_b_scrape gcp_b_health <<<"$(provider_target_readback)"
|
||||
printf 'provider_target_probe trace_id=%s run_id=%s attempt=%s ready=%s gcp_a_health=%s gcp_b_health=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${attempt}" "${providers_ready}" \
|
||||
"${gcp_a_health}" "${gcp_b_health}"
|
||||
if [ "${providers_ready}" = 1 ]; then
|
||||
FIRST_GCP_A_SCRAPE="${gcp_a_scrape}"
|
||||
FIRST_GCP_B_SCRAPE="${gcp_b_scrape}"
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
test -n "${FIRST_GCP_A_SCRAPE}"
|
||||
test -n "${FIRST_GCP_B_SCRAPE}"
|
||||
|
||||
sleep 31
|
||||
IFS='|' read -r count health SECOND_SCRAPE <<<"$(target_readback)"
|
||||
test "${count}" = 1
|
||||
test "${health}" = up
|
||||
test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
|
||||
IFS='|' read -r providers_ready SECOND_GCP_A_SCRAPE gcp_a_health \
|
||||
SECOND_GCP_B_SCRAPE gcp_b_health <<<"$(provider_target_readback)"
|
||||
test "${providers_ready}" = 1
|
||||
test "${SECOND_GCP_A_SCRAPE}" != "${FIRST_GCP_A_SCRAPE}"
|
||||
test "${SECOND_GCP_B_SCRAPE}" != "${FIRST_GCP_B_SCRAPE}"
|
||||
|
||||
SERIES_COUNT="$(curl -fsS --get \
|
||||
--data-urlencode 'query=alertmanager_notification_requests_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
|
||||
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
|
||||
import json
|
||||
import sys
|
||||
print(len(json.load(sys.stdin)["data"]["result"]))
|
||||
')"
|
||||
test "${SERIES_COUNT}" = 1
|
||||
|
||||
FAILED_SERIES_COUNT="$(curl -fsS --get \
|
||||
--data-urlencode 'query=alertmanager_notification_requests_failed_total{job="alertmanager-runtime",integration="webhook",receiver="awoooi-webhook"}' \
|
||||
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
|
||||
import json
|
||||
import sys
|
||||
print(len(json.load(sys.stdin)["data"]["result"]))
|
||||
')"
|
||||
test "${FAILED_SERIES_COUNT}" = 1
|
||||
|
||||
PROVIDER_SERIES_COUNT="$(curl -fsS --get \
|
||||
--data-urlencode 'query=probe_success{job="blackbox-http",provider=~"ollama_gcp_a|ollama_gcp_b",probe_origin="host110"}' \
|
||||
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
|
||||
import json
|
||||
import sys
|
||||
print(len(json.load(sys.stdin)["data"]["result"]))
|
||||
')"
|
||||
test "${PROVIDER_SERIES_COUNT}" = 2
|
||||
test "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${CANDIDATE_SHA}"
|
||||
|
||||
DEPLOY_VERIFIED=1
|
||||
printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=two_scrapes_up first=%s second=%s delivery_series_count=%s failed_series_count=%s provider_series_count=%s gcp_a_health=%s gcp_b_health=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
|
||||
"${SECOND_SCRAPE}" "${SERIES_COUNT}" "${FAILED_SERIES_COUNT}" \
|
||||
"${PROVIDER_SERIES_COUNT}" "${gcp_a_health}" "${gcp_b_health}"
|
||||
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
|
||||
REMOTE
|
||||
@@ -185,6 +185,16 @@ if [[ "$NO_ALERTS_QUERY" != *'source="alertmanager"'* ]]; then
|
||||
fi
|
||||
log "✅ NoAlertsReceived2Hours query 已限制 alertmanager 主鏈路"
|
||||
|
||||
ALERT_CHAIN_BROKEN_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'AlertChainBroken_Alertmanager'), ''))")
|
||||
if [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_failed_total'* ]] \
|
||||
|| [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'alertmanager_notification_requests_total'* ]] \
|
||||
|| [[ "$ALERT_CHAIN_BROKEN_QUERY" != *'receiver="awoooi-webhook"'* ]] \
|
||||
|| [[ "$ALERT_CHAIN_BROKEN_QUERY" == *'awoooi_webhook_requests_total'* ]]; then
|
||||
echo "ERROR: AlertChainBroken_Alertmanager 未使用 primary awoooi-webhook monotonic delivery truth: ${ALERT_CHAIN_BROKEN_QUERY}"
|
||||
exit 1
|
||||
fi
|
||||
log "✅ AlertChainBroken_Alertmanager 只使用 primary awoooi-webhook monotonic delivery truth"
|
||||
|
||||
SOURCE_PROVIDER_STALE_QUERY=$(ssh wooo@${TARGET_HOST} "curl -s ${PROMETHEUS_URL}/api/v1/rules" | python3 -c "import sys,json; r=json.load(sys.stdin); print(next((x.get('query','') for g in r['data']['groups'] for x in g['rules'] if x.get('name') == 'SourceProviderIngestionStale'), ''))")
|
||||
if [[ "$SOURCE_PROVIDER_STALE_QUERY" != *'source=~"sentry|signoz"'* ]]; then
|
||||
echo "ERROR: SourceProviderIngestionStale query 未限制 Sentry/SignOz provider freshness: ${SOURCE_PROVIDER_STALE_QUERY}"
|
||||
|
||||
99
scripts/ops/ollama111-fallback-proxy-diagnose.sh
Executable file → Normal file
99
scripts/ops/ollama111-fallback-proxy-diagnose.sh
Executable file → Normal file
@@ -1,78 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compatibility entrypoint retained for operators; the host110 proxy lane is
|
||||
# retired. All probes use the canonical direct provider identities.
|
||||
set -euo pipefail
|
||||
|
||||
# Read-only diagnosis for ADR-110 local fallback:
|
||||
# API Pod -> 110:11437 Nginx proxy -> 111:11434 Ollama allowlist proxy.
|
||||
GCP_A_URL="${GCP_A_URL:-http://34.143.170.20:11434}"
|
||||
GCP_B_URL="${GCP_B_URL:-http://34.21.145.224:11434}"
|
||||
HOST111_URL="${HOST111_URL:-http://192.168.0.111:11434}"
|
||||
PUBLIC_STATUS_URL="${PUBLIC_STATUS_URL:-https://awoooi.wooo.work/api/v1/ai/status}"
|
||||
CURL_TIMEOUT="${CURL_TIMEOUT:-8}"
|
||||
|
||||
NAMESPACE="${NAMESPACE:-awoooi-prod}"
|
||||
DEPLOYMENT="${DEPLOYMENT:-awoooi-api}"
|
||||
PROXY_HOST="${PROXY_HOST:-wooo@192.168.0.110}"
|
||||
LOCAL_HOST="${LOCAL_HOST:-ollama-111-gpu}"
|
||||
LOCAL_IP="${LOCAL_IP:-192.168.0.111}"
|
||||
PROXY_PORT="${PROXY_PORT:-11437}"
|
||||
LOCAL_PORT="${LOCAL_PORT:-11434}"
|
||||
SSH_TIMEOUT="${SSH_TIMEOUT:-8}"
|
||||
CURL_TIMEOUT="${CURL_TIMEOUT:-5}"
|
||||
|
||||
section() {
|
||||
printf '\n== %s ==\n' "$1"
|
||||
probe() {
|
||||
local provider="$1"
|
||||
local url="$2"
|
||||
local code
|
||||
code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \
|
||||
"${url}/api/tags" 2>/dev/null || true)"
|
||||
case "${code}" in
|
||||
200) printf 'provider=%s boundary=%s status=healthy http_code=200\n' \
|
||||
"${provider}" "$3" ;;
|
||||
*) printf 'provider=%s boundary=%s status=unavailable http_code=%s\n' \
|
||||
"${provider}" "$3" "${code:-000}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
run_local() {
|
||||
set +e
|
||||
"$@"
|
||||
local rc=$?
|
||||
set -e
|
||||
printf 'exit_code=%s\n' "$rc"
|
||||
}
|
||||
printf 'schema_version=ollama_direct_route_diagnosis_v1 writes_performed=0\n'
|
||||
probe ollama_gcp_a "${GCP_A_URL}" cloud
|
||||
probe ollama_gcp_b "${GCP_B_URL}" cloud
|
||||
probe ollama_local "${HOST111_URL}" local
|
||||
|
||||
section "Production health provider chain"
|
||||
run_local curl -sS -m 12 https://awoooi.wooo.work/api/v1/health
|
||||
|
||||
section "API pod view"
|
||||
set +e
|
||||
kubectl -n "${NAMESPACE}" exec -i "deploy/${DEPLOYMENT}" -- sh -lc "
|
||||
set +e
|
||||
echo OLLAMA_URL=\$OLLAMA_URL
|
||||
echo OLLAMA_SECONDARY_URL=\$OLLAMA_SECONDARY_URL
|
||||
echo OLLAMA_FALLBACK_URL=\$OLLAMA_FALLBACK_URL
|
||||
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://192.168.0.110:${PROXY_PORT}/api/tags | head -60
|
||||
"
|
||||
printf 'exit_code=%s\n' "$?"
|
||||
set -e
|
||||
|
||||
section "110 proxy upstream reachability"
|
||||
run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${PROXY_HOST}" "
|
||||
set +e
|
||||
echo route:
|
||||
ip route get ${LOCAL_IP}
|
||||
echo neigh:
|
||||
ip neigh show ${LOCAL_IP}
|
||||
echo ping:
|
||||
ping -c 3 -W 2 ${LOCAL_IP}
|
||||
echo direct_111:
|
||||
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://${LOCAL_IP}:${LOCAL_PORT}/api/tags | head -60
|
||||
echo proxy_11437:
|
||||
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${PROXY_PORT}/api/tags | head -60
|
||||
echo nginx_recent_errors:
|
||||
tail -20 /var/log/nginx/ollama-local-error.log 2>/dev/null || true
|
||||
"
|
||||
|
||||
section "111 host direct SSH view"
|
||||
run_local ssh -o BatchMode=yes -o ConnectTimeout="${SSH_TIMEOUT}" "${LOCAL_HOST}" "
|
||||
set +e
|
||||
hostname
|
||||
date
|
||||
curl -sS -m ${CURL_TIMEOUT} -w '\nHTTP=%{http_code}\n' http://127.0.0.1:${LOCAL_PORT}/api/tags | head -60
|
||||
launchctl print gui/501/com.momo.ollama111-allow-proxy 2>/dev/null | head -60 || true
|
||||
pmset -g custom 2>/dev/null | sed -n '1,80p' || true
|
||||
"
|
||||
|
||||
cat <<'EOF'
|
||||
|
||||
Interpretation:
|
||||
- If 110 shows "ip neigh INCOMPLETE", "Destination Host Unreachable", or "No route to host",
|
||||
the 111 host or LAN path is down. Do not restart API or change provider order.
|
||||
- If 110 can reach 111 directly but 11437 returns 502, inspect/reload the 110 Nginx proxy.
|
||||
- If 111 direct SSH works but 127.0.0.1:11434 fails, recover the 111 Ollama/allowlist LaunchAgent.
|
||||
EOF
|
||||
status_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${CURL_TIMEOUT}" \
|
||||
"${PUBLIC_STATUS_URL}" 2>/dev/null || true)"
|
||||
printf 'production_route_readback_http=%s\n' "${status_code:-000}"
|
||||
printf '%s\n' \
|
||||
'terminal=read_only provider_order=ollama_gcp_a,ollama_gcp_b,ollama_local,claude,gemini host110_transport_selected=false'
|
||||
|
||||
64
scripts/ops/run-paid-provider-canary.py
Normal file
64
scripts/ops/run-paid-provider-canary.py
Normal file
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Execute the bounded five-lane SRE comparison and paid-provider canary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_API_ROOT = _REPO_ROOT / "apps" / "api"
|
||||
if str(_API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_API_ROOT))
|
||||
|
||||
|
||||
async def _run_validation(**kwargs):
|
||||
from src.services.paid_provider_canary_validation import (
|
||||
run_paid_provider_canary_validation,
|
||||
)
|
||||
|
||||
return await run_paid_provider_canary_validation(**kwargs)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--run-ref", required=True)
|
||||
parser.add_argument("--operator-id", default="codex-controlled-canary")
|
||||
parser.add_argument("--authorization-ref", required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
receipt = asyncio.run(
|
||||
_run_validation(
|
||||
run_ref=args.run_ref,
|
||||
operator_id=args.operator_id,
|
||||
authorization_ref=args.authorization_ref,
|
||||
)
|
||||
)
|
||||
except BaseException as exc:
|
||||
error_code = str(exc)
|
||||
if not re.fullmatch(r"[a-z][a-z0-9_]{0,119}", error_code):
|
||||
error_code = f"canary_failed_{type(exc).__name__}"
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "paid_provider_sre_canary_error_v1",
|
||||
"status": "blocked_with_safe_next_action",
|
||||
"error_code": error_code[:120],
|
||||
"raw_prompt_persisted": False,
|
||||
"raw_response_persisted": False,
|
||||
"secret_value_returned": False,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 2
|
||||
print(json.dumps(receipt, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if receipt["independent_verifier"]["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-runtime-scrape.sh"
|
||||
|
||||
|
||||
def test_controlled_scrape_apply_has_check_rollback_and_independent_verifier() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'TRACE_ID="${TRACE_ID:?TRACE_ID is required}"' in text
|
||||
assert 'RUN_ID="${RUN_ID:?RUN_ID is required}"' in text
|
||||
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"' in text
|
||||
assert "promtool check config" in text
|
||||
assert "writes_performed=0" in text
|
||||
assert "rollback_receipt" in text
|
||||
assert "terminal=rollback_failed ready=false" in text
|
||||
assert "flock -n 9" in text
|
||||
assert '!= "${CANDIDATE_SHA}"' in text
|
||||
assert '!= "${SOURCE_SHA}"' in text
|
||||
assert "alertmanager_notification_requests_total" in text
|
||||
assert "alertmanager_notification_requests_failed_total" in text
|
||||
assert 'receiver="awoooi-webhook"' in text
|
||||
assert text.count('receiver="awoooi-webhook"') >= 4
|
||||
assert "two_scrapes_up" in text
|
||||
assert "delivery_series_count" in text
|
||||
|
||||
|
||||
def test_controlled_scrape_apply_never_replaces_the_full_live_config() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "BEGIN AWOOOI ALERTMANAGER RUNTIME DELIVERY TRUTH" in text
|
||||
assert "job_name: 'alertmanager-runtime'" in text
|
||||
assert "targets: ['alertmanager:9093']" in text
|
||||
assert "legacy_pattern" in text
|
||||
assert "k8s/monitoring/prometheus.yml" not in text
|
||||
@@ -7,6 +7,7 @@ import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ALERTMANAGER_CONFIG = ROOT / "ops" / "alertmanager" / "alertmanager.yml"
|
||||
DEPLOY_SCRIPT = ROOT / "scripts" / "ops" / "deploy-alertmanager-config.sh"
|
||||
|
||||
|
||||
def _config() -> dict:
|
||||
@@ -37,3 +38,70 @@ def test_emergency_direct_telegram_stays_limited_to_alert_chain() -> None:
|
||||
route_text = str(_config()["route"])
|
||||
assert "AWOOOIApiDown" in route_text
|
||||
assert 'severity="critical"' in route_text
|
||||
|
||||
|
||||
def test_emergency_card_is_bounded_and_has_honest_automation_attribution() -> None:
|
||||
receiver = _receiver("telegram-direct")
|
||||
telegram = receiver["telegram_configs"][0]
|
||||
message = telegram["message"]
|
||||
|
||||
assert "SRE BYPASS|告警主鏈異常" in message
|
||||
assert "Prometheus deterministic rule(未呼叫 LLM)" in message
|
||||
assert "Executor:none" in message
|
||||
assert "未宣稱已修復" in message
|
||||
assert "Verifier:source-specific delivery metric" in message
|
||||
assert "故障主機" not in message
|
||||
telegram_route = next(
|
||||
route
|
||||
for route in _config()["route"]["routes"]
|
||||
if route["receiver"] == "telegram-direct"
|
||||
)
|
||||
assert telegram_route["repeat_interval"] == "2h"
|
||||
|
||||
|
||||
def test_alertmanager_has_no_agent99_inbound_receiver_or_secret_first_hop() -> None:
|
||||
config = _config()
|
||||
assert all(
|
||||
item.get("receiver") != "agent99-alert-chain-relay"
|
||||
for item in config["route"]["routes"]
|
||||
)
|
||||
assert all(
|
||||
receiver.get("name") != "agent99-alert-chain-relay"
|
||||
for receiver in config["receivers"]
|
||||
)
|
||||
source = ALERTMANAGER_CONFIG.read_text(encoding="utf-8")
|
||||
assert "192.168.0.99:8787" not in source
|
||||
assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source
|
||||
assert "authorization:" not in source
|
||||
|
||||
|
||||
def test_alertmanager_deploy_does_not_inject_agent99_relay_secret() -> None:
|
||||
source = DEPLOY_SCRIPT.read_text(encoding="utf-8")
|
||||
assert "AGENT99_SRE_ALERT_RELAY_TOKEN" not in source
|
||||
assert 'os.environ["AGENT99_SRE_RELAY_TOKEN"]' not in source
|
||||
assert "AGENT99_SRE_RELAY_TOKEN_PLACEHOLDER" not in source
|
||||
assert "unreplaced secret placeholder remains" in source
|
||||
assert "set -x" not in source
|
||||
|
||||
|
||||
def test_secret_bearing_alertmanager_config_and_backups_are_not_world_readable() -> None:
|
||||
source = DEPLOY_SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "install -m 0600" in source
|
||||
assert 'chmod 0600 "\\$legacy_backup"' in source
|
||||
assert 'chmod 0400 "\\$target"' in source
|
||||
assert 'chmod 0000 "\\$target"' in source
|
||||
assert "chmod 0644" not in source
|
||||
assert "/proc/\\${container_pid}/status" in source
|
||||
assert "runtime_uid" in source
|
||||
assert "runtime_gid" in source
|
||||
assert "test -r /etc/alertmanager/alertmanager.yml" in source
|
||||
assert "rollback_secure" in source
|
||||
assert "secure rollback attempted" in source
|
||||
assert "amtool check-config /etc/alertmanager/alertmanager.yml" in source
|
||||
assert "amtool check-config /dev/stdin" in source
|
||||
assert "/tmp/alertmanager-rendered.yml" not in source
|
||||
assert "trap 'rm -f \"\\$staged\"' EXIT" in source
|
||||
assert source.index('chmod 0000 "\\$target"') < source.index(
|
||||
"sudo -n sh -c 'cat \"\\$1\" > \"\\$2\"'"
|
||||
)
|
||||
|
||||
@@ -456,7 +456,7 @@ $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-
|
||||
$decision = Get-AgentLivePreflightDecision `
|
||||
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
||||
-Manifest $manifest `
|
||||
-ExpectedRuntimeFileCount 14 `
|
||||
-ExpectedRuntimeFileCount 15 `
|
||||
-TaskFailureCount $taskFailures.Count `
|
||||
-RelayListenerCount $relayListeners.Count `
|
||||
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
|
||||
|
||||
@@ -24,6 +24,7 @@ $FixedRuntimeFiles = @(
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
"agent99-sre-alert-inbox.ps1",
|
||||
"agent99-sre-alert-relay.ps1",
|
||||
"agent99-submit-request.ps1",
|
||||
@@ -716,7 +717,7 @@ try {
|
||||
}
|
||||
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
|
||||
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 15) { throw "invalid_expected_runtime_file_count" }
|
||||
|
||||
$traceId = [string]$envelope.traceId
|
||||
$runId = [string]$envelope.runId
|
||||
@@ -751,7 +752,7 @@ try {
|
||||
$sourceManifest = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_source_manifest_v1"
|
||||
sourceRevision = $sourceRevision
|
||||
fileCount = 14
|
||||
fileCount = 15
|
||||
manifestSha256 = $manifestDigest
|
||||
files = @($FixedRuntimeFiles | ForEach-Object {
|
||||
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
|
||||
@@ -788,7 +789,7 @@ try {
|
||||
status = "check_ready"
|
||||
mode = "check"
|
||||
sourceRevision = $sourceRevision
|
||||
expectedRuntimeFileCount = 14
|
||||
expectedRuntimeFileCount = 15
|
||||
manifestSha256 = $manifestDigest
|
||||
plannedStagingPath = $stagePath
|
||||
runtimeManifest = $currentManifest
|
||||
@@ -875,7 +876,7 @@ try {
|
||||
if (
|
||||
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
|
||||
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
|
||||
[int]$existingSourceManifest.fileCount -ne 14 -or
|
||||
[int]$existingSourceManifest.fileCount -ne 15 -or
|
||||
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
|
||||
) { throw "existing_staging_manifest_mismatch" }
|
||||
} catch {
|
||||
@@ -899,7 +900,7 @@ try {
|
||||
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
|
||||
$live.runtimeMatched -and
|
||||
$live.sourceRevision -eq $sourceRevision -and
|
||||
$live.fileCount -eq 14 -and
|
||||
$live.fileCount -eq 15 -and
|
||||
$live.mismatchCount -eq 0 -and
|
||||
$liveLauncherContract.ok -and
|
||||
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
|
||||
@@ -1135,7 +1136,7 @@ try {
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 14 -and
|
||||
$runtimeManifest.fileCount -eq 15 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$launcherContract.ok -and
|
||||
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
|
||||
|
||||
@@ -19,6 +19,7 @@ RUNTIME_FILES=(
|
||||
"agent99-control-plane.ps1"
|
||||
"agent99-deploy.ps1"
|
||||
"agent99-register-tasks.ps1"
|
||||
"agent99-alertmanager-alertchain-poll.ps1"
|
||||
"agent99-sre-alert-inbox.ps1"
|
||||
"agent99-sre-alert-relay.ps1"
|
||||
"agent99-submit-request.ps1"
|
||||
@@ -234,7 +235,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
|
||||
output_path = Path(sys.argv[7])
|
||||
runtime_names = sys.argv[8:]
|
||||
|
||||
if len(runtime_names) != 14 or len(set(runtime_names)) != 14:
|
||||
if len(runtime_names) != 15 or len(set(runtime_names)) != 15:
|
||||
raise SystemExit("fixed_runtime_file_contract_failed")
|
||||
|
||||
files: list[dict[str, str]] = []
|
||||
@@ -273,7 +274,7 @@ envelope = {
|
||||
"runId": run_id,
|
||||
"workItemId": work_item_id,
|
||||
"sourceRevision": source_revision,
|
||||
"expectedRuntimeFileCount": 14,
|
||||
"expectedRuntimeFileCount": 15,
|
||||
"manifestSha256": manifest_sha256,
|
||||
"files": files,
|
||||
"livePreflight": {
|
||||
@@ -425,14 +426,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
|
||||
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
|
||||
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
|
||||
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 14-and$m.mismatchCount-eq 0-and$rows.Count-eq 14-and$seen.Count-eq 14-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 15-and$m.mismatchCount-eq 0-and$rows.Count-eq 15-and$seen.Count-eq 15-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
|
||||
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
|
||||
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
|
||||
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
|
||||
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
|
||||
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
|
||||
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=14;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=15;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
|
||||
if(-not$ready){exit 65}'''
|
||||
script = (
|
||||
script.replace("__SOURCE_REVISION__", source_revision)
|
||||
|
||||
@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
|
||||
"exists": True,
|
||||
"sourceRevision": "a" * 40,
|
||||
"runtimeMatched": True,
|
||||
"fileCount": 14,
|
||||
"fileCount": 15,
|
||||
"mismatchCount": 0,
|
||||
"parseError": "",
|
||||
},
|
||||
@@ -81,7 +81,7 @@ $case = @'
|
||||
$parameters = @{{
|
||||
AgentRootPresent = [bool]$case.agentRootPresent
|
||||
Manifest = $case.manifest
|
||||
ExpectedRuntimeFileCount = 14
|
||||
ExpectedRuntimeFileCount = 15
|
||||
TaskFailureCount = [int]$case.taskFailureCount
|
||||
RelayListenerCount = [int]$case.relayListenerCount
|
||||
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
|
||||
|
||||
@@ -25,6 +25,7 @@ EXPECTED_RUNTIME_FILES = (
|
||||
"agent99-control-plane.ps1",
|
||||
"agent99-deploy.ps1",
|
||||
"agent99-register-tasks.ps1",
|
||||
"agent99-alertmanager-alertchain-poll.ps1",
|
||||
"agent99-sre-alert-inbox.ps1",
|
||||
"agent99-sre-alert-relay.ps1",
|
||||
"agent99-submit-request.ps1",
|
||||
@@ -77,7 +78,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None
|
||||
assert sender_files == EXPECTED_RUNTIME_FILES
|
||||
assert receiver_files == EXPECTED_RUNTIME_FILES
|
||||
assert deployer_files == EXPECTED_RUNTIME_FILES
|
||||
assert "expectedRuntimeFileCount\": 14" in sender
|
||||
assert "expectedRuntimeFileCount\": 15" in sender
|
||||
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
|
||||
assert "required_runtime_file_missing" in receiver
|
||||
assert "duplicate_runtime_file" in receiver
|
||||
@@ -230,7 +231,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
|
||||
assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
|
||||
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
|
||||
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
|
||||
assert '$runtimeManifest.fileCount -eq 14' in source
|
||||
assert '$runtimeManifest.fileCount -eq 15' in source
|
||||
assert '$runtimeManifest.mismatchCount -eq 0' in source
|
||||
assert 'status = "deployed_verified"' in source
|
||||
assert '$failurePhase = "deploy"' in source
|
||||
@@ -699,8 +700,9 @@ def test_check_mode_uses_bounded_control_command_without_ssh_stdin(
|
||||
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
|
||||
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
|
||||
"printf '%040d\\n' 0; exit 0 ;;\n"
|
||||
" *\" cat-file -e \"*) exit 0 ;;\n"
|
||||
" *\" diff --quiet \"*) exit 0 ;;\n"
|
||||
" *\" cat-file -e \"*) exit 0 ;;\n"
|
||||
" *\" ls-files --error-unmatch \"*) exit 0 ;;\n"
|
||||
" *\" diff --quiet \"*) exit 0 ;;\n"
|
||||
"esac\n"
|
||||
f"exec {shlex.quote(real_git)} \"$@\"\n",
|
||||
encoding="utf-8",
|
||||
@@ -854,8 +856,9 @@ def test_apply_uses_ephemeral_file_transport_and_cleans_up_without_ssh_stdin(
|
||||
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
|
||||
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
|
||||
"printf '%040d\\n' 0; exit 0 ;;\n"
|
||||
" *\" cat-file -e \"*) exit 0 ;;\n"
|
||||
" *\" diff --quiet \"*) exit 0 ;;\n"
|
||||
" *\" cat-file -e \"*) exit 0 ;;\n"
|
||||
" *\" ls-files --error-unmatch \"*) exit 0 ;;\n"
|
||||
" *\" diff --quiet \"*) exit 0 ;;\n"
|
||||
"esac\n"
|
||||
f"exec {shlex.quote(real_git)} \"$@\"\n",
|
||||
encoding="utf-8",
|
||||
|
||||
@@ -189,19 +189,20 @@ SURFACES = [
|
||||
"requires_live_evidence": True,
|
||||
},
|
||||
{
|
||||
"surface_id": "ollama_proxy_gateway",
|
||||
"label": "Ollama proxy gateway / local fallback boundary",
|
||||
"expected_scope": "proxy route、local fallback、gateway health",
|
||||
"config_kind": "proxy_gateway",
|
||||
"surface_id": "ollama_provider_transport_boundary",
|
||||
"label": "Ollama direct route / mesh transport / local fallback boundary",
|
||||
"expected_scope": "GCP direct/mesh transport、host111 local boundary、provider health",
|
||||
"config_kind": "provider_transport",
|
||||
"source_refs": [
|
||||
"infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2",
|
||||
"ops/monitoring/service-registry.yaml",
|
||||
"k8s/awoooi-prod/04-configmap.yaml",
|
||||
"infra/ansible/playbooks/111-ollama-fallback.yml",
|
||||
"apps/api/src/services/ollama_endpoint_resolver.py",
|
||||
"apps/api/src/services/ollama_failover_manager.py",
|
||||
"apps/api/src/services/provider_proxy.py",
|
||||
],
|
||||
"write_capable_surface": True,
|
||||
"paid_provider_related": False,
|
||||
"data_egress_related": False,
|
||||
"data_egress_related": True,
|
||||
"requires_live_evidence": True,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -337,7 +337,8 @@ CATEGORIES = [
|
||||
"apps/api/src/services/ai_providers/**",
|
||||
"apps/api/src/services/**/*model*",
|
||||
"apps/api/src/services/**/*provider*",
|
||||
"infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2",
|
||||
"ops/monitoring/service-registry.yaml",
|
||||
"k8s/awoooi-prod/04-configmap.yaml",
|
||||
"docs/ai/**",
|
||||
"docs/**/*Ollama*",
|
||||
),
|
||||
|
||||
@@ -53,15 +53,6 @@ SOURCES = [
|
||||
control_tier="C0",
|
||||
owner_gate="public_tools_owner_response_required",
|
||||
),
|
||||
NginxSource(
|
||||
config_id="host110_ollama_proxy",
|
||||
host="192.168.0.110",
|
||||
role="ollama_proxy_gateway",
|
||||
source_path="infra/ansible/roles/nginx/templates/110-ollama-proxy.conf.j2",
|
||||
live_path="/etc/nginx/sites-enabled/110-ollama-proxy.conf",
|
||||
control_tier="C1",
|
||||
owner_gate="ai_provider_proxy_owner_response_required",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user