843 lines
33 KiB
Python
843 lines
33 KiB
Python
"""Independent asset-specific postconditions for controlled Ansible apply."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import time
|
|
from collections.abc import Awaitable, Callable, Mapping
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
from src.core.config import settings
|
|
|
|
SCHEMA_VERSION = "awoooi_ansible_asset_post_verifier_v1"
|
|
VERIFIER_ID = "asset_specific_read_only_host_postconditions"
|
|
INDEPENDENT_SOURCE = "broker_ssh_host_runtime_readback"
|
|
_SAFE_HOST_ALIAS_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
_SAFE_REMOTE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
_FIXED_PROXY_JUMP_BY_HOST = {
|
|
"host_111": "wooo@192.168.0.110",
|
|
}
|
|
_RUN_SCOPED_CLOSURE_BLOCKER_BY_CATALOG = {
|
|
"ansible:188-openclaw-callback-forwarder": (
|
|
"fresh_callback_receipt_run_scope_unavailable_in_static_postcondition_registry"
|
|
),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AssetPostcondition:
|
|
condition_id: str
|
|
condition_type: str
|
|
inventory_host: str
|
|
probe: str
|
|
timeout_seconds: int | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReadOnlyProbeResult:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
duration_ms: int
|
|
timed_out: bool = False
|
|
|
|
|
|
ProbeRunner = Callable[..., Awaitable[ReadOnlyProbeResult]]
|
|
|
|
|
|
_POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
|
|
"ansible:awoooi-auto-repair-canary": (
|
|
AssetPostcondition(
|
|
"host_121_awoooi_canary_rollout_ready",
|
|
"service",
|
|
"host_121",
|
|
"sudo -n /usr/local/bin/kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n awoooi-prod rollout status deployment/awoooi-auto-repair-canary --timeout=30s >/dev/null",
|
|
),
|
|
AssetPostcondition(
|
|
"host_121_awoooi_canary_ready_replicas",
|
|
"metric",
|
|
"host_121",
|
|
"test \"$(sudo -n /usr/local/bin/kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n awoooi-prod get deployment awoooi-auto-repair-canary -o jsonpath='{.status.readyReplicas}:{.status.replicas}')\" = '1:1'",
|
|
),
|
|
),
|
|
"ansible:wazuh-manager-posture-readback": (
|
|
AssetPostcondition(
|
|
"host_112_wazuh_manager_unit_loaded",
|
|
"security",
|
|
"host_112",
|
|
"systemctl show --property=LoadState --value wazuh-manager.service | grep -Fxq loaded",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_manager_runtime",
|
|
"service",
|
|
"host_112",
|
|
"systemctl is-active --quiet wazuh-manager.service",
|
|
),
|
|
),
|
|
"ansible:wazuh-alertmanager-integration": (
|
|
AssetPostcondition(
|
|
"host_112_wazuh_alertmanager_script",
|
|
"security",
|
|
"host_112",
|
|
"test -r /var/lib/awoooi-wazuh-alert-ingress/receipt.env && "
|
|
"grep -Fqx 'integration_name=custom-awoooi-alertmanager' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_alertmanager_config",
|
|
"security",
|
|
"host_112",
|
|
"grep -Fqx 'config_present=1' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env && "
|
|
"grep -Fqx 'raw_payload_persisted=0' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env && "
|
|
"grep -Fqx 'active_response_enabled=0' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_manager_runtime_after_ingress_convergence",
|
|
"service",
|
|
"host_112",
|
|
"systemctl is-active --quiet wazuh-manager.service",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_integrator_runtime",
|
|
"service",
|
|
"host_112",
|
|
"pgrep -x wazuh-integratord >/dev/null && "
|
|
"grep -Fqx 'integrator_running=1' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_awoooi_alertmanager_ingress_reachable",
|
|
"network",
|
|
"host_112",
|
|
"curl -fsS --max-time 10 https://awoooi.wooo.work/api/v1/webhooks/health >/dev/null",
|
|
),
|
|
),
|
|
"ansible:110-host-pressure-readonly": (
|
|
AssetPostcondition(
|
|
"host_110_pressure_controller_asset",
|
|
"service",
|
|
"host_110",
|
|
"test -x /home/wooo/scripts/host-sustained-load-controller.py",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_gitea_pressure_probe_asset",
|
|
"service",
|
|
"host_110",
|
|
"test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_pressure_metrics_freshness",
|
|
"metric",
|
|
"host_110",
|
|
"test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_docker_metrics_freshness",
|
|
"metric",
|
|
"host_110",
|
|
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
|
|
),
|
|
),
|
|
"ansible:110-alertmanager-delivery-recovery": (
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_container_running",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Running}}' alertmanager "
|
|
"2>/dev/null)\" = true",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_config_valid",
|
|
"configuration",
|
|
"host_110",
|
|
"docker exec alertmanager amtool check-config "
|
|
"/etc/alertmanager/alertmanager.yml >/dev/null",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_awoooi_webhook_network_boundary",
|
|
"network",
|
|
"host_110",
|
|
"code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 -I "
|
|
"https://awoooi.wooo.work/api/v1/webhooks/alertmanager); "
|
|
"test \"$code\" = 405 -o \"$code\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_runtime_target_up",
|
|
"metric",
|
|
"host_110",
|
|
"curl -fsS --get --data-urlencode "
|
|
"'query=up{job=\"alertmanager-runtime\"}' "
|
|
"http://127.0.0.1:9090/api/v1/query | python3 -c '"
|
|
"import json,sys; p=json.load(sys.stdin); r=p.get(\"data\",{}).get(\"result\",[]); "
|
|
"raise SystemExit(0 if p.get(\"status\")==\"success\" and len(r)==1 "
|
|
"and r[0].get(\"value\",[None,None])[1]==\"1\" else 1)'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_delivery_rule_healthy",
|
|
"metric",
|
|
"host_110",
|
|
"curl -fsS 'http://127.0.0.1:9090/api/v1/rules?type=alert' "
|
|
"| python3 -c 'import json,sys; p=json.load(sys.stdin); "
|
|
"rules=[r for g in p.get(\"data\",{}).get(\"groups\",[]) "
|
|
"for r in g.get(\"rules\",[]) "
|
|
"if r.get(\"name\")==\"AlertChainBroken_Alertmanager\"]; "
|
|
"raise SystemExit(0 if p.get(\"status\")==\"success\" and len(rules)==1 "
|
|
"and rules[0].get(\"health\")==\"ok\" else 1)'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_delivery_alert_resolved_after_window",
|
|
"metric",
|
|
"host_110",
|
|
"for attempt in $(seq 1 72); do "
|
|
"curl -fsS --get --data-urlencode "
|
|
"'query=ALERTS{alertname=\"AlertChainBroken_Alertmanager\","
|
|
"alertstate=\"firing\"}' http://127.0.0.1:9090/api/v1/query "
|
|
"| python3 -c 'import json,sys; p=json.load(sys.stdin); "
|
|
"raise SystemExit(0 if p.get(\"status\")==\"success\" "
|
|
"and p.get(\"data\",{}).get(\"result\")==[] else 1)' "
|
|
"&& exit 0; sleep 10; done; exit 1",
|
|
740,
|
|
),
|
|
),
|
|
"ansible:111-ollama-fallback": (
|
|
AssetPostcondition(
|
|
"host_111_ollama_launchagent_allowlist_exact",
|
|
"configuration",
|
|
"host_111",
|
|
"test \"$(/usr/libexec/PlistBuddy -c "
|
|
"'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' "
|
|
"/Users/ooo/Library/LaunchAgents/"
|
|
"com.momo.ollama111-allow-proxy.plist)\" = "
|
|
"'127.0.0.1/32,192.168.0.111/32,192.168.0.188/32,"
|
|
"192.168.0.120/32,192.168.0.121/32'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_111_ollama_local_tags_api",
|
|
"service",
|
|
"host_111",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://127.0.0.1:11434/api/tags)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_120_to_host111_ollama_tags_api",
|
|
"network",
|
|
"host_120",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://192.168.0.111:11434/api/tags)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_121_to_host111_ollama_tags_api",
|
|
"network",
|
|
"host_121",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://192.168.0.111:11434/api/tags)\" = 200",
|
|
),
|
|
),
|
|
"ansible:188-openclaw-callback-forwarder": (
|
|
AssetPostcondition(
|
|
"host_188_openclaw_overlay_manifest_identity",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"root=/home/ollama/.local/share/awoooi-controlled-apply/"
|
|
"openclaw-callback-forwarder/"
|
|
"payloads/8f367f8e7d6104c44b29bd5fa2d894db09ddf365 && "
|
|
"manifest=\"$root/manifest.json\" && "
|
|
"test -d \"$root\" && test ! -L \"$root\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$root\")\" = 'ollama:ollama:755' && "
|
|
"test -f \"$manifest\" && test ! -L \"$manifest\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$manifest\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$manifest\" | cut -d' ' -f1)\" = "
|
|
"0e5edaf700d264b44cc0eae3b9429af642be0b1563a0bffa633942d30d2c2e1c",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_callback_overlay_enabled",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"wd=$(systemctl show clawbot.service --property=WorkingDirectory "
|
|
"--value) && "
|
|
"test \"$wd\" = '/home/ollama/clawbot-v5' && "
|
|
"override=\"$wd/docker-compose.override.yml\" && "
|
|
"test -f \"$override\" && test ! -L \"$override\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$override\")\" = "
|
|
"'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$override\" | cut -d' ' -f1)\" = "
|
|
"33c4594d4e4092b2a2367583a67f4c9a82ead0d28ca5078eca4896276084da0d",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_callback_payload_runtime",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"root=/home/ollama/.local/share/awoooi-controlled-apply/"
|
|
"openclaw-callback-forwarder/"
|
|
"payloads/8f367f8e7d6104c44b29bd5fa2d894db09ddf365 && "
|
|
"telegram=\"$root/app/bot/telegram.py\" && "
|
|
"config=\"$root/app/core/config.py\" && "
|
|
"forwarder=\"$root/app/services/telegram_callback_forwarder.py\" && "
|
|
"test -f \"$telegram\" && test ! -L \"$telegram\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$telegram\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$telegram\" | cut -d' ' -f1)\" = "
|
|
"af9600f32305ae73184651a8d3437063d7c8d35e3c3eb0910a2e27ac874c9302 && "
|
|
"test -f \"$config\" && test ! -L \"$config\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$config\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$config\" | cut -d' ' -f1)\" = "
|
|
"e1b8b633f7b4780d3078d6c42a55e49d200255d3597a34e0cd2fd649b84eaea1 && "
|
|
"test -f \"$forwarder\" && test ! -L \"$forwarder\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$forwarder\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$forwarder\" | cut -d' ' -f1)\" = "
|
|
"8e8e0faf6322abb21d670ad9094f1805d7a4b3643da8fe88d74a8c9591066bf3 && "
|
|
"ids=$(docker ps -q "
|
|
"--filter label=com.docker.compose.project=clawbot "
|
|
"--filter label=com.docker.compose.service=clawbot) && "
|
|
"test \"$(printf '%s\\n' \"$ids\" | sed '/^$/d' | wc -l | "
|
|
"tr -d ' ')\" = 1 && "
|
|
"id=\"$ids\" && "
|
|
"test \"$(docker inspect --format='{{.Config.WorkingDir}}' \"$id\")\" "
|
|
"= '/app' && "
|
|
"test \"$(docker exec \"$ids\" sha256sum "
|
|
"/app/app/bot/telegram.py | cut -d' ' -f1)\" = "
|
|
"af9600f32305ae73184651a8d3437063d7c8d35e3c3eb0910a2e27ac874c9302 && "
|
|
"test \"$(docker exec \"$ids\" sha256sum "
|
|
"/app/app/core/config.py | cut -d' ' -f1)\" = "
|
|
"e1b8b633f7b4780d3078d6c42a55e49d200255d3597a34e0cd2fd649b84eaea1 && "
|
|
"test \"$(docker exec \"$ids\" sha256sum "
|
|
"/app/app/services/telegram_callback_forwarder.py | cut -d' ' -f1)\" = "
|
|
"8e8e0faf6322abb21d670ad9094f1805d7a4b3643da8fe88d74a8c9591066bf3 && "
|
|
"test \"$(docker inspect --format='{{range .Mounts}}{{if eq "
|
|
".Destination \"/app/app/bot/telegram.py\"}}{{.Source}}|"
|
|
"{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' \"$id\")\" = "
|
|
"\"$telegram|/app/app/bot/telegram.py|bind|false\" && "
|
|
"test \"$(docker inspect --format='{{range .Mounts}}{{if eq "
|
|
".Destination \"/app/app/core/config.py\"}}{{.Source}}|"
|
|
"{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' \"$id\")\" = "
|
|
"\"$config|/app/app/core/config.py|bind|false\" && "
|
|
"test \"$(docker inspect --format='{{range .Mounts}}{{if eq "
|
|
".Destination \"/app/app/services/telegram_callback_forwarder.py\"}}"
|
|
"{{.Source}}|{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' "
|
|
"\"$id\")\" = \"$forwarder|/app/app/services/"
|
|
"telegram_callback_forwarder.py|bind|false\"",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_callback_runtime_setting",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"ids=$(docker ps -q "
|
|
"--filter label=com.docker.compose.project=clawbot "
|
|
"--filter label=com.docker.compose.service=clawbot) && "
|
|
"test \"$(printf '%s\\n' \"$ids\" | sed '/^$/d' | wc -l | "
|
|
"tr -d ' ')\" = 1 && "
|
|
"docker exec \"$ids\" python -c \"import os; "
|
|
"from app.services.telegram_callback_forwarder import "
|
|
"TelegramCallbackForwarder; "
|
|
"TELEGRAM_CALLBACK_FORWARD_ENABLED = "
|
|
"os.environ.get('TELEGRAM_CALLBACK_FORWARD_ENABLED') == 'true'; "
|
|
"assert TELEGRAM_CALLBACK_FORWARD_ENABLED is True; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_URL') == "
|
|
"'https://awoooi.wooo.work/api/v1/telegram/callback-forward'; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_ALLOWED_HOSTS') == "
|
|
"'awoooi.wooo.work'; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_TIMEOUT_SECONDS') == "
|
|
"'3.0'; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_MAX_BODY_BYTES') == "
|
|
"'16384'\"",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_clawbot_systemd_runtime_active",
|
|
"service",
|
|
"host_188",
|
|
"set -euo pipefail; systemctl is-active --quiet clawbot.service",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_local_health",
|
|
"service",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://127.0.0.1:8088/health)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_awoooi_callback_receiver_public_health",
|
|
"network",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"https://awoooi.wooo.work/api/v1/telegram/health)\" = 200",
|
|
),
|
|
),
|
|
"ansible:110-disk-pressure": (
|
|
AssetPostcondition(
|
|
"host_110_root_disk_below_alert_threshold",
|
|
"metric",
|
|
"host_110",
|
|
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
|
|
"test -n \"$usage\" && test \"$usage\" -lt 85",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_docker_runtime_after_cleanup",
|
|
"service",
|
|
"host_110",
|
|
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
|
|
),
|
|
),
|
|
"ansible:188-disk-pressure": (
|
|
AssetPostcondition(
|
|
"host_188_root_disk_below_alert_threshold",
|
|
"metric",
|
|
"host_188",
|
|
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
|
|
"test -n \"$usage\" && test \"$usage\" -lt 85",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_docker_runtime_after_cleanup",
|
|
"service",
|
|
"host_188",
|
|
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
|
|
),
|
|
),
|
|
"ansible:110-devops": (
|
|
AssetPostcondition(
|
|
"host_110_node_exporter_container",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Running}}' node-exporter 2>/dev/null)\" = true",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_node_exporter_metrics",
|
|
"metric",
|
|
"host_110",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "
|
|
"http://127.0.0.1:9100/metrics)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_node_exporter_prometheus_scrape",
|
|
"metric",
|
|
"host_110",
|
|
"for attempt in 1 2 3 4 5 6; do "
|
|
"curl -fsS --max-time 5 "
|
|
"'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' "
|
|
"| grep -Eq '\"value\":\\[[^]]+,\"1\"\\]' && exit 0; "
|
|
"sleep 5; done; exit 1",
|
|
),
|
|
),
|
|
"ansible:110-sentry-profiling-consumer-recovery": (
|
|
AssetPostcondition(
|
|
"host_110_sentry_profiling_consumer_running",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Running}}' "
|
|
"sentry-self-hosted-snuba-profiling-functions-consumer-1 "
|
|
"2>/dev/null)\" = true",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_sentry_profiling_consumer_healthy",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Health.Status}}' "
|
|
"sentry-self-hosted-snuba-profiling-functions-consumer-1 "
|
|
"2>/dev/null)\" = healthy",
|
|
),
|
|
),
|
|
"ansible:110-devops-full-convergence": (
|
|
AssetPostcondition(
|
|
"host_110_docker_runtime",
|
|
"service",
|
|
"host_110",
|
|
"docker info >/dev/null 2>&1",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_harbor_route",
|
|
"route",
|
|
"host_110",
|
|
"status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://127.0.0.1:5000/v2/); "
|
|
"[ \"$status\" = 200 ] || [ \"$status\" = 401 ]",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_backup_assets",
|
|
"data",
|
|
"host_110",
|
|
"test -d /backup/scripts && find /backup/scripts -maxdepth 1 -type f | grep -q .",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_monitoring_guard",
|
|
"metric",
|
|
"host_110",
|
|
"test -x /home/wooo/scripts/prometheus-rule-drift-guard.sh",
|
|
),
|
|
),
|
|
"ansible:188-momo-backup-user": (
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_script",
|
|
"service",
|
|
"host_188",
|
|
"test -x /home/ollama/bin/momo-pg-backup.sh && bash -n /home/ollama/bin/momo-pg-backup.sh",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_schedule",
|
|
"scheduler",
|
|
"host_188",
|
|
"crontab -l | grep -Fq /home/ollama/bin/momo-pg-backup.sh",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_legacy_schedule_absent",
|
|
"safety",
|
|
"host_188",
|
|
"! crontab -l | grep -Fq /home/ollama/momo-pro/scripts/pg_backup.sh",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_directory",
|
|
"data",
|
|
"host_188",
|
|
"test -d /home/ollama/momo_backups",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_metric",
|
|
"metric",
|
|
"host_188",
|
|
"test -s /home/ollama/node_exporter_textfiles/backup_health.prom && grep -Fq momo_pg_daily /home/ollama/node_exporter_textfiles/backup_health.prom",
|
|
),
|
|
),
|
|
"ansible:188-ai-web": (
|
|
AssetPostcondition(
|
|
"host_188_openclaw_runtime",
|
|
"service",
|
|
"host_188",
|
|
"docker ps -q --filter name=openclaw | grep -q . || docker ps -q --filter name=clawbot | grep -q .",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_runtime",
|
|
"service",
|
|
"host_188",
|
|
"docker ps -q --filter name=momo | grep -q .",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_nginx_keepalived_runtime",
|
|
"route",
|
|
"host_188",
|
|
"systemctl is-active --quiet nginx && systemctl is-active --quiet keepalived",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_vip_runtime",
|
|
"metric",
|
|
"host_188",
|
|
"ip addr show | grep -Fq 192.168.0.200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_backup_assets",
|
|
"data",
|
|
"host_188",
|
|
"test -x /home/ollama/bin/momo-pg-backup.sh && test -d /home/ollama/momo_backups",
|
|
),
|
|
),
|
|
"ansible:nginx-sync": (
|
|
AssetPostcondition(
|
|
"host_188_nginx_runtime",
|
|
"service",
|
|
"host_188",
|
|
"systemctl is-active --quiet nginx",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_nginx_routes",
|
|
"route",
|
|
"host_188",
|
|
"test -s /etc/nginx/sites-enabled/all-sites.conf && test -s /etc/nginx/sites-enabled/188-internal-tools-https.conf",
|
|
),
|
|
),
|
|
"ansible:restore-password-auth": tuple(
|
|
condition
|
|
for host in ("host_110", "host_120", "host_121", "host_188")
|
|
for condition in (
|
|
AssetPostcondition(
|
|
f"{host}_password_auth_config",
|
|
"security",
|
|
host,
|
|
"grep -Eiq '^[[:space:]]*PasswordAuthentication[[:space:]]+yes' /etc/ssh/sshd_config",
|
|
),
|
|
AssetPostcondition(
|
|
f"{host}_sshd_runtime",
|
|
"service",
|
|
host,
|
|
"systemctl is-active --quiet ssh || systemctl is-active --quiet sshd",
|
|
),
|
|
)
|
|
),
|
|
}
|
|
|
|
|
|
def registered_catalog_ids() -> frozenset[str]:
|
|
return frozenset(_POSTCONDITION_REGISTRY)
|
|
|
|
|
|
def postconditions_for_catalog(catalog_id: str) -> tuple[AssetPostcondition, ...]:
|
|
return _POSTCONDITION_REGISTRY.get(str(catalog_id or ""), ())
|
|
|
|
|
|
def _playbook_roots(module_path: Path | None = None) -> list[Path]:
|
|
resolved_module_path = (module_path or Path(__file__)).resolve()
|
|
return [
|
|
Path("/app/infra/ansible"),
|
|
Path.cwd() / "infra" / "ansible",
|
|
*(parent / "infra" / "ansible" for parent in resolved_module_path.parents),
|
|
]
|
|
|
|
|
|
def _load_inventory_hosts(inventory_path: Path) -> dict[str, dict[str, str]]:
|
|
payload = yaml.safe_load(inventory_path.read_text(encoding="utf-8")) or {}
|
|
children = ((payload.get("all") or {}).get("children") or {})
|
|
hosts: dict[str, dict[str, str]] = {}
|
|
for group in children.values():
|
|
if not isinstance(group, Mapping):
|
|
continue
|
|
for alias, config in (group.get("hosts") or {}).items():
|
|
if not isinstance(config, Mapping):
|
|
continue
|
|
hosts[str(alias)] = {
|
|
"ansible_host": str(config.get("ansible_host") or ""),
|
|
"ansible_user": str(config.get("ansible_user") or ""),
|
|
"ansible_ssh_common_args": str(
|
|
config.get("ansible_ssh_common_args") or ""
|
|
),
|
|
}
|
|
return hosts
|
|
|
|
|
|
def build_read_only_probe_command(
|
|
condition: AssetPostcondition,
|
|
*,
|
|
inventory_path: Path,
|
|
ssh_key_path: Path,
|
|
known_hosts_path: Path,
|
|
) -> list[str]:
|
|
if not _SAFE_HOST_ALIAS_RE.fullmatch(condition.inventory_host):
|
|
raise ValueError("unsafe_postcondition_inventory_host")
|
|
inventory_hosts = _load_inventory_hosts(inventory_path)
|
|
target = inventory_hosts.get(condition.inventory_host) or {}
|
|
host = str(target.get("ansible_host") or "")
|
|
user = str(target.get("ansible_user") or "")
|
|
if not _SAFE_REMOTE_ID_RE.fullmatch(host) or not _SAFE_REMOTE_ID_RE.fullmatch(user):
|
|
raise ValueError("postcondition_inventory_target_missing")
|
|
command = [
|
|
"ssh",
|
|
"-i",
|
|
str(ssh_key_path),
|
|
"-o",
|
|
f"UserKnownHostsFile={known_hosts_path}",
|
|
"-o",
|
|
"StrictHostKeyChecking=yes",
|
|
"-o",
|
|
"IdentitiesOnly=yes",
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"ConnectTimeout=10",
|
|
]
|
|
proxy_jump = _FIXED_PROXY_JUMP_BY_HOST.get(condition.inventory_host)
|
|
if proxy_jump is not None:
|
|
expected_common_args = (
|
|
f"-o ProxyJump={proxy_jump} -o StrictHostKeyChecking=yes"
|
|
)
|
|
if target.get("ansible_ssh_common_args") != expected_common_args:
|
|
raise ValueError("postcondition_proxy_jump_policy_mismatch")
|
|
command.extend(["-o", f"ProxyJump={proxy_jump}"])
|
|
command.extend([f"{user}@{host}", condition.probe])
|
|
return command
|
|
|
|
|
|
async def _run_read_only_probe(
|
|
command: list[str],
|
|
*,
|
|
timeout_seconds: int,
|
|
) -> ReadOnlyProbeResult:
|
|
started = time.monotonic()
|
|
process = await asyncio.create_subprocess_exec(
|
|
*command,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
timed_out = False
|
|
try:
|
|
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
process.communicate(),
|
|
timeout=timeout_seconds,
|
|
)
|
|
except TimeoutError:
|
|
timed_out = True
|
|
process.kill()
|
|
stdout_bytes, stderr_bytes = await process.communicate()
|
|
except asyncio.CancelledError:
|
|
process.kill()
|
|
await process.communicate()
|
|
raise
|
|
return ReadOnlyProbeResult(
|
|
returncode=124 if timed_out else int(process.returncode or 0),
|
|
stdout=stdout_bytes.decode("utf-8", "replace"),
|
|
stderr=stderr_bytes.decode("utf-8", "replace"),
|
|
duration_ms=int((time.monotonic() - started) * 1000),
|
|
timed_out=timed_out,
|
|
)
|
|
|
|
|
|
def _output_fingerprint(result: ReadOnlyProbeResult) -> str:
|
|
digest = hashlib.sha256()
|
|
digest.update(result.stdout.encode("utf-8", "replace"))
|
|
digest.update(b"\0")
|
|
digest.update(result.stderr.encode("utf-8", "replace"))
|
|
return digest.hexdigest()
|
|
|
|
|
|
async def run_ansible_asset_post_verifier(
|
|
*,
|
|
catalog_id: str,
|
|
automation_run_id: str,
|
|
apply_op_id: str,
|
|
inventory_hosts: tuple[str, ...],
|
|
executor_returncode: int,
|
|
playbook_root: Path | None = None,
|
|
ssh_key_path: Path | None = None,
|
|
known_hosts_path: Path | None = None,
|
|
timeout_seconds: int = 20,
|
|
probe_runner: ProbeRunner = _run_read_only_probe,
|
|
) -> dict[str, Any]:
|
|
conditions = postconditions_for_catalog(catalog_id)
|
|
run_scoped_closure_blocker = _RUN_SCOPED_CLOSURE_BLOCKER_BY_CATALOG.get(
|
|
catalog_id
|
|
)
|
|
source_sha = os.getenv("AWOOOI_BUILD_COMMIT_SHA", "").strip().lower()
|
|
base = {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"verifier": VERIFIER_ID,
|
|
"independent_source": INDEPENDENT_SOURCE,
|
|
"automation_run_id": automation_run_id,
|
|
"apply_op_id": apply_op_id,
|
|
"catalog_id": catalog_id,
|
|
"verifier_source_sha": source_sha or None,
|
|
"executor_returncode": executor_returncode,
|
|
"executor_returncode_trusted": False,
|
|
"raw_output_stored": False,
|
|
"command_exposed": False,
|
|
"writes_on_verify": False,
|
|
"runtime_closure_receipt_required": bool(run_scoped_closure_blocker),
|
|
"runtime_closure_receipt_verified": (
|
|
False if run_scoped_closure_blocker else None
|
|
),
|
|
}
|
|
blockers: list[str] = []
|
|
if executor_returncode != 0:
|
|
blockers.append("executor_apply_failed_before_post_verifier")
|
|
if not conditions:
|
|
blockers.append("asset_postcondition_registry_missing")
|
|
condition_hosts = {condition.inventory_host for condition in conditions}
|
|
if not condition_hosts.issubset(set(inventory_hosts)):
|
|
blockers.append("asset_postcondition_target_outside_claim")
|
|
|
|
root = playbook_root or next(
|
|
(path for path in _playbook_roots() if path.exists()),
|
|
None,
|
|
)
|
|
inventory_path = (root / "inventory" / "hosts.yml") if root else None
|
|
resolved_key_path = ssh_key_path or Path(
|
|
settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH
|
|
)
|
|
resolved_known_hosts_path = known_hosts_path or Path(
|
|
settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH
|
|
)
|
|
if root is None or inventory_path is None or not inventory_path.is_file():
|
|
blockers.append("post_verifier_inventory_missing")
|
|
if not resolved_key_path.is_file() or not os.access(resolved_key_path, os.R_OK):
|
|
blockers.append("post_verifier_ssh_key_missing")
|
|
if not resolved_known_hosts_path.is_file() or not os.access(
|
|
resolved_known_hosts_path,
|
|
os.R_OK,
|
|
):
|
|
blockers.append("post_verifier_known_hosts_missing")
|
|
if blockers:
|
|
if run_scoped_closure_blocker:
|
|
blockers.append(run_scoped_closure_blocker)
|
|
return {
|
|
**base,
|
|
"verification_result": "failed",
|
|
"all_postconditions_passed": False,
|
|
"technical_postconditions_passed": False,
|
|
"required_postcondition_count": len(conditions),
|
|
"passed_postcondition_count": 0,
|
|
"postconditions": [],
|
|
"active_blockers": blockers,
|
|
}
|
|
|
|
receipts: list[dict[str, Any]] = []
|
|
for condition in conditions:
|
|
try:
|
|
command = build_read_only_probe_command(
|
|
condition,
|
|
inventory_path=inventory_path,
|
|
ssh_key_path=resolved_key_path,
|
|
known_hosts_path=resolved_known_hosts_path,
|
|
)
|
|
result = await probe_runner(
|
|
command,
|
|
timeout_seconds=max(
|
|
1,
|
|
condition.timeout_seconds or timeout_seconds,
|
|
),
|
|
)
|
|
except Exception as exc:
|
|
receipts.append({
|
|
"condition_id": condition.condition_id,
|
|
"condition_type": condition.condition_type,
|
|
"asset_id": condition.inventory_host,
|
|
"passed": False,
|
|
"returncode": None,
|
|
"timed_out": False,
|
|
"duration_ms": 0,
|
|
"output_sha256": None,
|
|
"error_type": type(exc).__name__,
|
|
"raw_output_stored": False,
|
|
})
|
|
continue
|
|
receipts.append({
|
|
"condition_id": condition.condition_id,
|
|
"condition_type": condition.condition_type,
|
|
"asset_id": condition.inventory_host,
|
|
"passed": result.returncode == 0 and not result.timed_out,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"duration_ms": result.duration_ms,
|
|
"output_sha256": _output_fingerprint(result),
|
|
"error_type": None,
|
|
"raw_output_stored": False,
|
|
})
|
|
passed_count = sum(receipt["passed"] is True for receipt in receipts)
|
|
if passed_count != len(conditions):
|
|
blockers.extend(
|
|
f"postcondition_failed:{receipt['condition_id']}"
|
|
for receipt in receipts
|
|
if receipt["passed"] is not True
|
|
)
|
|
technical_verified = bool(conditions and passed_count == len(conditions))
|
|
if run_scoped_closure_blocker:
|
|
blockers.append(run_scoped_closure_blocker)
|
|
verified = bool(technical_verified and not blockers)
|
|
return {
|
|
**base,
|
|
"verification_result": "success" if verified else "failed",
|
|
"all_postconditions_passed": verified,
|
|
"technical_postconditions_passed": technical_verified,
|
|
"required_postcondition_count": len(conditions),
|
|
"passed_postcondition_count": passed_count,
|
|
"postconditions": receipts,
|
|
"active_blockers": blockers,
|
|
}
|