"""Independent asset-specific postconditions for controlled Ansible apply.""" from __future__ import annotations import asyncio import hashlib import os import re import shlex import time from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any import yaml from src.core.config import settings from src.services.telegram_callback_correlation import ( controlled_callback_correlation_fingerprint, ) SCHEMA_VERSION = "awoooi_ansible_asset_post_verifier_v1" VERIFIER_ID = "asset_specific_read_only_host_postconditions" INDEPENDENT_SOURCE = "broker_ssh_host_runtime_readback" _SAFE_HOST_ALIAS_RE = re.compile(r"^[A-Za-z0-9_.-]+$") _SAFE_REMOTE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") _FIXED_PROXY_TARGET_BY_HOST = { "host_111": "wooo@192.168.0.110", } _INVENTORY_PROXY_KEY_PATH = Path("/run/secrets/ssh_mcp_key") _INVENTORY_PROXY_KNOWN_HOSTS_PATH = Path("/etc/ssh-mcp/known_hosts") _RUNTIME_CLOSURE_RECEIPT_CATALOGS = frozenset( {"ansible:188-openclaw-callback-forwarder"} ) _RUNTIME_RECEIPT_PENDING_BLOCKER = "fresh_callback_action_receipt_pending_or_mismatch" _RUNTIME_CORRELATION_BLOCKER = "controlled_callback_correlation_missing_or_invalid" @dataclass(frozen=True) class AssetPostcondition: condition_id: str condition_type: str inventory_host: str probe: str timeout_seconds: int | None = None @dataclass(frozen=True) class ReadOnlyProbeResult: returncode: int stdout: str stderr: str duration_ms: int timed_out: bool = False ProbeRunner = Callable[..., Awaitable[ReadOnlyProbeResult]] _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { "ansible:awoooi-auto-repair-canary": ( AssetPostcondition( "host_121_awoooi_canary_rollout_ready", "service", "host_121", "sudo -n /usr/local/bin/kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n awoooi-prod rollout status deployment/awoooi-auto-repair-canary --timeout=30s >/dev/null", ), AssetPostcondition( "host_121_awoooi_canary_ready_replicas", "metric", "host_121", "test \"$(sudo -n /usr/local/bin/kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n awoooi-prod get deployment awoooi-auto-repair-canary -o jsonpath='{.status.readyReplicas}:{.status.replicas}')\" = '1:1'", ), ), "ansible:wazuh-manager-posture-readback": ( AssetPostcondition( "host_112_wazuh_manager_unit_loaded", "security", "host_112", "systemctl show --property=LoadState --value wazuh-manager.service | grep -Fxq loaded", ), AssetPostcondition( "host_112_wazuh_manager_runtime", "service", "host_112", "systemctl is-active --quiet wazuh-manager.service", ), ), "ansible:wazuh-alertmanager-integration": ( AssetPostcondition( "host_112_wazuh_alertmanager_script", "security", "host_112", "test -r /var/lib/awoooi-wazuh-alert-ingress/receipt.env && " "grep -Fqx 'integration_name=custom-awoooi-alertmanager' " "/var/lib/awoooi-wazuh-alert-ingress/receipt.env", ), AssetPostcondition( "host_112_wazuh_alertmanager_config", "security", "host_112", "grep -Fqx 'config_present=1' " "/var/lib/awoooi-wazuh-alert-ingress/receipt.env && " "grep -Fqx 'raw_payload_persisted=0' " "/var/lib/awoooi-wazuh-alert-ingress/receipt.env && " "grep -Fqx 'active_response_enabled=0' " "/var/lib/awoooi-wazuh-alert-ingress/receipt.env", ), AssetPostcondition( "host_112_wazuh_manager_runtime_after_ingress_convergence", "service", "host_112", "systemctl is-active --quiet wazuh-manager.service", ), AssetPostcondition( "host_112_wazuh_integrator_runtime", "service", "host_112", "pgrep -x wazuh-integratord >/dev/null && " "grep -Fqx 'integrator_running=1' " "/var/lib/awoooi-wazuh-alert-ingress/receipt.env", ), AssetPostcondition( "host_112_awoooi_alertmanager_ingress_reachable", "network", "host_112", "curl -fsS --max-time 10 https://awoooi.wooo.work/api/v1/webhooks/health >/dev/null", ), ), "ansible:110-host-pressure-readonly": ( AssetPostcondition( "host_110_pressure_controller_asset", "service", "host_110", "test -x /home/wooo/scripts/host-sustained-load-controller.py", ), AssetPostcondition( "host_110_gitea_pressure_probe_asset", "service", "host_110", "test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py", ), AssetPostcondition( "host_110_pressure_metrics_freshness", "metric", "host_110", "test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom", ), AssetPostcondition( "host_110_docker_metrics_freshness", "metric", "host_110", "test -s /home/wooo/node_exporter_textfiles/docker_stats.prom", ), ), "ansible:110-alertmanager-delivery-recovery": ( AssetPostcondition( "host_110_alertmanager_container_running", "service", "host_110", "test \"$(docker inspect --format '{{.State.Running}}' alertmanager " "2>/dev/null)\" = true", ), AssetPostcondition( "host_110_alertmanager_config_valid", "configuration", "host_110", "docker exec alertmanager amtool check-config " "/etc/alertmanager/alertmanager.yml >/dev/null", ), AssetPostcondition( "host_110_awoooi_webhook_network_boundary", "network", "host_110", "code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 -I " "https://awoooi.wooo.work/api/v1/webhooks/alertmanager); " "test \"$code\" = 405 -o \"$code\" = 200", ), AssetPostcondition( "host_110_alertmanager_runtime_target_up", "metric", "host_110", "curl -fsS --get --data-urlencode " "'query=up{job=\"alertmanager-runtime\"}' " "http://127.0.0.1:9090/api/v1/query | python3 -c '" "import json,sys; p=json.load(sys.stdin); r=p.get(\"data\",{}).get(\"result\",[]); " "raise SystemExit(0 if p.get(\"status\")==\"success\" and len(r)==1 " "and r[0].get(\"value\",[None,None])[1]==\"1\" else 1)'", ), AssetPostcondition( "host_110_alertmanager_delivery_rule_healthy", "metric", "host_110", "curl -fsS 'http://127.0.0.1:9090/api/v1/rules?type=alert' " "| python3 -c 'import json,sys; p=json.load(sys.stdin); " "rules=[r for g in p.get(\"data\",{}).get(\"groups\",[]) " "for r in g.get(\"rules\",[]) " "if r.get(\"name\")==\"AlertChainBroken_Alertmanager\"]; " "raise SystemExit(0 if p.get(\"status\")==\"success\" and len(rules)==1 " "and rules[0].get(\"health\")==\"ok\" else 1)'", ), AssetPostcondition( "host_110_alertmanager_delivery_alert_resolved_after_window", "metric", "host_110", "for attempt in $(seq 1 72); do " "curl -fsS --get --data-urlencode " "'query=ALERTS{alertname=\"AlertChainBroken_Alertmanager\"," "alertstate=\"firing\"}' http://127.0.0.1:9090/api/v1/query " "| python3 -c 'import json,sys; p=json.load(sys.stdin); " "raise SystemExit(0 if p.get(\"status\")==\"success\" " "and p.get(\"data\",{}).get(\"result\")==[] else 1)' " "&& exit 0; sleep 10; done; exit 1", 740, ), ), "ansible:110-telegram-callback-ingress-sensor": ( AssetPostcondition( "host_110_telegram_callback_ingress_rule_healthy", "metric", "host_110", "curl -fsS --max-time 5 http://127.0.0.1:9090/api/v1/rules | " "python3 -c 'import json,sys; d=json.load(sys.stdin); " "r=[x for g in d.get(\"data\",{}).get(\"groups\",[]) " "for x in g.get(\"rules\",[]) " "if x.get(\"name\")==\"TelegramCallbackIngressUnverified\"]; " "assert len(r)==1 and r[0].get(\"health\")==\"ok\"; " "q=\"\".join(str(r[0].get(\"query\") or \"\").split()); " "assert \"awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds\" in q; " "assert \">300\" in q and \"absent(\" in q; " "labels=r[0].get(\"labels\") or {}; " "assert labels.get(\"component\")==\"openclaw\"; " "assert labels.get(\"canonical_asset_id\")==\"service:openclaw:host188\"; " "assert labels.get(\"domain_router\")==\"docker_container\"'", ), AssetPostcondition( "host_110_telegram_callback_ingress_metric_visible", "metric", "host_110", "curl -fsS --max-time 5 --get " "--data-urlencode " "'query=awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds' " "http://127.0.0.1:9090/api/v1/query | " "python3 -c 'import json,sys; d=json.load(sys.stdin); " "assert len(d.get(\"data\",{}).get(\"result\",[])) >= 1'", ), AssetPostcondition( "host_110_telegram_callback_ingress_rule_hash_converged", "metric", "host_110", "expected=$(sha256sum /home/wooo/monitoring/alerts.yml | awk '{print $1}'); " "test -n \"$expected\" && " "test \"$expected\" = \"$(sha256sum " "/home/wooo/monitoring/alerts-unified.canonical.yml | awk '{print $1}')\" && " "test \"$expected\" = \"$(docker exec prometheus sha256sum " "/etc/prometheus/alerts.yml | awk '{print $1}')\"", ), ), "ansible:111-ollama-fallback": ( AssetPostcondition( "host_111_ollama_launchagent_allowlist_exact", "configuration", "host_111", "test \"$(/usr/libexec/PlistBuddy -c " "'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' " "/Users/ooo/Library/LaunchAgents/" "com.momo.ollama111-allow-proxy.plist)\" = " "'127.0.0.1/32,192.168.0.111/32,192.168.0.188/32," "192.168.0.120/32,192.168.0.121/32'", ), AssetPostcondition( "host_111_ollama_local_tags_api", "service", "host_111", "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " "http://127.0.0.1:11434/api/tags)\" = 200", ), AssetPostcondition( "host_120_to_host111_ollama_tags_api", "network", "host_120", "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " "http://192.168.0.111:11434/api/tags)\" = 200", ), AssetPostcondition( "host_121_to_host111_ollama_tags_api", "network", "host_121", "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " "http://192.168.0.111:11434/api/tags)\" = 200", ), ), "ansible:188-openclaw-callback-forwarder": ( AssetPostcondition( "host_188_openclaw_overlay_manifest_identity", "configuration", "host_188", "set -euo pipefail; " "root=/home/ollama/.local/share/awoooi-controlled-apply/" "openclaw-callback-forwarder/" "payloads/8f367f8e7d6104c44b29bd5fa2d894db09ddf365 && " "identity=/home/ollama/.local/share/awoooi-controlled-apply/" "openclaw-callback-forwarder/receipts/current-deployment.json && " "manifest=\"$root/manifest.json\" && " "test -d \"$root\" && test ! -L \"$root\" && " "test \"$(stat -c '%U:%G:%a' \"$root\")\" = 'ollama:ollama:755' && " "test -f \"$manifest\" && test ! -L \"$manifest\" && " "test \"$(stat -c '%U:%G:%a' \"$manifest\")\" = 'ollama:ollama:444' && " "test \"$(sha256sum \"$manifest\" | cut -d' ' -f1)\" = " "0265a2c4c9b62aa571eefd56454896adf660edf5a03931bc19fac5e5d60e846e", ), AssetPostcondition( "host_188_openclaw_callback_overlay_enabled", "configuration", "host_188", "set -euo pipefail; " "wd=$(systemctl show clawbot.service --property=WorkingDirectory " "--value) && " "test \"$wd\" = '/home/ollama/clawbot-v5' && " "override=\"$wd/docker-compose.override.yml\" && " "test -f \"$override\" && test ! -L \"$override\" && " "test \"$(stat -c '%U:%G:%a' \"$override\")\" = " "'ollama:ollama:444' && " "test \"$(sha256sum \"$override\" | cut -d' ' -f1)\" = " "33c4594d4e4092b2a2367583a67f4c9a82ead0d28ca5078eca4896276084da0d", ), AssetPostcondition( "host_188_openclaw_callback_payload_runtime", "configuration", "host_188", "set -euo pipefail; " "root=/home/ollama/.local/share/awoooi-controlled-apply/" "openclaw-callback-forwarder/" "payloads/8f367f8e7d6104c44b29bd5fa2d894db09ddf365 && " "telegram=\"$root/app/bot/telegram.py\" && " "config=\"$root/app/core/config.py\" && " "forwarder=\"$root/app/services/telegram_callback_forwarder.py\" && " "test -f \"$telegram\" && test ! -L \"$telegram\" && " "test \"$(stat -c '%U:%G:%a' \"$telegram\")\" = 'ollama:ollama:444' && " "test \"$(sha256sum \"$telegram\" | cut -d' ' -f1)\" = " "c53bdce72634a581538d2af087eb920fa16fd7ee464a0dc5f3cb24ec3605a287 && " "test -f \"$config\" && test ! -L \"$config\" && " "test \"$(stat -c '%U:%G:%a' \"$config\")\" = 'ollama:ollama:444' && " "test \"$(sha256sum \"$config\" | cut -d' ' -f1)\" = " "e1b8b633f7b4780d3078d6c42a55e49d200255d3597a34e0cd2fd649b84eaea1 && " "test -f \"$forwarder\" && test ! -L \"$forwarder\" && " "test \"$(stat -c '%U:%G:%a' \"$forwarder\")\" = 'ollama:ollama:444' && " "test \"$(sha256sum \"$forwarder\" | cut -d' ' -f1)\" = " "8e8e0faf6322abb21d670ad9094f1805d7a4b3643da8fe88d74a8c9591066bf3 && " "test -f \"$identity\" && test ! -L \"$identity\" && " "test \"$(stat -c '%U:%G:%a' \"$identity\")\" = " "'ollama:ollama:444' && " "test \"$(stat -c '%s' \"$identity\")\" -le 4096 && " "ids=$(docker ps -q " "--filter label=com.docker.compose.project=clawbot " "--filter label=com.docker.compose.service=clawbot) && " "test \"$(printf '%s\\n' \"$ids\" | sed '/^$/d' | wc -l | " "tr -d ' ')\" = 1 && " "id=\"$ids\" && " "test \"$(docker inspect --format='{{.Config.WorkingDir}}' \"$id\")\" " "= '/app' && " "test \"$(docker exec \"$ids\" sha256sum " "/app/app/bot/telegram.py | cut -d' ' -f1)\" = " "c53bdce72634a581538d2af087eb920fa16fd7ee464a0dc5f3cb24ec3605a287 && " "test \"$(docker exec \"$ids\" sha256sum " "/app/app/core/config.py | cut -d' ' -f1)\" = " "e1b8b633f7b4780d3078d6c42a55e49d200255d3597a34e0cd2fd649b84eaea1 && " "test \"$(docker exec \"$ids\" sha256sum " "/app/app/services/telegram_callback_forwarder.py | cut -d' ' -f1)\" = " "8e8e0faf6322abb21d670ad9094f1805d7a4b3643da8fe88d74a8c9591066bf3 && " "test \"$(docker inspect --format='{{range .Mounts}}{{if eq " ".Destination \"/app/app/bot/telegram.py\"}}{{.Source}}|" "{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' \"$id\")\" = " "\"$telegram|/app/app/bot/telegram.py|bind|false\" && " "test \"$(docker inspect --format='{{range .Mounts}}{{if eq " ".Destination \"/app/app/core/config.py\"}}{{.Source}}|" "{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' \"$id\")\" = " "\"$config|/app/app/core/config.py|bind|false\" && " "test \"$(docker inspect --format='{{range .Mounts}}{{if eq " ".Destination \"/app/app/services/telegram_callback_forwarder.py\"}}" "{{.Source}}|{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' " "\"$id\")\" = \"$forwarder|/app/app/services/" "telegram_callback_forwarder.py|bind|false\" && " "test \"$(docker inspect --format='{{range .Mounts}}{{if eq " ".Destination \"/run/awoooi/openclaw-callback-forwarder/" "current-deployment.json\"}}{{.Source}}|{{.Destination}}|{{.Type}}|" "{{.RW}}{{end}}{{end}}' \"$id\")\" = \"$identity|/run/awoooi/" "openclaw-callback-forwarder/current-deployment.json|bind|false\"", ), AssetPostcondition( "host_188_openclaw_callback_runtime_setting", "configuration", "host_188", "set -euo pipefail; " "ids=$(docker ps -q " "--filter label=com.docker.compose.project=clawbot " "--filter label=com.docker.compose.service=clawbot) && " "test \"$(printf '%s\\n' \"$ids\" | sed '/^$/d' | wc -l | " "tr -d ' ')\" = 1 && " "docker exec \"$ids\" python -c \"import os; " "from app.services.telegram_callback_forwarder import " "TelegramCallbackForwarder; " "TELEGRAM_CALLBACK_FORWARD_ENABLED = " "os.environ.get('TELEGRAM_CALLBACK_FORWARD_ENABLED') == 'true'; " "assert TELEGRAM_CALLBACK_FORWARD_ENABLED is True; " "assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_URL') == " "'https://awoooi.wooo.work/api/v1/telegram/callback-forward'; " "assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_ALLOWED_HOSTS') == " "'awoooi.wooo.work'; " "assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_TIMEOUT_SECONDS') == " "'3.0'; " "assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_MAX_BODY_BYTES') == " "'16384'\"", ), AssetPostcondition( "host_188_clawbot_systemd_runtime_active", "service", "host_188", "set -euo pipefail; systemctl is-active --quiet clawbot.service", ), AssetPostcondition( "host_188_openclaw_local_health", "service", "host_188", "set -euo pipefail; " "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " "http://127.0.0.1:8088/health)\" = 200", ), AssetPostcondition( "host_188_awoooi_callback_receiver_public_health", "network", "host_188", "set -euo pipefail; " "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " "https://awoooi.wooo.work/api/v1/telegram/health)\" = 200", ), ), "ansible:110-disk-pressure": ( AssetPostcondition( "host_110_root_disk_below_alert_threshold", "metric", "host_110", "usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); " "test -n \"$usage\" && test \"$usage\" -lt 85", ), AssetPostcondition( "host_110_docker_runtime_after_cleanup", "service", "host_110", "docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0", ), ), "ansible:188-disk-pressure": ( AssetPostcondition( "host_188_root_disk_below_alert_threshold", "metric", "host_188", "usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); " "test -n \"$usage\" && test \"$usage\" -lt 85", ), AssetPostcondition( "host_188_docker_runtime_after_cleanup", "service", "host_188", "docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0", ), ), "ansible:110-devops": ( AssetPostcondition( "host_110_node_exporter_container", "service", "host_110", "test \"$(docker inspect --format '{{.State.Running}}' node-exporter 2>/dev/null)\" = true", ), AssetPostcondition( "host_110_node_exporter_metrics", "metric", "host_110", "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 " "http://127.0.0.1:9100/metrics)\" = 200", ), AssetPostcondition( "host_110_node_exporter_prometheus_scrape", "metric", "host_110", "for attempt in 1 2 3 4 5 6; do " "curl -fsS --max-time 5 " "'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' " "| grep -Eq '\"value\":\\[[^]]+,\"1\"\\]' && exit 0; " "sleep 5; done; exit 1", ), ), "ansible:110-sentry-profiling-consumer-recovery": ( AssetPostcondition( "host_110_sentry_profiling_consumer_running", "service", "host_110", "test \"$(docker inspect --format '{{.State.Running}}' " "sentry-self-hosted-snuba-profiling-functions-consumer-1 " "2>/dev/null)\" = true", ), AssetPostcondition( "host_110_sentry_profiling_consumer_healthy", "service", "host_110", "test \"$(docker inspect --format '{{.State.Health.Status}}' " "sentry-self-hosted-snuba-profiling-functions-consumer-1 " "2>/dev/null)\" = healthy", ), AssetPostcondition( "host_110_sentry_profiling_alert_resolved_after_window", "metric", "host_110", "clean_samples=0; for attempt in $(seq 1 43); do " "curl -fsS --connect-timeout 3 --max-time 5 --get " "--data-urlencode " "'query=ALERTS{alertname=\"DockerContainerUnhealthy\"," "alertstate=\"firing\",job=\"docker-health-monitor\"," "host=\"110\",container=\"" "sentry-self-hosted-snuba-profiling-functions-consumer-1\"} or " "ALERTS{alertname=\"DockerContainerUnhealthy\"," "alertstate=\"firing\",job=\"docker-health-monitor\"," "host=\"110\",name=\"" "sentry-self-hosted-snuba-profiling-functions-consumer-1\"} or " "ALERTS{alertname=\"DockerContainerUnhealthy\"," "alertstate=\"firing\",job=\"docker-health-monitor\"," "host=\"110\",container_name=\"" "sentry-self-hosted-snuba-profiling-functions-consumer-1\"}' " "http://127.0.0.1:9090/api/v1/query | python3 -c '" "import json,sys; p=json.load(sys.stdin); " "raise SystemExit(0 if p.get(\"status\")==\"success\" " "and p.get(\"data\",{}).get(\"result\")==[] else 1)' " "&& clean_samples=$((clean_samples + 1)) " "|| clean_samples=0; " "test \"$clean_samples\" -ge 37 && exit 0; " "sleep 10; done; exit 1", 660, ), ), "ansible:110-devops-full-convergence": ( AssetPostcondition( "host_110_docker_runtime", "service", "host_110", "docker info >/dev/null 2>&1", ), AssetPostcondition( "host_110_harbor_route", "route", "host_110", "status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " "http://127.0.0.1:5000/v2/); " "[ \"$status\" = 200 ] || [ \"$status\" = 401 ]", ), AssetPostcondition( "host_110_backup_assets", "data", "host_110", "test -d /backup/scripts && find /backup/scripts -maxdepth 1 -type f | grep -q .", ), AssetPostcondition( "host_110_monitoring_guard", "metric", "host_110", "test -x /home/wooo/scripts/prometheus-rule-drift-guard.sh", ), ), "ansible:188-momo-backup-user": ( AssetPostcondition( "host_188_momo_backup_script", "service", "host_188", "test -x /home/ollama/bin/momo-pg-backup.sh && bash -n /home/ollama/bin/momo-pg-backup.sh", ), AssetPostcondition( "host_188_momo_backup_schedule", "scheduler", "host_188", "crontab -l | grep -Fq /home/ollama/bin/momo-pg-backup.sh", ), AssetPostcondition( "host_188_momo_legacy_schedule_absent", "safety", "host_188", "! crontab -l | grep -Fq /home/ollama/momo-pro/scripts/pg_backup.sh", ), AssetPostcondition( "host_188_momo_backup_directory", "data", "host_188", "test -d /home/ollama/momo_backups", ), AssetPostcondition( "host_188_momo_backup_metric", "metric", "host_188", "test -s /home/ollama/node_exporter_textfiles/backup_health.prom && grep -Fq momo_pg_daily /home/ollama/node_exporter_textfiles/backup_health.prom", ), ), "ansible:188-ai-web": ( AssetPostcondition( "host_188_openclaw_runtime", "service", "host_188", "docker ps -q --filter name=openclaw | grep -q . || docker ps -q --filter name=clawbot | grep -q .", ), AssetPostcondition( "host_188_momo_runtime", "service", "host_188", "docker ps -q --filter name=momo | grep -q .", ), AssetPostcondition( "host_188_nginx_keepalived_runtime", "route", "host_188", "systemctl is-active --quiet nginx && systemctl is-active --quiet keepalived", ), AssetPostcondition( "host_188_vip_runtime", "metric", "host_188", "ip addr show | grep -Fq 192.168.0.200", ), AssetPostcondition( "host_188_backup_assets", "data", "host_188", "test -x /home/ollama/bin/momo-pg-backup.sh && test -d /home/ollama/momo_backups", ), ), "ansible:nginx-sync": ( AssetPostcondition( "host_188_nginx_runtime", "service", "host_188", "systemctl is-active --quiet nginx", ), AssetPostcondition( "host_188_nginx_routes", "route", "host_188", "test -s /etc/nginx/sites-enabled/all-sites.conf && test -s /etc/nginx/sites-enabled/188-internal-tools-https.conf", ), ), "ansible:restore-password-auth": tuple( condition for host in ("host_110", "host_120", "host_121", "host_188") for condition in ( AssetPostcondition( f"{host}_password_auth_config", "security", host, "grep -Eiq '^[[:space:]]*PasswordAuthentication[[:space:]]+yes' /etc/ssh/sshd_config", ), AssetPostcondition( f"{host}_sshd_runtime", "service", host, "systemctl is-active --quiet ssh || systemctl is-active --quiet sshd", ), ) ), } def registered_catalog_ids() -> frozenset[str]: return frozenset(_POSTCONDITION_REGISTRY) def postconditions_for_catalog(catalog_id: str) -> tuple[AssetPostcondition, ...]: return _POSTCONDITION_REGISTRY.get(str(catalog_id or ""), ()) def _playbook_roots(module_path: Path | None = None) -> list[Path]: resolved_module_path = (module_path or Path(__file__)).resolve() return [ Path("/app/infra/ansible"), Path.cwd() / "infra" / "ansible", *(parent / "infra" / "ansible" for parent in resolved_module_path.parents), ] def _load_inventory_hosts(inventory_path: Path) -> dict[str, dict[str, str]]: payload = yaml.safe_load(inventory_path.read_text(encoding="utf-8")) or {} children = ((payload.get("all") or {}).get("children") or {}) hosts: dict[str, dict[str, str]] = {} for group in children.values(): if not isinstance(group, Mapping): continue for alias, config in (group.get("hosts") or {}).items(): if not isinstance(config, Mapping): continue hosts[str(alias)] = { "ansible_host": str(config.get("ansible_host") or ""), "ansible_user": str(config.get("ansible_user") or ""), "ansible_ssh_common_args": str( config.get("ansible_ssh_common_args") or "" ), } return hosts def _pinned_proxy_command( *, proxy_target: str, ssh_key_path: Path, known_hosts_path: Path, ) -> str: if not re.fullmatch(r"[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+", proxy_target): raise ValueError("postcondition_proxy_target_invalid") return ( f"ssh -i {shlex.quote(str(ssh_key_path))} " f"-o UserKnownHostsFile={shlex.quote(str(known_hosts_path))} " "-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes " "-o BatchMode=yes -o ConnectTimeout=10 " f"-W %h:%p {proxy_target}" ) def build_read_only_probe_command( condition: AssetPostcondition, *, inventory_path: Path, ssh_key_path: Path, known_hosts_path: Path, ) -> list[str]: if not _SAFE_HOST_ALIAS_RE.fullmatch(condition.inventory_host): raise ValueError("unsafe_postcondition_inventory_host") inventory_hosts = _load_inventory_hosts(inventory_path) target = inventory_hosts.get(condition.inventory_host) or {} host = str(target.get("ansible_host") or "") user = str(target.get("ansible_user") or "") if not _SAFE_REMOTE_ID_RE.fullmatch(host) or not _SAFE_REMOTE_ID_RE.fullmatch(user): raise ValueError("postcondition_inventory_target_missing") command = [ "ssh", "-i", str(ssh_key_path), "-o", f"UserKnownHostsFile={known_hosts_path}", "-o", "StrictHostKeyChecking=yes", "-o", "IdentitiesOnly=yes", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", ] proxy_target = _FIXED_PROXY_TARGET_BY_HOST.get(condition.inventory_host) if proxy_target is not None: inventory_proxy_command = _pinned_proxy_command( proxy_target=proxy_target, ssh_key_path=_INVENTORY_PROXY_KEY_PATH, known_hosts_path=_INVENTORY_PROXY_KNOWN_HOSTS_PATH, ) expected_common_args = ( f'-o ProxyCommand="{inventory_proxy_command}" ' "-o StrictHostKeyChecking=yes" ) if target.get("ansible_ssh_common_args") != expected_common_args: raise ValueError("postcondition_proxy_command_policy_mismatch") command.extend( [ "-o", "ProxyCommand=" + _pinned_proxy_command( proxy_target=proxy_target, ssh_key_path=ssh_key_path, known_hosts_path=known_hosts_path, ), ] ) command.extend([f"{user}@{host}", condition.probe]) return command async def _run_read_only_probe( command: list[str], *, timeout_seconds: int, ) -> ReadOnlyProbeResult: started = time.monotonic() process = await asyncio.create_subprocess_exec( *command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) timed_out = False try: stdout_bytes, stderr_bytes = await asyncio.wait_for( process.communicate(), timeout=timeout_seconds, ) except TimeoutError: timed_out = True process.kill() stdout_bytes, stderr_bytes = await process.communicate() except asyncio.CancelledError: process.kill() await process.communicate() raise return ReadOnlyProbeResult( returncode=124 if timed_out else int(process.returncode or 0), stdout=stdout_bytes.decode("utf-8", "replace"), stderr=stderr_bytes.decode("utf-8", "replace"), duration_ms=int((time.monotonic() - started) * 1000), timed_out=timed_out, ) def _output_fingerprint(result: ReadOnlyProbeResult) -> str: digest = hashlib.sha256() digest.update(result.stdout.encode("utf-8", "replace")) digest.update(b"\0") digest.update(result.stderr.encode("utf-8", "replace")) return digest.hexdigest() def _controlled_callback_receipt_condition( expected_fingerprint: str, ) -> AssetPostcondition: return AssetPostcondition( "host_188_controlled_callback_action_receipt", "runtime_receipt", "host_188", "set -euo pipefail; " "curl -fsS --max-time 10 " "https://awoooi.wooo.work/api/v1/telegram/health | " "python3 -c 'import json,sys; d=json.load(sys.stdin); " "r=d.get(\"callback_ingress\", {}).get(\"controlled_action_receipt\", {}); " "assert r.get(\"durable\") is True; " "assert r.get(\"fresh\") is True; " "assert r.get(\"source\") == \"openclaw_188_hmac_forwarder\"; " f"assert r.get(\"correlation_fingerprint\") == \"{expected_fingerprint}\"'", ) async def run_ansible_asset_post_verifier( *, catalog_id: str, automation_run_id: str, apply_op_id: str, inventory_hosts: tuple[str, ...], executor_returncode: int, playbook_root: Path | None = None, ssh_key_path: Path | None = None, known_hosts_path: Path | None = None, timeout_seconds: int = 20, trace_id: str = "", run_id: str = "", work_item_id: str = "", probe_runner: ProbeRunner = _run_read_only_probe, ) -> dict[str, Any]: conditions = postconditions_for_catalog(catalog_id) runtime_receipt_required = catalog_id in _RUNTIME_CLOSURE_RECEIPT_CATALOGS source_sha = os.getenv("AWOOOI_BUILD_COMMIT_SHA", "").strip().lower() base = { "schema_version": SCHEMA_VERSION, "verifier": VERIFIER_ID, "independent_source": INDEPENDENT_SOURCE, "automation_run_id": automation_run_id, "apply_op_id": apply_op_id, "catalog_id": catalog_id, "verifier_source_sha": source_sha or None, "executor_returncode": executor_returncode, "executor_returncode_trusted": False, "raw_output_stored": False, "command_exposed": False, "writes_on_verify": False, "runtime_closure_receipt_required": runtime_receipt_required, "runtime_closure_receipt_verified": ( False if runtime_receipt_required else None ), } blockers: list[str] = [] if executor_returncode != 0: blockers.append("executor_apply_failed_before_post_verifier") if not conditions: blockers.append("asset_postcondition_registry_missing") condition_hosts = {condition.inventory_host for condition in conditions} if not condition_hosts.issubset(set(inventory_hosts)): blockers.append("asset_postcondition_target_outside_claim") root = playbook_root or next( (path for path in _playbook_roots() if path.exists()), None, ) inventory_path = (root / "inventory" / "hosts.yml") if root else None resolved_key_path = ssh_key_path or Path( settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH ) resolved_known_hosts_path = known_hosts_path or Path( settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH ) if root is None or inventory_path is None or not inventory_path.is_file(): blockers.append("post_verifier_inventory_missing") if not resolved_key_path.is_file() or not os.access(resolved_key_path, os.R_OK): blockers.append("post_verifier_ssh_key_missing") if not resolved_known_hosts_path.is_file() or not os.access( resolved_known_hosts_path, os.R_OK, ): blockers.append("post_verifier_known_hosts_missing") if blockers: return { **base, "verification_result": "failed", "all_postconditions_passed": False, "technical_postconditions_passed": False, "required_postcondition_count": len(conditions), "passed_postcondition_count": 0, "postconditions": [], "active_blockers": blockers, } receipts: list[dict[str, Any]] = [] for condition in conditions: try: command = build_read_only_probe_command( condition, inventory_path=inventory_path, ssh_key_path=resolved_key_path, known_hosts_path=resolved_known_hosts_path, ) result = await probe_runner( command, timeout_seconds=max( 1, condition.timeout_seconds or timeout_seconds, ), ) except Exception as exc: receipts.append({ "condition_id": condition.condition_id, "condition_type": condition.condition_type, "asset_id": condition.inventory_host, "passed": False, "returncode": None, "timed_out": False, "duration_ms": 0, "output_sha256": None, "error_type": type(exc).__name__, "raw_output_stored": False, }) continue receipts.append({ "condition_id": condition.condition_id, "condition_type": condition.condition_type, "asset_id": condition.inventory_host, "passed": result.returncode == 0 and not result.timed_out, "returncode": result.returncode, "timed_out": result.timed_out, "duration_ms": result.duration_ms, "output_sha256": _output_fingerprint(result), "error_type": None, "raw_output_stored": False, }) technical_passed_count = sum( receipt["passed"] is True for receipt in receipts ) if technical_passed_count != len(conditions): blockers.extend( f"postcondition_failed:{receipt['condition_id']}" for receipt in receipts if receipt["passed"] is not True ) technical_verified = bool( conditions and technical_passed_count == len(conditions) ) runtime_receipt_verified: bool | None = None if runtime_receipt_required and technical_verified: try: expected_fingerprint = controlled_callback_correlation_fingerprint({ "schema_version": "awoooi_controlled_callback_correlation_v1", "trace_id": trace_id, "run_id": run_id, "work_item_id": work_item_id, }) except ValueError: runtime_receipt_verified = False blockers.append(_RUNTIME_CORRELATION_BLOCKER) receipts.append({ "condition_id": "host_188_controlled_callback_action_receipt", "condition_type": "runtime_receipt", "asset_id": "host_188", "passed": False, "returncode": None, "timed_out": False, "duration_ms": 0, "output_sha256": None, "error_type": "correlation_missing_or_invalid", "raw_output_stored": False, }) else: condition = _controlled_callback_receipt_condition(expected_fingerprint) try: command = build_read_only_probe_command( condition, inventory_path=inventory_path, ssh_key_path=resolved_key_path, known_hosts_path=resolved_known_hosts_path, ) result = await probe_runner( command, timeout_seconds=max(1, timeout_seconds), ) runtime_receipt_verified = bool( result.returncode == 0 and not result.timed_out ) receipts.append({ "condition_id": condition.condition_id, "condition_type": condition.condition_type, "asset_id": condition.inventory_host, "passed": runtime_receipt_verified, "returncode": result.returncode, "timed_out": result.timed_out, "duration_ms": result.duration_ms, "output_sha256": _output_fingerprint(result), "error_type": None, "raw_output_stored": False, "expected_correlation_fingerprint": expected_fingerprint, }) except Exception as exc: runtime_receipt_verified = False receipts.append({ "condition_id": condition.condition_id, "condition_type": condition.condition_type, "asset_id": condition.inventory_host, "passed": False, "returncode": None, "timed_out": False, "duration_ms": 0, "output_sha256": None, "error_type": type(exc).__name__, "raw_output_stored": False, "expected_correlation_fingerprint": expected_fingerprint, }) if runtime_receipt_verified is not True: blockers.append(_RUNTIME_RECEIPT_PENDING_BLOCKER) verified = bool( technical_verified and ( runtime_receipt_verified is True if runtime_receipt_required else True ) and not blockers ) verification_result = ( "success" if verified else "pending" if technical_verified and runtime_receipt_required else "failed" ) passed_count = sum(receipt["passed"] is True for receipt in receipts) return { **base, "verification_result": verification_result, "all_postconditions_passed": verified, "technical_postconditions_passed": technical_verified, "runtime_closure_receipt_verified": runtime_receipt_verified, "required_postcondition_count": ( len(conditions) + (1 if runtime_receipt_required else 0) ), "passed_postcondition_count": passed_count, "postconditions": receipts, "active_blockers": blockers, }