7cf98f642c
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 38s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 1m17s
CD Pipeline / revalidate-deploy-carrier (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / revalidate-post-deploy-carrier (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
1074 lines
43 KiB
Python
1074 lines
43 KiB
Python
"""Independent asset-specific postconditions for controlled Ansible apply."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import shlex
|
|
import time
|
|
from collections.abc import Awaitable, Callable, Mapping
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
from src.core.config import settings
|
|
from src.services.telegram_callback_correlation import (
|
|
controlled_callback_correlation_fingerprint,
|
|
)
|
|
|
|
SCHEMA_VERSION = "awoooi_ansible_asset_post_verifier_v1"
|
|
VERIFIER_ID = "asset_specific_read_only_host_postconditions"
|
|
INDEPENDENT_SOURCE = "broker_ssh_host_runtime_readback"
|
|
_SAFE_HOST_ALIAS_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
_SAFE_REMOTE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
_FIXED_PROXY_TARGET_BY_HOST = {
|
|
"host_111": "wooo@192.168.0.110",
|
|
}
|
|
_INVENTORY_PROXY_KEY_PATH = Path("/run/secrets/ssh_mcp_key")
|
|
_INVENTORY_PROXY_KNOWN_HOSTS_PATH = Path("/etc/ssh-mcp/known_hosts")
|
|
_RUNTIME_CLOSURE_RECEIPT_CATALOGS = frozenset(
|
|
{"ansible:188-openclaw-callback-forwarder"}
|
|
)
|
|
_RUNTIME_RECEIPT_PENDING_BLOCKER = "fresh_callback_action_receipt_pending_or_mismatch"
|
|
_RUNTIME_CORRELATION_BLOCKER = "controlled_callback_correlation_missing_or_invalid"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AssetPostcondition:
|
|
condition_id: str
|
|
condition_type: str
|
|
inventory_host: str
|
|
probe: str
|
|
timeout_seconds: int | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReadOnlyProbeResult:
|
|
returncode: int
|
|
stdout: str
|
|
stderr: str
|
|
duration_ms: int
|
|
timed_out: bool = False
|
|
|
|
|
|
ProbeRunner = Callable[..., Awaitable[ReadOnlyProbeResult]]
|
|
|
|
|
|
_POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
|
|
"ansible:awoooi-auto-repair-canary": (
|
|
AssetPostcondition(
|
|
"host_121_awoooi_canary_rollout_ready",
|
|
"service",
|
|
"host_121",
|
|
"sudo -n /usr/local/bin/kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n awoooi-prod rollout status deployment/awoooi-auto-repair-canary --timeout=30s >/dev/null",
|
|
),
|
|
AssetPostcondition(
|
|
"host_121_awoooi_canary_ready_replicas",
|
|
"metric",
|
|
"host_121",
|
|
"test \"$(sudo -n /usr/local/bin/kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml -n awoooi-prod get deployment awoooi-auto-repair-canary -o jsonpath='{.status.readyReplicas}:{.status.replicas}')\" = '1:1'",
|
|
),
|
|
),
|
|
"ansible:wazuh-manager-posture-readback": (
|
|
AssetPostcondition(
|
|
"host_112_wazuh_manager_unit_loaded",
|
|
"security",
|
|
"host_112",
|
|
"systemctl show --property=LoadState --value wazuh-manager.service | grep -Fxq loaded",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_manager_runtime",
|
|
"service",
|
|
"host_112",
|
|
"systemctl is-active --quiet wazuh-manager.service",
|
|
),
|
|
),
|
|
"ansible:wazuh-alertmanager-integration": (
|
|
AssetPostcondition(
|
|
"host_112_wazuh_alertmanager_script",
|
|
"security",
|
|
"host_112",
|
|
"test -r /var/lib/awoooi-wazuh-alert-ingress/receipt.env && "
|
|
"grep -Fqx 'integration_name=custom-awoooi-alertmanager' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_alertmanager_config",
|
|
"security",
|
|
"host_112",
|
|
"grep -Fqx 'config_present=1' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env && "
|
|
"grep -Fqx 'raw_payload_persisted=0' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env && "
|
|
"grep -Fqx 'active_response_enabled=0' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_manager_runtime_after_ingress_convergence",
|
|
"service",
|
|
"host_112",
|
|
"systemctl is-active --quiet wazuh-manager.service",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_wazuh_integrator_runtime",
|
|
"service",
|
|
"host_112",
|
|
"pgrep -x wazuh-integratord >/dev/null && "
|
|
"grep -Fqx 'integrator_running=1' "
|
|
"/var/lib/awoooi-wazuh-alert-ingress/receipt.env",
|
|
),
|
|
AssetPostcondition(
|
|
"host_112_awoooi_alertmanager_ingress_reachable",
|
|
"network",
|
|
"host_112",
|
|
"curl -fsS --max-time 10 https://awoooi.wooo.work/api/v1/webhooks/health >/dev/null",
|
|
),
|
|
),
|
|
"ansible:110-host-pressure-readonly": (
|
|
AssetPostcondition(
|
|
"host_110_pressure_controller_asset",
|
|
"service",
|
|
"host_110",
|
|
"test -x /home/wooo/scripts/host-sustained-load-controller.py",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_gitea_pressure_probe_asset",
|
|
"service",
|
|
"host_110",
|
|
"test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_pressure_metrics_freshness",
|
|
"metric",
|
|
"host_110",
|
|
"test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_docker_metrics_freshness",
|
|
"metric",
|
|
"host_110",
|
|
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
|
|
),
|
|
),
|
|
"ansible:110-alertmanager-delivery-recovery": (
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_container_running",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Running}}' alertmanager "
|
|
"2>/dev/null)\" = true",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_config_valid",
|
|
"configuration",
|
|
"host_110",
|
|
"docker exec alertmanager amtool check-config "
|
|
"/etc/alertmanager/alertmanager.yml >/dev/null",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_awoooi_webhook_network_boundary",
|
|
"network",
|
|
"host_110",
|
|
"code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 -I "
|
|
"https://awoooi.wooo.work/api/v1/webhooks/alertmanager); "
|
|
"test \"$code\" = 405 -o \"$code\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_runtime_target_up",
|
|
"metric",
|
|
"host_110",
|
|
"curl -fsS --get --data-urlencode "
|
|
"'query=up{job=\"alertmanager-runtime\"}' "
|
|
"http://127.0.0.1:9090/api/v1/query | python3 -c '"
|
|
"import json,sys; p=json.load(sys.stdin); r=p.get(\"data\",{}).get(\"result\",[]); "
|
|
"raise SystemExit(0 if p.get(\"status\")==\"success\" and len(r)==1 "
|
|
"and r[0].get(\"value\",[None,None])[1]==\"1\" else 1)'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_delivery_rule_healthy",
|
|
"metric",
|
|
"host_110",
|
|
"curl -fsS 'http://127.0.0.1:9090/api/v1/rules?type=alert' "
|
|
"| python3 -c 'import json,sys; p=json.load(sys.stdin); "
|
|
"rules=[r for g in p.get(\"data\",{}).get(\"groups\",[]) "
|
|
"for r in g.get(\"rules\",[]) "
|
|
"if r.get(\"name\")==\"AlertChainBroken_Alertmanager\"]; "
|
|
"raise SystemExit(0 if p.get(\"status\")==\"success\" and len(rules)==1 "
|
|
"and rules[0].get(\"health\")==\"ok\" else 1)'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_alertmanager_delivery_alert_resolved_after_window",
|
|
"metric",
|
|
"host_110",
|
|
"for attempt in $(seq 1 72); do "
|
|
"curl -fsS --get --data-urlencode "
|
|
"'query=ALERTS{alertname=\"AlertChainBroken_Alertmanager\","
|
|
"alertstate=\"firing\"}' http://127.0.0.1:9090/api/v1/query "
|
|
"| python3 -c 'import json,sys; p=json.load(sys.stdin); "
|
|
"raise SystemExit(0 if p.get(\"status\")==\"success\" "
|
|
"and p.get(\"data\",{}).get(\"result\")==[] else 1)' "
|
|
"&& exit 0; sleep 10; done; exit 1",
|
|
740,
|
|
),
|
|
),
|
|
"ansible:110-telegram-callback-ingress-sensor": (
|
|
AssetPostcondition(
|
|
"host_110_telegram_callback_ingress_rule_healthy",
|
|
"metric",
|
|
"host_110",
|
|
"curl -fsS --max-time 5 http://127.0.0.1:9090/api/v1/rules | "
|
|
"python3 -c 'import json,sys; d=json.load(sys.stdin); "
|
|
"r=[x for g in d.get(\"data\",{}).get(\"groups\",[]) "
|
|
"for x in g.get(\"rules\",[]) "
|
|
"if x.get(\"name\")==\"TelegramCallbackIngressUnverified\"]; "
|
|
"assert len(r)==1 and r[0].get(\"health\")==\"ok\"; "
|
|
"q=\"\".join(str(r[0].get(\"query\") or \"\").split()); "
|
|
"assert \"awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds\" in q; "
|
|
"assert \">300\" in q and \"absent(\" in q; "
|
|
"labels=r[0].get(\"labels\") or {}; "
|
|
"assert labels.get(\"component\")==\"openclaw\"; "
|
|
"assert labels.get(\"canonical_asset_id\")==\"service:openclaw:host188\"; "
|
|
"assert labels.get(\"domain_router\")==\"docker_container\"'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_telegram_callback_ingress_metric_visible",
|
|
"metric",
|
|
"host_110",
|
|
"curl -fsS --max-time 5 --get "
|
|
"--data-urlencode "
|
|
"'query=awoooi_telegram_callback_ingress_last_receipt_timestamp_seconds' "
|
|
"http://127.0.0.1:9090/api/v1/query | "
|
|
"python3 -c 'import json,sys; d=json.load(sys.stdin); "
|
|
"assert len(d.get(\"data\",{}).get(\"result\",[])) >= 1'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_telegram_callback_ingress_rule_hash_converged",
|
|
"metric",
|
|
"host_110",
|
|
"expected=$(sha256sum /home/wooo/monitoring/alerts.yml | awk '{print $1}'); "
|
|
"test -n \"$expected\" && "
|
|
"test \"$expected\" = \"$(sha256sum "
|
|
"/home/wooo/monitoring/alerts-unified.canonical.yml | awk '{print $1}')\" && "
|
|
"test \"$expected\" = \"$(docker exec prometheus sha256sum "
|
|
"/etc/prometheus/alerts.yml | awk '{print $1}')\"",
|
|
),
|
|
),
|
|
"ansible:111-ollama-fallback": (
|
|
AssetPostcondition(
|
|
"host_111_ollama_launchagent_allowlist_exact",
|
|
"configuration",
|
|
"host_111",
|
|
"test \"$(/usr/libexec/PlistBuddy -c "
|
|
"'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' "
|
|
"/Users/ooo/Library/LaunchAgents/"
|
|
"com.momo.ollama111-allow-proxy.plist)\" = "
|
|
"'127.0.0.1/32,192.168.0.111/32,192.168.0.188/32,"
|
|
"192.168.0.120/32,192.168.0.121/32'",
|
|
),
|
|
AssetPostcondition(
|
|
"host_111_ollama_local_tags_api",
|
|
"service",
|
|
"host_111",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://127.0.0.1:11434/api/tags)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_120_to_host111_ollama_tags_api",
|
|
"network",
|
|
"host_120",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://192.168.0.111:11434/api/tags)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_121_to_host111_ollama_tags_api",
|
|
"network",
|
|
"host_121",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://192.168.0.111:11434/api/tags)\" = 200",
|
|
),
|
|
),
|
|
"ansible:188-openclaw-callback-forwarder": (
|
|
AssetPostcondition(
|
|
"host_188_openclaw_overlay_manifest_identity",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"root=/home/ollama/.local/share/awoooi-controlled-apply/"
|
|
"openclaw-callback-forwarder/"
|
|
"payloads/8f367f8e7d6104c44b29bd5fa2d894db09ddf365 && "
|
|
"identity=/home/ollama/.local/share/awoooi-controlled-apply/"
|
|
"openclaw-callback-forwarder/receipts/current-deployment.json && "
|
|
"manifest=\"$root/manifest.json\" && "
|
|
"test -d \"$root\" && test ! -L \"$root\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$root\")\" = 'ollama:ollama:755' && "
|
|
"test -f \"$manifest\" && test ! -L \"$manifest\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$manifest\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$manifest\" | cut -d' ' -f1)\" = "
|
|
"0265a2c4c9b62aa571eefd56454896adf660edf5a03931bc19fac5e5d60e846e",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_callback_overlay_enabled",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"wd=$(systemctl show clawbot.service --property=WorkingDirectory "
|
|
"--value) && "
|
|
"test \"$wd\" = '/home/ollama/clawbot-v5' && "
|
|
"override=\"$wd/docker-compose.override.yml\" && "
|
|
"test -f \"$override\" && test ! -L \"$override\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$override\")\" = "
|
|
"'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$override\" | cut -d' ' -f1)\" = "
|
|
"33c4594d4e4092b2a2367583a67f4c9a82ead0d28ca5078eca4896276084da0d",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_callback_payload_runtime",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"root=/home/ollama/.local/share/awoooi-controlled-apply/"
|
|
"openclaw-callback-forwarder/"
|
|
"payloads/8f367f8e7d6104c44b29bd5fa2d894db09ddf365 && "
|
|
"telegram=\"$root/app/bot/telegram.py\" && "
|
|
"config=\"$root/app/core/config.py\" && "
|
|
"forwarder=\"$root/app/services/telegram_callback_forwarder.py\" && "
|
|
"test -f \"$telegram\" && test ! -L \"$telegram\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$telegram\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$telegram\" | cut -d' ' -f1)\" = "
|
|
"c53bdce72634a581538d2af087eb920fa16fd7ee464a0dc5f3cb24ec3605a287 && "
|
|
"test -f \"$config\" && test ! -L \"$config\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$config\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$config\" | cut -d' ' -f1)\" = "
|
|
"e1b8b633f7b4780d3078d6c42a55e49d200255d3597a34e0cd2fd649b84eaea1 && "
|
|
"test -f \"$forwarder\" && test ! -L \"$forwarder\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$forwarder\")\" = 'ollama:ollama:444' && "
|
|
"test \"$(sha256sum \"$forwarder\" | cut -d' ' -f1)\" = "
|
|
"8e8e0faf6322abb21d670ad9094f1805d7a4b3643da8fe88d74a8c9591066bf3 && "
|
|
"test -f \"$identity\" && test ! -L \"$identity\" && "
|
|
"test \"$(stat -c '%U:%G:%a' \"$identity\")\" = "
|
|
"'ollama:ollama:444' && "
|
|
"test \"$(stat -c '%s' \"$identity\")\" -le 4096 && "
|
|
"ids=$(docker ps -q "
|
|
"--filter label=com.docker.compose.project=clawbot "
|
|
"--filter label=com.docker.compose.service=clawbot) && "
|
|
"test \"$(printf '%s\\n' \"$ids\" | sed '/^$/d' | wc -l | "
|
|
"tr -d ' ')\" = 1 && "
|
|
"id=\"$ids\" && "
|
|
"test \"$(docker inspect --format='{{.Config.WorkingDir}}' \"$id\")\" "
|
|
"= '/app' && "
|
|
"test \"$(docker exec \"$ids\" sha256sum "
|
|
"/app/app/bot/telegram.py | cut -d' ' -f1)\" = "
|
|
"c53bdce72634a581538d2af087eb920fa16fd7ee464a0dc5f3cb24ec3605a287 && "
|
|
"test \"$(docker exec \"$ids\" sha256sum "
|
|
"/app/app/core/config.py | cut -d' ' -f1)\" = "
|
|
"e1b8b633f7b4780d3078d6c42a55e49d200255d3597a34e0cd2fd649b84eaea1 && "
|
|
"test \"$(docker exec \"$ids\" sha256sum "
|
|
"/app/app/services/telegram_callback_forwarder.py | cut -d' ' -f1)\" = "
|
|
"8e8e0faf6322abb21d670ad9094f1805d7a4b3643da8fe88d74a8c9591066bf3 && "
|
|
"test \"$(docker inspect --format='{{range .Mounts}}{{if eq "
|
|
".Destination \"/app/app/bot/telegram.py\"}}{{.Source}}|"
|
|
"{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' \"$id\")\" = "
|
|
"\"$telegram|/app/app/bot/telegram.py|bind|false\" && "
|
|
"test \"$(docker inspect --format='{{range .Mounts}}{{if eq "
|
|
".Destination \"/app/app/core/config.py\"}}{{.Source}}|"
|
|
"{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' \"$id\")\" = "
|
|
"\"$config|/app/app/core/config.py|bind|false\" && "
|
|
"test \"$(docker inspect --format='{{range .Mounts}}{{if eq "
|
|
".Destination \"/app/app/services/telegram_callback_forwarder.py\"}}"
|
|
"{{.Source}}|{{.Destination}}|{{.Type}}|{{.RW}}{{end}}{{end}}' "
|
|
"\"$id\")\" = \"$forwarder|/app/app/services/"
|
|
"telegram_callback_forwarder.py|bind|false\" && "
|
|
"test \"$(docker inspect --format='{{range .Mounts}}{{if eq "
|
|
".Destination \"/run/awoooi/openclaw-callback-forwarder/"
|
|
"current-deployment.json\"}}{{.Source}}|{{.Destination}}|{{.Type}}|"
|
|
"{{.RW}}{{end}}{{end}}' \"$id\")\" = \"$identity|/run/awoooi/"
|
|
"openclaw-callback-forwarder/current-deployment.json|bind|false\"",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_callback_runtime_setting",
|
|
"configuration",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"ids=$(docker ps -q "
|
|
"--filter label=com.docker.compose.project=clawbot "
|
|
"--filter label=com.docker.compose.service=clawbot) && "
|
|
"test \"$(printf '%s\\n' \"$ids\" | sed '/^$/d' | wc -l | "
|
|
"tr -d ' ')\" = 1 && "
|
|
"docker exec \"$ids\" python -c \"import os; "
|
|
"from app.services.telegram_callback_forwarder import "
|
|
"TelegramCallbackForwarder; "
|
|
"TELEGRAM_CALLBACK_FORWARD_ENABLED = "
|
|
"os.environ.get('TELEGRAM_CALLBACK_FORWARD_ENABLED') == 'true'; "
|
|
"assert TELEGRAM_CALLBACK_FORWARD_ENABLED is True; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_URL') == "
|
|
"'https://awoooi.wooo.work/api/v1/telegram/callback-forward'; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_ALLOWED_HOSTS') == "
|
|
"'awoooi.wooo.work'; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_TIMEOUT_SECONDS') == "
|
|
"'3.0'; "
|
|
"assert os.environ.get('TELEGRAM_CALLBACK_FORWARD_MAX_BODY_BYTES') == "
|
|
"'16384'\"",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_clawbot_systemd_runtime_active",
|
|
"service",
|
|
"host_188",
|
|
"set -euo pipefail; systemctl is-active --quiet clawbot.service",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_openclaw_local_health",
|
|
"service",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://127.0.0.1:8088/health)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_awoooi_callback_receiver_public_health",
|
|
"network",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"https://awoooi.wooo.work/api/v1/telegram/health)\" = 200",
|
|
),
|
|
),
|
|
"ansible:110-disk-pressure": (
|
|
AssetPostcondition(
|
|
"host_110_root_disk_below_alert_threshold",
|
|
"metric",
|
|
"host_110",
|
|
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
|
|
"test -n \"$usage\" && test \"$usage\" -lt 85",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_docker_runtime_after_cleanup",
|
|
"service",
|
|
"host_110",
|
|
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
|
|
),
|
|
),
|
|
"ansible:188-disk-pressure": (
|
|
AssetPostcondition(
|
|
"host_188_root_disk_below_alert_threshold",
|
|
"metric",
|
|
"host_188",
|
|
"usage=$(df -P / | awk 'END {gsub(/%/, \"\", $5); print $5}'); "
|
|
"test -n \"$usage\" && test \"$usage\" -lt 85",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_docker_runtime_after_cleanup",
|
|
"service",
|
|
"host_188",
|
|
"docker info >/dev/null 2>&1 && test \"$(docker ps -q | wc -l)\" -gt 0",
|
|
),
|
|
),
|
|
"ansible:110-devops": (
|
|
AssetPostcondition(
|
|
"host_110_node_exporter_container",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Running}}' node-exporter 2>/dev/null)\" = true",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_node_exporter_metrics",
|
|
"metric",
|
|
"host_110",
|
|
"test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "
|
|
"http://127.0.0.1:9100/metrics)\" = 200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_node_exporter_prometheus_scrape",
|
|
"metric",
|
|
"host_110",
|
|
"for attempt in 1 2 3 4 5 6; do "
|
|
"curl -fsS --max-time 5 "
|
|
"'http://127.0.0.1:9090/api/v1/query?query=up%7Bjob%3D%22node-exporter-110%22%7D' "
|
|
"| grep -Eq '\"value\":\\[[^]]+,\"1\"\\]' && exit 0; "
|
|
"sleep 5; done; exit 1",
|
|
),
|
|
),
|
|
"ansible:110-sentry-profiling-consumer-recovery": (
|
|
AssetPostcondition(
|
|
"host_110_sentry_profiling_consumer_running",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Running}}' "
|
|
"sentry-self-hosted-snuba-profiling-functions-consumer-1 "
|
|
"2>/dev/null)\" = true",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_sentry_profiling_consumer_healthy",
|
|
"service",
|
|
"host_110",
|
|
"test \"$(docker inspect --format '{{.State.Health.Status}}' "
|
|
"sentry-self-hosted-snuba-profiling-functions-consumer-1 "
|
|
"2>/dev/null)\" = healthy",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_sentry_profiling_alert_resolved_after_window",
|
|
"metric",
|
|
"host_110",
|
|
"clean_samples=0; for attempt in $(seq 1 43); do "
|
|
"curl -fsS --connect-timeout 3 --max-time 5 --get "
|
|
"--data-urlencode "
|
|
"'query=ALERTS{alertname=\"DockerContainerUnhealthy\","
|
|
"alertstate=\"firing\",job=\"docker-health-monitor\","
|
|
"host=\"110\",container=\""
|
|
"sentry-self-hosted-snuba-profiling-functions-consumer-1\"} or "
|
|
"ALERTS{alertname=\"DockerContainerUnhealthy\","
|
|
"alertstate=\"firing\",job=\"docker-health-monitor\","
|
|
"host=\"110\",name=\""
|
|
"sentry-self-hosted-snuba-profiling-functions-consumer-1\"} or "
|
|
"ALERTS{alertname=\"DockerContainerUnhealthy\","
|
|
"alertstate=\"firing\",job=\"docker-health-monitor\","
|
|
"host=\"110\",container_name=\""
|
|
"sentry-self-hosted-snuba-profiling-functions-consumer-1\"}' "
|
|
"http://127.0.0.1:9090/api/v1/query | python3 -c '"
|
|
"import json,sys; p=json.load(sys.stdin); "
|
|
"raise SystemExit(0 if p.get(\"status\")==\"success\" "
|
|
"and p.get(\"data\",{}).get(\"result\")==[] else 1)' "
|
|
"&& clean_samples=$((clean_samples + 1)) "
|
|
"|| clean_samples=0; "
|
|
"test \"$clean_samples\" -ge 37 && exit 0; "
|
|
"sleep 10; done; exit 1",
|
|
660,
|
|
),
|
|
),
|
|
"ansible:110-devops-full-convergence": (
|
|
AssetPostcondition(
|
|
"host_110_docker_runtime",
|
|
"service",
|
|
"host_110",
|
|
"docker info >/dev/null 2>&1",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_harbor_route",
|
|
"route",
|
|
"host_110",
|
|
"status=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 "
|
|
"http://127.0.0.1:5000/v2/); "
|
|
"[ \"$status\" = 200 ] || [ \"$status\" = 401 ]",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_backup_assets",
|
|
"data",
|
|
"host_110",
|
|
"test -d /backup/scripts && find /backup/scripts -maxdepth 1 -type f | grep -q .",
|
|
),
|
|
AssetPostcondition(
|
|
"host_110_monitoring_guard",
|
|
"metric",
|
|
"host_110",
|
|
"test -x /home/wooo/scripts/prometheus-rule-drift-guard.sh",
|
|
),
|
|
),
|
|
"ansible:188-momo-backup-user": (
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_script",
|
|
"service",
|
|
"host_188",
|
|
"test -x /home/ollama/bin/momo-pg-backup.sh && bash -n /home/ollama/bin/momo-pg-backup.sh",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_schedule",
|
|
"scheduler",
|
|
"host_188",
|
|
"crontab -l | grep -Fq /home/ollama/bin/momo-pg-backup.sh",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_legacy_schedule_absent",
|
|
"safety",
|
|
"host_188",
|
|
"! crontab -l | grep -Fq /home/ollama/momo-pro/scripts/pg_backup.sh",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_directory",
|
|
"data",
|
|
"host_188",
|
|
"test -d /home/ollama/momo_backups",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_backup_metric",
|
|
"metric",
|
|
"host_188",
|
|
"test -s /home/ollama/node_exporter_textfiles/backup_health.prom && grep -Fq momo_pg_daily /home/ollama/node_exporter_textfiles/backup_health.prom",
|
|
),
|
|
),
|
|
"ansible:188-ai-web": (
|
|
AssetPostcondition(
|
|
"host_188_openclaw_runtime",
|
|
"service",
|
|
"host_188",
|
|
"docker ps -q --filter name=openclaw | grep -q . || docker ps -q --filter name=clawbot | grep -q .",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_momo_runtime",
|
|
"service",
|
|
"host_188",
|
|
"docker ps -q --filter name=momo | grep -q .",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_nginx_keepalived_runtime",
|
|
"route",
|
|
"host_188",
|
|
"systemctl is-active --quiet nginx && systemctl is-active --quiet keepalived",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_vip_runtime",
|
|
"metric",
|
|
"host_188",
|
|
"ip addr show | grep -Fq 192.168.0.200",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_backup_assets",
|
|
"data",
|
|
"host_188",
|
|
"test -x /home/ollama/bin/momo-pg-backup.sh && test -d /home/ollama/momo_backups",
|
|
),
|
|
),
|
|
"ansible:nginx-sync": (
|
|
AssetPostcondition(
|
|
"host_188_nginx_runtime",
|
|
"service",
|
|
"host_188",
|
|
"systemctl is-active --quiet nginx",
|
|
),
|
|
AssetPostcondition(
|
|
"host_188_nginx_routes",
|
|
"route",
|
|
"host_188",
|
|
"test -s /etc/nginx/sites-enabled/all-sites.conf && test -s /etc/nginx/sites-enabled/188-internal-tools-https.conf",
|
|
),
|
|
),
|
|
"ansible:restore-password-auth": tuple(
|
|
condition
|
|
for host in ("host_110", "host_120", "host_121", "host_188")
|
|
for condition in (
|
|
AssetPostcondition(
|
|
f"{host}_password_auth_config",
|
|
"security",
|
|
host,
|
|
"grep -Eiq '^[[:space:]]*PasswordAuthentication[[:space:]]+yes' /etc/ssh/sshd_config",
|
|
),
|
|
AssetPostcondition(
|
|
f"{host}_sshd_runtime",
|
|
"service",
|
|
host,
|
|
"systemctl is-active --quiet ssh || systemctl is-active --quiet sshd",
|
|
),
|
|
)
|
|
),
|
|
}
|
|
|
|
|
|
def registered_catalog_ids() -> frozenset[str]:
|
|
return frozenset(_POSTCONDITION_REGISTRY)
|
|
|
|
|
|
def postconditions_for_catalog(catalog_id: str) -> tuple[AssetPostcondition, ...]:
|
|
return _POSTCONDITION_REGISTRY.get(str(catalog_id or ""), ())
|
|
|
|
|
|
def _playbook_roots(module_path: Path | None = None) -> list[Path]:
|
|
resolved_module_path = (module_path or Path(__file__)).resolve()
|
|
return [
|
|
Path("/app/infra/ansible"),
|
|
Path.cwd() / "infra" / "ansible",
|
|
*(parent / "infra" / "ansible" for parent in resolved_module_path.parents),
|
|
]
|
|
|
|
|
|
def _load_inventory_hosts(inventory_path: Path) -> dict[str, dict[str, str]]:
|
|
payload = yaml.safe_load(inventory_path.read_text(encoding="utf-8")) or {}
|
|
children = ((payload.get("all") or {}).get("children") or {})
|
|
hosts: dict[str, dict[str, str]] = {}
|
|
for group in children.values():
|
|
if not isinstance(group, Mapping):
|
|
continue
|
|
for alias, config in (group.get("hosts") or {}).items():
|
|
if not isinstance(config, Mapping):
|
|
continue
|
|
hosts[str(alias)] = {
|
|
"ansible_host": str(config.get("ansible_host") or ""),
|
|
"ansible_user": str(config.get("ansible_user") or ""),
|
|
"ansible_ssh_common_args": str(
|
|
config.get("ansible_ssh_common_args") or ""
|
|
),
|
|
}
|
|
return hosts
|
|
|
|
|
|
def _pinned_proxy_command(
|
|
*,
|
|
proxy_target: str,
|
|
ssh_key_path: Path,
|
|
known_hosts_path: Path,
|
|
) -> str:
|
|
if not re.fullmatch(r"[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+", proxy_target):
|
|
raise ValueError("postcondition_proxy_target_invalid")
|
|
return (
|
|
f"ssh -i {shlex.quote(str(ssh_key_path))} "
|
|
f"-o UserKnownHostsFile={shlex.quote(str(known_hosts_path))} "
|
|
"-o StrictHostKeyChecking=yes -o IdentitiesOnly=yes "
|
|
"-o BatchMode=yes -o ConnectTimeout=10 "
|
|
f"-W %h:%p {proxy_target}"
|
|
)
|
|
|
|
|
|
def build_read_only_probe_command(
|
|
condition: AssetPostcondition,
|
|
*,
|
|
inventory_path: Path,
|
|
ssh_key_path: Path,
|
|
known_hosts_path: Path,
|
|
) -> list[str]:
|
|
if not _SAFE_HOST_ALIAS_RE.fullmatch(condition.inventory_host):
|
|
raise ValueError("unsafe_postcondition_inventory_host")
|
|
inventory_hosts = _load_inventory_hosts(inventory_path)
|
|
target = inventory_hosts.get(condition.inventory_host) or {}
|
|
host = str(target.get("ansible_host") or "")
|
|
user = str(target.get("ansible_user") or "")
|
|
if not _SAFE_REMOTE_ID_RE.fullmatch(host) or not _SAFE_REMOTE_ID_RE.fullmatch(user):
|
|
raise ValueError("postcondition_inventory_target_missing")
|
|
command = [
|
|
"ssh",
|
|
"-i",
|
|
str(ssh_key_path),
|
|
"-o",
|
|
f"UserKnownHostsFile={known_hosts_path}",
|
|
"-o",
|
|
"StrictHostKeyChecking=yes",
|
|
"-o",
|
|
"IdentitiesOnly=yes",
|
|
"-o",
|
|
"BatchMode=yes",
|
|
"-o",
|
|
"ConnectTimeout=10",
|
|
]
|
|
proxy_target = _FIXED_PROXY_TARGET_BY_HOST.get(condition.inventory_host)
|
|
if proxy_target is not None:
|
|
inventory_proxy_command = _pinned_proxy_command(
|
|
proxy_target=proxy_target,
|
|
ssh_key_path=_INVENTORY_PROXY_KEY_PATH,
|
|
known_hosts_path=_INVENTORY_PROXY_KNOWN_HOSTS_PATH,
|
|
)
|
|
expected_common_args = (
|
|
f'-o ProxyCommand="{inventory_proxy_command}" '
|
|
"-o StrictHostKeyChecking=yes"
|
|
)
|
|
if target.get("ansible_ssh_common_args") != expected_common_args:
|
|
raise ValueError("postcondition_proxy_command_policy_mismatch")
|
|
command.extend(
|
|
[
|
|
"-o",
|
|
"ProxyCommand="
|
|
+ _pinned_proxy_command(
|
|
proxy_target=proxy_target,
|
|
ssh_key_path=ssh_key_path,
|
|
known_hosts_path=known_hosts_path,
|
|
),
|
|
]
|
|
)
|
|
command.extend([f"{user}@{host}", condition.probe])
|
|
return command
|
|
|
|
|
|
async def _run_read_only_probe(
|
|
command: list[str],
|
|
*,
|
|
timeout_seconds: int,
|
|
) -> ReadOnlyProbeResult:
|
|
started = time.monotonic()
|
|
process = await asyncio.create_subprocess_exec(
|
|
*command,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
timed_out = False
|
|
try:
|
|
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
process.communicate(),
|
|
timeout=timeout_seconds,
|
|
)
|
|
except TimeoutError:
|
|
timed_out = True
|
|
process.kill()
|
|
stdout_bytes, stderr_bytes = await process.communicate()
|
|
except asyncio.CancelledError:
|
|
process.kill()
|
|
await process.communicate()
|
|
raise
|
|
return ReadOnlyProbeResult(
|
|
returncode=124 if timed_out else int(process.returncode or 0),
|
|
stdout=stdout_bytes.decode("utf-8", "replace"),
|
|
stderr=stderr_bytes.decode("utf-8", "replace"),
|
|
duration_ms=int((time.monotonic() - started) * 1000),
|
|
timed_out=timed_out,
|
|
)
|
|
|
|
|
|
def _output_fingerprint(result: ReadOnlyProbeResult) -> str:
|
|
digest = hashlib.sha256()
|
|
digest.update(result.stdout.encode("utf-8", "replace"))
|
|
digest.update(b"\0")
|
|
digest.update(result.stderr.encode("utf-8", "replace"))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _controlled_callback_receipt_condition(
|
|
expected_fingerprint: str,
|
|
) -> AssetPostcondition:
|
|
return AssetPostcondition(
|
|
"host_188_controlled_callback_action_receipt",
|
|
"runtime_receipt",
|
|
"host_188",
|
|
"set -euo pipefail; "
|
|
"curl -fsS --max-time 10 "
|
|
"https://awoooi.wooo.work/api/v1/telegram/health | "
|
|
"python3 -c 'import json,sys; d=json.load(sys.stdin); "
|
|
"r=d.get(\"callback_ingress\", {}).get(\"controlled_action_receipt\", {}); "
|
|
"assert r.get(\"durable\") is True; "
|
|
"assert r.get(\"fresh\") is True; "
|
|
"assert r.get(\"source\") == \"openclaw_188_hmac_forwarder\"; "
|
|
f"assert r.get(\"correlation_fingerprint\") == \"{expected_fingerprint}\"'",
|
|
)
|
|
|
|
|
|
async def run_ansible_asset_post_verifier(
|
|
*,
|
|
catalog_id: str,
|
|
automation_run_id: str,
|
|
apply_op_id: str,
|
|
inventory_hosts: tuple[str, ...],
|
|
executor_returncode: int,
|
|
playbook_root: Path | None = None,
|
|
ssh_key_path: Path | None = None,
|
|
known_hosts_path: Path | None = None,
|
|
timeout_seconds: int = 20,
|
|
trace_id: str = "",
|
|
run_id: str = "",
|
|
work_item_id: str = "",
|
|
probe_runner: ProbeRunner = _run_read_only_probe,
|
|
) -> dict[str, Any]:
|
|
conditions = postconditions_for_catalog(catalog_id)
|
|
runtime_receipt_required = catalog_id in _RUNTIME_CLOSURE_RECEIPT_CATALOGS
|
|
source_sha = os.getenv("AWOOOI_BUILD_COMMIT_SHA", "").strip().lower()
|
|
base = {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"verifier": VERIFIER_ID,
|
|
"independent_source": INDEPENDENT_SOURCE,
|
|
"automation_run_id": automation_run_id,
|
|
"apply_op_id": apply_op_id,
|
|
"catalog_id": catalog_id,
|
|
"verifier_source_sha": source_sha or None,
|
|
"executor_returncode": executor_returncode,
|
|
"executor_returncode_trusted": False,
|
|
"raw_output_stored": False,
|
|
"command_exposed": False,
|
|
"writes_on_verify": False,
|
|
"runtime_closure_receipt_required": runtime_receipt_required,
|
|
"runtime_closure_receipt_verified": (
|
|
False if runtime_receipt_required else None
|
|
),
|
|
}
|
|
blockers: list[str] = []
|
|
if executor_returncode != 0:
|
|
blockers.append("executor_apply_failed_before_post_verifier")
|
|
if not conditions:
|
|
blockers.append("asset_postcondition_registry_missing")
|
|
condition_hosts = {condition.inventory_host for condition in conditions}
|
|
if not condition_hosts.issubset(set(inventory_hosts)):
|
|
blockers.append("asset_postcondition_target_outside_claim")
|
|
|
|
root = playbook_root or next(
|
|
(path for path in _playbook_roots() if path.exists()),
|
|
None,
|
|
)
|
|
inventory_path = (root / "inventory" / "hosts.yml") if root else None
|
|
resolved_key_path = ssh_key_path or Path(
|
|
settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH
|
|
)
|
|
resolved_known_hosts_path = known_hosts_path or Path(
|
|
settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH
|
|
)
|
|
if root is None or inventory_path is None or not inventory_path.is_file():
|
|
blockers.append("post_verifier_inventory_missing")
|
|
if not resolved_key_path.is_file() or not os.access(resolved_key_path, os.R_OK):
|
|
blockers.append("post_verifier_ssh_key_missing")
|
|
if not resolved_known_hosts_path.is_file() or not os.access(
|
|
resolved_known_hosts_path,
|
|
os.R_OK,
|
|
):
|
|
blockers.append("post_verifier_known_hosts_missing")
|
|
if blockers:
|
|
return {
|
|
**base,
|
|
"verification_result": "failed",
|
|
"all_postconditions_passed": False,
|
|
"technical_postconditions_passed": False,
|
|
"required_postcondition_count": len(conditions),
|
|
"passed_postcondition_count": 0,
|
|
"postconditions": [],
|
|
"active_blockers": blockers,
|
|
}
|
|
|
|
receipts: list[dict[str, Any]] = []
|
|
for condition in conditions:
|
|
try:
|
|
command = build_read_only_probe_command(
|
|
condition,
|
|
inventory_path=inventory_path,
|
|
ssh_key_path=resolved_key_path,
|
|
known_hosts_path=resolved_known_hosts_path,
|
|
)
|
|
result = await probe_runner(
|
|
command,
|
|
timeout_seconds=max(
|
|
1,
|
|
condition.timeout_seconds or timeout_seconds,
|
|
),
|
|
)
|
|
except Exception as exc:
|
|
receipts.append({
|
|
"condition_id": condition.condition_id,
|
|
"condition_type": condition.condition_type,
|
|
"asset_id": condition.inventory_host,
|
|
"passed": False,
|
|
"returncode": None,
|
|
"timed_out": False,
|
|
"duration_ms": 0,
|
|
"output_sha256": None,
|
|
"error_type": type(exc).__name__,
|
|
"raw_output_stored": False,
|
|
})
|
|
continue
|
|
receipts.append({
|
|
"condition_id": condition.condition_id,
|
|
"condition_type": condition.condition_type,
|
|
"asset_id": condition.inventory_host,
|
|
"passed": result.returncode == 0 and not result.timed_out,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"duration_ms": result.duration_ms,
|
|
"output_sha256": _output_fingerprint(result),
|
|
"error_type": None,
|
|
"raw_output_stored": False,
|
|
})
|
|
technical_passed_count = sum(
|
|
receipt["passed"] is True for receipt in receipts
|
|
)
|
|
if technical_passed_count != len(conditions):
|
|
blockers.extend(
|
|
f"postcondition_failed:{receipt['condition_id']}"
|
|
for receipt in receipts
|
|
if receipt["passed"] is not True
|
|
)
|
|
technical_verified = bool(
|
|
conditions and technical_passed_count == len(conditions)
|
|
)
|
|
runtime_receipt_verified: bool | None = None
|
|
if runtime_receipt_required and technical_verified:
|
|
try:
|
|
expected_fingerprint = controlled_callback_correlation_fingerprint({
|
|
"schema_version": "awoooi_controlled_callback_correlation_v1",
|
|
"trace_id": trace_id,
|
|
"run_id": run_id,
|
|
"work_item_id": work_item_id,
|
|
})
|
|
except ValueError:
|
|
runtime_receipt_verified = False
|
|
blockers.append(_RUNTIME_CORRELATION_BLOCKER)
|
|
receipts.append({
|
|
"condition_id": "host_188_controlled_callback_action_receipt",
|
|
"condition_type": "runtime_receipt",
|
|
"asset_id": "host_188",
|
|
"passed": False,
|
|
"returncode": None,
|
|
"timed_out": False,
|
|
"duration_ms": 0,
|
|
"output_sha256": None,
|
|
"error_type": "correlation_missing_or_invalid",
|
|
"raw_output_stored": False,
|
|
})
|
|
else:
|
|
condition = _controlled_callback_receipt_condition(expected_fingerprint)
|
|
try:
|
|
command = build_read_only_probe_command(
|
|
condition,
|
|
inventory_path=inventory_path,
|
|
ssh_key_path=resolved_key_path,
|
|
known_hosts_path=resolved_known_hosts_path,
|
|
)
|
|
result = await probe_runner(
|
|
command,
|
|
timeout_seconds=max(1, timeout_seconds),
|
|
)
|
|
runtime_receipt_verified = bool(
|
|
result.returncode == 0 and not result.timed_out
|
|
)
|
|
receipts.append({
|
|
"condition_id": condition.condition_id,
|
|
"condition_type": condition.condition_type,
|
|
"asset_id": condition.inventory_host,
|
|
"passed": runtime_receipt_verified,
|
|
"returncode": result.returncode,
|
|
"timed_out": result.timed_out,
|
|
"duration_ms": result.duration_ms,
|
|
"output_sha256": _output_fingerprint(result),
|
|
"error_type": None,
|
|
"raw_output_stored": False,
|
|
"expected_correlation_fingerprint": expected_fingerprint,
|
|
})
|
|
except Exception as exc:
|
|
runtime_receipt_verified = False
|
|
receipts.append({
|
|
"condition_id": condition.condition_id,
|
|
"condition_type": condition.condition_type,
|
|
"asset_id": condition.inventory_host,
|
|
"passed": False,
|
|
"returncode": None,
|
|
"timed_out": False,
|
|
"duration_ms": 0,
|
|
"output_sha256": None,
|
|
"error_type": type(exc).__name__,
|
|
"raw_output_stored": False,
|
|
"expected_correlation_fingerprint": expected_fingerprint,
|
|
})
|
|
if runtime_receipt_verified is not True:
|
|
blockers.append(_RUNTIME_RECEIPT_PENDING_BLOCKER)
|
|
|
|
verified = bool(
|
|
technical_verified
|
|
and (
|
|
runtime_receipt_verified is True
|
|
if runtime_receipt_required
|
|
else True
|
|
)
|
|
and not blockers
|
|
)
|
|
verification_result = (
|
|
"success"
|
|
if verified
|
|
else "pending"
|
|
if technical_verified and runtime_receipt_required
|
|
else "failed"
|
|
)
|
|
passed_count = sum(receipt["passed"] is True for receipt in receipts)
|
|
return {
|
|
**base,
|
|
"verification_result": verification_result,
|
|
"all_postconditions_passed": verified,
|
|
"technical_postconditions_passed": technical_verified,
|
|
"runtime_closure_receipt_verified": runtime_receipt_verified,
|
|
"required_postcondition_count": (
|
|
len(conditions) + (1 if runtime_receipt_required else 0)
|
|
),
|
|
"passed_postcondition_count": passed_count,
|
|
"postconditions": receipts,
|
|
"active_blockers": blockers,
|
|
}
|