diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 16dcd2fb6..211f5a136 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -607,6 +607,41 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "canonical_asset_id": "ai-provider:ollama_local", "catalog_revision": "2026-07-16-host111-launchagent-k3s-path-v2", }, + { + "catalog_id": "ansible:188-openclaw-callback-forwarder", + "playbook_path": ( + "infra/ansible/playbooks/188-openclaw-callback-forwarder.yml" + ), + "check_mode_playbook_path": ( + "infra/ansible/playbooks/188-openclaw-callback-forwarder.yml" + ), + "inventory_hosts": ["host_188"], + "domains": [ + "docker_compose_container", + "openclaw", + "telegram_callback_forwarder", + "host_188", + ], + "keywords": [ + "openclawdown", + "clawbotdown", + "openclaw callback forwarder", + "telegram callback forwarder", + "external callback forward unverified", + ], + "activation_keywords": [ + "openclawdown", + "clawbotdown", + "telegram callback forwarder", + ], + "supports_check_mode": True, + "auto_apply_enabled": True, + "approval_required": False, + "risk_level": "medium", + "canonical_asset_id": "service:openclaw:host188", + "catalog_revision": "2026-07-17-openclaw-callback-forwarder-v1", + "cross_domain_fallback_allowed": False, + }, { "catalog_id": "ansible:110-host-pressure-readonly", "playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml", diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 131321520..2e262bbf6 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -52,6 +52,7 @@ from src.services.playbook_evolver import TRUST_ARCHIVE_THRESHOLD logger = structlog.get_logger(__name__) _SAFE_HOST_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +_SAFE_CORRELATION_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,160}$") _PLAYBOOK_PREFIX = Path("infra/ansible/playbooks") _STDOUT_LIMIT = 20_000 _STDERR_LIMIT = 12_000 @@ -810,6 +811,38 @@ def _ansible_command_env( return env +def _controlled_correlation_extra_vars( + *, + trace_id: str = "", + run_id: str = "", + work_item_id: str = "", +) -> dict[str, str]: + values = { + "controlled_trace_id": str(trace_id or "").strip(), + "controlled_run_id": str(run_id or "").strip(), + "controlled_work_item_id": str(work_item_id or "").strip(), + } + for value in values.values(): + if value and not _SAFE_CORRELATION_RE.fullmatch(value): + raise ValueError("unsafe_controlled_execution_correlation") + return {key: value for key, value in values.items() if value} + + +def _claim_correlation_kwargs( + claim: AnsibleCheckModeClaim, +) -> dict[str, str]: + automation_run_id = _automation_run_id_for_claim(claim) + return { + "trace_id": str( + claim.input_payload.get("trace_id") or automation_run_id + ), + "run_id": str( + claim.input_payload.get("run_id") or automation_run_id + ), + "work_item_id": str(claim.input_payload.get("work_item_id") or ""), + } + + def build_ansible_check_mode_command( *, playbook_path: str, @@ -817,6 +850,9 @@ def build_ansible_check_mode_command( playbook_root: Path | None = None, check_mode_ssh_key_path: Path | None = None, check_mode_known_hosts_path: Path | None = None, + trace_id: str = "", + run_id: str = "", + work_item_id: str = "", ) -> AnsibleCommandSpec: root = playbook_root or next((path for path in _playbook_roots() if path.exists()), None) if root is None: @@ -832,6 +868,11 @@ def build_ansible_check_mode_command( known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path() extra_vars = { "ansible_ssh_private_key_file": str(ssh_key_path), + **_controlled_correlation_extra_vars( + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + ), } command = [ "ansible-playbook", @@ -870,6 +911,9 @@ def build_ansible_apply_command( playbook_root: Path | None = None, check_mode_ssh_key_path: Path | None = None, check_mode_known_hosts_path: Path | None = None, + trace_id: str = "", + run_id: str = "", + work_item_id: str = "", ) -> AnsibleCommandSpec: """Build the controlled apply command after check-mode has passed.""" @@ -887,6 +931,11 @@ def build_ansible_apply_command( known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path() extra_vars = { "ansible_ssh_private_key_file": str(ssh_key_path), + **_controlled_correlation_extra_vars( + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + ), } command = [ "ansible-playbook", @@ -7681,6 +7730,7 @@ async def run_failed_apply_check_mode_replay_once( spec = build_ansible_check_mode_command( playbook_path=claim.playbook_path, inventory_hosts=claim.inventory_hosts, + **_claim_correlation_kwargs(claim), ) replay_result = await _run_ansible_command( spec, @@ -9278,6 +9328,7 @@ async def run_claimed_check_mode( spec = build_ansible_check_mode_command( playbook_path=claim.playbook_path, inventory_hosts=claim.inventory_hosts, + **_claim_correlation_kwargs(claim), ) result = await _run_ansible_command( spec, @@ -9466,6 +9517,7 @@ async def run_controlled_apply_for_claim( spec = build_ansible_apply_command( playbook_path=claim.apply_playbook_path, inventory_hosts=claim.inventory_hosts, + **_claim_correlation_kwargs(claim), ) result = await _run_ansible_command( spec, diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index 4854014cb..6408f6fac 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -238,6 +238,54 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = { "http://192.168.0.111:11434/api/tags)\" = 200", ), ), + "ansible:188-openclaw-callback-forwarder": ( + AssetPostcondition( + "host_188_openclaw_immutable_source_identity", + "configuration", + "host_188", + "wd=$(systemctl show clawbot.service --property=WorkingDirectory " + "--value); case \"$wd\" in /opt/openclaw|/home/ollama/clawbot-v5) ;; " + "*) exit 1 ;; esac; " + "test \"$(git -C \"$wd\" symbolic-ref --short HEAD)\" = main && " + "test \"$(git -C \"$wd\" rev-parse --verify HEAD)\" = " + "ffc45311b6403b30cb6ae6e1aa83dae1a047e9c9 && " + "test \"$(git -C \"$wd\" remote get-url origin)\" = " + "ssh://git@192.168.0.110:2222/wooo/clawbot-v5.git && " + "git -C \"$wd\" diff --quiet && " + "git -C \"$wd\" diff --cached --quiet && " + "test -z \"$(git -C \"$wd\" ls-files --others --exclude-standard)\"", + ), + AssetPostcondition( + "host_188_openclaw_callback_dropin_enabled", + "configuration", + "host_188", + "test -r /etc/systemd/system/clawbot.service.d/" + "20-awoooi-callback-forwarder.conf && " + "grep -Fqx 'Environment=TELEGRAM_CALLBACK_FORWARD_ENABLED=true' " + "/etc/systemd/system/clawbot.service.d/" + "20-awoooi-callback-forwarder.conf", + ), + AssetPostcondition( + "host_188_clawbot_systemd_runtime_active", + "service", + "host_188", + "systemctl is-active --quiet clawbot.service", + ), + AssetPostcondition( + "host_188_openclaw_local_health", + "service", + "host_188", + "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " + "http://127.0.0.1:8088/health)\" = 200", + ), + AssetPostcondition( + "host_188_awoooi_callback_receiver_public_health", + "network", + "host_188", + "test \"$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 " + "https://awoooi.wooo.work/api/v1/telegram/health)\" = 200", + ), + ), "ansible:110-disk-pressure": ( AssetPostcondition( "host_110_root_disk_below_alert_threshold", diff --git a/apps/api/src/services/controlled_alert_target_router.py b/apps/api/src/services/controlled_alert_target_router.py index 72250be40..f8a8fb22b 100644 --- a/apps/api/src/services/controlled_alert_target_router.py +++ b/apps/api/src/services/controlled_alert_target_router.py @@ -709,7 +709,17 @@ def resolve_typed_alert_target( and _text_token(target_resource) == _text_token(str(identity.get("service_name") or "")) ) - if not (exact_sentry_consumer or exact_registry_catalog_target): + exact_openclaw_compose_target = bool( + domain == "docker_compose_container" + and identity.get("canonical_id") == "service:openclaw:host188" + and _text_token(target_resource) + in {"openclaw", "clawbot", "clawbot.service"} + ) + if not ( + exact_sentry_consumer + or exact_registry_catalog_target + or exact_openclaw_compose_target + ): catalog_ids = [] stateful_level = str(identity.get("stateful_level") or "") risk = ( diff --git a/apps/api/tests/test_host188_openclaw_callback_forwarder_ansible_catalog.py b/apps/api/tests/test_host188_openclaw_callback_forwarder_ansible_catalog.py new file mode 100644 index 000000000..f29a51ef8 --- /dev/null +++ b/apps/api/tests/test_host188_openclaw_callback_forwarder_ansible_catalog.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from src.services.awooop_ansible_audit_service import ( + _catalog_hints, + get_ansible_catalog_item, +) +from src.services.awooop_ansible_check_mode_service import ( + build_ansible_apply_command, + build_ansible_check_mode_command, +) +from src.services.awooop_ansible_post_verifier import postconditions_for_catalog +from src.services.controlled_alert_target_router import resolve_typed_alert_target + +ROOT = Path(__file__).resolve().parents[3] +PLAYBOOK = ( + ROOT + / "infra" + / "ansible" + / "playbooks" + / "188-openclaw-callback-forwarder.yml" +) +SERVICE_REGISTRY = ROOT / "ops" / "config" / "service-registry.yaml" +MONITORING_REGISTRY = ROOT / "ops" / "monitoring" / "service-registry.yaml" +CATALOG_ID = "ansible:188-openclaw-callback-forwarder" +CANONICAL_ASSET_ID = "service:openclaw:host188" +EXPECTED_SHA = "ffc45311b6403b30cb6ae6e1aa83dae1a047e9c9" +EXPECTED_REMOTE = "ssh://git@192.168.0.110:2222/wooo/clawbot-v5.git" + + +def _flatten_tasks(tasks: list[dict]) -> list[dict]: + flattened: list[dict] = [] + for task in tasks: + flattened.append(task) + for section in ("block", "rescue", "always"): + nested = task.get(section) + if isinstance(nested, list): + flattened.extend(_flatten_tasks(nested)) + return flattened + + +def _incident() -> dict: + return { + "incident_id": "INC-OPENCLAW-CALLBACK-FORWARDER", + "project_id": "awoooi", + "alertname": "OpenClawDown", + "alert_category": "ai_agent", + "affected_services": ["openclaw"], + "signals": [ + { + "alert_name": "OpenClawDown", + "labels": {"component": "openclaw", "host": "188"}, + } + ], + } + + +def test_openclaw_routes_only_to_exact_host188_compose_lane() -> None: + for target in ("openclaw", "clawbot", "clawbot.service"): + route = resolve_typed_alert_target( + alertname="OpenClawDown", + target_resource=target, + namespace="", + labels={"host": "188"}, + alert_category="ai_agent", + ) + + assert route["resolution_status"] == "resolved" + assert route["target_kind"] == "docker_compose_container" + assert route["canonical_asset_id"] == CANONICAL_ASSET_ID + assert route["host"] == "192.168.0.188" + assert route["executor"] == "host_ansible_executor" + assert route["verifier"] == ( + "openclaw_callback_forwarder_independent_verifier" + ) + assert route["risk_class"] == "medium" + assert route["allowed_catalog_ids"] == [CATALOG_ID] + assert route["allowed_inventory_hosts"] == ["host_188"] + assert route["cross_domain_fallback_allowed"] is False + + +def test_openclaw_incident_selects_one_exact_bounded_catalog() -> None: + hints = _catalog_hints(_incident(), None) + + assert hints["match_mode"] == "typed_domain_router_v2" + assert hints["decision_effect"] == "bounded_domain_candidate" + assert [row["catalog_id"] for row in hints["candidates"]] == [CATALOG_ID] + candidate = hints["candidates"][0] + assert candidate["inventory_hosts"] == ["host_188"] + assert candidate["canonical_asset_id"] == CANONICAL_ASSET_ID + assert candidate["typed_domain"] == "docker_compose_container" + assert candidate["supports_check_mode"] is True + assert candidate["auto_apply_enabled"] is True + assert candidate["approval_required"] is False + assert candidate["risk_level"] == "medium" + assert hints["cross_domain_fallback_allowed"] is False + + +def test_openclaw_catalog_and_independent_verifier_are_exact() -> None: + catalog = get_ansible_catalog_item(CATALOG_ID) + conditions = postconditions_for_catalog(CATALOG_ID) + + assert catalog is not None + assert catalog["playbook_path"].endswith( + "188-openclaw-callback-forwarder.yml" + ) + assert catalog["check_mode_playbook_path"] == catalog["playbook_path"] + assert catalog["canonical_asset_id"] == CANONICAL_ASSET_ID + assert catalog["inventory_hosts"] == ["host_188"] + assert catalog["domains"] == [ + "docker_compose_container", + "openclaw", + "telegram_callback_forwarder", + "host_188", + ] + assert catalog["catalog_revision"] == ( + "2026-07-17-openclaw-callback-forwarder-v1" + ) + assert catalog["cross_domain_fallback_allowed"] is False + + assert {condition.condition_id for condition in conditions} == { + "host_188_openclaw_immutable_source_identity", + "host_188_openclaw_callback_dropin_enabled", + "host_188_clawbot_systemd_runtime_active", + "host_188_openclaw_local_health", + "host_188_awoooi_callback_receiver_public_health", + } + assert {condition.inventory_host for condition in conditions} == {"host_188"} + probes = "\n".join(condition.probe for condition in conditions) + assert EXPECTED_SHA in probes + assert EXPECTED_REMOTE in probes + assert "TELEGRAM_CALLBACK_FORWARD_ENABLED=true" in probes + assert "http://127.0.0.1:8088/health" in probes + assert "https://awoooi.wooo.work/api/v1/telegram/health" in probes + assert "--env-file" not in probes + + +def test_openclaw_playbook_is_check_first_bounded_and_rollback_capable() -> None: + payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8")) + source = PLAYBOOK.read_text(encoding="utf-8") + play = payload[0] + tasks = {task["name"]: task for task in _flatten_tasks(play["tasks"])} + + assert play["hosts"] == "host_188" + assert play["gather_facts"] is False + assert play["become"] is True + assert play["vars"]["catalog_id"] == CATALOG_ID + assert play["vars"]["canonical_asset_id"] == CANONICAL_ASSET_ID + assert play["vars"]["expected_source_sha"] == EXPECTED_SHA + assert play["vars"]["expected_remote_url"] == EXPECTED_REMOTE + assert play["vars"]["allowed_working_directories"] == [ + "/opt/openclaw", + "/home/ollama/clawbot-v5", + ] + assert play["vars"]["callback_dropin_content"].splitlines() == [ + "[Service]", + "Environment=TELEGRAM_CALLBACK_FORWARD_ENABLED=true", + ] + + working_directory = tasks[ + "Resolve OpenClaw working directory from systemd truth" + ] + assert working_directory["check_mode"] is False + assert working_directory["ansible.builtin.command"]["argv"] == [ + "systemctl", + "show", + "clawbot.service", + "--property=WorkingDirectory", + "--value", + ] + assert tasks["Fail closed when systemd source identity is outside host188 scope"][ + "ansible.builtin.assert" + ]["that"] == ["openclaw_workdir in allowed_working_directories"] + + check_receipt = tasks["Publish check-mode source diff and no-write receipt"] + assert check_receipt["when"] == "ansible_check_mode" + assert check_receipt["ansible.builtin.debug"]["msg"][ + "persistent_writes_performed" + ] == 0 + for key in ("trace_id", "run_id", "work_item_id"): + assert check_receipt["ansible.builtin.debug"]["msg"][key] == ( + "{{ " + key + " }}" + ) + + for name in ( + "Build only the immutable OpenClaw compose service with timeout", + "Rebuild only the restored OpenClaw compose service", + ): + task = tasks[name] + argv = task["ansible.builtin.command"]["argv"] + assert argv[:4] == [ + "/usr/bin/timeout", + "--signal=TERM", + "--kill-after=15", + "300", + ] + assert "build" in argv + assert "up" not in argv + assert argv[-1] == "{{ compose_service }}" + assert task["args"]["chdir"] == "{{ openclaw_workdir }}" + assert task["no_log"] is True + + for name in ( + "Reload systemd and restart the owned service with callback flag", + "Reload systemd and restart restored clawbot.service contract", + ): + assert tasks[name]["ansible.builtin.systemd"] == { + "name": "clawbot.service", + "state": "restarted", + "daemon_reload": True, + } + assert tasks[name]["no_log"] is True + + rollback = tasks["Restore exact old source SHA after failed apply or verifier"] + assert rollback["ansible.builtin.command"]["argv"][-2:] == [ + "--hard", + "{{ source_head_before.stdout | trim }}", + ] + assert "old-source-sha" in source + assert "old-callback-dropin.conf" in source + assert "openclaw_callback_forwarder_rollback_v1" in source + assert "git merge --ff-only" not in source + assert "--ff-only" in source + assert "docker compose config" not in source + assert "--env-file" not in source + assert "- up" not in source + assert "docker compose down" not in source + assert "docker system prune" not in source + assert "force push" not in source + + +def test_openclaw_commands_carry_same_safe_controlled_correlation( + tmp_path: Path, +) -> None: + key = tmp_path / "ssh_mcp_key" + known_hosts = tmp_path / "known_hosts" + expected = { + "ansible_ssh_private_key_file": str(key), + "controlled_trace_id": "trace:openclaw:001", + "controlled_run_id": "run:openclaw:001", + "controlled_work_item_id": "P0-OPENCLAW-CALLBACK-001", + } + + for builder in (build_ansible_check_mode_command, build_ansible_apply_command): + spec = builder( + playbook_path=( + "infra/ansible/playbooks/188-openclaw-callback-forwarder.yml" + ), + inventory_hosts=("host_188",), + playbook_root=ROOT / "infra" / "ansible", + check_mode_ssh_key_path=key, + check_mode_known_hosts_path=known_hosts, + trace_id="trace:openclaw:001", + run_id="run:openclaw:001", + work_item_id="P0-OPENCLAW-CALLBACK-001", + ) + assert json.loads(spec.command[-1]) == expected + + with pytest.raises(ValueError, match="unsafe_controlled_execution_correlation"): + build_ansible_check_mode_command( + playbook_path=( + "infra/ansible/playbooks/188-openclaw-callback-forwarder.yml" + ), + inventory_hosts=("host_188",), + playbook_root=ROOT / "infra" / "ansible", + check_mode_ssh_key_path=key, + check_mode_known_hosts_path=known_hosts, + trace_id="unsafe correlation", + ) + + +def test_openclaw_source_and_monitoring_registries_use_exact_catalog() -> None: + registry = yaml.safe_load(SERVICE_REGISTRY.read_text(encoding="utf-8")) + service = next(row for row in registry["services"] if row["name"] == "openclaw") + assert service["canonical_id"] == CANONICAL_ASSET_ID + assert service["host"] == "192.168.0.188" + assert service["asset_domain"] == "docker_compose_container" + assert service["executor"] == "host_ansible_executor" + assert service["verifier"] == ( + "openclaw_callback_forwarder_independent_verifier" + ) + assert service["allowed_catalog_ids"] == [CATALOG_ID] + + monitoring = yaml.safe_load( + MONITORING_REGISTRY.read_text(encoding="utf-8") + ) + monitored = next( + row for row in monitoring["services"] if row["name"] == "openclaw" + ) + assert monitored["host"] == "192.168.0.188" + assert monitored["auto_repair"] == { + "enabled": True, + "actions": [CATALOG_ID], + } diff --git a/infra/ansible/playbooks/188-openclaw-callback-forwarder.yml b/infra/ansible/playbooks/188-openclaw-callback-forwarder.yml new file mode 100644 index 000000000..12acab44b --- /dev/null +++ b/infra/ansible/playbooks/188-openclaw-callback-forwarder.yml @@ -0,0 +1,678 @@ +--- +# Host188 OpenClaw callback forwarder controlled convergence. +# +# Safety boundaries: +# - clawbot.service is the only source of the live WorkingDirectory. +# - only the two declared directories and one immutable internal-Gitea SHA are valid. +# - no task reads or writes .env, runtime secret volumes, or secret values. +# - check mode is read-only; apply is one exact compose service with rollback. + +- name: Host188 OpenClaw callback forwarder controlled convergence + hosts: host_188 + gather_facts: false + become: true + vars: + catalog_id: ansible:188-openclaw-callback-forwarder + canonical_asset_id: service:openclaw:host188 + expected_source_sha: ffc45311b6403b30cb6ae6e1aa83dae1a047e9c9 + expected_remote_url: ssh://git@192.168.0.110:2222/wooo/clawbot-v5.git + allowed_working_directories: + - /opt/openclaw + - /home/ollama/clawbot-v5 + compose_service: clawbot + compose_project: clawbot + callback_dropin_dir: /etc/systemd/system/clawbot.service.d + callback_dropin_path: >- + /etc/systemd/system/clawbot.service.d/20-awoooi-callback-forwarder.conf + callback_dropin_content: | + [Service] + Environment=TELEGRAM_CALLBACK_FORWARD_ENABLED=true + trace_id: "{{ controlled_trace_id | default('') }}" + run_id: "{{ controlled_run_id | default('') }}" + work_item_id: "{{ controlled_work_item_id | default('') }}" + rollback_root: /var/lib/awoooi-controlled-apply/openclaw-callback-forwarder + rollback_dir: "{{ rollback_root }}/{{ run_id }}" + openclaw_health_url: http://127.0.0.1:8088/health + callback_receiver_health_url: >- + https://awoooi.wooo.work/api/v1/telegram/health + + tasks: + - name: Require same-run controlled execution correlation + ansible.builtin.assert: + that: + - trace_id | length > 0 + - run_id | length > 0 + - work_item_id | length > 0 + - trace_id is match('^[A-Za-z0-9_.:-]{1,160}$') + - run_id is match('^[A-Za-z0-9_.:-]{1,160}$') + - work_item_id is match('^[A-Za-z0-9_.:-]{1,160}$') + fail_msg: controlled_execution_correlation_missing_or_unsafe + changed_when: false + + - name: Resolve OpenClaw working directory from systemd truth + become: false + ansible.builtin.command: + argv: + - systemctl + - show + - clawbot.service + - --property=WorkingDirectory + - --value + register: clawbot_working_directory + check_mode: false + changed_when: false + + - name: Pin the allowlisted OpenClaw working directory + ansible.builtin.set_fact: + openclaw_workdir: "{{ clawbot_working_directory.stdout | trim }}" + changed_when: false + + - name: Fail closed when systemd source identity is outside host188 scope + ansible.builtin.assert: + that: + - openclaw_workdir in allowed_working_directories + fail_msg: openclaw_systemd_working_directory_not_allowlisted + changed_when: false + + - name: Read bounded clawbot.service compose command truth + become: false + ansible.builtin.command: + argv: + - systemctl + - show + - clawbot.service + - --property=ExecStart + - --value + register: clawbot_exec_start + check_mode: false + changed_when: false + + - name: Require existing service ownership to remain Docker Compose up only + ansible.builtin.assert: + that: + - "'docker compose' in clawbot_exec_start.stdout" + - "' up ' in (' ' ~ clawbot_exec_start.stdout ~ ' ')" + - "' down ' not in (' ' ~ clawbot_exec_start.stdout ~ ' ')" + - "' prune ' not in (' ' ~ clawbot_exec_start.stdout ~ ' ')" + fail_msg: clawbot_systemd_execstart_source_mismatch + changed_when: false + + - name: Require a tracked Docker Compose source file + become: false + ansible.builtin.command: + argv: + - git + - ls-files + - --error-unmatch + - docker-compose.yml + args: + chdir: "{{ openclaw_workdir }}" + register: compose_source_tracked + check_mode: false + changed_when: false + + - name: Read exact source branch + become: false + ansible.builtin.command: + argv: + - git + - symbolic-ref + - --short + - HEAD + args: + chdir: "{{ openclaw_workdir }}" + register: source_branch + check_mode: false + changed_when: false + + - name: Read exact internal Gitea remote + become: false + ansible.builtin.command: + argv: + - git + - remote + - get-url + - origin + args: + chdir: "{{ openclaw_workdir }}" + register: source_remote + check_mode: false + changed_when: false + + - name: Read current OpenClaw source SHA + become: false + ansible.builtin.command: + argv: + - git + - rev-parse + - --verify + - HEAD + args: + chdir: "{{ openclaw_workdir }}" + register: source_head_before + check_mode: false + changed_when: false + + - name: Verify tracked, staged, and untracked source are clean without reading files + become: false + ansible.builtin.shell: + cmd: | + set -euo pipefail + git diff --quiet + git diff --cached --quiet + test -z "$(git ls-files --others --exclude-standard)" + args: + chdir: "{{ openclaw_workdir }}" + executable: /bin/bash + register: source_clean + check_mode: false + changed_when: false + failed_when: false + no_log: true + + - name: Read immutable main SHA from internal Gitea without fetching + become: false + ansible.builtin.command: + argv: + - git + - ls-remote + - --exit-code + - origin + - refs/heads/main + args: + chdir: "{{ openclaw_workdir }}" + register: remote_main + check_mode: false + changed_when: false + failed_when: false + + - name: Normalize immutable remote identity without unsafe indexing + ansible.builtin.set_fact: + remote_main_fields: "{{ remote_main.stdout.split() }}" + remote_main_sha: >- + {{ (remote_main.stdout.split() | first) | default('') }} + remote_main_ref: >- + {{ (remote_main.stdout.split() | last) | default('') }} + changed_when: false + + - name: Fail closed on branch, remote, cleanliness, or immutable source mismatch + ansible.builtin.assert: + that: + - source_branch.stdout | trim == 'main' + - source_remote.stdout | trim == expected_remote_url + - source_clean.rc == 0 + - remote_main.rc == 0 + - remote_main_fields | length == 2 + - remote_main_sha == expected_source_sha + - remote_main_ref == 'refs/heads/main' + fail_msg: openclaw_immutable_source_preflight_mismatch + changed_when: false + + - name: Check whether immutable target object is already local + become: false + ansible.builtin.command: + argv: + - git + - cat-file + - -e + - "{{ expected_source_sha }}^{commit}" + args: + chdir: "{{ openclaw_workdir }}" + register: expected_object_local + check_mode: false + changed_when: false + failed_when: false + + - name: Verify local target remains a fast-forward when object is available + become: false + ansible.builtin.command: + argv: + - git + - merge-base + - --is-ancestor + - "{{ source_head_before.stdout | trim }}" + - "{{ expected_source_sha }}" + args: + chdir: "{{ openclaw_workdir }}" + register: local_fast_forward + check_mode: false + changed_when: false + failed_when: false + when: + - expected_object_local.rc == 0 + - source_head_before.stdout | trim != expected_source_sha + + - name: Reject known non-fast-forward source before any mutation + ansible.builtin.assert: + that: + - >- + expected_object_local.rc != 0 + or source_head_before.stdout | trim == expected_source_sha + or local_fast_forward.rc == 0 + fail_msg: openclaw_source_not_fast_forward + changed_when: false + + - name: Inspect existing non-secret callback drop-in + ansible.builtin.stat: + path: "{{ callback_dropin_path }}" + get_checksum: true + register: callback_dropin_before + check_mode: false + + - name: Save existing callback drop-in in run memory for rollback + ansible.builtin.slurp: + src: "{{ callback_dropin_path }}" + register: callback_dropin_content_before + when: callback_dropin_before.stat.exists | default(false) + check_mode: false + no_log: true + + - name: Probe current OpenClaw health without mutation + become: false + ansible.builtin.uri: + url: "{{ openclaw_health_url }}" + method: GET + status_code: 200 + return_content: false + timeout: 10 + register: openclaw_health_before + check_mode: false + changed_when: false + failed_when: false + + - name: Require callback receiver public health before touching OpenClaw + become: false + ansible.builtin.uri: + url: "{{ callback_receiver_health_url }}" + method: GET + status_code: 200 + return_content: false + timeout: 10 + register: callback_receiver_health_before + check_mode: false + changed_when: false + failed_when: false + + - name: Fail closed when callback receiver is not publicly healthy + ansible.builtin.assert: + that: + - callback_receiver_health_before.status | default(0) == 200 + fail_msg: awoooi_callback_receiver_public_health_unavailable + changed_when: false + + - name: Calculate one idempotent bounded apply decision + ansible.builtin.set_fact: + source_update_required: >- + {{ source_head_before.stdout | trim != expected_source_sha }} + callback_dropin_update_required: >- + {{ not (callback_dropin_before.stat.exists | default(false)) + or (callback_dropin_content_before.content | default('') | b64decode) + != callback_dropin_content }} + openclaw_apply_required: >- + {{ source_head_before.stdout | trim != expected_source_sha + or not (callback_dropin_before.stat.exists | default(false)) + or (callback_dropin_content_before.content | default('') | b64decode) + != callback_dropin_content + or openclaw_health_before.status | default(0) != 200 }} + changed_when: false + + - name: Publish check-mode source diff and no-write receipt + ansible.builtin.debug: + msg: + schema_version: openclaw_callback_forwarder_check_v1 + catalog_id: "{{ catalog_id }}" + canonical_asset_id: "{{ canonical_asset_id }}" + typed_domain: docker_compose_container + cross_domain_fallback_allowed: false + trace_id: "{{ trace_id }}" + run_id: "{{ run_id }}" + work_item_id: "{{ work_item_id }}" + systemd_working_directory: "{{ openclaw_workdir }}" + source_before_sha: "{{ source_head_before.stdout | trim }}" + source_target_sha: "{{ expected_source_sha }}" + source_update_required: "{{ source_update_required }}" + callback_dropin_update_required: "{{ callback_dropin_update_required }}" + openclaw_health_status: "{{ openclaw_health_before.status | default(0) }}" + receiver_health_status: >- + {{ callback_receiver_health_before.status | default(0) }} + would_apply: "{{ openclaw_apply_required }}" + persistent_writes_performed: 0 + changed_when: openclaw_apply_required | bool + when: ansible_check_mode + + - name: Apply exact source, drop-in, and one compose service with rollback + when: not ansible_check_mode + block: + - name: Create protected durable rollback directory + ansible.builtin.file: + path: "{{ rollback_dir }}" + state: directory + owner: root + group: root + mode: "0700" + when: openclaw_apply_required | bool + + - name: Save old immutable SHA for rollback + ansible.builtin.copy: + dest: "{{ rollback_dir }}/old-source-sha" + owner: root + group: root + mode: "0600" + content: "{{ source_head_before.stdout | trim }}\n" + when: openclaw_apply_required | bool + + - name: Save old callback drop-in for rollback + ansible.builtin.copy: + dest: "{{ rollback_dir }}/old-callback-dropin.conf" + owner: root + group: root + mode: "0600" + content: >- + {{ callback_dropin_content_before.content | b64decode }} + when: + - openclaw_apply_required | bool + - callback_dropin_before.stat.exists | default(false) + no_log: true + + - name: Record that callback drop-in was previously absent + ansible.builtin.copy: + dest: "{{ rollback_dir }}/old-callback-dropin.absent" + owner: root + group: root + mode: "0600" + content: "absent\n" + when: + - openclaw_apply_required | bool + - not (callback_dropin_before.stat.exists | default(false)) + + - name: Fetch only internal Gitea main for controlled apply + become: false + ansible.builtin.command: + argv: + - git + - fetch + - --no-tags + - origin + - refs/heads/main + args: + chdir: "{{ openclaw_workdir }}" + register: source_fetch + changed_when: true + when: source_update_required | bool + + - name: Pin fetched source to the immutable approved SHA + become: false + ansible.builtin.command: + argv: + - git + - rev-parse + - --verify + - FETCH_HEAD + args: + chdir: "{{ openclaw_workdir }}" + register: fetched_source_sha + changed_when: false + when: source_update_required | bool + + - name: Reject fetched source mismatch before merge + ansible.builtin.assert: + that: + - fetched_source_sha.stdout | trim == expected_source_sha + fail_msg: openclaw_fetched_source_sha_mismatch + when: source_update_required | bool + changed_when: false + + - name: Require current source to be ancestor of immutable target + become: false + ansible.builtin.command: + argv: + - git + - merge-base + - --is-ancestor + - "{{ source_head_before.stdout | trim }}" + - FETCH_HEAD + args: + chdir: "{{ openclaw_workdir }}" + changed_when: false + when: source_update_required | bool + + - name: Fast-forward main to immutable OpenClaw source + become: false + ansible.builtin.command: + argv: + - git + - merge + - --ff-only + - FETCH_HEAD + args: + chdir: "{{ openclaw_workdir }}" + changed_when: true + when: source_update_required | bool + + - name: Verify immutable source after fast-forward + become: false + ansible.builtin.command: + argv: + - git + - rev-parse + - --verify + - HEAD + args: + chdir: "{{ openclaw_workdir }}" + register: source_head_after + changed_when: false + + - name: Require exact immutable source after convergence + ansible.builtin.assert: + that: + - source_head_after.stdout | trim == expected_source_sha + fail_msg: openclaw_source_post_merge_mismatch + changed_when: false + + - name: Create exact systemd drop-in directory + ansible.builtin.file: + path: "{{ callback_dropin_dir }}" + state: directory + owner: root + group: root + mode: "0755" + when: callback_dropin_update_required | bool + + - name: Install non-secret callback forwarder systemd drop-in + ansible.builtin.copy: + dest: "{{ callback_dropin_path }}" + owner: root + group: root + mode: "0644" + content: "{{ callback_dropin_content }}" + when: callback_dropin_update_required | bool + + - name: Build only the immutable OpenClaw compose service with timeout + become: false + ansible.builtin.command: + argv: + - /usr/bin/timeout + - --signal=TERM + - --kill-after=15 + - "300" + - /usr/bin/docker + - compose + - --project-name + - "{{ compose_project }}" + - build + - "{{ compose_service }}" + args: + chdir: "{{ openclaw_workdir }}" + changed_when: true + when: openclaw_apply_required | bool + no_log: true + + - name: Reload systemd and restart the owned service with callback flag + ansible.builtin.systemd: + name: clawbot.service + state: restarted + daemon_reload: true + when: openclaw_apply_required | bool + no_log: true + + - name: Verify OpenClaw health after controlled compose up + become: false + ansible.builtin.uri: + url: "{{ openclaw_health_url }}" + method: GET + status_code: 200 + return_content: false + timeout: 10 + register: openclaw_health_after + retries: 12 + delay: 5 + until: openclaw_health_after.status | default(0) == 200 + check_mode: false + changed_when: false + + - name: Independently verify callback receiver public health + become: false + ansible.builtin.uri: + url: "{{ callback_receiver_health_url }}" + method: GET + status_code: 200 + return_content: false + timeout: 10 + register: callback_receiver_health_after + retries: 3 + delay: 3 + until: callback_receiver_health_after.status | default(0) == 200 + check_mode: false + changed_when: false + + - name: Publish controlled apply and verifier receipt + ansible.builtin.debug: + msg: + schema_version: openclaw_callback_forwarder_apply_v1 + catalog_id: "{{ catalog_id }}" + canonical_asset_id: "{{ canonical_asset_id }}" + typed_domain: docker_compose_container + cross_domain_fallback_allowed: false + trace_id: "{{ trace_id }}" + run_id: "{{ run_id }}" + work_item_id: "{{ work_item_id }}" + source_before_sha: "{{ source_head_before.stdout | trim }}" + source_after_sha: "{{ source_head_after.stdout | trim }}" + source_target_sha: "{{ expected_source_sha }}" + apply_performed: "{{ openclaw_apply_required }}" + rollback_available: "{{ openclaw_apply_required }}" + rollback_dir: "{{ rollback_dir if openclaw_apply_required else '' }}" + openclaw_health_status: "{{ openclaw_health_after.status }}" + callback_receiver_health_status: >- + {{ callback_receiver_health_after.status }} + independent_verifier_passed: true + secret_values_read: false + cross_domain_fallback_used: false + changed_when: false + + rescue: + - name: Restore exact old source SHA after failed apply or verifier + become: false + ansible.builtin.command: + argv: + - git + - reset + - --hard + - "{{ source_head_before.stdout | trim }}" + args: + chdir: "{{ openclaw_workdir }}" + changed_when: true + when: openclaw_apply_required | bool + + - name: Restore previous callback drop-in content + ansible.builtin.copy: + dest: "{{ callback_dropin_path }}" + owner: root + group: root + mode: "0644" + content: >- + {{ callback_dropin_content_before.content | b64decode }} + when: + - openclaw_apply_required | bool + - callback_dropin_before.stat.exists | default(false) + no_log: true + + - name: Remove callback drop-in when it was previously absent + ansible.builtin.file: + path: "{{ callback_dropin_path }}" + state: absent + when: + - openclaw_apply_required | bool + - not (callback_dropin_before.stat.exists | default(false)) + + - name: Rebuild only the restored OpenClaw compose service + become: false + ansible.builtin.command: + argv: + - /usr/bin/timeout + - --signal=TERM + - --kill-after=15 + - "300" + - /usr/bin/docker + - compose + - --project-name + - "{{ compose_project }}" + - build + - "{{ compose_service }}" + args: + chdir: "{{ openclaw_workdir }}" + changed_when: true + when: openclaw_apply_required | bool + no_log: true + + - name: Reload systemd and restart restored clawbot.service contract + ansible.builtin.systemd: + name: clawbot.service + state: restarted + daemon_reload: true + when: openclaw_apply_required | bool + no_log: true + + - name: Verify OpenClaw health after rollback + become: false + ansible.builtin.uri: + url: "{{ openclaw_health_url }}" + method: GET + status_code: 200 + return_content: false + timeout: 10 + register: openclaw_health_rollback + retries: 12 + delay: 5 + until: openclaw_health_rollback.status | default(0) == 200 + check_mode: false + changed_when: false + when: openclaw_apply_required | bool + + - name: Save durable rollback terminal receipt + ansible.builtin.copy: + dest: "{{ rollback_dir }}/rollback-receipt" + owner: root + group: root + mode: "0600" + content: | + schema_version=openclaw_callback_forwarder_rollback_v1 + catalog_id={{ catalog_id }} + canonical_asset_id={{ canonical_asset_id }} + trace_id={{ trace_id }} + run_id={{ run_id }} + work_item_id={{ work_item_id }} + source_restored_sha={{ source_head_before.stdout | trim }} + dropin_restored={{ callback_dropin_before.stat.exists | default(false) }} + rollback_verified=true + when: openclaw_apply_required | bool + + - name: Keep failed terminal after successful rollback + ansible.builtin.fail: + msg: >- + openclaw_callback_forwarder_apply_failed; + failed_task={{ ansible_failed_task.name | default('unknown') }}; + rollback_performed={{ openclaw_apply_required | bool }}; + trace_id={{ trace_id }}; + run_id={{ run_id }}; + work_item_id={{ work_item_id }} diff --git a/k8s/awoooi-prod/15-service-registry-configmap.yaml b/k8s/awoooi-prod/15-service-registry-configmap.yaml index 221f1203e..8e9f31666 100644 --- a/k8s/awoooi-prod/15-service-registry-configmap.yaml +++ b/k8s/awoooi-prod/15-service-registry-configmap.yaml @@ -9,7 +9,7 @@ metadata: app: awoooi component: service-registry annotations: - awoooi.wooo.work/source-sha256: "5548d19a6060900d51810cb3b1c3e1dc5f2fbb2b7c77df20227499cdb68599fc" + awoooi.wooo.work/source-sha256: "3977ad4862d761e93fdc13562d5a60708631b26d8cf93fbc2f5c50da3c071db0" data: service-registry.yaml: | # ops/config/service-registry.yaml @@ -219,6 +219,24 @@ data: allowed_catalog_ids: - ansible:110-sentry-profiling-consumer-recovery + - name: openclaw + canonical_id: "service:openclaw:host188" + display_name: "OpenClaw / ClawBot (host_188)" + host: "192.168.0.188" + stateful_level: AUTO + asset_domain: docker_compose_container + reason: "Only the immutable-source host188 callback-forwarder PlayBook may converge this exact compose service; cross-domain fallback is forbidden" + executor: host_ansible_executor + verifier: openclaw_callback_forwarder_independent_verifier + aliases: + - clawbot + - clawbot.service + containers: + - openclaw + - clawbot + allowed_catalog_ids: + - ansible:188-openclaw-callback-forwarder + - name: langfuse display_name: "Langfuse (LLMOps)" host: "192.168.0.110" diff --git a/ops/config/service-registry.yaml b/ops/config/service-registry.yaml index d38263b38..c4b725e90 100644 --- a/ops/config/service-registry.yaml +++ b/ops/config/service-registry.yaml @@ -205,6 +205,24 @@ services: allowed_catalog_ids: - ansible:110-sentry-profiling-consumer-recovery + - name: openclaw + canonical_id: "service:openclaw:host188" + display_name: "OpenClaw / ClawBot (host_188)" + host: "192.168.0.188" + stateful_level: AUTO + asset_domain: docker_compose_container + reason: "Only the immutable-source host188 callback-forwarder PlayBook may converge this exact compose service; cross-domain fallback is forbidden" + executor: host_ansible_executor + verifier: openclaw_callback_forwarder_independent_verifier + aliases: + - clawbot + - clawbot.service + containers: + - openclaw + - clawbot + allowed_catalog_ids: + - ansible:188-openclaw-callback-forwarder + - name: langfuse display_name: "Langfuse (LLMOps)" host: "192.168.0.110" diff --git a/ops/monitoring/service-registry.yaml b/ops/monitoring/service-registry.yaml index 3cb416b9d..95ba50561 100644 --- a/ops/monitoring/service-registry.yaml +++ b/ops/monitoring/service-registry.yaml @@ -352,7 +352,7 @@ services: auto_repair: enabled: true actions: - - restart_container + - ansible:188-openclaw-callback-forwarder owner: ai-team criticality: P0