feat(observability): add guarded SigNoz migration wave A

2026-07-15 00:49:45 +08:00
parent abdf179f1b
commit 520c0bdb01
12 changed files with 664 additions and 27 deletions
--- a/scripts/reboot-recovery/signoz-110-static-ingest.sh
+++ b/scripts/reboot-recovery/signoz-110-static-ingest.sh
@@ -13,6 +13,7 @@ SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
 TRACE_ID="${TRACE_ID:-P0-OBS-001}"
 RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
 WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+MANAGED_PROMOTION_RECEIPT="${MANAGED_PROMOTION_RECEIPT:-}"
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 LOCAL_OVERRIDE="${LOCAL_OVERRIDE:-$ROOT_DIR/ops/signoz/docker-compose.otlp-static.override.yaml}"
 SSH_OPTS=(
@@ -30,7 +31,7 @@ Usage: signoz-110-static-ingest.sh [--check|--apply|--status|--rollback]
  --check      Validate the base stack and source-controlled override.
  --apply      Recreate only signoz-otel-collector with the static base config.
  --status     Verify the collector and OTLP listeners.
-  --rollback   Recreate only the collector from the original OpAMP command.
+  --rollback   Restore managed OpAMP only with a validated promotion receipt.

 All compose calls use --env-file /dev/null. Stateful services stay untouched.
 USAGE
@@ -75,6 +76,27 @@ printf 'SIGNOZ_STATIC_INGEST_STATUS running=%s restart_count=%s grpc_4317=%s htt
 [ \"\$running\" = true ] && [ \"\$grpc\" = 1 ] && [ \"\$http\" = 1 ] && [ \"\${http_code:-000}\" = 200 ]"
 }

+validate_managed_promotion_receipt() {
+  [ -n "$MANAGED_PROMOTION_RECEIPT" ]
+  [ -s "$MANAGED_PROMOTION_RECEIPT" ]
+  python3 - "$MANAGED_PROMOTION_RECEIPT" "$WORK_ITEM_ID" <<'PY'
+import json
+import sys
+
+path, work_item_id = sys.argv[1:]
+with open(path, encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+assert payload.get("schema") == "awoooi_signoz_managed_promotion_v1"
+assert payload.get("work_item_id") == work_item_id
+assert payload.get("terminal") == "pass"
+assert payload.get("organization", {}).get("setup_completed") is True
+assert payload.get("opamp", {}).get("nop_pipeline_count") == 0
+signals = payload.get("signals", {})
+assert all(signals.get(name) is True for name in ("logs", "metrics", "traces"))
+PY
+}
+
 case "$mode" in
  --check)
    test -s "$LOCAL_OVERRIDE"
@@ -116,6 +138,10 @@ printf 'SIGNOZ_STATIC_INGEST_APPLY before_id=%s after_id=%s stateful_services_to
    receipt verify pass "runtime_listener_check"
    ;;
  --rollback)
+    if ! validate_managed_promotion_receipt; then
+      receipt rollback blocked "managed_promotion_receipt_required"
+      exit 3
+    fi
    receipt decision allow "bounded_rollback_collector_only"
    remote "set -eu
 cd '$REMOTE_DIR'
@@ -123,6 +149,6 @@ docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' up -d
 cmd=\$(docker inspect -f '{{json .Config.Cmd}}' signoz-otel-collector)
 printf 'SIGNOZ_STATIC_INGEST_ROLLBACK command=%s\\n' \"\$cmd\"
 printf '%s' \"\$cmd\" | grep -q -- '--manager-config'"
-    receipt rollback pass "opamp_command_restored"
+    receipt rollback pass "validated_managed_opamp_command_restored"
    ;;
 esac
--- a/scripts/reboot-recovery/signoz-188-otlp-bridge.sh
+++ b/scripts/reboot-recovery/signoz-188-otlp-bridge.sh
@@ -3,10 +3,12 @@
 # Owner: platform-observability. Expiry: 2026-08-15 Asia/Taipei.
 # Replacement: repoint all producers and policies directly to host 110 OTLP.

-set -euo pipefail
+set -Eeuo pipefail

 TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}"
 REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-bridge.sh}"
+REMOTE_UNIT="${REMOTE_UNIT:-/home/ollama/.config/systemd/user/awoooi-signoz-otlp-bridge.service}"
+REMOTE_SERVICE="${REMOTE_SERVICE:-awoooi-signoz-otlp-bridge.service}"
 LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}"
 LISTEN_PORT="${LISTEN_PORT:-24318}"
 SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"
@@ -15,8 +17,12 @@ UPSTREAM_PORT="${UPSTREAM_PORT:-4318}"
 TRACE_ID="${TRACE_ID:-P0-OBS-001}"
 RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
 WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+RETIRE_RECEIPT="${RETIRE_RECEIPT:-}"
 VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}"
 SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCAL_UNIT="${LOCAL_UNIT:-$ROOT_DIR/ops/systemd/user/awoooi-signoz-otlp-bridge.service}"
+SUPERVISOR_TAKEOVER_ACTIVE=0
 SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT"
@@ -27,17 +33,25 @@ SSH_OPTS=(

 usage() {
  cat <<'USAGE'
-Usage: signoz-188-otlp-bridge.sh [--check|--apply|--status|--rollback]
+Usage: signoz-188-otlp-bridge.sh [mode]

 Creates a user-level 192.168.0.188:24318 -> 192.168.0.110:4318 bridge.
 Only source addresses 192.168.0.120 and 192.168.0.121 are accepted.
 No sudo, firewall, Nginx, Docker, database, secret, or volume action is used.
+
+  --check                Read-only prerequisite check.
+  --apply                Start the legacy bounded bridge (rollback target).
+  --status               Verify either supervised or legacy bridge runtime.
+  --supervise            Migrate the running bridge to a user-systemd unit.
+  --supervisor-status    Require enabled+active user-systemd supervision.
+  --supervisor-rollback  Restore the legacy process and reboot cron entry.
+  --rollback             Retire the bridge only with a validated receipt.
 USAGE
 }

 mode="${1:---check}"
 case "$mode" in
-  --check|--apply|--status|--rollback) ;;
+  --check|--apply|--status|--supervise|--supervisor-status|--supervisor-rollback|--rollback) ;;
  -h|--help) usage; exit 0 ;;
  *) usage >&2; exit 64 ;;
 esac
@@ -52,6 +66,26 @@ receipt() {
    "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail"
 }

+on_error() {
+  local rc=$?
+  trap - ERR
+  if [ "$SUPERVISOR_TAKEOVER_ACTIVE" -eq 1 ]; then
+    set +e
+    remote "systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true; sleep 1; $env_prefix '$REMOTE_SCRIPT' start"
+    install_legacy_cron
+    rollback_rc=$?
+    set -e
+    if [ "$rollback_rc" -eq 0 ]; then
+      receipt rollback pass "supervisor_takeover_failed_legacy_bridge_restored"
+    else
+      receipt rollback failed "supervisor_takeover_failed_legacy_restore_failed"
+    fi
+  fi
+  receipt terminal failed "command_exit_$rc"
+  exit "$rc"
+}
+trap on_error ERR
+
 remote_payload() {
  cat <<'REMOTE'
 #!/usr/bin/env bash
@@ -102,6 +136,18 @@ start() {
  status
 }

+run_foreground() {
+  command -v socat >/dev/null
+  mkdir -p "$RUN_DIR" "$LOG_DIR"
+  if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${LISTEN_PORT}$"; then
+    echo "SIGNOZ_OTLP_BRIDGE_BLOCKED port_busy=${LISTEN_PORT}" >&2
+    exit 2
+  fi
+  echo "$$" >"$PID_FILE"
+  exec socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \
+    "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}"
+}
+
 stop() {
  if is_running; then kill "$(sed -n '1p' "$PID_FILE")"; fi
  rm -f "$PID_FILE"
@@ -110,6 +156,7 @@ stop() {

 case "${1:-status}" in
  start) start ;;
+  run) run_foreground ;;
  stop) stop ;;
  status) status ;;
  *) exit 64 ;;
@@ -128,23 +175,54 @@ verify_nodes() {
  done
 }

+validate_retire_receipt() {
+  [ -n "$RETIRE_RECEIPT" ]
+  [ -s "$RETIRE_RECEIPT" ]
+  python3 - "$RETIRE_RECEIPT" "$WORK_ITEM_ID" <<'PY'
+import json
+import sys
+
+path, work_item_id = sys.argv[1:]
+with open(path, encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+assert payload.get("schema") == "awoooi_signoz_bridge_retirement_v1"
+assert payload.get("work_item_id") == work_item_id
+assert payload.get("terminal") == "pass"
+assert payload.get("active_direct_producers") == 5
+assert payload.get("queue_zero") is True
+assert payload.get("direct_post_verifier") is True
+PY
+}
+
+install_legacy_cron() {
+  remote "(crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d'; printf '%s\n' '# AWOOOI SignOz OTLP bridge start' '@reboot $env_prefix $REMOTE_SCRIPT start >/tmp/awoooi-signoz-otlp-bridge.cron.log 2>&1' '# AWOOOI SignOz OTLP bridge end') | crontab -"
+}
+
+remove_legacy_cron() {
+  remote "crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d' | crontab -"
+}
+
 case "$mode" in
  --check)
+    test -s "$LOCAL_UNIT"
    receipt sensor pass "canonical_110_upstream_selected"
    remote "set -eu
 command -v socat >/dev/null
+command -v systemctl >/dev/null
+linger=\$(loginctl show-user \"\$USER\" -p Linger --value 2>/dev/null || true)
 upstream=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs || true)
 busy=0
 if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '(^|:)${LISTEN_PORT}$'; then busy=1; fi
 printf 'SIGNOZ_OTLP_BRIDGE_CHECK upstream_code=%s listen_port_busy=%s source_range=%s\\n' \"\${upstream:-000}\" \"\$busy\" '$SOURCE_RANGE'
-[ \"\${upstream:-000}\" = 200 ]"
-    receipt check pass "upstream_otlp_200"
+[ \"\${upstream:-000}\" = 200 ] && [ \"\${linger:-}\" = yes ]"
+    receipt check pass "upstream_otlp_200_linger_yes_unit_source_present"
    ;;
  --apply)
    receipt decision allow "risk_medium_user_level_compatibility_bridge"
    remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload)
    remote "chmod 0755 '$REMOTE_SCRIPT.tmp' && mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'"
-    remote "(crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d'; printf '%s\n' '# AWOOOI SignOz OTLP bridge start' '@reboot $env_prefix $REMOTE_SCRIPT start >/tmp/awoooi-signoz-otlp-bridge.cron.log 2>&1' '# AWOOOI SignOz OTLP bridge end') | crontab -"
+    install_legacy_cron
    remote "$env_prefix '$REMOTE_SCRIPT' start"
    verify_nodes
    receipt execute pass "bridge_started_allowlist_120_121"
@@ -155,8 +233,75 @@ printf 'SIGNOZ_OTLP_BRIDGE_CHECK upstream_code=%s listen_port_busy=%s source_ran
    verify_nodes
    receipt verify pass "bridge_runtime_check"
    ;;
+  --supervise)
+    test -s "$LOCAL_UNIT"
+    receipt decision allow "risk_medium_user_systemd_supervisor_takeover"
+    remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log /home/ollama/.config/systemd/user; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload)
+    scp "${SSH_OPTS[@]}" "$LOCAL_UNIT" "$TARGET_HOST:$REMOTE_UNIT"
+    remote "set -eu
+chmod 0755 '$REMOTE_SCRIPT.tmp'
+chmod 0644 '$REMOTE_UNIT'
+mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'
+systemd-analyze --user verify '$REMOTE_UNIT' >/dev/null
+printf 'SIGNOZ_OTLP_BRIDGE_SUPERVISOR_DRY_RUN unit_verify=pass\n'"
+    receipt check pass "user_unit_verified_before_takeover"
+    SUPERVISOR_TAKEOVER_ACTIVE=1
+    remote "set -eu
+$env_prefix '$REMOTE_SCRIPT' stop || true
+sleep 1
+systemctl --user daemon-reload
+if ! systemctl --user enable --now '$REMOTE_SERVICE'; then
+  systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true
+  $env_prefix '$REMOTE_SCRIPT' start
+  exit 1
+fi
+ready=0
+for _ in 1 2 3 4 5 6 7 8 9 10; do
+  if systemctl --user is-active --quiet '$REMOTE_SERVICE' && $env_prefix '$REMOTE_SCRIPT' status; then
+    ready=1
+    break
+  fi
+  sleep 1
+done
+[ \"\$ready\" = 1 ]
+systemctl --user is-enabled --quiet '$REMOTE_SERVICE'
+printf 'SIGNOZ_OTLP_BRIDGE_SUPERVISOR_READY wait_seconds_lte=10\n'"
+    remove_legacy_cron
+    verify_nodes
+    SUPERVISOR_TAKEOVER_ACTIVE=0
+    receipt execute pass "bridge_migrated_to_user_systemd"
+    receipt verify pass "unit_enabled_active_nodes_120_121_otlp_200"
+    ;;
+  --supervisor-status)
+    remote "set -eu
+systemctl --user is-enabled --quiet '$REMOTE_SERVICE'
+systemctl --user is-active --quiet '$REMOTE_SERVICE'
+printf 'SIGNOZ_OTLP_BRIDGE_SUPERVISOR_STATUS enabled=1 active=1\n'
+$env_prefix '$REMOTE_SCRIPT' status"
+    verify_nodes
+    receipt verify pass "user_systemd_supervisor_runtime_check"
+    ;;
+  --supervisor-rollback)
+    receipt decision allow "bounded_supervisor_rollback_to_legacy_bridge"
+    remote "set -eu
+systemctl --user disable --now '$REMOTE_SERVICE'
+sleep 1
+if ! $env_prefix '$REMOTE_SCRIPT' start; then
+  systemctl --user enable --now '$REMOTE_SERVICE'
+  exit 1
+fi"
+    install_legacy_cron
+    remote "rm -f '$REMOTE_UNIT'; systemctl --user daemon-reload"
+    verify_nodes
+    receipt rollback pass "legacy_bridge_and_reboot_cron_restored"
+    ;;
  --rollback)
-    remote "if [ -x '$REMOTE_SCRIPT' ]; then $env_prefix '$REMOTE_SCRIPT' stop; fi; crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d' | crontab -"
-    receipt rollback pass "bridge_stopped_cron_removed"
+    if ! validate_retire_receipt; then
+      receipt rollback blocked "bridge_retirement_receipt_required"
+      exit 3
+    fi
+    remote "systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true; if [ -x '$REMOTE_SCRIPT' ]; then $env_prefix '$REMOTE_SCRIPT' stop; fi"
+    remove_legacy_cron
+    receipt rollback pass "validated_bridge_retirement_service_and_cron_stopped"
    ;;
 esac
--- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py
+++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""No-write source preflight for P0-OBS-002 Wave A.
+
+The checker intentionally does not access credentials, databases, runtime
+volumes, Docker, Kubernetes, or remote hosts. Runtime evidence belongs to a
+separate verifier receipt.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+SCHEMA = "awoooi_signoz_canonical_route_preflight_v1"
+EXPECTED_BLOCKERS = [
+    "organization_not_initialized",
+    "direct_ingress_policy_not_closed",
+    "traces_not_fresh",
+    "bridge_not_supervised",
+    "filelog_normalization_degraded",
+    "wave_b_not_authorized",
+]
+
+
+def _load_yaml(path: Path) -> Any:
+    with path.open(encoding="utf-8") as handle:
+        return yaml.safe_load(handle)
+
+
+def _read(root: Path, relative: str) -> str:
+    return (root / relative).read_text(encoding="utf-8")
+
+
+def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
+    contract = _load_yaml(contract_path)
+    checks: dict[str, bool] = {}
+
+    checks["contract_schema"] = (
+        contract.get("schema")
+        == "awoooi_signoz_organization_canonical_route_migration_v1"
+    )
+    checks["work_item"] = contract.get("work_item_id") == "P0-OBS-002"
+    checks["wave_a_phase"] = contract.get("phase") == "wave_a_dual_allow"
+    checks["active_route_incumbent"] = (
+        contract.get("routes", {}).get("active_route") == "incumbent"
+    )
+
+    producer_checks: list[bool] = []
+    for producer in contract.get("producers", []):
+        sources = producer.get("sources") or [producer.get("source")]
+        endpoint = str(producer.get("current_endpoint", ""))
+        producer_checks.extend(
+            bool(source) and endpoint in _read(root, str(source)) for source in sources
+        )
+    checks["active_producers_unchanged"] = bool(producer_checks) and all(
+        producer_checks
+    )
+
+    config = _read(root, "apps/api/src/core/config.py")
+    checks["dual_route_application_allow"] = (
+        'default=["192.168.0.188", "192.168.0.110"]' in config
+        and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]'
+        in config
+    )
+
+    network_documents = list(
+        yaml.safe_load_all(_read(root, "k8s/awoooi-prod/02-network-policy.yaml"))
+    )
+    egress_policy = next(
+        document
+        for document in network_documents
+        if document
+        and document.get("kind") == "NetworkPolicy"
+        and document.get("metadata", {}).get("name") == "allow-required-egress"
+    )
+    canonical_ports: set[int] = set()
+    for rule in egress_policy.get("spec", {}).get("egress", []):
+        cidrs = {
+            item.get("ipBlock", {}).get("cidr") for item in rule.get("to", [])
+        }
+        if "192.168.0.110/32" in cidrs:
+            canonical_ports.update(
+                int(item["port"])
+                for item in rule.get("ports", [])
+                if isinstance(item.get("port"), int)
+            )
+    checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset(
+        canonical_ports
+    )
+
+    unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-bridge.service")
+    checks["bridge_unit_hardened"] = all(
+        value in unit
+        for value in (
+            "Restart=on-failure",
+            "NoNewPrivileges=true",
+            "PrivateTmp=true",
+            "ProtectSystem=strict",
+            "SOURCE_RANGE=192.168.0.120/31",
+        )
+    )
+
+    verifier = _read(root, "scripts/reboot-recovery/verify-signoz-otel-ingestion.sh")
+    checks["route_specific_verifier"] = (
+        "OTLP_ENDPOINT" in verifier and "ROUTE_ID" in verifier
+    )
+    static_recovery = _read(
+        root, "scripts/reboot-recovery/signoz-110-static-ingest.sh"
+    )
+    checks["managed_promotion_fail_closed"] = all(
+        value in static_recovery
+        for value in (
+            "MANAGED_PROMOTION_RECEIPT",
+            "managed_promotion_receipt_required",
+            "nop_pipeline_count",
+        )
+    )
+
+    terminal = contract.get("terminal", {})
+    blockers = list(terminal.get("blockers", []))
+    checks["expected_blockers_explicit"] = blockers == EXPECTED_BLOCKERS
+    source_ready = all(checks.values())
+    return {
+        "schema": SCHEMA,
+        "work_item_id": "P0-OBS-002",
+        "mode": "check",
+        "phase": "wave_a_dual_allow",
+        "source_ready": source_ready,
+        "runtime_status": terminal.get("status", "unknown"),
+        "active_route": "incumbent",
+        "writes_performed": 0,
+        "checks": checks,
+        "blockers": blockers,
+        "terminal": "pass" if source_ready else "failed",
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2])
+    parser.add_argument(
+        "--contract",
+        type=Path,
+        default=None,
+    )
+    args = parser.parse_args()
+    root = args.root.resolve()
+    contract = args.contract or (
+        root / "ops/signoz/organization-canonical-route-migration.yaml"
+    )
+    payload = evaluate(root, contract.resolve())
+    print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
+    return 0 if payload["terminal"] == "pass" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
+++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+import yaml
+
+
+ROOT = Path(__file__).resolve().parents[3]
+CONTRACT = ROOT / "ops/signoz/organization-canonical-route-migration.yaml"
+PREFLIGHT = ROOT / "scripts/reboot-recovery/signoz-canonical-route-preflight.py"
+BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-bridge.sh"
+STATIC = ROOT / "scripts/reboot-recovery/signoz-110-static-ingest.sh"
+UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-bridge.service"
+
+
+def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
+    payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
+    assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
+    assert payload["work_item_id"] == "P0-OBS-002"
+    assert payload["phase"] == "wave_a_dual_allow"
+    assert payload["routes"]["active_route"] == "incumbent"
+    assert payload["organization"]["apply_allowed_in_wave_a"] is False
+    assert payload["policy"]["secret_values_forbidden_in_source"] is True
+    assert len(payload["producers"]) == 5
+
+    for producer in payload["producers"]:
+        sources = producer.get("sources") or [producer["source"]]
+        for source in sources:
+            assert producer["current_endpoint"] in (ROOT / source).read_text(
+                encoding="utf-8"
+            )
+
+
+def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None:
+    text = CONTRACT.read_text(encoding="utf-8").lower()
+    assert "signoz_user_root_password:" not in text
+    assert "signoz-api-key:" not in text
+    assert "authorization:" not in text
+    assert "raw_relational_database_update" in text
+    assert "stateful_volume_restore" in text
+    assert "apply_allowed_in_wave_a: false" in text
+
+
+def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
+    result = subprocess.run(
+        [sys.executable, str(PREFLIGHT), "--root", str(ROOT)],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    payload = json.loads(result.stdout)
+    assert payload["source_ready"] is True
+    assert payload["writes_performed"] == 0
+    assert payload["terminal"] == "pass"
+    assert payload["runtime_status"] == "partial_degraded"
+    assert payload["active_route"] == "incumbent"
+    assert payload["checks"] and all(payload["checks"].values())
+    assert "organization_not_initialized" in payload["blockers"]
+    assert "wave_b_not_authorized" in payload["blockers"]
+
+
+def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
+    text = UNIT.read_text(encoding="utf-8")
+    for expected in (
+        "Wants=network-online.target",
+        "After=network-online.target",
+        "SOURCE_RANGE=192.168.0.120/31",
+        "ExecStart=/home/ollama/bin/awoooi-signoz-otlp-bridge.sh run",
+        "Restart=on-failure",
+        "NoNewPrivileges=true",
+        "PrivateTmp=true",
+        "ProtectSystem=strict",
+        "ProtectHome=read-only",
+        "WantedBy=default.target",
+    ):
+        assert expected in text
+    assert "User=" not in text
+    assert "sudo" not in text
+
+
+def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
+    env = os.environ.copy()
+    env.pop("MANAGED_PROMOTION_RECEIPT", None)
+    env.pop("RETIRE_RECEIPT", None)
+
+    managed = subprocess.run(
+        ["bash", str(STATIC), "--rollback"],
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+    assert managed.returncode == 3
+    assert "managed_promotion_receipt_required" in managed.stdout
+    assert "ssh" not in managed.stderr.lower()
+
+    retired = subprocess.run(
+        ["bash", str(BRIDGE), "--rollback"],
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+    assert retired.returncode == 3
+    assert "bridge_retirement_receipt_required" in retired.stdout
+    assert "ssh" not in retired.stderr.lower()
--- a/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py
+++ b/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py
@@ -36,6 +36,10 @@ def test_static_apply_is_bounded_no_pull_and_never_reads_dotenv() -> None:
    assert "docker volume" not in executable
    assert "docker system prune" not in executable
    assert "github.com" not in text
+    assert "MANAGED_PROMOTION_RECEIPT" in text
+    assert "awoooi_signoz_managed_promotion_v1" in text
+    assert "managed_promotion_receipt_required" in text
+    assert "nop_pipeline_count" in text


 def test_otlp_bridge_is_allowlisted_reversible_and_has_sunset() -> None:
@@ -53,9 +57,25 @@ def test_otlp_bridge_is_allowlisted_reversible_and_has_sunset() -> None:
    assert "@reboot" in text
    assert "verify_nodes" in text
    assert "--rollback" in text
+    assert "--supervise" in text
+    assert "--supervisor-status" in text
+    assert "--supervisor-rollback" in text
+    assert "systemctl --user" in text
+    assert "run_foreground" in text
+    assert "RETIRE_RECEIPT" in text
+    assert "awoooi_signoz_bridge_retirement_v1" in text
+    assert "bridge_retirement_receipt_required" in text
+    assert "SUPERVISOR_TAKEOVER_ACTIVE=1" in text
+    assert "for _ in 1 2 3 4 5 6 7 8 9 10" in text
+    assert "SIGNOZ_OTLP_BRIDGE_SUPERVISOR_READY wait_seconds_lte=10" in text
+    assert text.index("SUPERVISOR_TAKEOVER_ACTIVE=1") < text.index(
+        "systemctl --user enable --now '$REMOTE_SERVICE'"
+    )
    assert "2026-08-15" in text
    assert "sudo " not in executable
-    assert "systemctl " not in executable
+    for line in executable.splitlines():
+        if "systemctl " in line and "command -v systemctl" not in line:
+            assert "systemctl --user" in line
    assert "iptables " not in executable
    assert "docker " not in executable

@@ -67,7 +87,10 @@ def test_ingestion_verifier_binds_queue_bridge_and_clickhouse() -> None:
    assert "otelcol_exporter_send_failed_log_records" in text
    assert "otelcol_exporter_sent_log_records" in text
    assert "otelcol_exporter_queue_size" in text
-    assert "192.168.0.188:24318/v1/logs" in text
+    assert "OTLP_ENDPOINT" in text
+    assert "ROUTE_ID" in text
+    assert "192.168.0.188:24318/v1/logs" in text  # compatibility default
+    assert "awoooi.route.id" in text
    assert "signoz_logs.logs_v2" in text
    assert "body={marker:String}" in text
    assert "declare -A" not in text
--- a/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh
+++ b/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh
@@ -1,20 +1,23 @@
 #!/usr/bin/env bash
-# Independent verifier for K3s -> 188 compatibility bridge -> 110 SigNoz -> ClickHouse.
+# Independent verifier for either the incumbent bridge or canonical SigNoz OTLP route.
 # It writes one non-sensitive synthetic log and queries only that exact marker.

 set -euo pipefail

-TRACE_ID="${TRACE_ID:-P0-OBS-001}"
-RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}"
-WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}"
+TRACE_ID="${TRACE_ID:-P0-OBS-002}"
+RUN_ID="${RUN_ID:-P0-OBS-002-$(date -u +%Y%m%dT%H%M%SZ)}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}"
 WINDOW_SECONDS="${WINDOW_SECONDS:-30}"
 EXPECTED_PODS="${EXPECTED_PODS:-2}"
 EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}"
 SOURCE_NODE="${SOURCE_NODE:-wooo@192.168.0.120}"
 SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}"
-BRIDGE_ENDPOINT="${BRIDGE_ENDPOINT:-http://192.168.0.188:24318/v1/logs}"
+# BRIDGE_ENDPOINT remains a compatibility alias for old callers. New migration
+# runs must set OTLP_ENDPOINT explicitly so route identity is present in the run.
+OTLP_ENDPOINT="${OTLP_ENDPOINT:-${BRIDGE_ENDPOINT:-http://192.168.0.188:24318/v1/logs}}"
+ROUTE_ID="${ROUTE_ID:-incumbent_bridge}"
 SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1)
-marker="P0OBS001_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)"
+marker="P0OBS_${ROUTE_ID//[^A-Za-z0-9_]/_}_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)"
 pod_list=()
 enqueue_before=()
 failed_before=()
@@ -59,9 +62,9 @@ for ((i = 0; i < pod_count; i++)); do
 done
 receipt sensor pass "collector_pods_${pod_count}_queue_capacity_${EXPECTED_QUEUE_CAPACITY}"

-http_code="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" "ts=\$(date +%s%N); curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"awoooi-p0-obs-verifier\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"awoooi.obs.verifier\"},\"logRecords\":[{\"timeUnixNano\":\"'\"\$ts\"'\",\"severityNumber\":9,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"${marker}\"}}]}]}]}]}' '$BRIDGE_ENDPOINT'")"
+http_code="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" "ts=\$(date +%s%N); curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"awoooi-p0-obs-verifier\"}},{\"key\":\"awoooi.route.id\",\"value\":{\"stringValue\":\"${ROUTE_ID}\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"awoooi.obs.verifier\"},\"logRecords\":[{\"timeUnixNano\":\"'\"\$ts\"'\",\"severityNumber\":9,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"${marker}\"}}]}]}]}]}' '$OTLP_ENDPOINT'")"
 [ "$http_code" = 200 ]
-receipt execute pass "synthetic_otlp_http_200_marker_${marker}"
+receipt execute pass "route_${ROUTE_ID}_synthetic_otlp_http_200_marker_${marker}"

 count=0
 for _ in $(seq 1 15); do