diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index e34d9496b..1d292c61c 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -466,8 +466,9 @@ class Settings(BaseSettings): # ========================================================================== # OpenTelemetry (可觀測性鐵律) - # 四主機架構強制校驗: OTEL 必須指向 192.168.0.188（AWOOOI 主站） - # ADR-121 + P0-08 修正：改為 config-driven，允許 EwoooC 指向不同 host + # P0-OBS-002 dual-route transition: active traffic remains on the bounded + # .188 bridge until replay/shadow/canary receipts exist, while .110 is the + # allowlisted canonical challenger. Promotion is a separate controlled run. # ========================================================================== OTEL_ENABLED: bool = Field( default=True, @@ -478,11 +479,11 @@ class Settings(BaseSettings): description="SigNoz OTLP gRPC endpoint (Host port 24317 -> Container 4317) - NO http:// prefix for gRPC", ) OTEL_ALLOWED_ENDPOINTS: list[str] = Field( - default=["192.168.0.188"], - description="允許的 OTEL endpoint host 列表（逗號分隔可用 env 覆寫）。EwoooC 可設自己的 SigNoz host。", + default=["192.168.0.188", "192.168.0.110"], + description="允許的 OTEL endpoint host 列表；.188 是 transition bridge，.110 是 canonical challenger。", ) OTEL_FORBIDDEN_ENDPOINTS: list[str] = Field( - default=["192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121"], + default=["192.168.0.112", "192.168.0.120", "192.168.0.121"], description="明確禁止的 OTEL endpoint host 列表（不允許誤指向非 SigNoz 主機）", ) AWOOOI_K8S_NAMESPACE: str = Field( diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index fad8a14f4..6fe865663 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -54739,3 +54739,11 @@ production browser smoke: - Agent99 SRE routing self-test `17/17`；Telegram ingest self-test `16/16`，不寫 alert、不送 Telegram。 - Agent99 tests `49 passed`；reboot-recovery suite `142 passed`；Windows staging contract/synthetic、JSON/YAML parse 通過。 - 112 runtime scope 已關閉。全主機 program 尚未關閉：只剩 fresh all-host reboot window 的三項歷史 blocker，必須由下一次真實 600 秒 drill 驗證，不能以事後綠燈清除。 +## 2026-07-15 — P0-OBS-002 SigNoz organization / canonical route Wave A + +- Current P0 固定為 `P0-OBS-002`；organization、canonical data route、Foundry/OpAMP、ClickHouse upgrade 拆成獨立 run，禁止一次切換。官方 self-hosted OTLP 4317/4318 不需 ingestion key，但首次 organization/root 需要 durable credential，因此本輪未建立帳號、未送 setup POST、未讀 secret/raw DB/raw volume。 +- 新增 `ops/signoz/organization-canonical-route-migration.yaml` 與 no-write `signoz-canonical-route-preflight.py`。Wave A source receipt 為 `source_ready=true`、`active_route=incumbent`、`writes_performed=0`；5 類 producer endpoint 全部保持 `.188`，只把 `.110` 加入 API allowlist 與 production NetworkPolicy 4317/4318 transition allow。 +- `verify-signoz-otel-ingestion.sh` 現可用 `OTLP_ENDPOINT` / `ROUTE_ID` 比較 route。`P0-OBS-002-20260715T003500+0800` 對 incumbent `.188`、`.120 -> .110 direct`、`.121 -> .110 direct` 三條 replay/shadow 都得到 HTTP 200、ClickHouse exact marker `1`、15 秒 queue zero/failure delta zero；這只證明 log data-plane challenger，不代表 security/org/traces gate 完成。 +- `.188` bridge supervisor source 新增 user-systemd unit、source-range、restart、filesystem/process hardening、legacy cron rollback 與 receipt-gated retirement。三次 bounded takeover 分別暴露 user-manager capability directive、ExecStartPre JSON quoting、listener readiness race；第三版 unit journal 已證明 `Started`，但 executor 於 socket ready 前判失敗。依三輪上限停止 apply，執行 `--supervisor-rollback`。 +- Rollback terminal：legacy process 與 reboot cron 已恢復；`.120/.121` 各自 OTLP HTTP 200，獨立 synthetic marker exact `1`，queue/failure delta `0`。Source 已把 takeover flag 提前並加入最多 10 秒 readiness loop，但本輪不再重試 runtime。 +- Focused source tests `17 passed`；bash syntax、Python compile、YAML preflight、`git diff --check` 通過。Runtime 仍為 `partial_degraded`：`setupCompleted=false`、direct ingress policy 未關閉、traces 不新鮮、bridge 尚未 supervised、filelog normalization degraded、Wave B 未授權。 diff --git a/docs/adr/ADR-053-observability-signoz-unified-architecture.md b/docs/adr/ADR-053-observability-signoz-unified-architecture.md index 842f81345..9a9c29629 100644 --- a/docs/adr/ADR-053-observability-signoz-unified-architecture.md +++ b/docs/adr/ADR-053-observability-signoz-unified-architecture.md @@ -10,6 +10,7 @@ | **2026-07-14 執行期修正** | `superseded_by=global_product_governance_v2`；`.188` SigNoz 主機配置已取代 | | **負責範圍 / 到期日** | `platform-observability` / 暫時 `.188:24318` 橋接於 `2026-08-15 Asia/Taipei` 到期 | | **替代方案 / 驗證器** | 正式後端 `.110:4318`；`scripts/reboot-recovery/verify-signoz-otel-ingestion.sh` | +| **2026-07-15 P0-OBS-002** | organization 與 canonical route 拆成獨立 gate；Wave A 只做 dual allow / replay / supervisor source，不切 active route | --- @@ -57,6 +58,12 @@ SigNoz OTEL Collector (Prometheus Receiver) 2026-07-14: `.188` 不再承載 SigNoz stateful runtime。相容 bridge 只允許 K3s `.120/.121` 來源，並必須在到期日前將所有產生端與 NetworkPolicy 改為直接 `.110`。 + +2026-07-15: `.110` self-hosted OTLP 不使用 ingestion key，且目前 4317/4318 +對內網的暴露面大於 `.188` bridge source range。因此 producer promotion 前新增獨立 +`direct_ingress_policy_gate`；organization 首位 Admin 需要 credential，屬 high/critical +secret boundary，不得與資料平面切流綁成一次不可回滾變更。正式狀態機以 +`ops/signoz/organization-canonical-route-migration.yaml` 為準。 ``` --- @@ -134,8 +141,13 @@ Warning/Error Event 全量保留。Normal/Scheduled/Pulling/Pulled/Created/Start ## 後續行動 -1. SSH 到 .188 部署剩餘配置 (prometheus-config-phase-o.yaml / prometheus-remote-write-signoz.yaml / minio-kali-alerts.yaml / SigNoz rules) -2. Wave D: NVIDIA Grafana Dashboard + 監控覆蓋率報告 +1. P0-OBS-002 Wave A：保留 `.188` active route，完成 machine-readable inventory、 + dual allow、route-specific replay 與 guarded bridge supervisor。 +2. organization/root bootstrap 使用 secret reference 且獨立授權；禁止 raw DB UPDATE、 + raw secret read 或 stateful volume restore。 +3. Wave B：direct ingress policy、logs/metrics/traces same-run shadow、單 producer canary、 + queue/normalization verifier 全綠後，才可逐步 promote 並退場 OTLP bridge。 +4. `.188:3301 -> .110:8080` UI bridge 是不同資產，不在 OTLP bridge retirement 範圍。 --- diff --git a/k8s/awoooi-prod/02-network-policy.yaml b/k8s/awoooi-prod/02-network-policy.yaml index 69c325bc1..b7a2f629d 100644 --- a/k8s/awoooi-prod/02-network-policy.yaml +++ b/k8s/awoooi-prod/02-network-policy.yaml @@ -158,6 +158,13 @@ spec: # Gitea — CI/CD 主倉 probe + monitoring - protocol: TCP port: 3001 + # P0-OBS-002 transition allow only. Active producers remain on the + # .188 compatibility bridge until replay/shadow/canary passes. + # SigNoz canonical OTLP gRPC / HTTP on host .110. + - protocol: TCP + port: 4317 + - protocol: TCP + port: 4318 # 2026-05-25 Codex: ADR-110 production Ollama proxy pool. # 11435=GCP-A, 11436=GCP-B, 11437=Local 111 fallback. - protocol: TCP diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml new file mode 100644 index 000000000..e73b40084 --- /dev/null +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -0,0 +1,108 @@ +schema: awoooi_signoz_organization_canonical_route_migration_v1 +work_item_id: P0-OBS-002 +owner_lane: platform-observability +phase: wave_a_dual_allow +generated_at: "2026-07-15T00:35:00+08:00" +policy: + superseded_by: global_product_governance_v2 + controlled_apply_allowed: true + secret_values_forbidden_in_source: true + raw_database_access_forbidden: true + raw_volume_access_forbidden: true + github_supply_chain_forbidden: true + +organization: + canonical_id: signoz-110-organization + observed_setup_completed: false + observed_version: v0.113.0 + risk: high + secret_reference_required: true + apply_allowed_in_wave_a: false + exit_conditions: + - setup_completed_true + - root_admin_readback_pass + - service_account_readback_pass + - metadata_snapshot_and_restore_verifier_pass + prohibited_actions: + - raw_relational_database_update + - raw_runtime_secret_read + - stateful_volume_restore + +routes: + incumbent: + canonical_id: signoz-188-otlp-compat-bridge + grpc: 192.168.0.188:24317 + http: http://192.168.0.188:24318 + source_range: 192.168.0.120/31 + expiry: "2026-08-15T23:59:59+08:00" + challenger: + canonical_id: signoz-110-otlp + grpc: 192.168.0.110:4317 + http: http://192.168.0.110:4318 + self_hosted_ingestion_key_required: false + ingress_policy_status: blocked_listener_broader_than_source_policy + active_route: incumbent + +producers: + - canonical_id: k3s-filelog-daemonset + source: k8s/observability/otel-collector-daemonset.yaml + protocol: otlp_http + current_endpoint: http://192.168.0.188:24318 + target_endpoint: http://192.168.0.110:4318 + - canonical_id: k3s-event-exporter + source: k8s/observability/event-exporter.yaml + protocol: otlp_http_logs + current_endpoint: http://192.168.0.188:24318/v1/logs + target_endpoint: http://192.168.0.110:4318/v1/logs + - canonical_id: awoooi-prod-api + source: k8s/awoooi-prod/04-configmap.yaml + protocol: otlp_grpc + current_endpoint: http://192.168.0.188:24317 + target_endpoint: http://192.168.0.110:4317 + - canonical_id: awoooi-dev-api + source: k8s/awoooi-dev/02-configmap.yaml + protocol: otlp_grpc + current_endpoint: http://192.168.0.188:24317 + target_endpoint: http://192.168.0.110:4317 + - canonical_id: gitea-workflows + sources: + - .gitea/workflows/cd.yaml + - .gitea/workflows/cd-dev.yaml + - .gitea/workflows/e2e-health.yaml + protocol: otlp_http + current_endpoint: http://192.168.0.188:24318 + target_endpoint: http://192.168.0.110:4318 + +wave_a_gates: + active_endpoints_unchanged: true + challenger_application_allowlisted: true + challenger_networkpolicy_allowed: true + bridge_user_systemd_supervision_required: true + route_specific_log_replay_required: true + +wave_b_gates: + organization_managed_collector_gate: independent + direct_ingress_policy_gate: required + logs_metrics_traces_same_run_gate: required + queue_and_failure_delta_gate: required + normalization_error_rate_gate: required + independent_post_verifier_gate: required + bridge_retirement_gate: required + rollback_route: incumbent + +runtime_observations: + logs_fresh: true + metrics_fresh: true + traces_fresh: false + bridge_supervisor: legacy_orphan_process + filelog_normalization: degraded + +terminal: + status: partial_degraded + blockers: + - organization_not_initialized + - direct_ingress_policy_not_closed + - traces_not_fresh + - bridge_not_supervised + - filelog_normalization_degraded + - wave_b_not_authorized diff --git a/ops/systemd/user/awoooi-signoz-otlp-bridge.service b/ops/systemd/user/awoooi-signoz-otlp-bridge.service new file mode 100644 index 000000000..be6415643 --- /dev/null +++ b/ops/systemd/user/awoooi-signoz-otlp-bridge.service @@ -0,0 +1,34 @@ +[Unit] +Description=AWOOOI SigNoz OTLP compatibility bridge +Wants=network-online.target +After=network-online.target + +[Service] +Type=simple +Environment=LISTEN_HOST=192.168.0.188 +Environment=LISTEN_PORT=24318 +Environment=SOURCE_RANGE=192.168.0.120/31 +Environment=UPSTREAM_HOST=192.168.0.110 +Environment=UPSTREAM_PORT=4318 +ExecStartPre=/usr/bin/curl -fsS --max-time 5 -H Content-Type:application/json --data-binary {\"resourceLogs\":[]} http://192.168.0.110:4318/v1/logs +ExecStart=/home/ollama/bin/awoooi-signoz-otlp-bridge.sh run +ExecStopPost=/usr/bin/rm -f /home/ollama/run/awoooi-signoz-otlp-bridge.pid +Restart=on-failure +RestartSec=5s +TimeoutStartSec=15s +TimeoutStopSec=15s +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=/home/ollama/run /home/ollama/log +# CapabilityBoundingSet-derived ProtectKernel*/RestrictSUIDSGID directives are +# intentionally omitted: this is a user manager and cannot apply them. The +# remaining namespace/filesystem/process controls are independently verified. +LockPersonality=true +MemoryDenyWriteExecute=true +RestrictAddressFamilies=AF_INET AF_INET6 + +[Install] +WantedBy=default.target diff --git a/scripts/reboot-recovery/signoz-110-static-ingest.sh b/scripts/reboot-recovery/signoz-110-static-ingest.sh index 0ff9ab0b5..2a6914d6f 100755 --- a/scripts/reboot-recovery/signoz-110-static-ingest.sh +++ b/scripts/reboot-recovery/signoz-110-static-ingest.sh @@ -13,6 +13,7 @@ SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}" TRACE_ID="${TRACE_ID:-P0-OBS-001}" RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}" WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}" +MANAGED_PROMOTION_RECEIPT="${MANAGED_PROMOTION_RECEIPT:-}" ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" LOCAL_OVERRIDE="${LOCAL_OVERRIDE:-$ROOT_DIR/ops/signoz/docker-compose.otlp-static.override.yaml}" SSH_OPTS=( @@ -30,7 +31,7 @@ Usage: signoz-110-static-ingest.sh [--check|--apply|--status|--rollback] --check Validate the base stack and source-controlled override. --apply Recreate only signoz-otel-collector with the static base config. --status Verify the collector and OTLP listeners. - --rollback Recreate only the collector from the original OpAMP command. + --rollback Restore managed OpAMP only with a validated promotion receipt. All compose calls use --env-file /dev/null. Stateful services stay untouched. USAGE @@ -75,6 +76,27 @@ printf 'SIGNOZ_STATIC_INGEST_STATUS running=%s restart_count=%s grpc_4317=%s htt [ \"\$running\" = true ] && [ \"\$grpc\" = 1 ] && [ \"\$http\" = 1 ] && [ \"\${http_code:-000}\" = 200 ]" } +validate_managed_promotion_receipt() { + [ -n "$MANAGED_PROMOTION_RECEIPT" ] + [ -s "$MANAGED_PROMOTION_RECEIPT" ] + python3 - "$MANAGED_PROMOTION_RECEIPT" "$WORK_ITEM_ID" <<'PY' +import json +import sys + +path, work_item_id = sys.argv[1:] +with open(path, encoding="utf-8") as handle: + payload = json.load(handle) + +assert payload.get("schema") == "awoooi_signoz_managed_promotion_v1" +assert payload.get("work_item_id") == work_item_id +assert payload.get("terminal") == "pass" +assert payload.get("organization", {}).get("setup_completed") is True +assert payload.get("opamp", {}).get("nop_pipeline_count") == 0 +signals = payload.get("signals", {}) +assert all(signals.get(name) is True for name in ("logs", "metrics", "traces")) +PY +} + case "$mode" in --check) test -s "$LOCAL_OVERRIDE" @@ -116,6 +138,10 @@ printf 'SIGNOZ_STATIC_INGEST_APPLY before_id=%s after_id=%s stateful_services_to receipt verify pass "runtime_listener_check" ;; --rollback) + if ! validate_managed_promotion_receipt; then + receipt rollback blocked "managed_promotion_receipt_required" + exit 3 + fi receipt decision allow "bounded_rollback_collector_only" remote "set -eu cd '$REMOTE_DIR' @@ -123,6 +149,6 @@ docker compose --env-file /dev/null -p '$REMOTE_PROJECT' -f '$REMOTE_BASE' up -d cmd=\$(docker inspect -f '{{json .Config.Cmd}}' signoz-otel-collector) printf 'SIGNOZ_STATIC_INGEST_ROLLBACK command=%s\\n' \"\$cmd\" printf '%s' \"\$cmd\" | grep -q -- '--manager-config'" - receipt rollback pass "opamp_command_restored" + receipt rollback pass "validated_managed_opamp_command_restored" ;; esac diff --git a/scripts/reboot-recovery/signoz-188-otlp-bridge.sh b/scripts/reboot-recovery/signoz-188-otlp-bridge.sh index e31512e85..76a7e77fb 100755 --- a/scripts/reboot-recovery/signoz-188-otlp-bridge.sh +++ b/scripts/reboot-recovery/signoz-188-otlp-bridge.sh @@ -3,10 +3,12 @@ # Owner: platform-observability. Expiry: 2026-08-15 Asia/Taipei. # Replacement: repoint all producers and policies directly to host 110 OTLP. -set -euo pipefail +set -Eeuo pipefail TARGET_HOST="${TARGET_HOST:-ollama@192.168.0.188}" REMOTE_SCRIPT="${REMOTE_SCRIPT:-/home/ollama/bin/awoooi-signoz-otlp-bridge.sh}" +REMOTE_UNIT="${REMOTE_UNIT:-/home/ollama/.config/systemd/user/awoooi-signoz-otlp-bridge.service}" +REMOTE_SERVICE="${REMOTE_SERVICE:-awoooi-signoz-otlp-bridge.service}" LISTEN_HOST="${LISTEN_HOST:-192.168.0.188}" LISTEN_PORT="${LISTEN_PORT:-24318}" SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}" @@ -15,8 +17,12 @@ UPSTREAM_PORT="${UPSTREAM_PORT:-4318}" TRACE_ID="${TRACE_ID:-P0-OBS-001}" RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}" WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}" +RETIRE_RECEIPT="${RETIRE_RECEIPT:-}" VERIFY_NODES="${VERIFY_NODES:-wooo@192.168.0.120 wooo@192.168.0.121}" SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LOCAL_UNIT="${LOCAL_UNIT:-$ROOT_DIR/ops/systemd/user/awoooi-signoz-otlp-bridge.service}" +SUPERVISOR_TAKEOVER_ACTIVE=0 SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" @@ -27,17 +33,25 @@ SSH_OPTS=( usage() { cat <<'USAGE' -Usage: signoz-188-otlp-bridge.sh [--check|--apply|--status|--rollback] +Usage: signoz-188-otlp-bridge.sh [mode] Creates a user-level 192.168.0.188:24318 -> 192.168.0.110:4318 bridge. Only source addresses 192.168.0.120 and 192.168.0.121 are accepted. No sudo, firewall, Nginx, Docker, database, secret, or volume action is used. + + --check Read-only prerequisite check. + --apply Start the legacy bounded bridge (rollback target). + --status Verify either supervised or legacy bridge runtime. + --supervise Migrate the running bridge to a user-systemd unit. + --supervisor-status Require enabled+active user-systemd supervision. + --supervisor-rollback Restore the legacy process and reboot cron entry. + --rollback Retire the bridge only with a validated receipt. USAGE } mode="${1:---check}" case "$mode" in - --check|--apply|--status|--rollback) ;; + --check|--apply|--status|--supervise|--supervisor-status|--supervisor-rollback|--rollback) ;; -h|--help) usage; exit 0 ;; *) usage >&2; exit 64 ;; esac @@ -52,6 +66,26 @@ receipt() { "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$phase" "$result" "$detail" } +on_error() { + local rc=$? + trap - ERR + if [ "$SUPERVISOR_TAKEOVER_ACTIVE" -eq 1 ]; then + set +e + remote "systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true; sleep 1; $env_prefix '$REMOTE_SCRIPT' start" + install_legacy_cron + rollback_rc=$? + set -e + if [ "$rollback_rc" -eq 0 ]; then + receipt rollback pass "supervisor_takeover_failed_legacy_bridge_restored" + else + receipt rollback failed "supervisor_takeover_failed_legacy_restore_failed" + fi + fi + receipt terminal failed "command_exit_$rc" + exit "$rc" +} +trap on_error ERR + remote_payload() { cat <<'REMOTE' #!/usr/bin/env bash @@ -102,6 +136,18 @@ start() { status } +run_foreground() { + command -v socat >/dev/null + mkdir -p "$RUN_DIR" "$LOG_DIR" + if ss -ltn 2>/dev/null | awk '{print $4}' | grep -Eq "(^|:)${LISTEN_PORT}$"; then + echo "SIGNOZ_OTLP_BRIDGE_BLOCKED port_busy=${LISTEN_PORT}" >&2 + exit 2 + fi + echo "$$" >"$PID_FILE" + exec socat "TCP-LISTEN:${LISTEN_PORT},bind=${LISTEN_HOST},reuseaddr,fork,range=${SOURCE_RANGE}" \ + "TCP:${UPSTREAM_HOST}:${UPSTREAM_PORT}" +} + stop() { if is_running; then kill "$(sed -n '1p' "$PID_FILE")"; fi rm -f "$PID_FILE" @@ -110,6 +156,7 @@ stop() { case "${1:-status}" in start) start ;; + run) run_foreground ;; stop) stop ;; status) status ;; *) exit 64 ;; @@ -128,23 +175,54 @@ verify_nodes() { done } +validate_retire_receipt() { + [ -n "$RETIRE_RECEIPT" ] + [ -s "$RETIRE_RECEIPT" ] + python3 - "$RETIRE_RECEIPT" "$WORK_ITEM_ID" <<'PY' +import json +import sys + +path, work_item_id = sys.argv[1:] +with open(path, encoding="utf-8") as handle: + payload = json.load(handle) + +assert payload.get("schema") == "awoooi_signoz_bridge_retirement_v1" +assert payload.get("work_item_id") == work_item_id +assert payload.get("terminal") == "pass" +assert payload.get("active_direct_producers") == 5 +assert payload.get("queue_zero") is True +assert payload.get("direct_post_verifier") is True +PY +} + +install_legacy_cron() { + remote "(crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d'; printf '%s\n' '# AWOOOI SignOz OTLP bridge start' '@reboot $env_prefix $REMOTE_SCRIPT start >/tmp/awoooi-signoz-otlp-bridge.cron.log 2>&1' '# AWOOOI SignOz OTLP bridge end') | crontab -" +} + +remove_legacy_cron() { + remote "crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d' | crontab -" +} + case "$mode" in --check) + test -s "$LOCAL_UNIT" receipt sensor pass "canonical_110_upstream_selected" remote "set -eu command -v socat >/dev/null +command -v systemctl >/dev/null +linger=\$(loginctl show-user \"\$USER\" -p Linger --value 2>/dev/null || true) upstream=\$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[]}' http://${UPSTREAM_HOST}:${UPSTREAM_PORT}/v1/logs || true) busy=0 if ss -ltn 2>/dev/null | awk '{print \$4}' | grep -Eq '(^|:)${LISTEN_PORT}$'; then busy=1; fi printf 'SIGNOZ_OTLP_BRIDGE_CHECK upstream_code=%s listen_port_busy=%s source_range=%s\\n' \"\${upstream:-000}\" \"\$busy\" '$SOURCE_RANGE' -[ \"\${upstream:-000}\" = 200 ]" - receipt check pass "upstream_otlp_200" +[ \"\${upstream:-000}\" = 200 ] && [ \"\${linger:-}\" = yes ]" + receipt check pass "upstream_otlp_200_linger_yes_unit_source_present" ;; --apply) receipt decision allow "risk_medium_user_level_compatibility_bridge" remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload) remote "chmod 0755 '$REMOTE_SCRIPT.tmp' && mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT'" - remote "(crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d'; printf '%s\n' '# AWOOOI SignOz OTLP bridge start' '@reboot $env_prefix $REMOTE_SCRIPT start >/tmp/awoooi-signoz-otlp-bridge.cron.log 2>&1' '# AWOOOI SignOz OTLP bridge end') | crontab -" + install_legacy_cron remote "$env_prefix '$REMOTE_SCRIPT' start" verify_nodes receipt execute pass "bridge_started_allowlist_120_121" @@ -155,8 +233,75 @@ printf 'SIGNOZ_OTLP_BRIDGE_CHECK upstream_code=%s listen_port_busy=%s source_ran verify_nodes receipt verify pass "bridge_runtime_check" ;; + --supervise) + test -s "$LOCAL_UNIT" + receipt decision allow "risk_medium_user_systemd_supervisor_takeover" + remote "mkdir -p /home/ollama/bin /home/ollama/run /home/ollama/log /home/ollama/.config/systemd/user; umask 077; tee '$REMOTE_SCRIPT.tmp' >/dev/null" < <(remote_payload) + scp "${SSH_OPTS[@]}" "$LOCAL_UNIT" "$TARGET_HOST:$REMOTE_UNIT" + remote "set -eu +chmod 0755 '$REMOTE_SCRIPT.tmp' +chmod 0644 '$REMOTE_UNIT' +mv '$REMOTE_SCRIPT.tmp' '$REMOTE_SCRIPT' +systemd-analyze --user verify '$REMOTE_UNIT' >/dev/null +printf 'SIGNOZ_OTLP_BRIDGE_SUPERVISOR_DRY_RUN unit_verify=pass\n'" + receipt check pass "user_unit_verified_before_takeover" + SUPERVISOR_TAKEOVER_ACTIVE=1 + remote "set -eu +$env_prefix '$REMOTE_SCRIPT' stop || true +sleep 1 +systemctl --user daemon-reload +if ! systemctl --user enable --now '$REMOTE_SERVICE'; then + systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true + $env_prefix '$REMOTE_SCRIPT' start + exit 1 +fi +ready=0 +for _ in 1 2 3 4 5 6 7 8 9 10; do + if systemctl --user is-active --quiet '$REMOTE_SERVICE' && $env_prefix '$REMOTE_SCRIPT' status; then + ready=1 + break + fi + sleep 1 +done +[ \"\$ready\" = 1 ] +systemctl --user is-enabled --quiet '$REMOTE_SERVICE' +printf 'SIGNOZ_OTLP_BRIDGE_SUPERVISOR_READY wait_seconds_lte=10\n'" + remove_legacy_cron + verify_nodes + SUPERVISOR_TAKEOVER_ACTIVE=0 + receipt execute pass "bridge_migrated_to_user_systemd" + receipt verify pass "unit_enabled_active_nodes_120_121_otlp_200" + ;; + --supervisor-status) + remote "set -eu +systemctl --user is-enabled --quiet '$REMOTE_SERVICE' +systemctl --user is-active --quiet '$REMOTE_SERVICE' +printf 'SIGNOZ_OTLP_BRIDGE_SUPERVISOR_STATUS enabled=1 active=1\n' +$env_prefix '$REMOTE_SCRIPT' status" + verify_nodes + receipt verify pass "user_systemd_supervisor_runtime_check" + ;; + --supervisor-rollback) + receipt decision allow "bounded_supervisor_rollback_to_legacy_bridge" + remote "set -eu +systemctl --user disable --now '$REMOTE_SERVICE' +sleep 1 +if ! $env_prefix '$REMOTE_SCRIPT' start; then + systemctl --user enable --now '$REMOTE_SERVICE' + exit 1 +fi" + install_legacy_cron + remote "rm -f '$REMOTE_UNIT'; systemctl --user daemon-reload" + verify_nodes + receipt rollback pass "legacy_bridge_and_reboot_cron_restored" + ;; --rollback) - remote "if [ -x '$REMOTE_SCRIPT' ]; then $env_prefix '$REMOTE_SCRIPT' stop; fi; crontab -l 2>/dev/null | sed '/# AWOOOI SignOz OTLP bridge start/,/# AWOOOI SignOz OTLP bridge end/d' | crontab -" - receipt rollback pass "bridge_stopped_cron_removed" + if ! validate_retire_receipt; then + receipt rollback blocked "bridge_retirement_receipt_required" + exit 3 + fi + remote "systemctl --user disable --now '$REMOTE_SERVICE' >/dev/null 2>&1 || true; if [ -x '$REMOTE_SCRIPT' ]; then $env_prefix '$REMOTE_SCRIPT' stop; fi" + remove_legacy_cron + receipt rollback pass "validated_bridge_retirement_service_and_cron_stopped" ;; esac diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py new file mode 100644 index 000000000..36b56c89c --- /dev/null +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""No-write source preflight for P0-OBS-002 Wave A. + +The checker intentionally does not access credentials, databases, runtime +volumes, Docker, Kubernetes, or remote hosts. Runtime evidence belongs to a +separate verifier receipt. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import yaml + + +SCHEMA = "awoooi_signoz_canonical_route_preflight_v1" +EXPECTED_BLOCKERS = [ + "organization_not_initialized", + "direct_ingress_policy_not_closed", + "traces_not_fresh", + "bridge_not_supervised", + "filelog_normalization_degraded", + "wave_b_not_authorized", +] + + +def _load_yaml(path: Path) -> Any: + with path.open(encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def _read(root: Path, relative: str) -> str: + return (root / relative).read_text(encoding="utf-8") + + +def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: + contract = _load_yaml(contract_path) + checks: dict[str, bool] = {} + + checks["contract_schema"] = ( + contract.get("schema") + == "awoooi_signoz_organization_canonical_route_migration_v1" + ) + checks["work_item"] = contract.get("work_item_id") == "P0-OBS-002" + checks["wave_a_phase"] = contract.get("phase") == "wave_a_dual_allow" + checks["active_route_incumbent"] = ( + contract.get("routes", {}).get("active_route") == "incumbent" + ) + + producer_checks: list[bool] = [] + for producer in contract.get("producers", []): + sources = producer.get("sources") or [producer.get("source")] + endpoint = str(producer.get("current_endpoint", "")) + producer_checks.extend( + bool(source) and endpoint in _read(root, str(source)) for source in sources + ) + checks["active_producers_unchanged"] = bool(producer_checks) and all( + producer_checks + ) + + config = _read(root, "apps/api/src/core/config.py") + checks["dual_route_application_allow"] = ( + 'default=["192.168.0.188", "192.168.0.110"]' in config + and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]' + in config + ) + + network_documents = list( + yaml.safe_load_all(_read(root, "k8s/awoooi-prod/02-network-policy.yaml")) + ) + egress_policy = next( + document + for document in network_documents + if document + and document.get("kind") == "NetworkPolicy" + and document.get("metadata", {}).get("name") == "allow-required-egress" + ) + canonical_ports: set[int] = set() + for rule in egress_policy.get("spec", {}).get("egress", []): + cidrs = { + item.get("ipBlock", {}).get("cidr") for item in rule.get("to", []) + } + if "192.168.0.110/32" in cidrs: + canonical_ports.update( + int(item["port"]) + for item in rule.get("ports", []) + if isinstance(item.get("port"), int) + ) + checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset( + canonical_ports + ) + + unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-bridge.service") + checks["bridge_unit_hardened"] = all( + value in unit + for value in ( + "Restart=on-failure", + "NoNewPrivileges=true", + "PrivateTmp=true", + "ProtectSystem=strict", + "SOURCE_RANGE=192.168.0.120/31", + ) + ) + + verifier = _read(root, "scripts/reboot-recovery/verify-signoz-otel-ingestion.sh") + checks["route_specific_verifier"] = ( + "OTLP_ENDPOINT" in verifier and "ROUTE_ID" in verifier + ) + static_recovery = _read( + root, "scripts/reboot-recovery/signoz-110-static-ingest.sh" + ) + checks["managed_promotion_fail_closed"] = all( + value in static_recovery + for value in ( + "MANAGED_PROMOTION_RECEIPT", + "managed_promotion_receipt_required", + "nop_pipeline_count", + ) + ) + + terminal = contract.get("terminal", {}) + blockers = list(terminal.get("blockers", [])) + checks["expected_blockers_explicit"] = blockers == EXPECTED_BLOCKERS + source_ready = all(checks.values()) + return { + "schema": SCHEMA, + "work_item_id": "P0-OBS-002", + "mode": "check", + "phase": "wave_a_dual_allow", + "source_ready": source_ready, + "runtime_status": terminal.get("status", "unknown"), + "active_route": "incumbent", + "writes_performed": 0, + "checks": checks, + "blockers": blockers, + "terminal": "pass" if source_ready else "failed", + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument( + "--contract", + type=Path, + default=None, + ) + args = parser.parse_args() + root = args.root.resolve() + contract = args.contract or ( + root / "ops/signoz/organization-canonical-route-migration.yaml" + ) + payload = evaluate(root, contract.resolve()) + print(json.dumps(payload, ensure_ascii=False, sort_keys=True)) + return 0 if payload["terminal"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py new file mode 100644 index 000000000..db6273459 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys + +import yaml + + +ROOT = Path(__file__).resolve().parents[3] +CONTRACT = ROOT / "ops/signoz/organization-canonical-route-migration.yaml" +PREFLIGHT = ROOT / "scripts/reboot-recovery/signoz-canonical-route-preflight.py" +BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-bridge.sh" +STATIC = ROOT / "scripts/reboot-recovery/signoz-110-static-ingest.sh" +UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-bridge.service" + + +def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None: + payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8")) + assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1" + assert payload["work_item_id"] == "P0-OBS-002" + assert payload["phase"] == "wave_a_dual_allow" + assert payload["routes"]["active_route"] == "incumbent" + assert payload["organization"]["apply_allowed_in_wave_a"] is False + assert payload["policy"]["secret_values_forbidden_in_source"] is True + assert len(payload["producers"]) == 5 + + for producer in payload["producers"]: + sources = producer.get("sources") or [producer["source"]] + for source in sources: + assert producer["current_endpoint"] in (ROOT / source).read_text( + encoding="utf-8" + ) + + +def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None: + text = CONTRACT.read_text(encoding="utf-8").lower() + assert "signoz_user_root_password:" not in text + assert "signoz-api-key:" not in text + assert "authorization:" not in text + assert "raw_relational_database_update" in text + assert "stateful_volume_restore" in text + assert "apply_allowed_in_wave_a: false" in text + + +def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: + result = subprocess.run( + [sys.executable, str(PREFLIGHT), "--root", str(ROOT)], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout) + assert payload["source_ready"] is True + assert payload["writes_performed"] == 0 + assert payload["terminal"] == "pass" + assert payload["runtime_status"] == "partial_degraded" + assert payload["active_route"] == "incumbent" + assert payload["checks"] and all(payload["checks"].values()) + assert "organization_not_initialized" in payload["blockers"] + assert "wave_b_not_authorized" in payload["blockers"] + + +def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None: + text = UNIT.read_text(encoding="utf-8") + for expected in ( + "Wants=network-online.target", + "After=network-online.target", + "SOURCE_RANGE=192.168.0.120/31", + "ExecStart=/home/ollama/bin/awoooi-signoz-otlp-bridge.sh run", + "Restart=on-failure", + "NoNewPrivileges=true", + "PrivateTmp=true", + "ProtectSystem=strict", + "ProtectHome=read-only", + "WantedBy=default.target", + ): + assert expected in text + assert "User=" not in text + assert "sudo" not in text + + +def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None: + env = os.environ.copy() + env.pop("MANAGED_PROMOTION_RECEIPT", None) + env.pop("RETIRE_RECEIPT", None) + + managed = subprocess.run( + ["bash", str(STATIC), "--rollback"], + env=env, + capture_output=True, + text=True, + ) + assert managed.returncode == 3 + assert "managed_promotion_receipt_required" in managed.stdout + assert "ssh" not in managed.stderr.lower() + + retired = subprocess.run( + ["bash", str(BRIDGE), "--rollback"], + env=env, + capture_output=True, + text=True, + ) + assert retired.returncode == 3 + assert "bridge_retirement_receipt_required" in retired.stdout + assert "ssh" not in retired.stderr.lower() diff --git a/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py b/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py index 1956243a0..908d4dfab 100644 --- a/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py +++ b/scripts/reboot-recovery/tests/test_signoz_static_ingest_recovery.py @@ -36,6 +36,10 @@ def test_static_apply_is_bounded_no_pull_and_never_reads_dotenv() -> None: assert "docker volume" not in executable assert "docker system prune" not in executable assert "github.com" not in text + assert "MANAGED_PROMOTION_RECEIPT" in text + assert "awoooi_signoz_managed_promotion_v1" in text + assert "managed_promotion_receipt_required" in text + assert "nop_pipeline_count" in text def test_otlp_bridge_is_allowlisted_reversible_and_has_sunset() -> None: @@ -53,9 +57,25 @@ def test_otlp_bridge_is_allowlisted_reversible_and_has_sunset() -> None: assert "@reboot" in text assert "verify_nodes" in text assert "--rollback" in text + assert "--supervise" in text + assert "--supervisor-status" in text + assert "--supervisor-rollback" in text + assert "systemctl --user" in text + assert "run_foreground" in text + assert "RETIRE_RECEIPT" in text + assert "awoooi_signoz_bridge_retirement_v1" in text + assert "bridge_retirement_receipt_required" in text + assert "SUPERVISOR_TAKEOVER_ACTIVE=1" in text + assert "for _ in 1 2 3 4 5 6 7 8 9 10" in text + assert "SIGNOZ_OTLP_BRIDGE_SUPERVISOR_READY wait_seconds_lte=10" in text + assert text.index("SUPERVISOR_TAKEOVER_ACTIVE=1") < text.index( + "systemctl --user enable --now '$REMOTE_SERVICE'" + ) assert "2026-08-15" in text assert "sudo " not in executable - assert "systemctl " not in executable + for line in executable.splitlines(): + if "systemctl " in line and "command -v systemctl" not in line: + assert "systemctl --user" in line assert "iptables " not in executable assert "docker " not in executable @@ -67,7 +87,10 @@ def test_ingestion_verifier_binds_queue_bridge_and_clickhouse() -> None: assert "otelcol_exporter_send_failed_log_records" in text assert "otelcol_exporter_sent_log_records" in text assert "otelcol_exporter_queue_size" in text - assert "192.168.0.188:24318/v1/logs" in text + assert "OTLP_ENDPOINT" in text + assert "ROUTE_ID" in text + assert "192.168.0.188:24318/v1/logs" in text # compatibility default + assert "awoooi.route.id" in text assert "signoz_logs.logs_v2" in text assert "body={marker:String}" in text assert "declare -A" not in text diff --git a/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh b/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh index ee8d3a402..84e3d8e17 100755 --- a/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh +++ b/scripts/reboot-recovery/verify-signoz-otel-ingestion.sh @@ -1,20 +1,23 @@ #!/usr/bin/env bash -# Independent verifier for K3s -> 188 compatibility bridge -> 110 SigNoz -> ClickHouse. +# Independent verifier for either the incumbent bridge or canonical SigNoz OTLP route. # It writes one non-sensitive synthetic log and queries only that exact marker. set -euo pipefail -TRACE_ID="${TRACE_ID:-P0-OBS-001}" -RUN_ID="${RUN_ID:-P0-OBS-001-$(date -u +%Y%m%dT%H%M%SZ)}" -WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-001}" +TRACE_ID="${TRACE_ID:-P0-OBS-002}" +RUN_ID="${RUN_ID:-P0-OBS-002-$(date -u +%Y%m%dT%H%M%SZ)}" +WORK_ITEM_ID="${WORK_ITEM_ID:-P0-OBS-002}" WINDOW_SECONDS="${WINDOW_SECONDS:-30}" EXPECTED_PODS="${EXPECTED_PODS:-2}" EXPECTED_QUEUE_CAPACITY="${EXPECTED_QUEUE_CAPACITY:-10000}" SOURCE_NODE="${SOURCE_NODE:-wooo@192.168.0.120}" SIGNOZ_HOST="${SIGNOZ_HOST:-wooo@192.168.0.110}" -BRIDGE_ENDPOINT="${BRIDGE_ENDPOINT:-http://192.168.0.188:24318/v1/logs}" +# BRIDGE_ENDPOINT remains a compatibility alias for old callers. New migration +# runs must set OTLP_ENDPOINT explicitly so route identity is present in the run. +OTLP_ENDPOINT="${OTLP_ENDPOINT:-${BRIDGE_ENDPOINT:-http://192.168.0.188:24318/v1/logs}}" +ROUTE_ID="${ROUTE_ID:-incumbent_bridge}" SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ConnectionAttempts=1) -marker="P0OBS001_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)" +marker="P0OBS_${ROUTE_ID//[^A-Za-z0-9_]/_}_${RUN_ID//[^A-Za-z0-9_]/_}_$(date -u +%Y%m%dT%H%M%SZ)" pod_list=() enqueue_before=() failed_before=() @@ -59,9 +62,9 @@ for ((i = 0; i < pod_count; i++)); do done receipt sensor pass "collector_pods_${pod_count}_queue_capacity_${EXPECTED_QUEUE_CAPACITY}" -http_code="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" "ts=\$(date +%s%N); curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"awoooi-p0-obs-verifier\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"awoooi.obs.verifier\"},\"logRecords\":[{\"timeUnixNano\":\"'\"\$ts\"'\",\"severityNumber\":9,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"${marker}\"}}]}]}]}]}' '$BRIDGE_ENDPOINT'")" +http_code="$(ssh "${SSH_OPTS[@]}" "$SOURCE_NODE" "ts=\$(date +%s%N); curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' --data-binary '{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"awoooi-p0-obs-verifier\"}},{\"key\":\"awoooi.route.id\",\"value\":{\"stringValue\":\"${ROUTE_ID}\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"awoooi.obs.verifier\"},\"logRecords\":[{\"timeUnixNano\":\"'\"\$ts\"'\",\"severityNumber\":9,\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"${marker}\"}}]}]}]}]}' '$OTLP_ENDPOINT'")" [ "$http_code" = 200 ] -receipt execute pass "synthetic_otlp_http_200_marker_${marker}" +receipt execute pass "route_${ROUTE_ID}_synthetic_otlp_http_200_marker_${marker}" count=0 for _ in $(seq 1 15); do