346 lines
14 KiB
Python
346 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
CONTRACT = ROOT / "ops/signoz/organization-canonical-route-migration.yaml"
|
|
PREFLIGHT = ROOT / "scripts/reboot-recovery/signoz-canonical-route-preflight.py"
|
|
BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-bridge.sh"
|
|
STATIC = ROOT / "scripts/reboot-recovery/signoz-110-static-ingest.sh"
|
|
UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-bridge.service"
|
|
SIGNAL_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
|
|
PARITY_SNAPSHOT = ROOT / "ops/signoz/p0-obs-002-three-signal-route-parity.yaml"
|
|
OTEL_DAEMONSET = ROOT / "k8s/observability/otel-collector-daemonset.yaml"
|
|
FILELOG_CANARY = ROOT / "ops/signoz/filelog-normalization-canary.yaml"
|
|
GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
|
|
GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
|
|
GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
|
|
GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml"
|
|
POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml"
|
|
|
|
|
|
def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
|
|
payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
|
|
assert payload["work_item_id"] == "P0-OBS-002"
|
|
assert payload["phase"] == "wave_a_dual_allow"
|
|
assert payload["routes"]["active_route"] == "incumbent"
|
|
assert payload["organization"]["apply_allowed_in_wave_a"] is False
|
|
assert payload["policy"]["secret_values_forbidden_in_source"] is True
|
|
assert len(payload["producers"]) == 5
|
|
|
|
for producer in payload["producers"]:
|
|
sources = producer.get("sources") or [producer["source"]]
|
|
for source in sources:
|
|
assert producer["current_endpoint"] in (ROOT / source).read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
|
|
def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None:
|
|
text = CONTRACT.read_text(encoding="utf-8").lower()
|
|
assert "signoz_user_root_password:" not in text
|
|
assert "signoz-api-key:" not in text
|
|
assert "authorization:" not in text
|
|
assert "raw_relational_database_update" in text
|
|
assert "stateful_volume_restore" in text
|
|
assert "apply_allowed_in_wave_a: false" in text
|
|
|
|
|
|
def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
|
result = subprocess.run(
|
|
[sys.executable, str(PREFLIGHT), "--root", str(ROOT)],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
payload = json.loads(result.stdout)
|
|
assert payload["source_ready"] is True
|
|
assert payload["writes_performed"] == 0
|
|
assert payload["terminal"] == "pass"
|
|
assert payload["runtime_status"] == "partial_degraded"
|
|
assert payload["active_route"] == "incumbent"
|
|
assert payload["checks"] and all(payload["checks"].values())
|
|
assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True
|
|
assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True
|
|
assert "organization_not_initialized" in payload["blockers"]
|
|
assert "signoz_control_plane_health_route_not_deployed" in payload["blockers"]
|
|
assert "wave_b_not_authorized" in payload["blockers"]
|
|
|
|
|
|
def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
|
|
text = UNIT.read_text(encoding="utf-8")
|
|
for expected in (
|
|
"Wants=network-online.target",
|
|
"After=network-online.target",
|
|
"SOURCE_RANGE=192.168.0.120/31",
|
|
"ExecStart=/home/ollama/bin/awoooi-signoz-otlp-bridge.sh run",
|
|
"Restart=on-failure",
|
|
"NoNewPrivileges=true",
|
|
"PrivateTmp=true",
|
|
"ProtectSystem=strict",
|
|
"ProtectHome=read-only",
|
|
"WantedBy=default.target",
|
|
):
|
|
assert expected in text
|
|
assert "User=" not in text
|
|
assert "sudo" not in text
|
|
|
|
|
|
def test_three_signal_verifier_is_route_scoped_and_exact() -> None:
|
|
text = SIGNAL_VERIFIER.read_text(encoding="utf-8")
|
|
for expected in (
|
|
"OTLP_BASE_ENDPOINT",
|
|
"ROUTE_ID",
|
|
"/v1/$signal",
|
|
"resourceLogs",
|
|
"resourceMetrics",
|
|
"resourceSpans",
|
|
"signoz_logs.logs_v2",
|
|
"signoz_metrics.time_series_v4",
|
|
"signoz_metrics.samples_v4",
|
|
"signoz_traces.signoz_index_v3",
|
|
"exact_log_1_metric_series_1_metric_sample_1_trace_1",
|
|
):
|
|
assert expected in text
|
|
assert "github.com" not in text
|
|
assert "SIGNOZ-API-KEY" not in text
|
|
assert "Authorization:" not in text
|
|
|
|
|
|
def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion() -> None:
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
parity = contract["runtime_observations"]["synthetic_three_signal_route_parity"]
|
|
assert parity["incumbent_bridge"] == "pass"
|
|
assert parity["direct_110_from_120"] == "pass"
|
|
assert parity["direct_110_from_121"] == "pass"
|
|
assert parity["exact_count_per_signal_and_route"] == 1
|
|
assert parity["real_application_trace_freshness_inferred"] is False
|
|
assert parity["producer_queue_failure_gate_inferred"] is False
|
|
assert parity["promotion_authorized"] is False
|
|
|
|
snapshot = yaml.safe_load(PARITY_SNAPSHOT.read_text(encoding="utf-8"))
|
|
assert snapshot["trace_id"] == parity["trace_id"]
|
|
assert snapshot["route_parity_terminal"] == "pass"
|
|
assert snapshot["program_terminal"] == "partial_degraded"
|
|
assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial"
|
|
assert (
|
|
snapshot["controlled_apply_contract"]["learning_writeback"][
|
|
"durable_gitea_feature_acknowledgement"
|
|
]
|
|
== "pass"
|
|
)
|
|
assert (
|
|
snapshot["controlled_apply_contract"]["learning_writeback"][
|
|
"durable_gitea_main_acknowledgement"
|
|
]
|
|
== "pending"
|
|
)
|
|
assert snapshot["promotion"]["authorized"] is False
|
|
assert snapshot["production_mutations"]["active_route_changed"] is False
|
|
assert snapshot["production_mutations"]["secret_read"] is False
|
|
|
|
|
|
def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() -> None:
|
|
text = OTEL_DAEMONSET.read_text(encoding="utf-8")
|
|
assert "id: cri_parser" in text
|
|
assert "on_error: send_quiet" not in text
|
|
assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text
|
|
assert "id: mark_cri_parsed" in text
|
|
assert 'value: parsed_cri' in text
|
|
assert "id: move_cri_body" in text
|
|
assert "if: attributes.log != nil" in text
|
|
assert "id: mark_cri_fallback" in text
|
|
assert 'value: fallback_unparsed' in text
|
|
assert 'if: attributes["awoooi.log.parse_status"] == nil' in text
|
|
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
observation = contract["runtime_observations"][
|
|
"filelog_normalization_error_readback"
|
|
]
|
|
assert observation["transport_queue_size"] == 0
|
|
assert observation["transport_send_failed_log_records"] == 0
|
|
assert observation["source_fix"]["preserves_unparsed_body"] is True
|
|
assert observation["source_fix"]["runtime_canary_passed"] is False
|
|
|
|
|
|
def test_filelog_canary_is_synthetic_bounded_and_has_no_network_exporter() -> None:
|
|
text = FILELOG_CANARY.read_text(encoding="utf-8")
|
|
documents = list(yaml.safe_load_all(text))
|
|
config_map, pod = documents
|
|
assert config_map["kind"] == "ConfigMap"
|
|
assert "P0OBS_FILELOG_PARSED_CRI" in config_map["data"]["input.log"]
|
|
assert "P0OBS_FILELOG_FALLBACK_UNPARSED" in config_map["data"]["input.log"]
|
|
config = config_map["data"]["config.yaml"]
|
|
assert "on_error: send_quiet" not in config
|
|
assert 'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"' in config
|
|
assert "fallback_unparsed" in config
|
|
assert "debug:" in config
|
|
assert "otlp" not in config.lower()
|
|
|
|
spec = pod["spec"]
|
|
assert spec["automountServiceAccountToken"] is False
|
|
assert spec["restartPolicy"] == "Never"
|
|
assert spec["activeDeadlineSeconds"] == 90
|
|
container = spec["containers"][0]
|
|
assert "@sha256:" in container["image"]
|
|
assert container["securityContext"]["allowPrivilegeEscalation"] is False
|
|
assert container["securityContext"]["readOnlyRootFilesystem"] is True
|
|
|
|
|
|
def test_grpc_bridge_is_independently_supervised_allowlisted_and_reversible() -> None:
|
|
bridge = GRPC_BRIDGE.read_text(encoding="utf-8")
|
|
unit = GRPC_UNIT.read_text(encoding="utf-8")
|
|
for expected in (
|
|
'LISTEN_PORT="${LISTEN_PORT:-24317}"',
|
|
'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"',
|
|
'UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"',
|
|
"--check|--apply|--status|--rollback",
|
|
"systemctl --user enable --now",
|
|
"rollback_runtime",
|
|
"verify_nodes",
|
|
"no_firewall_no_secret",
|
|
):
|
|
assert expected in bridge
|
|
for forbidden in ("sudo", "iptables", "nft ", "docker ", "kubectl "):
|
|
assert forbidden not in bridge
|
|
|
|
for expected in (
|
|
"LISTEN_PORT=24317",
|
|
"SOURCE_RANGE=192.168.0.120/31",
|
|
"UPSTREAM_PORT=4317",
|
|
"ExecStartPre=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh check-upstream",
|
|
"Restart=on-failure",
|
|
"NoNewPrivileges=true",
|
|
"ProtectSystem=strict",
|
|
"ProtectHome=read-only",
|
|
):
|
|
assert expected in unit
|
|
assert "User=" not in unit
|
|
|
|
|
|
def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier() -> None:
|
|
bridge = GRPC_BRIDGE.read_text(encoding="utf-8")
|
|
verifier = GRPC_VERIFIER.read_text(encoding="utf-8")
|
|
apply_block = bridge.split(" --apply)", 1)[1].split(" --status)", 1)[0]
|
|
|
|
learning = (
|
|
'receipt learning partial '
|
|
'"application_trace_metric_10m_post_verifier_pending"'
|
|
)
|
|
terminal = (
|
|
'receipt terminal partial '
|
|
'"grpc_transport_restored_independent_600s_post_verifier_pending"'
|
|
)
|
|
assert learning in apply_block
|
|
assert terminal in apply_block
|
|
assert apply_block.index(learning) < apply_block.index(terminal)
|
|
assert "receipt terminal pass" not in apply_block
|
|
assert (
|
|
'receipt terminal pass '
|
|
'"real_application_otlp_grpc_trace_metric_freshness_closed"'
|
|
in verifier
|
|
)
|
|
|
|
|
|
def test_contract_grpc_route_state_matches_verified_runtime_evidence() -> None:
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
incumbent = contract["routes"]["incumbent"]
|
|
observations = contract["runtime_observations"]
|
|
grpc = observations["application_otlp_grpc"]
|
|
|
|
assert (
|
|
incumbent["grpc_bridge_status"]
|
|
== "user_systemd_enabled_active_runtime_verified"
|
|
)
|
|
assert observations["grpc_bridge_supervisor"] == "user_systemd_enabled_active"
|
|
assert grpc["runtime_status"] == "restored_verified"
|
|
assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass"
|
|
assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"
|
|
assert grpc["source_fix"]["runtime_verifier_window_seconds"] >= 600
|
|
|
|
|
|
def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None:
|
|
text = GRPC_VERIFIER.read_text(encoding="utf-8")
|
|
for expected in (
|
|
'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
|
|
"https://awoooi.wooo.work/api/v1/openapi.json",
|
|
"serviceName={service:String}",
|
|
"signoz_metrics.samples_v4",
|
|
"resource_attrs['service.name']={service:String}",
|
|
"Failed to export traces",
|
|
"Failed to export metrics",
|
|
"restart_delta_0",
|
|
"real_application_otlp_grpc_trace_metric_freshness_closed",
|
|
):
|
|
assert expected in text
|
|
assert "SELECT body" not in text
|
|
assert "SELECT attributes" not in text
|
|
assert "/v1/traces" not in text
|
|
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
grpc = contract["runtime_observations"]["application_otlp_grpc"]
|
|
assert grpc["root_cause"] == "missing_incumbent_grpc_bridge"
|
|
assert grpc["runtime_status"] == "restored_verified"
|
|
assert grpc["confidence"] == 0.99
|
|
assert grpc["recent_awoooi_api_trace_count"] == 1098
|
|
assert grpc["recent_awoooi_api_metric_sample_count"] == 60180
|
|
assert grpc["http_only_verifier_blind_spot"] is True
|
|
assert grpc["source_fix"]["check_terminal"] == "pass"
|
|
assert (
|
|
grpc["source_fix"]["runtime_apply_status"]
|
|
== "applied_post_verifier_pass"
|
|
)
|
|
assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"
|
|
|
|
receipt = yaml.safe_load(GRPC_RUNTIME_RECEIPT.read_text(encoding="utf-8"))
|
|
assert receipt["run_id"] == grpc["source_fix"]["runtime_run_id"]
|
|
assert receipt["post_verifier"]["window_seconds"] == 600
|
|
assert receipt["post_verifier"]["exporter_trace_errors"] == 0
|
|
assert receipt["post_verifier"]["exporter_metric_errors"] == 0
|
|
assert receipt["post_verifier"]["awoooi_api_trace_count"] == 1098
|
|
assert receipt["post_verifier"]["awoooi_api_metric_sample_count"] == 60180
|
|
assert receipt["post_verifier"]["rollback"] == "not_required"
|
|
assert receipt["completion"]["grpc_runtime_asset"] == "closed"
|
|
assert receipt["completion"]["program"] == "partial_degraded"
|
|
|
|
regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
assert regression["baseline"]["verifier_terminal"] == "pass"
|
|
assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False
|
|
assert regression["controlled_recovery"]["stateful_services_touched"] == 0
|
|
assert regression["controlled_recovery"]["grpc_4317_listening"] is True
|
|
assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
|
|
assert regression["terminal"]["status"] == "partial_degraded"
|
|
|
|
|
|
def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
|
|
env = os.environ.copy()
|
|
env.pop("MANAGED_PROMOTION_RECEIPT", None)
|
|
env.pop("RETIRE_RECEIPT", None)
|
|
|
|
managed = subprocess.run(
|
|
["bash", str(STATIC), "--rollback"],
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert managed.returncode == 3
|
|
assert "managed_promotion_receipt_required" in managed.stdout
|
|
assert "ssh" not in managed.stderr.lower()
|
|
|
|
retired = subprocess.run(
|
|
["bash", str(BRIDGE), "--rollback"],
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert retired.returncode == 3
|
|
assert "bridge_retirement_receipt_required" in retired.stdout
|
|
assert "ssh" not in retired.stderr.lower()
|