1175 lines
52 KiB
Python
1175 lines
52 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
CONTRACT = ROOT / "ops/signoz/organization-canonical-route-migration.yaml"
|
|
PREFLIGHT = ROOT / "scripts/reboot-recovery/signoz-canonical-route-preflight.py"
|
|
BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-bridge.sh"
|
|
STATIC = ROOT / "scripts/reboot-recovery/signoz-110-static-ingest.sh"
|
|
UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-bridge.service"
|
|
SIGNAL_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
|
|
PARITY_SNAPSHOT = ROOT / "ops/signoz/p0-obs-002-three-signal-route-parity.yaml"
|
|
OTEL_DAEMONSET = ROOT / "k8s/observability/otel-collector-daemonset.yaml"
|
|
FILELOG_CANARY = ROOT / "ops/signoz/filelog-normalization-canary.yaml"
|
|
GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
|
|
GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
|
|
GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
|
|
GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml"
|
|
POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml"
|
|
|
|
|
|
def _load_preflight_module():
|
|
spec = importlib.util.spec_from_file_location(
|
|
"signoz_canonical_route_preflight_under_test", PREFLIGHT
|
|
)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
|
|
payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
assert (
|
|
payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
|
|
)
|
|
assert payload["work_item_id"] == "P0-OBS-002"
|
|
assert payload["phase"] == "wave_a_dual_allow"
|
|
assert payload["routes"]["active_route"] == "incumbent"
|
|
assert payload["organization"]["apply_allowed_in_wave_a"] is False
|
|
assert payload["policy"]["secret_values_forbidden_in_source"] is True
|
|
assert len(payload["producers"]) == 5
|
|
|
|
for producer in payload["producers"]:
|
|
sources = producer.get("sources") or [producer["source"]]
|
|
for source in sources:
|
|
assert producer["current_endpoint"] in (ROOT / source).read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
|
|
def test_contract_and_receipt_contain_no_secret_values_or_github_dependency() -> None:
|
|
contract_text = CONTRACT.read_text(encoding="utf-8").lower()
|
|
regression_text = POST_CLOSURE_REGRESSION.read_text(encoding="utf-8").lower()
|
|
forbidden = (
|
|
"signoz_user_root_password:",
|
|
"signoz-api-key:",
|
|
"authorization:",
|
|
"password:",
|
|
"private_key:",
|
|
"cookie:",
|
|
"session:",
|
|
"github.com",
|
|
"api.github.com",
|
|
"raw.githubusercontent.com",
|
|
"codeload.github.com",
|
|
"ghcr.io",
|
|
)
|
|
for text in (contract_text, regression_text):
|
|
assert all(value not in text for value in forbidden)
|
|
|
|
assert "raw_relational_database_update" in contract_text
|
|
assert "stateful_volume_restore" in contract_text
|
|
assert "apply_allowed_in_wave_a: false" in contract_text
|
|
|
|
regression = yaml.safe_load(regression_text)
|
|
assert regression["scope_boundaries"]["secret_read"] is False
|
|
assert regression["scope_boundaries"]["github_used"] is False
|
|
|
|
|
|
def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
|
result = subprocess.run(
|
|
[sys.executable, str(PREFLIGHT), "--root", str(ROOT)],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
payload = json.loads(result.stdout)
|
|
assert payload["source_ready"] is True
|
|
assert payload["writes_performed"] == 0
|
|
assert payload["terminal"] == "pass"
|
|
assert payload["runtime_status"] == "partial_degraded"
|
|
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
|
|
assert payload["backup_toolchain_current_source_deployed"] is True
|
|
assert payload["backup_toolchain_runtime_verifier_terminal"] == "pass"
|
|
assert payload["backup_toolchain_source_drift_fields"] == [
|
|
"backup_orchestrator_sha256",
|
|
"canary_wrapper_sha256",
|
|
"inventory_verifier_sha256",
|
|
]
|
|
assert payload["active_route"] == "incumbent"
|
|
assert payload["checks"] and all(payload["checks"].values())
|
|
assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True
|
|
assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True
|
|
assert payload["checks"]["post_closure_schema_v2"] is True
|
|
assert payload["checks"]["native_backup_current_source_hashes_bound"] is True
|
|
assert (
|
|
payload["checks"]["native_backup_current_source_deployment_state_explicit"]
|
|
is True
|
|
)
|
|
assert payload["checks"]["native_backup_current_source_runtime_closed"] is True
|
|
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
|
|
assert payload["checks"]["post_release_600s_verifier_consistent"] is True
|
|
assert (
|
|
payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"]
|
|
is True
|
|
)
|
|
assert payload["checks"]["service_registry_runtime_mirror_drift_explicit"]
|
|
assert payload["checks"]["github_freeze_legacy_asset_drift_explicit"]
|
|
assert payload["checks"]["monitoring_generator_duplicate_identity_drift_explicit"]
|
|
assert "organization_not_initialized" in payload["blockers"]
|
|
assert "wave_b_not_authorized" in payload["blockers"]
|
|
assert payload["blockers"] == [
|
|
"organization_not_initialized",
|
|
"direct_ingress_policy_not_closed",
|
|
"http_bridge_not_supervised",
|
|
"filelog_normalization_degraded",
|
|
"wave_b_not_authorized",
|
|
]
|
|
for closed_blocker in (
|
|
"api_runtime_cooldown_pending",
|
|
"cd_probe_safety_fix_not_deployed",
|
|
"backup_collector_restore_guard_not_deployed",
|
|
"signoz_control_plane_health_route_not_deployed",
|
|
):
|
|
assert closed_blocker not in payload["blockers"]
|
|
|
|
|
|
def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
|
|
text = UNIT.read_text(encoding="utf-8")
|
|
for expected in (
|
|
"Wants=network-online.target",
|
|
"After=network-online.target",
|
|
"SOURCE_RANGE=192.168.0.120/31",
|
|
"ExecStart=/home/ollama/bin/awoooi-signoz-otlp-bridge.sh run",
|
|
"Restart=on-failure",
|
|
"NoNewPrivileges=true",
|
|
"PrivateTmp=true",
|
|
"ProtectSystem=strict",
|
|
"ProtectHome=read-only",
|
|
"WantedBy=default.target",
|
|
):
|
|
assert expected in text
|
|
assert "User=" not in text
|
|
assert "sudo" not in text
|
|
|
|
|
|
def test_three_signal_verifier_is_route_scoped_and_exact() -> None:
|
|
text = SIGNAL_VERIFIER.read_text(encoding="utf-8")
|
|
for expected in (
|
|
"OTLP_BASE_ENDPOINT",
|
|
"ROUTE_ID",
|
|
"/v1/$signal",
|
|
"resourceLogs",
|
|
"resourceMetrics",
|
|
"resourceSpans",
|
|
"signoz_logs.logs_v2",
|
|
"signoz_metrics.time_series_v4",
|
|
"signoz_metrics.samples_v4",
|
|
"signoz_traces.signoz_index_v3",
|
|
"exact_log_1_metric_series_1_metric_sample_1_trace_1",
|
|
):
|
|
assert expected in text
|
|
assert "github.com" not in text
|
|
assert "SIGNOZ-API-KEY" not in text
|
|
assert "Authorization:" not in text
|
|
|
|
|
|
def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion() -> None:
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
parity = contract["runtime_observations"]["synthetic_three_signal_route_parity"]
|
|
assert parity["incumbent_bridge"] == "pass"
|
|
assert parity["direct_110_from_120"] == "pass"
|
|
assert parity["direct_110_from_121"] == "pass"
|
|
assert parity["exact_count_per_signal_and_route"] == 1
|
|
assert parity["real_application_trace_freshness_inferred"] is False
|
|
assert parity["producer_queue_failure_gate_inferred"] is False
|
|
assert parity["promotion_authorized"] is False
|
|
|
|
snapshot = yaml.safe_load(PARITY_SNAPSHOT.read_text(encoding="utf-8"))
|
|
assert snapshot["trace_id"] == parity["trace_id"]
|
|
assert snapshot["route_parity_terminal"] == "pass"
|
|
assert snapshot["program_terminal"] == "partial_degraded"
|
|
assert (
|
|
snapshot["controlled_apply_contract"]["learning_writeback"]["status"]
|
|
== "partial"
|
|
)
|
|
assert (
|
|
snapshot["controlled_apply_contract"]["learning_writeback"][
|
|
"durable_gitea_feature_acknowledgement"
|
|
]
|
|
== "pass"
|
|
)
|
|
assert (
|
|
snapshot["controlled_apply_contract"]["learning_writeback"][
|
|
"durable_gitea_main_acknowledgement"
|
|
]
|
|
== "pending"
|
|
)
|
|
assert snapshot["promotion"]["authorized"] is False
|
|
assert snapshot["production_mutations"]["active_route_changed"] is False
|
|
assert snapshot["production_mutations"]["secret_read"] is False
|
|
|
|
|
|
def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() -> None:
|
|
text = OTEL_DAEMONSET.read_text(encoding="utf-8")
|
|
assert "id: cri_parser" in text
|
|
assert "on_error: send_quiet" not in text
|
|
assert "if: 'body matches \"^[^ ]+ (stdout|stderr) [^ ]* .*$\"'" in text
|
|
assert "id: mark_cri_parsed" in text
|
|
assert "value: parsed_cri" in text
|
|
assert "id: move_cri_body" in text
|
|
assert "if: attributes.log != nil" in text
|
|
assert "id: mark_cri_fallback" in text
|
|
assert "value: fallback_unparsed" in text
|
|
assert 'if: attributes["awoooi.log.parse_status"] == nil' in text
|
|
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
observation = contract["runtime_observations"][
|
|
"filelog_normalization_error_readback"
|
|
]
|
|
assert observation["transport_queue_size"] == 0
|
|
assert observation["transport_send_failed_log_records"] == 0
|
|
assert observation["source_fix"]["preserves_unparsed_body"] is True
|
|
assert observation["source_fix"]["runtime_canary_passed"] is False
|
|
|
|
|
|
def test_filelog_canary_is_synthetic_bounded_and_has_no_network_exporter() -> None:
|
|
text = FILELOG_CANARY.read_text(encoding="utf-8")
|
|
documents = list(yaml.safe_load_all(text))
|
|
config_map, pod = documents
|
|
assert config_map["kind"] == "ConfigMap"
|
|
assert "P0OBS_FILELOG_PARSED_CRI" in config_map["data"]["input.log"]
|
|
assert "P0OBS_FILELOG_FALLBACK_UNPARSED" in config_map["data"]["input.log"]
|
|
config = config_map["data"]["config.yaml"]
|
|
assert "on_error: send_quiet" not in config
|
|
assert 'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"' in config
|
|
assert "fallback_unparsed" in config
|
|
assert "debug:" in config
|
|
assert "otlp" not in config.lower()
|
|
|
|
spec = pod["spec"]
|
|
assert spec["automountServiceAccountToken"] is False
|
|
assert spec["restartPolicy"] == "Never"
|
|
assert spec["activeDeadlineSeconds"] == 90
|
|
container = spec["containers"][0]
|
|
assert "@sha256:" in container["image"]
|
|
assert container["securityContext"]["allowPrivilegeEscalation"] is False
|
|
assert container["securityContext"]["readOnlyRootFilesystem"] is True
|
|
|
|
|
|
def test_grpc_bridge_is_independently_supervised_allowlisted_and_reversible() -> None:
|
|
bridge = GRPC_BRIDGE.read_text(encoding="utf-8")
|
|
unit = GRPC_UNIT.read_text(encoding="utf-8")
|
|
for expected in (
|
|
'LISTEN_PORT="${LISTEN_PORT:-24317}"',
|
|
'SOURCE_RANGE="${SOURCE_RANGE:-192.168.0.120/31}"',
|
|
'UPSTREAM_PORT="${UPSTREAM_PORT:-4317}"',
|
|
"--check|--apply|--status|--rollback",
|
|
"systemctl --user enable --now",
|
|
"rollback_runtime",
|
|
"verify_nodes",
|
|
"no_firewall_no_secret",
|
|
):
|
|
assert expected in bridge
|
|
for forbidden in ("sudo", "iptables", "nft ", "docker ", "kubectl "):
|
|
assert forbidden not in bridge
|
|
|
|
for expected in (
|
|
"LISTEN_PORT=24317",
|
|
"SOURCE_RANGE=192.168.0.120/31",
|
|
"UPSTREAM_PORT=4317",
|
|
"ExecStartPre=/home/ollama/bin/awoooi-signoz-otlp-grpc-bridge.sh check-upstream",
|
|
"Restart=on-failure",
|
|
"NoNewPrivileges=true",
|
|
"ProtectSystem=strict",
|
|
"ProtectHome=read-only",
|
|
):
|
|
assert expected in unit
|
|
assert "User=" not in unit
|
|
|
|
|
|
def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier() -> None:
|
|
bridge = GRPC_BRIDGE.read_text(encoding="utf-8")
|
|
verifier = GRPC_VERIFIER.read_text(encoding="utf-8")
|
|
apply_block = bridge.split(" --apply)", 1)[1].split(" --status)", 1)[0]
|
|
|
|
learning = (
|
|
'receipt learning partial "application_trace_metric_10m_post_verifier_pending"'
|
|
)
|
|
terminal = (
|
|
"receipt terminal partial "
|
|
'"grpc_transport_restored_independent_600s_post_verifier_pending"'
|
|
)
|
|
assert learning in apply_block
|
|
assert terminal in apply_block
|
|
assert apply_block.index(learning) < apply_block.index(terminal)
|
|
assert "receipt terminal pass" not in apply_block
|
|
assert (
|
|
"receipt terminal pass "
|
|
'"real_application_otlp_grpc_trace_metric_freshness_closed"' in verifier
|
|
)
|
|
|
|
|
|
def test_contract_grpc_route_state_matches_verified_runtime_evidence() -> None:
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
incumbent = contract["routes"]["incumbent"]
|
|
observations = contract["runtime_observations"]
|
|
grpc = observations["application_otlp_grpc"]
|
|
|
|
assert (
|
|
incumbent["grpc_bridge_status"]
|
|
== "user_systemd_enabled_active_runtime_verified"
|
|
)
|
|
assert observations["grpc_bridge_supervisor"] == "user_systemd_enabled_active"
|
|
assert grpc["runtime_status"] == "restored_verified"
|
|
assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass"
|
|
assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"
|
|
assert grpc["source_fix"]["runtime_verifier_window_seconds"] >= 600
|
|
|
|
|
|
def test_grpc_runtime_verifier_uses_real_aggregates_and_ten_minute_gate() -> None:
|
|
text = GRPC_VERIFIER.read_text(encoding="utf-8")
|
|
for expected in (
|
|
'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
|
|
"https://awoooi.wooo.work/api/v1/openapi.json",
|
|
"serviceName={service:String}",
|
|
"signoz_metrics.samples_v4",
|
|
"resource_attrs['service.name']={service:String}",
|
|
"Failed to export traces",
|
|
"Failed to export metrics",
|
|
"restart_delta_0",
|
|
"real_application_otlp_grpc_trace_metric_freshness_closed",
|
|
):
|
|
assert expected in text
|
|
assert "SELECT body" not in text
|
|
assert "SELECT attributes" not in text
|
|
assert "/v1/traces" not in text
|
|
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
grpc = contract["runtime_observations"]["application_otlp_grpc"]
|
|
assert grpc["root_cause"] == "missing_incumbent_grpc_bridge"
|
|
assert grpc["runtime_status"] == "restored_verified"
|
|
assert grpc["confidence"] == 0.99
|
|
assert grpc["recent_awoooi_api_trace_count"] == 1098
|
|
assert grpc["recent_awoooi_api_metric_sample_count"] == 60180
|
|
assert grpc["http_only_verifier_blind_spot"] is True
|
|
assert grpc["source_fix"]["check_terminal"] == "pass"
|
|
assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass"
|
|
assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"
|
|
|
|
receipt = yaml.safe_load(GRPC_RUNTIME_RECEIPT.read_text(encoding="utf-8"))
|
|
assert receipt["run_id"] == grpc["source_fix"]["runtime_run_id"]
|
|
assert receipt["post_verifier"]["window_seconds"] == 600
|
|
assert receipt["post_verifier"]["exporter_trace_errors"] == 0
|
|
assert receipt["post_verifier"]["exporter_metric_errors"] == 0
|
|
assert receipt["post_verifier"]["awoooi_api_trace_count"] == 1098
|
|
assert receipt["post_verifier"]["awoooi_api_metric_sample_count"] == 60180
|
|
assert receipt["post_verifier"]["rollback"] == "not_required"
|
|
assert receipt["completion"]["grpc_runtime_asset"] == "closed"
|
|
assert receipt["completion"]["program"] == "partial_degraded"
|
|
|
|
regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
assert regression["baseline"]["verifier_terminal"] == "pass"
|
|
assert (
|
|
regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"]
|
|
is True
|
|
)
|
|
assert regression["controlled_recovery"]["stateful_services_touched"] == 0
|
|
assert regression["controlled_recovery"]["grpc_4317_listening"] is True
|
|
assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
|
|
assert regression["terminal"]["status"] == "partial_degraded"
|
|
|
|
|
|
def test_post_closure_v2_records_cd_5140_and_config_route_patch() -> None:
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
assert receipt["schema"] == "awoooi_signoz_post_closure_regression_v2"
|
|
|
|
cd = receipt["release_receipts"]["gitea_cd_5140"]
|
|
assert cd["gitea_run_id"] == 5140
|
|
assert cd["source_sha"] == "e4da6570a645e504cc93647d396f45f167b3172a"
|
|
assert cd["workflow_terminal"] == "success"
|
|
assert cd["focused_tests_passed"] == 6
|
|
assert cd["unit_tests"] == {"passed": 4817, "skipped": 23, "warnings": 69}
|
|
assert cd["integration_tests_passed"] == 5
|
|
assert cd["production_readback"]["build_tag_matched"] is True
|
|
assert cd["production_readback"]["desired_tag_matched"] is True
|
|
assert cd["production_readback"]["rollout_terminal"] == "success"
|
|
assert cd["postdeploy"]["alert_chain_terminal"] == "pass"
|
|
assert cd["postdeploy"]["playwright_passed"] == 5
|
|
assert cd["cd_probe"]["receipt_provenance"] == "gitea_run_5140_job_log"
|
|
assert cd["cd_probe"]["restart_free"] is True
|
|
assert cd["cd_probe"]["workers"] == 4
|
|
assert cd["cd_probe"]["safety_reserve_connections"] == 4
|
|
|
|
route = receipt["controlled_applies"]["config_route_patch"]
|
|
assert route["run_id"] == ("P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1")
|
|
assert route["source_commit"] == "9a9d1464a586ba172d1613db5bc285def43eb3e8"
|
|
assert route["included_in_release_sha"] == cd["source_sha"]
|
|
assert route["normalized_asset"]["internal_url"] == "http://192.168.0.110:8080"
|
|
assert route["normalized_asset"]["public_url"] == "https://signoz.wooo.work"
|
|
assert route["source_truth_diff"]["otlp_producer_route_changed"] is False
|
|
assert route["execution_receipt"]["deployment_status"] == "deployed_verified"
|
|
assert route["terminal"] == "applied_post_verifier_pass"
|
|
|
|
|
|
def test_prometheus_target_first_receipt_has_independent_verifier() -> None:
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
route = receipt["controlled_applies"]["prometheus_target_route"]
|
|
check = route["check_receipt"]
|
|
execution = route["execution_receipt"]
|
|
post = route["post_verifier"]
|
|
independent = route["independent_verifier"]
|
|
|
|
assert route["run_id"] == ("P0-OBS-002-prometheus-route-20260714T215106Z-apply1")
|
|
assert check["promtool"] == "pass"
|
|
assert check["candidate_sha"] == execution["active_sha"] == execution["runtime_sha"]
|
|
assert execution["action"] == "target_first_config_apply"
|
|
assert execution["backup_sha"] == check["source_sha"]
|
|
assert post["terminal"] == "target_up_two_scrapes"
|
|
assert post["first_scrape_at"] == "2026-07-15T05:52:38.146935853+08:00"
|
|
assert post["second_scrape_at"] == "2026-07-15T05:52:53.146935853+08:00"
|
|
assert post["first_scrape_at"] < post["second_scrape_at"]
|
|
assert post["target_count"] == 1
|
|
assert post["target_health"] == "up"
|
|
assert post["probe_success"] == 1
|
|
assert post["legacy_target_count"] == 0
|
|
assert independent["mode"] == "independent_readback"
|
|
assert independent["active_sha_matches_runtime"] is True
|
|
assert independent["target_count"] == 1
|
|
assert independent["target_health"] == "up"
|
|
assert independent["probe_success"] == 1
|
|
assert independent["legacy_target_count"] == 0
|
|
assert independent["terminal"] == "pass"
|
|
assert route["rollback"] == {"required": False, "available": True}
|
|
assert route["terminal"] == "completed_target_first"
|
|
|
|
|
|
def test_alert_rules_receipt_hashes_and_runtime_rule_match() -> None:
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
alert = receipt["controlled_applies"]["alert_rules"]
|
|
source_sha = alert["check_receipt"]["source_sha"]
|
|
execution = alert["execution_receipt"]
|
|
verifier = alert["post_verifier"]
|
|
|
|
assert alert["run_id"] == "P0-OBS-002-alert-rules-20260714T215328Z-apply1"
|
|
assert alert["check_receipt"]["promtool"] == "pass"
|
|
assert source_sha == execution["remote_sha"]
|
|
assert source_sha == execution["canonical_sha"]
|
|
assert source_sha == execution["runtime_sha"]
|
|
assert execution["hashes_match"] is True
|
|
assert execution["prometheus_ready"] is True
|
|
assert verifier["window_seconds"] == 130
|
|
assert verifier["terminal"] == "pass"
|
|
assert verifier["signoz_rule_count"] == 1
|
|
assert verifier["signoz_rule_health"] == "ok"
|
|
assert verifier["signoz_rule_state"] == "inactive"
|
|
assert verifier["exact_query"] == (
|
|
'(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
|
|
'job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
|
|
'job="blackbox-http"})'
|
|
)
|
|
assert alert["terminal"] == "completed_verified"
|
|
|
|
|
|
def test_failed_backup_canary_preserves_failure_and_verified_rollback() -> None:
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
backup = receipt["controlled_applies"]["backup_canary"]
|
|
assert backup["guard_source"]["deployment_status"] == "deployed_verified"
|
|
assert len(backup["attempts"]) == 2
|
|
|
|
attempt = backup["attempts"][0]
|
|
assert attempt["run_id"] == ("P0-OBS-002-backup-canary-20260714T213210Z-retry1")
|
|
assert attempt["terminal"] == "failed_rolled_back"
|
|
assert attempt["classification"] == (
|
|
"failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention"
|
|
)
|
|
assert attempt["failed_stage"] == "clickhouse_archive_create"
|
|
assert attempt["archive_container_exit_code"] == 137
|
|
assert attempt["root_cause"] == (
|
|
"docker_health_monitor_restarted_collector_during_intentional_backup_stop"
|
|
)
|
|
for false_field in (
|
|
"consistent_archive_valid",
|
|
"clickhouse_archive_committed",
|
|
"sqlite_stage_reached",
|
|
"restic_stage_reached",
|
|
"retention_stage_reached",
|
|
"success_stage_reached",
|
|
):
|
|
assert attempt[false_field] is False
|
|
|
|
rollback = attempt["rollback"]
|
|
assert rollback["collector_running"] is True
|
|
assert rollback["collector_restart_count"] == 0
|
|
assert rollback["grpc_4317_listening"] is True
|
|
assert rollback["http_4318_listening"] is True
|
|
assert rollback["canary_processes_absent"] is True
|
|
assert rollback["canary_container_absent"] is True
|
|
assert rollback["current_temp_dir_absent"] is True
|
|
assert rollback["partial_artifacts_absent"] is True
|
|
assert not attempt["terminal"].startswith("completed")
|
|
assert "learning_recorded" not in attempt
|
|
|
|
retry2 = backup["attempts"][1]
|
|
assert retry2["run_id"] == ("P0-OBS-002-backup-canary-20260714T221230Z-retry2")
|
|
assert retry2["terminal"] == "failed_rolled_back"
|
|
assert retry2["backup_script_exit_code"] == 1
|
|
assert retry2["archive_container_exit_code"] == "unknown_not_durably_captured"
|
|
assert retry2["archive_stderr_receipt"] == "suppressed_by_legacy_pipeline"
|
|
assert retry2["root_cause"] == (
|
|
"raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract"
|
|
)
|
|
assert retry2["exact_tar_error_proven"] is False
|
|
assert retry2["monitor_window"]["cron_intervals_crossed"] == 2
|
|
assert retry2["monitor_window"]["auto_repair_count"] == 0
|
|
for false_field in (
|
|
"consistent_archive_valid",
|
|
"clickhouse_archive_committed",
|
|
"sqlite_stage_reached",
|
|
"restic_stage_reached",
|
|
"retention_stage_reached",
|
|
"success_stage_reached",
|
|
):
|
|
assert retry2[false_field] is False
|
|
retry2_rollback = retry2["rollback"]
|
|
assert retry2_rollback["cooldown_lease_restored"] is True
|
|
assert retry2_rollback["collector_running"] is True
|
|
assert retry2_rollback["collector_restart_count"] == 0
|
|
assert retry2_rollback["grpc_4317_listening"] is True
|
|
assert retry2_rollback["http_4318_listening"] is True
|
|
assert retry2_rollback["current_temp_dir_absent"] is True
|
|
assert retry2_rollback["partial_artifacts_absent"] is True
|
|
assert retry2["residual_cleanup_debt"] == []
|
|
|
|
|
|
def test_clickhouse_native_backup_disk_preserves_failed_attempt_and_closes_retry() -> (
|
|
None
|
|
):
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
native_disk = receipt["controlled_applies"]["clickhouse_native_backup_disk"]
|
|
|
|
assert native_disk["source"]["override_sha256"] == (
|
|
"cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488"
|
|
)
|
|
assert native_disk["source"]["config_sha256"] == (
|
|
"c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
|
|
)
|
|
assert native_disk["check_receipt"]["terminal"] == "check_pass_no_write"
|
|
assert len(native_disk["attempts"]) == 2
|
|
|
|
apply1, retry2 = native_disk["attempts"]
|
|
assert apply1["terminal"] == "failed_rolled_back"
|
|
assert apply1["exit_code"] == 2
|
|
assert apply1["rollback"]["terminal"] == "pass"
|
|
assert apply1["rollback"]["original_data_volume_preserved"] is True
|
|
assert apply1["rollback"]["residue_count"] == 0
|
|
|
|
assert retry2["run_id"] == (
|
|
"P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
|
|
)
|
|
assert retry2["terminal"] == "pass"
|
|
assert retry2["execution"]["image_digest_unchanged"] is True
|
|
assert retry2["execution"]["data_volume_unchanged"] is True
|
|
assert retry2["post_verifier"]["independent"] is True
|
|
assert retry2["post_verifier"]["server_write_access"] is True
|
|
assert retry2["post_verifier"]["dependent_restart_count_delta"] == 0
|
|
assert retry2["post_verifier"]["api_http_status"] == 200
|
|
assert retry2["post_verifier"]["active_backup_or_restore_count"] == 0
|
|
assert retry2["post_verifier"]["stage_candidate_process_residue_count"] == 0
|
|
assert native_disk["terminal"] == "deployed_verified"
|
|
|
|
|
|
def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> None:
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
|
|
assert canary["completion_scope"] == "clickhouse_native_only"
|
|
assert canary["source"]["post_canary_focused_tests_passed"] == 94
|
|
assert canary["source"]["post_format_focused_tests_passed"] == 94
|
|
assert canary["source"]["ruff_format"] == "pass"
|
|
assert canary["toolchain_controlled_apply"]["apply_terminal"] == "pass"
|
|
assert canary["toolchain_controlled_apply"]["postcheck_diff"] == (
|
|
"matching_7_drift_0_absent_0_active_operations_0"
|
|
)
|
|
assert len(canary["attempts"]) == 5
|
|
_, apply2, apply3, apply4, apply5 = canary["attempts"]
|
|
assert apply2["artifact_retention"] == "preserved_failed_run_evidence"
|
|
assert apply2["cleanup_verified"] is True
|
|
assert apply3["artifact_retention"] == "preserved_failed_run_evidence"
|
|
assert apply3["cleanup_verified"] is True
|
|
|
|
assert apply4["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
|
|
assert apply4["check_terminal"] == "check_pass_no_write"
|
|
assert apply4["terminal"] == "pass"
|
|
assert apply4["backup"]["terminal"] == "BACKUP_CREATED"
|
|
assert apply4["backup"]["error"] == ""
|
|
assert apply4["backup"]["artifact_sha256"] == (
|
|
"4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc"
|
|
)
|
|
|
|
restore = apply4["isolated_restore"]
|
|
assert restore["terminal"] == "verified"
|
|
assert restore["clickhouse_restore_terminal"] == "RESTORED"
|
|
assert restore["production_network_attached"] is False
|
|
assert restore["production_restore_performed"] is False
|
|
assert restore["source_staged_clickhouse_artifact_sha_match"] is True
|
|
assert restore["database_count"] == restore["restored_database_count"] == 6
|
|
assert restore["table_count"] == restore["restored_table_count"] == 100
|
|
assert restore["table_engine_schema_manifest_parity"] is True
|
|
assert restore["check_table_count"] == restore["check_table_pass_count"] == 44
|
|
assert restore["critical_nonzero_count"] == 4
|
|
assert restore["aggregate_counter_exact_parity_gate"] is False
|
|
|
|
restic = apply4["restic"]
|
|
assert restic["snapshot_id"] == "574e2a1a"
|
|
assert restic["snapshot_readback"] == "pass"
|
|
assert restic["repository_scope"] == "local_same_failure_domain"
|
|
assert restic["offsite_verified"] is False
|
|
|
|
post = apply4["independent_post_verifier"]
|
|
assert post["production_restart_count_delta"] == 0
|
|
assert post["api_http_status"] == 200
|
|
assert post["new_server_artifact_removed"] is True
|
|
assert post["apply2_apply3_preserved_artifact_hashes_unchanged"] is True
|
|
assert (
|
|
post["exact_ephemeral_container_volume_network_process_tmp_residue_count"] == 0
|
|
)
|
|
assert post["cleanup_verified"] is True
|
|
assert post["secret_values_collected"] is False
|
|
|
|
assert apply5["run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
|
)
|
|
assert apply5["check_run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
|
)
|
|
assert apply5["wrapper_terminal"] == "partial_degraded"
|
|
assert apply5["terminal"] == "partial_degraded"
|
|
assert apply5["runtime_operation"]["terminal"] == "pass"
|
|
assert apply5["nested_machine_receipt_verifier"] == {
|
|
"stage": "native_restore_receipt_verifier",
|
|
"terminal": "pass",
|
|
}
|
|
assert apply5["backup"]["terminal"] == "BACKUP_CREATED"
|
|
assert apply5["backup"]["file_count"] == 4885
|
|
assert apply5["backup"]["total_size_bytes"] == 108006575
|
|
assert apply5["backup"]["compressed_size_bytes"] == 99461897
|
|
assert apply5["backup"]["artifact_sha256"] == (
|
|
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
|
)
|
|
current_restore = apply5["isolated_restore"]
|
|
assert (
|
|
current_restore["database_count"]
|
|
== current_restore["restored_database_count"]
|
|
== 6
|
|
)
|
|
assert (
|
|
current_restore["table_count"] == current_restore["restored_table_count"] == 100
|
|
)
|
|
assert (
|
|
current_restore["check_table_count"]
|
|
== current_restore["check_table_pass_count"]
|
|
== 44
|
|
)
|
|
assert current_restore["critical_nonzero_count"] == 4
|
|
assert apply5["restic"]["snapshot_id"] == "135485ab"
|
|
current_post = apply5["independent_post_verifier"]
|
|
assert current_post["independent_monitor"] == "pass"
|
|
assert current_post["deployed_source_hash_drift_count"] == 0
|
|
assert current_post["runtime_drift_count"] == 0
|
|
assert (
|
|
current_post[
|
|
"exact_ephemeral_container_volume_network_process_tmp_residue_count"
|
|
]
|
|
== 0
|
|
)
|
|
assert current_post["cleanup_verified"] is True
|
|
assert canary["terminal"] == "production_closed_clickhouse_native_scope"
|
|
|
|
|
|
def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt() -> (
|
|
None
|
|
):
|
|
module = _load_preflight_module()
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
|
|
historical = canary["source"]
|
|
current = canary["current_source"]
|
|
|
|
historical_hashes = {
|
|
field: historical[field] for field in module.TOOLCHAIN_SOURCE_PATHS
|
|
}
|
|
assert historical["evidence_scope"] == "historical_production_receipt"
|
|
assert historical["deployment_status"] == "deployed_verified"
|
|
assert historical_hashes == module.HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES
|
|
assert historical["inventory_verifier_sha256"] == (
|
|
"79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c"
|
|
)
|
|
|
|
current_hashes = {
|
|
field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest()
|
|
for field, relative in module.TOOLCHAIN_SOURCE_PATHS.items()
|
|
}
|
|
assert current["hashes"] == current_hashes
|
|
assert current["hashes"]["inventory_verifier_sha256"] == (
|
|
"c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863"
|
|
)
|
|
assert current["deployed_source_hashes"] == current_hashes
|
|
assert current["status"] == "deployed_verified"
|
|
assert current["deployment_status"] == "deployed"
|
|
assert current["current_source_deployed"] is True
|
|
assert current["runtime_verifier_terminal"] == "pass"
|
|
assert current["historical_production_receipt_preserved"] is True
|
|
assert current["drift_from_historical_production_receipt"] == [
|
|
"backup_orchestrator_sha256",
|
|
"canary_wrapper_sha256",
|
|
"inventory_verifier_sha256",
|
|
]
|
|
assert current["toolchain_check_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
|
|
)
|
|
assert current["toolchain_apply_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
|
)
|
|
assert current["toolchain_postcheck_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
|
|
)
|
|
assert current["toolchain_postcanary_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
|
|
)
|
|
assert current["default_canary_check_run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
|
)
|
|
assert current["runtime_canary_run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
|
)
|
|
assert current["wrapper_terminal"] == "partial_degraded"
|
|
assert current["runtime_operation_terminal"] == "pass"
|
|
assert current["nested_machine_receipt_verifier_terminal"] == "pass"
|
|
|
|
|
|
def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale(
|
|
monkeypatch,
|
|
) -> None:
|
|
module = _load_preflight_module()
|
|
real_sha256_file = module._sha256_file
|
|
|
|
def stale_inventory_hash(path: Path) -> str:
|
|
if path.name == "clickhouse-restore-inventory.py":
|
|
return "0" * 64
|
|
return real_sha256_file(path)
|
|
|
|
monkeypatch.setattr(module, "_sha256_file", stale_inventory_hash)
|
|
payload = module.evaluate(ROOT, CONTRACT)
|
|
|
|
assert payload["source_ready"] is False
|
|
assert payload["terminal"] == "failed"
|
|
assert payload["checks"]["native_backup_current_source_hashes_bound"] is False
|
|
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
|
|
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
|
|
|
|
|
|
def test_preflight_fails_closed_when_current_runtime_receipt_is_incomplete(
|
|
monkeypatch,
|
|
) -> None:
|
|
module = _load_preflight_module()
|
|
real_load_yaml = module._load_yaml
|
|
|
|
def load_without_operation_identity(path: Path):
|
|
payload = real_load_yaml(path)
|
|
if path.name == POST_CLOSURE_REGRESSION.name:
|
|
attempts = payload["controlled_applies"][
|
|
"clickhouse_native_backup_restore_canary"
|
|
]["attempts"]
|
|
attempts[-1]["backup"].pop("operation_identity_hash")
|
|
return payload
|
|
|
|
monkeypatch.setattr(module, "_load_yaml", load_without_operation_identity)
|
|
payload = module.evaluate(ROOT, CONTRACT)
|
|
|
|
assert payload["source_ready"] is False
|
|
assert payload["terminal"] == "failed"
|
|
assert payload["checks"]["native_backup_current_source_runtime_closed"] is False
|
|
assert payload["runtime_status"] == "partial_degraded"
|
|
|
|
|
|
def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
|
|
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
|
verifier = receipt["post_release_runtime_verifier"]
|
|
assert verifier["run_id"] == "P0-OBS-002-post-release-20260714T215500Z"
|
|
assert verifier["window_seconds"] == 600
|
|
assert verifier["terminal"] == "pass"
|
|
assert verifier["restart_delta"] == 0
|
|
assert verifier["exporter_trace_errors"] == 0
|
|
assert verifier["exporter_metric_errors"] == 0
|
|
assert verifier["trace_count"] == 4026
|
|
assert verifier["metric_sample_count"] == 76369
|
|
assert verifier["api_runtime_cooldown_closed"] is True
|
|
assert (
|
|
receipt["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is True
|
|
)
|
|
|
|
closed = receipt["completed_verifiers"]["clickhouse_native_backup_migration"]
|
|
assert closed["prior_run_id"] == (
|
|
"P0-OBS-002-backup-canary-20260714T221230Z-retry2"
|
|
)
|
|
assert closed["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
|
|
assert closed["status"] == "production_closed_clickhouse_native_scope"
|
|
assert closed["native_backup_terminal"] == "BACKUP_CREATED"
|
|
assert closed["isolated_restore_terminal"] == "verified"
|
|
assert closed["restic_snapshot_id"] == "574e2a1a"
|
|
assert closed["independent_verifier"] == "pass"
|
|
assert closed["exact_cleanup"] == "pass"
|
|
sqlite_pending = receipt["pending_verifiers"][
|
|
"signoz_sqlite_application_consistent_backup"
|
|
]
|
|
assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False
|
|
assert sqlite_pending["sqlite_cli_present_in_runtime"] is False
|
|
assert sqlite_pending["source_contract_status"] == ("deployed_inactive_verified")
|
|
source_contract = sqlite_pending["source_contract"]
|
|
assert source_contract["policy"] == "config/signoz/metadata-export-policy.json"
|
|
assert source_contract["exporter"] == ("scripts/backup/signoz-metadata-export.py")
|
|
assert source_contract["independent_export_verifier"] == (
|
|
"scripts/backup/verify-signoz-metadata-export.py"
|
|
)
|
|
assert source_contract["isolated_restore_driver"] == (
|
|
"scripts/backup/signoz-metadata-restore-drill.py"
|
|
)
|
|
assert source_contract["controlled_deployer"] == (
|
|
"scripts/ops/deploy-signoz-metadata-toolchain.sh"
|
|
)
|
|
assert source_contract["deployer_tests"] == (
|
|
"scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py"
|
|
)
|
|
assert source_contract["expected_file_count"] == 8
|
|
metadata_paths = {
|
|
"policy_sha256": "config/signoz/metadata-export-policy.json",
|
|
"exporter_sha256": "scripts/backup/signoz-metadata-export.py",
|
|
"shared_contract_sha256": "scripts/backup/signoz_metadata_contract.py",
|
|
"independent_export_verifier_sha256": (
|
|
"scripts/backup/verify-signoz-metadata-export.py"
|
|
),
|
|
"isolated_restore_driver_sha256": (
|
|
"scripts/backup/signoz-metadata-restore-drill.py"
|
|
),
|
|
"contract_tests_sha256": (
|
|
"scripts/backup/tests/test_signoz_metadata_export_contract.py"
|
|
),
|
|
"controlled_deployer_sha256": (
|
|
"scripts/ops/deploy-signoz-metadata-toolchain.sh"
|
|
),
|
|
"deployer_tests_sha256": (
|
|
"scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py"
|
|
),
|
|
}
|
|
assert source_contract["hashes"] == {
|
|
field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest()
|
|
for field, relative in metadata_paths.items()
|
|
}
|
|
assert source_contract["test_terminal"] == "21_passed"
|
|
deployment = source_contract["deployment"]
|
|
assert deployment["status"] == "deployed_inactive_verified"
|
|
assert deployment["asset_id"] == "host110-signoz-metadata-toolchain"
|
|
assert deployment["target_host"] == "192.168.0.110"
|
|
assert deployment["bundle_id"] == (
|
|
"bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011"
|
|
)
|
|
assert deployment["directory_owner"] == "root:root"
|
|
assert deployment["directory_mode"] == "0750"
|
|
assert deployment["active_pointer_present"] is False
|
|
assert deployment["runtime_export_executed"] is False
|
|
assert deployment["initial_failed_attempt"]["terminal"] == (
|
|
"failed_no_active_pointer_write"
|
|
)
|
|
assert deployment["bounded_execution"]["durable_inner_terminal"] == (
|
|
"pass_source_deployed_inactive"
|
|
)
|
|
assert deployment["bounded_execution"]["outer_terminal"] == (
|
|
"failed_source_deploy_unverified"
|
|
)
|
|
assert deployment["independent_root_postcheck"]["terminal"] == (
|
|
"check_pass_no_write"
|
|
)
|
|
assert deployment["independent_root_postcheck"]["remote_state"] == "exact"
|
|
assert deployment["independent_root_postcheck"]["drift_count"] == 0
|
|
assert deployment["idempotent_closure"]["execution_terminal"] == (
|
|
"idempotent_no_write"
|
|
)
|
|
assert deployment["idempotent_closure"]["terminal"] == (
|
|
"pass_source_deployed_inactive"
|
|
)
|
|
assert deployment["idempotent_closure"]["independent_post_verifier"] == "pass"
|
|
assert deployment["learning_writeback"] == "partial_degraded"
|
|
assert source_contract["raw_sqlite_read"] is False
|
|
assert source_contract["raw_volume_read"] is False
|
|
assert source_contract["secret_value_persistence"] is False
|
|
assert source_contract["full_sqlite_completion_claim"] is False
|
|
assert sqlite_pending["runtime_export_terminal"] == (
|
|
"pending_authenticated_stable_export"
|
|
)
|
|
assert sqlite_pending["runtime_restore_terminal"] == (
|
|
"pending_isolated_same_version_target"
|
|
)
|
|
assert sqlite_pending["runtime_zero_residue_terminal"] == "pending"
|
|
assert len(sqlite_pending["required_preconditions"]) == 5
|
|
offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"]
|
|
assert offsite["current_repository_scope"] == "local_same_failure_domain"
|
|
assert offsite["offsite_verified"] is False
|
|
artifacts = receipt["pending_verifiers"][
|
|
"clickhouse_native_failed_artifact_retention"
|
|
]
|
|
assert artifacts["destructive_cleanup_authorized"] is False
|
|
assert artifacts["active_backup_or_restore_count"] == 0
|
|
assert len(artifacts["preserved_artifacts"]) == 2
|
|
resume = receipt["pending_verifiers"][
|
|
"clickhouse_native_resume_submitted_inventory"
|
|
]
|
|
assert resume["same_run_resume_safe"] is False
|
|
assert len(resume["required_preconditions"]) == 4
|
|
current_source = receipt["completed_verifiers"][
|
|
"clickhouse_native_current_source_toolchain"
|
|
]
|
|
assert current_source["status"] == "deployed_verified"
|
|
assert current_source["current_source_deployed"] is True
|
|
assert current_source["runtime_verifier_terminal"] == "pass"
|
|
assert current_source["historical_production_receipt_preserved"] is True
|
|
assert current_source["toolchain_apply_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
|
)
|
|
assert current_source["runtime_canary_run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
|
)
|
|
assert current_source["wrapper_terminal"] == "partial_degraded"
|
|
assert current_source["runtime_operation_terminal"] == "pass"
|
|
assert current_source["nested_machine_receipt_verifier_terminal"] == "pass"
|
|
assert current_source["restic_snapshot_id"] == "135485ab"
|
|
assert current_source["artifact_sha256"] == (
|
|
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
|
)
|
|
assert current_source["independent_monitor"] == "pass"
|
|
assert current_source["runtime_drift_count"] == 0
|
|
assert current_source["residue_count"] == 0
|
|
assert current_source["exact_cleanup"] == "pass"
|
|
assert (
|
|
"clickhouse_native_current_source_toolchain" not in receipt["pending_verifiers"]
|
|
)
|
|
assets = receipt["asset_reconciliation"]
|
|
assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
|
|
assert assets["source_service_count"] == 28
|
|
assert assets["runtime_mirror_service_count"] == 28
|
|
assert assets["exact_shared_service_count"] == 28
|
|
assert assets["shared_service_drift_count"] == 0
|
|
assert assets["source_only_services"] == []
|
|
assert assets["source_runtime_manifest_exact"] is True
|
|
assert assets["production_runtime_readback_status"] == (
|
|
"pending_current_sha_readback"
|
|
)
|
|
assert assets["signoz_clickhouse_exact_match"] is True
|
|
assert assets["live_signoz_query_host"] == "192.168.0.110"
|
|
assert assets["source_signoz_host"] == "192.168.0.188"
|
|
github_drift = assets["github_freeze_legacy_asset_drift"]
|
|
assert github_drift["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-002"
|
|
assert github_drift["superseded_by"] == "global_product_governance_v2"
|
|
assert github_drift["active_github_use_in_this_run"] is False
|
|
assert len(github_drift["source_references"]) == 3
|
|
assert github_drift["p0_p1_github_recovery_scheduled"] is False
|
|
duplicate_identity = assets["monitoring_generator_duplicate_identity_drift"]
|
|
assert duplicate_identity["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-003"
|
|
assert duplicate_identity["status"] == "pending_source_deduplication"
|
|
assert duplicate_identity["awoooi_api_registry_entry_count"] == 2
|
|
assert duplicate_identity["generated_scrape_config_count"] == 24
|
|
assert duplicate_identity["generated_awoooi_api_job_count"] == 2
|
|
assert duplicate_identity["generated_blackbox_target_count"] == 20
|
|
assert duplicate_identity["runtime_apply_performed"] is False
|
|
assert receipt["terminal"]["status"] == "partial_degraded"
|
|
assert receipt["terminal"]["blockers"] == [
|
|
"signoz_sqlite_application_consistent_backup_pending",
|
|
"signoz_backup_offsite_dr_pending",
|
|
"clickhouse_native_failed_artifact_retention_pending",
|
|
"clickhouse_native_resume_submitted_inventory_pending",
|
|
"service_registry_production_and_live_host_readback_pending",
|
|
"github_freeze_legacy_asset_retirement_pending",
|
|
"monitoring_generator_duplicate_awoooi_api_identity_pending",
|
|
]
|
|
|
|
|
|
def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> None:
|
|
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
|
post_closure = contract["runtime_observations"]["post_closure_regression"]
|
|
assert post_closure["snapshot_schema"] == (
|
|
"awoooi_signoz_post_closure_regression_v2"
|
|
)
|
|
assert post_closure["gitea_cd_run_id"] == 5140
|
|
assert post_closure["api_runtime_cooldown_closed"] is True
|
|
assert post_closure["cd_probe_safety_fix_deployed"] is True
|
|
assert post_closure["config_route_patch_deployed"] is True
|
|
assert post_closure["signoz_control_plane_health_route_deployed"] is True
|
|
assert post_closure["backup_collector_restore_guard_deployed"] is True
|
|
assert post_closure["backup_canary_scope"] == "clickhouse_native_only"
|
|
assert post_closure["backup_canary_verified"] is True
|
|
assert post_closure["backup_canary_last_terminal"] == "partial_degraded"
|
|
assert post_closure["backup_canary_last_run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
|
)
|
|
assert post_closure["backup_canary_runtime_operation_terminal"] == "pass"
|
|
assert post_closure["backup_canary_machine_receipt_verifier_terminal"] == "pass"
|
|
assert post_closure["backup_canary_cooldown_retry_status"] == (
|
|
"superseded_by_online_native_no_stop_contract"
|
|
)
|
|
assert post_closure["clickhouse_native_backup_migration_status"] == (
|
|
"production_closed"
|
|
)
|
|
assert post_closure["clickhouse_native_backup_disk_run_id"] == (
|
|
"P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
|
|
)
|
|
assert post_closure["clickhouse_native_backup_disk_status"] == "deployed_verified"
|
|
assert post_closure["clickhouse_native_backup_toolchain_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
|
|
)
|
|
assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
|
|
)
|
|
assert (
|
|
post_closure["clickhouse_native_current_source_status"] == "deployed_verified"
|
|
)
|
|
assert post_closure["clickhouse_native_current_source_deployed"] is True
|
|
assert (
|
|
post_closure["clickhouse_native_current_source_runtime_verifier_terminal"]
|
|
== "pass"
|
|
)
|
|
assert post_closure["clickhouse_native_current_source_toolchain_check_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
|
|
)
|
|
assert post_closure["clickhouse_native_current_source_toolchain_apply_run_id"] == (
|
|
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
|
)
|
|
assert (
|
|
post_closure["clickhouse_native_current_source_toolchain_postcheck_run_id"]
|
|
== "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
|
|
)
|
|
assert (
|
|
post_closure["clickhouse_native_current_source_toolchain_postcanary_run_id"]
|
|
== "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
|
|
)
|
|
assert post_closure["clickhouse_native_current_source_canary_check_run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
|
)
|
|
assert post_closure["clickhouse_native_current_source_canary_apply_run_id"] == (
|
|
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
|
)
|
|
assert post_closure["clickhouse_native_current_source_wrapper_terminal"] == (
|
|
"partial_degraded"
|
|
)
|
|
assert post_closure[
|
|
"clickhouse_native_current_source_runtime_operation_terminal"
|
|
] == ("pass")
|
|
assert (
|
|
post_closure[
|
|
"clickhouse_native_current_source_machine_receipt_verifier_terminal"
|
|
]
|
|
== "pass"
|
|
)
|
|
assert post_closure["clickhouse_native_current_source_artifact_sha256"] == (
|
|
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
|
)
|
|
assert post_closure["clickhouse_native_restore_terminal"] == "verified"
|
|
assert post_closure["clickhouse_native_restic_snapshot_id"] == "135485ab"
|
|
assert post_closure["clickhouse_native_offsite_status"] == "pending"
|
|
assert post_closure["clickhouse_native_failed_artifact_retention_status"] == (
|
|
"pending_bounded_retention_decision"
|
|
)
|
|
assert post_closure["clickhouse_native_resume_inventory_status"] == (
|
|
"pending_durable_submitted_inventory_contract"
|
|
)
|
|
assert (
|
|
post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending"
|
|
)
|
|
assert post_closure["signoz_metadata_export_source_contract_status"] == (
|
|
"deployed_inactive_verified"
|
|
)
|
|
assert post_closure["signoz_metadata_export_policy"] == (
|
|
"config/signoz/metadata-export-policy.json"
|
|
)
|
|
assert post_closure["signoz_metadata_exporter"] == (
|
|
"scripts/backup/signoz-metadata-export.py"
|
|
)
|
|
assert post_closure["signoz_metadata_export_verifier"] == (
|
|
"scripts/backup/verify-signoz-metadata-export.py"
|
|
)
|
|
assert post_closure["signoz_metadata_restore_driver"] == (
|
|
"scripts/backup/signoz-metadata-restore-drill.py"
|
|
)
|
|
assert post_closure["signoz_metadata_controlled_deployer"] == (
|
|
"scripts/ops/deploy-signoz-metadata-toolchain.sh"
|
|
)
|
|
assert post_closure["signoz_metadata_runtime_export_terminal"] == (
|
|
"pending_authenticated_stable_export"
|
|
)
|
|
assert post_closure["signoz_metadata_runtime_restore_terminal"] == (
|
|
"pending_isolated_same_version_target"
|
|
)
|
|
assert post_closure["signoz_metadata_zero_residue_terminal"] == "pending"
|
|
assert post_closure["signoz_metadata_full_sqlite_completion_claim"] is False
|
|
assert post_closure["service_registry_runtime_mirror_status"] == (
|
|
"source_manifest_exact_production_readback_pending"
|
|
)
|
|
assert post_closure["service_registry_runtime_mirror_work_item_id"] == (
|
|
"P0-OBS-002-ASSET-DRIFT-001"
|
|
)
|
|
assert post_closure["github_freeze_legacy_asset_status"] == (
|
|
"pending_controlled_retirement"
|
|
)
|
|
assert post_closure["github_freeze_legacy_asset_work_item_id"] == (
|
|
"P0-OBS-002-ASSET-DRIFT-002"
|
|
)
|
|
assert post_closure["monitoring_generator_duplicate_identity_status"] == (
|
|
"pending_source_deduplication"
|
|
)
|
|
assert post_closure["monitoring_generator_duplicate_identity_work_item_id"] == (
|
|
"P0-OBS-002-ASSET-DRIFT-003"
|
|
)
|
|
assert post_closure["post_release_otlp_grpc_600s_verifier_status"] == "pass"
|
|
assert contract["terminal"]["status"] == "partial_degraded"
|
|
assert contract["terminal"]["blockers"] == [
|
|
"organization_not_initialized",
|
|
"direct_ingress_policy_not_closed",
|
|
"http_bridge_not_supervised",
|
|
"filelog_normalization_degraded",
|
|
"wave_b_not_authorized",
|
|
]
|
|
|
|
|
|
def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
|
|
env = os.environ.copy()
|
|
env.pop("MANAGED_PROMOTION_RECEIPT", None)
|
|
env.pop("RETIRE_RECEIPT", None)
|
|
|
|
managed = subprocess.run(
|
|
["bash", str(STATIC), "--rollback"],
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert managed.returncode == 3
|
|
assert "managed_promotion_receipt_required" in managed.stdout
|
|
assert "ssh" not in managed.stderr.lower()
|
|
|
|
retired = subprocess.run(
|
|
["bash", str(BRIDGE), "--rollback"],
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert retired.returncode == 3
|
|
assert "bridge_retirement_receipt_required" in retired.stdout
|
|
assert "ssh" not in retired.stderr.lower()
|