feat(observability): close native SigNoz backup restore canary
This commit is contained in:
ogt
@@ -265,6 +265,12 @@ require_file scripts/backup/backup-awoooi-frequent.sh "AWOOOI high-frequency DB
|
||||
require_file scripts/backup/backup-langfuse.sh "Langfuse backup script"
|
||||
require_file scripts/backup/backup-monitoring.sh "Monitoring backup script"
|
||||
require_file scripts/backup/backup-signoz.sh "SignOz backup script"
|
||||
require_file scripts/backup/clickhouse-native-backup.sh "ClickHouse native backup adapter"
|
||||
require_file scripts/backup/clickhouse-native-restore-drill.sh "ClickHouse isolated native restore drill"
|
||||
require_file scripts/backup/clickhouse-restore-inventory.py "ClickHouse restore inventory verifier"
|
||||
require_file scripts/backup/run-signoz-backup-canary.sh "Guarded SignOz backup canary wrapper"
|
||||
require_file ops/signoz/clickhouse/config.d/backup_disk.xml "ClickHouse backup disk config"
|
||||
require_file ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml "ClickHouse isolated restore config"
|
||||
require_file scripts/backup/backup-open-webui.sh "Open-WebUI backup script"
|
||||
require_file scripts/backup/backup-clawbot.sh "ClawBot backup script"
|
||||
require_file scripts/backup/backup-configs.sh "Host/service config backup"
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
"""No-write source preflight for P0-OBS-002 Wave A.
|
||||
|
||||
The checker intentionally does not access credentials, databases, runtime
|
||||
volumes, Docker, Kubernetes, or remote hosts. Runtime evidence belongs to a
|
||||
separate verifier receipt.
|
||||
volumes, Docker, Kubernetes, or remote hosts. It validates runtime evidence
|
||||
only from the committed post-closure receipt.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,17 +16,24 @@ from typing import Any
|
||||
import yaml
|
||||
|
||||
SCHEMA = "awoooi_signoz_canonical_route_preflight_v1"
|
||||
POST_CLOSURE_SCHEMA = "awoooi_signoz_post_closure_regression_v2"
|
||||
POST_CLOSURE_SNAPSHOT = "ops/signoz/p0-obs-002-post-closure-regression.yaml"
|
||||
EXPECTED_BLOCKERS = [
|
||||
"organization_not_initialized",
|
||||
"direct_ingress_policy_not_closed",
|
||||
"http_bridge_not_supervised",
|
||||
"filelog_normalization_degraded",
|
||||
"api_runtime_cooldown_pending",
|
||||
"cd_probe_safety_fix_not_deployed",
|
||||
"backup_collector_restore_guard_not_deployed",
|
||||
"signoz_control_plane_health_route_not_deployed",
|
||||
"wave_b_not_authorized",
|
||||
]
|
||||
EXPECTED_POST_CLOSURE_BLOCKERS = [
|
||||
"signoz_sqlite_application_consistent_backup_pending",
|
||||
"signoz_backup_offsite_dr_pending",
|
||||
"clickhouse_native_failed_artifact_retention_pending",
|
||||
"clickhouse_native_resume_submitted_inventory_pending",
|
||||
"service_registry_runtime_mirror_reconciliation_pending",
|
||||
"github_freeze_legacy_asset_retirement_pending",
|
||||
"monitoring_generator_duplicate_awoooi_api_identity_pending",
|
||||
]
|
||||
|
||||
|
||||
def _load_yaml(path: Path) -> Any:
|
||||
@@ -40,6 +47,16 @@ def _read(root: Path, relative: str) -> str:
|
||||
|
||||
def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
contract = _load_yaml(contract_path)
|
||||
post_closure = contract.get("runtime_observations", {}).get(
|
||||
"post_closure_regression", {}
|
||||
)
|
||||
snapshot_path = str(post_closure.get("snapshot", ""))
|
||||
regression_path = root / POST_CLOSURE_SNAPSHOT
|
||||
regression = (
|
||||
_load_yaml(regression_path)
|
||||
if snapshot_path == POST_CLOSURE_SNAPSHOT and regression_path.is_file()
|
||||
else {}
|
||||
)
|
||||
checks: dict[str, bool] = {}
|
||||
|
||||
checks["contract_schema"] = (
|
||||
@@ -66,8 +83,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
config = _read(root, "apps/api/src/core/config.py")
|
||||
checks["dual_route_application_allow"] = (
|
||||
'default=["192.168.0.188", "192.168.0.110"]' in config
|
||||
and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]'
|
||||
in config
|
||||
and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]' in config
|
||||
)
|
||||
|
||||
network_documents = list(
|
||||
@@ -82,18 +98,14 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
)
|
||||
canonical_ports: set[int] = set()
|
||||
for rule in egress_policy.get("spec", {}).get("egress", []):
|
||||
cidrs = {
|
||||
item.get("ipBlock", {}).get("cidr") for item in rule.get("to", [])
|
||||
}
|
||||
cidrs = {item.get("ipBlock", {}).get("cidr") for item in rule.get("to", [])}
|
||||
if "192.168.0.110/32" in cidrs:
|
||||
canonical_ports.update(
|
||||
int(item["port"])
|
||||
for item in rule.get("ports", [])
|
||||
if isinstance(item.get("port"), int)
|
||||
)
|
||||
checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset(
|
||||
canonical_ports
|
||||
)
|
||||
checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset(canonical_ports)
|
||||
|
||||
unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-bridge.service")
|
||||
checks["bridge_unit_hardened"] = all(
|
||||
@@ -112,14 +124,15 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
root, "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
|
||||
)
|
||||
checks["route_specific_verifier"] = (
|
||||
"OTLP_ENDPOINT" in verifier and "ROUTE_ID" in verifier
|
||||
"OTLP_ENDPOINT" in verifier
|
||||
and "ROUTE_ID" in verifier
|
||||
and all(
|
||||
value in signal_verifier
|
||||
for value in (
|
||||
'send_signal logs',
|
||||
'send_signal metrics',
|
||||
'send_signal traces',
|
||||
'/v1/$signal',
|
||||
"send_signal logs",
|
||||
"send_signal metrics",
|
||||
"send_signal traces",
|
||||
"/v1/$signal",
|
||||
"signoz_logs.logs_v2",
|
||||
"signoz_metrics.time_series_v4",
|
||||
"signoz_metrics.samples_v4",
|
||||
@@ -127,57 +140,54 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
)
|
||||
)
|
||||
)
|
||||
grpc_bridge = _read(
|
||||
root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
|
||||
)
|
||||
grpc_unit = _read(
|
||||
root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
|
||||
)
|
||||
grpc_bridge = _read(root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh")
|
||||
grpc_unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service")
|
||||
grpc_verifier = _read(
|
||||
root, "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
|
||||
)
|
||||
_, grpc_apply_marker, grpc_after_apply = grpc_bridge.partition(" --apply)")
|
||||
grpc_apply_block, grpc_status_marker, _ = grpc_after_apply.partition(
|
||||
" --status)"
|
||||
)
|
||||
checks["grpc_transport_recovery_source_ready"] = all(
|
||||
value in grpc_bridge
|
||||
for value in (
|
||||
"LISTEN_PORT:-24317",
|
||||
"UPSTREAM_PORT:-4317",
|
||||
"SOURCE_RANGE:-192.168.0.120/31",
|
||||
"systemctl --user enable --now",
|
||||
"--rollback",
|
||||
grpc_apply_block, grpc_status_marker, _ = grpc_after_apply.partition(" --status)")
|
||||
checks["grpc_transport_recovery_source_ready"] = (
|
||||
all(
|
||||
value in grpc_bridge
|
||||
for value in (
|
||||
"LISTEN_PORT:-24317",
|
||||
"UPSTREAM_PORT:-4317",
|
||||
"SOURCE_RANGE:-192.168.0.120/31",
|
||||
"systemctl --user enable --now",
|
||||
"--rollback",
|
||||
)
|
||||
)
|
||||
) and all(
|
||||
value in grpc_unit
|
||||
for value in (
|
||||
"LISTEN_PORT=24317",
|
||||
"UPSTREAM_PORT=4317",
|
||||
"SOURCE_RANGE=192.168.0.120/31",
|
||||
"Restart=on-failure",
|
||||
"NoNewPrivileges=true",
|
||||
and all(
|
||||
value in grpc_unit
|
||||
for value in (
|
||||
"LISTEN_PORT=24317",
|
||||
"UPSTREAM_PORT=4317",
|
||||
"SOURCE_RANGE=192.168.0.120/31",
|
||||
"Restart=on-failure",
|
||||
"NoNewPrivileges=true",
|
||||
)
|
||||
)
|
||||
) and all(
|
||||
value in grpc_verifier
|
||||
for value in (
|
||||
"WINDOW_SECONDS:-600",
|
||||
"serviceName={service:String}",
|
||||
"signoz_metrics.samples_v4",
|
||||
"Failed to export traces",
|
||||
"restart_delta_0",
|
||||
and all(
|
||||
value in grpc_verifier
|
||||
for value in (
|
||||
"WINDOW_SECONDS:-600",
|
||||
"serviceName={service:String}",
|
||||
"signoz_metrics.samples_v4",
|
||||
"Failed to export traces",
|
||||
"restart_delta_0",
|
||||
)
|
||||
)
|
||||
)
|
||||
checks["grpc_apply_waits_for_independent_post_verifier"] = (
|
||||
bool(grpc_apply_marker)
|
||||
and bool(grpc_status_marker)
|
||||
and (
|
||||
'receipt learning partial '
|
||||
'"application_trace_metric_10m_post_verifier_pending"'
|
||||
in grpc_apply_block
|
||||
"receipt learning partial "
|
||||
'"application_trace_metric_10m_post_verifier_pending"' in grpc_apply_block
|
||||
)
|
||||
and (
|
||||
'receipt terminal partial '
|
||||
"receipt terminal partial "
|
||||
'"grpc_transport_restored_independent_600s_post_verifier_pending"'
|
||||
in grpc_apply_block
|
||||
)
|
||||
@@ -185,7 +195,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
and grpc_apply_block.index("receipt learning partial")
|
||||
< grpc_apply_block.index("receipt terminal partial")
|
||||
and (
|
||||
'receipt terminal pass '
|
||||
"receipt terminal pass "
|
||||
'"real_application_otlp_grpc_trace_metric_freshness_closed"'
|
||||
in grpc_verifier
|
||||
)
|
||||
@@ -202,14 +212,11 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
and contract.get("runtime_observations", {}).get("grpc_bridge_supervisor")
|
||||
== "user_systemd_enabled_active"
|
||||
and grpc_runtime.get("runtime_status") == "restored_verified"
|
||||
and grpc_source_fix.get("runtime_apply_status")
|
||||
== "applied_post_verifier_pass"
|
||||
and grpc_source_fix.get("runtime_apply_status") == "applied_post_verifier_pass"
|
||||
and grpc_source_fix.get("runtime_verifier_terminal") == "pass"
|
||||
and int(grpc_source_fix.get("runtime_verifier_window_seconds") or 0) >= 600
|
||||
)
|
||||
static_recovery = _read(
|
||||
root, "scripts/reboot-recovery/signoz-110-static-ingest.sh"
|
||||
)
|
||||
static_recovery = _read(root, "scripts/reboot-recovery/signoz-110-static-ingest.sh")
|
||||
checks["managed_promotion_fail_closed"] = all(
|
||||
value in static_recovery
|
||||
for value in (
|
||||
@@ -219,6 +226,453 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
)
|
||||
)
|
||||
|
||||
checks["post_closure_snapshot_path"] = snapshot_path == POST_CLOSURE_SNAPSHOT
|
||||
checks["post_closure_schema_v2"] = (
|
||||
regression.get("schema") == POST_CLOSURE_SCHEMA
|
||||
and post_closure.get("snapshot_schema") == POST_CLOSURE_SCHEMA
|
||||
and regression.get("work_item_id") == "P0-OBS-002"
|
||||
and regression.get("trace_id") == "P0-OBS-002"
|
||||
)
|
||||
|
||||
asset_reconciliation = regression.get("asset_reconciliation", {})
|
||||
source_registry = _load_yaml(root / "ops/config/service-registry.yaml")
|
||||
runtime_configmap = _load_yaml(
|
||||
root / "k8s/awoooi-prod/15-service-registry-configmap.yaml"
|
||||
)
|
||||
runtime_registry = yaml.safe_load(
|
||||
runtime_configmap.get("data", {}).get("service-registry.yaml", "")
|
||||
)
|
||||
source_services = {
|
||||
item["name"]: item for item in source_registry.get("services", [])
|
||||
}
|
||||
runtime_services = {
|
||||
item["name"]: item for item in runtime_registry.get("services", [])
|
||||
}
|
||||
shared_services = set(source_services) & set(runtime_services)
|
||||
exact_shared_services = {
|
||||
name
|
||||
for name in shared_services
|
||||
if source_services[name] == runtime_services[name]
|
||||
}
|
||||
source_only_services = sorted(set(source_services) - set(runtime_services))
|
||||
checks["service_registry_runtime_mirror_drift_explicit"] = (
|
||||
asset_reconciliation.get("drift_work_item_id") == "P0-OBS-002-ASSET-DRIFT-001"
|
||||
and asset_reconciliation.get("source_service_count")
|
||||
== len(source_services)
|
||||
== 27
|
||||
and asset_reconciliation.get("runtime_mirror_service_count")
|
||||
== len(runtime_services)
|
||||
== 24
|
||||
and asset_reconciliation.get("exact_shared_service_count")
|
||||
== len(exact_shared_services)
|
||||
== 24
|
||||
and asset_reconciliation.get("shared_service_drift_count") == 0
|
||||
and asset_reconciliation.get("source_only_services")
|
||||
== source_only_services
|
||||
== ["bitan-app", "sentry", "signoz"]
|
||||
and source_services.get("signoz-clickhouse")
|
||||
== runtime_services.get("signoz-clickhouse")
|
||||
and asset_reconciliation.get("signoz_clickhouse_exact_match") is True
|
||||
and asset_reconciliation.get("live_signoz_query_host") == "192.168.0.110"
|
||||
and asset_reconciliation.get("source_signoz_host") == "192.168.0.188"
|
||||
and asset_reconciliation.get("terminal") == "partial_degraded"
|
||||
)
|
||||
github_legacy_drift = asset_reconciliation.get(
|
||||
"github_freeze_legacy_asset_drift", {}
|
||||
)
|
||||
checks["github_freeze_legacy_asset_drift_explicit"] = (
|
||||
github_legacy_drift.get("drift_work_item_id") == "P0-OBS-002-ASSET-DRIFT-002"
|
||||
and github_legacy_drift.get("superseded_by") == "global_product_governance_v2"
|
||||
and github_legacy_drift.get("status") == "pending_controlled_retirement"
|
||||
and github_legacy_drift.get("active_github_use_in_this_run") is False
|
||||
and len(github_legacy_drift.get("source_references", [])) == 3
|
||||
and github_legacy_drift.get("replacement")
|
||||
== "gitea_runner_readiness_and_queue_receipts"
|
||||
and github_legacy_drift.get("p0_p1_github_recovery_scheduled") is False
|
||||
)
|
||||
duplicate_identity_drift = asset_reconciliation.get(
|
||||
"monitoring_generator_duplicate_identity_drift", {}
|
||||
)
|
||||
checks["monitoring_generator_duplicate_identity_drift_explicit"] = (
|
||||
duplicate_identity_drift.get("drift_work_item_id")
|
||||
== "P0-OBS-002-ASSET-DRIFT-003"
|
||||
and duplicate_identity_drift.get("status") == "pending_source_deduplication"
|
||||
and duplicate_identity_drift.get("registry_service_count") == 26
|
||||
and duplicate_identity_drift.get("awoooi_api_registry_entry_count") == 2
|
||||
and duplicate_identity_drift.get("generated_scrape_config_count") == 24
|
||||
and duplicate_identity_drift.get("generated_awoooi_api_job_count") == 2
|
||||
and duplicate_identity_drift.get("generated_blackbox_target_count") == 20
|
||||
and duplicate_identity_drift.get("runtime_apply_performed") is False
|
||||
)
|
||||
|
||||
cd = regression.get("release_receipts", {}).get("gitea_cd_5140", {})
|
||||
cd_probe = cd.get("cd_probe", {})
|
||||
checks["cd_5140_receipt_consistent"] = (
|
||||
cd.get("gitea_run_id") == 5140
|
||||
and cd.get("source_sha") == "e4da6570a645e504cc93647d396f45f167b3172a"
|
||||
and cd.get("workflow_terminal") == "success"
|
||||
and cd.get("unit_tests", {}).get("passed") == 4817
|
||||
and cd.get("integration_tests_passed") == 5
|
||||
and cd.get("production_readback", {}).get("build_tag_matched") is True
|
||||
and cd.get("production_readback", {}).get("desired_tag_matched") is True
|
||||
and cd_probe.get("receipt_provenance") == "gitea_run_5140_job_log"
|
||||
and cd_probe.get("passed") is True
|
||||
and cd_probe.get("workers") == 4
|
||||
and cd_probe.get("safety_reserve_connections") == 4
|
||||
and cd_probe.get("restart_free") is True
|
||||
)
|
||||
|
||||
applies = regression.get("controlled_applies", {})
|
||||
config_route = applies.get("config_route_patch", {})
|
||||
checks["config_route_patch_receipt_consistent"] = (
|
||||
config_route.get("run_id")
|
||||
== "P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1"
|
||||
and config_route.get("source_commit")
|
||||
== "9a9d1464a586ba172d1613db5bc285def43eb3e8"
|
||||
and config_route.get("included_in_release_sha") == cd.get("source_sha")
|
||||
and config_route.get("normalized_asset", {}).get("internal_url")
|
||||
== "http://192.168.0.110:8080"
|
||||
and config_route.get("source_truth_diff", {}).get("otlp_producer_route_changed")
|
||||
is False
|
||||
and config_route.get("execution_receipt", {}).get("deployment_status")
|
||||
== "deployed_verified"
|
||||
and config_route.get("terminal") == "applied_post_verifier_pass"
|
||||
)
|
||||
|
||||
prometheus = applies.get("prometheus_target_route", {})
|
||||
prometheus_check = prometheus.get("check_receipt", {})
|
||||
prometheus_execution = prometheus.get("execution_receipt", {})
|
||||
prometheus_post = prometheus.get("post_verifier", {})
|
||||
prometheus_independent = prometheus.get("independent_verifier", {})
|
||||
checks["prometheus_target_first_receipt_consistent"] = (
|
||||
prometheus.get("run_id")
|
||||
== "P0-OBS-002-prometheus-route-20260714T215106Z-apply1"
|
||||
and prometheus_check.get("promtool") == "pass"
|
||||
and prometheus_check.get("candidate_sha")
|
||||
== prometheus_execution.get("active_sha")
|
||||
== prometheus_execution.get("runtime_sha")
|
||||
and prometheus_execution.get("action") == "target_first_config_apply"
|
||||
and prometheus_post.get("terminal") == "target_up_two_scrapes"
|
||||
and prometheus_post.get("target_count") == 1
|
||||
and prometheus_post.get("target_health") == "up"
|
||||
and prometheus_post.get("probe_success") == 1
|
||||
and prometheus_post.get("legacy_target_count") == 0
|
||||
and prometheus_independent.get("active_sha_matches_runtime") is True
|
||||
and prometheus_independent.get("target_count") == 1
|
||||
and prometheus_independent.get("target_health") == "up"
|
||||
and prometheus_independent.get("legacy_target_count") == 0
|
||||
and prometheus_independent.get("terminal") == "pass"
|
||||
and prometheus.get("terminal") == "completed_target_first"
|
||||
)
|
||||
|
||||
alert_rules = applies.get("alert_rules", {})
|
||||
alert_check = alert_rules.get("check_receipt", {})
|
||||
alert_execution = alert_rules.get("execution_receipt", {})
|
||||
alert_post = alert_rules.get("post_verifier", {})
|
||||
checks["alert_rules_receipt_consistent"] = (
|
||||
alert_rules.get("run_id") == "P0-OBS-002-alert-rules-20260714T215328Z-apply1"
|
||||
and alert_check.get("promtool") == "pass"
|
||||
and alert_check.get("source_sha")
|
||||
== alert_execution.get("remote_sha")
|
||||
== alert_execution.get("canonical_sha")
|
||||
== alert_execution.get("runtime_sha")
|
||||
and alert_execution.get("hashes_match") is True
|
||||
and alert_post.get("window_seconds") == 130
|
||||
and alert_post.get("terminal") == "pass"
|
||||
and alert_post.get("signoz_rule_count") == 1
|
||||
and alert_post.get("signoz_rule_health") == "ok"
|
||||
and alert_rules.get("terminal") == "completed_verified"
|
||||
)
|
||||
|
||||
native_disk = applies.get("clickhouse_native_backup_disk", {})
|
||||
native_disk_attempts = native_disk.get("attempts", [])
|
||||
disk_apply1 = native_disk_attempts[0] if len(native_disk_attempts) == 2 else {}
|
||||
disk_retry2 = native_disk_attempts[1] if len(native_disk_attempts) == 2 else {}
|
||||
checks["clickhouse_native_backup_disk_receipt_consistent"] = (
|
||||
native_disk.get("source", {}).get("override_sha256")
|
||||
== "cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488"
|
||||
and native_disk.get("source", {}).get("config_sha256")
|
||||
== "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
|
||||
and native_disk.get("check_receipt", {}).get("terminal")
|
||||
== "check_pass_no_write"
|
||||
and disk_apply1.get("terminal") == "failed_rolled_back"
|
||||
and disk_apply1.get("exit_code") == 2
|
||||
and disk_apply1.get("rollback", {}).get("terminal") == "pass"
|
||||
and disk_apply1.get("rollback", {}).get("original_data_volume_preserved")
|
||||
is True
|
||||
and disk_retry2.get("run_id")
|
||||
== "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
|
||||
and disk_retry2.get("terminal") == "pass"
|
||||
and disk_retry2.get("execution", {}).get("image_digest_unchanged") is True
|
||||
and disk_retry2.get("execution", {}).get("data_volume_unchanged") is True
|
||||
and disk_retry2.get("post_verifier", {}).get("independent") is True
|
||||
and disk_retry2.get("post_verifier", {}).get("server_write_access") is True
|
||||
and disk_retry2.get("post_verifier", {}).get("dependent_restart_count_delta")
|
||||
== 0
|
||||
and disk_retry2.get("post_verifier", {}).get("api_http_status") == 200
|
||||
and disk_retry2.get("post_verifier", {}).get("active_backup_or_restore_count")
|
||||
== 0
|
||||
and native_disk.get("terminal") == "deployed_verified"
|
||||
)
|
||||
|
||||
native_canary = applies.get("clickhouse_native_backup_restore_canary", {})
|
||||
native_attempts = native_canary.get("attempts", [])
|
||||
native_apply2 = native_attempts[1] if len(native_attempts) == 4 else {}
|
||||
native_apply3 = native_attempts[2] if len(native_attempts) == 4 else {}
|
||||
native_apply4 = native_attempts[3] if len(native_attempts) == 4 else {}
|
||||
native_backup = native_apply4.get("backup", {})
|
||||
native_restore = native_apply4.get("isolated_restore", {})
|
||||
native_restic = native_apply4.get("restic", {})
|
||||
native_post = native_apply4.get("independent_post_verifier", {})
|
||||
toolchain_apply = native_canary.get("toolchain_controlled_apply", {})
|
||||
checks["native_backup_restore_canary_closed"] = (
|
||||
native_canary.get("completion_scope") == "clickhouse_native_only"
|
||||
and native_canary.get("source", {}).get("native_backup_sha256")
|
||||
== "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a"
|
||||
and native_canary.get("source", {}).get("isolated_restore_sha256")
|
||||
== "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7"
|
||||
and native_canary.get("source", {}).get("post_canary_focused_tests_passed")
|
||||
== 94
|
||||
and native_canary.get("source", {}).get("post_format_focused_tests_passed")
|
||||
== 94
|
||||
and native_canary.get("source", {}).get("ruff_format") == "pass"
|
||||
and toolchain_apply.get("apply_run_id")
|
||||
== "P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
|
||||
and toolchain_apply.get("apply_terminal") == "pass"
|
||||
and toolchain_apply.get("postcheck_terminal") == "check_pass_no_write"
|
||||
and toolchain_apply.get("postcheck_diff")
|
||||
== "matching_7_drift_0_absent_0_active_operations_0"
|
||||
and native_apply2.get("artifact_retention") == "preserved_failed_run_evidence"
|
||||
and native_apply2.get("cleanup_verified") is True
|
||||
and native_apply3.get("artifact_retention") == "preserved_failed_run_evidence"
|
||||
and native_apply3.get("cleanup_verified") is True
|
||||
and native_apply4.get("run_id")
|
||||
== "P0-OBS-002-native-canary-20260715T001329Z-apply4"
|
||||
and native_apply4.get("check_terminal") == "check_pass_no_write"
|
||||
and native_apply4.get("terminal") == "pass"
|
||||
and native_backup.get("terminal") == "BACKUP_CREATED"
|
||||
and native_backup.get("error") == ""
|
||||
and native_backup.get("artifact_sha256")
|
||||
== "4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc"
|
||||
and native_restore.get("terminal") == "verified"
|
||||
and native_restore.get("clickhouse_restore_terminal") == "RESTORED"
|
||||
and native_restore.get("production_network_attached") is False
|
||||
and native_restore.get("production_restore_performed") is False
|
||||
and native_restore.get("source_staged_clickhouse_artifact_sha_match") is True
|
||||
and native_restore.get("database_count")
|
||||
== native_restore.get("restored_database_count")
|
||||
== 6
|
||||
and native_restore.get("table_count")
|
||||
== native_restore.get("restored_table_count")
|
||||
== 100
|
||||
and native_restore.get("table_engine_schema_manifest_parity") is True
|
||||
and native_restore.get("check_table_count")
|
||||
== native_restore.get("check_table_pass_count")
|
||||
== 44
|
||||
and native_restore.get("critical_nonzero_count") == 4
|
||||
and native_restore.get("aggregate_counter_exact_parity_gate") is False
|
||||
and native_restic.get("snapshot_id") == "574e2a1a"
|
||||
and native_restic.get("snapshot_readback") == "pass"
|
||||
and native_restic.get("repository_scope") == "local_same_failure_domain"
|
||||
and native_restic.get("offsite_verified") is False
|
||||
and native_post.get("production_restart_count_delta") == 0
|
||||
and native_post.get("api_http_status") == 200
|
||||
and native_post.get("new_server_artifact_removed") is True
|
||||
and native_post.get("apply2_apply3_preserved_artifact_hashes_unchanged") is True
|
||||
and native_post.get(
|
||||
"exact_ephemeral_container_volume_network_process_tmp_residue_count"
|
||||
)
|
||||
== 0
|
||||
and native_post.get("cleanup_verified") is True
|
||||
and native_post.get("secret_values_collected") is False
|
||||
and native_canary.get("terminal") == "production_closed_clickhouse_native_scope"
|
||||
)
|
||||
|
||||
backup = applies.get("backup_canary", {})
|
||||
backup_attempts = backup.get("attempts", [])
|
||||
backup_attempt = backup_attempts[0] if len(backup_attempts) == 2 else {}
|
||||
retry2 = backup_attempts[1] if len(backup_attempts) == 2 else {}
|
||||
backup_rollback = backup_attempt.get("rollback", {})
|
||||
retry2_rollback = retry2.get("rollback", {})
|
||||
checks["backup_canary_failure_and_rollback_consistent"] = (
|
||||
backup.get("guard_source", {}).get("deployment_status") == "deployed_verified"
|
||||
and backup_attempt.get("run_id")
|
||||
== "P0-OBS-002-backup-canary-20260714T213210Z-retry1"
|
||||
and backup_attempt.get("terminal") == "failed_rolled_back"
|
||||
and backup_attempt.get("classification")
|
||||
== "failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention"
|
||||
and backup_attempt.get("archive_container_exit_code") == 137
|
||||
and backup_attempt.get("root_cause")
|
||||
== "docker_health_monitor_restarted_collector_during_intentional_backup_stop"
|
||||
and backup_attempt.get("clickhouse_archive_committed") is False
|
||||
and backup_attempt.get("sqlite_stage_reached") is False
|
||||
and backup_attempt.get("restic_stage_reached") is False
|
||||
and backup_attempt.get("retention_stage_reached") is False
|
||||
and backup_attempt.get("success_stage_reached") is False
|
||||
and backup_rollback.get("collector_running") is True
|
||||
and backup_rollback.get("collector_restart_count") == 0
|
||||
and backup_rollback.get("grpc_4317_listening") is True
|
||||
and backup_rollback.get("http_4318_listening") is True
|
||||
and backup_rollback.get("current_temp_dir_absent") is True
|
||||
and backup_rollback.get("partial_artifacts_absent") is True
|
||||
and retry2.get("run_id") == "P0-OBS-002-backup-canary-20260714T221230Z-retry2"
|
||||
and retry2.get("terminal") == "failed_rolled_back"
|
||||
and retry2.get("backup_script_exit_code") == 1
|
||||
and retry2.get("archive_container_exit_code") == "unknown_not_durably_captured"
|
||||
and retry2.get("root_cause")
|
||||
== "raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract"
|
||||
and retry2.get("exact_tar_error_proven") is False
|
||||
and retry2.get("clickhouse_archive_committed") is False
|
||||
and retry2.get("sqlite_stage_reached") is False
|
||||
and retry2.get("restic_stage_reached") is False
|
||||
and retry2.get("retention_stage_reached") is False
|
||||
and retry2.get("success_stage_reached") is False
|
||||
and retry2.get("monitor_window", {}).get("auto_repair_count") == 0
|
||||
and retry2_rollback.get("cooldown_lease_restored") is True
|
||||
and retry2_rollback.get("collector_running") is True
|
||||
and retry2_rollback.get("collector_restart_count") == 0
|
||||
and retry2_rollback.get("grpc_4317_listening") is True
|
||||
and retry2_rollback.get("http_4318_listening") is True
|
||||
and retry2_rollback.get("current_temp_dir_absent") is True
|
||||
and retry2_rollback.get("partial_artifacts_absent") is True
|
||||
)
|
||||
|
||||
post_release = regression.get("post_release_runtime_verifier", {})
|
||||
checks["post_release_600s_verifier_consistent"] = (
|
||||
post_release.get("run_id") == "P0-OBS-002-post-release-20260714T215500Z"
|
||||
and post_release.get("window_seconds") == 600
|
||||
and post_release.get("terminal") == "pass"
|
||||
and post_release.get("restart_delta") == 0
|
||||
and post_release.get("exporter_trace_errors") == 0
|
||||
and post_release.get("exporter_metric_errors") == 0
|
||||
and post_release.get("trace_count") == 4026
|
||||
and post_release.get("metric_sample_count") == 76369
|
||||
and post_release.get("api_runtime_cooldown_closed") is True
|
||||
and regression.get("regressions", {})
|
||||
.get("api_runtime", {})
|
||||
.get("last_observed_cooldown_closed")
|
||||
is True
|
||||
)
|
||||
|
||||
native_migration = regression.get("completed_verifiers", {}).get(
|
||||
"clickhouse_native_backup_migration", {}
|
||||
)
|
||||
pending_verifiers = regression.get("pending_verifiers", {})
|
||||
sqlite_backup = pending_verifiers.get(
|
||||
"signoz_sqlite_application_consistent_backup", {}
|
||||
)
|
||||
offsite_backup = pending_verifiers.get("signoz_backup_offsite_dr", {})
|
||||
failed_artifacts = pending_verifiers.get(
|
||||
"clickhouse_native_failed_artifact_retention", {}
|
||||
)
|
||||
resume_inventory = pending_verifiers.get(
|
||||
"clickhouse_native_resume_submitted_inventory", {}
|
||||
)
|
||||
checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = (
|
||||
native_migration.get("prior_run_id") == retry2.get("run_id")
|
||||
and native_migration.get("run_id") == native_apply4.get("run_id")
|
||||
and native_migration.get("status")
|
||||
== "production_closed_clickhouse_native_scope"
|
||||
and native_migration.get("native_backup_terminal") == "BACKUP_CREATED"
|
||||
and native_migration.get("isolated_restore_terminal") == "verified"
|
||||
and native_migration.get("restic_snapshot_id") == "574e2a1a"
|
||||
and native_migration.get("independent_verifier") == "pass"
|
||||
and native_migration.get("exact_cleanup") == "pass"
|
||||
and sqlite_backup.get("status")
|
||||
== "pending_supported_non_raw_export_or_coordinated_snapshot"
|
||||
and sqlite_backup.get("raw_sqlite_read_or_sync_allowed") is False
|
||||
and sqlite_backup.get("sqlite_cli_present_in_runtime") is False
|
||||
and offsite_backup.get("status")
|
||||
== "pending_offsite_snapshot_and_restore_verifier"
|
||||
and offsite_backup.get("current_repository_scope")
|
||||
== "local_same_failure_domain"
|
||||
and offsite_backup.get("offsite_verified") is False
|
||||
and failed_artifacts.get("status") == "pending_bounded_retention_decision"
|
||||
and failed_artifacts.get("destructive_cleanup_authorized") is False
|
||||
and failed_artifacts.get("active_backup_or_restore_count") == 0
|
||||
and len(failed_artifacts.get("preserved_artifacts", [])) == 2
|
||||
and resume_inventory.get("status")
|
||||
== "pending_durable_submitted_inventory_contract"
|
||||
and resume_inventory.get("same_run_resume_safe") is False
|
||||
and len(resume_inventory.get("required_preconditions", [])) == 4
|
||||
)
|
||||
|
||||
checks["post_closure_contract_mirror_consistent"] = (
|
||||
post_closure.get("gitea_cd_run_id") == cd.get("gitea_run_id")
|
||||
and post_closure.get("release_source_sha") == cd.get("source_sha")
|
||||
and post_closure.get("api_runtime_cooldown_closed") is True
|
||||
and post_closure.get("cd_probe_safety_fix_deployed") is True
|
||||
and post_closure.get("config_route_patch_deployed") is True
|
||||
and post_closure.get("signoz_control_plane_health_route_deployed") is True
|
||||
and post_closure.get("backup_collector_restore_guard_deployed") is True
|
||||
and post_closure.get("backup_canary_scope") == "clickhouse_native_only"
|
||||
and post_closure.get("backup_canary_verified") is True
|
||||
and post_closure.get("backup_canary_last_run_id") == native_apply4.get("run_id")
|
||||
and post_closure.get("backup_canary_last_terminal") == "pass"
|
||||
and post_closure.get("backup_canary_cooldown_retry_status")
|
||||
== "superseded_by_online_native_no_stop_contract"
|
||||
and post_closure.get("clickhouse_native_backup_migration_status")
|
||||
== "production_closed"
|
||||
and post_closure.get("clickhouse_native_backup_disk_run_id")
|
||||
== disk_retry2.get("run_id")
|
||||
and post_closure.get("clickhouse_native_backup_disk_status")
|
||||
== "deployed_verified"
|
||||
and post_closure.get("clickhouse_native_backup_toolchain_run_id")
|
||||
== toolchain_apply.get("apply_run_id")
|
||||
and post_closure.get("clickhouse_native_backup_toolchain_postcheck_run_id")
|
||||
== toolchain_apply.get("postcheck_run_id")
|
||||
and post_closure.get("clickhouse_native_restore_terminal") == "verified"
|
||||
and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a"
|
||||
and post_closure.get("clickhouse_native_offsite_status") == "pending"
|
||||
and post_closure.get("clickhouse_native_failed_artifact_retention_status")
|
||||
== "pending_bounded_retention_decision"
|
||||
and post_closure.get("clickhouse_native_resume_inventory_status")
|
||||
== "pending_durable_submitted_inventory_contract"
|
||||
and post_closure.get("signoz_sqlite_application_consistent_backup_status")
|
||||
== "pending"
|
||||
and post_closure.get("service_registry_runtime_mirror_status")
|
||||
== "partial_degraded"
|
||||
and post_closure.get("service_registry_runtime_mirror_work_item_id")
|
||||
== "P0-OBS-002-ASSET-DRIFT-001"
|
||||
and post_closure.get("github_freeze_legacy_asset_status")
|
||||
== "pending_controlled_retirement"
|
||||
and post_closure.get("github_freeze_legacy_asset_work_item_id")
|
||||
== "P0-OBS-002-ASSET-DRIFT-002"
|
||||
and post_closure.get("monitoring_generator_duplicate_identity_status")
|
||||
== "pending_source_deduplication"
|
||||
and post_closure.get("monitoring_generator_duplicate_identity_work_item_id")
|
||||
== "P0-OBS-002-ASSET-DRIFT-003"
|
||||
and post_closure.get("post_release_otlp_grpc_600s_verifier_status") == "pass"
|
||||
)
|
||||
|
||||
regression_terminal = regression.get("terminal", {})
|
||||
checks["post_closure_terminal_blocker_consistent"] = (
|
||||
regression_terminal.get("status") == "partial_degraded"
|
||||
and list(regression_terminal.get("blockers", []))
|
||||
== EXPECTED_POST_CLOSURE_BLOCKERS
|
||||
)
|
||||
|
||||
regression_text = _read(root, POST_CLOSURE_SNAPSHOT).lower()
|
||||
checks["post_closure_no_secret_or_github_dependency"] = (
|
||||
regression.get("scope_boundaries", {}).get("secret_read") is False
|
||||
and regression.get("scope_boundaries", {}).get("github_used") is False
|
||||
and all(
|
||||
forbidden not in regression_text
|
||||
for forbidden in (
|
||||
"github.com",
|
||||
"api.github.com",
|
||||
"raw.githubusercontent.com",
|
||||
"codeload.github.com",
|
||||
"ghcr.io",
|
||||
"authorization:",
|
||||
"password:",
|
||||
"private_key:",
|
||||
"cookie:",
|
||||
"session:",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
terminal = contract.get("terminal", {})
|
||||
blockers = list(terminal.get("blockers", []))
|
||||
checks["expected_blockers_explicit"] = blockers == EXPECTED_BLOCKERS
|
||||
@@ -240,7 +694,9 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2])
|
||||
parser.add_argument(
|
||||
"--root", type=Path, default=Path(__file__).resolve().parents[2]
|
||||
)
|
||||
parser.add_argument(
|
||||
"--contract",
|
||||
type=Path,
|
||||
|
||||
@@ -27,7 +27,9 @@ POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.
|
||||
|
||||
def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
|
||||
payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
||||
assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
|
||||
assert (
|
||||
payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
|
||||
)
|
||||
assert payload["work_item_id"] == "P0-OBS-002"
|
||||
assert payload["phase"] == "wave_a_dual_allow"
|
||||
assert payload["routes"]["active_route"] == "incumbent"
|
||||
@@ -43,14 +45,33 @@ def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() ->
|
||||
)
|
||||
|
||||
|
||||
def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None:
|
||||
text = CONTRACT.read_text(encoding="utf-8").lower()
|
||||
assert "signoz_user_root_password:" not in text
|
||||
assert "signoz-api-key:" not in text
|
||||
assert "authorization:" not in text
|
||||
assert "raw_relational_database_update" in text
|
||||
assert "stateful_volume_restore" in text
|
||||
assert "apply_allowed_in_wave_a: false" in text
|
||||
def test_contract_and_receipt_contain_no_secret_values_or_github_dependency() -> None:
|
||||
contract_text = CONTRACT.read_text(encoding="utf-8").lower()
|
||||
regression_text = POST_CLOSURE_REGRESSION.read_text(encoding="utf-8").lower()
|
||||
forbidden = (
|
||||
"signoz_user_root_password:",
|
||||
"signoz-api-key:",
|
||||
"authorization:",
|
||||
"password:",
|
||||
"private_key:",
|
||||
"cookie:",
|
||||
"session:",
|
||||
"github.com",
|
||||
"api.github.com",
|
||||
"raw.githubusercontent.com",
|
||||
"codeload.github.com",
|
||||
"ghcr.io",
|
||||
)
|
||||
for text in (contract_text, regression_text):
|
||||
assert all(value not in text for value in forbidden)
|
||||
|
||||
assert "raw_relational_database_update" in contract_text
|
||||
assert "stateful_volume_restore" in contract_text
|
||||
assert "apply_allowed_in_wave_a: false" in contract_text
|
||||
|
||||
regression = yaml.safe_load(regression_text)
|
||||
assert regression["scope_boundaries"]["secret_read"] is False
|
||||
assert regression["scope_boundaries"]["github_used"] is False
|
||||
|
||||
|
||||
def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
||||
@@ -69,9 +90,31 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
||||
assert payload["checks"] and all(payload["checks"].values())
|
||||
assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True
|
||||
assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True
|
||||
assert payload["checks"]["post_closure_schema_v2"] is True
|
||||
assert payload["checks"]["post_release_600s_verifier_consistent"] is True
|
||||
assert (
|
||||
payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"]
|
||||
is True
|
||||
)
|
||||
assert payload["checks"]["service_registry_runtime_mirror_drift_explicit"]
|
||||
assert payload["checks"]["github_freeze_legacy_asset_drift_explicit"]
|
||||
assert payload["checks"]["monitoring_generator_duplicate_identity_drift_explicit"]
|
||||
assert "organization_not_initialized" in payload["blockers"]
|
||||
assert "signoz_control_plane_health_route_not_deployed" in payload["blockers"]
|
||||
assert "wave_b_not_authorized" in payload["blockers"]
|
||||
assert payload["blockers"] == [
|
||||
"organization_not_initialized",
|
||||
"direct_ingress_policy_not_closed",
|
||||
"http_bridge_not_supervised",
|
||||
"filelog_normalization_degraded",
|
||||
"wave_b_not_authorized",
|
||||
]
|
||||
for closed_blocker in (
|
||||
"api_runtime_cooldown_pending",
|
||||
"cd_probe_safety_fix_not_deployed",
|
||||
"backup_collector_restore_guard_not_deployed",
|
||||
"signoz_control_plane_health_route_not_deployed",
|
||||
):
|
||||
assert closed_blocker not in payload["blockers"]
|
||||
|
||||
|
||||
def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
|
||||
@@ -129,7 +172,10 @@ def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion()
|
||||
assert snapshot["trace_id"] == parity["trace_id"]
|
||||
assert snapshot["route_parity_terminal"] == "pass"
|
||||
assert snapshot["program_terminal"] == "partial_degraded"
|
||||
assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial"
|
||||
assert (
|
||||
snapshot["controlled_apply_contract"]["learning_writeback"]["status"]
|
||||
== "partial"
|
||||
)
|
||||
assert (
|
||||
snapshot["controlled_apply_contract"]["learning_writeback"][
|
||||
"durable_gitea_feature_acknowledgement"
|
||||
@@ -151,13 +197,13 @@ def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() ->
|
||||
text = OTEL_DAEMONSET.read_text(encoding="utf-8")
|
||||
assert "id: cri_parser" in text
|
||||
assert "on_error: send_quiet" not in text
|
||||
assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text
|
||||
assert "if: 'body matches \"^[^ ]+ (stdout|stderr) [^ ]* .*$\"'" in text
|
||||
assert "id: mark_cri_parsed" in text
|
||||
assert 'value: parsed_cri' in text
|
||||
assert "value: parsed_cri" in text
|
||||
assert "id: move_cri_body" in text
|
||||
assert "if: attributes.log != nil" in text
|
||||
assert "id: mark_cri_fallback" in text
|
||||
assert 'value: fallback_unparsed' in text
|
||||
assert "value: fallback_unparsed" in text
|
||||
assert 'if: attributes["awoooi.log.parse_status"] == nil' in text
|
||||
|
||||
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
||||
@@ -231,11 +277,10 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
|
||||
apply_block = bridge.split(" --apply)", 1)[1].split(" --status)", 1)[0]
|
||||
|
||||
learning = (
|
||||
'receipt learning partial '
|
||||
'"application_trace_metric_10m_post_verifier_pending"'
|
||||
'receipt learning partial "application_trace_metric_10m_post_verifier_pending"'
|
||||
)
|
||||
terminal = (
|
||||
'receipt terminal partial '
|
||||
"receipt terminal partial "
|
||||
'"grpc_transport_restored_independent_600s_post_verifier_pending"'
|
||||
)
|
||||
assert learning in apply_block
|
||||
@@ -243,9 +288,8 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
|
||||
assert apply_block.index(learning) < apply_block.index(terminal)
|
||||
assert "receipt terminal pass" not in apply_block
|
||||
assert (
|
||||
'receipt terminal pass '
|
||||
'"real_application_otlp_grpc_trace_metric_freshness_closed"'
|
||||
in verifier
|
||||
"receipt terminal pass "
|
||||
'"real_application_otlp_grpc_trace_metric_freshness_closed"' in verifier
|
||||
)
|
||||
|
||||
|
||||
@@ -266,7 +310,7 @@ def test_contract_grpc_route_state_matches_verified_runtime_evidence() -> None:
|
||||
assert grpc["source_fix"]["runtime_verifier_window_seconds"] >= 600
|
||||
|
||||
|
||||
def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None:
|
||||
def test_grpc_runtime_verifier_uses_real_aggregates_and_ten_minute_gate() -> None:
|
||||
text = GRPC_VERIFIER.read_text(encoding="utf-8")
|
||||
for expected in (
|
||||
'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
|
||||
@@ -293,10 +337,7 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
|
||||
assert grpc["recent_awoooi_api_metric_sample_count"] == 60180
|
||||
assert grpc["http_only_verifier_blind_spot"] is True
|
||||
assert grpc["source_fix"]["check_terminal"] == "pass"
|
||||
assert (
|
||||
grpc["source_fix"]["runtime_apply_status"]
|
||||
== "applied_post_verifier_pass"
|
||||
)
|
||||
assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass"
|
||||
assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"
|
||||
|
||||
receipt = yaml.safe_load(GRPC_RUNTIME_RECEIPT.read_text(encoding="utf-8"))
|
||||
@@ -312,13 +353,431 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
|
||||
|
||||
regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
assert regression["baseline"]["verifier_terminal"] == "pass"
|
||||
assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False
|
||||
assert (
|
||||
regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"]
|
||||
is True
|
||||
)
|
||||
assert regression["controlled_recovery"]["stateful_services_touched"] == 0
|
||||
assert regression["controlled_recovery"]["grpc_4317_listening"] is True
|
||||
assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
|
||||
assert regression["terminal"]["status"] == "partial_degraded"
|
||||
|
||||
|
||||
def test_post_closure_v2_records_cd_5140_and_config_route_patch() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
assert receipt["schema"] == "awoooi_signoz_post_closure_regression_v2"
|
||||
|
||||
cd = receipt["release_receipts"]["gitea_cd_5140"]
|
||||
assert cd["gitea_run_id"] == 5140
|
||||
assert cd["source_sha"] == "e4da6570a645e504cc93647d396f45f167b3172a"
|
||||
assert cd["workflow_terminal"] == "success"
|
||||
assert cd["focused_tests_passed"] == 6
|
||||
assert cd["unit_tests"] == {"passed": 4817, "skipped": 23, "warnings": 69}
|
||||
assert cd["integration_tests_passed"] == 5
|
||||
assert cd["production_readback"]["build_tag_matched"] is True
|
||||
assert cd["production_readback"]["desired_tag_matched"] is True
|
||||
assert cd["production_readback"]["rollout_terminal"] == "success"
|
||||
assert cd["postdeploy"]["alert_chain_terminal"] == "pass"
|
||||
assert cd["postdeploy"]["playwright_passed"] == 5
|
||||
assert cd["cd_probe"]["receipt_provenance"] == "gitea_run_5140_job_log"
|
||||
assert cd["cd_probe"]["restart_free"] is True
|
||||
assert cd["cd_probe"]["workers"] == 4
|
||||
assert cd["cd_probe"]["safety_reserve_connections"] == 4
|
||||
|
||||
route = receipt["controlled_applies"]["config_route_patch"]
|
||||
assert route["run_id"] == ("P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1")
|
||||
assert route["source_commit"] == "9a9d1464a586ba172d1613db5bc285def43eb3e8"
|
||||
assert route["included_in_release_sha"] == cd["source_sha"]
|
||||
assert route["normalized_asset"]["internal_url"] == "http://192.168.0.110:8080"
|
||||
assert route["normalized_asset"]["public_url"] == "https://signoz.wooo.work"
|
||||
assert route["source_truth_diff"]["otlp_producer_route_changed"] is False
|
||||
assert route["execution_receipt"]["deployment_status"] == "deployed_verified"
|
||||
assert route["terminal"] == "applied_post_verifier_pass"
|
||||
|
||||
|
||||
def test_prometheus_target_first_receipt_has_independent_verifier() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
route = receipt["controlled_applies"]["prometheus_target_route"]
|
||||
check = route["check_receipt"]
|
||||
execution = route["execution_receipt"]
|
||||
post = route["post_verifier"]
|
||||
independent = route["independent_verifier"]
|
||||
|
||||
assert route["run_id"] == ("P0-OBS-002-prometheus-route-20260714T215106Z-apply1")
|
||||
assert check["promtool"] == "pass"
|
||||
assert check["candidate_sha"] == execution["active_sha"] == execution["runtime_sha"]
|
||||
assert execution["action"] == "target_first_config_apply"
|
||||
assert execution["backup_sha"] == check["source_sha"]
|
||||
assert post["terminal"] == "target_up_two_scrapes"
|
||||
assert post["first_scrape_at"] == "2026-07-15T05:52:38.146935853+08:00"
|
||||
assert post["second_scrape_at"] == "2026-07-15T05:52:53.146935853+08:00"
|
||||
assert post["first_scrape_at"] < post["second_scrape_at"]
|
||||
assert post["target_count"] == 1
|
||||
assert post["target_health"] == "up"
|
||||
assert post["probe_success"] == 1
|
||||
assert post["legacy_target_count"] == 0
|
||||
assert independent["mode"] == "independent_readback"
|
||||
assert independent["active_sha_matches_runtime"] is True
|
||||
assert independent["target_count"] == 1
|
||||
assert independent["target_health"] == "up"
|
||||
assert independent["probe_success"] == 1
|
||||
assert independent["legacy_target_count"] == 0
|
||||
assert independent["terminal"] == "pass"
|
||||
assert route["rollback"] == {"required": False, "available": True}
|
||||
assert route["terminal"] == "completed_target_first"
|
||||
|
||||
|
||||
def test_alert_rules_receipt_hashes_and_runtime_rule_match() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
alert = receipt["controlled_applies"]["alert_rules"]
|
||||
source_sha = alert["check_receipt"]["source_sha"]
|
||||
execution = alert["execution_receipt"]
|
||||
verifier = alert["post_verifier"]
|
||||
|
||||
assert alert["run_id"] == "P0-OBS-002-alert-rules-20260714T215328Z-apply1"
|
||||
assert alert["check_receipt"]["promtool"] == "pass"
|
||||
assert source_sha == execution["remote_sha"]
|
||||
assert source_sha == execution["canonical_sha"]
|
||||
assert source_sha == execution["runtime_sha"]
|
||||
assert execution["hashes_match"] is True
|
||||
assert execution["prometheus_ready"] is True
|
||||
assert verifier["window_seconds"] == 130
|
||||
assert verifier["terminal"] == "pass"
|
||||
assert verifier["signoz_rule_count"] == 1
|
||||
assert verifier["signoz_rule_health"] == "ok"
|
||||
assert verifier["signoz_rule_state"] == "inactive"
|
||||
assert verifier["exact_query"] == (
|
||||
'(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
|
||||
'job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
|
||||
'job="blackbox-http"})'
|
||||
)
|
||||
assert alert["terminal"] == "completed_verified"
|
||||
|
||||
|
||||
def test_failed_backup_canary_preserves_failure_and_verified_rollback() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
backup = receipt["controlled_applies"]["backup_canary"]
|
||||
assert backup["guard_source"]["deployment_status"] == "deployed_verified"
|
||||
assert len(backup["attempts"]) == 2
|
||||
|
||||
attempt = backup["attempts"][0]
|
||||
assert attempt["run_id"] == ("P0-OBS-002-backup-canary-20260714T213210Z-retry1")
|
||||
assert attempt["terminal"] == "failed_rolled_back"
|
||||
assert attempt["classification"] == (
|
||||
"failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention"
|
||||
)
|
||||
assert attempt["failed_stage"] == "clickhouse_archive_create"
|
||||
assert attempt["archive_container_exit_code"] == 137
|
||||
assert attempt["root_cause"] == (
|
||||
"docker_health_monitor_restarted_collector_during_intentional_backup_stop"
|
||||
)
|
||||
for false_field in (
|
||||
"consistent_archive_valid",
|
||||
"clickhouse_archive_committed",
|
||||
"sqlite_stage_reached",
|
||||
"restic_stage_reached",
|
||||
"retention_stage_reached",
|
||||
"success_stage_reached",
|
||||
):
|
||||
assert attempt[false_field] is False
|
||||
|
||||
rollback = attempt["rollback"]
|
||||
assert rollback["collector_running"] is True
|
||||
assert rollback["collector_restart_count"] == 0
|
||||
assert rollback["grpc_4317_listening"] is True
|
||||
assert rollback["http_4318_listening"] is True
|
||||
assert rollback["canary_processes_absent"] is True
|
||||
assert rollback["canary_container_absent"] is True
|
||||
assert rollback["current_temp_dir_absent"] is True
|
||||
assert rollback["partial_artifacts_absent"] is True
|
||||
assert not attempt["terminal"].startswith("completed")
|
||||
assert "learning_recorded" not in attempt
|
||||
|
||||
retry2 = backup["attempts"][1]
|
||||
assert retry2["run_id"] == ("P0-OBS-002-backup-canary-20260714T221230Z-retry2")
|
||||
assert retry2["terminal"] == "failed_rolled_back"
|
||||
assert retry2["backup_script_exit_code"] == 1
|
||||
assert retry2["archive_container_exit_code"] == "unknown_not_durably_captured"
|
||||
assert retry2["archive_stderr_receipt"] == "suppressed_by_legacy_pipeline"
|
||||
assert retry2["root_cause"] == (
|
||||
"raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract"
|
||||
)
|
||||
assert retry2["exact_tar_error_proven"] is False
|
||||
assert retry2["monitor_window"]["cron_intervals_crossed"] == 2
|
||||
assert retry2["monitor_window"]["auto_repair_count"] == 0
|
||||
for false_field in (
|
||||
"consistent_archive_valid",
|
||||
"clickhouse_archive_committed",
|
||||
"sqlite_stage_reached",
|
||||
"restic_stage_reached",
|
||||
"retention_stage_reached",
|
||||
"success_stage_reached",
|
||||
):
|
||||
assert retry2[false_field] is False
|
||||
retry2_rollback = retry2["rollback"]
|
||||
assert retry2_rollback["cooldown_lease_restored"] is True
|
||||
assert retry2_rollback["collector_running"] is True
|
||||
assert retry2_rollback["collector_restart_count"] == 0
|
||||
assert retry2_rollback["grpc_4317_listening"] is True
|
||||
assert retry2_rollback["http_4318_listening"] is True
|
||||
assert retry2_rollback["current_temp_dir_absent"] is True
|
||||
assert retry2_rollback["partial_artifacts_absent"] is True
|
||||
assert retry2["residual_cleanup_debt"] == []
|
||||
|
||||
|
||||
def test_clickhouse_native_backup_disk_preserves_failed_attempt_and_closes_retry() -> (
|
||||
None
|
||||
):
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
native_disk = receipt["controlled_applies"]["clickhouse_native_backup_disk"]
|
||||
|
||||
assert native_disk["source"]["override_sha256"] == (
|
||||
"cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488"
|
||||
)
|
||||
assert native_disk["source"]["config_sha256"] == (
|
||||
"c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
|
||||
)
|
||||
assert native_disk["check_receipt"]["terminal"] == "check_pass_no_write"
|
||||
assert len(native_disk["attempts"]) == 2
|
||||
|
||||
apply1, retry2 = native_disk["attempts"]
|
||||
assert apply1["terminal"] == "failed_rolled_back"
|
||||
assert apply1["exit_code"] == 2
|
||||
assert apply1["rollback"]["terminal"] == "pass"
|
||||
assert apply1["rollback"]["original_data_volume_preserved"] is True
|
||||
assert apply1["rollback"]["residue_count"] == 0
|
||||
|
||||
assert retry2["run_id"] == (
|
||||
"P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
|
||||
)
|
||||
assert retry2["terminal"] == "pass"
|
||||
assert retry2["execution"]["image_digest_unchanged"] is True
|
||||
assert retry2["execution"]["data_volume_unchanged"] is True
|
||||
assert retry2["post_verifier"]["independent"] is True
|
||||
assert retry2["post_verifier"]["server_write_access"] is True
|
||||
assert retry2["post_verifier"]["dependent_restart_count_delta"] == 0
|
||||
assert retry2["post_verifier"]["api_http_status"] == 200
|
||||
assert retry2["post_verifier"]["active_backup_or_restore_count"] == 0
|
||||
assert retry2["post_verifier"]["stage_candidate_process_residue_count"] == 0
|
||||
assert native_disk["terminal"] == "deployed_verified"
|
||||
|
||||
|
||||
def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
|
||||
assert canary["completion_scope"] == "clickhouse_native_only"
|
||||
assert canary["source"]["post_canary_focused_tests_passed"] == 94
|
||||
assert canary["source"]["post_format_focused_tests_passed"] == 94
|
||||
assert canary["source"]["ruff_format"] == "pass"
|
||||
assert canary["toolchain_controlled_apply"]["apply_terminal"] == "pass"
|
||||
assert canary["toolchain_controlled_apply"]["postcheck_diff"] == (
|
||||
"matching_7_drift_0_absent_0_active_operations_0"
|
||||
)
|
||||
assert len(canary["attempts"]) == 4
|
||||
_, apply2, apply3, apply4 = canary["attempts"]
|
||||
assert apply2["artifact_retention"] == "preserved_failed_run_evidence"
|
||||
assert apply2["cleanup_verified"] is True
|
||||
assert apply3["artifact_retention"] == "preserved_failed_run_evidence"
|
||||
assert apply3["cleanup_verified"] is True
|
||||
|
||||
assert apply4["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
|
||||
assert apply4["check_terminal"] == "check_pass_no_write"
|
||||
assert apply4["terminal"] == "pass"
|
||||
assert apply4["backup"]["terminal"] == "BACKUP_CREATED"
|
||||
assert apply4["backup"]["error"] == ""
|
||||
assert apply4["backup"]["artifact_sha256"] == (
|
||||
"4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc"
|
||||
)
|
||||
|
||||
restore = apply4["isolated_restore"]
|
||||
assert restore["terminal"] == "verified"
|
||||
assert restore["clickhouse_restore_terminal"] == "RESTORED"
|
||||
assert restore["production_network_attached"] is False
|
||||
assert restore["production_restore_performed"] is False
|
||||
assert restore["source_staged_clickhouse_artifact_sha_match"] is True
|
||||
assert restore["database_count"] == restore["restored_database_count"] == 6
|
||||
assert restore["table_count"] == restore["restored_table_count"] == 100
|
||||
assert restore["table_engine_schema_manifest_parity"] is True
|
||||
assert restore["check_table_count"] == restore["check_table_pass_count"] == 44
|
||||
assert restore["critical_nonzero_count"] == 4
|
||||
assert restore["aggregate_counter_exact_parity_gate"] is False
|
||||
|
||||
restic = apply4["restic"]
|
||||
assert restic["snapshot_id"] == "574e2a1a"
|
||||
assert restic["snapshot_readback"] == "pass"
|
||||
assert restic["repository_scope"] == "local_same_failure_domain"
|
||||
assert restic["offsite_verified"] is False
|
||||
|
||||
post = apply4["independent_post_verifier"]
|
||||
assert post["production_restart_count_delta"] == 0
|
||||
assert post["api_http_status"] == 200
|
||||
assert post["new_server_artifact_removed"] is True
|
||||
assert post["apply2_apply3_preserved_artifact_hashes_unchanged"] is True
|
||||
assert (
|
||||
post["exact_ephemeral_container_volume_network_process_tmp_residue_count"] == 0
|
||||
)
|
||||
assert post["cleanup_verified"] is True
|
||||
assert post["secret_values_collected"] is False
|
||||
assert canary["terminal"] == "production_closed_clickhouse_native_scope"
|
||||
|
||||
|
||||
def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
verifier = receipt["post_release_runtime_verifier"]
|
||||
assert verifier["run_id"] == "P0-OBS-002-post-release-20260714T215500Z"
|
||||
assert verifier["window_seconds"] == 600
|
||||
assert verifier["terminal"] == "pass"
|
||||
assert verifier["restart_delta"] == 0
|
||||
assert verifier["exporter_trace_errors"] == 0
|
||||
assert verifier["exporter_metric_errors"] == 0
|
||||
assert verifier["trace_count"] == 4026
|
||||
assert verifier["metric_sample_count"] == 76369
|
||||
assert verifier["api_runtime_cooldown_closed"] is True
|
||||
assert (
|
||||
receipt["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is True
|
||||
)
|
||||
|
||||
closed = receipt["completed_verifiers"]["clickhouse_native_backup_migration"]
|
||||
assert closed["prior_run_id"] == (
|
||||
"P0-OBS-002-backup-canary-20260714T221230Z-retry2"
|
||||
)
|
||||
assert closed["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
|
||||
assert closed["status"] == "production_closed_clickhouse_native_scope"
|
||||
assert closed["native_backup_terminal"] == "BACKUP_CREATED"
|
||||
assert closed["isolated_restore_terminal"] == "verified"
|
||||
assert closed["restic_snapshot_id"] == "574e2a1a"
|
||||
assert closed["independent_verifier"] == "pass"
|
||||
assert closed["exact_cleanup"] == "pass"
|
||||
sqlite_pending = receipt["pending_verifiers"][
|
||||
"signoz_sqlite_application_consistent_backup"
|
||||
]
|
||||
assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False
|
||||
assert sqlite_pending["sqlite_cli_present_in_runtime"] is False
|
||||
offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"]
|
||||
assert offsite["current_repository_scope"] == "local_same_failure_domain"
|
||||
assert offsite["offsite_verified"] is False
|
||||
artifacts = receipt["pending_verifiers"][
|
||||
"clickhouse_native_failed_artifact_retention"
|
||||
]
|
||||
assert artifacts["destructive_cleanup_authorized"] is False
|
||||
assert artifacts["active_backup_or_restore_count"] == 0
|
||||
assert len(artifacts["preserved_artifacts"]) == 2
|
||||
resume = receipt["pending_verifiers"][
|
||||
"clickhouse_native_resume_submitted_inventory"
|
||||
]
|
||||
assert resume["same_run_resume_safe"] is False
|
||||
assert len(resume["required_preconditions"]) == 4
|
||||
assets = receipt["asset_reconciliation"]
|
||||
assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
|
||||
assert assets["source_service_count"] == 27
|
||||
assert assets["runtime_mirror_service_count"] == 24
|
||||
assert assets["exact_shared_service_count"] == 24
|
||||
assert assets["shared_service_drift_count"] == 0
|
||||
assert assets["source_only_services"] == ["bitan-app", "sentry", "signoz"]
|
||||
assert assets["signoz_clickhouse_exact_match"] is True
|
||||
assert assets["live_signoz_query_host"] == "192.168.0.110"
|
||||
assert assets["source_signoz_host"] == "192.168.0.188"
|
||||
github_drift = assets["github_freeze_legacy_asset_drift"]
|
||||
assert github_drift["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-002"
|
||||
assert github_drift["superseded_by"] == "global_product_governance_v2"
|
||||
assert github_drift["active_github_use_in_this_run"] is False
|
||||
assert len(github_drift["source_references"]) == 3
|
||||
assert github_drift["p0_p1_github_recovery_scheduled"] is False
|
||||
duplicate_identity = assets["monitoring_generator_duplicate_identity_drift"]
|
||||
assert duplicate_identity["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-003"
|
||||
assert duplicate_identity["status"] == "pending_source_deduplication"
|
||||
assert duplicate_identity["awoooi_api_registry_entry_count"] == 2
|
||||
assert duplicate_identity["generated_scrape_config_count"] == 24
|
||||
assert duplicate_identity["generated_awoooi_api_job_count"] == 2
|
||||
assert duplicate_identity["generated_blackbox_target_count"] == 20
|
||||
assert duplicate_identity["runtime_apply_performed"] is False
|
||||
assert receipt["terminal"]["status"] == "partial_degraded"
|
||||
assert receipt["terminal"]["blockers"] == [
|
||||
"signoz_sqlite_application_consistent_backup_pending",
|
||||
"signoz_backup_offsite_dr_pending",
|
||||
"clickhouse_native_failed_artifact_retention_pending",
|
||||
"clickhouse_native_resume_submitted_inventory_pending",
|
||||
"service_registry_runtime_mirror_reconciliation_pending",
|
||||
"github_freeze_legacy_asset_retirement_pending",
|
||||
"monitoring_generator_duplicate_awoooi_api_identity_pending",
|
||||
]
|
||||
|
||||
|
||||
def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> None:
|
||||
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
||||
post_closure = contract["runtime_observations"]["post_closure_regression"]
|
||||
assert post_closure["snapshot_schema"] == (
|
||||
"awoooi_signoz_post_closure_regression_v2"
|
||||
)
|
||||
assert post_closure["gitea_cd_run_id"] == 5140
|
||||
assert post_closure["api_runtime_cooldown_closed"] is True
|
||||
assert post_closure["cd_probe_safety_fix_deployed"] is True
|
||||
assert post_closure["config_route_patch_deployed"] is True
|
||||
assert post_closure["signoz_control_plane_health_route_deployed"] is True
|
||||
assert post_closure["backup_collector_restore_guard_deployed"] is True
|
||||
assert post_closure["backup_canary_scope"] == "clickhouse_native_only"
|
||||
assert post_closure["backup_canary_verified"] is True
|
||||
assert post_closure["backup_canary_last_terminal"] == "pass"
|
||||
assert post_closure["backup_canary_last_run_id"] == (
|
||||
"P0-OBS-002-native-canary-20260715T001329Z-apply4"
|
||||
)
|
||||
assert post_closure["backup_canary_cooldown_retry_status"] == (
|
||||
"superseded_by_online_native_no_stop_contract"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_migration_status"] == (
|
||||
"production_closed"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_disk_run_id"] == (
|
||||
"P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_disk_status"] == "deployed_verified"
|
||||
assert post_closure["clickhouse_native_backup_toolchain_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
|
||||
)
|
||||
assert post_closure["clickhouse_native_restore_terminal"] == "verified"
|
||||
assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a"
|
||||
assert post_closure["clickhouse_native_offsite_status"] == "pending"
|
||||
assert post_closure["clickhouse_native_failed_artifact_retention_status"] == (
|
||||
"pending_bounded_retention_decision"
|
||||
)
|
||||
assert post_closure["clickhouse_native_resume_inventory_status"] == (
|
||||
"pending_durable_submitted_inventory_contract"
|
||||
)
|
||||
assert (
|
||||
post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending"
|
||||
)
|
||||
assert post_closure["service_registry_runtime_mirror_status"] == (
|
||||
"partial_degraded"
|
||||
)
|
||||
assert post_closure["service_registry_runtime_mirror_work_item_id"] == (
|
||||
"P0-OBS-002-ASSET-DRIFT-001"
|
||||
)
|
||||
assert post_closure["github_freeze_legacy_asset_status"] == (
|
||||
"pending_controlled_retirement"
|
||||
)
|
||||
assert post_closure["github_freeze_legacy_asset_work_item_id"] == (
|
||||
"P0-OBS-002-ASSET-DRIFT-002"
|
||||
)
|
||||
assert post_closure["monitoring_generator_duplicate_identity_status"] == (
|
||||
"pending_source_deduplication"
|
||||
)
|
||||
assert post_closure["monitoring_generator_duplicate_identity_work_item_id"] == (
|
||||
"P0-OBS-002-ASSET-DRIFT-003"
|
||||
)
|
||||
assert post_closure["post_release_otlp_grpc_600s_verifier_status"] == "pass"
|
||||
assert contract["terminal"]["status"] == "partial_degraded"
|
||||
assert contract["terminal"]["blockers"] == [
|
||||
"organization_not_initialized",
|
||||
"direct_ingress_policy_not_closed",
|
||||
"http_bridge_not_supervised",
|
||||
"filelog_normalization_degraded",
|
||||
"wave_b_not_authorized",
|
||||
]
|
||||
|
||||
|
||||
def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
|
||||
env = os.environ.copy()
|
||||
env.pop("MANAGED_PROMOTION_RECEIPT", None)
|
||||
|
||||
Reference in New Issue
Block a user