feat(observability): add safe SigNoz metadata backup lane

This commit is contained in:
ogt
2026-07-15 13:57:13 +08:00
parent 4dc5b51d0f
commit 1730c5bb71
12 changed files with 3485 additions and 28 deletions

View File

@@ -202,7 +202,17 @@ runtime_observations:
clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision
clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract
signoz_sqlite_application_consistent_backup_status: pending
service_registry_runtime_mirror_status: partial_degraded
signoz_metadata_export_source_contract_status: implemented_tested_not_deployed
signoz_metadata_export_policy: config/signoz/metadata-export-policy.json
signoz_metadata_exporter: scripts/backup/signoz-metadata-export.py
signoz_metadata_export_verifier: scripts/backup/verify-signoz-metadata-export.py
signoz_metadata_restore_driver: scripts/backup/signoz-metadata-restore-drill.py
signoz_metadata_controlled_deployer: scripts/ops/deploy-signoz-metadata-toolchain.sh
signoz_metadata_runtime_export_terminal: pending_authenticated_stable_export
signoz_metadata_runtime_restore_terminal: pending_isolated_same_version_target
signoz_metadata_zero_residue_terminal: pending
signoz_metadata_full_sqlite_completion_claim: false
service_registry_runtime_mirror_status: source_manifest_exact_production_readback_pending
service_registry_runtime_mirror_work_item_id: P0-OBS-002-ASSET-DRIFT-001
github_freeze_legacy_asset_status: pending_controlled_retirement
github_freeze_legacy_asset_work_item_id: P0-OBS-002-ASSET-DRIFT-002

View File

@@ -58,21 +58,20 @@ asset_reconciliation:
source: ops/config/service-registry.yaml
runtime_mirror: k8s/awoooi-prod/15-service-registry-configmap.yaml
source_service_count: 27
runtime_mirror_service_count: 24
exact_shared_service_count: 24
runtime_mirror_service_count: 27
exact_shared_service_count: 27
shared_service_drift_count: 0
source_only_services:
- bitan-app
- sentry
- signoz
source_only_services: []
source_runtime_manifest_exact: true
production_runtime_readback_status: pending_current_sha_readback
signoz_clickhouse_exact_match: true
live_signoz_query_host: 192.168.0.110
source_signoz_host: 192.168.0.188
terminal: partial_degraded
safe_next_action: >-
Reconcile the three source-only services and the stale SignOz query host
against production runtime before atomically regenerating the ConfigMap;
do not copy the stale 188 SignOz row into the runtime mirror.
Verify the exact 27-service source/runtime manifest against production,
then reconcile the SignOz query-host identity against live 110 readback;
source-manifest parity alone does not close the production drift item.
github_freeze_legacy_asset_drift:
drift_work_item_id: P0-OBS-002-ASSET-DRIFT-002
superseded_by: global_product_governance_v2
@@ -761,9 +760,57 @@ pending_verifiers:
status: pending_supported_non_raw_export_or_coordinated_snapshot
raw_sqlite_read_or_sync_allowed: false
sqlite_cli_present_in_runtime: false
source_contract_status: implemented_tested_not_deployed
source_contract:
policy: config/signoz/metadata-export-policy.json
exporter: scripts/backup/signoz-metadata-export.py
shared_contract: scripts/backup/signoz_metadata_contract.py
independent_export_verifier: scripts/backup/verify-signoz-metadata-export.py
isolated_restore_driver: scripts/backup/signoz-metadata-restore-drill.py
contract_tests: scripts/backup/tests/test_signoz_metadata_export_contract.py
controlled_deployer: scripts/ops/deploy-signoz-metadata-toolchain.sh
deployer_tests: scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py
expected_file_count: 8
hashes:
policy_sha256: df09dbf91d30bcf4d1a2119b1c301240a252fddd446c56ecba253d5399526533
exporter_sha256: b184d55bd5601377d25e78e422de172d01c3f8b8ea2de948b65c915201094956
shared_contract_sha256: f719d3f2ec86acfafe12be1883e57e6675b8b64caab0ee16d6054c8b383fa213
independent_export_verifier_sha256: cbab56dd3919866a96550c1dce46ec67783cd6f00473e4491861e17d1430048c
isolated_restore_driver_sha256: 38ce70b3f891a3ce8edfaeb880ccf69d0bb6817c3f801a0aefa0f06341901997
contract_tests_sha256: 474d9dfcfe170c62425356ea9f06ed0dc1629b39215ee80f395f721c7d43a923
controlled_deployer_sha256: b3edcb3629aa3069fcecde5812cc4f21e83e324eec89b3c75f61cc73444adb97
deployer_tests_sha256: cae135b2075d845f13d762c751d9e6de88bf806b37dcd5bd8837487211a20880
test_terminal: 19_passed
portable_assets:
- dashboards
- alert_rules
- explorer_views
export_only_assets:
- roles
- organization_preferences
- user_preferences
- global_config
forbidden_assets:
- personal_access_tokens
- authentication_domains
- notification_channels
- users_and_credentials
- invitations
- sessions
raw_sqlite_read: false
raw_volume_read: false
secret_value_persistence: false
full_sqlite_completion_claim: false
runtime_export_terminal: pending_authenticated_stable_export
runtime_restore_terminal: pending_isolated_same_version_target
runtime_zero_residue_terminal: pending
required_preconditions:
- supported_metadata_export_or_application_consistent_snapshot_design
- independent_restore_verifier_without_raw_session_or_secret_read
- deploy_exact_source_contract_with_hash_and_mode_readback
- trusted_service_account_secret_reference_without_value_readback
- authenticated_two_read_stable_production_export
- isolated_same_version_restore_semantic_readback
- zero_container_volume_network_listener_residue
- reissue_secret_bearing_assets_from_secret_references
signoz_backup_offsite_dr:
status: pending_offsite_snapshot_and_restore_verifier
current_repository_scope: local_same_failure_domain
@@ -816,7 +863,7 @@ terminal:
- signoz_backup_offsite_dr_pending
- clickhouse_native_failed_artifact_retention_pending
- clickhouse_native_resume_submitted_inventory_pending
- service_registry_runtime_mirror_reconciliation_pending
- service_registry_production_and_live_host_readback_pending
- github_freeze_legacy_asset_retirement_pending
- monitoring_generator_duplicate_awoooi_api_identity_pending
safe_next_action: >-
@@ -826,8 +873,9 @@ terminal:
target and restore it independently, then apply an explicit failed-artifact
retention decision. Persist the submitted source inventory before BACKUP so
same-run resume cannot bind an old artifact to newer live counters. Keep raw
SQLite reads and syncs disabled. Reconcile the source-only service registry
rows against live runtime before regenerating its K8s ConfigMap mirror, and
retire legacy GitHub runner assets only after the Gitea replacement verifier
is ready. Deduplicate the monitoring registry's awoooi-api identity before
applying regenerated scrape configuration.
SQLite reads and syncs disabled. Verify the exact source/runtime service
registry manifest against production and reconcile its SignOz query-host
identity against live runtime. Retire legacy GitHub runner assets only after
the Gitea replacement verifier is ready. Deduplicate the monitoring
registry's awoooi-api identity before applying regenerated scrape
configuration.