From 36b2446d0fe7799df5ebd94658f7ad380a20f9c3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 21:02:46 +0800 Subject: [PATCH] docs(signoz): bind current metadata candidate evidence --- ...rganization-canonical-route-migration.yaml | 6 +- .../p0-obs-002-post-closure-regression.yaml | 40 ++++++- .../signoz-canonical-route-preflight.py | 100 ++++++++++++++++-- .../test_signoz_canonical_route_migration.py | 84 +++++++++++---- 4 files changed, 191 insertions(+), 39 deletions(-) diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index d0c37435c..8c4c592ef 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -202,12 +202,14 @@ runtime_observations: clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract signoz_sqlite_application_consistent_backup_status: pending - signoz_metadata_export_source_contract_status: deployed_inactive_verified + signoz_metadata_export_source_contract_status: deployed_baseline_preserved_current_candidate_pending signoz_metadata_export_policy: config/signoz/metadata-export-policy.json signoz_metadata_exporter: scripts/backup/signoz-metadata-export.py signoz_metadata_export_verifier: scripts/backup/verify-signoz-metadata-export.py signoz_metadata_restore_driver: scripts/backup/signoz-metadata-restore-drill.py - signoz_metadata_controlled_deployer: scripts/ops/deploy-signoz-metadata-toolchain.sh + signoz_metadata_controlled_deployer: scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh + signoz_metadata_current_candidate_bundle_id: b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff + signoz_metadata_current_candidate_deployment_status: pending signoz_metadata_runtime_export_terminal: pending_authenticated_stable_export signoz_metadata_runtime_restore_terminal: pending_isolated_same_version_target signoz_metadata_zero_residue_terminal: pending diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 11c19cc3e..56044e40f 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -57,9 +57,9 @@ asset_reconciliation: drift_work_item_id: P0-OBS-002-ASSET-DRIFT-001 source: ops/config/service-registry.yaml runtime_mirror: k8s/awoooi-prod/15-service-registry-configmap.yaml - source_service_count: 29 - runtime_mirror_service_count: 29 - exact_shared_service_count: 29 + source_service_count: 31 + runtime_mirror_service_count: 31 + exact_shared_service_count: 31 shared_service_drift_count: 0 source_only_services: [] source_runtime_manifest_exact: true @@ -69,7 +69,7 @@ asset_reconciliation: source_signoz_host: 192.168.0.188 terminal: partial_degraded safe_next_action: >- - Verify the exact 28-service source/runtime manifest against production, + Verify the exact 31-service source/runtime manifest against production, then reconcile the SignOz query-host identity against live 110 readback; source-manifest parity alone does not close the production drift item. github_freeze_legacy_asset_drift: @@ -760,7 +760,7 @@ pending_verifiers: status: pending_supported_non_raw_export_or_coordinated_snapshot raw_sqlite_read_or_sync_allowed: false sqlite_cli_present_in_runtime: false - source_contract_status: deployed_inactive_verified + source_contract_status: deployed_baseline_preserved_current_candidate_pending source_contract: policy: config/signoz/metadata-export-policy.json exporter: scripts/backup/signoz-metadata-export.py @@ -771,6 +771,7 @@ pending_verifiers: controlled_deployer: scripts/ops/deploy-signoz-metadata-toolchain.sh deployer_tests: scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py expected_file_count: 8 + baseline_role: historical_deployed_baseline hashes: policy_sha256: 3ac9dbc1fb74bd7c6a15e99c54c8119e930df229e9d7cf8e3e323596bebe17fb exporter_sha256: b184d55bd5601377d25e78e422de172d01c3f8b8ea2de948b65c915201094956 @@ -781,6 +782,35 @@ pending_verifiers: controlled_deployer_sha256: e4183d1461fdce66af351d8bdcfe0626c3a240edd81eaee1d179498719f1cde0 deployer_tests_sha256: 4b4b8185fc456d6cbe4ab56affdfa919bf48b4cd6f330cdf7bf413451d331209 test_terminal: 24_passed + current_candidate: + status: source_bound_deployment_pending + current_source_deployed: false + exact_source_revision: 2bff3edcff38753d37d86d127cd3834572ffb91a + controlled_deployer: scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh + controller_tests: scripts/ops/tests/test_signoz_metadata_toolchain_agent99_transport.py + target_executor: agent99-signoz-metadata-executor.ps1 + target_program: agent99-signoz-toolchain-target.py + target_tests: scripts/ops/tests/test_agent99_signoz_toolchain_target.py + expected_file_count: 7 + bundle_id: b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff + hashes: + policy_sha256: a050943d50631d34d6eb2580db484197d6c7101b3b9cc2b47bb7ee3d87dd3822 + shared_contract_sha256: d2b2281da9ec89afc44d5153711b2d61e972545ad107611b74a5e6dfb01defe8 + exporter_sha256: 8cd5cabf245e3814da32e33920c431ec28ed7180edc0495a2d0b9379fb4025fb + independent_export_verifier_sha256: 0bac6d8ef8063f3584fc744cf205de08fc50d1085b292ffbd061f6090c344c86 + isolated_restore_driver_sha256: eb766f1209af08c3b339756d58653165d55b3c23bc8dbd66438dfb46d8098156 + isolated_restore_controller_sha256: dc57604919e220454f25955c1c652b7bd9dea632210019b0b4de527969e03c52 + isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b + focused_test_evidence: + passed: 130 + skipped: 10 + scope: metadata_export_restore_agent99_toolchain_runtime_transport_canonical_route + agent99_runtime_deploy_terminal: pending + toolchain_deploy_terminal: pending + authenticated_export_terminal: pending + isolated_restore_terminal: pending + zero_residue_terminal: pending + completion_claim: false deployment: status: deployed_inactive_verified asset_id: host110-signoz-metadata-toolchain diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index fa35689fa..2ee236df5 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -19,6 +19,7 @@ import yaml SCHEMA = "awoooi_signoz_canonical_route_preflight_v1" POST_CLOSURE_SCHEMA = "awoooi_signoz_post_closure_regression_v2" POST_CLOSURE_SNAPSHOT = "ops/signoz/p0-obs-002-post-closure-regression.yaml" +CURRENT_METADATA_CANDIDATE_SOURCE_REVISION = "2bff3edcff38753d37d86d127cd3834572ffb91a" EXPECTED_BLOCKERS = [ "organization_not_initialized", "direct_ingress_policy_not_closed", @@ -64,6 +65,33 @@ METADATA_EXPORT_SOURCE_PATHS = { "scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py" ), } +HISTORICAL_METADATA_EXPORT_HASHES = { + "policy_sha256": "3ac9dbc1fb74bd7c6a15e99c54c8119e930df229e9d7cf8e3e323596bebe17fb", + "exporter_sha256": "b184d55bd5601377d25e78e422de172d01c3f8b8ea2de948b65c915201094956", + "shared_contract_sha256": "f719d3f2ec86acfafe12be1883e57e6675b8b64caab0ee16d6054c8b383fa213", + "independent_export_verifier_sha256": "cbab56dd3919866a96550c1dce46ec67783cd6f00473e4491861e17d1430048c", + "isolated_restore_driver_sha256": "38ce70b3f891a3ce8edfaeb880ccf69d0bb6817c3f801a0aefa0f06341901997", + "contract_tests_sha256": "427093334d006f1a5d45c7f3691a22e1679c3279caf5b9f6d1b4954db2142b79", + "controlled_deployer_sha256": "e4183d1461fdce66af351d8bdcfe0626c3a240edd81eaee1d179498719f1cde0", + "deployer_tests_sha256": "4b4b8185fc456d6cbe4ab56affdfa919bf48b4cd6f330cdf7bf413451d331209", +} +METADATA_EXPORT_CANDIDATE_PATHS = { + "policy_sha256": "config/signoz/metadata-export-policy.json", + "shared_contract_sha256": "scripts/backup/signoz_metadata_contract.py", + "exporter_sha256": "scripts/backup/signoz-metadata-export.py", + "independent_export_verifier_sha256": ( + "scripts/backup/verify-signoz-metadata-export.py" + ), + "isolated_restore_driver_sha256": ( + "scripts/backup/signoz-metadata-restore-drill.py" + ), + "isolated_restore_controller_sha256": ( + "scripts/backup/signoz-metadata-isolated-restore.py" + ), + "isolated_cluster_config_sha256": ( + "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + ), +} HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES = { "backup_orchestrator_sha256": ( "2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520" @@ -121,7 +149,7 @@ def _current_toolchain_source_hashes(root: Path) -> dict[str, str]: def _current_metadata_export_source_hashes(root: Path) -> dict[str, str]: hashes: dict[str, str] = {} - for field, relative in METADATA_EXPORT_SOURCE_PATHS.items(): + for field, relative in METADATA_EXPORT_CANDIDATE_PATHS.items(): source = root / relative if not source.is_file() or source.is_symlink(): return {} @@ -346,13 +374,13 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: asset_reconciliation.get("drift_work_item_id") == "P0-OBS-002-ASSET-DRIFT-001" and asset_reconciliation.get("source_service_count") == len(source_services) - == 29 + == 31 and asset_reconciliation.get("runtime_mirror_service_count") == len(runtime_services) - == 29 + == 31 and asset_reconciliation.get("exact_shared_service_count") == len(exact_shared_services) - == 29 + == 31 and asset_reconciliation.get("shared_service_drift_count") == 0 and asset_reconciliation.get("source_only_services") == source_only_services @@ -862,14 +890,61 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: metadata_historical_deployment = metadata_export_source_contract.get( "historical_deployment", {} ) + metadata_current_candidate = metadata_export_source_contract.get( + "current_candidate", {} + ) declared_metadata_export_hashes = metadata_export_source_contract.get("hashes", {}) + declared_metadata_candidate_hashes = metadata_current_candidate.get("hashes", {}) actual_metadata_export_hashes = _current_metadata_export_source_hashes(root) + actual_metadata_bundle_id = hashlib.sha256( + "".join( + f"{actual_metadata_export_hashes.get(field, '')}\n" + for field in METADATA_EXPORT_CANDIDATE_PATHS + ).encode("ascii") + ).hexdigest() checks["signoz_metadata_export_source_hashes_bound"] = ( metadata_export_source_contract.get("expected_file_count") == len(METADATA_EXPORT_SOURCE_PATHS) and set(declared_metadata_export_hashes) == set(METADATA_EXPORT_SOURCE_PATHS) - and declared_metadata_export_hashes == actual_metadata_export_hashes + and declared_metadata_export_hashes == HISTORICAL_METADATA_EXPORT_HASHES + and metadata_export_source_contract.get("baseline_role") + == "historical_deployed_baseline" and metadata_export_source_contract.get("test_terminal") == "24_passed" + and metadata_current_candidate.get("status") + == "source_bound_deployment_pending" + and metadata_current_candidate.get("current_source_deployed") is False + and metadata_current_candidate.get("exact_source_revision") + == CURRENT_METADATA_CANDIDATE_SOURCE_REVISION + and metadata_current_candidate.get("controlled_deployer") + == "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" + and metadata_current_candidate.get("controller_tests") + == "scripts/ops/tests/test_signoz_metadata_toolchain_agent99_transport.py" + and metadata_current_candidate.get("target_executor") + == "agent99-signoz-metadata-executor.ps1" + and metadata_current_candidate.get("target_program") + == "agent99-signoz-toolchain-target.py" + and metadata_current_candidate.get("target_tests") + == "scripts/ops/tests/test_agent99_signoz_toolchain_target.py" + and metadata_current_candidate.get("expected_file_count") + == len(METADATA_EXPORT_CANDIDATE_PATHS) + == 7 + and set(declared_metadata_candidate_hashes) + == set(METADATA_EXPORT_CANDIDATE_PATHS) + and declared_metadata_candidate_hashes == actual_metadata_export_hashes + and metadata_current_candidate.get("bundle_id") == actual_metadata_bundle_id + and metadata_current_candidate.get("focused_test_evidence") + == { + "passed": 130, + "skipped": 10, + "scope": ( + "metadata_export_restore_agent99_toolchain_runtime_transport_" + "canonical_route" + ), + } + and metadata_current_candidate.get("agent99_runtime_deploy_terminal") + == "pending" + and metadata_current_candidate.get("toolchain_deploy_terminal") == "pending" + and metadata_current_candidate.get("completion_claim") is False ) offsite_backup = pending_verifiers.get("signoz_backup_offsite_dr", {}) failed_artifacts = pending_verifiers.get( @@ -892,7 +967,8 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == "pending_supported_non_raw_export_or_coordinated_snapshot" and sqlite_backup.get("raw_sqlite_read_or_sync_allowed") is False and sqlite_backup.get("sqlite_cli_present_in_runtime") is False - and sqlite_backup.get("source_contract_status") == "deployed_inactive_verified" + and sqlite_backup.get("source_contract_status") + == "deployed_baseline_preserved_current_candidate_pending" and metadata_export_source_contract.get("policy") == "config/signoz/metadata-export-policy.json" and metadata_export_source_contract.get("exporter") @@ -938,8 +1014,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == "check_pass_no_write" and metadata_deployment.get("independent_postcheck", {}).get("remote_state") == "exact" - and metadata_deployment.get("independent_postcheck", {}).get("drift_count") - == 0 + and metadata_deployment.get("independent_postcheck", {}).get("drift_count") == 0 and metadata_deployment.get("windows99_staging_cleanup", {}).get("terminal") == "pass" and metadata_deployment.get("windows99_staging_cleanup", {}).get( @@ -948,8 +1023,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == 0 and metadata_deployment.get("learning_writeback") == "partial_degraded" and metadata_deployment.get("completion_claim") is False - and metadata_historical_deployment.get("status") - == "deployed_inactive_verified" + and metadata_historical_deployment.get("status") == "deployed_inactive_verified" and metadata_historical_deployment.get("bundle_id") == "bb6d78b67f506072b2e45e92bc1d415364e25852282a9b068900665c578df011" and metadata_historical_deployment.get("initial_failed_attempt", {}).get( @@ -1115,7 +1189,11 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and post_closure.get("signoz_metadata_restore_driver") == sqlite_backup.get("source_contract", {}).get("isolated_restore_driver") and post_closure.get("signoz_metadata_controlled_deployer") - == sqlite_backup.get("source_contract", {}).get("controlled_deployer") + == metadata_current_candidate.get("controlled_deployer") + and post_closure.get("signoz_metadata_current_candidate_bundle_id") + == metadata_current_candidate.get("bundle_id") + and post_closure.get("signoz_metadata_current_candidate_deployment_status") + == "pending" and post_closure.get("signoz_metadata_runtime_export_terminal") == sqlite_backup.get("runtime_export_terminal") and post_closure.get("signoz_metadata_runtime_restore_terminal") diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 84d3e3605..935acb006 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -834,7 +834,9 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ] assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False assert sqlite_pending["sqlite_cli_present_in_runtime"] is False - assert sqlite_pending["source_contract_status"] == "deployed_inactive_verified" + assert sqlite_pending["source_contract_status"] == ( + "deployed_baseline_preserved_current_candidate_pending" + ) source_contract = sqlite_pending["source_contract"] assert source_contract["policy"] == "config/signoz/metadata-export-policy.json" assert source_contract["exporter"] == ("scripts/backup/signoz-metadata-export.py") @@ -851,31 +853,67 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> "scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py" ) assert source_contract["expected_file_count"] == 8 - metadata_paths = { + assert source_contract["baseline_role"] == "historical_deployed_baseline" + assert source_contract["hashes"] == { + "policy_sha256": "3ac9dbc1fb74bd7c6a15e99c54c8119e930df229e9d7cf8e3e323596bebe17fb", + "exporter_sha256": "b184d55bd5601377d25e78e422de172d01c3f8b8ea2de948b65c915201094956", + "shared_contract_sha256": "f719d3f2ec86acfafe12be1883e57e6675b8b64caab0ee16d6054c8b383fa213", + "independent_export_verifier_sha256": "cbab56dd3919866a96550c1dce46ec67783cd6f00473e4491861e17d1430048c", + "isolated_restore_driver_sha256": "38ce70b3f891a3ce8edfaeb880ccf69d0bb6817c3f801a0aefa0f06341901997", + "contract_tests_sha256": "427093334d006f1a5d45c7f3691a22e1679c3279caf5b9f6d1b4954db2142b79", + "controlled_deployer_sha256": "e4183d1461fdce66af351d8bdcfe0626c3a240edd81eaee1d179498719f1cde0", + "deployer_tests_sha256": "4b4b8185fc456d6cbe4ab56affdfa919bf48b4cd6f330cdf7bf413451d331209", + } + assert source_contract["test_terminal"] == "24_passed" + current_candidate = source_contract["current_candidate"] + candidate_paths = { "policy_sha256": "config/signoz/metadata-export-policy.json", - "exporter_sha256": "scripts/backup/signoz-metadata-export.py", "shared_contract_sha256": "scripts/backup/signoz_metadata_contract.py", + "exporter_sha256": "scripts/backup/signoz-metadata-export.py", "independent_export_verifier_sha256": ( "scripts/backup/verify-signoz-metadata-export.py" ), "isolated_restore_driver_sha256": ( "scripts/backup/signoz-metadata-restore-drill.py" ), - "contract_tests_sha256": ( - "scripts/backup/tests/test_signoz_metadata_export_contract.py" + "isolated_restore_controller_sha256": ( + "scripts/backup/signoz-metadata-isolated-restore.py" ), - "controlled_deployer_sha256": ( - "scripts/ops/deploy-signoz-metadata-toolchain.sh" - ), - "deployer_tests_sha256": ( - "scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py" + "isolated_cluster_config_sha256": ( + "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" ), } - assert source_contract["hashes"] == { + candidate_hashes = { field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() - for field, relative in metadata_paths.items() + for field, relative in candidate_paths.items() } - assert source_contract["test_terminal"] == "24_passed" + candidate_bundle = hashlib.sha256( + "".join(f"{candidate_hashes[field]}\n" for field in candidate_paths).encode( + "ascii" + ) + ).hexdigest() + assert current_candidate["status"] == "source_bound_deployment_pending" + assert current_candidate["current_source_deployed"] is False + assert current_candidate["exact_source_revision"] == ( + "2bff3edcff38753d37d86d127cd3834572ffb91a" + ) + assert current_candidate["controlled_deployer"] == ( + "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" + ) + assert current_candidate["expected_file_count"] == 7 + assert current_candidate["hashes"] == candidate_hashes + assert current_candidate["bundle_id"] == candidate_bundle + assert current_candidate["focused_test_evidence"] == { + "passed": 130, + "skipped": 10, + "scope": ( + "metadata_export_restore_agent99_toolchain_runtime_transport_" + "canonical_route" + ), + } + assert current_candidate["agent99_runtime_deploy_terminal"] == "pending" + assert current_candidate["toolchain_deploy_terminal"] == "pending" + assert current_candidate["completion_claim"] is False deployment = source_contract["deployment"] assert deployment["status"] == "deployed_inactive_verified" assert deployment["asset_id"] == "host110-signoz-metadata-toolchain" @@ -906,9 +944,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> assert deployment["independent_postcheck"]["run_id"] == ( "P0-OBS-002-signoz-toolchain-postcheck-20260722T181500P0800" ) - assert deployment["independent_postcheck"]["terminal"] == ( - "check_pass_no_write" - ) + assert deployment["independent_postcheck"]["terminal"] == ("check_pass_no_write") assert deployment["independent_postcheck"]["remote_state"] == "exact" assert deployment["independent_postcheck"]["drift_count"] == 0 assert deployment["windows99_staging_cleanup"]["terminal"] == "pass" @@ -985,9 +1021,9 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ) assets = receipt["asset_reconciliation"] assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001" - assert assets["source_service_count"] == 29 - assert assets["runtime_mirror_service_count"] == 29 - assert assets["exact_shared_service_count"] == 29 + assert assets["source_service_count"] == 31 + assert assets["runtime_mirror_service_count"] == 31 + assert assets["exact_shared_service_count"] == 31 assert assets["shared_service_drift_count"] == 0 assert assets["source_only_services"] == [] assert assets["source_runtime_manifest_exact"] is True @@ -1115,7 +1151,7 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending" ) assert post_closure["signoz_metadata_export_source_contract_status"] == ( - "deployed_inactive_verified" + "deployed_baseline_preserved_current_candidate_pending" ) assert post_closure["signoz_metadata_export_policy"] == ( "config/signoz/metadata-export-policy.json" @@ -1130,7 +1166,13 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N "scripts/backup/signoz-metadata-restore-drill.py" ) assert post_closure["signoz_metadata_controlled_deployer"] == ( - "scripts/ops/deploy-signoz-metadata-toolchain.sh" + "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" + ) + assert post_closure["signoz_metadata_current_candidate_bundle_id"] == ( + "b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff" + ) + assert post_closure["signoz_metadata_current_candidate_deployment_status"] == ( + "pending" ) assert post_closure["signoz_metadata_runtime_export_terminal"] == ( "pending_authenticated_stable_export"