import json import multiprocessing from pathlib import Path import subprocess import sys def _process_reserve(state_path, start_event, result_queue): from services.nemotron_dispatch_reservation_service import ( SharedFileReservationStore, ) store = SharedFileReservationStore( state_path, committed_ttl_sec=3600, lease_ttl_sec=60, ) start_event.wait(timeout=5) result_queue.put(store.reserve("SKU-PROCESS-SHARED")) def test_shared_store_coordinates_independent_instances_and_ttl(tmp_path): from services.nemotron_dispatch_reservation_service import ( SharedFileReservationStore, ) clock = {"now": 1000.0} path = tmp_path / "dedupe" / "state.json" first = SharedFileReservationStore( path, committed_ttl_sec=100, lease_ttl_sec=20, clock=lambda: clock["now"], ) second = SharedFileReservationStore( path, committed_ttl_sec=100, lease_ttl_sec=20, clock=lambda: clock["now"], ) token = first.reserve("SKU-SHARED") assert token assert second.reserve("SKU-SHARED") is None assert second.is_duplicate("SKU-SHARED") is True assert "SKU-SHARED" not in path.read_text(encoding="utf-8") assert second.release("SKU-SHARED", "stale-token") is False assert first.refresh("SKU-SHARED", token) is True assert first.commit("SKU-SHARED", token) is True assert second.reserve("SKU-SHARED") is None clock["now"] += 101 next_token = second.reserve("SKU-SHARED") assert next_token and next_token != token assert second.release("SKU-SHARED", next_token) is True def test_begin_side_effect_persists_extended_quarantine(tmp_path): from services.nemotron_dispatch_reservation_service import ( SharedFileReservationStore, ) clock = {"now": 1000.0} path = tmp_path / "state.json" first = SharedFileReservationStore( path, committed_ttl_sec=100, lease_ttl_sec=20, clock=lambda: clock["now"], ) second = SharedFileReservationStore( path, committed_ttl_sec=100, lease_ttl_sec=20, clock=lambda: clock["now"], ) token = first.reserve("SKU-SIDE-EFFECT") assert token assert first.begin_side_effect("SKU-SIDE-EFFECT", token) is True payload = json.loads(path.read_text(encoding="utf-8")) lease = next(iter(payload["leases"].values())) assert lease == { "token": token, "expires_at": 1100.0, "phase": "side_effect_started", } clock["now"] += 21 assert second.reserve("SKU-SIDE-EFFECT") is None clock["now"] += 80 assert second.reserve("SKU-SIDE-EFFECT") def test_shared_store_is_atomic_across_processes(tmp_path): state_path = str(tmp_path / "process" / "state.json") context = multiprocessing.get_context("fork") start_event = context.Event() result_queue = context.Queue() processes = [ context.Process( target=_process_reserve, args=(state_path, start_event, result_queue), ) for _index in range(6) ] for process in processes: process.start() start_event.set() tokens = [result_queue.get(timeout=10) for _process in processes] for process in processes: process.join(timeout=10) assert process.exitcode == 0 owners = [token for token in tokens if token] assert len(owners) == 1 def test_shared_store_corruption_fails_closed(tmp_path): from services.nemotron_dispatch_reservation_service import ( SharedFileReservationStore, ) path = tmp_path / "state.json" path.write_text("{not-json", encoding="utf-8") store = SharedFileReservationStore( path, committed_ttl_sec=3600, lease_ttl_sec=900, ) assert store.reserve("SKU-CORRUPT") is None assert store.refresh("SKU-CORRUPT", "token") is False assert store.commit("SKU-CORRUPT", "token") is False assert store.release("SKU-CORRUPT", "token") is False assert store.is_duplicate("SKU-CORRUPT") is True assert path.read_text(encoding="utf-8") == "{not-json" def test_atomic_write_fsyncs_file_and_parent_directory(tmp_path, monkeypatch): from services import nemotron_dispatch_reservation_service as service fsynced = [] real_fsync = service.os.fsync def record_fsync(fd): fsynced.append(fd) return real_fsync(fd) monkeypatch.setattr(service.os, "fsync", record_fsync) store = service.SharedFileReservationStore( tmp_path / "nested" / "state.json", committed_ttl_sec=3600, lease_ttl_sec=900, ) assert store.reserve("SKU-FSYNC") assert len(fsynced) >= 2 def test_missing_process_shared_lock_fails_closed(monkeypatch, tmp_path): from services import nemotron_dispatch_reservation_service as service monkeypatch.setattr(service, "fcntl", None) monkeypatch.delenv("FLASK_ENV", raising=False) monkeypatch.delenv("USE_POSTGRESQL", raising=False) assert service.build_production_shared_reservation_store( committed_ttl_sec=3600, lease_ttl_sec=900, ) is None store = service.SharedFileReservationStore( tmp_path / "state.json", committed_ttl_sec=3600, lease_ttl_sec=900, ) assert store.reserve("SKU-NO-FCNTL") is None assert store.is_duplicate("SKU-NO-FCNTL") is True def test_production_builder_forces_canonical_shared_data_path(monkeypatch): from services import nemotron_dispatch_reservation_service as service monkeypatch.setenv("FLASK_ENV", "production") store = service.build_production_shared_reservation_store( committed_ttl_sec=3600, lease_ttl_sec=900, ) assert store is not None assert store.state_path == Path( "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json" ) assert store.lock_path == Path( "/app/data/ai_automation/nemotron_dispatch_dedupe_state.json.lock" ) def test_postgres_runtime_cannot_fall_back_to_process_local_memory(monkeypatch): from services import nemotron_dispatch_reservation_service as service monkeypatch.delenv("FLASK_ENV", raising=False) monkeypatch.setenv("USE_POSTGRESQL", "true") assert service.production_shared_reservation_required() is True assert service.build_production_shared_reservation_store( committed_ttl_sec=3600, lease_ttl_sec=900, ) is not None def test_shared_state_contains_only_hashed_sku_and_timing_metadata(tmp_path): from services.nemotron_dispatch_reservation_service import ( SharedFileReservationStore, ) path = tmp_path / "state.json" store = SharedFileReservationStore( path, committed_ttl_sec=3600, lease_ttl_sec=900, ) token = store.reserve("RAW-SKU-MUST-NOT-APPEAR") payload = json.loads(path.read_text(encoding="utf-8")) assert token assert "RAW-SKU-MUST-NOT-APPEAR" not in path.read_text(encoding="utf-8") assert set(payload) == {"version", "leases", "committed"} assert len(next(iter(payload["leases"]))) == 64 def test_production_containers_share_data_mount_and_force_shared_backend(): compose = ( Path(__file__).resolve().parents[1] / "docker-compose.yml" ).read_text(encoding="utf-8") assert compose.count("- ./data:/app/data") >= 3 assert compose.count("- FLASK_ENV=production") >= 3 def test_shared_reservation_canary_has_no_business_side_effects(tmp_path): from services.nemotron_dispatch_reservation_service import ( run_shared_reservation_canary, ) payload = run_shared_reservation_canary( state_path=tmp_path / "canary" / "state.json", ) assert payload["success"] is True assert payload["check_pass_count"] == payload["check_count"] == 6 assert payload["controlled_apply"] == { "writes_shared_reservation_state": True, "terminal_state_clear": True, "writes_database": False, "sends_telegram": False, "executes_business_tool": False, "reads_secret": False, } def test_shared_reservation_canary_cli_is_machine_readable(tmp_path): completed = subprocess.run( [ sys.executable, "scripts/ops/run_nemotron_dispatch_reservation_canary.py", "--state-path", str(tmp_path / "cli" / "state.json"), ], cwd=Path(__file__).resolve().parents[1], capture_output=True, check=False, text=True, ) assert completed.returncode == 0, completed.stderr payload = json.loads(completed.stdout) assert payload["status"] == "shared_reservation_canary_passed" assert payload["controlled_apply"]["writes_database"] is False