use std::{ fs, os::unix::fs::{MetadataExt, PermissionsExt}, path::PathBuf, sync::atomic::{AtomicU64, Ordering}, sync::{Mutex, OnceLock}, time::{Duration, SystemTime}, }; #[cfg(debug_assertions)] use std::{process::Command, sync::Arc, thread}; use crank_artifacts::{ ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration, ReconciliationScanStop, }; #[cfg(debug_assertions)] use crank_artifacts::test_support::{ FaultAction, checkpoint_hits, clear_checkpoint, reset_traversal_calls, set_checkpoint, set_checkpoint_on_hit, traversal_calls, wait_until_held, }; static NEXT_ROOT: AtomicU64 = AtomicU64::new(0); const FULL_SCAN_BUDGET: usize = 2048; struct TestRoot(PathBuf); impl TestRoot { fn new(name: &str) -> Self { let path = std::env::temp_dir().join(format!( "crank-artifacts-reconciliation-{name}-{}-{}", std::process::id(), NEXT_ROOT.fetch_add(1, Ordering::Relaxed) )); fs::create_dir(&path).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); Self(path) } } impl Drop for TestRoot { fn drop(&mut self) { let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700)); let _ = fs::remove_dir_all(&self.0); } } fn fault_guard() -> std::sync::MutexGuard<'static, ()> { static GUARD: OnceLock> = OnceLock::new(); GUARD.get_or_init(|| Mutex::new(())).lock().unwrap() } #[test] fn paginates_final_entries_without_disclosing_locations() { let _guard = fault_guard(); let root = TestRoot::new("pages"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"reconciliation first").unwrap(); store.put(b"reconciliation second").unwrap(); let mut continuation = None; let mut candidates = Vec::new(); let mut final_entries = 0; let mut scanned = 0; loop { let (report, mut page) = store.scan_reconciliation(continuation, 1, 1).unwrap(); assert_eq!(report.traversal_syscalls, 1); assert!(report.scanned <= 1); assert_eq!(report.final_entries, page.len()); assert_eq!(report.quarantined_entries, 0); assert!(page.len() <= 1); scanned += report.scanned; final_entries += report.final_entries; if let Some(cursor) = report.continuation.as_ref() { assert_eq!(format!("{cursor:?}"), "ReconciliationCursor(..)"); } for candidate in &page { assert_eq!(format!("{candidate:?}"), "ReconciliationCandidate(..)"); } candidates.append(&mut page); continuation = report.continuation; if continuation.is_none() { break; } } assert_eq!(scanned, 2); assert_eq!(final_entries, 2); assert_eq!(candidates.len(), 2); assert!( candidates .iter() .all(|candidate| candidate.namespace() == ReconciliationNamespace::Final) ); for candidate in candidates { assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::Quarantined ); } let (report, quarantined_entries) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(quarantined_entries.len(), 2); assert_eq!(report.scanned, 2); assert_eq!(report.final_entries, 0); assert_eq!(report.quarantined_entries, 2); assert_eq!(report.stop, ReconciliationScanStop::Complete); assert!( quarantined_entries .iter() .all(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine) ); } #[test] fn classifies_malformed_and_unsafe_entries_with_a_usable_page() { let _guard = fault_guard(); let root = TestRoot::new("malformed"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation valid").unwrap(); let shard = root .0 .join("sha256") .join(&stored.artifact_ref.digest_hex()[..2]); fs::write(shard.join("not-a-digest"), b"junk").unwrap(); let unsafe_name = format!( "{}{}", &stored.artifact_ref.digest_hex()[..2], "f".repeat(62) ); std::os::unix::fs::symlink(shard.join("not-a-digest"), shard.join(unsafe_name)).unwrap(); let directory_name = format!( "{}{}", &stored.artifact_ref.digest_hex()[..2], "e".repeat(62) ); fs::create_dir(shard.join(directory_name)).unwrap(); let hardlink_name = format!( "{}{}", &stored.artifact_ref.digest_hex()[..2], "d".repeat(62) ); fs::hard_link(shard.join("not-a-digest"), shard.join(hardlink_name)).unwrap(); let (report, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.scanned, 5); assert_eq!(report.malformed, 1); assert_eq!(report.unsafe_entries, 3); assert_eq!(report.final_entries, 1); assert_eq!(report.quarantined_entries, 0); assert_eq!(candidates.len(), 1); assert!(report.continuation.is_none()); } #[test] fn skips_a_valid_publish_temp_without_classifying_it_as_malformed() { let _guard = fault_guard(); let root = TestRoot::new("valid-temp"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation with concurrent temp").unwrap(); let shard = root .0 .join("sha256") .join(&stored.artifact_ref.digest_hex()[..2]); fs::write( shard.join(".crank-artifact-tmp-v1-0123456789abcdef0123456789abcdef-7-9"), b"in-flight", ) .unwrap(); let (report, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.scanned, 2); assert_eq!(report.malformed, 0); assert_eq!(report.final_entries, 1); assert_eq!(candidates.len(), 1); } #[test] fn unsafe_canonical_shard_does_not_starve_later_shards() { let _guard = fault_guard(); let root = TestRoot::new("unsafe-shard"); let store = ArtifactStore::open(&root.0).unwrap(); let sha = root.0.join("sha256"); fs::create_dir(&sha).unwrap(); fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap(); let unsafe_shard = sha.join("00"); fs::create_dir(&unsafe_shard).unwrap(); fs::set_permissions(&unsafe_shard, fs::Permissions::from_mode(0o000)).unwrap(); let shard = sha.join("ff"); fs::create_dir(&shard).unwrap(); fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap(); let name = "f".repeat(64); fs::write(shard.join(&name), b"later valid entry").unwrap(); fs::set_permissions(shard.join(&name), fs::Permissions::from_mode(0o400)).unwrap(); let (report, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.stop, ReconciliationScanStop::Complete); assert_eq!(report.unsafe_entries, 1); assert_eq!(report.final_entries, 1); assert!(candidates.iter().any(|candidate| { candidate.namespace() == ReconciliationNamespace::Final && format!("{candidate:?}") == "ReconciliationCandidate(..)" })); fs::set_permissions(unsafe_shard, fs::Permissions::from_mode(0o700)).unwrap(); } #[test] #[cfg(debug_assertions)] fn traversal_never_exceeds_the_exact_syscall_budget() { let _guard = fault_guard(); let root = TestRoot::new("syscall-budget"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"bounded traversal syscall accounting").unwrap(); for budget in 0..=8 { reset_traversal_calls(); let (report, _) = store.scan_reconciliation(None, budget, 8).unwrap(); assert!(report.traversal_syscalls <= budget, "budget={budget}"); assert_eq!(report.traversal_syscalls, budget, "budget={budget}"); assert_eq!(traversal_calls(), report.traversal_syscalls); assert_eq!(report.stop, ReconciliationScanStop::ScanBudget); } reset_traversal_calls(); let (report, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 1) .unwrap(); assert_eq!(candidates.len(), 1); assert_eq!(report.stop, ReconciliationScanStop::ResultLimit); assert!(report.traversal_syscalls <= FULL_SCAN_BUDGET); assert_eq!(traversal_calls(), report.traversal_syscalls); } #[test] fn rejects_a_cursor_from_another_store() { let _guard = fault_guard(); let first_root = TestRoot::new("foreign-cursor-first"); let second_root = TestRoot::new("foreign-cursor-second"); let first = ArtifactStore::open(&first_root.0).unwrap(); let second = ArtifactStore::open(&second_root.0).unwrap(); first.put(b"cursor source").unwrap(); let (report, _) = first.scan_reconciliation(None, 1, 1).unwrap(); assert!(matches!( second.scan_reconciliation(report.continuation, 1, 1), Err(ArtifactError::UnsafeRoot) )); } #[test] #[cfg(debug_assertions)] fn disappearing_entry_after_readdir_is_skipped() { let _guard = fault_guard(); let root = TestRoot::new("disappearing"); let store = Arc::new(ArtifactStore::open(&root.0).unwrap()); let stored = store.put(b"disappearing reconciliation entry").unwrap(); let path = root .0 .join("sha256") .join(&stored.artifact_ref.digest_hex()[..2]) .join(stored.artifact_ref.digest_hex()); set_checkpoint("reconciliation_before_stat", FaultAction::Hold); let scan_store = Arc::clone(&store); let scan = thread::spawn(move || scan_store.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)); assert!(wait_until_held(Duration::from_secs(2))); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); fs::remove_file(&path).unwrap(); clear_checkpoint(); let (_, candidates) = scan.join().unwrap().unwrap(); assert!(candidates.is_empty()); } #[test] fn quarantine_and_delete_are_idempotent() { let _guard = fault_guard(); let root = TestRoot::new("mutation"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation mutation").unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); assert_eq!( store.quarantine_reconciliation(candidate.clone()).unwrap(), ReconciliationMutation::Quarantined ); assert_eq!( store.read(&stored.artifact_ref), Err(ArtifactError::NotFound) ); assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::AlreadyQuarantined ); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let quarantined = candidates .into_iter() .find(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine) .unwrap(); assert_eq!( store .delete_quarantined_reconciliation(quarantined.clone()) .unwrap(), ReconciliationMutation::Deleted ); assert_eq!( store .delete_quarantined_reconciliation(quarantined) .unwrap(), ReconciliationMutation::AlreadyAbsent ); } #[test] fn quarantine_preserves_old_inode_when_the_digest_is_republished() { let _guard = fault_guard(); let root = TestRoot::new("no-clobber"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation no-clobber").unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let original = candidates.into_iter().next().unwrap(); assert_eq!( store.quarantine_reconciliation(original.clone()).unwrap(), ReconciliationMutation::Quarantined ); store.put(b"reconciliation no-clobber").unwrap(); assert_eq!( store.quarantine_reconciliation(original).unwrap(), ReconciliationMutation::AlreadyQuarantined ); assert_eq!( store.read(&stored.artifact_ref).unwrap(), b"reconciliation no-clobber" ); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(candidates.len(), 2); assert!( candidates .iter() .any(|candidate| candidate.namespace() == ReconciliationNamespace::Final) ); assert!( candidates .iter() .any(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine) ); } #[test] fn quarantine_refuses_to_replace_an_unrelated_inode() { let _guard = fault_guard(); let root = TestRoot::new("unrelated-collision"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"canonical final survives collision").unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); let digest = stored.artifact_ref.digest_hex(); let quarantine = root.0.join("quarantine"); let shard = quarantine.join(&digest[..2]); fs::create_dir(&quarantine).unwrap(); fs::create_dir(&shard).unwrap(); fs::set_permissions(&quarantine, fs::Permissions::from_mode(0o700)).unwrap(); fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap(); let collision = shard.join(digest); fs::write(&collision, b"unrelated quarantine inode").unwrap(); fs::set_permissions(&collision, fs::Permissions::from_mode(0o400)).unwrap(); let collision_ino = fs::metadata(&collision).unwrap().ino(); assert_eq!( store.quarantine_reconciliation(candidate), Err(ArtifactError::UnsafeRoot) ); assert_eq!( store.read(&stored.artifact_ref).unwrap(), b"canonical final survives collision" ); assert_eq!(fs::read(collision).unwrap(), b"unrelated quarantine inode"); assert_eq!( fs::metadata(shard.join(digest)).unwrap().ino(), collision_ino ); } #[test] fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() { let _guard = fault_guard(); let root = TestRoot::new("quarantine-replaced"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store .put(b"reconciliation quarantined replacement") .unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); let path = root .0 .join("quarantine") .join(&stored.artifact_ref.digest_hex()[..2]) .join(stored.artifact_ref.digest_hex()); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); fs::remove_file(&path).unwrap(); fs::write(&path, b"replacement").unwrap(); let sibling = path.with_extension("link"); fs::hard_link(&path, &sibling).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap(); assert_eq!( store.delete_quarantined_reconciliation(candidate), Err(ArtifactError::UnsafeRoot) ); assert!(path.exists()); assert!(sibling.exists()); } #[test] fn delete_revalidates_a_post_scan_hardlink_without_inode_replacement() { let _guard = fault_guard(); let root = TestRoot::new("quarantine-hardlink"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"same quarantined inode gains a link").unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); let path = root .0 .join("quarantine") .join(&stored.artifact_ref.digest_hex()[..2]) .join(stored.artifact_ref.digest_hex()); let sibling = path.with_extension("link"); let original_ino = fs::metadata(&path).unwrap().ino(); fs::hard_link(&path, &sibling).unwrap(); assert_eq!( store.delete_quarantined_reconciliation(candidate), Err(ArtifactError::UnsafeRoot) ); assert_eq!(fs::metadata(&path).unwrap().nlink(), 2); assert_eq!(fs::metadata(&path).unwrap().ino(), original_ino); assert!(path.exists()); assert!(sibling.exists()); } #[test] #[cfg(debug_assertions)] fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() { let _guard = fault_guard(); let root = TestRoot::new("delete-fsync"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"reconciliation delete fsync").unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); set_checkpoint("reconciliation_delete_fsync", FaultAction::Fail); assert_eq!( store .delete_quarantined_reconciliation(candidate.clone()) .unwrap(), ReconciliationMutation::Retryable ); clear_checkpoint(); assert_eq!( store.delete_quarantined_reconciliation(candidate).unwrap(), ReconciliationMutation::AlreadyAbsent ); } #[test] fn rejects_a_replaced_final_inode() { let _guard = fault_guard(); let root = TestRoot::new("replaced"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation original").unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); let path = root .0 .join("sha256") .join(&stored.artifact_ref.digest_hex()[..2]) .join(stored.artifact_ref.digest_hex()); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); fs::remove_file(&path).unwrap(); fs::write(&path, b"replacement").unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap(); assert_eq!( store.quarantine_reconciliation(candidate), Err(ArtifactError::UnsafeRoot) ); assert!(path.exists()); } #[test] #[cfg(debug_assertions)] fn retries_after_post_rename_fsync_ambiguity() { let _guard = fault_guard(); for stage in [ "reconciliation_quarantine_source_fsync", "reconciliation_quarantine_destination_fsync", ] { let root = TestRoot::new(stage); let store = ArtifactStore::open(&root.0).unwrap(); store.put(stage.as_bytes()).unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); set_checkpoint(stage, FaultAction::Fail); assert_eq!( store.quarantine_reconciliation(candidate.clone()).unwrap(), ReconciliationMutation::Retryable, "stage={stage}" ); assert_eq!( checkpoint_hits("reconciliation_quarantine_source_fsync"), 1, "stage={stage}" ); assert_eq!( checkpoint_hits("reconciliation_quarantine_destination_fsync"), 1, "stage={stage}" ); clear_checkpoint(); assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::AlreadyQuarantined, "stage={stage}" ); } } #[test] #[cfg(debug_assertions)] fn mutation_syscall_failures_preserve_retryable_state() { let _guard = fault_guard(); let rename_root = TestRoot::new("rename-fail"); let rename_store = ArtifactStore::open(&rename_root.0).unwrap(); let stored = rename_store.put(b"rename failure bytes").unwrap(); let (_, candidates) = rename_store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); set_checkpoint("reconciliation_quarantine_rename", FaultAction::Fail); assert_eq!( rename_store.quarantine_reconciliation(candidate.clone()), Err(ArtifactError::Storage) ); assert_eq!( rename_store.read(&stored.artifact_ref).unwrap(), b"rename failure bytes" ); clear_checkpoint(); assert_eq!( rename_store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::Quarantined ); let delete_root = TestRoot::new("unlink-fail"); let delete_store = ArtifactStore::open(&delete_root.0).unwrap(); delete_store.put(b"unlink failure bytes").unwrap(); let (_, candidates) = delete_store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); delete_store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); let (_, candidates) = delete_store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); set_checkpoint("reconciliation_delete_unlink", FaultAction::Fail); assert_eq!( delete_store.delete_quarantined_reconciliation(candidate.clone()), Err(ArtifactError::Storage) ); clear_checkpoint(); assert_eq!( delete_store .delete_quarantined_reconciliation(candidate) .unwrap(), ReconciliationMutation::Deleted ); } #[test] #[cfg(debug_assertions)] fn storage_fault_returns_a_retryable_page_before_the_failed_entry() { let _guard = fault_guard(); let root = TestRoot::new("stat-storage"); let store = ArtifactStore::open(&root.0).unwrap(); let sha = root.0.join("sha256"); let shard = sha.join("aa"); fs::create_dir(&sha).unwrap(); fs::create_dir(&shard).unwrap(); fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap(); fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap(); for suffix in ['1', '2'] { let path = shard.join(format!("aa{}", suffix.to_string().repeat(62))); fs::write(&path, b"valid opaque candidate").unwrap(); fs::set_permissions(path, fs::Permissions::from_mode(0o400)).unwrap(); } set_checkpoint_on_hit("reconciliation_stat", FaultAction::Fail, 2); let (report, mut candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.stop, ReconciliationScanStop::Retryable); assert_eq!(report.scanned, 1); assert_eq!(report.final_entries, 1); assert_eq!(candidates.len(), 1); let continuation = report.continuation; clear_checkpoint(); let (report, resumed) = store .scan_reconciliation(continuation, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.stop, ReconciliationScanStop::Complete); assert_eq!(report.scanned, 1); assert_eq!(report.final_entries, 1); candidates.extend(resumed); assert_eq!(candidates.len(), 2); for candidate in candidates { assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::Quarantined ); } let (report, quarantined) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.final_entries, 0); assert_eq!(report.quarantined_entries, 2); assert_eq!(quarantined.len(), 2); } #[test] #[cfg(debug_assertions)] fn getdents_retry_restores_the_pre_call_directory_cookie() { let _guard = fault_guard(); let root = TestRoot::new("getdents-storage"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"getdents retry bytes").unwrap(); set_checkpoint("reconciliation_getdents", FaultAction::Fail); let (report, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.stop, ReconciliationScanStop::Retryable); assert!(candidates.is_empty()); clear_checkpoint(); let (report, candidates) = store .scan_reconciliation(report.continuation, 1, 8) .unwrap(); assert_eq!(report.stop, ReconciliationScanStop::ScanBudget); assert_eq!(report.traversal_syscalls, 1); assert!(candidates.is_empty()); let (report, candidates) = store .scan_reconciliation(report.continuation, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(report.stop, ReconciliationScanStop::Complete); assert_eq!(candidates.len(), 1); } #[test] #[cfg(debug_assertions)] fn crash_windows_are_recoverable() { if let Some(root) = std::env::var_os("CRANK_RECONCILIATION_CHILD_ROOT") { let action = std::env::var("CRANK_RECONCILIATION_CHILD_ACTION").unwrap(); set_checkpoint( std::env::var("CRANK_RECONCILIATION_CHILD_STAGE").unwrap(), FaultAction::Exit, ); let store = ArtifactStore::open(PathBuf::from(root)).unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.into_iter().next().unwrap(); if action == "quarantine" { let _ = store.quarantine_reconciliation(candidate); } else { let _ = store.delete_quarantined_reconciliation(candidate); } panic!("checkpoint did not terminate the child"); } for (stage, action) in [ ("reconciliation_quarantine_rename", "quarantine"), ("reconciliation_quarantine_source_fsync", "quarantine"), ("reconciliation_quarantine_destination_fsync", "quarantine"), ("reconciliation_delete_unlink", "delete"), ("reconciliation_delete_fsync", "delete"), ] { let root = TestRoot::new(&format!("crash-{stage}")); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation crash bytes").unwrap(); if action == "delete" { let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); } let status = Command::new(std::env::current_exe().unwrap()) .args(["--exact", "crash_windows_are_recoverable", "--nocapture"]) .env("CRANK_RECONCILIATION_CHILD_ROOT", &root.0) .env("CRANK_RECONCILIATION_CHILD_ACTION", action) .env("CRANK_RECONCILIATION_CHILD_STAGE", stage) .status() .unwrap(); assert_eq!(status.code(), Some(86), "stage={stage}"); let store = ArtifactStore::open(&root.0).unwrap(); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); for candidate in candidates { match candidate.namespace() { ReconciliationNamespace::Final => { let _ = store.quarantine_reconciliation(candidate).unwrap(); } ReconciliationNamespace::Quarantine => { let _ = store.delete_quarantined_reconciliation(candidate).unwrap(); } } } let (_, remaining) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); assert!(remaining.is_empty(), "stage={stage}"); assert_eq!( store .put(b"reconciliation crash bytes") .unwrap() .artifact_ref, stored.artifact_ref ); } } #[test] fn registration_revalidates_content_and_enforces_grace_without_candidate_identity_access() { let _guard = fault_guard(); let root = TestRoot::new("registration"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation registration").unwrap(); let (_, mut candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); let candidate = candidates.pop().unwrap(); assert_eq!(format!("{candidate:?}"), "ReconciliationCandidate(..)"); assert!(!candidate.is_grace_eligible(Duration::from_secs(3600), SystemTime::now())); assert_eq!( format!( "{:?}", store .register_reconciliation(&candidate, Duration::from_secs(3600), SystemTime::now()) .unwrap() ), "ReconciliationRegistration(..)" ); let registration = store .register_reconciliation( &candidate, Duration::ZERO, SystemTime::now() + Duration::from_secs(3600), ) .unwrap(); let registered = match registration { ReconciliationRegistration::Registered(registered) => registered, ReconciliationRegistration::NotEligible => panic!("zero grace must be eligible"), }; assert_eq!(registered.artifact_ref(), &stored.artifact_ref); assert_eq!( registered.size_bytes(), b"reconciliation registration".len() ); // The candidate remains usable for the physical mutation after the // registration bridge borrowed it. assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::Quarantined ); } #[test] fn namespace_scan_can_finish_quarantine_recovery_before_final_sweep() { let _guard = fault_guard(); let root = TestRoot::new("namespace-order"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"quarantine-first recovery").unwrap(); let (_, candidates) = store .scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8) .unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); store.put(b"quarantine-first recovery").unwrap(); let (quarantine_report, quarantine_candidates) = store .scan_reconciliation_namespace( ReconciliationNamespace::Quarantine, None, FULL_SCAN_BUDGET, 8, ) .unwrap(); assert_eq!(quarantine_report.stop, ReconciliationScanStop::Complete); assert_eq!(quarantine_report.final_entries, 0); assert_eq!(quarantine_report.quarantined_entries, 1); assert_eq!(quarantine_candidates.len(), 1); assert!(quarantine_report.continuation.is_none()); let (final_report, final_candidates) = store .scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8) .unwrap(); assert_eq!(final_report.stop, ReconciliationScanStop::Complete); assert_eq!(final_report.final_entries, 1); assert_eq!(final_report.quarantined_entries, 0); assert_eq!(final_candidates.len(), 1); } #[test] fn namespace_scan_rejects_a_cursor_from_a_broader_scan() { let _guard = fault_guard(); let root = TestRoot::new("namespace-cursor-scope"); let store = ArtifactStore::open(&root.0).unwrap(); let (report, _) = store.scan_reconciliation(None, 0, 1).unwrap(); let cursor = report .continuation .expect("zero-budget scan retains cursor"); assert!(matches!( store.scan_reconciliation_namespace( ReconciliationNamespace::Quarantine, Some(cursor), FULL_SCAN_BUDGET, 1, ), Err(ArtifactError::UnsafeRoot) )); } #[test] fn full_scan_rejects_a_quarantine_only_cursor() { let _guard = fault_guard(); let root = TestRoot::new("namespace-cursor-narrow"); let store = ArtifactStore::open(&root.0).unwrap(); let (report, _) = store .scan_reconciliation_namespace(ReconciliationNamespace::Quarantine, None, 0, 1) .unwrap(); let cursor = report .continuation .expect("zero-budget namespace scan retains cursor"); assert!(matches!( store.scan_reconciliation(Some(cursor), FULL_SCAN_BUDGET, 1), Err(ArtifactError::UnsafeRoot) )); } #[test] fn presence_probe_distinguishes_final_quarantine_and_absent_without_disclosure() { let _guard = fault_guard(); let root = TestRoot::new("presence"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"presence probe").unwrap(); assert_eq!( store.reconciliation_presence(&stored.artifact_ref).unwrap(), ReconciliationPresence::Final ); let (_, candidates) = store .scan_reconciliation(None, FULL_SCAN_BUDGET, 8) .unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); assert_eq!( store.reconciliation_presence(&stored.artifact_ref).unwrap(), ReconciliationPresence::Quarantine ); // A republish can leave both locations during recovery. Quarantine takes // precedence so the stale inode is still scheduled for deletion. store.put(b"presence probe").unwrap(); assert_eq!( store.reconciliation_presence(&stored.artifact_ref).unwrap(), ReconciliationPresence::Quarantine ); let (_, candidates) = store .scan_reconciliation_namespace( ReconciliationNamespace::Quarantine, None, FULL_SCAN_BUDGET, 8, ) .unwrap(); store .delete_quarantined_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); assert_eq!( store.reconciliation_presence(&stored.artifact_ref).unwrap(), ReconciliationPresence::Final ); let (_, candidates) = store .scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8) .unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); let (_, candidates) = store .scan_reconciliation_namespace( ReconciliationNamespace::Quarantine, None, FULL_SCAN_BUDGET, 8, ) .unwrap(); store .delete_quarantined_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); assert_eq!( store.reconciliation_presence(&stored.artifact_ref).unwrap(), ReconciliationPresence::Absent ); } #[cfg(debug_assertions)] #[test] fn presence_probe_classifies_storage_failure_as_retryable() { let _guard = fault_guard(); let root = TestRoot::new("presence-retryable"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"presence retryable").unwrap(); set_checkpoint("reconciliation_presence_open", FaultAction::Fail); assert_eq!( store.reconciliation_presence(&stored.artifact_ref).unwrap(), ReconciliationPresence::Retryable ); clear_checkpoint(); assert_eq!( store.reconciliation_presence(&stored.artifact_ref).unwrap(), ReconciliationPresence::Final ); }