use std::{ fs, os::unix::fs::PermissionsExt, path::PathBuf, process::Command, sync::{ Arc, atomic::{AtomicU64, Ordering}, }, thread, time::Duration, }; use crank_artifacts::{ ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace, }; #[cfg(debug_assertions)] use crank_artifacts::test_support::{ FaultAction, clear_checkpoint, set_checkpoint, wait_until_held, }; #[cfg(debug_assertions)] use std::sync::{Mutex, OnceLock}; static NEXT_ROOT: AtomicU64 = AtomicU64::new(0); struct TestRoot(PathBuf); impl TestRoot { fn new(name: &str) -> Self { let path = std::env::temp_dir().join(format!( "crank-artifacts-reconciliation-{name}-{}-{}", std::process::id(), NEXT_ROOT.fetch_add(1, Ordering::Relaxed) )); fs::create_dir(&path).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); Self(path) } } impl Drop for TestRoot { fn drop(&mut self) { let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700)); let _ = fs::remove_dir_all(&self.0); } } #[cfg(debug_assertions)] fn fault_guard() -> std::sync::MutexGuard<'static, ()> { static GUARD: OnceLock> = OnceLock::new(); GUARD.get_or_init(|| Mutex::new(())).lock().unwrap() } #[test] fn paginates_final_entries_without_disclosing_locations() { let root = TestRoot::new("pages"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"reconciliation first").unwrap(); store.put(b"reconciliation second").unwrap(); let mut continuation = None; let mut candidates = Vec::new(); loop { let (report, mut page) = store.scan_reconciliation(continuation, 1, 1).unwrap(); assert!(report.scanned <= 1); assert!(page.len() <= 1); candidates.append(&mut page); continuation = report.continuation; if continuation.is_none() { break; } } assert_eq!(candidates.len(), 2); assert!( candidates .iter() .all(|candidate| candidate.namespace() == ReconciliationNamespace::Final) ); for candidate in candidates { assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::Quarantined ); } let (_, quarantined_entries) = store.scan_reconciliation(None, 512, 8).unwrap(); assert_eq!(quarantined_entries.len(), 2); assert!( quarantined_entries .iter() .all(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine) ); } #[test] fn classifies_malformed_and_unsafe_entries_with_a_usable_page() { let root = TestRoot::new("malformed"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation valid").unwrap(); let shard = root .0 .join("sha256") .join(&stored.artifact_ref.digest_hex()[..2]); fs::write(shard.join("not-a-digest"), b"junk").unwrap(); let unsafe_name = format!( "{}{}", &stored.artifact_ref.digest_hex()[..2], "f".repeat(62) ); std::os::unix::fs::symlink(shard.join("not-a-digest"), shard.join(unsafe_name)).unwrap(); let directory_name = format!( "{}{}", &stored.artifact_ref.digest_hex()[..2], "e".repeat(62) ); fs::create_dir(shard.join(directory_name)).unwrap(); let hardlink_name = format!( "{}{}", &stored.artifact_ref.digest_hex()[..2], "d".repeat(62) ); fs::hard_link(shard.join("not-a-digest"), shard.join(hardlink_name)).unwrap(); let (report, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); assert!(report.malformed >= 1); assert!(report.unsafe_entries >= 3); assert_eq!(candidates.len(), 1); assert!(report.continuation.is_none()); } #[test] fn rejects_a_cursor_from_another_store() { let first_root = TestRoot::new("foreign-cursor-first"); let second_root = TestRoot::new("foreign-cursor-second"); let first = ArtifactStore::open(&first_root.0).unwrap(); let second = ArtifactStore::open(&second_root.0).unwrap(); first.put(b"cursor source").unwrap(); let (report, _) = first.scan_reconciliation(None, 1, 1).unwrap(); assert!(matches!( second.scan_reconciliation(report.continuation, 1, 1), Err(ArtifactError::UnsafeRoot) )); } #[test] #[cfg(debug_assertions)] fn disappearing_entry_after_readdir_is_skipped() { let _guard = fault_guard(); let root = TestRoot::new("disappearing"); let store = Arc::new(ArtifactStore::open(&root.0).unwrap()); let stored = store.put(b"disappearing reconciliation entry").unwrap(); let path = root .0 .join("sha256") .join(&stored.artifact_ref.digest_hex()[..2]) .join(stored.artifact_ref.digest_hex()); set_checkpoint("reconciliation_before_stat", FaultAction::Hold); let scan_store = Arc::clone(&store); let scan = thread::spawn(move || scan_store.scan_reconciliation(None, 512, 8)); assert!(wait_until_held(Duration::from_secs(2))); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); fs::remove_file(&path).unwrap(); clear_checkpoint(); let (_, candidates) = scan.join().unwrap().unwrap(); assert!(candidates.is_empty()); } #[test] fn quarantine_and_delete_are_idempotent() { let root = TestRoot::new("mutation"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation mutation").unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let candidate = candidates.into_iter().next().unwrap(); assert_eq!( store.quarantine_reconciliation(candidate.clone()).unwrap(), ReconciliationMutation::Quarantined ); assert_eq!( store.read(&stored.artifact_ref), Err(ArtifactError::NotFound) ); assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::AlreadyQuarantined ); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let quarantined = candidates .into_iter() .find(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine) .unwrap(); assert_eq!( store .delete_quarantined_reconciliation(quarantined.clone()) .unwrap(), ReconciliationMutation::Deleted ); assert_eq!( store .delete_quarantined_reconciliation(quarantined) .unwrap(), ReconciliationMutation::AlreadyAbsent ); } #[test] fn quarantine_preserves_old_inode_when_the_digest_is_republished() { let root = TestRoot::new("no-clobber"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation no-clobber").unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let original = candidates.into_iter().next().unwrap(); assert_eq!( store.quarantine_reconciliation(original.clone()).unwrap(), ReconciliationMutation::Quarantined ); store.put(b"reconciliation no-clobber").unwrap(); assert_eq!( store.quarantine_reconciliation(original).unwrap(), ReconciliationMutation::AlreadyQuarantined ); assert_eq!( store.read(&stored.artifact_ref).unwrap(), b"reconciliation no-clobber" ); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); assert_eq!(candidates.len(), 2); assert!( candidates .iter() .any(|candidate| candidate.namespace() == ReconciliationNamespace::Final) ); assert!( candidates .iter() .any(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine) ); } #[test] fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() { let root = TestRoot::new("quarantine-replaced"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store .put(b"reconciliation quarantined replacement") .unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let candidate = candidates.into_iter().next().unwrap(); let path = root .0 .join("quarantine") .join(&stored.artifact_ref.digest_hex()[..2]) .join(stored.artifact_ref.digest_hex()); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); fs::remove_file(&path).unwrap(); fs::write(&path, b"replacement").unwrap(); let sibling = path.with_extension("link"); fs::hard_link(&path, &sibling).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap(); assert_eq!( store.delete_quarantined_reconciliation(candidate), Err(ArtifactError::UnsafeRoot) ); assert!(path.exists()); assert!(sibling.exists()); } #[test] #[cfg(debug_assertions)] fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() { let _guard = fault_guard(); let root = TestRoot::new("delete-fsync"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"reconciliation delete fsync").unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let candidate = candidates.into_iter().next().unwrap(); set_checkpoint("reconciliation_delete_fsync", FaultAction::Fail); assert_eq!( store .delete_quarantined_reconciliation(candidate.clone()) .unwrap(), ReconciliationMutation::Retryable ); clear_checkpoint(); assert_eq!( store.delete_quarantined_reconciliation(candidate).unwrap(), ReconciliationMutation::AlreadyAbsent ); } #[test] fn rejects_a_replaced_final_inode() { let root = TestRoot::new("replaced"); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation original").unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let candidate = candidates.into_iter().next().unwrap(); let path = root .0 .join("sha256") .join(&stored.artifact_ref.digest_hex()[..2]) .join(stored.artifact_ref.digest_hex()); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); fs::remove_file(&path).unwrap(); fs::write(&path, b"replacement").unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap(); assert_eq!( store.quarantine_reconciliation(candidate), Err(ArtifactError::UnsafeRoot) ); assert!(path.exists()); } #[test] #[cfg(debug_assertions)] fn retries_after_post_rename_fsync_ambiguity() { let _guard = fault_guard(); let root = TestRoot::new("fsync"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"reconciliation fault").unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let candidate = candidates.into_iter().next().unwrap(); set_checkpoint("reconciliation_quarantine_source_fsync", FaultAction::Fail); assert_eq!( store.quarantine_reconciliation(candidate.clone()).unwrap(), ReconciliationMutation::Retryable ); clear_checkpoint(); assert_eq!( store.quarantine_reconciliation(candidate).unwrap(), ReconciliationMutation::AlreadyQuarantined ); } #[test] #[cfg(debug_assertions)] fn storage_fault_during_stat_propagates() { let _guard = fault_guard(); let root = TestRoot::new("stat-storage"); let store = ArtifactStore::open(&root.0).unwrap(); store.put(b"reconciliation stat storage").unwrap(); set_checkpoint("reconciliation_stat", FaultAction::Fail); assert!(matches!( store.scan_reconciliation(None, 512, 8), Err(ArtifactError::Storage) )); clear_checkpoint(); } #[test] #[cfg(debug_assertions)] fn crash_windows_are_recoverable() { if let Some(root) = std::env::var_os("CRANK_RECONCILIATION_CHILD_ROOT") { let action = std::env::var("CRANK_RECONCILIATION_CHILD_ACTION").unwrap(); set_checkpoint( std::env::var("CRANK_RECONCILIATION_CHILD_STAGE").unwrap(), FaultAction::Exit, ); let store = ArtifactStore::open(PathBuf::from(root)).unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); let candidate = candidates.into_iter().next().unwrap(); if action == "quarantine" { let _ = store.quarantine_reconciliation(candidate); } else { let _ = store.delete_quarantined_reconciliation(candidate); } panic!("checkpoint did not terminate the child"); } for (stage, action) in [ ("reconciliation_quarantine_rename", "quarantine"), ("reconciliation_quarantine_source_fsync", "quarantine"), ("reconciliation_quarantine_destination_fsync", "quarantine"), ("reconciliation_delete_unlink", "delete"), ("reconciliation_delete_fsync", "delete"), ] { let root = TestRoot::new(&format!("crash-{stage}")); let store = ArtifactStore::open(&root.0).unwrap(); let stored = store.put(b"reconciliation crash bytes").unwrap(); if action == "delete" { let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); store .quarantine_reconciliation(candidates.into_iter().next().unwrap()) .unwrap(); } let status = Command::new(std::env::current_exe().unwrap()) .args(["--exact", "crash_windows_are_recoverable", "--nocapture"]) .env("CRANK_RECONCILIATION_CHILD_ROOT", &root.0) .env("CRANK_RECONCILIATION_CHILD_ACTION", action) .env("CRANK_RECONCILIATION_CHILD_STAGE", stage) .status() .unwrap(); assert_eq!(status.code(), Some(86), "stage={stage}"); let store = ArtifactStore::open(&root.0).unwrap(); let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap(); for candidate in candidates { match candidate.namespace() { ReconciliationNamespace::Final => { let _ = store.quarantine_reconciliation(candidate).unwrap(); } ReconciliationNamespace::Quarantine => { let _ = store.delete_quarantined_reconciliation(candidate).unwrap(); } } } let (_, remaining) = store.scan_reconciliation(None, 512, 8).unwrap(); assert!(remaining.is_empty(), "stage={stage}"); assert_eq!( store .put(b"reconciliation crash bytes") .unwrap() .artifact_ref, stored.artifact_ref ); } }