772 lines
27 KiB
Rust
772 lines
27 KiB
Rust
use std::{
|
|
fs,
|
|
os::unix::fs::{MetadataExt, PermissionsExt},
|
|
path::PathBuf,
|
|
sync::atomic::{AtomicU64, Ordering},
|
|
};
|
|
|
|
#[cfg(debug_assertions)]
|
|
use std::{process::Command, sync::Arc, thread, time::Duration};
|
|
|
|
use crank_artifacts::{
|
|
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
|
|
ReconciliationScanStop,
|
|
};
|
|
|
|
#[cfg(debug_assertions)]
|
|
use crank_artifacts::test_support::{
|
|
FaultAction, checkpoint_hits, clear_checkpoint, reset_traversal_calls, set_checkpoint,
|
|
set_checkpoint_on_hit, traversal_calls, wait_until_held,
|
|
};
|
|
#[cfg(debug_assertions)]
|
|
use std::sync::{Mutex, OnceLock};
|
|
|
|
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
|
|
const FULL_SCAN_BUDGET: usize = 2048;
|
|
|
|
struct TestRoot(PathBuf);
|
|
|
|
impl TestRoot {
|
|
fn new(name: &str) -> Self {
|
|
let path = std::env::temp_dir().join(format!(
|
|
"crank-artifacts-reconciliation-{name}-{}-{}",
|
|
std::process::id(),
|
|
NEXT_ROOT.fetch_add(1, Ordering::Relaxed)
|
|
));
|
|
fs::create_dir(&path).unwrap();
|
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
|
|
Self(path)
|
|
}
|
|
}
|
|
|
|
impl Drop for TestRoot {
|
|
fn drop(&mut self) {
|
|
let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
|
|
let _ = fs::remove_dir_all(&self.0);
|
|
}
|
|
}
|
|
|
|
#[cfg(debug_assertions)]
|
|
fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
|
|
static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
|
|
GUARD.get_or_init(|| Mutex::new(())).lock().unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn paginates_final_entries_without_disclosing_locations() {
|
|
let root = TestRoot::new("pages");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
store.put(b"reconciliation first").unwrap();
|
|
store.put(b"reconciliation second").unwrap();
|
|
|
|
let mut continuation = None;
|
|
let mut candidates = Vec::new();
|
|
let mut final_entries = 0;
|
|
let mut scanned = 0;
|
|
loop {
|
|
let (report, mut page) = store.scan_reconciliation(continuation, 1, 1).unwrap();
|
|
assert_eq!(report.traversal_syscalls, 1);
|
|
assert!(report.scanned <= 1);
|
|
assert_eq!(report.final_entries, page.len());
|
|
assert_eq!(report.quarantined_entries, 0);
|
|
assert!(page.len() <= 1);
|
|
scanned += report.scanned;
|
|
final_entries += report.final_entries;
|
|
if let Some(cursor) = report.continuation.as_ref() {
|
|
assert_eq!(format!("{cursor:?}"), "ReconciliationCursor(..)");
|
|
}
|
|
for candidate in &page {
|
|
assert_eq!(format!("{candidate:?}"), "ReconciliationCandidate(..)");
|
|
}
|
|
candidates.append(&mut page);
|
|
continuation = report.continuation;
|
|
if continuation.is_none() {
|
|
break;
|
|
}
|
|
}
|
|
assert_eq!(scanned, 2);
|
|
assert_eq!(final_entries, 2);
|
|
assert_eq!(candidates.len(), 2);
|
|
assert!(
|
|
candidates
|
|
.iter()
|
|
.all(|candidate| candidate.namespace() == ReconciliationNamespace::Final)
|
|
);
|
|
for candidate in candidates {
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate).unwrap(),
|
|
ReconciliationMutation::Quarantined
|
|
);
|
|
}
|
|
let (report, quarantined_entries) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(quarantined_entries.len(), 2);
|
|
assert_eq!(report.scanned, 2);
|
|
assert_eq!(report.final_entries, 0);
|
|
assert_eq!(report.quarantined_entries, 2);
|
|
assert_eq!(report.stop, ReconciliationScanStop::Complete);
|
|
assert!(
|
|
quarantined_entries
|
|
.iter()
|
|
.all(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
|
|
let root = TestRoot::new("malformed");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"reconciliation valid").unwrap();
|
|
let shard = root
|
|
.0
|
|
.join("sha256")
|
|
.join(&stored.artifact_ref.digest_hex()[..2]);
|
|
fs::write(shard.join("not-a-digest"), b"junk").unwrap();
|
|
let unsafe_name = format!(
|
|
"{}{}",
|
|
&stored.artifact_ref.digest_hex()[..2],
|
|
"f".repeat(62)
|
|
);
|
|
std::os::unix::fs::symlink(shard.join("not-a-digest"), shard.join(unsafe_name)).unwrap();
|
|
let directory_name = format!(
|
|
"{}{}",
|
|
&stored.artifact_ref.digest_hex()[..2],
|
|
"e".repeat(62)
|
|
);
|
|
fs::create_dir(shard.join(directory_name)).unwrap();
|
|
let hardlink_name = format!(
|
|
"{}{}",
|
|
&stored.artifact_ref.digest_hex()[..2],
|
|
"d".repeat(62)
|
|
);
|
|
fs::hard_link(shard.join("not-a-digest"), shard.join(hardlink_name)).unwrap();
|
|
|
|
let (report, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.scanned, 5);
|
|
assert_eq!(report.malformed, 1);
|
|
assert_eq!(report.unsafe_entries, 3);
|
|
assert_eq!(report.final_entries, 1);
|
|
assert_eq!(report.quarantined_entries, 0);
|
|
assert_eq!(candidates.len(), 1);
|
|
assert!(report.continuation.is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn skips_a_valid_publish_temp_without_classifying_it_as_malformed() {
|
|
let root = TestRoot::new("valid-temp");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"reconciliation with concurrent temp").unwrap();
|
|
let shard = root
|
|
.0
|
|
.join("sha256")
|
|
.join(&stored.artifact_ref.digest_hex()[..2]);
|
|
fs::write(
|
|
shard.join(".crank-artifact-tmp-v1-0123456789abcdef0123456789abcdef-7-9"),
|
|
b"in-flight",
|
|
)
|
|
.unwrap();
|
|
|
|
let (report, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.scanned, 2);
|
|
assert_eq!(report.malformed, 0);
|
|
assert_eq!(report.final_entries, 1);
|
|
assert_eq!(candidates.len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn unsafe_canonical_shard_does_not_starve_later_shards() {
|
|
let root = TestRoot::new("unsafe-shard");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let sha = root.0.join("sha256");
|
|
fs::create_dir(&sha).unwrap();
|
|
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
|
|
let unsafe_shard = sha.join("00");
|
|
fs::create_dir(&unsafe_shard).unwrap();
|
|
fs::set_permissions(&unsafe_shard, fs::Permissions::from_mode(0o000)).unwrap();
|
|
let shard = sha.join("ff");
|
|
fs::create_dir(&shard).unwrap();
|
|
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
|
|
let name = "f".repeat(64);
|
|
fs::write(shard.join(&name), b"later valid entry").unwrap();
|
|
fs::set_permissions(shard.join(&name), fs::Permissions::from_mode(0o400)).unwrap();
|
|
|
|
let (report, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.stop, ReconciliationScanStop::Complete);
|
|
assert_eq!(report.unsafe_entries, 1);
|
|
assert_eq!(report.final_entries, 1);
|
|
assert!(candidates.iter().any(|candidate| {
|
|
candidate.namespace() == ReconciliationNamespace::Final
|
|
&& format!("{candidate:?}") == "ReconciliationCandidate(..)"
|
|
}));
|
|
fs::set_permissions(unsafe_shard, fs::Permissions::from_mode(0o700)).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn traversal_never_exceeds_the_exact_syscall_budget() {
|
|
let root = TestRoot::new("syscall-budget");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
store.put(b"bounded traversal syscall accounting").unwrap();
|
|
|
|
for budget in 0..=8 {
|
|
reset_traversal_calls();
|
|
let (report, _) = store.scan_reconciliation(None, budget, 8).unwrap();
|
|
assert!(report.traversal_syscalls <= budget, "budget={budget}");
|
|
assert_eq!(report.traversal_syscalls, budget, "budget={budget}");
|
|
assert_eq!(traversal_calls(), report.traversal_syscalls);
|
|
assert_eq!(report.stop, ReconciliationScanStop::ScanBudget);
|
|
}
|
|
|
|
reset_traversal_calls();
|
|
let (report, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 1)
|
|
.unwrap();
|
|
assert_eq!(candidates.len(), 1);
|
|
assert_eq!(report.stop, ReconciliationScanStop::ResultLimit);
|
|
assert!(report.traversal_syscalls <= FULL_SCAN_BUDGET);
|
|
assert_eq!(traversal_calls(), report.traversal_syscalls);
|
|
}
|
|
|
|
#[test]
|
|
fn rejects_a_cursor_from_another_store() {
|
|
let first_root = TestRoot::new("foreign-cursor-first");
|
|
let second_root = TestRoot::new("foreign-cursor-second");
|
|
let first = ArtifactStore::open(&first_root.0).unwrap();
|
|
let second = ArtifactStore::open(&second_root.0).unwrap();
|
|
first.put(b"cursor source").unwrap();
|
|
let (report, _) = first.scan_reconciliation(None, 1, 1).unwrap();
|
|
|
|
assert!(matches!(
|
|
second.scan_reconciliation(report.continuation, 1, 1),
|
|
Err(ArtifactError::UnsafeRoot)
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn disappearing_entry_after_readdir_is_skipped() {
|
|
let _guard = fault_guard();
|
|
let root = TestRoot::new("disappearing");
|
|
let store = Arc::new(ArtifactStore::open(&root.0).unwrap());
|
|
let stored = store.put(b"disappearing reconciliation entry").unwrap();
|
|
let path = root
|
|
.0
|
|
.join("sha256")
|
|
.join(&stored.artifact_ref.digest_hex()[..2])
|
|
.join(stored.artifact_ref.digest_hex());
|
|
set_checkpoint("reconciliation_before_stat", FaultAction::Hold);
|
|
let scan_store = Arc::clone(&store);
|
|
let scan = thread::spawn(move || scan_store.scan_reconciliation(None, FULL_SCAN_BUDGET, 8));
|
|
assert!(wait_until_held(Duration::from_secs(2)));
|
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
|
|
fs::remove_file(&path).unwrap();
|
|
clear_checkpoint();
|
|
|
|
let (_, candidates) = scan.join().unwrap().unwrap();
|
|
assert!(candidates.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn quarantine_and_delete_are_idempotent() {
|
|
let root = TestRoot::new("mutation");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"reconciliation mutation").unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate.clone()).unwrap(),
|
|
ReconciliationMutation::Quarantined
|
|
);
|
|
assert_eq!(
|
|
store.read(&stored.artifact_ref),
|
|
Err(ArtifactError::NotFound)
|
|
);
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate).unwrap(),
|
|
ReconciliationMutation::AlreadyQuarantined
|
|
);
|
|
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let quarantined = candidates
|
|
.into_iter()
|
|
.find(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
|
|
.unwrap();
|
|
assert_eq!(
|
|
store
|
|
.delete_quarantined_reconciliation(quarantined.clone())
|
|
.unwrap(),
|
|
ReconciliationMutation::Deleted
|
|
);
|
|
assert_eq!(
|
|
store
|
|
.delete_quarantined_reconciliation(quarantined)
|
|
.unwrap(),
|
|
ReconciliationMutation::AlreadyAbsent
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
|
|
let root = TestRoot::new("no-clobber");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"reconciliation no-clobber").unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let original = candidates.into_iter().next().unwrap();
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(original.clone()).unwrap(),
|
|
ReconciliationMutation::Quarantined
|
|
);
|
|
store.put(b"reconciliation no-clobber").unwrap();
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(original).unwrap(),
|
|
ReconciliationMutation::AlreadyQuarantined
|
|
);
|
|
assert_eq!(
|
|
store.read(&stored.artifact_ref).unwrap(),
|
|
b"reconciliation no-clobber"
|
|
);
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(candidates.len(), 2);
|
|
assert!(
|
|
candidates
|
|
.iter()
|
|
.any(|candidate| candidate.namespace() == ReconciliationNamespace::Final)
|
|
);
|
|
assert!(
|
|
candidates
|
|
.iter()
|
|
.any(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn quarantine_refuses_to_replace_an_unrelated_inode() {
|
|
let root = TestRoot::new("unrelated-collision");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"canonical final survives collision").unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
let digest = stored.artifact_ref.digest_hex();
|
|
let quarantine = root.0.join("quarantine");
|
|
let shard = quarantine.join(&digest[..2]);
|
|
fs::create_dir(&quarantine).unwrap();
|
|
fs::create_dir(&shard).unwrap();
|
|
fs::set_permissions(&quarantine, fs::Permissions::from_mode(0o700)).unwrap();
|
|
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
|
|
let collision = shard.join(digest);
|
|
fs::write(&collision, b"unrelated quarantine inode").unwrap();
|
|
fs::set_permissions(&collision, fs::Permissions::from_mode(0o400)).unwrap();
|
|
let collision_ino = fs::metadata(&collision).unwrap().ino();
|
|
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate),
|
|
Err(ArtifactError::UnsafeRoot)
|
|
);
|
|
assert_eq!(
|
|
store.read(&stored.artifact_ref).unwrap(),
|
|
b"canonical final survives collision"
|
|
);
|
|
assert_eq!(fs::read(collision).unwrap(), b"unrelated quarantine inode");
|
|
assert_eq!(
|
|
fs::metadata(shard.join(digest)).unwrap().ino(),
|
|
collision_ino
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
|
|
let root = TestRoot::new("quarantine-replaced");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store
|
|
.put(b"reconciliation quarantined replacement")
|
|
.unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
store
|
|
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
|
.unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
let path = root
|
|
.0
|
|
.join("quarantine")
|
|
.join(&stored.artifact_ref.digest_hex()[..2])
|
|
.join(stored.artifact_ref.digest_hex());
|
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
|
|
fs::remove_file(&path).unwrap();
|
|
fs::write(&path, b"replacement").unwrap();
|
|
let sibling = path.with_extension("link");
|
|
fs::hard_link(&path, &sibling).unwrap();
|
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap();
|
|
|
|
assert_eq!(
|
|
store.delete_quarantined_reconciliation(candidate),
|
|
Err(ArtifactError::UnsafeRoot)
|
|
);
|
|
assert!(path.exists());
|
|
assert!(sibling.exists());
|
|
}
|
|
|
|
#[test]
|
|
fn delete_revalidates_a_post_scan_hardlink_without_inode_replacement() {
|
|
let root = TestRoot::new("quarantine-hardlink");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"same quarantined inode gains a link").unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
store
|
|
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
|
.unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
let path = root
|
|
.0
|
|
.join("quarantine")
|
|
.join(&stored.artifact_ref.digest_hex()[..2])
|
|
.join(stored.artifact_ref.digest_hex());
|
|
let sibling = path.with_extension("link");
|
|
let original_ino = fs::metadata(&path).unwrap().ino();
|
|
fs::hard_link(&path, &sibling).unwrap();
|
|
|
|
assert_eq!(
|
|
store.delete_quarantined_reconciliation(candidate),
|
|
Err(ArtifactError::UnsafeRoot)
|
|
);
|
|
assert_eq!(fs::metadata(&path).unwrap().nlink(), 2);
|
|
assert_eq!(fs::metadata(&path).unwrap().ino(), original_ino);
|
|
assert!(path.exists());
|
|
assert!(sibling.exists());
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() {
|
|
let _guard = fault_guard();
|
|
let root = TestRoot::new("delete-fsync");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
store.put(b"reconciliation delete fsync").unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
store
|
|
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
|
.unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
set_checkpoint("reconciliation_delete_fsync", FaultAction::Fail);
|
|
assert_eq!(
|
|
store
|
|
.delete_quarantined_reconciliation(candidate.clone())
|
|
.unwrap(),
|
|
ReconciliationMutation::Retryable
|
|
);
|
|
clear_checkpoint();
|
|
assert_eq!(
|
|
store.delete_quarantined_reconciliation(candidate).unwrap(),
|
|
ReconciliationMutation::AlreadyAbsent
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn rejects_a_replaced_final_inode() {
|
|
let root = TestRoot::new("replaced");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"reconciliation original").unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
let path = root
|
|
.0
|
|
.join("sha256")
|
|
.join(&stored.artifact_ref.digest_hex()[..2])
|
|
.join(stored.artifact_ref.digest_hex());
|
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
|
|
fs::remove_file(&path).unwrap();
|
|
fs::write(&path, b"replacement").unwrap();
|
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap();
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate),
|
|
Err(ArtifactError::UnsafeRoot)
|
|
);
|
|
assert!(path.exists());
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn retries_after_post_rename_fsync_ambiguity() {
|
|
let _guard = fault_guard();
|
|
for stage in [
|
|
"reconciliation_quarantine_source_fsync",
|
|
"reconciliation_quarantine_destination_fsync",
|
|
] {
|
|
let root = TestRoot::new(stage);
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
store.put(stage.as_bytes()).unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
set_checkpoint(stage, FaultAction::Fail);
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate.clone()).unwrap(),
|
|
ReconciliationMutation::Retryable,
|
|
"stage={stage}"
|
|
);
|
|
assert_eq!(
|
|
checkpoint_hits("reconciliation_quarantine_source_fsync"),
|
|
1,
|
|
"stage={stage}"
|
|
);
|
|
assert_eq!(
|
|
checkpoint_hits("reconciliation_quarantine_destination_fsync"),
|
|
1,
|
|
"stage={stage}"
|
|
);
|
|
clear_checkpoint();
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate).unwrap(),
|
|
ReconciliationMutation::AlreadyQuarantined,
|
|
"stage={stage}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn mutation_syscall_failures_preserve_retryable_state() {
|
|
let _guard = fault_guard();
|
|
|
|
let rename_root = TestRoot::new("rename-fail");
|
|
let rename_store = ArtifactStore::open(&rename_root.0).unwrap();
|
|
let stored = rename_store.put(b"rename failure bytes").unwrap();
|
|
let (_, candidates) = rename_store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
set_checkpoint("reconciliation_quarantine_rename", FaultAction::Fail);
|
|
assert_eq!(
|
|
rename_store.quarantine_reconciliation(candidate.clone()),
|
|
Err(ArtifactError::Storage)
|
|
);
|
|
assert_eq!(
|
|
rename_store.read(&stored.artifact_ref).unwrap(),
|
|
b"rename failure bytes"
|
|
);
|
|
clear_checkpoint();
|
|
assert_eq!(
|
|
rename_store.quarantine_reconciliation(candidate).unwrap(),
|
|
ReconciliationMutation::Quarantined
|
|
);
|
|
|
|
let delete_root = TestRoot::new("unlink-fail");
|
|
let delete_store = ArtifactStore::open(&delete_root.0).unwrap();
|
|
delete_store.put(b"unlink failure bytes").unwrap();
|
|
let (_, candidates) = delete_store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
delete_store
|
|
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
|
.unwrap();
|
|
let (_, candidates) = delete_store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
set_checkpoint("reconciliation_delete_unlink", FaultAction::Fail);
|
|
assert_eq!(
|
|
delete_store.delete_quarantined_reconciliation(candidate.clone()),
|
|
Err(ArtifactError::Storage)
|
|
);
|
|
clear_checkpoint();
|
|
assert_eq!(
|
|
delete_store
|
|
.delete_quarantined_reconciliation(candidate)
|
|
.unwrap(),
|
|
ReconciliationMutation::Deleted
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn storage_fault_returns_a_retryable_page_before_the_failed_entry() {
|
|
let _guard = fault_guard();
|
|
let root = TestRoot::new("stat-storage");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let sha = root.0.join("sha256");
|
|
let shard = sha.join("aa");
|
|
fs::create_dir(&sha).unwrap();
|
|
fs::create_dir(&shard).unwrap();
|
|
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
|
|
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
|
|
for suffix in ['1', '2'] {
|
|
let path = shard.join(format!("aa{}", suffix.to_string().repeat(62)));
|
|
fs::write(&path, b"valid opaque candidate").unwrap();
|
|
fs::set_permissions(path, fs::Permissions::from_mode(0o400)).unwrap();
|
|
}
|
|
|
|
set_checkpoint_on_hit("reconciliation_stat", FaultAction::Fail, 2);
|
|
let (report, mut candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.stop, ReconciliationScanStop::Retryable);
|
|
assert_eq!(report.scanned, 1);
|
|
assert_eq!(report.final_entries, 1);
|
|
assert_eq!(candidates.len(), 1);
|
|
let continuation = report.continuation;
|
|
clear_checkpoint();
|
|
|
|
let (report, resumed) = store
|
|
.scan_reconciliation(continuation, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.stop, ReconciliationScanStop::Complete);
|
|
assert_eq!(report.scanned, 1);
|
|
assert_eq!(report.final_entries, 1);
|
|
candidates.extend(resumed);
|
|
assert_eq!(candidates.len(), 2);
|
|
for candidate in candidates {
|
|
assert_eq!(
|
|
store.quarantine_reconciliation(candidate).unwrap(),
|
|
ReconciliationMutation::Quarantined
|
|
);
|
|
}
|
|
let (report, quarantined) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.final_entries, 0);
|
|
assert_eq!(report.quarantined_entries, 2);
|
|
assert_eq!(quarantined.len(), 2);
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn getdents_retry_restores_the_pre_call_directory_cookie() {
|
|
let _guard = fault_guard();
|
|
let root = TestRoot::new("getdents-storage");
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
store.put(b"getdents retry bytes").unwrap();
|
|
|
|
set_checkpoint("reconciliation_getdents", FaultAction::Fail);
|
|
let (report, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.stop, ReconciliationScanStop::Retryable);
|
|
assert!(candidates.is_empty());
|
|
clear_checkpoint();
|
|
|
|
let (report, candidates) = store
|
|
.scan_reconciliation(report.continuation, 1, 8)
|
|
.unwrap();
|
|
assert_eq!(report.stop, ReconciliationScanStop::ScanBudget);
|
|
assert_eq!(report.traversal_syscalls, 1);
|
|
assert!(candidates.is_empty());
|
|
|
|
let (report, candidates) = store
|
|
.scan_reconciliation(report.continuation, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert_eq!(report.stop, ReconciliationScanStop::Complete);
|
|
assert_eq!(candidates.len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
#[cfg(debug_assertions)]
|
|
fn crash_windows_are_recoverable() {
|
|
if let Some(root) = std::env::var_os("CRANK_RECONCILIATION_CHILD_ROOT") {
|
|
let action = std::env::var("CRANK_RECONCILIATION_CHILD_ACTION").unwrap();
|
|
set_checkpoint(
|
|
std::env::var("CRANK_RECONCILIATION_CHILD_STAGE").unwrap(),
|
|
FaultAction::Exit,
|
|
);
|
|
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
let candidate = candidates.into_iter().next().unwrap();
|
|
if action == "quarantine" {
|
|
let _ = store.quarantine_reconciliation(candidate);
|
|
} else {
|
|
let _ = store.delete_quarantined_reconciliation(candidate);
|
|
}
|
|
panic!("checkpoint did not terminate the child");
|
|
}
|
|
|
|
for (stage, action) in [
|
|
("reconciliation_quarantine_rename", "quarantine"),
|
|
("reconciliation_quarantine_source_fsync", "quarantine"),
|
|
("reconciliation_quarantine_destination_fsync", "quarantine"),
|
|
("reconciliation_delete_unlink", "delete"),
|
|
("reconciliation_delete_fsync", "delete"),
|
|
] {
|
|
let root = TestRoot::new(&format!("crash-{stage}"));
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let stored = store.put(b"reconciliation crash bytes").unwrap();
|
|
if action == "delete" {
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
store
|
|
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
|
.unwrap();
|
|
}
|
|
let status = Command::new(std::env::current_exe().unwrap())
|
|
.args(["--exact", "crash_windows_are_recoverable", "--nocapture"])
|
|
.env("CRANK_RECONCILIATION_CHILD_ROOT", &root.0)
|
|
.env("CRANK_RECONCILIATION_CHILD_ACTION", action)
|
|
.env("CRANK_RECONCILIATION_CHILD_STAGE", stage)
|
|
.status()
|
|
.unwrap();
|
|
assert_eq!(status.code(), Some(86), "stage={stage}");
|
|
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let (_, candidates) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
for candidate in candidates {
|
|
match candidate.namespace() {
|
|
ReconciliationNamespace::Final => {
|
|
let _ = store.quarantine_reconciliation(candidate).unwrap();
|
|
}
|
|
ReconciliationNamespace::Quarantine => {
|
|
let _ = store.delete_quarantined_reconciliation(candidate).unwrap();
|
|
}
|
|
}
|
|
}
|
|
let (_, remaining) = store
|
|
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
|
.unwrap();
|
|
assert!(remaining.is_empty(), "stage={stage}");
|
|
assert_eq!(
|
|
store
|
|
.put(b"reconciliation crash bytes")
|
|
.unwrap()
|
|
.artifact_ref,
|
|
stored.artifact_ref
|
|
);
|
|
}
|
|
}
|