bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

fix(artifacts): harden reconciliation scanner

Browse Source
This commit is contained in:
bsodfather
2026-08-27 12:48:49 +03:00
parent 889b1bdb57
commit 8784964fb2
6 changed files with 1087 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/crank-artifacts/tests/reconciliation.rs
+384 -43
View File
@@ -1,28 +1,28 @@
use std::{
fs,
os::unix::fs::PermissionsExt,
os::unix::fs::{MetadataExt, PermissionsExt},
path::PathBuf,
process::Command,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
thread,
time::Duration,
sync::atomic::{AtomicU64, Ordering},
};
#[cfg(debug_assertions)]
use std::{process::Command, sync::Arc, thread, time::Duration};
use crank_artifacts::{
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
ReconciliationScanStop,
};
#[cfg(debug_assertions)]
use crank_artifacts::test_support::{
FaultAction, clear_checkpoint, set_checkpoint, wait_until_held,
FaultAction, checkpoint_hits, clear_checkpoint, reset_traversal_calls, set_checkpoint,
set_checkpoint_on_hit, traversal_calls, wait_until_held,
};
#[cfg(debug_assertions)]
use std::sync::{Mutex, OnceLock};
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
const FULL_SCAN_BUDGET: usize = 2048;
struct TestRoot(PathBuf);
@@ -61,16 +61,31 @@ fn paginates_final_entries_without_disclosing_locations() {
let mut continuation = None;
let mut candidates = Vec::new();
let mut final_entries = 0;
let mut scanned = 0;
loop {
let (report, mut page) = store.scan_reconciliation(continuation, 1, 1).unwrap();
assert_eq!(report.traversal_syscalls, 1);
assert!(report.scanned <= 1);
assert_eq!(report.final_entries, page.len());
assert_eq!(report.quarantined_entries, 0);
assert!(page.len() <= 1);
scanned += report.scanned;
final_entries += report.final_entries;
if let Some(cursor) = report.continuation.as_ref() {
assert_eq!(format!("{cursor:?}"), "ReconciliationCursor(..)");
}
for candidate in &page {
assert_eq!(format!("{candidate:?}"), "ReconciliationCandidate(..)");
}
candidates.append(&mut page);
continuation = report.continuation;
if continuation.is_none() {
break;
}
}
assert_eq!(scanned, 2);
assert_eq!(final_entries, 2);
assert_eq!(candidates.len(), 2);
assert!(
candidates
@@ -83,8 +98,14 @@ fn paginates_final_entries_without_disclosing_locations() {
ReconciliationMutation::Quarantined
);
}
let (_, quarantined_entries) = store.scan_reconciliation(None, 512, 8).unwrap();
let (report, quarantined_entries) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(quarantined_entries.len(), 2);
assert_eq!(report.scanned, 2);
assert_eq!(report.final_entries, 0);
assert_eq!(report.quarantined_entries, 2);
assert_eq!(report.stop, ReconciliationScanStop::Complete);
assert!(
quarantined_entries
.iter()
@@ -121,13 +142,98 @@ fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
);
fs::hard_link(shard.join("not-a-digest"), shard.join(hardlink_name)).unwrap();
let (report, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
assert!(report.malformed >= 1);
assert!(report.unsafe_entries >= 3);
let (report, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.scanned, 5);
assert_eq!(report.malformed, 1);
assert_eq!(report.unsafe_entries, 3);
assert_eq!(report.final_entries, 1);
assert_eq!(report.quarantined_entries, 0);
assert_eq!(candidates.len(), 1);
assert!(report.continuation.is_none());
}
#[test]
fn skips_a_valid_publish_temp_without_classifying_it_as_malformed() {
let root = TestRoot::new("valid-temp");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation with concurrent temp").unwrap();
let shard = root
.0
.join("sha256")
.join(&stored.artifact_ref.digest_hex()[..2]);
fs::write(
shard.join(".crank-artifact-tmp-v1-0123456789abcdef0123456789abcdef-7-9"),
b"in-flight",
)
.unwrap();
let (report, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.scanned, 2);
assert_eq!(report.malformed, 0);
assert_eq!(report.final_entries, 1);
assert_eq!(candidates.len(), 1);
}
#[test]
fn unsafe_canonical_shard_does_not_starve_later_shards() {
let root = TestRoot::new("unsafe-shard");
let store = ArtifactStore::open(&root.0).unwrap();
let sha = root.0.join("sha256");
fs::create_dir(&sha).unwrap();
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
let unsafe_shard = sha.join("00");
fs::create_dir(&unsafe_shard).unwrap();
fs::set_permissions(&unsafe_shard, fs::Permissions::from_mode(0o000)).unwrap();
let shard = sha.join("ff");
fs::create_dir(&shard).unwrap();
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
let name = "f".repeat(64);
fs::write(shard.join(&name), b"later valid entry").unwrap();
fs::set_permissions(shard.join(&name), fs::Permissions::from_mode(0o400)).unwrap();
let (report, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.stop, ReconciliationScanStop::Complete);
assert_eq!(report.unsafe_entries, 1);
assert_eq!(report.final_entries, 1);
assert!(candidates.iter().any(|candidate| {
candidate.namespace() == ReconciliationNamespace::Final
&& format!("{candidate:?}") == "ReconciliationCandidate(..)"
}));
fs::set_permissions(unsafe_shard, fs::Permissions::from_mode(0o700)).unwrap();
}
#[test]
#[cfg(debug_assertions)]
fn traversal_never_exceeds_the_exact_syscall_budget() {
let root = TestRoot::new("syscall-budget");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"bounded traversal syscall accounting").unwrap();
for budget in 0..=8 {
reset_traversal_calls();
let (report, _) = store.scan_reconciliation(None, budget, 8).unwrap();
assert!(report.traversal_syscalls <= budget, "budget={budget}");
assert_eq!(report.traversal_syscalls, budget, "budget={budget}");
assert_eq!(traversal_calls(), report.traversal_syscalls);
assert_eq!(report.stop, ReconciliationScanStop::ScanBudget);
}
reset_traversal_calls();
let (report, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 1)
.unwrap();
assert_eq!(candidates.len(), 1);
assert_eq!(report.stop, ReconciliationScanStop::ResultLimit);
assert!(report.traversal_syscalls <= FULL_SCAN_BUDGET);
assert_eq!(traversal_calls(), report.traversal_syscalls);
}
#[test]
fn rejects_a_cursor_from_another_store() {
let first_root = TestRoot::new("foreign-cursor-first");
@@ -157,7 +263,7 @@ fn disappearing_entry_after_readdir_is_skipped() {
.join(stored.artifact_ref.digest_hex());
set_checkpoint("reconciliation_before_stat", FaultAction::Hold);
let scan_store = Arc::clone(&store);
let scan = thread::spawn(move || scan_store.scan_reconciliation(None, 512, 8));
let scan = thread::spawn(move || scan_store.scan_reconciliation(None, FULL_SCAN_BUDGET, 8));
assert!(wait_until_held(Duration::from_secs(2)));
fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
fs::remove_file(&path).unwrap();
@@ -172,7 +278,9 @@ fn quarantine_and_delete_are_idempotent() {
let root = TestRoot::new("mutation");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation mutation").unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
assert_eq!(
@@ -188,7 +296,9 @@ fn quarantine_and_delete_are_idempotent() {
ReconciliationMutation::AlreadyQuarantined
);
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let quarantined = candidates
.into_iter()
.find(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
@@ -212,7 +322,9 @@ fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
let root = TestRoot::new("no-clobber");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation no-clobber").unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let original = candidates.into_iter().next().unwrap();
assert_eq!(
store.quarantine_reconciliation(original.clone()).unwrap(),
@@ -227,7 +339,9 @@ fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
store.read(&stored.artifact_ref).unwrap(),
b"reconciliation no-clobber"
);
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(candidates.len(), 2);
assert!(
candidates
@@ -241,6 +355,42 @@ fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
);
}
#[test]
fn quarantine_refuses_to_replace_an_unrelated_inode() {
let root = TestRoot::new("unrelated-collision");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"canonical final survives collision").unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
let digest = stored.artifact_ref.digest_hex();
let quarantine = root.0.join("quarantine");
let shard = quarantine.join(&digest[..2]);
fs::create_dir(&quarantine).unwrap();
fs::create_dir(&shard).unwrap();
fs::set_permissions(&quarantine, fs::Permissions::from_mode(0o700)).unwrap();
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
let collision = shard.join(digest);
fs::write(&collision, b"unrelated quarantine inode").unwrap();
fs::set_permissions(&collision, fs::Permissions::from_mode(0o400)).unwrap();
let collision_ino = fs::metadata(&collision).unwrap().ino();
assert_eq!(
store.quarantine_reconciliation(candidate),
Err(ArtifactError::UnsafeRoot)
);
assert_eq!(
store.read(&stored.artifact_ref).unwrap(),
b"canonical final survives collision"
);
assert_eq!(fs::read(collision).unwrap(), b"unrelated quarantine inode");
assert_eq!(
fs::metadata(shard.join(digest)).unwrap().ino(),
collision_ino
);
}
#[test]
fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
let root = TestRoot::new("quarantine-replaced");
@@ -248,11 +398,15 @@ fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
let stored = store
.put(b"reconciliation quarantined replacement")
.unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
let path = root
.0
@@ -274,6 +428,40 @@ fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
assert!(sibling.exists());
}
#[test]
fn delete_revalidates_a_post_scan_hardlink_without_inode_replacement() {
let root = TestRoot::new("quarantine-hardlink");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"same quarantined inode gains a link").unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
let path = root
.0
.join("quarantine")
.join(&stored.artifact_ref.digest_hex()[..2])
.join(stored.artifact_ref.digest_hex());
let sibling = path.with_extension("link");
let original_ino = fs::metadata(&path).unwrap().ino();
fs::hard_link(&path, &sibling).unwrap();
assert_eq!(
store.delete_quarantined_reconciliation(candidate),
Err(ArtifactError::UnsafeRoot)
);
assert_eq!(fs::metadata(&path).unwrap().nlink(), 2);
assert_eq!(fs::metadata(&path).unwrap().ino(), original_ino);
assert!(path.exists());
assert!(sibling.exists());
}
#[test]
#[cfg(debug_assertions)]
fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() {
@@ -281,11 +469,15 @@ fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() {
let root = TestRoot::new("delete-fsync");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"reconciliation delete fsync").unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
set_checkpoint("reconciliation_delete_fsync", FaultAction::Fail);
assert_eq!(
@@ -306,7 +498,9 @@ fn rejects_a_replaced_final_inode() {
let root = TestRoot::new("replaced");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation original").unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
let path = root
.0
@@ -328,36 +522,175 @@ fn rejects_a_replaced_final_inode() {
#[cfg(debug_assertions)]
fn retries_after_post_rename_fsync_ambiguity() {
let _guard = fault_guard();
let root = TestRoot::new("fsync");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"reconciliation fault").unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
for stage in [
"reconciliation_quarantine_source_fsync",
"reconciliation_quarantine_destination_fsync",
] {
let root = TestRoot::new(stage);
let store = ArtifactStore::open(&root.0).unwrap();
store.put(stage.as_bytes()).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
set_checkpoint(stage, FaultAction::Fail);
assert_eq!(
store.quarantine_reconciliation(candidate.clone()).unwrap(),
ReconciliationMutation::Retryable,
"stage={stage}"
);
assert_eq!(
checkpoint_hits("reconciliation_quarantine_source_fsync"),
1,
"stage={stage}"
);
assert_eq!(
checkpoint_hits("reconciliation_quarantine_destination_fsync"),
1,
"stage={stage}"
);
clear_checkpoint();
assert_eq!(
store.quarantine_reconciliation(candidate).unwrap(),
ReconciliationMutation::AlreadyQuarantined,
"stage={stage}"
);
}
}
#[test]
#[cfg(debug_assertions)]
fn mutation_syscall_failures_preserve_retryable_state() {
let _guard = fault_guard();
let rename_root = TestRoot::new("rename-fail");
let rename_store = ArtifactStore::open(&rename_root.0).unwrap();
let stored = rename_store.put(b"rename failure bytes").unwrap();
let (_, candidates) = rename_store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
set_checkpoint("reconciliation_quarantine_source_fsync", FaultAction::Fail);
set_checkpoint("reconciliation_quarantine_rename", FaultAction::Fail);
assert_eq!(
store.quarantine_reconciliation(candidate.clone()).unwrap(),
ReconciliationMutation::Retryable
rename_store.quarantine_reconciliation(candidate.clone()),
Err(ArtifactError::Storage)
);
assert_eq!(
rename_store.read(&stored.artifact_ref).unwrap(),
b"rename failure bytes"
);
clear_checkpoint();
assert_eq!(
store.quarantine_reconciliation(candidate).unwrap(),
ReconciliationMutation::AlreadyQuarantined
rename_store.quarantine_reconciliation(candidate).unwrap(),
ReconciliationMutation::Quarantined
);
let delete_root = TestRoot::new("unlink-fail");
let delete_store = ArtifactStore::open(&delete_root.0).unwrap();
delete_store.put(b"unlink failure bytes").unwrap();
let (_, candidates) = delete_store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
delete_store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
let (_, candidates) = delete_store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
set_checkpoint("reconciliation_delete_unlink", FaultAction::Fail);
assert_eq!(
delete_store.delete_quarantined_reconciliation(candidate.clone()),
Err(ArtifactError::Storage)
);
clear_checkpoint();
assert_eq!(
delete_store
.delete_quarantined_reconciliation(candidate)
.unwrap(),
ReconciliationMutation::Deleted
);
}
#[test]
#[cfg(debug_assertions)]
fn storage_fault_during_stat_propagates() {
fn storage_fault_returns_a_retryable_page_before_the_failed_entry() {
let _guard = fault_guard();
let root = TestRoot::new("stat-storage");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"reconciliation stat storage").unwrap();
set_checkpoint("reconciliation_stat", FaultAction::Fail);
assert!(matches!(
store.scan_reconciliation(None, 512, 8),
Err(ArtifactError::Storage)
));
let sha = root.0.join("sha256");
let shard = sha.join("aa");
fs::create_dir(&sha).unwrap();
fs::create_dir(&shard).unwrap();
fs::set_permissions(&sha, fs::Permissions::from_mode(0o700)).unwrap();
fs::set_permissions(&shard, fs::Permissions::from_mode(0o700)).unwrap();
for suffix in ['1', '2'] {
let path = shard.join(format!("aa{}", suffix.to_string().repeat(62)));
fs::write(&path, b"valid opaque candidate").unwrap();
fs::set_permissions(path, fs::Permissions::from_mode(0o400)).unwrap();
}
set_checkpoint_on_hit("reconciliation_stat", FaultAction::Fail, 2);
let (report, mut candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.stop, ReconciliationScanStop::Retryable);
assert_eq!(report.scanned, 1);
assert_eq!(report.final_entries, 1);
assert_eq!(candidates.len(), 1);
let continuation = report.continuation;
clear_checkpoint();
let (report, resumed) = store
.scan_reconciliation(continuation, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.stop, ReconciliationScanStop::Complete);
assert_eq!(report.scanned, 1);
assert_eq!(report.final_entries, 1);
candidates.extend(resumed);
assert_eq!(candidates.len(), 2);
for candidate in candidates {
assert_eq!(
store.quarantine_reconciliation(candidate).unwrap(),
ReconciliationMutation::Quarantined
);
}
let (report, quarantined) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.final_entries, 0);
assert_eq!(report.quarantined_entries, 2);
assert_eq!(quarantined.len(), 2);
}
#[test]
#[cfg(debug_assertions)]
fn getdents_retry_restores_the_pre_call_directory_cookie() {
let _guard = fault_guard();
let root = TestRoot::new("getdents-storage");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"getdents retry bytes").unwrap();
set_checkpoint("reconciliation_getdents", FaultAction::Fail);
let (report, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.stop, ReconciliationScanStop::Retryable);
assert!(candidates.is_empty());
clear_checkpoint();
let (report, candidates) = store
.scan_reconciliation(report.continuation, 1, 8)
.unwrap();
assert_eq!(report.stop, ReconciliationScanStop::ScanBudget);
assert_eq!(report.traversal_syscalls, 1);
assert!(candidates.is_empty());
let (report, candidates) = store
.scan_reconciliation(report.continuation, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(report.stop, ReconciliationScanStop::Complete);
assert_eq!(candidates.len(), 1);
}
#[test]
@@ -370,7 +703,9 @@ fn crash_windows_are_recoverable() {
FaultAction::Exit,
);
let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.into_iter().next().unwrap();
if action == "quarantine" {
let _ = store.quarantine_reconciliation(candidate);
@@ -391,7 +726,9 @@ fn crash_windows_are_recoverable() {
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation crash bytes").unwrap();
if action == "delete" {
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
@@ -406,7 +743,9 @@ fn crash_windows_are_recoverable() {
assert_eq!(status.code(), Some(86), "stage={stage}");
let store = ArtifactStore::open(&root.0).unwrap();
let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
for candidate in candidates {
match candidate.namespace() {
ReconciliationNamespace::Final => {
@@ -417,7 +756,9 @@ fn crash_windows_are_recoverable() {
}
}
}
let (_, remaining) = store.scan_reconciliation(None, 512, 8).unwrap();
let (_, remaining) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert!(remaining.is_empty(), "stage={stage}");
assert_eq!(
store