bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(artifacts): add fenced reconciliation recovery

Browse Source
This commit is contained in:
bsodfather
2026-08-28 00:44:27 +03:00
parent 8784964fb2
commit d2849ea3fe
21 changed files with 3322 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/crank-artifacts/src/housekeeping.rs
+41 -44
View File
@@ -8,21 +8,21 @@ use crate::temp_scan::{list_names, valid_temp_name};
use crate::{
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
ReconciliationScan, ReconciliationScanStop,
recovery::{FINAL_NAMESPACE, namespace_name, namespace_number},
store::{
QUARANTINE_DIR, RootLock, check_file, checkpointed, ensure_root_unchanged, fsync_fd,
open_existing_dir, open_or_create_dir, rename_no_replace_at, stat_fd, unlinkat,
},
};
const FINAL_NAMESPACE: u8 = 0;
const QUARANTINE_NAMESPACE: u8 = 1;
/// Opaque bounded-scan continuation, valid only for an unchanged namespace.
/// Discard it after any put, quarantine, delete, or external mutation.
pub struct ReconciliationCursor {
root_dev: u64,
root_ino: u64,
namespace_start: u8,
namespace: u8,
namespace_end: u8,
shard: u8,
state: ReconciliationCursorState,
}
@@ -67,29 +67,17 @@ impl fmt::Debug for ReconciliationCursor {
/// Opaque inode-bound evidence that exposes neither digest nor filesystem path.
#[derive(Clone)]
pub struct ReconciliationCandidate {
root_dev: u64,
root_ino: u64,
namespace: ReconciliationNamespace,
shard: String,
shard_dev: u64,
shard_ino: u64,
name: String,
dev: u64,
ino: u64,
modified_seconds: i64,
modified_nanoseconds: i64,
}
impl fmt::Debug for ReconciliationCandidate {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ReconciliationCandidate(..)")
}
}
impl ReconciliationCandidate {
pub fn namespace(&self) -> ReconciliationNamespace {
self.namespace
}
pub(crate) root_dev: u64,
pub(crate) root_ino: u64,
pub(crate) namespace: ReconciliationNamespace,
pub(crate) shard: String,
pub(crate) shard_dev: u64,
pub(crate) shard_ino: u64,
pub(crate) name: String,
pub(crate) dev: u64,
pub(crate) ino: u64,
pub(crate) modified_seconds: i64,
pub(crate) modified_nanoseconds: i64,
}
/// Opaque single-use evidence for a stale temporary inode under the pinned root.
@@ -113,25 +101,31 @@ pub struct TempScan {
}
impl ArtifactStore {
/// Streams entries without disclosing paths/digests, with opaque continuation.
pub fn scan_reconciliation(
pub(crate) fn scan_reconciliation_bounded(
&self,
start_namespace: ReconciliationNamespace,
end_namespace: ReconciliationNamespace,
continuation: Option<ReconciliationCursor>,
scan_budget: usize,
result_limit: usize,
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
let root = self.root()?;
let namespace = namespace_number(start_namespace);
let namespace_end = namespace_number(end_namespace);
let mut cursor = match continuation {
Some(cursor) => {
if cursor.root_dev != root.dev
|| cursor.root_ino != root.ino
|| cursor.namespace > QUARANTINE_NAMESPACE
|| cursor.namespace_start != namespace
|| cursor.namespace < namespace
|| cursor.namespace > cursor.namespace_end
|| cursor.namespace_end != namespace_end
{
return Err(ArtifactError::UnsafeRoot);
}
cursor
}
None => reconciliation_cursor(root.dev, root.ino),
None => reconciliation_cursor(root.dev, root.ino, namespace, namespace_end),
};
let report = ReconciliationScan::empty(None);
let _lock = match RootLock::shared(root) {
@@ -177,7 +171,7 @@ impl ArtifactStore {
}
let mut entries = Vec::new();
while cursor.namespace <= QUARANTINE_NAMESPACE {
while cursor.namespace <= cursor.namespace_end {
if entries.len() == result_limit {
return Ok(reconciliation_page(
report,
@@ -438,11 +432,18 @@ impl ArtifactStore {
}
}
fn reconciliation_cursor(root_dev: u64, root_ino: u64) -> ReconciliationCursor {
fn reconciliation_cursor(
root_dev: u64,
root_ino: u64,
namespace: u8,
namespace_end: u8,
) -> ReconciliationCursor {
ReconciliationCursor {
root_dev,
root_ino,
namespace: FINAL_NAMESPACE,
namespace_start: namespace,
namespace,
namespace_end,
shard: 0,
state: ReconciliationCursorState::OpenNamespace,
}
@@ -865,14 +866,6 @@ fn parse_dirent(buffer: &[u8], offset: usize) -> Result<(Vec<u8>, usize, i64), A
Ok((raw_name[..end].to_vec(), after, cookie))
}
fn namespace_name(namespace: u8) -> &'static [u8] {
match namespace {
FINAL_NAMESPACE => b"sha256",
QUARANTINE_NAMESPACE => QUARANTINE_DIR,
_ => unreachable!("validated reconciliation namespace"),
}
}
fn advance_reconciliation_shard(cursor: &mut ReconciliationCursor, namespace: OwnedFd) {
if cursor.shard == u8::MAX {
advance_namespace(cursor);
@@ -883,12 +876,16 @@ fn advance_reconciliation_shard(cursor: &mut ReconciliationCursor, namespace: Ow
}
fn advance_namespace(cursor: &mut ReconciliationCursor) {
cursor.namespace = cursor.namespace.saturating_add(1);
cursor.namespace = if cursor.namespace >= cursor.namespace_end {
cursor.namespace_end.saturating_add(1)
} else {
cursor.namespace + 1
};
cursor.shard = 0;
cursor.state = ReconciliationCursorState::OpenNamespace;
}
fn ensure_reconciliation_root(
pub(crate) fn ensure_reconciliation_root(
root: &crate::store::Root,
candidate: &ReconciliationCandidate,
) -> Result<(), ArtifactError> {
@@ -904,7 +901,7 @@ fn ensure_reconciliation_root(
Ok(())
}
fn revalidate_candidate(
pub(crate) fn revalidate_candidate(
candidate: &ReconciliationCandidate,
shard: i32,
) -> Result<(), ArtifactError> {
crates/crank-artifacts/src/lib.rs
+3 -2
View File
@@ -11,6 +11,7 @@ mod error;
mod housekeeping;
mod legacy;
mod model;
mod recovery;
mod store;
mod temp_scan;
@@ -22,7 +23,7 @@ pub use error::ArtifactError;
pub use housekeeping::{ReconciliationCandidate, ReconciliationCursor, StaleTemp, TempScan};
pub use model::{
ArtifactRef, MAX_ARTIFACT_BYTES, MAX_SOURCE_BYTES, ReconciliationMutation,
ReconciliationNamespace, ReconciliationScan, ReconciliationScanStop, RegisteredArtifact,
StoredArtifact,
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
ReconciliationScan, ReconciliationScanStop, RegisteredArtifact, StoredArtifact,
};
pub use store::ArtifactStore;
crates/crank-artifacts/src/model.rs
+60
View File
@@ -77,6 +77,66 @@ pub struct RegisteredArtifact {
pub(crate) size_bytes: usize,
}
/// Result of validating a scanned reconciliation candidate for registry use.
///
/// A candidate that is younger than the configured grace period never yields
/// a registration capability. The enum is intentionally redacted by its
/// `Debug` implementation so an eligibility report cannot disclose the
/// physical artifact identity.
#[derive(Clone)]
pub enum ReconciliationRegistration {
NotEligible,
Registered(RegisteredArtifact),
}
/// Redacted physical state observed by an expired-claim recovery probe.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ReconciliationPresence {
Final,
Quarantine,
Absent,
Unsafe,
Retryable,
}
impl fmt::Debug for ReconciliationRegistration {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ReconciliationRegistration(..)")
}
}
impl PartialEq for ReconciliationRegistration {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::NotEligible, Self::NotEligible) => true,
(Self::Registered(left), Self::Registered(right)) => left == right,
_ => false,
}
}
}
impl Eq for ReconciliationRegistration {}
impl ReconciliationRegistration {
pub fn is_eligible(&self) -> bool {
matches!(self, Self::Registered(_))
}
pub fn artifact(&self) -> Option<&RegisteredArtifact> {
match self {
Self::NotEligible => None,
Self::Registered(artifact) => Some(artifact),
}
}
pub fn into_artifact(self) -> Option<RegisteredArtifact> {
match self {
Self::NotEligible => None,
Self::Registered(artifact) => Some(artifact),
}
}
}
/// Namespace in which reconciliation observed an artifact inode. It does not
/// disclose the artifact's digest or filesystem location.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
crates/crank-artifacts/src/recovery.rs
+224
View File
@@ -0,0 +1,224 @@
use std::{
fmt,
os::fd::{AsRawFd, OwnedFd},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use crate::{
ArtifactError, ArtifactRef, ArtifactStore, ReconciliationCandidate, ReconciliationCursor,
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
ReconciliationScan,
housekeeping::{ensure_reconciliation_root, revalidate_candidate},
store::{
QUARANTINE_DIR, Root, RootLock, checkpointed, ensure_root_unchanged, open_existing_dir,
open_existing_file, read_verified, stat_fd,
},
};
pub(crate) const FINAL_NAMESPACE: u8 = 0;
pub(crate) const QUARANTINE_NAMESPACE: u8 = 1;
pub(crate) fn namespace_name(namespace: u8) -> &'static [u8] {
match namespace {
FINAL_NAMESPACE => b"sha256",
QUARANTINE_NAMESPACE => QUARANTINE_DIR,
_ => unreachable!("validated reconciliation namespace"),
}
}
pub(crate) fn namespace_number(namespace: ReconciliationNamespace) -> u8 {
match namespace {
ReconciliationNamespace::Final => FINAL_NAMESPACE,
ReconciliationNamespace::Quarantine => QUARANTINE_NAMESPACE,
}
}
impl fmt::Debug for ReconciliationCandidate {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ReconciliationCandidate(..)")
}
}
impl ReconciliationCandidate {
pub fn namespace(&self) -> ReconciliationNamespace {
self.namespace
}
/// Returns whether the candidate has aged past `grace` at `now`.
///
/// This only reports the timestamp captured by the scanner. Callers that
/// need a registration capability must use [`ArtifactStore::register_reconciliation`],
/// which repeats inode and content validation while holding the store lock.
pub fn is_grace_eligible(&self, grace: Duration, now: SystemTime) -> bool {
let modified = UNIX_EPOCH
.checked_add(Duration::new(
self.modified_seconds.max(0) as u64,
self.modified_nanoseconds.max(0) as u32,
))
.unwrap_or(SystemTime::UNIX_EPOCH);
now.duration_since(modified).is_ok_and(|age| age >= grace)
}
}
impl ArtifactStore {
/// Scans only one reconciliation namespace with a bounded opaque cursor.
///
/// A quarantine scan ends after quarantine and never falls through to
/// final entries, so recovery can finish before the normal final sweep.
pub fn scan_reconciliation_namespace(
&self,
namespace: ReconciliationNamespace,
continuation: Option<ReconciliationCursor>,
scan_budget: usize,
result_limit: usize,
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
self.scan_reconciliation_bounded(
namespace,
namespace,
continuation,
scan_budget,
result_limit,
)
}
/// Streams entries without disclosing paths/digests, with opaque continuation.
pub fn scan_reconciliation(
&self,
continuation: Option<ReconciliationCursor>,
scan_budget: usize,
result_limit: usize,
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
self.scan_reconciliation_bounded(
ReconciliationNamespace::Final,
ReconciliationNamespace::Quarantine,
continuation,
scan_budget,
result_limit,
)
}
/// Revalidates a scanned candidate and mints a registration capability
/// only for a file that is old enough and whose bytes still match its
/// canonical name. The candidate remains usable for a later mutation.
pub fn register_reconciliation(
&self,
candidate: &ReconciliationCandidate,
grace: Duration,
now: SystemTime,
) -> Result<ReconciliationRegistration, ArtifactError> {
let root = self.root()?;
let _lock = RootLock::shared(root)?;
ensure_reconciliation_root(root, candidate)?;
let namespace = namespace_number(candidate.namespace);
let namespace_root = open_existing_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
let shard = open_existing_dir(namespace_root.as_raw_fd(), candidate.shard.as_bytes())?;
revalidate_candidate(candidate, shard.as_raw_fd())?;
let file = open_existing_file(shard.as_raw_fd(), candidate.name.as_bytes())?;
let stat = stat_fd(file.as_raw_fd())?;
if stat.st_dev != candidate.dev
|| stat.st_ino != candidate.ino
|| stat.st_mtime != candidate.modified_seconds
|| stat.st_mtime_nsec != candidate.modified_nanoseconds
{
return Err(ArtifactError::UnsafeRoot);
}
if !candidate.is_grace_eligible(grace, now) {
return Ok(ReconciliationRegistration::NotEligible);
}
let artifact_ref =
ArtifactRef::from_digest_hex(&candidate.name).map_err(|_| ArtifactError::UnsafeRoot)?;
let bytes = read_verified(&file, &artifact_ref)?;
Ok(ReconciliationRegistration::Registered(
crate::RegisteredArtifact {
artifact_ref,
size_bytes: bytes.len(),
},
))
}
/// Convenience form that uses the current wall clock for grace checks.
pub fn register_reconciliation_candidate(
&self,
candidate: &ReconciliationCandidate,
grace: Duration,
) -> Result<ReconciliationRegistration, ArtifactError> {
self.register_reconciliation(candidate, grace, SystemTime::now())
}
/// Probes final and quarantine for an expired claim without disclosing its
/// physical identity. Quarantine takes precedence over final.
pub fn reconciliation_presence(
&self,
artifact_ref: &ArtifactRef,
) -> Result<ReconciliationPresence, ArtifactError> {
let root = self.root()?;
let _lock = match RootLock::shared(root) {
Ok(lock) => lock,
Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
Err(error) => return Err(error),
};
match ensure_root_unchanged(root) {
Ok(()) => {}
Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
Err(error) => return Err(error),
}
match probe_namespace(root, QUARANTINE_NAMESPACE, artifact_ref) {
Ok(Some(presence)) => return Ok(presence),
Ok(None) => {}
Err(presence) => return Ok(presence),
}
match probe_namespace(root, FINAL_NAMESPACE, artifact_ref) {
Ok(Some(presence)) => Ok(presence),
Ok(None) => Ok(ReconciliationPresence::Absent),
Err(presence) => Ok(presence),
}
}
}
fn probe_namespace(
root: &Root,
namespace: u8,
artifact_ref: &ArtifactRef,
) -> Result<Option<ReconciliationPresence>, ReconciliationPresence> {
let namespace_root = probe_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
let Some(namespace_root) = namespace_root else {
return Ok(None);
};
let shard = probe_dir(namespace_root.as_raw_fd(), artifact_ref.shard().as_bytes())?;
let Some(shard) = shard else {
return Ok(None);
};
let file = match checkpointed("reconciliation_presence_open", || {
open_existing_file(shard.as_raw_fd(), artifact_ref.digest_hex().as_bytes())
}) {
Ok(fd) => fd,
Err(ArtifactError::NotFound) => return Ok(None),
Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
return Err(ReconciliationPresence::Unsafe);
}
Err(_) => return Err(ReconciliationPresence::Retryable),
};
match read_verified(&file, artifact_ref) {
Ok(_) => Ok(Some(if namespace == FINAL_NAMESPACE {
ReconciliationPresence::Final
} else {
ReconciliationPresence::Quarantine
})),
Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
Err(ReconciliationPresence::Unsafe)
}
Err(_) => Err(ReconciliationPresence::Retryable),
}
}
fn probe_dir(parent: i32, name: &[u8]) -> Result<Option<OwnedFd>, ReconciliationPresence> {
match open_existing_dir(parent, name) {
Ok(fd) => Ok(Some(fd)),
Err(ArtifactError::NotFound) => Ok(None),
Err(ArtifactError::UnsafeRoot) => Err(ReconciliationPresence::Unsafe),
Err(_) => Err(ReconciliationPresence::Retryable),
}
}
crates/crank-artifacts/tests/reconciliation.rs
+208 -2
View File
@@ -3,14 +3,15 @@ use std::{
os::unix::fs::{MetadataExt, PermissionsExt},
path::PathBuf,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, SystemTime},
};
#[cfg(debug_assertions)]
use std::{process::Command, sync::Arc, thread, time::Duration};
use std::{process::Command, sync::Arc, thread};
use crank_artifacts::{
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
ReconciliationScanStop,
ReconciliationPresence, ReconciliationRegistration, ReconciliationScanStop,
};
#[cfg(debug_assertions)]
@@ -769,3 +770,208 @@ fn crash_windows_are_recoverable() {
);
}
}
#[test]
fn registration_revalidates_content_and_enforces_grace_without_candidate_identity_access() {
let root = TestRoot::new("registration");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation registration").unwrap();
let (_, mut candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
let candidate = candidates.pop().unwrap();
assert_eq!(format!("{candidate:?}"), "ReconciliationCandidate(..)");
assert!(!candidate.is_grace_eligible(Duration::from_secs(3600), SystemTime::now()));
assert_eq!(
format!(
"{:?}",
store
.register_reconciliation(&candidate, Duration::from_secs(3600), SystemTime::now())
.unwrap()
),
"ReconciliationRegistration(..)"
);
let registration = store
.register_reconciliation(
&candidate,
Duration::ZERO,
SystemTime::now() + Duration::from_secs(3600),
)
.unwrap();
let registered = match registration {
ReconciliationRegistration::Registered(registered) => registered,
ReconciliationRegistration::NotEligible => panic!("zero grace must be eligible"),
};
assert_eq!(registered.artifact_ref(), &stored.artifact_ref);
assert_eq!(
registered.size_bytes(),
b"reconciliation registration".len()
);
// The candidate remains usable for the physical mutation after the
// registration bridge borrowed it.
assert_eq!(
store.quarantine_reconciliation(candidate).unwrap(),
ReconciliationMutation::Quarantined
);
}
#[test]
fn namespace_scan_can_finish_quarantine_recovery_before_final_sweep() {
let root = TestRoot::new("namespace-order");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"quarantine-first recovery").unwrap();
let (_, candidates) = store
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8)
.unwrap();
store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
store.put(b"quarantine-first recovery").unwrap();
let (quarantine_report, quarantine_candidates) = store
.scan_reconciliation_namespace(
ReconciliationNamespace::Quarantine,
None,
FULL_SCAN_BUDGET,
8,
)
.unwrap();
assert_eq!(quarantine_report.stop, ReconciliationScanStop::Complete);
assert_eq!(quarantine_report.final_entries, 0);
assert_eq!(quarantine_report.quarantined_entries, 1);
assert_eq!(quarantine_candidates.len(), 1);
assert!(quarantine_report.continuation.is_none());
let (final_report, final_candidates) = store
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8)
.unwrap();
assert_eq!(final_report.stop, ReconciliationScanStop::Complete);
assert_eq!(final_report.final_entries, 1);
assert_eq!(final_report.quarantined_entries, 0);
assert_eq!(final_candidates.len(), 1);
}
#[test]
fn namespace_scan_rejects_a_cursor_from_a_broader_scan() {
let root = TestRoot::new("namespace-cursor-scope");
let store = ArtifactStore::open(&root.0).unwrap();
let (report, _) = store.scan_reconciliation(None, 0, 1).unwrap();
let cursor = report
.continuation
.expect("zero-budget scan retains cursor");
assert!(matches!(
store.scan_reconciliation_namespace(
ReconciliationNamespace::Quarantine,
Some(cursor),
FULL_SCAN_BUDGET,
1,
),
Err(ArtifactError::UnsafeRoot)
));
}
#[test]
fn full_scan_rejects_a_quarantine_only_cursor() {
let root = TestRoot::new("namespace-cursor-narrow");
let store = ArtifactStore::open(&root.0).unwrap();
let (report, _) = store
.scan_reconciliation_namespace(ReconciliationNamespace::Quarantine, None, 0, 1)
.unwrap();
let cursor = report
.continuation
.expect("zero-budget namespace scan retains cursor");
assert!(matches!(
store.scan_reconciliation(Some(cursor), FULL_SCAN_BUDGET, 1),
Err(ArtifactError::UnsafeRoot)
));
}
#[test]
fn presence_probe_distinguishes_final_quarantine_and_absent_without_disclosure() {
let root = TestRoot::new("presence");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"presence probe").unwrap();
assert_eq!(
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
ReconciliationPresence::Final
);
let (_, candidates) = store
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
.unwrap();
store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
assert_eq!(
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
ReconciliationPresence::Quarantine
);
// A republish can leave both locations during recovery. Quarantine takes
// precedence so the stale inode is still scheduled for deletion.
store.put(b"presence probe").unwrap();
assert_eq!(
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
ReconciliationPresence::Quarantine
);
let (_, candidates) = store
.scan_reconciliation_namespace(
ReconciliationNamespace::Quarantine,
None,
FULL_SCAN_BUDGET,
8,
)
.unwrap();
store
.delete_quarantined_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
assert_eq!(
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
ReconciliationPresence::Final
);
let (_, candidates) = store
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8)
.unwrap();
store
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
let (_, candidates) = store
.scan_reconciliation_namespace(
ReconciliationNamespace::Quarantine,
None,
FULL_SCAN_BUDGET,
8,
)
.unwrap();
store
.delete_quarantined_reconciliation(candidates.into_iter().next().unwrap())
.unwrap();
assert_eq!(
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
ReconciliationPresence::Absent
);
}
#[cfg(debug_assertions)]
#[test]
fn presence_probe_classifies_storage_failure_as_retryable() {
let _guard = fault_guard();
let root = TestRoot::new("presence-retryable");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"presence retryable").unwrap();
set_checkpoint("reconciliation_presence_open", FaultAction::Fail);
assert_eq!(
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
ReconciliationPresence::Retryable
);
clear_checkpoint();
assert_eq!(
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
ReconciliationPresence::Final
);
}