feat(artifacts): add fenced reconciliation recovery

crates/crank-artifacts/src/housekeeping.rs

@@ -8,21 +8,21 @@ use crate::temp_scan::{list_names, valid_temp_name};
 use crate::{
     ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
     ReconciliationScan, ReconciliationScanStop,
+    recovery::{FINAL_NAMESPACE, namespace_name, namespace_number},
     store::{
         QUARANTINE_DIR, RootLock, check_file, checkpointed, ensure_root_unchanged, fsync_fd,
         open_existing_dir, open_or_create_dir, rename_no_replace_at, stat_fd, unlinkat,
     },
 };
 
-const FINAL_NAMESPACE: u8 = 0;
-const QUARANTINE_NAMESPACE: u8 = 1;
-
 /// Opaque bounded-scan continuation, valid only for an unchanged namespace.
 /// Discard it after any put, quarantine, delete, or external mutation.
 pub struct ReconciliationCursor {
     root_dev: u64,
     root_ino: u64,
+    namespace_start: u8,
     namespace: u8,
+    namespace_end: u8,
     shard: u8,
     state: ReconciliationCursorState,
 }
@@ -67,29 +67,17 @@ impl fmt::Debug for ReconciliationCursor {
 /// Opaque inode-bound evidence that exposes neither digest nor filesystem path.
 #[derive(Clone)]
 pub struct ReconciliationCandidate {
-    root_dev: u64,
-    root_ino: u64,
-    namespace: ReconciliationNamespace,
-    shard: String,
-    shard_dev: u64,
-    shard_ino: u64,
-    name: String,
-    dev: u64,
-    ino: u64,
-    modified_seconds: i64,
-    modified_nanoseconds: i64,
-}
-
-impl fmt::Debug for ReconciliationCandidate {
-    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-        formatter.write_str("ReconciliationCandidate(..)")
-    }
-}
-
-impl ReconciliationCandidate {
-    pub fn namespace(&self) -> ReconciliationNamespace {
-        self.namespace
-    }
+    pub(crate) root_dev: u64,
+    pub(crate) root_ino: u64,
+    pub(crate) namespace: ReconciliationNamespace,
+    pub(crate) shard: String,
+    pub(crate) shard_dev: u64,
+    pub(crate) shard_ino: u64,
+    pub(crate) name: String,
+    pub(crate) dev: u64,
+    pub(crate) ino: u64,
+    pub(crate) modified_seconds: i64,
+    pub(crate) modified_nanoseconds: i64,
 }
 
 /// Opaque single-use evidence for a stale temporary inode under the pinned root.
@@ -113,25 +101,31 @@ pub struct TempScan {
 }
 
 impl ArtifactStore {
-    /// Streams entries without disclosing paths/digests, with opaque continuation.
-    pub fn scan_reconciliation(
+    pub(crate) fn scan_reconciliation_bounded(
         &self,
+        start_namespace: ReconciliationNamespace,
+        end_namespace: ReconciliationNamespace,
         continuation: Option<ReconciliationCursor>,
         scan_budget: usize,
         result_limit: usize,
     ) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
         let root = self.root()?;
+        let namespace = namespace_number(start_namespace);
+        let namespace_end = namespace_number(end_namespace);
         let mut cursor = match continuation {
             Some(cursor) => {
                 if cursor.root_dev != root.dev
                     || cursor.root_ino != root.ino
-                    || cursor.namespace > QUARANTINE_NAMESPACE
+                    || cursor.namespace_start != namespace
+                    || cursor.namespace < namespace
+                    || cursor.namespace > cursor.namespace_end
+                    || cursor.namespace_end != namespace_end
                 {
                     return Err(ArtifactError::UnsafeRoot);
                 }
                 cursor
             }
-            None => reconciliation_cursor(root.dev, root.ino),
+            None => reconciliation_cursor(root.dev, root.ino, namespace, namespace_end),
         };
         let report = ReconciliationScan::empty(None);
         let _lock = match RootLock::shared(root) {
@@ -177,7 +171,7 @@ impl ArtifactStore {
         }
 
         let mut entries = Vec::new();
-        while cursor.namespace <= QUARANTINE_NAMESPACE {
+        while cursor.namespace <= cursor.namespace_end {
             if entries.len() == result_limit {
                 return Ok(reconciliation_page(
                     report,
@@ -438,11 +432,18 @@ impl ArtifactStore {
     }
 }
 
-fn reconciliation_cursor(root_dev: u64, root_ino: u64) -> ReconciliationCursor {
+fn reconciliation_cursor(
+    root_dev: u64,
+    root_ino: u64,
+    namespace: u8,
+    namespace_end: u8,
+) -> ReconciliationCursor {
     ReconciliationCursor {
         root_dev,
         root_ino,
-        namespace: FINAL_NAMESPACE,
+        namespace_start: namespace,
+        namespace,
+        namespace_end,
         shard: 0,
         state: ReconciliationCursorState::OpenNamespace,
     }
@@ -865,14 +866,6 @@ fn parse_dirent(buffer: &[u8], offset: usize) -> Result<(Vec<u8>, usize, i64), A
     Ok((raw_name[..end].to_vec(), after, cookie))
 }
 
-fn namespace_name(namespace: u8) -> &'static [u8] {
-    match namespace {
-        FINAL_NAMESPACE => b"sha256",
-        QUARANTINE_NAMESPACE => QUARANTINE_DIR,
-        _ => unreachable!("validated reconciliation namespace"),
-    }
-}
-
 fn advance_reconciliation_shard(cursor: &mut ReconciliationCursor, namespace: OwnedFd) {
     if cursor.shard == u8::MAX {
         advance_namespace(cursor);
@@ -883,12 +876,16 @@ fn advance_reconciliation_shard(cursor: &mut ReconciliationCursor, namespace: Ow
 }
 
 fn advance_namespace(cursor: &mut ReconciliationCursor) {
-    cursor.namespace = cursor.namespace.saturating_add(1);
+    cursor.namespace = if cursor.namespace >= cursor.namespace_end {
+        cursor.namespace_end.saturating_add(1)
+    } else {
+        cursor.namespace + 1
+    };
     cursor.shard = 0;
     cursor.state = ReconciliationCursorState::OpenNamespace;
 }
 
-fn ensure_reconciliation_root(
+pub(crate) fn ensure_reconciliation_root(
     root: &crate::store::Root,
     candidate: &ReconciliationCandidate,
 ) -> Result<(), ArtifactError> {
@@ -904,7 +901,7 @@ fn ensure_reconciliation_root(
     Ok(())
 }
 
-fn revalidate_candidate(
+pub(crate) fn revalidate_candidate(
     candidate: &ReconciliationCandidate,
     shard: i32,
 ) -> Result<(), ArtifactError> {

crates/crank-artifacts/src/lib.rs

@@ -11,6 +11,7 @@ mod error;
 mod housekeeping;
 mod legacy;
 mod model;
+mod recovery;
 mod store;
 mod temp_scan;
 
@@ -22,7 +23,7 @@ pub use error::ArtifactError;
 pub use housekeeping::{ReconciliationCandidate, ReconciliationCursor, StaleTemp, TempScan};
 pub use model::{
     ArtifactRef, MAX_ARTIFACT_BYTES, MAX_SOURCE_BYTES, ReconciliationMutation,
-    ReconciliationNamespace, ReconciliationScan, ReconciliationScanStop, RegisteredArtifact,
-    StoredArtifact,
+    ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
+    ReconciliationScan, ReconciliationScanStop, RegisteredArtifact, StoredArtifact,
 };
 pub use store::ArtifactStore;

crates/crank-artifacts/src/model.rs

@@ -77,6 +77,66 @@ pub struct RegisteredArtifact {
     pub(crate) size_bytes: usize,
 }
 
+/// Result of validating a scanned reconciliation candidate for registry use.
+///
+/// A candidate that is younger than the configured grace period never yields
+/// a registration capability.  The enum is intentionally redacted by its
+/// `Debug` implementation so an eligibility report cannot disclose the
+/// physical artifact identity.
+#[derive(Clone)]
+pub enum ReconciliationRegistration {
+    NotEligible,
+    Registered(RegisteredArtifact),
+}
+
+/// Redacted physical state observed by an expired-claim recovery probe.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ReconciliationPresence {
+    Final,
+    Quarantine,
+    Absent,
+    Unsafe,
+    Retryable,
+}
+
+impl fmt::Debug for ReconciliationRegistration {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("ReconciliationRegistration(..)")
+    }
+}
+
+impl PartialEq for ReconciliationRegistration {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::NotEligible, Self::NotEligible) => true,
+            (Self::Registered(left), Self::Registered(right)) => left == right,
+            _ => false,
+        }
+    }
+}
+
+impl Eq for ReconciliationRegistration {}
+
+impl ReconciliationRegistration {
+    pub fn is_eligible(&self) -> bool {
+        matches!(self, Self::Registered(_))
+    }
+
+    pub fn artifact(&self) -> Option<&RegisteredArtifact> {
+        match self {
+            Self::NotEligible => None,
+            Self::Registered(artifact) => Some(artifact),
+        }
+    }
+
+    pub fn into_artifact(self) -> Option<RegisteredArtifact> {
+        match self {
+            Self::NotEligible => None,
+            Self::Registered(artifact) => Some(artifact),
+        }
+    }
+}
+
 /// Namespace in which reconciliation observed an artifact inode.  It does not
 /// disclose the artifact's digest or filesystem location.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]

crates/crank-artifacts/src/recovery.rs

@@ -0,0 +1,224 @@
+use std::{
+    fmt,
+    os::fd::{AsRawFd, OwnedFd},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use crate::{
+    ArtifactError, ArtifactRef, ArtifactStore, ReconciliationCandidate, ReconciliationCursor,
+    ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
+    ReconciliationScan,
+    housekeeping::{ensure_reconciliation_root, revalidate_candidate},
+    store::{
+        QUARANTINE_DIR, Root, RootLock, checkpointed, ensure_root_unchanged, open_existing_dir,
+        open_existing_file, read_verified, stat_fd,
+    },
+};
+
+pub(crate) const FINAL_NAMESPACE: u8 = 0;
+pub(crate) const QUARANTINE_NAMESPACE: u8 = 1;
+
+pub(crate) fn namespace_name(namespace: u8) -> &'static [u8] {
+    match namespace {
+        FINAL_NAMESPACE => b"sha256",
+        QUARANTINE_NAMESPACE => QUARANTINE_DIR,
+        _ => unreachable!("validated reconciliation namespace"),
+    }
+}
+
+pub(crate) fn namespace_number(namespace: ReconciliationNamespace) -> u8 {
+    match namespace {
+        ReconciliationNamespace::Final => FINAL_NAMESPACE,
+        ReconciliationNamespace::Quarantine => QUARANTINE_NAMESPACE,
+    }
+}
+
+impl fmt::Debug for ReconciliationCandidate {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("ReconciliationCandidate(..)")
+    }
+}
+
+impl ReconciliationCandidate {
+    pub fn namespace(&self) -> ReconciliationNamespace {
+        self.namespace
+    }
+
+    /// Returns whether the candidate has aged past `grace` at `now`.
+    ///
+    /// This only reports the timestamp captured by the scanner. Callers that
+    /// need a registration capability must use [`ArtifactStore::register_reconciliation`],
+    /// which repeats inode and content validation while holding the store lock.
+    pub fn is_grace_eligible(&self, grace: Duration, now: SystemTime) -> bool {
+        let modified = UNIX_EPOCH
+            .checked_add(Duration::new(
+                self.modified_seconds.max(0) as u64,
+                self.modified_nanoseconds.max(0) as u32,
+            ))
+            .unwrap_or(SystemTime::UNIX_EPOCH);
+        now.duration_since(modified).is_ok_and(|age| age >= grace)
+    }
+}
+
+impl ArtifactStore {
+    /// Scans only one reconciliation namespace with a bounded opaque cursor.
+    ///
+    /// A quarantine scan ends after quarantine and never falls through to
+    /// final entries, so recovery can finish before the normal final sweep.
+    pub fn scan_reconciliation_namespace(
+        &self,
+        namespace: ReconciliationNamespace,
+        continuation: Option<ReconciliationCursor>,
+        scan_budget: usize,
+        result_limit: usize,
+    ) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
+        self.scan_reconciliation_bounded(
+            namespace,
+            namespace,
+            continuation,
+            scan_budget,
+            result_limit,
+        )
+    }
+
+    /// Streams entries without disclosing paths/digests, with opaque continuation.
+    pub fn scan_reconciliation(
+        &self,
+        continuation: Option<ReconciliationCursor>,
+        scan_budget: usize,
+        result_limit: usize,
+    ) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
+        self.scan_reconciliation_bounded(
+            ReconciliationNamespace::Final,
+            ReconciliationNamespace::Quarantine,
+            continuation,
+            scan_budget,
+            result_limit,
+        )
+    }
+
+    /// Revalidates a scanned candidate and mints a registration capability
+    /// only for a file that is old enough and whose bytes still match its
+    /// canonical name. The candidate remains usable for a later mutation.
+    pub fn register_reconciliation(
+        &self,
+        candidate: &ReconciliationCandidate,
+        grace: Duration,
+        now: SystemTime,
+    ) -> Result<ReconciliationRegistration, ArtifactError> {
+        let root = self.root()?;
+        let _lock = RootLock::shared(root)?;
+        ensure_reconciliation_root(root, candidate)?;
+        let namespace = namespace_number(candidate.namespace);
+        let namespace_root = open_existing_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
+        let shard = open_existing_dir(namespace_root.as_raw_fd(), candidate.shard.as_bytes())?;
+        revalidate_candidate(candidate, shard.as_raw_fd())?;
+
+        let file = open_existing_file(shard.as_raw_fd(), candidate.name.as_bytes())?;
+        let stat = stat_fd(file.as_raw_fd())?;
+        if stat.st_dev != candidate.dev
+            || stat.st_ino != candidate.ino
+            || stat.st_mtime != candidate.modified_seconds
+            || stat.st_mtime_nsec != candidate.modified_nanoseconds
+        {
+            return Err(ArtifactError::UnsafeRoot);
+        }
+        if !candidate.is_grace_eligible(grace, now) {
+            return Ok(ReconciliationRegistration::NotEligible);
+        }
+
+        let artifact_ref =
+            ArtifactRef::from_digest_hex(&candidate.name).map_err(|_| ArtifactError::UnsafeRoot)?;
+        let bytes = read_verified(&file, &artifact_ref)?;
+        Ok(ReconciliationRegistration::Registered(
+            crate::RegisteredArtifact {
+                artifact_ref,
+                size_bytes: bytes.len(),
+            },
+        ))
+    }
+
+    /// Convenience form that uses the current wall clock for grace checks.
+    pub fn register_reconciliation_candidate(
+        &self,
+        candidate: &ReconciliationCandidate,
+        grace: Duration,
+    ) -> Result<ReconciliationRegistration, ArtifactError> {
+        self.register_reconciliation(candidate, grace, SystemTime::now())
+    }
+
+    /// Probes final and quarantine for an expired claim without disclosing its
+    /// physical identity. Quarantine takes precedence over final.
+    pub fn reconciliation_presence(
+        &self,
+        artifact_ref: &ArtifactRef,
+    ) -> Result<ReconciliationPresence, ArtifactError> {
+        let root = self.root()?;
+        let _lock = match RootLock::shared(root) {
+            Ok(lock) => lock,
+            Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
+            Err(error) => return Err(error),
+        };
+        match ensure_root_unchanged(root) {
+            Ok(()) => {}
+            Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
+            Err(error) => return Err(error),
+        }
+
+        match probe_namespace(root, QUARANTINE_NAMESPACE, artifact_ref) {
+            Ok(Some(presence)) => return Ok(presence),
+            Ok(None) => {}
+            Err(presence) => return Ok(presence),
+        }
+        match probe_namespace(root, FINAL_NAMESPACE, artifact_ref) {
+            Ok(Some(presence)) => Ok(presence),
+            Ok(None) => Ok(ReconciliationPresence::Absent),
+            Err(presence) => Ok(presence),
+        }
+    }
+}
+
+fn probe_namespace(
+    root: &Root,
+    namespace: u8,
+    artifact_ref: &ArtifactRef,
+) -> Result<Option<ReconciliationPresence>, ReconciliationPresence> {
+    let namespace_root = probe_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
+    let Some(namespace_root) = namespace_root else {
+        return Ok(None);
+    };
+    let shard = probe_dir(namespace_root.as_raw_fd(), artifact_ref.shard().as_bytes())?;
+    let Some(shard) = shard else {
+        return Ok(None);
+    };
+    let file = match checkpointed("reconciliation_presence_open", || {
+        open_existing_file(shard.as_raw_fd(), artifact_ref.digest_hex().as_bytes())
+    }) {
+        Ok(fd) => fd,
+        Err(ArtifactError::NotFound) => return Ok(None),
+        Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
+            return Err(ReconciliationPresence::Unsafe);
+        }
+        Err(_) => return Err(ReconciliationPresence::Retryable),
+    };
+    match read_verified(&file, artifact_ref) {
+        Ok(_) => Ok(Some(if namespace == FINAL_NAMESPACE {
+            ReconciliationPresence::Final
+        } else {
+            ReconciliationPresence::Quarantine
+        })),
+        Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
+            Err(ReconciliationPresence::Unsafe)
+        }
+        Err(_) => Err(ReconciliationPresence::Retryable),
+    }
+}
+
+fn probe_dir(parent: i32, name: &[u8]) -> Result<Option<OwnedFd>, ReconciliationPresence> {
+    match open_existing_dir(parent, name) {
+        Ok(fd) => Ok(Some(fd)),
+        Err(ArtifactError::NotFound) => Ok(None),
+        Err(ArtifactError::UnsafeRoot) => Err(ReconciliationPresence::Unsafe),
+        Err(_) => Err(ReconciliationPresence::Retryable),
+    }
+}