feat(artifacts): add fenced reconciliation recovery
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
use std::{
|
||||
fmt,
|
||||
os::fd::{AsRawFd, OwnedFd},
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
ArtifactError, ArtifactRef, ArtifactStore, ReconciliationCandidate, ReconciliationCursor,
|
||||
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
|
||||
ReconciliationScan,
|
||||
housekeeping::{ensure_reconciliation_root, revalidate_candidate},
|
||||
store::{
|
||||
QUARANTINE_DIR, Root, RootLock, checkpointed, ensure_root_unchanged, open_existing_dir,
|
||||
open_existing_file, read_verified, stat_fd,
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) const FINAL_NAMESPACE: u8 = 0;
|
||||
pub(crate) const QUARANTINE_NAMESPACE: u8 = 1;
|
||||
|
||||
pub(crate) fn namespace_name(namespace: u8) -> &'static [u8] {
|
||||
match namespace {
|
||||
FINAL_NAMESPACE => b"sha256",
|
||||
QUARANTINE_NAMESPACE => QUARANTINE_DIR,
|
||||
_ => unreachable!("validated reconciliation namespace"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_number(namespace: ReconciliationNamespace) -> u8 {
|
||||
match namespace {
|
||||
ReconciliationNamespace::Final => FINAL_NAMESPACE,
|
||||
ReconciliationNamespace::Quarantine => QUARANTINE_NAMESPACE,
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ReconciliationCandidate {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ReconciliationCandidate(..)")
|
||||
}
|
||||
}
|
||||
|
||||
impl ReconciliationCandidate {
|
||||
pub fn namespace(&self) -> ReconciliationNamespace {
|
||||
self.namespace
|
||||
}
|
||||
|
||||
/// Returns whether the candidate has aged past `grace` at `now`.
|
||||
///
|
||||
/// This only reports the timestamp captured by the scanner. Callers that
|
||||
/// need a registration capability must use [`ArtifactStore::register_reconciliation`],
|
||||
/// which repeats inode and content validation while holding the store lock.
|
||||
pub fn is_grace_eligible(&self, grace: Duration, now: SystemTime) -> bool {
|
||||
let modified = UNIX_EPOCH
|
||||
.checked_add(Duration::new(
|
||||
self.modified_seconds.max(0) as u64,
|
||||
self.modified_nanoseconds.max(0) as u32,
|
||||
))
|
||||
.unwrap_or(SystemTime::UNIX_EPOCH);
|
||||
now.duration_since(modified).is_ok_and(|age| age >= grace)
|
||||
}
|
||||
}
|
||||
|
||||
impl ArtifactStore {
|
||||
/// Scans only one reconciliation namespace with a bounded opaque cursor.
|
||||
///
|
||||
/// A quarantine scan ends after quarantine and never falls through to
|
||||
/// final entries, so recovery can finish before the normal final sweep.
|
||||
pub fn scan_reconciliation_namespace(
|
||||
&self,
|
||||
namespace: ReconciliationNamespace,
|
||||
continuation: Option<ReconciliationCursor>,
|
||||
scan_budget: usize,
|
||||
result_limit: usize,
|
||||
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
|
||||
self.scan_reconciliation_bounded(
|
||||
namespace,
|
||||
namespace,
|
||||
continuation,
|
||||
scan_budget,
|
||||
result_limit,
|
||||
)
|
||||
}
|
||||
|
||||
/// Streams entries without disclosing paths/digests, with opaque continuation.
|
||||
pub fn scan_reconciliation(
|
||||
&self,
|
||||
continuation: Option<ReconciliationCursor>,
|
||||
scan_budget: usize,
|
||||
result_limit: usize,
|
||||
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
|
||||
self.scan_reconciliation_bounded(
|
||||
ReconciliationNamespace::Final,
|
||||
ReconciliationNamespace::Quarantine,
|
||||
continuation,
|
||||
scan_budget,
|
||||
result_limit,
|
||||
)
|
||||
}
|
||||
|
||||
/// Revalidates a scanned candidate and mints a registration capability
|
||||
/// only for a file that is old enough and whose bytes still match its
|
||||
/// canonical name. The candidate remains usable for a later mutation.
|
||||
pub fn register_reconciliation(
|
||||
&self,
|
||||
candidate: &ReconciliationCandidate,
|
||||
grace: Duration,
|
||||
now: SystemTime,
|
||||
) -> Result<ReconciliationRegistration, ArtifactError> {
|
||||
let root = self.root()?;
|
||||
let _lock = RootLock::shared(root)?;
|
||||
ensure_reconciliation_root(root, candidate)?;
|
||||
let namespace = namespace_number(candidate.namespace);
|
||||
let namespace_root = open_existing_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
|
||||
let shard = open_existing_dir(namespace_root.as_raw_fd(), candidate.shard.as_bytes())?;
|
||||
revalidate_candidate(candidate, shard.as_raw_fd())?;
|
||||
|
||||
let file = open_existing_file(shard.as_raw_fd(), candidate.name.as_bytes())?;
|
||||
let stat = stat_fd(file.as_raw_fd())?;
|
||||
if stat.st_dev != candidate.dev
|
||||
|| stat.st_ino != candidate.ino
|
||||
|| stat.st_mtime != candidate.modified_seconds
|
||||
|| stat.st_mtime_nsec != candidate.modified_nanoseconds
|
||||
{
|
||||
return Err(ArtifactError::UnsafeRoot);
|
||||
}
|
||||
if !candidate.is_grace_eligible(grace, now) {
|
||||
return Ok(ReconciliationRegistration::NotEligible);
|
||||
}
|
||||
|
||||
let artifact_ref =
|
||||
ArtifactRef::from_digest_hex(&candidate.name).map_err(|_| ArtifactError::UnsafeRoot)?;
|
||||
let bytes = read_verified(&file, &artifact_ref)?;
|
||||
Ok(ReconciliationRegistration::Registered(
|
||||
crate::RegisteredArtifact {
|
||||
artifact_ref,
|
||||
size_bytes: bytes.len(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
/// Convenience form that uses the current wall clock for grace checks.
|
||||
pub fn register_reconciliation_candidate(
|
||||
&self,
|
||||
candidate: &ReconciliationCandidate,
|
||||
grace: Duration,
|
||||
) -> Result<ReconciliationRegistration, ArtifactError> {
|
||||
self.register_reconciliation(candidate, grace, SystemTime::now())
|
||||
}
|
||||
|
||||
/// Probes final and quarantine for an expired claim without disclosing its
|
||||
/// physical identity. Quarantine takes precedence over final.
|
||||
pub fn reconciliation_presence(
|
||||
&self,
|
||||
artifact_ref: &ArtifactRef,
|
||||
) -> Result<ReconciliationPresence, ArtifactError> {
|
||||
let root = self.root()?;
|
||||
let _lock = match RootLock::shared(root) {
|
||||
Ok(lock) => lock,
|
||||
Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
match ensure_root_unchanged(root) {
|
||||
Ok(()) => {}
|
||||
Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
|
||||
match probe_namespace(root, QUARANTINE_NAMESPACE, artifact_ref) {
|
||||
Ok(Some(presence)) => return Ok(presence),
|
||||
Ok(None) => {}
|
||||
Err(presence) => return Ok(presence),
|
||||
}
|
||||
match probe_namespace(root, FINAL_NAMESPACE, artifact_ref) {
|
||||
Ok(Some(presence)) => Ok(presence),
|
||||
Ok(None) => Ok(ReconciliationPresence::Absent),
|
||||
Err(presence) => Ok(presence),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn probe_namespace(
|
||||
root: &Root,
|
||||
namespace: u8,
|
||||
artifact_ref: &ArtifactRef,
|
||||
) -> Result<Option<ReconciliationPresence>, ReconciliationPresence> {
|
||||
let namespace_root = probe_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
|
||||
let Some(namespace_root) = namespace_root else {
|
||||
return Ok(None);
|
||||
};
|
||||
let shard = probe_dir(namespace_root.as_raw_fd(), artifact_ref.shard().as_bytes())?;
|
||||
let Some(shard) = shard else {
|
||||
return Ok(None);
|
||||
};
|
||||
let file = match checkpointed("reconciliation_presence_open", || {
|
||||
open_existing_file(shard.as_raw_fd(), artifact_ref.digest_hex().as_bytes())
|
||||
}) {
|
||||
Ok(fd) => fd,
|
||||
Err(ArtifactError::NotFound) => return Ok(None),
|
||||
Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
|
||||
return Err(ReconciliationPresence::Unsafe);
|
||||
}
|
||||
Err(_) => return Err(ReconciliationPresence::Retryable),
|
||||
};
|
||||
match read_verified(&file, artifact_ref) {
|
||||
Ok(_) => Ok(Some(if namespace == FINAL_NAMESPACE {
|
||||
ReconciliationPresence::Final
|
||||
} else {
|
||||
ReconciliationPresence::Quarantine
|
||||
})),
|
||||
Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
|
||||
Err(ReconciliationPresence::Unsafe)
|
||||
}
|
||||
Err(_) => Err(ReconciliationPresence::Retryable),
|
||||
}
|
||||
}
|
||||
|
||||
fn probe_dir(parent: i32, name: &[u8]) -> Result<Option<OwnedFd>, ReconciliationPresence> {
|
||||
match open_existing_dir(parent, name) {
|
||||
Ok(fd) => Ok(Some(fd)),
|
||||
Err(ArtifactError::NotFound) => Ok(None),
|
||||
Err(ArtifactError::UnsafeRoot) => Err(ReconciliationPresence::Unsafe),
|
||||
Err(_) => Err(ReconciliationPresence::Retryable),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user