bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

fix(artifacts): harden reconciliation scanner

Browse Source
This commit is contained in:
bsodfather
2026-08-27 12:48:49 +03:00
parent 889b1bdb57
commit 8784964fb2
6 changed files with 1087 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/crank-artifacts/src/housekeeping.rs
+550 -282
View File
@@ -1,12 +1,13 @@
use std::{
fmt,
os::fd::AsRawFd,
os::fd::{AsRawFd, FromRawFd, OwnedFd},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use crate::temp_scan::{list_names, valid_temp_name};
use crate::{
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
ReconciliationScan,
ReconciliationScan, ReconciliationScanStop,
store::{
QUARANTINE_DIR, RootLock, check_file, checkpointed, ensure_root_unchanged, fsync_fd,
open_existing_dir, open_or_create_dir, rename_no_replace_at, stat_fd, unlinkat,
@@ -16,24 +17,54 @@ use crate::{
const FINAL_NAMESPACE: u8 = 0;
const QUARANTINE_NAMESPACE: u8 = 1;
/// Opaque continuation for a bounded reconciliation traversal. It is valid
/// only for the unchanged pinned root from which it was returned.
/// Opaque bounded-scan continuation, valid only for an unchanged namespace.
/// Discard it after any put, quarantine, delete, or external mutation.
pub struct ReconciliationCursor {
root_dev: u64,
root_ino: u64,
namespace: u8,
shard: u8,
position: i64,
state: ReconciliationCursorState,
}
enum ReconciliationCursorState {
OpenNamespace,
ValidateNamespace(OwnedFd),
OpenShard(OwnedFd),
ValidateShard {
namespace: OwnedFd,
shard: OwnedFd,
},
ReadShard {
namespace: OwnedFd,
shard: OwnedFd,
shard_dev: u64,
shard_ino: u64,
buffer: Vec<u8>,
offset: usize,
filled: usize,
cookie: i64,
},
SeekShard {
namespace: OwnedFd,
shard: OwnedFd,
shard_dev: u64,
shard_ino: u64,
buffer: Vec<u8>,
cookie: i64,
},
Transition,
}
struct ShardIdentity(u64, u64);
impl fmt::Debug for ReconciliationCursor {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ReconciliationCursor(..)")
}
}
/// Opaque, inode-bound evidence for one valid final or quarantined artifact.
/// It intentionally exposes neither its digest nor filesystem path.
/// Opaque inode-bound evidence that exposes neither digest nor filesystem path.
#[derive(Clone)]
pub struct ReconciliationCandidate {
root_dev: u64,
@@ -61,8 +92,7 @@ impl ReconciliationCandidate {
}
}
/// Opaque, single-use evidence that a particular stale temporary inode was
/// observed under this store's pinned root. It is not a blob reference.
/// Opaque single-use evidence for a stale temporary inode under the pinned root.
#[derive(Debug)]
pub struct StaleTemp {
name: String,
@@ -83,8 +113,7 @@ pub struct TempScan {
}
impl ArtifactStore {
/// Streams valid final and quarantine entries with an opaque continuation.
/// Neither the report nor the returned capabilities disclose paths/digests.
/// Streams entries without disclosing paths/digests, with opaque continuation.
pub fn scan_reconciliation(
&self,
continuation: Option<ReconciliationCursor>,
@@ -92,8 +121,6 @@ impl ArtifactStore {
result_limit: usize,
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
let root = self.root()?;
let _lock = RootLock::shared(root)?;
ensure_root_unchanged(root)?;
let mut cursor = match continuation {
Some(cursor) => {
if cursor.root_dev != root.dev
@@ -104,128 +131,89 @@ impl ArtifactStore {
}
cursor
}
None => reconciliation_cursor(root.dev, root.ino, FINAL_NAMESPACE, 0, 0),
None => reconciliation_cursor(root.dev, root.ino),
};
let mut report = ReconciliationScan::empty(None);
if scan_budget == 0 || result_limit == 0 {
report.continuation = Some(cursor);
return Ok((report, Vec::new()));
let report = ReconciliationScan::empty(None);
let _lock = match RootLock::shared(root) {
Ok(lock) => lock,
Err(ArtifactError::Storage) => {
return Ok(reconciliation_page(
report,
Vec::new(),
cursor,
ReconciliationScanStop::Retryable,
));
}
Err(error) => return Err(error),
};
match ensure_root_unchanged(root) {
Ok(()) => {}
Err(ArtifactError::Storage) => {
return Ok(reconciliation_page(
report,
Vec::new(),
cursor,
ReconciliationScanStop::Retryable,
));
}
Err(error) => return Err(error),
}
let mut report = report;
if scan_budget == 0 {
return Ok(reconciliation_page(
report,
Vec::new(),
cursor,
ReconciliationScanStop::ScanBudget,
));
}
if result_limit == 0 {
return Ok(reconciliation_page(
report,
Vec::new(),
cursor,
ReconciliationScanStop::ResultLimit,
));
}
let mut remaining = scan_budget;
let mut entries = Vec::new();
while cursor.namespace <= QUARANTINE_NAMESPACE {
if remaining == 0 || entries.len() == result_limit {
report.continuation = Some(cursor);
return Ok((report, entries));
if entries.len() == result_limit {
return Ok(reconciliation_page(
report,
entries,
cursor,
ReconciliationScanStop::ResultLimit,
));
}
let namespace_name = namespace_name(cursor.namespace);
let namespace = match open_existing_dir(root.fd.as_raw_fd(), namespace_name) {
Ok(fd) => fd,
Err(ArtifactError::NotFound) => {
advance_namespace(&mut cursor);
continue;
}
Err(error) => return Err(error),
};
loop {
if remaining == 0 {
report.continuation = Some(cursor);
return Ok((report, entries));
}
let shard_name = format!("{:02x}", cursor.shard);
let shard = match open_existing_dir(namespace.as_raw_fd(), shard_name.as_bytes()) {
Ok(fd) => fd,
Err(ArtifactError::NotFound) => {
consume_scan_work(&mut report, &mut remaining);
if cursor.shard == u8::MAX {
break;
}
advance_shard(&mut cursor);
continue;
}
Err(error) => return Err(error),
};
let shard_stat = stat_fd(shard.as_raw_fd())?;
let mut directory = DirectoryStream::open(shard.as_raw_fd(), cursor.position)?;
loop {
if remaining == 0 || entries.len() == result_limit {
report.continuation = Some(cursor);
return Ok((report, entries));
}
let Some((name, after)) = directory.next()? else {
// An empty directory still consumes one bounded
// traversal unit, so a sparse namespace cannot make
// us probe all 256 shards in one page.
consume_scan_work(&mut report, &mut remaining);
break;
};
cursor.position = after;
consume_scan_work(&mut report, &mut remaining);
let Ok(name) = String::from_utf8(name) else {
report.malformed += 1;
continue;
};
if !valid_final_name(&name, &shard_name) {
report.malformed += 1;
continue;
}
#[cfg(debug_assertions)]
let _ = crate::test_support::checkpoint("reconciliation_before_stat");
let stat = match checkpointed("reconciliation_stat", || {
nofollow_stat(shard.as_raw_fd(), &name)
}) {
Ok(stat) => stat,
Err(ArtifactError::NotFound) => {
// The entry was concurrently removed. It is no
// longer retryable, so the cursor may advance.
continue;
}
Err(ArtifactError::UnsafeRoot) => {
report.unsafe_entries += 1;
continue;
}
Err(error) => return Err(error),
};
if check_file(&stat).is_err() {
report.unsafe_entries += 1;
continue;
}
let namespace = if cursor.namespace == FINAL_NAMESPACE {
report.final_entries += 1;
ReconciliationNamespace::Final
} else {
report.quarantined_entries += 1;
ReconciliationNamespace::Quarantine
};
entries.push(ReconciliationCandidate {
root_dev: root.dev,
root_ino: root.ino,
namespace,
shard: shard_name.clone(),
shard_dev: shard_stat.st_dev,
shard_ino: shard_stat.st_ino,
name,
dev: stat.st_dev,
ino: stat.st_ino,
modified_seconds: stat.st_mtime,
modified_nanoseconds: stat.st_mtime_nsec,
});
// The continuation is deliberately kept at `after`,
// immediately before the next unissued item.
}
if cursor.shard == u8::MAX {
break;
}
advance_shard(&mut cursor);
if report.traversal_syscalls == scan_budget {
return Ok(reconciliation_page(
report,
entries,
cursor,
ReconciliationScanStop::ScanBudget,
));
}
let state = std::mem::replace(&mut cursor.state, ReconciliationCursorState::Transition);
match reconciliation_step(root, &mut cursor, state, &mut report, &mut entries) {
Ok(()) => {}
Err((state, ArtifactError::Storage)) => {
cursor.state = state;
return Ok(reconciliation_page(
report,
entries,
cursor,
ReconciliationScanStop::Retryable,
));
}
Err((_, error)) => return Err(error),
}
advance_namespace(&mut cursor);
}
report.stop = ReconciliationScanStop::Complete;
Ok((report, entries))
}
/// Moves a revalidated final inode into the private quarantine namespace
/// without replacing any existing quarantine inode.
/// Moves a revalidated final inode to quarantine without replacing an inode.
pub fn quarantine_reconciliation(
&self,
candidate: ReconciliationCandidate,
@@ -271,8 +259,7 @@ impl ArtifactStore {
}
}
/// Idempotently removes a revalidated quarantined inode and fsyncs its
/// directory. An fsync ambiguity is explicitly retryable.
/// Idempotently removes a revalidated quarantined inode and fsyncs its directory.
pub fn delete_quarantined_reconciliation(
&self,
candidate: ReconciliationCandidate,
@@ -336,8 +323,7 @@ impl ArtifactStore {
Err(_) => Ok(ReconciliationMutation::Retryable),
}
}
/// Streams temporary entries, reading at most `scan_budget` directory
/// entries and returning at most `result_limit` stale capabilities.
/// Scans at most `scan_budget` entries and returns `result_limit` stale capabilities.
pub fn scan_stale_temps(
&self,
grace: Duration,
@@ -409,8 +395,7 @@ impl ArtifactStore {
Ok((report, result))
}
/// Deletes only a previously discovered temporary inode after an exclusive
/// cooperative root lock and inode revalidation, then fsyncs its shard.
/// Deletes a revalidated stale inode under an exclusive lock, then fsyncs.
pub fn delete_stale_temp(&self, candidate: StaleTemp) -> Result<(), ArtifactError> {
let root = self.root()?;
let _lock = RootLock::exclusive(root)?;
@@ -453,22 +438,433 @@ impl ArtifactStore {
}
}
fn reconciliation_cursor(
root_dev: u64,
root_ino: u64,
namespace: u8,
shard: u8,
position: i64,
) -> ReconciliationCursor {
fn reconciliation_cursor(root_dev: u64, root_ino: u64) -> ReconciliationCursor {
ReconciliationCursor {
root_dev,
root_ino,
namespace: FINAL_NAMESPACE,
shard: 0,
state: ReconciliationCursorState::OpenNamespace,
}
}
fn reconciliation_page(
mut report: ReconciliationScan,
entries: Vec<ReconciliationCandidate>,
cursor: ReconciliationCursor,
stop: ReconciliationScanStop,
) -> (ReconciliationScan, Vec<ReconciliationCandidate>) {
report.stop = stop;
report.continuation = Some(cursor);
(report, entries)
}
fn reconciliation_step(
root: &crate::store::Root,
cursor: &mut ReconciliationCursor,
state: ReconciliationCursorState,
report: &mut ReconciliationScan,
entries: &mut Vec<ReconciliationCandidate>,
) -> Result<(), (ReconciliationCursorState, ArtifactError)> {
match state {
ReconciliationCursorState::OpenNamespace => {
count_traversal_syscall(report);
match open_traversal_dir(root.fd.as_raw_fd(), namespace_name(cursor.namespace)) {
Ok(namespace) => {
cursor.state = ReconciliationCursorState::ValidateNamespace(namespace);
}
Err(ArtifactError::NotFound) => advance_namespace(cursor),
Err(ArtifactError::UnsafeRoot) => {
report.unsafe_entries += 1;
advance_namespace(cursor);
}
Err(error) => {
return Err((ReconciliationCursorState::OpenNamespace, error));
}
}
}
ReconciliationCursorState::ValidateNamespace(namespace) => {
count_traversal_syscall(report);
match validate_traversal_dir(&namespace) {
Ok(_) => {
cursor.state = ReconciliationCursorState::OpenShard(namespace);
}
Err(ArtifactError::UnsafeRoot) => {
report.unsafe_entries += 1;
advance_namespace(cursor);
}
Err(error) => {
return Err((
ReconciliationCursorState::ValidateNamespace(namespace),
error,
));
}
}
}
ReconciliationCursorState::OpenShard(namespace) => {
let shard_name = format!("{:02x}", cursor.shard);
count_traversal_syscall(report);
match open_traversal_dir(namespace.as_raw_fd(), shard_name.as_bytes()) {
Ok(shard) => {
cursor.state = ReconciliationCursorState::ValidateShard { namespace, shard };
}
Err(ArtifactError::NotFound) => advance_reconciliation_shard(cursor, namespace),
Err(ArtifactError::UnsafeRoot) => {
report.unsafe_entries += 1;
advance_reconciliation_shard(cursor, namespace);
}
Err(error) => {
return Err((ReconciliationCursorState::OpenShard(namespace), error));
}
}
}
ReconciliationCursorState::ValidateShard { namespace, shard } => {
count_traversal_syscall(report);
match validate_traversal_dir(&shard) {
Ok(stat) => {
cursor.state = read_shard_state(
namespace,
shard,
ShardIdentity(stat.st_dev, stat.st_ino),
vec![0; 8192],
0,
0,
0,
);
}
Err(ArtifactError::UnsafeRoot) => {
report.unsafe_entries += 1;
advance_reconciliation_shard(cursor, namespace);
}
Err(error) => {
return Err((
ReconciliationCursorState::ValidateShard { namespace, shard },
error,
));
}
}
}
ReconciliationCursorState::ReadShard {
namespace,
shard,
shard_dev,
shard_ino,
mut buffer,
mut offset,
mut filled,
mut cookie,
} => {
if offset == filled {
count_traversal_syscall(report);
match checkpointed("reconciliation_getdents", || {
getdents64(shard.as_raw_fd(), &mut buffer)
}) {
Ok(0) => advance_reconciliation_shard(cursor, namespace),
Ok(count) => {
offset = 0;
filled = count;
cursor.state = read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
offset,
filled,
cookie,
);
}
Err(error) => {
return Err((
ReconciliationCursorState::SeekShard {
namespace,
shard,
shard_dev,
shard_ino,
buffer,
cookie,
},
error,
));
}
}
return Ok(());
}
let (name, after, after_cookie) = match parse_dirent(&buffer[..filled], offset) {
Ok(entry) => entry,
Err(error) => {
return Err((
read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
offset,
filled,
cookie,
),
error,
));
}
};
if name == b"." || name == b".." {
offset = after;
cookie = after_cookie;
cursor.state = read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
offset,
filled,
cookie,
);
return Ok(());
}
let shard_name = format!("{:02x}", cursor.shard);
let Ok(name) = String::from_utf8(name) else {
report.scanned += 1;
report.malformed += 1;
offset = after;
cookie = after_cookie;
cursor.state = read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
offset,
filled,
cookie,
);
return Ok(());
};
if !valid_final_name(&name, &shard_name) {
report.scanned += 1;
if cursor.namespace != FINAL_NAMESPACE || !valid_temp_name(&name) {
report.malformed += 1;
}
offset = after;
cookie = after_cookie;
cursor.state = read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
offset,
filled,
cookie,
);
return Ok(());
}
#[cfg(debug_assertions)]
let _ = crate::test_support::checkpoint("reconciliation_before_stat");
count_traversal_syscall(report);
let stat = match checkpointed("reconciliation_stat", || {
record_traversal_call();
nofollow_stat(shard.as_raw_fd(), &name)
}) {
Ok(stat) => Some(stat),
Err(ArtifactError::NotFound) => None,
Err(ArtifactError::UnsafeRoot) => {
report.unsafe_entries += 1;
None
}
Err(error) => {
return Err((
read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
offset,
filled,
cookie,
),
error,
));
}
};
report.scanned += 1;
offset = after;
cookie = after_cookie;
if let Some(stat) = stat {
if check_file(&stat).is_err() {
report.unsafe_entries += 1;
} else {
let candidate_namespace = if cursor.namespace == FINAL_NAMESPACE {
report.final_entries += 1;
ReconciliationNamespace::Final
} else {
report.quarantined_entries += 1;
ReconciliationNamespace::Quarantine
};
entries.push(ReconciliationCandidate {
root_dev: root.dev,
root_ino: root.ino,
namespace: candidate_namespace,
shard: shard_name,
shard_dev,
shard_ino,
name,
dev: stat.st_dev,
ino: stat.st_ino,
modified_seconds: stat.st_mtime,
modified_nanoseconds: stat.st_mtime_nsec,
});
}
}
cursor.state = read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
offset,
filled,
cookie,
);
}
ReconciliationCursorState::SeekShard {
namespace,
shard,
shard_dev,
shard_ino,
buffer,
cookie,
} => {
count_traversal_syscall(report);
record_traversal_call();
if unsafe { libc::lseek(shard.as_raw_fd(), cookie, libc::SEEK_SET) } != cookie {
return Err((
ReconciliationCursorState::SeekShard {
namespace,
shard,
shard_dev,
shard_ino,
buffer,
cookie,
},
ArtifactError::Storage,
));
}
cursor.state = read_shard_state(
namespace,
shard,
ShardIdentity(shard_dev, shard_ino),
buffer,
0,
0,
cookie,
);
}
ReconciliationCursorState::Transition => unreachable!("temporary scanner state"),
}
Ok(())
}
fn count_traversal_syscall(report: &mut ReconciliationScan) {
report.traversal_syscalls += 1;
}
fn read_shard_state(
namespace: OwnedFd,
shard: OwnedFd,
shard_identity: ShardIdentity,
buffer: Vec<u8>,
offset: usize,
filled: usize,
cookie: i64,
) -> ReconciliationCursorState {
let ShardIdentity(shard_dev, shard_ino) = shard_identity;
ReconciliationCursorState::ReadShard {
namespace,
shard,
position,
shard_dev,
shard_ino,
buffer,
offset,
filled,
cookie,
}
}
fn open_traversal_dir(parent: i32, name: &[u8]) -> Result<OwnedFd, ArtifactError> {
let name = crate::store::c_name(name)?;
record_traversal_call();
let raw = unsafe {
libc::openat(
parent,
name.as_ptr(),
libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW,
)
};
if raw < 0 {
match unsafe { *libc::__errno_location() } {
libc::ENOENT => Err(ArtifactError::NotFound),
libc::EACCES | libc::ELOOP | libc::ENOTDIR => Err(ArtifactError::UnsafeRoot),
_ => Err(ArtifactError::Storage),
}
} else {
Ok(unsafe { OwnedFd::from_raw_fd(raw) })
}
}
fn validate_traversal_dir(directory: &OwnedFd) -> Result<libc::stat, ArtifactError> {
record_traversal_call();
let stat = stat_fd(directory.as_raw_fd())?;
if stat.st_uid != unsafe { libc::geteuid() }
|| (stat.st_mode & libc::S_IFMT) != libc::S_IFDIR
|| (stat.st_mode & 0o777) != 0o700
{
Err(ArtifactError::UnsafeRoot)
} else {
Ok(stat)
}
}
fn getdents64(directory: i32, buffer: &mut [u8]) -> Result<usize, ArtifactError> {
record_traversal_call();
let count = unsafe {
libc::syscall(
libc::SYS_getdents64,
directory,
buffer.as_mut_ptr(),
buffer.len(),
)
};
if count < 0 {
Err(ArtifactError::Storage)
} else {
usize::try_from(count).map_err(|_| ArtifactError::Storage)
}
}
fn parse_dirent(buffer: &[u8], offset: usize) -> Result<(Vec<u8>, usize, i64), ArtifactError> {
const HEADER: usize = 19;
if buffer.len().saturating_sub(offset) < HEADER {
return Err(ArtifactError::Storage);
}
let record_length = usize::from(u16::from_ne_bytes([
buffer[offset + 16],
buffer[offset + 17],
]));
let after = offset
.checked_add(record_length)
.filter(|after| record_length >= HEADER && *after <= buffer.len())
.ok_or(ArtifactError::Storage)?;
let raw_name = &buffer[offset + HEADER..after];
let end = raw_name
.iter()
.position(|byte| *byte == 0)
.ok_or(ArtifactError::Storage)?;
let cookie = i64::from_ne_bytes(
buffer[offset + 8..offset + 16]
.try_into()
.map_err(|_| ArtifactError::Storage)?,
);
Ok((raw_name[..end].to_vec(), after, cookie))
}
fn namespace_name(namespace: u8) -> &'static [u8] {
match namespace {
FINAL_NAMESPACE => b"sha256",
@@ -477,22 +873,19 @@ fn namespace_name(namespace: u8) -> &'static [u8] {
}
}
fn advance_shard(cursor: &mut ReconciliationCursor) {
debug_assert_ne!(cursor.shard, u8::MAX);
cursor.shard += 1;
cursor.position = 0;
fn advance_reconciliation_shard(cursor: &mut ReconciliationCursor, namespace: OwnedFd) {
if cursor.shard == u8::MAX {
advance_namespace(cursor);
} else {
cursor.shard += 1;
cursor.state = ReconciliationCursorState::OpenShard(namespace);
}
}
fn advance_namespace(cursor: &mut ReconciliationCursor) {
cursor.namespace = cursor.namespace.saturating_add(1);
cursor.shard = 0;
cursor.position = 0;
}
fn consume_scan_work(report: &mut ReconciliationScan, remaining: &mut usize) {
debug_assert!(*remaining > 0);
*remaining -= 1;
report.scanned += 1;
cursor.state = ReconciliationCursorState::OpenNamespace;
}
fn ensure_reconciliation_root(
@@ -530,10 +923,8 @@ fn revalidate_candidate(
check_file(&stat).map_err(|_| ArtifactError::UnsafeRoot)
}
/// Recovery follows a successful cross-directory rename. The candidate remains
/// bound to the original final shard, so the quarantine shard identity cannot
/// equal its recorded shard identity; only the canonical name and inode are
/// revalidated here.
/// After cross-directory rename, only canonical name and inode remain common;
/// the quarantine shard cannot equal the capability's recorded final shard.
fn revalidate_moved_candidate(
candidate: &ReconciliationCandidate,
shard: i32,
@@ -561,9 +952,7 @@ fn fsync_mutation(
}
}
/// Persist both directory entry changes even if the first fsync reports an
/// error. Either failure remains an ambiguity, but skipping the second fsync
/// would needlessly enlarge the crash window.
/// Attempts both directory fsyncs; either failure leaves a retryable ambiguity.
fn fsync_quarantine_dirs(
final_shard: &std::os::fd::OwnedFd,
quarantine_shard: &std::os::fd::OwnedFd,
@@ -581,132 +970,6 @@ fn valid_final_name(name: &str, shard: &str) -> bool {
name.len() == 64 && name.starts_with(shard) && name.bytes().all(is_lower_hex)
}
struct DirectoryStream {
directory: *mut libc::DIR,
}
impl DirectoryStream {
fn open(parent: i32, position: i64) -> Result<Self, ArtifactError> {
let duplicate = crate::store::duplicate_fd(parent)?;
let raw = std::os::fd::IntoRawFd::into_raw_fd(duplicate);
let directory = unsafe { libc::fdopendir(raw) };
if directory.is_null() {
unsafe {
libc::close(raw);
}
return Err(ArtifactError::Storage);
}
if position != 0 {
unsafe {
libc::seekdir(directory, position as libc::c_long);
}
}
Ok(Self { directory })
}
fn next(&mut self) -> Result<Option<(Vec<u8>, i64)>, ArtifactError> {
loop {
unsafe {
*libc::__errno_location() = 0;
}
let entry = unsafe { libc::readdir(self.directory) };
if entry.is_null() {
if unsafe { *libc::__errno_location() } != 0 {
return Err(ArtifactError::Storage);
}
return Ok(None);
}
let after = unsafe { libc::telldir(self.directory) } as i64;
let name = unsafe { std::ffi::CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes();
if name == b"." || name == b".." {
continue;
}
return Ok(Some((name.to_vec(), after)));
}
}
}
impl Drop for DirectoryStream {
fn drop(&mut self) {
unsafe {
libc::closedir(self.directory);
}
}
}
fn list_names(
parent: i32,
remaining: &mut usize,
report: &mut TempScan,
) -> Result<Vec<String>, ArtifactError> {
let dot = std::ffi::CString::new(".").map_err(|_| ArtifactError::Storage)?;
let raw = unsafe {
libc::openat(
parent,
dot.as_ptr(),
libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC,
)
};
if raw < 0 {
return Err(ArtifactError::Storage);
}
let directory = unsafe { libc::fdopendir(raw) };
if directory.is_null() {
unsafe {
libc::close(raw);
}
return Err(ArtifactError::Storage);
}
let mut names = Vec::new();
loop {
if *remaining == 0 {
break;
}
unsafe {
*libc::__errno_location() = 0;
}
let entry = unsafe { libc::readdir(directory) };
if entry.is_null() {
if unsafe { *libc::__errno_location() } != 0 {
unsafe {
libc::closedir(directory);
}
return Err(ArtifactError::Storage);
}
break;
}
let name = unsafe { std::ffi::CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes();
if name == b"." || name == b".." {
continue;
}
report.scanned += 1;
*remaining -= 1;
if let Ok(name) = std::str::from_utf8(name) {
names.push(name.to_owned());
}
}
unsafe {
libc::closedir(directory);
}
Ok(names)
}
fn valid_temp_name(name: &str) -> bool {
let Some(rest) = name.strip_prefix(ArtifactStore::temp_prefix()) else {
return false;
};
let mut fields = rest.split('-');
let (Some(nonce), Some(pid), Some(sequence), None) =
(fields.next(), fields.next(), fields.next(), fields.next())
else {
return false;
};
nonce.len() == 32
&& nonce.bytes().all(is_lower_hex)
&& !pid.is_empty()
&& pid.bytes().all(|byte| byte.is_ascii_digit())
&& !sequence.is_empty()
&& sequence.bytes().all(|byte| byte.is_ascii_digit())
}
fn is_lower_hex(byte: u8) -> bool {
byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)
}
@@ -719,3 +982,8 @@ fn nofollow_stat(parent: i32, name: &str) -> Result<libc::stat, ArtifactError> {
Err(crate::store::classify_errno())
}
}
fn record_traversal_call() {
#[cfg(debug_assertions)]
crate::test_support::record_traversal_call();
}
crates/crank-artifacts/src/lib.rs
+3 -1
View File
@@ -12,6 +12,7 @@ mod housekeeping;
mod legacy;
mod model;
mod store;
mod temp_scan;
#[cfg(debug_assertions)]
#[doc(hidden)]
@@ -21,6 +22,7 @@ pub use error::ArtifactError;
pub use housekeeping::{ReconciliationCandidate, ReconciliationCursor, StaleTemp, TempScan};
pub use model::{
ArtifactRef, MAX_ARTIFACT_BYTES, MAX_SOURCE_BYTES, ReconciliationMutation,
ReconciliationNamespace, ReconciliationScan, RegisteredArtifact, StoredArtifact,
ReconciliationNamespace, ReconciliationScan, ReconciliationScanStop, RegisteredArtifact,
StoredArtifact,
};
pub use store::ArtifactStore;
crates/crank-artifacts/src/model.rs
+17
View File
@@ -85,17 +85,32 @@ pub enum ReconciliationNamespace {
Quarantine,
}
/// Why a bounded reconciliation page stopped.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ReconciliationScanStop {
Complete,
ScanBudget,
ResultLimit,
Retryable,
}
/// Bounded, redacted progress report from a reconciliation page.
///
/// `continuation` is opaque evidence for resuming an unchanged namespace. It
/// intentionally cannot be created from a caller-supplied path or digest.
#[derive(Debug)]
pub struct ReconciliationScan {
/// Non-dot directory entries classified by this page.
pub scanned: usize,
/// Traversal syscall budget consumed by this page. It equals syscall
/// attempts in production; debug fault injection may stop an attempt just
/// before the syscall. Root validation and locking are fixed overhead.
pub traversal_syscalls: usize,
pub malformed: usize,
pub unsafe_entries: usize,
pub final_entries: usize,
pub quarantined_entries: usize,
pub stop: ReconciliationScanStop,
pub continuation: Option<crate::ReconciliationCursor>,
}
@@ -103,10 +118,12 @@ impl ReconciliationScan {
pub(crate) fn empty(continuation: Option<crate::ReconciliationCursor>) -> Self {
Self {
scanned: 0,
traversal_syscalls: 0,
malformed: 0,
unsafe_entries: 0,
final_entries: 0,
quarantined_entries: 0,
stop: ReconciliationScanStop::Complete,
continuation,
}
}
crates/crank-artifacts/src/temp_scan.rs
+78
View File
@@ -0,0 +1,78 @@
use crate::{ArtifactError, ArtifactStore, TempScan};
pub(crate) fn valid_temp_name(name: &str) -> bool {
let Some(rest) = name.strip_prefix(ArtifactStore::temp_prefix()) else {
return false;
};
let mut fields = rest.split('-');
let (Some(nonce), Some(pid), Some(sequence), None) =
(fields.next(), fields.next(), fields.next(), fields.next())
else {
return false;
};
nonce.len() == 32
&& nonce
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
&& !pid.is_empty()
&& pid.bytes().all(|byte| byte.is_ascii_digit())
&& !sequence.is_empty()
&& sequence.bytes().all(|byte| byte.is_ascii_digit())
}
pub(crate) fn list_names(
parent: i32,
remaining: &mut usize,
report: &mut TempScan,
) -> Result<Vec<String>, ArtifactError> {
let dot = std::ffi::CString::new(".").map_err(|_| ArtifactError::Storage)?;
let raw = unsafe {
libc::openat(
parent,
dot.as_ptr(),
libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC,
)
};
if raw < 0 {
return Err(ArtifactError::Storage);
}
let directory = unsafe { libc::fdopendir(raw) };
if directory.is_null() {
unsafe {
libc::close(raw);
}
return Err(ArtifactError::Storage);
}
let mut names = Vec::new();
loop {
if *remaining == 0 {
break;
}
unsafe {
*libc::__errno_location() = 0;
}
let entry = unsafe { libc::readdir(directory) };
if entry.is_null() {
if unsafe { *libc::__errno_location() } != 0 {
unsafe {
libc::closedir(directory);
}
return Err(ArtifactError::Storage);
}
break;
}
let name = unsafe { std::ffi::CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes();
if name == b"." || name == b".." {
continue;
}
report.scanned += 1;
*remaining -= 1;
if let Ok(name) = std::str::from_utf8(name) {
names.push(name.to_owned());
}
}
unsafe {
libc::closedir(directory);
}
Ok(names)
}
crates/crank-artifacts/src/test_support.rs
+55 -9
View File
@@ -1,10 +1,28 @@
//! Test-only crash and fault control for syscall-coupled storage checkpoints.
use std::{
sync::{Condvar, Mutex, OnceLock},
collections::BTreeMap,
sync::{
Condvar, Mutex, OnceLock,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
static TRAVERSAL_CALLS: AtomicUsize = AtomicUsize::new(0);
pub fn reset_traversal_calls() {
TRAVERSAL_CALLS.store(0, Ordering::SeqCst);
}
pub fn traversal_calls() -> usize {
TRAVERSAL_CALLS.load(Ordering::SeqCst)
}
pub(crate) fn record_traversal_call() {
TRAVERSAL_CALLS.fetch_add(1, Ordering::SeqCst);
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum FaultAction {
Exit,
@@ -13,7 +31,8 @@ pub enum FaultAction {
}
struct State {
checkpoint: Option<(String, FaultAction)>,
checkpoint: Option<(String, FaultAction, usize)>,
hits: BTreeMap<String, usize>,
held: bool,
}
fn slot() -> &'static (Mutex<State>, Condvar) {
@@ -22,6 +41,7 @@ fn slot() -> &'static (Mutex<State>, Condvar) {
(
Mutex::new(State {
checkpoint: None,
hits: BTreeMap::new(),
held: false,
}),
Condvar::new(),
@@ -30,10 +50,25 @@ fn slot() -> &'static (Mutex<State>, Condvar) {
}
pub fn set_checkpoint(stage: impl Into<String>, action: FaultAction) {
set_checkpoint_on_hit(stage, action, 1);
}
pub fn set_checkpoint_on_hit(stage: impl Into<String>, action: FaultAction, hit: usize) {
assert!(hit > 0, "checkpoint hit is one-based");
let (lock, _) = slot();
let mut state = lock.lock().expect("artifact test checkpoint lock poisoned");
state.checkpoint = Some((stage.into(), action, hit));
state.hits.clear();
}
pub fn checkpoint_hits(stage: &str) -> usize {
let (lock, _) = slot();
lock.lock()
.expect("artifact test checkpoint lock poisoned")
.checkpoint = Some((stage.into(), action));
.hits
.get(stage)
.copied()
.unwrap_or(0)
}
pub fn clear_checkpoint() {
@@ -55,10 +90,17 @@ pub fn wait_until_held(timeout: Duration) -> bool {
pub(crate) fn checkpoint(stage: &str) -> Option<FaultAction> {
let (lock, wake) = slot();
let mut state = lock.lock().expect("artifact test checkpoint lock poisoned");
let action = state
.checkpoint
.as_ref()
.and_then(|(expected, action)| (expected == stage).then_some(*action));
let configured = state.checkpoint.clone();
let action = configured.and_then(|(expected, action, target_hit)| {
if expected != stage {
return None;
}
let hit = state.hits.entry(stage.to_owned()).or_insert(0);
if *hit == 0 {
*hit = 1;
}
(*hit == target_hit).then_some(action)
});
if action == Some(FaultAction::Hold) {
state.held = true;
wake.notify_all();
@@ -73,9 +115,13 @@ pub(crate) fn checkpoint(stage: &str) -> Option<FaultAction> {
pub(crate) fn action(stage: &str) -> Option<FaultAction> {
let (lock, _) = slot();
let state = lock.lock().expect("artifact test checkpoint lock poisoned");
let mut state = lock.lock().expect("artifact test checkpoint lock poisoned");
*state.hits.entry(stage.to_owned()).or_insert(0) += 1;
let hit = state.hits[stage];
state
.checkpoint
.as_ref()
.and_then(|(expected, action)| (expected == stage).then_some(*action))
.and_then(|(expected, action, target_hit)| {
(expected == stage && hit == *target_hit).then_some(*action)
})
}