fix(openapi): harden story 2.1 production lifecycle
CI / Rust Checks (push) Failing after 4m6s
CI / UI Checks (push) Has been skipped
CI / Frontend E2E (push) Has been skipped
CI / Community Image Smoke (push) Has been skipped
CI / Deploy (push) Has been skipped

This commit is contained in:
2026-08-29 00:48:22 +03:00
parent 2c94af6791
commit bc03c33387
46 changed files with 2198 additions and 276 deletions
+139 -6
View File
@@ -4,7 +4,7 @@ use std::{
time::{Duration, SystemTime, UNIX_EPOCH},
};
use crate::temp_scan::{list_names, valid_temp_name};
use crate::temp_scan::{list_names_after, valid_temp_name};
use crate::{
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
ReconciliationScan, ReconciliationScanStop,
@@ -94,10 +94,33 @@ pub struct StaleTemp {
grace: Duration,
}
/// Opaque, inode-bound progress marker for the bounded stale-temp sweeper.
///
/// It deliberately advances by shard instead of retaining a directory offset:
/// directory offsets are invalidated by a concurrent writer, while round-robin
/// shard progress prevents a busy low-numbered shard from starving all others.
#[derive(Clone, Debug)]
pub struct TempScanCursor {
root_dev: u64,
root_ino: u64,
next_shard: u8,
shard_continuation: Option<TempShardContinuation>,
}
#[derive(Clone, Debug)]
struct TempShardContinuation {
shard: u8,
dev: u64,
ino: u64,
cookie: i64,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct TempScan {
pub scanned: usize,
pub omitted: usize,
/// True when this page completed a full round-robin traversal.
pub complete: bool,
}
impl ArtifactStore {
@@ -324,27 +347,93 @@ impl ArtifactStore {
scan_budget: usize,
result_limit: usize,
) -> Result<(TempScan, Vec<StaleTemp>), ArtifactError> {
self.scan_stale_temps_after(grace, None, scan_budget, result_limit)
.map(|(report, candidates, _)| (report, candidates))
}
/// Resumes stale-temp cleanup from an opaque round-robin marker. A
/// continuation is valid only for this exact pinned root.
pub fn scan_stale_temps_after(
&self,
grace: Duration,
continuation: Option<TempScanCursor>,
scan_budget: usize,
result_limit: usize,
) -> Result<(TempScan, Vec<StaleTemp>, TempScanCursor), ArtifactError> {
let root = self.root()?;
let _lock = RootLock::shared(root)?;
ensure_root_unchanged(root)?;
let sha = match open_existing_dir(root.fd.as_raw_fd(), b"sha256") {
Ok(fd) => fd,
Err(ArtifactError::NotFound) => return Ok((TempScan::default(), Vec::new())),
Err(ArtifactError::NotFound) => {
return Ok((
TempScan {
complete: true,
..TempScan::default()
},
Vec::new(),
TempScanCursor {
root_dev: root.dev,
root_ino: root.ino,
next_shard: 0,
shard_continuation: None,
},
));
}
Err(error) => return Err(error),
};
let (start_shard, mut shard_continuation) = match continuation {
Some(cursor) if cursor.root_dev == root.dev && cursor.root_ino == root.ino => {
(cursor.next_shard, cursor.shard_continuation)
}
Some(_) => return Err(ArtifactError::UnsafeRoot),
None => (0, None),
};
let mut report = TempScan::default();
let mut remaining = scan_budget;
let mut result = Vec::new();
for shard in (0_u8..=255).map(|value| format!("{value:02x}")) {
if remaining == 0 {
return Ok((
report,
result,
TempScanCursor {
root_dev: root.dev,
root_ino: root.ino,
next_shard: start_shard,
shard_continuation,
},
));
}
let mut next_shard = start_shard;
for offset in 0_u16..=255 {
if remaining == 0 {
break;
}
let shard_number = start_shard.wrapping_add(offset as u8);
next_shard = shard_number.wrapping_add(1);
let shard = format!("{shard_number:02x}");
let shard_fd = match open_existing_dir(sha.as_raw_fd(), shard.as_bytes()) {
Ok(fd) => fd,
Err(ArtifactError::NotFound) => continue,
Err(ArtifactError::NotFound) => {
shard_continuation = None;
continue;
}
Err(error) => return Err(error),
};
for name in list_names(shard_fd.as_raw_fd(), &mut remaining, &mut report)? {
let stat = stat_fd(shard_fd.as_raw_fd())?;
let resume = shard_continuation.take().and_then(|continuation| {
if continuation.shard == shard_number
&& continuation.dev == stat.st_dev
&& continuation.ino == stat.st_ino
{
Some(continuation.cookie)
} else {
None
}
});
let page = list_names_after(shard_fd.as_raw_fd(), &mut remaining, &mut report, resume)?;
for entry in page.names {
let name = entry.name;
if !valid_temp_name(&name) {
continue;
}
@@ -370,6 +459,23 @@ impl ArtifactStore {
{
if result.len() == result_limit {
report.omitted += 1;
if result_limit > 0 {
return Ok((
report,
result,
TempScanCursor {
root_dev: root.dev,
root_ino: root.ino,
next_shard: shard_number,
shard_continuation: Some(TempShardContinuation {
shard: shard_number,
dev: stat.st_dev,
ino: stat.st_ino,
cookie: entry.cookie_before,
}),
},
));
}
continue;
}
result.push(StaleTemp {
@@ -385,8 +491,35 @@ impl ArtifactStore {
});
}
}
if let Some(cookie) = page.continuation {
return Ok((
report,
result,
TempScanCursor {
root_dev: root.dev,
root_ino: root.ino,
next_shard: shard_number,
shard_continuation: Some(TempShardContinuation {
shard: shard_number,
dev: stat.st_dev,
ino: stat.st_ino,
cookie,
}),
},
));
}
}
Ok((report, result))
report.complete = true;
Ok((
report,
result,
TempScanCursor {
root_dev: root.dev,
root_ino: root.ino,
next_shard,
shard_continuation: None,
},
))
}
/// Deletes a revalidated stale inode under an exclusive lock, then fsyncs.
+3 -1
View File
@@ -20,7 +20,9 @@ mod temp_scan;
pub mod test_support;
pub use error::ArtifactError;
pub use housekeeping::{ReconciliationCandidate, ReconciliationCursor, StaleTemp, TempScan};
pub use housekeeping::{
ReconciliationCandidate, ReconciliationCursor, StaleTemp, TempScan, TempScanCursor,
};
pub use model::{
ArtifactRef, MAX_ARTIFACT_BYTES, MAX_SOURCE_BYTES, ReconciliationMutation,
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
+9
View File
@@ -80,6 +80,15 @@ impl ArtifactStore {
})
}
/// Verifies that the pinned root is still the same private directory that
/// was accepted at startup. This does not open data by pathname and is
/// suitable for a process readiness probe.
pub fn check_health(&self) -> Result<(), ArtifactError> {
let root = self.root()?;
let _lock = RootLock::shared(root)?;
ensure_root_unchanged(root)
}
/// Compatibility constructor. I/O returns the opening error fail-closed.
pub fn new(root: impl AsRef<Path>) -> Self {
Self {
+48 -9
View File
@@ -1,5 +1,19 @@
use crate::{ArtifactError, ArtifactStore, TempScan};
pub(crate) struct NamePage {
pub(crate) names: Vec<NameEntry>,
/// Opaque `telldir` position immediately after the last consumed entry.
/// It is meaningful only for the same opened directory inode.
pub(crate) continuation: Option<i64>,
}
pub(crate) struct NameEntry {
pub(crate) name: String,
/// Directory position immediately before this entry. Resuming here
/// guarantees a candidate omitted by a result limit is reconsidered.
pub(crate) cookie_before: i64,
}
pub(crate) fn valid_temp_name(name: &str) -> bool {
let Some(rest) = name.strip_prefix(ArtifactStore::temp_prefix()) else {
return false;
@@ -20,11 +34,12 @@ pub(crate) fn valid_temp_name(name: &str) -> bool {
&& sequence.bytes().all(|byte| byte.is_ascii_digit())
}
pub(crate) fn list_names(
pub(crate) fn list_names_after(
parent: i32,
remaining: &mut usize,
report: &mut TempScan,
) -> Result<Vec<String>, ArtifactError> {
continuation: Option<i64>,
) -> Result<NamePage, ArtifactError> {
let dot = std::ffi::CString::new(".").map_err(|_| ArtifactError::Storage)?;
let raw = unsafe {
libc::openat(
@@ -44,13 +59,32 @@ pub(crate) fn list_names(
return Err(ArtifactError::Storage);
}
let mut names = Vec::new();
if let Some(cookie) = continuation {
unsafe {
libc::seekdir(directory, cookie as libc::c_long);
}
}
loop {
if *remaining == 0 {
break;
let continuation = unsafe { libc::telldir(directory) };
unsafe {
libc::closedir(directory);
}
return Ok(NamePage {
names,
continuation: (continuation >= 0).then_some(continuation as i64),
});
}
unsafe {
*libc::__errno_location() = 0;
}
let cookie_before = unsafe { libc::telldir(directory) };
if cookie_before < 0 {
unsafe {
libc::closedir(directory);
}
return Err(ArtifactError::Storage);
}
let entry = unsafe { libc::readdir(directory) };
if entry.is_null() {
if unsafe { *libc::__errno_location() } != 0 {
@@ -59,7 +93,13 @@ pub(crate) fn list_names(
}
return Err(ArtifactError::Storage);
}
break;
unsafe {
libc::closedir(directory);
}
return Ok(NamePage {
names,
continuation: None,
});
}
let name = unsafe { std::ffi::CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes();
if name == b"." || name == b".." {
@@ -68,11 +108,10 @@ pub(crate) fn list_names(
report.scanned += 1;
*remaining -= 1;
if let Ok(name) = std::str::from_utf8(name) {
names.push(name.to_owned());
names.push(NameEntry {
name: name.to_owned(),
cookie_before: cookie_before as i64,
});
}
}
unsafe {
libc::closedir(directory);
}
Ok(names)
}
@@ -360,6 +360,119 @@ fn housekeeping_enforces_result_limit_after_stale_validation() {
assert!(temp_paths.iter().all(|path| path.exists()));
}
#[test]
fn stale_temp_cleanup_round_robins_past_a_busy_early_shard() {
let root = TestRoot::new("cleanup-round-robin");
let store = ArtifactStore::open(&root.0).unwrap();
let sha = root.0.join("sha256");
let early = sha.join("00");
let late = sha.join("ff");
fs::create_dir(&sha).unwrap();
fs::create_dir(&early).unwrap();
fs::create_dir(&late).unwrap();
for directory in [&sha, &early, &late] {
fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).unwrap();
}
for index in 0..5 {
let entry = early.join(format!("unrelated-{index}"));
fs::write(&entry, b"not a temp").unwrap();
fs::set_permissions(entry, fs::Permissions::from_mode(0o400)).unwrap();
}
let stale = late.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-1");
fs::write(&stale, b"partial").unwrap();
fs::set_permissions(&stale, fs::Permissions::from_mode(0o400)).unwrap();
let (_, first, cursor) = store
.scan_stale_temps_after(Duration::ZERO, None, 5, 1)
.unwrap();
assert!(first.is_empty());
let (_, second, _) = store
.scan_stale_temps_after(Duration::ZERO, Some(cursor), 5, 1)
.unwrap();
assert_eq!(second.len(), 1);
store
.delete_stale_temp(second.into_iter().next().unwrap())
.unwrap();
assert!(!stale.exists());
}
#[test]
fn stale_temp_cleanup_resumes_inside_a_busy_shard() {
let root = TestRoot::new("cleanup-intra-shard");
let store = ArtifactStore::open(&root.0).unwrap();
let sha = root.0.join("sha256");
let shard = sha.join("00");
fs::create_dir(&sha).unwrap();
fs::create_dir(&shard).unwrap();
for directory in [&sha, &shard] {
fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).unwrap();
}
for index in 0..12 {
let entry = shard.join(format!("unrelated-{index}"));
fs::write(&entry, b"not a temp").unwrap();
fs::set_permissions(entry, fs::Permissions::from_mode(0o400)).unwrap();
}
let stale = shard.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-2");
fs::write(&stale, b"partial").unwrap();
fs::set_permissions(&stale, fs::Permissions::from_mode(0o400)).unwrap();
let mut cursor = None;
let mut discovered = None;
for _ in 0..4 {
let (_, candidates, next_cursor) = store
.scan_stale_temps_after(Duration::ZERO, cursor, 5, 1)
.unwrap();
if let Some(candidate) = candidates.into_iter().next() {
discovered = Some(candidate);
break;
}
cursor = Some(next_cursor);
}
let candidate = discovered.expect("the temp behind one scan budget is eventually discovered");
store.delete_stale_temp(candidate).unwrap();
assert!(!stale.exists());
}
#[test]
fn stale_temp_cleanup_reconsiders_candidates_beyond_each_result_page() {
let root = TestRoot::new("cleanup-result-continuation");
let store = ArtifactStore::open(&root.0).unwrap();
let sha = root.0.join("sha256");
let shard = sha.join("00");
fs::create_dir(&sha).unwrap();
fs::create_dir(&shard).unwrap();
for directory in [&sha, &shard] {
fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).unwrap();
}
for index in 0..20 {
let temp = shard.join(format!(
".crank-artifact-tmp-v1-00000000000000000000000000000000-1-{index}"
));
fs::write(&temp, b"partial").unwrap();
fs::set_permissions(temp, fs::Permissions::from_mode(0o400)).unwrap();
}
let mut cursor = None;
let mut deleted = 0;
for _ in 0..16 {
let (scan, candidates, next_cursor) = store
.scan_stale_temps_after(Duration::ZERO, cursor, 512, 3)
.unwrap();
for candidate in candidates {
store.delete_stale_temp(candidate).unwrap();
deleted += 1;
}
cursor = Some(next_cursor);
if scan.complete {
break;
}
}
assert_eq!(deleted, 20, "every stale candidate must remain reachable");
assert!(fs::read_dir(shard).unwrap().next().is_none());
}
#[test]
fn housekeeping_revalidates_subsecond_mtime() {
let root = TestRoot::new("cleanup-subsecond");
@@ -479,6 +592,16 @@ fn open_rejects_non_private_root_and_existing_finals_must_be_immutable() {
);
}
#[test]
fn health_check_revalidates_the_pinned_root_permissions() {
let root = TestRoot::new("health-check");
let store = ArtifactStore::open(&root.0).unwrap();
assert_eq!(store.check_health(), Ok(()));
fs::set_permissions(&root.0, fs::Permissions::from_mode(0o755)).unwrap();
assert_eq!(store.check_health(), Err(ArtifactError::UnsafeRoot));
}
#[test]
fn open_rejects_symlinked_root() {
let root = TestRoot::new("root-symlink-target");
+18 -3
View File
@@ -3,6 +3,7 @@ use std::{
os::unix::fs::{MetadataExt, PermissionsExt},
path::PathBuf,
sync::atomic::{AtomicU64, Ordering},
sync::{Mutex, OnceLock},
time::{Duration, SystemTime},
};
@@ -19,8 +20,6 @@ use crank_artifacts::test_support::{
FaultAction, checkpoint_hits, clear_checkpoint, reset_traversal_calls, set_checkpoint,
set_checkpoint_on_hit, traversal_calls, wait_until_held,
};
#[cfg(debug_assertions)]
use std::sync::{Mutex, OnceLock};
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
const FULL_SCAN_BUDGET: usize = 2048;
@@ -47,7 +46,6 @@ impl Drop for TestRoot {
}
}
#[cfg(debug_assertions)]
fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
GUARD.get_or_init(|| Mutex::new(())).lock().unwrap()
@@ -55,6 +53,7 @@ fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
#[test]
fn paginates_final_entries_without_disclosing_locations() {
let _guard = fault_guard();
let root = TestRoot::new("pages");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"reconciliation first").unwrap();
@@ -116,6 +115,7 @@ fn paginates_final_entries_without_disclosing_locations() {
#[test]
fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
let _guard = fault_guard();
let root = TestRoot::new("malformed");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation valid").unwrap();
@@ -157,6 +157,7 @@ fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
#[test]
fn skips_a_valid_publish_temp_without_classifying_it_as_malformed() {
let _guard = fault_guard();
let root = TestRoot::new("valid-temp");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation with concurrent temp").unwrap();
@@ -181,6 +182,7 @@ fn skips_a_valid_publish_temp_without_classifying_it_as_malformed() {
#[test]
fn unsafe_canonical_shard_does_not_starve_later_shards() {
let _guard = fault_guard();
let root = TestRoot::new("unsafe-shard");
let store = ArtifactStore::open(&root.0).unwrap();
let sha = root.0.join("sha256");
@@ -212,6 +214,7 @@ fn unsafe_canonical_shard_does_not_starve_later_shards() {
#[test]
#[cfg(debug_assertions)]
fn traversal_never_exceeds_the_exact_syscall_budget() {
let _guard = fault_guard();
let root = TestRoot::new("syscall-budget");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"bounded traversal syscall accounting").unwrap();
@@ -237,6 +240,7 @@ fn traversal_never_exceeds_the_exact_syscall_budget() {
#[test]
fn rejects_a_cursor_from_another_store() {
let _guard = fault_guard();
let first_root = TestRoot::new("foreign-cursor-first");
let second_root = TestRoot::new("foreign-cursor-second");
let first = ArtifactStore::open(&first_root.0).unwrap();
@@ -276,6 +280,7 @@ fn disappearing_entry_after_readdir_is_skipped() {
#[test]
fn quarantine_and_delete_are_idempotent() {
let _guard = fault_guard();
let root = TestRoot::new("mutation");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation mutation").unwrap();
@@ -320,6 +325,7 @@ fn quarantine_and_delete_are_idempotent() {
#[test]
fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
let _guard = fault_guard();
let root = TestRoot::new("no-clobber");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation no-clobber").unwrap();
@@ -358,6 +364,7 @@ fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
#[test]
fn quarantine_refuses_to_replace_an_unrelated_inode() {
let _guard = fault_guard();
let root = TestRoot::new("unrelated-collision");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"canonical final survives collision").unwrap();
@@ -394,6 +401,7 @@ fn quarantine_refuses_to_replace_an_unrelated_inode() {
#[test]
fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
let _guard = fault_guard();
let root = TestRoot::new("quarantine-replaced");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store
@@ -431,6 +439,7 @@ fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
#[test]
fn delete_revalidates_a_post_scan_hardlink_without_inode_replacement() {
let _guard = fault_guard();
let root = TestRoot::new("quarantine-hardlink");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"same quarantined inode gains a link").unwrap();
@@ -496,6 +505,7 @@ fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() {
#[test]
fn rejects_a_replaced_final_inode() {
let _guard = fault_guard();
let root = TestRoot::new("replaced");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation original").unwrap();
@@ -773,6 +783,7 @@ fn crash_windows_are_recoverable() {
#[test]
fn registration_revalidates_content_and_enforces_grace_without_candidate_identity_access() {
let _guard = fault_guard();
let root = TestRoot::new("registration");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"reconciliation registration").unwrap();
@@ -820,6 +831,7 @@ fn registration_revalidates_content_and_enforces_grace_without_candidate_identit
#[test]
fn namespace_scan_can_finish_quarantine_recovery_before_final_sweep() {
let _guard = fault_guard();
let root = TestRoot::new("namespace-order");
let store = ArtifactStore::open(&root.0).unwrap();
store.put(b"quarantine-first recovery").unwrap();
@@ -856,6 +868,7 @@ fn namespace_scan_can_finish_quarantine_recovery_before_final_sweep() {
#[test]
fn namespace_scan_rejects_a_cursor_from_a_broader_scan() {
let _guard = fault_guard();
let root = TestRoot::new("namespace-cursor-scope");
let store = ArtifactStore::open(&root.0).unwrap();
let (report, _) = store.scan_reconciliation(None, 0, 1).unwrap();
@@ -876,6 +889,7 @@ fn namespace_scan_rejects_a_cursor_from_a_broader_scan() {
#[test]
fn full_scan_rejects_a_quarantine_only_cursor() {
let _guard = fault_guard();
let root = TestRoot::new("namespace-cursor-narrow");
let store = ArtifactStore::open(&root.0).unwrap();
let (report, _) = store
@@ -893,6 +907,7 @@ fn full_scan_rejects_a_quarantine_only_cursor() {
#[test]
fn presence_probe_distinguishes_final_quarantine_and_absent_without_disclosure() {
let _guard = fault_guard();
let root = TestRoot::new("presence");
let store = ArtifactStore::open(&root.0).unwrap();
let stored = store.put(b"presence probe").unwrap();