fix(openapi): harden story 2.1 production lifecycle

crates/crank-artifacts/tests/artifact_store.rs

@@ -360,6 +360,119 @@ fn housekeeping_enforces_result_limit_after_stale_validation() {
     assert!(temp_paths.iter().all(|path| path.exists()));
 }
 
+#[test]
+fn stale_temp_cleanup_round_robins_past_a_busy_early_shard() {
+    let root = TestRoot::new("cleanup-round-robin");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let sha = root.0.join("sha256");
+    let early = sha.join("00");
+    let late = sha.join("ff");
+    fs::create_dir(&sha).unwrap();
+    fs::create_dir(&early).unwrap();
+    fs::create_dir(&late).unwrap();
+    for directory in [&sha, &early, &late] {
+        fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).unwrap();
+    }
+    for index in 0..5 {
+        let entry = early.join(format!("unrelated-{index}"));
+        fs::write(&entry, b"not a temp").unwrap();
+        fs::set_permissions(entry, fs::Permissions::from_mode(0o400)).unwrap();
+    }
+    let stale = late.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-1");
+    fs::write(&stale, b"partial").unwrap();
+    fs::set_permissions(&stale, fs::Permissions::from_mode(0o400)).unwrap();
+
+    let (_, first, cursor) = store
+        .scan_stale_temps_after(Duration::ZERO, None, 5, 1)
+        .unwrap();
+    assert!(first.is_empty());
+
+    let (_, second, _) = store
+        .scan_stale_temps_after(Duration::ZERO, Some(cursor), 5, 1)
+        .unwrap();
+    assert_eq!(second.len(), 1);
+    store
+        .delete_stale_temp(second.into_iter().next().unwrap())
+        .unwrap();
+    assert!(!stale.exists());
+}
+
+#[test]
+fn stale_temp_cleanup_resumes_inside_a_busy_shard() {
+    let root = TestRoot::new("cleanup-intra-shard");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let sha = root.0.join("sha256");
+    let shard = sha.join("00");
+    fs::create_dir(&sha).unwrap();
+    fs::create_dir(&shard).unwrap();
+    for directory in [&sha, &shard] {
+        fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).unwrap();
+    }
+    for index in 0..12 {
+        let entry = shard.join(format!("unrelated-{index}"));
+        fs::write(&entry, b"not a temp").unwrap();
+        fs::set_permissions(entry, fs::Permissions::from_mode(0o400)).unwrap();
+    }
+    let stale = shard.join(".crank-artifact-tmp-v1-00000000000000000000000000000000-1-2");
+    fs::write(&stale, b"partial").unwrap();
+    fs::set_permissions(&stale, fs::Permissions::from_mode(0o400)).unwrap();
+
+    let mut cursor = None;
+    let mut discovered = None;
+    for _ in 0..4 {
+        let (_, candidates, next_cursor) = store
+            .scan_stale_temps_after(Duration::ZERO, cursor, 5, 1)
+            .unwrap();
+        if let Some(candidate) = candidates.into_iter().next() {
+            discovered = Some(candidate);
+            break;
+        }
+        cursor = Some(next_cursor);
+    }
+    let candidate = discovered.expect("the temp behind one scan budget is eventually discovered");
+    store.delete_stale_temp(candidate).unwrap();
+    assert!(!stale.exists());
+}
+
+#[test]
+fn stale_temp_cleanup_reconsiders_candidates_beyond_each_result_page() {
+    let root = TestRoot::new("cleanup-result-continuation");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let sha = root.0.join("sha256");
+    let shard = sha.join("00");
+    fs::create_dir(&sha).unwrap();
+    fs::create_dir(&shard).unwrap();
+    for directory in [&sha, &shard] {
+        fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).unwrap();
+    }
+    for index in 0..20 {
+        let temp = shard.join(format!(
+            ".crank-artifact-tmp-v1-00000000000000000000000000000000-1-{index}"
+        ));
+        fs::write(&temp, b"partial").unwrap();
+        fs::set_permissions(temp, fs::Permissions::from_mode(0o400)).unwrap();
+    }
+
+    let mut cursor = None;
+    let mut deleted = 0;
+    for _ in 0..16 {
+        let (scan, candidates, next_cursor) = store
+            .scan_stale_temps_after(Duration::ZERO, cursor, 512, 3)
+            .unwrap();
+        for candidate in candidates {
+            store.delete_stale_temp(candidate).unwrap();
+            deleted += 1;
+        }
+        cursor = Some(next_cursor);
+        if scan.complete {
+            break;
+        }
+    }
+
+    assert_eq!(deleted, 20, "every stale candidate must remain reachable");
+    assert!(fs::read_dir(shard).unwrap().next().is_none());
+}
+
 #[test]
 fn housekeeping_revalidates_subsecond_mtime() {
     let root = TestRoot::new("cleanup-subsecond");
@@ -479,6 +592,16 @@ fn open_rejects_non_private_root_and_existing_finals_must_be_immutable() {
     );
 }
 
+#[test]
+fn health_check_revalidates_the_pinned_root_permissions() {
+    let root = TestRoot::new("health-check");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    assert_eq!(store.check_health(), Ok(()));
+
+    fs::set_permissions(&root.0, fs::Permissions::from_mode(0o755)).unwrap();
+    assert_eq!(store.check_health(), Err(ArtifactError::UnsafeRoot));
+}
+
 #[test]
 fn open_rejects_symlinked_root() {
     let root = TestRoot::new("root-symlink-target");

crates/crank-artifacts/tests/reconciliation.rs

@@ -3,6 +3,7 @@ use std::{
     os::unix::fs::{MetadataExt, PermissionsExt},
     path::PathBuf,
     sync::atomic::{AtomicU64, Ordering},
+    sync::{Mutex, OnceLock},
     time::{Duration, SystemTime},
 };
 
@@ -19,8 +20,6 @@ use crank_artifacts::test_support::{
     FaultAction, checkpoint_hits, clear_checkpoint, reset_traversal_calls, set_checkpoint,
     set_checkpoint_on_hit, traversal_calls, wait_until_held,
 };
-#[cfg(debug_assertions)]
-use std::sync::{Mutex, OnceLock};
 
 static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
 const FULL_SCAN_BUDGET: usize = 2048;
@@ -47,7 +46,6 @@ impl Drop for TestRoot {
     }
 }
 
-#[cfg(debug_assertions)]
 fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
     static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
     GUARD.get_or_init(|| Mutex::new(())).lock().unwrap()
@@ -55,6 +53,7 @@ fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
 
 #[test]
 fn paginates_final_entries_without_disclosing_locations() {
+    let _guard = fault_guard();
     let root = TestRoot::new("pages");
     let store = ArtifactStore::open(&root.0).unwrap();
     store.put(b"reconciliation first").unwrap();
@@ -116,6 +115,7 @@ fn paginates_final_entries_without_disclosing_locations() {
 
 #[test]
 fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
+    let _guard = fault_guard();
     let root = TestRoot::new("malformed");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"reconciliation valid").unwrap();
@@ -157,6 +157,7 @@ fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
 
 #[test]
 fn skips_a_valid_publish_temp_without_classifying_it_as_malformed() {
+    let _guard = fault_guard();
     let root = TestRoot::new("valid-temp");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"reconciliation with concurrent temp").unwrap();
@@ -181,6 +182,7 @@ fn skips_a_valid_publish_temp_without_classifying_it_as_malformed() {
 
 #[test]
 fn unsafe_canonical_shard_does_not_starve_later_shards() {
+    let _guard = fault_guard();
     let root = TestRoot::new("unsafe-shard");
     let store = ArtifactStore::open(&root.0).unwrap();
     let sha = root.0.join("sha256");
@@ -212,6 +214,7 @@ fn unsafe_canonical_shard_does_not_starve_later_shards() {
 #[test]
 #[cfg(debug_assertions)]
 fn traversal_never_exceeds_the_exact_syscall_budget() {
+    let _guard = fault_guard();
     let root = TestRoot::new("syscall-budget");
     let store = ArtifactStore::open(&root.0).unwrap();
     store.put(b"bounded traversal syscall accounting").unwrap();
@@ -237,6 +240,7 @@ fn traversal_never_exceeds_the_exact_syscall_budget() {
 
 #[test]
 fn rejects_a_cursor_from_another_store() {
+    let _guard = fault_guard();
     let first_root = TestRoot::new("foreign-cursor-first");
     let second_root = TestRoot::new("foreign-cursor-second");
     let first = ArtifactStore::open(&first_root.0).unwrap();
@@ -276,6 +280,7 @@ fn disappearing_entry_after_readdir_is_skipped() {
 
 #[test]
 fn quarantine_and_delete_are_idempotent() {
+    let _guard = fault_guard();
     let root = TestRoot::new("mutation");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"reconciliation mutation").unwrap();
@@ -320,6 +325,7 @@ fn quarantine_and_delete_are_idempotent() {
 
 #[test]
 fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
+    let _guard = fault_guard();
     let root = TestRoot::new("no-clobber");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"reconciliation no-clobber").unwrap();
@@ -358,6 +364,7 @@ fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
 
 #[test]
 fn quarantine_refuses_to_replace_an_unrelated_inode() {
+    let _guard = fault_guard();
     let root = TestRoot::new("unrelated-collision");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"canonical final survives collision").unwrap();
@@ -394,6 +401,7 @@ fn quarantine_refuses_to_replace_an_unrelated_inode() {
 
 #[test]
 fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
+    let _guard = fault_guard();
     let root = TestRoot::new("quarantine-replaced");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store
@@ -431,6 +439,7 @@ fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
 
 #[test]
 fn delete_revalidates_a_post_scan_hardlink_without_inode_replacement() {
+    let _guard = fault_guard();
     let root = TestRoot::new("quarantine-hardlink");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"same quarantined inode gains a link").unwrap();
@@ -496,6 +505,7 @@ fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() {
 
 #[test]
 fn rejects_a_replaced_final_inode() {
+    let _guard = fault_guard();
     let root = TestRoot::new("replaced");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"reconciliation original").unwrap();
@@ -773,6 +783,7 @@ fn crash_windows_are_recoverable() {
 
 #[test]
 fn registration_revalidates_content_and_enforces_grace_without_candidate_identity_access() {
+    let _guard = fault_guard();
     let root = TestRoot::new("registration");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"reconciliation registration").unwrap();
@@ -820,6 +831,7 @@ fn registration_revalidates_content_and_enforces_grace_without_candidate_identit
 
 #[test]
 fn namespace_scan_can_finish_quarantine_recovery_before_final_sweep() {
+    let _guard = fault_guard();
     let root = TestRoot::new("namespace-order");
     let store = ArtifactStore::open(&root.0).unwrap();
     store.put(b"quarantine-first recovery").unwrap();
@@ -856,6 +868,7 @@ fn namespace_scan_can_finish_quarantine_recovery_before_final_sweep() {
 
 #[test]
 fn namespace_scan_rejects_a_cursor_from_a_broader_scan() {
+    let _guard = fault_guard();
     let root = TestRoot::new("namespace-cursor-scope");
     let store = ArtifactStore::open(&root.0).unwrap();
     let (report, _) = store.scan_reconciliation(None, 0, 1).unwrap();
@@ -876,6 +889,7 @@ fn namespace_scan_rejects_a_cursor_from_a_broader_scan() {
 
 #[test]
 fn full_scan_rejects_a_quarantine_only_cursor() {
+    let _guard = fault_guard();
     let root = TestRoot::new("namespace-cursor-narrow");
     let store = ArtifactStore::open(&root.0).unwrap();
     let (report, _) = store
@@ -893,6 +907,7 @@ fn full_scan_rejects_a_quarantine_only_cursor() {
 
 #[test]
 fn presence_probe_distinguishes_final_quarantine_and_absent_without_disclosure() {
+    let _guard = fault_guard();
     let root = TestRoot::new("presence");
     let store = ArtifactStore::open(&root.0).unwrap();
     let stored = store.put(b"presence probe").unwrap();