feat(artifacts): add bounded reconciliation

2026-08-27 07:04:22 +03:00
parent 38ba898b72
commit 889b1bdb57
5 changed files with 985 additions and 9 deletions
@@ -0,0 +1,430 @@
+use std::{
+    fs,
+    os::unix::fs::PermissionsExt,
+    path::PathBuf,
+    process::Command,
+    sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    },
+    thread,
+    time::Duration,
+};
+
+use crank_artifacts::{
+    ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
+};
+
+#[cfg(debug_assertions)]
+use crank_artifacts::test_support::{
+    FaultAction, clear_checkpoint, set_checkpoint, wait_until_held,
+};
+#[cfg(debug_assertions)]
+use std::sync::{Mutex, OnceLock};
+
+static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
+
+struct TestRoot(PathBuf);
+
+impl TestRoot {
+    fn new(name: &str) -> Self {
+        let path = std::env::temp_dir().join(format!(
+            "crank-artifacts-reconciliation-{name}-{}-{}",
+            std::process::id(),
+            NEXT_ROOT.fetch_add(1, Ordering::Relaxed)
+        ));
+        fs::create_dir(&path).unwrap();
+        fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
+        Self(path)
+    }
+}
+
+impl Drop for TestRoot {
+    fn drop(&mut self) {
+        let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
+        let _ = fs::remove_dir_all(&self.0);
+    }
+}
+
+#[cfg(debug_assertions)]
+fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
+    static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
+    GUARD.get_or_init(|| Mutex::new(())).lock().unwrap()
+}
+
+#[test]
+fn paginates_final_entries_without_disclosing_locations() {
+    let root = TestRoot::new("pages");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    store.put(b"reconciliation first").unwrap();
+    store.put(b"reconciliation second").unwrap();
+
+    let mut continuation = None;
+    let mut candidates = Vec::new();
+    loop {
+        let (report, mut page) = store.scan_reconciliation(continuation, 1, 1).unwrap();
+        assert!(report.scanned <= 1);
+        assert!(page.len() <= 1);
+        candidates.append(&mut page);
+        continuation = report.continuation;
+        if continuation.is_none() {
+            break;
+        }
+    }
+    assert_eq!(candidates.len(), 2);
+    assert!(
+        candidates
+            .iter()
+            .all(|candidate| candidate.namespace() == ReconciliationNamespace::Final)
+    );
+    for candidate in candidates {
+        assert_eq!(
+            store.quarantine_reconciliation(candidate).unwrap(),
+            ReconciliationMutation::Quarantined
+        );
+    }
+    let (_, quarantined_entries) = store.scan_reconciliation(None, 512, 8).unwrap();
+    assert_eq!(quarantined_entries.len(), 2);
+    assert!(
+        quarantined_entries
+            .iter()
+            .all(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
+    );
+}
+
+#[test]
+fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
+    let root = TestRoot::new("malformed");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let stored = store.put(b"reconciliation valid").unwrap();
+    let shard = root
+        .0
+        .join("sha256")
+        .join(&stored.artifact_ref.digest_hex()[..2]);
+    fs::write(shard.join("not-a-digest"), b"junk").unwrap();
+    let unsafe_name = format!(
+        "{}{}",
+        &stored.artifact_ref.digest_hex()[..2],
+        "f".repeat(62)
+    );
+    std::os::unix::fs::symlink(shard.join("not-a-digest"), shard.join(unsafe_name)).unwrap();
+    let directory_name = format!(
+        "{}{}",
+        &stored.artifact_ref.digest_hex()[..2],
+        "e".repeat(62)
+    );
+    fs::create_dir(shard.join(directory_name)).unwrap();
+    let hardlink_name = format!(
+        "{}{}",
+        &stored.artifact_ref.digest_hex()[..2],
+        "d".repeat(62)
+    );
+    fs::hard_link(shard.join("not-a-digest"), shard.join(hardlink_name)).unwrap();
+
+    let (report, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    assert!(report.malformed >= 1);
+    assert!(report.unsafe_entries >= 3);
+    assert_eq!(candidates.len(), 1);
+    assert!(report.continuation.is_none());
+}
+
+#[test]
+fn rejects_a_cursor_from_another_store() {
+    let first_root = TestRoot::new("foreign-cursor-first");
+    let second_root = TestRoot::new("foreign-cursor-second");
+    let first = ArtifactStore::open(&first_root.0).unwrap();
+    let second = ArtifactStore::open(&second_root.0).unwrap();
+    first.put(b"cursor source").unwrap();
+    let (report, _) = first.scan_reconciliation(None, 1, 1).unwrap();
+
+    assert!(matches!(
+        second.scan_reconciliation(report.continuation, 1, 1),
+        Err(ArtifactError::UnsafeRoot)
+    ));
+}
+
+#[test]
+#[cfg(debug_assertions)]
+fn disappearing_entry_after_readdir_is_skipped() {
+    let _guard = fault_guard();
+    let root = TestRoot::new("disappearing");
+    let store = Arc::new(ArtifactStore::open(&root.0).unwrap());
+    let stored = store.put(b"disappearing reconciliation entry").unwrap();
+    let path = root
+        .0
+        .join("sha256")
+        .join(&stored.artifact_ref.digest_hex()[..2])
+        .join(stored.artifact_ref.digest_hex());
+    set_checkpoint("reconciliation_before_stat", FaultAction::Hold);
+    let scan_store = Arc::clone(&store);
+    let scan = thread::spawn(move || scan_store.scan_reconciliation(None, 512, 8));
+    assert!(wait_until_held(Duration::from_secs(2)));
+    fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
+    fs::remove_file(&path).unwrap();
+    clear_checkpoint();
+
+    let (_, candidates) = scan.join().unwrap().unwrap();
+    assert!(candidates.is_empty());
+}
+
+#[test]
+fn quarantine_and_delete_are_idempotent() {
+    let root = TestRoot::new("mutation");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let stored = store.put(b"reconciliation mutation").unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    let candidate = candidates.into_iter().next().unwrap();
+
+    assert_eq!(
+        store.quarantine_reconciliation(candidate.clone()).unwrap(),
+        ReconciliationMutation::Quarantined
+    );
+    assert_eq!(
+        store.read(&stored.artifact_ref),
+        Err(ArtifactError::NotFound)
+    );
+    assert_eq!(
+        store.quarantine_reconciliation(candidate).unwrap(),
+        ReconciliationMutation::AlreadyQuarantined
+    );
+
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    let quarantined = candidates
+        .into_iter()
+        .find(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
+        .unwrap();
+    assert_eq!(
+        store
+            .delete_quarantined_reconciliation(quarantined.clone())
+            .unwrap(),
+        ReconciliationMutation::Deleted
+    );
+    assert_eq!(
+        store
+            .delete_quarantined_reconciliation(quarantined)
+            .unwrap(),
+        ReconciliationMutation::AlreadyAbsent
+    );
+}
+
+#[test]
+fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
+    let root = TestRoot::new("no-clobber");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let stored = store.put(b"reconciliation no-clobber").unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    let original = candidates.into_iter().next().unwrap();
+    assert_eq!(
+        store.quarantine_reconciliation(original.clone()).unwrap(),
+        ReconciliationMutation::Quarantined
+    );
+    store.put(b"reconciliation no-clobber").unwrap();
+    assert_eq!(
+        store.quarantine_reconciliation(original).unwrap(),
+        ReconciliationMutation::AlreadyQuarantined
+    );
+    assert_eq!(
+        store.read(&stored.artifact_ref).unwrap(),
+        b"reconciliation no-clobber"
+    );
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    assert_eq!(candidates.len(), 2);
+    assert!(
+        candidates
+            .iter()
+            .any(|candidate| candidate.namespace() == ReconciliationNamespace::Final)
+    );
+    assert!(
+        candidates
+            .iter()
+            .any(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
+    );
+}
+
+#[test]
+fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
+    let root = TestRoot::new("quarantine-replaced");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let stored = store
+        .put(b"reconciliation quarantined replacement")
+        .unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    store
+        .quarantine_reconciliation(candidates.into_iter().next().unwrap())
+        .unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    let candidate = candidates.into_iter().next().unwrap();
+    let path = root
+        .0
+        .join("quarantine")
+        .join(&stored.artifact_ref.digest_hex()[..2])
+        .join(stored.artifact_ref.digest_hex());
+    fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
+    fs::remove_file(&path).unwrap();
+    fs::write(&path, b"replacement").unwrap();
+    let sibling = path.with_extension("link");
+    fs::hard_link(&path, &sibling).unwrap();
+    fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap();
+
+    assert_eq!(
+        store.delete_quarantined_reconciliation(candidate),
+        Err(ArtifactError::UnsafeRoot)
+    );
+    assert!(path.exists());
+    assert!(sibling.exists());
+}
+
+#[test]
+#[cfg(debug_assertions)]
+fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() {
+    let _guard = fault_guard();
+    let root = TestRoot::new("delete-fsync");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    store.put(b"reconciliation delete fsync").unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    store
+        .quarantine_reconciliation(candidates.into_iter().next().unwrap())
+        .unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    let candidate = candidates.into_iter().next().unwrap();
+    set_checkpoint("reconciliation_delete_fsync", FaultAction::Fail);
+    assert_eq!(
+        store
+            .delete_quarantined_reconciliation(candidate.clone())
+            .unwrap(),
+        ReconciliationMutation::Retryable
+    );
+    clear_checkpoint();
+    assert_eq!(
+        store.delete_quarantined_reconciliation(candidate).unwrap(),
+        ReconciliationMutation::AlreadyAbsent
+    );
+}
+
+#[test]
+fn rejects_a_replaced_final_inode() {
+    let root = TestRoot::new("replaced");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let stored = store.put(b"reconciliation original").unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    let candidate = candidates.into_iter().next().unwrap();
+    let path = root
+        .0
+        .join("sha256")
+        .join(&stored.artifact_ref.digest_hex()[..2])
+        .join(stored.artifact_ref.digest_hex());
+    fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
+    fs::remove_file(&path).unwrap();
+    fs::write(&path, b"replacement").unwrap();
+    fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap();
+    assert_eq!(
+        store.quarantine_reconciliation(candidate),
+        Err(ArtifactError::UnsafeRoot)
+    );
+    assert!(path.exists());
+}
+
+#[test]
+#[cfg(debug_assertions)]
+fn retries_after_post_rename_fsync_ambiguity() {
+    let _guard = fault_guard();
+    let root = TestRoot::new("fsync");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    store.put(b"reconciliation fault").unwrap();
+    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+    let candidate = candidates.into_iter().next().unwrap();
+    set_checkpoint("reconciliation_quarantine_source_fsync", FaultAction::Fail);
+    assert_eq!(
+        store.quarantine_reconciliation(candidate.clone()).unwrap(),
+        ReconciliationMutation::Retryable
+    );
+    clear_checkpoint();
+    assert_eq!(
+        store.quarantine_reconciliation(candidate).unwrap(),
+        ReconciliationMutation::AlreadyQuarantined
+    );
+}
+
+#[test]
+#[cfg(debug_assertions)]
+fn storage_fault_during_stat_propagates() {
+    let _guard = fault_guard();
+    let root = TestRoot::new("stat-storage");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    store.put(b"reconciliation stat storage").unwrap();
+    set_checkpoint("reconciliation_stat", FaultAction::Fail);
+    assert!(matches!(
+        store.scan_reconciliation(None, 512, 8),
+        Err(ArtifactError::Storage)
+    ));
+    clear_checkpoint();
+}
+
+#[test]
+#[cfg(debug_assertions)]
+fn crash_windows_are_recoverable() {
+    if let Some(root) = std::env::var_os("CRANK_RECONCILIATION_CHILD_ROOT") {
+        let action = std::env::var("CRANK_RECONCILIATION_CHILD_ACTION").unwrap();
+        set_checkpoint(
+            std::env::var("CRANK_RECONCILIATION_CHILD_STAGE").unwrap(),
+            FaultAction::Exit,
+        );
+        let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
+        let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+        let candidate = candidates.into_iter().next().unwrap();
+        if action == "quarantine" {
+            let _ = store.quarantine_reconciliation(candidate);
+        } else {
+            let _ = store.delete_quarantined_reconciliation(candidate);
+        }
+        panic!("checkpoint did not terminate the child");
+    }
+
+    for (stage, action) in [
+        ("reconciliation_quarantine_rename", "quarantine"),
+        ("reconciliation_quarantine_source_fsync", "quarantine"),
+        ("reconciliation_quarantine_destination_fsync", "quarantine"),
+        ("reconciliation_delete_unlink", "delete"),
+        ("reconciliation_delete_fsync", "delete"),
+    ] {
+        let root = TestRoot::new(&format!("crash-{stage}"));
+        let store = ArtifactStore::open(&root.0).unwrap();
+        let stored = store.put(b"reconciliation crash bytes").unwrap();
+        if action == "delete" {
+            let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+            store
+                .quarantine_reconciliation(candidates.into_iter().next().unwrap())
+                .unwrap();
+        }
+        let status = Command::new(std::env::current_exe().unwrap())
+            .args(["--exact", "crash_windows_are_recoverable", "--nocapture"])
+            .env("CRANK_RECONCILIATION_CHILD_ROOT", &root.0)
+            .env("CRANK_RECONCILIATION_CHILD_ACTION", action)
+            .env("CRANK_RECONCILIATION_CHILD_STAGE", stage)
+            .status()
+            .unwrap();
+        assert_eq!(status.code(), Some(86), "stage={stage}");
+
+        let store = ArtifactStore::open(&root.0).unwrap();
+        let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
+        for candidate in candidates {
+            match candidate.namespace() {
+                ReconciliationNamespace::Final => {
+                    let _ = store.quarantine_reconciliation(candidate).unwrap();
+                }
+                ReconciliationNamespace::Quarantine => {
+                    let _ = store.delete_quarantined_reconciliation(candidate).unwrap();
+                }
+            }
+        }
+        let (_, remaining) = store.scan_reconciliation(None, 512, 8).unwrap();
+        assert!(remaining.is_empty(), "stage={stage}");
+        assert_eq!(
+            store
+                .put(b"reconciliation crash bytes")
+                .unwrap()
+                .artifact_ref,
+            stored.artifact_ref
+        );
+    }
+}