crank/crates/crank-artifacts/tests/reconciliation.rs

use std::{
    fs,
    os::unix::fs::PermissionsExt,
    path::PathBuf,
    process::Command,
    sync::{
        Arc,
        atomic::{AtomicU64, Ordering},
    },
    thread,
    time::Duration,
};

use crank_artifacts::{
    ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
};

#[cfg(debug_assertions)]
use crank_artifacts::test_support::{
    FaultAction, clear_checkpoint, set_checkpoint, wait_until_held,
};
#[cfg(debug_assertions)]
use std::sync::{Mutex, OnceLock};

static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);

struct TestRoot(PathBuf);

impl TestRoot {
    fn new(name: &str) -> Self {
        let path = std::env::temp_dir().join(format!(
            "crank-artifacts-reconciliation-{name}-{}-{}",
            std::process::id(),
            NEXT_ROOT.fetch_add(1, Ordering::Relaxed)
        ));
        fs::create_dir(&path).unwrap();
        fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
        Self(path)
    }
}

impl Drop for TestRoot {
    fn drop(&mut self) {
        let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
        let _ = fs::remove_dir_all(&self.0);
    }
}

#[cfg(debug_assertions)]
fn fault_guard() -> std::sync::MutexGuard<'static, ()> {
    static GUARD: OnceLock<Mutex<()>> = OnceLock::new();
    GUARD.get_or_init(|| Mutex::new(())).lock().unwrap()
}

#[test]
fn paginates_final_entries_without_disclosing_locations() {
    let root = TestRoot::new("pages");
    let store = ArtifactStore::open(&root.0).unwrap();
    store.put(b"reconciliation first").unwrap();
    store.put(b"reconciliation second").unwrap();

    let mut continuation = None;
    let mut candidates = Vec::new();
    loop {
        let (report, mut page) = store.scan_reconciliation(continuation, 1, 1).unwrap();
        assert!(report.scanned <= 1);
        assert!(page.len() <= 1);
        candidates.append(&mut page);
        continuation = report.continuation;
        if continuation.is_none() {
            break;
        }
    }
    assert_eq!(candidates.len(), 2);
    assert!(
        candidates
            .iter()
            .all(|candidate| candidate.namespace() == ReconciliationNamespace::Final)
    );
    for candidate in candidates {
        assert_eq!(
            store.quarantine_reconciliation(candidate).unwrap(),
            ReconciliationMutation::Quarantined
        );
    }
    let (_, quarantined_entries) = store.scan_reconciliation(None, 512, 8).unwrap();
    assert_eq!(quarantined_entries.len(), 2);
    assert!(
        quarantined_entries
            .iter()
            .all(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
    );
}

#[test]
fn classifies_malformed_and_unsafe_entries_with_a_usable_page() {
    let root = TestRoot::new("malformed");
    let store = ArtifactStore::open(&root.0).unwrap();
    let stored = store.put(b"reconciliation valid").unwrap();
    let shard = root
        .0
        .join("sha256")
        .join(&stored.artifact_ref.digest_hex()[..2]);
    fs::write(shard.join("not-a-digest"), b"junk").unwrap();
    let unsafe_name = format!(
        "{}{}",
        &stored.artifact_ref.digest_hex()[..2],
        "f".repeat(62)
    );
    std::os::unix::fs::symlink(shard.join("not-a-digest"), shard.join(unsafe_name)).unwrap();
    let directory_name = format!(
        "{}{}",
        &stored.artifact_ref.digest_hex()[..2],
        "e".repeat(62)
    );
    fs::create_dir(shard.join(directory_name)).unwrap();
    let hardlink_name = format!(
        "{}{}",
        &stored.artifact_ref.digest_hex()[..2],
        "d".repeat(62)
    );
    fs::hard_link(shard.join("not-a-digest"), shard.join(hardlink_name)).unwrap();

    let (report, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    assert!(report.malformed >= 1);
    assert!(report.unsafe_entries >= 3);
    assert_eq!(candidates.len(), 1);
    assert!(report.continuation.is_none());
}

#[test]
fn rejects_a_cursor_from_another_store() {
    let first_root = TestRoot::new("foreign-cursor-first");
    let second_root = TestRoot::new("foreign-cursor-second");
    let first = ArtifactStore::open(&first_root.0).unwrap();
    let second = ArtifactStore::open(&second_root.0).unwrap();
    first.put(b"cursor source").unwrap();
    let (report, _) = first.scan_reconciliation(None, 1, 1).unwrap();

    assert!(matches!(
        second.scan_reconciliation(report.continuation, 1, 1),
        Err(ArtifactError::UnsafeRoot)
    ));
}

#[test]
#[cfg(debug_assertions)]
fn disappearing_entry_after_readdir_is_skipped() {
    let _guard = fault_guard();
    let root = TestRoot::new("disappearing");
    let store = Arc::new(ArtifactStore::open(&root.0).unwrap());
    let stored = store.put(b"disappearing reconciliation entry").unwrap();
    let path = root
        .0
        .join("sha256")
        .join(&stored.artifact_ref.digest_hex()[..2])
        .join(stored.artifact_ref.digest_hex());
    set_checkpoint("reconciliation_before_stat", FaultAction::Hold);
    let scan_store = Arc::clone(&store);
    let scan = thread::spawn(move || scan_store.scan_reconciliation(None, 512, 8));
    assert!(wait_until_held(Duration::from_secs(2)));
    fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
    fs::remove_file(&path).unwrap();
    clear_checkpoint();

    let (_, candidates) = scan.join().unwrap().unwrap();
    assert!(candidates.is_empty());
}

#[test]
fn quarantine_and_delete_are_idempotent() {
    let root = TestRoot::new("mutation");
    let store = ArtifactStore::open(&root.0).unwrap();
    let stored = store.put(b"reconciliation mutation").unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    let candidate = candidates.into_iter().next().unwrap();

    assert_eq!(
        store.quarantine_reconciliation(candidate.clone()).unwrap(),
        ReconciliationMutation::Quarantined
    );
    assert_eq!(
        store.read(&stored.artifact_ref),
        Err(ArtifactError::NotFound)
    );
    assert_eq!(
        store.quarantine_reconciliation(candidate).unwrap(),
        ReconciliationMutation::AlreadyQuarantined
    );

    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    let quarantined = candidates
        .into_iter()
        .find(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
        .unwrap();
    assert_eq!(
        store
            .delete_quarantined_reconciliation(quarantined.clone())
            .unwrap(),
        ReconciliationMutation::Deleted
    );
    assert_eq!(
        store
            .delete_quarantined_reconciliation(quarantined)
            .unwrap(),
        ReconciliationMutation::AlreadyAbsent
    );
}

#[test]
fn quarantine_preserves_old_inode_when_the_digest_is_republished() {
    let root = TestRoot::new("no-clobber");
    let store = ArtifactStore::open(&root.0).unwrap();
    let stored = store.put(b"reconciliation no-clobber").unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    let original = candidates.into_iter().next().unwrap();
    assert_eq!(
        store.quarantine_reconciliation(original.clone()).unwrap(),
        ReconciliationMutation::Quarantined
    );
    store.put(b"reconciliation no-clobber").unwrap();
    assert_eq!(
        store.quarantine_reconciliation(original).unwrap(),
        ReconciliationMutation::AlreadyQuarantined
    );
    assert_eq!(
        store.read(&stored.artifact_ref).unwrap(),
        b"reconciliation no-clobber"
    );
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    assert_eq!(candidates.len(), 2);
    assert!(
        candidates
            .iter()
            .any(|candidate| candidate.namespace() == ReconciliationNamespace::Final)
    );
    assert!(
        candidates
            .iter()
            .any(|candidate| candidate.namespace() == ReconciliationNamespace::Quarantine)
    );
}

#[test]
fn delete_fails_closed_for_a_replaced_or_hardlinked_quarantined_inode() {
    let root = TestRoot::new("quarantine-replaced");
    let store = ArtifactStore::open(&root.0).unwrap();
    let stored = store
        .put(b"reconciliation quarantined replacement")
        .unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    store
        .quarantine_reconciliation(candidates.into_iter().next().unwrap())
        .unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    let candidate = candidates.into_iter().next().unwrap();
    let path = root
        .0
        .join("quarantine")
        .join(&stored.artifact_ref.digest_hex()[..2])
        .join(stored.artifact_ref.digest_hex());
    fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
    fs::remove_file(&path).unwrap();
    fs::write(&path, b"replacement").unwrap();
    let sibling = path.with_extension("link");
    fs::hard_link(&path, &sibling).unwrap();
    fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap();

    assert_eq!(
        store.delete_quarantined_reconciliation(candidate),
        Err(ArtifactError::UnsafeRoot)
    );
    assert!(path.exists());
    assert!(sibling.exists());
}

#[test]
#[cfg(debug_assertions)]
fn delete_fsync_ambiguity_is_retryable_and_recovers_as_absent() {
    let _guard = fault_guard();
    let root = TestRoot::new("delete-fsync");
    let store = ArtifactStore::open(&root.0).unwrap();
    store.put(b"reconciliation delete fsync").unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    store
        .quarantine_reconciliation(candidates.into_iter().next().unwrap())
        .unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    let candidate = candidates.into_iter().next().unwrap();
    set_checkpoint("reconciliation_delete_fsync", FaultAction::Fail);
    assert_eq!(
        store
            .delete_quarantined_reconciliation(candidate.clone())
            .unwrap(),
        ReconciliationMutation::Retryable
    );
    clear_checkpoint();
    assert_eq!(
        store.delete_quarantined_reconciliation(candidate).unwrap(),
        ReconciliationMutation::AlreadyAbsent
    );
}

#[test]
fn rejects_a_replaced_final_inode() {
    let root = TestRoot::new("replaced");
    let store = ArtifactStore::open(&root.0).unwrap();
    let stored = store.put(b"reconciliation original").unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    let candidate = candidates.into_iter().next().unwrap();
    let path = root
        .0
        .join("sha256")
        .join(&stored.artifact_ref.digest_hex()[..2])
        .join(stored.artifact_ref.digest_hex());
    fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap();
    fs::remove_file(&path).unwrap();
    fs::write(&path, b"replacement").unwrap();
    fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap();
    assert_eq!(
        store.quarantine_reconciliation(candidate),
        Err(ArtifactError::UnsafeRoot)
    );
    assert!(path.exists());
}

#[test]
#[cfg(debug_assertions)]
fn retries_after_post_rename_fsync_ambiguity() {
    let _guard = fault_guard();
    let root = TestRoot::new("fsync");
    let store = ArtifactStore::open(&root.0).unwrap();
    store.put(b"reconciliation fault").unwrap();
    let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
    let candidate = candidates.into_iter().next().unwrap();
    set_checkpoint("reconciliation_quarantine_source_fsync", FaultAction::Fail);
    assert_eq!(
        store.quarantine_reconciliation(candidate.clone()).unwrap(),
        ReconciliationMutation::Retryable
    );
    clear_checkpoint();
    assert_eq!(
        store.quarantine_reconciliation(candidate).unwrap(),
        ReconciliationMutation::AlreadyQuarantined
    );
}

#[test]
#[cfg(debug_assertions)]
fn storage_fault_during_stat_propagates() {
    let _guard = fault_guard();
    let root = TestRoot::new("stat-storage");
    let store = ArtifactStore::open(&root.0).unwrap();
    store.put(b"reconciliation stat storage").unwrap();
    set_checkpoint("reconciliation_stat", FaultAction::Fail);
    assert!(matches!(
        store.scan_reconciliation(None, 512, 8),
        Err(ArtifactError::Storage)
    ));
    clear_checkpoint();
}

#[test]
#[cfg(debug_assertions)]
fn crash_windows_are_recoverable() {
    if let Some(root) = std::env::var_os("CRANK_RECONCILIATION_CHILD_ROOT") {
        let action = std::env::var("CRANK_RECONCILIATION_CHILD_ACTION").unwrap();
        set_checkpoint(
            std::env::var("CRANK_RECONCILIATION_CHILD_STAGE").unwrap(),
            FaultAction::Exit,
        );
        let store = ArtifactStore::open(PathBuf::from(root)).unwrap();
        let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
        let candidate = candidates.into_iter().next().unwrap();
        if action == "quarantine" {
            let _ = store.quarantine_reconciliation(candidate);
        } else {
            let _ = store.delete_quarantined_reconciliation(candidate);
        }
        panic!("checkpoint did not terminate the child");
    }

    for (stage, action) in [
        ("reconciliation_quarantine_rename", "quarantine"),
        ("reconciliation_quarantine_source_fsync", "quarantine"),
        ("reconciliation_quarantine_destination_fsync", "quarantine"),
        ("reconciliation_delete_unlink", "delete"),
        ("reconciliation_delete_fsync", "delete"),
    ] {
        let root = TestRoot::new(&format!("crash-{stage}"));
        let store = ArtifactStore::open(&root.0).unwrap();
        let stored = store.put(b"reconciliation crash bytes").unwrap();
        if action == "delete" {
            let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
            store
                .quarantine_reconciliation(candidates.into_iter().next().unwrap())
                .unwrap();
        }
        let status = Command::new(std::env::current_exe().unwrap())
            .args(["--exact", "crash_windows_are_recoverable", "--nocapture"])
            .env("CRANK_RECONCILIATION_CHILD_ROOT", &root.0)
            .env("CRANK_RECONCILIATION_CHILD_ACTION", action)
            .env("CRANK_RECONCILIATION_CHILD_STAGE", stage)
            .status()
            .unwrap();
        assert_eq!(status.code(), Some(86), "stage={stage}");

        let store = ArtifactStore::open(&root.0).unwrap();
        let (_, candidates) = store.scan_reconciliation(None, 512, 8).unwrap();
        for candidate in candidates {
            match candidate.namespace() {
                ReconciliationNamespace::Final => {
                    let _ = store.quarantine_reconciliation(candidate).unwrap();
                }
                ReconciliationNamespace::Quarantine => {
                    let _ = store.delete_quarantined_reconciliation(candidate).unwrap();
                }
            }
        }
        let (_, remaining) = store.scan_reconciliation(None, 512, 8).unwrap();
        assert!(remaining.is_empty(), "stage={stage}");
        assert_eq!(
            store
                .put(b"reconciliation crash bytes")
                .unwrap()
                .artifact_ref,
            stored.artifact_ref
        );
    }
}