use std::{ collections::VecDeque, fs, os::unix::fs::{PermissionsExt, symlink}, path::{Path, PathBuf}, sync::{ Arc, Mutex, atomic::{AtomicUsize, Ordering}, }, }; use async_trait::async_trait; use crank_artifacts::{ReconciliationRegistration, RegisteredArtifact}; use crank_registry::MigrationAuthority; use sqlx::Row; use super::*; static NEXT_TEST_ROOT: AtomicUsize = AtomicUsize::new(0); struct TestRoot(PathBuf); impl TestRoot { fn new() -> Self { let path = std::env::temp_dir().join(format!( "crank-admin-reconciliation-{}-{}", std::process::id(), NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed) )); fs::create_dir(&path).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); Self(path) } } impl Drop for TestRoot { fn drop(&mut self) { let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700)); let _ = fs::remove_dir_all(&self.0); } } fn artifact_path(root: &Path, artifact: &RegisteredArtifact) -> PathBuf { let digest = artifact.artifact_ref().digest_hex(); root.join("sha256").join(&digest[..2]).join(digest) } fn age_artifact(root: &Path, artifact: &RegisteredArtifact) { fs::File::open(artifact_path(root, artifact)) .unwrap() .set_modified(SystemTime::now() - RECONCILIATION_GRACE - Duration::from_secs(60)) .unwrap(); } fn remove_registered_artifact(store: &ArtifactStore, artifact: &RegisteredArtifact) { let (_, candidates) = store .scan_reconciliation_namespace(ReconciliationNamespace::Final, None, 4_096, 32) .unwrap(); let candidate = candidates .into_iter() .find(|candidate| { matches!( store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()), Ok(ReconciliationRegistration::Registered(found)) if found == *artifact ) }) .expect("the final artifact is discoverable"); assert!(matches!( store.quarantine_reconciliation(candidate), Ok(ReconciliationMutation::Quarantined | ReconciliationMutation::AlreadyQuarantined) )); let (_, candidates) = store .scan_reconciliation_namespace(ReconciliationNamespace::Quarantine, None, 4_096, 32) .unwrap(); let candidate = candidates .into_iter() .find(|candidate| { matches!( store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()), Ok(ReconciliationRegistration::Registered(found)) if found == *artifact ) }) .expect("the quarantined artifact is discoverable"); assert!(matches!( store.delete_quarantined_reconciliation(candidate), Ok(ReconciliationMutation::Deleted | ReconciliationMutation::AlreadyAbsent) )); } #[derive(Clone, Copy)] struct StepTemplate { scan_complete: bool, traversal_syscalls: usize, candidates: usize, mutation_attempts: usize, classifications: ReconciliationClassificationCounters, physical_mutation_possible: bool, retryable: bool, return_cursor: bool, error: bool, } impl StepTemplate { fn complete() -> Self { Self { scan_complete: true, traversal_syscalls: 1, candidates: 0, mutation_attempts: 0, classifications: ReconciliationClassificationCounters::default(), physical_mutation_possible: false, retryable: false, return_cursor: false, error: false, } } } struct CursorEvidence { alive: Arc, } impl CursorEvidence { fn new(alive: Arc, maximum: &AtomicUsize) -> Self { let current = alive.fetch_add(1, Ordering::SeqCst) + 1; maximum.fetch_max(current, Ordering::SeqCst); Self { alive } } } impl Drop for CursorEvidence { fn drop(&mut self) { self.alive.fetch_sub(1, Ordering::SeqCst); } } struct ScriptedBackend { steps: Mutex>, calls: Mutex>, alive: Arc, maximum: AtomicUsize, } impl ScriptedBackend { fn new(steps: impl IntoIterator) -> Self { Self { steps: Mutex::new(steps.into_iter().collect()), calls: Mutex::new(Vec::new()), alive: Arc::new(AtomicUsize::new(0)), maximum: AtomicUsize::new(0), } } } #[async_trait] impl ReconciliationBackend for Arc { type Cursor = CursorEvidence; async fn step( &self, phase: Phase, cursor: Option, limits: StepLimits, ) -> Result, ReconciliationCoordinatorError> { self.calls .lock() .unwrap() .push((phase, cursor.is_some(), limits)); drop(cursor); let template = self.steps.lock().unwrap().pop_front().unwrap(); if template.error { return Err(ReconciliationCoordinatorError::Backend); } let cursor = template .return_cursor .then(|| CursorEvidence::new(self.alive.clone(), &self.maximum)); Ok(Step { cursor, scan_complete: template.scan_complete, traversal_syscalls: template.traversal_syscalls, candidates: template.candidates, mutation_attempts: template.mutation_attempts, classifications: template.classifications, physical_mutation_possible: template.physical_mutation_possible, retryable: template.retryable, }) } } #[tokio::test] async fn recovery_always_completes_before_final_sweep() { let backend = Arc::new(ScriptedBackend::new([ StepTemplate::complete(), StepTemplate::complete(), ])); let mut coordinator = ReconciliationCoordinator::new(backend.clone()); let report = coordinator.tick().await.unwrap(); assert!(report.recovery_completed); assert!(report.cycle_completed); let phases = backend .calls .lock() .unwrap() .iter() .map(|call| call.0) .collect::>(); assert_eq!(phases, vec![Phase::Recovery, Phase::Sweep]); } #[tokio::test] async fn possible_mutation_discards_cursor_and_restarts_recovery() { let paged = StepTemplate { scan_complete: false, traversal_syscalls: 2, candidates: 1, mutation_attempts: 0, classifications: ReconciliationClassificationCounters::default(), physical_mutation_possible: false, retryable: true, return_cursor: true, error: false, }; let mutation = StepTemplate { scan_complete: false, traversal_syscalls: 1, candidates: 1, mutation_attempts: 1, classifications: ReconciliationClassificationCounters::default(), physical_mutation_possible: true, retryable: false, return_cursor: true, error: false, }; let backend = Arc::new(ScriptedBackend::new([ StepTemplate::complete(), paged, mutation, StepTemplate::complete(), StepTemplate::complete(), ])); let mut coordinator = ReconciliationCoordinator::new(backend.clone()); let first = coordinator.tick().await.unwrap(); assert!(first.retryable); assert_eq!(backend.alive.load(Ordering::SeqCst), 1); let second = coordinator.tick().await.unwrap(); assert!(second.cycle_completed); let calls = backend.calls.lock().unwrap(); assert_eq!( calls .iter() .map(|(phase, had_cursor, _)| (*phase, *had_cursor)) .collect::>(), vec![ (Phase::Recovery, false), (Phase::Sweep, false), (Phase::Sweep, true), (Phase::Recovery, false), (Phase::Sweep, false), ] ); assert_eq!(backend.alive.load(Ordering::SeqCst), 0); assert_eq!(backend.maximum.load(Ordering::SeqCst), 1); } #[tokio::test] async fn backend_error_discards_cursor_and_restarts_recovery() { let paged = StepTemplate { scan_complete: false, traversal_syscalls: 1, candidates: 1, mutation_attempts: 0, classifications: ReconciliationClassificationCounters::default(), physical_mutation_possible: false, retryable: true, return_cursor: true, error: false, }; let failure = StepTemplate { error: true, ..StepTemplate::complete() }; let backend = Arc::new(ScriptedBackend::new([ StepTemplate::complete(), paged, failure, StepTemplate::complete(), StepTemplate::complete(), ])); let mut coordinator = ReconciliationCoordinator::new(backend.clone()); assert!(coordinator.tick().await.unwrap().retryable); assert_eq!(backend.alive.load(Ordering::SeqCst), 1); assert_eq!( coordinator.tick().await, Err(ReconciliationCoordinatorError::Backend) ); assert_eq!(backend.alive.load(Ordering::SeqCst), 0); assert!(coordinator.tick().await.unwrap().cycle_completed); let calls = backend.calls.lock().unwrap(); assert_eq!( calls .iter() .map(|(phase, had_cursor, _)| (*phase, *had_cursor)) .collect::>(), vec![ (Phase::Recovery, false), (Phase::Sweep, false), (Phase::Sweep, true), (Phase::Recovery, false), (Phase::Sweep, false), ] ); } #[tokio::test] async fn backend_cannot_exceed_tick_limits() { let backend = Arc::new(ScriptedBackend::new([StepTemplate { scan_complete: false, traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT + 1, candidates: 0, mutation_attempts: 0, classifications: ReconciliationClassificationCounters::default(), physical_mutation_possible: false, retryable: false, return_cursor: false, error: false, }])); let mut coordinator = ReconciliationCoordinator::new(backend); assert_eq!( coordinator.tick().await, Err(ReconciliationCoordinatorError::Bounds) ); } #[tokio::test] async fn tick_stops_at_the_exact_candidate_and_mutation_limits() { let backend = Arc::new(ScriptedBackend::new( (0..RECONCILIATION_MUTATION_LIMIT).map(|_| StepTemplate { scan_complete: false, traversal_syscalls: 1, candidates: 1, mutation_attempts: 1, classifications: ReconciliationClassificationCounters::default(), physical_mutation_possible: true, retryable: false, return_cursor: true, error: false, }), )); let mut coordinator = ReconciliationCoordinator::new(backend.clone()); let report = coordinator.tick().await.unwrap(); assert_eq!(report.candidates, RECONCILIATION_CANDIDATE_LIMIT); assert_eq!(report.mutation_attempts, RECONCILIATION_MUTATION_LIMIT); assert_eq!(report.traversal_syscalls, RECONCILIATION_MUTATION_LIMIT); assert_eq!( backend.calls.lock().unwrap().len(), RECONCILIATION_MUTATION_LIMIT ); assert_eq!(backend.alive.load(Ordering::SeqCst), 0); assert_eq!(backend.maximum.load(Ordering::SeqCst), 1); } #[tokio::test] async fn coordinator_retains_at_most_one_cursor_between_ticks() { let backend = Arc::new(ScriptedBackend::new((0..128).map(|_| StepTemplate { scan_complete: false, traversal_syscalls: 1, candidates: 0, mutation_attempts: 0, classifications: ReconciliationClassificationCounters::default(), physical_mutation_possible: false, retryable: true, return_cursor: true, error: false, }))); let mut coordinator = ReconciliationCoordinator::new(backend.clone()); for _ in 0..128 { coordinator.tick().await.unwrap(); assert_eq!(backend.alive.load(Ordering::SeqCst), 1); } assert_eq!(backend.maximum.load(Ordering::SeqCst), 1); drop(coordinator); assert_eq!(backend.alive.load(Ordering::SeqCst), 0); } #[test] fn registration_failures_have_a_closed_redacted_classification() { let mut counters = ReconciliationClassificationCounters::default(); assert!(!counters.record_registration_error(ArtifactError::Integrity)); assert!(!counters.record_registration_error(ArtifactError::UnsafeRoot)); assert!(counters.record_registration_error(ArtifactError::Storage)); assert!(!counters.record_registration_error(ArtifactError::NotFound)); assert!(!counters.record_registration_error(ArtifactError::InvalidReference)); assert!(!counters.record_registration_error(ArtifactError::EmptySource)); assert!(!counters.record_registration_error(ArtifactError::SourceTooLarge)); assert_eq!( counters, ReconciliationClassificationCounters { scanner_malformed: 0, scanner_unsafe: 0, recovery_present: 0, recovery_safety: 0, recovery_retryable: 0, registration_integrity: 3, registration_safety: 3, registration_retryable: 1, claim_metadata_conflict: 0, mutation_integrity: 0, mutation_safety: 0, mutation_retryable: 0, } ); let debug = format!("{counters:?}"); assert!(!debug.contains("sha256:")); assert!(!debug.contains('/')); } #[test] fn mutation_failures_are_classified_and_always_retryable() { let mut counters = ReconciliationClassificationCounters::default(); assert!(counters.record_mutation_error(ArtifactError::Integrity)); assert!(counters.record_mutation_error(ArtifactError::UnsafeRoot)); assert!(counters.record_mutation_error(ArtifactError::Storage)); assert!(counters.record_mutation_error(ArtifactError::NotFound)); assert!(counters.record_mutation_error(ArtifactError::InvalidReference)); assert!(counters.record_mutation_error(ArtifactError::EmptySource)); assert!(counters.record_mutation_error(ArtifactError::SourceTooLarge)); assert_eq!(counters.mutation_integrity, 3); assert_eq!(counters.mutation_safety, 3); assert_eq!(counters.mutation_retryable, 1); } #[tokio::test] async fn production_backend_recovers_absence_before_sweep_and_restarts_after_mutation() { let database_url = crank_test_support::postgres_schema_url("admin_reconciliation_backend").await; let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); MigrationAuthority::apply(&pool).await.unwrap(); let registry = PostgresRegistry::connect(&database_url).await.unwrap(); let root = TestRoot::new(); let store = ArtifactStore::open(&root.0).unwrap(); let now = OffsetDateTime::now_utc(); let expired = store.put_registered(b"expired physical absence\n").unwrap(); let expired_claim = registry .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest { artifact: &expired, token: ArtifactClaimToken::generate(), claimed_at: now - time::Duration::minutes(10), lease_expires_at: now - time::Duration::minutes(5), detached_grace: time::Duration::hours(24), }) .await .unwrap(); assert!(matches!(expired_claim, ArtifactClaimOutcome::Claimed(_))); remove_registered_artifact(&store, &expired); sqlx::query( "update artifact_blobs set claim_expires_at = clock_timestamp() - interval '1 second' where digest = $1", ) .bind(expired.artifact_ref().digest_hex()) .execute(registry.pool()) .await .unwrap(); let corrupted = store .put_registered(b"integrity-blocked-candidate\n") .unwrap(); let corrupted_path = artifact_path(&root.0, &corrupted); let corrupted_size = fs::metadata(&corrupted_path).unwrap().len() as usize; fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o600)).unwrap(); fs::write(&corrupted_path, vec![b'x'; corrupted_size]).unwrap(); fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o400)).unwrap(); age_artifact(&root.0, &corrupted); let later = store.put_registered(b"later-valid-candidate\n").unwrap(); age_artifact(&root.0, &later); let metadata_conflict = store .put_registered(b"metadata-conflict-candidate\n") .unwrap(); age_artifact(&root.0, &metadata_conflict); sqlx::query( "insert into artifact_blobs (digest, artifact_ref, size_bytes, storage_lifecycle, created_at, updated_at) values ($1, $2, $3, 'available', $4, $4)", ) .bind(metadata_conflict.artifact_ref().digest_hex()) .bind(metadata_conflict.artifact_ref().as_str()) .bind(i64::try_from(metadata_conflict.size_bytes()).unwrap() + 1) .bind(now) .execute(registry.pool()) .await .unwrap(); let shard = artifact_path(&root.0, &later) .parent() .unwrap() .to_path_buf(); fs::write(shard.join("not-a-digest"), b"malformed").unwrap(); let unsafe_name = format!( "{}{}", &later.artifact_ref().digest_hex()[..2], "f".repeat(62) ); assert_ne!(unsafe_name, later.artifact_ref().digest_hex()); symlink("not-a-digest", shard.join(unsafe_name)).unwrap(); let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone())); let mut coordinator = ReconciliationCoordinator::with_tick_limits( backend, StepLimits { traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT, candidates: RECONCILIATION_CANDIDATE_LIMIT, mutations: 1, }, ); // The single mutation slot is consumed by expired-claim recovery. The old // final candidate proves Sweep did not run ahead of Recovery. let first = coordinator.tick().await.unwrap(); assert_eq!(first.mutation_attempts, 1); assert!(!first.recovery_completed); assert_eq!( store.reconciliation_presence(later.artifact_ref()).unwrap(), ReconciliationPresence::Final ); let lifecycle: String = sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1") .bind(expired.artifact_ref().digest_hex()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!(lifecycle, "unavailable"); // The real final sweep classifies blocked entries and still reaches the // later valid item. Its physical mutation invalidates the final cursor. let second = coordinator.tick().await.unwrap(); assert!(second.recovery_completed); assert_eq!(second.mutation_attempts, 1); assert_eq!( store.reconciliation_presence(later.artifact_ref()).unwrap(), ReconciliationPresence::Quarantine ); // Model the next scheduled/restarted lease window. Because the coordinator // reset to Recovery and discarded its final cursor, quarantine is handled // before another final sweep. sqlx::query("update artifact_blobs set claim_expires_at = $1 where digest = $2") .bind(OffsetDateTime::now_utc() - time::Duration::seconds(1)) .bind(later.artifact_ref().digest_hex()) .execute(registry.pool()) .await .unwrap(); let third = coordinator.tick().await.unwrap(); assert_eq!(third.mutation_attempts, 1); assert_eq!( store.reconciliation_presence(later.artifact_ref()).unwrap(), ReconciliationPresence::Absent ); let fourth = coordinator.tick().await.unwrap(); assert!(fourth.cycle_completed); let classifications = first .classifications .checked_add(second.classifications) .and_then(|value| value.checked_add(third.classifications)) .and_then(|value| value.checked_add(fourth.classifications)) .unwrap(); assert!(classifications.scanner_malformed > 0); assert!(classifications.scanner_unsafe > 0); assert!(classifications.registration_integrity > 0); assert!(classifications.claim_metadata_conflict > 0); assert_eq!( store .reconciliation_presence(corrupted.artifact_ref()) .unwrap(), ReconciliationPresence::Unsafe ); assert_eq!(classifications.registration_safety, 0); assert_eq!(classifications.registration_retryable, 0); } #[tokio::test] async fn expired_claim_cursor_prevents_unsafe_prefix_starvation_across_ticks() { let database_url = crank_test_support::postgres_schema_url("admin_reconciliation_starvation").await; let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); MigrationAuthority::apply(&pool).await.unwrap(); let registry = PostgresRegistry::connect(&database_url).await.unwrap(); let root = TestRoot::new(); let store = ArtifactStore::open(&root.0).unwrap(); let now = OffsetDateTime::now_utc(); for index in 0..=RECONCILIATION_CANDIDATE_LIMIT { let artifact = store .put_registered(format!("unsafe expired claim {index}\n").as_bytes()) .unwrap(); let outcome = registry .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest { artifact: &artifact, token: ArtifactClaimToken::generate(), claimed_at: now - time::Duration::minutes(20), lease_expires_at: now - time::Duration::minutes(15), detached_grace: time::Duration::hours(24), }) .await .unwrap(); assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_))); sqlx::query( "update artifact_blobs set claim_expires_at = clock_timestamp() - interval '10 minutes' where digest = $1", ) .bind(artifact.artifact_ref().digest_hex()) .execute(registry.pool()) .await .unwrap(); fs::set_permissions( artifact_path(&root.0, &artifact), fs::Permissions::from_mode(0o600), ) .unwrap(); } let absent = store .put_registered(b"absent after unsafe prefix\n") .unwrap(); let outcome = registry .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest { artifact: &absent, token: ArtifactClaimToken::generate(), claimed_at: now - time::Duration::minutes(15), lease_expires_at: now - time::Duration::minutes(10), detached_grace: time::Duration::hours(24), }) .await .unwrap(); assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_))); sqlx::query( "update artifact_blobs set claim_expires_at = clock_timestamp() - interval '5 minutes' where digest = $1", ) .bind(absent.artifact_ref().digest_hex()) .execute(registry.pool()) .await .unwrap(); remove_registered_artifact(&store, &absent); let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store)); let mut coordinator = ReconciliationCoordinator::new(backend); let first = coordinator.tick().await.unwrap(); assert_eq!(first.candidates, RECONCILIATION_CANDIDATE_LIMIT); assert_eq!(first.mutation_attempts, 0); assert_eq!( first.classifications.recovery_safety, RECONCILIATION_CANDIDATE_LIMIT ); let first_lifecycle: String = sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1") .bind(absent.artifact_ref().digest_hex()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!(first_lifecycle, "available"); let second = coordinator.tick().await.unwrap(); assert!(second.classifications.recovery_safety > 0); assert!(second.mutation_attempts > 0); let second_lifecycle: String = sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1") .bind(absent.artifact_ref().digest_hex()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!(second_lifecycle, "unavailable"); } #[path = "tests/commit_failure.rs"] mod commit_failure;