695 lines
24 KiB
Rust
695 lines
24 KiB
Rust
use std::{
|
|
collections::VecDeque,
|
|
fs,
|
|
os::unix::fs::{PermissionsExt, symlink},
|
|
path::{Path, PathBuf},
|
|
sync::{
|
|
Arc, Mutex,
|
|
atomic::{AtomicUsize, Ordering},
|
|
},
|
|
};
|
|
|
|
use async_trait::async_trait;
|
|
use crank_artifacts::{ReconciliationRegistration, RegisteredArtifact};
|
|
use crank_registry::MigrationAuthority;
|
|
use sqlx::Row;
|
|
|
|
use super::*;
|
|
|
|
static NEXT_TEST_ROOT: AtomicUsize = AtomicUsize::new(0);
|
|
|
|
struct TestRoot(PathBuf);
|
|
|
|
impl TestRoot {
|
|
fn new() -> Self {
|
|
let path = std::env::temp_dir().join(format!(
|
|
"crank-admin-reconciliation-{}-{}",
|
|
std::process::id(),
|
|
NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed)
|
|
));
|
|
fs::create_dir(&path).unwrap();
|
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
|
|
Self(path)
|
|
}
|
|
}
|
|
|
|
impl Drop for TestRoot {
|
|
fn drop(&mut self) {
|
|
let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
|
|
let _ = fs::remove_dir_all(&self.0);
|
|
}
|
|
}
|
|
|
|
fn artifact_path(root: &Path, artifact: &RegisteredArtifact) -> PathBuf {
|
|
let digest = artifact.artifact_ref().digest_hex();
|
|
root.join("sha256").join(&digest[..2]).join(digest)
|
|
}
|
|
|
|
fn age_artifact(root: &Path, artifact: &RegisteredArtifact) {
|
|
fs::File::open(artifact_path(root, artifact))
|
|
.unwrap()
|
|
.set_modified(SystemTime::now() - RECONCILIATION_GRACE - Duration::from_secs(60))
|
|
.unwrap();
|
|
}
|
|
|
|
fn remove_registered_artifact(store: &ArtifactStore, artifact: &RegisteredArtifact) {
|
|
let (_, candidates) = store
|
|
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, 4_096, 32)
|
|
.unwrap();
|
|
let candidate = candidates
|
|
.into_iter()
|
|
.find(|candidate| {
|
|
matches!(
|
|
store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()),
|
|
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
|
|
)
|
|
})
|
|
.expect("the final artifact is discoverable");
|
|
assert!(matches!(
|
|
store.quarantine_reconciliation(candidate),
|
|
Ok(ReconciliationMutation::Quarantined | ReconciliationMutation::AlreadyQuarantined)
|
|
));
|
|
|
|
let (_, candidates) = store
|
|
.scan_reconciliation_namespace(ReconciliationNamespace::Quarantine, None, 4_096, 32)
|
|
.unwrap();
|
|
let candidate = candidates
|
|
.into_iter()
|
|
.find(|candidate| {
|
|
matches!(
|
|
store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()),
|
|
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
|
|
)
|
|
})
|
|
.expect("the quarantined artifact is discoverable");
|
|
assert!(matches!(
|
|
store.delete_quarantined_reconciliation(candidate),
|
|
Ok(ReconciliationMutation::Deleted | ReconciliationMutation::AlreadyAbsent)
|
|
));
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
struct StepTemplate {
|
|
scan_complete: bool,
|
|
traversal_syscalls: usize,
|
|
candidates: usize,
|
|
mutation_attempts: usize,
|
|
classifications: ReconciliationClassificationCounters,
|
|
physical_mutation_possible: bool,
|
|
retryable: bool,
|
|
return_cursor: bool,
|
|
error: bool,
|
|
}
|
|
|
|
impl StepTemplate {
|
|
fn complete() -> Self {
|
|
Self {
|
|
scan_complete: true,
|
|
traversal_syscalls: 1,
|
|
candidates: 0,
|
|
mutation_attempts: 0,
|
|
classifications: ReconciliationClassificationCounters::default(),
|
|
physical_mutation_possible: false,
|
|
retryable: false,
|
|
return_cursor: false,
|
|
error: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
struct CursorEvidence {
|
|
alive: Arc<AtomicUsize>,
|
|
}
|
|
|
|
impl CursorEvidence {
|
|
fn new(alive: Arc<AtomicUsize>, maximum: &AtomicUsize) -> Self {
|
|
let current = alive.fetch_add(1, Ordering::SeqCst) + 1;
|
|
maximum.fetch_max(current, Ordering::SeqCst);
|
|
Self { alive }
|
|
}
|
|
}
|
|
|
|
impl Drop for CursorEvidence {
|
|
fn drop(&mut self) {
|
|
self.alive.fetch_sub(1, Ordering::SeqCst);
|
|
}
|
|
}
|
|
|
|
struct ScriptedBackend {
|
|
steps: Mutex<VecDeque<StepTemplate>>,
|
|
calls: Mutex<Vec<(Phase, bool, StepLimits)>>,
|
|
alive: Arc<AtomicUsize>,
|
|
maximum: AtomicUsize,
|
|
}
|
|
|
|
impl ScriptedBackend {
|
|
fn new(steps: impl IntoIterator<Item = StepTemplate>) -> Self {
|
|
Self {
|
|
steps: Mutex::new(steps.into_iter().collect()),
|
|
calls: Mutex::new(Vec::new()),
|
|
alive: Arc::new(AtomicUsize::new(0)),
|
|
maximum: AtomicUsize::new(0),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl ReconciliationBackend for Arc<ScriptedBackend> {
|
|
type Cursor = CursorEvidence;
|
|
|
|
async fn step(
|
|
&self,
|
|
phase: Phase,
|
|
cursor: Option<Self::Cursor>,
|
|
limits: StepLimits,
|
|
) -> Result<Step<Self::Cursor>, ReconciliationCoordinatorError> {
|
|
self.calls
|
|
.lock()
|
|
.unwrap()
|
|
.push((phase, cursor.is_some(), limits));
|
|
drop(cursor);
|
|
let template = self.steps.lock().unwrap().pop_front().unwrap();
|
|
if template.error {
|
|
return Err(ReconciliationCoordinatorError::Backend);
|
|
}
|
|
let cursor = template
|
|
.return_cursor
|
|
.then(|| CursorEvidence::new(self.alive.clone(), &self.maximum));
|
|
Ok(Step {
|
|
cursor,
|
|
scan_complete: template.scan_complete,
|
|
traversal_syscalls: template.traversal_syscalls,
|
|
candidates: template.candidates,
|
|
mutation_attempts: template.mutation_attempts,
|
|
classifications: template.classifications,
|
|
physical_mutation_possible: template.physical_mutation_possible,
|
|
retryable: template.retryable,
|
|
})
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn recovery_always_completes_before_final_sweep() {
|
|
let backend = Arc::new(ScriptedBackend::new([
|
|
StepTemplate::complete(),
|
|
StepTemplate::complete(),
|
|
]));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
|
|
|
let report = coordinator.tick().await.unwrap();
|
|
|
|
assert!(report.recovery_completed);
|
|
assert!(report.cycle_completed);
|
|
let phases = backend
|
|
.calls
|
|
.lock()
|
|
.unwrap()
|
|
.iter()
|
|
.map(|call| call.0)
|
|
.collect::<Vec<_>>();
|
|
assert_eq!(phases, vec![Phase::Recovery, Phase::Sweep]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn possible_mutation_discards_cursor_and_restarts_recovery() {
|
|
let paged = StepTemplate {
|
|
scan_complete: false,
|
|
traversal_syscalls: 2,
|
|
candidates: 1,
|
|
mutation_attempts: 0,
|
|
classifications: ReconciliationClassificationCounters::default(),
|
|
physical_mutation_possible: false,
|
|
retryable: true,
|
|
return_cursor: true,
|
|
error: false,
|
|
};
|
|
let mutation = StepTemplate {
|
|
scan_complete: false,
|
|
traversal_syscalls: 1,
|
|
candidates: 1,
|
|
mutation_attempts: 1,
|
|
classifications: ReconciliationClassificationCounters::default(),
|
|
physical_mutation_possible: true,
|
|
retryable: false,
|
|
return_cursor: true,
|
|
error: false,
|
|
};
|
|
let backend = Arc::new(ScriptedBackend::new([
|
|
StepTemplate::complete(),
|
|
paged,
|
|
mutation,
|
|
StepTemplate::complete(),
|
|
StepTemplate::complete(),
|
|
]));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
|
|
|
let first = coordinator.tick().await.unwrap();
|
|
assert!(first.retryable);
|
|
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
|
|
|
|
let second = coordinator.tick().await.unwrap();
|
|
assert!(second.cycle_completed);
|
|
let calls = backend.calls.lock().unwrap();
|
|
assert_eq!(
|
|
calls
|
|
.iter()
|
|
.map(|(phase, had_cursor, _)| (*phase, *had_cursor))
|
|
.collect::<Vec<_>>(),
|
|
vec![
|
|
(Phase::Recovery, false),
|
|
(Phase::Sweep, false),
|
|
(Phase::Sweep, true),
|
|
(Phase::Recovery, false),
|
|
(Phase::Sweep, false),
|
|
]
|
|
);
|
|
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
|
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn backend_error_discards_cursor_and_restarts_recovery() {
|
|
let paged = StepTemplate {
|
|
scan_complete: false,
|
|
traversal_syscalls: 1,
|
|
candidates: 1,
|
|
mutation_attempts: 0,
|
|
classifications: ReconciliationClassificationCounters::default(),
|
|
physical_mutation_possible: false,
|
|
retryable: true,
|
|
return_cursor: true,
|
|
error: false,
|
|
};
|
|
let failure = StepTemplate {
|
|
error: true,
|
|
..StepTemplate::complete()
|
|
};
|
|
let backend = Arc::new(ScriptedBackend::new([
|
|
StepTemplate::complete(),
|
|
paged,
|
|
failure,
|
|
StepTemplate::complete(),
|
|
StepTemplate::complete(),
|
|
]));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
|
|
|
assert!(coordinator.tick().await.unwrap().retryable);
|
|
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
|
|
assert_eq!(
|
|
coordinator.tick().await,
|
|
Err(ReconciliationCoordinatorError::Backend)
|
|
);
|
|
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
|
assert!(coordinator.tick().await.unwrap().cycle_completed);
|
|
|
|
let calls = backend.calls.lock().unwrap();
|
|
assert_eq!(
|
|
calls
|
|
.iter()
|
|
.map(|(phase, had_cursor, _)| (*phase, *had_cursor))
|
|
.collect::<Vec<_>>(),
|
|
vec![
|
|
(Phase::Recovery, false),
|
|
(Phase::Sweep, false),
|
|
(Phase::Sweep, true),
|
|
(Phase::Recovery, false),
|
|
(Phase::Sweep, false),
|
|
]
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn backend_cannot_exceed_tick_limits() {
|
|
let backend = Arc::new(ScriptedBackend::new([StepTemplate {
|
|
scan_complete: false,
|
|
traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT + 1,
|
|
candidates: 0,
|
|
mutation_attempts: 0,
|
|
classifications: ReconciliationClassificationCounters::default(),
|
|
physical_mutation_possible: false,
|
|
retryable: false,
|
|
return_cursor: false,
|
|
error: false,
|
|
}]));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend);
|
|
|
|
assert_eq!(
|
|
coordinator.tick().await,
|
|
Err(ReconciliationCoordinatorError::Bounds)
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn tick_stops_at_the_exact_candidate_and_mutation_limits() {
|
|
let backend = Arc::new(ScriptedBackend::new(
|
|
(0..RECONCILIATION_MUTATION_LIMIT).map(|_| StepTemplate {
|
|
scan_complete: false,
|
|
traversal_syscalls: 1,
|
|
candidates: 1,
|
|
mutation_attempts: 1,
|
|
classifications: ReconciliationClassificationCounters::default(),
|
|
physical_mutation_possible: true,
|
|
retryable: false,
|
|
return_cursor: true,
|
|
error: false,
|
|
}),
|
|
));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
|
|
|
let report = coordinator.tick().await.unwrap();
|
|
|
|
assert_eq!(report.candidates, RECONCILIATION_CANDIDATE_LIMIT);
|
|
assert_eq!(report.mutation_attempts, RECONCILIATION_MUTATION_LIMIT);
|
|
assert_eq!(report.traversal_syscalls, RECONCILIATION_MUTATION_LIMIT);
|
|
assert_eq!(
|
|
backend.calls.lock().unwrap().len(),
|
|
RECONCILIATION_MUTATION_LIMIT
|
|
);
|
|
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
|
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn coordinator_retains_at_most_one_cursor_between_ticks() {
|
|
let backend = Arc::new(ScriptedBackend::new((0..128).map(|_| StepTemplate {
|
|
scan_complete: false,
|
|
traversal_syscalls: 1,
|
|
candidates: 0,
|
|
mutation_attempts: 0,
|
|
classifications: ReconciliationClassificationCounters::default(),
|
|
physical_mutation_possible: false,
|
|
retryable: true,
|
|
return_cursor: true,
|
|
error: false,
|
|
})));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
|
|
|
for _ in 0..128 {
|
|
coordinator.tick().await.unwrap();
|
|
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
|
|
}
|
|
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
|
|
drop(coordinator);
|
|
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
|
}
|
|
|
|
#[test]
|
|
fn registration_failures_have_a_closed_redacted_classification() {
|
|
let mut counters = ReconciliationClassificationCounters::default();
|
|
|
|
assert!(!counters.record_registration_error(ArtifactError::Integrity));
|
|
assert!(!counters.record_registration_error(ArtifactError::UnsafeRoot));
|
|
assert!(counters.record_registration_error(ArtifactError::Storage));
|
|
assert!(!counters.record_registration_error(ArtifactError::NotFound));
|
|
assert!(!counters.record_registration_error(ArtifactError::InvalidReference));
|
|
assert!(!counters.record_registration_error(ArtifactError::EmptySource));
|
|
assert!(!counters.record_registration_error(ArtifactError::SourceTooLarge));
|
|
|
|
assert_eq!(
|
|
counters,
|
|
ReconciliationClassificationCounters {
|
|
scanner_malformed: 0,
|
|
scanner_unsafe: 0,
|
|
recovery_present: 0,
|
|
recovery_safety: 0,
|
|
recovery_retryable: 0,
|
|
registration_integrity: 3,
|
|
registration_safety: 3,
|
|
registration_retryable: 1,
|
|
claim_metadata_conflict: 0,
|
|
mutation_integrity: 0,
|
|
mutation_safety: 0,
|
|
mutation_retryable: 0,
|
|
}
|
|
);
|
|
let debug = format!("{counters:?}");
|
|
assert!(!debug.contains("sha256:"));
|
|
assert!(!debug.contains('/'));
|
|
}
|
|
|
|
#[test]
|
|
fn mutation_failures_are_classified_and_always_retryable() {
|
|
let mut counters = ReconciliationClassificationCounters::default();
|
|
|
|
assert!(counters.record_mutation_error(ArtifactError::Integrity));
|
|
assert!(counters.record_mutation_error(ArtifactError::UnsafeRoot));
|
|
assert!(counters.record_mutation_error(ArtifactError::Storage));
|
|
assert!(counters.record_mutation_error(ArtifactError::NotFound));
|
|
assert!(counters.record_mutation_error(ArtifactError::InvalidReference));
|
|
assert!(counters.record_mutation_error(ArtifactError::EmptySource));
|
|
assert!(counters.record_mutation_error(ArtifactError::SourceTooLarge));
|
|
|
|
assert_eq!(counters.mutation_integrity, 3);
|
|
assert_eq!(counters.mutation_safety, 3);
|
|
assert_eq!(counters.mutation_retryable, 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn production_backend_recovers_absence_before_sweep_and_restarts_after_mutation() {
|
|
let database_url =
|
|
crank_test_support::postgres_schema_url("admin_reconciliation_backend").await;
|
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
|
MigrationAuthority::apply(&pool).await.unwrap();
|
|
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
|
let root = TestRoot::new();
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let now = OffsetDateTime::now_utc();
|
|
|
|
let expired = store.put_registered(b"expired physical absence\n").unwrap();
|
|
let expired_claim = registry
|
|
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
|
artifact: &expired,
|
|
token: ArtifactClaimToken::generate(),
|
|
claimed_at: now - time::Duration::minutes(10),
|
|
lease_expires_at: now - time::Duration::minutes(5),
|
|
detached_grace: time::Duration::hours(24),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(expired_claim, ArtifactClaimOutcome::Claimed(_)));
|
|
remove_registered_artifact(&store, &expired);
|
|
sqlx::query(
|
|
"update artifact_blobs
|
|
set claim_expires_at = clock_timestamp() - interval '1 second'
|
|
where digest = $1",
|
|
)
|
|
.bind(expired.artifact_ref().digest_hex())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
|
|
let corrupted = store
|
|
.put_registered(b"integrity-blocked-candidate\n")
|
|
.unwrap();
|
|
let corrupted_path = artifact_path(&root.0, &corrupted);
|
|
let corrupted_size = fs::metadata(&corrupted_path).unwrap().len() as usize;
|
|
fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o600)).unwrap();
|
|
fs::write(&corrupted_path, vec![b'x'; corrupted_size]).unwrap();
|
|
fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o400)).unwrap();
|
|
age_artifact(&root.0, &corrupted);
|
|
|
|
let later = store.put_registered(b"later-valid-candidate\n").unwrap();
|
|
age_artifact(&root.0, &later);
|
|
let metadata_conflict = store
|
|
.put_registered(b"metadata-conflict-candidate\n")
|
|
.unwrap();
|
|
age_artifact(&root.0, &metadata_conflict);
|
|
sqlx::query(
|
|
"insert into artifact_blobs
|
|
(digest, artifact_ref, size_bytes, storage_lifecycle, created_at, updated_at)
|
|
values ($1, $2, $3, 'available', $4, $4)",
|
|
)
|
|
.bind(metadata_conflict.artifact_ref().digest_hex())
|
|
.bind(metadata_conflict.artifact_ref().as_str())
|
|
.bind(i64::try_from(metadata_conflict.size_bytes()).unwrap() + 1)
|
|
.bind(now)
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
let shard = artifact_path(&root.0, &later)
|
|
.parent()
|
|
.unwrap()
|
|
.to_path_buf();
|
|
fs::write(shard.join("not-a-digest"), b"malformed").unwrap();
|
|
let unsafe_name = format!(
|
|
"{}{}",
|
|
&later.artifact_ref().digest_hex()[..2],
|
|
"f".repeat(62)
|
|
);
|
|
assert_ne!(unsafe_name, later.artifact_ref().digest_hex());
|
|
symlink("not-a-digest", shard.join(unsafe_name)).unwrap();
|
|
|
|
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone()));
|
|
let mut coordinator = ReconciliationCoordinator::with_tick_limits(
|
|
backend,
|
|
StepLimits {
|
|
traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT,
|
|
candidates: RECONCILIATION_CANDIDATE_LIMIT,
|
|
mutations: 1,
|
|
},
|
|
);
|
|
|
|
// The single mutation slot is consumed by expired-claim recovery. The old
|
|
// final candidate proves Sweep did not run ahead of Recovery.
|
|
let first = coordinator.tick().await.unwrap();
|
|
assert_eq!(first.mutation_attempts, 1);
|
|
assert!(!first.recovery_completed);
|
|
assert_eq!(
|
|
store.reconciliation_presence(later.artifact_ref()).unwrap(),
|
|
ReconciliationPresence::Final
|
|
);
|
|
let lifecycle: String =
|
|
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
|
|
.bind(expired.artifact_ref().digest_hex())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(lifecycle, "unavailable");
|
|
|
|
// The real final sweep classifies blocked entries and still reaches the
|
|
// later valid item. Its physical mutation invalidates the final cursor.
|
|
let second = coordinator.tick().await.unwrap();
|
|
assert!(second.recovery_completed);
|
|
assert_eq!(second.mutation_attempts, 1);
|
|
assert_eq!(
|
|
store.reconciliation_presence(later.artifact_ref()).unwrap(),
|
|
ReconciliationPresence::Quarantine
|
|
);
|
|
|
|
// Model the next scheduled/restarted lease window. Because the coordinator
|
|
// reset to Recovery and discarded its final cursor, quarantine is handled
|
|
// before another final sweep.
|
|
sqlx::query("update artifact_blobs set claim_expires_at = $1 where digest = $2")
|
|
.bind(OffsetDateTime::now_utc() - time::Duration::seconds(1))
|
|
.bind(later.artifact_ref().digest_hex())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
let third = coordinator.tick().await.unwrap();
|
|
assert_eq!(third.mutation_attempts, 1);
|
|
assert_eq!(
|
|
store.reconciliation_presence(later.artifact_ref()).unwrap(),
|
|
ReconciliationPresence::Absent
|
|
);
|
|
|
|
let fourth = coordinator.tick().await.unwrap();
|
|
assert!(fourth.cycle_completed);
|
|
let classifications = first
|
|
.classifications
|
|
.checked_add(second.classifications)
|
|
.and_then(|value| value.checked_add(third.classifications))
|
|
.and_then(|value| value.checked_add(fourth.classifications))
|
|
.unwrap();
|
|
assert!(classifications.scanner_malformed > 0);
|
|
assert!(classifications.scanner_unsafe > 0);
|
|
assert!(classifications.registration_integrity > 0);
|
|
assert!(classifications.claim_metadata_conflict > 0);
|
|
assert_eq!(
|
|
store
|
|
.reconciliation_presence(corrupted.artifact_ref())
|
|
.unwrap(),
|
|
ReconciliationPresence::Unsafe
|
|
);
|
|
assert_eq!(classifications.registration_safety, 0);
|
|
assert_eq!(classifications.registration_retryable, 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn expired_claim_cursor_prevents_unsafe_prefix_starvation_across_ticks() {
|
|
let database_url =
|
|
crank_test_support::postgres_schema_url("admin_reconciliation_starvation").await;
|
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
|
MigrationAuthority::apply(&pool).await.unwrap();
|
|
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
|
let root = TestRoot::new();
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let now = OffsetDateTime::now_utc();
|
|
|
|
for index in 0..=RECONCILIATION_CANDIDATE_LIMIT {
|
|
let artifact = store
|
|
.put_registered(format!("unsafe expired claim {index}\n").as_bytes())
|
|
.unwrap();
|
|
let outcome = registry
|
|
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
|
artifact: &artifact,
|
|
token: ArtifactClaimToken::generate(),
|
|
claimed_at: now - time::Duration::minutes(20),
|
|
lease_expires_at: now - time::Duration::minutes(15),
|
|
detached_grace: time::Duration::hours(24),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
|
|
sqlx::query(
|
|
"update artifact_blobs
|
|
set claim_expires_at = clock_timestamp() - interval '10 minutes'
|
|
where digest = $1",
|
|
)
|
|
.bind(artifact.artifact_ref().digest_hex())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
fs::set_permissions(
|
|
artifact_path(&root.0, &artifact),
|
|
fs::Permissions::from_mode(0o600),
|
|
)
|
|
.unwrap();
|
|
}
|
|
|
|
let absent = store
|
|
.put_registered(b"absent after unsafe prefix\n")
|
|
.unwrap();
|
|
let outcome = registry
|
|
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
|
artifact: &absent,
|
|
token: ArtifactClaimToken::generate(),
|
|
claimed_at: now - time::Duration::minutes(15),
|
|
lease_expires_at: now - time::Duration::minutes(10),
|
|
detached_grace: time::Duration::hours(24),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
|
|
sqlx::query(
|
|
"update artifact_blobs
|
|
set claim_expires_at = clock_timestamp() - interval '5 minutes'
|
|
where digest = $1",
|
|
)
|
|
.bind(absent.artifact_ref().digest_hex())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
remove_registered_artifact(&store, &absent);
|
|
|
|
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend);
|
|
let first = coordinator.tick().await.unwrap();
|
|
assert_eq!(first.candidates, RECONCILIATION_CANDIDATE_LIMIT);
|
|
assert_eq!(first.mutation_attempts, 0);
|
|
assert_eq!(
|
|
first.classifications.recovery_safety,
|
|
RECONCILIATION_CANDIDATE_LIMIT
|
|
);
|
|
let first_lifecycle: String =
|
|
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
|
|
.bind(absent.artifact_ref().digest_hex())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(first_lifecycle, "available");
|
|
|
|
let second = coordinator.tick().await.unwrap();
|
|
assert!(second.classifications.recovery_safety > 0);
|
|
assert!(second.mutation_attempts > 0);
|
|
let second_lifecycle: String =
|
|
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
|
|
.bind(absent.artifact_ref().digest_hex())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(second_lifecycle, "unavailable");
|
|
}
|
|
|
|
#[path = "tests/commit_failure.rs"]
|
|
mod commit_failure;
|