feat(artifacts): add fenced reconciliation recovery
This commit is contained in:
1 parent
8784964fb2
commit
d2849ea3fe
21 files changed
+3322
-97
No files matched your search
@@ -0,0 +1,690 @@
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
fs,
|
||||
os::unix::fs::{PermissionsExt, symlink},
|
||||
path::{Path, PathBuf},
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_artifacts::{ReconciliationRegistration, RegisteredArtifact};
|
||||
use crank_registry::MigrationAuthority;
|
||||
|
||||
use super::*;
|
||||
|
||||
static NEXT_TEST_ROOT: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
struct TestRoot(PathBuf);
|
||||
|
||||
impl TestRoot {
|
||||
fn new() -> Self {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"crank-admin-reconciliation-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
fs::create_dir(&path).unwrap();
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
Self(path)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestRoot {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact_path(root: &Path, artifact: &RegisteredArtifact) -> PathBuf {
|
||||
let digest = artifact.artifact_ref().digest_hex();
|
||||
root.join("sha256").join(&digest[..2]).join(digest)
|
||||
}
|
||||
|
||||
fn age_artifact(root: &Path, artifact: &RegisteredArtifact) {
|
||||
fs::File::open(artifact_path(root, artifact))
|
||||
.unwrap()
|
||||
.set_modified(SystemTime::now() - RECONCILIATION_GRACE - Duration::from_secs(60))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn remove_registered_artifact(store: &ArtifactStore, artifact: &RegisteredArtifact) {
|
||||
let (_, candidates) = store
|
||||
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, 4_096, 32)
|
||||
.unwrap();
|
||||
let candidate = candidates
|
||||
.into_iter()
|
||||
.find(|candidate| {
|
||||
matches!(
|
||||
store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()),
|
||||
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
|
||||
)
|
||||
})
|
||||
.expect("the final artifact is discoverable");
|
||||
assert!(matches!(
|
||||
store.quarantine_reconciliation(candidate),
|
||||
Ok(ReconciliationMutation::Quarantined | ReconciliationMutation::AlreadyQuarantined)
|
||||
));
|
||||
|
||||
let (_, candidates) = store
|
||||
.scan_reconciliation_namespace(ReconciliationNamespace::Quarantine, None, 4_096, 32)
|
||||
.unwrap();
|
||||
let candidate = candidates
|
||||
.into_iter()
|
||||
.find(|candidate| {
|
||||
matches!(
|
||||
store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()),
|
||||
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
|
||||
)
|
||||
})
|
||||
.expect("the quarantined artifact is discoverable");
|
||||
assert!(matches!(
|
||||
store.delete_quarantined_reconciliation(candidate),
|
||||
Ok(ReconciliationMutation::Deleted | ReconciliationMutation::AlreadyAbsent)
|
||||
));
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct StepTemplate {
|
||||
scan_complete: bool,
|
||||
traversal_syscalls: usize,
|
||||
candidates: usize,
|
||||
mutation_attempts: usize,
|
||||
classifications: ReconciliationClassificationCounters,
|
||||
physical_mutation_possible: bool,
|
||||
retryable: bool,
|
||||
return_cursor: bool,
|
||||
error: bool,
|
||||
}
|
||||
|
||||
impl StepTemplate {
|
||||
fn complete() -> Self {
|
||||
Self {
|
||||
scan_complete: true,
|
||||
traversal_syscalls: 1,
|
||||
candidates: 0,
|
||||
mutation_attempts: 0,
|
||||
classifications: ReconciliationClassificationCounters::default(),
|
||||
physical_mutation_possible: false,
|
||||
retryable: false,
|
||||
return_cursor: false,
|
||||
error: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CursorEvidence {
|
||||
alive: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl CursorEvidence {
|
||||
fn new(alive: Arc<AtomicUsize>, maximum: &AtomicUsize) -> Self {
|
||||
let current = alive.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
maximum.fetch_max(current, Ordering::SeqCst);
|
||||
Self { alive }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CursorEvidence {
|
||||
fn drop(&mut self) {
|
||||
self.alive.fetch_sub(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
struct ScriptedBackend {
|
||||
steps: Mutex<VecDeque<StepTemplate>>,
|
||||
calls: Mutex<Vec<(Phase, bool, StepLimits)>>,
|
||||
alive: Arc<AtomicUsize>,
|
||||
maximum: AtomicUsize,
|
||||
}
|
||||
|
||||
impl ScriptedBackend {
|
||||
fn new(steps: impl IntoIterator<Item = StepTemplate>) -> Self {
|
||||
Self {
|
||||
steps: Mutex::new(steps.into_iter().collect()),
|
||||
calls: Mutex::new(Vec::new()),
|
||||
alive: Arc::new(AtomicUsize::new(0)),
|
||||
maximum: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ReconciliationBackend for Arc<ScriptedBackend> {
|
||||
type Cursor = CursorEvidence;
|
||||
|
||||
async fn step(
|
||||
&self,
|
||||
phase: Phase,
|
||||
cursor: Option<Self::Cursor>,
|
||||
limits: StepLimits,
|
||||
) -> Result<Step<Self::Cursor>, ReconciliationCoordinatorError> {
|
||||
self.calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push((phase, cursor.is_some(), limits));
|
||||
drop(cursor);
|
||||
let template = self.steps.lock().unwrap().pop_front().unwrap();
|
||||
if template.error {
|
||||
return Err(ReconciliationCoordinatorError::Backend);
|
||||
}
|
||||
let cursor = template
|
||||
.return_cursor
|
||||
.then(|| CursorEvidence::new(self.alive.clone(), &self.maximum));
|
||||
Ok(Step {
|
||||
cursor,
|
||||
scan_complete: template.scan_complete,
|
||||
traversal_syscalls: template.traversal_syscalls,
|
||||
candidates: template.candidates,
|
||||
mutation_attempts: template.mutation_attempts,
|
||||
classifications: template.classifications,
|
||||
physical_mutation_possible: template.physical_mutation_possible,
|
||||
retryable: template.retryable,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_always_completes_before_final_sweep() {
|
||||
let backend = Arc::new(ScriptedBackend::new([
|
||||
StepTemplate::complete(),
|
||||
StepTemplate::complete(),
|
||||
]));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
||||
|
||||
let report = coordinator.tick().await.unwrap();
|
||||
|
||||
assert!(report.recovery_completed);
|
||||
assert!(report.cycle_completed);
|
||||
let phases = backend
|
||||
.calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|call| call.0)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(phases, vec![Phase::Recovery, Phase::Sweep]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn possible_mutation_discards_cursor_and_restarts_recovery() {
|
||||
let paged = StepTemplate {
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 2,
|
||||
candidates: 1,
|
||||
mutation_attempts: 0,
|
||||
classifications: ReconciliationClassificationCounters::default(),
|
||||
physical_mutation_possible: false,
|
||||
retryable: true,
|
||||
return_cursor: true,
|
||||
error: false,
|
||||
};
|
||||
let mutation = StepTemplate {
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 1,
|
||||
candidates: 1,
|
||||
mutation_attempts: 1,
|
||||
classifications: ReconciliationClassificationCounters::default(),
|
||||
physical_mutation_possible: true,
|
||||
retryable: false,
|
||||
return_cursor: true,
|
||||
error: false,
|
||||
};
|
||||
let backend = Arc::new(ScriptedBackend::new([
|
||||
StepTemplate::complete(),
|
||||
paged,
|
||||
mutation,
|
||||
StepTemplate::complete(),
|
||||
StepTemplate::complete(),
|
||||
]));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
||||
|
||||
let first = coordinator.tick().await.unwrap();
|
||||
assert!(first.retryable);
|
||||
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
|
||||
|
||||
let second = coordinator.tick().await.unwrap();
|
||||
assert!(second.cycle_completed);
|
||||
let calls = backend.calls.lock().unwrap();
|
||||
assert_eq!(
|
||||
calls
|
||||
.iter()
|
||||
.map(|(phase, had_cursor, _)| (*phase, *had_cursor))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(Phase::Recovery, false),
|
||||
(Phase::Sweep, false),
|
||||
(Phase::Sweep, true),
|
||||
(Phase::Recovery, false),
|
||||
(Phase::Sweep, false),
|
||||
]
|
||||
);
|
||||
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_error_discards_cursor_and_restarts_recovery() {
|
||||
let paged = StepTemplate {
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 1,
|
||||
candidates: 1,
|
||||
mutation_attempts: 0,
|
||||
classifications: ReconciliationClassificationCounters::default(),
|
||||
physical_mutation_possible: false,
|
||||
retryable: true,
|
||||
return_cursor: true,
|
||||
error: false,
|
||||
};
|
||||
let failure = StepTemplate {
|
||||
error: true,
|
||||
..StepTemplate::complete()
|
||||
};
|
||||
let backend = Arc::new(ScriptedBackend::new([
|
||||
StepTemplate::complete(),
|
||||
paged,
|
||||
failure,
|
||||
StepTemplate::complete(),
|
||||
StepTemplate::complete(),
|
||||
]));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
||||
|
||||
assert!(coordinator.tick().await.unwrap().retryable);
|
||||
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
coordinator.tick().await,
|
||||
Err(ReconciliationCoordinatorError::Backend)
|
||||
);
|
||||
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
||||
assert!(coordinator.tick().await.unwrap().cycle_completed);
|
||||
|
||||
let calls = backend.calls.lock().unwrap();
|
||||
assert_eq!(
|
||||
calls
|
||||
.iter()
|
||||
.map(|(phase, had_cursor, _)| (*phase, *had_cursor))
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(Phase::Recovery, false),
|
||||
(Phase::Sweep, false),
|
||||
(Phase::Sweep, true),
|
||||
(Phase::Recovery, false),
|
||||
(Phase::Sweep, false),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_cannot_exceed_tick_limits() {
|
||||
let backend = Arc::new(ScriptedBackend::new([StepTemplate {
|
||||
scan_complete: false,
|
||||
traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT + 1,
|
||||
candidates: 0,
|
||||
mutation_attempts: 0,
|
||||
classifications: ReconciliationClassificationCounters::default(),
|
||||
physical_mutation_possible: false,
|
||||
retryable: false,
|
||||
return_cursor: false,
|
||||
error: false,
|
||||
}]));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend);
|
||||
|
||||
assert_eq!(
|
||||
coordinator.tick().await,
|
||||
Err(ReconciliationCoordinatorError::Bounds)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tick_stops_at_the_exact_candidate_and_mutation_limits() {
|
||||
let backend = Arc::new(ScriptedBackend::new(
|
||||
(0..RECONCILIATION_MUTATION_LIMIT).map(|_| StepTemplate {
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 1,
|
||||
candidates: 1,
|
||||
mutation_attempts: 1,
|
||||
classifications: ReconciliationClassificationCounters::default(),
|
||||
physical_mutation_possible: true,
|
||||
retryable: false,
|
||||
return_cursor: true,
|
||||
error: false,
|
||||
}),
|
||||
));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
||||
|
||||
let report = coordinator.tick().await.unwrap();
|
||||
|
||||
assert_eq!(report.candidates, RECONCILIATION_CANDIDATE_LIMIT);
|
||||
assert_eq!(report.mutation_attempts, RECONCILIATION_MUTATION_LIMIT);
|
||||
assert_eq!(report.traversal_syscalls, RECONCILIATION_MUTATION_LIMIT);
|
||||
assert_eq!(
|
||||
backend.calls.lock().unwrap().len(),
|
||||
RECONCILIATION_MUTATION_LIMIT
|
||||
);
|
||||
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coordinator_retains_at_most_one_cursor_between_ticks() {
|
||||
let backend = Arc::new(ScriptedBackend::new((0..128).map(|_| StepTemplate {
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 1,
|
||||
candidates: 0,
|
||||
mutation_attempts: 0,
|
||||
classifications: ReconciliationClassificationCounters::default(),
|
||||
physical_mutation_possible: false,
|
||||
retryable: true,
|
||||
return_cursor: true,
|
||||
error: false,
|
||||
})));
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
|
||||
|
||||
for _ in 0..128 {
|
||||
coordinator.tick().await.unwrap();
|
||||
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
|
||||
drop(coordinator);
|
||||
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registration_failures_have_a_closed_redacted_classification() {
|
||||
let mut counters = ReconciliationClassificationCounters::default();
|
||||
|
||||
assert!(!counters.record_registration_error(ArtifactError::Integrity));
|
||||
assert!(!counters.record_registration_error(ArtifactError::UnsafeRoot));
|
||||
assert!(counters.record_registration_error(ArtifactError::Storage));
|
||||
assert!(!counters.record_registration_error(ArtifactError::NotFound));
|
||||
assert!(!counters.record_registration_error(ArtifactError::InvalidReference));
|
||||
assert!(!counters.record_registration_error(ArtifactError::EmptySource));
|
||||
assert!(!counters.record_registration_error(ArtifactError::SourceTooLarge));
|
||||
|
||||
assert_eq!(
|
||||
counters,
|
||||
ReconciliationClassificationCounters {
|
||||
scanner_malformed: 0,
|
||||
scanner_unsafe: 0,
|
||||
recovery_present: 0,
|
||||
recovery_safety: 0,
|
||||
recovery_retryable: 0,
|
||||
registration_integrity: 3,
|
||||
registration_safety: 3,
|
||||
registration_retryable: 1,
|
||||
claim_metadata_conflict: 0,
|
||||
mutation_integrity: 0,
|
||||
mutation_safety: 0,
|
||||
mutation_retryable: 0,
|
||||
}
|
||||
);
|
||||
let debug = format!("{counters:?}");
|
||||
assert!(!debug.contains("sha256:"));
|
||||
assert!(!debug.contains('/'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutation_failures_are_classified_and_always_retryable() {
|
||||
let mut counters = ReconciliationClassificationCounters::default();
|
||||
|
||||
assert!(counters.record_mutation_error(ArtifactError::Integrity));
|
||||
assert!(counters.record_mutation_error(ArtifactError::UnsafeRoot));
|
||||
assert!(counters.record_mutation_error(ArtifactError::Storage));
|
||||
assert!(counters.record_mutation_error(ArtifactError::NotFound));
|
||||
assert!(counters.record_mutation_error(ArtifactError::InvalidReference));
|
||||
assert!(counters.record_mutation_error(ArtifactError::EmptySource));
|
||||
assert!(counters.record_mutation_error(ArtifactError::SourceTooLarge));
|
||||
|
||||
assert_eq!(counters.mutation_integrity, 3);
|
||||
assert_eq!(counters.mutation_safety, 3);
|
||||
assert_eq!(counters.mutation_retryable, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_backend_recovers_absence_before_sweep_and_restarts_after_mutation() {
|
||||
let database_url =
|
||||
crank_test_support::postgres_schema_url("admin_reconciliation_backend").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
||||
let root = TestRoot::new();
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
let expired = store.put_registered(b"expired physical absence\n").unwrap();
|
||||
let expired_claim = registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &expired,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: now - time::Duration::minutes(10),
|
||||
lease_expires_at: now - time::Duration::minutes(5),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(expired_claim, ArtifactClaimOutcome::Claimed(_)));
|
||||
remove_registered_artifact(&store, &expired);
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set claim_expires_at = clock_timestamp() - interval '1 second'
|
||||
where digest = $1",
|
||||
)
|
||||
.bind(expired.artifact_ref().digest_hex())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let corrupted = store
|
||||
.put_registered(b"integrity-blocked-candidate\n")
|
||||
.unwrap();
|
||||
let corrupted_path = artifact_path(&root.0, &corrupted);
|
||||
let corrupted_size = fs::metadata(&corrupted_path).unwrap().len() as usize;
|
||||
fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o600)).unwrap();
|
||||
fs::write(&corrupted_path, vec![b'x'; corrupted_size]).unwrap();
|
||||
fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o400)).unwrap();
|
||||
age_artifact(&root.0, &corrupted);
|
||||
|
||||
let later = store.put_registered(b"later-valid-candidate\n").unwrap();
|
||||
age_artifact(&root.0, &later);
|
||||
let metadata_conflict = store
|
||||
.put_registered(b"metadata-conflict-candidate\n")
|
||||
.unwrap();
|
||||
age_artifact(&root.0, &metadata_conflict);
|
||||
sqlx::query(
|
||||
"insert into artifact_blobs
|
||||
(digest, artifact_ref, size_bytes, storage_lifecycle, created_at, updated_at)
|
||||
values ($1, $2, $3, 'available', $4, $4)",
|
||||
)
|
||||
.bind(metadata_conflict.artifact_ref().digest_hex())
|
||||
.bind(metadata_conflict.artifact_ref().as_str())
|
||||
.bind(i64::try_from(metadata_conflict.size_bytes()).unwrap() + 1)
|
||||
.bind(now)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let shard = artifact_path(&root.0, &later)
|
||||
.parent()
|
||||
.unwrap()
|
||||
.to_path_buf();
|
||||
fs::write(shard.join("not-a-digest"), b"malformed").unwrap();
|
||||
let unsafe_name = format!(
|
||||
"{}{}",
|
||||
&later.artifact_ref().digest_hex()[..2],
|
||||
"f".repeat(62)
|
||||
);
|
||||
assert_ne!(unsafe_name, later.artifact_ref().digest_hex());
|
||||
symlink("not-a-digest", shard.join(unsafe_name)).unwrap();
|
||||
|
||||
let backend = PostgresReconciliationBackend::new(registry.clone(), store.clone());
|
||||
let mut coordinator = ReconciliationCoordinator::with_tick_limits(
|
||||
backend,
|
||||
StepLimits {
|
||||
traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT,
|
||||
candidates: RECONCILIATION_CANDIDATE_LIMIT,
|
||||
mutations: 1,
|
||||
},
|
||||
);
|
||||
|
||||
// The single mutation slot is consumed by expired-claim recovery. The old
|
||||
// final candidate proves Sweep did not run ahead of Recovery.
|
||||
let first = coordinator.tick().await.unwrap();
|
||||
assert_eq!(first.mutation_attempts, 1);
|
||||
assert!(!first.recovery_completed);
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(later.artifact_ref()).unwrap(),
|
||||
ReconciliationPresence::Final
|
||||
);
|
||||
let lifecycle: String =
|
||||
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
|
||||
.bind(expired.artifact_ref().digest_hex())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(lifecycle, "unavailable");
|
||||
|
||||
// The real final sweep classifies blocked entries and still reaches the
|
||||
// later valid item. Its physical mutation invalidates the final cursor.
|
||||
let second = coordinator.tick().await.unwrap();
|
||||
assert!(second.recovery_completed);
|
||||
assert_eq!(second.mutation_attempts, 1);
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(later.artifact_ref()).unwrap(),
|
||||
ReconciliationPresence::Quarantine
|
||||
);
|
||||
|
||||
// Model the next scheduled/restarted lease window. Because the coordinator
|
||||
// reset to Recovery and discarded its final cursor, quarantine is handled
|
||||
// before another final sweep.
|
||||
sqlx::query("update artifact_blobs set claim_expires_at = $1 where digest = $2")
|
||||
.bind(OffsetDateTime::now_utc() - time::Duration::seconds(1))
|
||||
.bind(later.artifact_ref().digest_hex())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let third = coordinator.tick().await.unwrap();
|
||||
assert_eq!(third.mutation_attempts, 1);
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(later.artifact_ref()).unwrap(),
|
||||
ReconciliationPresence::Absent
|
||||
);
|
||||
|
||||
let fourth = coordinator.tick().await.unwrap();
|
||||
assert!(fourth.cycle_completed);
|
||||
let classifications = first
|
||||
.classifications
|
||||
.checked_add(second.classifications)
|
||||
.and_then(|value| value.checked_add(third.classifications))
|
||||
.and_then(|value| value.checked_add(fourth.classifications))
|
||||
.unwrap();
|
||||
assert!(classifications.scanner_malformed > 0);
|
||||
assert!(classifications.scanner_unsafe > 0);
|
||||
assert!(classifications.registration_integrity > 0);
|
||||
assert!(classifications.claim_metadata_conflict > 0);
|
||||
assert_eq!(
|
||||
store
|
||||
.reconciliation_presence(corrupted.artifact_ref())
|
||||
.unwrap(),
|
||||
ReconciliationPresence::Unsafe
|
||||
);
|
||||
assert_eq!(classifications.registration_safety, 0);
|
||||
assert_eq!(classifications.registration_retryable, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_claim_cursor_prevents_unsafe_prefix_starvation_across_ticks() {
|
||||
let database_url =
|
||||
crank_test_support::postgres_schema_url("admin_reconciliation_starvation").await;
|
||||
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
||||
let root = TestRoot::new();
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
for index in 0..=RECONCILIATION_CANDIDATE_LIMIT {
|
||||
let artifact = store
|
||||
.put_registered(format!("unsafe expired claim {index}\n").as_bytes())
|
||||
.unwrap();
|
||||
let outcome = registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &artifact,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: now - time::Duration::minutes(20),
|
||||
lease_expires_at: now - time::Duration::minutes(15),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set claim_expires_at = clock_timestamp() - interval '10 minutes'
|
||||
where digest = $1",
|
||||
)
|
||||
.bind(artifact.artifact_ref().digest_hex())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
fs::set_permissions(
|
||||
artifact_path(&root.0, &artifact),
|
||||
fs::Permissions::from_mode(0o600),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let absent = store
|
||||
.put_registered(b"absent after unsafe prefix\n")
|
||||
.unwrap();
|
||||
let outcome = registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &absent,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: now - time::Duration::minutes(15),
|
||||
lease_expires_at: now - time::Duration::minutes(10),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set claim_expires_at = clock_timestamp() - interval '5 minutes'
|
||||
where digest = $1",
|
||||
)
|
||||
.bind(absent.artifact_ref().digest_hex())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
remove_registered_artifact(&store, &absent);
|
||||
|
||||
let backend = PostgresReconciliationBackend::new(registry.clone(), store);
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend);
|
||||
let first = coordinator.tick().await.unwrap();
|
||||
assert_eq!(first.candidates, RECONCILIATION_CANDIDATE_LIMIT);
|
||||
assert_eq!(first.mutation_attempts, 0);
|
||||
assert_eq!(
|
||||
first.classifications.recovery_safety,
|
||||
RECONCILIATION_CANDIDATE_LIMIT
|
||||
);
|
||||
let first_lifecycle: String =
|
||||
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
|
||||
.bind(absent.artifact_ref().digest_hex())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first_lifecycle, "available");
|
||||
|
||||
let second = coordinator.tick().await.unwrap();
|
||||
assert!(second.classifications.recovery_safety > 0);
|
||||
assert!(second.mutation_attempts > 0);
|
||||
let second_lifecycle: String =
|
||||
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
|
||||
.bind(absent.artifact_ref().digest_hex())
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(second_lifecycle, "unavailable");
|
||||
}
|
||||
Reference in new issue
Block a user