Files
crank/apps/admin-api/src/reconciliation/tests.rs
T
bsodfather ad23cd9a46
CI / Rust Checks (pull_request) Successful in 24m43s
CI / UI Checks (pull_request) Successful in 6s
CI / Community Image Smoke (pull_request) Successful in 9m5s
CI / Frontend E2E (pull_request) Successful in 12m43s
CI / Deploy (pull_request) Skipped
fix(import): подтвердить обязательные гарантии второго эпика
2026-10-10 13:25:05 +03:00

695 lines
24 KiB
Rust

use std::{
collections::VecDeque,
fs,
os::unix::fs::{PermissionsExt, symlink},
path::{Path, PathBuf},
sync::{
Arc, Mutex,
atomic::{AtomicUsize, Ordering},
},
};
use async_trait::async_trait;
use crank_artifacts::{ReconciliationRegistration, RegisteredArtifact};
use crank_registry::MigrationAuthority;
use sqlx::Row;
use super::*;
static NEXT_TEST_ROOT: AtomicUsize = AtomicUsize::new(0);
struct TestRoot(PathBuf);
impl TestRoot {
fn new() -> Self {
let path = std::env::temp_dir().join(format!(
"crank-admin-reconciliation-{}-{}",
std::process::id(),
NEXT_TEST_ROOT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&path).unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
Self(path)
}
}
impl Drop for TestRoot {
fn drop(&mut self) {
let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700));
let _ = fs::remove_dir_all(&self.0);
}
}
fn artifact_path(root: &Path, artifact: &RegisteredArtifact) -> PathBuf {
let digest = artifact.artifact_ref().digest_hex();
root.join("sha256").join(&digest[..2]).join(digest)
}
fn age_artifact(root: &Path, artifact: &RegisteredArtifact) {
fs::File::open(artifact_path(root, artifact))
.unwrap()
.set_modified(SystemTime::now() - RECONCILIATION_GRACE - Duration::from_secs(60))
.unwrap();
}
fn remove_registered_artifact(store: &ArtifactStore, artifact: &RegisteredArtifact) {
let (_, candidates) = store
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, 4_096, 32)
.unwrap();
let candidate = candidates
.into_iter()
.find(|candidate| {
matches!(
store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()),
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
)
})
.expect("the final artifact is discoverable");
assert!(matches!(
store.quarantine_reconciliation(candidate),
Ok(ReconciliationMutation::Quarantined | ReconciliationMutation::AlreadyQuarantined)
));
let (_, candidates) = store
.scan_reconciliation_namespace(ReconciliationNamespace::Quarantine, None, 4_096, 32)
.unwrap();
let candidate = candidates
.into_iter()
.find(|candidate| {
matches!(
store.register_reconciliation(candidate, Duration::ZERO, SystemTime::now()),
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
)
})
.expect("the quarantined artifact is discoverable");
assert!(matches!(
store.delete_quarantined_reconciliation(candidate),
Ok(ReconciliationMutation::Deleted | ReconciliationMutation::AlreadyAbsent)
));
}
#[derive(Clone, Copy)]
struct StepTemplate {
scan_complete: bool,
traversal_syscalls: usize,
candidates: usize,
mutation_attempts: usize,
classifications: ReconciliationClassificationCounters,
physical_mutation_possible: bool,
retryable: bool,
return_cursor: bool,
error: bool,
}
impl StepTemplate {
fn complete() -> Self {
Self {
scan_complete: true,
traversal_syscalls: 1,
candidates: 0,
mutation_attempts: 0,
classifications: ReconciliationClassificationCounters::default(),
physical_mutation_possible: false,
retryable: false,
return_cursor: false,
error: false,
}
}
}
struct CursorEvidence {
alive: Arc<AtomicUsize>,
}
impl CursorEvidence {
fn new(alive: Arc<AtomicUsize>, maximum: &AtomicUsize) -> Self {
let current = alive.fetch_add(1, Ordering::SeqCst) + 1;
maximum.fetch_max(current, Ordering::SeqCst);
Self { alive }
}
}
impl Drop for CursorEvidence {
fn drop(&mut self) {
self.alive.fetch_sub(1, Ordering::SeqCst);
}
}
struct ScriptedBackend {
steps: Mutex<VecDeque<StepTemplate>>,
calls: Mutex<Vec<(Phase, bool, StepLimits)>>,
alive: Arc<AtomicUsize>,
maximum: AtomicUsize,
}
impl ScriptedBackend {
fn new(steps: impl IntoIterator<Item = StepTemplate>) -> Self {
Self {
steps: Mutex::new(steps.into_iter().collect()),
calls: Mutex::new(Vec::new()),
alive: Arc::new(AtomicUsize::new(0)),
maximum: AtomicUsize::new(0),
}
}
}
#[async_trait]
impl ReconciliationBackend for Arc<ScriptedBackend> {
type Cursor = CursorEvidence;
async fn step(
&self,
phase: Phase,
cursor: Option<Self::Cursor>,
limits: StepLimits,
) -> Result<Step<Self::Cursor>, ReconciliationCoordinatorError> {
self.calls
.lock()
.unwrap()
.push((phase, cursor.is_some(), limits));
drop(cursor);
let template = self.steps.lock().unwrap().pop_front().unwrap();
if template.error {
return Err(ReconciliationCoordinatorError::Backend);
}
let cursor = template
.return_cursor
.then(|| CursorEvidence::new(self.alive.clone(), &self.maximum));
Ok(Step {
cursor,
scan_complete: template.scan_complete,
traversal_syscalls: template.traversal_syscalls,
candidates: template.candidates,
mutation_attempts: template.mutation_attempts,
classifications: template.classifications,
physical_mutation_possible: template.physical_mutation_possible,
retryable: template.retryable,
})
}
}
#[tokio::test]
async fn recovery_always_completes_before_final_sweep() {
let backend = Arc::new(ScriptedBackend::new([
StepTemplate::complete(),
StepTemplate::complete(),
]));
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
let report = coordinator.tick().await.unwrap();
assert!(report.recovery_completed);
assert!(report.cycle_completed);
let phases = backend
.calls
.lock()
.unwrap()
.iter()
.map(|call| call.0)
.collect::<Vec<_>>();
assert_eq!(phases, vec![Phase::Recovery, Phase::Sweep]);
}
#[tokio::test]
async fn possible_mutation_discards_cursor_and_restarts_recovery() {
let paged = StepTemplate {
scan_complete: false,
traversal_syscalls: 2,
candidates: 1,
mutation_attempts: 0,
classifications: ReconciliationClassificationCounters::default(),
physical_mutation_possible: false,
retryable: true,
return_cursor: true,
error: false,
};
let mutation = StepTemplate {
scan_complete: false,
traversal_syscalls: 1,
candidates: 1,
mutation_attempts: 1,
classifications: ReconciliationClassificationCounters::default(),
physical_mutation_possible: true,
retryable: false,
return_cursor: true,
error: false,
};
let backend = Arc::new(ScriptedBackend::new([
StepTemplate::complete(),
paged,
mutation,
StepTemplate::complete(),
StepTemplate::complete(),
]));
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
let first = coordinator.tick().await.unwrap();
assert!(first.retryable);
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
let second = coordinator.tick().await.unwrap();
assert!(second.cycle_completed);
let calls = backend.calls.lock().unwrap();
assert_eq!(
calls
.iter()
.map(|(phase, had_cursor, _)| (*phase, *had_cursor))
.collect::<Vec<_>>(),
vec![
(Phase::Recovery, false),
(Phase::Sweep, false),
(Phase::Sweep, true),
(Phase::Recovery, false),
(Phase::Sweep, false),
]
);
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn backend_error_discards_cursor_and_restarts_recovery() {
let paged = StepTemplate {
scan_complete: false,
traversal_syscalls: 1,
candidates: 1,
mutation_attempts: 0,
classifications: ReconciliationClassificationCounters::default(),
physical_mutation_possible: false,
retryable: true,
return_cursor: true,
error: false,
};
let failure = StepTemplate {
error: true,
..StepTemplate::complete()
};
let backend = Arc::new(ScriptedBackend::new([
StepTemplate::complete(),
paged,
failure,
StepTemplate::complete(),
StepTemplate::complete(),
]));
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
assert!(coordinator.tick().await.unwrap().retryable);
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
assert_eq!(
coordinator.tick().await,
Err(ReconciliationCoordinatorError::Backend)
);
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
assert!(coordinator.tick().await.unwrap().cycle_completed);
let calls = backend.calls.lock().unwrap();
assert_eq!(
calls
.iter()
.map(|(phase, had_cursor, _)| (*phase, *had_cursor))
.collect::<Vec<_>>(),
vec![
(Phase::Recovery, false),
(Phase::Sweep, false),
(Phase::Sweep, true),
(Phase::Recovery, false),
(Phase::Sweep, false),
]
);
}
#[tokio::test]
async fn backend_cannot_exceed_tick_limits() {
let backend = Arc::new(ScriptedBackend::new([StepTemplate {
scan_complete: false,
traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT + 1,
candidates: 0,
mutation_attempts: 0,
classifications: ReconciliationClassificationCounters::default(),
physical_mutation_possible: false,
retryable: false,
return_cursor: false,
error: false,
}]));
let mut coordinator = ReconciliationCoordinator::new(backend);
assert_eq!(
coordinator.tick().await,
Err(ReconciliationCoordinatorError::Bounds)
);
}
#[tokio::test]
async fn tick_stops_at_the_exact_candidate_and_mutation_limits() {
let backend = Arc::new(ScriptedBackend::new(
(0..RECONCILIATION_MUTATION_LIMIT).map(|_| StepTemplate {
scan_complete: false,
traversal_syscalls: 1,
candidates: 1,
mutation_attempts: 1,
classifications: ReconciliationClassificationCounters::default(),
physical_mutation_possible: true,
retryable: false,
return_cursor: true,
error: false,
}),
));
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
let report = coordinator.tick().await.unwrap();
assert_eq!(report.candidates, RECONCILIATION_CANDIDATE_LIMIT);
assert_eq!(report.mutation_attempts, RECONCILIATION_MUTATION_LIMIT);
assert_eq!(report.traversal_syscalls, RECONCILIATION_MUTATION_LIMIT);
assert_eq!(
backend.calls.lock().unwrap().len(),
RECONCILIATION_MUTATION_LIMIT
);
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn coordinator_retains_at_most_one_cursor_between_ticks() {
let backend = Arc::new(ScriptedBackend::new((0..128).map(|_| StepTemplate {
scan_complete: false,
traversal_syscalls: 1,
candidates: 0,
mutation_attempts: 0,
classifications: ReconciliationClassificationCounters::default(),
physical_mutation_possible: false,
retryable: true,
return_cursor: true,
error: false,
})));
let mut coordinator = ReconciliationCoordinator::new(backend.clone());
for _ in 0..128 {
coordinator.tick().await.unwrap();
assert_eq!(backend.alive.load(Ordering::SeqCst), 1);
}
assert_eq!(backend.maximum.load(Ordering::SeqCst), 1);
drop(coordinator);
assert_eq!(backend.alive.load(Ordering::SeqCst), 0);
}
#[test]
fn registration_failures_have_a_closed_redacted_classification() {
let mut counters = ReconciliationClassificationCounters::default();
assert!(!counters.record_registration_error(ArtifactError::Integrity));
assert!(!counters.record_registration_error(ArtifactError::UnsafeRoot));
assert!(counters.record_registration_error(ArtifactError::Storage));
assert!(!counters.record_registration_error(ArtifactError::NotFound));
assert!(!counters.record_registration_error(ArtifactError::InvalidReference));
assert!(!counters.record_registration_error(ArtifactError::EmptySource));
assert!(!counters.record_registration_error(ArtifactError::SourceTooLarge));
assert_eq!(
counters,
ReconciliationClassificationCounters {
scanner_malformed: 0,
scanner_unsafe: 0,
recovery_present: 0,
recovery_safety: 0,
recovery_retryable: 0,
registration_integrity: 3,
registration_safety: 3,
registration_retryable: 1,
claim_metadata_conflict: 0,
mutation_integrity: 0,
mutation_safety: 0,
mutation_retryable: 0,
}
);
let debug = format!("{counters:?}");
assert!(!debug.contains("sha256:"));
assert!(!debug.contains('/'));
}
#[test]
fn mutation_failures_are_classified_and_always_retryable() {
let mut counters = ReconciliationClassificationCounters::default();
assert!(counters.record_mutation_error(ArtifactError::Integrity));
assert!(counters.record_mutation_error(ArtifactError::UnsafeRoot));
assert!(counters.record_mutation_error(ArtifactError::Storage));
assert!(counters.record_mutation_error(ArtifactError::NotFound));
assert!(counters.record_mutation_error(ArtifactError::InvalidReference));
assert!(counters.record_mutation_error(ArtifactError::EmptySource));
assert!(counters.record_mutation_error(ArtifactError::SourceTooLarge));
assert_eq!(counters.mutation_integrity, 3);
assert_eq!(counters.mutation_safety, 3);
assert_eq!(counters.mutation_retryable, 1);
}
#[tokio::test]
async fn production_backend_recovers_absence_before_sweep_and_restarts_after_mutation() {
let database_url =
crank_test_support::postgres_schema_url("admin_reconciliation_backend").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let root = TestRoot::new();
let store = ArtifactStore::open(&root.0).unwrap();
let now = OffsetDateTime::now_utc();
let expired = store.put_registered(b"expired physical absence\n").unwrap();
let expired_claim = registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &expired,
token: ArtifactClaimToken::generate(),
claimed_at: now - time::Duration::minutes(10),
lease_expires_at: now - time::Duration::minutes(5),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap();
assert!(matches!(expired_claim, ArtifactClaimOutcome::Claimed(_)));
remove_registered_artifact(&store, &expired);
sqlx::query(
"update artifact_blobs
set claim_expires_at = clock_timestamp() - interval '1 second'
where digest = $1",
)
.bind(expired.artifact_ref().digest_hex())
.execute(registry.pool())
.await
.unwrap();
let corrupted = store
.put_registered(b"integrity-blocked-candidate\n")
.unwrap();
let corrupted_path = artifact_path(&root.0, &corrupted);
let corrupted_size = fs::metadata(&corrupted_path).unwrap().len() as usize;
fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o600)).unwrap();
fs::write(&corrupted_path, vec![b'x'; corrupted_size]).unwrap();
fs::set_permissions(&corrupted_path, fs::Permissions::from_mode(0o400)).unwrap();
age_artifact(&root.0, &corrupted);
let later = store.put_registered(b"later-valid-candidate\n").unwrap();
age_artifact(&root.0, &later);
let metadata_conflict = store
.put_registered(b"metadata-conflict-candidate\n")
.unwrap();
age_artifact(&root.0, &metadata_conflict);
sqlx::query(
"insert into artifact_blobs
(digest, artifact_ref, size_bytes, storage_lifecycle, created_at, updated_at)
values ($1, $2, $3, 'available', $4, $4)",
)
.bind(metadata_conflict.artifact_ref().digest_hex())
.bind(metadata_conflict.artifact_ref().as_str())
.bind(i64::try_from(metadata_conflict.size_bytes()).unwrap() + 1)
.bind(now)
.execute(registry.pool())
.await
.unwrap();
let shard = artifact_path(&root.0, &later)
.parent()
.unwrap()
.to_path_buf();
fs::write(shard.join("not-a-digest"), b"malformed").unwrap();
let unsafe_name = format!(
"{}{}",
&later.artifact_ref().digest_hex()[..2],
"f".repeat(62)
);
assert_ne!(unsafe_name, later.artifact_ref().digest_hex());
symlink("not-a-digest", shard.join(unsafe_name)).unwrap();
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone()));
let mut coordinator = ReconciliationCoordinator::with_tick_limits(
backend,
StepLimits {
traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT,
candidates: RECONCILIATION_CANDIDATE_LIMIT,
mutations: 1,
},
);
// The single mutation slot is consumed by expired-claim recovery. The old
// final candidate proves Sweep did not run ahead of Recovery.
let first = coordinator.tick().await.unwrap();
assert_eq!(first.mutation_attempts, 1);
assert!(!first.recovery_completed);
assert_eq!(
store.reconciliation_presence(later.artifact_ref()).unwrap(),
ReconciliationPresence::Final
);
let lifecycle: String =
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
.bind(expired.artifact_ref().digest_hex())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(lifecycle, "unavailable");
// The real final sweep classifies blocked entries and still reaches the
// later valid item. Its physical mutation invalidates the final cursor.
let second = coordinator.tick().await.unwrap();
assert!(second.recovery_completed);
assert_eq!(second.mutation_attempts, 1);
assert_eq!(
store.reconciliation_presence(later.artifact_ref()).unwrap(),
ReconciliationPresence::Quarantine
);
// Model the next scheduled/restarted lease window. Because the coordinator
// reset to Recovery and discarded its final cursor, quarantine is handled
// before another final sweep.
sqlx::query("update artifact_blobs set claim_expires_at = $1 where digest = $2")
.bind(OffsetDateTime::now_utc() - time::Duration::seconds(1))
.bind(later.artifact_ref().digest_hex())
.execute(registry.pool())
.await
.unwrap();
let third = coordinator.tick().await.unwrap();
assert_eq!(third.mutation_attempts, 1);
assert_eq!(
store.reconciliation_presence(later.artifact_ref()).unwrap(),
ReconciliationPresence::Absent
);
let fourth = coordinator.tick().await.unwrap();
assert!(fourth.cycle_completed);
let classifications = first
.classifications
.checked_add(second.classifications)
.and_then(|value| value.checked_add(third.classifications))
.and_then(|value| value.checked_add(fourth.classifications))
.unwrap();
assert!(classifications.scanner_malformed > 0);
assert!(classifications.scanner_unsafe > 0);
assert!(classifications.registration_integrity > 0);
assert!(classifications.claim_metadata_conflict > 0);
assert_eq!(
store
.reconciliation_presence(corrupted.artifact_ref())
.unwrap(),
ReconciliationPresence::Unsafe
);
assert_eq!(classifications.registration_safety, 0);
assert_eq!(classifications.registration_retryable, 0);
}
#[tokio::test]
async fn expired_claim_cursor_prevents_unsafe_prefix_starvation_across_ticks() {
let database_url =
crank_test_support::postgres_schema_url("admin_reconciliation_starvation").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let root = TestRoot::new();
let store = ArtifactStore::open(&root.0).unwrap();
let now = OffsetDateTime::now_utc();
for index in 0..=RECONCILIATION_CANDIDATE_LIMIT {
let artifact = store
.put_registered(format!("unsafe expired claim {index}\n").as_bytes())
.unwrap();
let outcome = registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &artifact,
token: ArtifactClaimToken::generate(),
claimed_at: now - time::Duration::minutes(20),
lease_expires_at: now - time::Duration::minutes(15),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap();
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
sqlx::query(
"update artifact_blobs
set claim_expires_at = clock_timestamp() - interval '10 minutes'
where digest = $1",
)
.bind(artifact.artifact_ref().digest_hex())
.execute(registry.pool())
.await
.unwrap();
fs::set_permissions(
artifact_path(&root.0, &artifact),
fs::Permissions::from_mode(0o600),
)
.unwrap();
}
let absent = store
.put_registered(b"absent after unsafe prefix\n")
.unwrap();
let outcome = registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &absent,
token: ArtifactClaimToken::generate(),
claimed_at: now - time::Duration::minutes(15),
lease_expires_at: now - time::Duration::minutes(10),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap();
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
sqlx::query(
"update artifact_blobs
set claim_expires_at = clock_timestamp() - interval '5 minutes'
where digest = $1",
)
.bind(absent.artifact_ref().digest_hex())
.execute(registry.pool())
.await
.unwrap();
remove_registered_artifact(&store, &absent);
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store));
let mut coordinator = ReconciliationCoordinator::new(backend);
let first = coordinator.tick().await.unwrap();
assert_eq!(first.candidates, RECONCILIATION_CANDIDATE_LIMIT);
assert_eq!(first.mutation_attempts, 0);
assert_eq!(
first.classifications.recovery_safety,
RECONCILIATION_CANDIDATE_LIMIT
);
let first_lifecycle: String =
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
.bind(absent.artifact_ref().digest_hex())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(first_lifecycle, "available");
let second = coordinator.tick().await.unwrap();
assert!(second.classifications.recovery_safety > 0);
assert!(second.mutation_attempts > 0);
let second_lifecycle: String =
sqlx::query_scalar("select storage_lifecycle from artifact_blobs where digest = $1")
.bind(absent.artifact_ref().digest_hex())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(second_lifecycle, "unavailable");
}
#[path = "tests/commit_failure.rs"]
mod commit_failure;