150 lines
5.1 KiB
Rust
150 lines
5.1 KiB
Rust
use super::*;
|
|
|
|
#[tokio::test]
|
|
async fn finalize_commit_failure_stays_retryable_and_recovers_on_a_fresh_cycle() {
|
|
let database_url =
|
|
crank_test_support::postgres_schema_url("admin_reconciliation_commit_failure").await;
|
|
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
|
|
MigrationAuthority::apply(&pool).await.unwrap();
|
|
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
|
|
let root = TestRoot::new();
|
|
let store = ArtifactStore::open(&root.0).unwrap();
|
|
let now = OffsetDateTime::now_utc();
|
|
|
|
let artifact = store
|
|
.put_registered(b"commit-failure physical absence\n")
|
|
.unwrap();
|
|
let outcome = registry
|
|
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
|
artifact: &artifact,
|
|
token: ArtifactClaimToken::generate(),
|
|
claimed_at: now - time::Duration::minutes(10),
|
|
lease_expires_at: now - time::Duration::minutes(5),
|
|
detached_grace: time::Duration::hours(24),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
|
|
remove_registered_artifact(&store, &artifact);
|
|
sqlx::query(
|
|
"update artifact_blobs
|
|
set claim_expires_at = clock_timestamp() - interval '1 second'
|
|
where digest = $1",
|
|
)
|
|
.bind(artifact.artifact_ref().digest_hex())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
|
|
// A deferred constraint trigger fails the COMMIT that records
|
|
// `unavailable`, modelling a real failure after the physical unlink that
|
|
// removed the last bytes already succeeded. It lives only in this isolated
|
|
// test schema.
|
|
sqlx::raw_sql(
|
|
"create function crank_test_fail_unavailable() returns trigger language plpgsql as $$
|
|
begin
|
|
if new.storage_lifecycle = 'unavailable' then
|
|
raise exception 'injected finalize commit failure';
|
|
end if;
|
|
return new;
|
|
end $$;
|
|
create constraint trigger crank_test_fail_unavailable_guard
|
|
after update on artifact_blobs
|
|
deferrable initially deferred
|
|
for each row execute function crank_test_fail_unavailable();",
|
|
)
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
|
|
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone()));
|
|
let mut coordinator = ReconciliationCoordinator::new(backend);
|
|
let failed = coordinator.tick().await.unwrap();
|
|
|
|
// A real commit failure must surface as retryable work, never as idle.
|
|
assert!(failed.retryable);
|
|
assert_eq!(failed.mutation_attempts, 1);
|
|
assert_eq!(
|
|
store
|
|
.reconciliation_presence(artifact.artifact_ref())
|
|
.unwrap(),
|
|
ReconciliationPresence::Absent
|
|
);
|
|
let retained = sqlx::query(
|
|
"select claim_token, claim_expires_at, storage_lifecycle
|
|
from artifact_blobs where digest = $1",
|
|
)
|
|
.bind(artifact.artifact_ref().digest_hex())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert!(
|
|
retained
|
|
.try_get::<Option<String>, _>("claim_token")
|
|
.unwrap()
|
|
.is_some(),
|
|
"the failed commit must roll back the claim clear"
|
|
);
|
|
assert!(
|
|
retained
|
|
.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")
|
|
.unwrap()
|
|
.is_some()
|
|
);
|
|
assert_eq!(
|
|
retained.get::<String, _>("storage_lifecycle"),
|
|
"available",
|
|
"the failed commit must roll back the unavailable transition"
|
|
);
|
|
|
|
// Removing the injected trigger and retrying a fresh coordinator clears the
|
|
// retained claim and records the confirmed absence as `unavailable`.
|
|
sqlx::raw_sql(
|
|
"drop trigger crank_test_fail_unavailable_guard on artifact_blobs;
|
|
drop function crank_test_fail_unavailable();",
|
|
)
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"update artifact_blobs
|
|
set claim_expires_at = clock_timestamp() - interval '1 second'
|
|
where digest = $1",
|
|
)
|
|
.bind(artifact.artifact_ref().digest_hex())
|
|
.execute(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
|
|
let mut recovered_coordinator = ReconciliationCoordinator::new(
|
|
PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone())),
|
|
);
|
|
let recovered = recovered_coordinator.tick().await.unwrap();
|
|
assert!(!recovered.retryable);
|
|
assert_eq!(
|
|
store
|
|
.reconciliation_presence(artifact.artifact_ref())
|
|
.unwrap(),
|
|
ReconciliationPresence::Absent
|
|
);
|
|
let cleared = sqlx::query(
|
|
"select claim_token, claim_expires_at, storage_lifecycle
|
|
from artifact_blobs where digest = $1",
|
|
)
|
|
.bind(artifact.artifact_ref().digest_hex())
|
|
.fetch_one(registry.pool())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
cleared.try_get::<Option<String>, _>("claim_token").unwrap(),
|
|
None
|
|
);
|
|
assert_eq!(
|
|
cleared
|
|
.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")
|
|
.unwrap(),
|
|
None
|
|
);
|
|
assert_eq!(cleared.get::<String, _>("storage_lifecycle"), "unavailable");
|
|
}
|