use super::*; #[tokio::test] async fn finalize_commit_failure_stays_retryable_and_recovers_on_a_fresh_cycle() { let database_url = crank_test_support::postgres_schema_url("admin_reconciliation_commit_failure").await; let pool = sqlx::PgPool::connect(&database_url).await.unwrap(); MigrationAuthority::apply(&pool).await.unwrap(); let registry = PostgresRegistry::connect(&database_url).await.unwrap(); let root = TestRoot::new(); let store = ArtifactStore::open(&root.0).unwrap(); let now = OffsetDateTime::now_utc(); let artifact = store .put_registered(b"commit-failure physical absence\n") .unwrap(); let outcome = registry .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest { artifact: &artifact, token: ArtifactClaimToken::generate(), claimed_at: now - time::Duration::minutes(10), lease_expires_at: now - time::Duration::minutes(5), detached_grace: time::Duration::hours(24), }) .await .unwrap(); assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_))); remove_registered_artifact(&store, &artifact); sqlx::query( "update artifact_blobs set claim_expires_at = clock_timestamp() - interval '1 second' where digest = $1", ) .bind(artifact.artifact_ref().digest_hex()) .execute(registry.pool()) .await .unwrap(); // A deferred constraint trigger fails the COMMIT that records // `unavailable`, modelling a real failure after the physical unlink that // removed the last bytes already succeeded. It lives only in this isolated // test schema. sqlx::raw_sql( "create function crank_test_fail_unavailable() returns trigger language plpgsql as $$ begin if new.storage_lifecycle = 'unavailable' then raise exception 'injected finalize commit failure'; end if; return new; end $$; create constraint trigger crank_test_fail_unavailable_guard after update on artifact_blobs deferrable initially deferred for each row execute function crank_test_fail_unavailable();", ) .execute(registry.pool()) .await .unwrap(); let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone())); let mut coordinator = ReconciliationCoordinator::new(backend); let failed = coordinator.tick().await.unwrap(); // A real commit failure must surface as retryable work, never as idle. assert!(failed.retryable); assert_eq!(failed.mutation_attempts, 1); assert_eq!( store .reconciliation_presence(artifact.artifact_ref()) .unwrap(), ReconciliationPresence::Absent ); let retained = sqlx::query( "select claim_token, claim_expires_at, storage_lifecycle from artifact_blobs where digest = $1", ) .bind(artifact.artifact_ref().digest_hex()) .fetch_one(registry.pool()) .await .unwrap(); assert!( retained .try_get::, _>("claim_token") .unwrap() .is_some(), "the failed commit must roll back the claim clear" ); assert!( retained .try_get::, _>("claim_expires_at") .unwrap() .is_some() ); assert_eq!( retained.get::("storage_lifecycle"), "available", "the failed commit must roll back the unavailable transition" ); // Removing the injected trigger and retrying a fresh coordinator clears the // retained claim and records the confirmed absence as `unavailable`. sqlx::raw_sql( "drop trigger crank_test_fail_unavailable_guard on artifact_blobs; drop function crank_test_fail_unavailable();", ) .execute(registry.pool()) .await .unwrap(); sqlx::query( "update artifact_blobs set claim_expires_at = clock_timestamp() - interval '1 second' where digest = $1", ) .bind(artifact.artifact_ref().digest_hex()) .execute(registry.pool()) .await .unwrap(); let mut recovered_coordinator = ReconciliationCoordinator::new( PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone())), ); let recovered = recovered_coordinator.tick().await.unwrap(); assert!(!recovered.retryable); assert_eq!( store .reconciliation_presence(artifact.artifact_ref()) .unwrap(), ReconciliationPresence::Absent ); let cleared = sqlx::query( "select claim_token, claim_expires_at, storage_lifecycle from artifact_blobs where digest = $1", ) .bind(artifact.artifact_ref().digest_hex()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!( cleared.try_get::, _>("claim_token").unwrap(), None ); assert_eq!( cleared .try_get::, _>("claim_expires_at") .unwrap(), None ); assert_eq!(cleared.get::("storage_lifecycle"), "unavailable"); }