feat(artifacts): add fenced reconciliation recovery

2026-08-28 00:44:27 +03:00
parent 8784964fb2
commit d2849ea3fe
21 changed files with 3322 additions and 97 deletions
@@ -0,0 +1,214 @@
+use std::time::Duration as StdDuration;
+
+use crank_artifacts::ArtifactStore;
+use crank_registry::{
+    ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
+    ArtifactClaimToken, ClaimArtifactReconciliationRequest,
+    ClaimExpiredArtifactReconciliationRequest, RegistryError,
+};
+
+use super::{
+    artifact_sources::{TestRoot, timestamp},
+    common::TestDatabase,
+};
+
+#[tokio::test]
+async fn locked_database_time_and_expired_recovery_are_fenced() {
+    let database = TestDatabase::new().await;
+    let registry = database.registry().await;
+    let raw_pool = database.raw_pool().await;
+    let root = TestRoot::new("database-time");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let caller_time = timestamp("2020-01-01T00:00:00Z");
+
+    let blocked = store.put_registered(b"row lock expiry\n").unwrap();
+    assert!(matches!(
+        registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &blocked,
+                token: ArtifactClaimToken::generate(),
+                claimed_at: caller_time,
+                lease_expires_at: caller_time + time::Duration::seconds(1),
+                detached_grace: time::Duration::hours(24),
+            })
+            .await
+            .unwrap(),
+        ArtifactClaimOutcome::Claimed(_)
+    ));
+    let mut blocker = raw_pool.begin().await.unwrap();
+    sqlx::query("select digest from artifact_blobs where digest = $1 for update")
+        .bind(blocked.artifact_ref().digest_hex())
+        .fetch_one(&mut *blocker)
+        .await
+        .unwrap();
+    let waiting_registry = registry.clone();
+    let waiting_artifact = blocked.clone();
+    let waiting = tokio::spawn(async move {
+        waiting_registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &waiting_artifact,
+                token: ArtifactClaimToken::generate(),
+                claimed_at: caller_time,
+                lease_expires_at: caller_time + time::Duration::minutes(5),
+                detached_grace: time::Duration::hours(24),
+            })
+            .await
+    });
+    tokio::time::sleep(StdDuration::from_millis(1_100)).await;
+    blocker.commit().await.unwrap();
+    assert!(matches!(
+        waiting.await.unwrap().unwrap(),
+        ArtifactClaimOutcome::Claimed(_)
+    ));
+
+    let recoverable = store.put_registered(b"single winner\n").unwrap();
+    let prior_token = ArtifactClaimToken::generate();
+    let prior_claim = match registry
+        .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+            artifact: &recoverable,
+            token: prior_token.clone(),
+            claimed_at: caller_time,
+            lease_expires_at: caller_time + time::Duration::minutes(5),
+            detached_grace: time::Duration::hours(24),
+        })
+        .await
+        .unwrap()
+    {
+        ArtifactClaimOutcome::Claimed(claim) => claim,
+        outcome => panic!("unexpected claim outcome: {outcome:?}"),
+    };
+    sqlx::query(
+        "update artifact_blobs
+            set claim_expires_at = clock_timestamp() - interval '1 second'
+          where digest = $1",
+    )
+    .bind(recoverable.artifact_ref().digest_hex())
+    .execute(&raw_pool)
+    .await
+    .unwrap();
+    assert!(matches!(
+        registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &recoverable,
+                token: prior_token.clone(),
+                claimed_at: caller_time,
+                lease_expires_at: caller_time + time::Duration::minutes(5),
+                detached_grace: time::Duration::hours(24),
+            })
+            .await
+            .unwrap(),
+        ArtifactClaimOutcome::HeldByOther
+    ));
+    let recovery_now = registry.artifact_reconciliation_now().await.unwrap();
+    let probe = registry
+        .list_expired_artifact_reconciliation_probes(recovery_now, 1)
+        .await
+        .unwrap()
+        .pop()
+        .unwrap();
+    assert!(matches!(
+        registry
+            .claim_expired_artifact_reconciliation(
+                &probe,
+                ClaimExpiredArtifactReconciliationRequest {
+                    token: prior_token,
+                    claimed_at: caller_time,
+                    lease_expires_at: caller_time + time::Duration::minutes(5),
+                },
+            )
+            .await,
+        Err(RegistryError::InvalidArtifactSource {
+            field: "claim_token"
+        })
+    ));
+
+    let first_registry = registry.clone();
+    let second_registry = registry.clone();
+    let first_probe = probe.clone();
+    let (first, second) = tokio::join!(
+        async move {
+            first_registry
+                .claim_expired_artifact_reconciliation(
+                    &first_probe,
+                    ClaimExpiredArtifactReconciliationRequest {
+                        token: ArtifactClaimToken::generate(),
+                        claimed_at: caller_time,
+                        lease_expires_at: caller_time + time::Duration::minutes(5),
+                    },
+                )
+                .await
+        },
+        async move {
+            second_registry
+                .claim_expired_artifact_reconciliation(
+                    &probe,
+                    ClaimExpiredArtifactReconciliationRequest {
+                        token: ArtifactClaimToken::generate(),
+                        claimed_at: caller_time,
+                        lease_expires_at: caller_time + time::Duration::minutes(5),
+                    },
+                )
+                .await
+        }
+    );
+    let first = first.unwrap();
+    let second = second.unwrap();
+    assert_eq!(
+        usize::from(first.is_some()) + usize::from(second.is_some()),
+        1
+    );
+    let winner = first.or(second).unwrap();
+
+    sqlx::query("update artifact_blobs set claim_expires_at = clock_timestamp() where digest = $1")
+        .bind(recoverable.artifact_ref().digest_hex())
+        .execute(&raw_pool)
+        .await
+        .unwrap();
+    for claim in [&winner, &prior_claim] {
+        assert_eq!(
+            registry
+                .finalize_artifact_reconciliation_claim(
+                    claim,
+                    ArtifactClaimFinalization::AlreadyAbsent,
+                    caller_time,
+                )
+                .await
+                .unwrap(),
+            ArtifactClaimFinalizeOutcome::Stale
+        );
+    }
+
+    let invalid = store.put_registered(b"invalid bounds\n").unwrap();
+    assert!(matches!(
+        registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &invalid,
+                token: ArtifactClaimToken::generate(),
+                claimed_at: caller_time,
+                lease_expires_at: caller_time + time::Duration::minutes(5),
+                detached_grace: time::Duration::MAX,
+            })
+            .await,
+        Err(RegistryError::InvalidArtifactSource {
+            field: "detached_grace"
+        })
+    ));
+    assert!(matches!(
+        registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &invalid,
+                token: ArtifactClaimToken::generate(),
+                claimed_at: caller_time,
+                lease_expires_at: caller_time
+                    + time::Duration::minutes(5)
+                    + time::Duration::seconds(1),
+                detached_grace: time::Duration::ZERO,
+            })
+            .await,
+        Err(RegistryError::InvalidArtifactSource {
+            field: "claim_lease"
+        })
+    ));
+
+    database.cleanup().await;
+}
@@ -5,24 +5,30 @@ use std::{
    os::unix::fs::PermissionsExt,
    path::PathBuf,
    sync::atomic::{AtomicU64, Ordering},
+    time::{Duration as StdDuration, SystemTime},
 };

-use crank_artifacts::{ArtifactRef, ArtifactStore};
+use crank_artifacts::{
+    ArtifactRef, ArtifactStore, ReconciliationMutation, ReconciliationPresence,
+    ReconciliationRegistration, RegisteredArtifact,
+};
 use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
 use crank_registry::{
-    ArtifactSourceId, ArtifactSourceLifecycle, ArtifactSourceSensitivity,
-    CreateArtifactSourceRequest, CreateWorkspaceRequest, DetachArtifactSourceRequest,
-    ListArtifactSourcesQuery, RegistryError,
+    ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
+    ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactSourceId, ArtifactSourceLifecycle,
+    ArtifactSourceSensitivity, ClaimArtifactReconciliationRequest,
+    ClaimExpiredArtifactReconciliationRequest, CreateArtifactSourceRequest, CreateWorkspaceRequest,
+    DetachArtifactSourceRequest, ListArtifactSourcesQuery, RegistryError,
 };
 use serde_json::json;
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};

 static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);

-struct TestRoot(PathBuf);
+pub(super) struct TestRoot(pub(super) PathBuf);

 impl TestRoot {
-    fn new(name: &str) -> Self {
+    pub(super) fn new(name: &str) -> Self {
        let path = std::env::temp_dir().join(format!(
            "crank-registry-artifacts-{name}-{}-{}",
            std::process::id(),
@@ -41,10 +47,26 @@ impl Drop for TestRoot {
    }
 }

-fn timestamp(value: &str) -> OffsetDateTime {
+pub(super) fn timestamp(value: &str) -> OffsetDateTime {
    OffsetDateTime::parse(value, &Rfc3339).unwrap()
 }

+fn reconciliation_candidate_for(
+    store: &ArtifactStore,
+    artifact: &RegisteredArtifact,
+) -> crank_artifacts::ReconciliationCandidate {
+    let (_, candidates) = store.scan_reconciliation(None, 4096, 32).unwrap();
+    candidates
+        .into_iter()
+        .find(|candidate| {
+            matches!(
+                store.register_reconciliation(candidate, StdDuration::ZERO, SystemTime::now()),
+                Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
+            )
+        })
+        .expect("registered reconciliation candidate")
+}
+
 async fn create_workspace(registry: &crank_registry::PostgresRegistry, id: &str) -> WorkspaceId {
    let workspace_id = WorkspaceId::new(id);
    let created_at = timestamp("2026-08-26T10:00:00Z");
@@ -534,3 +556,300 @@ async fn verified_read_returns_only_digest_and_size_verified_bytes() {

    database.cleanup().await;
 }
+
+#[tokio::test]
+async fn reconciliation_claims_are_fenced_global_and_allow_verified_revival() {
+    let database = TestDatabase::new().await;
+    let registry = database.registry().await;
+    let raw_pool = database.raw_pool().await;
+    let workspace_a = create_workspace(&registry, "ws_claim_a").await;
+    let workspace_b = create_workspace(&registry, "ws_claim_b").await;
+    let root = TestRoot::new("claims");
+    let store = ArtifactStore::open(&root.0).unwrap();
+    let unreferenced = store.put_registered(b"unreferenced\n").unwrap();
+    let started = timestamp("2026-08-27T10:00:00Z");
+    let lease = timestamp("2026-08-27T10:05:00Z");
+    let token = ArtifactClaimToken::generate();
+    assert_eq!(format!("{token:?}"), "ArtifactClaimToken(..)");
+
+    let request = ClaimArtifactReconciliationRequest {
+        artifact: &unreferenced,
+        token: token.clone(),
+        claimed_at: started,
+        lease_expires_at: lease,
+        detached_grace: time::Duration::hours(24),
+    };
+    assert_eq!(
+        format!("{request:?}"),
+        "ClaimArtifactReconciliationRequest(..)"
+    );
+    let claim = match registry
+        .claim_artifact_reconciliation(request)
+        .await
+        .unwrap()
+    {
+        ArtifactClaimOutcome::Claimed(claim) => claim,
+        outcome => panic!("unexpected claim outcome: {outcome:?}"),
+    };
+    assert_eq!(format!("{claim:?}"), "ArtifactReconciliationClaim(..)");
+    assert!(matches!(
+        registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &unreferenced,
+                token,
+                claimed_at: timestamp("2026-08-27T10:01:00Z"),
+                lease_expires_at: lease,
+                detached_grace: time::Duration::hours(24),
+            })
+            .await
+            .unwrap(),
+        ArtifactClaimOutcome::Claimed(_)
+    ));
+    assert_eq!(
+        registry
+            .recheck_artifact_reconciliation_claim(&claim, started, time::Duration::hours(24))
+            .await
+            .unwrap(),
+        ArtifactClaimRecheckOutcome::Mutate
+    );
+    assert!(matches!(
+        registry
+            .create_artifact_source(CreateArtifactSourceRequest {
+                workspace_id: &workspace_a,
+                source_id: &ArtifactSourceId::new("src_blocked"),
+                artifact: &unreferenced,
+                mime_type: "text/plain",
+                sensitivity: ArtifactSourceSensitivity::Internal,
+                created_at: started,
+            })
+            .await,
+        Err(RegistryError::ArtifactClaimInProgress)
+    ));
+    assert_eq!(
+        registry
+            .finalize_artifact_reconciliation_claim(
+                &claim,
+                ArtifactClaimFinalization::Quarantined,
+                timestamp("2026-08-27T10:01:00Z"),
+            )
+            .await
+            .unwrap(),
+        ArtifactClaimFinalizeOutcome::Retained
+    );
+
+    let final_candidate = reconciliation_candidate_for(&store, &unreferenced);
+    assert!(matches!(
+        store.quarantine_reconciliation(final_candidate),
+        Ok(ReconciliationMutation::Quarantined | ReconciliationMutation::AlreadyQuarantined)
+    ));
+    let quarantined_candidate = reconciliation_candidate_for(&store, &unreferenced);
+    assert!(matches!(
+        store.delete_quarantined_reconciliation(quarantined_candidate),
+        Ok(ReconciliationMutation::Deleted | ReconciliationMutation::AlreadyAbsent)
+    ));
+    assert_eq!(
+        store
+            .reconciliation_presence(unreferenced.artifact_ref())
+            .unwrap(),
+        ReconciliationPresence::Absent
+    );
+
+    sqlx::query(
+        "update artifact_blobs
+            set claim_expires_at = clock_timestamp() - interval '1 second'
+          where digest = $1",
+    )
+    .bind(unreferenced.artifact_ref().digest_hex())
+    .execute(&raw_pool)
+    .await
+    .unwrap();
+    let recovery_now = registry.artifact_reconciliation_now().await.unwrap();
+    let probes = registry
+        .list_expired_artifact_reconciliation_probes(recovery_now, 1)
+        .await
+        .unwrap();
+    assert_eq!(
+        format!("{:?}", probes.first().expect("expired claim probe")),
+        "ArtifactExpiredClaimProbe(..)"
+    );
+    let winner = registry
+        .claim_expired_artifact_reconciliation(
+            probes.first().expect("expired claim probe"),
+            ClaimExpiredArtifactReconciliationRequest {
+                token: ArtifactClaimToken::generate(),
+                claimed_at: recovery_now,
+                lease_expires_at: recovery_now + time::Duration::minutes(5),
+            },
+        )
+        .await
+        .unwrap()
+        .expect("expired claim was not recovered");
+    assert_eq!(
+        registry
+            .finalize_artifact_reconciliation_claim(
+                &claim,
+                ArtifactClaimFinalization::Deleted,
+                timestamp("2026-08-27T10:05:01Z"),
+            )
+            .await
+            .unwrap(),
+        ArtifactClaimFinalizeOutcome::Stale
+    );
+    assert_eq!(
+        registry
+            .recover_artifact_reconciliation_claim(
+                &winner,
+                ArtifactClaimFinalization::AlreadyAbsent,
+                timestamp("2026-08-27T10:05:02Z"),
+            )
+            .await
+            .unwrap(),
+        ArtifactClaimFinalizeOutcome::Unavailable
+    );
+    let republished = store.put_registered(b"unreferenced\n").unwrap();
+    let revived = registry
+        .create_artifact_source(CreateArtifactSourceRequest {
+            workspace_id: &workspace_a,
+            source_id: &ArtifactSourceId::new("src_revived"),
+            artifact: &republished,
+            mime_type: "text/plain",
+            sensitivity: ArtifactSourceSensitivity::Internal,
+            created_at: timestamp("2026-08-27T10:06:00Z"),
+        })
+        .await
+        .unwrap();
+    assert_eq!(
+        revived.blob.lifecycle,
+        crank_registry::ArtifactBlobLifecycle::Available
+    );
+
+    let referenced = store.put_registered(b"referenced\n").unwrap();
+    let active_id = ArtifactSourceId::new("src_active");
+    let active = registry
+        .create_artifact_source(CreateArtifactSourceRequest {
+            workspace_id: &workspace_b,
+            source_id: &active_id,
+            artifact: &referenced,
+            mime_type: "text/plain",
+            sensitivity: ArtifactSourceSensitivity::Internal,
+            created_at: started,
+        })
+        .await
+        .unwrap();
+    assert!(matches!(
+        registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &referenced,
+                token: ArtifactClaimToken::generate(),
+                claimed_at: timestamp("2026-08-27T10:01:00Z"),
+                lease_expires_at: lease,
+                detached_grace: time::Duration::hours(24),
+            })
+            .await
+            .unwrap(),
+        ArtifactClaimOutcome::ActiveReference
+    ));
+    registry
+        .detach_artifact_source(DetachArtifactSourceRequest {
+            workspace_id: &workspace_b,
+            source_id: &active_id,
+            expected_updated_at: Some(active.updated_at),
+            detached_at: timestamp("2026-08-27T10:02:00Z"),
+        })
+        .await
+        .unwrap();
+    assert!(matches!(
+        registry
+            .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                artifact: &referenced,
+                token: ArtifactClaimToken::generate(),
+                claimed_at: timestamp("2026-08-27T10:03:00Z"),
+                lease_expires_at: lease,
+                detached_grace: time::Duration::hours(24),
+            })
+            .await
+            .unwrap(),
+        ArtifactClaimOutcome::DetachedReferenceInGrace
+    ));
+
+    let concurrent = store.put_registered(b"concurrent claim\n").unwrap();
+    let first_registry = registry.clone();
+    let second_registry = registry.clone();
+    let first_artifact = concurrent.clone();
+    let second_artifact = concurrent.clone();
+    let (first, second) = tokio::join!(
+        async move {
+            first_registry
+                .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                    artifact: &first_artifact,
+                    token: ArtifactClaimToken::generate(),
+                    claimed_at: timestamp("2026-08-27T11:00:00Z"),
+                    lease_expires_at: timestamp("2026-08-27T11:05:00Z"),
+                    detached_grace: time::Duration::hours(24),
+                })
+                .await
+        },
+        async move {
+            second_registry
+                .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                    artifact: &second_artifact,
+                    token: ArtifactClaimToken::generate(),
+                    claimed_at: timestamp("2026-08-27T11:00:00Z"),
+                    lease_expires_at: timestamp("2026-08-27T11:05:00Z"),
+                    detached_grace: time::Duration::hours(24),
+                })
+                .await
+        }
+    );
+    let first = first.unwrap();
+    let second = second.unwrap();
+    assert_eq!(
+        usize::from(matches!(first, ArtifactClaimOutcome::Claimed(_)))
+            + usize::from(matches!(second, ArtifactClaimOutcome::Claimed(_))),
+        1
+    );
+    assert_eq!(
+        usize::from(matches!(first, ArtifactClaimOutcome::HeldByOther))
+            + usize::from(matches!(second, ArtifactClaimOutcome::HeldByOther)),
+        1
+    );
+
+    let attach_race = store.put_registered(b"attach race\n").unwrap();
+    let claim_registry = registry.clone();
+    let attach_registry = registry.clone();
+    let claim_artifact = attach_race.clone();
+    let attach_artifact = attach_race.clone();
+    let attach_workspace = workspace_a.clone();
+    let (claim_race, attach_race) = tokio::join!(
+        async move {
+            claim_registry
+                .claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
+                    artifact: &claim_artifact,
+                    token: ArtifactClaimToken::generate(),
+                    claimed_at: timestamp("2026-08-27T12:00:00Z"),
+                    lease_expires_at: timestamp("2026-08-27T12:05:00Z"),
+                    detached_grace: time::Duration::hours(24),
+                })
+                .await
+        },
+        async move {
+            attach_registry
+                .create_artifact_source(CreateArtifactSourceRequest {
+                    workspace_id: &attach_workspace,
+                    source_id: &ArtifactSourceId::new("src_attach_race"),
+                    artifact: &attach_artifact,
+                    mime_type: "text/plain",
+                    sensitivity: ArtifactSourceSensitivity::Internal,
+                    created_at: timestamp("2026-08-27T12:00:00Z"),
+                })
+                .await
+        }
+    );
+    match (claim_race.unwrap(), attach_race) {
+        (ArtifactClaimOutcome::ActiveReference, Ok(_)) => {}
+        (ArtifactClaimOutcome::Claimed(_), Err(RegistryError::ArtifactClaimInProgress)) => {}
+        (claim, attach) => panic!("unsafe claim/attach race outcome: {claim:?}, {attach:?}"),
+    }
+
+    database.cleanup().await;
+}