feat(artifacts): add fenced reconciliation recovery

This commit is contained in:
2026-08-28 00:44:27 +03:00
parent 8784964fb2
commit d2849ea3fe
21 changed files with 3322 additions and 97 deletions
+1
View File
@@ -11,6 +11,7 @@ crank-artifacts = { path = "../crank-artifacts" }
crank-core = { path = "../crank-core" }
crank-mapping = { path = "../crank-mapping" }
crank-schema = { path = "../crank-schema" }
rand.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
+2
View File
@@ -156,6 +156,8 @@ pub enum RegistryError {
SourceUnavailable,
#[error("artifact source integrity verification failed")]
SourceIntegrity,
#[error("artifact reconciliation is already claimed")]
ArtifactClaimInProgress,
#[error("artifact source metadata is invalid for field {field}")]
InvalidArtifactSource { field: &'static str },
}
+32 -24
View File
@@ -15,28 +15,32 @@ pub mod records {
pub use crate::model::{
AdminBootstrapContractRecord, AgentSummary, AgentVersionRecord, AppendProductEventOutcome,
AppliedImportOperation, ApprovalRequestRecord, ArtifactBlobLifecycle, ArtifactBlobRecord,
ArtifactDigest, ArtifactSourceCursor, ArtifactSourceId, ArtifactSourceLifecycle,
ArtifactSourcePage, ArtifactSourceRecord, ArtifactSourceSensitivity, AuthUserRecord,
DescriptorKind, DescriptorMetadata, ImportJob, ImportJobApplyResult, ImportJobId,
ImportJobKind, ImportJobStatus, InvitationRecord, InvocationHistoryLoss,
InvocationHistoryLossCategory, InvocationHistoryWriteOutcome, InvocationLogRecord,
InvocationRetentionOutcome, InvocationRetentionPolicy, InvocationRetentionStatus,
MasterKeyIdentityRecord, MasterKeyRotationRecord, MasterKeyRotationStatus,
MembershipRecord, OnboardingMilestoneResult, OnboardingPresentationMilestone,
OperationAgentRef, OperationSampleMetadata, OperationSummary, OperationUsageSummary,
OperationVersionRecord, Page, PlatformApiKeyRecord, ProductEventRecord,
PublishedAgentCatalog, PublishedAgentTool, RegistryOperation, SampleKind, SecretRecord,
SecretVersionRecord, SessionRecord, SkippedImportOperation, UsageAgentBreakdown,
UsageBucket, UsageOperationBreakdown, UsageOutcomeBreakdown, UsageOutcomeGroup,
UsageRollupRecord, UsageSummary, UsageTimelinePoint, VerifiedArtifactSource,
WorkspaceMembershipRecord, WorkspaceRecord, WorkspaceUpstream, WorkspaceUpstreamId,
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactDigest, ArtifactExpiredClaimProbe,
ArtifactReconciliationClaim, ArtifactSourceCursor, ArtifactSourceId,
ArtifactSourceLifecycle, ArtifactSourcePage, ArtifactSourceRecord,
ArtifactSourceSensitivity, AuthUserRecord, DescriptorKind, DescriptorMetadata, ImportJob,
ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobStatus, InvitationRecord,
InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
InvocationLogRecord, InvocationRetentionOutcome, InvocationRetentionPolicy,
InvocationRetentionStatus, MasterKeyIdentityRecord, MasterKeyRotationRecord,
MasterKeyRotationStatus, MembershipRecord, OnboardingMilestoneResult,
OnboardingPresentationMilestone, OperationAgentRef, OperationSampleMetadata,
OperationSummary, OperationUsageSummary, OperationVersionRecord, Page,
PlatformApiKeyRecord, ProductEventRecord, PublishedAgentCatalog, PublishedAgentTool,
RegistryOperation, SampleKind, SecretRecord, SecretVersionRecord, SessionRecord,
SkippedImportOperation, UsageAgentBreakdown, UsageBucket, UsageOperationBreakdown,
UsageOutcomeBreakdown, UsageOutcomeGroup, UsageRollupRecord, UsageSummary,
UsageTimelinePoint, VerifiedArtifactSource, WorkspaceMembershipRecord, WorkspaceRecord,
WorkspaceUpstream, WorkspaceUpstreamId, YamlImportJob, YamlImportJobCompletion,
YamlImportJobId, YamlImportJobStatus,
};
}
pub mod requests {
pub use crate::model::{
AdminSecurityAuditRequest, AppendProductEventRequest, ApplyImportJobRequest,
ClaimArtifactReconciliationRequest, ClaimExpiredArtifactReconciliationRequest,
ConsumeAdminBootstrapContractRequest, CreateAdminBootstrapContractRequest,
CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest,
CreateArtifactSourceRequest, CreateImportJobRequest, CreateInvitationRequest,
@@ -67,9 +71,12 @@ pub use model::{
AdminBootstrapContractRecord, AdminSecurityAuditRequest, AgentStateExpectation, AgentSummary,
AgentVersionRecord, AppendProductEventOutcome, AppendProductEventRequest,
AppliedImportOperation, ApplyImportJobRequest, ApprovalRequestRecord, ArtifactBlobLifecycle,
ArtifactBlobRecord, ArtifactDigest, ArtifactSourceCursor, ArtifactSourceId,
ArtifactBlobRecord, ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome,
ArtifactClaimOutcome, ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactDigest,
ArtifactExpiredClaimProbe, ArtifactReconciliationClaim, ArtifactSourceCursor, ArtifactSourceId,
ArtifactSourceLifecycle, ArtifactSourcePage, ArtifactSourceRecord, ArtifactSourceSensitivity,
AuthUserRecord, ConsumeAdminBootstrapContractRequest, CreateAdminBootstrapContractRequest,
AuthUserRecord, ClaimArtifactReconciliationRequest, ClaimExpiredArtifactReconciliationRequest,
ConsumeAdminBootstrapContractRequest, CreateAdminBootstrapContractRequest,
CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest,
CreateArtifactSourceRequest, CreateImportJobRequest, CreateInvitationRequest,
CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest,
@@ -81,12 +88,13 @@ pub use model::{
InvocationHistoryWriteOutcome, InvocationLogRecord, InvocationRetentionOutcome,
InvocationRetentionPolicy, InvocationRetentionStatus, ListApprovalRequestsQuery,
ListArtifactSourcesQuery, ListInvocationLogsQuery, ListProductEventsQuery,
MASTER_KEY_CIPHER_CONTRACT, MAX_ARTIFACT_SOURCE_PAGE_SIZE, MasterKeyIdentityCandidate,
MasterKeyIdentityRecord, MasterKeyRotationRecord, MasterKeyRotationStatus, MembershipRecord,
OnboardingMilestoneResult, OnboardingPresentationMilestone, OperationAgentRef,
OperationSampleMetadata, OperationStateExpectation, OperationSummary, OperationUsageSummary,
OperationVersionRecord, Page, PlatformApiKeyRecord, ProductEventRecord, PublishAgentRequest,
PublishRequest, PublishedAgentCatalog, PublishedAgentTool, RecordOnboardingCompletionRequest,
MASTER_KEY_CIPHER_CONTRACT, MAX_ARTIFACT_CLAIM_RECOVERY_BATCH, MAX_ARTIFACT_SOURCE_PAGE_SIZE,
MasterKeyIdentityCandidate, MasterKeyIdentityRecord, MasterKeyRotationRecord,
MasterKeyRotationStatus, MembershipRecord, OnboardingMilestoneResult,
OnboardingPresentationMilestone, OperationAgentRef, OperationSampleMetadata,
OperationStateExpectation, OperationSummary, OperationUsageSummary, OperationVersionRecord,
Page, PlatformApiKeyRecord, ProductEventRecord, PublishAgentRequest, PublishRequest,
PublishedAgentCatalog, PublishedAgentTool, RecordOnboardingCompletionRequest,
RecordOnboardingMilestoneRequest, RecoverAdminPasswordRequest, RegistryOperation,
RotateSecretRequest, SampleKind, SaveAgentBindingsRequest, SaveAgentCatalogConfigRequest,
SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest,
@@ -1,5 +1,8 @@
use std::fmt;
use crank_artifacts::{ArtifactRef, RegisteredArtifact};
use crank_core::WorkspaceId;
use rand::random;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
@@ -7,6 +10,189 @@ define_registry_id!(ArtifactSourceId);
define_registry_id!(ArtifactDigest);
pub const MAX_ARTIFACT_SOURCE_PAGE_SIZE: u32 = 100;
pub const MAX_ARTIFACT_CLAIM_RECOVERY_BATCH: u32 = 32;
const ARTIFACT_CLAIM_TOKEN_BYTES: usize = 32;
/// An unpredictable, bounded fencing token for one reconciliation attempt.
///
/// It is intentionally opaque: callers can retain it in an
/// [`ArtifactReconciliationClaim`], but cannot format or extract it for logs,
/// metrics, or another process.
#[derive(Clone, PartialEq, Eq)]
pub struct ArtifactClaimToken(String);
impl ArtifactClaimToken {
pub fn generate() -> Self {
let bytes = random::<[u8; ARTIFACT_CLAIM_TOKEN_BYTES]>();
let mut value = String::with_capacity(ARTIFACT_CLAIM_TOKEN_BYTES * 2);
for byte in bytes {
use std::fmt::Write as _;
let _ = write!(value, "{byte:02x}");
}
Self(value)
}
pub(crate) fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Debug for ArtifactClaimToken {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ArtifactClaimToken(..)")
}
}
/// Opaque transient authority to recheck and finalize a reconciliation item.
///
/// It has no `Display` implementation and its `Debug` form is redacted so a
/// durable claim token cannot cross an observability boundary by accident.
#[derive(Clone)]
pub struct ArtifactReconciliationClaim {
artifact_ref: ArtifactRef,
token: ArtifactClaimToken,
}
impl ArtifactReconciliationClaim {
pub(crate) fn new(artifact_ref: ArtifactRef, token: ArtifactClaimToken) -> Self {
Self {
artifact_ref,
token,
}
}
/// Opaque content-addressed identity for a trusted artifact-store probe.
///
/// This must not be formatted into telemetry or an external response.
pub fn artifact_ref(&self) -> &ArtifactRef {
&self.artifact_ref
}
pub(crate) fn token(&self) -> &ArtifactClaimToken {
&self.token
}
}
impl fmt::Debug for ArtifactReconciliationClaim {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ArtifactReconciliationClaim(..)")
}
}
#[derive(Clone)]
pub struct ClaimArtifactReconciliationRequest<'a> {
pub artifact: &'a RegisteredArtifact,
pub token: ArtifactClaimToken,
pub claimed_at: OffsetDateTime,
pub lease_expires_at: OffsetDateTime,
pub detached_grace: time::Duration,
}
impl fmt::Debug for ClaimArtifactReconciliationRequest<'_> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ClaimArtifactReconciliationRequest(..)")
}
}
/// Bounded recovery lease acquisition for one expired durable claim.
///
/// Recovery needs no `RegisteredArtifact`: it recovers an identity validated
/// when the previous claim was acquired. The caller must still revalidate
/// physical presence through the artifact store before finalization.
#[derive(Clone, Debug)]
pub struct ClaimExpiredArtifactReconciliationRequest {
pub token: ArtifactClaimToken,
pub claimed_at: OffsetDateTime,
pub lease_expires_at: OffsetDateTime,
}
/// Opaque compare-and-swap evidence for an expired claim discovered by a
/// bounded recovery read. It does not acquire a lease by itself.
#[derive(Clone)]
pub struct ArtifactExpiredClaimProbe {
artifact_ref: ArtifactRef,
prior_token: String,
claim_expires_at: OffsetDateTime,
}
impl ArtifactExpiredClaimProbe {
pub(crate) fn new(
artifact_ref: ArtifactRef,
prior_token: String,
claim_expires_at: OffsetDateTime,
) -> Self {
Self {
artifact_ref,
prior_token,
claim_expires_at,
}
}
/// Opaque content-addressed identity for a trusted artifact-store probe.
/// This must not be formatted into telemetry or an external response.
pub fn artifact_ref(&self) -> &ArtifactRef {
&self.artifact_ref
}
pub(crate) fn prior_token(&self) -> &str {
&self.prior_token
}
pub(crate) fn claim_expires_at(&self) -> OffsetDateTime {
self.claim_expires_at
}
}
impl fmt::Debug for ArtifactExpiredClaimProbe {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("ArtifactExpiredClaimProbe(..)")
}
}
#[derive(Clone, Debug)]
pub enum ArtifactClaimOutcome {
/// This caller owns the fenced claim and may perform the next recheck.
Claimed(ArtifactReconciliationClaim),
/// A current token already owns the artifact; retry after its lease.
HeldByOther,
/// The artifact is still referenced by at least one active source.
ActiveReference,
/// A detached source remains within the caller's grace window.
DetachedReferenceInGrace,
/// A row with the same digest does not match store-validated metadata.
MetadataConflict,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ArtifactClaimRecheckOutcome {
/// The caller still owns the claim and no global reference blocks I/O.
Mutate,
/// An active relation now protects the artifact.
ActiveReference,
/// A detached relation is still protected by grace.
DetachedReferenceInGrace,
/// The token was finalized, replaced, or expired.
Stale,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ArtifactClaimFinalization {
Deleted,
AlreadyAbsent,
Quarantined,
Retryable,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ArtifactClaimFinalizeOutcome {
/// Confirmed absence was durably recorded as V12 `unavailable`.
Unavailable,
/// An ambiguous/mid-flight mutation remains fenced for recovery.
Retained,
/// The supplied token no longer owns the row.
Stale,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -3,19 +3,373 @@ use crank_core::WorkspaceId;
use sqlx::{Postgres, Row, Transaction, postgres::PgRow};
use crate::{
ArtifactBlobLifecycle, ArtifactBlobRecord, ArtifactDigest, ArtifactSourceCursor,
ArtifactBlobLifecycle, ArtifactBlobRecord, ArtifactClaimFinalization,
ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome, ArtifactClaimRecheckOutcome,
ArtifactDigest, ArtifactExpiredClaimProbe, ArtifactReconciliationClaim, ArtifactSourceCursor,
ArtifactSourceId, ArtifactSourceLifecycle, ArtifactSourcePage, ArtifactSourceRecord,
ArtifactSourceSensitivity, CreateArtifactSourceRequest, DetachArtifactSourceRequest,
ListArtifactSourcesQuery, MAX_ARTIFACT_SOURCE_PAGE_SIZE, RegistryError, VerifiedArtifactSource,
ArtifactSourceSensitivity, ClaimArtifactReconciliationRequest,
ClaimExpiredArtifactReconciliationRequest, CreateArtifactSourceRequest,
DetachArtifactSourceRequest, ListArtifactSourcesQuery, MAX_ARTIFACT_CLAIM_RECOVERY_BATCH,
MAX_ARTIFACT_SOURCE_PAGE_SIZE, RegistryError, VerifiedArtifactSource,
};
use super::PostgresRegistry;
mod reconciliation;
use reconciliation::{
checked_cutoff, database_now, insert_blob_if_missing_at_database_time, validate_claim_lease,
};
const SOURCE_ID_PREFIX: &str = "src_";
const MAX_SOURCE_ID_SUFFIX_BYTES: usize = 128;
const MAX_MIME_TYPE_BYTES: usize = 255;
impl PostgresRegistry {
/// Acquires a short, fenced reconciliation lease after checking references
/// across every workspace. A lease may be replayed by its original token,
/// or stolen only after expiry.
pub async fn claim_artifact_reconciliation(
&self,
request: ClaimArtifactReconciliationRequest<'_>,
) -> Result<ArtifactClaimOutcome, RegistryError> {
let lease_duration = validate_claim_lease(request.claimed_at, request.lease_expires_at)?;
if request.detached_grace.is_negative() {
return Err(RegistryError::InvalidArtifactSource {
field: "detached_grace",
});
}
if !valid_artifact_size(request.artifact.size_bytes()) {
return Err(RegistryError::InvalidArtifactSource {
field: "size_bytes",
});
}
let mut transaction = self.pool().begin().await?;
let artifact_ref = request.artifact.artifact_ref();
let size_bytes = i64::try_from(request.artifact.size_bytes()).map_err(|_| {
RegistryError::InvalidArtifactSource {
field: "size_bytes",
}
})?;
insert_blob_if_missing_at_database_time(&mut transaction, artifact_ref, size_bytes).await?;
let row = lock_blob(&mut transaction, artifact_ref.digest_hex()).await?;
let database_now = database_now(&mut transaction).await?;
if !blob_metadata_matches(&row, artifact_ref, size_bytes)? {
transaction.rollback().await?;
return Ok(ArtifactClaimOutcome::MetadataConflict);
}
if global_active_reference(&mut transaction, artifact_ref.digest_hex()).await? {
transaction.rollback().await?;
return Ok(ArtifactClaimOutcome::ActiveReference);
}
if global_detached_reference_in_grace(
&mut transaction,
artifact_ref.digest_hex(),
checked_cutoff(database_now, request.detached_grace)?,
)
.await?
{
transaction.rollback().await?;
return Ok(ArtifactClaimOutcome::DetachedReferenceInGrace);
}
let claim_token = row.try_get::<Option<String>, _>("claim_token")?;
let claim_expires_at =
row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
if let (Some(current_token), Some(expires_at)) = (claim_token, claim_expires_at) {
if current_token == request.token.as_str() && expires_at > database_now {
transaction.commit().await?;
return Ok(ArtifactClaimOutcome::Claimed(
ArtifactReconciliationClaim::new(artifact_ref.clone(), request.token),
));
}
if expires_at > database_now || current_token == request.token.as_str() {
transaction.rollback().await?;
return Ok(ArtifactClaimOutcome::HeldByOther);
}
}
let updated = sqlx::query(
"update artifact_blobs
set claim_token = $1, claim_expires_at = $2, updated_at = greatest(updated_at, $3)
where digest = $4",
)
.bind(request.token.as_str())
.bind(database_now + lease_duration)
.bind(database_now)
.bind(artifact_ref.digest_hex())
.execute(&mut *transaction)
.await?
.rows_affected();
debug_assert_eq!(updated, 1);
transaction.commit().await?;
Ok(ArtifactClaimOutcome::Claimed(
ArtifactReconciliationClaim::new(artifact_ref.clone(), request.token),
))
}
/// Reads a bounded set of expired claims without taking their leases.
///
/// Recovery first probes physical presence through the artifact store. Only
/// a confirmed absence should call `claim_expired_artifact_reconciliation`.
/// Leaving final or quarantine entries untouched lets the ordinary scanner
/// reclaim them with a store-validated `RegisteredArtifact`.
pub async fn list_expired_artifact_reconciliation_probes(
&self,
now: time::OffsetDateTime,
limit: u32,
) -> Result<Vec<ArtifactExpiredClaimProbe>, RegistryError> {
self.list_expired_artifact_reconciliation_probes_after(now, None, limit)
.await
}
/// Continues the bounded expired-claim read after an opaque prior probe.
///
/// Keyset pagination prevents an unsafe or still-present first claim from
/// starving later absent claims without persisting worker state in V12.
pub async fn list_expired_artifact_reconciliation_probes_after(
&self,
now: time::OffsetDateTime,
after: Option<&ArtifactExpiredClaimProbe>,
limit: u32,
) -> Result<Vec<ArtifactExpiredClaimProbe>, RegistryError> {
let limit = limit.min(MAX_ARTIFACT_CLAIM_RECOVERY_BATCH);
if limit == 0 {
return Ok(Vec::new());
}
let after_expires_at = after.map(ArtifactExpiredClaimProbe::claim_expires_at);
let after_digest = after.map(|probe| probe.artifact_ref().digest_hex());
let rows = sqlx::query(
"select artifact_ref, claim_token, claim_expires_at
from artifact_blobs
where claim_token is not null
and claim_expires_at <= $1
and (
$2::timestamptz is null
or (claim_expires_at, digest) > ($2::timestamptz, $3::text)
)
order by claim_expires_at asc, digest asc
limit $4",
)
.bind(now)
.bind(after_expires_at)
.bind(after_digest)
.bind(i64::from(limit))
.fetch_all(self.pool())
.await?;
rows.into_iter()
.map(|row| {
let artifact_ref = ArtifactRef::parse(&row.try_get::<String, _>("artifact_ref")?)
.map_err(|_| RegistryError::InvalidArtifactSource {
field: "artifact_ref",
})?;
let prior_token = row.try_get::<Option<String>, _>("claim_token")?.ok_or(
RegistryError::InvalidArtifactSource {
field: "claim_token",
},
)?;
let claim_expires_at = row
.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?
.ok_or(RegistryError::InvalidArtifactSource {
field: "claim_expires_at",
})?;
Ok(ArtifactExpiredClaimProbe::new(
artifact_ref,
prior_token,
claim_expires_at,
))
})
.collect()
}
/// Compare-and-swap steals an expired probe after recovery observed absence.
///
/// The caller must probe physical state again after a successful claim: a
/// concurrent scanner may have changed the inode between the read and CAS.
pub async fn claim_expired_artifact_reconciliation(
&self,
probe: &ArtifactExpiredClaimProbe,
request: ClaimExpiredArtifactReconciliationRequest,
) -> Result<Option<ArtifactReconciliationClaim>, RegistryError> {
let lease_duration = validate_claim_lease(request.claimed_at, request.lease_expires_at)?;
if request.token.as_str() == probe.prior_token() {
return Err(RegistryError::InvalidArtifactSource {
field: "claim_token",
});
}
let mut transaction = self.pool().begin().await?;
let Some(row) =
lock_blob_optional(&mut transaction, probe.artifact_ref().digest_hex()).await?
else {
transaction.rollback().await?;
return Ok(None);
};
let database_now = database_now(&mut transaction).await?;
let current_token = row.try_get::<Option<String>, _>("claim_token")?;
let expires_at = row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
if current_token.as_deref() != Some(probe.prior_token())
|| expires_at.is_none_or(|value| value > database_now)
{
transaction.rollback().await?;
return Ok(None);
}
let claimed = sqlx::query(
"update artifact_blobs
set claim_token = $1,
claim_expires_at = $2,
updated_at = greatest(updated_at, $3)
where digest = $4
and claim_token = $5
and claim_expires_at <= $3",
)
.bind(request.token.as_str())
.bind(database_now + lease_duration)
.bind(database_now)
.bind(probe.artifact_ref().digest_hex())
.bind(probe.prior_token())
.execute(&mut *transaction)
.await?
.rows_affected()
== 1;
transaction.commit().await?;
Ok(claimed
.then(|| ArtifactReconciliationClaim::new(probe.artifact_ref().clone(), request.token)))
}
/// Rechecks an owned claim immediately before blocking filesystem I/O.
///
/// `create_artifact_source` holds the same blob lock and rejects any active
/// claim, so a `Mutate` outcome fences the recheck-to-filesystem window.
pub async fn recheck_artifact_reconciliation_claim(
&self,
claim: &ArtifactReconciliationClaim,
_now: time::OffsetDateTime,
detached_grace: time::Duration,
) -> Result<ArtifactClaimRecheckOutcome, RegistryError> {
if detached_grace.is_negative() {
return Err(RegistryError::InvalidArtifactSource {
field: "detached_grace",
});
}
let mut transaction = self.pool().begin().await?;
let Some(row) =
lock_blob_optional(&mut transaction, claim.artifact_ref().digest_hex()).await?
else {
transaction.rollback().await?;
return Ok(ArtifactClaimRecheckOutcome::Stale);
};
let database_now = database_now(&mut transaction).await?;
let current_token = row.try_get::<Option<String>, _>("claim_token")?;
let expires_at = row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
if current_token.as_deref() != Some(claim.token().as_str())
|| expires_at.is_none_or(|value| value <= database_now)
{
transaction.rollback().await?;
return Ok(ArtifactClaimRecheckOutcome::Stale);
}
if global_active_reference(&mut transaction, claim.artifact_ref().digest_hex()).await? {
transaction.commit().await?;
return Ok(ArtifactClaimRecheckOutcome::ActiveReference);
}
if global_detached_reference_in_grace(
&mut transaction,
claim.artifact_ref().digest_hex(),
checked_cutoff(database_now, detached_grace)?,
)
.await?
{
transaction.commit().await?;
return Ok(ArtifactClaimRecheckOutcome::DetachedReferenceInGrace);
}
transaction.commit().await?;
Ok(ArtifactClaimRecheckOutcome::Mutate)
}
/// Records a filesystem observation without performing filesystem I/O.
///
/// Only confirmed absence clears the claim and transitions V12 metadata to
/// `unavailable`; every ambiguous observation deliberately retains it for
/// a later bounded recovery attempt.
pub async fn finalize_artifact_reconciliation_claim(
&self,
claim: &ArtifactReconciliationClaim,
observation: ArtifactClaimFinalization,
_observed_at: time::OffsetDateTime,
) -> Result<ArtifactClaimFinalizeOutcome, RegistryError> {
let mut transaction = self.pool().begin().await?;
let Some(row) =
lock_blob_optional(&mut transaction, claim.artifact_ref().digest_hex()).await?
else {
transaction.rollback().await?;
return Ok(ArtifactClaimFinalizeOutcome::Stale);
};
let database_now = database_now(&mut transaction).await?;
let current_token = row.try_get::<Option<String>, _>("claim_token")?;
let expires_at = row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
if current_token.as_deref() != Some(claim.token().as_str())
|| expires_at.is_none_or(|value| value <= database_now)
{
transaction.rollback().await?;
return Ok(ArtifactClaimFinalizeOutcome::Stale);
}
let (lifecycle, clear_claim, outcome) = match observation {
ArtifactClaimFinalization::Deleted | ArtifactClaimFinalization::AlreadyAbsent => (
"unavailable",
true,
ArtifactClaimFinalizeOutcome::Unavailable,
),
ArtifactClaimFinalization::Quarantined | ArtifactClaimFinalization::Retryable => {
("available", false, ArtifactClaimFinalizeOutcome::Retained)
}
};
let updated = if clear_claim {
sqlx::query(
"update artifact_blobs
set storage_lifecycle = $1, claim_token = null, claim_expires_at = null,
updated_at = greatest(updated_at, $2)
where digest = $3 and claim_token = $4 and claim_expires_at > $2",
)
.bind(lifecycle)
.bind(database_now)
.bind(claim.artifact_ref().digest_hex())
.bind(claim.token().as_str())
.execute(&mut *transaction)
.await?
.rows_affected()
} else {
sqlx::query(
"update artifact_blobs
set updated_at = greatest(updated_at, $1)
where digest = $2 and claim_token = $3 and claim_expires_at > $1",
)
.bind(database_now)
.bind(claim.artifact_ref().digest_hex())
.bind(claim.token().as_str())
.execute(&mut *transaction)
.await?
.rows_affected()
};
if updated != 1 {
transaction.rollback().await?;
return Ok(ArtifactClaimFinalizeOutcome::Stale);
}
transaction.commit().await?;
Ok(outcome)
}
/// Alias used by startup recovery once its filesystem probe has produced a
/// typed observation. The registry never performs the probe itself.
pub async fn recover_artifact_reconciliation_claim(
&self,
claim: &ArtifactReconciliationClaim,
observation: ArtifactClaimFinalization,
recovered_at: time::OffsetDateTime,
) -> Result<ArtifactClaimFinalizeOutcome, RegistryError> {
self.finalize_artifact_reconciliation_claim(claim, observation, recovered_at)
.await
}
pub async fn create_artifact_source(
&self,
request: CreateArtifactSourceRequest<'_>,
@@ -254,6 +608,41 @@ async fn ensure_blob(
field: "size_bytes",
}
})?;
insert_blob_if_missing(
transaction,
request.artifact.artifact_ref(),
size_bytes,
request.created_at,
)
.await?;
let row = lock_blob(transaction, digest).await?;
if !blob_metadata_matches(&row, request.artifact.artifact_ref(), size_bytes)? {
return Err(source_conflict(request.source_id));
}
if row.try_get::<Option<String>, _>("claim_token")?.is_some() {
return Err(RegistryError::ArtifactClaimInProgress);
}
if row.try_get::<String, _>("storage_lifecycle")? == "unavailable" {
sqlx::query(
"update artifact_blobs
set storage_lifecycle = 'available', updated_at = greatest(updated_at, $1)
where digest = $2 and claim_token is null",
)
.bind(request.created_at)
.bind(digest)
.execute(&mut **transaction)
.await?;
}
Ok(())
}
async fn insert_blob_if_missing(
transaction: &mut Transaction<'_, Postgres>,
artifact_ref: &ArtifactRef,
size_bytes: i64,
created_at: time::OffsetDateTime,
) -> Result<(), RegistryError> {
sqlx::query(
"insert into artifact_blobs (
digest, artifact_ref, size_bytes, storage_lifecycle,
@@ -261,25 +650,87 @@ async fn ensure_blob(
) values ($1, $2, $3, 'available', null, null, $4, $4)
on conflict (digest) do nothing",
)
.bind(digest)
.bind(request.artifact.artifact_ref().as_str())
.bind(artifact_ref.digest_hex())
.bind(artifact_ref.as_str())
.bind(size_bytes)
.bind(request.created_at)
.bind(created_at)
.execute(&mut **transaction)
.await?;
Ok(())
}
let row = sqlx::query(
"select artifact_ref, size_bytes from artifact_blobs where digest = $1 for update",
async fn lock_blob(
transaction: &mut Transaction<'_, Postgres>,
digest: &str,
) -> Result<PgRow, RegistryError> {
lock_blob_optional(transaction, digest)
.await?
.ok_or(RegistryError::InvalidArtifactSource {
field: "artifact_blob",
})
}
async fn lock_blob_optional(
transaction: &mut Transaction<'_, Postgres>,
digest: &str,
) -> Result<Option<PgRow>, RegistryError> {
sqlx::query(
"select artifact_ref, size_bytes, storage_lifecycle, claim_token, claim_expires_at
from artifact_blobs
where digest = $1
for update",
)
.bind(digest)
.fetch_optional(&mut **transaction)
.await
.map_err(Into::into)
}
fn blob_metadata_matches(
row: &PgRow,
artifact_ref: &ArtifactRef,
size_bytes: i64,
) -> Result<bool, RegistryError> {
Ok(
row.try_get::<String, _>("artifact_ref")? == artifact_ref.as_str()
&& row.try_get::<i64, _>("size_bytes")? == size_bytes,
)
}
async fn global_active_reference(
transaction: &mut Transaction<'_, Postgres>,
digest: &str,
) -> Result<bool, RegistryError> {
sqlx::query_scalar::<_, bool>(
"select exists(
select 1 from artifact_sources
where blob_digest = $1 and lifecycle = 'active'
)",
)
.bind(digest)
.fetch_one(&mut **transaction)
.await?;
let recorded_ref = row.try_get::<String, _>("artifact_ref")?;
let recorded_size = row.try_get::<i64, _>("size_bytes")?;
if recorded_ref != request.artifact.artifact_ref().as_str() || recorded_size != size_bytes {
return Err(source_conflict(request.source_id));
}
Ok(())
.await
.map_err(Into::into)
}
async fn global_detached_reference_in_grace(
transaction: &mut Transaction<'_, Postgres>,
digest: &str,
detached_after: time::OffsetDateTime,
) -> Result<bool, RegistryError> {
sqlx::query_scalar::<_, bool>(
"select exists(
select 1 from artifact_sources
where blob_digest = $1
and lifecycle = 'detached'
and detached_at > $2
)",
)
.bind(digest)
.bind(detached_after)
.fetch_one(&mut **transaction)
.await
.map_err(Into::into)
}
async fn get_source_in_transaction(
@@ -0,0 +1,68 @@
use crank_artifacts::ArtifactRef;
use sqlx::{Postgres, Transaction};
use crate::{PostgresRegistry, RegistryError};
const MAX_ARTIFACT_CLAIM_LEASE: time::Duration = time::Duration::minutes(5);
impl PostgresRegistry {
/// Returns the PostgreSQL clock used by production reconciliation leases.
pub async fn artifact_reconciliation_now(&self) -> Result<time::OffsetDateTime, RegistryError> {
sqlx::query_scalar("select clock_timestamp()")
.fetch_one(self.pool())
.await
.map_err(RegistryError::from)
}
}
pub(super) async fn insert_blob_if_missing_at_database_time(
transaction: &mut Transaction<'_, Postgres>,
artifact_ref: &ArtifactRef,
size_bytes: i64,
) -> Result<(), RegistryError> {
sqlx::query(
"insert into artifact_blobs (
digest, artifact_ref, size_bytes, storage_lifecycle,
claim_token, claim_expires_at, created_at, updated_at
) values ($1, $2, $3, 'available', null, null, clock_timestamp(), clock_timestamp())
on conflict (digest) do nothing",
)
.bind(artifact_ref.digest_hex())
.bind(artifact_ref.as_str())
.bind(size_bytes)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub(super) fn validate_claim_lease(
claimed_at: time::OffsetDateTime,
lease_expires_at: time::OffsetDateTime,
) -> Result<time::Duration, RegistryError> {
let duration = lease_expires_at - claimed_at;
if duration <= time::Duration::ZERO || duration > MAX_ARTIFACT_CLAIM_LEASE {
return Err(RegistryError::InvalidArtifactSource {
field: "claim_lease",
});
}
Ok(duration)
}
pub(super) fn checked_cutoff(
now: time::OffsetDateTime,
grace: time::Duration,
) -> Result<time::OffsetDateTime, RegistryError> {
now.checked_sub(grace)
.ok_or(RegistryError::InvalidArtifactSource {
field: "detached_grace",
})
}
pub(super) async fn database_now(
transaction: &mut Transaction<'_, Postgres>,
) -> Result<time::OffsetDateTime, RegistryError> {
sqlx::query_scalar("select clock_timestamp()")
.fetch_one(&mut **transaction)
.await
.map_err(Into::into)
}
@@ -1,6 +1,7 @@
mod integration {
mod agents_usage;
mod approval;
mod artifact_reconciliation;
mod artifact_sources;
mod common;
mod credential_touch;
@@ -0,0 +1,214 @@
use std::time::Duration as StdDuration;
use crank_artifacts::ArtifactStore;
use crank_registry::{
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
ArtifactClaimToken, ClaimArtifactReconciliationRequest,
ClaimExpiredArtifactReconciliationRequest, RegistryError,
};
use super::{
artifact_sources::{TestRoot, timestamp},
common::TestDatabase,
};
#[tokio::test]
async fn locked_database_time_and_expired_recovery_are_fenced() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let raw_pool = database.raw_pool().await;
let root = TestRoot::new("database-time");
let store = ArtifactStore::open(&root.0).unwrap();
let caller_time = timestamp("2020-01-01T00:00:00Z");
let blocked = store.put_registered(b"row lock expiry\n").unwrap();
assert!(matches!(
registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &blocked,
token: ArtifactClaimToken::generate(),
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::seconds(1),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap(),
ArtifactClaimOutcome::Claimed(_)
));
let mut blocker = raw_pool.begin().await.unwrap();
sqlx::query("select digest from artifact_blobs where digest = $1 for update")
.bind(blocked.artifact_ref().digest_hex())
.fetch_one(&mut *blocker)
.await
.unwrap();
let waiting_registry = registry.clone();
let waiting_artifact = blocked.clone();
let waiting = tokio::spawn(async move {
waiting_registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &waiting_artifact,
token: ArtifactClaimToken::generate(),
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::minutes(5),
detached_grace: time::Duration::hours(24),
})
.await
});
tokio::time::sleep(StdDuration::from_millis(1_100)).await;
blocker.commit().await.unwrap();
assert!(matches!(
waiting.await.unwrap().unwrap(),
ArtifactClaimOutcome::Claimed(_)
));
let recoverable = store.put_registered(b"single winner\n").unwrap();
let prior_token = ArtifactClaimToken::generate();
let prior_claim = match registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &recoverable,
token: prior_token.clone(),
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::minutes(5),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap()
{
ArtifactClaimOutcome::Claimed(claim) => claim,
outcome => panic!("unexpected claim outcome: {outcome:?}"),
};
sqlx::query(
"update artifact_blobs
set claim_expires_at = clock_timestamp() - interval '1 second'
where digest = $1",
)
.bind(recoverable.artifact_ref().digest_hex())
.execute(&raw_pool)
.await
.unwrap();
assert!(matches!(
registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &recoverable,
token: prior_token.clone(),
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::minutes(5),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap(),
ArtifactClaimOutcome::HeldByOther
));
let recovery_now = registry.artifact_reconciliation_now().await.unwrap();
let probe = registry
.list_expired_artifact_reconciliation_probes(recovery_now, 1)
.await
.unwrap()
.pop()
.unwrap();
assert!(matches!(
registry
.claim_expired_artifact_reconciliation(
&probe,
ClaimExpiredArtifactReconciliationRequest {
token: prior_token,
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::minutes(5),
},
)
.await,
Err(RegistryError::InvalidArtifactSource {
field: "claim_token"
})
));
let first_registry = registry.clone();
let second_registry = registry.clone();
let first_probe = probe.clone();
let (first, second) = tokio::join!(
async move {
first_registry
.claim_expired_artifact_reconciliation(
&first_probe,
ClaimExpiredArtifactReconciliationRequest {
token: ArtifactClaimToken::generate(),
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::minutes(5),
},
)
.await
},
async move {
second_registry
.claim_expired_artifact_reconciliation(
&probe,
ClaimExpiredArtifactReconciliationRequest {
token: ArtifactClaimToken::generate(),
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::minutes(5),
},
)
.await
}
);
let first = first.unwrap();
let second = second.unwrap();
assert_eq!(
usize::from(first.is_some()) + usize::from(second.is_some()),
1
);
let winner = first.or(second).unwrap();
sqlx::query("update artifact_blobs set claim_expires_at = clock_timestamp() where digest = $1")
.bind(recoverable.artifact_ref().digest_hex())
.execute(&raw_pool)
.await
.unwrap();
for claim in [&winner, &prior_claim] {
assert_eq!(
registry
.finalize_artifact_reconciliation_claim(
claim,
ArtifactClaimFinalization::AlreadyAbsent,
caller_time,
)
.await
.unwrap(),
ArtifactClaimFinalizeOutcome::Stale
);
}
let invalid = store.put_registered(b"invalid bounds\n").unwrap();
assert!(matches!(
registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &invalid,
token: ArtifactClaimToken::generate(),
claimed_at: caller_time,
lease_expires_at: caller_time + time::Duration::minutes(5),
detached_grace: time::Duration::MAX,
})
.await,
Err(RegistryError::InvalidArtifactSource {
field: "detached_grace"
})
));
assert!(matches!(
registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &invalid,
token: ArtifactClaimToken::generate(),
claimed_at: caller_time,
lease_expires_at: caller_time
+ time::Duration::minutes(5)
+ time::Duration::seconds(1),
detached_grace: time::Duration::ZERO,
})
.await,
Err(RegistryError::InvalidArtifactSource {
field: "claim_lease"
})
));
database.cleanup().await;
}
@@ -5,24 +5,30 @@ use std::{
os::unix::fs::PermissionsExt,
path::PathBuf,
sync::atomic::{AtomicU64, Ordering},
time::{Duration as StdDuration, SystemTime},
};
use crank_artifacts::{ArtifactRef, ArtifactStore};
use crank_artifacts::{
ArtifactRef, ArtifactStore, ReconciliationMutation, ReconciliationPresence,
ReconciliationRegistration, RegisteredArtifact,
};
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
use crank_registry::{
ArtifactSourceId, ArtifactSourceLifecycle, ArtifactSourceSensitivity,
CreateArtifactSourceRequest, CreateWorkspaceRequest, DetachArtifactSourceRequest,
ListArtifactSourcesQuery, RegistryError,
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactSourceId, ArtifactSourceLifecycle,
ArtifactSourceSensitivity, ClaimArtifactReconciliationRequest,
ClaimExpiredArtifactReconciliationRequest, CreateArtifactSourceRequest, CreateWorkspaceRequest,
DetachArtifactSourceRequest, ListArtifactSourcesQuery, RegistryError,
};
use serde_json::json;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
struct TestRoot(PathBuf);
pub(super) struct TestRoot(pub(super) PathBuf);
impl TestRoot {
fn new(name: &str) -> Self {
pub(super) fn new(name: &str) -> Self {
let path = std::env::temp_dir().join(format!(
"crank-registry-artifacts-{name}-{}-{}",
std::process::id(),
@@ -41,10 +47,26 @@ impl Drop for TestRoot {
}
}
fn timestamp(value: &str) -> OffsetDateTime {
pub(super) fn timestamp(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).unwrap()
}
fn reconciliation_candidate_for(
store: &ArtifactStore,
artifact: &RegisteredArtifact,
) -> crank_artifacts::ReconciliationCandidate {
let (_, candidates) = store.scan_reconciliation(None, 4096, 32).unwrap();
candidates
.into_iter()
.find(|candidate| {
matches!(
store.register_reconciliation(candidate, StdDuration::ZERO, SystemTime::now()),
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
)
})
.expect("registered reconciliation candidate")
}
async fn create_workspace(registry: &crank_registry::PostgresRegistry, id: &str) -> WorkspaceId {
let workspace_id = WorkspaceId::new(id);
let created_at = timestamp("2026-08-26T10:00:00Z");
@@ -534,3 +556,300 @@ async fn verified_read_returns_only_digest_and_size_verified_bytes() {
database.cleanup().await;
}
#[tokio::test]
async fn reconciliation_claims_are_fenced_global_and_allow_verified_revival() {
let database = TestDatabase::new().await;
let registry = database.registry().await;
let raw_pool = database.raw_pool().await;
let workspace_a = create_workspace(&registry, "ws_claim_a").await;
let workspace_b = create_workspace(&registry, "ws_claim_b").await;
let root = TestRoot::new("claims");
let store = ArtifactStore::open(&root.0).unwrap();
let unreferenced = store.put_registered(b"unreferenced\n").unwrap();
let started = timestamp("2026-08-27T10:00:00Z");
let lease = timestamp("2026-08-27T10:05:00Z");
let token = ArtifactClaimToken::generate();
assert_eq!(format!("{token:?}"), "ArtifactClaimToken(..)");
let request = ClaimArtifactReconciliationRequest {
artifact: &unreferenced,
token: token.clone(),
claimed_at: started,
lease_expires_at: lease,
detached_grace: time::Duration::hours(24),
};
assert_eq!(
format!("{request:?}"),
"ClaimArtifactReconciliationRequest(..)"
);
let claim = match registry
.claim_artifact_reconciliation(request)
.await
.unwrap()
{
ArtifactClaimOutcome::Claimed(claim) => claim,
outcome => panic!("unexpected claim outcome: {outcome:?}"),
};
assert_eq!(format!("{claim:?}"), "ArtifactReconciliationClaim(..)");
assert!(matches!(
registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &unreferenced,
token,
claimed_at: timestamp("2026-08-27T10:01:00Z"),
lease_expires_at: lease,
detached_grace: time::Duration::hours(24),
})
.await
.unwrap(),
ArtifactClaimOutcome::Claimed(_)
));
assert_eq!(
registry
.recheck_artifact_reconciliation_claim(&claim, started, time::Duration::hours(24))
.await
.unwrap(),
ArtifactClaimRecheckOutcome::Mutate
);
assert!(matches!(
registry
.create_artifact_source(CreateArtifactSourceRequest {
workspace_id: &workspace_a,
source_id: &ArtifactSourceId::new("src_blocked"),
artifact: &unreferenced,
mime_type: "text/plain",
sensitivity: ArtifactSourceSensitivity::Internal,
created_at: started,
})
.await,
Err(RegistryError::ArtifactClaimInProgress)
));
assert_eq!(
registry
.finalize_artifact_reconciliation_claim(
&claim,
ArtifactClaimFinalization::Quarantined,
timestamp("2026-08-27T10:01:00Z"),
)
.await
.unwrap(),
ArtifactClaimFinalizeOutcome::Retained
);
let final_candidate = reconciliation_candidate_for(&store, &unreferenced);
assert!(matches!(
store.quarantine_reconciliation(final_candidate),
Ok(ReconciliationMutation::Quarantined | ReconciliationMutation::AlreadyQuarantined)
));
let quarantined_candidate = reconciliation_candidate_for(&store, &unreferenced);
assert!(matches!(
store.delete_quarantined_reconciliation(quarantined_candidate),
Ok(ReconciliationMutation::Deleted | ReconciliationMutation::AlreadyAbsent)
));
assert_eq!(
store
.reconciliation_presence(unreferenced.artifact_ref())
.unwrap(),
ReconciliationPresence::Absent
);
sqlx::query(
"update artifact_blobs
set claim_expires_at = clock_timestamp() - interval '1 second'
where digest = $1",
)
.bind(unreferenced.artifact_ref().digest_hex())
.execute(&raw_pool)
.await
.unwrap();
let recovery_now = registry.artifact_reconciliation_now().await.unwrap();
let probes = registry
.list_expired_artifact_reconciliation_probes(recovery_now, 1)
.await
.unwrap();
assert_eq!(
format!("{:?}", probes.first().expect("expired claim probe")),
"ArtifactExpiredClaimProbe(..)"
);
let winner = registry
.claim_expired_artifact_reconciliation(
probes.first().expect("expired claim probe"),
ClaimExpiredArtifactReconciliationRequest {
token: ArtifactClaimToken::generate(),
claimed_at: recovery_now,
lease_expires_at: recovery_now + time::Duration::minutes(5),
},
)
.await
.unwrap()
.expect("expired claim was not recovered");
assert_eq!(
registry
.finalize_artifact_reconciliation_claim(
&claim,
ArtifactClaimFinalization::Deleted,
timestamp("2026-08-27T10:05:01Z"),
)
.await
.unwrap(),
ArtifactClaimFinalizeOutcome::Stale
);
assert_eq!(
registry
.recover_artifact_reconciliation_claim(
&winner,
ArtifactClaimFinalization::AlreadyAbsent,
timestamp("2026-08-27T10:05:02Z"),
)
.await
.unwrap(),
ArtifactClaimFinalizeOutcome::Unavailable
);
let republished = store.put_registered(b"unreferenced\n").unwrap();
let revived = registry
.create_artifact_source(CreateArtifactSourceRequest {
workspace_id: &workspace_a,
source_id: &ArtifactSourceId::new("src_revived"),
artifact: &republished,
mime_type: "text/plain",
sensitivity: ArtifactSourceSensitivity::Internal,
created_at: timestamp("2026-08-27T10:06:00Z"),
})
.await
.unwrap();
assert_eq!(
revived.blob.lifecycle,
crank_registry::ArtifactBlobLifecycle::Available
);
let referenced = store.put_registered(b"referenced\n").unwrap();
let active_id = ArtifactSourceId::new("src_active");
let active = registry
.create_artifact_source(CreateArtifactSourceRequest {
workspace_id: &workspace_b,
source_id: &active_id,
artifact: &referenced,
mime_type: "text/plain",
sensitivity: ArtifactSourceSensitivity::Internal,
created_at: started,
})
.await
.unwrap();
assert!(matches!(
registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &referenced,
token: ArtifactClaimToken::generate(),
claimed_at: timestamp("2026-08-27T10:01:00Z"),
lease_expires_at: lease,
detached_grace: time::Duration::hours(24),
})
.await
.unwrap(),
ArtifactClaimOutcome::ActiveReference
));
registry
.detach_artifact_source(DetachArtifactSourceRequest {
workspace_id: &workspace_b,
source_id: &active_id,
expected_updated_at: Some(active.updated_at),
detached_at: timestamp("2026-08-27T10:02:00Z"),
})
.await
.unwrap();
assert!(matches!(
registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &referenced,
token: ArtifactClaimToken::generate(),
claimed_at: timestamp("2026-08-27T10:03:00Z"),
lease_expires_at: lease,
detached_grace: time::Duration::hours(24),
})
.await
.unwrap(),
ArtifactClaimOutcome::DetachedReferenceInGrace
));
let concurrent = store.put_registered(b"concurrent claim\n").unwrap();
let first_registry = registry.clone();
let second_registry = registry.clone();
let first_artifact = concurrent.clone();
let second_artifact = concurrent.clone();
let (first, second) = tokio::join!(
async move {
first_registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &first_artifact,
token: ArtifactClaimToken::generate(),
claimed_at: timestamp("2026-08-27T11:00:00Z"),
lease_expires_at: timestamp("2026-08-27T11:05:00Z"),
detached_grace: time::Duration::hours(24),
})
.await
},
async move {
second_registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &second_artifact,
token: ArtifactClaimToken::generate(),
claimed_at: timestamp("2026-08-27T11:00:00Z"),
lease_expires_at: timestamp("2026-08-27T11:05:00Z"),
detached_grace: time::Duration::hours(24),
})
.await
}
);
let first = first.unwrap();
let second = second.unwrap();
assert_eq!(
usize::from(matches!(first, ArtifactClaimOutcome::Claimed(_)))
+ usize::from(matches!(second, ArtifactClaimOutcome::Claimed(_))),
1
);
assert_eq!(
usize::from(matches!(first, ArtifactClaimOutcome::HeldByOther))
+ usize::from(matches!(second, ArtifactClaimOutcome::HeldByOther)),
1
);
let attach_race = store.put_registered(b"attach race\n").unwrap();
let claim_registry = registry.clone();
let attach_registry = registry.clone();
let claim_artifact = attach_race.clone();
let attach_artifact = attach_race.clone();
let attach_workspace = workspace_a.clone();
let (claim_race, attach_race) = tokio::join!(
async move {
claim_registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &claim_artifact,
token: ArtifactClaimToken::generate(),
claimed_at: timestamp("2026-08-27T12:00:00Z"),
lease_expires_at: timestamp("2026-08-27T12:05:00Z"),
detached_grace: time::Duration::hours(24),
})
.await
},
async move {
attach_registry
.create_artifact_source(CreateArtifactSourceRequest {
workspace_id: &attach_workspace,
source_id: &ArtifactSourceId::new("src_attach_race"),
artifact: &attach_artifact,
mime_type: "text/plain",
sensitivity: ArtifactSourceSensitivity::Internal,
created_at: timestamp("2026-08-27T12:00:00Z"),
})
.await
}
);
match (claim_race.unwrap(), attach_race) {
(ArtifactClaimOutcome::ActiveReference, Ok(_)) => {}
(ArtifactClaimOutcome::Claimed(_), Err(RegistryError::ArtifactClaimInProgress)) => {}
(claim, attach) => panic!("unsafe claim/attach race outcome: {claim:?}, {attach:?}"),
}
database.cleanup().await;
}