feat(artifacts): add fenced reconciliation recovery
This commit is contained in:
@@ -8,21 +8,21 @@ use crate::temp_scan::{list_names, valid_temp_name};
|
||||
use crate::{
|
||||
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
|
||||
ReconciliationScan, ReconciliationScanStop,
|
||||
recovery::{FINAL_NAMESPACE, namespace_name, namespace_number},
|
||||
store::{
|
||||
QUARANTINE_DIR, RootLock, check_file, checkpointed, ensure_root_unchanged, fsync_fd,
|
||||
open_existing_dir, open_or_create_dir, rename_no_replace_at, stat_fd, unlinkat,
|
||||
},
|
||||
};
|
||||
|
||||
const FINAL_NAMESPACE: u8 = 0;
|
||||
const QUARANTINE_NAMESPACE: u8 = 1;
|
||||
|
||||
/// Opaque bounded-scan continuation, valid only for an unchanged namespace.
|
||||
/// Discard it after any put, quarantine, delete, or external mutation.
|
||||
pub struct ReconciliationCursor {
|
||||
root_dev: u64,
|
||||
root_ino: u64,
|
||||
namespace_start: u8,
|
||||
namespace: u8,
|
||||
namespace_end: u8,
|
||||
shard: u8,
|
||||
state: ReconciliationCursorState,
|
||||
}
|
||||
@@ -67,29 +67,17 @@ impl fmt::Debug for ReconciliationCursor {
|
||||
/// Opaque inode-bound evidence that exposes neither digest nor filesystem path.
|
||||
#[derive(Clone)]
|
||||
pub struct ReconciliationCandidate {
|
||||
root_dev: u64,
|
||||
root_ino: u64,
|
||||
namespace: ReconciliationNamespace,
|
||||
shard: String,
|
||||
shard_dev: u64,
|
||||
shard_ino: u64,
|
||||
name: String,
|
||||
dev: u64,
|
||||
ino: u64,
|
||||
modified_seconds: i64,
|
||||
modified_nanoseconds: i64,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ReconciliationCandidate {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ReconciliationCandidate(..)")
|
||||
}
|
||||
}
|
||||
|
||||
impl ReconciliationCandidate {
|
||||
pub fn namespace(&self) -> ReconciliationNamespace {
|
||||
self.namespace
|
||||
}
|
||||
pub(crate) root_dev: u64,
|
||||
pub(crate) root_ino: u64,
|
||||
pub(crate) namespace: ReconciliationNamespace,
|
||||
pub(crate) shard: String,
|
||||
pub(crate) shard_dev: u64,
|
||||
pub(crate) shard_ino: u64,
|
||||
pub(crate) name: String,
|
||||
pub(crate) dev: u64,
|
||||
pub(crate) ino: u64,
|
||||
pub(crate) modified_seconds: i64,
|
||||
pub(crate) modified_nanoseconds: i64,
|
||||
}
|
||||
|
||||
/// Opaque single-use evidence for a stale temporary inode under the pinned root.
|
||||
@@ -113,25 +101,31 @@ pub struct TempScan {
|
||||
}
|
||||
|
||||
impl ArtifactStore {
|
||||
/// Streams entries without disclosing paths/digests, with opaque continuation.
|
||||
pub fn scan_reconciliation(
|
||||
pub(crate) fn scan_reconciliation_bounded(
|
||||
&self,
|
||||
start_namespace: ReconciliationNamespace,
|
||||
end_namespace: ReconciliationNamespace,
|
||||
continuation: Option<ReconciliationCursor>,
|
||||
scan_budget: usize,
|
||||
result_limit: usize,
|
||||
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
|
||||
let root = self.root()?;
|
||||
let namespace = namespace_number(start_namespace);
|
||||
let namespace_end = namespace_number(end_namespace);
|
||||
let mut cursor = match continuation {
|
||||
Some(cursor) => {
|
||||
if cursor.root_dev != root.dev
|
||||
|| cursor.root_ino != root.ino
|
||||
|| cursor.namespace > QUARANTINE_NAMESPACE
|
||||
|| cursor.namespace_start != namespace
|
||||
|| cursor.namespace < namespace
|
||||
|| cursor.namespace > cursor.namespace_end
|
||||
|| cursor.namespace_end != namespace_end
|
||||
{
|
||||
return Err(ArtifactError::UnsafeRoot);
|
||||
}
|
||||
cursor
|
||||
}
|
||||
None => reconciliation_cursor(root.dev, root.ino),
|
||||
None => reconciliation_cursor(root.dev, root.ino, namespace, namespace_end),
|
||||
};
|
||||
let report = ReconciliationScan::empty(None);
|
||||
let _lock = match RootLock::shared(root) {
|
||||
@@ -177,7 +171,7 @@ impl ArtifactStore {
|
||||
}
|
||||
|
||||
let mut entries = Vec::new();
|
||||
while cursor.namespace <= QUARANTINE_NAMESPACE {
|
||||
while cursor.namespace <= cursor.namespace_end {
|
||||
if entries.len() == result_limit {
|
||||
return Ok(reconciliation_page(
|
||||
report,
|
||||
@@ -438,11 +432,18 @@ impl ArtifactStore {
|
||||
}
|
||||
}
|
||||
|
||||
fn reconciliation_cursor(root_dev: u64, root_ino: u64) -> ReconciliationCursor {
|
||||
fn reconciliation_cursor(
|
||||
root_dev: u64,
|
||||
root_ino: u64,
|
||||
namespace: u8,
|
||||
namespace_end: u8,
|
||||
) -> ReconciliationCursor {
|
||||
ReconciliationCursor {
|
||||
root_dev,
|
||||
root_ino,
|
||||
namespace: FINAL_NAMESPACE,
|
||||
namespace_start: namespace,
|
||||
namespace,
|
||||
namespace_end,
|
||||
shard: 0,
|
||||
state: ReconciliationCursorState::OpenNamespace,
|
||||
}
|
||||
@@ -865,14 +866,6 @@ fn parse_dirent(buffer: &[u8], offset: usize) -> Result<(Vec<u8>, usize, i64), A
|
||||
Ok((raw_name[..end].to_vec(), after, cookie))
|
||||
}
|
||||
|
||||
fn namespace_name(namespace: u8) -> &'static [u8] {
|
||||
match namespace {
|
||||
FINAL_NAMESPACE => b"sha256",
|
||||
QUARANTINE_NAMESPACE => QUARANTINE_DIR,
|
||||
_ => unreachable!("validated reconciliation namespace"),
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_reconciliation_shard(cursor: &mut ReconciliationCursor, namespace: OwnedFd) {
|
||||
if cursor.shard == u8::MAX {
|
||||
advance_namespace(cursor);
|
||||
@@ -883,12 +876,16 @@ fn advance_reconciliation_shard(cursor: &mut ReconciliationCursor, namespace: Ow
|
||||
}
|
||||
|
||||
fn advance_namespace(cursor: &mut ReconciliationCursor) {
|
||||
cursor.namespace = cursor.namespace.saturating_add(1);
|
||||
cursor.namespace = if cursor.namespace >= cursor.namespace_end {
|
||||
cursor.namespace_end.saturating_add(1)
|
||||
} else {
|
||||
cursor.namespace + 1
|
||||
};
|
||||
cursor.shard = 0;
|
||||
cursor.state = ReconciliationCursorState::OpenNamespace;
|
||||
}
|
||||
|
||||
fn ensure_reconciliation_root(
|
||||
pub(crate) fn ensure_reconciliation_root(
|
||||
root: &crate::store::Root,
|
||||
candidate: &ReconciliationCandidate,
|
||||
) -> Result<(), ArtifactError> {
|
||||
@@ -904,7 +901,7 @@ fn ensure_reconciliation_root(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn revalidate_candidate(
|
||||
pub(crate) fn revalidate_candidate(
|
||||
candidate: &ReconciliationCandidate,
|
||||
shard: i32,
|
||||
) -> Result<(), ArtifactError> {
|
||||
|
||||
@@ -11,6 +11,7 @@ mod error;
|
||||
mod housekeeping;
|
||||
mod legacy;
|
||||
mod model;
|
||||
mod recovery;
|
||||
mod store;
|
||||
mod temp_scan;
|
||||
|
||||
@@ -22,7 +23,7 @@ pub use error::ArtifactError;
|
||||
pub use housekeeping::{ReconciliationCandidate, ReconciliationCursor, StaleTemp, TempScan};
|
||||
pub use model::{
|
||||
ArtifactRef, MAX_ARTIFACT_BYTES, MAX_SOURCE_BYTES, ReconciliationMutation,
|
||||
ReconciliationNamespace, ReconciliationScan, ReconciliationScanStop, RegisteredArtifact,
|
||||
StoredArtifact,
|
||||
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
|
||||
ReconciliationScan, ReconciliationScanStop, RegisteredArtifact, StoredArtifact,
|
||||
};
|
||||
pub use store::ArtifactStore;
|
||||
|
||||
@@ -77,6 +77,66 @@ pub struct RegisteredArtifact {
|
||||
pub(crate) size_bytes: usize,
|
||||
}
|
||||
|
||||
/// Result of validating a scanned reconciliation candidate for registry use.
|
||||
///
|
||||
/// A candidate that is younger than the configured grace period never yields
|
||||
/// a registration capability. The enum is intentionally redacted by its
|
||||
/// `Debug` implementation so an eligibility report cannot disclose the
|
||||
/// physical artifact identity.
|
||||
#[derive(Clone)]
|
||||
pub enum ReconciliationRegistration {
|
||||
NotEligible,
|
||||
Registered(RegisteredArtifact),
|
||||
}
|
||||
|
||||
/// Redacted physical state observed by an expired-claim recovery probe.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ReconciliationPresence {
|
||||
Final,
|
||||
Quarantine,
|
||||
Absent,
|
||||
Unsafe,
|
||||
Retryable,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ReconciliationRegistration {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ReconciliationRegistration(..)")
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for ReconciliationRegistration {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
match (self, other) {
|
||||
(Self::NotEligible, Self::NotEligible) => true,
|
||||
(Self::Registered(left), Self::Registered(right)) => left == right,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for ReconciliationRegistration {}
|
||||
|
||||
impl ReconciliationRegistration {
|
||||
pub fn is_eligible(&self) -> bool {
|
||||
matches!(self, Self::Registered(_))
|
||||
}
|
||||
|
||||
pub fn artifact(&self) -> Option<&RegisteredArtifact> {
|
||||
match self {
|
||||
Self::NotEligible => None,
|
||||
Self::Registered(artifact) => Some(artifact),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_artifact(self) -> Option<RegisteredArtifact> {
|
||||
match self {
|
||||
Self::NotEligible => None,
|
||||
Self::Registered(artifact) => Some(artifact),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Namespace in which reconciliation observed an artifact inode. It does not
|
||||
/// disclose the artifact's digest or filesystem location.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
use std::{
|
||||
fmt,
|
||||
os::fd::{AsRawFd, OwnedFd},
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
ArtifactError, ArtifactRef, ArtifactStore, ReconciliationCandidate, ReconciliationCursor,
|
||||
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
|
||||
ReconciliationScan,
|
||||
housekeeping::{ensure_reconciliation_root, revalidate_candidate},
|
||||
store::{
|
||||
QUARANTINE_DIR, Root, RootLock, checkpointed, ensure_root_unchanged, open_existing_dir,
|
||||
open_existing_file, read_verified, stat_fd,
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) const FINAL_NAMESPACE: u8 = 0;
|
||||
pub(crate) const QUARANTINE_NAMESPACE: u8 = 1;
|
||||
|
||||
pub(crate) fn namespace_name(namespace: u8) -> &'static [u8] {
|
||||
match namespace {
|
||||
FINAL_NAMESPACE => b"sha256",
|
||||
QUARANTINE_NAMESPACE => QUARANTINE_DIR,
|
||||
_ => unreachable!("validated reconciliation namespace"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_number(namespace: ReconciliationNamespace) -> u8 {
|
||||
match namespace {
|
||||
ReconciliationNamespace::Final => FINAL_NAMESPACE,
|
||||
ReconciliationNamespace::Quarantine => QUARANTINE_NAMESPACE,
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ReconciliationCandidate {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ReconciliationCandidate(..)")
|
||||
}
|
||||
}
|
||||
|
||||
impl ReconciliationCandidate {
|
||||
pub fn namespace(&self) -> ReconciliationNamespace {
|
||||
self.namespace
|
||||
}
|
||||
|
||||
/// Returns whether the candidate has aged past `grace` at `now`.
|
||||
///
|
||||
/// This only reports the timestamp captured by the scanner. Callers that
|
||||
/// need a registration capability must use [`ArtifactStore::register_reconciliation`],
|
||||
/// which repeats inode and content validation while holding the store lock.
|
||||
pub fn is_grace_eligible(&self, grace: Duration, now: SystemTime) -> bool {
|
||||
let modified = UNIX_EPOCH
|
||||
.checked_add(Duration::new(
|
||||
self.modified_seconds.max(0) as u64,
|
||||
self.modified_nanoseconds.max(0) as u32,
|
||||
))
|
||||
.unwrap_or(SystemTime::UNIX_EPOCH);
|
||||
now.duration_since(modified).is_ok_and(|age| age >= grace)
|
||||
}
|
||||
}
|
||||
|
||||
impl ArtifactStore {
|
||||
/// Scans only one reconciliation namespace with a bounded opaque cursor.
|
||||
///
|
||||
/// A quarantine scan ends after quarantine and never falls through to
|
||||
/// final entries, so recovery can finish before the normal final sweep.
|
||||
pub fn scan_reconciliation_namespace(
|
||||
&self,
|
||||
namespace: ReconciliationNamespace,
|
||||
continuation: Option<ReconciliationCursor>,
|
||||
scan_budget: usize,
|
||||
result_limit: usize,
|
||||
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
|
||||
self.scan_reconciliation_bounded(
|
||||
namespace,
|
||||
namespace,
|
||||
continuation,
|
||||
scan_budget,
|
||||
result_limit,
|
||||
)
|
||||
}
|
||||
|
||||
/// Streams entries without disclosing paths/digests, with opaque continuation.
|
||||
pub fn scan_reconciliation(
|
||||
&self,
|
||||
continuation: Option<ReconciliationCursor>,
|
||||
scan_budget: usize,
|
||||
result_limit: usize,
|
||||
) -> Result<(ReconciliationScan, Vec<ReconciliationCandidate>), ArtifactError> {
|
||||
self.scan_reconciliation_bounded(
|
||||
ReconciliationNamespace::Final,
|
||||
ReconciliationNamespace::Quarantine,
|
||||
continuation,
|
||||
scan_budget,
|
||||
result_limit,
|
||||
)
|
||||
}
|
||||
|
||||
/// Revalidates a scanned candidate and mints a registration capability
|
||||
/// only for a file that is old enough and whose bytes still match its
|
||||
/// canonical name. The candidate remains usable for a later mutation.
|
||||
pub fn register_reconciliation(
|
||||
&self,
|
||||
candidate: &ReconciliationCandidate,
|
||||
grace: Duration,
|
||||
now: SystemTime,
|
||||
) -> Result<ReconciliationRegistration, ArtifactError> {
|
||||
let root = self.root()?;
|
||||
let _lock = RootLock::shared(root)?;
|
||||
ensure_reconciliation_root(root, candidate)?;
|
||||
let namespace = namespace_number(candidate.namespace);
|
||||
let namespace_root = open_existing_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
|
||||
let shard = open_existing_dir(namespace_root.as_raw_fd(), candidate.shard.as_bytes())?;
|
||||
revalidate_candidate(candidate, shard.as_raw_fd())?;
|
||||
|
||||
let file = open_existing_file(shard.as_raw_fd(), candidate.name.as_bytes())?;
|
||||
let stat = stat_fd(file.as_raw_fd())?;
|
||||
if stat.st_dev != candidate.dev
|
||||
|| stat.st_ino != candidate.ino
|
||||
|| stat.st_mtime != candidate.modified_seconds
|
||||
|| stat.st_mtime_nsec != candidate.modified_nanoseconds
|
||||
{
|
||||
return Err(ArtifactError::UnsafeRoot);
|
||||
}
|
||||
if !candidate.is_grace_eligible(grace, now) {
|
||||
return Ok(ReconciliationRegistration::NotEligible);
|
||||
}
|
||||
|
||||
let artifact_ref =
|
||||
ArtifactRef::from_digest_hex(&candidate.name).map_err(|_| ArtifactError::UnsafeRoot)?;
|
||||
let bytes = read_verified(&file, &artifact_ref)?;
|
||||
Ok(ReconciliationRegistration::Registered(
|
||||
crate::RegisteredArtifact {
|
||||
artifact_ref,
|
||||
size_bytes: bytes.len(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
/// Convenience form that uses the current wall clock for grace checks.
|
||||
pub fn register_reconciliation_candidate(
|
||||
&self,
|
||||
candidate: &ReconciliationCandidate,
|
||||
grace: Duration,
|
||||
) -> Result<ReconciliationRegistration, ArtifactError> {
|
||||
self.register_reconciliation(candidate, grace, SystemTime::now())
|
||||
}
|
||||
|
||||
/// Probes final and quarantine for an expired claim without disclosing its
|
||||
/// physical identity. Quarantine takes precedence over final.
|
||||
pub fn reconciliation_presence(
|
||||
&self,
|
||||
artifact_ref: &ArtifactRef,
|
||||
) -> Result<ReconciliationPresence, ArtifactError> {
|
||||
let root = self.root()?;
|
||||
let _lock = match RootLock::shared(root) {
|
||||
Ok(lock) => lock,
|
||||
Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
match ensure_root_unchanged(root) {
|
||||
Ok(()) => {}
|
||||
Err(ArtifactError::Storage) => return Ok(ReconciliationPresence::Retryable),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
|
||||
match probe_namespace(root, QUARANTINE_NAMESPACE, artifact_ref) {
|
||||
Ok(Some(presence)) => return Ok(presence),
|
||||
Ok(None) => {}
|
||||
Err(presence) => return Ok(presence),
|
||||
}
|
||||
match probe_namespace(root, FINAL_NAMESPACE, artifact_ref) {
|
||||
Ok(Some(presence)) => Ok(presence),
|
||||
Ok(None) => Ok(ReconciliationPresence::Absent),
|
||||
Err(presence) => Ok(presence),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn probe_namespace(
|
||||
root: &Root,
|
||||
namespace: u8,
|
||||
artifact_ref: &ArtifactRef,
|
||||
) -> Result<Option<ReconciliationPresence>, ReconciliationPresence> {
|
||||
let namespace_root = probe_dir(root.fd.as_raw_fd(), namespace_name(namespace))?;
|
||||
let Some(namespace_root) = namespace_root else {
|
||||
return Ok(None);
|
||||
};
|
||||
let shard = probe_dir(namespace_root.as_raw_fd(), artifact_ref.shard().as_bytes())?;
|
||||
let Some(shard) = shard else {
|
||||
return Ok(None);
|
||||
};
|
||||
let file = match checkpointed("reconciliation_presence_open", || {
|
||||
open_existing_file(shard.as_raw_fd(), artifact_ref.digest_hex().as_bytes())
|
||||
}) {
|
||||
Ok(fd) => fd,
|
||||
Err(ArtifactError::NotFound) => return Ok(None),
|
||||
Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
|
||||
return Err(ReconciliationPresence::Unsafe);
|
||||
}
|
||||
Err(_) => return Err(ReconciliationPresence::Retryable),
|
||||
};
|
||||
match read_verified(&file, artifact_ref) {
|
||||
Ok(_) => Ok(Some(if namespace == FINAL_NAMESPACE {
|
||||
ReconciliationPresence::Final
|
||||
} else {
|
||||
ReconciliationPresence::Quarantine
|
||||
})),
|
||||
Err(ArtifactError::Integrity | ArtifactError::UnsafeRoot) => {
|
||||
Err(ReconciliationPresence::Unsafe)
|
||||
}
|
||||
Err(_) => Err(ReconciliationPresence::Retryable),
|
||||
}
|
||||
}
|
||||
|
||||
fn probe_dir(parent: i32, name: &[u8]) -> Result<Option<OwnedFd>, ReconciliationPresence> {
|
||||
match open_existing_dir(parent, name) {
|
||||
Ok(fd) => Ok(Some(fd)),
|
||||
Err(ArtifactError::NotFound) => Ok(None),
|
||||
Err(ArtifactError::UnsafeRoot) => Err(ReconciliationPresence::Unsafe),
|
||||
Err(_) => Err(ReconciliationPresence::Retryable),
|
||||
}
|
||||
}
|
||||
@@ -3,14 +3,15 @@ use std::{
|
||||
os::unix::fs::{MetadataExt, PermissionsExt},
|
||||
path::PathBuf,
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
use std::{process::Command, sync::Arc, thread, time::Duration};
|
||||
use std::{process::Command, sync::Arc, thread};
|
||||
|
||||
use crank_artifacts::{
|
||||
ArtifactError, ArtifactStore, ReconciliationMutation, ReconciliationNamespace,
|
||||
ReconciliationScanStop,
|
||||
ReconciliationPresence, ReconciliationRegistration, ReconciliationScanStop,
|
||||
};
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
@@ -769,3 +770,208 @@ fn crash_windows_are_recoverable() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registration_revalidates_content_and_enforces_grace_without_candidate_identity_access() {
|
||||
let root = TestRoot::new("registration");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let stored = store.put(b"reconciliation registration").unwrap();
|
||||
let (_, mut candidates) = store
|
||||
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
||||
.unwrap();
|
||||
let candidate = candidates.pop().unwrap();
|
||||
|
||||
assert_eq!(format!("{candidate:?}"), "ReconciliationCandidate(..)");
|
||||
assert!(!candidate.is_grace_eligible(Duration::from_secs(3600), SystemTime::now()));
|
||||
assert_eq!(
|
||||
format!(
|
||||
"{:?}",
|
||||
store
|
||||
.register_reconciliation(&candidate, Duration::from_secs(3600), SystemTime::now())
|
||||
.unwrap()
|
||||
),
|
||||
"ReconciliationRegistration(..)"
|
||||
);
|
||||
|
||||
let registration = store
|
||||
.register_reconciliation(
|
||||
&candidate,
|
||||
Duration::ZERO,
|
||||
SystemTime::now() + Duration::from_secs(3600),
|
||||
)
|
||||
.unwrap();
|
||||
let registered = match registration {
|
||||
ReconciliationRegistration::Registered(registered) => registered,
|
||||
ReconciliationRegistration::NotEligible => panic!("zero grace must be eligible"),
|
||||
};
|
||||
assert_eq!(registered.artifact_ref(), &stored.artifact_ref);
|
||||
assert_eq!(
|
||||
registered.size_bytes(),
|
||||
b"reconciliation registration".len()
|
||||
);
|
||||
|
||||
// The candidate remains usable for the physical mutation after the
|
||||
// registration bridge borrowed it.
|
||||
assert_eq!(
|
||||
store.quarantine_reconciliation(candidate).unwrap(),
|
||||
ReconciliationMutation::Quarantined
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scan_can_finish_quarantine_recovery_before_final_sweep() {
|
||||
let root = TestRoot::new("namespace-order");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
store.put(b"quarantine-first recovery").unwrap();
|
||||
let (_, candidates) = store
|
||||
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8)
|
||||
.unwrap();
|
||||
store
|
||||
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
||||
.unwrap();
|
||||
store.put(b"quarantine-first recovery").unwrap();
|
||||
|
||||
let (quarantine_report, quarantine_candidates) = store
|
||||
.scan_reconciliation_namespace(
|
||||
ReconciliationNamespace::Quarantine,
|
||||
None,
|
||||
FULL_SCAN_BUDGET,
|
||||
8,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(quarantine_report.stop, ReconciliationScanStop::Complete);
|
||||
assert_eq!(quarantine_report.final_entries, 0);
|
||||
assert_eq!(quarantine_report.quarantined_entries, 1);
|
||||
assert_eq!(quarantine_candidates.len(), 1);
|
||||
assert!(quarantine_report.continuation.is_none());
|
||||
|
||||
let (final_report, final_candidates) = store
|
||||
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8)
|
||||
.unwrap();
|
||||
assert_eq!(final_report.stop, ReconciliationScanStop::Complete);
|
||||
assert_eq!(final_report.final_entries, 1);
|
||||
assert_eq!(final_report.quarantined_entries, 0);
|
||||
assert_eq!(final_candidates.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scan_rejects_a_cursor_from_a_broader_scan() {
|
||||
let root = TestRoot::new("namespace-cursor-scope");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let (report, _) = store.scan_reconciliation(None, 0, 1).unwrap();
|
||||
let cursor = report
|
||||
.continuation
|
||||
.expect("zero-budget scan retains cursor");
|
||||
|
||||
assert!(matches!(
|
||||
store.scan_reconciliation_namespace(
|
||||
ReconciliationNamespace::Quarantine,
|
||||
Some(cursor),
|
||||
FULL_SCAN_BUDGET,
|
||||
1,
|
||||
),
|
||||
Err(ArtifactError::UnsafeRoot)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_scan_rejects_a_quarantine_only_cursor() {
|
||||
let root = TestRoot::new("namespace-cursor-narrow");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let (report, _) = store
|
||||
.scan_reconciliation_namespace(ReconciliationNamespace::Quarantine, None, 0, 1)
|
||||
.unwrap();
|
||||
let cursor = report
|
||||
.continuation
|
||||
.expect("zero-budget namespace scan retains cursor");
|
||||
|
||||
assert!(matches!(
|
||||
store.scan_reconciliation(Some(cursor), FULL_SCAN_BUDGET, 1),
|
||||
Err(ArtifactError::UnsafeRoot)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presence_probe_distinguishes_final_quarantine_and_absent_without_disclosure() {
|
||||
let root = TestRoot::new("presence");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let stored = store.put(b"presence probe").unwrap();
|
||||
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
|
||||
ReconciliationPresence::Final
|
||||
);
|
||||
let (_, candidates) = store
|
||||
.scan_reconciliation(None, FULL_SCAN_BUDGET, 8)
|
||||
.unwrap();
|
||||
store
|
||||
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
|
||||
ReconciliationPresence::Quarantine
|
||||
);
|
||||
// A republish can leave both locations during recovery. Quarantine takes
|
||||
// precedence so the stale inode is still scheduled for deletion.
|
||||
store.put(b"presence probe").unwrap();
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
|
||||
ReconciliationPresence::Quarantine
|
||||
);
|
||||
let (_, candidates) = store
|
||||
.scan_reconciliation_namespace(
|
||||
ReconciliationNamespace::Quarantine,
|
||||
None,
|
||||
FULL_SCAN_BUDGET,
|
||||
8,
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.delete_quarantined_reconciliation(candidates.into_iter().next().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
|
||||
ReconciliationPresence::Final
|
||||
);
|
||||
let (_, candidates) = store
|
||||
.scan_reconciliation_namespace(ReconciliationNamespace::Final, None, FULL_SCAN_BUDGET, 8)
|
||||
.unwrap();
|
||||
store
|
||||
.quarantine_reconciliation(candidates.into_iter().next().unwrap())
|
||||
.unwrap();
|
||||
let (_, candidates) = store
|
||||
.scan_reconciliation_namespace(
|
||||
ReconciliationNamespace::Quarantine,
|
||||
None,
|
||||
FULL_SCAN_BUDGET,
|
||||
8,
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.delete_quarantined_reconciliation(candidates.into_iter().next().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
|
||||
ReconciliationPresence::Absent
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
#[test]
|
||||
fn presence_probe_classifies_storage_failure_as_retryable() {
|
||||
let _guard = fault_guard();
|
||||
let root = TestRoot::new("presence-retryable");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let stored = store.put(b"presence retryable").unwrap();
|
||||
|
||||
set_checkpoint("reconciliation_presence_open", FaultAction::Fail);
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
|
||||
ReconciliationPresence::Retryable
|
||||
);
|
||||
clear_checkpoint();
|
||||
assert_eq!(
|
||||
store.reconciliation_presence(&stored.artifact_ref).unwrap(),
|
||||
ReconciliationPresence::Final
|
||||
);
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ crank-artifacts = { path = "../crank-artifacts" }
|
||||
crank-core = { path = "../crank-core" }
|
||||
crank-mapping = { path = "../crank-mapping" }
|
||||
crank-schema = { path = "../crank-schema" }
|
||||
rand.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
@@ -156,6 +156,8 @@ pub enum RegistryError {
|
||||
SourceUnavailable,
|
||||
#[error("artifact source integrity verification failed")]
|
||||
SourceIntegrity,
|
||||
#[error("artifact reconciliation is already claimed")]
|
||||
ArtifactClaimInProgress,
|
||||
#[error("artifact source metadata is invalid for field {field}")]
|
||||
InvalidArtifactSource { field: &'static str },
|
||||
}
|
||||
|
||||
@@ -15,28 +15,32 @@ pub mod records {
|
||||
pub use crate::model::{
|
||||
AdminBootstrapContractRecord, AgentSummary, AgentVersionRecord, AppendProductEventOutcome,
|
||||
AppliedImportOperation, ApprovalRequestRecord, ArtifactBlobLifecycle, ArtifactBlobRecord,
|
||||
ArtifactDigest, ArtifactSourceCursor, ArtifactSourceId, ArtifactSourceLifecycle,
|
||||
ArtifactSourcePage, ArtifactSourceRecord, ArtifactSourceSensitivity, AuthUserRecord,
|
||||
DescriptorKind, DescriptorMetadata, ImportJob, ImportJobApplyResult, ImportJobId,
|
||||
ImportJobKind, ImportJobStatus, InvitationRecord, InvocationHistoryLoss,
|
||||
InvocationHistoryLossCategory, InvocationHistoryWriteOutcome, InvocationLogRecord,
|
||||
InvocationRetentionOutcome, InvocationRetentionPolicy, InvocationRetentionStatus,
|
||||
MasterKeyIdentityRecord, MasterKeyRotationRecord, MasterKeyRotationStatus,
|
||||
MembershipRecord, OnboardingMilestoneResult, OnboardingPresentationMilestone,
|
||||
OperationAgentRef, OperationSampleMetadata, OperationSummary, OperationUsageSummary,
|
||||
OperationVersionRecord, Page, PlatformApiKeyRecord, ProductEventRecord,
|
||||
PublishedAgentCatalog, PublishedAgentTool, RegistryOperation, SampleKind, SecretRecord,
|
||||
SecretVersionRecord, SessionRecord, SkippedImportOperation, UsageAgentBreakdown,
|
||||
UsageBucket, UsageOperationBreakdown, UsageOutcomeBreakdown, UsageOutcomeGroup,
|
||||
UsageRollupRecord, UsageSummary, UsageTimelinePoint, VerifiedArtifactSource,
|
||||
WorkspaceMembershipRecord, WorkspaceRecord, WorkspaceUpstream, WorkspaceUpstreamId,
|
||||
YamlImportJob, YamlImportJobCompletion, YamlImportJobId, YamlImportJobStatus,
|
||||
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
|
||||
ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactDigest, ArtifactExpiredClaimProbe,
|
||||
ArtifactReconciliationClaim, ArtifactSourceCursor, ArtifactSourceId,
|
||||
ArtifactSourceLifecycle, ArtifactSourcePage, ArtifactSourceRecord,
|
||||
ArtifactSourceSensitivity, AuthUserRecord, DescriptorKind, DescriptorMetadata, ImportJob,
|
||||
ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobStatus, InvitationRecord,
|
||||
InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
|
||||
InvocationLogRecord, InvocationRetentionOutcome, InvocationRetentionPolicy,
|
||||
InvocationRetentionStatus, MasterKeyIdentityRecord, MasterKeyRotationRecord,
|
||||
MasterKeyRotationStatus, MembershipRecord, OnboardingMilestoneResult,
|
||||
OnboardingPresentationMilestone, OperationAgentRef, OperationSampleMetadata,
|
||||
OperationSummary, OperationUsageSummary, OperationVersionRecord, Page,
|
||||
PlatformApiKeyRecord, ProductEventRecord, PublishedAgentCatalog, PublishedAgentTool,
|
||||
RegistryOperation, SampleKind, SecretRecord, SecretVersionRecord, SessionRecord,
|
||||
SkippedImportOperation, UsageAgentBreakdown, UsageBucket, UsageOperationBreakdown,
|
||||
UsageOutcomeBreakdown, UsageOutcomeGroup, UsageRollupRecord, UsageSummary,
|
||||
UsageTimelinePoint, VerifiedArtifactSource, WorkspaceMembershipRecord, WorkspaceRecord,
|
||||
WorkspaceUpstream, WorkspaceUpstreamId, YamlImportJob, YamlImportJobCompletion,
|
||||
YamlImportJobId, YamlImportJobStatus,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod requests {
|
||||
pub use crate::model::{
|
||||
AdminSecurityAuditRequest, AppendProductEventRequest, ApplyImportJobRequest,
|
||||
ClaimArtifactReconciliationRequest, ClaimExpiredArtifactReconciliationRequest,
|
||||
ConsumeAdminBootstrapContractRequest, CreateAdminBootstrapContractRequest,
|
||||
CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest,
|
||||
CreateArtifactSourceRequest, CreateImportJobRequest, CreateInvitationRequest,
|
||||
@@ -67,9 +71,12 @@ pub use model::{
|
||||
AdminBootstrapContractRecord, AdminSecurityAuditRequest, AgentStateExpectation, AgentSummary,
|
||||
AgentVersionRecord, AppendProductEventOutcome, AppendProductEventRequest,
|
||||
AppliedImportOperation, ApplyImportJobRequest, ApprovalRequestRecord, ArtifactBlobLifecycle,
|
||||
ArtifactBlobRecord, ArtifactDigest, ArtifactSourceCursor, ArtifactSourceId,
|
||||
ArtifactBlobRecord, ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome,
|
||||
ArtifactClaimOutcome, ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactDigest,
|
||||
ArtifactExpiredClaimProbe, ArtifactReconciliationClaim, ArtifactSourceCursor, ArtifactSourceId,
|
||||
ArtifactSourceLifecycle, ArtifactSourcePage, ArtifactSourceRecord, ArtifactSourceSensitivity,
|
||||
AuthUserRecord, ConsumeAdminBootstrapContractRequest, CreateAdminBootstrapContractRequest,
|
||||
AuthUserRecord, ClaimArtifactReconciliationRequest, ClaimExpiredArtifactReconciliationRequest,
|
||||
ConsumeAdminBootstrapContractRequest, CreateAdminBootstrapContractRequest,
|
||||
CreateAgentDraftVersionRequest, CreateAgentRequest, CreateApprovalRequest,
|
||||
CreateArtifactSourceRequest, CreateImportJobRequest, CreateInvitationRequest,
|
||||
CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest,
|
||||
@@ -81,12 +88,13 @@ pub use model::{
|
||||
InvocationHistoryWriteOutcome, InvocationLogRecord, InvocationRetentionOutcome,
|
||||
InvocationRetentionPolicy, InvocationRetentionStatus, ListApprovalRequestsQuery,
|
||||
ListArtifactSourcesQuery, ListInvocationLogsQuery, ListProductEventsQuery,
|
||||
MASTER_KEY_CIPHER_CONTRACT, MAX_ARTIFACT_SOURCE_PAGE_SIZE, MasterKeyIdentityCandidate,
|
||||
MasterKeyIdentityRecord, MasterKeyRotationRecord, MasterKeyRotationStatus, MembershipRecord,
|
||||
OnboardingMilestoneResult, OnboardingPresentationMilestone, OperationAgentRef,
|
||||
OperationSampleMetadata, OperationStateExpectation, OperationSummary, OperationUsageSummary,
|
||||
OperationVersionRecord, Page, PlatformApiKeyRecord, ProductEventRecord, PublishAgentRequest,
|
||||
PublishRequest, PublishedAgentCatalog, PublishedAgentTool, RecordOnboardingCompletionRequest,
|
||||
MASTER_KEY_CIPHER_CONTRACT, MAX_ARTIFACT_CLAIM_RECOVERY_BATCH, MAX_ARTIFACT_SOURCE_PAGE_SIZE,
|
||||
MasterKeyIdentityCandidate, MasterKeyIdentityRecord, MasterKeyRotationRecord,
|
||||
MasterKeyRotationStatus, MembershipRecord, OnboardingMilestoneResult,
|
||||
OnboardingPresentationMilestone, OperationAgentRef, OperationSampleMetadata,
|
||||
OperationStateExpectation, OperationSummary, OperationUsageSummary, OperationVersionRecord,
|
||||
Page, PlatformApiKeyRecord, ProductEventRecord, PublishAgentRequest, PublishRequest,
|
||||
PublishedAgentCatalog, PublishedAgentTool, RecordOnboardingCompletionRequest,
|
||||
RecordOnboardingMilestoneRequest, RecoverAdminPasswordRequest, RegistryOperation,
|
||||
RotateSecretRequest, SampleKind, SaveAgentBindingsRequest, SaveAgentCatalogConfigRequest,
|
||||
SaveAuthProfileRequest, SaveDescriptorMetadataRequest, SaveSampleMetadataRequest,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use std::fmt;
|
||||
|
||||
use crank_artifacts::{ArtifactRef, RegisteredArtifact};
|
||||
use crank_core::WorkspaceId;
|
||||
use rand::random;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -7,6 +10,189 @@ define_registry_id!(ArtifactSourceId);
|
||||
define_registry_id!(ArtifactDigest);
|
||||
|
||||
pub const MAX_ARTIFACT_SOURCE_PAGE_SIZE: u32 = 100;
|
||||
pub const MAX_ARTIFACT_CLAIM_RECOVERY_BATCH: u32 = 32;
|
||||
const ARTIFACT_CLAIM_TOKEN_BYTES: usize = 32;
|
||||
|
||||
/// An unpredictable, bounded fencing token for one reconciliation attempt.
|
||||
///
|
||||
/// It is intentionally opaque: callers can retain it in an
|
||||
/// [`ArtifactReconciliationClaim`], but cannot format or extract it for logs,
|
||||
/// metrics, or another process.
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ArtifactClaimToken(String);
|
||||
|
||||
impl ArtifactClaimToken {
|
||||
pub fn generate() -> Self {
|
||||
let bytes = random::<[u8; ARTIFACT_CLAIM_TOKEN_BYTES]>();
|
||||
let mut value = String::with_capacity(ARTIFACT_CLAIM_TOKEN_BYTES * 2);
|
||||
for byte in bytes {
|
||||
use std::fmt::Write as _;
|
||||
|
||||
let _ = write!(value, "{byte:02x}");
|
||||
}
|
||||
Self(value)
|
||||
}
|
||||
|
||||
pub(crate) fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ArtifactClaimToken {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ArtifactClaimToken(..)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque transient authority to recheck and finalize a reconciliation item.
|
||||
///
|
||||
/// It has no `Display` implementation and its `Debug` form is redacted so a
|
||||
/// durable claim token cannot cross an observability boundary by accident.
|
||||
#[derive(Clone)]
|
||||
pub struct ArtifactReconciliationClaim {
|
||||
artifact_ref: ArtifactRef,
|
||||
token: ArtifactClaimToken,
|
||||
}
|
||||
|
||||
impl ArtifactReconciliationClaim {
|
||||
pub(crate) fn new(artifact_ref: ArtifactRef, token: ArtifactClaimToken) -> Self {
|
||||
Self {
|
||||
artifact_ref,
|
||||
token,
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque content-addressed identity for a trusted artifact-store probe.
|
||||
///
|
||||
/// This must not be formatted into telemetry or an external response.
|
||||
pub fn artifact_ref(&self) -> &ArtifactRef {
|
||||
&self.artifact_ref
|
||||
}
|
||||
|
||||
pub(crate) fn token(&self) -> &ArtifactClaimToken {
|
||||
&self.token
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ArtifactReconciliationClaim {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ArtifactReconciliationClaim(..)")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ClaimArtifactReconciliationRequest<'a> {
|
||||
pub artifact: &'a RegisteredArtifact,
|
||||
pub token: ArtifactClaimToken,
|
||||
pub claimed_at: OffsetDateTime,
|
||||
pub lease_expires_at: OffsetDateTime,
|
||||
pub detached_grace: time::Duration,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ClaimArtifactReconciliationRequest<'_> {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ClaimArtifactReconciliationRequest(..)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded recovery lease acquisition for one expired durable claim.
|
||||
///
|
||||
/// Recovery needs no `RegisteredArtifact`: it recovers an identity validated
|
||||
/// when the previous claim was acquired. The caller must still revalidate
|
||||
/// physical presence through the artifact store before finalization.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ClaimExpiredArtifactReconciliationRequest {
|
||||
pub token: ArtifactClaimToken,
|
||||
pub claimed_at: OffsetDateTime,
|
||||
pub lease_expires_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
/// Opaque compare-and-swap evidence for an expired claim discovered by a
|
||||
/// bounded recovery read. It does not acquire a lease by itself.
|
||||
#[derive(Clone)]
|
||||
pub struct ArtifactExpiredClaimProbe {
|
||||
artifact_ref: ArtifactRef,
|
||||
prior_token: String,
|
||||
claim_expires_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
impl ArtifactExpiredClaimProbe {
|
||||
pub(crate) fn new(
|
||||
artifact_ref: ArtifactRef,
|
||||
prior_token: String,
|
||||
claim_expires_at: OffsetDateTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
artifact_ref,
|
||||
prior_token,
|
||||
claim_expires_at,
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque content-addressed identity for a trusted artifact-store probe.
|
||||
/// This must not be formatted into telemetry or an external response.
|
||||
pub fn artifact_ref(&self) -> &ArtifactRef {
|
||||
&self.artifact_ref
|
||||
}
|
||||
|
||||
pub(crate) fn prior_token(&self) -> &str {
|
||||
&self.prior_token
|
||||
}
|
||||
|
||||
pub(crate) fn claim_expires_at(&self) -> OffsetDateTime {
|
||||
self.claim_expires_at
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for ArtifactExpiredClaimProbe {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("ArtifactExpiredClaimProbe(..)")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum ArtifactClaimOutcome {
|
||||
/// This caller owns the fenced claim and may perform the next recheck.
|
||||
Claimed(ArtifactReconciliationClaim),
|
||||
/// A current token already owns the artifact; retry after its lease.
|
||||
HeldByOther,
|
||||
/// The artifact is still referenced by at least one active source.
|
||||
ActiveReference,
|
||||
/// A detached source remains within the caller's grace window.
|
||||
DetachedReferenceInGrace,
|
||||
/// A row with the same digest does not match store-validated metadata.
|
||||
MetadataConflict,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ArtifactClaimRecheckOutcome {
|
||||
/// The caller still owns the claim and no global reference blocks I/O.
|
||||
Mutate,
|
||||
/// An active relation now protects the artifact.
|
||||
ActiveReference,
|
||||
/// A detached relation is still protected by grace.
|
||||
DetachedReferenceInGrace,
|
||||
/// The token was finalized, replaced, or expired.
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ArtifactClaimFinalization {
|
||||
Deleted,
|
||||
AlreadyAbsent,
|
||||
Quarantined,
|
||||
Retryable,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ArtifactClaimFinalizeOutcome {
|
||||
/// Confirmed absence was durably recorded as V12 `unavailable`.
|
||||
Unavailable,
|
||||
/// An ambiguous/mid-flight mutation remains fenced for recovery.
|
||||
Retained,
|
||||
/// The supplied token no longer owns the row.
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
|
||||
@@ -3,19 +3,373 @@ use crank_core::WorkspaceId;
|
||||
use sqlx::{Postgres, Row, Transaction, postgres::PgRow};
|
||||
|
||||
use crate::{
|
||||
ArtifactBlobLifecycle, ArtifactBlobRecord, ArtifactDigest, ArtifactSourceCursor,
|
||||
ArtifactBlobLifecycle, ArtifactBlobRecord, ArtifactClaimFinalization,
|
||||
ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome, ArtifactClaimRecheckOutcome,
|
||||
ArtifactDigest, ArtifactExpiredClaimProbe, ArtifactReconciliationClaim, ArtifactSourceCursor,
|
||||
ArtifactSourceId, ArtifactSourceLifecycle, ArtifactSourcePage, ArtifactSourceRecord,
|
||||
ArtifactSourceSensitivity, CreateArtifactSourceRequest, DetachArtifactSourceRequest,
|
||||
ListArtifactSourcesQuery, MAX_ARTIFACT_SOURCE_PAGE_SIZE, RegistryError, VerifiedArtifactSource,
|
||||
ArtifactSourceSensitivity, ClaimArtifactReconciliationRequest,
|
||||
ClaimExpiredArtifactReconciliationRequest, CreateArtifactSourceRequest,
|
||||
DetachArtifactSourceRequest, ListArtifactSourcesQuery, MAX_ARTIFACT_CLAIM_RECOVERY_BATCH,
|
||||
MAX_ARTIFACT_SOURCE_PAGE_SIZE, RegistryError, VerifiedArtifactSource,
|
||||
};
|
||||
|
||||
use super::PostgresRegistry;
|
||||
|
||||
mod reconciliation;
|
||||
use reconciliation::{
|
||||
checked_cutoff, database_now, insert_blob_if_missing_at_database_time, validate_claim_lease,
|
||||
};
|
||||
|
||||
const SOURCE_ID_PREFIX: &str = "src_";
|
||||
const MAX_SOURCE_ID_SUFFIX_BYTES: usize = 128;
|
||||
const MAX_MIME_TYPE_BYTES: usize = 255;
|
||||
|
||||
impl PostgresRegistry {
|
||||
/// Acquires a short, fenced reconciliation lease after checking references
|
||||
/// across every workspace. A lease may be replayed by its original token,
|
||||
/// or stolen only after expiry.
|
||||
pub async fn claim_artifact_reconciliation(
|
||||
&self,
|
||||
request: ClaimArtifactReconciliationRequest<'_>,
|
||||
) -> Result<ArtifactClaimOutcome, RegistryError> {
|
||||
let lease_duration = validate_claim_lease(request.claimed_at, request.lease_expires_at)?;
|
||||
if request.detached_grace.is_negative() {
|
||||
return Err(RegistryError::InvalidArtifactSource {
|
||||
field: "detached_grace",
|
||||
});
|
||||
}
|
||||
if !valid_artifact_size(request.artifact.size_bytes()) {
|
||||
return Err(RegistryError::InvalidArtifactSource {
|
||||
field: "size_bytes",
|
||||
});
|
||||
}
|
||||
|
||||
let mut transaction = self.pool().begin().await?;
|
||||
let artifact_ref = request.artifact.artifact_ref();
|
||||
let size_bytes = i64::try_from(request.artifact.size_bytes()).map_err(|_| {
|
||||
RegistryError::InvalidArtifactSource {
|
||||
field: "size_bytes",
|
||||
}
|
||||
})?;
|
||||
insert_blob_if_missing_at_database_time(&mut transaction, artifact_ref, size_bytes).await?;
|
||||
let row = lock_blob(&mut transaction, artifact_ref.digest_hex()).await?;
|
||||
let database_now = database_now(&mut transaction).await?;
|
||||
if !blob_metadata_matches(&row, artifact_ref, size_bytes)? {
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimOutcome::MetadataConflict);
|
||||
}
|
||||
if global_active_reference(&mut transaction, artifact_ref.digest_hex()).await? {
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimOutcome::ActiveReference);
|
||||
}
|
||||
if global_detached_reference_in_grace(
|
||||
&mut transaction,
|
||||
artifact_ref.digest_hex(),
|
||||
checked_cutoff(database_now, request.detached_grace)?,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimOutcome::DetachedReferenceInGrace);
|
||||
}
|
||||
|
||||
let claim_token = row.try_get::<Option<String>, _>("claim_token")?;
|
||||
let claim_expires_at =
|
||||
row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
|
||||
if let (Some(current_token), Some(expires_at)) = (claim_token, claim_expires_at) {
|
||||
if current_token == request.token.as_str() && expires_at > database_now {
|
||||
transaction.commit().await?;
|
||||
return Ok(ArtifactClaimOutcome::Claimed(
|
||||
ArtifactReconciliationClaim::new(artifact_ref.clone(), request.token),
|
||||
));
|
||||
}
|
||||
if expires_at > database_now || current_token == request.token.as_str() {
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimOutcome::HeldByOther);
|
||||
}
|
||||
}
|
||||
|
||||
let updated = sqlx::query(
|
||||
"update artifact_blobs
|
||||
set claim_token = $1, claim_expires_at = $2, updated_at = greatest(updated_at, $3)
|
||||
where digest = $4",
|
||||
)
|
||||
.bind(request.token.as_str())
|
||||
.bind(database_now + lease_duration)
|
||||
.bind(database_now)
|
||||
.bind(artifact_ref.digest_hex())
|
||||
.execute(&mut *transaction)
|
||||
.await?
|
||||
.rows_affected();
|
||||
debug_assert_eq!(updated, 1);
|
||||
transaction.commit().await?;
|
||||
Ok(ArtifactClaimOutcome::Claimed(
|
||||
ArtifactReconciliationClaim::new(artifact_ref.clone(), request.token),
|
||||
))
|
||||
}
|
||||
|
||||
/// Reads a bounded set of expired claims without taking their leases.
|
||||
///
|
||||
/// Recovery first probes physical presence through the artifact store. Only
|
||||
/// a confirmed absence should call `claim_expired_artifact_reconciliation`.
|
||||
/// Leaving final or quarantine entries untouched lets the ordinary scanner
|
||||
/// reclaim them with a store-validated `RegisteredArtifact`.
|
||||
pub async fn list_expired_artifact_reconciliation_probes(
|
||||
&self,
|
||||
now: time::OffsetDateTime,
|
||||
limit: u32,
|
||||
) -> Result<Vec<ArtifactExpiredClaimProbe>, RegistryError> {
|
||||
self.list_expired_artifact_reconciliation_probes_after(now, None, limit)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Continues the bounded expired-claim read after an opaque prior probe.
|
||||
///
|
||||
/// Keyset pagination prevents an unsafe or still-present first claim from
|
||||
/// starving later absent claims without persisting worker state in V12.
|
||||
pub async fn list_expired_artifact_reconciliation_probes_after(
|
||||
&self,
|
||||
now: time::OffsetDateTime,
|
||||
after: Option<&ArtifactExpiredClaimProbe>,
|
||||
limit: u32,
|
||||
) -> Result<Vec<ArtifactExpiredClaimProbe>, RegistryError> {
|
||||
let limit = limit.min(MAX_ARTIFACT_CLAIM_RECOVERY_BATCH);
|
||||
if limit == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let after_expires_at = after.map(ArtifactExpiredClaimProbe::claim_expires_at);
|
||||
let after_digest = after.map(|probe| probe.artifact_ref().digest_hex());
|
||||
let rows = sqlx::query(
|
||||
"select artifact_ref, claim_token, claim_expires_at
|
||||
from artifact_blobs
|
||||
where claim_token is not null
|
||||
and claim_expires_at <= $1
|
||||
and (
|
||||
$2::timestamptz is null
|
||||
or (claim_expires_at, digest) > ($2::timestamptz, $3::text)
|
||||
)
|
||||
order by claim_expires_at asc, digest asc
|
||||
limit $4",
|
||||
)
|
||||
.bind(now)
|
||||
.bind(after_expires_at)
|
||||
.bind(after_digest)
|
||||
.bind(i64::from(limit))
|
||||
.fetch_all(self.pool())
|
||||
.await?;
|
||||
rows.into_iter()
|
||||
.map(|row| {
|
||||
let artifact_ref = ArtifactRef::parse(&row.try_get::<String, _>("artifact_ref")?)
|
||||
.map_err(|_| RegistryError::InvalidArtifactSource {
|
||||
field: "artifact_ref",
|
||||
})?;
|
||||
let prior_token = row.try_get::<Option<String>, _>("claim_token")?.ok_or(
|
||||
RegistryError::InvalidArtifactSource {
|
||||
field: "claim_token",
|
||||
},
|
||||
)?;
|
||||
let claim_expires_at = row
|
||||
.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?
|
||||
.ok_or(RegistryError::InvalidArtifactSource {
|
||||
field: "claim_expires_at",
|
||||
})?;
|
||||
Ok(ArtifactExpiredClaimProbe::new(
|
||||
artifact_ref,
|
||||
prior_token,
|
||||
claim_expires_at,
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Compare-and-swap steals an expired probe after recovery observed absence.
|
||||
///
|
||||
/// The caller must probe physical state again after a successful claim: a
|
||||
/// concurrent scanner may have changed the inode between the read and CAS.
|
||||
pub async fn claim_expired_artifact_reconciliation(
|
||||
&self,
|
||||
probe: &ArtifactExpiredClaimProbe,
|
||||
request: ClaimExpiredArtifactReconciliationRequest,
|
||||
) -> Result<Option<ArtifactReconciliationClaim>, RegistryError> {
|
||||
let lease_duration = validate_claim_lease(request.claimed_at, request.lease_expires_at)?;
|
||||
if request.token.as_str() == probe.prior_token() {
|
||||
return Err(RegistryError::InvalidArtifactSource {
|
||||
field: "claim_token",
|
||||
});
|
||||
}
|
||||
let mut transaction = self.pool().begin().await?;
|
||||
let Some(row) =
|
||||
lock_blob_optional(&mut transaction, probe.artifact_ref().digest_hex()).await?
|
||||
else {
|
||||
transaction.rollback().await?;
|
||||
return Ok(None);
|
||||
};
|
||||
let database_now = database_now(&mut transaction).await?;
|
||||
let current_token = row.try_get::<Option<String>, _>("claim_token")?;
|
||||
let expires_at = row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
|
||||
if current_token.as_deref() != Some(probe.prior_token())
|
||||
|| expires_at.is_none_or(|value| value > database_now)
|
||||
{
|
||||
transaction.rollback().await?;
|
||||
return Ok(None);
|
||||
}
|
||||
let claimed = sqlx::query(
|
||||
"update artifact_blobs
|
||||
set claim_token = $1,
|
||||
claim_expires_at = $2,
|
||||
updated_at = greatest(updated_at, $3)
|
||||
where digest = $4
|
||||
and claim_token = $5
|
||||
and claim_expires_at <= $3",
|
||||
)
|
||||
.bind(request.token.as_str())
|
||||
.bind(database_now + lease_duration)
|
||||
.bind(database_now)
|
||||
.bind(probe.artifact_ref().digest_hex())
|
||||
.bind(probe.prior_token())
|
||||
.execute(&mut *transaction)
|
||||
.await?
|
||||
.rows_affected()
|
||||
== 1;
|
||||
transaction.commit().await?;
|
||||
Ok(claimed
|
||||
.then(|| ArtifactReconciliationClaim::new(probe.artifact_ref().clone(), request.token)))
|
||||
}
|
||||
|
||||
/// Rechecks an owned claim immediately before blocking filesystem I/O.
|
||||
///
|
||||
/// `create_artifact_source` holds the same blob lock and rejects any active
|
||||
/// claim, so a `Mutate` outcome fences the recheck-to-filesystem window.
|
||||
pub async fn recheck_artifact_reconciliation_claim(
|
||||
&self,
|
||||
claim: &ArtifactReconciliationClaim,
|
||||
_now: time::OffsetDateTime,
|
||||
detached_grace: time::Duration,
|
||||
) -> Result<ArtifactClaimRecheckOutcome, RegistryError> {
|
||||
if detached_grace.is_negative() {
|
||||
return Err(RegistryError::InvalidArtifactSource {
|
||||
field: "detached_grace",
|
||||
});
|
||||
}
|
||||
let mut transaction = self.pool().begin().await?;
|
||||
let Some(row) =
|
||||
lock_blob_optional(&mut transaction, claim.artifact_ref().digest_hex()).await?
|
||||
else {
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimRecheckOutcome::Stale);
|
||||
};
|
||||
let database_now = database_now(&mut transaction).await?;
|
||||
let current_token = row.try_get::<Option<String>, _>("claim_token")?;
|
||||
let expires_at = row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
|
||||
if current_token.as_deref() != Some(claim.token().as_str())
|
||||
|| expires_at.is_none_or(|value| value <= database_now)
|
||||
{
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimRecheckOutcome::Stale);
|
||||
}
|
||||
if global_active_reference(&mut transaction, claim.artifact_ref().digest_hex()).await? {
|
||||
transaction.commit().await?;
|
||||
return Ok(ArtifactClaimRecheckOutcome::ActiveReference);
|
||||
}
|
||||
if global_detached_reference_in_grace(
|
||||
&mut transaction,
|
||||
claim.artifact_ref().digest_hex(),
|
||||
checked_cutoff(database_now, detached_grace)?,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
transaction.commit().await?;
|
||||
return Ok(ArtifactClaimRecheckOutcome::DetachedReferenceInGrace);
|
||||
}
|
||||
transaction.commit().await?;
|
||||
Ok(ArtifactClaimRecheckOutcome::Mutate)
|
||||
}
|
||||
|
||||
/// Records a filesystem observation without performing filesystem I/O.
|
||||
///
|
||||
/// Only confirmed absence clears the claim and transitions V12 metadata to
|
||||
/// `unavailable`; every ambiguous observation deliberately retains it for
|
||||
/// a later bounded recovery attempt.
|
||||
pub async fn finalize_artifact_reconciliation_claim(
|
||||
&self,
|
||||
claim: &ArtifactReconciliationClaim,
|
||||
observation: ArtifactClaimFinalization,
|
||||
_observed_at: time::OffsetDateTime,
|
||||
) -> Result<ArtifactClaimFinalizeOutcome, RegistryError> {
|
||||
let mut transaction = self.pool().begin().await?;
|
||||
let Some(row) =
|
||||
lock_blob_optional(&mut transaction, claim.artifact_ref().digest_hex()).await?
|
||||
else {
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimFinalizeOutcome::Stale);
|
||||
};
|
||||
let database_now = database_now(&mut transaction).await?;
|
||||
let current_token = row.try_get::<Option<String>, _>("claim_token")?;
|
||||
let expires_at = row.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")?;
|
||||
if current_token.as_deref() != Some(claim.token().as_str())
|
||||
|| expires_at.is_none_or(|value| value <= database_now)
|
||||
{
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimFinalizeOutcome::Stale);
|
||||
}
|
||||
|
||||
let (lifecycle, clear_claim, outcome) = match observation {
|
||||
ArtifactClaimFinalization::Deleted | ArtifactClaimFinalization::AlreadyAbsent => (
|
||||
"unavailable",
|
||||
true,
|
||||
ArtifactClaimFinalizeOutcome::Unavailable,
|
||||
),
|
||||
ArtifactClaimFinalization::Quarantined | ArtifactClaimFinalization::Retryable => {
|
||||
("available", false, ArtifactClaimFinalizeOutcome::Retained)
|
||||
}
|
||||
};
|
||||
let updated = if clear_claim {
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set storage_lifecycle = $1, claim_token = null, claim_expires_at = null,
|
||||
updated_at = greatest(updated_at, $2)
|
||||
where digest = $3 and claim_token = $4 and claim_expires_at > $2",
|
||||
)
|
||||
.bind(lifecycle)
|
||||
.bind(database_now)
|
||||
.bind(claim.artifact_ref().digest_hex())
|
||||
.bind(claim.token().as_str())
|
||||
.execute(&mut *transaction)
|
||||
.await?
|
||||
.rows_affected()
|
||||
} else {
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set updated_at = greatest(updated_at, $1)
|
||||
where digest = $2 and claim_token = $3 and claim_expires_at > $1",
|
||||
)
|
||||
.bind(database_now)
|
||||
.bind(claim.artifact_ref().digest_hex())
|
||||
.bind(claim.token().as_str())
|
||||
.execute(&mut *transaction)
|
||||
.await?
|
||||
.rows_affected()
|
||||
};
|
||||
if updated != 1 {
|
||||
transaction.rollback().await?;
|
||||
return Ok(ArtifactClaimFinalizeOutcome::Stale);
|
||||
}
|
||||
transaction.commit().await?;
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
/// Alias used by startup recovery once its filesystem probe has produced a
|
||||
/// typed observation. The registry never performs the probe itself.
|
||||
pub async fn recover_artifact_reconciliation_claim(
|
||||
&self,
|
||||
claim: &ArtifactReconciliationClaim,
|
||||
observation: ArtifactClaimFinalization,
|
||||
recovered_at: time::OffsetDateTime,
|
||||
) -> Result<ArtifactClaimFinalizeOutcome, RegistryError> {
|
||||
self.finalize_artifact_reconciliation_claim(claim, observation, recovered_at)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn create_artifact_source(
|
||||
&self,
|
||||
request: CreateArtifactSourceRequest<'_>,
|
||||
@@ -254,6 +608,41 @@ async fn ensure_blob(
|
||||
field: "size_bytes",
|
||||
}
|
||||
})?;
|
||||
insert_blob_if_missing(
|
||||
transaction,
|
||||
request.artifact.artifact_ref(),
|
||||
size_bytes,
|
||||
request.created_at,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let row = lock_blob(transaction, digest).await?;
|
||||
if !blob_metadata_matches(&row, request.artifact.artifact_ref(), size_bytes)? {
|
||||
return Err(source_conflict(request.source_id));
|
||||
}
|
||||
if row.try_get::<Option<String>, _>("claim_token")?.is_some() {
|
||||
return Err(RegistryError::ArtifactClaimInProgress);
|
||||
}
|
||||
if row.try_get::<String, _>("storage_lifecycle")? == "unavailable" {
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set storage_lifecycle = 'available', updated_at = greatest(updated_at, $1)
|
||||
where digest = $2 and claim_token is null",
|
||||
)
|
||||
.bind(request.created_at)
|
||||
.bind(digest)
|
||||
.execute(&mut **transaction)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn insert_blob_if_missing(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
artifact_ref: &ArtifactRef,
|
||||
size_bytes: i64,
|
||||
created_at: time::OffsetDateTime,
|
||||
) -> Result<(), RegistryError> {
|
||||
sqlx::query(
|
||||
"insert into artifact_blobs (
|
||||
digest, artifact_ref, size_bytes, storage_lifecycle,
|
||||
@@ -261,25 +650,87 @@ async fn ensure_blob(
|
||||
) values ($1, $2, $3, 'available', null, null, $4, $4)
|
||||
on conflict (digest) do nothing",
|
||||
)
|
||||
.bind(digest)
|
||||
.bind(request.artifact.artifact_ref().as_str())
|
||||
.bind(artifact_ref.digest_hex())
|
||||
.bind(artifact_ref.as_str())
|
||||
.bind(size_bytes)
|
||||
.bind(request.created_at)
|
||||
.bind(created_at)
|
||||
.execute(&mut **transaction)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
let row = sqlx::query(
|
||||
"select artifact_ref, size_bytes from artifact_blobs where digest = $1 for update",
|
||||
async fn lock_blob(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
digest: &str,
|
||||
) -> Result<PgRow, RegistryError> {
|
||||
lock_blob_optional(transaction, digest)
|
||||
.await?
|
||||
.ok_or(RegistryError::InvalidArtifactSource {
|
||||
field: "artifact_blob",
|
||||
})
|
||||
}
|
||||
|
||||
async fn lock_blob_optional(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
digest: &str,
|
||||
) -> Result<Option<PgRow>, RegistryError> {
|
||||
sqlx::query(
|
||||
"select artifact_ref, size_bytes, storage_lifecycle, claim_token, claim_expires_at
|
||||
from artifact_blobs
|
||||
where digest = $1
|
||||
for update",
|
||||
)
|
||||
.bind(digest)
|
||||
.fetch_optional(&mut **transaction)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn blob_metadata_matches(
|
||||
row: &PgRow,
|
||||
artifact_ref: &ArtifactRef,
|
||||
size_bytes: i64,
|
||||
) -> Result<bool, RegistryError> {
|
||||
Ok(
|
||||
row.try_get::<String, _>("artifact_ref")? == artifact_ref.as_str()
|
||||
&& row.try_get::<i64, _>("size_bytes")? == size_bytes,
|
||||
)
|
||||
}
|
||||
|
||||
async fn global_active_reference(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
digest: &str,
|
||||
) -> Result<bool, RegistryError> {
|
||||
sqlx::query_scalar::<_, bool>(
|
||||
"select exists(
|
||||
select 1 from artifact_sources
|
||||
where blob_digest = $1 and lifecycle = 'active'
|
||||
)",
|
||||
)
|
||||
.bind(digest)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await?;
|
||||
let recorded_ref = row.try_get::<String, _>("artifact_ref")?;
|
||||
let recorded_size = row.try_get::<i64, _>("size_bytes")?;
|
||||
if recorded_ref != request.artifact.artifact_ref().as_str() || recorded_size != size_bytes {
|
||||
return Err(source_conflict(request.source_id));
|
||||
}
|
||||
Ok(())
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn global_detached_reference_in_grace(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
digest: &str,
|
||||
detached_after: time::OffsetDateTime,
|
||||
) -> Result<bool, RegistryError> {
|
||||
sqlx::query_scalar::<_, bool>(
|
||||
"select exists(
|
||||
select 1 from artifact_sources
|
||||
where blob_digest = $1
|
||||
and lifecycle = 'detached'
|
||||
and detached_at > $2
|
||||
)",
|
||||
)
|
||||
.bind(digest)
|
||||
.bind(detached_after)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn get_source_in_transaction(
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
use crank_artifacts::ArtifactRef;
|
||||
use sqlx::{Postgres, Transaction};
|
||||
|
||||
use crate::{PostgresRegistry, RegistryError};
|
||||
|
||||
const MAX_ARTIFACT_CLAIM_LEASE: time::Duration = time::Duration::minutes(5);
|
||||
|
||||
impl PostgresRegistry {
|
||||
/// Returns the PostgreSQL clock used by production reconciliation leases.
|
||||
pub async fn artifact_reconciliation_now(&self) -> Result<time::OffsetDateTime, RegistryError> {
|
||||
sqlx::query_scalar("select clock_timestamp()")
|
||||
.fetch_one(self.pool())
|
||||
.await
|
||||
.map_err(RegistryError::from)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn insert_blob_if_missing_at_database_time(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
artifact_ref: &ArtifactRef,
|
||||
size_bytes: i64,
|
||||
) -> Result<(), RegistryError> {
|
||||
sqlx::query(
|
||||
"insert into artifact_blobs (
|
||||
digest, artifact_ref, size_bytes, storage_lifecycle,
|
||||
claim_token, claim_expires_at, created_at, updated_at
|
||||
) values ($1, $2, $3, 'available', null, null, clock_timestamp(), clock_timestamp())
|
||||
on conflict (digest) do nothing",
|
||||
)
|
||||
.bind(artifact_ref.digest_hex())
|
||||
.bind(artifact_ref.as_str())
|
||||
.bind(size_bytes)
|
||||
.execute(&mut **transaction)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_claim_lease(
|
||||
claimed_at: time::OffsetDateTime,
|
||||
lease_expires_at: time::OffsetDateTime,
|
||||
) -> Result<time::Duration, RegistryError> {
|
||||
let duration = lease_expires_at - claimed_at;
|
||||
if duration <= time::Duration::ZERO || duration > MAX_ARTIFACT_CLAIM_LEASE {
|
||||
return Err(RegistryError::InvalidArtifactSource {
|
||||
field: "claim_lease",
|
||||
});
|
||||
}
|
||||
Ok(duration)
|
||||
}
|
||||
|
||||
pub(super) fn checked_cutoff(
|
||||
now: time::OffsetDateTime,
|
||||
grace: time::Duration,
|
||||
) -> Result<time::OffsetDateTime, RegistryError> {
|
||||
now.checked_sub(grace)
|
||||
.ok_or(RegistryError::InvalidArtifactSource {
|
||||
field: "detached_grace",
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn database_now(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
) -> Result<time::OffsetDateTime, RegistryError> {
|
||||
sqlx::query_scalar("select clock_timestamp()")
|
||||
.fetch_one(&mut **transaction)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
mod integration {
|
||||
mod agents_usage;
|
||||
mod approval;
|
||||
mod artifact_reconciliation;
|
||||
mod artifact_sources;
|
||||
mod common;
|
||||
mod credential_touch;
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
use std::time::Duration as StdDuration;
|
||||
|
||||
use crank_artifacts::ArtifactStore;
|
||||
use crank_registry::{
|
||||
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
|
||||
ArtifactClaimToken, ClaimArtifactReconciliationRequest,
|
||||
ClaimExpiredArtifactReconciliationRequest, RegistryError,
|
||||
};
|
||||
|
||||
use super::{
|
||||
artifact_sources::{TestRoot, timestamp},
|
||||
common::TestDatabase,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
async fn locked_database_time_and_expired_recovery_are_fenced() {
|
||||
let database = TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let raw_pool = database.raw_pool().await;
|
||||
let root = TestRoot::new("database-time");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let caller_time = timestamp("2020-01-01T00:00:00Z");
|
||||
|
||||
let blocked = store.put_registered(b"row lock expiry\n").unwrap();
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &blocked,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::seconds(1),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimOutcome::Claimed(_)
|
||||
));
|
||||
let mut blocker = raw_pool.begin().await.unwrap();
|
||||
sqlx::query("select digest from artifact_blobs where digest = $1 for update")
|
||||
.bind(blocked.artifact_ref().digest_hex())
|
||||
.fetch_one(&mut *blocker)
|
||||
.await
|
||||
.unwrap();
|
||||
let waiting_registry = registry.clone();
|
||||
let waiting_artifact = blocked.clone();
|
||||
let waiting = tokio::spawn(async move {
|
||||
waiting_registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &waiting_artifact,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::minutes(5),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::sleep(StdDuration::from_millis(1_100)).await;
|
||||
blocker.commit().await.unwrap();
|
||||
assert!(matches!(
|
||||
waiting.await.unwrap().unwrap(),
|
||||
ArtifactClaimOutcome::Claimed(_)
|
||||
));
|
||||
|
||||
let recoverable = store.put_registered(b"single winner\n").unwrap();
|
||||
let prior_token = ArtifactClaimToken::generate();
|
||||
let prior_claim = match registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &recoverable,
|
||||
token: prior_token.clone(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::minutes(5),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
ArtifactClaimOutcome::Claimed(claim) => claim,
|
||||
outcome => panic!("unexpected claim outcome: {outcome:?}"),
|
||||
};
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set claim_expires_at = clock_timestamp() - interval '1 second'
|
||||
where digest = $1",
|
||||
)
|
||||
.bind(recoverable.artifact_ref().digest_hex())
|
||||
.execute(&raw_pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &recoverable,
|
||||
token: prior_token.clone(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::minutes(5),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimOutcome::HeldByOther
|
||||
));
|
||||
let recovery_now = registry.artifact_reconciliation_now().await.unwrap();
|
||||
let probe = registry
|
||||
.list_expired_artifact_reconciliation_probes(recovery_now, 1)
|
||||
.await
|
||||
.unwrap()
|
||||
.pop()
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_expired_artifact_reconciliation(
|
||||
&probe,
|
||||
ClaimExpiredArtifactReconciliationRequest {
|
||||
token: prior_token,
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::minutes(5),
|
||||
},
|
||||
)
|
||||
.await,
|
||||
Err(RegistryError::InvalidArtifactSource {
|
||||
field: "claim_token"
|
||||
})
|
||||
));
|
||||
|
||||
let first_registry = registry.clone();
|
||||
let second_registry = registry.clone();
|
||||
let first_probe = probe.clone();
|
||||
let (first, second) = tokio::join!(
|
||||
async move {
|
||||
first_registry
|
||||
.claim_expired_artifact_reconciliation(
|
||||
&first_probe,
|
||||
ClaimExpiredArtifactReconciliationRequest {
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::minutes(5),
|
||||
},
|
||||
)
|
||||
.await
|
||||
},
|
||||
async move {
|
||||
second_registry
|
||||
.claim_expired_artifact_reconciliation(
|
||||
&probe,
|
||||
ClaimExpiredArtifactReconciliationRequest {
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::minutes(5),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
);
|
||||
let first = first.unwrap();
|
||||
let second = second.unwrap();
|
||||
assert_eq!(
|
||||
usize::from(first.is_some()) + usize::from(second.is_some()),
|
||||
1
|
||||
);
|
||||
let winner = first.or(second).unwrap();
|
||||
|
||||
sqlx::query("update artifact_blobs set claim_expires_at = clock_timestamp() where digest = $1")
|
||||
.bind(recoverable.artifact_ref().digest_hex())
|
||||
.execute(&raw_pool)
|
||||
.await
|
||||
.unwrap();
|
||||
for claim in [&winner, &prior_claim] {
|
||||
assert_eq!(
|
||||
registry
|
||||
.finalize_artifact_reconciliation_claim(
|
||||
claim,
|
||||
ArtifactClaimFinalization::AlreadyAbsent,
|
||||
caller_time,
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimFinalizeOutcome::Stale
|
||||
);
|
||||
}
|
||||
|
||||
let invalid = store.put_registered(b"invalid bounds\n").unwrap();
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &invalid,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time + time::Duration::minutes(5),
|
||||
detached_grace: time::Duration::MAX,
|
||||
})
|
||||
.await,
|
||||
Err(RegistryError::InvalidArtifactSource {
|
||||
field: "detached_grace"
|
||||
})
|
||||
));
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &invalid,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: caller_time,
|
||||
lease_expires_at: caller_time
|
||||
+ time::Duration::minutes(5)
|
||||
+ time::Duration::seconds(1),
|
||||
detached_grace: time::Duration::ZERO,
|
||||
})
|
||||
.await,
|
||||
Err(RegistryError::InvalidArtifactSource {
|
||||
field: "claim_lease"
|
||||
})
|
||||
));
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
@@ -5,24 +5,30 @@ use std::{
|
||||
os::unix::fs::PermissionsExt,
|
||||
path::PathBuf,
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
time::{Duration as StdDuration, SystemTime},
|
||||
};
|
||||
|
||||
use crank_artifacts::{ArtifactRef, ArtifactStore};
|
||||
use crank_artifacts::{
|
||||
ArtifactRef, ArtifactStore, ReconciliationMutation, ReconciliationPresence,
|
||||
ReconciliationRegistration, RegisteredArtifact,
|
||||
};
|
||||
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
||||
use crank_registry::{
|
||||
ArtifactSourceId, ArtifactSourceLifecycle, ArtifactSourceSensitivity,
|
||||
CreateArtifactSourceRequest, CreateWorkspaceRequest, DetachArtifactSourceRequest,
|
||||
ListArtifactSourcesQuery, RegistryError,
|
||||
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
|
||||
ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactSourceId, ArtifactSourceLifecycle,
|
||||
ArtifactSourceSensitivity, ClaimArtifactReconciliationRequest,
|
||||
ClaimExpiredArtifactReconciliationRequest, CreateArtifactSourceRequest, CreateWorkspaceRequest,
|
||||
DetachArtifactSourceRequest, ListArtifactSourcesQuery, RegistryError,
|
||||
};
|
||||
use serde_json::json;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
static NEXT_ROOT: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
struct TestRoot(PathBuf);
|
||||
pub(super) struct TestRoot(pub(super) PathBuf);
|
||||
|
||||
impl TestRoot {
|
||||
fn new(name: &str) -> Self {
|
||||
pub(super) fn new(name: &str) -> Self {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"crank-registry-artifacts-{name}-{}-{}",
|
||||
std::process::id(),
|
||||
@@ -41,10 +47,26 @@ impl Drop for TestRoot {
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp(value: &str) -> OffsetDateTime {
|
||||
pub(super) fn timestamp(value: &str) -> OffsetDateTime {
|
||||
OffsetDateTime::parse(value, &Rfc3339).unwrap()
|
||||
}
|
||||
|
||||
fn reconciliation_candidate_for(
|
||||
store: &ArtifactStore,
|
||||
artifact: &RegisteredArtifact,
|
||||
) -> crank_artifacts::ReconciliationCandidate {
|
||||
let (_, candidates) = store.scan_reconciliation(None, 4096, 32).unwrap();
|
||||
candidates
|
||||
.into_iter()
|
||||
.find(|candidate| {
|
||||
matches!(
|
||||
store.register_reconciliation(candidate, StdDuration::ZERO, SystemTime::now()),
|
||||
Ok(ReconciliationRegistration::Registered(found)) if found == *artifact
|
||||
)
|
||||
})
|
||||
.expect("registered reconciliation candidate")
|
||||
}
|
||||
|
||||
async fn create_workspace(registry: &crank_registry::PostgresRegistry, id: &str) -> WorkspaceId {
|
||||
let workspace_id = WorkspaceId::new(id);
|
||||
let created_at = timestamp("2026-08-26T10:00:00Z");
|
||||
@@ -534,3 +556,300 @@ async fn verified_read_returns_only_digest_and_size_verified_bytes() {
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconciliation_claims_are_fenced_global_and_allow_verified_revival() {
|
||||
let database = TestDatabase::new().await;
|
||||
let registry = database.registry().await;
|
||||
let raw_pool = database.raw_pool().await;
|
||||
let workspace_a = create_workspace(®istry, "ws_claim_a").await;
|
||||
let workspace_b = create_workspace(®istry, "ws_claim_b").await;
|
||||
let root = TestRoot::new("claims");
|
||||
let store = ArtifactStore::open(&root.0).unwrap();
|
||||
let unreferenced = store.put_registered(b"unreferenced\n").unwrap();
|
||||
let started = timestamp("2026-08-27T10:00:00Z");
|
||||
let lease = timestamp("2026-08-27T10:05:00Z");
|
||||
let token = ArtifactClaimToken::generate();
|
||||
assert_eq!(format!("{token:?}"), "ArtifactClaimToken(..)");
|
||||
|
||||
let request = ClaimArtifactReconciliationRequest {
|
||||
artifact: &unreferenced,
|
||||
token: token.clone(),
|
||||
claimed_at: started,
|
||||
lease_expires_at: lease,
|
||||
detached_grace: time::Duration::hours(24),
|
||||
};
|
||||
assert_eq!(
|
||||
format!("{request:?}"),
|
||||
"ClaimArtifactReconciliationRequest(..)"
|
||||
);
|
||||
let claim = match registry
|
||||
.claim_artifact_reconciliation(request)
|
||||
.await
|
||||
.unwrap()
|
||||
{
|
||||
ArtifactClaimOutcome::Claimed(claim) => claim,
|
||||
outcome => panic!("unexpected claim outcome: {outcome:?}"),
|
||||
};
|
||||
assert_eq!(format!("{claim:?}"), "ArtifactReconciliationClaim(..)");
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &unreferenced,
|
||||
token,
|
||||
claimed_at: timestamp("2026-08-27T10:01:00Z"),
|
||||
lease_expires_at: lease,
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimOutcome::Claimed(_)
|
||||
));
|
||||
assert_eq!(
|
||||
registry
|
||||
.recheck_artifact_reconciliation_claim(&claim, started, time::Duration::hours(24))
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimRecheckOutcome::Mutate
|
||||
);
|
||||
assert!(matches!(
|
||||
registry
|
||||
.create_artifact_source(CreateArtifactSourceRequest {
|
||||
workspace_id: &workspace_a,
|
||||
source_id: &ArtifactSourceId::new("src_blocked"),
|
||||
artifact: &unreferenced,
|
||||
mime_type: "text/plain",
|
||||
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||
created_at: started,
|
||||
})
|
||||
.await,
|
||||
Err(RegistryError::ArtifactClaimInProgress)
|
||||
));
|
||||
assert_eq!(
|
||||
registry
|
||||
.finalize_artifact_reconciliation_claim(
|
||||
&claim,
|
||||
ArtifactClaimFinalization::Quarantined,
|
||||
timestamp("2026-08-27T10:01:00Z"),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimFinalizeOutcome::Retained
|
||||
);
|
||||
|
||||
let final_candidate = reconciliation_candidate_for(&store, &unreferenced);
|
||||
assert!(matches!(
|
||||
store.quarantine_reconciliation(final_candidate),
|
||||
Ok(ReconciliationMutation::Quarantined | ReconciliationMutation::AlreadyQuarantined)
|
||||
));
|
||||
let quarantined_candidate = reconciliation_candidate_for(&store, &unreferenced);
|
||||
assert!(matches!(
|
||||
store.delete_quarantined_reconciliation(quarantined_candidate),
|
||||
Ok(ReconciliationMutation::Deleted | ReconciliationMutation::AlreadyAbsent)
|
||||
));
|
||||
assert_eq!(
|
||||
store
|
||||
.reconciliation_presence(unreferenced.artifact_ref())
|
||||
.unwrap(),
|
||||
ReconciliationPresence::Absent
|
||||
);
|
||||
|
||||
sqlx::query(
|
||||
"update artifact_blobs
|
||||
set claim_expires_at = clock_timestamp() - interval '1 second'
|
||||
where digest = $1",
|
||||
)
|
||||
.bind(unreferenced.artifact_ref().digest_hex())
|
||||
.execute(&raw_pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let recovery_now = registry.artifact_reconciliation_now().await.unwrap();
|
||||
let probes = registry
|
||||
.list_expired_artifact_reconciliation_probes(recovery_now, 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
format!("{:?}", probes.first().expect("expired claim probe")),
|
||||
"ArtifactExpiredClaimProbe(..)"
|
||||
);
|
||||
let winner = registry
|
||||
.claim_expired_artifact_reconciliation(
|
||||
probes.first().expect("expired claim probe"),
|
||||
ClaimExpiredArtifactReconciliationRequest {
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: recovery_now,
|
||||
lease_expires_at: recovery_now + time::Duration::minutes(5),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("expired claim was not recovered");
|
||||
assert_eq!(
|
||||
registry
|
||||
.finalize_artifact_reconciliation_claim(
|
||||
&claim,
|
||||
ArtifactClaimFinalization::Deleted,
|
||||
timestamp("2026-08-27T10:05:01Z"),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimFinalizeOutcome::Stale
|
||||
);
|
||||
assert_eq!(
|
||||
registry
|
||||
.recover_artifact_reconciliation_claim(
|
||||
&winner,
|
||||
ArtifactClaimFinalization::AlreadyAbsent,
|
||||
timestamp("2026-08-27T10:05:02Z"),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimFinalizeOutcome::Unavailable
|
||||
);
|
||||
let republished = store.put_registered(b"unreferenced\n").unwrap();
|
||||
let revived = registry
|
||||
.create_artifact_source(CreateArtifactSourceRequest {
|
||||
workspace_id: &workspace_a,
|
||||
source_id: &ArtifactSourceId::new("src_revived"),
|
||||
artifact: &republished,
|
||||
mime_type: "text/plain",
|
||||
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||
created_at: timestamp("2026-08-27T10:06:00Z"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
revived.blob.lifecycle,
|
||||
crank_registry::ArtifactBlobLifecycle::Available
|
||||
);
|
||||
|
||||
let referenced = store.put_registered(b"referenced\n").unwrap();
|
||||
let active_id = ArtifactSourceId::new("src_active");
|
||||
let active = registry
|
||||
.create_artifact_source(CreateArtifactSourceRequest {
|
||||
workspace_id: &workspace_b,
|
||||
source_id: &active_id,
|
||||
artifact: &referenced,
|
||||
mime_type: "text/plain",
|
||||
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||
created_at: started,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &referenced,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: timestamp("2026-08-27T10:01:00Z"),
|
||||
lease_expires_at: lease,
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimOutcome::ActiveReference
|
||||
));
|
||||
registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &workspace_b,
|
||||
source_id: &active_id,
|
||||
expected_updated_at: Some(active.updated_at),
|
||||
detached_at: timestamp("2026-08-27T10:02:00Z"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &referenced,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: timestamp("2026-08-27T10:03:00Z"),
|
||||
lease_expires_at: lease,
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
ArtifactClaimOutcome::DetachedReferenceInGrace
|
||||
));
|
||||
|
||||
let concurrent = store.put_registered(b"concurrent claim\n").unwrap();
|
||||
let first_registry = registry.clone();
|
||||
let second_registry = registry.clone();
|
||||
let first_artifact = concurrent.clone();
|
||||
let second_artifact = concurrent.clone();
|
||||
let (first, second) = tokio::join!(
|
||||
async move {
|
||||
first_registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &first_artifact,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: timestamp("2026-08-27T11:00:00Z"),
|
||||
lease_expires_at: timestamp("2026-08-27T11:05:00Z"),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
},
|
||||
async move {
|
||||
second_registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &second_artifact,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: timestamp("2026-08-27T11:00:00Z"),
|
||||
lease_expires_at: timestamp("2026-08-27T11:05:00Z"),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
}
|
||||
);
|
||||
let first = first.unwrap();
|
||||
let second = second.unwrap();
|
||||
assert_eq!(
|
||||
usize::from(matches!(first, ArtifactClaimOutcome::Claimed(_)))
|
||||
+ usize::from(matches!(second, ArtifactClaimOutcome::Claimed(_))),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
usize::from(matches!(first, ArtifactClaimOutcome::HeldByOther))
|
||||
+ usize::from(matches!(second, ArtifactClaimOutcome::HeldByOther)),
|
||||
1
|
||||
);
|
||||
|
||||
let attach_race = store.put_registered(b"attach race\n").unwrap();
|
||||
let claim_registry = registry.clone();
|
||||
let attach_registry = registry.clone();
|
||||
let claim_artifact = attach_race.clone();
|
||||
let attach_artifact = attach_race.clone();
|
||||
let attach_workspace = workspace_a.clone();
|
||||
let (claim_race, attach_race) = tokio::join!(
|
||||
async move {
|
||||
claim_registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &claim_artifact,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at: timestamp("2026-08-27T12:00:00Z"),
|
||||
lease_expires_at: timestamp("2026-08-27T12:05:00Z"),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
},
|
||||
async move {
|
||||
attach_registry
|
||||
.create_artifact_source(CreateArtifactSourceRequest {
|
||||
workspace_id: &attach_workspace,
|
||||
source_id: &ArtifactSourceId::new("src_attach_race"),
|
||||
artifact: &attach_artifact,
|
||||
mime_type: "text/plain",
|
||||
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||
created_at: timestamp("2026-08-27T12:00:00Z"),
|
||||
})
|
||||
.await
|
||||
}
|
||||
);
|
||||
match (claim_race.unwrap(), attach_race) {
|
||||
(ArtifactClaimOutcome::ActiveReference, Ok(_)) => {}
|
||||
(ArtifactClaimOutcome::Claimed(_), Err(RegistryError::ArtifactClaimInProgress)) => {}
|
||||
(claim, attach) => panic!("unsafe claim/attach race outcome: {claim:?}, {attach:?}"),
|
||||
}
|
||||
|
||||
database.cleanup().await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user