feat(artifacts): add fenced reconciliation recovery
This commit is contained in:
@@ -0,0 +1,783 @@
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use crank_artifacts::{
|
||||
ArtifactError, ArtifactStore, ReconciliationCursor, ReconciliationMutation,
|
||||
ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
|
||||
ReconciliationScanStop,
|
||||
};
|
||||
use crank_registry::{
|
||||
ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
|
||||
ArtifactClaimRecheckOutcome, ArtifactClaimToken, ArtifactExpiredClaimProbe,
|
||||
ClaimArtifactReconciliationRequest, ClaimExpiredArtifactReconciliationRequest,
|
||||
PostgresRegistry,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::MissedTickBehavior;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub const RECONCILIATION_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
||||
pub const RECONCILIATION_GRACE: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
pub const RECONCILIATION_LEASE: Duration = Duration::from_secs(5 * 60);
|
||||
pub const RECONCILIATION_TRAVERSAL_LIMIT: usize = 4_096;
|
||||
pub const RECONCILIATION_CANDIDATE_LIMIT: usize = 32;
|
||||
pub const RECONCILIATION_MUTATION_LIMIT: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum Phase {
|
||||
Recovery,
|
||||
Sweep,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct StepLimits {
|
||||
traversal_syscalls: usize,
|
||||
candidates: usize,
|
||||
mutations: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Step<C> {
|
||||
cursor: Option<C>,
|
||||
scan_complete: bool,
|
||||
traversal_syscalls: usize,
|
||||
candidates: usize,
|
||||
mutation_attempts: usize,
|
||||
classifications: ReconciliationClassificationCounters,
|
||||
physical_mutation_possible: bool,
|
||||
retryable: bool,
|
||||
}
|
||||
|
||||
/// Bounded, identity-free classifications observed during one or more scans.
|
||||
///
|
||||
/// The fixed fields are the complete telemetry vocabulary: no digest, path,
|
||||
/// token, source or workspace value can be attached to an item classification.
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub struct ReconciliationClassificationCounters {
|
||||
pub scanner_malformed: usize,
|
||||
pub scanner_unsafe: usize,
|
||||
pub recovery_present: usize,
|
||||
pub recovery_safety: usize,
|
||||
pub recovery_retryable: usize,
|
||||
pub registration_integrity: usize,
|
||||
pub registration_safety: usize,
|
||||
pub registration_retryable: usize,
|
||||
pub claim_metadata_conflict: usize,
|
||||
pub mutation_integrity: usize,
|
||||
pub mutation_safety: usize,
|
||||
pub mutation_retryable: usize,
|
||||
}
|
||||
|
||||
impl ReconciliationClassificationCounters {
|
||||
fn checked_add(self, other: Self) -> Option<Self> {
|
||||
Some(Self {
|
||||
scanner_malformed: self
|
||||
.scanner_malformed
|
||||
.checked_add(other.scanner_malformed)?,
|
||||
scanner_unsafe: self.scanner_unsafe.checked_add(other.scanner_unsafe)?,
|
||||
recovery_present: self.recovery_present.checked_add(other.recovery_present)?,
|
||||
recovery_safety: self.recovery_safety.checked_add(other.recovery_safety)?,
|
||||
recovery_retryable: self
|
||||
.recovery_retryable
|
||||
.checked_add(other.recovery_retryable)?,
|
||||
registration_integrity: self
|
||||
.registration_integrity
|
||||
.checked_add(other.registration_integrity)?,
|
||||
registration_safety: self
|
||||
.registration_safety
|
||||
.checked_add(other.registration_safety)?,
|
||||
registration_retryable: self
|
||||
.registration_retryable
|
||||
.checked_add(other.registration_retryable)?,
|
||||
claim_metadata_conflict: self
|
||||
.claim_metadata_conflict
|
||||
.checked_add(other.claim_metadata_conflict)?,
|
||||
mutation_integrity: self
|
||||
.mutation_integrity
|
||||
.checked_add(other.mutation_integrity)?,
|
||||
mutation_safety: self.mutation_safety.checked_add(other.mutation_safety)?,
|
||||
mutation_retryable: self
|
||||
.mutation_retryable
|
||||
.checked_add(other.mutation_retryable)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn record_registration_error(&mut self, error: ArtifactError) -> bool {
|
||||
match error {
|
||||
ArtifactError::Integrity
|
||||
| ArtifactError::EmptySource
|
||||
| ArtifactError::SourceTooLarge => {
|
||||
self.registration_integrity += 1;
|
||||
false
|
||||
}
|
||||
ArtifactError::UnsafeRoot
|
||||
| ArtifactError::InvalidReference
|
||||
| ArtifactError::NotFound => {
|
||||
self.registration_safety += 1;
|
||||
false
|
||||
}
|
||||
ArtifactError::Storage => {
|
||||
self.registration_retryable += 1;
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_mutation_error(&mut self, error: ArtifactError) -> bool {
|
||||
match error {
|
||||
ArtifactError::Integrity
|
||||
| ArtifactError::EmptySource
|
||||
| ArtifactError::SourceTooLarge => self.mutation_integrity += 1,
|
||||
ArtifactError::UnsafeRoot
|
||||
| ArtifactError::InvalidReference
|
||||
| ArtifactError::NotFound => self.mutation_safety += 1,
|
||||
ArtifactError::Storage => self.mutation_retryable += 1,
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn record_recovery_presence(&mut self, presence: ReconciliationPresence) -> bool {
|
||||
match presence {
|
||||
ReconciliationPresence::Final | ReconciliationPresence::Quarantine => {
|
||||
self.recovery_present += 1;
|
||||
false
|
||||
}
|
||||
ReconciliationPresence::Unsafe => {
|
||||
self.recovery_safety += 1;
|
||||
false
|
||||
}
|
||||
ReconciliationPresence::Retryable => {
|
||||
self.recovery_retryable += 1;
|
||||
true
|
||||
}
|
||||
ReconciliationPresence::Absent => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_mutation_presence(&mut self, presence: ReconciliationPresence) -> bool {
|
||||
match presence {
|
||||
ReconciliationPresence::Absent => false,
|
||||
ReconciliationPresence::Unsafe => {
|
||||
self.mutation_safety += 1;
|
||||
true
|
||||
}
|
||||
ReconciliationPresence::Final
|
||||
| ReconciliationPresence::Quarantine
|
||||
| ReconciliationPresence::Retryable => {
|
||||
self.mutation_retryable += 1;
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub struct ReconciliationTickReport {
|
||||
pub traversal_syscalls: usize,
|
||||
pub candidates: usize,
|
||||
pub mutation_attempts: usize,
|
||||
pub classifications: ReconciliationClassificationCounters,
|
||||
pub recovery_completed: bool,
|
||||
pub cycle_completed: bool,
|
||||
pub retryable: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
|
||||
pub enum ReconciliationCoordinatorError {
|
||||
#[error("artifact reconciliation backend is unavailable")]
|
||||
Backend,
|
||||
#[error("artifact reconciliation backend exceeded its bounded contract")]
|
||||
Bounds,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
trait ReconciliationBackend: Send + Sync {
|
||||
type Cursor: Send;
|
||||
|
||||
async fn step(
|
||||
&self,
|
||||
phase: Phase,
|
||||
cursor: Option<Self::Cursor>,
|
||||
limits: StepLimits,
|
||||
) -> Result<Step<Self::Cursor>, ReconciliationCoordinatorError>;
|
||||
}
|
||||
|
||||
struct ReconciliationCoordinator<B: ReconciliationBackend> {
|
||||
backend: B,
|
||||
phase: Phase,
|
||||
cursor: Option<B::Cursor>,
|
||||
tick_limits: StepLimits,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PostgresReconciliationBackend {
|
||||
registry: PostgresRegistry,
|
||||
store: ArtifactStore,
|
||||
}
|
||||
|
||||
enum PostgresReconciliationCursor {
|
||||
ExpiredClaim(ArtifactExpiredClaimProbe),
|
||||
Filesystem(ReconciliationCursor),
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PostgresReconciliationCursor {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("PostgresReconciliationCursor(..)")
|
||||
}
|
||||
}
|
||||
|
||||
impl PostgresReconciliationBackend {
|
||||
fn new(registry: PostgresRegistry, store: ArtifactStore) -> Self {
|
||||
Self { registry, store }
|
||||
}
|
||||
|
||||
async fn database_now(&self) -> Result<OffsetDateTime, ReconciliationCoordinatorError> {
|
||||
self.registry
|
||||
.artifact_reconciliation_now()
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)
|
||||
}
|
||||
|
||||
async fn presence(
|
||||
&self,
|
||||
artifact_ref: crank_artifacts::ArtifactRef,
|
||||
) -> Result<ReconciliationPresence, ReconciliationCoordinatorError> {
|
||||
let store = self.store.clone();
|
||||
let result =
|
||||
tokio::task::spawn_blocking(move || store.reconciliation_presence(&artifact_ref))
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?;
|
||||
Ok(match result {
|
||||
Ok(presence) => presence,
|
||||
Err(ArtifactError::Storage) => ReconciliationPresence::Retryable,
|
||||
Err(_) => ReconciliationPresence::Unsafe,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn reconciliation_lease_expires_at(claimed_at: OffsetDateTime) -> OffsetDateTime {
|
||||
claimed_at
|
||||
+ time::Duration::seconds(
|
||||
i64::try_from(RECONCILIATION_LEASE.as_secs())
|
||||
.expect("the fixed reconciliation lease fits i64"),
|
||||
)
|
||||
}
|
||||
|
||||
impl<B: ReconciliationBackend> ReconciliationCoordinator<B> {
|
||||
fn new(backend: B) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
phase: Phase::Recovery,
|
||||
cursor: None,
|
||||
tick_limits: StepLimits {
|
||||
traversal_syscalls: RECONCILIATION_TRAVERSAL_LIMIT,
|
||||
candidates: RECONCILIATION_CANDIDATE_LIMIT,
|
||||
mutations: RECONCILIATION_MUTATION_LIMIT,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn with_tick_limits(backend: B, tick_limits: StepLimits) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
phase: Phase::Recovery,
|
||||
cursor: None,
|
||||
tick_limits,
|
||||
}
|
||||
}
|
||||
|
||||
async fn tick(&mut self) -> Result<ReconciliationTickReport, ReconciliationCoordinatorError> {
|
||||
let mut report = ReconciliationTickReport::default();
|
||||
|
||||
loop {
|
||||
let limits = StepLimits {
|
||||
traversal_syscalls: self
|
||||
.tick_limits
|
||||
.traversal_syscalls
|
||||
.saturating_sub(report.traversal_syscalls),
|
||||
candidates: self
|
||||
.tick_limits
|
||||
.candidates
|
||||
.saturating_sub(report.candidates),
|
||||
mutations: self
|
||||
.tick_limits
|
||||
.mutations
|
||||
.saturating_sub(report.mutation_attempts),
|
||||
};
|
||||
if limits.traversal_syscalls == 0 || limits.candidates == 0 || limits.mutations == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let step = match self
|
||||
.backend
|
||||
.step(self.phase, self.cursor.take(), limits)
|
||||
.await
|
||||
{
|
||||
Ok(step) => step,
|
||||
Err(error) => {
|
||||
self.cursor = None;
|
||||
self.phase = Phase::Recovery;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
if step.traversal_syscalls > limits.traversal_syscalls
|
||||
|| step.candidates > limits.candidates
|
||||
|| step.mutation_attempts > limits.mutations
|
||||
|| step.mutation_attempts > step.candidates
|
||||
{
|
||||
self.cursor = None;
|
||||
self.phase = Phase::Recovery;
|
||||
return Err(ReconciliationCoordinatorError::Bounds);
|
||||
}
|
||||
|
||||
report.traversal_syscalls += step.traversal_syscalls;
|
||||
report.candidates += step.candidates;
|
||||
report.mutation_attempts += step.mutation_attempts;
|
||||
report.classifications = report
|
||||
.classifications
|
||||
.checked_add(step.classifications)
|
||||
.ok_or_else(|| {
|
||||
self.cursor = None;
|
||||
self.phase = Phase::Recovery;
|
||||
ReconciliationCoordinatorError::Bounds
|
||||
})?;
|
||||
report.retryable |= step.retryable;
|
||||
|
||||
let made_progress =
|
||||
step.traversal_syscalls != 0 || step.candidates != 0 || step.mutation_attempts != 0;
|
||||
if step.physical_mutation_possible {
|
||||
// A cursor is valid only while the namespace is unchanged. A
|
||||
// possible rename/unlink includes ambiguous fsync outcomes.
|
||||
drop(step.cursor);
|
||||
self.cursor = None;
|
||||
self.phase = Phase::Recovery;
|
||||
} else {
|
||||
self.cursor = step.cursor;
|
||||
if step.scan_complete {
|
||||
self.cursor = None;
|
||||
match self.phase {
|
||||
Phase::Recovery => {
|
||||
report.recovery_completed = true;
|
||||
self.phase = Phase::Sweep;
|
||||
}
|
||||
Phase::Sweep => {
|
||||
report.cycle_completed = true;
|
||||
self.phase = Phase::Recovery;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if step.retryable || (!made_progress && !step.scan_complete) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ReconciliationBackend for PostgresReconciliationBackend {
|
||||
type Cursor = PostgresReconciliationCursor;
|
||||
|
||||
async fn step(
|
||||
&self,
|
||||
phase: Phase,
|
||||
cursor: Option<Self::Cursor>,
|
||||
limits: StepLimits,
|
||||
) -> Result<Step<Self::Cursor>, ReconciliationCoordinatorError> {
|
||||
let mut recovered_candidates = 0;
|
||||
let mut classifications = ReconciliationClassificationCounters::default();
|
||||
let mut recovery_retryable = false;
|
||||
let mut filesystem_cursor = None;
|
||||
if phase == Phase::Recovery {
|
||||
let after = match cursor {
|
||||
Some(PostgresReconciliationCursor::ExpiredClaim(probe)) => Some(probe),
|
||||
Some(PostgresReconciliationCursor::Filesystem(cursor)) => {
|
||||
filesystem_cursor = Some(cursor);
|
||||
None
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
if filesystem_cursor.is_none() {
|
||||
let database_now = self.database_now().await?;
|
||||
let probe_limit = u32::try_from(limits.candidates).unwrap_or(u32::MAX);
|
||||
let probes = self
|
||||
.registry
|
||||
.list_expired_artifact_reconciliation_probes_after(
|
||||
database_now,
|
||||
after.as_ref(),
|
||||
probe_limit,
|
||||
)
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?;
|
||||
let page_full = probes.len() == probe_limit as usize;
|
||||
let mut last_processed = after;
|
||||
for probe in probes {
|
||||
recovered_candidates += 1;
|
||||
last_processed = Some(probe.clone());
|
||||
let presence = self.presence(probe.artifact_ref().clone()).await?;
|
||||
if presence != ReconciliationPresence::Absent {
|
||||
recovery_retryable |= classifications.record_recovery_presence(presence);
|
||||
continue;
|
||||
}
|
||||
|
||||
let claimed_at = self.database_now().await?;
|
||||
let claim = self
|
||||
.registry
|
||||
.claim_expired_artifact_reconciliation(
|
||||
&probe,
|
||||
ClaimExpiredArtifactReconciliationRequest {
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at,
|
||||
lease_expires_at: reconciliation_lease_expires_at(claimed_at),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?;
|
||||
let Some(claim) = claim else {
|
||||
continue;
|
||||
};
|
||||
|
||||
// Recheck after the DB CAS. A concurrent filesystem put
|
||||
// does not consult PostgreSQL, so the first probe alone
|
||||
// cannot prove absence at finalization time.
|
||||
let presence = self.presence(claim.artifact_ref().clone()).await?;
|
||||
if presence != ReconciliationPresence::Absent {
|
||||
classifications.record_recovery_presence(presence);
|
||||
recovery_retryable = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
let recovered_at = self.database_now().await?;
|
||||
let result = self
|
||||
.registry
|
||||
.recover_artifact_reconciliation_claim(
|
||||
&claim,
|
||||
ArtifactClaimFinalization::AlreadyAbsent,
|
||||
recovered_at,
|
||||
)
|
||||
.await;
|
||||
let recovered = matches!(result, Ok(ArtifactClaimFinalizeOutcome::Unavailable));
|
||||
if recovered {
|
||||
let presence = self.presence(claim.artifact_ref().clone()).await?;
|
||||
if presence != ReconciliationPresence::Absent {
|
||||
classifications.record_recovery_presence(presence);
|
||||
recovery_retryable = true;
|
||||
}
|
||||
}
|
||||
return Ok(Step {
|
||||
cursor: last_processed.map(PostgresReconciliationCursor::ExpiredClaim),
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 0,
|
||||
candidates: recovered_candidates,
|
||||
mutation_attempts: 1,
|
||||
classifications,
|
||||
physical_mutation_possible: false,
|
||||
retryable: recovery_retryable || !recovered,
|
||||
});
|
||||
}
|
||||
|
||||
if page_full {
|
||||
return Ok(Step {
|
||||
cursor: last_processed.map(PostgresReconciliationCursor::ExpiredClaim),
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 0,
|
||||
candidates: recovered_candidates,
|
||||
mutation_attempts: 0,
|
||||
classifications,
|
||||
physical_mutation_possible: false,
|
||||
retryable: recovery_retryable,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if let Some(PostgresReconciliationCursor::Filesystem(cursor)) = cursor {
|
||||
filesystem_cursor = Some(cursor);
|
||||
}
|
||||
|
||||
let scan_candidate_limit = limits.candidates.saturating_sub(recovered_candidates);
|
||||
if scan_candidate_limit == 0 {
|
||||
return Ok(Step {
|
||||
cursor: filesystem_cursor.map(PostgresReconciliationCursor::Filesystem),
|
||||
scan_complete: false,
|
||||
traversal_syscalls: 0,
|
||||
candidates: recovered_candidates,
|
||||
mutation_attempts: 0,
|
||||
classifications,
|
||||
physical_mutation_possible: false,
|
||||
retryable: recovery_retryable,
|
||||
});
|
||||
}
|
||||
let namespace = match phase {
|
||||
Phase::Recovery => ReconciliationNamespace::Quarantine,
|
||||
Phase::Sweep => ReconciliationNamespace::Final,
|
||||
};
|
||||
let store = self.store.clone();
|
||||
let (mut scan, candidates) = tokio::task::spawn_blocking(move || {
|
||||
store.scan_reconciliation_namespace(
|
||||
namespace,
|
||||
filesystem_cursor,
|
||||
limits.traversal_syscalls,
|
||||
scan_candidate_limit,
|
||||
)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?;
|
||||
|
||||
let mut step = Step {
|
||||
cursor: scan
|
||||
.continuation
|
||||
.take()
|
||||
.map(PostgresReconciliationCursor::Filesystem),
|
||||
scan_complete: scan.stop == ReconciliationScanStop::Complete,
|
||||
traversal_syscalls: scan.traversal_syscalls,
|
||||
candidates: recovered_candidates,
|
||||
mutation_attempts: 0,
|
||||
classifications: classifications
|
||||
.checked_add(ReconciliationClassificationCounters {
|
||||
scanner_malformed: scan.malformed,
|
||||
scanner_unsafe: scan.unsafe_entries,
|
||||
..ReconciliationClassificationCounters::default()
|
||||
})
|
||||
.ok_or(ReconciliationCoordinatorError::Bounds)?,
|
||||
physical_mutation_possible: false,
|
||||
retryable: recovery_retryable || scan.stop == ReconciliationScanStop::Retryable,
|
||||
};
|
||||
|
||||
for candidate in candidates {
|
||||
if step.candidates == limits.candidates || step.mutation_attempts == limits.mutations {
|
||||
break;
|
||||
}
|
||||
step.candidates += 1;
|
||||
let registration_grace = match phase {
|
||||
Phase::Recovery => Duration::ZERO,
|
||||
Phase::Sweep => RECONCILIATION_GRACE,
|
||||
};
|
||||
let store = self.store.clone();
|
||||
let registration = tokio::task::spawn_blocking(move || {
|
||||
let registration = store.register_reconciliation(
|
||||
&candidate,
|
||||
registration_grace,
|
||||
SystemTime::now(),
|
||||
);
|
||||
(candidate, registration)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?;
|
||||
let (candidate, registration) = registration;
|
||||
let artifact = match registration {
|
||||
Ok(ReconciliationRegistration::Registered(artifact)) => artifact,
|
||||
Ok(ReconciliationRegistration::NotEligible) => continue,
|
||||
Err(error) => {
|
||||
step.retryable |= step.classifications.record_registration_error(error);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let claimed_at = self.database_now().await?;
|
||||
let lease_expires_at = reconciliation_lease_expires_at(claimed_at);
|
||||
let detached_grace = time::Duration::seconds(
|
||||
i64::try_from(RECONCILIATION_GRACE.as_secs())
|
||||
.expect("the fixed reconciliation grace fits i64"),
|
||||
);
|
||||
let claim = self
|
||||
.registry
|
||||
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
|
||||
artifact: &artifact,
|
||||
token: ArtifactClaimToken::generate(),
|
||||
claimed_at,
|
||||
lease_expires_at,
|
||||
detached_grace,
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?;
|
||||
let claim = match claim {
|
||||
ArtifactClaimOutcome::Claimed(claim) => claim,
|
||||
ArtifactClaimOutcome::HeldByOther
|
||||
| ArtifactClaimOutcome::ActiveReference
|
||||
| ArtifactClaimOutcome::DetachedReferenceInGrace => continue,
|
||||
ArtifactClaimOutcome::MetadataConflict => {
|
||||
step.classifications.claim_metadata_conflict += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let rechecked_at = self.database_now().await?;
|
||||
let recheck = self
|
||||
.registry
|
||||
.recheck_artifact_reconciliation_claim(&claim, rechecked_at, detached_grace)
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?;
|
||||
if recheck != ArtifactClaimRecheckOutcome::Mutate {
|
||||
continue;
|
||||
}
|
||||
|
||||
step.mutation_attempts += 1;
|
||||
step.physical_mutation_possible = true;
|
||||
let store = self.store.clone();
|
||||
let mutation = match tokio::task::spawn_blocking(move || match phase {
|
||||
Phase::Recovery => store.delete_quarantined_reconciliation(candidate),
|
||||
Phase::Sweep => store.quarantine_reconciliation(candidate),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(mutation) => mutation,
|
||||
Err(_) => Err(ArtifactError::Storage),
|
||||
};
|
||||
let mut observation = match mutation {
|
||||
Ok(ReconciliationMutation::Quarantined)
|
||||
| Ok(ReconciliationMutation::AlreadyQuarantined) => {
|
||||
ArtifactClaimFinalization::Quarantined
|
||||
}
|
||||
Ok(ReconciliationMutation::Deleted) => ArtifactClaimFinalization::Deleted,
|
||||
Ok(ReconciliationMutation::AlreadyAbsent) => {
|
||||
ArtifactClaimFinalization::AlreadyAbsent
|
||||
}
|
||||
Ok(ReconciliationMutation::Retryable) => {
|
||||
step.classifications.mutation_retryable += 1;
|
||||
step.retryable = true;
|
||||
ArtifactClaimFinalization::Retryable
|
||||
}
|
||||
Err(error) => {
|
||||
step.retryable |= step.classifications.record_mutation_error(error);
|
||||
ArtifactClaimFinalization::Retryable
|
||||
}
|
||||
};
|
||||
if matches!(
|
||||
observation,
|
||||
ArtifactClaimFinalization::Deleted | ArtifactClaimFinalization::AlreadyAbsent
|
||||
) {
|
||||
// `unavailable` means both canonical locations are confirmed
|
||||
// absent. A concurrent republish can recreate final bytes
|
||||
// without consulting PostgreSQL, so unlink alone is not proof.
|
||||
let presence = self.presence(claim.artifact_ref().clone()).await?;
|
||||
observation = match presence {
|
||||
ReconciliationPresence::Absent => ArtifactClaimFinalization::AlreadyAbsent,
|
||||
presence => {
|
||||
step.retryable |= step.classifications.record_mutation_presence(presence);
|
||||
ArtifactClaimFinalization::Retryable
|
||||
}
|
||||
};
|
||||
}
|
||||
let observed_at = self.database_now().await?;
|
||||
let finalization = match phase {
|
||||
Phase::Recovery => {
|
||||
self.registry
|
||||
.recover_artifact_reconciliation_claim(&claim, observation, observed_at)
|
||||
.await
|
||||
}
|
||||
Phase::Sweep => {
|
||||
self.registry
|
||||
.finalize_artifact_reconciliation_claim(&claim, observation, observed_at)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let finalized_as_expected =
|
||||
match observation {
|
||||
ArtifactClaimFinalization::Deleted
|
||||
| ArtifactClaimFinalization::AlreadyAbsent => matches!(
|
||||
finalization.as_ref(),
|
||||
Ok(ArtifactClaimFinalizeOutcome::Unavailable)
|
||||
),
|
||||
ArtifactClaimFinalization::Quarantined
|
||||
| ArtifactClaimFinalization::Retryable => matches!(
|
||||
finalization.as_ref(),
|
||||
Ok(ArtifactClaimFinalizeOutcome::Retained)
|
||||
),
|
||||
};
|
||||
if !finalized_as_expected {
|
||||
step.retryable = true;
|
||||
}
|
||||
if matches!(
|
||||
finalization.as_ref(),
|
||||
Ok(ArtifactClaimFinalizeOutcome::Unavailable)
|
||||
) {
|
||||
let presence = self.presence(claim.artifact_ref().clone()).await?;
|
||||
step.retryable |= step.classifications.record_mutation_presence(presence);
|
||||
}
|
||||
|
||||
// The scan cursor and every remaining observation came from the
|
||||
// pre-mutation namespace. Return immediately so the coordinator
|
||||
// drops them and restarts a recovery cycle from `None`.
|
||||
break;
|
||||
}
|
||||
|
||||
Ok(step)
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens the protected root without running filesystem syscalls on Tokio's
|
||||
/// async executor.
|
||||
pub async fn open_reconciliation_store(
|
||||
root: PathBuf,
|
||||
) -> Result<ArtifactStore, ReconciliationCoordinatorError> {
|
||||
tokio::task::spawn_blocking(move || ArtifactStore::open(root))
|
||||
.await
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)?
|
||||
.map_err(|_| ReconciliationCoordinatorError::Backend)
|
||||
}
|
||||
|
||||
/// Runs the immediate bounded startup cycle, then schedules 15-minute ticks.
|
||||
/// Recovery remains ahead of the final sweep even when it spans several ticks.
|
||||
pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: ArtifactStore) {
|
||||
let backend = PostgresReconciliationBackend::new(registry, store);
|
||||
let mut coordinator = ReconciliationCoordinator::new(backend);
|
||||
observe_tick(coordinator.tick().await);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(RECONCILIATION_INTERVAL);
|
||||
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
|
||||
// The immediate tick was run above as part of startup composition.
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
observe_tick(coordinator.tick().await);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn observe_tick(result: Result<ReconciliationTickReport, ReconciliationCoordinatorError>) {
|
||||
match result {
|
||||
Ok(report) => info!(
|
||||
name: "admin.artifact_reconciliation.tick",
|
||||
traversal_syscalls = report.traversal_syscalls,
|
||||
candidates = report.candidates,
|
||||
mutation_attempts = report.mutation_attempts,
|
||||
scanner_malformed = report.classifications.scanner_malformed,
|
||||
scanner_unsafe = report.classifications.scanner_unsafe,
|
||||
recovery_present = report.classifications.recovery_present,
|
||||
recovery_safety = report.classifications.recovery_safety,
|
||||
recovery_retryable = report.classifications.recovery_retryable,
|
||||
registration_integrity = report.classifications.registration_integrity,
|
||||
registration_safety = report.classifications.registration_safety,
|
||||
registration_retryable = report.classifications.registration_retryable,
|
||||
claim_metadata_conflict = report.classifications.claim_metadata_conflict,
|
||||
mutation_integrity = report.classifications.mutation_integrity,
|
||||
mutation_safety = report.classifications.mutation_safety,
|
||||
mutation_retryable = report.classifications.mutation_retryable,
|
||||
recovery_completed = report.recovery_completed,
|
||||
cycle_completed = report.cycle_completed,
|
||||
retryable = report.retryable,
|
||||
"artifact reconciliation tick completed"
|
||||
),
|
||||
Err(error) => warn!(
|
||||
name: "admin.artifact_reconciliation.failed",
|
||||
error_category = match error {
|
||||
ReconciliationCoordinatorError::Backend => "backend",
|
||||
ReconciliationCoordinatorError::Bounds => "bounds",
|
||||
},
|
||||
"artifact reconciliation tick will be retried"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "reconciliation/tests.rs"]
|
||||
mod tests;
|
||||
Reference in New Issue
Block a user