fix(openapi): harden story 2.1 production lifecycle

apps/admin-api/src/reconciliation.rs

@@ -1,5 +1,6 @@
 use std::{
     path::PathBuf,
+    sync::Arc,
     time::{Duration, SystemTime},
 };
 
@@ -7,7 +8,7 @@ use async_trait::async_trait;
 use crank_artifacts::{
     ArtifactError, ArtifactStore, ReconciliationCursor, ReconciliationMutation,
     ReconciliationNamespace, ReconciliationPresence, ReconciliationRegistration,
-    ReconciliationScanStop,
+    ReconciliationScanStop, TempScanCursor,
 };
 use crank_registry::{
     ArtifactClaimFinalization, ArtifactClaimFinalizeOutcome, ArtifactClaimOutcome,
@@ -25,6 +26,11 @@ pub const RECONCILIATION_LEASE: Duration = Duration::from_secs(5 * 60);
 pub const RECONCILIATION_TRAVERSAL_LIMIT: usize = 4_096;
 pub const RECONCILIATION_CANDIDATE_LIMIT: usize = 32;
 pub const RECONCILIATION_MUTATION_LIMIT: usize = 32;
+pub const RECONCILIATION_TICK_DEADLINE: Duration = Duration::from_secs(30);
+pub const TEMP_CLEANUP_GRACE: Duration = Duration::from_secs(60 * 60);
+pub const TEMP_CLEANUP_SCAN_LIMIT: usize = 512;
+pub const TEMP_CLEANUP_DELETE_LIMIT: usize = 16;
+pub const IMPORT_JOB_CLEANUP_LIMIT: u32 = 128;
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 enum Phase {
@@ -191,6 +197,8 @@ pub enum ReconciliationCoordinatorError {
     Backend,
     #[error("artifact reconciliation backend exceeded its bounded contract")]
     Bounds,
+    #[error("artifact reconciliation backend exceeded its deadline")]
+    Deadline,
 }
 
 #[async_trait]
@@ -215,7 +223,7 @@ struct ReconciliationCoordinator<B: ReconciliationBackend> {
 #[derive(Clone)]
 struct PostgresReconciliationBackend {
     registry: PostgresRegistry,
-    store: ArtifactStore,
+    store: Arc<ArtifactStore>,
 }
 
 enum PostgresReconciliationCursor {
@@ -230,7 +238,7 @@ impl std::fmt::Debug for PostgresReconciliationCursor {
 }
 
 impl PostgresReconciliationBackend {
-    fn new(registry: PostgresRegistry, store: ArtifactStore) -> Self {
+    fn new(registry: PostgresRegistry, store: Arc<ArtifactStore>) -> Self {
         Self { registry, store }
     }
 
@@ -245,7 +253,7 @@ impl PostgresReconciliationBackend {
         &self,
         artifact_ref: crank_artifacts::ArtifactRef,
     ) -> Result<ReconciliationPresence, ReconciliationCoordinatorError> {
-        let store = self.store.clone();
+        let store = Arc::clone(&self.store);
         let result =
             tokio::task::spawn_blocking(move || store.reconciliation_presence(&artifact_ref))
                 .await
@@ -518,7 +526,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
             Phase::Recovery => ReconciliationNamespace::Quarantine,
             Phase::Sweep => ReconciliationNamespace::Final,
         };
-        let store = self.store.clone();
+        let store = Arc::clone(&self.store);
         let (mut scan, candidates) = tokio::task::spawn_blocking(move || {
             store.scan_reconciliation_namespace(
                 namespace,
@@ -560,7 +568,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
                 Phase::Recovery => Duration::ZERO,
                 Phase::Sweep => RECONCILIATION_GRACE,
             };
-            let store = self.store.clone();
+            let store = Arc::clone(&self.store);
             let registration = tokio::task::spawn_blocking(move || {
                 let registration = store.register_reconciliation(
                     &candidate,
@@ -621,7 +629,7 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
 
             step.mutation_attempts += 1;
             step.physical_mutation_possible = true;
-            let store = self.store.clone();
+            let store = Arc::clone(&self.store);
             let mutation = match tokio::task::spawn_blocking(move || match phase {
                 Phase::Recovery => store.delete_quarantined_reconciliation(candidate),
                 Phase::Sweep => store.quarantine_reconciliation(candidate),
@@ -717,8 +725,8 @@ impl ReconciliationBackend for PostgresReconciliationBackend {
 /// async executor.
 pub async fn open_reconciliation_store(
     root: PathBuf,
-) -> Result<ArtifactStore, ReconciliationCoordinatorError> {
-    tokio::task::spawn_blocking(move || ArtifactStore::open(root))
+) -> Result<Arc<ArtifactStore>, ReconciliationCoordinatorError> {
+    tokio::task::spawn_blocking(move || ArtifactStore::open(root).map(Arc::new))
         .await
         .map_err(|_| ReconciliationCoordinatorError::Backend)?
         .map_err(|_| ReconciliationCoordinatorError::Backend)
@@ -726,10 +734,25 @@ pub async fn open_reconciliation_store(
 
 /// Runs the immediate bounded startup cycle, then schedules 15-minute ticks.
 /// Recovery remains ahead of the final sweep even when it spans several ticks.
-pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: ArtifactStore) {
-    let backend = PostgresReconciliationBackend::new(registry, store);
+pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: Arc<ArtifactStore>) {
+    let cleanup_registry = registry.clone();
+    let backend = PostgresReconciliationBackend::new(registry, Arc::clone(&store));
     let mut coordinator = ReconciliationCoordinator::new(backend);
-    observe_tick(coordinator.tick().await);
+    let mut temp_cursor = None;
+    let startup_maintenance = async {
+        observe_import_job_cleanup(&cleanup_registry).await;
+        observe_tick(run_bounded_reconciliation_tick(&mut coordinator).await);
+        temp_cursor = observe_stale_temp_cleanup(Arc::clone(&store), None).await;
+    };
+    if tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, startup_maintenance)
+        .await
+        .is_err()
+    {
+        warn!(
+            name: "admin.artifact_maintenance.startup_deadline",
+            "startup maintenance exceeded its shared deadline; remaining work will resume in background"
+        );
+    }
 
     tokio::spawn(async move {
         let mut interval = tokio::time::interval(RECONCILIATION_INTERVAL);
@@ -738,11 +761,148 @@ pub async fn spawn_artifact_reconciliation(registry: PostgresRegistry, store: Ar
         interval.tick().await;
         loop {
             interval.tick().await;
-            observe_tick(coordinator.tick().await);
+            observe_import_job_cleanup(&cleanup_registry).await;
+            observe_tick(run_bounded_reconciliation_tick(&mut coordinator).await);
+            temp_cursor = observe_stale_temp_cleanup(Arc::clone(&store), temp_cursor).await;
         }
     });
 }
 
+async fn observe_import_job_cleanup(registry: &PostgresRegistry) {
+    let cleanup = async {
+        let mut deleted_jobs = 0_u64;
+        let mut detached_sources = 0_u64;
+        loop {
+            let report = registry
+                .cleanup_expired_import_jobs(IMPORT_JOB_CLEANUP_LIMIT)
+                .await?;
+            deleted_jobs = deleted_jobs.saturating_add(report.deleted_jobs);
+            detached_sources = detached_sources.saturating_add(report.detached_sources);
+            if !report.more_work {
+                return Ok::<_, crank_registry::RegistryError>((deleted_jobs, detached_sources));
+            }
+            // Each pass commits independently. Yield before requesting the
+            // next lock batch so normal import traffic can make progress.
+            tokio::task::yield_now().await;
+        }
+    };
+    match tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, cleanup).await {
+        Ok(Ok((deleted_jobs, detached_sources))) => info!(
+            name: "admin.openapi_import_cleanup.tick",
+            deleted_jobs,
+            detached_sources,
+            more_work = false,
+            "OpenAPI import cleanup tick completed"
+        ),
+        Ok(Err(_)) => warn!(
+            name: "admin.openapi_import_cleanup.failed",
+            error_category = "registry",
+            "OpenAPI import cleanup tick will be retried"
+        ),
+        Err(_) => warn!(
+            name: "admin.openapi_import_cleanup.failed",
+            error_category = "deadline",
+            "OpenAPI import cleanup tick exceeded its deadline"
+        ),
+    }
+}
+
+async fn run_bounded_reconciliation_tick<B: ReconciliationBackend>(
+    coordinator: &mut ReconciliationCoordinator<B>,
+) -> Result<ReconciliationTickReport, ReconciliationCoordinatorError> {
+    tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, coordinator.tick())
+        .await
+        .map_err(|_| ReconciliationCoordinatorError::Deadline)?
+}
+
+async fn observe_stale_temp_cleanup(
+    store: Arc<ArtifactStore>,
+    cursor: Option<TempScanCursor>,
+) -> Option<TempScanCursor> {
+    let progress = Arc::new(std::sync::Mutex::new(cursor));
+    let cleanup_progress = Arc::clone(&progress);
+    let cleanup = async move {
+        let mut scanned = 0_usize;
+        let mut omitted = 0_usize;
+        let mut deleted = 0_usize;
+        let mut retryable = 0_usize;
+        loop {
+            let scanner = Arc::clone(&store);
+            let cursor = cleanup_progress
+                .lock()
+                .map_err(|_| ArtifactError::Storage)?
+                .clone();
+            let (scan, candidates, next_cursor) = tokio::task::spawn_blocking(move || {
+                scanner.scan_stale_temps_after(
+                    TEMP_CLEANUP_GRACE,
+                    cursor,
+                    TEMP_CLEANUP_SCAN_LIMIT,
+                    TEMP_CLEANUP_DELETE_LIMIT,
+                )
+            })
+            .await
+            .map_err(|_| ArtifactError::Storage)??;
+            scanned = scanned.saturating_add(scan.scanned);
+            omitted = omitted.saturating_add(scan.omitted);
+            *cleanup_progress
+                .lock()
+                .map_err(|_| ArtifactError::Storage)? = Some(next_cursor);
+
+            for candidate in candidates {
+                let store = Arc::clone(&store);
+                match tokio::task::spawn_blocking(move || store.delete_stale_temp(candidate)).await
+                {
+                    Ok(Ok(())) => deleted += 1,
+                    Ok(Err(_)) | Err(_) => retryable += 1,
+                }
+            }
+            if scan.complete {
+                return Ok::<_, ArtifactError>((
+                    scanned,
+                    omitted,
+                    deleted,
+                    retryable,
+                    cleanup_progress
+                        .lock()
+                        .map_err(|_| ArtifactError::Storage)?
+                        .clone(),
+                ));
+            }
+            tokio::task::yield_now().await;
+        }
+    };
+
+    match tokio::time::timeout(RECONCILIATION_TICK_DEADLINE, cleanup).await {
+        Ok(Ok((scanned, omitted, deleted, retryable, next_cursor))) => {
+            info!(
+                name: "admin.artifact_temp_cleanup.tick",
+                scanned,
+                omitted,
+                deleted,
+                retryable,
+                "artifact temporary-file cleanup tick completed"
+            );
+            next_cursor
+        }
+        Ok(Err(_)) => {
+            warn!(
+                name: "admin.artifact_temp_cleanup.failed",
+                error_category = "backend",
+                "artifact temporary-file cleanup tick will be retried"
+            );
+            progress.lock().ok().and_then(|cursor| cursor.clone())
+        }
+        Err(_) => {
+            warn!(
+                name: "admin.artifact_temp_cleanup.failed",
+                error_category = "deadline",
+                "artifact temporary-file cleanup tick exceeded its deadline"
+            );
+            progress.lock().ok().and_then(|cursor| cursor.clone())
+        }
+    }
+}
+
 fn observe_tick(result: Result<ReconciliationTickReport, ReconciliationCoordinatorError>) {
     match result {
         Ok(report) => info!(
@@ -772,6 +932,7 @@ fn observe_tick(result: Result<ReconciliationTickReport, ReconciliationCoordinat
             error_category = match error {
                 ReconciliationCoordinatorError::Backend => "backend",
                 ReconciliationCoordinatorError::Bounds => "bounds",
+                ReconciliationCoordinatorError::Deadline => "deadline",
             },
             "artifact reconciliation tick will be retried"
         ),