feat(openapi): complete upload preview and UI evidence

crates/crank-registry/src/lib.rs

@@ -20,14 +20,14 @@ pub mod records {
         ArtifactReconciliationClaim, ArtifactSourceCursor, ArtifactSourceId,
         ArtifactSourceLifecycle, ArtifactSourcePage, ArtifactSourceRecord,
         ArtifactSourceSensitivity, AuthUserRecord, DescriptorKind, DescriptorMetadata, ImportJob,
-        ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobStatus, InvitationRecord,
-        InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
-        InvocationLogRecord, InvocationRetentionOutcome, InvocationRetentionPolicy,
-        InvocationRetentionStatus, MasterKeyIdentityRecord, MasterKeyRotationRecord,
-        MasterKeyRotationStatus, MembershipRecord, OnboardingMilestoneResult,
-        OnboardingPresentationMilestone, OperationAgentRef, OperationSampleMetadata,
-        OperationSummary, OperationUsageSummary, OperationVersionRecord, Page,
-        PlatformApiKeyRecord, ProductEventRecord, PublishedAgentCatalog, PublishedAgentTool,
+        ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope, ImportJobStatus,
+        InvitationRecord, InvocationHistoryLoss, InvocationHistoryLossCategory,
+        InvocationHistoryWriteOutcome, InvocationLogRecord, InvocationRetentionOutcome,
+        InvocationRetentionPolicy, InvocationRetentionStatus, MasterKeyIdentityRecord,
+        MasterKeyRotationRecord, MasterKeyRotationStatus, MembershipRecord,
+        OnboardingMilestoneResult, OnboardingPresentationMilestone, OperationAgentRef,
+        OperationSampleMetadata, OperationSummary, OperationUsageSummary, OperationVersionRecord,
+        Page, PlatformApiKeyRecord, ProductEventRecord, PublishedAgentCatalog, PublishedAgentTool,
         RegistryOperation, SampleKind, SecretRecord, SecretVersionRecord, SessionRecord,
         SkippedImportOperation, UsageAgentBreakdown, UsageBucket, UsageOperationBreakdown,
         UsageOutcomeBreakdown, UsageOutcomeGroup, UsageRollupRecord, UsageSummary,
@@ -47,14 +47,14 @@ pub mod requests {
         CreateInvocationLogRequest, CreatePlatformApiKeyRequest, CreateSecretRequest,
         CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest,
         DecideApprovalRequest, DetachArtifactSourceRequest, ExpireApprovalRequest,
-        FinishApprovalRequest, FinishImportJobRequest, ImportConflictMode, ImportOperationDraft,
-        ListApprovalRequestsQuery, ListArtifactSourcesQuery, ListInvocationLogsQuery,
-        ListProductEventsQuery, MasterKeyIdentityCandidate, PublishAgentRequest, PublishRequest,
-        RecordOnboardingCompletionRequest, RecordOnboardingMilestoneRequest,
-        RecoverAdminPasswordRequest, RotateSecretRequest, SaveAgentBindingsRequest,
-        SaveAgentCatalogConfigRequest, SaveAuthProfileRequest, SaveDescriptorMetadataRequest,
-        SaveSampleMetadataRequest, SaveWorkspaceUpstreamRequest, UpdateAgentSummaryRequest,
-        UpdateWorkspaceRequest, UsageQuery,
+        FinishApprovalRequest, FinishImportJobRequest, ImportConflictMode, ImportJobSourceEnvelope,
+        ImportOperationDraft, ListApprovalRequestsQuery, ListArtifactSourcesQuery,
+        ListInvocationLogsQuery, ListProductEventsQuery, MasterKeyIdentityCandidate,
+        PublishAgentRequest, PublishRequest, RecordOnboardingCompletionRequest,
+        RecordOnboardingMilestoneRequest, RecoverAdminPasswordRequest, RotateSecretRequest,
+        SaveAgentBindingsRequest, SaveAgentCatalogConfigRequest, SaveAuthProfileRequest,
+        SaveDescriptorMetadataRequest, SaveSampleMetadataRequest, SaveWorkspaceUpstreamRequest,
+        UpdateAgentSummaryRequest, UpdateWorkspaceRequest, UsageQuery,
     };
 }
 
@@ -83,14 +83,14 @@ pub use model::{
     CreateVersionRequest, CreateWorkspaceRequest, CreateYamlImportJobRequest,
     DecideApprovalRequest, DescriptorKind, DescriptorMetadata, DetachArtifactSourceRequest,
     ExpireApprovalRequest, FinishApprovalRequest, FinishImportJobRequest, ImportConflictMode,
-    ImportJob, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobStatus,
-    ImportOperationDraft, InvitationRecord, InvocationHistoryLoss, InvocationHistoryLossCategory,
-    InvocationHistoryWriteOutcome, InvocationLogRecord, InvocationRetentionOutcome,
-    InvocationRetentionPolicy, InvocationRetentionStatus, ListApprovalRequestsQuery,
-    ListArtifactSourcesQuery, ListInvocationLogsQuery, ListProductEventsQuery,
-    MASTER_KEY_CIPHER_CONTRACT, MAX_ARTIFACT_CLAIM_RECOVERY_BATCH, MAX_ARTIFACT_SOURCE_PAGE_SIZE,
-    MasterKeyIdentityCandidate, MasterKeyIdentityRecord, MasterKeyRotationRecord,
-    MasterKeyRotationStatus, MembershipRecord, OnboardingMilestoneResult,
+    ImportJob, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope,
+    ImportJobStatus, ImportOperationDraft, InvitationRecord, InvocationHistoryLoss,
+    InvocationHistoryLossCategory, InvocationHistoryWriteOutcome, InvocationLogRecord,
+    InvocationRetentionOutcome, InvocationRetentionPolicy, InvocationRetentionStatus,
+    ListApprovalRequestsQuery, ListArtifactSourcesQuery, ListInvocationLogsQuery,
+    ListProductEventsQuery, MASTER_KEY_CIPHER_CONTRACT, MAX_ARTIFACT_CLAIM_RECOVERY_BATCH,
+    MAX_ARTIFACT_SOURCE_PAGE_SIZE, MasterKeyIdentityCandidate, MasterKeyIdentityRecord,
+    MasterKeyRotationRecord, MasterKeyRotationStatus, MembershipRecord, OnboardingMilestoneResult,
     OnboardingPresentationMilestone, OperationAgentRef, OperationSampleMetadata,
     OperationStateExpectation, OperationSummary, OperationUsageSummary, OperationVersionRecord,
     Page, PlatformApiKeyRecord, ProductEventRecord, PublishAgentRequest, PublishRequest,

crates/crank-registry/src/model.rs

@@ -1,3 +1,4 @@
+use crank_artifacts::ArtifactRef;
 use crank_core::{
     Agent, AgentId, AgentOperationBinding, AgentStatus, AgentVersion, ApprovalRequest,
     ApprovalRequestId, ApprovalRequestStatus, AuthProfile, DescriptorId, ExecutionErrorCode,
@@ -584,6 +585,16 @@ pub enum ImportJobKind {
     OpenApi,
 }
 
+/// The bounded, opaque authority retained with an OpenAPI import job.
+/// It is persisted inside the existing JSONB payload, never exposed by the
+/// admin response, and is rechecked against workspace-scoped metadata at
+/// apply time.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ImportJobSourceEnvelope {
+    pub source_id: ArtifactSourceId,
+    pub digest: ArtifactRef,
+}
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct ImportJob {
     pub id: ImportJobId,
@@ -611,6 +622,7 @@ pub struct CreateImportJobRequest<'a> {
     pub source_format: &'a str,
     pub source_version: Option<&'a str>,
     pub status: ImportJobStatus,
+    pub source: &'a ImportJobSourceEnvelope,
     pub preview_payload: &'a Value,
     pub created_at: &'a OffsetDateTime,
     pub expires_at: &'a OffsetDateTime,

crates/crank-registry/src/postgres/import_job.rs

@@ -1,12 +1,18 @@
 use super::*;
+use crate::{ArtifactSourceId, ImportJobSourceEnvelope};
+use crank_artifacts::ArtifactRef;
 
 const APPLICATION_RESULT_KEY: &str = "_crank_application_result";
+const IMPORT_JOB_CLEANUP_BATCH: i64 = 128;
+const DANGLING_OPENAPI_SOURCE_GRACE: time::Duration = time::Duration::minutes(5);
 
 impl PostgresRegistry {
     pub async fn create_import_job(
         &self,
         request: CreateImportJobRequest<'_>,
     ) -> Result<(), RegistryError> {
+        validate_source_envelope(request.source, request.preview_payload)?;
+        let mut transaction = self.pool.begin().await?;
         sqlx::query(
             "insert into import_jobs (
                 id,
@@ -34,8 +40,9 @@ impl PostgresRegistry {
         .bind(request.preview_payload)
         .bind(request.created_at)
         .bind(request.expires_at)
-        .execute(&self.pool)
+        .execute(&mut *transaction)
         .await?;
+        transaction.commit().await?;
 
         Ok(())
     }
@@ -137,11 +144,79 @@ impl PostgresRegistry {
     }
 
     pub async fn delete_expired_import_jobs(&self) -> Result<u64, RegistryError> {
-        let result = sqlx::query("delete from import_jobs where expires_at < now()")
-            .execute(&self.pool)
+        let mut transaction = self.pool.begin().await?;
+        let now = sqlx::query_scalar::<_, OffsetDateTime>("select now()")
+            .fetch_one(&mut *transaction)
             .await?;
+        let expired = sqlx::query(
+            "select id, workspace_id, preview_payload
+               from import_jobs
+              where expires_at < now()
+              order by expires_at, id
+              limit $1
+              for update skip locked",
+        )
+        .bind(IMPORT_JOB_CLEANUP_BATCH)
+        .fetch_all(&mut *transaction)
+        .await?;
+        let mut expired_ids = Vec::with_capacity(expired.len());
+        for row in &expired {
+            expired_ids.push(row.try_get::<String, _>("id")?);
+            let workspace_id = WorkspaceId::new(row.try_get::<String, _>("workspace_id")?);
+            let payload = row.try_get::<Value, _>("preview_payload")?;
+            if let Some(source) = source_from_payload(&payload)? {
+                detach_source_in_transaction(
+                    &mut transaction,
+                    &workspace_id,
+                    &source.source_id,
+                    now,
+                )
+                .await?;
+            }
+        }
+        let deleted = if expired_ids.is_empty() {
+            0
+        } else {
+            sqlx::query("delete from import_jobs where id = any($1)")
+                .bind(&expired_ids)
+                .execute(&mut *transaction)
+                .await?
+                .rows_affected()
+        };
 
-        Ok(result.rows_affected())
+        let dangling_cutoff = now.checked_sub(DANGLING_OPENAPI_SOURCE_GRACE).ok_or(
+            RegistryError::InvalidArtifactSource {
+                field: "import_job_source",
+            },
+        )?;
+        let dangling = sqlx::query(
+            "select s.workspace_id, s.source_id
+               from artifact_sources s
+              where s.lifecycle = 'active'
+                and left(s.source_id, 12) = 'src_openapi_'
+                and s.created_at < $1
+                and not exists (
+                    select 1
+                      from import_jobs j
+                     where j.workspace_id = s.workspace_id
+                       and j.preview_payload -> 'source' ->> 'source_id' = s.source_id
+                )
+              order by s.created_at, s.workspace_id, s.source_id
+              limit $2
+              for update of s skip locked",
+        )
+        .bind(dangling_cutoff)
+        .bind(IMPORT_JOB_CLEANUP_BATCH)
+        .fetch_all(&mut *transaction)
+        .await?;
+        for row in dangling {
+            let workspace_id = WorkspaceId::new(row.try_get::<String, _>("workspace_id")?);
+            let source_id = ArtifactSourceId::new(row.try_get::<String, _>("source_id")?);
+            detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now).await?;
+        }
+        transaction.commit().await?;
+
+        Ok(deleted)
     }
 }
 
@@ -164,6 +239,10 @@ async fn apply_import_job_transaction(
     })?;
     let status = deserialize_enum_text::<ImportJobStatus>(row.try_get("status")?, "status")?;
     let mut preview_payload = row.try_get::<Value, _>("preview_payload")?;
+    let source =
+        source_from_payload(&preview_payload)?.ok_or(RegistryError::InvalidArtifactSource {
+            field: "import_job_source",
+        })?;
 
     if status == ImportJobStatus::Completed
         && let Some(result) = stored_application_result(&preview_payload)?
@@ -181,6 +260,30 @@ async fn apply_import_job_transaction(
         .fetch_one(&mut **tx)
         .await?;
 
+    let recorded_source = sqlx::query(
+        "select s.lifecycle, b.artifact_ref
+           from artifact_sources s
+           join artifact_blobs b on b.digest = s.blob_digest
+          where s.workspace_id = $1 and s.source_id = $2
+          for update",
+    )
+    .bind(request.workspace_id.as_str())
+    .bind(source.source_id.as_str())
+    .fetch_optional(&mut **tx)
+    .await?;
+    let source_is_current = recorded_source
+        .as_ref()
+        .map(|row| {
+            let lifecycle = row.try_get::<String, _>("lifecycle").ok();
+            let artifact_ref = row.try_get::<String, _>("artifact_ref").ok();
+            lifecycle.as_deref() == Some("active")
+                && artifact_ref.as_deref() == Some(source.digest.as_str())
+        })
+        .unwrap_or(false);
+    if !source_is_current {
+        return Err(RegistryError::SourceUnavailable);
+    }
+
     let mut result = ImportJobApplyResult {
         application_key: request.application_key.to_owned(),
         ..ImportJobApplyResult::default()
@@ -257,9 +360,91 @@ async fn apply_import_job_transaction(
     .execute(&mut **tx)
     .await?;
 
+    detach_source_in_transaction(
+        tx,
+        request.workspace_id,
+        &source.source_id,
+        *request.finished_at,
+    )
+    .await?;
+
     Ok(result)
 }
 
+fn validate_source_envelope(
+    source: &ImportJobSourceEnvelope,
+    preview_payload: &Value,
+) -> Result<(), RegistryError> {
+    if source.source_id.as_str().len() > 132 {
+        return Err(RegistryError::InvalidArtifactSource {
+            field: "import_job_source",
+        });
+    }
+    let parsed =
+        source_from_payload(preview_payload)?.ok_or(RegistryError::InvalidArtifactSource {
+            field: "import_job_source",
+        })?;
+    if parsed != *source {
+        return Err(RegistryError::InvalidArtifactSource {
+            field: "import_job_source",
+        });
+    }
+    Ok(())
+}
+
+fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope>, RegistryError> {
+    let Some(source) = payload.get("source") else {
+        return Ok(None);
+    };
+    let has_source_id = source.get("source_id").is_some();
+    let has_digest = source.get("digest").is_some();
+    if !has_source_id && !has_digest {
+        // Jobs created before the verified-source envelope used `source` for
+        // OpenAPI format/version metadata. Expiry cleanup must remain able to
+        // delete those jobs during a rolling upgrade.
+        return Ok(None);
+    }
+    let source_id = source
+        .get("source_id")
+        .and_then(Value::as_str)
+        .filter(|value| value.len() <= 132)
+        .ok_or(RegistryError::InvalidArtifactSource {
+            field: "import_job_source",
+        })?;
+    let digest = source.get("digest").and_then(Value::as_str).ok_or(
+        RegistryError::InvalidArtifactSource {
+            field: "import_job_source",
+        },
+    )?;
+    let digest = ArtifactRef::parse(digest).map_err(|_| RegistryError::InvalidArtifactSource {
+        field: "import_job_source",
+    })?;
+    Ok(Some(ImportJobSourceEnvelope {
+        source_id: ArtifactSourceId::new(source_id),
+        digest,
+    }))
+}
+
+async fn detach_source_in_transaction(
+    transaction: &mut Transaction<'_, Postgres>,
+    workspace_id: &WorkspaceId,
+    source_id: &ArtifactSourceId,
+    detached_at: OffsetDateTime,
+) -> Result<(), RegistryError> {
+    sqlx::query(
+        "update artifact_sources
+            set lifecycle = 'detached', updated_at = $1, detached_at = $1
+          where workspace_id = $2 and source_id = $3
+            and lifecycle = 'active'",
+    )
+    .bind(detached_at)
+    .bind(workspace_id.as_str())
+    .bind(source_id.as_str())
+    .execute(&mut **transaction)
+    .await?;
+    Ok(())
+}
+
 fn stored_application_result(
     preview_payload: &Value,
 ) -> Result<Option<ImportJobApplyResult>, RegistryError> {