use super::*; use std::collections::VecDeque; use tracing::warn; pub(super) struct MaterializedDependencies { pub(super) snapshots: Vec, envelopes: Vec, canonical_uris: Vec, guards: Vec, pub(super) unsafe_source: bool, } struct StagedDependency { canonical_uri: String, bytes: Vec, document: String, } pub(super) struct ImportReplayContext<'a> { pub(super) workspace_id: &'a WorkspaceId, pub(super) job_id: &'a ImportJobId, pub(super) application_key: &'a str, pub(super) conflict_mode: ImportConflictMode, pub(super) finished_at: &'a OffsetDateTime, } impl MaterializedDependencies { fn empty() -> Self { Self { snapshots: Vec::new(), envelopes: Vec::new(), canonical_uris: Vec::new(), guards: Vec::new(), unsafe_source: false, } } pub(super) fn payload(&self) -> Vec { self.envelopes .iter() .zip(&self.canonical_uris) .map(|(dependency, canonical_uri)| { json!({ "source_id": dependency.source_id.as_str(), "digest": dependency.digest.as_str(), "canonical_uri": canonical_uri, }) }) .collect() } pub(super) fn snapshot_payload(&self) -> Vec { self.snapshots .iter() .filter_map(|snapshot| { self.envelopes .iter() .find(|dependency| dependency.digest.digest_hex() == snapshot.digest.as_str()) .map(|dependency| { json!({ "source_id": dependency.source_id.as_str(), "digest": dependency.digest.as_str(), "canonical_uri": snapshot.canonical_uri, }) }) }) .collect() } pub(super) fn disarm(&mut self) { for guard in &mut self.guards { guard.disarm(); } } pub(super) async fn detach_all(&mut self) { for guard in &mut self.guards { guard.detach_now().await; } } } impl AdminService { pub(super) async fn fail_openapi_import_or_replay( &self, context: &ImportReplayContext<'_>, error_text: &'static str, error: ApiError, ) -> Result { let empty = json!([]); self.registry .finish_import_job(FinishImportJobRequest { id: context.job_id, status: ImportJobStatus::Failed, created_operation_ids: &empty, error_text: Some(error_text), finished_at: context.finished_at, }) .await?; // `finish_import_job` preserves an already Completed row under its // lock. This read distinguishes that race from the terminal Failed // transition and returns the immutable canonical application result. let latest = self .registry .get_import_job(context.workspace_id, context.job_id) .await?; if latest.is_some_and(|job| job.status == ImportJobStatus::Completed) { let applied = self .registry .apply_import_job(ApplyImportJobRequest { id: context.job_id, workspace_id: context.workspace_id, application_key: context.application_key, conflict_mode: context.conflict_mode, operations: &[], pre_skipped: &[], finished_at: context.finished_at, }) .await?; return Ok(openapi_import_response(applied)); } Err(error) } pub(super) async fn materialize_external_reference_snapshots( &self, workspace_id: &WorkspaceId, primary_bytes: &[u8], created_at: OffsetDateTime, ) -> MaterializedDependencies { let materialized_count = std::sync::atomic::AtomicUsize::new(0); match tokio::time::timeout( self.external_reference_materialization_timeout, self.materialize_external_reference_snapshots_inner( workspace_id, primary_bytes, created_at, &materialized_count, ), ) .await { Ok(dependencies) => dependencies, Err(_) => { materialization_failure( "chain", "timeout", materialized_count.load(std::sync::atomic::Ordering::Relaxed), ); MaterializedDependencies::empty() } } } async fn materialize_external_reference_snapshots_inner( &self, workspace_id: &WorkspaceId, primary_bytes: &[u8], created_at: OffsetDateTime, materialized_count: &std::sync::atomic::AtomicUsize, ) -> MaterializedDependencies { if !self .external_reference_normalization .external_references_enabled { return MaterializedDependencies::empty(); } let Ok(primary) = std::str::from_utf8(primary_bytes) else { materialization_failure("primary_decode", "invalid_utf8", 0); return MaterializedDependencies::empty(); }; let Ok(primary_references) = crank_import::rest::reference_uris(primary, &self.external_reference_normalization) else { materialization_failure("primary_scan", "invalid_document", 0); return MaterializedDependencies::empty(); }; let mut queue = VecDeque::new(); for reference in primary_references { if let Some(uri) = canonical_external_document_uri(None, &reference) { queue.push_back((uri, 1usize)); } } let mut seen = BTreeSet::new(); let mut staged = Vec::::new(); let mut staged_documents = vec![primary.to_owned()]; while let Some((canonical_uri, depth)) = queue.pop_front() { if !seen.insert(canonical_uri.clone()) || depth > self.external_reference_normalization.max_reference_depth || seen.len() > self .external_reference_normalization .max_reference_documents { continue; } let bytes = match self.external_reference_fetcher.get(&canonical_uri).await { Ok(bytes) => bytes, Err(error) => { materialization_failure( "fetch", external_fetch_error_code(&error), staged.len(), ); continue; } }; let Ok(document) = std::str::from_utf8(&bytes).map(str::to_owned) else { materialization_failure("dependency_decode", "invalid_utf8", staged.len()); continue; }; let Ok(references) = crank_import::rest::external_reference_uris( &document, &self.external_reference_normalization, ) else { materialization_failure("dependency_scan", "invalid_document", staged.len()); continue; }; staged_documents.push(document.clone()); staged.push(StagedDependency { canonical_uri: canonical_uri.clone(), bytes, document, }); for reference in references { if let Some(uri) = canonical_external_document_uri(Some(&canonical_uri), &reference) { queue.push_back((uri, depth.saturating_add(1))); } } } let staged_snapshots = staged .iter() .filter_map(|dependency| { let digest = format!("{:x}", Sha256::digest(&dependency.bytes)); let digest = crank_import::rest::SourceDigest::parse(digest).ok()?; Some(crank_import::rest::ExternalDocumentSnapshot { canonical_uri: dependency.canonical_uri.clone(), digest, document: dependency.document.clone(), }) }) .collect::>(); let primary_digest = crank_import::rest::SourceDigest::parse(format!("{:x}", Sha256::digest(primary_bytes))) .expect("SHA-256 digest is valid"); let raw_bundle_is_unsafe = crank_import::rest::validate_document_bundle_source_safe( &staged_documents .iter() .map(String::as_str) .collect::>(), &self.external_reference_normalization, ) .is_err(); let resolved_bundle_is_unsafe = crank_import::rest::validate_resolved_bundle_source_safe( primary, primary_digest, &staged_snapshots, &self.external_reference_normalization, ) .is_err(); if raw_bundle_is_unsafe || resolved_bundle_is_unsafe { materialization_failure("dependency_redaction", "invalid_document", staged.len()); let mut rejected = MaterializedDependencies::empty(); rejected.unsafe_source = true; return rejected; } let mut result = MaterializedDependencies::empty(); let mut dependency_by_digest = BTreeMap::::new(); for dependency in staged { let store = self.artifact_store.clone(); let artifact_bytes = dependency.bytes.clone(); let Ok(Ok(artifact)) = tokio::task::spawn_blocking(move || store.put_registered(&artifact_bytes)).await else { materialization_failure( "artifact_store", "storage_unavailable", result.snapshots.len(), ); continue; }; let digest_key = artifact.artifact_ref().as_str().to_owned(); let envelope_index = if let Some(index) = dependency_by_digest.get(&digest_key) { *index } else { let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi_dep")); let Ok(source) = self .registry .create_artifact_source(CreateArtifactSourceRequest { workspace_id, source_id: &source_id, artifact: &artifact, mime_type: "application/octet-stream", sensitivity: ArtifactSourceSensitivity::Internal, created_at, }) .await else { materialization_failure( "source_create", "registry_unavailable", result.snapshots.len(), ); continue; }; let mut guard = SourceDetachGuard::new( self.registry.clone(), workspace_id.clone(), source_id.clone(), source.updated_at, ); let Ok(verified) = self .registry .read_artifact_source( std::sync::Arc::clone(&self.artifact_store), workspace_id, &source_id, ) .await else { materialization_failure( "source_verify", "source_unavailable", result.snapshots.len(), ); guard.detach_now().await; continue; }; if verified.source.blob.artifact_ref != *artifact.artifact_ref() { materialization_failure( "source_verify", "source_integrity", result.snapshots.len(), ); guard.detach_now().await; continue; } let index = result.envelopes.len(); result.envelopes.push(ImportJobSourceEnvelope { source_id: source_id.clone(), digest: artifact.artifact_ref().clone(), }); result.guards.push(guard); dependency_by_digest.insert(digest_key, index); index }; let dependency_envelope = &result.envelopes[envelope_index]; let Ok(snapshot_digest) = crank_import::rest::SourceDigest::parse( dependency_envelope.digest.digest_hex().to_owned(), ) else { materialization_failure("snapshot", "source_integrity", result.snapshots.len()); continue; }; result .snapshots .push(crank_import::rest::ExternalDocumentSnapshot { canonical_uri: dependency.canonical_uri.clone(), digest: snapshot_digest, document: dependency.document, }); materialized_count.store(result.snapshots.len(), std::sync::atomic::Ordering::Relaxed); result.canonical_uris.push(dependency.canonical_uri); } // Registry ownership is one envelope per immutable digest, while the // resolver may map several canonical URIs to that same snapshot. Keep // payload rows aligned with envelopes and add aliases separately. if result.envelopes.len() != result.canonical_uris.len() { let mut canonical_by_digest = BTreeMap::new(); for snapshot in &result.snapshots { canonical_by_digest .entry(snapshot.digest.as_str().to_owned()) .or_insert_with(|| snapshot.canonical_uri.clone()); } result.canonical_uris = result .envelopes .iter() .map(|envelope| { canonical_by_digest .get(envelope.digest.digest_hex()) .cloned() .unwrap_or_default() }) .collect(); } result } pub(super) async fn read_import_job_dependencies( &self, workspace_id: &WorkspaceId, payload: &serde_json::Value, locale: OpenApiUploadLocale, ) -> Result, ApiError> { let dependencies = import_job_dependencies(payload, locale)?; let mut snapshots = Vec::with_capacity(dependencies.len()); for (canonical_uri, dependency) in dependencies { let verified = self .registry .read_artifact_source( std::sync::Arc::clone(&self.artifact_store), workspace_id, &dependency.source_id, ) .await .map_err(|error| openapi_source_error(locale, error))?; if verified.source.blob.artifact_ref != dependency.digest { return Err(ApiError::openapi_upload(locale, "source_integrity")); } let document = String::from_utf8(verified.bytes) .map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?; let digest = crank_import::rest::SourceDigest::parse(dependency.digest.digest_hex().to_owned()) .map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?; snapshots.push(crank_import::rest::ExternalDocumentSnapshot { canonical_uri, digest, document, }); } Ok(snapshots) } } fn external_fetch_error_code(error: &crank_runtime::ExternalReferenceFetchError) -> &'static str { match error { crank_runtime::ExternalReferenceFetchError::Disabled => "disabled", crank_runtime::ExternalReferenceFetchError::InvalidUrl => "invalid_url", crank_runtime::ExternalReferenceFetchError::TargetNotAllowed => "target_not_allowed", crank_runtime::ExternalReferenceFetchError::RedirectNotAllowed => "redirect_not_allowed", crank_runtime::ExternalReferenceFetchError::ResponseTooLarge { .. } => "response_too_large", crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { .. } => "unexpected_status", crank_runtime::ExternalReferenceFetchError::Transport { timeout: true, .. } => "timeout", crank_runtime::ExternalReferenceFetchError::Transport { .. } => "transport", crank_runtime::ExternalReferenceFetchError::InvalidConfiguration => "invalid_configuration", } } fn materialization_failure(stage: &'static str, error_code: &'static str, count: usize) { warn!( name: "admin.openapi_import.materialization_failed", stage, error_code, count, "external OpenAPI materialization failed" ); } #[cfg(test)] mod tests { use super::external_fetch_error_code; #[test] fn unexpected_status_has_a_stable_safe_error_code() { let error = crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { status: 500 }; assert_eq!(external_fetch_error_code(&error), "unexpected_status"); } }