feat(import): resolve references and schema composition
This commit is contained in:
@@ -0,0 +1,405 @@
|
||||
use super::*;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::warn;
|
||||
|
||||
pub(super) struct MaterializedDependencies {
|
||||
pub(super) snapshots: Vec<crank_import::rest::ExternalDocumentSnapshot>,
|
||||
envelopes: Vec<ImportJobSourceEnvelope>,
|
||||
canonical_uris: Vec<String>,
|
||||
guards: Vec<SourceDetachGuard>,
|
||||
}
|
||||
|
||||
pub(super) struct ImportReplayContext<'a> {
|
||||
pub(super) workspace_id: &'a WorkspaceId,
|
||||
pub(super) job_id: &'a ImportJobId,
|
||||
pub(super) application_key: &'a str,
|
||||
pub(super) conflict_mode: ImportConflictMode,
|
||||
pub(super) finished_at: &'a OffsetDateTime,
|
||||
}
|
||||
|
||||
impl MaterializedDependencies {
|
||||
fn empty() -> Self {
|
||||
Self {
|
||||
snapshots: Vec::new(),
|
||||
envelopes: Vec::new(),
|
||||
canonical_uris: Vec::new(),
|
||||
guards: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn payload(&self) -> Vec<serde_json::Value> {
|
||||
self.envelopes
|
||||
.iter()
|
||||
.zip(&self.canonical_uris)
|
||||
.map(|(dependency, canonical_uri)| {
|
||||
json!({
|
||||
"source_id": dependency.source_id.as_str(),
|
||||
"digest": dependency.digest.as_str(),
|
||||
"canonical_uri": canonical_uri,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn snapshot_payload(&self) -> Vec<serde_json::Value> {
|
||||
self.snapshots
|
||||
.iter()
|
||||
.filter_map(|snapshot| {
|
||||
self.envelopes
|
||||
.iter()
|
||||
.find(|dependency| dependency.digest.digest_hex() == snapshot.digest.as_str())
|
||||
.map(|dependency| {
|
||||
json!({
|
||||
"source_id": dependency.source_id.as_str(),
|
||||
"digest": dependency.digest.as_str(),
|
||||
"canonical_uri": snapshot.canonical_uri,
|
||||
})
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn disarm(&mut self) {
|
||||
for guard in &mut self.guards {
|
||||
guard.disarm();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn detach_all(&mut self) {
|
||||
for guard in &mut self.guards {
|
||||
guard.detach_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminService {
|
||||
pub(super) async fn fail_openapi_import_or_replay(
|
||||
&self,
|
||||
context: &ImportReplayContext<'_>,
|
||||
error_text: &'static str,
|
||||
error: ApiError,
|
||||
) -> Result<OpenApiImportCreateResponse, ApiError> {
|
||||
let empty = json!([]);
|
||||
self.registry
|
||||
.finish_import_job(FinishImportJobRequest {
|
||||
id: context.job_id,
|
||||
status: ImportJobStatus::Failed,
|
||||
created_operation_ids: &empty,
|
||||
error_text: Some(error_text),
|
||||
finished_at: context.finished_at,
|
||||
})
|
||||
.await?;
|
||||
|
||||
// `finish_import_job` preserves an already Completed row under its
|
||||
// lock. This read distinguishes that race from the terminal Failed
|
||||
// transition and returns the immutable canonical application result.
|
||||
let latest = self
|
||||
.registry
|
||||
.get_import_job(context.workspace_id, context.job_id)
|
||||
.await?;
|
||||
if latest.is_some_and(|job| job.status == ImportJobStatus::Completed) {
|
||||
let applied = self
|
||||
.registry
|
||||
.apply_import_job(ApplyImportJobRequest {
|
||||
id: context.job_id,
|
||||
workspace_id: context.workspace_id,
|
||||
application_key: context.application_key,
|
||||
conflict_mode: context.conflict_mode,
|
||||
operations: &[],
|
||||
pre_skipped: &[],
|
||||
finished_at: context.finished_at,
|
||||
})
|
||||
.await?;
|
||||
return Ok(openapi_import_response(applied));
|
||||
}
|
||||
|
||||
Err(error)
|
||||
}
|
||||
|
||||
pub(super) async fn materialize_external_reference_snapshots(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
primary_bytes: &[u8],
|
||||
created_at: OffsetDateTime,
|
||||
) -> MaterializedDependencies {
|
||||
let materialized_count = std::sync::atomic::AtomicUsize::new(0);
|
||||
match tokio::time::timeout(
|
||||
self.external_reference_materialization_timeout,
|
||||
self.materialize_external_reference_snapshots_inner(
|
||||
workspace_id,
|
||||
primary_bytes,
|
||||
created_at,
|
||||
&materialized_count,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dependencies) => dependencies,
|
||||
Err(_) => {
|
||||
materialization_failure(
|
||||
"chain",
|
||||
"timeout",
|
||||
materialized_count.load(std::sync::atomic::Ordering::Relaxed),
|
||||
);
|
||||
MaterializedDependencies::empty()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn materialize_external_reference_snapshots_inner(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
primary_bytes: &[u8],
|
||||
created_at: OffsetDateTime,
|
||||
materialized_count: &std::sync::atomic::AtomicUsize,
|
||||
) -> MaterializedDependencies {
|
||||
if !self
|
||||
.external_reference_normalization
|
||||
.external_references_enabled
|
||||
{
|
||||
return MaterializedDependencies::empty();
|
||||
}
|
||||
let Ok(primary) = std::str::from_utf8(primary_bytes) else {
|
||||
materialization_failure("primary_decode", "invalid_utf8", 0);
|
||||
return MaterializedDependencies::empty();
|
||||
};
|
||||
let Ok(primary_references) =
|
||||
crank_import::rest::reference_uris(primary, &self.external_reference_normalization)
|
||||
else {
|
||||
materialization_failure("primary_scan", "invalid_document", 0);
|
||||
return MaterializedDependencies::empty();
|
||||
};
|
||||
let mut queue = VecDeque::new();
|
||||
for reference in primary_references {
|
||||
if let Some(uri) = canonical_external_document_uri(None, &reference) {
|
||||
queue.push_back((uri, 1usize));
|
||||
}
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut result = MaterializedDependencies::empty();
|
||||
let mut dependency_by_digest = BTreeMap::<String, usize>::new();
|
||||
while let Some((canonical_uri, depth)) = queue.pop_front() {
|
||||
if !seen.insert(canonical_uri.clone())
|
||||
|| depth > self.external_reference_normalization.max_reference_depth
|
||||
|| seen.len()
|
||||
> self
|
||||
.external_reference_normalization
|
||||
.max_reference_documents
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let bytes = match self.external_reference_fetcher.get(&canonical_uri).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
materialization_failure(
|
||||
"fetch",
|
||||
external_fetch_error_code(&error),
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Ok(document) = std::str::from_utf8(&bytes).map(str::to_owned) else {
|
||||
materialization_failure(
|
||||
"dependency_decode",
|
||||
"invalid_utf8",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let Ok(references) = crank_import::rest::external_reference_uris(
|
||||
&document,
|
||||
&self.external_reference_normalization,
|
||||
) else {
|
||||
materialization_failure(
|
||||
"dependency_scan",
|
||||
"invalid_document",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let store = self.artifact_store.clone();
|
||||
let artifact_bytes = bytes.clone();
|
||||
let Ok(Ok(artifact)) =
|
||||
tokio::task::spawn_blocking(move || store.put_registered(&artifact_bytes)).await
|
||||
else {
|
||||
materialization_failure(
|
||||
"artifact_store",
|
||||
"storage_unavailable",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let digest_key = artifact.artifact_ref().as_str().to_owned();
|
||||
let envelope_index = if let Some(index) = dependency_by_digest.get(&digest_key) {
|
||||
*index
|
||||
} else {
|
||||
let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi_dep"));
|
||||
let Ok(source) = self
|
||||
.registry
|
||||
.create_artifact_source(CreateArtifactSourceRequest {
|
||||
workspace_id,
|
||||
source_id: &source_id,
|
||||
artifact: &artifact,
|
||||
mime_type: "application/octet-stream",
|
||||
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||
created_at,
|
||||
})
|
||||
.await
|
||||
else {
|
||||
materialization_failure(
|
||||
"source_create",
|
||||
"registry_unavailable",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let mut guard = SourceDetachGuard::new(
|
||||
self.registry.clone(),
|
||||
workspace_id.clone(),
|
||||
source_id.clone(),
|
||||
source.updated_at,
|
||||
);
|
||||
let Ok(verified) = self
|
||||
.registry
|
||||
.read_artifact_source(
|
||||
std::sync::Arc::clone(&self.artifact_store),
|
||||
workspace_id,
|
||||
&source_id,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
materialization_failure(
|
||||
"source_verify",
|
||||
"source_unavailable",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
guard.detach_now().await;
|
||||
continue;
|
||||
};
|
||||
if verified.source.blob.artifact_ref != *artifact.artifact_ref() {
|
||||
materialization_failure(
|
||||
"source_verify",
|
||||
"source_integrity",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
guard.detach_now().await;
|
||||
continue;
|
||||
}
|
||||
let index = result.envelopes.len();
|
||||
result.envelopes.push(ImportJobSourceEnvelope {
|
||||
source_id: source_id.clone(),
|
||||
digest: artifact.artifact_ref().clone(),
|
||||
});
|
||||
result.guards.push(guard);
|
||||
dependency_by_digest.insert(digest_key, index);
|
||||
index
|
||||
};
|
||||
let dependency = &result.envelopes[envelope_index];
|
||||
let snapshot_digest = match crank_import::rest::SourceDigest::parse(
|
||||
dependency.digest.digest_hex().to_owned(),
|
||||
) {
|
||||
Ok(digest) => digest,
|
||||
Err(_) => {
|
||||
materialization_failure("snapshot", "source_integrity", result.snapshots.len());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
result
|
||||
.snapshots
|
||||
.push(crank_import::rest::ExternalDocumentSnapshot {
|
||||
canonical_uri: canonical_uri.clone(),
|
||||
digest: snapshot_digest,
|
||||
document,
|
||||
});
|
||||
materialized_count.store(result.snapshots.len(), std::sync::atomic::Ordering::Relaxed);
|
||||
result.canonical_uris.push(canonical_uri.clone());
|
||||
for reference in references {
|
||||
if let Some(uri) = canonical_external_document_uri(Some(&canonical_uri), &reference)
|
||||
{
|
||||
queue.push_back((uri, depth.saturating_add(1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Registry ownership is one envelope per immutable digest, while the
|
||||
// resolver may map several canonical URIs to that same snapshot. Keep
|
||||
// payload rows aligned with envelopes and add aliases separately.
|
||||
if result.envelopes.len() != result.canonical_uris.len() {
|
||||
let mut canonical_by_digest = BTreeMap::new();
|
||||
for snapshot in &result.snapshots {
|
||||
canonical_by_digest
|
||||
.entry(snapshot.digest.as_str().to_owned())
|
||||
.or_insert_with(|| snapshot.canonical_uri.clone());
|
||||
}
|
||||
result.canonical_uris = result
|
||||
.envelopes
|
||||
.iter()
|
||||
.map(|envelope| {
|
||||
canonical_by_digest
|
||||
.get(envelope.digest.digest_hex())
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) async fn read_import_job_dependencies(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<Vec<crank_import::rest::ExternalDocumentSnapshot>, ApiError> {
|
||||
let dependencies = import_job_dependencies(payload, locale)?;
|
||||
let mut snapshots = Vec::with_capacity(dependencies.len());
|
||||
for (canonical_uri, dependency) in dependencies {
|
||||
let verified = self
|
||||
.registry
|
||||
.read_artifact_source(
|
||||
std::sync::Arc::clone(&self.artifact_store),
|
||||
workspace_id,
|
||||
&dependency.source_id,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| openapi_source_error(locale, error))?;
|
||||
if verified.source.blob.artifact_ref != dependency.digest {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let document = String::from_utf8(verified.bytes)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let digest =
|
||||
crank_import::rest::SourceDigest::parse(dependency.digest.digest_hex().to_owned())
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
snapshots.push(crank_import::rest::ExternalDocumentSnapshot {
|
||||
canonical_uri,
|
||||
digest,
|
||||
document,
|
||||
});
|
||||
}
|
||||
Ok(snapshots)
|
||||
}
|
||||
}
|
||||
fn external_fetch_error_code(error: &crank_runtime::ExternalReferenceFetchError) -> &'static str {
|
||||
match error {
|
||||
crank_runtime::ExternalReferenceFetchError::Disabled => "disabled",
|
||||
crank_runtime::ExternalReferenceFetchError::InvalidUrl => "invalid_url",
|
||||
crank_runtime::ExternalReferenceFetchError::TargetNotAllowed => "target_not_allowed",
|
||||
crank_runtime::ExternalReferenceFetchError::RedirectNotAllowed => "redirect_not_allowed",
|
||||
crank_runtime::ExternalReferenceFetchError::ResponseTooLarge { .. } => "response_too_large",
|
||||
crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { .. } => "unexpected_status",
|
||||
crank_runtime::ExternalReferenceFetchError::Transport { timeout: true, .. } => "timeout",
|
||||
crank_runtime::ExternalReferenceFetchError::Transport { .. } => "transport",
|
||||
crank_runtime::ExternalReferenceFetchError::InvalidConfiguration => "invalid_configuration",
|
||||
}
|
||||
}
|
||||
|
||||
fn materialization_failure(stage: &'static str, error_code: &'static str, count: usize) {
|
||||
warn!(
|
||||
name: "admin.openapi_import.materialization_failed",
|
||||
stage,
|
||||
error_code,
|
||||
count,
|
||||
"external OpenAPI materialization failed"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crank_core::WorkspaceId;
|
||||
use crank_registry::{ArtifactSourceId, DetachArtifactSourceRequest, ImportJobSourceEnvelope};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::{error::ApiError, service::OpenApiUploadLocale};
|
||||
|
||||
use super::canonical_external_document_uri;
|
||||
|
||||
const MAX_IMPORT_JOB_DOCUMENTS: usize = 32;
|
||||
|
||||
pub(super) fn import_job_source(
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<ImportJobSourceEnvelope, ApiError> {
|
||||
let source = payload
|
||||
.get("source")
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let source_id = source
|
||||
.get("source_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let digest = source
|
||||
.get("digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| value.parse().ok())
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
Ok(ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn import_job_dependencies(
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<Vec<(String, ImportJobSourceEnvelope)>, ApiError> {
|
||||
let empty = Vec::new();
|
||||
let items = payload
|
||||
.get("dependency_snapshots")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.unwrap_or(&empty);
|
||||
if items.len() > MAX_IMPORT_JOB_DOCUMENTS {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let mut canonical_uris = BTreeSet::new();
|
||||
items
|
||||
.iter()
|
||||
.map(|item| {
|
||||
let canonical_uri = item
|
||||
.get("canonical_uri")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| {
|
||||
canonical_external_document_uri(None, value).as_deref() == Some(*value)
|
||||
})
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
if !canonical_uris.insert(canonical_uri.to_owned()) {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let source_id = item
|
||||
.get("source_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let digest = item
|
||||
.get("digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| value.parse().ok())
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
Ok((
|
||||
canonical_uri.to_owned(),
|
||||
ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
},
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn import_job_normalization_config(
|
||||
payload: &serde_json::Value,
|
||||
current: &crank_import::rest::NormalizationConfig,
|
||||
legacy_v1: bool,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<crank_import::rest::NormalizationConfig, ApiError> {
|
||||
if legacy_v1 {
|
||||
return Ok(current.clone());
|
||||
}
|
||||
payload
|
||||
.pointer("/normalization/config")
|
||||
.cloned()
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))
|
||||
.and_then(|value| {
|
||||
serde_json::from_value(value)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) struct SourceDetachGuard {
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl SourceDetachGuard {
|
||||
pub(super) fn new(
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
workspace_id,
|
||||
source_id,
|
||||
expected_updated_at,
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
pub(super) async fn detach_now(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
self.armed = false;
|
||||
let _ = self
|
||||
.registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &self.workspace_id,
|
||||
source_id: &self.source_id,
|
||||
expected_updated_at: Some(self.expected_updated_at),
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SourceDetachGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
let registry = self.registry.clone();
|
||||
let workspace_id = self.workspace_id.clone();
|
||||
let source_id = self.source_id.clone();
|
||||
let expected_updated_at = Some(self.expected_updated_at);
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
let _ = registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &workspace_id,
|
||||
source_id: &source_id,
|
||||
expected_updated_at,
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user