Files
crank/apps/admin-api/src/service/imports/external_references.rs
T
bsodfather e93de7015a
CI / Rust Checks (pull_request) Failing after 1s
CI / UI Checks (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Deploy (pull_request) Skipped
feat(import): complete OpenAPI onboarding and verify delivery
2026-10-09 17:50:38 +03:00

461 lines
18 KiB
Rust

use super::*;
use std::collections::VecDeque;
use tracing::warn;
pub(super) struct MaterializedDependencies {
pub(super) snapshots: Vec<crank_import::rest::ExternalDocumentSnapshot>,
envelopes: Vec<ImportJobSourceEnvelope>,
canonical_uris: Vec<String>,
guards: Vec<SourceDetachGuard>,
pub(super) unsafe_source: bool,
}
struct StagedDependency {
canonical_uri: String,
bytes: Vec<u8>,
document: String,
}
pub(super) struct ImportReplayContext<'a> {
pub(super) workspace_id: &'a WorkspaceId,
pub(super) job_id: &'a ImportJobId,
pub(super) application_key: &'a str,
pub(super) conflict_mode: ImportConflictMode,
pub(super) finished_at: &'a OffsetDateTime,
}
impl MaterializedDependencies {
fn empty() -> Self {
Self {
snapshots: Vec::new(),
envelopes: Vec::new(),
canonical_uris: Vec::new(),
guards: Vec::new(),
unsafe_source: false,
}
}
pub(super) fn payload(&self) -> Vec<serde_json::Value> {
self.envelopes
.iter()
.zip(&self.canonical_uris)
.map(|(dependency, canonical_uri)| {
json!({
"source_id": dependency.source_id.as_str(),
"digest": dependency.digest.as_str(),
"canonical_uri": canonical_uri,
})
})
.collect()
}
pub(super) fn snapshot_payload(&self) -> Vec<serde_json::Value> {
self.snapshots
.iter()
.filter_map(|snapshot| {
self.envelopes
.iter()
.find(|dependency| dependency.digest.digest_hex() == snapshot.digest.as_str())
.map(|dependency| {
json!({
"source_id": dependency.source_id.as_str(),
"digest": dependency.digest.as_str(),
"canonical_uri": snapshot.canonical_uri,
})
})
})
.collect()
}
pub(super) fn disarm(&mut self) {
for guard in &mut self.guards {
guard.disarm();
}
}
pub(super) async fn detach_all(&mut self) {
for guard in &mut self.guards {
guard.detach_now().await;
}
}
}
impl AdminService {
pub(super) async fn fail_openapi_import_or_replay(
&self,
context: &ImportReplayContext<'_>,
error_text: &'static str,
error: ApiError,
) -> Result<OpenApiImportCreateResponse, ApiError> {
let empty = json!([]);
self.registry
.finish_import_job(FinishImportJobRequest {
id: context.job_id,
status: ImportJobStatus::Failed,
created_operation_ids: &empty,
error_text: Some(error_text),
finished_at: context.finished_at,
})
.await?;
// `finish_import_job` preserves an already Completed row under its
// lock. This read distinguishes that race from the terminal Failed
// transition and returns the immutable canonical application result.
let latest = self
.registry
.get_import_job(context.workspace_id, context.job_id)
.await?;
if latest.is_some_and(|job| job.status == ImportJobStatus::Completed) {
let applied = self
.registry
.apply_import_job(ApplyImportJobRequest {
id: context.job_id,
workspace_id: context.workspace_id,
application_key: context.application_key,
conflict_mode: context.conflict_mode,
operations: &[],
pre_skipped: &[],
finished_at: context.finished_at,
})
.await?;
return Ok(openapi_import_response(applied));
}
Err(error)
}
pub(super) async fn materialize_external_reference_snapshots(
&self,
workspace_id: &WorkspaceId,
primary_bytes: &[u8],
created_at: OffsetDateTime,
) -> MaterializedDependencies {
let materialized_count = std::sync::atomic::AtomicUsize::new(0);
match tokio::time::timeout(
self.external_reference_materialization_timeout,
self.materialize_external_reference_snapshots_inner(
workspace_id,
primary_bytes,
created_at,
&materialized_count,
),
)
.await
{
Ok(dependencies) => dependencies,
Err(_) => {
materialization_failure(
"chain",
"timeout",
materialized_count.load(std::sync::atomic::Ordering::Relaxed),
);
MaterializedDependencies::empty()
}
}
}
async fn materialize_external_reference_snapshots_inner(
&self,
workspace_id: &WorkspaceId,
primary_bytes: &[u8],
created_at: OffsetDateTime,
materialized_count: &std::sync::atomic::AtomicUsize,
) -> MaterializedDependencies {
if !self
.external_reference_normalization
.external_references_enabled
{
return MaterializedDependencies::empty();
}
let Ok(primary) = std::str::from_utf8(primary_bytes) else {
materialization_failure("primary_decode", "invalid_utf8", 0);
return MaterializedDependencies::empty();
};
let Ok(primary_references) =
crank_import::rest::reference_uris(primary, &self.external_reference_normalization)
else {
materialization_failure("primary_scan", "invalid_document", 0);
return MaterializedDependencies::empty();
};
let mut queue = VecDeque::new();
for reference in primary_references {
if let Some(uri) = canonical_external_document_uri(None, &reference) {
queue.push_back((uri, 1usize));
}
}
let mut seen = BTreeSet::new();
let mut staged = Vec::<StagedDependency>::new();
let mut staged_documents = vec![primary.to_owned()];
while let Some((canonical_uri, depth)) = queue.pop_front() {
if !seen.insert(canonical_uri.clone())
|| depth > self.external_reference_normalization.max_reference_depth
|| seen.len()
> self
.external_reference_normalization
.max_reference_documents
{
continue;
}
let bytes = match self.external_reference_fetcher.get(&canonical_uri).await {
Ok(bytes) => bytes,
Err(error) => {
materialization_failure(
"fetch",
external_fetch_error_code(&error),
staged.len(),
);
continue;
}
};
let Ok(document) = std::str::from_utf8(&bytes).map(str::to_owned) else {
materialization_failure("dependency_decode", "invalid_utf8", staged.len());
continue;
};
let Ok(references) = crank_import::rest::external_reference_uris(
&document,
&self.external_reference_normalization,
) else {
materialization_failure("dependency_scan", "invalid_document", staged.len());
continue;
};
staged_documents.push(document.clone());
staged.push(StagedDependency {
canonical_uri: canonical_uri.clone(),
bytes,
document,
});
for reference in references {
if let Some(uri) = canonical_external_document_uri(Some(&canonical_uri), &reference)
{
queue.push_back((uri, depth.saturating_add(1)));
}
}
}
let staged_snapshots = staged
.iter()
.filter_map(|dependency| {
let digest = format!("{:x}", Sha256::digest(&dependency.bytes));
let digest = crank_import::rest::SourceDigest::parse(digest).ok()?;
Some(crank_import::rest::ExternalDocumentSnapshot {
canonical_uri: dependency.canonical_uri.clone(),
digest,
document: dependency.document.clone(),
})
})
.collect::<Vec<_>>();
let primary_digest =
crank_import::rest::SourceDigest::parse(format!("{:x}", Sha256::digest(primary_bytes)))
.expect("SHA-256 digest is valid");
let raw_bundle_is_unsafe = crank_import::rest::validate_document_bundle_source_safe(
&staged_documents
.iter()
.map(String::as_str)
.collect::<Vec<_>>(),
&self.external_reference_normalization,
)
.is_err();
let resolved_bundle_is_unsafe = crank_import::rest::validate_resolved_bundle_source_safe(
primary,
primary_digest,
&staged_snapshots,
&self.external_reference_normalization,
)
.is_err();
if raw_bundle_is_unsafe || resolved_bundle_is_unsafe {
materialization_failure("dependency_redaction", "invalid_document", staged.len());
let mut rejected = MaterializedDependencies::empty();
rejected.unsafe_source = true;
return rejected;
}
let mut result = MaterializedDependencies::empty();
let mut dependency_by_digest = BTreeMap::<String, usize>::new();
for dependency in staged {
let store = self.artifact_store.clone();
let artifact_bytes = dependency.bytes.clone();
let Ok(Ok(artifact)) =
tokio::task::spawn_blocking(move || store.put_registered(&artifact_bytes)).await
else {
materialization_failure(
"artifact_store",
"storage_unavailable",
result.snapshots.len(),
);
continue;
};
let digest_key = artifact.artifact_ref().as_str().to_owned();
let envelope_index = if let Some(index) = dependency_by_digest.get(&digest_key) {
*index
} else {
let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi_dep"));
let Ok(source) = self
.registry
.create_artifact_source(CreateArtifactSourceRequest {
workspace_id,
source_id: &source_id,
artifact: &artifact,
mime_type: "application/octet-stream",
sensitivity: ArtifactSourceSensitivity::Internal,
created_at,
})
.await
else {
materialization_failure(
"source_create",
"registry_unavailable",
result.snapshots.len(),
);
continue;
};
let mut guard = SourceDetachGuard::new(
self.registry.clone(),
workspace_id.clone(),
source_id.clone(),
source.updated_at,
);
let Ok(verified) = self
.registry
.read_artifact_source(
std::sync::Arc::clone(&self.artifact_store),
workspace_id,
&source_id,
)
.await
else {
materialization_failure(
"source_verify",
"source_unavailable",
result.snapshots.len(),
);
guard.detach_now().await;
continue;
};
if verified.source.blob.artifact_ref != *artifact.artifact_ref() {
materialization_failure(
"source_verify",
"source_integrity",
result.snapshots.len(),
);
guard.detach_now().await;
continue;
}
let index = result.envelopes.len();
result.envelopes.push(ImportJobSourceEnvelope {
source_id: source_id.clone(),
digest: artifact.artifact_ref().clone(),
});
result.guards.push(guard);
dependency_by_digest.insert(digest_key, index);
index
};
let dependency_envelope = &result.envelopes[envelope_index];
let Ok(snapshot_digest) = crank_import::rest::SourceDigest::parse(
dependency_envelope.digest.digest_hex().to_owned(),
) else {
materialization_failure("snapshot", "source_integrity", result.snapshots.len());
continue;
};
result
.snapshots
.push(crank_import::rest::ExternalDocumentSnapshot {
canonical_uri: dependency.canonical_uri.clone(),
digest: snapshot_digest,
document: dependency.document,
});
materialized_count.store(result.snapshots.len(), std::sync::atomic::Ordering::Relaxed);
result.canonical_uris.push(dependency.canonical_uri);
}
// Registry ownership is one envelope per immutable digest, while the
// resolver may map several canonical URIs to that same snapshot. Keep
// payload rows aligned with envelopes and add aliases separately.
if result.envelopes.len() != result.canonical_uris.len() {
let mut canonical_by_digest = BTreeMap::new();
for snapshot in &result.snapshots {
canonical_by_digest
.entry(snapshot.digest.as_str().to_owned())
.or_insert_with(|| snapshot.canonical_uri.clone());
}
result.canonical_uris = result
.envelopes
.iter()
.map(|envelope| {
canonical_by_digest
.get(envelope.digest.digest_hex())
.cloned()
.unwrap_or_default()
})
.collect();
}
result
}
pub(super) async fn read_import_job_dependencies(
&self,
workspace_id: &WorkspaceId,
payload: &serde_json::Value,
locale: OpenApiUploadLocale,
) -> Result<Vec<crank_import::rest::ExternalDocumentSnapshot>, ApiError> {
let dependencies = import_job_dependencies(payload, locale)?;
let mut snapshots = Vec::with_capacity(dependencies.len());
for (canonical_uri, dependency) in dependencies {
let verified = self
.registry
.read_artifact_source(
std::sync::Arc::clone(&self.artifact_store),
workspace_id,
&dependency.source_id,
)
.await
.map_err(|error| openapi_source_error(locale, error))?;
if verified.source.blob.artifact_ref != dependency.digest {
return Err(ApiError::openapi_upload(locale, "source_integrity"));
}
let document = String::from_utf8(verified.bytes)
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
let digest =
crank_import::rest::SourceDigest::parse(dependency.digest.digest_hex().to_owned())
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
snapshots.push(crank_import::rest::ExternalDocumentSnapshot {
canonical_uri,
digest,
document,
});
}
Ok(snapshots)
}
}
fn external_fetch_error_code(error: &crank_runtime::ExternalReferenceFetchError) -> &'static str {
match error {
crank_runtime::ExternalReferenceFetchError::Disabled => "disabled",
crank_runtime::ExternalReferenceFetchError::InvalidUrl => "invalid_url",
crank_runtime::ExternalReferenceFetchError::TargetNotAllowed => "target_not_allowed",
crank_runtime::ExternalReferenceFetchError::RedirectNotAllowed => "redirect_not_allowed",
crank_runtime::ExternalReferenceFetchError::ResponseTooLarge { .. } => "response_too_large",
crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { .. } => "unexpected_status",
crank_runtime::ExternalReferenceFetchError::Transport { timeout: true, .. } => "timeout",
crank_runtime::ExternalReferenceFetchError::Transport { .. } => "transport",
crank_runtime::ExternalReferenceFetchError::InvalidConfiguration => "invalid_configuration",
}
}
fn materialization_failure(stage: &'static str, error_code: &'static str, count: usize) {
warn!(
name: "admin.openapi_import.materialization_failed",
stage,
error_code,
count,
"external OpenAPI materialization failed"
);
}
#[cfg(test)]
mod tests {
use super::external_fetch_error_code;
#[test]
fn unexpected_status_has_a_stable_safe_error_code() {
let error = crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { status: 500 };
assert_eq!(external_fetch_error_code(&error), "unexpected_status");
}
}