bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(import): resolve references and schema composition

Browse Source
This commit is contained in:
bsodfather
2026-08-29 08:54:26 +03:00
parent 6c2a3712d8
commit 55209a9bbc
46 changed files with 4848 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/admin-api/src/service/imports.rs
+255 -146
View File
@@ -11,7 +11,7 @@ use crank_import::rest::{
};
use crank_registry::{
ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity,
CreateArtifactSourceRequest, CreateImportJobRequest, DetachArtifactSourceRequest,
CreateArtifactSourceRequest, CreateImportJobRequest, FinishImportJobRequest,
ImportConflictMode, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope,
ImportJobStatus, ImportOperationDraft, RegistryError,
};
@@ -29,6 +29,13 @@ use crate::{
},
};
mod external_references;
mod job_contract;
use external_references::ImportReplayContext;
use job_contract::{
SourceDetachGuard, import_job_dependencies, import_job_normalization_config, import_job_source,
};
const IMPORT_JOB_TTL_HOURS: i64 = 24;
const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30);
const OPENAPI_PARSE_CONCURRENCY: usize = 4;
@@ -95,9 +102,14 @@ impl AdminService {
detach_guard.detach_now().await;
return Err(ApiError::openapi_upload(locale, "source_integrity"));
}
let mut dependencies = self
.materialize_external_reference_snapshots(workspace_id, &verified.bytes, now)
.await;
let parsed = match parse_verified_preview(
verified.bytes,
artifact.artifact_ref().digest_hex().to_owned(),
dependencies.snapshots.clone(),
self.external_reference_normalization.clone(),
locale,
false,
)
@@ -105,6 +117,7 @@ impl AdminService {
{
Ok(preview) => preview,
Err(error) => {
dependencies.detach_all().await;
detach_guard.detach_now().await;
return Err(error);
}
@@ -115,6 +128,7 @@ impl AdminService {
.iter()
.all(|group| group.operations.is_empty())
{
dependencies.detach_all().await;
detach_guard.detach_now().await;
return Err(ApiError::openapi_upload(locale, "no_methods"));
}
@@ -130,12 +144,15 @@ impl AdminService {
"source_id": source_envelope.source_id.as_str(),
"digest": source_envelope.digest.as_str(),
},
"dependencies": dependencies.payload(),
"dependency_snapshots": dependencies.snapshot_payload(),
"preview": preview_value,
"preview_digest": preview_digest,
"normalization": {
"normalizer_version": NORMALIZER_VERSION,
"projection_version": PROJECTION_VERSION,
"ir_fingerprint": parsed.ir_fingerprint,
"config": self.external_reference_normalization,
},
});
@@ -155,9 +172,11 @@ impl AdminService {
})
.await
{
dependencies.detach_all().await;
detach_guard.detach_now().await;
return Err(ApiError::from(error));
}
dependencies.disarm();
detach_guard.disarm();
Ok(OpenApiImportPreviewResponse {
@@ -234,6 +253,13 @@ impl AdminService {
} else {
ImportConflictMode::Rename
};
let replay_context = ImportReplayContext {
workspace_id,
job_id,
application_key: &application_key,
conflict_mode,
finished_at: &finished_at,
};
if job.status == ImportJobStatus::Completed {
let applied = self
.registry
@@ -250,7 +276,18 @@ impl AdminService {
return Ok(openapi_import_response(applied));
}
let source = import_job_source(&job.preview_payload, locale)?;
let source = match import_job_source(&job.preview_payload, locale) {
Ok(source) => source,
Err(error) => {
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_source_verification_failed",
error,
)
.await;
}
};
let verified = match self
.registry
.read_artifact_source(
@@ -264,44 +301,123 @@ impl AdminService {
Err(
error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable),
) => {
// Another request may have finished and detached the source
// after our unlocked Pending read. Re-read the job and ask
// the registry for its canonical, locked replay instead of
// exposing a false `source_unavailable` outcome.
let latest = self.registry.get_import_job(workspace_id, job_id).await?;
if latest.is_some_and(|latest| latest.status == ImportJobStatus::Completed) {
let applied = self
.registry
.apply_import_job(ApplyImportJobRequest {
id: job_id,
workspace_id,
application_key: &application_key,
conflict_mode,
operations: &[],
pre_skipped: &[],
finished_at: &finished_at,
})
.await?;
return Ok(openapi_import_response(applied));
}
return Err(openapi_source_error(locale, error));
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_source_verification_failed",
openapi_source_error(locale, error),
)
.await;
}
Err(error @ RegistryError::SourceIntegrity) => {
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_source_verification_failed",
openapi_source_error(locale, error),
)
.await;
}
Err(error) => return Err(openapi_source_error(locale, error)),
};
if verified.source.blob.artifact_ref != source.digest {
return Err(ApiError::openapi_upload(locale, "source_integrity"));
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_source_verification_failed",
ApiError::openapi_upload(locale, "source_integrity"),
)
.await;
}
let legacy_v1 = job.preview_payload.get("normalization").is_none();
let parsed = parse_verified_preview(
verified.bytes,
source.digest.digest_hex().to_owned(),
locale,
legacy_v1,
)
.await?;
verify_preview_contract(&job.preview_payload, &parsed, locale)?;
if preview_has_blocker(&parsed.preview) {
return Err(ApiError::openapi_upload(locale, "invalid_document"));
let replay_contract = match import_replay_contract(&job.preview_payload, locale) {
Ok(contract) => contract,
Err(error) => {
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_replay_verification_failed",
error,
)
.await;
}
};
let parsed = match replay_contract {
ImportReplayContract::PersistedV2(parsed) => parsed,
ImportReplayContract::LegacyV1 | ImportReplayContract::Current => {
let legacy_v1 = matches!(replay_contract, ImportReplayContract::LegacyV1);
let dependency_snapshots = match self
.read_import_job_dependencies(workspace_id, &job.preview_payload, locale)
.await
{
Ok(snapshots) => snapshots,
Err(error) => {
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_dependency_verification_failed",
error,
)
.await;
}
};
let normalization_config = match import_job_normalization_config(
&job.preview_payload,
&self.external_reference_normalization,
legacy_v1,
locale,
) {
Ok(config) => config,
Err(error) => {
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_replay_verification_failed",
error,
)
.await;
}
};
let parsed = match parse_verified_preview(
verified.bytes,
source.digest.digest_hex().to_owned(),
dependency_snapshots,
normalization_config,
locale,
legacy_v1,
)
.await
{
Ok(parsed) => parsed,
Err(error) => {
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_replay_verification_failed",
error,
)
.await;
}
};
if let Err(error) = verify_preview_contract(&job.preview_payload, &parsed, locale) {
return self
.fail_openapi_import_or_replay(
&replay_context,
"import_replay_verification_failed",
error,
)
.await;
}
parsed
}
};
if preview_has_blocker(&parsed.preview, &selected) {
return self
.fail_openapi_import_or_replay(
&replay_context,
"reference_resolution_blocked",
ApiError::openapi_upload(locale, "invalid_document"),
)
.await;
}
let mut candidates = BTreeMap::new();
for group in &parsed.preview.groups {
@@ -372,6 +488,24 @@ impl AdminService {
}
}
fn canonical_external_document_uri(base: Option<&str>, reference: &str) -> Option<String> {
if reference.starts_with('#') {
return None;
}
let mut url = match base {
Some(base) => url::Url::parse(base).ok()?.join(reference).ok()?,
None => url::Url::parse(reference).ok()?,
};
if !matches!(url.scheme(), "http" | "https")
|| !url.username().is_empty()
|| url.password().is_some()
{
return None;
}
url.set_fragment(None);
Some(url.to_string())
}
fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse {
let created = applied
.created
@@ -470,9 +604,72 @@ struct ParsedOpenApiPreview {
ir_fingerprint: String,
}
enum ImportReplayContract {
LegacyV1,
PersistedV2(ParsedOpenApiPreview),
Current,
}
fn import_replay_contract(
payload: &serde_json::Value,
locale: OpenApiUploadLocale,
) -> Result<ImportReplayContract, ApiError> {
let Some(normalization) = payload.get("normalization") else {
return Ok(ImportReplayContract::LegacyV1);
};
let normalizer = normalization
.get("normalizer_version")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
let projection = normalization
.get("projection_version")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
let ir_fingerprint = normalization
.get("ir_fingerprint")
.and_then(serde_json::Value::as_str)
.filter(|value| is_lower_sha256(value))
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
let expected_preview_digest = payload
.get("preview_digest")
.and_then(serde_json::Value::as_str)
.filter(|value| is_lower_sha256(value))
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
if normalizer == NORMALIZER_VERSION && projection == PROJECTION_VERSION {
return Ok(ImportReplayContract::Current);
}
if normalizer != "normalized-ir-v2" || projection != "preview-v2" {
return Err(ApiError::openapi_upload(locale, "source_integrity"));
}
let preview_value = payload
.get("preview")
.cloned()
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
if preview_digest(&preview_value)? != expected_preview_digest {
return Err(ApiError::openapi_upload(locale, "source_integrity"));
}
let preview = serde_json::from_value(preview_value)
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
Ok(ImportReplayContract::PersistedV2(ParsedOpenApiPreview {
preview,
ir_fingerprint: ir_fingerprint.to_owned(),
}))
}
fn is_lower_sha256(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
async fn parse_verified_preview(
bytes: Vec<u8>,
digest: String,
snapshots: Vec<crank_import::rest::ExternalDocumentSnapshot>,
normalization_config: crank_import::rest::NormalizationConfig,
locale: OpenApiUploadLocale,
legacy_v1: bool,
) -> Result<ParsedOpenApiPreview, ApiError> {
@@ -498,14 +695,18 @@ async fn parse_verified_preview(
}
let digest = crank_import::rest::SourceDigest::parse(digest)
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
let ir =
crank_import::rest::normalize_verified_document(document, digest, &Default::default())
.map_err(|error| match error {
crank_import::rest::ImportParseError::NoMethods => {
ApiError::openapi_upload(locale, "no_methods")
}
_ => ApiError::openapi_upload(locale, "invalid_document"),
})?;
let ir = crank_import::rest::normalize_verified_bundle(
document,
digest,
&snapshots,
&normalization_config,
)
.map_err(|error| match error {
crank_import::rest::ImportParseError::NoMethods => {
ApiError::openapi_upload(locale, "no_methods")
}
_ => ApiError::openapi_upload(locale, "invalid_document"),
})?;
let ir_fingerprint = preview_digest(
&serde_json::to_value(&ir)
.map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))?,
@@ -597,113 +798,21 @@ fn verify_preview_contract(
}
}
fn preview_has_blocker(preview: &crank_import::rest::ImportPreview) -> bool {
fn preview_has_blocker(
preview: &crank_import::rest::ImportPreview,
selected_operation_keys: &BTreeSet<String>,
) -> bool {
preview
.findings
.iter()
.chain(
preview
.groups
.iter()
.flat_map(|group| group.operations.iter())
.flat_map(|operation| operation.findings.iter()),
)
.any(|finding| finding.severity == ImportFindingSeverity::Error)
}
fn import_job_source(
payload: &serde_json::Value,
locale: OpenApiUploadLocale,
) -> Result<ImportJobSourceEnvelope, ApiError> {
let source = payload
.get("source")
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
let source_id = source
.get("source_id")
.and_then(serde_json::Value::as_str)
.filter(|value| value.len() <= 132)
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
let digest = source
.get("digest")
.and_then(serde_json::Value::as_str)
.and_then(|value| value.parse().ok())
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
Ok(ImportJobSourceEnvelope {
source_id: ArtifactSourceId::new(source_id),
digest,
})
}
struct SourceDetachGuard {
registry: crank_registry::PostgresRegistry,
workspace_id: WorkspaceId,
source_id: ArtifactSourceId,
expected_updated_at: OffsetDateTime,
armed: bool,
}
impl SourceDetachGuard {
fn new(
registry: crank_registry::PostgresRegistry,
workspace_id: WorkspaceId,
source_id: ArtifactSourceId,
expected_updated_at: OffsetDateTime,
) -> Self {
Self {
registry,
workspace_id,
source_id,
expected_updated_at,
armed: true,
}
}
fn disarm(&mut self) {
self.armed = false;
}
async fn detach_now(&mut self) {
if !self.armed {
return;
}
self.armed = false;
// This is the normal error path: await the database transition so a
// caller receives a completed cleanup before it can retry. `Drop`
// remains only the cancellation/shutdown safety net.
let _ = self
.registry
.detach_artifact_source(DetachArtifactSourceRequest {
workspace_id: &self.workspace_id,
source_id: &self.source_id,
expected_updated_at: Some(self.expected_updated_at),
detached_at: OffsetDateTime::now_utc(),
})
.await;
}
}
impl Drop for SourceDetachGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
let registry = self.registry.clone();
let workspace_id = self.workspace_id.clone();
let source_id = self.source_id.clone();
let expected_updated_at = Some(self.expected_updated_at);
if let Ok(handle) = tokio::runtime::Handle::try_current() {
handle.spawn(async move {
let _ = registry
.detach_artifact_source(DetachArtifactSourceRequest {
workspace_id: &workspace_id,
source_id: &source_id,
expected_updated_at,
detached_at: OffsetDateTime::now_utc(),
})
.await;
});
}
}
|| preview
.groups
.iter()
.flat_map(|group| group.operations.iter())
.filter(|operation| selected_operation_keys.contains(&operation.key))
.flat_map(|operation| operation.findings.iter())
.any(|finding| finding.severity == ImportFindingSeverity::Error)
}
fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {