bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(import): resolve references and schema composition

Browse Source
This commit is contained in:
bsodfather
2026-08-29 08:54:26 +03:00
parent 6c2a3712d8
commit 55209a9bbc
46 changed files with 4848 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/crank-registry/src/postgres/import_job.rs
+175 -30
View File
@@ -4,6 +4,7 @@ use crank_artifacts::ArtifactRef;
const APPLICATION_RESULT_KEY: &str = "_crank_application_result";
const IMPORT_JOB_CLEANUP_BATCH: u32 = 128;
const MAX_IMPORT_JOB_DOCUMENTS: usize = 32;
const DANGLING_OPENAPI_SOURCE_GRACE: time::Duration = time::Duration::minutes(5);
impl PostgresRegistry {
@@ -12,6 +13,7 @@ impl PostgresRegistry {
request: CreateImportJobRequest<'_>,
) -> Result<(), RegistryError> {
validate_source_envelope(request.source, request.preview_payload)?;
validate_dependency_envelopes(request.preview_payload)?;
let mut transaction = self.pool.begin().await?;
sqlx::query(
"insert into import_jobs (
@@ -81,6 +83,26 @@ impl PostgresRegistry {
&self,
request: FinishImportJobRequest<'_>,
) -> Result<(), RegistryError> {
let mut transaction = self.pool.begin().await?;
let row = sqlx::query(
"select workspace_id, status, preview_payload
from import_jobs
where id = $1
for update",
)
.bind(request.id.as_str())
.fetch_optional(&mut *transaction)
.await?
.ok_or_else(|| RegistryError::ImportJobNotFound {
job_id: request.id.as_str().to_owned(),
})?;
let current_status =
deserialize_enum_text::<ImportJobStatus>(row.try_get("status")?, "status")?;
if request.status == ImportJobStatus::Failed && current_status == ImportJobStatus::Completed
{
transaction.commit().await?;
return Ok(());
}
let result = sqlx::query(
"update import_jobs
set status = $2,
@@ -94,7 +116,7 @@ impl PostgresRegistry {
.bind(request.created_operation_ids)
.bind(request.error_text)
.bind(request.finished_at)
.execute(&self.pool)
.execute(&mut *transaction)
.await?;
if result.rows_affected() == 0 {
@@ -103,6 +125,25 @@ impl PostgresRegistry {
});
}
if request.status == ImportJobStatus::Failed {
let workspace_id = WorkspaceId::new(row.try_get::<String, _>("workspace_id")?);
let payload = row.try_get::<Value, _>("preview_payload")?;
// Failure finalization must itself be fail-safe. The strict
// envelope parsers are used before Apply; cleanup only needs the
// bounded source identities and must not roll back the terminal
// state because some other payload field was corrupted.
for source_id in cleanup_source_ids(&payload) {
let _ = detach_source_in_transaction(
&mut transaction,
&workspace_id,
&source_id,
*request.finished_at,
)
.await?;
}
}
transaction.commit().await?;
Ok(())
}
@@ -120,24 +161,18 @@ impl PostgresRegistry {
}
Err(error) => {
tx.rollback().await?;
let error_text = error.to_string();
let _ = sqlx::query(
"update import_jobs
set status = $3,
error_text = $4,
finished_at = $5::timestamptz
where id = $1
and workspace_id = $2
and status <> $6",
)
.bind(request.id.as_str())
.bind(request.workspace_id.as_str())
.bind(serialize_enum_text(&ImportJobStatus::Failed, "status")?)
.bind(error_text)
.bind(request.finished_at)
.bind(serialize_enum_text(&ImportJobStatus::Completed, "status")?)
.execute(&self.pool)
.await;
if !matches!(error, RegistryError::ImportJobAlreadyApplied { .. }) {
let empty = serde_json::json!([]);
let _ = self
.finish_import_job(FinishImportJobRequest {
id: request.id,
status: ImportJobStatus::Failed,
created_operation_ids: &empty,
error_text: Some("import_apply_failed"),
finished_at: request.finished_at,
})
.await;
}
Err(error)
}
}
@@ -177,16 +212,12 @@ impl PostgresRegistry {
// The job itself is expired regardless of whether a legacy or
// corrupt payload can be decoded. Do not let one bad row roll
// back cleanup for every tenant.
if let Ok(Some(source)) = source_from_payload(&payload)
&& detach_source_in_transaction(
&mut transaction,
&workspace_id,
&source.source_id,
now,
)
.await?
{
detached_sources += 1;
for source_id in cleanup_source_ids(&payload) {
if detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now)
.await?
{
detached_sources += 1;
}
}
}
let deleted = if expired_ids.is_empty() {
@@ -214,7 +245,14 @@ impl PostgresRegistry {
select 1
from import_jobs j
where j.workspace_id = s.workspace_id
and j.preview_payload -> 'source' ->> 'source_id' = s.source_id
and (
j.preview_payload -> 'source' ->> 'source_id' = s.source_id
or exists (
select 1
from jsonb_array_elements(coalesce(j.preview_payload -> 'dependencies', '[]'::jsonb)) d
where d ->> 'source_id' = s.source_id
)
)
)
order by s.created_at, s.workspace_id, s.source_id
limit $2
@@ -283,6 +321,7 @@ async fn apply_import_job_transaction(
source_from_payload(&preview_payload)?.ok_or(RegistryError::InvalidArtifactSource {
field: "import_job_source",
})?;
let dependencies = dependencies_from_payload(&preview_payload)?;
if status == ImportJobStatus::Completed
&& let Some(result) = stored_application_result(&preview_payload)?
@@ -323,6 +362,31 @@ async fn apply_import_job_transaction(
if !source_is_current {
return Err(RegistryError::SourceUnavailable);
}
for dependency in &dependencies {
let recorded = sqlx::query(
"select s.lifecycle, b.artifact_ref
from artifact_sources s
join artifact_blobs b on b.digest = s.blob_digest
where s.workspace_id = $1 and s.source_id = $2
for update",
)
.bind(request.workspace_id.as_str())
.bind(dependency.source_id.as_str())
.fetch_optional(&mut **tx)
.await?;
let current = recorded
.as_ref()
.map(|row| {
let lifecycle = row.try_get::<String, _>("lifecycle").ok();
let artifact_ref = row.try_get::<String, _>("artifact_ref").ok();
lifecycle.as_deref() == Some("active")
&& artifact_ref.as_deref() == Some(dependency.digest.as_str())
})
.unwrap_or(false);
if !current {
return Err(RegistryError::SourceUnavailable);
}
}
let mut result = ImportJobApplyResult {
application_key: request.application_key.to_owned(),
@@ -408,6 +472,15 @@ async fn apply_import_job_transaction(
*request.finished_at,
)
.await?;
for dependency in dependencies {
let _ = detach_source_in_transaction(
tx,
request.workspace_id,
&dependency.source_id,
*request.finished_at,
)
.await?;
}
Ok(result)
}
@@ -433,6 +506,11 @@ fn validate_source_envelope(
Ok(())
}
fn validate_dependency_envelopes(preview_payload: &Value) -> Result<(), RegistryError> {
dependencies_from_payload(preview_payload)?;
Ok(())
}
fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope>, RegistryError> {
let Some(source) = payload.get("source") else {
return Ok(None);
@@ -466,6 +544,73 @@ fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope
}))
}
fn dependencies_from_payload(
payload: &Value,
) -> Result<Vec<ImportJobSourceEnvelope>, RegistryError> {
let Some(value) = payload.get("dependencies") else {
return Ok(Vec::new());
};
let items = value
.as_array()
.filter(|items| items.len() <= MAX_IMPORT_JOB_DOCUMENTS)
.ok_or(RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
})?;
let mut dependencies = Vec::with_capacity(items.len());
let mut identities = std::collections::BTreeSet::new();
for item in items {
let source_id = item
.get("source_id")
.and_then(Value::as_str)
.filter(|value| value.len() <= 132)
.ok_or(RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
})?;
let digest = item.get("digest").and_then(Value::as_str).ok_or(
RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
},
)?;
let digest =
ArtifactRef::parse(digest).map_err(|_| RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
})?;
if !identities.insert((source_id.to_owned(), digest.as_str().to_owned())) {
return Err(RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
});
}
dependencies.push(ImportJobSourceEnvelope {
source_id: ArtifactSourceId::new(source_id),
digest,
});
}
Ok(dependencies)
}
fn cleanup_source_ids(payload: &Value) -> Vec<ArtifactSourceId> {
let mut identities = std::collections::BTreeSet::new();
if let Some(source_id) = payload
.pointer("/source/source_id")
.and_then(Value::as_str)
.filter(|value| value.len() <= 132)
{
identities.insert(source_id.to_owned());
}
if let Some(dependencies) = payload.get("dependencies").and_then(Value::as_array) {
for source_id in dependencies
.iter()
.take(MAX_IMPORT_JOB_DOCUMENTS)
.filter_map(|item| item.get("source_id"))
.filter_map(Value::as_str)
.filter(|value| value.len() <= 132)
{
identities.insert(source_id.to_owned());
}
}
identities.into_iter().map(ArtifactSourceId::new).collect()
}
async fn detach_source_in_transaction(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &WorkspaceId,