feat(import): resolve references and schema composition

2026-08-29 08:54:26 +03:00
parent 6c2a3712d8
commit 55209a9bbc
46 changed files with 4848 additions and 437 deletions
@@ -108,6 +108,24 @@ pub(super) fn build_test_app(
    })
 }

+pub(super) fn build_test_app_with_external_references(
+    registry: PostgresRegistry,
+    storage_root: std::path::PathBuf,
+    allowed_url_prefixes: Vec<String>,
+) -> Router {
+    build_app(AppState {
+        service: test_service_with_external_references(
+            registry,
+            storage_root,
+            allowed_url_prefixes,
+        ),
+        api_rate_limiter: crank_runtime::RequestRateLimiter::new(
+            crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
+        ),
+        trusted_proxy_ips: Vec::new(),
+    })
+}
+
 pub(super) fn build_test_app_with_audit_sink(
    registry: PostgresRegistry,
    storage_root: std::path::PathBuf,
@@ -152,6 +170,33 @@ pub(super) fn test_service(
    .build()
 }

+pub(super) fn test_service_with_external_references(
+    registry: PostgresRegistry,
+    storage_root: std::path::PathBuf,
+    allowed_url_prefixes: Vec<String>,
+) -> AdminService {
+    let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
+    let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
+    AdminServiceBuilder::new(
+        registry,
+        storage_root,
+        test_auth_settings(),
+        test_secret_crypto(),
+        runtime,
+    )
+    .with_external_reference_import(&crank_config::ExternalReferenceSettings {
+        allowed_url_prefixes,
+        max_depth: 8,
+        max_documents: 32,
+        max_fetch_bytes: 64 * 1024,
+        fetch_timeout_ms: 2_000,
+        max_expanded_nodes: 10_000,
+    })
+    .unwrap()
+    .with_outbound_http_policy(outbound_policy)
+    .build()
+}
+
 pub(super) async fn spawn_upstream_server() -> String {
    let app = Router::new().route("/crm/leads", post(create_lead));
    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -1,4 +1,6 @@
-use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
+use admin_api::service::{
+    AdminServiceBuilder, OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale,
+};
 use crank_core::WorkspaceId;
 use crank_registry::ImportJobStatus;
 use serial_test::serial;
@@ -7,6 +9,8 @@ use super::common::{
    test_auth_settings, test_registry, test_secret_crypto, test_service, test_storage_root,
 };

+mod external_references;
+
 const OPENAPI3: &str = r#"
 openapi: 3.0.3
 info:
@@ -73,11 +77,11 @@ async fn previews_openapi_and_creates_draft_operations() {
    assert_eq!(preview_job.status, ImportJobStatus::Pending);
    assert_eq!(
        preview_job.preview_payload["normalization"]["normalizer_version"],
-        "normalized-ir-v2"
+        "normalized-ir-v3"
    );
    assert_eq!(
        preview_job.preview_payload["normalization"]["projection_version"],
-        "preview-v2"
+        "preview-v3"
    );
    assert!(preview_job.preview_payload["normalization"]["ir_fingerprint"].is_string());

@@ -236,7 +240,7 @@ async fn unknown_selected_operations_are_persisted_in_the_canonical_replay() {
 async fn concurrent_openapi_import_replays_the_same_atomic_result() {
    let registry = test_registry().await;
    let service = test_service(
-        registry,
+        registry.clone(),
        test_storage_root("openapi_import_replay"),
        test_auth_settings(),
        test_secret_crypto(),
@@ -271,6 +275,31 @@ async fn concurrent_openapi_import_replays_the_same_atomic_result() {
        service.list_operations(&workspace_id).await.unwrap().len(),
        1
    );
+    let completed_before = registry
+        .get_import_job(&workspace_id, &job_id)
+        .await
+        .unwrap()
+        .unwrap();
+    let empty = serde_json::json!([]);
+    let failed_at = time::OffsetDateTime::now_utc();
+    registry
+        .finish_import_job(crank_registry::FinishImportJobRequest {
+            id: &job_id,
+            status: ImportJobStatus::Failed,
+            created_operation_ids: &empty,
+            error_text: Some("late_verification_failure"),
+            finished_at: &failed_at,
+        })
+        .await
+        .unwrap();
+    let completed_after = registry
+        .get_import_job(&workspace_id, &job_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(completed_after.status, ImportJobStatus::Completed);
+    assert_eq!(completed_after.error_text, completed_before.error_text);
+    assert_eq!(completed_after.finished_at, completed_before.finished_at);

    let conflicting_replay = service
        .create_openapi_import(
@@ -326,6 +355,13 @@ async fn apply_fails_closed_when_the_preview_parser_contract_drifts() {
        .await;

    assert!(result.is_err());
+    assert_failed_job_and_detached_sources(
+        &registry,
+        &workspace_id,
+        &job_id,
+        "import_replay_verification_failed",
+    )
+    .await;
    assert!(
        service
            .list_operations(&workspace_id)
@@ -395,6 +431,13 @@ async fn apply_fails_closed_for_each_normalization_contract_field_and_digest_sha
                .await
                .is_err()
        );
+        assert_failed_job_and_detached_sources(
+            &registry,
+            &workspace_id,
+            &job_id,
+            "import_replay_verification_failed",
+        )
+        .await;
        assert!(
            service
                .list_operations(&workspace_id)
@@ -449,7 +492,7 @@ async fn legacy_job_without_normalization_contract_replays_until_expiry() {
 async fn blocker_finding_keeps_valid_preview_but_prevents_draft_mutation() {
    let registry = test_registry().await;
    let service = test_service(
-        registry,
+        registry.clone(),
        test_storage_root("openapi_import_blocker"),
        test_auth_settings(),
        test_secret_crypto(),
@@ -503,11 +546,124 @@ paths:
            .unwrap()
            .is_empty()
    );
+    assert_failed_job_and_detached_sources(
+        &registry,
+        &workspace_id,
+        &preview.job_id.as_str().into(),
+        "reference_resolution_blocked",
+    )
+    .await;
 }

 #[tokio::test]
 #[serial]
-async fn operation_level_reference_blocker_prevents_draft_mutation() {
+async fn operation_blockers_apply_only_to_selected_keys_and_full_selection_fails_atomically() {
+    let registry = test_registry().await;
+    let service = test_service(
+        registry.clone(),
+        test_storage_root("openapi_import_selected_blockers"),
+        test_auth_settings(),
+        test_secret_crypto(),
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let upload = OpenApiUpload {
+        bytes: br#"
+openapi: 3.1.0
+info: { title: Selected blockers }
+servers: [{ url: https://api.example.test }]
+paths:
+  /valid:
+    get:
+      operationId: validOperation
+      responses: { '200': { description: ok } }
+  /broken:
+    get:
+      operationId: brokenOperation
+      responses:
+        '200':
+          description: unresolved external schema
+          content:
+            application/json:
+              schema: { $ref: 'https://schemas.example.test/missing.yaml#/Result' }
+"#
+        .to_vec(),
+        mime_type: "application/yaml".to_owned(),
+        locale: OpenApiUploadLocale::En,
+    };
+
+    let valid_preview = service
+        .preview_openapi_import(&workspace_id, upload.clone())
+        .await
+        .unwrap();
+    assert!(
+        valid_preview.preview.findings.iter().all(|finding| {
+            finding.severity != crank_import::rest::ImportFindingSeverity::Error
+        })
+    );
+    let broken = valid_preview
+        .preview
+        .groups
+        .iter()
+        .flat_map(|group| &group.operations)
+        .find(|operation| operation.key == "GET /broken")
+        .unwrap();
+    assert!(
+        broken.findings.iter().any(|finding| {
+            finding.severity == crank_import::rest::ImportFindingSeverity::Error
+        })
+    );
+
+    let valid = service
+        .create_openapi_import(
+            &workspace_id,
+            &valid_preview.job_id.as_str().into(),
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /valid".to_owned()],
+                server_url: None,
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    assert_eq!(valid.created.len(), 1);
+
+    let full_preview = service
+        .preview_openapi_import(&workspace_id, upload)
+        .await
+        .unwrap();
+    let result = service
+        .create_openapi_import(
+            &workspace_id,
+            &full_preview.job_id.as_str().into(),
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /valid".to_owned(), "GET /broken".to_owned()],
+                server_url: None,
+                conflict_mode: "rename".to_owned(),
+            },
+        )
+        .await;
+    assert!(result.is_err());
+    assert_eq!(
+        service.list_operations(&workspace_id).await.unwrap().len(),
+        1,
+        "the full selection must not create either selected Draft"
+    );
+    let failed = registry
+        .get_import_job(&workspace_id, &full_preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(failed.status, ImportJobStatus::Failed);
+    assert_eq!(
+        failed.error_text.as_deref(),
+        Some("reference_resolution_blocked")
+    );
+    assert!(failed.finished_at.is_some());
+}
+
+#[tokio::test]
+#[serial]
+async fn local_operation_reference_resolves_and_creates_draft() {
    let registry = test_registry().await;
    let service = test_service(
        registry,
@@ -520,6 +676,7 @@ async fn operation_level_reference_blocker_prevents_draft_mutation() {
        bytes: br#"
 openapi: 3.0.3
 info: { title: References }
+servers: [{ url: https://api.example.test }]
 paths:
  /referenced:
    get:
@@ -551,32 +708,26 @@ components:
        preview.preview.groups[0].operations[0]
            .findings
            .iter()
-            .any(|finding| {
-                finding.code == "unresolved_reference"
-                    && finding.severity == crank_import::rest::ImportFindingSeverity::Error
-            })
+            .all(|finding| finding.code != "unresolved_reference")
    );
+    assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);

-    assert!(
-        service
-            .create_openapi_import(
-                &workspace_id,
-                &preview.job_id.as_str().into(),
-                OpenApiImportCreatePayload {
-                    selected_operation_keys: vec!["GET /referenced".to_owned()],
-                    server_url: None,
-                    conflict_mode: "skip".to_owned(),
-                },
-            )
-            .await
-            .is_err()
-    );
-    assert!(
-        service
-            .list_operations(&workspace_id)
-            .await
-            .unwrap()
-            .is_empty()
+    let created = service
+        .create_openapi_import(
+            &workspace_id,
+            &preview.job_id.as_str().into(),
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /referenced".to_owned()],
+                server_url: None,
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    assert_eq!(created.created.len(), 1);
+    assert_eq!(
+        service.list_operations(&workspace_id).await.unwrap().len(),
+        1
    );
 }

@@ -587,3 +738,44 @@ fn openapi_upload() -> OpenApiUpload {
        locale: OpenApiUploadLocale::En,
    }
 }
+
+async fn assert_failed_job_and_detached_sources(
+    registry: &crank_registry::PostgresRegistry,
+    workspace_id: &WorkspaceId,
+    job_id: &crank_registry::ImportJobId,
+    expected_error: &str,
+) {
+    let job = registry
+        .get_import_job(workspace_id, job_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(job.status, ImportJobStatus::Failed);
+    assert!(job.finished_at.is_some());
+    assert_eq!(job.error_text.as_deref(), Some(expected_error));
+
+    let mut source_ids = vec![
+        job.preview_payload["source"]["source_id"]
+            .as_str()
+            .unwrap()
+            .to_owned(),
+    ];
+    source_ids.extend(
+        job.preview_payload["dependencies"]
+            .as_array()
+            .into_iter()
+            .flatten()
+            .map(|dependency| dependency["source_id"].as_str().unwrap().to_owned()),
+    );
+    for source_id in source_ids {
+        let lifecycle: String = sqlx::query_scalar(
+            "select lifecycle from artifact_sources where workspace_id = $1 and source_id = $2",
+        )
+        .bind(workspace_id.as_str())
+        .bind(source_id)
+        .fetch_one(registry.pool())
+        .await
+        .unwrap();
+        assert_eq!(lifecycle, "detached");
+    }
+}
@@ -0,0 +1,538 @@
+use super::super::common::test_service_with_external_references;
+use super::*;
+use axum::{Router, routing::get};
+use std::sync::{
+    Arc,
+    atomic::{AtomicUsize, Ordering},
+};
+
+#[tokio::test]
+#[serial]
+async fn external_relative_chain_survives_reversed_builder_order_and_apply_never_refetches() {
+    let fetches = Arc::new(AtomicUsize::new(0));
+    let root_fetches = Arc::clone(&fetches);
+    let child_fetches = Arc::clone(&fetches);
+    let app = Router::new()
+        .route(
+            "/root.yaml",
+            get(move || {
+                let fetches = Arc::clone(&root_fetches);
+                async move {
+                    fetches.fetch_add(1, Ordering::SeqCst);
+                    "Item: { $ref: './child.yaml#/Item' }"
+                }
+            }),
+        )
+        .route(
+            "/child.yaml",
+            get(move || {
+                let fetches = Arc::clone(&child_fetches);
+                async move {
+                    fetches.fetch_add(1, Ordering::SeqCst);
+                    "Item: { type: object, required: [id], properties: { id: { type: string } } }"
+                }
+            }),
+        );
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+    let origin = format!("http://{address}");
+
+    let registry = test_registry().await;
+    let service = test_service_with_external_references(
+        registry.clone(),
+        test_storage_root("openapi_import_external_snapshot"),
+        vec![format!("{origin}/")],
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let upload = OpenApiUpload {
+        bytes: format!(
+            r#"
+openapi: 3.1.0
+info: {{ title: External }}
+servers: [{{ url: https://api.example.test }}]
+paths:
+  /items:
+    get:
+      operationId: listItems
+      responses:
+        '200':
+          description: ok
+          content:
+            application/json:
+              schema: {{ $ref: '{origin}/root.yaml#/Item' }}
+"#
+        )
+        .into_bytes(),
+        mime_type: "application/yaml".to_owned(),
+        locale: OpenApiUploadLocale::En,
+    };
+    let preview = service
+        .preview_openapi_import(&workspace_id, upload)
+        .await
+        .unwrap();
+    assert_eq!(fetches.load(Ordering::SeqCst), 2);
+    assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);
+    let job = registry
+        .get_import_job(&workspace_id, &preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(
+        job.preview_payload["dependencies"]
+            .as_array()
+            .unwrap()
+            .len(),
+        2
+    );
+    assert_eq!(
+        job.preview_payload["dependency_snapshots"]
+            .as_array()
+            .unwrap()
+            .len(),
+        2
+    );
+
+    let applied = service
+        .create_openapi_import(
+            &workspace_id,
+            &preview.job_id.as_str().into(),
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /items".to_owned()],
+                server_url: None,
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    assert_eq!(applied.created.len(), 1);
+    assert_eq!(fetches.load(Ordering::SeqCst), 2);
+    let active_sources: i64 = sqlx::query_scalar(
+        "select count(*) from artifact_sources
+          where source_id like 'src_openapi_%' and lifecycle = 'active'",
+    )
+    .fetch_one(registry.pool())
+    .await
+    .unwrap();
+    assert_eq!(active_sources, 0);
+}
+
+#[tokio::test]
+#[serial]
+async fn missing_external_snapshot_fails_closed_before_draft_mutation() {
+    let app = Router::new().route(
+        "/schemas.yaml",
+        get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
+    );
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+    let origin = format!("http://{address}");
+    let registry = test_registry().await;
+    let storage_root = test_storage_root("openapi_import_missing_external_snapshot");
+    let service = test_service_with_external_references(
+        registry.clone(),
+        storage_root.clone(),
+        vec![format!("{origin}/")],
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiUpload {
+                bytes: format!(
+                    r#"
+openapi: 3.1.0
+info: {{ title: External integrity }}
+servers: [{{ url: https://api.example.test }}]
+paths:
+  /items:
+    get:
+      operationId: listItems
+      responses:
+        '200':
+          description: ok
+          content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
+"#
+                )
+                .into_bytes(),
+                mime_type: "application/yaml".to_owned(),
+                locale: OpenApiUploadLocale::En,
+            },
+        )
+        .await
+        .unwrap();
+    let job = registry
+        .get_import_job(&workspace_id, &preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    let dependency_source_id = job.preview_payload["dependencies"][0]["source_id"]
+        .as_str()
+        .unwrap();
+    sqlx::query(
+        "update artifact_sources
+            set lifecycle = 'detached', updated_at = now(), detached_at = now()
+          where workspace_id = $1 and source_id = $2",
+    )
+    .bind(workspace_id.as_str())
+    .bind(dependency_source_id)
+    .execute(registry.pool())
+    .await
+    .unwrap();
+
+    assert!(
+        service
+            .create_openapi_import(
+                &workspace_id,
+                &preview.job_id.as_str().into(),
+                OpenApiImportCreatePayload {
+                    selected_operation_keys: vec!["GET /items".to_owned()],
+                    server_url: None,
+                    conflict_mode: "skip".to_owned(),
+                },
+            )
+            .await
+            .is_err()
+    );
+    assert!(
+        service
+            .list_operations(&workspace_id)
+            .await
+            .unwrap()
+            .is_empty()
+    );
+    assert_failed_job_and_detached_sources(
+        &registry,
+        &workspace_id,
+        &preview.job_id.as_str().into(),
+        "import_dependency_verification_failed",
+    )
+    .await;
+
+    let corrupt_preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiUpload {
+                bytes: format!(
+                    r#"
+openapi: 3.1.0
+info: {{ title: Corrupt external integrity }}
+servers: [{{ url: https://api.example.test }}]
+paths:
+  /items:
+    get:
+      operationId: listItemsCorrupt
+      responses:
+        '200':
+          description: ok
+          content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
+"#
+                )
+                .into_bytes(),
+                mime_type: "application/yaml".to_owned(),
+                locale: OpenApiUploadLocale::En,
+            },
+        )
+        .await
+        .unwrap();
+    let corrupt_job = registry
+        .get_import_job(&workspace_id, &corrupt_preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    let dependency_ref: crank_artifacts::ArtifactRef = corrupt_job.preview_payload["dependencies"]
+        [0]["digest"]
+        .as_str()
+        .unwrap()
+        .parse()
+        .unwrap();
+    let dependency_path = storage_root
+        .join("sha256")
+        .join(&dependency_ref.digest_hex()[..2])
+        .join(dependency_ref.digest_hex());
+    std::fs::set_permissions(
+        &dependency_path,
+        std::os::unix::fs::PermissionsExt::from_mode(0o600),
+    )
+    .unwrap();
+    std::fs::write(&dependency_path, b"corrupt external dependency").unwrap();
+    std::fs::set_permissions(
+        &dependency_path,
+        std::os::unix::fs::PermissionsExt::from_mode(0o400),
+    )
+    .unwrap();
+
+    assert!(
+        service
+            .create_openapi_import(
+                &workspace_id,
+                &corrupt_preview.job_id.as_str().into(),
+                OpenApiImportCreatePayload {
+                    selected_operation_keys: vec!["GET /items".to_owned()],
+                    server_url: None,
+                    conflict_mode: "skip".to_owned(),
+                },
+            )
+            .await
+            .is_err()
+    );
+    assert_failed_job_and_detached_sources(
+        &registry,
+        &workspace_id,
+        &corrupt_preview.job_id.as_str().into(),
+        "import_dependency_verification_failed",
+    )
+    .await;
+}
+
+#[tokio::test]
+#[serial]
+async fn external_materialization_uses_one_wall_clock_timeout_and_cleans_cancelled_sources() {
+    let fetches = Arc::new(AtomicUsize::new(0));
+    let root_fetches = Arc::clone(&fetches);
+    let child_fetches = Arc::clone(&fetches);
+    let app = Router::new()
+        .route(
+            "/root.yaml",
+            get(move || {
+                let fetches = Arc::clone(&root_fetches);
+                async move {
+                    fetches.fetch_add(1, Ordering::SeqCst);
+                    "Item: { $ref: './slow-child.yaml#/Item' }"
+                }
+            }),
+        )
+        .route(
+            "/slow-child.yaml",
+            get(move || {
+                let fetches = Arc::clone(&child_fetches);
+                async move {
+                    fetches.fetch_add(1, Ordering::SeqCst);
+                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                    "Item: { type: object, properties: { id: { type: string } } }"
+                }
+            }),
+        );
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+    let origin = format!("http://{address}");
+
+    let registry = test_registry().await;
+    let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
+    let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
+    let service = AdminServiceBuilder::new(
+        registry.clone(),
+        test_storage_root("openapi_import_chain_timeout"),
+        test_auth_settings(),
+        test_secret_crypto(),
+        runtime,
+    )
+    .with_external_reference_import(&crank_config::ExternalReferenceSettings {
+        allowed_url_prefixes: vec![format!("{origin}/")],
+        max_depth: 8,
+        max_documents: 32,
+        max_fetch_bytes: 64 * 1024,
+        fetch_timeout_ms: 200,
+        max_expanded_nodes: 10_000,
+    })
+    .unwrap()
+    .with_outbound_http_policy(outbound_policy)
+    .build();
+    let workspace_id = WorkspaceId::new("ws_default");
+    let started = tokio::time::Instant::now();
+    let preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiUpload {
+                bytes: format!(
+                    r#"
+openapi: 3.1.0
+info: {{ title: Timed chain }}
+servers: [{{ url: https://api.example.test }}]
+paths:
+  /items:
+    get:
+      operationId: timedItems
+      responses:
+        '200':
+          description: ok
+          content: {{ application/json: {{ schema: {{ $ref: '{origin}/root.yaml#/Item' }} }} }}
+"#
+                )
+                .into_bytes(),
+                mime_type: "application/yaml".to_owned(),
+                locale: OpenApiUploadLocale::En,
+            },
+        )
+        .await
+        .unwrap();
+    assert!(started.elapsed() < std::time::Duration::from_millis(800));
+    assert_eq!(fetches.load(Ordering::SeqCst), 2);
+    let job = registry
+        .get_import_job(&workspace_id, &preview.job_id.as_str().into())
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(
+        job.preview_payload["dependencies"]
+            .as_array()
+            .unwrap()
+            .len(),
+        0
+    );
+
+    for _ in 0..100 {
+        let active_dependencies: i64 = sqlx::query_scalar(
+            "select count(*) from artifact_sources
+              where source_id like 'src_openapi_dep_%' and lifecycle = 'active'",
+        )
+        .fetch_one(registry.pool())
+        .await
+        .unwrap();
+        if active_dependencies == 0 {
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+    }
+    let detached_dependencies: i64 = sqlx::query_scalar(
+        "select count(*) from artifact_sources
+          where source_id like 'src_openapi_dep_%' and lifecycle = 'detached'",
+    )
+    .fetch_one(registry.pool())
+    .await
+    .unwrap();
+    assert_eq!(detached_dependencies, 1);
+}
+
+#[tokio::test]
+#[serial]
+async fn expired_job_detaches_primary_and_external_dependency_without_orphan() {
+    let app = Router::new().route(
+        "/schemas.yaml",
+        get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
+    );
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+    let origin = format!("http://{address}");
+    let registry = test_registry().await;
+    let service = test_service_with_external_references(
+        registry.clone(),
+        test_storage_root("openapi_import_external_expiry"),
+        vec![format!("{origin}/")],
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let preview = service
+        .preview_openapi_import(
+            &workspace_id,
+            OpenApiUpload {
+                bytes: format!(
+                    r#"
+openapi: 3.1.0
+info: {{ title: Expiring external }}
+servers: [{{ url: https://api.example.test }}]
+paths:
+  /items:
+    get:
+      operationId: listItems
+      responses:
+        '200':
+          description: ok
+          content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
+"#
+                )
+                .into_bytes(),
+                mime_type: "application/yaml".to_owned(),
+                locale: OpenApiUploadLocale::En,
+            },
+        )
+        .await
+        .unwrap();
+    sqlx::query("update import_jobs set expires_at = now() - interval '1 second' where id = $1")
+        .bind(preview.job_id.as_str())
+        .execute(registry.pool())
+        .await
+        .unwrap();
+    let report = registry.cleanup_expired_import_jobs(16).await.unwrap();
+    assert_eq!(report.deleted_jobs, 1);
+    assert_eq!(report.detached_sources, 2);
+    let active_sources: i64 = sqlx::query_scalar(
+        "select count(*) from artifact_sources
+          where source_id like 'src_openapi_%' and lifecycle = 'active'",
+    )
+    .fetch_one(registry.pool())
+    .await
+    .unwrap();
+    assert_eq!(active_sources, 0);
+}
+#[tokio::test]
+#[serial]
+async fn exact_v2_contract_replays_persisted_preview_without_v3_or_dependency_reads() {
+    let registry = test_registry().await;
+    let service = test_service(
+        registry.clone(),
+        test_storage_root("openapi_import_v2_compatibility"),
+        test_auth_settings(),
+        test_secret_crypto(),
+    );
+    let workspace_id = WorkspaceId::new("ws_default");
+    let preview = service
+        .preview_openapi_import(&workspace_id, openapi_upload())
+        .await
+        .unwrap();
+    let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
+    let missing_dependency = serde_json::json!([{
+        "source_id": "src_openapi_dep_v2_must_not_be_read",
+        "digest": format!("sha256:{}", "0".repeat(64)),
+        "canonical_uri": "https://schemas.example.test/v2.yaml"
+    }]);
+    sqlx::query(
+        "update import_jobs
+            set preview_payload = jsonb_set(
+                jsonb_set(
+                    jsonb_set(
+                        jsonb_set(
+                            preview_payload,
+                            '{normalization,normalizer_version}',
+                            to_jsonb('normalized-ir-v2'::text)
+                        ),
+                        '{normalization,projection_version}',
+                        to_jsonb('preview-v2'::text)
+                    ),
+                    '{normalization,ir_fingerprint}',
+                    to_jsonb($1::text)
+                ),
+                '{dependency_snapshots}',
+                $2::jsonb
+            )
+          where id = $3",
+    )
+    .bind("a".repeat(64))
+    .bind(missing_dependency)
+    .bind(job_id.as_str())
+    .execute(registry.pool())
+    .await
+    .unwrap();
+
+    let result = service
+        .create_openapi_import(
+            &workspace_id,
+            &job_id,
+            OpenApiImportCreatePayload {
+                selected_operation_keys: vec!["GET /v2/latest".to_owned()],
+                server_url: Some("https://api.frankfurter.dev".to_owned()),
+                conflict_mode: "skip".to_owned(),
+            },
+        )
+        .await
+        .unwrap();
+    assert_eq!(result.created.len(), 1);
+    let completed = registry
+        .get_import_job(&workspace_id, &job_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(completed.status, ImportJobStatus::Completed);
+}
@@ -1,30 +1,18 @@
-use std::{
-    io,
-    sync::{Arc, Mutex},
-};
-
-use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
-use crank_artifacts::MAX_ARTIFACT_BYTES;
-use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
-use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest};
-use metrics_util::debugging::DebuggingRecorder;
-use opentelemetry::trace::TracerProvider as _;
-use opentelemetry_sdk::{
-    error::OTelSdkResult,
-    trace::{SdkTracerProvider, SpanData, SpanExporter},
-};
-use reqwest::multipart::{Form, Part};
-use serde_json::Value;
-use serial_test::serial;
-use time::{Duration, OffsetDateTime};
-use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
-
 use super::common::{
    authorized_client, build_test_app, spawn_admin_api, test_auth_settings, test_registry,
    test_secret_crypto, test_service, test_storage_root,
 };
+use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
+use crank_artifacts::MAX_ARTIFACT_BYTES;
+use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
+use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest};
+use reqwest::multipart::{Form, Part};
+use serde_json::Value;
+use serial_test::serial;
+use time::{Duration, OffsetDateTime};

 mod apply_failures;
+mod telemetry;

 const OPENAPI: &str = r#"
 openapi: 3.0.3
@@ -136,78 +124,6 @@ async fn multipart_requires_an_owner_membership_before_reading_the_file() {
    );
 }

-#[tokio::test(flavor = "multi_thread")]
-#[serial]
-async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
-    const CANARY: &str = "openapi-telemetry-secret-canary";
-
-    let recorder = DebuggingRecorder::new();
-    let snapshotter = recorder.snapshotter();
-    recorder
-        .install()
-        .expect("isolated integration test metrics recorder");
-
-    let writer = SharedLogWriter::default();
-    let exported = Arc::new(Mutex::new(Vec::new()));
-    let provider = SdkTracerProvider::builder()
-        .with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
-        .build();
-    let tracer = provider.tracer("admin-openapi-source-test");
-    let subscriber = tracing_subscriber::registry()
-        .with(tracing_subscriber::fmt::layer().with_writer(writer.clone()))
-        .with(tracing_opentelemetry::layer().with_tracer(tracer));
-    let dispatch = tracing::Dispatch::new(subscriber);
-    tracing::dispatcher::set_global_default(dispatch)
-        .expect("isolated integration test tracing subscriber");
-
-    let registry = test_registry().await;
-    let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary"));
-    let server = spawn_admin_api(app).await;
-    let client = authorized_client(&server).await;
-    let document = format!(
-        "openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n  /broken:\n    get:\n      description: {CANARY}\n      responses: ["
-    );
-    let response = client
-        .post(format!("{server}/imports/openapi/preview"))
-        .multipart(Form::new().part(
-            "file",
-            file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
-        ))
-        .send()
-        .await
-        .unwrap();
-    assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
-    let trace_id = response
-        .headers()
-        .get("x-trace-id")
-        .unwrap()
-        .to_str()
-        .unwrap()
-        .to_owned();
-    let body = response.text().await.unwrap();
-    assert!(!body.contains(CANARY));
-    let diagnostics: Value = serde_json::from_str(&body).unwrap();
-    assert_eq!(diagnostics["error"]["trace_id"], trace_id);
-
-    provider.force_flush().unwrap();
-    let logs = writer.output();
-    assert!(!logs.contains(CANARY));
-    assert!(logs.contains(&trace_id));
-    let spans = exported.lock().unwrap();
-    let rendered_spans = format!("{spans:?}");
-    assert!(!rendered_spans.contains(CANARY));
-    drop(spans);
-    for (key, _, _, _) in snapshotter.snapshot().into_vec() {
-        assert!(!key.key().name().contains(CANARY));
-        assert!(
-            !key.key()
-                .labels()
-                .any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) })
-        );
-    }
-    provider.shutdown().unwrap();
-}
-
 #[tokio::test(flavor = "multi_thread")]
 #[serial]
 async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() {
@@ -939,49 +855,3 @@ async fn wait_for_specific_source_lifecycle(
    }
    panic!("artifact source {source_id:?} did not reach {expected}");
 }
-
-#[derive(Clone, Default)]
-struct SharedLogWriter {
-    buffer: Arc<Mutex<Vec<u8>>>,
-}
-
-impl SharedLogWriter {
-    fn output(&self) -> String {
-        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
-    }
-}
-
-impl<'a> MakeWriter<'a> for SharedLogWriter {
-    type Writer = SharedLogGuard;
-
-    fn make_writer(&'a self) -> Self::Writer {
-        SharedLogGuard {
-            buffer: Arc::clone(&self.buffer),
-        }
-    }
-}
-
-struct SharedLogGuard {
-    buffer: Arc<Mutex<Vec<u8>>>,
-}
-
-impl io::Write for SharedLogGuard {
-    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
-        self.buffer.lock().unwrap().extend_from_slice(bytes);
-        Ok(bytes.len())
-    }
-
-    fn flush(&mut self) -> io::Result<()> {
-        Ok(())
-    }
-}
-
-#[derive(Clone, Debug)]
-struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
-
-impl SpanExporter for CapturingExporter {
-    async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
-        self.0.lock().unwrap().extend(batch);
-        Ok(())
-    }
-}
@@ -0,0 +1,184 @@
+use super::super::common::build_test_app_with_external_references;
+use super::*;
+use std::{
+    io,
+    sync::{Arc, Mutex},
+};
+
+use metrics_util::debugging::DebuggingRecorder;
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry_sdk::{
+    error::OTelSdkResult,
+    trace::{SdkTracerProvider, SpanData, SpanExporter},
+};
+use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
+
+#[tokio::test(flavor = "multi_thread")]
+#[serial]
+async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
+    const CANARY: &str = "openapi-telemetry-secret-canary";
+
+    let recorder = DebuggingRecorder::new();
+    let snapshotter = recorder.snapshotter();
+    recorder
+        .install()
+        .expect("isolated integration test metrics recorder");
+
+    let writer = SharedLogWriter::default();
+    let exported = Arc::new(Mutex::new(Vec::new()));
+    let provider = SdkTracerProvider::builder()
+        .with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
+        .build();
+    let tracer = provider.tracer("admin-openapi-source-test");
+    let subscriber = tracing_subscriber::registry()
+        .with(tracing_subscriber::fmt::layer().with_writer(writer.clone()))
+        .with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let dispatch = tracing::Dispatch::new(subscriber);
+    tracing::dispatcher::set_global_default(dispatch)
+        .expect("isolated integration test tracing subscriber");
+
+    let registry = test_registry().await;
+    let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary"));
+    let server = spawn_admin_api(app).await;
+    let client = authorized_client(&server).await;
+    let document = format!(
+        "openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n  /broken:\n    get:\n      description: {CANARY}\n      responses: ["
+    );
+    let response = client
+        .post(format!("{server}/imports/openapi/preview"))
+        .multipart(Form::new().part(
+            "file",
+            file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
+        ))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
+    let trace_id = response
+        .headers()
+        .get("x-trace-id")
+        .unwrap()
+        .to_str()
+        .unwrap()
+        .to_owned();
+    let body = response.text().await.unwrap();
+    assert!(!body.contains(CANARY));
+    let diagnostics: Value = serde_json::from_str(&body).unwrap();
+    assert_eq!(diagnostics["error"]["trace_id"], trace_id);
+
+    let route = format!("/{CANARY}.yaml");
+    let external = axum::Router::new().route(
+        &route,
+        axum::routing::get(|| async { (axum::http::StatusCode::INTERNAL_SERVER_ERROR, CANARY) }),
+    );
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let address = listener.local_addr().unwrap();
+    tokio::spawn(async move { axum::serve(listener, external).await.unwrap() });
+    let origin = format!("http://{address}");
+    let external_registry = test_registry().await;
+    let external_app = build_test_app_with_external_references(
+        external_registry,
+        test_storage_root("openapi_materialization_telemetry_canary"),
+        vec![format!("{origin}/")],
+    );
+    let external_server = spawn_admin_api(external_app).await;
+    let external_client = authorized_client(&external_server).await;
+    let external_document = format!(
+        r#"
+openapi: 3.1.0
+info: {{ title: Safe materialization }}
+servers: [{{ url: https://api.example.test }}]
+paths:
+  /items:
+    get:
+      responses:
+        '200':
+          description: ok
+          content:
+            application/json:
+              schema: {{ $ref: '{origin}/{CANARY}.yaml#/Item' }}
+"#
+    );
+    let response = external_client
+        .post(format!("{external_server}/imports/openapi/preview"))
+        .multipart(Form::new().part(
+            "file",
+            file_part(
+                external_document.as_bytes(),
+                "external.yaml",
+                "application/yaml",
+            ),
+        ))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(response.status(), reqwest::StatusCode::OK);
+
+    provider.force_flush().unwrap();
+    let logs = writer.output();
+    assert!(!logs.contains(CANARY));
+    assert!(logs.contains(&trace_id));
+    assert!(logs.contains("external OpenAPI materialization failed"));
+    assert!(logs.contains("stage=\"fetch\""));
+    assert!(logs.contains("error_code=\"unexpected_status\""));
+    assert!(logs.contains("count=0"));
+    let spans = exported.lock().unwrap();
+    let rendered_spans = format!("{spans:?}");
+    assert!(!rendered_spans.contains(CANARY));
+    drop(spans);
+    for (key, _, _, _) in snapshotter.snapshot().into_vec() {
+        assert!(!key.key().name().contains(CANARY));
+        assert!(
+            !key.key()
+                .labels()
+                .any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) })
+        );
+    }
+    provider.shutdown().unwrap();
+}
+
+#[derive(Clone, Default)]
+struct SharedLogWriter {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl SharedLogWriter {
+    fn output(&self) -> String {
+        String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
+    }
+}
+
+impl<'a> MakeWriter<'a> for SharedLogWriter {
+    type Writer = SharedLogGuard;
+
+    fn make_writer(&'a self) -> Self::Writer {
+        SharedLogGuard {
+            buffer: Arc::clone(&self.buffer),
+        }
+    }
+}
+
+struct SharedLogGuard {
+    buffer: Arc<Mutex<Vec<u8>>>,
+}
+
+impl io::Write for SharedLogGuard {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.buffer.lock().unwrap().extend_from_slice(bytes);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug)]
+struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
+
+impl SpanExporter for CapturingExporter {
+    async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
+        self.0.lock().unwrap().extend(batch);
+        Ok(())
+    }
+}