diff --git a/.env.example b/.env.example index 5c68761..b3e8261 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS= CRANK_OUTBOUND_DENIED_HOSTS= CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304 CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304 +CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES= +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144 +CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000 CRANK_ENVIRONMENT=development CRANK_LOG_LEVEL= CRANK_SENTRY_DSN= diff --git a/apps/admin-api/src/main.rs b/apps/admin-api/src/main.rs index 5270131..1bb35b6 100644 --- a/apps/admin-api/src/main.rs +++ b/apps/admin-api/src/main.rs @@ -227,6 +227,7 @@ async fn run( .with_artifact_store(artifact_store.clone()) .with_public_base_url(base_url) .with_outbound_http_policy(outbound_http_policy) + .with_external_reference_import(&config.external_references)? .with_identity_provider(std::sync::Arc::new(identity_provider)) .build(); if config.demo_seed { diff --git a/apps/admin-api/src/service.rs b/apps/admin-api/src/service.rs index 9ef758c..da30d52 100644 --- a/apps/admin-api/src/service.rs +++ b/apps/admin-api/src/service.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use crank_artifacts::ArtifactStore; +use crank_config::ExternalReferenceSettings; use crank_core::{ AuditActor, AuditEvent, AuditEventId, AuditSink, AuditTarget, AuditTargetKind, AuthProfile, CapabilityProfile, CommunityCapabilityProfile, CorrelationContext, EditionCapabilities, @@ -19,7 +20,8 @@ use crank_registry::{ UsageBucket, }; use crank_runtime::{ - OutboundHttpPolicy, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto, + ExternalReferenceFetcher, OutboundHttpPolicy, ResolvedAuth, RuntimeError, RuntimeExecutor, + SecretCrypto, }; use crank_schema::{Schema, SchemaKind}; use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query}; @@ -68,6 +70,9 @@ pub struct AdminService { audit_sink: Arc, capability_profile: Arc, outbound_http_policy: OutboundHttpPolicy, + external_reference_fetcher: ExternalReferenceFetcher, + external_reference_normalization: crank_import::rest::NormalizationConfig, + external_reference_materialization_timeout: std::time::Duration, public_base_url: String, } @@ -83,6 +88,9 @@ pub struct AdminServiceBuilder { audit_sink: Option>, capability_profile: Option>, outbound_http_policy: OutboundHttpPolicy, + external_reference_fetcher: ExternalReferenceFetcher, + external_reference_normalization: crank_import::rest::NormalizationConfig, + external_reference_settings: ExternalReferenceSettings, public_base_url: String, } @@ -220,6 +228,22 @@ impl AdminServiceBuilder { secret_crypto: SecretCrypto, runtime: RuntimeExecutor, ) -> Self { + let outbound_http_policy = OutboundHttpPolicy::default(); + let external_reference_settings = ExternalReferenceSettings { + allowed_url_prefixes: Vec::new(), + max_depth: 8, + max_documents: 32, + max_fetch_bytes: 256 * 1024, + fetch_timeout_ms: 5_000, + max_expanded_nodes: 10_000, + }; + let external_reference_fetcher = ExternalReferenceFetcher::try_new( + outbound_http_policy.clone(), + external_reference_settings.allowed_url_prefixes.clone(), + external_reference_settings.max_fetch_bytes, + std::time::Duration::from_millis(external_reference_settings.fetch_timeout_ms), + ) + .expect("static external reference defaults are valid"); Self { registry, storage_root, @@ -231,7 +255,10 @@ impl AdminServiceBuilder { policy_engine: None, audit_sink: None, capability_profile: None, - outbound_http_policy: OutboundHttpPolicy::default(), + outbound_http_policy, + external_reference_fetcher, + external_reference_normalization: Default::default(), + external_reference_settings, public_base_url: "http://localhost:3000".to_owned(), } } @@ -242,10 +269,41 @@ impl AdminServiceBuilder { } pub fn with_outbound_http_policy(mut self, policy: OutboundHttpPolicy) -> Self { + self.external_reference_fetcher = ExternalReferenceFetcher::try_new( + policy.clone(), + self.external_reference_settings + .allowed_url_prefixes + .clone(), + self.external_reference_settings.max_fetch_bytes, + std::time::Duration::from_millis(self.external_reference_settings.fetch_timeout_ms), + ) + .expect("validated external reference settings remain valid"); self.outbound_http_policy = policy; self } + pub fn with_external_reference_import( + mut self, + settings: &ExternalReferenceSettings, + ) -> Result { + self.external_reference_fetcher = ExternalReferenceFetcher::try_new( + self.outbound_http_policy.clone(), + settings.allowed_url_prefixes.clone(), + settings.max_fetch_bytes, + std::time::Duration::from_millis(settings.fetch_timeout_ms), + )?; + self.external_reference_normalization.max_reference_depth = settings.max_depth; + self.external_reference_normalization + .max_reference_documents = settings.max_documents; + self.external_reference_normalization + .max_external_document_bytes = settings.max_fetch_bytes; + self.external_reference_normalization.max_expanded_nodes = settings.max_expanded_nodes; + self.external_reference_normalization + .external_references_enabled = !settings.allowed_url_prefixes.is_empty(); + self.external_reference_settings = settings.clone(); + Ok(self) + } + /// Reuses the process-wide immutable artifact authority for OpenAPI /// ingress and reconciliation. Tests may omit this and use their private /// storage root instead. @@ -299,6 +357,11 @@ impl AdminServiceBuilder { .capability_profile .unwrap_or_else(|| Arc::new(CommunityCapabilityProfile)), outbound_http_policy: self.outbound_http_policy, + external_reference_fetcher: self.external_reference_fetcher, + external_reference_normalization: self.external_reference_normalization, + external_reference_materialization_timeout: std::time::Duration::from_millis( + self.external_reference_settings.fetch_timeout_ms, + ), public_base_url: self.public_base_url, } } diff --git a/apps/admin-api/src/service/imports.rs b/apps/admin-api/src/service/imports.rs index b57b128..36be097 100644 --- a/apps/admin-api/src/service/imports.rs +++ b/apps/admin-api/src/service/imports.rs @@ -11,7 +11,7 @@ use crank_import::rest::{ }; use crank_registry::{ ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity, - CreateArtifactSourceRequest, CreateImportJobRequest, DetachArtifactSourceRequest, + CreateArtifactSourceRequest, CreateImportJobRequest, FinishImportJobRequest, ImportConflictMode, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope, ImportJobStatus, ImportOperationDraft, RegistryError, }; @@ -29,6 +29,13 @@ use crate::{ }, }; +mod external_references; +mod job_contract; +use external_references::ImportReplayContext; +use job_contract::{ + SourceDetachGuard, import_job_dependencies, import_job_normalization_config, import_job_source, +}; + const IMPORT_JOB_TTL_HOURS: i64 = 24; const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); const OPENAPI_PARSE_CONCURRENCY: usize = 4; @@ -95,9 +102,14 @@ impl AdminService { detach_guard.detach_now().await; return Err(ApiError::openapi_upload(locale, "source_integrity")); } + let mut dependencies = self + .materialize_external_reference_snapshots(workspace_id, &verified.bytes, now) + .await; let parsed = match parse_verified_preview( verified.bytes, artifact.artifact_ref().digest_hex().to_owned(), + dependencies.snapshots.clone(), + self.external_reference_normalization.clone(), locale, false, ) @@ -105,6 +117,7 @@ impl AdminService { { Ok(preview) => preview, Err(error) => { + dependencies.detach_all().await; detach_guard.detach_now().await; return Err(error); } @@ -115,6 +128,7 @@ impl AdminService { .iter() .all(|group| group.operations.is_empty()) { + dependencies.detach_all().await; detach_guard.detach_now().await; return Err(ApiError::openapi_upload(locale, "no_methods")); } @@ -130,12 +144,15 @@ impl AdminService { "source_id": source_envelope.source_id.as_str(), "digest": source_envelope.digest.as_str(), }, + "dependencies": dependencies.payload(), + "dependency_snapshots": dependencies.snapshot_payload(), "preview": preview_value, "preview_digest": preview_digest, "normalization": { "normalizer_version": NORMALIZER_VERSION, "projection_version": PROJECTION_VERSION, "ir_fingerprint": parsed.ir_fingerprint, + "config": self.external_reference_normalization, }, }); @@ -155,9 +172,11 @@ impl AdminService { }) .await { + dependencies.detach_all().await; detach_guard.detach_now().await; return Err(ApiError::from(error)); } + dependencies.disarm(); detach_guard.disarm(); Ok(OpenApiImportPreviewResponse { @@ -234,6 +253,13 @@ impl AdminService { } else { ImportConflictMode::Rename }; + let replay_context = ImportReplayContext { + workspace_id, + job_id, + application_key: &application_key, + conflict_mode, + finished_at: &finished_at, + }; if job.status == ImportJobStatus::Completed { let applied = self .registry @@ -250,7 +276,18 @@ impl AdminService { return Ok(openapi_import_response(applied)); } - let source = import_job_source(&job.preview_payload, locale)?; + let source = match import_job_source(&job.preview_payload, locale) { + Ok(source) => source, + Err(error) => { + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_source_verification_failed", + error, + ) + .await; + } + }; let verified = match self .registry .read_artifact_source( @@ -264,44 +301,123 @@ impl AdminService { Err( error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable), ) => { - // Another request may have finished and detached the source - // after our unlocked Pending read. Re-read the job and ask - // the registry for its canonical, locked replay instead of - // exposing a false `source_unavailable` outcome. - let latest = self.registry.get_import_job(workspace_id, job_id).await?; - if latest.is_some_and(|latest| latest.status == ImportJobStatus::Completed) { - let applied = self - .registry - .apply_import_job(ApplyImportJobRequest { - id: job_id, - workspace_id, - application_key: &application_key, - conflict_mode, - operations: &[], - pre_skipped: &[], - finished_at: &finished_at, - }) - .await?; - return Ok(openapi_import_response(applied)); - } - return Err(openapi_source_error(locale, error)); + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_source_verification_failed", + openapi_source_error(locale, error), + ) + .await; + } + Err(error @ RegistryError::SourceIntegrity) => { + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_source_verification_failed", + openapi_source_error(locale, error), + ) + .await; } Err(error) => return Err(openapi_source_error(locale, error)), }; if verified.source.blob.artifact_ref != source.digest { - return Err(ApiError::openapi_upload(locale, "source_integrity")); + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_source_verification_failed", + ApiError::openapi_upload(locale, "source_integrity"), + ) + .await; } - let legacy_v1 = job.preview_payload.get("normalization").is_none(); - let parsed = parse_verified_preview( - verified.bytes, - source.digest.digest_hex().to_owned(), - locale, - legacy_v1, - ) - .await?; - verify_preview_contract(&job.preview_payload, &parsed, locale)?; - if preview_has_blocker(&parsed.preview) { - return Err(ApiError::openapi_upload(locale, "invalid_document")); + let replay_contract = match import_replay_contract(&job.preview_payload, locale) { + Ok(contract) => contract, + Err(error) => { + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_replay_verification_failed", + error, + ) + .await; + } + }; + let parsed = match replay_contract { + ImportReplayContract::PersistedV2(parsed) => parsed, + ImportReplayContract::LegacyV1 | ImportReplayContract::Current => { + let legacy_v1 = matches!(replay_contract, ImportReplayContract::LegacyV1); + let dependency_snapshots = match self + .read_import_job_dependencies(workspace_id, &job.preview_payload, locale) + .await + { + Ok(snapshots) => snapshots, + Err(error) => { + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_dependency_verification_failed", + error, + ) + .await; + } + }; + let normalization_config = match import_job_normalization_config( + &job.preview_payload, + &self.external_reference_normalization, + legacy_v1, + locale, + ) { + Ok(config) => config, + Err(error) => { + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_replay_verification_failed", + error, + ) + .await; + } + }; + let parsed = match parse_verified_preview( + verified.bytes, + source.digest.digest_hex().to_owned(), + dependency_snapshots, + normalization_config, + locale, + legacy_v1, + ) + .await + { + Ok(parsed) => parsed, + Err(error) => { + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_replay_verification_failed", + error, + ) + .await; + } + }; + if let Err(error) = verify_preview_contract(&job.preview_payload, &parsed, locale) { + return self + .fail_openapi_import_or_replay( + &replay_context, + "import_replay_verification_failed", + error, + ) + .await; + } + parsed + } + }; + if preview_has_blocker(&parsed.preview, &selected) { + return self + .fail_openapi_import_or_replay( + &replay_context, + "reference_resolution_blocked", + ApiError::openapi_upload(locale, "invalid_document"), + ) + .await; } let mut candidates = BTreeMap::new(); for group in &parsed.preview.groups { @@ -372,6 +488,24 @@ impl AdminService { } } +fn canonical_external_document_uri(base: Option<&str>, reference: &str) -> Option { + if reference.starts_with('#') { + return None; + } + let mut url = match base { + Some(base) => url::Url::parse(base).ok()?.join(reference).ok()?, + None => url::Url::parse(reference).ok()?, + }; + if !matches!(url.scheme(), "http" | "https") + || !url.username().is_empty() + || url.password().is_some() + { + return None; + } + url.set_fragment(None); + Some(url.to_string()) +} + fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse { let created = applied .created @@ -470,9 +604,72 @@ struct ParsedOpenApiPreview { ir_fingerprint: String, } +enum ImportReplayContract { + LegacyV1, + PersistedV2(ParsedOpenApiPreview), + Current, +} + +fn import_replay_contract( + payload: &serde_json::Value, + locale: OpenApiUploadLocale, +) -> Result { + let Some(normalization) = payload.get("normalization") else { + return Ok(ImportReplayContract::LegacyV1); + }; + let normalizer = normalization + .get("normalizer_version") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + let projection = normalization + .get("projection_version") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + let ir_fingerprint = normalization + .get("ir_fingerprint") + .and_then(serde_json::Value::as_str) + .filter(|value| is_lower_sha256(value)) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + let expected_preview_digest = payload + .get("preview_digest") + .and_then(serde_json::Value::as_str) + .filter(|value| is_lower_sha256(value)) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + + if normalizer == NORMALIZER_VERSION && projection == PROJECTION_VERSION { + return Ok(ImportReplayContract::Current); + } + if normalizer != "normalized-ir-v2" || projection != "preview-v2" { + return Err(ApiError::openapi_upload(locale, "source_integrity")); + } + + let preview_value = payload + .get("preview") + .cloned() + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + if preview_digest(&preview_value)? != expected_preview_digest { + return Err(ApiError::openapi_upload(locale, "source_integrity")); + } + let preview = serde_json::from_value(preview_value) + .map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?; + Ok(ImportReplayContract::PersistedV2(ParsedOpenApiPreview { + preview, + ir_fingerprint: ir_fingerprint.to_owned(), + })) +} + +fn is_lower_sha256(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + async fn parse_verified_preview( bytes: Vec, digest: String, + snapshots: Vec, + normalization_config: crank_import::rest::NormalizationConfig, locale: OpenApiUploadLocale, legacy_v1: bool, ) -> Result { @@ -498,14 +695,18 @@ async fn parse_verified_preview( } let digest = crank_import::rest::SourceDigest::parse(digest) .map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?; - let ir = - crank_import::rest::normalize_verified_document(document, digest, &Default::default()) - .map_err(|error| match error { - crank_import::rest::ImportParseError::NoMethods => { - ApiError::openapi_upload(locale, "no_methods") - } - _ => ApiError::openapi_upload(locale, "invalid_document"), - })?; + let ir = crank_import::rest::normalize_verified_bundle( + document, + digest, + &snapshots, + &normalization_config, + ) + .map_err(|error| match error { + crank_import::rest::ImportParseError::NoMethods => { + ApiError::openapi_upload(locale, "no_methods") + } + _ => ApiError::openapi_upload(locale, "invalid_document"), + })?; let ir_fingerprint = preview_digest( &serde_json::to_value(&ir) .map_err(|_| ApiError::openapi_upload(locale, "invalid_document"))?, @@ -597,113 +798,21 @@ fn verify_preview_contract( } } -fn preview_has_blocker(preview: &crank_import::rest::ImportPreview) -> bool { +fn preview_has_blocker( + preview: &crank_import::rest::ImportPreview, + selected_operation_keys: &BTreeSet, +) -> bool { preview .findings .iter() - .chain( - preview - .groups - .iter() - .flat_map(|group| group.operations.iter()) - .flat_map(|operation| operation.findings.iter()), - ) .any(|finding| finding.severity == ImportFindingSeverity::Error) -} - -fn import_job_source( - payload: &serde_json::Value, - locale: OpenApiUploadLocale, -) -> Result { - let source = payload - .get("source") - .ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?; - let source_id = source - .get("source_id") - .and_then(serde_json::Value::as_str) - .filter(|value| value.len() <= 132) - .ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?; - let digest = source - .get("digest") - .and_then(serde_json::Value::as_str) - .and_then(|value| value.parse().ok()) - .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; - Ok(ImportJobSourceEnvelope { - source_id: ArtifactSourceId::new(source_id), - digest, - }) -} - -struct SourceDetachGuard { - registry: crank_registry::PostgresRegistry, - workspace_id: WorkspaceId, - source_id: ArtifactSourceId, - expected_updated_at: OffsetDateTime, - armed: bool, -} - -impl SourceDetachGuard { - fn new( - registry: crank_registry::PostgresRegistry, - workspace_id: WorkspaceId, - source_id: ArtifactSourceId, - expected_updated_at: OffsetDateTime, - ) -> Self { - Self { - registry, - workspace_id, - source_id, - expected_updated_at, - armed: true, - } - } - - fn disarm(&mut self) { - self.armed = false; - } - - async fn detach_now(&mut self) { - if !self.armed { - return; - } - self.armed = false; - // This is the normal error path: await the database transition so a - // caller receives a completed cleanup before it can retry. `Drop` - // remains only the cancellation/shutdown safety net. - let _ = self - .registry - .detach_artifact_source(DetachArtifactSourceRequest { - workspace_id: &self.workspace_id, - source_id: &self.source_id, - expected_updated_at: Some(self.expected_updated_at), - detached_at: OffsetDateTime::now_utc(), - }) - .await; - } -} - -impl Drop for SourceDetachGuard { - fn drop(&mut self) { - if !self.armed { - return; - } - let registry = self.registry.clone(); - let workspace_id = self.workspace_id.clone(); - let source_id = self.source_id.clone(); - let expected_updated_at = Some(self.expected_updated_at); - if let Ok(handle) = tokio::runtime::Handle::try_current() { - handle.spawn(async move { - let _ = registry - .detach_artifact_source(DetachArtifactSourceRequest { - workspace_id: &workspace_id, - source_id: &source_id, - expected_updated_at, - detached_at: OffsetDateTime::now_utc(), - }) - .await; - }); - } - } + || preview + .groups + .iter() + .flat_map(|group| group.operations.iter()) + .filter(|operation| selected_operation_keys.contains(&operation.key)) + .flat_map(|operation| operation.findings.iter()) + .any(|finding| finding.severity == ImportFindingSeverity::Error) } fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result { diff --git a/apps/admin-api/src/service/imports/external_references.rs b/apps/admin-api/src/service/imports/external_references.rs new file mode 100644 index 0000000..b225ed8 --- /dev/null +++ b/apps/admin-api/src/service/imports/external_references.rs @@ -0,0 +1,405 @@ +use super::*; +use std::collections::VecDeque; +use tracing::warn; + +pub(super) struct MaterializedDependencies { + pub(super) snapshots: Vec, + envelopes: Vec, + canonical_uris: Vec, + guards: Vec, +} + +pub(super) struct ImportReplayContext<'a> { + pub(super) workspace_id: &'a WorkspaceId, + pub(super) job_id: &'a ImportJobId, + pub(super) application_key: &'a str, + pub(super) conflict_mode: ImportConflictMode, + pub(super) finished_at: &'a OffsetDateTime, +} + +impl MaterializedDependencies { + fn empty() -> Self { + Self { + snapshots: Vec::new(), + envelopes: Vec::new(), + canonical_uris: Vec::new(), + guards: Vec::new(), + } + } + + pub(super) fn payload(&self) -> Vec { + self.envelopes + .iter() + .zip(&self.canonical_uris) + .map(|(dependency, canonical_uri)| { + json!({ + "source_id": dependency.source_id.as_str(), + "digest": dependency.digest.as_str(), + "canonical_uri": canonical_uri, + }) + }) + .collect() + } + + pub(super) fn snapshot_payload(&self) -> Vec { + self.snapshots + .iter() + .filter_map(|snapshot| { + self.envelopes + .iter() + .find(|dependency| dependency.digest.digest_hex() == snapshot.digest.as_str()) + .map(|dependency| { + json!({ + "source_id": dependency.source_id.as_str(), + "digest": dependency.digest.as_str(), + "canonical_uri": snapshot.canonical_uri, + }) + }) + }) + .collect() + } + + pub(super) fn disarm(&mut self) { + for guard in &mut self.guards { + guard.disarm(); + } + } + + pub(super) async fn detach_all(&mut self) { + for guard in &mut self.guards { + guard.detach_now().await; + } + } +} + +impl AdminService { + pub(super) async fn fail_openapi_import_or_replay( + &self, + context: &ImportReplayContext<'_>, + error_text: &'static str, + error: ApiError, + ) -> Result { + let empty = json!([]); + self.registry + .finish_import_job(FinishImportJobRequest { + id: context.job_id, + status: ImportJobStatus::Failed, + created_operation_ids: &empty, + error_text: Some(error_text), + finished_at: context.finished_at, + }) + .await?; + + // `finish_import_job` preserves an already Completed row under its + // lock. This read distinguishes that race from the terminal Failed + // transition and returns the immutable canonical application result. + let latest = self + .registry + .get_import_job(context.workspace_id, context.job_id) + .await?; + if latest.is_some_and(|job| job.status == ImportJobStatus::Completed) { + let applied = self + .registry + .apply_import_job(ApplyImportJobRequest { + id: context.job_id, + workspace_id: context.workspace_id, + application_key: context.application_key, + conflict_mode: context.conflict_mode, + operations: &[], + pre_skipped: &[], + finished_at: context.finished_at, + }) + .await?; + return Ok(openapi_import_response(applied)); + } + + Err(error) + } + + pub(super) async fn materialize_external_reference_snapshots( + &self, + workspace_id: &WorkspaceId, + primary_bytes: &[u8], + created_at: OffsetDateTime, + ) -> MaterializedDependencies { + let materialized_count = std::sync::atomic::AtomicUsize::new(0); + match tokio::time::timeout( + self.external_reference_materialization_timeout, + self.materialize_external_reference_snapshots_inner( + workspace_id, + primary_bytes, + created_at, + &materialized_count, + ), + ) + .await + { + Ok(dependencies) => dependencies, + Err(_) => { + materialization_failure( + "chain", + "timeout", + materialized_count.load(std::sync::atomic::Ordering::Relaxed), + ); + MaterializedDependencies::empty() + } + } + } + + async fn materialize_external_reference_snapshots_inner( + &self, + workspace_id: &WorkspaceId, + primary_bytes: &[u8], + created_at: OffsetDateTime, + materialized_count: &std::sync::atomic::AtomicUsize, + ) -> MaterializedDependencies { + if !self + .external_reference_normalization + .external_references_enabled + { + return MaterializedDependencies::empty(); + } + let Ok(primary) = std::str::from_utf8(primary_bytes) else { + materialization_failure("primary_decode", "invalid_utf8", 0); + return MaterializedDependencies::empty(); + }; + let Ok(primary_references) = + crank_import::rest::reference_uris(primary, &self.external_reference_normalization) + else { + materialization_failure("primary_scan", "invalid_document", 0); + return MaterializedDependencies::empty(); + }; + let mut queue = VecDeque::new(); + for reference in primary_references { + if let Some(uri) = canonical_external_document_uri(None, &reference) { + queue.push_back((uri, 1usize)); + } + } + let mut seen = BTreeSet::new(); + let mut result = MaterializedDependencies::empty(); + let mut dependency_by_digest = BTreeMap::::new(); + while let Some((canonical_uri, depth)) = queue.pop_front() { + if !seen.insert(canonical_uri.clone()) + || depth > self.external_reference_normalization.max_reference_depth + || seen.len() + > self + .external_reference_normalization + .max_reference_documents + { + continue; + } + let bytes = match self.external_reference_fetcher.get(&canonical_uri).await { + Ok(bytes) => bytes, + Err(error) => { + materialization_failure( + "fetch", + external_fetch_error_code(&error), + result.snapshots.len(), + ); + continue; + } + }; + let Ok(document) = std::str::from_utf8(&bytes).map(str::to_owned) else { + materialization_failure( + "dependency_decode", + "invalid_utf8", + result.snapshots.len(), + ); + continue; + }; + let Ok(references) = crank_import::rest::external_reference_uris( + &document, + &self.external_reference_normalization, + ) else { + materialization_failure( + "dependency_scan", + "invalid_document", + result.snapshots.len(), + ); + continue; + }; + let store = self.artifact_store.clone(); + let artifact_bytes = bytes.clone(); + let Ok(Ok(artifact)) = + tokio::task::spawn_blocking(move || store.put_registered(&artifact_bytes)).await + else { + materialization_failure( + "artifact_store", + "storage_unavailable", + result.snapshots.len(), + ); + continue; + }; + let digest_key = artifact.artifact_ref().as_str().to_owned(); + let envelope_index = if let Some(index) = dependency_by_digest.get(&digest_key) { + *index + } else { + let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi_dep")); + let Ok(source) = self + .registry + .create_artifact_source(CreateArtifactSourceRequest { + workspace_id, + source_id: &source_id, + artifact: &artifact, + mime_type: "application/octet-stream", + sensitivity: ArtifactSourceSensitivity::Internal, + created_at, + }) + .await + else { + materialization_failure( + "source_create", + "registry_unavailable", + result.snapshots.len(), + ); + continue; + }; + let mut guard = SourceDetachGuard::new( + self.registry.clone(), + workspace_id.clone(), + source_id.clone(), + source.updated_at, + ); + let Ok(verified) = self + .registry + .read_artifact_source( + std::sync::Arc::clone(&self.artifact_store), + workspace_id, + &source_id, + ) + .await + else { + materialization_failure( + "source_verify", + "source_unavailable", + result.snapshots.len(), + ); + guard.detach_now().await; + continue; + }; + if verified.source.blob.artifact_ref != *artifact.artifact_ref() { + materialization_failure( + "source_verify", + "source_integrity", + result.snapshots.len(), + ); + guard.detach_now().await; + continue; + } + let index = result.envelopes.len(); + result.envelopes.push(ImportJobSourceEnvelope { + source_id: source_id.clone(), + digest: artifact.artifact_ref().clone(), + }); + result.guards.push(guard); + dependency_by_digest.insert(digest_key, index); + index + }; + let dependency = &result.envelopes[envelope_index]; + let snapshot_digest = match crank_import::rest::SourceDigest::parse( + dependency.digest.digest_hex().to_owned(), + ) { + Ok(digest) => digest, + Err(_) => { + materialization_failure("snapshot", "source_integrity", result.snapshots.len()); + continue; + } + }; + result + .snapshots + .push(crank_import::rest::ExternalDocumentSnapshot { + canonical_uri: canonical_uri.clone(), + digest: snapshot_digest, + document, + }); + materialized_count.store(result.snapshots.len(), std::sync::atomic::Ordering::Relaxed); + result.canonical_uris.push(canonical_uri.clone()); + for reference in references { + if let Some(uri) = canonical_external_document_uri(Some(&canonical_uri), &reference) + { + queue.push_back((uri, depth.saturating_add(1))); + } + } + } + // Registry ownership is one envelope per immutable digest, while the + // resolver may map several canonical URIs to that same snapshot. Keep + // payload rows aligned with envelopes and add aliases separately. + if result.envelopes.len() != result.canonical_uris.len() { + let mut canonical_by_digest = BTreeMap::new(); + for snapshot in &result.snapshots { + canonical_by_digest + .entry(snapshot.digest.as_str().to_owned()) + .or_insert_with(|| snapshot.canonical_uri.clone()); + } + result.canonical_uris = result + .envelopes + .iter() + .map(|envelope| { + canonical_by_digest + .get(envelope.digest.digest_hex()) + .cloned() + .unwrap_or_default() + }) + .collect(); + } + result + } + + pub(super) async fn read_import_job_dependencies( + &self, + workspace_id: &WorkspaceId, + payload: &serde_json::Value, + locale: OpenApiUploadLocale, + ) -> Result, ApiError> { + let dependencies = import_job_dependencies(payload, locale)?; + let mut snapshots = Vec::with_capacity(dependencies.len()); + for (canonical_uri, dependency) in dependencies { + let verified = self + .registry + .read_artifact_source( + std::sync::Arc::clone(&self.artifact_store), + workspace_id, + &dependency.source_id, + ) + .await + .map_err(|error| openapi_source_error(locale, error))?; + if verified.source.blob.artifact_ref != dependency.digest { + return Err(ApiError::openapi_upload(locale, "source_integrity")); + } + let document = String::from_utf8(verified.bytes) + .map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?; + let digest = + crank_import::rest::SourceDigest::parse(dependency.digest.digest_hex().to_owned()) + .map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?; + snapshots.push(crank_import::rest::ExternalDocumentSnapshot { + canonical_uri, + digest, + document, + }); + } + Ok(snapshots) + } +} +fn external_fetch_error_code(error: &crank_runtime::ExternalReferenceFetchError) -> &'static str { + match error { + crank_runtime::ExternalReferenceFetchError::Disabled => "disabled", + crank_runtime::ExternalReferenceFetchError::InvalidUrl => "invalid_url", + crank_runtime::ExternalReferenceFetchError::TargetNotAllowed => "target_not_allowed", + crank_runtime::ExternalReferenceFetchError::RedirectNotAllowed => "redirect_not_allowed", + crank_runtime::ExternalReferenceFetchError::ResponseTooLarge { .. } => "response_too_large", + crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { .. } => "unexpected_status", + crank_runtime::ExternalReferenceFetchError::Transport { timeout: true, .. } => "timeout", + crank_runtime::ExternalReferenceFetchError::Transport { .. } => "transport", + crank_runtime::ExternalReferenceFetchError::InvalidConfiguration => "invalid_configuration", + } +} + +fn materialization_failure(stage: &'static str, error_code: &'static str, count: usize) { + warn!( + name: "admin.openapi_import.materialization_failed", + stage, + error_code, + count, + "external OpenAPI materialization failed" + ); +} diff --git a/apps/admin-api/src/service/imports/job_contract.rs b/apps/admin-api/src/service/imports/job_contract.rs new file mode 100644 index 0000000..62b7ec3 --- /dev/null +++ b/apps/admin-api/src/service/imports/job_contract.rs @@ -0,0 +1,169 @@ +use std::collections::BTreeSet; + +use crank_core::WorkspaceId; +use crank_registry::{ArtifactSourceId, DetachArtifactSourceRequest, ImportJobSourceEnvelope}; +use time::OffsetDateTime; + +use crate::{error::ApiError, service::OpenApiUploadLocale}; + +use super::canonical_external_document_uri; + +const MAX_IMPORT_JOB_DOCUMENTS: usize = 32; + +pub(super) fn import_job_source( + payload: &serde_json::Value, + locale: OpenApiUploadLocale, +) -> Result { + let source = payload + .get("source") + .ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?; + let source_id = source + .get("source_id") + .and_then(serde_json::Value::as_str) + .filter(|value| value.len() <= 132) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?; + let digest = source + .get("digest") + .and_then(serde_json::Value::as_str) + .and_then(|value| value.parse().ok()) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + Ok(ImportJobSourceEnvelope { + source_id: ArtifactSourceId::new(source_id), + digest, + }) +} + +pub(super) fn import_job_dependencies( + payload: &serde_json::Value, + locale: OpenApiUploadLocale, +) -> Result, ApiError> { + let empty = Vec::new(); + let items = payload + .get("dependency_snapshots") + .and_then(serde_json::Value::as_array) + .unwrap_or(&empty); + if items.len() > MAX_IMPORT_JOB_DOCUMENTS { + return Err(ApiError::openapi_upload(locale, "source_integrity")); + } + let mut canonical_uris = BTreeSet::new(); + items + .iter() + .map(|item| { + let canonical_uri = item + .get("canonical_uri") + .and_then(serde_json::Value::as_str) + .filter(|value| { + canonical_external_document_uri(None, value).as_deref() == Some(*value) + }) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + if !canonical_uris.insert(canonical_uri.to_owned()) { + return Err(ApiError::openapi_upload(locale, "source_integrity")); + } + let source_id = item + .get("source_id") + .and_then(serde_json::Value::as_str) + .filter(|value| value.len() <= 132) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + let digest = item + .get("digest") + .and_then(serde_json::Value::as_str) + .and_then(|value| value.parse().ok()) + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?; + Ok(( + canonical_uri.to_owned(), + ImportJobSourceEnvelope { + source_id: ArtifactSourceId::new(source_id), + digest, + }, + )) + }) + .collect() +} + +pub(super) fn import_job_normalization_config( + payload: &serde_json::Value, + current: &crank_import::rest::NormalizationConfig, + legacy_v1: bool, + locale: OpenApiUploadLocale, +) -> Result { + if legacy_v1 { + return Ok(current.clone()); + } + payload + .pointer("/normalization/config") + .cloned() + .ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity")) + .and_then(|value| { + serde_json::from_value(value) + .map_err(|_| ApiError::openapi_upload(locale, "source_integrity")) + }) +} + +pub(super) struct SourceDetachGuard { + registry: crank_registry::PostgresRegistry, + workspace_id: WorkspaceId, + source_id: ArtifactSourceId, + expected_updated_at: OffsetDateTime, + armed: bool, +} + +impl SourceDetachGuard { + pub(super) fn new( + registry: crank_registry::PostgresRegistry, + workspace_id: WorkspaceId, + source_id: ArtifactSourceId, + expected_updated_at: OffsetDateTime, + ) -> Self { + Self { + registry, + workspace_id, + source_id, + expected_updated_at, + armed: true, + } + } + + pub(super) fn disarm(&mut self) { + self.armed = false; + } + + pub(super) async fn detach_now(&mut self) { + if !self.armed { + return; + } + self.armed = false; + let _ = self + .registry + .detach_artifact_source(DetachArtifactSourceRequest { + workspace_id: &self.workspace_id, + source_id: &self.source_id, + expected_updated_at: Some(self.expected_updated_at), + detached_at: OffsetDateTime::now_utc(), + }) + .await; + } +} + +impl Drop for SourceDetachGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + let registry = self.registry.clone(); + let workspace_id = self.workspace_id.clone(); + let source_id = self.source_id.clone(); + let expected_updated_at = Some(self.expected_updated_at); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn(async move { + let _ = registry + .detach_artifact_source(DetachArtifactSourceRequest { + workspace_id: &workspace_id, + source_id: &source_id, + expected_updated_at, + detached_at: OffsetDateTime::now_utc(), + }) + .await; + }); + } + } +} diff --git a/apps/admin-api/tests/integration/common.rs b/apps/admin-api/tests/integration/common.rs index c04ad7e..b8f8dae 100644 --- a/apps/admin-api/tests/integration/common.rs +++ b/apps/admin-api/tests/integration/common.rs @@ -108,6 +108,24 @@ pub(super) fn build_test_app( }) } +pub(super) fn build_test_app_with_external_references( + registry: PostgresRegistry, + storage_root: std::path::PathBuf, + allowed_url_prefixes: Vec, +) -> Router { + build_app(AppState { + service: test_service_with_external_references( + registry, + storage_root, + allowed_url_prefixes, + ), + api_rate_limiter: crank_runtime::RequestRateLimiter::new( + crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(), + ), + trusted_proxy_ips: Vec::new(), + }) +} + pub(super) fn build_test_app_with_audit_sink( registry: PostgresRegistry, storage_root: std::path::PathBuf, @@ -152,6 +170,33 @@ pub(super) fn test_service( .build() } +pub(super) fn test_service_with_external_references( + registry: PostgresRegistry, + storage_root: std::path::PathBuf, + allowed_url_prefixes: Vec, +) -> AdminService { + let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]); + let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build(); + AdminServiceBuilder::new( + registry, + storage_root, + test_auth_settings(), + test_secret_crypto(), + runtime, + ) + .with_external_reference_import(&crank_config::ExternalReferenceSettings { + allowed_url_prefixes, + max_depth: 8, + max_documents: 32, + max_fetch_bytes: 64 * 1024, + fetch_timeout_ms: 2_000, + max_expanded_nodes: 10_000, + }) + .unwrap() + .with_outbound_http_policy(outbound_policy) + .build() +} + pub(super) async fn spawn_upstream_server() -> String { let app = Router::new().route("/crm/leads", post(create_lead)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/apps/admin-api/tests/integration/openapi_import.rs b/apps/admin-api/tests/integration/openapi_import.rs index 572254c..650c6b3 100644 --- a/apps/admin-api/tests/integration/openapi_import.rs +++ b/apps/admin-api/tests/integration/openapi_import.rs @@ -1,4 +1,6 @@ -use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale}; +use admin_api::service::{ + AdminServiceBuilder, OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale, +}; use crank_core::WorkspaceId; use crank_registry::ImportJobStatus; use serial_test::serial; @@ -7,6 +9,8 @@ use super::common::{ test_auth_settings, test_registry, test_secret_crypto, test_service, test_storage_root, }; +mod external_references; + const OPENAPI3: &str = r#" openapi: 3.0.3 info: @@ -73,11 +77,11 @@ async fn previews_openapi_and_creates_draft_operations() { assert_eq!(preview_job.status, ImportJobStatus::Pending); assert_eq!( preview_job.preview_payload["normalization"]["normalizer_version"], - "normalized-ir-v2" + "normalized-ir-v3" ); assert_eq!( preview_job.preview_payload["normalization"]["projection_version"], - "preview-v2" + "preview-v3" ); assert!(preview_job.preview_payload["normalization"]["ir_fingerprint"].is_string()); @@ -236,7 +240,7 @@ async fn unknown_selected_operations_are_persisted_in_the_canonical_replay() { async fn concurrent_openapi_import_replays_the_same_atomic_result() { let registry = test_registry().await; let service = test_service( - registry, + registry.clone(), test_storage_root("openapi_import_replay"), test_auth_settings(), test_secret_crypto(), @@ -271,6 +275,31 @@ async fn concurrent_openapi_import_replays_the_same_atomic_result() { service.list_operations(&workspace_id).await.unwrap().len(), 1 ); + let completed_before = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + let empty = serde_json::json!([]); + let failed_at = time::OffsetDateTime::now_utc(); + registry + .finish_import_job(crank_registry::FinishImportJobRequest { + id: &job_id, + status: ImportJobStatus::Failed, + created_operation_ids: &empty, + error_text: Some("late_verification_failure"), + finished_at: &failed_at, + }) + .await + .unwrap(); + let completed_after = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + assert_eq!(completed_after.status, ImportJobStatus::Completed); + assert_eq!(completed_after.error_text, completed_before.error_text); + assert_eq!(completed_after.finished_at, completed_before.finished_at); let conflicting_replay = service .create_openapi_import( @@ -326,6 +355,13 @@ async fn apply_fails_closed_when_the_preview_parser_contract_drifts() { .await; assert!(result.is_err()); + assert_failed_job_and_detached_sources( + ®istry, + &workspace_id, + &job_id, + "import_replay_verification_failed", + ) + .await; assert!( service .list_operations(&workspace_id) @@ -395,6 +431,13 @@ async fn apply_fails_closed_for_each_normalization_contract_field_and_digest_sha .await .is_err() ); + assert_failed_job_and_detached_sources( + ®istry, + &workspace_id, + &job_id, + "import_replay_verification_failed", + ) + .await; assert!( service .list_operations(&workspace_id) @@ -449,7 +492,7 @@ async fn legacy_job_without_normalization_contract_replays_until_expiry() { async fn blocker_finding_keeps_valid_preview_but_prevents_draft_mutation() { let registry = test_registry().await; let service = test_service( - registry, + registry.clone(), test_storage_root("openapi_import_blocker"), test_auth_settings(), test_secret_crypto(), @@ -503,11 +546,124 @@ paths: .unwrap() .is_empty() ); + assert_failed_job_and_detached_sources( + ®istry, + &workspace_id, + &preview.job_id.as_str().into(), + "reference_resolution_blocked", + ) + .await; } #[tokio::test] #[serial] -async fn operation_level_reference_blocker_prevents_draft_mutation() { +async fn operation_blockers_apply_only_to_selected_keys_and_full_selection_fails_atomically() { + let registry = test_registry().await; + let service = test_service( + registry.clone(), + test_storage_root("openapi_import_selected_blockers"), + test_auth_settings(), + test_secret_crypto(), + ); + let workspace_id = WorkspaceId::new("ws_default"); + let upload = OpenApiUpload { + bytes: br#" +openapi: 3.1.0 +info: { title: Selected blockers } +servers: [{ url: https://api.example.test }] +paths: + /valid: + get: + operationId: validOperation + responses: { '200': { description: ok } } + /broken: + get: + operationId: brokenOperation + responses: + '200': + description: unresolved external schema + content: + application/json: + schema: { $ref: 'https://schemas.example.test/missing.yaml#/Result' } +"# + .to_vec(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }; + + let valid_preview = service + .preview_openapi_import(&workspace_id, upload.clone()) + .await + .unwrap(); + assert!( + valid_preview.preview.findings.iter().all(|finding| { + finding.severity != crank_import::rest::ImportFindingSeverity::Error + }) + ); + let broken = valid_preview + .preview + .groups + .iter() + .flat_map(|group| &group.operations) + .find(|operation| operation.key == "GET /broken") + .unwrap(); + assert!( + broken.findings.iter().any(|finding| { + finding.severity == crank_import::rest::ImportFindingSeverity::Error + }) + ); + + let valid = service + .create_openapi_import( + &workspace_id, + &valid_preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /valid".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + assert_eq!(valid.created.len(), 1); + + let full_preview = service + .preview_openapi_import(&workspace_id, upload) + .await + .unwrap(); + let result = service + .create_openapi_import( + &workspace_id, + &full_preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /valid".to_owned(), "GET /broken".to_owned()], + server_url: None, + conflict_mode: "rename".to_owned(), + }, + ) + .await; + assert!(result.is_err()); + assert_eq!( + service.list_operations(&workspace_id).await.unwrap().len(), + 1, + "the full selection must not create either selected Draft" + ); + let failed = registry + .get_import_job(&workspace_id, &full_preview.job_id.as_str().into()) + .await + .unwrap() + .unwrap(); + assert_eq!(failed.status, ImportJobStatus::Failed); + assert_eq!( + failed.error_text.as_deref(), + Some("reference_resolution_blocked") + ); + assert!(failed.finished_at.is_some()); +} + +#[tokio::test] +#[serial] +async fn local_operation_reference_resolves_and_creates_draft() { let registry = test_registry().await; let service = test_service( registry, @@ -520,6 +676,7 @@ async fn operation_level_reference_blocker_prevents_draft_mutation() { bytes: br#" openapi: 3.0.3 info: { title: References } +servers: [{ url: https://api.example.test }] paths: /referenced: get: @@ -551,32 +708,26 @@ components: preview.preview.groups[0].operations[0] .findings .iter() - .any(|finding| { - finding.code == "unresolved_reference" - && finding.severity == crank_import::rest::ImportFindingSeverity::Error - }) + .all(|finding| finding.code != "unresolved_reference") ); + assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1); - assert!( - service - .create_openapi_import( - &workspace_id, - &preview.job_id.as_str().into(), - OpenApiImportCreatePayload { - selected_operation_keys: vec!["GET /referenced".to_owned()], - server_url: None, - conflict_mode: "skip".to_owned(), - }, - ) - .await - .is_err() - ); - assert!( - service - .list_operations(&workspace_id) - .await - .unwrap() - .is_empty() + let created = service + .create_openapi_import( + &workspace_id, + &preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /referenced".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + assert_eq!(created.created.len(), 1); + assert_eq!( + service.list_operations(&workspace_id).await.unwrap().len(), + 1 ); } @@ -587,3 +738,44 @@ fn openapi_upload() -> OpenApiUpload { locale: OpenApiUploadLocale::En, } } + +async fn assert_failed_job_and_detached_sources( + registry: &crank_registry::PostgresRegistry, + workspace_id: &WorkspaceId, + job_id: &crank_registry::ImportJobId, + expected_error: &str, +) { + let job = registry + .get_import_job(workspace_id, job_id) + .await + .unwrap() + .unwrap(); + assert_eq!(job.status, ImportJobStatus::Failed); + assert!(job.finished_at.is_some()); + assert_eq!(job.error_text.as_deref(), Some(expected_error)); + + let mut source_ids = vec![ + job.preview_payload["source"]["source_id"] + .as_str() + .unwrap() + .to_owned(), + ]; + source_ids.extend( + job.preview_payload["dependencies"] + .as_array() + .into_iter() + .flatten() + .map(|dependency| dependency["source_id"].as_str().unwrap().to_owned()), + ); + for source_id in source_ids { + let lifecycle: String = sqlx::query_scalar( + "select lifecycle from artifact_sources where workspace_id = $1 and source_id = $2", + ) + .bind(workspace_id.as_str()) + .bind(source_id) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(lifecycle, "detached"); + } +} diff --git a/apps/admin-api/tests/integration/openapi_import/external_references.rs b/apps/admin-api/tests/integration/openapi_import/external_references.rs new file mode 100644 index 0000000..24d0e5b --- /dev/null +++ b/apps/admin-api/tests/integration/openapi_import/external_references.rs @@ -0,0 +1,538 @@ +use super::super::common::test_service_with_external_references; +use super::*; +use axum::{Router, routing::get}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +#[tokio::test] +#[serial] +async fn external_relative_chain_survives_reversed_builder_order_and_apply_never_refetches() { + let fetches = Arc::new(AtomicUsize::new(0)); + let root_fetches = Arc::clone(&fetches); + let child_fetches = Arc::clone(&fetches); + let app = Router::new() + .route( + "/root.yaml", + get(move || { + let fetches = Arc::clone(&root_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + "Item: { $ref: './child.yaml#/Item' }" + } + }), + ) + .route( + "/child.yaml", + get(move || { + let fetches = Arc::clone(&child_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + "Item: { type: object, required: [id], properties: { id: { type: string } } }" + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let origin = format!("http://{address}"); + + let registry = test_registry().await; + let service = test_service_with_external_references( + registry.clone(), + test_storage_root("openapi_import_external_snapshot"), + vec![format!("{origin}/")], + ); + let workspace_id = WorkspaceId::new("ws_default"); + let upload = OpenApiUpload { + bytes: format!( + r#" +openapi: 3.1.0 +info: {{ title: External }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: + application/json: + schema: {{ $ref: '{origin}/root.yaml#/Item' }} +"# + ) + .into_bytes(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }; + let preview = service + .preview_openapi_import(&workspace_id, upload) + .await + .unwrap(); + assert_eq!(fetches.load(Ordering::SeqCst), 2); + assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1); + let job = registry + .get_import_job(&workspace_id, &preview.job_id.as_str().into()) + .await + .unwrap() + .unwrap(); + assert_eq!( + job.preview_payload["dependencies"] + .as_array() + .unwrap() + .len(), + 2 + ); + assert_eq!( + job.preview_payload["dependency_snapshots"] + .as_array() + .unwrap() + .len(), + 2 + ); + + let applied = service + .create_openapi_import( + &workspace_id, + &preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /items".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + assert_eq!(applied.created.len(), 1); + assert_eq!(fetches.load(Ordering::SeqCst), 2); + let active_sources: i64 = sqlx::query_scalar( + "select count(*) from artifact_sources + where source_id like 'src_openapi_%' and lifecycle = 'active'", + ) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(active_sources, 0); +} + +#[tokio::test] +#[serial] +async fn missing_external_snapshot_fails_closed_before_draft_mutation() { + let app = Router::new().route( + "/schemas.yaml", + get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let origin = format!("http://{address}"); + let registry = test_registry().await; + let storage_root = test_storage_root("openapi_import_missing_external_snapshot"); + let service = test_service_with_external_references( + registry.clone(), + storage_root.clone(), + vec![format!("{origin}/")], + ); + let workspace_id = WorkspaceId::new("ws_default"); + let preview = service + .preview_openapi_import( + &workspace_id, + OpenApiUpload { + bytes: format!( + r#" +openapi: 3.1.0 +info: {{ title: External integrity }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }} +"# + ) + .into_bytes(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }, + ) + .await + .unwrap(); + let job = registry + .get_import_job(&workspace_id, &preview.job_id.as_str().into()) + .await + .unwrap() + .unwrap(); + let dependency_source_id = job.preview_payload["dependencies"][0]["source_id"] + .as_str() + .unwrap(); + sqlx::query( + "update artifact_sources + set lifecycle = 'detached', updated_at = now(), detached_at = now() + where workspace_id = $1 and source_id = $2", + ) + .bind(workspace_id.as_str()) + .bind(dependency_source_id) + .execute(registry.pool()) + .await + .unwrap(); + + assert!( + service + .create_openapi_import( + &workspace_id, + &preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /items".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .is_err() + ); + assert!( + service + .list_operations(&workspace_id) + .await + .unwrap() + .is_empty() + ); + assert_failed_job_and_detached_sources( + ®istry, + &workspace_id, + &preview.job_id.as_str().into(), + "import_dependency_verification_failed", + ) + .await; + + let corrupt_preview = service + .preview_openapi_import( + &workspace_id, + OpenApiUpload { + bytes: format!( + r#" +openapi: 3.1.0 +info: {{ title: Corrupt external integrity }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: listItemsCorrupt + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }} +"# + ) + .into_bytes(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }, + ) + .await + .unwrap(); + let corrupt_job = registry + .get_import_job(&workspace_id, &corrupt_preview.job_id.as_str().into()) + .await + .unwrap() + .unwrap(); + let dependency_ref: crank_artifacts::ArtifactRef = corrupt_job.preview_payload["dependencies"] + [0]["digest"] + .as_str() + .unwrap() + .parse() + .unwrap(); + let dependency_path = storage_root + .join("sha256") + .join(&dependency_ref.digest_hex()[..2]) + .join(dependency_ref.digest_hex()); + std::fs::set_permissions( + &dependency_path, + std::os::unix::fs::PermissionsExt::from_mode(0o600), + ) + .unwrap(); + std::fs::write(&dependency_path, b"corrupt external dependency").unwrap(); + std::fs::set_permissions( + &dependency_path, + std::os::unix::fs::PermissionsExt::from_mode(0o400), + ) + .unwrap(); + + assert!( + service + .create_openapi_import( + &workspace_id, + &corrupt_preview.job_id.as_str().into(), + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /items".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .is_err() + ); + assert_failed_job_and_detached_sources( + ®istry, + &workspace_id, + &corrupt_preview.job_id.as_str().into(), + "import_dependency_verification_failed", + ) + .await; +} + +#[tokio::test] +#[serial] +async fn external_materialization_uses_one_wall_clock_timeout_and_cleans_cancelled_sources() { + let fetches = Arc::new(AtomicUsize::new(0)); + let root_fetches = Arc::clone(&fetches); + let child_fetches = Arc::clone(&fetches); + let app = Router::new() + .route( + "/root.yaml", + get(move || { + let fetches = Arc::clone(&root_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + "Item: { $ref: './slow-child.yaml#/Item' }" + } + }), + ) + .route( + "/slow-child.yaml", + get(move || { + let fetches = Arc::clone(&child_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + "Item: { type: object, properties: { id: { type: string } } }" + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let origin = format!("http://{address}"); + + let registry = test_registry().await; + let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]); + let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build(); + let service = AdminServiceBuilder::new( + registry.clone(), + test_storage_root("openapi_import_chain_timeout"), + test_auth_settings(), + test_secret_crypto(), + runtime, + ) + .with_external_reference_import(&crank_config::ExternalReferenceSettings { + allowed_url_prefixes: vec![format!("{origin}/")], + max_depth: 8, + max_documents: 32, + max_fetch_bytes: 64 * 1024, + fetch_timeout_ms: 200, + max_expanded_nodes: 10_000, + }) + .unwrap() + .with_outbound_http_policy(outbound_policy) + .build(); + let workspace_id = WorkspaceId::new("ws_default"); + let started = tokio::time::Instant::now(); + let preview = service + .preview_openapi_import( + &workspace_id, + OpenApiUpload { + bytes: format!( + r#" +openapi: 3.1.0 +info: {{ title: Timed chain }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: timedItems + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: '{origin}/root.yaml#/Item' }} }} }} +"# + ) + .into_bytes(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }, + ) + .await + .unwrap(); + assert!(started.elapsed() < std::time::Duration::from_millis(800)); + assert_eq!(fetches.load(Ordering::SeqCst), 2); + let job = registry + .get_import_job(&workspace_id, &preview.job_id.as_str().into()) + .await + .unwrap() + .unwrap(); + assert_eq!( + job.preview_payload["dependencies"] + .as_array() + .unwrap() + .len(), + 0 + ); + + for _ in 0..100 { + let active_dependencies: i64 = sqlx::query_scalar( + "select count(*) from artifact_sources + where source_id like 'src_openapi_dep_%' and lifecycle = 'active'", + ) + .fetch_one(registry.pool()) + .await + .unwrap(); + if active_dependencies == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let detached_dependencies: i64 = sqlx::query_scalar( + "select count(*) from artifact_sources + where source_id like 'src_openapi_dep_%' and lifecycle = 'detached'", + ) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(detached_dependencies, 1); +} + +#[tokio::test] +#[serial] +async fn expired_job_detaches_primary_and_external_dependency_without_orphan() { + let app = Router::new().route( + "/schemas.yaml", + get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let origin = format!("http://{address}"); + let registry = test_registry().await; + let service = test_service_with_external_references( + registry.clone(), + test_storage_root("openapi_import_external_expiry"), + vec![format!("{origin}/")], + ); + let workspace_id = WorkspaceId::new("ws_default"); + let preview = service + .preview_openapi_import( + &workspace_id, + OpenApiUpload { + bytes: format!( + r#" +openapi: 3.1.0 +info: {{ title: Expiring external }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }} +"# + ) + .into_bytes(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }, + ) + .await + .unwrap(); + sqlx::query("update import_jobs set expires_at = now() - interval '1 second' where id = $1") + .bind(preview.job_id.as_str()) + .execute(registry.pool()) + .await + .unwrap(); + let report = registry.cleanup_expired_import_jobs(16).await.unwrap(); + assert_eq!(report.deleted_jobs, 1); + assert_eq!(report.detached_sources, 2); + let active_sources: i64 = sqlx::query_scalar( + "select count(*) from artifact_sources + where source_id like 'src_openapi_%' and lifecycle = 'active'", + ) + .fetch_one(registry.pool()) + .await + .unwrap(); + assert_eq!(active_sources, 0); +} +#[tokio::test] +#[serial] +async fn exact_v2_contract_replays_persisted_preview_without_v3_or_dependency_reads() { + let registry = test_registry().await; + let service = test_service( + registry.clone(), + test_storage_root("openapi_import_v2_compatibility"), + test_auth_settings(), + test_secret_crypto(), + ); + let workspace_id = WorkspaceId::new("ws_default"); + let preview = service + .preview_openapi_import(&workspace_id, openapi_upload()) + .await + .unwrap(); + let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into(); + let missing_dependency = serde_json::json!([{ + "source_id": "src_openapi_dep_v2_must_not_be_read", + "digest": format!("sha256:{}", "0".repeat(64)), + "canonical_uri": "https://schemas.example.test/v2.yaml" + }]); + sqlx::query( + "update import_jobs + set preview_payload = jsonb_set( + jsonb_set( + jsonb_set( + jsonb_set( + preview_payload, + '{normalization,normalizer_version}', + to_jsonb('normalized-ir-v2'::text) + ), + '{normalization,projection_version}', + to_jsonb('preview-v2'::text) + ), + '{normalization,ir_fingerprint}', + to_jsonb($1::text) + ), + '{dependency_snapshots}', + $2::jsonb + ) + where id = $3", + ) + .bind("a".repeat(64)) + .bind(missing_dependency) + .bind(job_id.as_str()) + .execute(registry.pool()) + .await + .unwrap(); + + let result = service + .create_openapi_import( + &workspace_id, + &job_id, + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /v2/latest".to_owned()], + server_url: Some("https://api.frankfurter.dev".to_owned()), + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + assert_eq!(result.created.len(), 1); + let completed = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.status, ImportJobStatus::Completed); +} diff --git a/apps/admin-api/tests/integration/openapi_source.rs b/apps/admin-api/tests/integration/openapi_source.rs index 08fd211..47d32cb 100644 --- a/apps/admin-api/tests/integration/openapi_source.rs +++ b/apps/admin-api/tests/integration/openapi_source.rs @@ -1,30 +1,18 @@ -use std::{ - io, - sync::{Arc, Mutex}, -}; - -use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale}; -use crank_artifacts::MAX_ARTIFACT_BYTES; -use crank_core::{Workspace, WorkspaceId, WorkspaceStatus}; -use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest}; -use metrics_util::debugging::DebuggingRecorder; -use opentelemetry::trace::TracerProvider as _; -use opentelemetry_sdk::{ - error::OTelSdkResult, - trace::{SdkTracerProvider, SpanData, SpanExporter}, -}; -use reqwest::multipart::{Form, Part}; -use serde_json::Value; -use serial_test::serial; -use time::{Duration, OffsetDateTime}; -use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt}; - use super::common::{ authorized_client, build_test_app, spawn_admin_api, test_auth_settings, test_registry, test_secret_crypto, test_service, test_storage_root, }; +use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale}; +use crank_artifacts::MAX_ARTIFACT_BYTES; +use crank_core::{Workspace, WorkspaceId, WorkspaceStatus}; +use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest}; +use reqwest::multipart::{Form, Part}; +use serde_json::Value; +use serial_test::serial; +use time::{Duration, OffsetDateTime}; mod apply_failures; +mod telemetry; const OPENAPI: &str = r#" openapi: 3.0.3 @@ -136,78 +124,6 @@ async fn multipart_requires_an_owner_membership_before_reading_the_file() { ); } -#[tokio::test(flavor = "multi_thread")] -#[serial] -async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() { - const CANARY: &str = "openapi-telemetry-secret-canary"; - - let recorder = DebuggingRecorder::new(); - let snapshotter = recorder.snapshotter(); - recorder - .install() - .expect("isolated integration test metrics recorder"); - - let writer = SharedLogWriter::default(); - let exported = Arc::new(Mutex::new(Vec::new())); - let provider = SdkTracerProvider::builder() - .with_simple_exporter(CapturingExporter(Arc::clone(&exported))) - .build(); - let tracer = provider.tracer("admin-openapi-source-test"); - let subscriber = tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().with_writer(writer.clone())) - .with(tracing_opentelemetry::layer().with_tracer(tracer)); - let dispatch = tracing::Dispatch::new(subscriber); - tracing::dispatcher::set_global_default(dispatch) - .expect("isolated integration test tracing subscriber"); - - let registry = test_registry().await; - let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary")); - let server = spawn_admin_api(app).await; - let client = authorized_client(&server).await; - let document = format!( - "openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n /broken:\n get:\n description: {CANARY}\n responses: [" - ); - let response = client - .post(format!("{server}/imports/openapi/preview")) - .multipart(Form::new().part( - "file", - file_part(document.as_bytes(), "openapi.yaml", "application/yaml"), - )) - .send() - .await - .unwrap(); - assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); - let trace_id = response - .headers() - .get("x-trace-id") - .unwrap() - .to_str() - .unwrap() - .to_owned(); - let body = response.text().await.unwrap(); - assert!(!body.contains(CANARY)); - let diagnostics: Value = serde_json::from_str(&body).unwrap(); - assert_eq!(diagnostics["error"]["trace_id"], trace_id); - - provider.force_flush().unwrap(); - let logs = writer.output(); - assert!(!logs.contains(CANARY)); - assert!(logs.contains(&trace_id)); - let spans = exported.lock().unwrap(); - let rendered_spans = format!("{spans:?}"); - assert!(!rendered_spans.contains(CANARY)); - drop(spans); - for (key, _, _, _) in snapshotter.snapshot().into_vec() { - assert!(!key.key().name().contains(CANARY)); - assert!( - !key.key() - .labels() - .any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) }) - ); - } - provider.shutdown().unwrap(); -} - #[tokio::test(flavor = "multi_thread")] #[serial] async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() { @@ -939,49 +855,3 @@ async fn wait_for_specific_source_lifecycle( } panic!("artifact source {source_id:?} did not reach {expected}"); } - -#[derive(Clone, Default)] -struct SharedLogWriter { - buffer: Arc>>, -} - -impl SharedLogWriter { - fn output(&self) -> String { - String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap() - } -} - -impl<'a> MakeWriter<'a> for SharedLogWriter { - type Writer = SharedLogGuard; - - fn make_writer(&'a self) -> Self::Writer { - SharedLogGuard { - buffer: Arc::clone(&self.buffer), - } - } -} - -struct SharedLogGuard { - buffer: Arc>>, -} - -impl io::Write for SharedLogGuard { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.buffer.lock().unwrap().extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -#[derive(Clone, Debug)] -struct CapturingExporter(Arc>>); - -impl SpanExporter for CapturingExporter { - async fn export(&self, batch: Vec) -> OTelSdkResult { - self.0.lock().unwrap().extend(batch); - Ok(()) - } -} diff --git a/apps/admin-api/tests/integration/openapi_source/telemetry.rs b/apps/admin-api/tests/integration/openapi_source/telemetry.rs new file mode 100644 index 0000000..566aefb --- /dev/null +++ b/apps/admin-api/tests/integration/openapi_source/telemetry.rs @@ -0,0 +1,184 @@ +use super::super::common::build_test_app_with_external_references; +use super::*; +use std::{ + io, + sync::{Arc, Mutex}, +}; + +use metrics_util::debugging::DebuggingRecorder; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::{ + error::OTelSdkResult, + trace::{SdkTracerProvider, SpanData, SpanExporter}, +}; +use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt}; + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() { + const CANARY: &str = "openapi-telemetry-secret-canary"; + + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + recorder + .install() + .expect("isolated integration test metrics recorder"); + + let writer = SharedLogWriter::default(); + let exported = Arc::new(Mutex::new(Vec::new())); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(CapturingExporter(Arc::clone(&exported))) + .build(); + let tracer = provider.tracer("admin-openapi-source-test"); + let subscriber = tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(writer.clone())) + .with(tracing_opentelemetry::layer().with_tracer(tracer)); + let dispatch = tracing::Dispatch::new(subscriber); + tracing::dispatcher::set_global_default(dispatch) + .expect("isolated integration test tracing subscriber"); + + let registry = test_registry().await; + let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary")); + let server = spawn_admin_api(app).await; + let client = authorized_client(&server).await; + let document = format!( + "openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n /broken:\n get:\n description: {CANARY}\n responses: [" + ); + let response = client + .post(format!("{server}/imports/openapi/preview")) + .multipart(Form::new().part( + "file", + file_part(document.as_bytes(), "openapi.yaml", "application/yaml"), + )) + .send() + .await + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); + let trace_id = response + .headers() + .get("x-trace-id") + .unwrap() + .to_str() + .unwrap() + .to_owned(); + let body = response.text().await.unwrap(); + assert!(!body.contains(CANARY)); + let diagnostics: Value = serde_json::from_str(&body).unwrap(); + assert_eq!(diagnostics["error"]["trace_id"], trace_id); + + let route = format!("/{CANARY}.yaml"); + let external = axum::Router::new().route( + &route, + axum::routing::get(|| async { (axum::http::StatusCode::INTERNAL_SERVER_ERROR, CANARY) }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, external).await.unwrap() }); + let origin = format!("http://{address}"); + let external_registry = test_registry().await; + let external_app = build_test_app_with_external_references( + external_registry, + test_storage_root("openapi_materialization_telemetry_canary"), + vec![format!("{origin}/")], + ); + let external_server = spawn_admin_api(external_app).await; + let external_client = authorized_client(&external_server).await; + let external_document = format!( + r#" +openapi: 3.1.0 +info: {{ title: Safe materialization }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + responses: + '200': + description: ok + content: + application/json: + schema: {{ $ref: '{origin}/{CANARY}.yaml#/Item' }} +"# + ); + let response = external_client + .post(format!("{external_server}/imports/openapi/preview")) + .multipart(Form::new().part( + "file", + file_part( + external_document.as_bytes(), + "external.yaml", + "application/yaml", + ), + )) + .send() + .await + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::OK); + + provider.force_flush().unwrap(); + let logs = writer.output(); + assert!(!logs.contains(CANARY)); + assert!(logs.contains(&trace_id)); + assert!(logs.contains("external OpenAPI materialization failed")); + assert!(logs.contains("stage=\"fetch\"")); + assert!(logs.contains("error_code=\"unexpected_status\"")); + assert!(logs.contains("count=0")); + let spans = exported.lock().unwrap(); + let rendered_spans = format!("{spans:?}"); + assert!(!rendered_spans.contains(CANARY)); + drop(spans); + for (key, _, _, _) in snapshotter.snapshot().into_vec() { + assert!(!key.key().name().contains(CANARY)); + assert!( + !key.key() + .labels() + .any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) }) + ); + } + provider.shutdown().unwrap(); +} + +#[derive(Clone, Default)] +struct SharedLogWriter { + buffer: Arc>>, +} + +impl SharedLogWriter { + fn output(&self) -> String { + String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap() + } +} + +impl<'a> MakeWriter<'a> for SharedLogWriter { + type Writer = SharedLogGuard; + + fn make_writer(&'a self) -> Self::Writer { + SharedLogGuard { + buffer: Arc::clone(&self.buffer), + } + } +} + +struct SharedLogGuard { + buffer: Arc>>, +} + +impl io::Write for SharedLogGuard { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.buffer.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[derive(Clone, Debug)] +struct CapturingExporter(Arc>>); + +impl SpanExporter for CapturingExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + self.0.lock().unwrap().extend(batch); + Ok(()) + } +} diff --git a/crates/crank-adapter-rest/src/client.rs b/crates/crank-adapter-rest/src/client.rs index ffd70f6..274d4c0 100644 --- a/crates/crank-adapter-rest/src/client.rs +++ b/crates/crank-adapter-rest/src/client.rs @@ -25,7 +25,7 @@ use serde_json::Value; use tracing::{Instrument, Span}; use tracing_opentelemetry::OpenTelemetrySpanExt; -use crate::{RestAdapterError, RestRequest, RestResponse}; +use crate::{ExternalReferenceFetchError, RestAdapterError, RestRequest, RestResponse}; #[derive(Clone, Debug)] pub struct RestAdapter { @@ -33,6 +33,18 @@ pub struct RestAdapter { policy: OutboundHttpPolicy, } +/// A deliberately separate, GET-only boundary for materializing external +/// OpenAPI documents. It has no execution metrics, tracing propagation, or +/// request construction semantics from [`RestAdapter`]. +#[derive(Clone, Debug)] +pub struct ExternalReferenceFetcher { + client: Result>, + policy: OutboundHttpPolicy, + allowed_url_prefixes: Vec, + max_response_bytes: usize, + timeout: Duration, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct OutboundHttpPolicy { allowed_hosts: Vec, @@ -56,15 +68,7 @@ impl RestAdapter { } pub fn with_policy(policy: OutboundHttpPolicy) -> Self { - let resolver = Arc::new(PolicyDnsResolver { - policy: policy.clone(), - }); - let client = Client::builder() - .redirect(redirect::Policy::none()) - .no_proxy() - .dns_resolver(resolver) - .build() - .map_err(|error| Arc::::from(error.to_string())); + let client = outbound_client(&policy); Self { client, policy } } @@ -195,6 +199,153 @@ impl RestAdapter { } } +impl ExternalReferenceFetcher { + pub fn try_new( + policy: OutboundHttpPolicy, + allowed_url_prefixes: Vec, + max_response_bytes: usize, + timeout: Duration, + ) -> Result { + if max_response_bytes == 0 || timeout.is_zero() { + return Err(ExternalReferenceFetchError::InvalidConfiguration); + } + let allowed_url_prefixes = allowed_url_prefixes + .into_iter() + .map(|prefix| canonical_external_reference_prefix(&prefix)) + .collect::, _>>()?; + Ok(Self { + client: outbound_client(&policy), + policy, + allowed_url_prefixes, + max_response_bytes, + timeout, + }) + } + + /// Fetches one document with a bounded, headerless `GET`. + /// + /// URL fragments are stripped because they address JSON Pointer targets in + /// the fetched document rather than a network resource. + pub async fn get(&self, url: &str) -> Result, ExternalReferenceFetchError> { + if self.allowed_url_prefixes.is_empty() { + return Err(ExternalReferenceFetchError::Disabled); + } + let mut url = + reqwest::Url::parse(url).map_err(|_| ExternalReferenceFetchError::InvalidUrl)?; + url.set_fragment(None); + if !self + .allowed_url_prefixes + .iter() + .any(|prefix| matches_external_reference_prefix(&url, prefix)) + { + return Err(ExternalReferenceFetchError::TargetNotAllowed); + } + self.policy + .validate_url(&url) + .map_err(external_policy_error)?; + let client = self + .client + .as_ref() + .map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?; + let response = client + .get(url) + .timeout(self.timeout) + .send() + .await + .map_err(external_transport_error)?; + let status = response.status(); + if status.is_redirection() { + return Err(ExternalReferenceFetchError::RedirectNotAllowed); + } + if !status.is_success() { + return Err(ExternalReferenceFetchError::UnexpectedStatus { + status: status.as_u16(), + }); + } + read_external_response_bytes(response, self.max_response_bytes).await + } +} + +fn outbound_client(policy: &OutboundHttpPolicy) -> Result> { + let resolver = Arc::new(PolicyDnsResolver { + policy: policy.clone(), + }); + Client::builder() + .redirect(redirect::Policy::none()) + .no_proxy() + .dns_resolver(resolver) + .build() + .map_err(|error| Arc::::from(error.to_string())) +} + +fn canonical_external_reference_prefix( + prefix: &str, +) -> Result { + let url = reqwest::Url::parse(prefix) + .map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(ExternalReferenceFetchError::InvalidConfiguration); + } + Ok(url.to_string()) +} + +fn matches_external_reference_prefix(url: &reqwest::Url, prefix: &str) -> bool { + let url = url.as_str(); + if !url.starts_with(prefix) { + return false; + } + let Some(next) = url.as_bytes().get(prefix.len()) else { + return true; + }; + prefix.ends_with('/') || matches!(next, b'/' | b'?') +} + +fn external_policy_error(error: RestAdapterError) -> ExternalReferenceFetchError { + match error { + RestAdapterError::TargetNotAllowed { .. } => ExternalReferenceFetchError::TargetNotAllowed, + _ => ExternalReferenceFetchError::InvalidConfiguration, + } +} + +fn external_transport_error(error: reqwest::Error) -> ExternalReferenceFetchError { + ExternalReferenceFetchError::Transport { + timeout: error.is_timeout(), + connect: error.is_connect(), + } +} + +async fn read_external_response_bytes( + response: reqwest::Response, + max_response_bytes: usize, +) -> Result, ExternalReferenceFetchError> { + if response + .content_length() + .is_some_and(|length| length > max_response_bytes as u64) + { + return Err(ExternalReferenceFetchError::ResponseTooLarge { + limit_bytes: max_response_bytes, + }); + } + let mut stream = response.bytes_stream(); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(external_transport_error)?; + if bytes.len().saturating_add(chunk.len()) > max_response_bytes { + return Err(ExternalReferenceFetchError::ResponseTooLarge { + limit_bytes: max_response_bytes, + }); + } + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + fn upstream_outcome(error: &RestAdapterError) -> UpstreamOutcome { match error { RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => { diff --git a/crates/crank-adapter-rest/src/error.rs b/crates/crank-adapter-rest/src/error.rs index f1531c8..3c739f4 100644 --- a/crates/crank-adapter-rest/src/error.rs +++ b/crates/crank-adapter-rest/src/error.rs @@ -1,6 +1,26 @@ use serde_json::Value; use thiserror::Error; +#[derive(Debug, Error)] +pub enum ExternalReferenceFetchError { + #[error("external references are disabled")] + Disabled, + #[error("external reference URL is invalid")] + InvalidUrl, + #[error("external reference target is not allowed")] + TargetNotAllowed, + #[error("external reference redirects are not allowed")] + RedirectNotAllowed, + #[error("external reference response exceeds the configured limit of {limit_bytes} bytes")] + ResponseTooLarge { limit_bytes: usize }, + #[error("external reference endpoint returned status {status}")] + UnexpectedStatus { status: u16 }, + #[error("external reference request failed")] + Transport { timeout: bool, connect: bool }, + #[error("external reference fetch configuration is invalid")] + InvalidConfiguration, +} + #[derive(Debug, Error)] pub enum RestAdapterError { #[error("invalid base url: {url}")] diff --git a/crates/crank-adapter-rest/src/lib.rs b/crates/crank-adapter-rest/src/lib.rs index 6df2bd4..e026041 100644 --- a/crates/crank-adapter-rest/src/lib.rs +++ b/crates/crank-adapter-rest/src/lib.rs @@ -8,8 +8,8 @@ use crank_core::{ ProtocolAdapterError, RestTarget, RuntimeRequestContext, Target, }; -pub use client::{OutboundHttpPolicy, RestAdapter}; -pub use error::RestAdapterError; +pub use client::{ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter}; +pub use error::{ExternalReferenceFetchError, RestAdapterError}; pub use model::{RestRequest, RestResponse}; #[async_trait] diff --git a/crates/crank-adapter-rest/tests/integration.rs b/crates/crank-adapter-rest/tests/integration.rs index f9b8b9c..1c95304 100644 --- a/crates/crank-adapter-rest/tests/integration.rs +++ b/crates/crank-adapter-rest/tests/integration.rs @@ -1,4 +1,5 @@ mod integration { mod client; + mod external_reference_fetcher; mod outbound_security; } diff --git a/crates/crank-adapter-rest/tests/integration/external_reference_fetcher.rs b/crates/crank-adapter-rest/tests/integration/external_reference_fetcher.rs new file mode 100644 index 0000000..8b5e2c6 --- /dev/null +++ b/crates/crank-adapter-rest/tests/integration/external_reference_fetcher.rs @@ -0,0 +1,151 @@ +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use axum::{Router, http::StatusCode, response::Redirect, routing::get}; +use crank_adapter_rest::{ + ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy, +}; +use tokio::net::TcpListener; + +#[tokio::test] +async fn external_references_are_default_off_before_any_request() { + let requests = Arc::new(AtomicUsize::new(0)); + let base_url = spawn_server(Arc::clone(&requests)).await; + let fetcher = ExternalReferenceFetcher::try_new( + OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + Vec::new(), + 1024, + Duration::from_secs(1), + ) + .unwrap(); + + let error = fetcher + .get(&format!("{base_url}/document")) + .await + .unwrap_err(); + + assert!(matches!(error, ExternalReferenceFetchError::Disabled)); + assert_eq!(requests.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn fetcher_uses_prefix_and_actual_address_policy_then_returns_bounded_bytes() { + let base_url = spawn_server(Arc::new(AtomicUsize::new(0))).await; + let fetcher = ExternalReferenceFetcher::try_new( + OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + vec![base_url.clone()], + 8, + Duration::from_secs(1), + ) + .unwrap(); + + assert_eq!( + fetcher + .get(&format!("{base_url}/document#/components/schemas/A")) + .await + .unwrap(), + b"openapi".to_vec() + ); + let error = fetcher.get(&format!("{base_url}/large")).await.unwrap_err(); + assert!(matches!( + error, + ExternalReferenceFetchError::ResponseTooLarge { limit_bytes: 8 } + )); + + let private_without_explicit_outbound_allow = ExternalReferenceFetcher::try_new( + OutboundHttpPolicy::default(), + vec![base_url.clone()], + 1024, + Duration::from_secs(1), + ) + .unwrap(); + let error = private_without_explicit_outbound_allow + .get(&format!("{base_url}/document")) + .await + .unwrap_err(); + assert!(matches!( + error, + ExternalReferenceFetchError::TargetNotAllowed + )); + + let exact_path_fetcher = ExternalReferenceFetcher::try_new( + OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + vec![format!("{base_url}/document")], + 1024, + Duration::from_secs(1), + ) + .unwrap(); + let error = exact_path_fetcher + .get(&format!("{base_url}/document-unrelated")) + .await + .unwrap_err(); + assert!(matches!( + error, + ExternalReferenceFetchError::TargetNotAllowed + )); +} + +#[tokio::test] +async fn fetcher_rejects_redirects_and_userinfo_without_exposing_the_url() { + let requests = Arc::new(AtomicUsize::new(0)); + let base_url = spawn_server(Arc::clone(&requests)).await; + let fetcher = ExternalReferenceFetcher::try_new( + OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), + vec![base_url.clone()], + 1024, + Duration::from_secs(1), + ) + .unwrap(); + + let error = fetcher + .get(&format!("{base_url}/redirect")) + .await + .unwrap_err(); + assert!(matches!( + error, + ExternalReferenceFetchError::RedirectNotAllowed + )); + + let userinfo_url = base_url.replacen("http://", "http://user:credential@", 1); + let error = fetcher + .get(&format!("{userinfo_url}/document")) + .await + .unwrap_err(); + assert!(matches!( + error, + ExternalReferenceFetchError::TargetNotAllowed + )); + let rendered = format!("{error:?} {error}"); + assert!(!rendered.contains("credential")); + assert_eq!(requests.load(Ordering::SeqCst), 0); +} + +async fn spawn_server(requests: Arc) -> String { + let app = Router::new() + .route( + "/document", + get({ + let requests = Arc::clone(&requests); + move || { + let requests = Arc::clone(&requests); + async move { + requests.fetch_add(1, Ordering::SeqCst); + (StatusCode::OK, "openapi") + } + } + }), + ) + .route("/large", get(|| async { "response too large" })) + .route("/redirect", get(|| async { Redirect::to("/document") })); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{address}") +} diff --git a/crates/crank-adapter-rest/tests/integration/outbound_security.rs b/crates/crank-adapter-rest/tests/integration/outbound_security.rs index 102f61e..d324946 100644 --- a/crates/crank-adapter-rest/tests/integration/outbound_security.rs +++ b/crates/crank-adapter-rest/tests/integration/outbound_security.rs @@ -6,6 +6,7 @@ use std::{ Arc, atomic::{AtomicUsize, Ordering}, }, + time::Duration, }; use axum::{ @@ -14,7 +15,9 @@ use axum::{ http::StatusCode, routing::{any, post}, }; -use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest}; +use crank_adapter_rest::{ + ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest, +}; use crank_core::{HttpMethod, ProtocolAdapterError, RestTarget}; use serde_json::{Value, json}; use tokio::net::TcpListener; @@ -94,7 +97,7 @@ async fn proxy_environment_is_ignored_by_default() { let adapter = RestAdapter::default(); let request = json_request(json!({"payload": "proxy-env-canary"})); let rest_target = RestTarget { - base_url: target, + base_url: target.clone(), method: HttpMethod::Post, path_template: "/capture".to_owned(), static_headers: BTreeMap::new(), @@ -102,6 +105,17 @@ async fn proxy_environment_is_ignored_by_default() { let _ = adapter.execute(&rest_target, &request).await.expect_err( "unresolvable target should fail locally instead of being sent through proxy env", ); + let fetcher = ExternalReferenceFetcher::try_new( + OutboundHttpPolicy::default(), + vec!["http://public.example.test/".to_owned()], + 1024, + Duration::from_secs(1), + ) + .expect("valid external reference fetcher"); + let _ = fetcher + .get(&target) + .await + .expect_err("external reference fetcher must not use proxy environment variables"); return; } diff --git a/crates/crank-config/src/debug.rs b/crates/crank-config/src/debug.rs index e211f9a..a28ea4e 100644 --- a/crates/crank-config/src/debug.rs +++ b/crates/crank-config/src/debug.rs @@ -1,8 +1,9 @@ use std::fmt; use crate::{ - AdminProcessConfig, CacheSettings, DatabaseSettings, McpProcessConfig, MetricsSettings, - MigratorConfig, ObservabilitySettings, OtlpSettings, OutboundSettings, RuntimeSettings, + AdminProcessConfig, CacheSettings, DatabaseSettings, ExternalReferenceSettings, + McpProcessConfig, MetricsSettings, MigratorConfig, ObservabilitySettings, OtlpSettings, + OutboundSettings, RuntimeSettings, }; impl fmt::Debug for DatabaseSettings { @@ -32,6 +33,18 @@ impl fmt::Debug for OutboundSettings { .finish() } } +impl fmt::Debug for ExternalReferenceSettings { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExternalReferenceSettings") + .field("allowed_url_prefix_count", &self.allowed_url_prefixes.len()) + .field("max_depth", &self.max_depth) + .field("max_documents", &self.max_documents) + .field("max_fetch_bytes", &self.max_fetch_bytes) + .field("fetch_timeout_ms", &self.fetch_timeout_ms) + .field("max_expanded_nodes", &self.max_expanded_nodes) + .finish() + } +} impl fmt::Debug for RuntimeSettings { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RuntimeSettings") @@ -93,6 +106,7 @@ impl fmt::Debug for AdminProcessConfig { f.debug_struct("AdminProcessConfig") .field("database", &self.database) .field("runtime", &self.runtime) + .field("external_references", &self.external_references) .field("observability", &self.observability) .field("storage_root", &"configured") .field("session_secret", &self.session_secret) diff --git a/crates/crank-config/src/lib.rs b/crates/crank-config/src/lib.rs index 8a6beb3..9d47515 100644 --- a/crates/crank-config/src/lib.rs +++ b/crates/crank-config/src/lib.rs @@ -15,8 +15,9 @@ pub use diagnostic::{ConfigError, Diagnostic, DiagnosticCode}; pub use migrator::{MigratorConfig, parse_migrator}; pub use process::{ AdminProcessConfig, CacheBackend, CacheSettings, DatabaseSettings, DeprecationRecord, - EffectiveConfig, McpProcessConfig, MetricsSettings, ObservabilitySettings, OtlpSettings, - OutboundSettings, PoolSettings, ProcessKind, RateLimitSettings, RuntimeSettings, parse_process, + EffectiveConfig, ExternalReferenceSettings, McpProcessConfig, MetricsSettings, + ObservabilitySettings, OtlpSettings, OutboundSettings, PoolSettings, ProcessKind, + RateLimitSettings, RuntimeSettings, parse_process, }; pub use schema::{ FieldMode, FieldSpec, ProcessScope, Sensitivity, deployment_field_registry, field_registry, diff --git a/crates/crank-config/src/process.rs b/crates/crank-config/src/process.rs index 6b5ea94..939c283 100644 --- a/crates/crank-config/src/process.rs +++ b/crates/crank-config/src/process.rs @@ -12,7 +12,9 @@ use std::{ path::PathBuf, }; use url::Url; +mod external_references; mod list_parsers; +pub use external_references::ExternalReferenceSettings; const MAX_ENV_VALUE_BYTES: usize = 8_192; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ProcessKind { @@ -127,6 +129,7 @@ pub struct ObservabilitySettings { pub struct AdminProcessConfig { pub database: DatabaseSettings, pub runtime: RuntimeSettings, + pub external_references: ExternalReferenceSettings, pub observability: ObservabilitySettings, pub bind_addr: SocketAddr, pub storage_root: PathBuf, @@ -154,8 +157,8 @@ pub struct McpProcessConfig { #[derive(Clone)] enum Projection { - Admin(AdminProcessConfig), - Mcp(McpProcessConfig), + Admin(Box), + Mcp(Box), } #[derive(Clone)] @@ -790,9 +793,10 @@ pub fn parse_process( { parser.push(DiagnosticCode::UnsafeCombination, "admin.exposure.tls"); } - Projection::Admin(AdminProcessConfig { + Projection::Admin(Box::new(AdminProcessConfig { database, runtime, + external_references: external_references::parse(&mut parser), observability, bind_addr, storage_root: parser.absolute_path("CRANK_STORAGE_ROOT", "/var/lib/crank/storage"), @@ -811,7 +815,7 @@ pub fn parse_process( bootstrap_display_name: parser .string("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", Some("Crank Owner")), demo_seed: parser.boolean("CRANK_DEMO_SEED"), - }) + })) } ProcessKind::McpServer => { let rps = parser.number("CRANK_MCP_RATE_LIMIT_RPS") as u32; @@ -819,7 +823,7 @@ pub fn parse_process( if burst < rps { parser.push(DiagnosticCode::UnsafeCombination, "mcp.rate_limit.burst"); } - Projection::Mcp(McpProcessConfig { + Projection::Mcp(Box::new(McpProcessConfig { database, runtime, observability, @@ -829,7 +833,7 @@ pub fn parse_process( requests_per_second: rps, burst, }, - }) + })) } }; @@ -867,6 +871,18 @@ fn fingerprint_parts(kind: ProcessKind, projection: &Projection) -> Vec format!("session_ttl={}", config.session_ttl_hours), format!("trusted_proxies={:?}", config.trusted_proxy_ips), format!("demo={}", config.demo_seed), + format!( + "external_reference_prefixes={}", + config.external_references.allowed_url_prefixes.join(",") + ), + format!( + "external_reference_limits={}:{}:{}:{}:{}", + config.external_references.max_depth, + config.external_references.max_documents, + config.external_references.max_fetch_bytes, + config.external_references.fetch_timeout_ms, + config.external_references.max_expanded_nodes, + ), "storage=path-configured".to_owned(), format!("session_secret={}", config.session_secret.is_configured()), format!("pepper={}", config.password_pepper.is_configured()), diff --git a/crates/crank-config/src/process/external_references.rs b/crates/crank-config/src/process/external_references.rs new file mode 100644 index 0000000..95e64f6 --- /dev/null +++ b/crates/crank-config/src/process/external_references.rs @@ -0,0 +1,24 @@ +#[derive(Clone, Eq, PartialEq)] +pub struct ExternalReferenceSettings { + /// Canonical HTTP(S) URL prefixes which opt an operator into remote `$ref` fetches. + /// An empty list is a deliberate default-deny switch. + pub allowed_url_prefixes: Vec, + pub max_depth: usize, + pub max_documents: usize, + pub max_fetch_bytes: usize, + pub fetch_timeout_ms: u64, + pub max_expanded_nodes: usize, +} + +pub(super) fn parse(parser: &mut super::Parser<'_>) -> ExternalReferenceSettings { + ExternalReferenceSettings { + allowed_url_prefixes: parser + .url_prefix_list("CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES"), + max_depth: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH") as usize, + max_documents: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS") as usize, + max_fetch_bytes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES") as usize, + fetch_timeout_ms: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS"), + max_expanded_nodes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES") + as usize, + } +} diff --git a/crates/crank-config/src/process/list_parsers.rs b/crates/crank-config/src/process/list_parsers.rs index 020d555..6884306 100644 --- a/crates/crank-config/src/process/list_parsers.rs +++ b/crates/crank-config/src/process/list_parsers.rs @@ -4,6 +4,7 @@ use crate::{ DiagnosticCode, validation::{parse_host_list, parse_ip_list}, }; +use url::Url; impl super::Parser<'_> { pub(super) fn host_list(&mut self, name: &'static str) -> Vec { @@ -33,4 +34,41 @@ impl super::Parser<'_> { } parsed.items } + + pub(super) fn url_prefix_list(&mut self, name: &'static str) -> Vec { + let Some(raw) = self.optional(name) else { + return Vec::new(); + }; + let mut prefixes = Vec::new(); + let mut invalid = false; + for item in raw.split(',') { + let item = item.trim(); + let Ok(url) = Url::parse(item) else { + invalid = true; + continue; + }; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + invalid = true; + continue; + } + let canonical = url.to_string(); + if !prefixes.contains(&canonical) { + prefixes.push(canonical); + } + } + if invalid { + self.push(DiagnosticCode::InvalidType, name); + } + if prefixes.len() > 64 { + self.push(DiagnosticCode::OutOfRange, name); + prefixes.truncate(64); + } + prefixes + } } diff --git a/crates/crank-config/src/schema.rs b/crates/crank-config/src/schema.rs index 00ddda1..f5ef2c0 100644 --- a/crates/crank-config/src/schema.rs +++ b/crates/crank-config/src/schema.rs @@ -78,7 +78,7 @@ macro_rules! f { }; } -static FIELDS: [FieldSpec; 59] = [ +static FIELDS: [FieldSpec; 65] = [ FieldSpec { compatibility: Some("legacy URL form"), rules: &[ @@ -339,6 +339,78 @@ static FIELDS: [FieldSpec; 59] = [ Some(67108864), Public ), + FieldSpec { + rules: &[ + "empty list disables external OpenAPI reference fetching", + "each prefix must be canonical HTTP(S) without userinfo, query, or fragment", + ], + ..f!( + "import.external_references.allowed_url_prefixes", + "CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES", + AdminApi, + "url_prefix_list", + None, + Some(""), + None, + None, + Internal + ) + }, + f!( + "import.external_references.max_depth", + "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH", + AdminApi, + "u32", + Some("edges"), + Some("8"), + Some(1), + Some(32), + Public + ), + f!( + "import.external_references.max_documents", + "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS", + AdminApi, + "u32", + Some("documents"), + Some("32"), + Some(1), + Some(32), + Public + ), + f!( + "import.external_references.max_fetch_bytes", + "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES", + AdminApi, + "u64", + Some("bytes"), + Some("262144"), + Some(1), + Some(4194304), + Public + ), + f!( + "import.external_references.fetch_timeout_ms", + "CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS", + AdminApi, + "u64", + Some("milliseconds"), + Some("10000"), + Some(1), + Some(300000), + Public + ), + f!( + "import.external_references.max_expanded_nodes", + "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES", + AdminApi, + "u32", + Some("nodes"), + Some("10000"), + Some(1), + Some(100000), + Public + ), f!( "observability.environment", "CRANK_ENVIRONMENT", diff --git a/crates/crank-config/tests/contract.rs b/crates/crank-config/tests/contract.rs index a9d9865..5f224c6 100644 --- a/crates/crank-config/tests/contract.rs +++ b/crates/crank-config/tests/contract.rs @@ -108,9 +108,64 @@ fn source_for( } #[test] -fn registry_covers_exactly_the_59_observed_runtime_names() { +fn external_reference_contract_is_default_off_and_validates_prefixes_and_limits() { + let config = parse_process( + ProcessKind::AdminApi, + ConfigSource::from_utf8(required_admin()), + ) + .expect("the external reference contract has safe defaults"); + let references = &config.admin().unwrap().external_references; + assert!(references.allowed_url_prefixes.is_empty()); + assert_eq!(references.max_depth, 8); + assert_eq!(references.max_documents, 32); + assert_eq!(references.max_fetch_bytes, 262_144); + assert_eq!(references.fetch_timeout_ms, 10_000); + assert_eq!(references.max_expanded_nodes, 10_000); + + let mut allowed = required_admin(); + allowed.insert( + "CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(), + "https://schemas.example.test/openapi/,https://schemas.example.test/openapi/".to_owned(), + ); + let config = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(allowed)).unwrap(); + assert_eq!( + config + .admin() + .unwrap() + .external_references + .allowed_url_prefixes, + vec!["https://schemas.example.test/openapi/".to_owned()] + ); + + let mut invalid = required_admin(); + invalid.insert( + "CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(), + "https://user:secret@schemas.example.test/openapi/".to_owned(), + ); + let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(invalid)) + .expect_err("userinfo must never become an operator allow rule"); + assert!(error.diagnostics().iter().any(|diagnostic| { + diagnostic.code == DiagnosticCode::InvalidType + && diagnostic.field == "import.external_references.allowed_url_prefixes" + })); + + let mut out_of_range = required_admin(); + out_of_range.insert( + "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS".to_owned(), + "33".to_owned(), + ); + let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(out_of_range)) + .expect_err("graph document limit must be bounded"); + assert!(error.diagnostics().iter().any(|diagnostic| { + diagnostic.code == DiagnosticCode::OutOfRange + && diagnostic.field == "import.external_references.max_documents" + })); +} + +#[test] +fn registry_covers_exactly_the_65_observed_runtime_names() { let registry = field_registry(); - assert_eq!(registry.len(), 59); + assert_eq!(registry.len(), 65); let unique = registry .iter() .map(|field| field.env_name) diff --git a/crates/crank-config/tests/generation.rs b/crates/crank-config/tests/generation.rs index f198eb6..f72c797 100644 --- a/crates/crank-config/tests/generation.rs +++ b/crates/crank-config/tests/generation.rs @@ -30,6 +30,9 @@ fn generated_reference_distinguishes_required_and_optional_fields() { assert!(reference.contains( "| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` |" )); + assert!(reference.contains( + "| `CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES` | `import.external_references.allowed_url_prefixes` | `AdminApi` | `url_prefix_list/-` | `` |" + )); } #[test] diff --git a/crates/crank-import/src/rest/mod.rs b/crates/crank-import/src/rest/mod.rs index 6b19b63..a919347 100644 --- a/crates/crank-import/src/rest/mod.rs +++ b/crates/crank-import/src/rest/mod.rs @@ -8,19 +8,22 @@ mod normalize_schema; mod openapi3; mod payload; mod recommendations; +mod reference; mod schema; mod swagger2; pub use model::{ - ImportFinding, ImportFindingSeverity, ImportGroupPreview, ImportOperationCandidate, - ImportPreview, ImportSourcePreview, NORMALIZER_VERSION, NormalizationConfig, NormalizedFinding, - NormalizedIr, NormalizedOperation, NormalizedParameter, NormalizedReference, NormalizedSchema, - NormalizedSchemaKind, PROJECTION_VERSION, RestImportCandidate, RestImportDocument, - RestImportOperation, RestImportParameter, RestParameterLocation, SourceDigest, SourceIdentity, - SourceLocation, UnresolvedReference, + ExternalDocumentSnapshot, ImportFinding, ImportFindingSeverity, ImportGroupPreview, + ImportOperationCandidate, ImportPreview, ImportSourcePreview, NORMALIZER_VERSION, + NormalizationConfig, NormalizedFinding, NormalizedIr, NormalizedOperation, NormalizedParameter, + NormalizedReference, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind, + PROJECTION_VERSION, ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode, + RestImportCandidate, RestImportDocument, RestImportOperation, RestImportParameter, + RestParameterLocation, SourceDigest, SourceIdentity, SourceLocation, UnresolvedReference, }; pub use normalize::{ - ImportParseError, normalize_verified_document, preview_document, preview_document_legacy_v1, - preview_from_ir, validate_normalized_ir, + ImportParseError, external_reference_uris, normalize_verified_bundle, + normalize_verified_document, preview_document, preview_document_legacy_v1, preview_from_ir, + reference_uris, validate_normalized_ir, }; pub use payload::operation_draft_from_candidate; diff --git a/crates/crank-import/src/rest/model.rs b/crates/crank-import/src/rest/model.rs index 05f5176..11b19fa 100644 --- a/crates/crank-import/src/rest/model.rs +++ b/crates/crank-import/src/rest/model.rs @@ -9,10 +9,10 @@ use serde_json::Value; /// The immutable contract used to normalize an OpenAPI source. These names /// deliberately travel with an import job: changing either contract must not /// silently reinterpret a pending preview. -pub const NORMALIZER_VERSION: &str = "normalized-ir-v2"; -pub const PROJECTION_VERSION: &str = "preview-v2"; +pub const NORMALIZER_VERSION: &str = "normalized-ir-v3"; +pub const PROJECTION_VERSION: &str = "preview-v3"; -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] pub struct SourceDigest(String); impl SourceDigest { @@ -97,6 +97,19 @@ pub struct NormalizationConfig { pub max_collection_items: usize, pub max_aliases: usize, pub max_scalar_bytes: usize, + /// Maximum number of reference hops followed from one source location. + pub max_reference_depth: usize, + /// Maximum number of `$ref` occurrences inspected across the bundle. + pub max_references: usize, + /// Maximum number of immutable external documents supplied to the pure resolver. + pub max_reference_documents: usize, + /// Maximum number of nodes copied while expanding resolved references. + pub max_expanded_nodes: usize, + pub max_external_document_bytes: usize, + /// Signals that orchestration enabled external fetching. It never permits + /// I/O in this crate; it only distinguishes default-deny from a missing or + /// rejected supplied snapshot in exact findings. + pub external_references_enabled: bool, } impl Default for NormalizationConfig { @@ -110,6 +123,12 @@ impl Default for NormalizationConfig { max_collection_items: 10_000, max_aliases: 128, max_scalar_bytes: 256 * 1024, + max_reference_depth: 32, + max_references: 4_096, + max_reference_documents: 32, + max_expanded_nodes: 100_000, + max_external_document_bytes: 256 * 1024, + external_references_enabled: false, } } } @@ -139,6 +158,38 @@ pub struct UnresolvedReference { pub location: SourceLocation, } +/// Immutable external input for the pure reference resolver. The caller owns +/// URL policy and I/O; `crank-import` only consumes already verified bytes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExternalDocumentSnapshot { + pub canonical_uri: String, + pub digest: SourceDigest, + pub document: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolvedReferenceNode { + pub snapshot_digest: SourceDigest, + pub location: SourceLocation, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolvedReferenceEdge { + pub source: ResolvedReferenceNode, + pub target: ResolvedReferenceNode, + pub recursive: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolvedReferenceGraph { + /// Sorted, deduplicated immutable dependency identities. Canonical URLs + /// deliberately do not enter the IR or public diagnostics. + #[serde(default)] + pub dependency_digests: Vec, + #[serde(default)] + pub edges: Vec, +} + /// An unresolved `$ref` preserved from the decoded source. This is deliberately /// broader than schema references: path items, reusable parameters and other /// object-level references remain available to a later resolution phase. @@ -213,9 +264,34 @@ pub struct NormalizedSchema { pub location: SourceLocation, #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub discriminator: Option, + #[serde(default)] + pub constraints: NormalizedSchemaConstraints, pub kind: NormalizedSchemaKind, } +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct NormalizedSchemaConstraints { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub minimum: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub maximum: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_length: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_length: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pattern: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NormalizedDiscriminator { + pub property_name: String, + #[serde(default)] + pub mapping: BTreeMap, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] pub enum NormalizedSchemaKind { @@ -280,6 +356,8 @@ pub struct NormalizedIr { pub paths: Vec, #[serde(default)] pub unresolved_references: Vec, + #[serde(default)] + pub reference_graph: ResolvedReferenceGraph, pub source: ImportSourcePreview, #[serde(default)] pub operations: Vec, @@ -416,7 +494,7 @@ fn empty_source_location() -> SourceLocation { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum RestParameterLocation { Path, diff --git a/crates/crank-import/src/rest/normalize.rs b/crates/crank-import/src/rest/normalize.rs index 46f90b6..5ccaf00 100644 --- a/crates/crank-import/src/rest/normalize.rs +++ b/crates/crank-import/src/rest/normalize.rs @@ -5,13 +5,13 @@ use thiserror::Error; use crate::rest::{ model::{ - CoverageDisposition, CoverageEntry, ImportFinding, ImportFindingSeverity, - ImportGroupPreview, ImportPreview, ImportSourcePreview, NORMALIZER_VERSION, - NormalizationConfig, NormalizedApiMetadata, NormalizedFinding, NormalizedIr, - NormalizedLiteral, NormalizedOperation, NormalizedParameter, NormalizedScalarKind, - NormalizedSchema, NormalizedSchemaKind, PROJECTION_VERSION, RestImportDocument, - RestImportOperation, RestImportParameter, SourceDigest, SourceIdentity, SourceLocation, - SourceSyntax, + CoverageDisposition, CoverageEntry, ExternalDocumentSnapshot, ImportFinding, + ImportFindingSeverity, ImportGroupPreview, ImportPreview, ImportSourcePreview, + NORMALIZER_VERSION, NormalizationConfig, NormalizedApiMetadata, NormalizedFinding, + NormalizedIr, NormalizedLiteral, NormalizedOperation, NormalizedParameter, + NormalizedScalarKind, NormalizedSchema, NormalizedSchemaKind, PROJECTION_VERSION, + RestImportDocument, RestImportOperation, RestImportParameter, SourceDigest, SourceIdentity, + SourceLocation, SourceSyntax, }, openapi3, payload::candidate_from_operation, @@ -39,6 +39,25 @@ pub fn preview_document(document: &str) -> Result Result, ImportParseError> { + super::reference::reference_uris(document, config) +} + +/// Enumerates references in an already materialized external document. This +/// keeps the historical primary-document API intact while applying the +/// external-document byte limit at an explicit call site. +pub fn external_reference_uris( + document: &str, + config: &NormalizationConfig, +) -> Result, ImportParseError> { + super::reference::external_reference_uris(document, config) +} + /// Compatibility projection for jobs created before `NormalizedIr`. It is /// deliberately isolated from the v2 pipeline and may be removed only after /// the import-job TTL has elapsed. @@ -83,6 +102,18 @@ pub fn normalize_verified_document( document: &str, digest: SourceDigest, config: &NormalizationConfig, +) -> Result { + normalize_verified_bundle(document, digest, &[], config) +} + +/// Normalizes a verified primary document together with immutable external +/// snapshots. This function is pure: callers must perform all URL policy, +/// fetching, artifact persistence and digest verification beforehand. +pub fn normalize_verified_bundle( + document: &str, + digest: SourceDigest, + snapshots: &[ExternalDocumentSnapshot], + config: &NormalizationConfig, ) -> Result { config .validate_versions() @@ -99,6 +130,14 @@ pub fn normalize_verified_document( }; let root = decode(document)?; validate_value_limits(&root, config)?; + for snapshot in snapshots { + if snapshot.document.len() > config.max_external_document_bytes { + return Err(ImportParseError::LimitExceeded); + } + } + let resolution = super::reference::resolve(root, digest.clone(), snapshots, config)?; + let root = resolution.root; + validate_value_limits(&root, config)?; let parsed = match root.get("openapi").and_then(Value::as_str) { Some(version) if supported_oas_version(version) => openapi3::parse_document(&root)?, @@ -112,7 +151,15 @@ pub fn normalize_verified_document( if parsed.operations.is_empty() { return Err(ImportParseError::NoMethods); } - canonicalize(parsed, digest, config, root, source_syntax) + canonicalize( + parsed, + digest, + config, + root, + source_syntax, + resolution.graph, + resolution.findings, + ) } pub fn preview_from_ir(ir: &NormalizedIr) -> ImportPreview { @@ -202,6 +249,8 @@ fn canonicalize( config: &NormalizationConfig, root: Value, source_syntax: SourceSyntax, + reference_graph: crate::rest::model::ResolvedReferenceGraph, + resolution_findings: Vec, ) -> Result { let source = ImportSourcePreview { format: document.format, @@ -260,6 +309,7 @@ fn canonicalize( } }) .collect::>(); + findings.extend(resolution_findings); let mut operations = document .operations .into_iter() @@ -466,6 +516,40 @@ fn canonicalize( right.location.pointer.as_str(), )) }); + let mut document_findings = Vec::with_capacity(findings.len()); + for mut finding in findings.drain(..) { + let resolution_finding = matches!( + finding.code.as_str(), + "external_reference_disabled" + | "reference_target_missing" + | "reference_uri_malformed" + | "external_reference_unavailable" + | "reference_type_mismatch" + | "reference_graph_limit" + | "all_of_conflict" + | "unsupported_composition" + | "unsupported_discriminator" + ); + let target = resolution_finding + .then(|| { + operations.iter_mut().find(|operation| { + !finding.location.pointer.is_empty() + && (finding.location.pointer == operation.location.pointer + || finding + .location + .pointer + .starts_with(&format!("{}/", operation.location.pointer))) + }) + }) + .flatten(); + if let Some(operation) = target { + finding.operation_key = Some(operation.key.clone()); + operation.findings.push(finding); + } else { + document_findings.push(finding); + } + } + findings = document_findings; for operation in &mut operations { sort_normalized_findings(&mut operation.findings); } @@ -501,10 +585,9 @@ fn canonicalize( } sort_normalized_findings(&mut operation.findings); } - // Resolution is deliberately deferred. References already represented by - // an operation schema use that operation's blocker; all other references - // need a document-level blocker so omitted object-level semantics can - // never reach Apply silently. + // References left in the finite projection are precisely failures which + // the bounded resolver could not safely expand. Keep a blocker at the + // closest affected operation; unrelated operations remain actionable. findings.extend( unresolved_references .iter() @@ -529,7 +612,11 @@ fn canonicalize( }), ); sort_normalized_findings(&mut findings); - for finding in &findings { + for finding in findings.iter().chain( + operations + .iter() + .flat_map(|operation| operation.findings.iter()), + ) { if !coverage.iter().any(|entry| { entry.construct_id == finding.construct_id && entry.location == finding.location @@ -580,6 +667,7 @@ fn canonicalize( .unwrap_or_default(), paths: normalize_coverage::paths_from_operations(&operations, version), unresolved_references, + reference_graph, source, operations, findings, @@ -593,9 +681,56 @@ pub fn validate_normalized_ir(ir: &NormalizedIr) -> Result<(), ImportParseError> if ir.normalizer_version != NORMALIZER_VERSION || ir.projection_version != PROJECTION_VERSION { return Err(ImportParseError::InvalidDocument); } + validate_reference_graph(ir)?; normalize_coverage::validate_coverage(ir) } +fn validate_reference_graph(ir: &NormalizedIr) -> Result<(), ImportParseError> { + if ir + .reference_graph + .dependency_digests + .windows(2) + .any(|pair| pair[0] >= pair[1]) + { + return Err(ImportParseError::InvalidDocument); + } + let allowed = ir + .reference_graph + .dependency_digests + .iter() + .chain(std::iter::once(&ir.source_identity.digest)) + .map(SourceDigest::as_str) + .collect::>(); + let edges = &ir.reference_graph.edges; + if edges + .windows(2) + .any(|pair| reference_edge_key(&pair[0]) >= reference_edge_key(&pair[1])) + || edges.iter().any(|edge| { + !allowed.contains(edge.source.snapshot_digest.as_str()) + || !allowed.contains(edge.target.snapshot_digest.as_str()) + || !(edge.source.location.pointer.is_empty() + || edge.source.location.pointer.starts_with('/')) + || !(edge.target.location.pointer.is_empty() + || edge.target.location.pointer.starts_with('/')) + }) + { + return Err(ImportParseError::InvalidDocument); + } + Ok(()) +} + +fn reference_edge_key( + edge: &crate::rest::model::ResolvedReferenceEdge, +) -> (&str, &str, &str, &str, bool) { + ( + edge.source.snapshot_digest.as_str(), + edge.source.location.pointer.as_str(), + edge.target.snapshot_digest.as_str(), + edge.target.location.pointer.as_str(), + edge.recursive, + ) +} + fn coverage_disposition_rank(disposition: &CoverageDisposition) -> u8 { match disposition { CoverageDisposition::Mapped => 0, @@ -650,7 +785,10 @@ fn legacy_schema_value(schema: &NormalizedSchema) -> Value { let mut value = match &schema.kind { NormalizedSchemaKind::Reference { reference } => serde_json::json!({"$ref": reference.uri}), NormalizedSchemaKind::Composition { operator, variants } => { - serde_json::json!({operator: variants.iter().map(legacy_schema_value).collect::>() }) + serde_json::json!({ + operator: variants.iter().map(legacy_schema_value).collect::>(), + "x-crank-lossless-composition": true, + }) } NormalizedSchemaKind::Object { properties, @@ -718,7 +856,7 @@ fn schema_contains_reference(schema: &NormalizedSchema, location: &SourceLocatio } } -fn validate_value_limits( +pub(super) fn validate_value_limits( value: &Value, config: &NormalizationConfig, ) -> Result<(), ImportParseError> { diff --git a/crates/crank-import/src/rest/normalize_schema.rs b/crates/crank-import/src/rest/normalize_schema.rs index 81dee1c..776b481 100644 --- a/crates/crank-import/src/rest/normalize_schema.rs +++ b/crates/crank-import/src/rest/normalize_schema.rs @@ -1,8 +1,9 @@ use serde_json::Value; use crate::rest::model::{ - CoverageDisposition, CoverageEntry, NormalizedLiteral, NormalizedScalarKind, NormalizedSchema, - NormalizedSchemaKind, SourceLocation, UnresolvedReference, + CoverageDisposition, CoverageEntry, NormalizedDiscriminator, NormalizedLiteral, + NormalizedScalarKind, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind, + SourceLocation, UnresolvedReference, }; use super::normalize_coverage::escape_pointer; @@ -150,6 +151,35 @@ pub(super) fn typed_schema( .get("description") .and_then(Value::as_str) .map(ToOwned::to_owned), + discriminator: value.get("discriminator").and_then(|value| { + let property_name = value.get("propertyName")?.as_str()?.to_owned(); + let mapping = value + .get("mapping") + .and_then(Value::as_object) + .map(|mapping| { + mapping + .iter() + .filter_map(|(key, value)| { + value.as_str().map(|value| (key.clone(), value.to_owned())) + }) + .collect() + }) + .unwrap_or_default(); + Some(NormalizedDiscriminator { + property_name, + mapping, + }) + }), + constraints: NormalizedSchemaConstraints { + minimum: value.get("minimum").and_then(Value::as_f64), + maximum: value.get("maximum").and_then(Value::as_f64), + min_length: value.get("minLength").and_then(Value::as_u64), + max_length: value.get("maxLength").and_then(Value::as_u64), + pattern: value + .get("pattern") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + }, kind, } } diff --git a/crates/crank-import/src/rest/openapi3.rs b/crates/crank-import/src/rest/openapi3.rs index 9d23cac..66e8206 100644 --- a/crates/crank-import/src/rest/openapi3.rs +++ b/crates/crank-import/src/rest/openapi3.rs @@ -133,6 +133,7 @@ fn parse_document_v2(root: &Value) -> Result Result) { + let mut seen = std::collections::BTreeSet::new(); + parameters.reverse(); + parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location))); + parameters.reverse(); +} + pub fn parse_document_legacy_v1(root: &Value) -> Result { let version = root .get("openapi") diff --git a/crates/crank-import/src/rest/reference.rs b/crates/crank-import/src/rest/reference.rs new file mode 100644 index 0000000..0fdd09c --- /dev/null +++ b/crates/crank-import/src/rest/reference.rs @@ -0,0 +1,872 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Map, Value}; + +use crate::rest::model::{ + ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedFinding, + ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode, SourceDigest, + SourceLocation, +}; + +use super::{ImportParseError, normalize_coverage::escape_pointer}; + +pub(super) struct ResolutionResult { + pub root: Value, + pub graph: ResolvedReferenceGraph, + pub findings: Vec, +} + +pub(super) fn reference_uris( + document: &str, + config: &NormalizationConfig, +) -> Result, ImportParseError> { + reference_uris_with_max_bytes(document, config, config.max_bytes) +} + +pub(super) fn external_reference_uris( + document: &str, + config: &NormalizationConfig, +) -> Result, ImportParseError> { + reference_uris_with_max_bytes(document, config, config.max_external_document_bytes) +} + +fn reference_uris_with_max_bytes( + document: &str, + config: &NormalizationConfig, + max_bytes: usize, +) -> Result, ImportParseError> { + fn collect(value: &Value, uris: &mut BTreeSet) { + match value { + Value::Object(object) => { + if let Some(reference) = object.get("$ref").and_then(Value::as_str) { + uris.insert(reference.to_owned()); + } + for (key, child) in object { + if is_literal_payload_key(key) { + continue; + } + collect(child, uris); + } + } + Value::Array(items) => { + for child in items { + collect(child, uris); + } + } + _ => {} + } + } + if document.len() > max_bytes + || super::normalize_limits::alias_count(document) > config.max_aliases + { + return Err(ImportParseError::LimitExceeded); + } + let root = decode_snapshot(document)?; + super::normalize::validate_value_limits(&root, config)?; + let mut uris = BTreeSet::new(); + collect(&root, &mut uris); + Ok(uris.into_iter().collect()) +} + +struct Document { + uri: Option, + digest: SourceDigest, + root: Value, + oas31: bool, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct NodeKey { + digest: String, + pointer: String, +} + +#[derive(Clone, Copy)] +struct TraversalLocation<'a> { + projection: &'a str, + origin: &'a str, +} + +struct Resolver<'a> { + config: &'a NormalizationConfig, + documents: Vec, + by_uri: BTreeMap, + graph: ResolvedReferenceGraph, + findings: Vec, + references: usize, + expanded_nodes: usize, +} + +pub(super) fn resolve( + root: Value, + primary_digest: SourceDigest, + snapshots: &[ExternalDocumentSnapshot], + config: &NormalizationConfig, +) -> Result { + if snapshots.len() > config.max_reference_documents { + return Err(ImportParseError::LimitExceeded); + } + let mut documents = vec![Document { + uri: None, + digest: primary_digest, + oas31: is_oas31(&root), + root, + }]; + let mut by_uri = BTreeMap::new(); + let mut dependency_digests = BTreeSet::new(); + let primary_oas31 = documents[0].oas31; + for snapshot in snapshots { + if snapshot.canonical_uri.is_empty() + || by_uri + .insert(snapshot.canonical_uri.clone(), documents.len()) + .is_some() + { + return Err(ImportParseError::InvalidDocument); + } + if super::normalize_limits::alias_count(&snapshot.document) > config.max_aliases { + return Err(ImportParseError::LimitExceeded); + } + let root = decode_snapshot(&snapshot.document)?; + super::normalize::validate_value_limits(&root, config)?; + dependency_digests.insert(snapshot.digest.clone()); + documents.push(Document { + uri: Some(snapshot.canonical_uri.clone()), + digest: snapshot.digest.clone(), + // External documents are fragments of the primary contract. In a + // 3.1 bundle they therefore use 3.1 `$ref` sibling semantics even + // when the fragment itself omits an `openapi` declaration. + oas31: primary_oas31 || is_oas31(&root), + root, + }); + } + let mut resolver = Resolver { + config, + documents, + by_uri, + graph: ResolvedReferenceGraph { + dependency_digests: dependency_digests.into_iter().collect(), + edges: Vec::new(), + }, + findings: Vec::new(), + references: 0, + expanded_nodes: 0, + }; + let root = resolver.documents[0].root.clone(); + let root = resolver.expand_value( + 0, + root, + TraversalLocation { + projection: "", + origin: "", + }, + 0, + &mut Vec::new(), + )?; + resolver + .graph + .edges + .sort_by(|left, right| edge_key(left).cmp(&edge_key(right))); + resolver.graph.edges.dedup(); + resolver.findings.sort_by(|left, right| { + ( + &left.operation_key, + &left.construct_id, + &left.code, + &left.location.pointer, + ) + .cmp(&( + &right.operation_key, + &right.construct_id, + &right.code, + &right.location.pointer, + )) + }); + resolver.findings.dedup(); + Ok(ResolutionResult { + root, + graph: resolver.graph, + findings: resolver.findings, + }) +} + +impl Resolver<'_> { + fn expand_value( + &mut self, + document_index: usize, + value: Value, + location: TraversalLocation<'_>, + depth: usize, + stack: &mut Vec, + ) -> Result { + if depth > 0 { + self.expanded_nodes = self.expanded_nodes.saturating_add(1); + if self.expanded_nodes > self.config.max_expanded_nodes { + self.push_finding("reference_graph_limit", location.projection); + return Ok(Value::Object(Map::new())); + } + } + match value { + Value::Object(mut object) => { + if let Some(reference) = object + .get("$ref") + .and_then(Value::as_str) + .map(str::to_owned) + { + return self.expand_reference( + document_index, + object, + &reference, + location, + depth, + stack, + ); + } + let keys = object.keys().cloned().collect::>(); + for key in keys { + if let Some(child) = object.remove(&key) { + if is_literal_payload_key(&key) { + object.insert(key, child); + continue; + } + let child_pointer = + format!("{}/{}", location.projection, escape_pointer(&key)); + let child_origin_pointer = + format!("{}/{}", location.origin, escape_pointer(&key)); + object.insert( + key, + self.expand_value( + document_index, + child, + TraversalLocation { + projection: &child_pointer, + origin: &child_origin_pointer, + }, + depth, + stack, + )?, + ); + } + } + let composition_count = ["allOf", "oneOf", "anyOf"] + .into_iter() + .filter(|operator| object.contains_key(*operator)) + .count(); + if composition_count > 1 { + self.push_finding("unsupported_composition", location.projection); + } + if let Some(discriminator) = object.get("discriminator") { + let valid = discriminator + .get("propertyName") + .and_then(Value::as_str) + .is_some_and(|value| !value.is_empty()) + && discriminator.get("mapping").is_none_or(|mapping| { + mapping + .as_object() + .is_some_and(|mapping| mapping.values().all(Value::is_string)) + }) + && (object.contains_key("oneOf") || object.contains_key("anyOf")); + if !valid { + self.push_finding("unsupported_discriminator", location.projection); + } + } + self.merge_all_of(object, location.projection) + } + Value::Array(items) => Ok(Value::Array( + items + .into_iter() + .enumerate() + .map(|(index, child)| { + self.expand_value( + document_index, + child, + TraversalLocation { + projection: &format!("{}/{index}", location.projection), + origin: &format!("{}/{index}", location.origin), + }, + depth, + stack, + ) + }) + .collect::, _>>()?, + )), + other => Ok(other), + } + } + + fn expand_reference( + &mut self, + document_index: usize, + mut source_object: Map, + reference: &str, + location: TraversalLocation<'_>, + depth: usize, + stack: &mut Vec, + ) -> Result { + self.references = self.references.saturating_add(1); + if self.references > self.config.max_references || depth >= self.config.max_reference_depth + { + self.push_finding( + "reference_graph_limit", + &format!("{}/$ref", location.projection), + ); + return Ok(Value::Object(source_object)); + } + let target = self.target(document_index, reference); + if matches!(target, Err(TargetError::MalformedFragment)) { + self.push_finding( + "reference_uri_malformed", + &format!("{}/$ref", location.projection), + ); + return Ok(Value::Object(source_object)); + } + let Some((target_document, target_pointer)) = target.ok().flatten() else { + let external = reference.starts_with("http://") || reference.starts_with("https://"); + let code = if external { + if self.config.external_references_enabled { + "external_reference_unavailable" + } else { + "external_reference_disabled" + } + } else { + "reference_target_missing" + }; + self.push_finding(code, &format!("{}/$ref", location.projection)); + return Ok(Value::Object(source_object)); + }; + let source = ResolvedReferenceNode { + snapshot_digest: self.documents[document_index].digest.clone(), + location: SourceLocation { + pointer: format!("{}/$ref", location.origin), + }, + }; + let target = ResolvedReferenceNode { + snapshot_digest: self.documents[target_document].digest.clone(), + location: SourceLocation { + pointer: target_pointer.clone(), + }, + }; + let key = NodeKey { + digest: target.snapshot_digest.as_str().to_owned(), + pointer: target_pointer.clone(), + }; + let recursive = stack.contains(&key); + self.graph.edges.push(ResolvedReferenceEdge { + source, + target, + recursive, + }); + if recursive { + // The edge is the lossless representation. Expansion stops here, + // producing an opaque object for the finite preview projection. + return Ok(Value::Object(Map::new())); + } + let Some(target_value) = self.documents[target_document] + .root + .pointer(&target_pointer) + .cloned() + else { + self.push_finding( + "reference_target_missing", + &format!("{}/$ref", location.projection), + ); + return Ok(Value::Object(source_object)); + }; + if !target_value.is_object() { + self.push_finding( + "reference_type_mismatch", + &format!("{}/$ref", location.projection), + ); + return Ok(Value::Object(source_object)); + } + stack.push(key); + let mut expanded = self.expand_value( + target_document, + target_value, + TraversalLocation { + projection: location.projection, + origin: &target_pointer, + }, + depth + 1, + stack, + )?; + stack.pop(); + + // OAS 3.1 Schema Objects permit siblings next to `$ref`; OAS 3.0 and + // Swagger Reference Objects ignore them. The source document version + // controls the semantics at the reference site. + source_object.remove("$ref"); + if self.documents[document_index].oas31 && !source_object.is_empty() { + let Value::Object(expanded_object) = &mut expanded else { + self.push_finding( + "reference_type_mismatch", + &format!("{}/$ref", location.projection), + ); + return Ok(Value::Object(source_object)); + }; + for (key, sibling) in source_object { + let child_pointer = format!("{}/{}", location.projection, escape_pointer(&key)); + let child_origin_pointer = format!("{}/{}", location.origin, escape_pointer(&key)); + expanded_object.insert( + key, + self.expand_value( + document_index, + sibling, + TraversalLocation { + projection: &child_pointer, + origin: &child_origin_pointer, + }, + depth, + stack, + )?, + ); + } + } + Ok(expanded) + } + + fn target( + &self, + current: usize, + reference: &str, + ) -> Result, TargetError> { + let (document, fragment) = reference.split_once('#').unwrap_or((reference, "")); + let fragment = percent_decode(fragment)?; + let pointer = if fragment.is_empty() { + String::new() + } else if fragment.starts_with('/') { + fragment + } else { + return Ok(None); + }; + if document.is_empty() { + return Ok(Some((current, pointer))); + } + let canonical = if has_uri_scheme(document) { + canonical_absolute_uri(document) + } else { + let Some(base) = self.documents[current].uri.as_deref() else { + return Ok(None); + }; + join_relative(base, document) + }; + Ok(canonical.and_then(|canonical| { + self.by_uri + .get(&canonical) + .copied() + .map(|index| (index, pointer)) + })) + } + + fn merge_all_of( + &mut self, + mut object: Map, + pointer: &str, + ) -> Result { + let branches = match object.remove("allOf") { + None => return Ok(Value::Object(object)), + Some(Value::Array(branches)) => branches, + Some(value) => { + object.insert("allOf".to_owned(), value); + self.push_finding("unsupported_composition", &format!("{pointer}/allOf")); + return Ok(Value::Object(object)); + } + }; + let original = branches.clone(); + for branch in branches { + let Some(fields) = branch.as_object() else { + object.insert("allOf".to_owned(), Value::Array(original)); + self.push_finding("all_of_conflict", &format!("{pointer}/allOf")); + return Ok(Value::Object(object)); + }; + if !merge_object(&mut object, fields) { + object.insert("allOf".to_owned(), Value::Array(original)); + self.push_finding("all_of_conflict", &format!("{pointer}/allOf")); + return Ok(Value::Object(object)); + } + } + Ok(Value::Object(object)) + } + + fn push_finding(&mut self, code: &str, pointer: &str) { + self.findings.push(NormalizedFinding { + code: code.to_owned(), + severity: ImportFindingSeverity::Error, + message: match code { + "external_reference_disabled" => "Внешняя ссылка отключена политикой импорта.", + "reference_target_missing" => { + "Цель ссылки отсутствует или имеет неверный JSON Pointer." + } + "external_reference_unavailable" => { + "Внешний snapshot недоступен или отклонён политикой импорта." + } + "reference_type_mismatch" => "Цель ссылки имеет неподдерживаемый тип.", + "reference_graph_limit" => "Граф ссылок превышает установленный предел.", + "all_of_conflict" => "Ветки allOf содержат несовместимые определения.", + "reference_uri_malformed" => "URI fragment ссылки содержит некорректное percent-кодирование.", + "unsupported_composition" => "Несколько операторов composition в одной schema не могут быть спроецированы без потерь.", + "unsupported_discriminator" => "Discriminator имеет неподдерживаемую или неполную структуру.", + _ => "Ссылка не может быть безопасно разрешена.", + } + .to_owned(), + construct_id: format!("source:{pointer}"), + location: SourceLocation { + pointer: pointer.to_owned(), + }, + operation_key: None, + }); + } +} + +fn merge_object(target: &mut Map, source: &Map) -> bool { + for (key, value) in source { + match key.as_str() { + "properties" => { + let Some(source_properties) = value.as_object() else { + return false; + }; + let properties = target + .entry(key.clone()) + .or_insert_with(|| Value::Object(Map::new())); + let Some(target_properties) = properties.as_object_mut() else { + return false; + }; + for (name, schema) in source_properties { + if let Some(existing) = target_properties.get(name) { + let (Some(existing), Some(schema)) = + (existing.as_object(), schema.as_object()) + else { + return false; + }; + let mut merged = existing.clone(); + if !merge_object(&mut merged, schema) { + return false; + } + target_properties.insert(name.clone(), Value::Object(merged)); + } else { + target_properties.insert(name.clone(), schema.clone()); + } + } + } + "required" => { + let Some(source_required) = value.as_array() else { + return false; + }; + let required = target + .entry(key.clone()) + .or_insert_with(|| Value::Array(Vec::new())); + let Some(target_required) = required.as_array_mut() else { + return false; + }; + target_required.extend(source_required.iter().cloned()); + target_required.sort_by(|left, right| left.as_str().cmp(&right.as_str())); + target_required.dedup(); + } + "minimum" | "maximum" => { + let Some(source_value) = value.as_f64() else { + return false; + }; + if target.contains_key(key) && target.get(key).and_then(Value::as_f64).is_none() { + return false; + } + let merged = match (key.as_str(), target.get(key).and_then(Value::as_f64)) { + ("minimum", Some(current)) => current.max(source_value), + ("maximum", Some(current)) => current.min(source_value), + _ => source_value, + }; + let Some(number) = serde_json::Number::from_f64(merged) else { + return false; + }; + target.insert(key.clone(), Value::Number(number)); + if constraint_contradiction(target, "minimum", "maximum") { + return false; + } + } + "minLength" | "maxLength" => { + let Some(source_value) = value.as_u64() else { + return false; + }; + if target.contains_key(key) && target.get(key).and_then(Value::as_u64).is_none() { + return false; + } + let merged = match (key.as_str(), target.get(key).and_then(Value::as_u64)) { + ("minLength", Some(current)) => current.max(source_value), + ("maxLength", Some(current)) => current.min(source_value), + _ => source_value, + }; + target.insert(key.clone(), Value::Number(merged.into())); + if integer_constraint_contradiction(target, "minLength", "maxLength") { + return false; + } + } + _ => { + if target.get(key).is_some_and(|existing| existing != value) { + return false; + } + target.insert(key.clone(), value.clone()); + } + } + } + true +} + +fn constraint_contradiction(target: &Map, minimum: &str, maximum: &str) -> bool { + match ( + target.get(minimum).and_then(Value::as_f64), + target.get(maximum).and_then(Value::as_f64), + ) { + (Some(minimum), Some(maximum)) => minimum > maximum, + _ => false, + } +} + +fn integer_constraint_contradiction( + target: &Map, + minimum: &str, + maximum: &str, +) -> bool { + match ( + target.get(minimum).and_then(Value::as_u64), + target.get(maximum).and_then(Value::as_u64), + ) { + (Some(minimum), Some(maximum)) => minimum > maximum, + _ => false, + } +} + +fn edge_key(edge: &ResolvedReferenceEdge) -> (&str, &str, &str, &str, bool) { + ( + edge.source.snapshot_digest.as_str(), + &edge.source.location.pointer, + edge.target.snapshot_digest.as_str(), + &edge.target.location.pointer, + edge.recursive, + ) +} + +fn decode_snapshot(document: &str) -> Result { + if let Ok(value) = serde_json::from_str(document) { + return Ok(value); + } + let yaml: serde_yaml::Value = + serde_yaml::from_str(document).map_err(|_| ImportParseError::InvalidDocument)?; + serde_json::to_value(yaml).map_err(|_| ImportParseError::InvalidDocument) +} + +fn is_oas31(root: &Value) -> bool { + root.get("openapi") + .and_then(Value::as_str) + .is_some_and(|version| version.starts_with("3.1.")) +} + +fn join_relative(base: &str, relative: &str) -> Option { + let base = UriReference::parse(base)?; + let relative = UriReference::parse(relative)?; + let scheme = relative.scheme.or(base.scheme)?; + let authority = if relative.scheme.is_some() || relative.authority.is_some() { + relative.authority + } else { + base.authority + }; + let (path, query) = if relative.scheme.is_some() || relative.authority.is_some() { + (remove_dot_segments(&relative.path), relative.query) + } else if relative.path.is_empty() { + (base.path, relative.query.or(base.query)) + } else if relative.path.starts_with('/') { + (remove_dot_segments(&relative.path), relative.query) + } else { + ( + remove_dot_segments(&merge_paths( + &base.path, + authority.is_some(), + &relative.path, + )), + relative.query, + ) + }; + UriReference { + scheme: Some(scheme), + authority, + path, + query, + } + .render() +} + +fn canonical_absolute_uri(uri: &str) -> Option { + let reference = UriReference::parse(uri)?; + let scheme = reference.scheme?; + UriReference { + scheme: Some(scheme), + authority: reference.authority, + path: remove_dot_segments(&reference.path), + query: reference.query, + } + .render() +} + +#[derive(Clone, Debug)] +struct UriReference<'a> { + scheme: Option<&'a str>, + authority: Option<&'a str>, + path: String, + query: Option<&'a str>, +} + +impl<'a> UriReference<'a> { + fn parse(value: &'a str) -> Option { + if value.contains('\\') || value.contains('#') { + return None; + } + let (without_query, query) = value + .split_once('?') + .map_or((value, None), |(path, query)| (path, Some(query))); + let (scheme, rest) = if let Some(index) = without_query.find(':') { + let candidate = &without_query[..index]; + if is_uri_scheme(candidate) { + (Some(candidate), &without_query[index + 1..]) + } else { + (None, without_query) + } + } else { + (None, without_query) + }; + let (authority, path) = if let Some(rest) = rest.strip_prefix("//") { + match rest.find('/') { + Some(index) => (Some(&rest[..index]), rest[index..].to_owned()), + None => (Some(rest), String::new()), + } + } else { + (None, rest.to_owned()) + }; + Some(Self { + scheme, + authority, + path, + query, + }) + } + + fn render(&self) -> Option { + let scheme = self.scheme?; + let scheme = scheme.to_ascii_lowercase(); + let mut value = format!("{scheme}:"); + if let Some(authority) = self.authority { + value.push_str("//"); + value.push_str(&canonical_authority(authority, &scheme)); + } + if self.authority.is_some() && self.path.is_empty() { + value.push('/'); + } else { + value.push_str(&self.path); + } + if let Some(query) = self.query { + value.push('?'); + value.push_str(query); + } + Some(value) + } +} + +fn canonical_authority(authority: &str, scheme: &str) -> String { + let authority = authority.to_ascii_lowercase(); + let default_port = match scheme { + "http" => Some("80"), + "https" => Some("443"), + _ => None, + }; + if let Some(default_port) = default_port + && let Some((host, port)) = authority.rsplit_once(':') + && port == default_port + && (!host.contains(':') || host.ends_with(']')) + { + return host.to_owned(); + } + authority +} + +fn has_uri_scheme(value: &str) -> bool { + value + .split_once(':') + .is_some_and(|(candidate, _)| is_uri_scheme(candidate)) +} + +fn is_uri_scheme(candidate: &str) -> bool { + let Some(first) = candidate.as_bytes().first() else { + return false; + }; + first.is_ascii_alphabetic() + && candidate + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.')) +} + +fn merge_paths(base_path: &str, has_authority: bool, relative_path: &str) -> String { + match base_path.rfind('/') { + Some(index) => format!("{}{}", &base_path[..=index], relative_path), + None if has_authority => format!("/{relative_path}"), + None => relative_path.to_owned(), + } +} + +fn remove_dot_segments(path: &str) -> String { + let leading_slash = path.starts_with('/'); + let trailing_slash = path.ends_with('/'); + let mut output = Vec::new(); + for segment in path.split('/') { + match segment { + "." => {} + ".." => { + output.pop(); + } + _ => output.push(segment), + } + } + let mut result = output.join("/"); + if leading_slash && !result.starts_with('/') { + result.insert(0, '/'); + } + if trailing_slash && !result.ends_with('/') { + result.push('/'); + } + result +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TargetError { + MalformedFragment, +} + +fn percent_decode(fragment: &str) -> Result { + let bytes = fragment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] != b'%' { + decoded.push(bytes[index]); + index += 1; + continue; + } + let Some(high) = bytes.get(index + 1).and_then(|byte| hex_value(*byte)) else { + return Err(TargetError::MalformedFragment); + }; + let Some(low) = bytes.get(index + 2).and_then(|byte| hex_value(*byte)) else { + return Err(TargetError::MalformedFragment); + }; + decoded.push((high << 4) | low); + index += 3; + } + String::from_utf8(decoded).map_err(|_| TargetError::MalformedFragment) +} + +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +fn is_literal_payload_key(key: &str) -> bool { + matches!(key, "example" | "examples" | "default" | "enum" | "const") || key.starts_with("x-") +} diff --git a/crates/crank-import/src/rest/schema.rs b/crates/crank-import/src/rest/schema.rs index 96ea705..e2b5abb 100644 --- a/crates/crank-import/src/rest/schema.rs +++ b/crates/crank-import/src/rest/schema.rs @@ -32,8 +32,33 @@ pub fn schema_from_openapi( let Some(value) = value else { return primitive(SchemaKind::String, required, description); }; - // NormalizedIR keeps composition typed; the legacy preview adapter retains - // its historical first-branch projection for pending v1 jobs. + // Only v3 NormalizedIR emits the private marker. Pending legacy-v1 jobs + // retain their historical first-branch behavior, while modern oneOf/anyOf + // is projected losslessly through crank-schema's existing Oneof shape. + if value + .get("x-crank-lossless-composition") + .and_then(Value::as_bool) + == Some(true) + && let Some(items) = value + .get("oneOf") + .or_else(|| value.get("anyOf")) + .and_then(Value::as_array) + { + return Schema { + kind: SchemaKind::Oneof, + description: description.or_else(|| text(value, "description")), + required, + nullable: nullable(value), + default_value: value.get("default").cloned(), + fields: BTreeMap::new(), + items: None, + enum_values: Vec::new(), + variants: items + .iter() + .map(|item| schema_from_openapi(Some(item), true, None)) + .collect(), + }; + } let resolved = collapse_composition(value); if let Some(values) = resolved.get("enum").and_then(Value::as_array) { diff --git a/crates/crank-import/src/rest/swagger2.rs b/crates/crank-import/src/rest/swagger2.rs index 5d736d6..cffe9c9 100644 --- a/crates/crank-import/src/rest/swagger2.rs +++ b/crates/crank-import/src/rest/swagger2.rs @@ -107,6 +107,7 @@ fn parse_document_v2(root: &Value) -> Result Result) { + let mut seen = std::collections::BTreeSet::new(); + parameters.reverse(); + parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location))); + parameters.reverse(); +} + pub fn parse_document_legacy_v1(root: &Value) -> Result { let title = root .pointer("/info/title") diff --git a/crates/crank-import/tests/normalization_details.rs b/crates/crank-import/tests/normalization_details.rs index 6b7f082..26a5922 100644 --- a/crates/crank-import/tests/normalization_details.rs +++ b/crates/crank-import/tests/normalization_details.rs @@ -161,6 +161,58 @@ paths: assert!(ir.operations[0].request_body_schema.is_some()); } + #[test] + fn operation_parameters_override_path_parameters_by_name_and_location() { + let openapi = r#" +openapi: 3.1.0 +info: { title: OAS parameter overrides } +servers: [{ url: https://example.test }] +paths: + /items: + parameters: + - { name: page, in: query, description: path, schema: { type: integer } } + get: + operationId: listItems + parameters: + - { name: page, in: query, description: operation, schema: { type: string } } + responses: { '200': { description: ok } } +"#; + let ir = normalize_document(openapi, &NormalizationConfig::default()).unwrap(); + assert_eq!(ir.operations[0].parameters.len(), 1); + assert_eq!( + ir.operations[0].parameters[0].description.as_deref(), + Some("operation") + ); + assert_eq!( + ir.operations[0].parameters[0].source_location.pointer, + "/paths/~1items/get/parameters/0" + ); + + let swagger = r#" +swagger: '2.0' +info: { title: Swagger parameter overrides } +paths: + /items: + parameters: + - { name: page, in: query, description: path, type: integer } + get: + operationId: listItems + parameters: + - { name: page, in: query, description: operation, type: string } + responses: { '200': { description: ok } } +"#; + let ir = normalize_document(swagger, &NormalizationConfig::default()).unwrap(); + assert_eq!(ir.operations[0].parameters.len(), 1); + assert_eq!( + ir.operations[0].parameters[0].description.as_deref(), + Some("operation") + ); + assert_eq!( + ir.operations[0].parameters[0].source_location.pointer, + "/paths/~1items/get/parameters/0" + ); + } + #[test] fn openapi_parameter_omissions_are_errors_at_the_dropped_item_pointer() { let document = r#" @@ -220,10 +272,6 @@ components: "/paths/~1items~1{id}/parameters/1", ), ("invalid_parameter", "/paths/~1items~1{id}/parameters/2",), - ( - "unresolved_parameter_reference", - "/paths/~1items~1{id}/parameters/3", - ), ( "missing_parameter_name", "/paths/~1items~1{id}/get/parameters/0", @@ -336,10 +384,6 @@ parameters: "/paths/~1items~1{id}/parameters/1", ), ("invalid_parameter", "/paths/~1items~1{id}/parameters/2",), - ( - "unresolved_parameter_reference", - "/paths/~1items~1{id}/parameters/3", - ), ( "missing_parameter_name", "/paths/~1items~1{id}/get/parameters/0", @@ -688,7 +732,7 @@ paths: } #[test] - fn preserves_every_unresolved_reference_from_the_full_source_tree() { + fn resolves_local_reference_graph_and_preserves_external_blocker() { let document = r#" openapi: 3.1.0 info: { title: References } @@ -722,32 +766,24 @@ components: serde_json::to_vec(&first).unwrap(), serde_json::to_vec(&second).unwrap() ); - assert_eq!(first.unresolved_references.len(), 10); + assert_eq!(first.unresolved_references.len(), 1); let references = first .unresolved_references .iter() .map(|reference| (reference.uri.as_str(), reference.location.pointer.as_str())) .collect::>(); - assert!(references.contains(&( - "#/components/parameters/Id", - "/paths/~1ok/parameters/0/$ref" - ))); - assert!(references.contains(&( - "#/components/requestBodies/Body", - "/paths/~1ok/get/requestBody/$ref" - ))); - assert!(references.contains(&( - "#/components/responses/Ok", - "/paths/~1ok/get/responses/200/$ref" - ))); - assert!(references.contains(&("#/components/pathItems/Reusable", "/paths/~1reused/$ref"))); - assert!( - references.contains(&("#/components/schemas/Loop", "/components/schemas/Loop/$ref")) - ); assert!(references.contains(&( "https://example.test/schema.json#/Remote", "/components/schemas/Remote/$ref" ))); + assert!(first.reference_graph.edges.len() >= 5); + assert!( + first + .reference_graph + .edges + .iter() + .any(|edge| edge.recursive) + ); assert!(first.unresolved_references.iter().all(|reference| { reference.construct_id == format!( diff --git a/crates/crank-import/tests/preview.rs b/crates/crank-import/tests/preview.rs index be83a0f..4107901 100644 --- a/crates/crank-import/tests/preview.rs +++ b/crates/crank-import/tests/preview.rs @@ -94,7 +94,7 @@ definitions: } #[test] - fn previews_swagger2_and_preserves_unresolved_definitions() { + fn previews_swagger2_and_resolves_local_definitions() { let preview = preview_document(SWAGGER2).unwrap(); assert_eq!(preview.source.format, "swagger"); @@ -105,7 +105,7 @@ definitions: let operation = &preview.groups[0].operations[0]; assert_eq!(operation.suggested_name, "get_pet"); assert_eq!(operation.input_fields, 1); - assert_eq!(operation.output_fields, 0); + assert_eq!(operation.output_fields, 2); assert_eq!(operation.draft.target.path_template, "/pets/{id}"); assert_eq!( operation.draft.input_mapping.rules[0].target, diff --git a/crates/crank-import/tests/reference_resolution.rs b/crates/crank-import/tests/reference_resolution.rs new file mode 100644 index 0000000..5293c44 --- /dev/null +++ b/crates/crank-import/tests/reference_resolution.rs @@ -0,0 +1,793 @@ +use crank_import::rest::{ + ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedSchemaKind, + SourceDigest, external_reference_uris, normalize_verified_bundle, normalize_verified_document, + preview_from_ir, reference_uris, +}; + +fn digest(byte: char) -> SourceDigest { + SourceDigest::parse(byte.to_string().repeat(64)).unwrap() +} + +fn normalize(document: &str) -> crank_import::rest::NormalizedIr { + normalize_verified_document(document, digest('a'), &NormalizationConfig::default()).unwrap() +} + +#[test] +fn resolves_local_schema_and_object_references_with_rfc6901_escaping() { + let document = r#" +openapi: 3.1.0 +info: { title: Local refs } +servers: [{ url: https://api.example.test }] +paths: + /items/{id}: + get: + operationId: getItem + parameters: + - { $ref: '#/components/parameters/Id' } + responses: + '200': { $ref: '#/components/responses/Ok' } +components: + parameters: + Id: { name: id, in: path, required: true, schema: { type: string } } + responses: + Ok: + description: ok + content: + application/json: + schema: { $ref: '#/components/schemas/a~1b~0c' } + schemas: + a/b~c: + type: object + required: [id] + properties: { id: { type: string } } +"#; + let ir = normalize(document); + let operation = &ir.operations[0]; + assert_eq!(operation.parameters.len(), 1); + assert_eq!(operation.parameters[0].name, "id"); + assert!(matches!( + operation.response_schema.as_ref().map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id") + )); + assert!(ir.unresolved_references.is_empty()); + assert_eq!(ir.reference_graph.edges.len(), 3); + assert!( + ir.findings + .iter() + .chain(operation.findings.iter()) + .all(|finding| finding.code != "unresolved_reference") + ); +} + +#[test] +fn broken_reference_blocks_only_affected_candidate_and_full_preview_remains_visible() { + let document = r#" +openapi: 3.1.0 +info: { title: Partial graph } +servers: [{ url: https://api.example.test }] +paths: + /broken: + get: + operationId: broken + responses: + '200': + description: nope + content: { application/json: { schema: { $ref: '#/components/schemas/Missing' } } } + /healthy: + get: + operationId: healthy + responses: { '204': { description: ok } } +"#; + let ir = normalize(document); + let broken = ir + .operations + .iter() + .find(|operation| operation.path == "/broken") + .unwrap(); + let healthy = ir + .operations + .iter() + .find(|operation| operation.path == "/healthy") + .unwrap(); + assert!(broken.findings.iter().any(|finding| { + finding.code == "reference_target_missing" + && finding.severity == ImportFindingSeverity::Error + })); + assert!(healthy.findings.is_empty()); + let preview = preview_from_ir(&ir); + assert_eq!( + preview + .groups + .iter() + .map(|group| group.operations.len()) + .sum::(), + 2 + ); +} + +#[test] +fn external_references_are_default_deny_without_network_or_snapshot() { + let document = r#" +openapi: 3.1.0 +info: { title: External deny } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } } +"#; + let ir = normalize(document); + assert!( + ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "external_reference_disabled") + ); + assert!(ir.reference_graph.edges.is_empty()); + assert!(ir.reference_graph.dependency_digests.is_empty()); +} + +#[test] +fn resolves_supplied_external_snapshot_and_relative_chain_deterministically() { + let document = r#" +openapi: 3.1.0 +info: { title: External snapshots } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } } +"#; + let snapshots = vec![ + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/root.yaml".to_owned(), + digest: digest('b'), + document: "Item: { $ref: 'child.yaml#/Child' }".to_owned(), + }, + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/child.yaml".to_owned(), + digest: digest('c'), + document: "Child: { type: object, properties: { value: { type: integer } } }" + .to_owned(), + }, + ]; + let first = normalize_verified_bundle( + document, + digest('a'), + &snapshots, + &NormalizationConfig::default(), + ) + .unwrap(); + let second = normalize_verified_bundle( + document, + digest('a'), + &snapshots, + &NormalizationConfig::default(), + ) + .unwrap(); + assert_eq!( + serde_json::to_vec(&first).unwrap(), + serde_json::to_vec(&second).unwrap() + ); + assert_eq!( + first.reference_graph.dependency_digests, + vec![digest('b'), digest('c')] + ); + assert_eq!(first.reference_graph.edges.len(), 2); + assert_eq!( + first.reference_graph.edges[1].source.snapshot_digest, + digest('b') + ); + assert!(first.unresolved_references.is_empty()); +} + +#[test] +fn recursion_is_a_stable_graph_edge_without_unbounded_expansion() { + let document = r#" +openapi: 3.1.0 +info: { title: Recursive } +servers: [{ url: https://api.example.test }] +paths: + /nodes: + get: + operationId: getNode + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: '#/components/schemas/Node' } } } +components: + schemas: + Node: + type: object + properties: + child: { $ref: '#/components/schemas/Node' } +"#; + let ir = normalize(document); + assert!(ir.reference_graph.edges.iter().any(|edge| edge.recursive)); + assert!(ir.unresolved_references.is_empty()); + assert!( + ir.operations[0] + .findings + .iter() + .all(|finding| finding.code != "reference_graph_limit") + ); +} + +#[test] +fn merges_compatible_all_of_and_blocks_conflicts_without_first_branch_loss() { + let compatible = r#" +openapi: 3.1.0 +info: { title: AllOf } +servers: [{ url: https://api.example.test }] +paths: + /items: + post: + operationId: createItem + requestBody: + content: + application/json: + schema: + allOf: + - { type: object, required: [id], properties: { id: { type: string } } } + - { type: object, required: [name], properties: { name: { type: string } } } + responses: { '204': { description: ok } } +"#; + let ir = normalize(compatible); + assert!(matches!( + ir.operations[0].request_body_schema.as_ref().map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Object { properties, required }) + if properties.len() == 2 && required == &vec!["id".to_owned(), "name".to_owned()] + )); + + let conflict = compatible.replace("{ name: { type: string } }", "{ id: { type: integer } }"); + let ir = normalize(&conflict); + assert!( + ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "all_of_conflict") + ); +} + +#[test] +fn preserves_one_of_discriminator_and_projects_all_alternatives() { + let document = r#" +openapi: 3.1.0 +info: { title: Alternatives } +servers: [{ url: https://api.example.test }] +paths: + /events: + post: + operationId: createEvent + requestBody: + content: + application/json: + schema: + discriminator: + propertyName: kind + mapping: { text: '#/components/schemas/Text' } + oneOf: + - { type: object, properties: { text: { type: string } } } + - { type: object, properties: { count: { type: integer } } } + responses: { '204': { description: ok } } +"#; + let ir = normalize(document); + let schema = ir.operations[0].request_body_schema.as_ref().unwrap(); + assert_eq!(schema.discriminator.as_ref().unwrap().property_name, "kind"); + assert!(matches!( + &schema.kind, + NormalizedSchemaKind::Composition { operator, variants } + if operator == "oneOf" && variants.len() == 2 + )); + let preview = preview_from_ir(&ir); + let candidate = &preview.groups[0].operations[0]; + assert_eq!( + candidate + .draft + .input_schema + .fields + .get("body") + .unwrap() + .variants + .len(), + 2 + ); + + let any_of = document + .replace("discriminator:\n propertyName: kind\n mapping: { text: '#/components/schemas/Text' }\n oneOf:", "anyOf:"); + let ir = normalize(&any_of); + assert!(matches!( + &ir.operations[0].request_body_schema.as_ref().unwrap().kind, + NormalizedSchemaKind::Composition { operator, variants } + if operator == "anyOf" && variants.len() == 2 + )); + let preview = preview_from_ir(&ir); + assert_eq!( + preview.groups[0].operations[0].draft.input_schema.fields["body"] + .variants + .len(), + 2 + ); +} + +#[test] +fn oas_31_applies_ref_siblings_while_oas_30_ignores_them() { + let template = |version: &str| { + format!( + r#" +openapi: {version} +info: {{ title: Siblings }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: + application/json: + schema: + $ref: '#/components/schemas/Item' + description: sibling-description +components: + schemas: + Item: {{ type: string, description: target-description }} +"# + ) + }; + let v30 = normalize(&template("3.0.3")); + let v31 = normalize(&template("3.1.0")); + assert_eq!( + v30.operations[0] + .response_schema + .as_ref() + .unwrap() + .description + .as_deref(), + Some("target-description") + ); + assert_eq!( + v31.operations[0] + .response_schema + .as_ref() + .unwrap() + .description + .as_deref(), + Some("sibling-description") + ); +} + +#[test] +fn invalid_pointer_and_type_mismatch_are_exact_blockers_without_panic() { + let document = r#" +openapi: 3.1.0 +info: { title: Invalid targets } +servers: [{ url: https://api.example.test }] +paths: + /missing: + get: + operationId: missing + responses: + '200': { description: ok, content: { application/json: { schema: { $ref: '#not-a-pointer' } } } } + /scalar: + get: + operationId: scalar + responses: + '200': { description: ok, content: { application/json: { schema: { $ref: '#/info/title' } } } } +"#; + let ir = normalize(document); + let codes = ir + .operations + .iter() + .flat_map(|operation| { + operation + .findings + .iter() + .map(move |finding| (operation.path.as_str(), finding.code.as_str())) + }) + .collect::>(); + assert!(codes.contains(&("/missing", "reference_target_missing"))); + assert!(codes.contains(&("/scalar", "reference_type_mismatch"))); +} + +#[test] +fn reference_depth_and_expanded_node_limits_fail_closed() { + let document = r#" +openapi: 3.1.0 +info: { title: Bounded } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': { description: ok, content: { application/json: { schema: { $ref: '#/components/schemas/A' } } } } +components: + schemas: + A: { $ref: '#/components/schemas/B' } + B: { $ref: '#/components/schemas/C' } + C: { type: object, properties: { id: { type: string } } } +"#; + let config = NormalizationConfig { + max_reference_depth: 1, + ..NormalizationConfig::default() + }; + let ir = normalize_verified_document(document, digest('a'), &config).unwrap(); + assert!( + ir.operations + .iter() + .flat_map(|operation| &operation.findings) + .any(|finding| finding.code == "reference_graph_limit") + ); + + let config = NormalizationConfig { + max_expanded_nodes: 8, + ..NormalizationConfig::default() + }; + let ir = normalize_verified_document(document, digest('a'), &config).unwrap(); + assert!( + ir.findings + .iter() + .chain( + ir.operations + .iter() + .flat_map(|operation| &operation.findings) + ) + .any(|finding| finding.code == "reference_graph_limit") + ); +} + +#[test] +fn invalid_discriminator_and_multiple_composition_operators_are_blockers() { + let document = r#" +openapi: 3.1.0 +info: { title: Unsupported composition } +servers: [{ url: https://api.example.test }] +paths: + /events: + post: + operationId: createEvent + requestBody: + content: + application/json: + schema: + discriminator: { mapping: { bad: 42 } } + oneOf: [{ type: string }, { type: integer }] + anyOf: [{ type: boolean }, { type: string }] + responses: { '204': { description: ok } } +"#; + let ir = normalize(document); + let codes = ir.operations[0] + .findings + .iter() + .map(|finding| finding.code.as_str()) + .collect::>(); + assert!(codes.contains("unsupported_discriminator")); + assert!(codes.contains("unsupported_composition")); +} + +#[test] +fn reference_uri_scan_applies_external_size_depth_and_alias_limits_before_traversal() { + let config = NormalizationConfig { + max_bytes: 8, + max_external_document_bytes: 8, + ..NormalizationConfig::default() + }; + assert_eq!( + reference_uris("external-document", &config), + Err(crank_import::rest::ImportParseError::LimitExceeded) + ); + + let config = NormalizationConfig { + max_bytes: 8, + max_external_document_bytes: 32, + ..NormalizationConfig::default() + }; + assert_eq!( + external_reference_uris("external-document", &config), + Ok(Vec::new()) + ); + + let config = NormalizationConfig { + max_depth: 1, + ..NormalizationConfig::default() + }; + assert_eq!( + reference_uris("a: { b: { $ref: '#/x' } }", &config), + Err(crank_import::rest::ImportParseError::LimitExceeded) + ); + + let aliases = format!( + "items: [{}]", + std::iter::repeat_n("*a", 129) + .collect::>() + .join(", ") + ); + assert_eq!( + reference_uris(&aliases, &NormalizationConfig::default()), + Err(crank_import::rest::ImportParseError::LimitExceeded) + ); +} + +#[test] +fn resolves_relative_references_using_rfc3986_paths_and_decoded_fragments() { + let document = r#" +openapi: 3.1.0 +info: { title: Relative refs } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: 'HTTPS://SCHEMAS.EXAMPLE.TEST:443/a/b/root.yaml#/Item' } } } +"#; + let snapshots = vec![ + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/a/b/root.yaml".to_owned(), + digest: digest('b'), + document: r#" +Item: + type: object + properties: + dot: { $ref: './child.yaml#/Value' } + parent: { $ref: '../common.yaml#/Value' } + absolute: { $ref: '/shared.yaml#/Value' } +"# + .to_owned(), + }, + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/a/b/child.yaml".to_owned(), + digest: digest('c'), + document: "Value: { type: string }".to_owned(), + }, + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/a/common.yaml".to_owned(), + digest: digest('d'), + document: "Value: { type: integer }".to_owned(), + }, + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/shared.yaml".to_owned(), + digest: digest('e'), + document: "Value: { $ref: '#/a%7E1b' }\na/b: { type: boolean }".to_owned(), + }, + ]; + let ir = normalize_verified_bundle( + document, + digest('a'), + &snapshots, + &NormalizationConfig::default(), + ) + .unwrap(); + let Some(NormalizedSchemaKind::Object { properties, .. }) = ir.operations[0] + .response_schema + .as_ref() + .map(|schema| &schema.kind) + else { + panic!("response should be an expanded object schema"); + }; + assert_eq!(properties.len(), 3); + assert_eq!(ir.reference_graph.edges.len(), 5); + assert!(ir.findings.is_empty()); +} + +#[test] +fn malformed_percent_encoded_fragment_is_a_blocker() { + let document = r#" +openapi: 3.1.0 +info: { title: Malformed reference } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': { description: ok, content: { application/json: { schema: { $ref: '#/components/%ZZ' } } } } +"#; + let ir = normalize(document); + assert!( + ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "reference_uri_malformed") + ); +} + +#[test] +fn does_not_follow_references_inside_literal_payloads() { + let document = r#" +openapi: 3.1.0 +info: { title: Literal payloads } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: + application/json: + schema: + type: object + example: { $ref: '#/components/schemas/Missing' } + examples: { sample: { value: { $ref: '#/components/schemas/Missing' } } } + default: { $ref: '#/components/schemas/Missing' } + enum: [{ $ref: '#/components/schemas/Missing' }] + const: { $ref: '#/components/schemas/Missing' } + x-fixture: { $ref: '#/components/schemas/Missing' } + properties: { known: { $ref: '#/components/schemas/Known' } } +components: + schemas: + Known: { type: string } +"#; + assert_eq!( + reference_uris(document, &NormalizationConfig::default()), + Ok(vec!["#/components/schemas/Known".to_owned()]) + ); + let ir = normalize(document); + assert_eq!(ir.reference_graph.edges.len(), 1); + assert!( + ir.operations[0] + .findings + .iter() + .all(|finding| finding.code != "reference_target_missing") + ); +} + +#[test] +fn primary_oas31_applies_ref_siblings_in_external_snapshots_without_openapi_field() { + let source = |version: &str| { + format!( + r#" +openapi: {version} +info: {{ title: External siblings }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: listItems + responses: + '200': {{ description: ok, content: {{ application/json: {{ schema: {{ $ref: 'https://schemas.example.test/root.yaml#/Item' }} }} }} }} +"# + ) + }; + let snapshots = vec![ + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/root.yaml".to_owned(), + digest: digest('b'), + document: "Item: { $ref: 'child.yaml#/Base', description: sibling }".to_owned(), + }, + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/child.yaml".to_owned(), + digest: digest('c'), + document: "Base: { type: string, description: target }".to_owned(), + }, + ]; + let v31 = normalize_verified_bundle( + &source("3.1.0"), + digest('a'), + &snapshots, + &NormalizationConfig::default(), + ) + .unwrap(); + let v30 = normalize_verified_bundle( + &source("3.0.3"), + digest('a'), + &snapshots, + &NormalizationConfig::default(), + ) + .unwrap(); + assert_eq!( + v31.operations[0] + .response_schema + .as_ref() + .unwrap() + .description + .as_deref(), + Some("sibling") + ); + assert_eq!( + v30.operations[0] + .response_schema + .as_ref() + .unwrap() + .description + .as_deref(), + Some("target") + ); +} + +#[test] +fn all_of_intersects_constraints_and_nested_properties_without_losing_conflicts() { + let source = r#" +openapi: 3.1.0 +info: { title: allOf intersections } +servers: [{ url: https://api.example.test }] +paths: + /items: + post: + operationId: createItem + requestBody: + content: + application/json: + schema: + allOf: + - type: object + minimum: 1 + maximum: 10 + minLength: 2 + maxLength: 12 + properties: { nested: { type: object, properties: { left: { type: string } } } } + - type: object + minimum: 4 + maximum: 8 + minLength: 5 + maxLength: 9 + properties: { nested: { type: object, properties: { right: { type: integer } } } } + responses: { '204': { description: ok } } +"#; + let ir = normalize(source); + let schema = ir.operations[0].request_body_schema.as_ref().unwrap(); + assert_eq!(schema.constraints.minimum, Some(4.0)); + assert_eq!(schema.constraints.maximum, Some(8.0)); + assert_eq!(schema.constraints.min_length, Some(5)); + assert_eq!(schema.constraints.max_length, Some(9)); + let NormalizedSchemaKind::Object { properties, .. } = &schema.kind else { + panic!("merged schema should remain an object"); + }; + let NormalizedSchemaKind::Object { properties, .. } = &properties["nested"].kind else { + panic!("nested property should remain an object"); + }; + assert!(properties.contains_key("left") && properties.contains_key("right")); + + let conflicting = source.replace("maximum: 8", "maximum: 3"); + let ir = normalize(&conflicting); + assert!( + ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "all_of_conflict") + ); +} + +#[test] +fn non_array_all_of_is_preserved_and_reported() { + let document = r#" +openapi: 3.1.0 +info: { title: Invalid allOf } +servers: [{ url: https://api.example.test }] +paths: + /items: + post: + operationId: createItem + requestBody: + content: { application/json: { schema: { allOf: { type: string } } } } + responses: { '204': { description: ok } } +"#; + let ir = normalize(document); + assert!( + ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "unsupported_composition") + ); + assert!(matches!( + ir.operations[0] + .request_body_schema + .as_ref() + .map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Unknown) + )); +} diff --git a/crates/crank-import/tests/unit.rs b/crates/crank-import/tests/unit.rs index 840d9b0..2767bc9 100644 --- a/crates/crank-import/tests/unit.rs +++ b/crates/crank-import/tests/unit.rs @@ -122,7 +122,7 @@ paths: } #[test] - fn keeps_references_typed_without_resolving_them() { + fn resolves_local_references_into_typed_graph() { let ir = normalize_document(SWAGGER2, &NormalizationConfig::default()).unwrap(); let operation = &ir.operations[0]; assert!(matches!( @@ -130,13 +130,15 @@ paths: .response_schema .as_ref() .map(|schema| &schema.kind), - Some(crank_import::rest::NormalizedSchemaKind::Reference { .. }) + Some(crank_import::rest::NormalizedSchemaKind::Object { .. }) )); + assert!(ir.unresolved_references.is_empty()); + assert!(!ir.reference_graph.edges.is_empty()); assert!( operation .findings .iter() - .any(|finding| finding.code == "unresolved_reference") + .all(|finding| finding.code != "unresolved_reference") ); } diff --git a/crates/crank-registry/src/postgres/import_job.rs b/crates/crank-registry/src/postgres/import_job.rs index e64a6a9..7e1faf4 100644 --- a/crates/crank-registry/src/postgres/import_job.rs +++ b/crates/crank-registry/src/postgres/import_job.rs @@ -4,6 +4,7 @@ use crank_artifacts::ArtifactRef; const APPLICATION_RESULT_KEY: &str = "_crank_application_result"; const IMPORT_JOB_CLEANUP_BATCH: u32 = 128; +const MAX_IMPORT_JOB_DOCUMENTS: usize = 32; const DANGLING_OPENAPI_SOURCE_GRACE: time::Duration = time::Duration::minutes(5); impl PostgresRegistry { @@ -12,6 +13,7 @@ impl PostgresRegistry { request: CreateImportJobRequest<'_>, ) -> Result<(), RegistryError> { validate_source_envelope(request.source, request.preview_payload)?; + validate_dependency_envelopes(request.preview_payload)?; let mut transaction = self.pool.begin().await?; sqlx::query( "insert into import_jobs ( @@ -81,6 +83,26 @@ impl PostgresRegistry { &self, request: FinishImportJobRequest<'_>, ) -> Result<(), RegistryError> { + let mut transaction = self.pool.begin().await?; + let row = sqlx::query( + "select workspace_id, status, preview_payload + from import_jobs + where id = $1 + for update", + ) + .bind(request.id.as_str()) + .fetch_optional(&mut *transaction) + .await? + .ok_or_else(|| RegistryError::ImportJobNotFound { + job_id: request.id.as_str().to_owned(), + })?; + let current_status = + deserialize_enum_text::(row.try_get("status")?, "status")?; + if request.status == ImportJobStatus::Failed && current_status == ImportJobStatus::Completed + { + transaction.commit().await?; + return Ok(()); + } let result = sqlx::query( "update import_jobs set status = $2, @@ -94,7 +116,7 @@ impl PostgresRegistry { .bind(request.created_operation_ids) .bind(request.error_text) .bind(request.finished_at) - .execute(&self.pool) + .execute(&mut *transaction) .await?; if result.rows_affected() == 0 { @@ -103,6 +125,25 @@ impl PostgresRegistry { }); } + if request.status == ImportJobStatus::Failed { + let workspace_id = WorkspaceId::new(row.try_get::("workspace_id")?); + let payload = row.try_get::("preview_payload")?; + // Failure finalization must itself be fail-safe. The strict + // envelope parsers are used before Apply; cleanup only needs the + // bounded source identities and must not roll back the terminal + // state because some other payload field was corrupted. + for source_id in cleanup_source_ids(&payload) { + let _ = detach_source_in_transaction( + &mut transaction, + &workspace_id, + &source_id, + *request.finished_at, + ) + .await?; + } + } + transaction.commit().await?; + Ok(()) } @@ -120,24 +161,18 @@ impl PostgresRegistry { } Err(error) => { tx.rollback().await?; - let error_text = error.to_string(); - let _ = sqlx::query( - "update import_jobs - set status = $3, - error_text = $4, - finished_at = $5::timestamptz - where id = $1 - and workspace_id = $2 - and status <> $6", - ) - .bind(request.id.as_str()) - .bind(request.workspace_id.as_str()) - .bind(serialize_enum_text(&ImportJobStatus::Failed, "status")?) - .bind(error_text) - .bind(request.finished_at) - .bind(serialize_enum_text(&ImportJobStatus::Completed, "status")?) - .execute(&self.pool) - .await; + if !matches!(error, RegistryError::ImportJobAlreadyApplied { .. }) { + let empty = serde_json::json!([]); + let _ = self + .finish_import_job(FinishImportJobRequest { + id: request.id, + status: ImportJobStatus::Failed, + created_operation_ids: &empty, + error_text: Some("import_apply_failed"), + finished_at: request.finished_at, + }) + .await; + } Err(error) } } @@ -177,16 +212,12 @@ impl PostgresRegistry { // The job itself is expired regardless of whether a legacy or // corrupt payload can be decoded. Do not let one bad row roll // back cleanup for every tenant. - if let Ok(Some(source)) = source_from_payload(&payload) - && detach_source_in_transaction( - &mut transaction, - &workspace_id, - &source.source_id, - now, - ) - .await? - { - detached_sources += 1; + for source_id in cleanup_source_ids(&payload) { + if detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now) + .await? + { + detached_sources += 1; + } } } let deleted = if expired_ids.is_empty() { @@ -214,7 +245,14 @@ impl PostgresRegistry { select 1 from import_jobs j where j.workspace_id = s.workspace_id - and j.preview_payload -> 'source' ->> 'source_id' = s.source_id + and ( + j.preview_payload -> 'source' ->> 'source_id' = s.source_id + or exists ( + select 1 + from jsonb_array_elements(coalesce(j.preview_payload -> 'dependencies', '[]'::jsonb)) d + where d ->> 'source_id' = s.source_id + ) + ) ) order by s.created_at, s.workspace_id, s.source_id limit $2 @@ -283,6 +321,7 @@ async fn apply_import_job_transaction( source_from_payload(&preview_payload)?.ok_or(RegistryError::InvalidArtifactSource { field: "import_job_source", })?; + let dependencies = dependencies_from_payload(&preview_payload)?; if status == ImportJobStatus::Completed && let Some(result) = stored_application_result(&preview_payload)? @@ -323,6 +362,31 @@ async fn apply_import_job_transaction( if !source_is_current { return Err(RegistryError::SourceUnavailable); } + for dependency in &dependencies { + let recorded = sqlx::query( + "select s.lifecycle, b.artifact_ref + from artifact_sources s + join artifact_blobs b on b.digest = s.blob_digest + where s.workspace_id = $1 and s.source_id = $2 + for update", + ) + .bind(request.workspace_id.as_str()) + .bind(dependency.source_id.as_str()) + .fetch_optional(&mut **tx) + .await?; + let current = recorded + .as_ref() + .map(|row| { + let lifecycle = row.try_get::("lifecycle").ok(); + let artifact_ref = row.try_get::("artifact_ref").ok(); + lifecycle.as_deref() == Some("active") + && artifact_ref.as_deref() == Some(dependency.digest.as_str()) + }) + .unwrap_or(false); + if !current { + return Err(RegistryError::SourceUnavailable); + } + } let mut result = ImportJobApplyResult { application_key: request.application_key.to_owned(), @@ -408,6 +472,15 @@ async fn apply_import_job_transaction( *request.finished_at, ) .await?; + for dependency in dependencies { + let _ = detach_source_in_transaction( + tx, + request.workspace_id, + &dependency.source_id, + *request.finished_at, + ) + .await?; + } Ok(result) } @@ -433,6 +506,11 @@ fn validate_source_envelope( Ok(()) } +fn validate_dependency_envelopes(preview_payload: &Value) -> Result<(), RegistryError> { + dependencies_from_payload(preview_payload)?; + Ok(()) +} + fn source_from_payload(payload: &Value) -> Result, RegistryError> { let Some(source) = payload.get("source") else { return Ok(None); @@ -466,6 +544,73 @@ fn source_from_payload(payload: &Value) -> Result Result, RegistryError> { + let Some(value) = payload.get("dependencies") else { + return Ok(Vec::new()); + }; + let items = value + .as_array() + .filter(|items| items.len() <= MAX_IMPORT_JOB_DOCUMENTS) + .ok_or(RegistryError::InvalidArtifactSource { + field: "import_job_dependencies", + })?; + let mut dependencies = Vec::with_capacity(items.len()); + let mut identities = std::collections::BTreeSet::new(); + for item in items { + let source_id = item + .get("source_id") + .and_then(Value::as_str) + .filter(|value| value.len() <= 132) + .ok_or(RegistryError::InvalidArtifactSource { + field: "import_job_dependencies", + })?; + let digest = item.get("digest").and_then(Value::as_str).ok_or( + RegistryError::InvalidArtifactSource { + field: "import_job_dependencies", + }, + )?; + let digest = + ArtifactRef::parse(digest).map_err(|_| RegistryError::InvalidArtifactSource { + field: "import_job_dependencies", + })?; + if !identities.insert((source_id.to_owned(), digest.as_str().to_owned())) { + return Err(RegistryError::InvalidArtifactSource { + field: "import_job_dependencies", + }); + } + dependencies.push(ImportJobSourceEnvelope { + source_id: ArtifactSourceId::new(source_id), + digest, + }); + } + Ok(dependencies) +} + +fn cleanup_source_ids(payload: &Value) -> Vec { + let mut identities = std::collections::BTreeSet::new(); + if let Some(source_id) = payload + .pointer("/source/source_id") + .and_then(Value::as_str) + .filter(|value| value.len() <= 132) + { + identities.insert(source_id.to_owned()); + } + if let Some(dependencies) = payload.get("dependencies").and_then(Value::as_array) { + for source_id in dependencies + .iter() + .take(MAX_IMPORT_JOB_DOCUMENTS) + .filter_map(|item| item.get("source_id")) + .filter_map(Value::as_str) + .filter(|value| value.len() <= 132) + { + identities.insert(source_id.to_owned()); + } + } + identities.into_iter().map(ArtifactSourceId::new).collect() +} + async fn detach_source_in_transaction( transaction: &mut Transaction<'_, Postgres>, workspace_id: &WorkspaceId, diff --git a/crates/crank-runtime/src/lib.rs b/crates/crank-runtime/src/lib.rs index db8f6f9..921ae72 100644 --- a/crates/crank-runtime/src/lib.rs +++ b/crates/crank-runtime/src/lib.rs @@ -25,7 +25,9 @@ pub use cache::{ pub use cache_factory::{ BuiltinCacheBackendFactory, CacheBackendFactory, SharedCacheBackendFactory, }; -pub use crank_adapter_rest::OutboundHttpPolicy; +pub use crank_adapter_rest::{ + ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy, +}; pub use error::RuntimeError; pub use execution_failure::{normalize_runtime_error, normalize_runtime_error_for_operation}; pub use execution_request::{ diff --git a/deploy/community/.env.example b/deploy/community/.env.example index 867c52f..2a5f89a 100644 --- a/deploy/community/.env.example +++ b/deploy/community/.env.example @@ -25,6 +25,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS= CRANK_OUTBOUND_DENIED_HOSTS= CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304 CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304 +CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES= +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144 +CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000 CRANK_ENVIRONMENT=production CRANK_LOG_LEVEL= CRANK_SENTRY_DSN= diff --git a/deploy/community/.env.images.example b/deploy/community/.env.images.example index e217795..56f6fe3 100644 --- a/deploy/community/.env.images.example +++ b/deploy/community/.env.images.example @@ -27,6 +27,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS= CRANK_OUTBOUND_DENIED_HOSTS= CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304 CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304 +CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES= +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144 +CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000 +CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000 CRANK_ENVIRONMENT=production CRANK_LOG_LEVEL= CRANK_SENTRY_DSN= diff --git a/deploy/community/docker-compose.images.yml b/deploy/community/docker-compose.images.yml index 30a4690..9b2f536 100644 --- a/deploy/community/docker-compose.images.yml +++ b/deploy/community/docker-compose.images.yml @@ -127,6 +127,12 @@ services: CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-} CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304} CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304} + CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144} + CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000} volumes: - artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage} ports: diff --git a/deploy/community/docker-compose.yml b/deploy/community/docker-compose.yml index 484c70b..3711408 100644 --- a/deploy/community/docker-compose.yml +++ b/deploy/community/docker-compose.yml @@ -130,6 +130,12 @@ services: CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-} CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304} CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304} + CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144} + CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000} volumes: - artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage} ports: diff --git a/docker-compose.yml b/docker-compose.yml index c7d3e25..1d0d2d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,6 +109,12 @@ services: CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-} CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304} CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304} + CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144} + CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000} + CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000} depends_on: migrate: condition: service_completed_successfully diff --git a/docs/runtime-config.md b/docs/runtime-config.md index 86234d2..1fbd817 100644 --- a/docs/runtime-config.md +++ b/docs/runtime-config.md @@ -27,6 +27,12 @@ Crank настраивается через переменные окружен | `CRANK_OUTBOUND_DENIED_HOSTS` | `outbound.denied_hosts` | `Shared` | `host_list/-` | `` | `-` | `Internal` | `Effective` | | `CRANK_OUTBOUND_MAX_REQUEST_BYTES` | `outbound.max_request_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` | | `CRANK_OUTBOUND_MAX_RESPONSE_BYTES` | `outbound.max_response_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` | +| `CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES` | `import.external_references.allowed_url_prefixes` | `AdminApi` | `url_prefix_list/-` | `` | `-` | `Internal` | `Effective` | +| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH` | `import.external_references.max_depth` | `AdminApi` | `u32/edges` | `8` | `1..=32` | `Public` | `Effective` | +| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS` | `import.external_references.max_documents` | `AdminApi` | `u32/documents` | `32` | `1..=32` | `Public` | `Effective` | +| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES` | `import.external_references.max_fetch_bytes` | `AdminApi` | `u64/bytes` | `262144` | `1..=4194304` | `Public` | `Effective` | +| `CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS` | `import.external_references.fetch_timeout_ms` | `AdminApi` | `u64/milliseconds` | `10000` | `1..=300000` | `Public` | `Effective` | +| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES` | `import.external_references.max_expanded_nodes` | `AdminApi` | `u32/nodes` | `10000` | `1..=100000` | `Public` | `Effective` | | `CRANK_ENVIRONMENT` | `observability.environment` | `Shared` | `label/-` | `development` | `-` | `Public` | `Effective` | | `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` | `-` | `Public` | `Effective` | | `CRANK_SENTRY_DSN` | `observability.sentry_dsn` | `Shared` | `url/-` | `blank` | `-` | `Secret` | `Effective` | diff --git a/docs/schemas/runtime-config.schema.json b/docs/schemas/runtime-config.schema.json index 1965bd5..5c4ed97 100644 --- a/docs/schemas/runtime-config.schema.json +++ b/docs/schemas/runtime-config.schema.json @@ -330,6 +330,99 @@ "compatibility": null, "rules": [] }, + { + "semantic_path": "import.external_references.allowed_url_prefixes", + "env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES", + "process": "admin_api", + "value_type": "url_prefix_list", + "unit": null, + "default": "", + "required": false, + "minimum": null, + "maximum": null, + "sensitivity": "internal", + "mode": "effective", + "compatibility": null, + "rules": [ + "empty list disables external OpenAPI reference fetching", + "each prefix must be canonical HTTP(S) without userinfo, query, or fragment" + ] + }, + { + "semantic_path": "import.external_references.max_depth", + "env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH", + "process": "admin_api", + "value_type": "u32", + "unit": "edges", + "default": "8", + "required": false, + "minimum": 1, + "maximum": 32, + "sensitivity": "public", + "mode": "effective", + "compatibility": null, + "rules": [] + }, + { + "semantic_path": "import.external_references.max_documents", + "env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS", + "process": "admin_api", + "value_type": "u32", + "unit": "documents", + "default": "32", + "required": false, + "minimum": 1, + "maximum": 32, + "sensitivity": "public", + "mode": "effective", + "compatibility": null, + "rules": [] + }, + { + "semantic_path": "import.external_references.max_fetch_bytes", + "env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES", + "process": "admin_api", + "value_type": "u64", + "unit": "bytes", + "default": "262144", + "required": false, + "minimum": 1, + "maximum": 4194304, + "sensitivity": "public", + "mode": "effective", + "compatibility": null, + "rules": [] + }, + { + "semantic_path": "import.external_references.fetch_timeout_ms", + "env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS", + "process": "admin_api", + "value_type": "u64", + "unit": "milliseconds", + "default": "10000", + "required": false, + "minimum": 1, + "maximum": 300000, + "sensitivity": "public", + "mode": "effective", + "compatibility": null, + "rules": [] + }, + { + "semantic_path": "import.external_references.max_expanded_nodes", + "env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES", + "process": "admin_api", + "value_type": "u32", + "unit": "nodes", + "default": "10000", + "required": false, + "minimum": 1, + "maximum": 100000, + "sensitivity": "public", + "mode": "effective", + "compatibility": null, + "rules": [] + }, { "semantic_path": "observability.environment", "env_name": "CRANK_ENVIRONMENT",