feat(import): resolve references and schema composition

This commit is contained in:
2026-08-29 08:54:26 +03:00
parent 6c2a3712d8
commit 55209a9bbc
46 changed files with 4848 additions and 437 deletions
+161 -10
View File
@@ -25,7 +25,7 @@ use serde_json::Value;
use tracing::{Instrument, Span};
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::{RestAdapterError, RestRequest, RestResponse};
use crate::{ExternalReferenceFetchError, RestAdapterError, RestRequest, RestResponse};
#[derive(Clone, Debug)]
pub struct RestAdapter {
@@ -33,6 +33,18 @@ pub struct RestAdapter {
policy: OutboundHttpPolicy,
}
/// A deliberately separate, GET-only boundary for materializing external
/// OpenAPI documents. It has no execution metrics, tracing propagation, or
/// request construction semantics from [`RestAdapter`].
#[derive(Clone, Debug)]
pub struct ExternalReferenceFetcher {
client: Result<Client, Arc<str>>,
policy: OutboundHttpPolicy,
allowed_url_prefixes: Vec<String>,
max_response_bytes: usize,
timeout: Duration,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OutboundHttpPolicy {
allowed_hosts: Vec<String>,
@@ -56,15 +68,7 @@ impl RestAdapter {
}
pub fn with_policy(policy: OutboundHttpPolicy) -> Self {
let resolver = Arc::new(PolicyDnsResolver {
policy: policy.clone(),
});
let client = Client::builder()
.redirect(redirect::Policy::none())
.no_proxy()
.dns_resolver(resolver)
.build()
.map_err(|error| Arc::<str>::from(error.to_string()));
let client = outbound_client(&policy);
Self { client, policy }
}
@@ -195,6 +199,153 @@ impl RestAdapter {
}
}
impl ExternalReferenceFetcher {
pub fn try_new(
policy: OutboundHttpPolicy,
allowed_url_prefixes: Vec<String>,
max_response_bytes: usize,
timeout: Duration,
) -> Result<Self, ExternalReferenceFetchError> {
if max_response_bytes == 0 || timeout.is_zero() {
return Err(ExternalReferenceFetchError::InvalidConfiguration);
}
let allowed_url_prefixes = allowed_url_prefixes
.into_iter()
.map(|prefix| canonical_external_reference_prefix(&prefix))
.collect::<Result<Vec<_>, _>>()?;
Ok(Self {
client: outbound_client(&policy),
policy,
allowed_url_prefixes,
max_response_bytes,
timeout,
})
}
/// Fetches one document with a bounded, headerless `GET`.
///
/// URL fragments are stripped because they address JSON Pointer targets in
/// the fetched document rather than a network resource.
pub async fn get(&self, url: &str) -> Result<Vec<u8>, ExternalReferenceFetchError> {
if self.allowed_url_prefixes.is_empty() {
return Err(ExternalReferenceFetchError::Disabled);
}
let mut url =
reqwest::Url::parse(url).map_err(|_| ExternalReferenceFetchError::InvalidUrl)?;
url.set_fragment(None);
if !self
.allowed_url_prefixes
.iter()
.any(|prefix| matches_external_reference_prefix(&url, prefix))
{
return Err(ExternalReferenceFetchError::TargetNotAllowed);
}
self.policy
.validate_url(&url)
.map_err(external_policy_error)?;
let client = self
.client
.as_ref()
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
let response = client
.get(url)
.timeout(self.timeout)
.send()
.await
.map_err(external_transport_error)?;
let status = response.status();
if status.is_redirection() {
return Err(ExternalReferenceFetchError::RedirectNotAllowed);
}
if !status.is_success() {
return Err(ExternalReferenceFetchError::UnexpectedStatus {
status: status.as_u16(),
});
}
read_external_response_bytes(response, self.max_response_bytes).await
}
}
fn outbound_client(policy: &OutboundHttpPolicy) -> Result<Client, Arc<str>> {
let resolver = Arc::new(PolicyDnsResolver {
policy: policy.clone(),
});
Client::builder()
.redirect(redirect::Policy::none())
.no_proxy()
.dns_resolver(resolver)
.build()
.map_err(|error| Arc::<str>::from(error.to_string()))
}
fn canonical_external_reference_prefix(
prefix: &str,
) -> Result<String, ExternalReferenceFetchError> {
let url = reqwest::Url::parse(prefix)
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
if !matches!(url.scheme(), "http" | "https")
|| url.host_str().is_none()
|| !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(ExternalReferenceFetchError::InvalidConfiguration);
}
Ok(url.to_string())
}
fn matches_external_reference_prefix(url: &reqwest::Url, prefix: &str) -> bool {
let url = url.as_str();
if !url.starts_with(prefix) {
return false;
}
let Some(next) = url.as_bytes().get(prefix.len()) else {
return true;
};
prefix.ends_with('/') || matches!(next, b'/' | b'?')
}
fn external_policy_error(error: RestAdapterError) -> ExternalReferenceFetchError {
match error {
RestAdapterError::TargetNotAllowed { .. } => ExternalReferenceFetchError::TargetNotAllowed,
_ => ExternalReferenceFetchError::InvalidConfiguration,
}
}
fn external_transport_error(error: reqwest::Error) -> ExternalReferenceFetchError {
ExternalReferenceFetchError::Transport {
timeout: error.is_timeout(),
connect: error.is_connect(),
}
}
async fn read_external_response_bytes(
response: reqwest::Response,
max_response_bytes: usize,
) -> Result<Vec<u8>, ExternalReferenceFetchError> {
if response
.content_length()
.is_some_and(|length| length > max_response_bytes as u64)
{
return Err(ExternalReferenceFetchError::ResponseTooLarge {
limit_bytes: max_response_bytes,
});
}
let mut stream = response.bytes_stream();
let mut bytes = Vec::new();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(external_transport_error)?;
if bytes.len().saturating_add(chunk.len()) > max_response_bytes {
return Err(ExternalReferenceFetchError::ResponseTooLarge {
limit_bytes: max_response_bytes,
});
}
bytes.extend_from_slice(&chunk);
}
Ok(bytes)
}
fn upstream_outcome(error: &RestAdapterError) -> UpstreamOutcome {
match error {
RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
+20
View File
@@ -1,6 +1,26 @@
use serde_json::Value;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ExternalReferenceFetchError {
#[error("external references are disabled")]
Disabled,
#[error("external reference URL is invalid")]
InvalidUrl,
#[error("external reference target is not allowed")]
TargetNotAllowed,
#[error("external reference redirects are not allowed")]
RedirectNotAllowed,
#[error("external reference response exceeds the configured limit of {limit_bytes} bytes")]
ResponseTooLarge { limit_bytes: usize },
#[error("external reference endpoint returned status {status}")]
UnexpectedStatus { status: u16 },
#[error("external reference request failed")]
Transport { timeout: bool, connect: bool },
#[error("external reference fetch configuration is invalid")]
InvalidConfiguration,
}
#[derive(Debug, Error)]
pub enum RestAdapterError {
#[error("invalid base url: {url}")]
+2 -2
View File
@@ -8,8 +8,8 @@ use crank_core::{
ProtocolAdapterError, RestTarget, RuntimeRequestContext, Target,
};
pub use client::{OutboundHttpPolicy, RestAdapter};
pub use error::RestAdapterError;
pub use client::{ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter};
pub use error::{ExternalReferenceFetchError, RestAdapterError};
pub use model::{RestRequest, RestResponse};
#[async_trait]
@@ -1,4 +1,5 @@
mod integration {
mod client;
mod external_reference_fetcher;
mod outbound_security;
}
@@ -0,0 +1,151 @@
use std::{
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{Router, http::StatusCode, response::Redirect, routing::get};
use crank_adapter_rest::{
ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy,
};
use tokio::net::TcpListener;
#[tokio::test]
async fn external_references_are_default_off_before_any_request() {
let requests = Arc::new(AtomicUsize::new(0));
let base_url = spawn_server(Arc::clone(&requests)).await;
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
Vec::new(),
1024,
Duration::from_secs(1),
)
.unwrap();
let error = fetcher
.get(&format!("{base_url}/document"))
.await
.unwrap_err();
assert!(matches!(error, ExternalReferenceFetchError::Disabled));
assert_eq!(requests.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn fetcher_uses_prefix_and_actual_address_policy_then_returns_bounded_bytes() {
let base_url = spawn_server(Arc::new(AtomicUsize::new(0))).await;
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
vec![base_url.clone()],
8,
Duration::from_secs(1),
)
.unwrap();
assert_eq!(
fetcher
.get(&format!("{base_url}/document#/components/schemas/A"))
.await
.unwrap(),
b"openapi".to_vec()
);
let error = fetcher.get(&format!("{base_url}/large")).await.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::ResponseTooLarge { limit_bytes: 8 }
));
let private_without_explicit_outbound_allow = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::default(),
vec![base_url.clone()],
1024,
Duration::from_secs(1),
)
.unwrap();
let error = private_without_explicit_outbound_allow
.get(&format!("{base_url}/document"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::TargetNotAllowed
));
let exact_path_fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
vec![format!("{base_url}/document")],
1024,
Duration::from_secs(1),
)
.unwrap();
let error = exact_path_fetcher
.get(&format!("{base_url}/document-unrelated"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::TargetNotAllowed
));
}
#[tokio::test]
async fn fetcher_rejects_redirects_and_userinfo_without_exposing_the_url() {
let requests = Arc::new(AtomicUsize::new(0));
let base_url = spawn_server(Arc::clone(&requests)).await;
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
vec![base_url.clone()],
1024,
Duration::from_secs(1),
)
.unwrap();
let error = fetcher
.get(&format!("{base_url}/redirect"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::RedirectNotAllowed
));
let userinfo_url = base_url.replacen("http://", "http://user:credential@", 1);
let error = fetcher
.get(&format!("{userinfo_url}/document"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::TargetNotAllowed
));
let rendered = format!("{error:?} {error}");
assert!(!rendered.contains("credential"));
assert_eq!(requests.load(Ordering::SeqCst), 0);
}
async fn spawn_server(requests: Arc<AtomicUsize>) -> String {
let app = Router::new()
.route(
"/document",
get({
let requests = Arc::clone(&requests);
move || {
let requests = Arc::clone(&requests);
async move {
requests.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, "openapi")
}
}
}),
)
.route("/large", get(|| async { "response too large" }))
.route("/redirect", get(|| async { Redirect::to("/document") }));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{address}")
}
@@ -6,6 +6,7 @@ use std::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{
@@ -14,7 +15,9 @@ use axum::{
http::StatusCode,
routing::{any, post},
};
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
use crank_adapter_rest::{
ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest,
};
use crank_core::{HttpMethod, ProtocolAdapterError, RestTarget};
use serde_json::{Value, json};
use tokio::net::TcpListener;
@@ -94,7 +97,7 @@ async fn proxy_environment_is_ignored_by_default() {
let adapter = RestAdapter::default();
let request = json_request(json!({"payload": "proxy-env-canary"}));
let rest_target = RestTarget {
base_url: target,
base_url: target.clone(),
method: HttpMethod::Post,
path_template: "/capture".to_owned(),
static_headers: BTreeMap::new(),
@@ -102,6 +105,17 @@ async fn proxy_environment_is_ignored_by_default() {
let _ = adapter.execute(&rest_target, &request).await.expect_err(
"unresolvable target should fail locally instead of being sent through proxy env",
);
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::default(),
vec!["http://public.example.test/".to_owned()],
1024,
Duration::from_secs(1),
)
.expect("valid external reference fetcher");
let _ = fetcher
.get(&target)
.await
.expect_err("external reference fetcher must not use proxy environment variables");
return;
}
+16 -2
View File
@@ -1,8 +1,9 @@
use std::fmt;
use crate::{
AdminProcessConfig, CacheSettings, DatabaseSettings, McpProcessConfig, MetricsSettings,
MigratorConfig, ObservabilitySettings, OtlpSettings, OutboundSettings, RuntimeSettings,
AdminProcessConfig, CacheSettings, DatabaseSettings, ExternalReferenceSettings,
McpProcessConfig, MetricsSettings, MigratorConfig, ObservabilitySettings, OtlpSettings,
OutboundSettings, RuntimeSettings,
};
impl fmt::Debug for DatabaseSettings {
@@ -32,6 +33,18 @@ impl fmt::Debug for OutboundSettings {
.finish()
}
}
impl fmt::Debug for ExternalReferenceSettings {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ExternalReferenceSettings")
.field("allowed_url_prefix_count", &self.allowed_url_prefixes.len())
.field("max_depth", &self.max_depth)
.field("max_documents", &self.max_documents)
.field("max_fetch_bytes", &self.max_fetch_bytes)
.field("fetch_timeout_ms", &self.fetch_timeout_ms)
.field("max_expanded_nodes", &self.max_expanded_nodes)
.finish()
}
}
impl fmt::Debug for RuntimeSettings {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RuntimeSettings")
@@ -93,6 +106,7 @@ impl fmt::Debug for AdminProcessConfig {
f.debug_struct("AdminProcessConfig")
.field("database", &self.database)
.field("runtime", &self.runtime)
.field("external_references", &self.external_references)
.field("observability", &self.observability)
.field("storage_root", &"configured")
.field("session_secret", &self.session_secret)
+3 -2
View File
@@ -15,8 +15,9 @@ pub use diagnostic::{ConfigError, Diagnostic, DiagnosticCode};
pub use migrator::{MigratorConfig, parse_migrator};
pub use process::{
AdminProcessConfig, CacheBackend, CacheSettings, DatabaseSettings, DeprecationRecord,
EffectiveConfig, McpProcessConfig, MetricsSettings, ObservabilitySettings, OtlpSettings,
OutboundSettings, PoolSettings, ProcessKind, RateLimitSettings, RuntimeSettings, parse_process,
EffectiveConfig, ExternalReferenceSettings, McpProcessConfig, MetricsSettings,
ObservabilitySettings, OtlpSettings, OutboundSettings, PoolSettings, ProcessKind,
RateLimitSettings, RuntimeSettings, parse_process,
};
pub use schema::{
FieldMode, FieldSpec, ProcessScope, Sensitivity, deployment_field_registry, field_registry,
+22 -6
View File
@@ -12,7 +12,9 @@ use std::{
path::PathBuf,
};
use url::Url;
mod external_references;
mod list_parsers;
pub use external_references::ExternalReferenceSettings;
const MAX_ENV_VALUE_BYTES: usize = 8_192;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ProcessKind {
@@ -127,6 +129,7 @@ pub struct ObservabilitySettings {
pub struct AdminProcessConfig {
pub database: DatabaseSettings,
pub runtime: RuntimeSettings,
pub external_references: ExternalReferenceSettings,
pub observability: ObservabilitySettings,
pub bind_addr: SocketAddr,
pub storage_root: PathBuf,
@@ -154,8 +157,8 @@ pub struct McpProcessConfig {
#[derive(Clone)]
enum Projection {
Admin(AdminProcessConfig),
Mcp(McpProcessConfig),
Admin(Box<AdminProcessConfig>),
Mcp(Box<McpProcessConfig>),
}
#[derive(Clone)]
@@ -790,9 +793,10 @@ pub fn parse_process(
{
parser.push(DiagnosticCode::UnsafeCombination, "admin.exposure.tls");
}
Projection::Admin(AdminProcessConfig {
Projection::Admin(Box::new(AdminProcessConfig {
database,
runtime,
external_references: external_references::parse(&mut parser),
observability,
bind_addr,
storage_root: parser.absolute_path("CRANK_STORAGE_ROOT", "/var/lib/crank/storage"),
@@ -811,7 +815,7 @@ pub fn parse_process(
bootstrap_display_name: parser
.string("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", Some("Crank Owner")),
demo_seed: parser.boolean("CRANK_DEMO_SEED"),
})
}))
}
ProcessKind::McpServer => {
let rps = parser.number("CRANK_MCP_RATE_LIMIT_RPS") as u32;
@@ -819,7 +823,7 @@ pub fn parse_process(
if burst < rps {
parser.push(DiagnosticCode::UnsafeCombination, "mcp.rate_limit.burst");
}
Projection::Mcp(McpProcessConfig {
Projection::Mcp(Box::new(McpProcessConfig {
database,
runtime,
observability,
@@ -829,7 +833,7 @@ pub fn parse_process(
requests_per_second: rps,
burst,
},
})
}))
}
};
@@ -867,6 +871,18 @@ fn fingerprint_parts(kind: ProcessKind, projection: &Projection) -> Vec<String>
format!("session_ttl={}", config.session_ttl_hours),
format!("trusted_proxies={:?}", config.trusted_proxy_ips),
format!("demo={}", config.demo_seed),
format!(
"external_reference_prefixes={}",
config.external_references.allowed_url_prefixes.join(",")
),
format!(
"external_reference_limits={}:{}:{}:{}:{}",
config.external_references.max_depth,
config.external_references.max_documents,
config.external_references.max_fetch_bytes,
config.external_references.fetch_timeout_ms,
config.external_references.max_expanded_nodes,
),
"storage=path-configured".to_owned(),
format!("session_secret={}", config.session_secret.is_configured()),
format!("pepper={}", config.password_pepper.is_configured()),
@@ -0,0 +1,24 @@
#[derive(Clone, Eq, PartialEq)]
pub struct ExternalReferenceSettings {
/// Canonical HTTP(S) URL prefixes which opt an operator into remote `$ref` fetches.
/// An empty list is a deliberate default-deny switch.
pub allowed_url_prefixes: Vec<String>,
pub max_depth: usize,
pub max_documents: usize,
pub max_fetch_bytes: usize,
pub fetch_timeout_ms: u64,
pub max_expanded_nodes: usize,
}
pub(super) fn parse(parser: &mut super::Parser<'_>) -> ExternalReferenceSettings {
ExternalReferenceSettings {
allowed_url_prefixes: parser
.url_prefix_list("CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES"),
max_depth: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH") as usize,
max_documents: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS") as usize,
max_fetch_bytes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES") as usize,
fetch_timeout_ms: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS"),
max_expanded_nodes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES")
as usize,
}
}
@@ -4,6 +4,7 @@ use crate::{
DiagnosticCode,
validation::{parse_host_list, parse_ip_list},
};
use url::Url;
impl super::Parser<'_> {
pub(super) fn host_list(&mut self, name: &'static str) -> Vec<String> {
@@ -33,4 +34,41 @@ impl super::Parser<'_> {
}
parsed.items
}
pub(super) fn url_prefix_list(&mut self, name: &'static str) -> Vec<String> {
let Some(raw) = self.optional(name) else {
return Vec::new();
};
let mut prefixes = Vec::new();
let mut invalid = false;
for item in raw.split(',') {
let item = item.trim();
let Ok(url) = Url::parse(item) else {
invalid = true;
continue;
};
if !matches!(url.scheme(), "http" | "https")
|| url.host_str().is_none()
|| !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
{
invalid = true;
continue;
}
let canonical = url.to_string();
if !prefixes.contains(&canonical) {
prefixes.push(canonical);
}
}
if invalid {
self.push(DiagnosticCode::InvalidType, name);
}
if prefixes.len() > 64 {
self.push(DiagnosticCode::OutOfRange, name);
prefixes.truncate(64);
}
prefixes
}
}
+73 -1
View File
@@ -78,7 +78,7 @@ macro_rules! f {
};
}
static FIELDS: [FieldSpec; 59] = [
static FIELDS: [FieldSpec; 65] = [
FieldSpec {
compatibility: Some("legacy URL form"),
rules: &[
@@ -339,6 +339,78 @@ static FIELDS: [FieldSpec; 59] = [
Some(67108864),
Public
),
FieldSpec {
rules: &[
"empty list disables external OpenAPI reference fetching",
"each prefix must be canonical HTTP(S) without userinfo, query, or fragment",
],
..f!(
"import.external_references.allowed_url_prefixes",
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES",
AdminApi,
"url_prefix_list",
None,
Some(""),
None,
None,
Internal
)
},
f!(
"import.external_references.max_depth",
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH",
AdminApi,
"u32",
Some("edges"),
Some("8"),
Some(1),
Some(32),
Public
),
f!(
"import.external_references.max_documents",
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS",
AdminApi,
"u32",
Some("documents"),
Some("32"),
Some(1),
Some(32),
Public
),
f!(
"import.external_references.max_fetch_bytes",
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES",
AdminApi,
"u64",
Some("bytes"),
Some("262144"),
Some(1),
Some(4194304),
Public
),
f!(
"import.external_references.fetch_timeout_ms",
"CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS",
AdminApi,
"u64",
Some("milliseconds"),
Some("10000"),
Some(1),
Some(300000),
Public
),
f!(
"import.external_references.max_expanded_nodes",
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES",
AdminApi,
"u32",
Some("nodes"),
Some("10000"),
Some(1),
Some(100000),
Public
),
f!(
"observability.environment",
"CRANK_ENVIRONMENT",
+57 -2
View File
@@ -108,9 +108,64 @@ fn source_for(
}
#[test]
fn registry_covers_exactly_the_59_observed_runtime_names() {
fn external_reference_contract_is_default_off_and_validates_prefixes_and_limits() {
let config = parse_process(
ProcessKind::AdminApi,
ConfigSource::from_utf8(required_admin()),
)
.expect("the external reference contract has safe defaults");
let references = &config.admin().unwrap().external_references;
assert!(references.allowed_url_prefixes.is_empty());
assert_eq!(references.max_depth, 8);
assert_eq!(references.max_documents, 32);
assert_eq!(references.max_fetch_bytes, 262_144);
assert_eq!(references.fetch_timeout_ms, 10_000);
assert_eq!(references.max_expanded_nodes, 10_000);
let mut allowed = required_admin();
allowed.insert(
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(),
"https://schemas.example.test/openapi/,https://schemas.example.test/openapi/".to_owned(),
);
let config = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(allowed)).unwrap();
assert_eq!(
config
.admin()
.unwrap()
.external_references
.allowed_url_prefixes,
vec!["https://schemas.example.test/openapi/".to_owned()]
);
let mut invalid = required_admin();
invalid.insert(
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(),
"https://user:secret@schemas.example.test/openapi/".to_owned(),
);
let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(invalid))
.expect_err("userinfo must never become an operator allow rule");
assert!(error.diagnostics().iter().any(|diagnostic| {
diagnostic.code == DiagnosticCode::InvalidType
&& diagnostic.field == "import.external_references.allowed_url_prefixes"
}));
let mut out_of_range = required_admin();
out_of_range.insert(
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS".to_owned(),
"33".to_owned(),
);
let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(out_of_range))
.expect_err("graph document limit must be bounded");
assert!(error.diagnostics().iter().any(|diagnostic| {
diagnostic.code == DiagnosticCode::OutOfRange
&& diagnostic.field == "import.external_references.max_documents"
}));
}
#[test]
fn registry_covers_exactly_the_65_observed_runtime_names() {
let registry = field_registry();
assert_eq!(registry.len(), 59);
assert_eq!(registry.len(), 65);
let unique = registry
.iter()
.map(|field| field.env_name)
+3
View File
@@ -30,6 +30,9 @@ fn generated_reference_distinguishes_required_and_optional_fields() {
assert!(reference.contains(
"| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` |"
));
assert!(reference.contains(
"| `CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES` | `import.external_references.allowed_url_prefixes` | `AdminApi` | `url_prefix_list/-` | `` |"
));
}
#[test]
+11 -8
View File
@@ -8,19 +8,22 @@ mod normalize_schema;
mod openapi3;
mod payload;
mod recommendations;
mod reference;
mod schema;
mod swagger2;
pub use model::{
ImportFinding, ImportFindingSeverity, ImportGroupPreview, ImportOperationCandidate,
ImportPreview, ImportSourcePreview, NORMALIZER_VERSION, NormalizationConfig, NormalizedFinding,
NormalizedIr, NormalizedOperation, NormalizedParameter, NormalizedReference, NormalizedSchema,
NormalizedSchemaKind, PROJECTION_VERSION, RestImportCandidate, RestImportDocument,
RestImportOperation, RestImportParameter, RestParameterLocation, SourceDigest, SourceIdentity,
SourceLocation, UnresolvedReference,
ExternalDocumentSnapshot, ImportFinding, ImportFindingSeverity, ImportGroupPreview,
ImportOperationCandidate, ImportPreview, ImportSourcePreview, NORMALIZER_VERSION,
NormalizationConfig, NormalizedFinding, NormalizedIr, NormalizedOperation, NormalizedParameter,
NormalizedReference, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind,
PROJECTION_VERSION, ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode,
RestImportCandidate, RestImportDocument, RestImportOperation, RestImportParameter,
RestParameterLocation, SourceDigest, SourceIdentity, SourceLocation, UnresolvedReference,
};
pub use normalize::{
ImportParseError, normalize_verified_document, preview_document, preview_document_legacy_v1,
preview_from_ir, validate_normalized_ir,
ImportParseError, external_reference_uris, normalize_verified_bundle,
normalize_verified_document, preview_document, preview_document_legacy_v1, preview_from_ir,
reference_uris, validate_normalized_ir,
};
pub use payload::operation_draft_from_candidate;
+82 -4
View File
@@ -9,10 +9,10 @@ use serde_json::Value;
/// The immutable contract used to normalize an OpenAPI source. These names
/// deliberately travel with an import job: changing either contract must not
/// silently reinterpret a pending preview.
pub const NORMALIZER_VERSION: &str = "normalized-ir-v2";
pub const PROJECTION_VERSION: &str = "preview-v2";
pub const NORMALIZER_VERSION: &str = "normalized-ir-v3";
pub const PROJECTION_VERSION: &str = "preview-v3";
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct SourceDigest(String);
impl SourceDigest {
@@ -97,6 +97,19 @@ pub struct NormalizationConfig {
pub max_collection_items: usize,
pub max_aliases: usize,
pub max_scalar_bytes: usize,
/// Maximum number of reference hops followed from one source location.
pub max_reference_depth: usize,
/// Maximum number of `$ref` occurrences inspected across the bundle.
pub max_references: usize,
/// Maximum number of immutable external documents supplied to the pure resolver.
pub max_reference_documents: usize,
/// Maximum number of nodes copied while expanding resolved references.
pub max_expanded_nodes: usize,
pub max_external_document_bytes: usize,
/// Signals that orchestration enabled external fetching. It never permits
/// I/O in this crate; it only distinguishes default-deny from a missing or
/// rejected supplied snapshot in exact findings.
pub external_references_enabled: bool,
}
impl Default for NormalizationConfig {
@@ -110,6 +123,12 @@ impl Default for NormalizationConfig {
max_collection_items: 10_000,
max_aliases: 128,
max_scalar_bytes: 256 * 1024,
max_reference_depth: 32,
max_references: 4_096,
max_reference_documents: 32,
max_expanded_nodes: 100_000,
max_external_document_bytes: 256 * 1024,
external_references_enabled: false,
}
}
}
@@ -139,6 +158,38 @@ pub struct UnresolvedReference {
pub location: SourceLocation,
}
/// Immutable external input for the pure reference resolver. The caller owns
/// URL policy and I/O; `crank-import` only consumes already verified bytes.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExternalDocumentSnapshot {
pub canonical_uri: String,
pub digest: SourceDigest,
pub document: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResolvedReferenceNode {
pub snapshot_digest: SourceDigest,
pub location: SourceLocation,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResolvedReferenceEdge {
pub source: ResolvedReferenceNode,
pub target: ResolvedReferenceNode,
pub recursive: bool,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResolvedReferenceGraph {
/// Sorted, deduplicated immutable dependency identities. Canonical URLs
/// deliberately do not enter the IR or public diagnostics.
#[serde(default)]
pub dependency_digests: Vec<SourceDigest>,
#[serde(default)]
pub edges: Vec<ResolvedReferenceEdge>,
}
/// An unresolved `$ref` preserved from the decoded source. This is deliberately
/// broader than schema references: path items, reusable parameters and other
/// object-level references remain available to a later resolution phase.
@@ -213,9 +264,34 @@ pub struct NormalizedSchema {
pub location: SourceLocation,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub discriminator: Option<NormalizedDiscriminator>,
#[serde(default)]
pub constraints: NormalizedSchemaConstraints,
pub kind: NormalizedSchemaKind,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct NormalizedSchemaConstraints {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub minimum: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub maximum: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub min_length: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_length: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pattern: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NormalizedDiscriminator {
pub property_name: String,
#[serde(default)]
pub mapping: BTreeMap<String, String>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "type")]
pub enum NormalizedSchemaKind {
@@ -280,6 +356,8 @@ pub struct NormalizedIr {
pub paths: Vec<NormalizedPath>,
#[serde(default)]
pub unresolved_references: Vec<NormalizedReference>,
#[serde(default)]
pub reference_graph: ResolvedReferenceGraph,
pub source: ImportSourcePreview,
#[serde(default)]
pub operations: Vec<NormalizedOperation>,
@@ -416,7 +494,7 @@ fn empty_source_location() -> SourceLocation {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RestParameterLocation {
Path,
+153 -15
View File
@@ -5,13 +5,13 @@ use thiserror::Error;
use crate::rest::{
model::{
CoverageDisposition, CoverageEntry, ImportFinding, ImportFindingSeverity,
ImportGroupPreview, ImportPreview, ImportSourcePreview, NORMALIZER_VERSION,
NormalizationConfig, NormalizedApiMetadata, NormalizedFinding, NormalizedIr,
NormalizedLiteral, NormalizedOperation, NormalizedParameter, NormalizedScalarKind,
NormalizedSchema, NormalizedSchemaKind, PROJECTION_VERSION, RestImportDocument,
RestImportOperation, RestImportParameter, SourceDigest, SourceIdentity, SourceLocation,
SourceSyntax,
CoverageDisposition, CoverageEntry, ExternalDocumentSnapshot, ImportFinding,
ImportFindingSeverity, ImportGroupPreview, ImportPreview, ImportSourcePreview,
NORMALIZER_VERSION, NormalizationConfig, NormalizedApiMetadata, NormalizedFinding,
NormalizedIr, NormalizedLiteral, NormalizedOperation, NormalizedParameter,
NormalizedScalarKind, NormalizedSchema, NormalizedSchemaKind, PROJECTION_VERSION,
RestImportDocument, RestImportOperation, RestImportParameter, SourceDigest, SourceIdentity,
SourceLocation, SourceSyntax,
},
openapi3,
payload::candidate_from_operation,
@@ -39,6 +39,25 @@ pub fn preview_document(document: &str) -> Result<ImportPreview, ImportParseErro
Ok(preview_from_ir(&ir))
}
/// Enumerates reference URIs without resolving or performing I/O. Intended
/// for an orchestration layer that materializes a bounded immutable bundle.
pub fn reference_uris(
document: &str,
config: &NormalizationConfig,
) -> Result<Vec<String>, ImportParseError> {
super::reference::reference_uris(document, config)
}
/// Enumerates references in an already materialized external document. This
/// keeps the historical primary-document API intact while applying the
/// external-document byte limit at an explicit call site.
pub fn external_reference_uris(
document: &str,
config: &NormalizationConfig,
) -> Result<Vec<String>, ImportParseError> {
super::reference::external_reference_uris(document, config)
}
/// Compatibility projection for jobs created before `NormalizedIr`. It is
/// deliberately isolated from the v2 pipeline and may be removed only after
/// the import-job TTL has elapsed.
@@ -83,6 +102,18 @@ pub fn normalize_verified_document(
document: &str,
digest: SourceDigest,
config: &NormalizationConfig,
) -> Result<NormalizedIr, ImportParseError> {
normalize_verified_bundle(document, digest, &[], config)
}
/// Normalizes a verified primary document together with immutable external
/// snapshots. This function is pure: callers must perform all URL policy,
/// fetching, artifact persistence and digest verification beforehand.
pub fn normalize_verified_bundle(
document: &str,
digest: SourceDigest,
snapshots: &[ExternalDocumentSnapshot],
config: &NormalizationConfig,
) -> Result<NormalizedIr, ImportParseError> {
config
.validate_versions()
@@ -99,6 +130,14 @@ pub fn normalize_verified_document(
};
let root = decode(document)?;
validate_value_limits(&root, config)?;
for snapshot in snapshots {
if snapshot.document.len() > config.max_external_document_bytes {
return Err(ImportParseError::LimitExceeded);
}
}
let resolution = super::reference::resolve(root, digest.clone(), snapshots, config)?;
let root = resolution.root;
validate_value_limits(&root, config)?;
let parsed = match root.get("openapi").and_then(Value::as_str) {
Some(version) if supported_oas_version(version) => openapi3::parse_document(&root)?,
@@ -112,7 +151,15 @@ pub fn normalize_verified_document(
if parsed.operations.is_empty() {
return Err(ImportParseError::NoMethods);
}
canonicalize(parsed, digest, config, root, source_syntax)
canonicalize(
parsed,
digest,
config,
root,
source_syntax,
resolution.graph,
resolution.findings,
)
}
pub fn preview_from_ir(ir: &NormalizedIr) -> ImportPreview {
@@ -202,6 +249,8 @@ fn canonicalize(
config: &NormalizationConfig,
root: Value,
source_syntax: SourceSyntax,
reference_graph: crate::rest::model::ResolvedReferenceGraph,
resolution_findings: Vec<NormalizedFinding>,
) -> Result<NormalizedIr, ImportParseError> {
let source = ImportSourcePreview {
format: document.format,
@@ -260,6 +309,7 @@ fn canonicalize(
}
})
.collect::<Vec<_>>();
findings.extend(resolution_findings);
let mut operations = document
.operations
.into_iter()
@@ -466,6 +516,40 @@ fn canonicalize(
right.location.pointer.as_str(),
))
});
let mut document_findings = Vec::with_capacity(findings.len());
for mut finding in findings.drain(..) {
let resolution_finding = matches!(
finding.code.as_str(),
"external_reference_disabled"
| "reference_target_missing"
| "reference_uri_malformed"
| "external_reference_unavailable"
| "reference_type_mismatch"
| "reference_graph_limit"
| "all_of_conflict"
| "unsupported_composition"
| "unsupported_discriminator"
);
let target = resolution_finding
.then(|| {
operations.iter_mut().find(|operation| {
!finding.location.pointer.is_empty()
&& (finding.location.pointer == operation.location.pointer
|| finding
.location
.pointer
.starts_with(&format!("{}/", operation.location.pointer)))
})
})
.flatten();
if let Some(operation) = target {
finding.operation_key = Some(operation.key.clone());
operation.findings.push(finding);
} else {
document_findings.push(finding);
}
}
findings = document_findings;
for operation in &mut operations {
sort_normalized_findings(&mut operation.findings);
}
@@ -501,10 +585,9 @@ fn canonicalize(
}
sort_normalized_findings(&mut operation.findings);
}
// Resolution is deliberately deferred. References already represented by
// an operation schema use that operation's blocker; all other references
// need a document-level blocker so omitted object-level semantics can
// never reach Apply silently.
// References left in the finite projection are precisely failures which
// the bounded resolver could not safely expand. Keep a blocker at the
// closest affected operation; unrelated operations remain actionable.
findings.extend(
unresolved_references
.iter()
@@ -529,7 +612,11 @@ fn canonicalize(
}),
);
sort_normalized_findings(&mut findings);
for finding in &findings {
for finding in findings.iter().chain(
operations
.iter()
.flat_map(|operation| operation.findings.iter()),
) {
if !coverage.iter().any(|entry| {
entry.construct_id == finding.construct_id
&& entry.location == finding.location
@@ -580,6 +667,7 @@ fn canonicalize(
.unwrap_or_default(),
paths: normalize_coverage::paths_from_operations(&operations, version),
unresolved_references,
reference_graph,
source,
operations,
findings,
@@ -593,9 +681,56 @@ pub fn validate_normalized_ir(ir: &NormalizedIr) -> Result<(), ImportParseError>
if ir.normalizer_version != NORMALIZER_VERSION || ir.projection_version != PROJECTION_VERSION {
return Err(ImportParseError::InvalidDocument);
}
validate_reference_graph(ir)?;
normalize_coverage::validate_coverage(ir)
}
fn validate_reference_graph(ir: &NormalizedIr) -> Result<(), ImportParseError> {
if ir
.reference_graph
.dependency_digests
.windows(2)
.any(|pair| pair[0] >= pair[1])
{
return Err(ImportParseError::InvalidDocument);
}
let allowed = ir
.reference_graph
.dependency_digests
.iter()
.chain(std::iter::once(&ir.source_identity.digest))
.map(SourceDigest::as_str)
.collect::<BTreeSet<_>>();
let edges = &ir.reference_graph.edges;
if edges
.windows(2)
.any(|pair| reference_edge_key(&pair[0]) >= reference_edge_key(&pair[1]))
|| edges.iter().any(|edge| {
!allowed.contains(edge.source.snapshot_digest.as_str())
|| !allowed.contains(edge.target.snapshot_digest.as_str())
|| !(edge.source.location.pointer.is_empty()
|| edge.source.location.pointer.starts_with('/'))
|| !(edge.target.location.pointer.is_empty()
|| edge.target.location.pointer.starts_with('/'))
})
{
return Err(ImportParseError::InvalidDocument);
}
Ok(())
}
fn reference_edge_key(
edge: &crate::rest::model::ResolvedReferenceEdge,
) -> (&str, &str, &str, &str, bool) {
(
edge.source.snapshot_digest.as_str(),
edge.source.location.pointer.as_str(),
edge.target.snapshot_digest.as_str(),
edge.target.location.pointer.as_str(),
edge.recursive,
)
}
fn coverage_disposition_rank(disposition: &CoverageDisposition) -> u8 {
match disposition {
CoverageDisposition::Mapped => 0,
@@ -650,7 +785,10 @@ fn legacy_schema_value(schema: &NormalizedSchema) -> Value {
let mut value = match &schema.kind {
NormalizedSchemaKind::Reference { reference } => serde_json::json!({"$ref": reference.uri}),
NormalizedSchemaKind::Composition { operator, variants } => {
serde_json::json!({operator: variants.iter().map(legacy_schema_value).collect::<Vec<_>>() })
serde_json::json!({
operator: variants.iter().map(legacy_schema_value).collect::<Vec<_>>(),
"x-crank-lossless-composition": true,
})
}
NormalizedSchemaKind::Object {
properties,
@@ -718,7 +856,7 @@ fn schema_contains_reference(schema: &NormalizedSchema, location: &SourceLocatio
}
}
fn validate_value_limits(
pub(super) fn validate_value_limits(
value: &Value,
config: &NormalizationConfig,
) -> Result<(), ImportParseError> {
@@ -1,8 +1,9 @@
use serde_json::Value;
use crate::rest::model::{
CoverageDisposition, CoverageEntry, NormalizedLiteral, NormalizedScalarKind, NormalizedSchema,
NormalizedSchemaKind, SourceLocation, UnresolvedReference,
CoverageDisposition, CoverageEntry, NormalizedDiscriminator, NormalizedLiteral,
NormalizedScalarKind, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind,
SourceLocation, UnresolvedReference,
};
use super::normalize_coverage::escape_pointer;
@@ -150,6 +151,35 @@ pub(super) fn typed_schema(
.get("description")
.and_then(Value::as_str)
.map(ToOwned::to_owned),
discriminator: value.get("discriminator").and_then(|value| {
let property_name = value.get("propertyName")?.as_str()?.to_owned();
let mapping = value
.get("mapping")
.and_then(Value::as_object)
.map(|mapping| {
mapping
.iter()
.filter_map(|(key, value)| {
value.as_str().map(|value| (key.clone(), value.to_owned()))
})
.collect()
})
.unwrap_or_default();
Some(NormalizedDiscriminator {
property_name,
mapping,
})
}),
constraints: NormalizedSchemaConstraints {
minimum: value.get("minimum").and_then(Value::as_f64),
maximum: value.get("maximum").and_then(Value::as_f64),
min_length: value.get("minLength").and_then(Value::as_u64),
max_length: value.get("maxLength").and_then(Value::as_u64),
pattern: value
.get("pattern")
.and_then(Value::as_str)
.map(ToOwned::to_owned),
},
kind,
}
}
+8
View File
@@ -133,6 +133,7 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
parsed_parameters.findings,
);
operation_parameters.extend(parsed_parameters.parameters);
deduplicate_parameters(&mut operation_parameters);
let operation_servers = parse_servers(
operation_value.get("servers"),
&format!("{operation_pointer}/servers"),
@@ -203,6 +204,13 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
})
}
fn deduplicate_parameters(parameters: &mut Vec<RestImportParameter>) {
let mut seen = std::collections::BTreeSet::new();
parameters.reverse();
parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location)));
parameters.reverse();
}
pub fn parse_document_legacy_v1(root: &Value) -> Result<RestImportDocument, ImportParseError> {
let version = root
.get("openapi")
+872
View File
@@ -0,0 +1,872 @@
use std::collections::{BTreeMap, BTreeSet};
use serde_json::{Map, Value};
use crate::rest::model::{
ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedFinding,
ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode, SourceDigest,
SourceLocation,
};
use super::{ImportParseError, normalize_coverage::escape_pointer};
pub(super) struct ResolutionResult {
pub root: Value,
pub graph: ResolvedReferenceGraph,
pub findings: Vec<NormalizedFinding>,
}
pub(super) fn reference_uris(
document: &str,
config: &NormalizationConfig,
) -> Result<Vec<String>, ImportParseError> {
reference_uris_with_max_bytes(document, config, config.max_bytes)
}
pub(super) fn external_reference_uris(
document: &str,
config: &NormalizationConfig,
) -> Result<Vec<String>, ImportParseError> {
reference_uris_with_max_bytes(document, config, config.max_external_document_bytes)
}
fn reference_uris_with_max_bytes(
document: &str,
config: &NormalizationConfig,
max_bytes: usize,
) -> Result<Vec<String>, ImportParseError> {
fn collect(value: &Value, uris: &mut BTreeSet<String>) {
match value {
Value::Object(object) => {
if let Some(reference) = object.get("$ref").and_then(Value::as_str) {
uris.insert(reference.to_owned());
}
for (key, child) in object {
if is_literal_payload_key(key) {
continue;
}
collect(child, uris);
}
}
Value::Array(items) => {
for child in items {
collect(child, uris);
}
}
_ => {}
}
}
if document.len() > max_bytes
|| super::normalize_limits::alias_count(document) > config.max_aliases
{
return Err(ImportParseError::LimitExceeded);
}
let root = decode_snapshot(document)?;
super::normalize::validate_value_limits(&root, config)?;
let mut uris = BTreeSet::new();
collect(&root, &mut uris);
Ok(uris.into_iter().collect())
}
struct Document {
uri: Option<String>,
digest: SourceDigest,
root: Value,
oas31: bool,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct NodeKey {
digest: String,
pointer: String,
}
#[derive(Clone, Copy)]
struct TraversalLocation<'a> {
projection: &'a str,
origin: &'a str,
}
struct Resolver<'a> {
config: &'a NormalizationConfig,
documents: Vec<Document>,
by_uri: BTreeMap<String, usize>,
graph: ResolvedReferenceGraph,
findings: Vec<NormalizedFinding>,
references: usize,
expanded_nodes: usize,
}
pub(super) fn resolve(
root: Value,
primary_digest: SourceDigest,
snapshots: &[ExternalDocumentSnapshot],
config: &NormalizationConfig,
) -> Result<ResolutionResult, ImportParseError> {
if snapshots.len() > config.max_reference_documents {
return Err(ImportParseError::LimitExceeded);
}
let mut documents = vec![Document {
uri: None,
digest: primary_digest,
oas31: is_oas31(&root),
root,
}];
let mut by_uri = BTreeMap::new();
let mut dependency_digests = BTreeSet::new();
let primary_oas31 = documents[0].oas31;
for snapshot in snapshots {
if snapshot.canonical_uri.is_empty()
|| by_uri
.insert(snapshot.canonical_uri.clone(), documents.len())
.is_some()
{
return Err(ImportParseError::InvalidDocument);
}
if super::normalize_limits::alias_count(&snapshot.document) > config.max_aliases {
return Err(ImportParseError::LimitExceeded);
}
let root = decode_snapshot(&snapshot.document)?;
super::normalize::validate_value_limits(&root, config)?;
dependency_digests.insert(snapshot.digest.clone());
documents.push(Document {
uri: Some(snapshot.canonical_uri.clone()),
digest: snapshot.digest.clone(),
// External documents are fragments of the primary contract. In a
// 3.1 bundle they therefore use 3.1 `$ref` sibling semantics even
// when the fragment itself omits an `openapi` declaration.
oas31: primary_oas31 || is_oas31(&root),
root,
});
}
let mut resolver = Resolver {
config,
documents,
by_uri,
graph: ResolvedReferenceGraph {
dependency_digests: dependency_digests.into_iter().collect(),
edges: Vec::new(),
},
findings: Vec::new(),
references: 0,
expanded_nodes: 0,
};
let root = resolver.documents[0].root.clone();
let root = resolver.expand_value(
0,
root,
TraversalLocation {
projection: "",
origin: "",
},
0,
&mut Vec::new(),
)?;
resolver
.graph
.edges
.sort_by(|left, right| edge_key(left).cmp(&edge_key(right)));
resolver.graph.edges.dedup();
resolver.findings.sort_by(|left, right| {
(
&left.operation_key,
&left.construct_id,
&left.code,
&left.location.pointer,
)
.cmp(&(
&right.operation_key,
&right.construct_id,
&right.code,
&right.location.pointer,
))
});
resolver.findings.dedup();
Ok(ResolutionResult {
root,
graph: resolver.graph,
findings: resolver.findings,
})
}
impl Resolver<'_> {
fn expand_value(
&mut self,
document_index: usize,
value: Value,
location: TraversalLocation<'_>,
depth: usize,
stack: &mut Vec<NodeKey>,
) -> Result<Value, ImportParseError> {
if depth > 0 {
self.expanded_nodes = self.expanded_nodes.saturating_add(1);
if self.expanded_nodes > self.config.max_expanded_nodes {
self.push_finding("reference_graph_limit", location.projection);
return Ok(Value::Object(Map::new()));
}
}
match value {
Value::Object(mut object) => {
if let Some(reference) = object
.get("$ref")
.and_then(Value::as_str)
.map(str::to_owned)
{
return self.expand_reference(
document_index,
object,
&reference,
location,
depth,
stack,
);
}
let keys = object.keys().cloned().collect::<Vec<_>>();
for key in keys {
if let Some(child) = object.remove(&key) {
if is_literal_payload_key(&key) {
object.insert(key, child);
continue;
}
let child_pointer =
format!("{}/{}", location.projection, escape_pointer(&key));
let child_origin_pointer =
format!("{}/{}", location.origin, escape_pointer(&key));
object.insert(
key,
self.expand_value(
document_index,
child,
TraversalLocation {
projection: &child_pointer,
origin: &child_origin_pointer,
},
depth,
stack,
)?,
);
}
}
let composition_count = ["allOf", "oneOf", "anyOf"]
.into_iter()
.filter(|operator| object.contains_key(*operator))
.count();
if composition_count > 1 {
self.push_finding("unsupported_composition", location.projection);
}
if let Some(discriminator) = object.get("discriminator") {
let valid = discriminator
.get("propertyName")
.and_then(Value::as_str)
.is_some_and(|value| !value.is_empty())
&& discriminator.get("mapping").is_none_or(|mapping| {
mapping
.as_object()
.is_some_and(|mapping| mapping.values().all(Value::is_string))
})
&& (object.contains_key("oneOf") || object.contains_key("anyOf"));
if !valid {
self.push_finding("unsupported_discriminator", location.projection);
}
}
self.merge_all_of(object, location.projection)
}
Value::Array(items) => Ok(Value::Array(
items
.into_iter()
.enumerate()
.map(|(index, child)| {
self.expand_value(
document_index,
child,
TraversalLocation {
projection: &format!("{}/{index}", location.projection),
origin: &format!("{}/{index}", location.origin),
},
depth,
stack,
)
})
.collect::<Result<Vec<_>, _>>()?,
)),
other => Ok(other),
}
}
fn expand_reference(
&mut self,
document_index: usize,
mut source_object: Map<String, Value>,
reference: &str,
location: TraversalLocation<'_>,
depth: usize,
stack: &mut Vec<NodeKey>,
) -> Result<Value, ImportParseError> {
self.references = self.references.saturating_add(1);
if self.references > self.config.max_references || depth >= self.config.max_reference_depth
{
self.push_finding(
"reference_graph_limit",
&format!("{}/$ref", location.projection),
);
return Ok(Value::Object(source_object));
}
let target = self.target(document_index, reference);
if matches!(target, Err(TargetError::MalformedFragment)) {
self.push_finding(
"reference_uri_malformed",
&format!("{}/$ref", location.projection),
);
return Ok(Value::Object(source_object));
}
let Some((target_document, target_pointer)) = target.ok().flatten() else {
let external = reference.starts_with("http://") || reference.starts_with("https://");
let code = if external {
if self.config.external_references_enabled {
"external_reference_unavailable"
} else {
"external_reference_disabled"
}
} else {
"reference_target_missing"
};
self.push_finding(code, &format!("{}/$ref", location.projection));
return Ok(Value::Object(source_object));
};
let source = ResolvedReferenceNode {
snapshot_digest: self.documents[document_index].digest.clone(),
location: SourceLocation {
pointer: format!("{}/$ref", location.origin),
},
};
let target = ResolvedReferenceNode {
snapshot_digest: self.documents[target_document].digest.clone(),
location: SourceLocation {
pointer: target_pointer.clone(),
},
};
let key = NodeKey {
digest: target.snapshot_digest.as_str().to_owned(),
pointer: target_pointer.clone(),
};
let recursive = stack.contains(&key);
self.graph.edges.push(ResolvedReferenceEdge {
source,
target,
recursive,
});
if recursive {
// The edge is the lossless representation. Expansion stops here,
// producing an opaque object for the finite preview projection.
return Ok(Value::Object(Map::new()));
}
let Some(target_value) = self.documents[target_document]
.root
.pointer(&target_pointer)
.cloned()
else {
self.push_finding(
"reference_target_missing",
&format!("{}/$ref", location.projection),
);
return Ok(Value::Object(source_object));
};
if !target_value.is_object() {
self.push_finding(
"reference_type_mismatch",
&format!("{}/$ref", location.projection),
);
return Ok(Value::Object(source_object));
}
stack.push(key);
let mut expanded = self.expand_value(
target_document,
target_value,
TraversalLocation {
projection: location.projection,
origin: &target_pointer,
},
depth + 1,
stack,
)?;
stack.pop();
// OAS 3.1 Schema Objects permit siblings next to `$ref`; OAS 3.0 and
// Swagger Reference Objects ignore them. The source document version
// controls the semantics at the reference site.
source_object.remove("$ref");
if self.documents[document_index].oas31 && !source_object.is_empty() {
let Value::Object(expanded_object) = &mut expanded else {
self.push_finding(
"reference_type_mismatch",
&format!("{}/$ref", location.projection),
);
return Ok(Value::Object(source_object));
};
for (key, sibling) in source_object {
let child_pointer = format!("{}/{}", location.projection, escape_pointer(&key));
let child_origin_pointer = format!("{}/{}", location.origin, escape_pointer(&key));
expanded_object.insert(
key,
self.expand_value(
document_index,
sibling,
TraversalLocation {
projection: &child_pointer,
origin: &child_origin_pointer,
},
depth,
stack,
)?,
);
}
}
Ok(expanded)
}
fn target(
&self,
current: usize,
reference: &str,
) -> Result<Option<(usize, String)>, TargetError> {
let (document, fragment) = reference.split_once('#').unwrap_or((reference, ""));
let fragment = percent_decode(fragment)?;
let pointer = if fragment.is_empty() {
String::new()
} else if fragment.starts_with('/') {
fragment
} else {
return Ok(None);
};
if document.is_empty() {
return Ok(Some((current, pointer)));
}
let canonical = if has_uri_scheme(document) {
canonical_absolute_uri(document)
} else {
let Some(base) = self.documents[current].uri.as_deref() else {
return Ok(None);
};
join_relative(base, document)
};
Ok(canonical.and_then(|canonical| {
self.by_uri
.get(&canonical)
.copied()
.map(|index| (index, pointer))
}))
}
fn merge_all_of(
&mut self,
mut object: Map<String, Value>,
pointer: &str,
) -> Result<Value, ImportParseError> {
let branches = match object.remove("allOf") {
None => return Ok(Value::Object(object)),
Some(Value::Array(branches)) => branches,
Some(value) => {
object.insert("allOf".to_owned(), value);
self.push_finding("unsupported_composition", &format!("{pointer}/allOf"));
return Ok(Value::Object(object));
}
};
let original = branches.clone();
for branch in branches {
let Some(fields) = branch.as_object() else {
object.insert("allOf".to_owned(), Value::Array(original));
self.push_finding("all_of_conflict", &format!("{pointer}/allOf"));
return Ok(Value::Object(object));
};
if !merge_object(&mut object, fields) {
object.insert("allOf".to_owned(), Value::Array(original));
self.push_finding("all_of_conflict", &format!("{pointer}/allOf"));
return Ok(Value::Object(object));
}
}
Ok(Value::Object(object))
}
fn push_finding(&mut self, code: &str, pointer: &str) {
self.findings.push(NormalizedFinding {
code: code.to_owned(),
severity: ImportFindingSeverity::Error,
message: match code {
"external_reference_disabled" => "Внешняя ссылка отключена политикой импорта.",
"reference_target_missing" => {
"Цель ссылки отсутствует или имеет неверный JSON Pointer."
}
"external_reference_unavailable" => {
"Внешний snapshot недоступен или отклонён политикой импорта."
}
"reference_type_mismatch" => "Цель ссылки имеет неподдерживаемый тип.",
"reference_graph_limit" => "Граф ссылок превышает установленный предел.",
"all_of_conflict" => "Ветки allOf содержат несовместимые определения.",
"reference_uri_malformed" => "URI fragment ссылки содержит некорректное percent-кодирование.",
"unsupported_composition" => "Несколько операторов composition в одной schema не могут быть спроецированы без потерь.",
"unsupported_discriminator" => "Discriminator имеет неподдерживаемую или неполную структуру.",
_ => "Ссылка не может быть безопасно разрешена.",
}
.to_owned(),
construct_id: format!("source:{pointer}"),
location: SourceLocation {
pointer: pointer.to_owned(),
},
operation_key: None,
});
}
}
fn merge_object(target: &mut Map<String, Value>, source: &Map<String, Value>) -> bool {
for (key, value) in source {
match key.as_str() {
"properties" => {
let Some(source_properties) = value.as_object() else {
return false;
};
let properties = target
.entry(key.clone())
.or_insert_with(|| Value::Object(Map::new()));
let Some(target_properties) = properties.as_object_mut() else {
return false;
};
for (name, schema) in source_properties {
if let Some(existing) = target_properties.get(name) {
let (Some(existing), Some(schema)) =
(existing.as_object(), schema.as_object())
else {
return false;
};
let mut merged = existing.clone();
if !merge_object(&mut merged, schema) {
return false;
}
target_properties.insert(name.clone(), Value::Object(merged));
} else {
target_properties.insert(name.clone(), schema.clone());
}
}
}
"required" => {
let Some(source_required) = value.as_array() else {
return false;
};
let required = target
.entry(key.clone())
.or_insert_with(|| Value::Array(Vec::new()));
let Some(target_required) = required.as_array_mut() else {
return false;
};
target_required.extend(source_required.iter().cloned());
target_required.sort_by(|left, right| left.as_str().cmp(&right.as_str()));
target_required.dedup();
}
"minimum" | "maximum" => {
let Some(source_value) = value.as_f64() else {
return false;
};
if target.contains_key(key) && target.get(key).and_then(Value::as_f64).is_none() {
return false;
}
let merged = match (key.as_str(), target.get(key).and_then(Value::as_f64)) {
("minimum", Some(current)) => current.max(source_value),
("maximum", Some(current)) => current.min(source_value),
_ => source_value,
};
let Some(number) = serde_json::Number::from_f64(merged) else {
return false;
};
target.insert(key.clone(), Value::Number(number));
if constraint_contradiction(target, "minimum", "maximum") {
return false;
}
}
"minLength" | "maxLength" => {
let Some(source_value) = value.as_u64() else {
return false;
};
if target.contains_key(key) && target.get(key).and_then(Value::as_u64).is_none() {
return false;
}
let merged = match (key.as_str(), target.get(key).and_then(Value::as_u64)) {
("minLength", Some(current)) => current.max(source_value),
("maxLength", Some(current)) => current.min(source_value),
_ => source_value,
};
target.insert(key.clone(), Value::Number(merged.into()));
if integer_constraint_contradiction(target, "minLength", "maxLength") {
return false;
}
}
_ => {
if target.get(key).is_some_and(|existing| existing != value) {
return false;
}
target.insert(key.clone(), value.clone());
}
}
}
true
}
fn constraint_contradiction(target: &Map<String, Value>, minimum: &str, maximum: &str) -> bool {
match (
target.get(minimum).and_then(Value::as_f64),
target.get(maximum).and_then(Value::as_f64),
) {
(Some(minimum), Some(maximum)) => minimum > maximum,
_ => false,
}
}
fn integer_constraint_contradiction(
target: &Map<String, Value>,
minimum: &str,
maximum: &str,
) -> bool {
match (
target.get(minimum).and_then(Value::as_u64),
target.get(maximum).and_then(Value::as_u64),
) {
(Some(minimum), Some(maximum)) => minimum > maximum,
_ => false,
}
}
fn edge_key(edge: &ResolvedReferenceEdge) -> (&str, &str, &str, &str, bool) {
(
edge.source.snapshot_digest.as_str(),
&edge.source.location.pointer,
edge.target.snapshot_digest.as_str(),
&edge.target.location.pointer,
edge.recursive,
)
}
fn decode_snapshot(document: &str) -> Result<Value, ImportParseError> {
if let Ok(value) = serde_json::from_str(document) {
return Ok(value);
}
let yaml: serde_yaml::Value =
serde_yaml::from_str(document).map_err(|_| ImportParseError::InvalidDocument)?;
serde_json::to_value(yaml).map_err(|_| ImportParseError::InvalidDocument)
}
fn is_oas31(root: &Value) -> bool {
root.get("openapi")
.and_then(Value::as_str)
.is_some_and(|version| version.starts_with("3.1."))
}
fn join_relative(base: &str, relative: &str) -> Option<String> {
let base = UriReference::parse(base)?;
let relative = UriReference::parse(relative)?;
let scheme = relative.scheme.or(base.scheme)?;
let authority = if relative.scheme.is_some() || relative.authority.is_some() {
relative.authority
} else {
base.authority
};
let (path, query) = if relative.scheme.is_some() || relative.authority.is_some() {
(remove_dot_segments(&relative.path), relative.query)
} else if relative.path.is_empty() {
(base.path, relative.query.or(base.query))
} else if relative.path.starts_with('/') {
(remove_dot_segments(&relative.path), relative.query)
} else {
(
remove_dot_segments(&merge_paths(
&base.path,
authority.is_some(),
&relative.path,
)),
relative.query,
)
};
UriReference {
scheme: Some(scheme),
authority,
path,
query,
}
.render()
}
fn canonical_absolute_uri(uri: &str) -> Option<String> {
let reference = UriReference::parse(uri)?;
let scheme = reference.scheme?;
UriReference {
scheme: Some(scheme),
authority: reference.authority,
path: remove_dot_segments(&reference.path),
query: reference.query,
}
.render()
}
#[derive(Clone, Debug)]
struct UriReference<'a> {
scheme: Option<&'a str>,
authority: Option<&'a str>,
path: String,
query: Option<&'a str>,
}
impl<'a> UriReference<'a> {
fn parse(value: &'a str) -> Option<Self> {
if value.contains('\\') || value.contains('#') {
return None;
}
let (without_query, query) = value
.split_once('?')
.map_or((value, None), |(path, query)| (path, Some(query)));
let (scheme, rest) = if let Some(index) = without_query.find(':') {
let candidate = &without_query[..index];
if is_uri_scheme(candidate) {
(Some(candidate), &without_query[index + 1..])
} else {
(None, without_query)
}
} else {
(None, without_query)
};
let (authority, path) = if let Some(rest) = rest.strip_prefix("//") {
match rest.find('/') {
Some(index) => (Some(&rest[..index]), rest[index..].to_owned()),
None => (Some(rest), String::new()),
}
} else {
(None, rest.to_owned())
};
Some(Self {
scheme,
authority,
path,
query,
})
}
fn render(&self) -> Option<String> {
let scheme = self.scheme?;
let scheme = scheme.to_ascii_lowercase();
let mut value = format!("{scheme}:");
if let Some(authority) = self.authority {
value.push_str("//");
value.push_str(&canonical_authority(authority, &scheme));
}
if self.authority.is_some() && self.path.is_empty() {
value.push('/');
} else {
value.push_str(&self.path);
}
if let Some(query) = self.query {
value.push('?');
value.push_str(query);
}
Some(value)
}
}
fn canonical_authority(authority: &str, scheme: &str) -> String {
let authority = authority.to_ascii_lowercase();
let default_port = match scheme {
"http" => Some("80"),
"https" => Some("443"),
_ => None,
};
if let Some(default_port) = default_port
&& let Some((host, port)) = authority.rsplit_once(':')
&& port == default_port
&& (!host.contains(':') || host.ends_with(']'))
{
return host.to_owned();
}
authority
}
fn has_uri_scheme(value: &str) -> bool {
value
.split_once(':')
.is_some_and(|(candidate, _)| is_uri_scheme(candidate))
}
fn is_uri_scheme(candidate: &str) -> bool {
let Some(first) = candidate.as_bytes().first() else {
return false;
};
first.is_ascii_alphabetic()
&& candidate
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.'))
}
fn merge_paths(base_path: &str, has_authority: bool, relative_path: &str) -> String {
match base_path.rfind('/') {
Some(index) => format!("{}{}", &base_path[..=index], relative_path),
None if has_authority => format!("/{relative_path}"),
None => relative_path.to_owned(),
}
}
fn remove_dot_segments(path: &str) -> String {
let leading_slash = path.starts_with('/');
let trailing_slash = path.ends_with('/');
let mut output = Vec::new();
for segment in path.split('/') {
match segment {
"." => {}
".." => {
output.pop();
}
_ => output.push(segment),
}
}
let mut result = output.join("/");
if leading_slash && !result.starts_with('/') {
result.insert(0, '/');
}
if trailing_slash && !result.ends_with('/') {
result.push('/');
}
result
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum TargetError {
MalformedFragment,
}
fn percent_decode(fragment: &str) -> Result<String, TargetError> {
let bytes = fragment.as_bytes();
let mut decoded = Vec::with_capacity(bytes.len());
let mut index = 0;
while index < bytes.len() {
if bytes[index] != b'%' {
decoded.push(bytes[index]);
index += 1;
continue;
}
let Some(high) = bytes.get(index + 1).and_then(|byte| hex_value(*byte)) else {
return Err(TargetError::MalformedFragment);
};
let Some(low) = bytes.get(index + 2).and_then(|byte| hex_value(*byte)) else {
return Err(TargetError::MalformedFragment);
};
decoded.push((high << 4) | low);
index += 3;
}
String::from_utf8(decoded).map_err(|_| TargetError::MalformedFragment)
}
fn hex_value(byte: u8) -> Option<u8> {
match byte {
b'0'..=b'9' => Some(byte - b'0'),
b'a'..=b'f' => Some(byte - b'a' + 10),
b'A'..=b'F' => Some(byte - b'A' + 10),
_ => None,
}
}
fn is_literal_payload_key(key: &str) -> bool {
matches!(key, "example" | "examples" | "default" | "enum" | "const") || key.starts_with("x-")
}
+27 -2
View File
@@ -32,8 +32,33 @@ pub fn schema_from_openapi(
let Some(value) = value else {
return primitive(SchemaKind::String, required, description);
};
// NormalizedIR keeps composition typed; the legacy preview adapter retains
// its historical first-branch projection for pending v1 jobs.
// Only v3 NormalizedIR emits the private marker. Pending legacy-v1 jobs
// retain their historical first-branch behavior, while modern oneOf/anyOf
// is projected losslessly through crank-schema's existing Oneof shape.
if value
.get("x-crank-lossless-composition")
.and_then(Value::as_bool)
== Some(true)
&& let Some(items) = value
.get("oneOf")
.or_else(|| value.get("anyOf"))
.and_then(Value::as_array)
{
return Schema {
kind: SchemaKind::Oneof,
description: description.or_else(|| text(value, "description")),
required,
nullable: nullable(value),
default_value: value.get("default").cloned(),
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: items
.iter()
.map(|item| schema_from_openapi(Some(item), true, None))
.collect(),
};
}
let resolved = collapse_composition(value);
if let Some(values) = resolved.get("enum").and_then(Value::as_array) {
+8
View File
@@ -107,6 +107,7 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
parsed_parameters.findings,
);
operation_parameters.extend(parsed_parameters.parameters);
deduplicate_parameters(&mut operation_parameters);
let tags = tags(
operation_value.get("tags"),
&format!("{path_pointer}/{method_name}/tags"),
@@ -183,6 +184,13 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
})
}
fn deduplicate_parameters(parameters: &mut Vec<RestImportParameter>) {
let mut seen = std::collections::BTreeSet::new();
parameters.reverse();
parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location)));
parameters.reverse();
}
pub fn parse_document_legacy_v1(root: &Value) -> Result<RestImportDocument, ImportParseError> {
let title = root
.pointer("/info/title")
@@ -161,6 +161,58 @@ paths:
assert!(ir.operations[0].request_body_schema.is_some());
}
#[test]
fn operation_parameters_override_path_parameters_by_name_and_location() {
let openapi = r#"
openapi: 3.1.0
info: { title: OAS parameter overrides }
servers: [{ url: https://example.test }]
paths:
/items:
parameters:
- { name: page, in: query, description: path, schema: { type: integer } }
get:
operationId: listItems
parameters:
- { name: page, in: query, description: operation, schema: { type: string } }
responses: { '200': { description: ok } }
"#;
let ir = normalize_document(openapi, &NormalizationConfig::default()).unwrap();
assert_eq!(ir.operations[0].parameters.len(), 1);
assert_eq!(
ir.operations[0].parameters[0].description.as_deref(),
Some("operation")
);
assert_eq!(
ir.operations[0].parameters[0].source_location.pointer,
"/paths/~1items/get/parameters/0"
);
let swagger = r#"
swagger: '2.0'
info: { title: Swagger parameter overrides }
paths:
/items:
parameters:
- { name: page, in: query, description: path, type: integer }
get:
operationId: listItems
parameters:
- { name: page, in: query, description: operation, type: string }
responses: { '200': { description: ok } }
"#;
let ir = normalize_document(swagger, &NormalizationConfig::default()).unwrap();
assert_eq!(ir.operations[0].parameters.len(), 1);
assert_eq!(
ir.operations[0].parameters[0].description.as_deref(),
Some("operation")
);
assert_eq!(
ir.operations[0].parameters[0].source_location.pointer,
"/paths/~1items/get/parameters/0"
);
}
#[test]
fn openapi_parameter_omissions_are_errors_at_the_dropped_item_pointer() {
let document = r#"
@@ -220,10 +272,6 @@ components:
"/paths/~1items~1{id}/parameters/1",
),
("invalid_parameter", "/paths/~1items~1{id}/parameters/2",),
(
"unresolved_parameter_reference",
"/paths/~1items~1{id}/parameters/3",
),
(
"missing_parameter_name",
"/paths/~1items~1{id}/get/parameters/0",
@@ -336,10 +384,6 @@ parameters:
"/paths/~1items~1{id}/parameters/1",
),
("invalid_parameter", "/paths/~1items~1{id}/parameters/2",),
(
"unresolved_parameter_reference",
"/paths/~1items~1{id}/parameters/3",
),
(
"missing_parameter_name",
"/paths/~1items~1{id}/get/parameters/0",
@@ -688,7 +732,7 @@ paths:
}
#[test]
fn preserves_every_unresolved_reference_from_the_full_source_tree() {
fn resolves_local_reference_graph_and_preserves_external_blocker() {
let document = r#"
openapi: 3.1.0
info: { title: References }
@@ -722,32 +766,24 @@ components:
serde_json::to_vec(&first).unwrap(),
serde_json::to_vec(&second).unwrap()
);
assert_eq!(first.unresolved_references.len(), 10);
assert_eq!(first.unresolved_references.len(), 1);
let references = first
.unresolved_references
.iter()
.map(|reference| (reference.uri.as_str(), reference.location.pointer.as_str()))
.collect::<std::collections::BTreeSet<_>>();
assert!(references.contains(&(
"#/components/parameters/Id",
"/paths/~1ok/parameters/0/$ref"
)));
assert!(references.contains(&(
"#/components/requestBodies/Body",
"/paths/~1ok/get/requestBody/$ref"
)));
assert!(references.contains(&(
"#/components/responses/Ok",
"/paths/~1ok/get/responses/200/$ref"
)));
assert!(references.contains(&("#/components/pathItems/Reusable", "/paths/~1reused/$ref")));
assert!(
references.contains(&("#/components/schemas/Loop", "/components/schemas/Loop/$ref"))
);
assert!(references.contains(&(
"https://example.test/schema.json#/Remote",
"/components/schemas/Remote/$ref"
)));
assert!(first.reference_graph.edges.len() >= 5);
assert!(
first
.reference_graph
.edges
.iter()
.any(|edge| edge.recursive)
);
assert!(first.unresolved_references.iter().all(|reference| {
reference.construct_id
== format!(
+2 -2
View File
@@ -94,7 +94,7 @@ definitions:
}
#[test]
fn previews_swagger2_and_preserves_unresolved_definitions() {
fn previews_swagger2_and_resolves_local_definitions() {
let preview = preview_document(SWAGGER2).unwrap();
assert_eq!(preview.source.format, "swagger");
@@ -105,7 +105,7 @@ definitions:
let operation = &preview.groups[0].operations[0];
assert_eq!(operation.suggested_name, "get_pet");
assert_eq!(operation.input_fields, 1);
assert_eq!(operation.output_fields, 0);
assert_eq!(operation.output_fields, 2);
assert_eq!(operation.draft.target.path_template, "/pets/{id}");
assert_eq!(
operation.draft.input_mapping.rules[0].target,
@@ -0,0 +1,793 @@
use crank_import::rest::{
ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedSchemaKind,
SourceDigest, external_reference_uris, normalize_verified_bundle, normalize_verified_document,
preview_from_ir, reference_uris,
};
fn digest(byte: char) -> SourceDigest {
SourceDigest::parse(byte.to_string().repeat(64)).unwrap()
}
fn normalize(document: &str) -> crank_import::rest::NormalizedIr {
normalize_verified_document(document, digest('a'), &NormalizationConfig::default()).unwrap()
}
#[test]
fn resolves_local_schema_and_object_references_with_rfc6901_escaping() {
let document = r#"
openapi: 3.1.0
info: { title: Local refs }
servers: [{ url: https://api.example.test }]
paths:
/items/{id}:
get:
operationId: getItem
parameters:
- { $ref: '#/components/parameters/Id' }
responses:
'200': { $ref: '#/components/responses/Ok' }
components:
parameters:
Id: { name: id, in: path, required: true, schema: { type: string } }
responses:
Ok:
description: ok
content:
application/json:
schema: { $ref: '#/components/schemas/a~1b~0c' }
schemas:
a/b~c:
type: object
required: [id]
properties: { id: { type: string } }
"#;
let ir = normalize(document);
let operation = &ir.operations[0];
assert_eq!(operation.parameters.len(), 1);
assert_eq!(operation.parameters[0].name, "id");
assert!(matches!(
operation.response_schema.as_ref().map(|schema| &schema.kind),
Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id")
));
assert!(ir.unresolved_references.is_empty());
assert_eq!(ir.reference_graph.edges.len(), 3);
assert!(
ir.findings
.iter()
.chain(operation.findings.iter())
.all(|finding| finding.code != "unresolved_reference")
);
}
#[test]
fn broken_reference_blocks_only_affected_candidate_and_full_preview_remains_visible() {
let document = r#"
openapi: 3.1.0
info: { title: Partial graph }
servers: [{ url: https://api.example.test }]
paths:
/broken:
get:
operationId: broken
responses:
'200':
description: nope
content: { application/json: { schema: { $ref: '#/components/schemas/Missing' } } }
/healthy:
get:
operationId: healthy
responses: { '204': { description: ok } }
"#;
let ir = normalize(document);
let broken = ir
.operations
.iter()
.find(|operation| operation.path == "/broken")
.unwrap();
let healthy = ir
.operations
.iter()
.find(|operation| operation.path == "/healthy")
.unwrap();
assert!(broken.findings.iter().any(|finding| {
finding.code == "reference_target_missing"
&& finding.severity == ImportFindingSeverity::Error
}));
assert!(healthy.findings.is_empty());
let preview = preview_from_ir(&ir);
assert_eq!(
preview
.groups
.iter()
.map(|group| group.operations.len())
.sum::<usize>(),
2
);
}
#[test]
fn external_references_are_default_deny_without_network_or_snapshot() {
let document = r#"
openapi: 3.1.0
info: { title: External deny }
servers: [{ url: https://api.example.test }]
paths:
/items:
get:
operationId: listItems
responses:
'200':
description: ok
content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } }
"#;
let ir = normalize(document);
assert!(
ir.operations[0]
.findings
.iter()
.any(|finding| finding.code == "external_reference_disabled")
);
assert!(ir.reference_graph.edges.is_empty());
assert!(ir.reference_graph.dependency_digests.is_empty());
}
#[test]
fn resolves_supplied_external_snapshot_and_relative_chain_deterministically() {
let document = r#"
openapi: 3.1.0
info: { title: External snapshots }
servers: [{ url: https://api.example.test }]
paths:
/items:
get:
operationId: listItems
responses:
'200':
description: ok
content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } }
"#;
let snapshots = vec![
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/root.yaml".to_owned(),
digest: digest('b'),
document: "Item: { $ref: 'child.yaml#/Child' }".to_owned(),
},
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/child.yaml".to_owned(),
digest: digest('c'),
document: "Child: { type: object, properties: { value: { type: integer } } }"
.to_owned(),
},
];
let first = normalize_verified_bundle(
document,
digest('a'),
&snapshots,
&NormalizationConfig::default(),
)
.unwrap();
let second = normalize_verified_bundle(
document,
digest('a'),
&snapshots,
&NormalizationConfig::default(),
)
.unwrap();
assert_eq!(
serde_json::to_vec(&first).unwrap(),
serde_json::to_vec(&second).unwrap()
);
assert_eq!(
first.reference_graph.dependency_digests,
vec![digest('b'), digest('c')]
);
assert_eq!(first.reference_graph.edges.len(), 2);
assert_eq!(
first.reference_graph.edges[1].source.snapshot_digest,
digest('b')
);
assert!(first.unresolved_references.is_empty());
}
#[test]
fn recursion_is_a_stable_graph_edge_without_unbounded_expansion() {
let document = r#"
openapi: 3.1.0
info: { title: Recursive }
servers: [{ url: https://api.example.test }]
paths:
/nodes:
get:
operationId: getNode
responses:
'200':
description: ok
content: { application/json: { schema: { $ref: '#/components/schemas/Node' } } }
components:
schemas:
Node:
type: object
properties:
child: { $ref: '#/components/schemas/Node' }
"#;
let ir = normalize(document);
assert!(ir.reference_graph.edges.iter().any(|edge| edge.recursive));
assert!(ir.unresolved_references.is_empty());
assert!(
ir.operations[0]
.findings
.iter()
.all(|finding| finding.code != "reference_graph_limit")
);
}
#[test]
fn merges_compatible_all_of_and_blocks_conflicts_without_first_branch_loss() {
let compatible = r#"
openapi: 3.1.0
info: { title: AllOf }
servers: [{ url: https://api.example.test }]
paths:
/items:
post:
operationId: createItem
requestBody:
content:
application/json:
schema:
allOf:
- { type: object, required: [id], properties: { id: { type: string } } }
- { type: object, required: [name], properties: { name: { type: string } } }
responses: { '204': { description: ok } }
"#;
let ir = normalize(compatible);
assert!(matches!(
ir.operations[0].request_body_schema.as_ref().map(|schema| &schema.kind),
Some(NormalizedSchemaKind::Object { properties, required })
if properties.len() == 2 && required == &vec!["id".to_owned(), "name".to_owned()]
));
let conflict = compatible.replace("{ name: { type: string } }", "{ id: { type: integer } }");
let ir = normalize(&conflict);
assert!(
ir.operations[0]
.findings
.iter()
.any(|finding| finding.code == "all_of_conflict")
);
}
#[test]
fn preserves_one_of_discriminator_and_projects_all_alternatives() {
let document = r#"
openapi: 3.1.0
info: { title: Alternatives }
servers: [{ url: https://api.example.test }]
paths:
/events:
post:
operationId: createEvent
requestBody:
content:
application/json:
schema:
discriminator:
propertyName: kind
mapping: { text: '#/components/schemas/Text' }
oneOf:
- { type: object, properties: { text: { type: string } } }
- { type: object, properties: { count: { type: integer } } }
responses: { '204': { description: ok } }
"#;
let ir = normalize(document);
let schema = ir.operations[0].request_body_schema.as_ref().unwrap();
assert_eq!(schema.discriminator.as_ref().unwrap().property_name, "kind");
assert!(matches!(
&schema.kind,
NormalizedSchemaKind::Composition { operator, variants }
if operator == "oneOf" && variants.len() == 2
));
let preview = preview_from_ir(&ir);
let candidate = &preview.groups[0].operations[0];
assert_eq!(
candidate
.draft
.input_schema
.fields
.get("body")
.unwrap()
.variants
.len(),
2
);
let any_of = document
.replace("discriminator:\n propertyName: kind\n mapping: { text: '#/components/schemas/Text' }\n oneOf:", "anyOf:");
let ir = normalize(&any_of);
assert!(matches!(
&ir.operations[0].request_body_schema.as_ref().unwrap().kind,
NormalizedSchemaKind::Composition { operator, variants }
if operator == "anyOf" && variants.len() == 2
));
let preview = preview_from_ir(&ir);
assert_eq!(
preview.groups[0].operations[0].draft.input_schema.fields["body"]
.variants
.len(),
2
);
}
#[test]
fn oas_31_applies_ref_siblings_while_oas_30_ignores_them() {
let template = |version: &str| {
format!(
r#"
openapi: {version}
info: {{ title: Siblings }}
servers: [{{ url: https://api.example.test }}]
paths:
/items:
get:
operationId: listItems
responses:
'200':
description: ok
content:
application/json:
schema:
$ref: '#/components/schemas/Item'
description: sibling-description
components:
schemas:
Item: {{ type: string, description: target-description }}
"#
)
};
let v30 = normalize(&template("3.0.3"));
let v31 = normalize(&template("3.1.0"));
assert_eq!(
v30.operations[0]
.response_schema
.as_ref()
.unwrap()
.description
.as_deref(),
Some("target-description")
);
assert_eq!(
v31.operations[0]
.response_schema
.as_ref()
.unwrap()
.description
.as_deref(),
Some("sibling-description")
);
}
#[test]
fn invalid_pointer_and_type_mismatch_are_exact_blockers_without_panic() {
let document = r#"
openapi: 3.1.0
info: { title: Invalid targets }
servers: [{ url: https://api.example.test }]
paths:
/missing:
get:
operationId: missing
responses:
'200': { description: ok, content: { application/json: { schema: { $ref: '#not-a-pointer' } } } }
/scalar:
get:
operationId: scalar
responses:
'200': { description: ok, content: { application/json: { schema: { $ref: '#/info/title' } } } }
"#;
let ir = normalize(document);
let codes = ir
.operations
.iter()
.flat_map(|operation| {
operation
.findings
.iter()
.map(move |finding| (operation.path.as_str(), finding.code.as_str()))
})
.collect::<std::collections::BTreeSet<_>>();
assert!(codes.contains(&("/missing", "reference_target_missing")));
assert!(codes.contains(&("/scalar", "reference_type_mismatch")));
}
#[test]
fn reference_depth_and_expanded_node_limits_fail_closed() {
let document = r#"
openapi: 3.1.0
info: { title: Bounded }
servers: [{ url: https://api.example.test }]
paths:
/items:
get:
operationId: listItems
responses:
'200': { description: ok, content: { application/json: { schema: { $ref: '#/components/schemas/A' } } } }
components:
schemas:
A: { $ref: '#/components/schemas/B' }
B: { $ref: '#/components/schemas/C' }
C: { type: object, properties: { id: { type: string } } }
"#;
let config = NormalizationConfig {
max_reference_depth: 1,
..NormalizationConfig::default()
};
let ir = normalize_verified_document(document, digest('a'), &config).unwrap();
assert!(
ir.operations
.iter()
.flat_map(|operation| &operation.findings)
.any(|finding| finding.code == "reference_graph_limit")
);
let config = NormalizationConfig {
max_expanded_nodes: 8,
..NormalizationConfig::default()
};
let ir = normalize_verified_document(document, digest('a'), &config).unwrap();
assert!(
ir.findings
.iter()
.chain(
ir.operations
.iter()
.flat_map(|operation| &operation.findings)
)
.any(|finding| finding.code == "reference_graph_limit")
);
}
#[test]
fn invalid_discriminator_and_multiple_composition_operators_are_blockers() {
let document = r#"
openapi: 3.1.0
info: { title: Unsupported composition }
servers: [{ url: https://api.example.test }]
paths:
/events:
post:
operationId: createEvent
requestBody:
content:
application/json:
schema:
discriminator: { mapping: { bad: 42 } }
oneOf: [{ type: string }, { type: integer }]
anyOf: [{ type: boolean }, { type: string }]
responses: { '204': { description: ok } }
"#;
let ir = normalize(document);
let codes = ir.operations[0]
.findings
.iter()
.map(|finding| finding.code.as_str())
.collect::<std::collections::BTreeSet<_>>();
assert!(codes.contains("unsupported_discriminator"));
assert!(codes.contains("unsupported_composition"));
}
#[test]
fn reference_uri_scan_applies_external_size_depth_and_alias_limits_before_traversal() {
let config = NormalizationConfig {
max_bytes: 8,
max_external_document_bytes: 8,
..NormalizationConfig::default()
};
assert_eq!(
reference_uris("external-document", &config),
Err(crank_import::rest::ImportParseError::LimitExceeded)
);
let config = NormalizationConfig {
max_bytes: 8,
max_external_document_bytes: 32,
..NormalizationConfig::default()
};
assert_eq!(
external_reference_uris("external-document", &config),
Ok(Vec::new())
);
let config = NormalizationConfig {
max_depth: 1,
..NormalizationConfig::default()
};
assert_eq!(
reference_uris("a: { b: { $ref: '#/x' } }", &config),
Err(crank_import::rest::ImportParseError::LimitExceeded)
);
let aliases = format!(
"items: [{}]",
std::iter::repeat_n("*a", 129)
.collect::<Vec<_>>()
.join(", ")
);
assert_eq!(
reference_uris(&aliases, &NormalizationConfig::default()),
Err(crank_import::rest::ImportParseError::LimitExceeded)
);
}
#[test]
fn resolves_relative_references_using_rfc3986_paths_and_decoded_fragments() {
let document = r#"
openapi: 3.1.0
info: { title: Relative refs }
servers: [{ url: https://api.example.test }]
paths:
/items:
get:
operationId: listItems
responses:
'200':
description: ok
content: { application/json: { schema: { $ref: 'HTTPS://SCHEMAS.EXAMPLE.TEST:443/a/b/root.yaml#/Item' } } }
"#;
let snapshots = vec![
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/a/b/root.yaml".to_owned(),
digest: digest('b'),
document: r#"
Item:
type: object
properties:
dot: { $ref: './child.yaml#/Value' }
parent: { $ref: '../common.yaml#/Value' }
absolute: { $ref: '/shared.yaml#/Value' }
"#
.to_owned(),
},
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/a/b/child.yaml".to_owned(),
digest: digest('c'),
document: "Value: { type: string }".to_owned(),
},
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/a/common.yaml".to_owned(),
digest: digest('d'),
document: "Value: { type: integer }".to_owned(),
},
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/shared.yaml".to_owned(),
digest: digest('e'),
document: "Value: { $ref: '#/a%7E1b' }\na/b: { type: boolean }".to_owned(),
},
];
let ir = normalize_verified_bundle(
document,
digest('a'),
&snapshots,
&NormalizationConfig::default(),
)
.unwrap();
let Some(NormalizedSchemaKind::Object { properties, .. }) = ir.operations[0]
.response_schema
.as_ref()
.map(|schema| &schema.kind)
else {
panic!("response should be an expanded object schema");
};
assert_eq!(properties.len(), 3);
assert_eq!(ir.reference_graph.edges.len(), 5);
assert!(ir.findings.is_empty());
}
#[test]
fn malformed_percent_encoded_fragment_is_a_blocker() {
let document = r#"
openapi: 3.1.0
info: { title: Malformed reference }
servers: [{ url: https://api.example.test }]
paths:
/items:
get:
operationId: listItems
responses:
'200': { description: ok, content: { application/json: { schema: { $ref: '#/components/%ZZ' } } } }
"#;
let ir = normalize(document);
assert!(
ir.operations[0]
.findings
.iter()
.any(|finding| finding.code == "reference_uri_malformed")
);
}
#[test]
fn does_not_follow_references_inside_literal_payloads() {
let document = r#"
openapi: 3.1.0
info: { title: Literal payloads }
servers: [{ url: https://api.example.test }]
paths:
/items:
get:
operationId: listItems
responses:
'200':
description: ok
content:
application/json:
schema:
type: object
example: { $ref: '#/components/schemas/Missing' }
examples: { sample: { value: { $ref: '#/components/schemas/Missing' } } }
default: { $ref: '#/components/schemas/Missing' }
enum: [{ $ref: '#/components/schemas/Missing' }]
const: { $ref: '#/components/schemas/Missing' }
x-fixture: { $ref: '#/components/schemas/Missing' }
properties: { known: { $ref: '#/components/schemas/Known' } }
components:
schemas:
Known: { type: string }
"#;
assert_eq!(
reference_uris(document, &NormalizationConfig::default()),
Ok(vec!["#/components/schemas/Known".to_owned()])
);
let ir = normalize(document);
assert_eq!(ir.reference_graph.edges.len(), 1);
assert!(
ir.operations[0]
.findings
.iter()
.all(|finding| finding.code != "reference_target_missing")
);
}
#[test]
fn primary_oas31_applies_ref_siblings_in_external_snapshots_without_openapi_field() {
let source = |version: &str| {
format!(
r#"
openapi: {version}
info: {{ title: External siblings }}
servers: [{{ url: https://api.example.test }}]
paths:
/items:
get:
operationId: listItems
responses:
'200': {{ description: ok, content: {{ application/json: {{ schema: {{ $ref: 'https://schemas.example.test/root.yaml#/Item' }} }} }} }}
"#
)
};
let snapshots = vec![
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/root.yaml".to_owned(),
digest: digest('b'),
document: "Item: { $ref: 'child.yaml#/Base', description: sibling }".to_owned(),
},
ExternalDocumentSnapshot {
canonical_uri: "https://schemas.example.test/child.yaml".to_owned(),
digest: digest('c'),
document: "Base: { type: string, description: target }".to_owned(),
},
];
let v31 = normalize_verified_bundle(
&source("3.1.0"),
digest('a'),
&snapshots,
&NormalizationConfig::default(),
)
.unwrap();
let v30 = normalize_verified_bundle(
&source("3.0.3"),
digest('a'),
&snapshots,
&NormalizationConfig::default(),
)
.unwrap();
assert_eq!(
v31.operations[0]
.response_schema
.as_ref()
.unwrap()
.description
.as_deref(),
Some("sibling")
);
assert_eq!(
v30.operations[0]
.response_schema
.as_ref()
.unwrap()
.description
.as_deref(),
Some("target")
);
}
#[test]
fn all_of_intersects_constraints_and_nested_properties_without_losing_conflicts() {
let source = r#"
openapi: 3.1.0
info: { title: allOf intersections }
servers: [{ url: https://api.example.test }]
paths:
/items:
post:
operationId: createItem
requestBody:
content:
application/json:
schema:
allOf:
- type: object
minimum: 1
maximum: 10
minLength: 2
maxLength: 12
properties: { nested: { type: object, properties: { left: { type: string } } } }
- type: object
minimum: 4
maximum: 8
minLength: 5
maxLength: 9
properties: { nested: { type: object, properties: { right: { type: integer } } } }
responses: { '204': { description: ok } }
"#;
let ir = normalize(source);
let schema = ir.operations[0].request_body_schema.as_ref().unwrap();
assert_eq!(schema.constraints.minimum, Some(4.0));
assert_eq!(schema.constraints.maximum, Some(8.0));
assert_eq!(schema.constraints.min_length, Some(5));
assert_eq!(schema.constraints.max_length, Some(9));
let NormalizedSchemaKind::Object { properties, .. } = &schema.kind else {
panic!("merged schema should remain an object");
};
let NormalizedSchemaKind::Object { properties, .. } = &properties["nested"].kind else {
panic!("nested property should remain an object");
};
assert!(properties.contains_key("left") && properties.contains_key("right"));
let conflicting = source.replace("maximum: 8", "maximum: 3");
let ir = normalize(&conflicting);
assert!(
ir.operations[0]
.findings
.iter()
.any(|finding| finding.code == "all_of_conflict")
);
}
#[test]
fn non_array_all_of_is_preserved_and_reported() {
let document = r#"
openapi: 3.1.0
info: { title: Invalid allOf }
servers: [{ url: https://api.example.test }]
paths:
/items:
post:
operationId: createItem
requestBody:
content: { application/json: { schema: { allOf: { type: string } } } }
responses: { '204': { description: ok } }
"#;
let ir = normalize(document);
assert!(
ir.operations[0]
.findings
.iter()
.any(|finding| finding.code == "unsupported_composition")
);
assert!(matches!(
ir.operations[0]
.request_body_schema
.as_ref()
.map(|schema| &schema.kind),
Some(NormalizedSchemaKind::Unknown)
));
}
+5 -3
View File
@@ -122,7 +122,7 @@ paths:
}
#[test]
fn keeps_references_typed_without_resolving_them() {
fn resolves_local_references_into_typed_graph() {
let ir = normalize_document(SWAGGER2, &NormalizationConfig::default()).unwrap();
let operation = &ir.operations[0];
assert!(matches!(
@@ -130,13 +130,15 @@ paths:
.response_schema
.as_ref()
.map(|schema| &schema.kind),
Some(crank_import::rest::NormalizedSchemaKind::Reference { .. })
Some(crank_import::rest::NormalizedSchemaKind::Object { .. })
));
assert!(ir.unresolved_references.is_empty());
assert!(!ir.reference_graph.edges.is_empty());
assert!(
operation
.findings
.iter()
.any(|finding| finding.code == "unresolved_reference")
.all(|finding| finding.code != "unresolved_reference")
);
}
+175 -30
View File
@@ -4,6 +4,7 @@ use crank_artifacts::ArtifactRef;
const APPLICATION_RESULT_KEY: &str = "_crank_application_result";
const IMPORT_JOB_CLEANUP_BATCH: u32 = 128;
const MAX_IMPORT_JOB_DOCUMENTS: usize = 32;
const DANGLING_OPENAPI_SOURCE_GRACE: time::Duration = time::Duration::minutes(5);
impl PostgresRegistry {
@@ -12,6 +13,7 @@ impl PostgresRegistry {
request: CreateImportJobRequest<'_>,
) -> Result<(), RegistryError> {
validate_source_envelope(request.source, request.preview_payload)?;
validate_dependency_envelopes(request.preview_payload)?;
let mut transaction = self.pool.begin().await?;
sqlx::query(
"insert into import_jobs (
@@ -81,6 +83,26 @@ impl PostgresRegistry {
&self,
request: FinishImportJobRequest<'_>,
) -> Result<(), RegistryError> {
let mut transaction = self.pool.begin().await?;
let row = sqlx::query(
"select workspace_id, status, preview_payload
from import_jobs
where id = $1
for update",
)
.bind(request.id.as_str())
.fetch_optional(&mut *transaction)
.await?
.ok_or_else(|| RegistryError::ImportJobNotFound {
job_id: request.id.as_str().to_owned(),
})?;
let current_status =
deserialize_enum_text::<ImportJobStatus>(row.try_get("status")?, "status")?;
if request.status == ImportJobStatus::Failed && current_status == ImportJobStatus::Completed
{
transaction.commit().await?;
return Ok(());
}
let result = sqlx::query(
"update import_jobs
set status = $2,
@@ -94,7 +116,7 @@ impl PostgresRegistry {
.bind(request.created_operation_ids)
.bind(request.error_text)
.bind(request.finished_at)
.execute(&self.pool)
.execute(&mut *transaction)
.await?;
if result.rows_affected() == 0 {
@@ -103,6 +125,25 @@ impl PostgresRegistry {
});
}
if request.status == ImportJobStatus::Failed {
let workspace_id = WorkspaceId::new(row.try_get::<String, _>("workspace_id")?);
let payload = row.try_get::<Value, _>("preview_payload")?;
// Failure finalization must itself be fail-safe. The strict
// envelope parsers are used before Apply; cleanup only needs the
// bounded source identities and must not roll back the terminal
// state because some other payload field was corrupted.
for source_id in cleanup_source_ids(&payload) {
let _ = detach_source_in_transaction(
&mut transaction,
&workspace_id,
&source_id,
*request.finished_at,
)
.await?;
}
}
transaction.commit().await?;
Ok(())
}
@@ -120,24 +161,18 @@ impl PostgresRegistry {
}
Err(error) => {
tx.rollback().await?;
let error_text = error.to_string();
let _ = sqlx::query(
"update import_jobs
set status = $3,
error_text = $4,
finished_at = $5::timestamptz
where id = $1
and workspace_id = $2
and status <> $6",
)
.bind(request.id.as_str())
.bind(request.workspace_id.as_str())
.bind(serialize_enum_text(&ImportJobStatus::Failed, "status")?)
.bind(error_text)
.bind(request.finished_at)
.bind(serialize_enum_text(&ImportJobStatus::Completed, "status")?)
.execute(&self.pool)
.await;
if !matches!(error, RegistryError::ImportJobAlreadyApplied { .. }) {
let empty = serde_json::json!([]);
let _ = self
.finish_import_job(FinishImportJobRequest {
id: request.id,
status: ImportJobStatus::Failed,
created_operation_ids: &empty,
error_text: Some("import_apply_failed"),
finished_at: request.finished_at,
})
.await;
}
Err(error)
}
}
@@ -177,16 +212,12 @@ impl PostgresRegistry {
// The job itself is expired regardless of whether a legacy or
// corrupt payload can be decoded. Do not let one bad row roll
// back cleanup for every tenant.
if let Ok(Some(source)) = source_from_payload(&payload)
&& detach_source_in_transaction(
&mut transaction,
&workspace_id,
&source.source_id,
now,
)
.await?
{
detached_sources += 1;
for source_id in cleanup_source_ids(&payload) {
if detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now)
.await?
{
detached_sources += 1;
}
}
}
let deleted = if expired_ids.is_empty() {
@@ -214,7 +245,14 @@ impl PostgresRegistry {
select 1
from import_jobs j
where j.workspace_id = s.workspace_id
and j.preview_payload -> 'source' ->> 'source_id' = s.source_id
and (
j.preview_payload -> 'source' ->> 'source_id' = s.source_id
or exists (
select 1
from jsonb_array_elements(coalesce(j.preview_payload -> 'dependencies', '[]'::jsonb)) d
where d ->> 'source_id' = s.source_id
)
)
)
order by s.created_at, s.workspace_id, s.source_id
limit $2
@@ -283,6 +321,7 @@ async fn apply_import_job_transaction(
source_from_payload(&preview_payload)?.ok_or(RegistryError::InvalidArtifactSource {
field: "import_job_source",
})?;
let dependencies = dependencies_from_payload(&preview_payload)?;
if status == ImportJobStatus::Completed
&& let Some(result) = stored_application_result(&preview_payload)?
@@ -323,6 +362,31 @@ async fn apply_import_job_transaction(
if !source_is_current {
return Err(RegistryError::SourceUnavailable);
}
for dependency in &dependencies {
let recorded = sqlx::query(
"select s.lifecycle, b.artifact_ref
from artifact_sources s
join artifact_blobs b on b.digest = s.blob_digest
where s.workspace_id = $1 and s.source_id = $2
for update",
)
.bind(request.workspace_id.as_str())
.bind(dependency.source_id.as_str())
.fetch_optional(&mut **tx)
.await?;
let current = recorded
.as_ref()
.map(|row| {
let lifecycle = row.try_get::<String, _>("lifecycle").ok();
let artifact_ref = row.try_get::<String, _>("artifact_ref").ok();
lifecycle.as_deref() == Some("active")
&& artifact_ref.as_deref() == Some(dependency.digest.as_str())
})
.unwrap_or(false);
if !current {
return Err(RegistryError::SourceUnavailable);
}
}
let mut result = ImportJobApplyResult {
application_key: request.application_key.to_owned(),
@@ -408,6 +472,15 @@ async fn apply_import_job_transaction(
*request.finished_at,
)
.await?;
for dependency in dependencies {
let _ = detach_source_in_transaction(
tx,
request.workspace_id,
&dependency.source_id,
*request.finished_at,
)
.await?;
}
Ok(result)
}
@@ -433,6 +506,11 @@ fn validate_source_envelope(
Ok(())
}
fn validate_dependency_envelopes(preview_payload: &Value) -> Result<(), RegistryError> {
dependencies_from_payload(preview_payload)?;
Ok(())
}
fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope>, RegistryError> {
let Some(source) = payload.get("source") else {
return Ok(None);
@@ -466,6 +544,73 @@ fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope
}))
}
fn dependencies_from_payload(
payload: &Value,
) -> Result<Vec<ImportJobSourceEnvelope>, RegistryError> {
let Some(value) = payload.get("dependencies") else {
return Ok(Vec::new());
};
let items = value
.as_array()
.filter(|items| items.len() <= MAX_IMPORT_JOB_DOCUMENTS)
.ok_or(RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
})?;
let mut dependencies = Vec::with_capacity(items.len());
let mut identities = std::collections::BTreeSet::new();
for item in items {
let source_id = item
.get("source_id")
.and_then(Value::as_str)
.filter(|value| value.len() <= 132)
.ok_or(RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
})?;
let digest = item.get("digest").and_then(Value::as_str).ok_or(
RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
},
)?;
let digest =
ArtifactRef::parse(digest).map_err(|_| RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
})?;
if !identities.insert((source_id.to_owned(), digest.as_str().to_owned())) {
return Err(RegistryError::InvalidArtifactSource {
field: "import_job_dependencies",
});
}
dependencies.push(ImportJobSourceEnvelope {
source_id: ArtifactSourceId::new(source_id),
digest,
});
}
Ok(dependencies)
}
fn cleanup_source_ids(payload: &Value) -> Vec<ArtifactSourceId> {
let mut identities = std::collections::BTreeSet::new();
if let Some(source_id) = payload
.pointer("/source/source_id")
.and_then(Value::as_str)
.filter(|value| value.len() <= 132)
{
identities.insert(source_id.to_owned());
}
if let Some(dependencies) = payload.get("dependencies").and_then(Value::as_array) {
for source_id in dependencies
.iter()
.take(MAX_IMPORT_JOB_DOCUMENTS)
.filter_map(|item| item.get("source_id"))
.filter_map(Value::as_str)
.filter(|value| value.len() <= 132)
{
identities.insert(source_id.to_owned());
}
}
identities.into_iter().map(ArtifactSourceId::new).collect()
}
async fn detach_source_in_transaction(
transaction: &mut Transaction<'_, Postgres>,
workspace_id: &WorkspaceId,
+3 -1
View File
@@ -25,7 +25,9 @@ pub use cache::{
pub use cache_factory::{
BuiltinCacheBackendFactory, CacheBackendFactory, SharedCacheBackendFactory,
};
pub use crank_adapter_rest::OutboundHttpPolicy;
pub use crank_adapter_rest::{
ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy,
};
pub use error::RuntimeError;
pub use execution_failure::{normalize_runtime_error, normalize_runtime_error_for_operation};
pub use execution_request::{