bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(import): resolve references and schema composition

Browse Source
This commit is contained in:
bsodfather
2026-08-29 08:54:26 +03:00
parent 6c2a3712d8
commit 55209a9bbc
46 changed files with 4848 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/crank-adapter-rest/src/client.rs
+161 -10
View File
@@ -25,7 +25,7 @@ use serde_json::Value;
use tracing::{Instrument, Span};
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::{RestAdapterError, RestRequest, RestResponse};
use crate::{ExternalReferenceFetchError, RestAdapterError, RestRequest, RestResponse};
#[derive(Clone, Debug)]
pub struct RestAdapter {
@@ -33,6 +33,18 @@ pub struct RestAdapter {
policy: OutboundHttpPolicy,
}
/// A deliberately separate, GET-only boundary for materializing external
/// OpenAPI documents. It has no execution metrics, tracing propagation, or
/// request construction semantics from [`RestAdapter`].
#[derive(Clone, Debug)]
pub struct ExternalReferenceFetcher {
client: Result<Client, Arc<str>>,
policy: OutboundHttpPolicy,
allowed_url_prefixes: Vec<String>,
max_response_bytes: usize,
timeout: Duration,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OutboundHttpPolicy {
allowed_hosts: Vec<String>,
@@ -56,15 +68,7 @@ impl RestAdapter {
}
pub fn with_policy(policy: OutboundHttpPolicy) -> Self {
let resolver = Arc::new(PolicyDnsResolver {
policy: policy.clone(),
});
let client = Client::builder()
.redirect(redirect::Policy::none())
.no_proxy()
.dns_resolver(resolver)
.build()
.map_err(|error| Arc::<str>::from(error.to_string()));
let client = outbound_client(&policy);
Self { client, policy }
}
@@ -195,6 +199,153 @@ impl RestAdapter {
}
}
impl ExternalReferenceFetcher {
pub fn try_new(
policy: OutboundHttpPolicy,
allowed_url_prefixes: Vec<String>,
max_response_bytes: usize,
timeout: Duration,
) -> Result<Self, ExternalReferenceFetchError> {
if max_response_bytes == 0 || timeout.is_zero() {
return Err(ExternalReferenceFetchError::InvalidConfiguration);
}
let allowed_url_prefixes = allowed_url_prefixes
.into_iter()
.map(|prefix| canonical_external_reference_prefix(&prefix))
.collect::<Result<Vec<_>, _>>()?;
Ok(Self {
client: outbound_client(&policy),
policy,
allowed_url_prefixes,
max_response_bytes,
timeout,
})
}
/// Fetches one document with a bounded, headerless `GET`.
///
/// URL fragments are stripped because they address JSON Pointer targets in
/// the fetched document rather than a network resource.
pub async fn get(&self, url: &str) -> Result<Vec<u8>, ExternalReferenceFetchError> {
if self.allowed_url_prefixes.is_empty() {
return Err(ExternalReferenceFetchError::Disabled);
}
let mut url =
reqwest::Url::parse(url).map_err(|_| ExternalReferenceFetchError::InvalidUrl)?;
url.set_fragment(None);
if !self
.allowed_url_prefixes
.iter()
.any(|prefix| matches_external_reference_prefix(&url, prefix))
{
return Err(ExternalReferenceFetchError::TargetNotAllowed);
}
self.policy
.validate_url(&url)
.map_err(external_policy_error)?;
let client = self
.client
.as_ref()
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
let response = client
.get(url)
.timeout(self.timeout)
.send()
.await
.map_err(external_transport_error)?;
let status = response.status();
if status.is_redirection() {
return Err(ExternalReferenceFetchError::RedirectNotAllowed);
}
if !status.is_success() {
return Err(ExternalReferenceFetchError::UnexpectedStatus {
status: status.as_u16(),
});
}
read_external_response_bytes(response, self.max_response_bytes).await
}
}
fn outbound_client(policy: &OutboundHttpPolicy) -> Result<Client, Arc<str>> {
let resolver = Arc::new(PolicyDnsResolver {
policy: policy.clone(),
});
Client::builder()
.redirect(redirect::Policy::none())
.no_proxy()
.dns_resolver(resolver)
.build()
.map_err(|error| Arc::<str>::from(error.to_string()))
}
fn canonical_external_reference_prefix(
prefix: &str,
) -> Result<String, ExternalReferenceFetchError> {
let url = reqwest::Url::parse(prefix)
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
if !matches!(url.scheme(), "http" | "https")
|| url.host_str().is_none()
|| !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(ExternalReferenceFetchError::InvalidConfiguration);
}
Ok(url.to_string())
}
fn matches_external_reference_prefix(url: &reqwest::Url, prefix: &str) -> bool {
let url = url.as_str();
if !url.starts_with(prefix) {
return false;
}
let Some(next) = url.as_bytes().get(prefix.len()) else {
return true;
};
prefix.ends_with('/') || matches!(next, b'/' | b'?')
}
fn external_policy_error(error: RestAdapterError) -> ExternalReferenceFetchError {
match error {
RestAdapterError::TargetNotAllowed { .. } => ExternalReferenceFetchError::TargetNotAllowed,
_ => ExternalReferenceFetchError::InvalidConfiguration,
}
}
fn external_transport_error(error: reqwest::Error) -> ExternalReferenceFetchError {
ExternalReferenceFetchError::Transport {
timeout: error.is_timeout(),
connect: error.is_connect(),
}
}
async fn read_external_response_bytes(
response: reqwest::Response,
max_response_bytes: usize,
) -> Result<Vec<u8>, ExternalReferenceFetchError> {
if response
.content_length()
.is_some_and(|length| length > max_response_bytes as u64)
{
return Err(ExternalReferenceFetchError::ResponseTooLarge {
limit_bytes: max_response_bytes,
});
}
let mut stream = response.bytes_stream();
let mut bytes = Vec::new();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(external_transport_error)?;
if bytes.len().saturating_add(chunk.len()) > max_response_bytes {
return Err(ExternalReferenceFetchError::ResponseTooLarge {
limit_bytes: max_response_bytes,
});
}
bytes.extend_from_slice(&chunk);
}
Ok(bytes)
}
fn upstream_outcome(error: &RestAdapterError) -> UpstreamOutcome {
match error {
RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
crates/crank-adapter-rest/src/error.rs
+20
View File
@@ -1,6 +1,26 @@
use serde_json::Value;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ExternalReferenceFetchError {
#[error("external references are disabled")]
Disabled,
#[error("external reference URL is invalid")]
InvalidUrl,
#[error("external reference target is not allowed")]
TargetNotAllowed,
#[error("external reference redirects are not allowed")]
RedirectNotAllowed,
#[error("external reference response exceeds the configured limit of {limit_bytes} bytes")]
ResponseTooLarge { limit_bytes: usize },
#[error("external reference endpoint returned status {status}")]
UnexpectedStatus { status: u16 },
#[error("external reference request failed")]
Transport { timeout: bool, connect: bool },
#[error("external reference fetch configuration is invalid")]
InvalidConfiguration,
}
#[derive(Debug, Error)]
pub enum RestAdapterError {
#[error("invalid base url: {url}")]
crates/crank-adapter-rest/src/lib.rs
+2 -2
View File
@@ -8,8 +8,8 @@ use crank_core::{
ProtocolAdapterError, RestTarget, RuntimeRequestContext, Target,
};
pub use client::{OutboundHttpPolicy, RestAdapter};
pub use error::RestAdapterError;
pub use client::{ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter};
pub use error::{ExternalReferenceFetchError, RestAdapterError};
pub use model::{RestRequest, RestResponse};
#[async_trait]
crates/crank-adapter-rest/tests/integration.rs
+1
View File
@@ -1,4 +1,5 @@
mod integration {
mod client;
mod external_reference_fetcher;
mod outbound_security;
}
crates/crank-adapter-rest/tests/integration/external_reference_fetcher.rs
+151
View File
@@ -0,0 +1,151 @@
use std::{
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{Router, http::StatusCode, response::Redirect, routing::get};
use crank_adapter_rest::{
ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy,
};
use tokio::net::TcpListener;
#[tokio::test]
async fn external_references_are_default_off_before_any_request() {
let requests = Arc::new(AtomicUsize::new(0));
let base_url = spawn_server(Arc::clone(&requests)).await;
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
Vec::new(),
1024,
Duration::from_secs(1),
)
.unwrap();
let error = fetcher
.get(&format!("{base_url}/document"))
.await
.unwrap_err();
assert!(matches!(error, ExternalReferenceFetchError::Disabled));
assert_eq!(requests.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn fetcher_uses_prefix_and_actual_address_policy_then_returns_bounded_bytes() {
let base_url = spawn_server(Arc::new(AtomicUsize::new(0))).await;
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
vec![base_url.clone()],
8,
Duration::from_secs(1),
)
.unwrap();
assert_eq!(
fetcher
.get(&format!("{base_url}/document#/components/schemas/A"))
.await
.unwrap(),
b"openapi".to_vec()
);
let error = fetcher.get(&format!("{base_url}/large")).await.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::ResponseTooLarge { limit_bytes: 8 }
));
let private_without_explicit_outbound_allow = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::default(),
vec![base_url.clone()],
1024,
Duration::from_secs(1),
)
.unwrap();
let error = private_without_explicit_outbound_allow
.get(&format!("{base_url}/document"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::TargetNotAllowed
));
let exact_path_fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
vec![format!("{base_url}/document")],
1024,
Duration::from_secs(1),
)
.unwrap();
let error = exact_path_fetcher
.get(&format!("{base_url}/document-unrelated"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::TargetNotAllowed
));
}
#[tokio::test]
async fn fetcher_rejects_redirects_and_userinfo_without_exposing_the_url() {
let requests = Arc::new(AtomicUsize::new(0));
let base_url = spawn_server(Arc::clone(&requests)).await;
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
vec![base_url.clone()],
1024,
Duration::from_secs(1),
)
.unwrap();
let error = fetcher
.get(&format!("{base_url}/redirect"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::RedirectNotAllowed
));
let userinfo_url = base_url.replacen("http://", "http://user:credential@", 1);
let error = fetcher
.get(&format!("{userinfo_url}/document"))
.await
.unwrap_err();
assert!(matches!(
error,
ExternalReferenceFetchError::TargetNotAllowed
));
let rendered = format!("{error:?} {error}");
assert!(!rendered.contains("credential"));
assert_eq!(requests.load(Ordering::SeqCst), 0);
}
async fn spawn_server(requests: Arc<AtomicUsize>) -> String {
let app = Router::new()
.route(
"/document",
get({
let requests = Arc::clone(&requests);
move || {
let requests = Arc::clone(&requests);
async move {
requests.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, "openapi")
}
}
}),
)
.route("/large", get(|| async { "response too large" }))
.route("/redirect", get(|| async { Redirect::to("/document") }));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{address}")
}
crates/crank-adapter-rest/tests/integration/outbound_security.rs
+16 -2
View File
@@ -6,6 +6,7 @@ use std::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use axum::{
@@ -14,7 +15,9 @@ use axum::{
http::StatusCode,
routing::{any, post},
};
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
use crank_adapter_rest::{
ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest,
};
use crank_core::{HttpMethod, ProtocolAdapterError, RestTarget};
use serde_json::{Value, json};
use tokio::net::TcpListener;
@@ -94,7 +97,7 @@ async fn proxy_environment_is_ignored_by_default() {
let adapter = RestAdapter::default();
let request = json_request(json!({"payload": "proxy-env-canary"}));
let rest_target = RestTarget {
base_url: target,
base_url: target.clone(),
method: HttpMethod::Post,
path_template: "/capture".to_owned(),
static_headers: BTreeMap::new(),
@@ -102,6 +105,17 @@ async fn proxy_environment_is_ignored_by_default() {
let _ = adapter.execute(&rest_target, &request).await.expect_err(
"unresolvable target should fail locally instead of being sent through proxy env",
);
let fetcher = ExternalReferenceFetcher::try_new(
OutboundHttpPolicy::default(),
vec!["http://public.example.test/".to_owned()],
1024,
Duration::from_secs(1),
)
.expect("valid external reference fetcher");
let _ = fetcher
.get(&target)
.await
.expect_err("external reference fetcher must not use proxy environment variables");
return;
}