feat(import): resolve references and schema composition
This commit is contained in:
@@ -25,6 +25,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS=
|
||||
CRANK_OUTBOUND_DENIED_HOSTS=
|
||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES=
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000
|
||||
CRANK_ENVIRONMENT=development
|
||||
CRANK_LOG_LEVEL=
|
||||
CRANK_SENTRY_DSN=
|
||||
|
||||
@@ -227,6 +227,7 @@ async fn run(
|
||||
.with_artifact_store(artifact_store.clone())
|
||||
.with_public_base_url(base_url)
|
||||
.with_outbound_http_policy(outbound_http_policy)
|
||||
.with_external_reference_import(&config.external_references)?
|
||||
.with_identity_provider(std::sync::Arc::new(identity_provider))
|
||||
.build();
|
||||
if config.demo_seed {
|
||||
|
||||
@@ -4,6 +4,7 @@ use std::sync::Arc;
|
||||
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use crank_artifacts::ArtifactStore;
|
||||
use crank_config::ExternalReferenceSettings;
|
||||
use crank_core::{
|
||||
AuditActor, AuditEvent, AuditEventId, AuditSink, AuditTarget, AuditTargetKind, AuthProfile,
|
||||
CapabilityProfile, CommunityCapabilityProfile, CorrelationContext, EditionCapabilities,
|
||||
@@ -19,7 +20,8 @@ use crank_registry::{
|
||||
UsageBucket,
|
||||
};
|
||||
use crank_runtime::{
|
||||
OutboundHttpPolicy, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
|
||||
ExternalReferenceFetcher, OutboundHttpPolicy, ResolvedAuth, RuntimeError, RuntimeExecutor,
|
||||
SecretCrypto,
|
||||
};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
|
||||
@@ -68,6 +70,9 @@ pub struct AdminService {
|
||||
audit_sink: Arc<dyn AuditSink>,
|
||||
capability_profile: Arc<dyn CapabilityProfile>,
|
||||
outbound_http_policy: OutboundHttpPolicy,
|
||||
external_reference_fetcher: ExternalReferenceFetcher,
|
||||
external_reference_normalization: crank_import::rest::NormalizationConfig,
|
||||
external_reference_materialization_timeout: std::time::Duration,
|
||||
public_base_url: String,
|
||||
}
|
||||
|
||||
@@ -83,6 +88,9 @@ pub struct AdminServiceBuilder {
|
||||
audit_sink: Option<Arc<dyn AuditSink>>,
|
||||
capability_profile: Option<Arc<dyn CapabilityProfile>>,
|
||||
outbound_http_policy: OutboundHttpPolicy,
|
||||
external_reference_fetcher: ExternalReferenceFetcher,
|
||||
external_reference_normalization: crank_import::rest::NormalizationConfig,
|
||||
external_reference_settings: ExternalReferenceSettings,
|
||||
public_base_url: String,
|
||||
}
|
||||
|
||||
@@ -220,6 +228,22 @@ impl AdminServiceBuilder {
|
||||
secret_crypto: SecretCrypto,
|
||||
runtime: RuntimeExecutor,
|
||||
) -> Self {
|
||||
let outbound_http_policy = OutboundHttpPolicy::default();
|
||||
let external_reference_settings = ExternalReferenceSettings {
|
||||
allowed_url_prefixes: Vec::new(),
|
||||
max_depth: 8,
|
||||
max_documents: 32,
|
||||
max_fetch_bytes: 256 * 1024,
|
||||
fetch_timeout_ms: 5_000,
|
||||
max_expanded_nodes: 10_000,
|
||||
};
|
||||
let external_reference_fetcher = ExternalReferenceFetcher::try_new(
|
||||
outbound_http_policy.clone(),
|
||||
external_reference_settings.allowed_url_prefixes.clone(),
|
||||
external_reference_settings.max_fetch_bytes,
|
||||
std::time::Duration::from_millis(external_reference_settings.fetch_timeout_ms),
|
||||
)
|
||||
.expect("static external reference defaults are valid");
|
||||
Self {
|
||||
registry,
|
||||
storage_root,
|
||||
@@ -231,7 +255,10 @@ impl AdminServiceBuilder {
|
||||
policy_engine: None,
|
||||
audit_sink: None,
|
||||
capability_profile: None,
|
||||
outbound_http_policy: OutboundHttpPolicy::default(),
|
||||
outbound_http_policy,
|
||||
external_reference_fetcher,
|
||||
external_reference_normalization: Default::default(),
|
||||
external_reference_settings,
|
||||
public_base_url: "http://localhost:3000".to_owned(),
|
||||
}
|
||||
}
|
||||
@@ -242,10 +269,41 @@ impl AdminServiceBuilder {
|
||||
}
|
||||
|
||||
pub fn with_outbound_http_policy(mut self, policy: OutboundHttpPolicy) -> Self {
|
||||
self.external_reference_fetcher = ExternalReferenceFetcher::try_new(
|
||||
policy.clone(),
|
||||
self.external_reference_settings
|
||||
.allowed_url_prefixes
|
||||
.clone(),
|
||||
self.external_reference_settings.max_fetch_bytes,
|
||||
std::time::Duration::from_millis(self.external_reference_settings.fetch_timeout_ms),
|
||||
)
|
||||
.expect("validated external reference settings remain valid");
|
||||
self.outbound_http_policy = policy;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_external_reference_import(
|
||||
mut self,
|
||||
settings: &ExternalReferenceSettings,
|
||||
) -> Result<Self, crank_runtime::ExternalReferenceFetchError> {
|
||||
self.external_reference_fetcher = ExternalReferenceFetcher::try_new(
|
||||
self.outbound_http_policy.clone(),
|
||||
settings.allowed_url_prefixes.clone(),
|
||||
settings.max_fetch_bytes,
|
||||
std::time::Duration::from_millis(settings.fetch_timeout_ms),
|
||||
)?;
|
||||
self.external_reference_normalization.max_reference_depth = settings.max_depth;
|
||||
self.external_reference_normalization
|
||||
.max_reference_documents = settings.max_documents;
|
||||
self.external_reference_normalization
|
||||
.max_external_document_bytes = settings.max_fetch_bytes;
|
||||
self.external_reference_normalization.max_expanded_nodes = settings.max_expanded_nodes;
|
||||
self.external_reference_normalization
|
||||
.external_references_enabled = !settings.allowed_url_prefixes.is_empty();
|
||||
self.external_reference_settings = settings.clone();
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
/// Reuses the process-wide immutable artifact authority for OpenAPI
|
||||
/// ingress and reconciliation. Tests may omit this and use their private
|
||||
/// storage root instead.
|
||||
@@ -299,6 +357,11 @@ impl AdminServiceBuilder {
|
||||
.capability_profile
|
||||
.unwrap_or_else(|| Arc::new(CommunityCapabilityProfile)),
|
||||
outbound_http_policy: self.outbound_http_policy,
|
||||
external_reference_fetcher: self.external_reference_fetcher,
|
||||
external_reference_normalization: self.external_reference_normalization,
|
||||
external_reference_materialization_timeout: std::time::Duration::from_millis(
|
||||
self.external_reference_settings.fetch_timeout_ms,
|
||||
),
|
||||
public_base_url: self.public_base_url,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ use crank_import::rest::{
|
||||
};
|
||||
use crank_registry::{
|
||||
ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity,
|
||||
CreateArtifactSourceRequest, CreateImportJobRequest, DetachArtifactSourceRequest,
|
||||
CreateArtifactSourceRequest, CreateImportJobRequest, FinishImportJobRequest,
|
||||
ImportConflictMode, ImportJobApplyResult, ImportJobId, ImportJobKind, ImportJobSourceEnvelope,
|
||||
ImportJobStatus, ImportOperationDraft, RegistryError,
|
||||
};
|
||||
@@ -29,6 +29,13 @@ use crate::{
|
||||
},
|
||||
};
|
||||
|
||||
mod external_references;
|
||||
mod job_contract;
|
||||
use external_references::ImportReplayContext;
|
||||
use job_contract::{
|
||||
SourceDetachGuard, import_job_dependencies, import_job_normalization_config, import_job_source,
|
||||
};
|
||||
|
||||
const IMPORT_JOB_TTL_HOURS: i64 = 24;
|
||||
const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30);
|
||||
const OPENAPI_PARSE_CONCURRENCY: usize = 4;
|
||||
@@ -95,9 +102,14 @@ impl AdminService {
|
||||
detach_guard.detach_now().await;
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let mut dependencies = self
|
||||
.materialize_external_reference_snapshots(workspace_id, &verified.bytes, now)
|
||||
.await;
|
||||
let parsed = match parse_verified_preview(
|
||||
verified.bytes,
|
||||
artifact.artifact_ref().digest_hex().to_owned(),
|
||||
dependencies.snapshots.clone(),
|
||||
self.external_reference_normalization.clone(),
|
||||
locale,
|
||||
false,
|
||||
)
|
||||
@@ -105,6 +117,7 @@ impl AdminService {
|
||||
{
|
||||
Ok(preview) => preview,
|
||||
Err(error) => {
|
||||
dependencies.detach_all().await;
|
||||
detach_guard.detach_now().await;
|
||||
return Err(error);
|
||||
}
|
||||
@@ -115,6 +128,7 @@ impl AdminService {
|
||||
.iter()
|
||||
.all(|group| group.operations.is_empty())
|
||||
{
|
||||
dependencies.detach_all().await;
|
||||
detach_guard.detach_now().await;
|
||||
return Err(ApiError::openapi_upload(locale, "no_methods"));
|
||||
}
|
||||
@@ -130,12 +144,15 @@ impl AdminService {
|
||||
"source_id": source_envelope.source_id.as_str(),
|
||||
"digest": source_envelope.digest.as_str(),
|
||||
},
|
||||
"dependencies": dependencies.payload(),
|
||||
"dependency_snapshots": dependencies.snapshot_payload(),
|
||||
"preview": preview_value,
|
||||
"preview_digest": preview_digest,
|
||||
"normalization": {
|
||||
"normalizer_version": NORMALIZER_VERSION,
|
||||
"projection_version": PROJECTION_VERSION,
|
||||
"ir_fingerprint": parsed.ir_fingerprint,
|
||||
"config": self.external_reference_normalization,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -155,9 +172,11 @@ impl AdminService {
|
||||
})
|
||||
.await
|
||||
{
|
||||
dependencies.detach_all().await;
|
||||
detach_guard.detach_now().await;
|
||||
return Err(ApiError::from(error));
|
||||
}
|
||||
dependencies.disarm();
|
||||
detach_guard.disarm();
|
||||
|
||||
Ok(OpenApiImportPreviewResponse {
|
||||
@@ -234,6 +253,13 @@ impl AdminService {
|
||||
} else {
|
||||
ImportConflictMode::Rename
|
||||
};
|
||||
let replay_context = ImportReplayContext {
|
||||
workspace_id,
|
||||
job_id,
|
||||
application_key: &application_key,
|
||||
conflict_mode,
|
||||
finished_at: &finished_at,
|
||||
};
|
||||
if job.status == ImportJobStatus::Completed {
|
||||
let applied = self
|
||||
.registry
|
||||
@@ -250,7 +276,18 @@ impl AdminService {
|
||||
return Ok(openapi_import_response(applied));
|
||||
}
|
||||
|
||||
let source = import_job_source(&job.preview_payload, locale)?;
|
||||
let source = match import_job_source(&job.preview_payload, locale) {
|
||||
Ok(source) => source,
|
||||
Err(error) => {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_source_verification_failed",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
let verified = match self
|
||||
.registry
|
||||
.read_artifact_source(
|
||||
@@ -264,44 +301,123 @@ impl AdminService {
|
||||
Err(
|
||||
error @ (RegistryError::SourceNotFound { .. } | RegistryError::SourceUnavailable),
|
||||
) => {
|
||||
// Another request may have finished and detached the source
|
||||
// after our unlocked Pending read. Re-read the job and ask
|
||||
// the registry for its canonical, locked replay instead of
|
||||
// exposing a false `source_unavailable` outcome.
|
||||
let latest = self.registry.get_import_job(workspace_id, job_id).await?;
|
||||
if latest.is_some_and(|latest| latest.status == ImportJobStatus::Completed) {
|
||||
let applied = self
|
||||
.registry
|
||||
.apply_import_job(ApplyImportJobRequest {
|
||||
id: job_id,
|
||||
workspace_id,
|
||||
application_key: &application_key,
|
||||
conflict_mode,
|
||||
operations: &[],
|
||||
pre_skipped: &[],
|
||||
finished_at: &finished_at,
|
||||
})
|
||||
.await?;
|
||||
return Ok(openapi_import_response(applied));
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_source_verification_failed",
|
||||
openapi_source_error(locale, error),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
return Err(openapi_source_error(locale, error));
|
||||
Err(error @ RegistryError::SourceIntegrity) => {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_source_verification_failed",
|
||||
openapi_source_error(locale, error),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(error) => return Err(openapi_source_error(locale, error)),
|
||||
};
|
||||
if verified.source.blob.artifact_ref != source.digest {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_source_verification_failed",
|
||||
ApiError::openapi_upload(locale, "source_integrity"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let legacy_v1 = job.preview_payload.get("normalization").is_none();
|
||||
let parsed = parse_verified_preview(
|
||||
let replay_contract = match import_replay_contract(&job.preview_payload, locale) {
|
||||
Ok(contract) => contract,
|
||||
Err(error) => {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_replay_verification_failed",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
let parsed = match replay_contract {
|
||||
ImportReplayContract::PersistedV2(parsed) => parsed,
|
||||
ImportReplayContract::LegacyV1 | ImportReplayContract::Current => {
|
||||
let legacy_v1 = matches!(replay_contract, ImportReplayContract::LegacyV1);
|
||||
let dependency_snapshots = match self
|
||||
.read_import_job_dependencies(workspace_id, &job.preview_payload, locale)
|
||||
.await
|
||||
{
|
||||
Ok(snapshots) => snapshots,
|
||||
Err(error) => {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_dependency_verification_failed",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
let normalization_config = match import_job_normalization_config(
|
||||
&job.preview_payload,
|
||||
&self.external_reference_normalization,
|
||||
legacy_v1,
|
||||
locale,
|
||||
) {
|
||||
Ok(config) => config,
|
||||
Err(error) => {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_replay_verification_failed",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
let parsed = match parse_verified_preview(
|
||||
verified.bytes,
|
||||
source.digest.digest_hex().to_owned(),
|
||||
dependency_snapshots,
|
||||
normalization_config,
|
||||
locale,
|
||||
legacy_v1,
|
||||
)
|
||||
.await?;
|
||||
verify_preview_contract(&job.preview_payload, &parsed, locale)?;
|
||||
if preview_has_blocker(&parsed.preview) {
|
||||
return Err(ApiError::openapi_upload(locale, "invalid_document"));
|
||||
.await
|
||||
{
|
||||
Ok(parsed) => parsed,
|
||||
Err(error) => {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_replay_verification_failed",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
};
|
||||
if let Err(error) = verify_preview_contract(&job.preview_payload, &parsed, locale) {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"import_replay_verification_failed",
|
||||
error,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
parsed
|
||||
}
|
||||
};
|
||||
if preview_has_blocker(&parsed.preview, &selected) {
|
||||
return self
|
||||
.fail_openapi_import_or_replay(
|
||||
&replay_context,
|
||||
"reference_resolution_blocked",
|
||||
ApiError::openapi_upload(locale, "invalid_document"),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let mut candidates = BTreeMap::new();
|
||||
for group in &parsed.preview.groups {
|
||||
@@ -372,6 +488,24 @@ impl AdminService {
|
||||
}
|
||||
}
|
||||
|
||||
fn canonical_external_document_uri(base: Option<&str>, reference: &str) -> Option<String> {
|
||||
if reference.starts_with('#') {
|
||||
return None;
|
||||
}
|
||||
let mut url = match base {
|
||||
Some(base) => url::Url::parse(base).ok()?.join(reference).ok()?,
|
||||
None => url::Url::parse(reference).ok()?,
|
||||
};
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
url.set_fragment(None);
|
||||
Some(url.to_string())
|
||||
}
|
||||
|
||||
fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse {
|
||||
let created = applied
|
||||
.created
|
||||
@@ -470,9 +604,72 @@ struct ParsedOpenApiPreview {
|
||||
ir_fingerprint: String,
|
||||
}
|
||||
|
||||
enum ImportReplayContract {
|
||||
LegacyV1,
|
||||
PersistedV2(ParsedOpenApiPreview),
|
||||
Current,
|
||||
}
|
||||
|
||||
fn import_replay_contract(
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<ImportReplayContract, ApiError> {
|
||||
let Some(normalization) = payload.get("normalization") else {
|
||||
return Ok(ImportReplayContract::LegacyV1);
|
||||
};
|
||||
let normalizer = normalization
|
||||
.get("normalizer_version")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let projection = normalization
|
||||
.get("projection_version")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let ir_fingerprint = normalization
|
||||
.get("ir_fingerprint")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| is_lower_sha256(value))
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let expected_preview_digest = payload
|
||||
.get("preview_digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| is_lower_sha256(value))
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
|
||||
if normalizer == NORMALIZER_VERSION && projection == PROJECTION_VERSION {
|
||||
return Ok(ImportReplayContract::Current);
|
||||
}
|
||||
if normalizer != "normalized-ir-v2" || projection != "preview-v2" {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
|
||||
let preview_value = payload
|
||||
.get("preview")
|
||||
.cloned()
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
if preview_digest(&preview_value)? != expected_preview_digest {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let preview = serde_json::from_value(preview_value)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
Ok(ImportReplayContract::PersistedV2(ParsedOpenApiPreview {
|
||||
preview,
|
||||
ir_fingerprint: ir_fingerprint.to_owned(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn is_lower_sha256(value: &str) -> bool {
|
||||
value.len() == 64
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}
|
||||
|
||||
async fn parse_verified_preview(
|
||||
bytes: Vec<u8>,
|
||||
digest: String,
|
||||
snapshots: Vec<crank_import::rest::ExternalDocumentSnapshot>,
|
||||
normalization_config: crank_import::rest::NormalizationConfig,
|
||||
locale: OpenApiUploadLocale,
|
||||
legacy_v1: bool,
|
||||
) -> Result<ParsedOpenApiPreview, ApiError> {
|
||||
@@ -498,8 +695,12 @@ async fn parse_verified_preview(
|
||||
}
|
||||
let digest = crank_import::rest::SourceDigest::parse(digest)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let ir =
|
||||
crank_import::rest::normalize_verified_document(document, digest, &Default::default())
|
||||
let ir = crank_import::rest::normalize_verified_bundle(
|
||||
document,
|
||||
digest,
|
||||
&snapshots,
|
||||
&normalization_config,
|
||||
)
|
||||
.map_err(|error| match error {
|
||||
crank_import::rest::ImportParseError::NoMethods => {
|
||||
ApiError::openapi_upload(locale, "no_methods")
|
||||
@@ -597,115 +798,23 @@ fn verify_preview_contract(
|
||||
}
|
||||
}
|
||||
|
||||
fn preview_has_blocker(preview: &crank_import::rest::ImportPreview) -> bool {
|
||||
fn preview_has_blocker(
|
||||
preview: &crank_import::rest::ImportPreview,
|
||||
selected_operation_keys: &BTreeSet<String>,
|
||||
) -> bool {
|
||||
preview
|
||||
.findings
|
||||
.iter()
|
||||
.chain(
|
||||
preview
|
||||
.any(|finding| finding.severity == ImportFindingSeverity::Error)
|
||||
|| preview
|
||||
.groups
|
||||
.iter()
|
||||
.flat_map(|group| group.operations.iter())
|
||||
.flat_map(|operation| operation.findings.iter()),
|
||||
)
|
||||
.filter(|operation| selected_operation_keys.contains(&operation.key))
|
||||
.flat_map(|operation| operation.findings.iter())
|
||||
.any(|finding| finding.severity == ImportFindingSeverity::Error)
|
||||
}
|
||||
|
||||
fn import_job_source(
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<ImportJobSourceEnvelope, ApiError> {
|
||||
let source = payload
|
||||
.get("source")
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let source_id = source
|
||||
.get("source_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let digest = source
|
||||
.get("digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| value.parse().ok())
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
Ok(ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
})
|
||||
}
|
||||
|
||||
struct SourceDetachGuard {
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl SourceDetachGuard {
|
||||
fn new(
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
workspace_id,
|
||||
source_id,
|
||||
expected_updated_at,
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
async fn detach_now(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
self.armed = false;
|
||||
// This is the normal error path: await the database transition so a
|
||||
// caller receives a completed cleanup before it can retry. `Drop`
|
||||
// remains only the cancellation/shutdown safety net.
|
||||
let _ = self
|
||||
.registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &self.workspace_id,
|
||||
source_id: &self.source_id,
|
||||
expected_updated_at: Some(self.expected_updated_at),
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SourceDetachGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
let registry = self.registry.clone();
|
||||
let workspace_id = self.workspace_id.clone();
|
||||
let source_id = self.source_id.clone();
|
||||
let expected_updated_at = Some(self.expected_updated_at);
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
let _ = registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &workspace_id,
|
||||
source_id: &source_id,
|
||||
expected_updated_at,
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
|
||||
let selected_operation_keys = payload
|
||||
.selected_operation_keys
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
use super::*;
|
||||
use std::collections::VecDeque;
|
||||
use tracing::warn;
|
||||
|
||||
pub(super) struct MaterializedDependencies {
|
||||
pub(super) snapshots: Vec<crank_import::rest::ExternalDocumentSnapshot>,
|
||||
envelopes: Vec<ImportJobSourceEnvelope>,
|
||||
canonical_uris: Vec<String>,
|
||||
guards: Vec<SourceDetachGuard>,
|
||||
}
|
||||
|
||||
pub(super) struct ImportReplayContext<'a> {
|
||||
pub(super) workspace_id: &'a WorkspaceId,
|
||||
pub(super) job_id: &'a ImportJobId,
|
||||
pub(super) application_key: &'a str,
|
||||
pub(super) conflict_mode: ImportConflictMode,
|
||||
pub(super) finished_at: &'a OffsetDateTime,
|
||||
}
|
||||
|
||||
impl MaterializedDependencies {
|
||||
fn empty() -> Self {
|
||||
Self {
|
||||
snapshots: Vec::new(),
|
||||
envelopes: Vec::new(),
|
||||
canonical_uris: Vec::new(),
|
||||
guards: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn payload(&self) -> Vec<serde_json::Value> {
|
||||
self.envelopes
|
||||
.iter()
|
||||
.zip(&self.canonical_uris)
|
||||
.map(|(dependency, canonical_uri)| {
|
||||
json!({
|
||||
"source_id": dependency.source_id.as_str(),
|
||||
"digest": dependency.digest.as_str(),
|
||||
"canonical_uri": canonical_uri,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn snapshot_payload(&self) -> Vec<serde_json::Value> {
|
||||
self.snapshots
|
||||
.iter()
|
||||
.filter_map(|snapshot| {
|
||||
self.envelopes
|
||||
.iter()
|
||||
.find(|dependency| dependency.digest.digest_hex() == snapshot.digest.as_str())
|
||||
.map(|dependency| {
|
||||
json!({
|
||||
"source_id": dependency.source_id.as_str(),
|
||||
"digest": dependency.digest.as_str(),
|
||||
"canonical_uri": snapshot.canonical_uri,
|
||||
})
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn disarm(&mut self) {
|
||||
for guard in &mut self.guards {
|
||||
guard.disarm();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn detach_all(&mut self) {
|
||||
for guard in &mut self.guards {
|
||||
guard.detach_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AdminService {
|
||||
pub(super) async fn fail_openapi_import_or_replay(
|
||||
&self,
|
||||
context: &ImportReplayContext<'_>,
|
||||
error_text: &'static str,
|
||||
error: ApiError,
|
||||
) -> Result<OpenApiImportCreateResponse, ApiError> {
|
||||
let empty = json!([]);
|
||||
self.registry
|
||||
.finish_import_job(FinishImportJobRequest {
|
||||
id: context.job_id,
|
||||
status: ImportJobStatus::Failed,
|
||||
created_operation_ids: &empty,
|
||||
error_text: Some(error_text),
|
||||
finished_at: context.finished_at,
|
||||
})
|
||||
.await?;
|
||||
|
||||
// `finish_import_job` preserves an already Completed row under its
|
||||
// lock. This read distinguishes that race from the terminal Failed
|
||||
// transition and returns the immutable canonical application result.
|
||||
let latest = self
|
||||
.registry
|
||||
.get_import_job(context.workspace_id, context.job_id)
|
||||
.await?;
|
||||
if latest.is_some_and(|job| job.status == ImportJobStatus::Completed) {
|
||||
let applied = self
|
||||
.registry
|
||||
.apply_import_job(ApplyImportJobRequest {
|
||||
id: context.job_id,
|
||||
workspace_id: context.workspace_id,
|
||||
application_key: context.application_key,
|
||||
conflict_mode: context.conflict_mode,
|
||||
operations: &[],
|
||||
pre_skipped: &[],
|
||||
finished_at: context.finished_at,
|
||||
})
|
||||
.await?;
|
||||
return Ok(openapi_import_response(applied));
|
||||
}
|
||||
|
||||
Err(error)
|
||||
}
|
||||
|
||||
pub(super) async fn materialize_external_reference_snapshots(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
primary_bytes: &[u8],
|
||||
created_at: OffsetDateTime,
|
||||
) -> MaterializedDependencies {
|
||||
let materialized_count = std::sync::atomic::AtomicUsize::new(0);
|
||||
match tokio::time::timeout(
|
||||
self.external_reference_materialization_timeout,
|
||||
self.materialize_external_reference_snapshots_inner(
|
||||
workspace_id,
|
||||
primary_bytes,
|
||||
created_at,
|
||||
&materialized_count,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(dependencies) => dependencies,
|
||||
Err(_) => {
|
||||
materialization_failure(
|
||||
"chain",
|
||||
"timeout",
|
||||
materialized_count.load(std::sync::atomic::Ordering::Relaxed),
|
||||
);
|
||||
MaterializedDependencies::empty()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn materialize_external_reference_snapshots_inner(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
primary_bytes: &[u8],
|
||||
created_at: OffsetDateTime,
|
||||
materialized_count: &std::sync::atomic::AtomicUsize,
|
||||
) -> MaterializedDependencies {
|
||||
if !self
|
||||
.external_reference_normalization
|
||||
.external_references_enabled
|
||||
{
|
||||
return MaterializedDependencies::empty();
|
||||
}
|
||||
let Ok(primary) = std::str::from_utf8(primary_bytes) else {
|
||||
materialization_failure("primary_decode", "invalid_utf8", 0);
|
||||
return MaterializedDependencies::empty();
|
||||
};
|
||||
let Ok(primary_references) =
|
||||
crank_import::rest::reference_uris(primary, &self.external_reference_normalization)
|
||||
else {
|
||||
materialization_failure("primary_scan", "invalid_document", 0);
|
||||
return MaterializedDependencies::empty();
|
||||
};
|
||||
let mut queue = VecDeque::new();
|
||||
for reference in primary_references {
|
||||
if let Some(uri) = canonical_external_document_uri(None, &reference) {
|
||||
queue.push_back((uri, 1usize));
|
||||
}
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut result = MaterializedDependencies::empty();
|
||||
let mut dependency_by_digest = BTreeMap::<String, usize>::new();
|
||||
while let Some((canonical_uri, depth)) = queue.pop_front() {
|
||||
if !seen.insert(canonical_uri.clone())
|
||||
|| depth > self.external_reference_normalization.max_reference_depth
|
||||
|| seen.len()
|
||||
> self
|
||||
.external_reference_normalization
|
||||
.max_reference_documents
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let bytes = match self.external_reference_fetcher.get(&canonical_uri).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => {
|
||||
materialization_failure(
|
||||
"fetch",
|
||||
external_fetch_error_code(&error),
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Ok(document) = std::str::from_utf8(&bytes).map(str::to_owned) else {
|
||||
materialization_failure(
|
||||
"dependency_decode",
|
||||
"invalid_utf8",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let Ok(references) = crank_import::rest::external_reference_uris(
|
||||
&document,
|
||||
&self.external_reference_normalization,
|
||||
) else {
|
||||
materialization_failure(
|
||||
"dependency_scan",
|
||||
"invalid_document",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let store = self.artifact_store.clone();
|
||||
let artifact_bytes = bytes.clone();
|
||||
let Ok(Ok(artifact)) =
|
||||
tokio::task::spawn_blocking(move || store.put_registered(&artifact_bytes)).await
|
||||
else {
|
||||
materialization_failure(
|
||||
"artifact_store",
|
||||
"storage_unavailable",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let digest_key = artifact.artifact_ref().as_str().to_owned();
|
||||
let envelope_index = if let Some(index) = dependency_by_digest.get(&digest_key) {
|
||||
*index
|
||||
} else {
|
||||
let source_id = ArtifactSourceId::new(new_prefixed_id("src_openapi_dep"));
|
||||
let Ok(source) = self
|
||||
.registry
|
||||
.create_artifact_source(CreateArtifactSourceRequest {
|
||||
workspace_id,
|
||||
source_id: &source_id,
|
||||
artifact: &artifact,
|
||||
mime_type: "application/octet-stream",
|
||||
sensitivity: ArtifactSourceSensitivity::Internal,
|
||||
created_at,
|
||||
})
|
||||
.await
|
||||
else {
|
||||
materialization_failure(
|
||||
"source_create",
|
||||
"registry_unavailable",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
continue;
|
||||
};
|
||||
let mut guard = SourceDetachGuard::new(
|
||||
self.registry.clone(),
|
||||
workspace_id.clone(),
|
||||
source_id.clone(),
|
||||
source.updated_at,
|
||||
);
|
||||
let Ok(verified) = self
|
||||
.registry
|
||||
.read_artifact_source(
|
||||
std::sync::Arc::clone(&self.artifact_store),
|
||||
workspace_id,
|
||||
&source_id,
|
||||
)
|
||||
.await
|
||||
else {
|
||||
materialization_failure(
|
||||
"source_verify",
|
||||
"source_unavailable",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
guard.detach_now().await;
|
||||
continue;
|
||||
};
|
||||
if verified.source.blob.artifact_ref != *artifact.artifact_ref() {
|
||||
materialization_failure(
|
||||
"source_verify",
|
||||
"source_integrity",
|
||||
result.snapshots.len(),
|
||||
);
|
||||
guard.detach_now().await;
|
||||
continue;
|
||||
}
|
||||
let index = result.envelopes.len();
|
||||
result.envelopes.push(ImportJobSourceEnvelope {
|
||||
source_id: source_id.clone(),
|
||||
digest: artifact.artifact_ref().clone(),
|
||||
});
|
||||
result.guards.push(guard);
|
||||
dependency_by_digest.insert(digest_key, index);
|
||||
index
|
||||
};
|
||||
let dependency = &result.envelopes[envelope_index];
|
||||
let snapshot_digest = match crank_import::rest::SourceDigest::parse(
|
||||
dependency.digest.digest_hex().to_owned(),
|
||||
) {
|
||||
Ok(digest) => digest,
|
||||
Err(_) => {
|
||||
materialization_failure("snapshot", "source_integrity", result.snapshots.len());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
result
|
||||
.snapshots
|
||||
.push(crank_import::rest::ExternalDocumentSnapshot {
|
||||
canonical_uri: canonical_uri.clone(),
|
||||
digest: snapshot_digest,
|
||||
document,
|
||||
});
|
||||
materialized_count.store(result.snapshots.len(), std::sync::atomic::Ordering::Relaxed);
|
||||
result.canonical_uris.push(canonical_uri.clone());
|
||||
for reference in references {
|
||||
if let Some(uri) = canonical_external_document_uri(Some(&canonical_uri), &reference)
|
||||
{
|
||||
queue.push_back((uri, depth.saturating_add(1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Registry ownership is one envelope per immutable digest, while the
|
||||
// resolver may map several canonical URIs to that same snapshot. Keep
|
||||
// payload rows aligned with envelopes and add aliases separately.
|
||||
if result.envelopes.len() != result.canonical_uris.len() {
|
||||
let mut canonical_by_digest = BTreeMap::new();
|
||||
for snapshot in &result.snapshots {
|
||||
canonical_by_digest
|
||||
.entry(snapshot.digest.as_str().to_owned())
|
||||
.or_insert_with(|| snapshot.canonical_uri.clone());
|
||||
}
|
||||
result.canonical_uris = result
|
||||
.envelopes
|
||||
.iter()
|
||||
.map(|envelope| {
|
||||
canonical_by_digest
|
||||
.get(envelope.digest.digest_hex())
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(super) async fn read_import_job_dependencies(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<Vec<crank_import::rest::ExternalDocumentSnapshot>, ApiError> {
|
||||
let dependencies = import_job_dependencies(payload, locale)?;
|
||||
let mut snapshots = Vec::with_capacity(dependencies.len());
|
||||
for (canonical_uri, dependency) in dependencies {
|
||||
let verified = self
|
||||
.registry
|
||||
.read_artifact_source(
|
||||
std::sync::Arc::clone(&self.artifact_store),
|
||||
workspace_id,
|
||||
&dependency.source_id,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| openapi_source_error(locale, error))?;
|
||||
if verified.source.blob.artifact_ref != dependency.digest {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let document = String::from_utf8(verified.bytes)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let digest =
|
||||
crank_import::rest::SourceDigest::parse(dependency.digest.digest_hex().to_owned())
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
snapshots.push(crank_import::rest::ExternalDocumentSnapshot {
|
||||
canonical_uri,
|
||||
digest,
|
||||
document,
|
||||
});
|
||||
}
|
||||
Ok(snapshots)
|
||||
}
|
||||
}
|
||||
fn external_fetch_error_code(error: &crank_runtime::ExternalReferenceFetchError) -> &'static str {
|
||||
match error {
|
||||
crank_runtime::ExternalReferenceFetchError::Disabled => "disabled",
|
||||
crank_runtime::ExternalReferenceFetchError::InvalidUrl => "invalid_url",
|
||||
crank_runtime::ExternalReferenceFetchError::TargetNotAllowed => "target_not_allowed",
|
||||
crank_runtime::ExternalReferenceFetchError::RedirectNotAllowed => "redirect_not_allowed",
|
||||
crank_runtime::ExternalReferenceFetchError::ResponseTooLarge { .. } => "response_too_large",
|
||||
crank_runtime::ExternalReferenceFetchError::UnexpectedStatus { .. } => "unexpected_status",
|
||||
crank_runtime::ExternalReferenceFetchError::Transport { timeout: true, .. } => "timeout",
|
||||
crank_runtime::ExternalReferenceFetchError::Transport { .. } => "transport",
|
||||
crank_runtime::ExternalReferenceFetchError::InvalidConfiguration => "invalid_configuration",
|
||||
}
|
||||
}
|
||||
|
||||
fn materialization_failure(stage: &'static str, error_code: &'static str, count: usize) {
|
||||
warn!(
|
||||
name: "admin.openapi_import.materialization_failed",
|
||||
stage,
|
||||
error_code,
|
||||
count,
|
||||
"external OpenAPI materialization failed"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use crank_core::WorkspaceId;
|
||||
use crank_registry::{ArtifactSourceId, DetachArtifactSourceRequest, ImportJobSourceEnvelope};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::{error::ApiError, service::OpenApiUploadLocale};
|
||||
|
||||
use super::canonical_external_document_uri;
|
||||
|
||||
const MAX_IMPORT_JOB_DOCUMENTS: usize = 32;
|
||||
|
||||
pub(super) fn import_job_source(
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<ImportJobSourceEnvelope, ApiError> {
|
||||
let source = payload
|
||||
.get("source")
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let source_id = source
|
||||
.get("source_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_unavailable"))?;
|
||||
let digest = source
|
||||
.get("digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| value.parse().ok())
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
Ok(ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn import_job_dependencies(
|
||||
payload: &serde_json::Value,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<Vec<(String, ImportJobSourceEnvelope)>, ApiError> {
|
||||
let empty = Vec::new();
|
||||
let items = payload
|
||||
.get("dependency_snapshots")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.unwrap_or(&empty);
|
||||
if items.len() > MAX_IMPORT_JOB_DOCUMENTS {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let mut canonical_uris = BTreeSet::new();
|
||||
items
|
||||
.iter()
|
||||
.map(|item| {
|
||||
let canonical_uri = item
|
||||
.get("canonical_uri")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| {
|
||||
canonical_external_document_uri(None, value).as_deref() == Some(*value)
|
||||
})
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
if !canonical_uris.insert(canonical_uri.to_owned()) {
|
||||
return Err(ApiError::openapi_upload(locale, "source_integrity"));
|
||||
}
|
||||
let source_id = item
|
||||
.get("source_id")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
let digest = item
|
||||
.get("digest")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.and_then(|value| value.parse().ok())
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))?;
|
||||
Ok((
|
||||
canonical_uri.to_owned(),
|
||||
ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
},
|
||||
))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn import_job_normalization_config(
|
||||
payload: &serde_json::Value,
|
||||
current: &crank_import::rest::NormalizationConfig,
|
||||
legacy_v1: bool,
|
||||
locale: OpenApiUploadLocale,
|
||||
) -> Result<crank_import::rest::NormalizationConfig, ApiError> {
|
||||
if legacy_v1 {
|
||||
return Ok(current.clone());
|
||||
}
|
||||
payload
|
||||
.pointer("/normalization/config")
|
||||
.cloned()
|
||||
.ok_or_else(|| ApiError::openapi_upload(locale, "source_integrity"))
|
||||
.and_then(|value| {
|
||||
serde_json::from_value(value)
|
||||
.map_err(|_| ApiError::openapi_upload(locale, "source_integrity"))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) struct SourceDetachGuard {
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl SourceDetachGuard {
|
||||
pub(super) fn new(
|
||||
registry: crank_registry::PostgresRegistry,
|
||||
workspace_id: WorkspaceId,
|
||||
source_id: ArtifactSourceId,
|
||||
expected_updated_at: OffsetDateTime,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry,
|
||||
workspace_id,
|
||||
source_id,
|
||||
expected_updated_at,
|
||||
armed: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn disarm(&mut self) {
|
||||
self.armed = false;
|
||||
}
|
||||
|
||||
pub(super) async fn detach_now(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
self.armed = false;
|
||||
let _ = self
|
||||
.registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &self.workspace_id,
|
||||
source_id: &self.source_id,
|
||||
expected_updated_at: Some(self.expected_updated_at),
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SourceDetachGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
return;
|
||||
}
|
||||
let registry = self.registry.clone();
|
||||
let workspace_id = self.workspace_id.clone();
|
||||
let source_id = self.source_id.clone();
|
||||
let expected_updated_at = Some(self.expected_updated_at);
|
||||
if let Ok(handle) = tokio::runtime::Handle::try_current() {
|
||||
handle.spawn(async move {
|
||||
let _ = registry
|
||||
.detach_artifact_source(DetachArtifactSourceRequest {
|
||||
workspace_id: &workspace_id,
|
||||
source_id: &source_id,
|
||||
expected_updated_at,
|
||||
detached_at: OffsetDateTime::now_utc(),
|
||||
})
|
||||
.await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,6 +108,24 @@ pub(super) fn build_test_app(
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn build_test_app_with_external_references(
|
||||
registry: PostgresRegistry,
|
||||
storage_root: std::path::PathBuf,
|
||||
allowed_url_prefixes: Vec<String>,
|
||||
) -> Router {
|
||||
build_app(AppState {
|
||||
service: test_service_with_external_references(
|
||||
registry,
|
||||
storage_root,
|
||||
allowed_url_prefixes,
|
||||
),
|
||||
api_rate_limiter: crank_runtime::RequestRateLimiter::new(
|
||||
crank_runtime::RequestRateLimitConfig::new(10_000, 10_000).unwrap(),
|
||||
),
|
||||
trusted_proxy_ips: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn build_test_app_with_audit_sink(
|
||||
registry: PostgresRegistry,
|
||||
storage_root: std::path::PathBuf,
|
||||
@@ -152,6 +170,33 @@ pub(super) fn test_service(
|
||||
.build()
|
||||
}
|
||||
|
||||
pub(super) fn test_service_with_external_references(
|
||||
registry: PostgresRegistry,
|
||||
storage_root: std::path::PathBuf,
|
||||
allowed_url_prefixes: Vec<String>,
|
||||
) -> AdminService {
|
||||
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
|
||||
AdminServiceBuilder::new(
|
||||
registry,
|
||||
storage_root,
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
runtime,
|
||||
)
|
||||
.with_external_reference_import(&crank_config::ExternalReferenceSettings {
|
||||
allowed_url_prefixes,
|
||||
max_depth: 8,
|
||||
max_documents: 32,
|
||||
max_fetch_bytes: 64 * 1024,
|
||||
fetch_timeout_ms: 2_000,
|
||||
max_expanded_nodes: 10_000,
|
||||
})
|
||||
.unwrap()
|
||||
.with_outbound_http_policy(outbound_policy)
|
||||
.build()
|
||||
}
|
||||
|
||||
pub(super) async fn spawn_upstream_server() -> String {
|
||||
let app = Router::new().route("/crm/leads", post(create_lead));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
||||
use admin_api::service::{
|
||||
AdminServiceBuilder, OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale,
|
||||
};
|
||||
use crank_core::WorkspaceId;
|
||||
use crank_registry::ImportJobStatus;
|
||||
use serial_test::serial;
|
||||
@@ -7,6 +9,8 @@ use super::common::{
|
||||
test_auth_settings, test_registry, test_secret_crypto, test_service, test_storage_root,
|
||||
};
|
||||
|
||||
mod external_references;
|
||||
|
||||
const OPENAPI3: &str = r#"
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
@@ -73,11 +77,11 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
assert_eq!(preview_job.status, ImportJobStatus::Pending);
|
||||
assert_eq!(
|
||||
preview_job.preview_payload["normalization"]["normalizer_version"],
|
||||
"normalized-ir-v2"
|
||||
"normalized-ir-v3"
|
||||
);
|
||||
assert_eq!(
|
||||
preview_job.preview_payload["normalization"]["projection_version"],
|
||||
"preview-v2"
|
||||
"preview-v3"
|
||||
);
|
||||
assert!(preview_job.preview_payload["normalization"]["ir_fingerprint"].is_string());
|
||||
|
||||
@@ -236,7 +240,7 @@ async fn unknown_selected_operations_are_persisted_in_the_canonical_replay() {
|
||||
async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry,
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_replay"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
@@ -271,6 +275,31 @@ async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
||||
service.list_operations(&workspace_id).await.unwrap().len(),
|
||||
1
|
||||
);
|
||||
let completed_before = registry
|
||||
.get_import_job(&workspace_id, &job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let empty = serde_json::json!([]);
|
||||
let failed_at = time::OffsetDateTime::now_utc();
|
||||
registry
|
||||
.finish_import_job(crank_registry::FinishImportJobRequest {
|
||||
id: &job_id,
|
||||
status: ImportJobStatus::Failed,
|
||||
created_operation_ids: &empty,
|
||||
error_text: Some("late_verification_failure"),
|
||||
finished_at: &failed_at,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let completed_after = registry
|
||||
.get_import_job(&workspace_id, &job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed_after.status, ImportJobStatus::Completed);
|
||||
assert_eq!(completed_after.error_text, completed_before.error_text);
|
||||
assert_eq!(completed_after.finished_at, completed_before.finished_at);
|
||||
|
||||
let conflicting_replay = service
|
||||
.create_openapi_import(
|
||||
@@ -326,6 +355,13 @@ async fn apply_fails_closed_when_the_preview_parser_contract_drifts() {
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_failed_job_and_detached_sources(
|
||||
®istry,
|
||||
&workspace_id,
|
||||
&job_id,
|
||||
"import_replay_verification_failed",
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
service
|
||||
.list_operations(&workspace_id)
|
||||
@@ -395,6 +431,13 @@ async fn apply_fails_closed_for_each_normalization_contract_field_and_digest_sha
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_failed_job_and_detached_sources(
|
||||
®istry,
|
||||
&workspace_id,
|
||||
&job_id,
|
||||
"import_replay_verification_failed",
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
service
|
||||
.list_operations(&workspace_id)
|
||||
@@ -449,7 +492,7 @@ async fn legacy_job_without_normalization_contract_replays_until_expiry() {
|
||||
async fn blocker_finding_keeps_valid_preview_but_prevents_draft_mutation() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry,
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_blocker"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
@@ -503,11 +546,124 @@ paths:
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_failed_job_and_detached_sources(
|
||||
®istry,
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
"reference_resolution_blocked",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn operation_level_reference_blocker_prevents_draft_mutation() {
|
||||
async fn operation_blockers_apply_only_to_selected_keys_and_full_selection_fails_atomically() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_selected_blockers"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let upload = OpenApiUpload {
|
||||
bytes: br#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Selected blockers }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/valid:
|
||||
get:
|
||||
operationId: validOperation
|
||||
responses: { '200': { description: ok } }
|
||||
/broken:
|
||||
get:
|
||||
operationId: brokenOperation
|
||||
responses:
|
||||
'200':
|
||||
description: unresolved external schema
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: 'https://schemas.example.test/missing.yaml#/Result' }
|
||||
"#
|
||||
.to_vec(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
};
|
||||
|
||||
let valid_preview = service
|
||||
.preview_openapi_import(&workspace_id, upload.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
valid_preview.preview.findings.iter().all(|finding| {
|
||||
finding.severity != crank_import::rest::ImportFindingSeverity::Error
|
||||
})
|
||||
);
|
||||
let broken = valid_preview
|
||||
.preview
|
||||
.groups
|
||||
.iter()
|
||||
.flat_map(|group| &group.operations)
|
||||
.find(|operation| operation.key == "GET /broken")
|
||||
.unwrap();
|
||||
assert!(
|
||||
broken.findings.iter().any(|finding| {
|
||||
finding.severity == crank_import::rest::ImportFindingSeverity::Error
|
||||
})
|
||||
);
|
||||
|
||||
let valid = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&valid_preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /valid".to_owned()],
|
||||
server_url: None,
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(valid.created.len(), 1);
|
||||
|
||||
let full_preview = service
|
||||
.preview_openapi_import(&workspace_id, upload)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&full_preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /valid".to_owned(), "GET /broken".to_owned()],
|
||||
server_url: None,
|
||||
conflict_mode: "rename".to_owned(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
service.list_operations(&workspace_id).await.unwrap().len(),
|
||||
1,
|
||||
"the full selection must not create either selected Draft"
|
||||
);
|
||||
let failed = registry
|
||||
.get_import_job(&workspace_id, &full_preview.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(failed.status, ImportJobStatus::Failed);
|
||||
assert_eq!(
|
||||
failed.error_text.as_deref(),
|
||||
Some("reference_resolution_blocked")
|
||||
);
|
||||
assert!(failed.finished_at.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn local_operation_reference_resolves_and_creates_draft() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry,
|
||||
@@ -520,6 +676,7 @@ async fn operation_level_reference_blocker_prevents_draft_mutation() {
|
||||
bytes: br#"
|
||||
openapi: 3.0.3
|
||||
info: { title: References }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/referenced:
|
||||
get:
|
||||
@@ -551,14 +708,11 @@ components:
|
||||
preview.preview.groups[0].operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| {
|
||||
finding.code == "unresolved_reference"
|
||||
&& finding.severity == crank_import::rest::ImportFindingSeverity::Error
|
||||
})
|
||||
.all(|finding| finding.code != "unresolved_reference")
|
||||
);
|
||||
assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);
|
||||
|
||||
assert!(
|
||||
service
|
||||
let created = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
@@ -569,14 +723,11 @@ components:
|
||||
},
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
service
|
||||
.list_operations(&workspace_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
.unwrap();
|
||||
assert_eq!(created.created.len(), 1);
|
||||
assert_eq!(
|
||||
service.list_operations(&workspace_id).await.unwrap().len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
@@ -587,3 +738,44 @@ fn openapi_upload() -> OpenApiUpload {
|
||||
locale: OpenApiUploadLocale::En,
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_failed_job_and_detached_sources(
|
||||
registry: &crank_registry::PostgresRegistry,
|
||||
workspace_id: &WorkspaceId,
|
||||
job_id: &crank_registry::ImportJobId,
|
||||
expected_error: &str,
|
||||
) {
|
||||
let job = registry
|
||||
.get_import_job(workspace_id, job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(job.status, ImportJobStatus::Failed);
|
||||
assert!(job.finished_at.is_some());
|
||||
assert_eq!(job.error_text.as_deref(), Some(expected_error));
|
||||
|
||||
let mut source_ids = vec![
|
||||
job.preview_payload["source"]["source_id"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_owned(),
|
||||
];
|
||||
source_ids.extend(
|
||||
job.preview_payload["dependencies"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|dependency| dependency["source_id"].as_str().unwrap().to_owned()),
|
||||
);
|
||||
for source_id in source_ids {
|
||||
let lifecycle: String = sqlx::query_scalar(
|
||||
"select lifecycle from artifact_sources where workspace_id = $1 and source_id = $2",
|
||||
)
|
||||
.bind(workspace_id.as_str())
|
||||
.bind(source_id)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(lifecycle, "detached");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
use super::super::common::test_service_with_external_references;
|
||||
use super::*;
|
||||
use axum::{Router, routing::get};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn external_relative_chain_survives_reversed_builder_order_and_apply_never_refetches() {
|
||||
let fetches = Arc::new(AtomicUsize::new(0));
|
||||
let root_fetches = Arc::clone(&fetches);
|
||||
let child_fetches = Arc::clone(&fetches);
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/root.yaml",
|
||||
get(move || {
|
||||
let fetches = Arc::clone(&root_fetches);
|
||||
async move {
|
||||
fetches.fetch_add(1, Ordering::SeqCst);
|
||||
"Item: { $ref: './child.yaml#/Item' }"
|
||||
}
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/child.yaml",
|
||||
get(move || {
|
||||
let fetches = Arc::clone(&child_fetches);
|
||||
async move {
|
||||
fetches.fetch_add(1, Ordering::SeqCst);
|
||||
"Item: { type: object, required: [id], properties: { id: { type: string } } }"
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
|
||||
let registry = test_registry().await;
|
||||
let service = test_service_with_external_references(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_external_snapshot"),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let upload = OpenApiUpload {
|
||||
bytes: format!(
|
||||
r#"
|
||||
openapi: 3.1.0
|
||||
info: {{ title: External }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content:
|
||||
application/json:
|
||||
schema: {{ $ref: '{origin}/root.yaml#/Item' }}
|
||||
"#
|
||||
)
|
||||
.into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
};
|
||||
let preview = service
|
||||
.preview_openapi_import(&workspace_id, upload)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);
|
||||
let job = registry
|
||||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job.preview_payload["dependencies"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
job.preview_payload["dependency_snapshots"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
|
||||
let applied = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /items".to_owned()],
|
||||
server_url: None,
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(applied.created.len(), 1);
|
||||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||
let active_sources: i64 = sqlx::query_scalar(
|
||||
"select count(*) from artifact_sources
|
||||
where source_id like 'src_openapi_%' and lifecycle = 'active'",
|
||||
)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(active_sources, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn missing_external_snapshot_fails_closed_before_draft_mutation() {
|
||||
let app = Router::new().route(
|
||||
"/schemas.yaml",
|
||||
get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
let registry = test_registry().await;
|
||||
let storage_root = test_storage_root("openapi_import_missing_external_snapshot");
|
||||
let service = test_service_with_external_references(
|
||||
registry.clone(),
|
||||
storage_root.clone(),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: format!(
|
||||
r#"
|
||||
openapi: 3.1.0
|
||||
info: {{ title: External integrity }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||||
"#
|
||||
)
|
||||
.into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let job = registry
|
||||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let dependency_source_id = job.preview_payload["dependencies"][0]["source_id"]
|
||||
.as_str()
|
||||
.unwrap();
|
||||
sqlx::query(
|
||||
"update artifact_sources
|
||||
set lifecycle = 'detached', updated_at = now(), detached_at = now()
|
||||
where workspace_id = $1 and source_id = $2",
|
||||
)
|
||||
.bind(workspace_id.as_str())
|
||||
.bind(dependency_source_id)
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /items".to_owned()],
|
||||
server_url: None,
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
service
|
||||
.list_operations(&workspace_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_failed_job_and_detached_sources(
|
||||
®istry,
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
"import_dependency_verification_failed",
|
||||
)
|
||||
.await;
|
||||
|
||||
let corrupt_preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: format!(
|
||||
r#"
|
||||
openapi: 3.1.0
|
||||
info: {{ title: Corrupt external integrity }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItemsCorrupt
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||||
"#
|
||||
)
|
||||
.into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let corrupt_job = registry
|
||||
.get_import_job(&workspace_id, &corrupt_preview.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let dependency_ref: crank_artifacts::ArtifactRef = corrupt_job.preview_payload["dependencies"]
|
||||
[0]["digest"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.parse()
|
||||
.unwrap();
|
||||
let dependency_path = storage_root
|
||||
.join("sha256")
|
||||
.join(&dependency_ref.digest_hex()[..2])
|
||||
.join(dependency_ref.digest_hex());
|
||||
std::fs::set_permissions(
|
||||
&dependency_path,
|
||||
std::os::unix::fs::PermissionsExt::from_mode(0o600),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(&dependency_path, b"corrupt external dependency").unwrap();
|
||||
std::fs::set_permissions(
|
||||
&dependency_path,
|
||||
std::os::unix::fs::PermissionsExt::from_mode(0o400),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&corrupt_preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /items".to_owned()],
|
||||
server_url: None,
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_failed_job_and_detached_sources(
|
||||
®istry,
|
||||
&workspace_id,
|
||||
&corrupt_preview.job_id.as_str().into(),
|
||||
"import_dependency_verification_failed",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn external_materialization_uses_one_wall_clock_timeout_and_cleans_cancelled_sources() {
|
||||
let fetches = Arc::new(AtomicUsize::new(0));
|
||||
let root_fetches = Arc::clone(&fetches);
|
||||
let child_fetches = Arc::clone(&fetches);
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/root.yaml",
|
||||
get(move || {
|
||||
let fetches = Arc::clone(&root_fetches);
|
||||
async move {
|
||||
fetches.fetch_add(1, Ordering::SeqCst);
|
||||
"Item: { $ref: './slow-child.yaml#/Item' }"
|
||||
}
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/slow-child.yaml",
|
||||
get(move || {
|
||||
let fetches = Arc::clone(&child_fetches);
|
||||
async move {
|
||||
fetches.fetch_add(1, Ordering::SeqCst);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
"Item: { type: object, properties: { id: { type: string } } }"
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
|
||||
let registry = test_registry().await;
|
||||
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
|
||||
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
|
||||
let service = AdminServiceBuilder::new(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_chain_timeout"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
runtime,
|
||||
)
|
||||
.with_external_reference_import(&crank_config::ExternalReferenceSettings {
|
||||
allowed_url_prefixes: vec![format!("{origin}/")],
|
||||
max_depth: 8,
|
||||
max_documents: 32,
|
||||
max_fetch_bytes: 64 * 1024,
|
||||
fetch_timeout_ms: 200,
|
||||
max_expanded_nodes: 10_000,
|
||||
})
|
||||
.unwrap()
|
||||
.with_outbound_http_policy(outbound_policy)
|
||||
.build();
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let started = tokio::time::Instant::now();
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: format!(
|
||||
r#"
|
||||
openapi: 3.1.0
|
||||
info: {{ title: Timed chain }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: timedItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/root.yaml#/Item' }} }} }}
|
||||
"#
|
||||
)
|
||||
.into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(started.elapsed() < std::time::Duration::from_millis(800));
|
||||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||
let job = registry
|
||||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
job.preview_payload["dependencies"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
|
||||
for _ in 0..100 {
|
||||
let active_dependencies: i64 = sqlx::query_scalar(
|
||||
"select count(*) from artifact_sources
|
||||
where source_id like 'src_openapi_dep_%' and lifecycle = 'active'",
|
||||
)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
if active_dependencies == 0 {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
let detached_dependencies: i64 = sqlx::query_scalar(
|
||||
"select count(*) from artifact_sources
|
||||
where source_id like 'src_openapi_dep_%' and lifecycle = 'detached'",
|
||||
)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(detached_dependencies, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expired_job_detaches_primary_and_external_dependency_without_orphan() {
|
||||
let app = Router::new().route(
|
||||
"/schemas.yaml",
|
||||
get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
let registry = test_registry().await;
|
||||
let service = test_service_with_external_references(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_external_expiry"),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiUpload {
|
||||
bytes: format!(
|
||||
r#"
|
||||
openapi: 3.1.0
|
||||
info: {{ title: Expiring external }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||||
"#
|
||||
)
|
||||
.into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("update import_jobs set expires_at = now() - interval '1 second' where id = $1")
|
||||
.bind(preview.job_id.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
let report = registry.cleanup_expired_import_jobs(16).await.unwrap();
|
||||
assert_eq!(report.deleted_jobs, 1);
|
||||
assert_eq!(report.detached_sources, 2);
|
||||
let active_sources: i64 = sqlx::query_scalar(
|
||||
"select count(*) from artifact_sources
|
||||
where source_id like 'src_openapi_%' and lifecycle = 'active'",
|
||||
)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(active_sources, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn exact_v2_contract_replays_persisted_preview_without_v3_or_dependency_reads() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_v2_compatibility"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let preview = service
|
||||
.preview_openapi_import(&workspace_id, openapi_upload())
|
||||
.await
|
||||
.unwrap();
|
||||
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||||
let missing_dependency = serde_json::json!([{
|
||||
"source_id": "src_openapi_dep_v2_must_not_be_read",
|
||||
"digest": format!("sha256:{}", "0".repeat(64)),
|
||||
"canonical_uri": "https://schemas.example.test/v2.yaml"
|
||||
}]);
|
||||
sqlx::query(
|
||||
"update import_jobs
|
||||
set preview_payload = jsonb_set(
|
||||
jsonb_set(
|
||||
jsonb_set(
|
||||
jsonb_set(
|
||||
preview_payload,
|
||||
'{normalization,normalizer_version}',
|
||||
to_jsonb('normalized-ir-v2'::text)
|
||||
),
|
||||
'{normalization,projection_version}',
|
||||
to_jsonb('preview-v2'::text)
|
||||
),
|
||||
'{normalization,ir_fingerprint}',
|
||||
to_jsonb($1::text)
|
||||
),
|
||||
'{dependency_snapshots}',
|
||||
$2::jsonb
|
||||
)
|
||||
where id = $3",
|
||||
)
|
||||
.bind("a".repeat(64))
|
||||
.bind(missing_dependency)
|
||||
.bind(job_id.as_str())
|
||||
.execute(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&job_id,
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.created.len(), 1);
|
||||
let completed = registry
|
||||
.get_import_job(&workspace_id, &job_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(completed.status, ImportJobStatus::Completed);
|
||||
}
|
||||
@@ -1,30 +1,18 @@
|
||||
use std::{
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
||||
use crank_artifacts::MAX_ARTIFACT_BYTES;
|
||||
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
||||
use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
use opentelemetry::trace::TracerProvider as _;
|
||||
use opentelemetry_sdk::{
|
||||
error::OTelSdkResult,
|
||||
trace::{SdkTracerProvider, SpanData, SpanExporter},
|
||||
};
|
||||
use reqwest::multipart::{Form, Part};
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
use super::common::{
|
||||
authorized_client, build_test_app, spawn_admin_api, test_auth_settings, test_registry,
|
||||
test_secret_crypto, test_service, test_storage_root,
|
||||
};
|
||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiUpload, OpenApiUploadLocale};
|
||||
use crank_artifacts::MAX_ARTIFACT_BYTES;
|
||||
use crank_core::{Workspace, WorkspaceId, WorkspaceStatus};
|
||||
use crank_registry::{ArtifactSourceId, ArtifactSourceLifecycle, CreateWorkspaceRequest};
|
||||
use reqwest::multipart::{Form, Part};
|
||||
use serde_json::Value;
|
||||
use serial_test::serial;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
|
||||
mod apply_failures;
|
||||
mod telemetry;
|
||||
|
||||
const OPENAPI: &str = r#"
|
||||
openapi: 3.0.3
|
||||
@@ -136,78 +124,6 @@ async fn multipart_requires_an_owner_membership_before_reading_the_file() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
|
||||
const CANARY: &str = "openapi-telemetry-secret-canary";
|
||||
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
recorder
|
||||
.install()
|
||||
.expect("isolated integration test metrics recorder");
|
||||
|
||||
let writer = SharedLogWriter::default();
|
||||
let exported = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = SdkTracerProvider::builder()
|
||||
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
|
||||
.build();
|
||||
let tracer = provider.tracer("admin-openapi-source-test");
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(tracing_subscriber::fmt::layer().with_writer(writer.clone()))
|
||||
.with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
tracing::dispatcher::set_global_default(dispatch)
|
||||
.expect("isolated integration test tracing subscriber");
|
||||
|
||||
let registry = test_registry().await;
|
||||
let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary"));
|
||||
let server = spawn_admin_api(app).await;
|
||||
let client = authorized_client(&server).await;
|
||||
let document = format!(
|
||||
"openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n /broken:\n get:\n description: {CANARY}\n responses: ["
|
||||
);
|
||||
let response = client
|
||||
.post(format!("{server}/imports/openapi/preview"))
|
||||
.multipart(Form::new().part(
|
||||
"file",
|
||||
file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
let trace_id = response
|
||||
.headers()
|
||||
.get("x-trace-id")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let body = response.text().await.unwrap();
|
||||
assert!(!body.contains(CANARY));
|
||||
let diagnostics: Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(diagnostics["error"]["trace_id"], trace_id);
|
||||
|
||||
provider.force_flush().unwrap();
|
||||
let logs = writer.output();
|
||||
assert!(!logs.contains(CANARY));
|
||||
assert!(logs.contains(&trace_id));
|
||||
let spans = exported.lock().unwrap();
|
||||
let rendered_spans = format!("{spans:?}");
|
||||
assert!(!rendered_spans.contains(CANARY));
|
||||
drop(spans);
|
||||
for (key, _, _, _) in snapshotter.snapshot().into_vec() {
|
||||
assert!(!key.key().name().contains(CANARY));
|
||||
assert!(
|
||||
!key.key()
|
||||
.labels()
|
||||
.any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) })
|
||||
);
|
||||
}
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn multipart_boundary_rejections_are_localized_and_leave_no_entities() {
|
||||
@@ -939,49 +855,3 @@ async fn wait_for_specific_source_lifecycle(
|
||||
}
|
||||
panic!("artifact source {source_id:?} did not reach {expected}");
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
||||
|
||||
impl SpanExporter for CapturingExporter {
|
||||
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
|
||||
self.0.lock().unwrap().extend(batch);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
use super::super::common::build_test_app_with_external_references;
|
||||
use super::*;
|
||||
use std::{
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
use opentelemetry::trace::TracerProvider as _;
|
||||
use opentelemetry_sdk::{
|
||||
error::OTelSdkResult,
|
||||
trace::{SdkTracerProvider, SpanData, SpanExporter},
|
||||
};
|
||||
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn parser_canary_never_reaches_diagnostics_logs_traces_or_metrics() {
|
||||
const CANARY: &str = "openapi-telemetry-secret-canary";
|
||||
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
recorder
|
||||
.install()
|
||||
.expect("isolated integration test metrics recorder");
|
||||
|
||||
let writer = SharedLogWriter::default();
|
||||
let exported = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = SdkTracerProvider::builder()
|
||||
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
|
||||
.build();
|
||||
let tracer = provider.tracer("admin-openapi-source-test");
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(tracing_subscriber::fmt::layer().with_writer(writer.clone()))
|
||||
.with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
tracing::dispatcher::set_global_default(dispatch)
|
||||
.expect("isolated integration test tracing subscriber");
|
||||
|
||||
let registry = test_registry().await;
|
||||
let app = build_test_app(registry, test_storage_root("openapi_telemetry_canary"));
|
||||
let server = spawn_admin_api(app).await;
|
||||
let client = authorized_client(&server).await;
|
||||
let document = format!(
|
||||
"openapi: 3.0.3\ninfo: {{ title: Canary }}\npaths:\n /broken:\n get:\n description: {CANARY}\n responses: ["
|
||||
);
|
||||
let response = client
|
||||
.post(format!("{server}/imports/openapi/preview"))
|
||||
.multipart(Form::new().part(
|
||||
"file",
|
||||
file_part(document.as_bytes(), "openapi.yaml", "application/yaml"),
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
let trace_id = response
|
||||
.headers()
|
||||
.get("x-trace-id")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let body = response.text().await.unwrap();
|
||||
assert!(!body.contains(CANARY));
|
||||
let diagnostics: Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(diagnostics["error"]["trace_id"], trace_id);
|
||||
|
||||
let route = format!("/{CANARY}.yaml");
|
||||
let external = axum::Router::new().route(
|
||||
&route,
|
||||
axum::routing::get(|| async { (axum::http::StatusCode::INTERNAL_SERVER_ERROR, CANARY) }),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move { axum::serve(listener, external).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
let external_registry = test_registry().await;
|
||||
let external_app = build_test_app_with_external_references(
|
||||
external_registry,
|
||||
test_storage_root("openapi_materialization_telemetry_canary"),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let external_server = spawn_admin_api(external_app).await;
|
||||
let external_client = authorized_client(&external_server).await;
|
||||
let external_document = format!(
|
||||
r#"
|
||||
openapi: 3.1.0
|
||||
info: {{ title: Safe materialization }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content:
|
||||
application/json:
|
||||
schema: {{ $ref: '{origin}/{CANARY}.yaml#/Item' }}
|
||||
"#
|
||||
);
|
||||
let response = external_client
|
||||
.post(format!("{external_server}/imports/openapi/preview"))
|
||||
.multipart(Form::new().part(
|
||||
"file",
|
||||
file_part(
|
||||
external_document.as_bytes(),
|
||||
"external.yaml",
|
||||
"application/yaml",
|
||||
),
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||
|
||||
provider.force_flush().unwrap();
|
||||
let logs = writer.output();
|
||||
assert!(!logs.contains(CANARY));
|
||||
assert!(logs.contains(&trace_id));
|
||||
assert!(logs.contains("external OpenAPI materialization failed"));
|
||||
assert!(logs.contains("stage=\"fetch\""));
|
||||
assert!(logs.contains("error_code=\"unexpected_status\""));
|
||||
assert!(logs.contains("count=0"));
|
||||
let spans = exported.lock().unwrap();
|
||||
let rendered_spans = format!("{spans:?}");
|
||||
assert!(!rendered_spans.contains(CANARY));
|
||||
drop(spans);
|
||||
for (key, _, _, _) in snapshotter.snapshot().into_vec() {
|
||||
assert!(!key.key().name().contains(CANARY));
|
||||
assert!(
|
||||
!key.key()
|
||||
.labels()
|
||||
.any(|label| { label.key().contains(CANARY) || label.value().contains(CANARY) })
|
||||
);
|
||||
}
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
||||
|
||||
impl SpanExporter for CapturingExporter {
|
||||
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
|
||||
self.0.lock().unwrap().extend(batch);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,7 @@ use serde_json::Value;
|
||||
use tracing::{Instrument, Span};
|
||||
use tracing_opentelemetry::OpenTelemetrySpanExt;
|
||||
|
||||
use crate::{RestAdapterError, RestRequest, RestResponse};
|
||||
use crate::{ExternalReferenceFetchError, RestAdapterError, RestRequest, RestResponse};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RestAdapter {
|
||||
@@ -33,6 +33,18 @@ pub struct RestAdapter {
|
||||
policy: OutboundHttpPolicy,
|
||||
}
|
||||
|
||||
/// A deliberately separate, GET-only boundary for materializing external
|
||||
/// OpenAPI documents. It has no execution metrics, tracing propagation, or
|
||||
/// request construction semantics from [`RestAdapter`].
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ExternalReferenceFetcher {
|
||||
client: Result<Client, Arc<str>>,
|
||||
policy: OutboundHttpPolicy,
|
||||
allowed_url_prefixes: Vec<String>,
|
||||
max_response_bytes: usize,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct OutboundHttpPolicy {
|
||||
allowed_hosts: Vec<String>,
|
||||
@@ -56,15 +68,7 @@ impl RestAdapter {
|
||||
}
|
||||
|
||||
pub fn with_policy(policy: OutboundHttpPolicy) -> Self {
|
||||
let resolver = Arc::new(PolicyDnsResolver {
|
||||
policy: policy.clone(),
|
||||
});
|
||||
let client = Client::builder()
|
||||
.redirect(redirect::Policy::none())
|
||||
.no_proxy()
|
||||
.dns_resolver(resolver)
|
||||
.build()
|
||||
.map_err(|error| Arc::<str>::from(error.to_string()));
|
||||
let client = outbound_client(&policy);
|
||||
|
||||
Self { client, policy }
|
||||
}
|
||||
@@ -195,6 +199,153 @@ impl RestAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
impl ExternalReferenceFetcher {
|
||||
pub fn try_new(
|
||||
policy: OutboundHttpPolicy,
|
||||
allowed_url_prefixes: Vec<String>,
|
||||
max_response_bytes: usize,
|
||||
timeout: Duration,
|
||||
) -> Result<Self, ExternalReferenceFetchError> {
|
||||
if max_response_bytes == 0 || timeout.is_zero() {
|
||||
return Err(ExternalReferenceFetchError::InvalidConfiguration);
|
||||
}
|
||||
let allowed_url_prefixes = allowed_url_prefixes
|
||||
.into_iter()
|
||||
.map(|prefix| canonical_external_reference_prefix(&prefix))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(Self {
|
||||
client: outbound_client(&policy),
|
||||
policy,
|
||||
allowed_url_prefixes,
|
||||
max_response_bytes,
|
||||
timeout,
|
||||
})
|
||||
}
|
||||
|
||||
/// Fetches one document with a bounded, headerless `GET`.
|
||||
///
|
||||
/// URL fragments are stripped because they address JSON Pointer targets in
|
||||
/// the fetched document rather than a network resource.
|
||||
pub async fn get(&self, url: &str) -> Result<Vec<u8>, ExternalReferenceFetchError> {
|
||||
if self.allowed_url_prefixes.is_empty() {
|
||||
return Err(ExternalReferenceFetchError::Disabled);
|
||||
}
|
||||
let mut url =
|
||||
reqwest::Url::parse(url).map_err(|_| ExternalReferenceFetchError::InvalidUrl)?;
|
||||
url.set_fragment(None);
|
||||
if !self
|
||||
.allowed_url_prefixes
|
||||
.iter()
|
||||
.any(|prefix| matches_external_reference_prefix(&url, prefix))
|
||||
{
|
||||
return Err(ExternalReferenceFetchError::TargetNotAllowed);
|
||||
}
|
||||
self.policy
|
||||
.validate_url(&url)
|
||||
.map_err(external_policy_error)?;
|
||||
let client = self
|
||||
.client
|
||||
.as_ref()
|
||||
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
|
||||
let response = client
|
||||
.get(url)
|
||||
.timeout(self.timeout)
|
||||
.send()
|
||||
.await
|
||||
.map_err(external_transport_error)?;
|
||||
let status = response.status();
|
||||
if status.is_redirection() {
|
||||
return Err(ExternalReferenceFetchError::RedirectNotAllowed);
|
||||
}
|
||||
if !status.is_success() {
|
||||
return Err(ExternalReferenceFetchError::UnexpectedStatus {
|
||||
status: status.as_u16(),
|
||||
});
|
||||
}
|
||||
read_external_response_bytes(response, self.max_response_bytes).await
|
||||
}
|
||||
}
|
||||
|
||||
fn outbound_client(policy: &OutboundHttpPolicy) -> Result<Client, Arc<str>> {
|
||||
let resolver = Arc::new(PolicyDnsResolver {
|
||||
policy: policy.clone(),
|
||||
});
|
||||
Client::builder()
|
||||
.redirect(redirect::Policy::none())
|
||||
.no_proxy()
|
||||
.dns_resolver(resolver)
|
||||
.build()
|
||||
.map_err(|error| Arc::<str>::from(error.to_string()))
|
||||
}
|
||||
|
||||
fn canonical_external_reference_prefix(
|
||||
prefix: &str,
|
||||
) -> Result<String, ExternalReferenceFetchError> {
|
||||
let url = reqwest::Url::parse(prefix)
|
||||
.map_err(|_| ExternalReferenceFetchError::InvalidConfiguration)?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| url.host_str().is_none()
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
{
|
||||
return Err(ExternalReferenceFetchError::InvalidConfiguration);
|
||||
}
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
fn matches_external_reference_prefix(url: &reqwest::Url, prefix: &str) -> bool {
|
||||
let url = url.as_str();
|
||||
if !url.starts_with(prefix) {
|
||||
return false;
|
||||
}
|
||||
let Some(next) = url.as_bytes().get(prefix.len()) else {
|
||||
return true;
|
||||
};
|
||||
prefix.ends_with('/') || matches!(next, b'/' | b'?')
|
||||
}
|
||||
|
||||
fn external_policy_error(error: RestAdapterError) -> ExternalReferenceFetchError {
|
||||
match error {
|
||||
RestAdapterError::TargetNotAllowed { .. } => ExternalReferenceFetchError::TargetNotAllowed,
|
||||
_ => ExternalReferenceFetchError::InvalidConfiguration,
|
||||
}
|
||||
}
|
||||
|
||||
fn external_transport_error(error: reqwest::Error) -> ExternalReferenceFetchError {
|
||||
ExternalReferenceFetchError::Transport {
|
||||
timeout: error.is_timeout(),
|
||||
connect: error.is_connect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_external_response_bytes(
|
||||
response: reqwest::Response,
|
||||
max_response_bytes: usize,
|
||||
) -> Result<Vec<u8>, ExternalReferenceFetchError> {
|
||||
if response
|
||||
.content_length()
|
||||
.is_some_and(|length| length > max_response_bytes as u64)
|
||||
{
|
||||
return Err(ExternalReferenceFetchError::ResponseTooLarge {
|
||||
limit_bytes: max_response_bytes,
|
||||
});
|
||||
}
|
||||
let mut stream = response.bytes_stream();
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(external_transport_error)?;
|
||||
if bytes.len().saturating_add(chunk.len()) > max_response_bytes {
|
||||
return Err(ExternalReferenceFetchError::ResponseTooLarge {
|
||||
limit_bytes: max_response_bytes,
|
||||
});
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn upstream_outcome(error: &RestAdapterError) -> UpstreamOutcome {
|
||||
match error {
|
||||
RestAdapterError::UnexpectedStatus { status, .. } if (400..500).contains(status) => {
|
||||
|
||||
@@ -1,6 +1,26 @@
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum ExternalReferenceFetchError {
|
||||
#[error("external references are disabled")]
|
||||
Disabled,
|
||||
#[error("external reference URL is invalid")]
|
||||
InvalidUrl,
|
||||
#[error("external reference target is not allowed")]
|
||||
TargetNotAllowed,
|
||||
#[error("external reference redirects are not allowed")]
|
||||
RedirectNotAllowed,
|
||||
#[error("external reference response exceeds the configured limit of {limit_bytes} bytes")]
|
||||
ResponseTooLarge { limit_bytes: usize },
|
||||
#[error("external reference endpoint returned status {status}")]
|
||||
UnexpectedStatus { status: u16 },
|
||||
#[error("external reference request failed")]
|
||||
Transport { timeout: bool, connect: bool },
|
||||
#[error("external reference fetch configuration is invalid")]
|
||||
InvalidConfiguration,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum RestAdapterError {
|
||||
#[error("invalid base url: {url}")]
|
||||
|
||||
@@ -8,8 +8,8 @@ use crank_core::{
|
||||
ProtocolAdapterError, RestTarget, RuntimeRequestContext, Target,
|
||||
};
|
||||
|
||||
pub use client::{OutboundHttpPolicy, RestAdapter};
|
||||
pub use error::RestAdapterError;
|
||||
pub use client::{ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter};
|
||||
pub use error::{ExternalReferenceFetchError, RestAdapterError};
|
||||
pub use model::{RestRequest, RestResponse};
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
mod integration {
|
||||
mod client;
|
||||
mod external_reference_fetcher;
|
||||
mod outbound_security;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{Router, http::StatusCode, response::Redirect, routing::get};
|
||||
use crank_adapter_rest::{
|
||||
ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy,
|
||||
};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[tokio::test]
|
||||
async fn external_references_are_default_off_before_any_request() {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let base_url = spawn_server(Arc::clone(&requests)).await;
|
||||
let fetcher = ExternalReferenceFetcher::try_new(
|
||||
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
Vec::new(),
|
||||
1024,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let error = fetcher
|
||||
.get(&format!("{base_url}/document"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(error, ExternalReferenceFetchError::Disabled));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetcher_uses_prefix_and_actual_address_policy_then_returns_bounded_bytes() {
|
||||
let base_url = spawn_server(Arc::new(AtomicUsize::new(0))).await;
|
||||
let fetcher = ExternalReferenceFetcher::try_new(
|
||||
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
vec![base_url.clone()],
|
||||
8,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
fetcher
|
||||
.get(&format!("{base_url}/document#/components/schemas/A"))
|
||||
.await
|
||||
.unwrap(),
|
||||
b"openapi".to_vec()
|
||||
);
|
||||
let error = fetcher.get(&format!("{base_url}/large")).await.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
ExternalReferenceFetchError::ResponseTooLarge { limit_bytes: 8 }
|
||||
));
|
||||
|
||||
let private_without_explicit_outbound_allow = ExternalReferenceFetcher::try_new(
|
||||
OutboundHttpPolicy::default(),
|
||||
vec![base_url.clone()],
|
||||
1024,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.unwrap();
|
||||
let error = private_without_explicit_outbound_allow
|
||||
.get(&format!("{base_url}/document"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
ExternalReferenceFetchError::TargetNotAllowed
|
||||
));
|
||||
|
||||
let exact_path_fetcher = ExternalReferenceFetcher::try_new(
|
||||
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
vec![format!("{base_url}/document")],
|
||||
1024,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.unwrap();
|
||||
let error = exact_path_fetcher
|
||||
.get(&format!("{base_url}/document-unrelated"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
ExternalReferenceFetchError::TargetNotAllowed
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetcher_rejects_redirects_and_userinfo_without_exposing_the_url() {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let base_url = spawn_server(Arc::clone(&requests)).await;
|
||||
let fetcher = ExternalReferenceFetcher::try_new(
|
||||
OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
vec![base_url.clone()],
|
||||
1024,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let error = fetcher
|
||||
.get(&format!("{base_url}/redirect"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
ExternalReferenceFetchError::RedirectNotAllowed
|
||||
));
|
||||
|
||||
let userinfo_url = base_url.replacen("http://", "http://user:credential@", 1);
|
||||
let error = fetcher
|
||||
.get(&format!("{userinfo_url}/document"))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
ExternalReferenceFetchError::TargetNotAllowed
|
||||
));
|
||||
let rendered = format!("{error:?} {error}");
|
||||
assert!(!rendered.contains("credential"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
async fn spawn_server(requests: Arc<AtomicUsize>) -> String {
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/document",
|
||||
get({
|
||||
let requests = Arc::clone(&requests);
|
||||
move || {
|
||||
let requests = Arc::clone(&requests);
|
||||
async move {
|
||||
requests.fetch_add(1, Ordering::SeqCst);
|
||||
(StatusCode::OK, "openapi")
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.route("/large", get(|| async { "response too large" }))
|
||||
.route("/redirect", get(|| async { Redirect::to("/document") }));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{address}")
|
||||
}
|
||||
@@ -6,6 +6,7 @@ use std::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
@@ -14,7 +15,9 @@ use axum::{
|
||||
http::StatusCode,
|
||||
routing::{any, post},
|
||||
};
|
||||
use crank_adapter_rest::{OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest};
|
||||
use crank_adapter_rest::{
|
||||
ExternalReferenceFetcher, OutboundHttpPolicy, RestAdapter, RestAdapterError, RestRequest,
|
||||
};
|
||||
use crank_core::{HttpMethod, ProtocolAdapterError, RestTarget};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::net::TcpListener;
|
||||
@@ -94,7 +97,7 @@ async fn proxy_environment_is_ignored_by_default() {
|
||||
let adapter = RestAdapter::default();
|
||||
let request = json_request(json!({"payload": "proxy-env-canary"}));
|
||||
let rest_target = RestTarget {
|
||||
base_url: target,
|
||||
base_url: target.clone(),
|
||||
method: HttpMethod::Post,
|
||||
path_template: "/capture".to_owned(),
|
||||
static_headers: BTreeMap::new(),
|
||||
@@ -102,6 +105,17 @@ async fn proxy_environment_is_ignored_by_default() {
|
||||
let _ = adapter.execute(&rest_target, &request).await.expect_err(
|
||||
"unresolvable target should fail locally instead of being sent through proxy env",
|
||||
);
|
||||
let fetcher = ExternalReferenceFetcher::try_new(
|
||||
OutboundHttpPolicy::default(),
|
||||
vec!["http://public.example.test/".to_owned()],
|
||||
1024,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.expect("valid external reference fetcher");
|
||||
let _ = fetcher
|
||||
.get(&target)
|
||||
.await
|
||||
.expect_err("external reference fetcher must not use proxy environment variables");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
use std::fmt;
|
||||
|
||||
use crate::{
|
||||
AdminProcessConfig, CacheSettings, DatabaseSettings, McpProcessConfig, MetricsSettings,
|
||||
MigratorConfig, ObservabilitySettings, OtlpSettings, OutboundSettings, RuntimeSettings,
|
||||
AdminProcessConfig, CacheSettings, DatabaseSettings, ExternalReferenceSettings,
|
||||
McpProcessConfig, MetricsSettings, MigratorConfig, ObservabilitySettings, OtlpSettings,
|
||||
OutboundSettings, RuntimeSettings,
|
||||
};
|
||||
|
||||
impl fmt::Debug for DatabaseSettings {
|
||||
@@ -32,6 +33,18 @@ impl fmt::Debug for OutboundSettings {
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
impl fmt::Debug for ExternalReferenceSettings {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("ExternalReferenceSettings")
|
||||
.field("allowed_url_prefix_count", &self.allowed_url_prefixes.len())
|
||||
.field("max_depth", &self.max_depth)
|
||||
.field("max_documents", &self.max_documents)
|
||||
.field("max_fetch_bytes", &self.max_fetch_bytes)
|
||||
.field("fetch_timeout_ms", &self.fetch_timeout_ms)
|
||||
.field("max_expanded_nodes", &self.max_expanded_nodes)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
impl fmt::Debug for RuntimeSettings {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RuntimeSettings")
|
||||
@@ -93,6 +106,7 @@ impl fmt::Debug for AdminProcessConfig {
|
||||
f.debug_struct("AdminProcessConfig")
|
||||
.field("database", &self.database)
|
||||
.field("runtime", &self.runtime)
|
||||
.field("external_references", &self.external_references)
|
||||
.field("observability", &self.observability)
|
||||
.field("storage_root", &"configured")
|
||||
.field("session_secret", &self.session_secret)
|
||||
|
||||
@@ -15,8 +15,9 @@ pub use diagnostic::{ConfigError, Diagnostic, DiagnosticCode};
|
||||
pub use migrator::{MigratorConfig, parse_migrator};
|
||||
pub use process::{
|
||||
AdminProcessConfig, CacheBackend, CacheSettings, DatabaseSettings, DeprecationRecord,
|
||||
EffectiveConfig, McpProcessConfig, MetricsSettings, ObservabilitySettings, OtlpSettings,
|
||||
OutboundSettings, PoolSettings, ProcessKind, RateLimitSettings, RuntimeSettings, parse_process,
|
||||
EffectiveConfig, ExternalReferenceSettings, McpProcessConfig, MetricsSettings,
|
||||
ObservabilitySettings, OtlpSettings, OutboundSettings, PoolSettings, ProcessKind,
|
||||
RateLimitSettings, RuntimeSettings, parse_process,
|
||||
};
|
||||
pub use schema::{
|
||||
FieldMode, FieldSpec, ProcessScope, Sensitivity, deployment_field_registry, field_registry,
|
||||
|
||||
@@ -12,7 +12,9 @@ use std::{
|
||||
path::PathBuf,
|
||||
};
|
||||
use url::Url;
|
||||
mod external_references;
|
||||
mod list_parsers;
|
||||
pub use external_references::ExternalReferenceSettings;
|
||||
const MAX_ENV_VALUE_BYTES: usize = 8_192;
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ProcessKind {
|
||||
@@ -127,6 +129,7 @@ pub struct ObservabilitySettings {
|
||||
pub struct AdminProcessConfig {
|
||||
pub database: DatabaseSettings,
|
||||
pub runtime: RuntimeSettings,
|
||||
pub external_references: ExternalReferenceSettings,
|
||||
pub observability: ObservabilitySettings,
|
||||
pub bind_addr: SocketAddr,
|
||||
pub storage_root: PathBuf,
|
||||
@@ -154,8 +157,8 @@ pub struct McpProcessConfig {
|
||||
|
||||
#[derive(Clone)]
|
||||
enum Projection {
|
||||
Admin(AdminProcessConfig),
|
||||
Mcp(McpProcessConfig),
|
||||
Admin(Box<AdminProcessConfig>),
|
||||
Mcp(Box<McpProcessConfig>),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -790,9 +793,10 @@ pub fn parse_process(
|
||||
{
|
||||
parser.push(DiagnosticCode::UnsafeCombination, "admin.exposure.tls");
|
||||
}
|
||||
Projection::Admin(AdminProcessConfig {
|
||||
Projection::Admin(Box::new(AdminProcessConfig {
|
||||
database,
|
||||
runtime,
|
||||
external_references: external_references::parse(&mut parser),
|
||||
observability,
|
||||
bind_addr,
|
||||
storage_root: parser.absolute_path("CRANK_STORAGE_ROOT", "/var/lib/crank/storage"),
|
||||
@@ -811,7 +815,7 @@ pub fn parse_process(
|
||||
bootstrap_display_name: parser
|
||||
.string("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", Some("Crank Owner")),
|
||||
demo_seed: parser.boolean("CRANK_DEMO_SEED"),
|
||||
})
|
||||
}))
|
||||
}
|
||||
ProcessKind::McpServer => {
|
||||
let rps = parser.number("CRANK_MCP_RATE_LIMIT_RPS") as u32;
|
||||
@@ -819,7 +823,7 @@ pub fn parse_process(
|
||||
if burst < rps {
|
||||
parser.push(DiagnosticCode::UnsafeCombination, "mcp.rate_limit.burst");
|
||||
}
|
||||
Projection::Mcp(McpProcessConfig {
|
||||
Projection::Mcp(Box::new(McpProcessConfig {
|
||||
database,
|
||||
runtime,
|
||||
observability,
|
||||
@@ -829,7 +833,7 @@ pub fn parse_process(
|
||||
requests_per_second: rps,
|
||||
burst,
|
||||
},
|
||||
})
|
||||
}))
|
||||
}
|
||||
};
|
||||
|
||||
@@ -867,6 +871,18 @@ fn fingerprint_parts(kind: ProcessKind, projection: &Projection) -> Vec<String>
|
||||
format!("session_ttl={}", config.session_ttl_hours),
|
||||
format!("trusted_proxies={:?}", config.trusted_proxy_ips),
|
||||
format!("demo={}", config.demo_seed),
|
||||
format!(
|
||||
"external_reference_prefixes={}",
|
||||
config.external_references.allowed_url_prefixes.join(",")
|
||||
),
|
||||
format!(
|
||||
"external_reference_limits={}:{}:{}:{}:{}",
|
||||
config.external_references.max_depth,
|
||||
config.external_references.max_documents,
|
||||
config.external_references.max_fetch_bytes,
|
||||
config.external_references.fetch_timeout_ms,
|
||||
config.external_references.max_expanded_nodes,
|
||||
),
|
||||
"storage=path-configured".to_owned(),
|
||||
format!("session_secret={}", config.session_secret.is_configured()),
|
||||
format!("pepper={}", config.password_pepper.is_configured()),
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
#[derive(Clone, Eq, PartialEq)]
|
||||
pub struct ExternalReferenceSettings {
|
||||
/// Canonical HTTP(S) URL prefixes which opt an operator into remote `$ref` fetches.
|
||||
/// An empty list is a deliberate default-deny switch.
|
||||
pub allowed_url_prefixes: Vec<String>,
|
||||
pub max_depth: usize,
|
||||
pub max_documents: usize,
|
||||
pub max_fetch_bytes: usize,
|
||||
pub fetch_timeout_ms: u64,
|
||||
pub max_expanded_nodes: usize,
|
||||
}
|
||||
|
||||
pub(super) fn parse(parser: &mut super::Parser<'_>) -> ExternalReferenceSettings {
|
||||
ExternalReferenceSettings {
|
||||
allowed_url_prefixes: parser
|
||||
.url_prefix_list("CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES"),
|
||||
max_depth: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH") as usize,
|
||||
max_documents: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS") as usize,
|
||||
max_fetch_bytes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES") as usize,
|
||||
fetch_timeout_ms: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS"),
|
||||
max_expanded_nodes: parser.number("CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES")
|
||||
as usize,
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use crate::{
|
||||
DiagnosticCode,
|
||||
validation::{parse_host_list, parse_ip_list},
|
||||
};
|
||||
use url::Url;
|
||||
|
||||
impl super::Parser<'_> {
|
||||
pub(super) fn host_list(&mut self, name: &'static str) -> Vec<String> {
|
||||
@@ -33,4 +34,41 @@ impl super::Parser<'_> {
|
||||
}
|
||||
parsed.items
|
||||
}
|
||||
|
||||
pub(super) fn url_prefix_list(&mut self, name: &'static str) -> Vec<String> {
|
||||
let Some(raw) = self.optional(name) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut prefixes = Vec::new();
|
||||
let mut invalid = false;
|
||||
for item in raw.split(',') {
|
||||
let item = item.trim();
|
||||
let Ok(url) = Url::parse(item) else {
|
||||
invalid = true;
|
||||
continue;
|
||||
};
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
|| url.host_str().is_none()
|
||||
|| !url.username().is_empty()
|
||||
|| url.password().is_some()
|
||||
|| url.query().is_some()
|
||||
|| url.fragment().is_some()
|
||||
{
|
||||
invalid = true;
|
||||
continue;
|
||||
}
|
||||
let canonical = url.to_string();
|
||||
if !prefixes.contains(&canonical) {
|
||||
prefixes.push(canonical);
|
||||
}
|
||||
}
|
||||
if invalid {
|
||||
self.push(DiagnosticCode::InvalidType, name);
|
||||
}
|
||||
if prefixes.len() > 64 {
|
||||
self.push(DiagnosticCode::OutOfRange, name);
|
||||
prefixes.truncate(64);
|
||||
}
|
||||
prefixes
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ macro_rules! f {
|
||||
};
|
||||
}
|
||||
|
||||
static FIELDS: [FieldSpec; 59] = [
|
||||
static FIELDS: [FieldSpec; 65] = [
|
||||
FieldSpec {
|
||||
compatibility: Some("legacy URL form"),
|
||||
rules: &[
|
||||
@@ -339,6 +339,78 @@ static FIELDS: [FieldSpec; 59] = [
|
||||
Some(67108864),
|
||||
Public
|
||||
),
|
||||
FieldSpec {
|
||||
rules: &[
|
||||
"empty list disables external OpenAPI reference fetching",
|
||||
"each prefix must be canonical HTTP(S) without userinfo, query, or fragment",
|
||||
],
|
||||
..f!(
|
||||
"import.external_references.allowed_url_prefixes",
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES",
|
||||
AdminApi,
|
||||
"url_prefix_list",
|
||||
None,
|
||||
Some(""),
|
||||
None,
|
||||
None,
|
||||
Internal
|
||||
)
|
||||
},
|
||||
f!(
|
||||
"import.external_references.max_depth",
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH",
|
||||
AdminApi,
|
||||
"u32",
|
||||
Some("edges"),
|
||||
Some("8"),
|
||||
Some(1),
|
||||
Some(32),
|
||||
Public
|
||||
),
|
||||
f!(
|
||||
"import.external_references.max_documents",
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS",
|
||||
AdminApi,
|
||||
"u32",
|
||||
Some("documents"),
|
||||
Some("32"),
|
||||
Some(1),
|
||||
Some(32),
|
||||
Public
|
||||
),
|
||||
f!(
|
||||
"import.external_references.max_fetch_bytes",
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES",
|
||||
AdminApi,
|
||||
"u64",
|
||||
Some("bytes"),
|
||||
Some("262144"),
|
||||
Some(1),
|
||||
Some(4194304),
|
||||
Public
|
||||
),
|
||||
f!(
|
||||
"import.external_references.fetch_timeout_ms",
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS",
|
||||
AdminApi,
|
||||
"u64",
|
||||
Some("milliseconds"),
|
||||
Some("10000"),
|
||||
Some(1),
|
||||
Some(300000),
|
||||
Public
|
||||
),
|
||||
f!(
|
||||
"import.external_references.max_expanded_nodes",
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES",
|
||||
AdminApi,
|
||||
"u32",
|
||||
Some("nodes"),
|
||||
Some("10000"),
|
||||
Some(1),
|
||||
Some(100000),
|
||||
Public
|
||||
),
|
||||
f!(
|
||||
"observability.environment",
|
||||
"CRANK_ENVIRONMENT",
|
||||
|
||||
@@ -108,9 +108,64 @@ fn source_for(
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_covers_exactly_the_59_observed_runtime_names() {
|
||||
fn external_reference_contract_is_default_off_and_validates_prefixes_and_limits() {
|
||||
let config = parse_process(
|
||||
ProcessKind::AdminApi,
|
||||
ConfigSource::from_utf8(required_admin()),
|
||||
)
|
||||
.expect("the external reference contract has safe defaults");
|
||||
let references = &config.admin().unwrap().external_references;
|
||||
assert!(references.allowed_url_prefixes.is_empty());
|
||||
assert_eq!(references.max_depth, 8);
|
||||
assert_eq!(references.max_documents, 32);
|
||||
assert_eq!(references.max_fetch_bytes, 262_144);
|
||||
assert_eq!(references.fetch_timeout_ms, 10_000);
|
||||
assert_eq!(references.max_expanded_nodes, 10_000);
|
||||
|
||||
let mut allowed = required_admin();
|
||||
allowed.insert(
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(),
|
||||
"https://schemas.example.test/openapi/,https://schemas.example.test/openapi/".to_owned(),
|
||||
);
|
||||
let config = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(allowed)).unwrap();
|
||||
assert_eq!(
|
||||
config
|
||||
.admin()
|
||||
.unwrap()
|
||||
.external_references
|
||||
.allowed_url_prefixes,
|
||||
vec!["https://schemas.example.test/openapi/".to_owned()]
|
||||
);
|
||||
|
||||
let mut invalid = required_admin();
|
||||
invalid.insert(
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES".to_owned(),
|
||||
"https://user:secret@schemas.example.test/openapi/".to_owned(),
|
||||
);
|
||||
let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(invalid))
|
||||
.expect_err("userinfo must never become an operator allow rule");
|
||||
assert!(error.diagnostics().iter().any(|diagnostic| {
|
||||
diagnostic.code == DiagnosticCode::InvalidType
|
||||
&& diagnostic.field == "import.external_references.allowed_url_prefixes"
|
||||
}));
|
||||
|
||||
let mut out_of_range = required_admin();
|
||||
out_of_range.insert(
|
||||
"CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS".to_owned(),
|
||||
"33".to_owned(),
|
||||
);
|
||||
let error = parse_process(ProcessKind::AdminApi, ConfigSource::from_utf8(out_of_range))
|
||||
.expect_err("graph document limit must be bounded");
|
||||
assert!(error.diagnostics().iter().any(|diagnostic| {
|
||||
diagnostic.code == DiagnosticCode::OutOfRange
|
||||
&& diagnostic.field == "import.external_references.max_documents"
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_covers_exactly_the_65_observed_runtime_names() {
|
||||
let registry = field_registry();
|
||||
assert_eq!(registry.len(), 59);
|
||||
assert_eq!(registry.len(), 65);
|
||||
let unique = registry
|
||||
.iter()
|
||||
.map(|field| field.env_name)
|
||||
|
||||
@@ -30,6 +30,9 @@ fn generated_reference_distinguishes_required_and_optional_fields() {
|
||||
assert!(reference.contains(
|
||||
"| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` |"
|
||||
));
|
||||
assert!(reference.contains(
|
||||
"| `CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES` | `import.external_references.allowed_url_prefixes` | `AdminApi` | `url_prefix_list/-` | `` |"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -8,19 +8,22 @@ mod normalize_schema;
|
||||
mod openapi3;
|
||||
mod payload;
|
||||
mod recommendations;
|
||||
mod reference;
|
||||
mod schema;
|
||||
mod swagger2;
|
||||
|
||||
pub use model::{
|
||||
ImportFinding, ImportFindingSeverity, ImportGroupPreview, ImportOperationCandidate,
|
||||
ImportPreview, ImportSourcePreview, NORMALIZER_VERSION, NormalizationConfig, NormalizedFinding,
|
||||
NormalizedIr, NormalizedOperation, NormalizedParameter, NormalizedReference, NormalizedSchema,
|
||||
NormalizedSchemaKind, PROJECTION_VERSION, RestImportCandidate, RestImportDocument,
|
||||
RestImportOperation, RestImportParameter, RestParameterLocation, SourceDigest, SourceIdentity,
|
||||
SourceLocation, UnresolvedReference,
|
||||
ExternalDocumentSnapshot, ImportFinding, ImportFindingSeverity, ImportGroupPreview,
|
||||
ImportOperationCandidate, ImportPreview, ImportSourcePreview, NORMALIZER_VERSION,
|
||||
NormalizationConfig, NormalizedFinding, NormalizedIr, NormalizedOperation, NormalizedParameter,
|
||||
NormalizedReference, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind,
|
||||
PROJECTION_VERSION, ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode,
|
||||
RestImportCandidate, RestImportDocument, RestImportOperation, RestImportParameter,
|
||||
RestParameterLocation, SourceDigest, SourceIdentity, SourceLocation, UnresolvedReference,
|
||||
};
|
||||
pub use normalize::{
|
||||
ImportParseError, normalize_verified_document, preview_document, preview_document_legacy_v1,
|
||||
preview_from_ir, validate_normalized_ir,
|
||||
ImportParseError, external_reference_uris, normalize_verified_bundle,
|
||||
normalize_verified_document, preview_document, preview_document_legacy_v1, preview_from_ir,
|
||||
reference_uris, validate_normalized_ir,
|
||||
};
|
||||
pub use payload::operation_draft_from_candidate;
|
||||
|
||||
@@ -9,10 +9,10 @@ use serde_json::Value;
|
||||
/// The immutable contract used to normalize an OpenAPI source. These names
|
||||
/// deliberately travel with an import job: changing either contract must not
|
||||
/// silently reinterpret a pending preview.
|
||||
pub const NORMALIZER_VERSION: &str = "normalized-ir-v2";
|
||||
pub const PROJECTION_VERSION: &str = "preview-v2";
|
||||
pub const NORMALIZER_VERSION: &str = "normalized-ir-v3";
|
||||
pub const PROJECTION_VERSION: &str = "preview-v3";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
|
||||
pub struct SourceDigest(String);
|
||||
|
||||
impl SourceDigest {
|
||||
@@ -97,6 +97,19 @@ pub struct NormalizationConfig {
|
||||
pub max_collection_items: usize,
|
||||
pub max_aliases: usize,
|
||||
pub max_scalar_bytes: usize,
|
||||
/// Maximum number of reference hops followed from one source location.
|
||||
pub max_reference_depth: usize,
|
||||
/// Maximum number of `$ref` occurrences inspected across the bundle.
|
||||
pub max_references: usize,
|
||||
/// Maximum number of immutable external documents supplied to the pure resolver.
|
||||
pub max_reference_documents: usize,
|
||||
/// Maximum number of nodes copied while expanding resolved references.
|
||||
pub max_expanded_nodes: usize,
|
||||
pub max_external_document_bytes: usize,
|
||||
/// Signals that orchestration enabled external fetching. It never permits
|
||||
/// I/O in this crate; it only distinguishes default-deny from a missing or
|
||||
/// rejected supplied snapshot in exact findings.
|
||||
pub external_references_enabled: bool,
|
||||
}
|
||||
|
||||
impl Default for NormalizationConfig {
|
||||
@@ -110,6 +123,12 @@ impl Default for NormalizationConfig {
|
||||
max_collection_items: 10_000,
|
||||
max_aliases: 128,
|
||||
max_scalar_bytes: 256 * 1024,
|
||||
max_reference_depth: 32,
|
||||
max_references: 4_096,
|
||||
max_reference_documents: 32,
|
||||
max_expanded_nodes: 100_000,
|
||||
max_external_document_bytes: 256 * 1024,
|
||||
external_references_enabled: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -139,6 +158,38 @@ pub struct UnresolvedReference {
|
||||
pub location: SourceLocation,
|
||||
}
|
||||
|
||||
/// Immutable external input for the pure reference resolver. The caller owns
|
||||
/// URL policy and I/O; `crank-import` only consumes already verified bytes.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ExternalDocumentSnapshot {
|
||||
pub canonical_uri: String,
|
||||
pub digest: SourceDigest,
|
||||
pub document: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ResolvedReferenceNode {
|
||||
pub snapshot_digest: SourceDigest,
|
||||
pub location: SourceLocation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ResolvedReferenceEdge {
|
||||
pub source: ResolvedReferenceNode,
|
||||
pub target: ResolvedReferenceNode,
|
||||
pub recursive: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ResolvedReferenceGraph {
|
||||
/// Sorted, deduplicated immutable dependency identities. Canonical URLs
|
||||
/// deliberately do not enter the IR or public diagnostics.
|
||||
#[serde(default)]
|
||||
pub dependency_digests: Vec<SourceDigest>,
|
||||
#[serde(default)]
|
||||
pub edges: Vec<ResolvedReferenceEdge>,
|
||||
}
|
||||
|
||||
/// An unresolved `$ref` preserved from the decoded source. This is deliberately
|
||||
/// broader than schema references: path items, reusable parameters and other
|
||||
/// object-level references remain available to a later resolution phase.
|
||||
@@ -213,9 +264,34 @@ pub struct NormalizedSchema {
|
||||
pub location: SourceLocation,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub discriminator: Option<NormalizedDiscriminator>,
|
||||
#[serde(default)]
|
||||
pub constraints: NormalizedSchemaConstraints,
|
||||
pub kind: NormalizedSchemaKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct NormalizedSchemaConstraints {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub minimum: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub maximum: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub min_length: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_length: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub pattern: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct NormalizedDiscriminator {
|
||||
pub property_name: String,
|
||||
#[serde(default)]
|
||||
pub mapping: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case", tag = "type")]
|
||||
pub enum NormalizedSchemaKind {
|
||||
@@ -280,6 +356,8 @@ pub struct NormalizedIr {
|
||||
pub paths: Vec<NormalizedPath>,
|
||||
#[serde(default)]
|
||||
pub unresolved_references: Vec<NormalizedReference>,
|
||||
#[serde(default)]
|
||||
pub reference_graph: ResolvedReferenceGraph,
|
||||
pub source: ImportSourcePreview,
|
||||
#[serde(default)]
|
||||
pub operations: Vec<NormalizedOperation>,
|
||||
@@ -416,7 +494,7 @@ fn empty_source_location() -> SourceLocation {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RestParameterLocation {
|
||||
Path,
|
||||
|
||||
@@ -5,13 +5,13 @@ use thiserror::Error;
|
||||
|
||||
use crate::rest::{
|
||||
model::{
|
||||
CoverageDisposition, CoverageEntry, ImportFinding, ImportFindingSeverity,
|
||||
ImportGroupPreview, ImportPreview, ImportSourcePreview, NORMALIZER_VERSION,
|
||||
NormalizationConfig, NormalizedApiMetadata, NormalizedFinding, NormalizedIr,
|
||||
NormalizedLiteral, NormalizedOperation, NormalizedParameter, NormalizedScalarKind,
|
||||
NormalizedSchema, NormalizedSchemaKind, PROJECTION_VERSION, RestImportDocument,
|
||||
RestImportOperation, RestImportParameter, SourceDigest, SourceIdentity, SourceLocation,
|
||||
SourceSyntax,
|
||||
CoverageDisposition, CoverageEntry, ExternalDocumentSnapshot, ImportFinding,
|
||||
ImportFindingSeverity, ImportGroupPreview, ImportPreview, ImportSourcePreview,
|
||||
NORMALIZER_VERSION, NormalizationConfig, NormalizedApiMetadata, NormalizedFinding,
|
||||
NormalizedIr, NormalizedLiteral, NormalizedOperation, NormalizedParameter,
|
||||
NormalizedScalarKind, NormalizedSchema, NormalizedSchemaKind, PROJECTION_VERSION,
|
||||
RestImportDocument, RestImportOperation, RestImportParameter, SourceDigest, SourceIdentity,
|
||||
SourceLocation, SourceSyntax,
|
||||
},
|
||||
openapi3,
|
||||
payload::candidate_from_operation,
|
||||
@@ -39,6 +39,25 @@ pub fn preview_document(document: &str) -> Result<ImportPreview, ImportParseErro
|
||||
Ok(preview_from_ir(&ir))
|
||||
}
|
||||
|
||||
/// Enumerates reference URIs without resolving or performing I/O. Intended
|
||||
/// for an orchestration layer that materializes a bounded immutable bundle.
|
||||
pub fn reference_uris(
|
||||
document: &str,
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<Vec<String>, ImportParseError> {
|
||||
super::reference::reference_uris(document, config)
|
||||
}
|
||||
|
||||
/// Enumerates references in an already materialized external document. This
|
||||
/// keeps the historical primary-document API intact while applying the
|
||||
/// external-document byte limit at an explicit call site.
|
||||
pub fn external_reference_uris(
|
||||
document: &str,
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<Vec<String>, ImportParseError> {
|
||||
super::reference::external_reference_uris(document, config)
|
||||
}
|
||||
|
||||
/// Compatibility projection for jobs created before `NormalizedIr`. It is
|
||||
/// deliberately isolated from the v2 pipeline and may be removed only after
|
||||
/// the import-job TTL has elapsed.
|
||||
@@ -83,6 +102,18 @@ pub fn normalize_verified_document(
|
||||
document: &str,
|
||||
digest: SourceDigest,
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<NormalizedIr, ImportParseError> {
|
||||
normalize_verified_bundle(document, digest, &[], config)
|
||||
}
|
||||
|
||||
/// Normalizes a verified primary document together with immutable external
|
||||
/// snapshots. This function is pure: callers must perform all URL policy,
|
||||
/// fetching, artifact persistence and digest verification beforehand.
|
||||
pub fn normalize_verified_bundle(
|
||||
document: &str,
|
||||
digest: SourceDigest,
|
||||
snapshots: &[ExternalDocumentSnapshot],
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<NormalizedIr, ImportParseError> {
|
||||
config
|
||||
.validate_versions()
|
||||
@@ -99,6 +130,14 @@ pub fn normalize_verified_document(
|
||||
};
|
||||
let root = decode(document)?;
|
||||
validate_value_limits(&root, config)?;
|
||||
for snapshot in snapshots {
|
||||
if snapshot.document.len() > config.max_external_document_bytes {
|
||||
return Err(ImportParseError::LimitExceeded);
|
||||
}
|
||||
}
|
||||
let resolution = super::reference::resolve(root, digest.clone(), snapshots, config)?;
|
||||
let root = resolution.root;
|
||||
validate_value_limits(&root, config)?;
|
||||
|
||||
let parsed = match root.get("openapi").and_then(Value::as_str) {
|
||||
Some(version) if supported_oas_version(version) => openapi3::parse_document(&root)?,
|
||||
@@ -112,7 +151,15 @@ pub fn normalize_verified_document(
|
||||
if parsed.operations.is_empty() {
|
||||
return Err(ImportParseError::NoMethods);
|
||||
}
|
||||
canonicalize(parsed, digest, config, root, source_syntax)
|
||||
canonicalize(
|
||||
parsed,
|
||||
digest,
|
||||
config,
|
||||
root,
|
||||
source_syntax,
|
||||
resolution.graph,
|
||||
resolution.findings,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn preview_from_ir(ir: &NormalizedIr) -> ImportPreview {
|
||||
@@ -202,6 +249,8 @@ fn canonicalize(
|
||||
config: &NormalizationConfig,
|
||||
root: Value,
|
||||
source_syntax: SourceSyntax,
|
||||
reference_graph: crate::rest::model::ResolvedReferenceGraph,
|
||||
resolution_findings: Vec<NormalizedFinding>,
|
||||
) -> Result<NormalizedIr, ImportParseError> {
|
||||
let source = ImportSourcePreview {
|
||||
format: document.format,
|
||||
@@ -260,6 +309,7 @@ fn canonicalize(
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
findings.extend(resolution_findings);
|
||||
let mut operations = document
|
||||
.operations
|
||||
.into_iter()
|
||||
@@ -466,6 +516,40 @@ fn canonicalize(
|
||||
right.location.pointer.as_str(),
|
||||
))
|
||||
});
|
||||
let mut document_findings = Vec::with_capacity(findings.len());
|
||||
for mut finding in findings.drain(..) {
|
||||
let resolution_finding = matches!(
|
||||
finding.code.as_str(),
|
||||
"external_reference_disabled"
|
||||
| "reference_target_missing"
|
||||
| "reference_uri_malformed"
|
||||
| "external_reference_unavailable"
|
||||
| "reference_type_mismatch"
|
||||
| "reference_graph_limit"
|
||||
| "all_of_conflict"
|
||||
| "unsupported_composition"
|
||||
| "unsupported_discriminator"
|
||||
);
|
||||
let target = resolution_finding
|
||||
.then(|| {
|
||||
operations.iter_mut().find(|operation| {
|
||||
!finding.location.pointer.is_empty()
|
||||
&& (finding.location.pointer == operation.location.pointer
|
||||
|| finding
|
||||
.location
|
||||
.pointer
|
||||
.starts_with(&format!("{}/", operation.location.pointer)))
|
||||
})
|
||||
})
|
||||
.flatten();
|
||||
if let Some(operation) = target {
|
||||
finding.operation_key = Some(operation.key.clone());
|
||||
operation.findings.push(finding);
|
||||
} else {
|
||||
document_findings.push(finding);
|
||||
}
|
||||
}
|
||||
findings = document_findings;
|
||||
for operation in &mut operations {
|
||||
sort_normalized_findings(&mut operation.findings);
|
||||
}
|
||||
@@ -501,10 +585,9 @@ fn canonicalize(
|
||||
}
|
||||
sort_normalized_findings(&mut operation.findings);
|
||||
}
|
||||
// Resolution is deliberately deferred. References already represented by
|
||||
// an operation schema use that operation's blocker; all other references
|
||||
// need a document-level blocker so omitted object-level semantics can
|
||||
// never reach Apply silently.
|
||||
// References left in the finite projection are precisely failures which
|
||||
// the bounded resolver could not safely expand. Keep a blocker at the
|
||||
// closest affected operation; unrelated operations remain actionable.
|
||||
findings.extend(
|
||||
unresolved_references
|
||||
.iter()
|
||||
@@ -529,7 +612,11 @@ fn canonicalize(
|
||||
}),
|
||||
);
|
||||
sort_normalized_findings(&mut findings);
|
||||
for finding in &findings {
|
||||
for finding in findings.iter().chain(
|
||||
operations
|
||||
.iter()
|
||||
.flat_map(|operation| operation.findings.iter()),
|
||||
) {
|
||||
if !coverage.iter().any(|entry| {
|
||||
entry.construct_id == finding.construct_id
|
||||
&& entry.location == finding.location
|
||||
@@ -580,6 +667,7 @@ fn canonicalize(
|
||||
.unwrap_or_default(),
|
||||
paths: normalize_coverage::paths_from_operations(&operations, version),
|
||||
unresolved_references,
|
||||
reference_graph,
|
||||
source,
|
||||
operations,
|
||||
findings,
|
||||
@@ -593,9 +681,56 @@ pub fn validate_normalized_ir(ir: &NormalizedIr) -> Result<(), ImportParseError>
|
||||
if ir.normalizer_version != NORMALIZER_VERSION || ir.projection_version != PROJECTION_VERSION {
|
||||
return Err(ImportParseError::InvalidDocument);
|
||||
}
|
||||
validate_reference_graph(ir)?;
|
||||
normalize_coverage::validate_coverage(ir)
|
||||
}
|
||||
|
||||
fn validate_reference_graph(ir: &NormalizedIr) -> Result<(), ImportParseError> {
|
||||
if ir
|
||||
.reference_graph
|
||||
.dependency_digests
|
||||
.windows(2)
|
||||
.any(|pair| pair[0] >= pair[1])
|
||||
{
|
||||
return Err(ImportParseError::InvalidDocument);
|
||||
}
|
||||
let allowed = ir
|
||||
.reference_graph
|
||||
.dependency_digests
|
||||
.iter()
|
||||
.chain(std::iter::once(&ir.source_identity.digest))
|
||||
.map(SourceDigest::as_str)
|
||||
.collect::<BTreeSet<_>>();
|
||||
let edges = &ir.reference_graph.edges;
|
||||
if edges
|
||||
.windows(2)
|
||||
.any(|pair| reference_edge_key(&pair[0]) >= reference_edge_key(&pair[1]))
|
||||
|| edges.iter().any(|edge| {
|
||||
!allowed.contains(edge.source.snapshot_digest.as_str())
|
||||
|| !allowed.contains(edge.target.snapshot_digest.as_str())
|
||||
|| !(edge.source.location.pointer.is_empty()
|
||||
|| edge.source.location.pointer.starts_with('/'))
|
||||
|| !(edge.target.location.pointer.is_empty()
|
||||
|| edge.target.location.pointer.starts_with('/'))
|
||||
})
|
||||
{
|
||||
return Err(ImportParseError::InvalidDocument);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reference_edge_key(
|
||||
edge: &crate::rest::model::ResolvedReferenceEdge,
|
||||
) -> (&str, &str, &str, &str, bool) {
|
||||
(
|
||||
edge.source.snapshot_digest.as_str(),
|
||||
edge.source.location.pointer.as_str(),
|
||||
edge.target.snapshot_digest.as_str(),
|
||||
edge.target.location.pointer.as_str(),
|
||||
edge.recursive,
|
||||
)
|
||||
}
|
||||
|
||||
fn coverage_disposition_rank(disposition: &CoverageDisposition) -> u8 {
|
||||
match disposition {
|
||||
CoverageDisposition::Mapped => 0,
|
||||
@@ -650,7 +785,10 @@ fn legacy_schema_value(schema: &NormalizedSchema) -> Value {
|
||||
let mut value = match &schema.kind {
|
||||
NormalizedSchemaKind::Reference { reference } => serde_json::json!({"$ref": reference.uri}),
|
||||
NormalizedSchemaKind::Composition { operator, variants } => {
|
||||
serde_json::json!({operator: variants.iter().map(legacy_schema_value).collect::<Vec<_>>() })
|
||||
serde_json::json!({
|
||||
operator: variants.iter().map(legacy_schema_value).collect::<Vec<_>>(),
|
||||
"x-crank-lossless-composition": true,
|
||||
})
|
||||
}
|
||||
NormalizedSchemaKind::Object {
|
||||
properties,
|
||||
@@ -718,7 +856,7 @@ fn schema_contains_reference(schema: &NormalizedSchema, location: &SourceLocatio
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_value_limits(
|
||||
pub(super) fn validate_value_limits(
|
||||
value: &Value,
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<(), ImportParseError> {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::rest::model::{
|
||||
CoverageDisposition, CoverageEntry, NormalizedLiteral, NormalizedScalarKind, NormalizedSchema,
|
||||
NormalizedSchemaKind, SourceLocation, UnresolvedReference,
|
||||
CoverageDisposition, CoverageEntry, NormalizedDiscriminator, NormalizedLiteral,
|
||||
NormalizedScalarKind, NormalizedSchema, NormalizedSchemaConstraints, NormalizedSchemaKind,
|
||||
SourceLocation, UnresolvedReference,
|
||||
};
|
||||
|
||||
use super::normalize_coverage::escape_pointer;
|
||||
@@ -150,6 +151,35 @@ pub(super) fn typed_schema(
|
||||
.get("description")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
discriminator: value.get("discriminator").and_then(|value| {
|
||||
let property_name = value.get("propertyName")?.as_str()?.to_owned();
|
||||
let mapping = value
|
||||
.get("mapping")
|
||||
.and_then(Value::as_object)
|
||||
.map(|mapping| {
|
||||
mapping
|
||||
.iter()
|
||||
.filter_map(|(key, value)| {
|
||||
value.as_str().map(|value| (key.clone(), value.to_owned()))
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
Some(NormalizedDiscriminator {
|
||||
property_name,
|
||||
mapping,
|
||||
})
|
||||
}),
|
||||
constraints: NormalizedSchemaConstraints {
|
||||
minimum: value.get("minimum").and_then(Value::as_f64),
|
||||
maximum: value.get("maximum").and_then(Value::as_f64),
|
||||
min_length: value.get("minLength").and_then(Value::as_u64),
|
||||
max_length: value.get("maxLength").and_then(Value::as_u64),
|
||||
pattern: value
|
||||
.get("pattern")
|
||||
.and_then(Value::as_str)
|
||||
.map(ToOwned::to_owned),
|
||||
},
|
||||
kind,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,6 +133,7 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
|
||||
parsed_parameters.findings,
|
||||
);
|
||||
operation_parameters.extend(parsed_parameters.parameters);
|
||||
deduplicate_parameters(&mut operation_parameters);
|
||||
let operation_servers = parse_servers(
|
||||
operation_value.get("servers"),
|
||||
&format!("{operation_pointer}/servers"),
|
||||
@@ -203,6 +204,13 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
|
||||
})
|
||||
}
|
||||
|
||||
fn deduplicate_parameters(parameters: &mut Vec<RestImportParameter>) {
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
parameters.reverse();
|
||||
parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location)));
|
||||
parameters.reverse();
|
||||
}
|
||||
|
||||
pub fn parse_document_legacy_v1(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||
let version = root
|
||||
.get("openapi")
|
||||
|
||||
@@ -0,0 +1,872 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::rest::model::{
|
||||
ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedFinding,
|
||||
ResolvedReferenceEdge, ResolvedReferenceGraph, ResolvedReferenceNode, SourceDigest,
|
||||
SourceLocation,
|
||||
};
|
||||
|
||||
use super::{ImportParseError, normalize_coverage::escape_pointer};
|
||||
|
||||
pub(super) struct ResolutionResult {
|
||||
pub root: Value,
|
||||
pub graph: ResolvedReferenceGraph,
|
||||
pub findings: Vec<NormalizedFinding>,
|
||||
}
|
||||
|
||||
pub(super) fn reference_uris(
|
||||
document: &str,
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<Vec<String>, ImportParseError> {
|
||||
reference_uris_with_max_bytes(document, config, config.max_bytes)
|
||||
}
|
||||
|
||||
pub(super) fn external_reference_uris(
|
||||
document: &str,
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<Vec<String>, ImportParseError> {
|
||||
reference_uris_with_max_bytes(document, config, config.max_external_document_bytes)
|
||||
}
|
||||
|
||||
fn reference_uris_with_max_bytes(
|
||||
document: &str,
|
||||
config: &NormalizationConfig,
|
||||
max_bytes: usize,
|
||||
) -> Result<Vec<String>, ImportParseError> {
|
||||
fn collect(value: &Value, uris: &mut BTreeSet<String>) {
|
||||
match value {
|
||||
Value::Object(object) => {
|
||||
if let Some(reference) = object.get("$ref").and_then(Value::as_str) {
|
||||
uris.insert(reference.to_owned());
|
||||
}
|
||||
for (key, child) in object {
|
||||
if is_literal_payload_key(key) {
|
||||
continue;
|
||||
}
|
||||
collect(child, uris);
|
||||
}
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for child in items {
|
||||
collect(child, uris);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if document.len() > max_bytes
|
||||
|| super::normalize_limits::alias_count(document) > config.max_aliases
|
||||
{
|
||||
return Err(ImportParseError::LimitExceeded);
|
||||
}
|
||||
let root = decode_snapshot(document)?;
|
||||
super::normalize::validate_value_limits(&root, config)?;
|
||||
let mut uris = BTreeSet::new();
|
||||
collect(&root, &mut uris);
|
||||
Ok(uris.into_iter().collect())
|
||||
}
|
||||
|
||||
struct Document {
|
||||
uri: Option<String>,
|
||||
digest: SourceDigest,
|
||||
root: Value,
|
||||
oas31: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
struct NodeKey {
|
||||
digest: String,
|
||||
pointer: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TraversalLocation<'a> {
|
||||
projection: &'a str,
|
||||
origin: &'a str,
|
||||
}
|
||||
|
||||
struct Resolver<'a> {
|
||||
config: &'a NormalizationConfig,
|
||||
documents: Vec<Document>,
|
||||
by_uri: BTreeMap<String, usize>,
|
||||
graph: ResolvedReferenceGraph,
|
||||
findings: Vec<NormalizedFinding>,
|
||||
references: usize,
|
||||
expanded_nodes: usize,
|
||||
}
|
||||
|
||||
pub(super) fn resolve(
|
||||
root: Value,
|
||||
primary_digest: SourceDigest,
|
||||
snapshots: &[ExternalDocumentSnapshot],
|
||||
config: &NormalizationConfig,
|
||||
) -> Result<ResolutionResult, ImportParseError> {
|
||||
if snapshots.len() > config.max_reference_documents {
|
||||
return Err(ImportParseError::LimitExceeded);
|
||||
}
|
||||
let mut documents = vec![Document {
|
||||
uri: None,
|
||||
digest: primary_digest,
|
||||
oas31: is_oas31(&root),
|
||||
root,
|
||||
}];
|
||||
let mut by_uri = BTreeMap::new();
|
||||
let mut dependency_digests = BTreeSet::new();
|
||||
let primary_oas31 = documents[0].oas31;
|
||||
for snapshot in snapshots {
|
||||
if snapshot.canonical_uri.is_empty()
|
||||
|| by_uri
|
||||
.insert(snapshot.canonical_uri.clone(), documents.len())
|
||||
.is_some()
|
||||
{
|
||||
return Err(ImportParseError::InvalidDocument);
|
||||
}
|
||||
if super::normalize_limits::alias_count(&snapshot.document) > config.max_aliases {
|
||||
return Err(ImportParseError::LimitExceeded);
|
||||
}
|
||||
let root = decode_snapshot(&snapshot.document)?;
|
||||
super::normalize::validate_value_limits(&root, config)?;
|
||||
dependency_digests.insert(snapshot.digest.clone());
|
||||
documents.push(Document {
|
||||
uri: Some(snapshot.canonical_uri.clone()),
|
||||
digest: snapshot.digest.clone(),
|
||||
// External documents are fragments of the primary contract. In a
|
||||
// 3.1 bundle they therefore use 3.1 `$ref` sibling semantics even
|
||||
// when the fragment itself omits an `openapi` declaration.
|
||||
oas31: primary_oas31 || is_oas31(&root),
|
||||
root,
|
||||
});
|
||||
}
|
||||
let mut resolver = Resolver {
|
||||
config,
|
||||
documents,
|
||||
by_uri,
|
||||
graph: ResolvedReferenceGraph {
|
||||
dependency_digests: dependency_digests.into_iter().collect(),
|
||||
edges: Vec::new(),
|
||||
},
|
||||
findings: Vec::new(),
|
||||
references: 0,
|
||||
expanded_nodes: 0,
|
||||
};
|
||||
let root = resolver.documents[0].root.clone();
|
||||
let root = resolver.expand_value(
|
||||
0,
|
||||
root,
|
||||
TraversalLocation {
|
||||
projection: "",
|
||||
origin: "",
|
||||
},
|
||||
0,
|
||||
&mut Vec::new(),
|
||||
)?;
|
||||
resolver
|
||||
.graph
|
||||
.edges
|
||||
.sort_by(|left, right| edge_key(left).cmp(&edge_key(right)));
|
||||
resolver.graph.edges.dedup();
|
||||
resolver.findings.sort_by(|left, right| {
|
||||
(
|
||||
&left.operation_key,
|
||||
&left.construct_id,
|
||||
&left.code,
|
||||
&left.location.pointer,
|
||||
)
|
||||
.cmp(&(
|
||||
&right.operation_key,
|
||||
&right.construct_id,
|
||||
&right.code,
|
||||
&right.location.pointer,
|
||||
))
|
||||
});
|
||||
resolver.findings.dedup();
|
||||
Ok(ResolutionResult {
|
||||
root,
|
||||
graph: resolver.graph,
|
||||
findings: resolver.findings,
|
||||
})
|
||||
}
|
||||
|
||||
impl Resolver<'_> {
|
||||
fn expand_value(
|
||||
&mut self,
|
||||
document_index: usize,
|
||||
value: Value,
|
||||
location: TraversalLocation<'_>,
|
||||
depth: usize,
|
||||
stack: &mut Vec<NodeKey>,
|
||||
) -> Result<Value, ImportParseError> {
|
||||
if depth > 0 {
|
||||
self.expanded_nodes = self.expanded_nodes.saturating_add(1);
|
||||
if self.expanded_nodes > self.config.max_expanded_nodes {
|
||||
self.push_finding("reference_graph_limit", location.projection);
|
||||
return Ok(Value::Object(Map::new()));
|
||||
}
|
||||
}
|
||||
match value {
|
||||
Value::Object(mut object) => {
|
||||
if let Some(reference) = object
|
||||
.get("$ref")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
{
|
||||
return self.expand_reference(
|
||||
document_index,
|
||||
object,
|
||||
&reference,
|
||||
location,
|
||||
depth,
|
||||
stack,
|
||||
);
|
||||
}
|
||||
let keys = object.keys().cloned().collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
if let Some(child) = object.remove(&key) {
|
||||
if is_literal_payload_key(&key) {
|
||||
object.insert(key, child);
|
||||
continue;
|
||||
}
|
||||
let child_pointer =
|
||||
format!("{}/{}", location.projection, escape_pointer(&key));
|
||||
let child_origin_pointer =
|
||||
format!("{}/{}", location.origin, escape_pointer(&key));
|
||||
object.insert(
|
||||
key,
|
||||
self.expand_value(
|
||||
document_index,
|
||||
child,
|
||||
TraversalLocation {
|
||||
projection: &child_pointer,
|
||||
origin: &child_origin_pointer,
|
||||
},
|
||||
depth,
|
||||
stack,
|
||||
)?,
|
||||
);
|
||||
}
|
||||
}
|
||||
let composition_count = ["allOf", "oneOf", "anyOf"]
|
||||
.into_iter()
|
||||
.filter(|operator| object.contains_key(*operator))
|
||||
.count();
|
||||
if composition_count > 1 {
|
||||
self.push_finding("unsupported_composition", location.projection);
|
||||
}
|
||||
if let Some(discriminator) = object.get("discriminator") {
|
||||
let valid = discriminator
|
||||
.get("propertyName")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|value| !value.is_empty())
|
||||
&& discriminator.get("mapping").is_none_or(|mapping| {
|
||||
mapping
|
||||
.as_object()
|
||||
.is_some_and(|mapping| mapping.values().all(Value::is_string))
|
||||
})
|
||||
&& (object.contains_key("oneOf") || object.contains_key("anyOf"));
|
||||
if !valid {
|
||||
self.push_finding("unsupported_discriminator", location.projection);
|
||||
}
|
||||
}
|
||||
self.merge_all_of(object, location.projection)
|
||||
}
|
||||
Value::Array(items) => Ok(Value::Array(
|
||||
items
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, child)| {
|
||||
self.expand_value(
|
||||
document_index,
|
||||
child,
|
||||
TraversalLocation {
|
||||
projection: &format!("{}/{index}", location.projection),
|
||||
origin: &format!("{}/{index}", location.origin),
|
||||
},
|
||||
depth,
|
||||
stack,
|
||||
)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?,
|
||||
)),
|
||||
other => Ok(other),
|
||||
}
|
||||
}
|
||||
|
||||
fn expand_reference(
|
||||
&mut self,
|
||||
document_index: usize,
|
||||
mut source_object: Map<String, Value>,
|
||||
reference: &str,
|
||||
location: TraversalLocation<'_>,
|
||||
depth: usize,
|
||||
stack: &mut Vec<NodeKey>,
|
||||
) -> Result<Value, ImportParseError> {
|
||||
self.references = self.references.saturating_add(1);
|
||||
if self.references > self.config.max_references || depth >= self.config.max_reference_depth
|
||||
{
|
||||
self.push_finding(
|
||||
"reference_graph_limit",
|
||||
&format!("{}/$ref", location.projection),
|
||||
);
|
||||
return Ok(Value::Object(source_object));
|
||||
}
|
||||
let target = self.target(document_index, reference);
|
||||
if matches!(target, Err(TargetError::MalformedFragment)) {
|
||||
self.push_finding(
|
||||
"reference_uri_malformed",
|
||||
&format!("{}/$ref", location.projection),
|
||||
);
|
||||
return Ok(Value::Object(source_object));
|
||||
}
|
||||
let Some((target_document, target_pointer)) = target.ok().flatten() else {
|
||||
let external = reference.starts_with("http://") || reference.starts_with("https://");
|
||||
let code = if external {
|
||||
if self.config.external_references_enabled {
|
||||
"external_reference_unavailable"
|
||||
} else {
|
||||
"external_reference_disabled"
|
||||
}
|
||||
} else {
|
||||
"reference_target_missing"
|
||||
};
|
||||
self.push_finding(code, &format!("{}/$ref", location.projection));
|
||||
return Ok(Value::Object(source_object));
|
||||
};
|
||||
let source = ResolvedReferenceNode {
|
||||
snapshot_digest: self.documents[document_index].digest.clone(),
|
||||
location: SourceLocation {
|
||||
pointer: format!("{}/$ref", location.origin),
|
||||
},
|
||||
};
|
||||
let target = ResolvedReferenceNode {
|
||||
snapshot_digest: self.documents[target_document].digest.clone(),
|
||||
location: SourceLocation {
|
||||
pointer: target_pointer.clone(),
|
||||
},
|
||||
};
|
||||
let key = NodeKey {
|
||||
digest: target.snapshot_digest.as_str().to_owned(),
|
||||
pointer: target_pointer.clone(),
|
||||
};
|
||||
let recursive = stack.contains(&key);
|
||||
self.graph.edges.push(ResolvedReferenceEdge {
|
||||
source,
|
||||
target,
|
||||
recursive,
|
||||
});
|
||||
if recursive {
|
||||
// The edge is the lossless representation. Expansion stops here,
|
||||
// producing an opaque object for the finite preview projection.
|
||||
return Ok(Value::Object(Map::new()));
|
||||
}
|
||||
let Some(target_value) = self.documents[target_document]
|
||||
.root
|
||||
.pointer(&target_pointer)
|
||||
.cloned()
|
||||
else {
|
||||
self.push_finding(
|
||||
"reference_target_missing",
|
||||
&format!("{}/$ref", location.projection),
|
||||
);
|
||||
return Ok(Value::Object(source_object));
|
||||
};
|
||||
if !target_value.is_object() {
|
||||
self.push_finding(
|
||||
"reference_type_mismatch",
|
||||
&format!("{}/$ref", location.projection),
|
||||
);
|
||||
return Ok(Value::Object(source_object));
|
||||
}
|
||||
stack.push(key);
|
||||
let mut expanded = self.expand_value(
|
||||
target_document,
|
||||
target_value,
|
||||
TraversalLocation {
|
||||
projection: location.projection,
|
||||
origin: &target_pointer,
|
||||
},
|
||||
depth + 1,
|
||||
stack,
|
||||
)?;
|
||||
stack.pop();
|
||||
|
||||
// OAS 3.1 Schema Objects permit siblings next to `$ref`; OAS 3.0 and
|
||||
// Swagger Reference Objects ignore them. The source document version
|
||||
// controls the semantics at the reference site.
|
||||
source_object.remove("$ref");
|
||||
if self.documents[document_index].oas31 && !source_object.is_empty() {
|
||||
let Value::Object(expanded_object) = &mut expanded else {
|
||||
self.push_finding(
|
||||
"reference_type_mismatch",
|
||||
&format!("{}/$ref", location.projection),
|
||||
);
|
||||
return Ok(Value::Object(source_object));
|
||||
};
|
||||
for (key, sibling) in source_object {
|
||||
let child_pointer = format!("{}/{}", location.projection, escape_pointer(&key));
|
||||
let child_origin_pointer = format!("{}/{}", location.origin, escape_pointer(&key));
|
||||
expanded_object.insert(
|
||||
key,
|
||||
self.expand_value(
|
||||
document_index,
|
||||
sibling,
|
||||
TraversalLocation {
|
||||
projection: &child_pointer,
|
||||
origin: &child_origin_pointer,
|
||||
},
|
||||
depth,
|
||||
stack,
|
||||
)?,
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(expanded)
|
||||
}
|
||||
|
||||
fn target(
|
||||
&self,
|
||||
current: usize,
|
||||
reference: &str,
|
||||
) -> Result<Option<(usize, String)>, TargetError> {
|
||||
let (document, fragment) = reference.split_once('#').unwrap_or((reference, ""));
|
||||
let fragment = percent_decode(fragment)?;
|
||||
let pointer = if fragment.is_empty() {
|
||||
String::new()
|
||||
} else if fragment.starts_with('/') {
|
||||
fragment
|
||||
} else {
|
||||
return Ok(None);
|
||||
};
|
||||
if document.is_empty() {
|
||||
return Ok(Some((current, pointer)));
|
||||
}
|
||||
let canonical = if has_uri_scheme(document) {
|
||||
canonical_absolute_uri(document)
|
||||
} else {
|
||||
let Some(base) = self.documents[current].uri.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
join_relative(base, document)
|
||||
};
|
||||
Ok(canonical.and_then(|canonical| {
|
||||
self.by_uri
|
||||
.get(&canonical)
|
||||
.copied()
|
||||
.map(|index| (index, pointer))
|
||||
}))
|
||||
}
|
||||
|
||||
fn merge_all_of(
|
||||
&mut self,
|
||||
mut object: Map<String, Value>,
|
||||
pointer: &str,
|
||||
) -> Result<Value, ImportParseError> {
|
||||
let branches = match object.remove("allOf") {
|
||||
None => return Ok(Value::Object(object)),
|
||||
Some(Value::Array(branches)) => branches,
|
||||
Some(value) => {
|
||||
object.insert("allOf".to_owned(), value);
|
||||
self.push_finding("unsupported_composition", &format!("{pointer}/allOf"));
|
||||
return Ok(Value::Object(object));
|
||||
}
|
||||
};
|
||||
let original = branches.clone();
|
||||
for branch in branches {
|
||||
let Some(fields) = branch.as_object() else {
|
||||
object.insert("allOf".to_owned(), Value::Array(original));
|
||||
self.push_finding("all_of_conflict", &format!("{pointer}/allOf"));
|
||||
return Ok(Value::Object(object));
|
||||
};
|
||||
if !merge_object(&mut object, fields) {
|
||||
object.insert("allOf".to_owned(), Value::Array(original));
|
||||
self.push_finding("all_of_conflict", &format!("{pointer}/allOf"));
|
||||
return Ok(Value::Object(object));
|
||||
}
|
||||
}
|
||||
Ok(Value::Object(object))
|
||||
}
|
||||
|
||||
fn push_finding(&mut self, code: &str, pointer: &str) {
|
||||
self.findings.push(NormalizedFinding {
|
||||
code: code.to_owned(),
|
||||
severity: ImportFindingSeverity::Error,
|
||||
message: match code {
|
||||
"external_reference_disabled" => "Внешняя ссылка отключена политикой импорта.",
|
||||
"reference_target_missing" => {
|
||||
"Цель ссылки отсутствует или имеет неверный JSON Pointer."
|
||||
}
|
||||
"external_reference_unavailable" => {
|
||||
"Внешний snapshot недоступен или отклонён политикой импорта."
|
||||
}
|
||||
"reference_type_mismatch" => "Цель ссылки имеет неподдерживаемый тип.",
|
||||
"reference_graph_limit" => "Граф ссылок превышает установленный предел.",
|
||||
"all_of_conflict" => "Ветки allOf содержат несовместимые определения.",
|
||||
"reference_uri_malformed" => "URI fragment ссылки содержит некорректное percent-кодирование.",
|
||||
"unsupported_composition" => "Несколько операторов composition в одной schema не могут быть спроецированы без потерь.",
|
||||
"unsupported_discriminator" => "Discriminator имеет неподдерживаемую или неполную структуру.",
|
||||
_ => "Ссылка не может быть безопасно разрешена.",
|
||||
}
|
||||
.to_owned(),
|
||||
construct_id: format!("source:{pointer}"),
|
||||
location: SourceLocation {
|
||||
pointer: pointer.to_owned(),
|
||||
},
|
||||
operation_key: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_object(target: &mut Map<String, Value>, source: &Map<String, Value>) -> bool {
|
||||
for (key, value) in source {
|
||||
match key.as_str() {
|
||||
"properties" => {
|
||||
let Some(source_properties) = value.as_object() else {
|
||||
return false;
|
||||
};
|
||||
let properties = target
|
||||
.entry(key.clone())
|
||||
.or_insert_with(|| Value::Object(Map::new()));
|
||||
let Some(target_properties) = properties.as_object_mut() else {
|
||||
return false;
|
||||
};
|
||||
for (name, schema) in source_properties {
|
||||
if let Some(existing) = target_properties.get(name) {
|
||||
let (Some(existing), Some(schema)) =
|
||||
(existing.as_object(), schema.as_object())
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
let mut merged = existing.clone();
|
||||
if !merge_object(&mut merged, schema) {
|
||||
return false;
|
||||
}
|
||||
target_properties.insert(name.clone(), Value::Object(merged));
|
||||
} else {
|
||||
target_properties.insert(name.clone(), schema.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
"required" => {
|
||||
let Some(source_required) = value.as_array() else {
|
||||
return false;
|
||||
};
|
||||
let required = target
|
||||
.entry(key.clone())
|
||||
.or_insert_with(|| Value::Array(Vec::new()));
|
||||
let Some(target_required) = required.as_array_mut() else {
|
||||
return false;
|
||||
};
|
||||
target_required.extend(source_required.iter().cloned());
|
||||
target_required.sort_by(|left, right| left.as_str().cmp(&right.as_str()));
|
||||
target_required.dedup();
|
||||
}
|
||||
"minimum" | "maximum" => {
|
||||
let Some(source_value) = value.as_f64() else {
|
||||
return false;
|
||||
};
|
||||
if target.contains_key(key) && target.get(key).and_then(Value::as_f64).is_none() {
|
||||
return false;
|
||||
}
|
||||
let merged = match (key.as_str(), target.get(key).and_then(Value::as_f64)) {
|
||||
("minimum", Some(current)) => current.max(source_value),
|
||||
("maximum", Some(current)) => current.min(source_value),
|
||||
_ => source_value,
|
||||
};
|
||||
let Some(number) = serde_json::Number::from_f64(merged) else {
|
||||
return false;
|
||||
};
|
||||
target.insert(key.clone(), Value::Number(number));
|
||||
if constraint_contradiction(target, "minimum", "maximum") {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
"minLength" | "maxLength" => {
|
||||
let Some(source_value) = value.as_u64() else {
|
||||
return false;
|
||||
};
|
||||
if target.contains_key(key) && target.get(key).and_then(Value::as_u64).is_none() {
|
||||
return false;
|
||||
}
|
||||
let merged = match (key.as_str(), target.get(key).and_then(Value::as_u64)) {
|
||||
("minLength", Some(current)) => current.max(source_value),
|
||||
("maxLength", Some(current)) => current.min(source_value),
|
||||
_ => source_value,
|
||||
};
|
||||
target.insert(key.clone(), Value::Number(merged.into()));
|
||||
if integer_constraint_contradiction(target, "minLength", "maxLength") {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if target.get(key).is_some_and(|existing| existing != value) {
|
||||
return false;
|
||||
}
|
||||
target.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn constraint_contradiction(target: &Map<String, Value>, minimum: &str, maximum: &str) -> bool {
|
||||
match (
|
||||
target.get(minimum).and_then(Value::as_f64),
|
||||
target.get(maximum).and_then(Value::as_f64),
|
||||
) {
|
||||
(Some(minimum), Some(maximum)) => minimum > maximum,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn integer_constraint_contradiction(
|
||||
target: &Map<String, Value>,
|
||||
minimum: &str,
|
||||
maximum: &str,
|
||||
) -> bool {
|
||||
match (
|
||||
target.get(minimum).and_then(Value::as_u64),
|
||||
target.get(maximum).and_then(Value::as_u64),
|
||||
) {
|
||||
(Some(minimum), Some(maximum)) => minimum > maximum,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn edge_key(edge: &ResolvedReferenceEdge) -> (&str, &str, &str, &str, bool) {
|
||||
(
|
||||
edge.source.snapshot_digest.as_str(),
|
||||
&edge.source.location.pointer,
|
||||
edge.target.snapshot_digest.as_str(),
|
||||
&edge.target.location.pointer,
|
||||
edge.recursive,
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_snapshot(document: &str) -> Result<Value, ImportParseError> {
|
||||
if let Ok(value) = serde_json::from_str(document) {
|
||||
return Ok(value);
|
||||
}
|
||||
let yaml: serde_yaml::Value =
|
||||
serde_yaml::from_str(document).map_err(|_| ImportParseError::InvalidDocument)?;
|
||||
serde_json::to_value(yaml).map_err(|_| ImportParseError::InvalidDocument)
|
||||
}
|
||||
|
||||
fn is_oas31(root: &Value) -> bool {
|
||||
root.get("openapi")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|version| version.starts_with("3.1."))
|
||||
}
|
||||
|
||||
fn join_relative(base: &str, relative: &str) -> Option<String> {
|
||||
let base = UriReference::parse(base)?;
|
||||
let relative = UriReference::parse(relative)?;
|
||||
let scheme = relative.scheme.or(base.scheme)?;
|
||||
let authority = if relative.scheme.is_some() || relative.authority.is_some() {
|
||||
relative.authority
|
||||
} else {
|
||||
base.authority
|
||||
};
|
||||
let (path, query) = if relative.scheme.is_some() || relative.authority.is_some() {
|
||||
(remove_dot_segments(&relative.path), relative.query)
|
||||
} else if relative.path.is_empty() {
|
||||
(base.path, relative.query.or(base.query))
|
||||
} else if relative.path.starts_with('/') {
|
||||
(remove_dot_segments(&relative.path), relative.query)
|
||||
} else {
|
||||
(
|
||||
remove_dot_segments(&merge_paths(
|
||||
&base.path,
|
||||
authority.is_some(),
|
||||
&relative.path,
|
||||
)),
|
||||
relative.query,
|
||||
)
|
||||
};
|
||||
UriReference {
|
||||
scheme: Some(scheme),
|
||||
authority,
|
||||
path,
|
||||
query,
|
||||
}
|
||||
.render()
|
||||
}
|
||||
|
||||
fn canonical_absolute_uri(uri: &str) -> Option<String> {
|
||||
let reference = UriReference::parse(uri)?;
|
||||
let scheme = reference.scheme?;
|
||||
UriReference {
|
||||
scheme: Some(scheme),
|
||||
authority: reference.authority,
|
||||
path: remove_dot_segments(&reference.path),
|
||||
query: reference.query,
|
||||
}
|
||||
.render()
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct UriReference<'a> {
|
||||
scheme: Option<&'a str>,
|
||||
authority: Option<&'a str>,
|
||||
path: String,
|
||||
query: Option<&'a str>,
|
||||
}
|
||||
|
||||
impl<'a> UriReference<'a> {
|
||||
fn parse(value: &'a str) -> Option<Self> {
|
||||
if value.contains('\\') || value.contains('#') {
|
||||
return None;
|
||||
}
|
||||
let (without_query, query) = value
|
||||
.split_once('?')
|
||||
.map_or((value, None), |(path, query)| (path, Some(query)));
|
||||
let (scheme, rest) = if let Some(index) = without_query.find(':') {
|
||||
let candidate = &without_query[..index];
|
||||
if is_uri_scheme(candidate) {
|
||||
(Some(candidate), &without_query[index + 1..])
|
||||
} else {
|
||||
(None, without_query)
|
||||
}
|
||||
} else {
|
||||
(None, without_query)
|
||||
};
|
||||
let (authority, path) = if let Some(rest) = rest.strip_prefix("//") {
|
||||
match rest.find('/') {
|
||||
Some(index) => (Some(&rest[..index]), rest[index..].to_owned()),
|
||||
None => (Some(rest), String::new()),
|
||||
}
|
||||
} else {
|
||||
(None, rest.to_owned())
|
||||
};
|
||||
Some(Self {
|
||||
scheme,
|
||||
authority,
|
||||
path,
|
||||
query,
|
||||
})
|
||||
}
|
||||
|
||||
fn render(&self) -> Option<String> {
|
||||
let scheme = self.scheme?;
|
||||
let scheme = scheme.to_ascii_lowercase();
|
||||
let mut value = format!("{scheme}:");
|
||||
if let Some(authority) = self.authority {
|
||||
value.push_str("//");
|
||||
value.push_str(&canonical_authority(authority, &scheme));
|
||||
}
|
||||
if self.authority.is_some() && self.path.is_empty() {
|
||||
value.push('/');
|
||||
} else {
|
||||
value.push_str(&self.path);
|
||||
}
|
||||
if let Some(query) = self.query {
|
||||
value.push('?');
|
||||
value.push_str(query);
|
||||
}
|
||||
Some(value)
|
||||
}
|
||||
}
|
||||
|
||||
fn canonical_authority(authority: &str, scheme: &str) -> String {
|
||||
let authority = authority.to_ascii_lowercase();
|
||||
let default_port = match scheme {
|
||||
"http" => Some("80"),
|
||||
"https" => Some("443"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(default_port) = default_port
|
||||
&& let Some((host, port)) = authority.rsplit_once(':')
|
||||
&& port == default_port
|
||||
&& (!host.contains(':') || host.ends_with(']'))
|
||||
{
|
||||
return host.to_owned();
|
||||
}
|
||||
authority
|
||||
}
|
||||
|
||||
fn has_uri_scheme(value: &str) -> bool {
|
||||
value
|
||||
.split_once(':')
|
||||
.is_some_and(|(candidate, _)| is_uri_scheme(candidate))
|
||||
}
|
||||
|
||||
fn is_uri_scheme(candidate: &str) -> bool {
|
||||
let Some(first) = candidate.as_bytes().first() else {
|
||||
return false;
|
||||
};
|
||||
first.is_ascii_alphabetic()
|
||||
&& candidate
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.'))
|
||||
}
|
||||
|
||||
fn merge_paths(base_path: &str, has_authority: bool, relative_path: &str) -> String {
|
||||
match base_path.rfind('/') {
|
||||
Some(index) => format!("{}{}", &base_path[..=index], relative_path),
|
||||
None if has_authority => format!("/{relative_path}"),
|
||||
None => relative_path.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_dot_segments(path: &str) -> String {
|
||||
let leading_slash = path.starts_with('/');
|
||||
let trailing_slash = path.ends_with('/');
|
||||
let mut output = Vec::new();
|
||||
for segment in path.split('/') {
|
||||
match segment {
|
||||
"." => {}
|
||||
".." => {
|
||||
output.pop();
|
||||
}
|
||||
_ => output.push(segment),
|
||||
}
|
||||
}
|
||||
let mut result = output.join("/");
|
||||
if leading_slash && !result.starts_with('/') {
|
||||
result.insert(0, '/');
|
||||
}
|
||||
if trailing_slash && !result.ends_with('/') {
|
||||
result.push('/');
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TargetError {
|
||||
MalformedFragment,
|
||||
}
|
||||
|
||||
fn percent_decode(fragment: &str) -> Result<String, TargetError> {
|
||||
let bytes = fragment.as_bytes();
|
||||
let mut decoded = Vec::with_capacity(bytes.len());
|
||||
let mut index = 0;
|
||||
while index < bytes.len() {
|
||||
if bytes[index] != b'%' {
|
||||
decoded.push(bytes[index]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
let Some(high) = bytes.get(index + 1).and_then(|byte| hex_value(*byte)) else {
|
||||
return Err(TargetError::MalformedFragment);
|
||||
};
|
||||
let Some(low) = bytes.get(index + 2).and_then(|byte| hex_value(*byte)) else {
|
||||
return Err(TargetError::MalformedFragment);
|
||||
};
|
||||
decoded.push((high << 4) | low);
|
||||
index += 3;
|
||||
}
|
||||
String::from_utf8(decoded).map_err(|_| TargetError::MalformedFragment)
|
||||
}
|
||||
|
||||
fn hex_value(byte: u8) -> Option<u8> {
|
||||
match byte {
|
||||
b'0'..=b'9' => Some(byte - b'0'),
|
||||
b'a'..=b'f' => Some(byte - b'a' + 10),
|
||||
b'A'..=b'F' => Some(byte - b'A' + 10),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_literal_payload_key(key: &str) -> bool {
|
||||
matches!(key, "example" | "examples" | "default" | "enum" | "const") || key.starts_with("x-")
|
||||
}
|
||||
@@ -32,8 +32,33 @@ pub fn schema_from_openapi(
|
||||
let Some(value) = value else {
|
||||
return primitive(SchemaKind::String, required, description);
|
||||
};
|
||||
// NormalizedIR keeps composition typed; the legacy preview adapter retains
|
||||
// its historical first-branch projection for pending v1 jobs.
|
||||
// Only v3 NormalizedIR emits the private marker. Pending legacy-v1 jobs
|
||||
// retain their historical first-branch behavior, while modern oneOf/anyOf
|
||||
// is projected losslessly through crank-schema's existing Oneof shape.
|
||||
if value
|
||||
.get("x-crank-lossless-composition")
|
||||
.and_then(Value::as_bool)
|
||||
== Some(true)
|
||||
&& let Some(items) = value
|
||||
.get("oneOf")
|
||||
.or_else(|| value.get("anyOf"))
|
||||
.and_then(Value::as_array)
|
||||
{
|
||||
return Schema {
|
||||
kind: SchemaKind::Oneof,
|
||||
description: description.or_else(|| text(value, "description")),
|
||||
required,
|
||||
nullable: nullable(value),
|
||||
default_value: value.get("default").cloned(),
|
||||
fields: BTreeMap::new(),
|
||||
items: None,
|
||||
enum_values: Vec::new(),
|
||||
variants: items
|
||||
.iter()
|
||||
.map(|item| schema_from_openapi(Some(item), true, None))
|
||||
.collect(),
|
||||
};
|
||||
}
|
||||
let resolved = collapse_composition(value);
|
||||
|
||||
if let Some(values) = resolved.get("enum").and_then(Value::as_array) {
|
||||
|
||||
@@ -107,6 +107,7 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
|
||||
parsed_parameters.findings,
|
||||
);
|
||||
operation_parameters.extend(parsed_parameters.parameters);
|
||||
deduplicate_parameters(&mut operation_parameters);
|
||||
let tags = tags(
|
||||
operation_value.get("tags"),
|
||||
&format!("{path_pointer}/{method_name}/tags"),
|
||||
@@ -183,6 +184,13 @@ fn parse_document_v2(root: &Value) -> Result<RestImportDocument, ImportParseErro
|
||||
})
|
||||
}
|
||||
|
||||
fn deduplicate_parameters(parameters: &mut Vec<RestImportParameter>) {
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
parameters.reverse();
|
||||
parameters.retain(|parameter| seen.insert((parameter.name.clone(), parameter.location)));
|
||||
parameters.reverse();
|
||||
}
|
||||
|
||||
pub fn parse_document_legacy_v1(root: &Value) -> Result<RestImportDocument, ImportParseError> {
|
||||
let title = root
|
||||
.pointer("/info/title")
|
||||
|
||||
@@ -161,6 +161,58 @@ paths:
|
||||
assert!(ir.operations[0].request_body_schema.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_parameters_override_path_parameters_by_name_and_location() {
|
||||
let openapi = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: OAS parameter overrides }
|
||||
servers: [{ url: https://example.test }]
|
||||
paths:
|
||||
/items:
|
||||
parameters:
|
||||
- { name: page, in: query, description: path, schema: { type: integer } }
|
||||
get:
|
||||
operationId: listItems
|
||||
parameters:
|
||||
- { name: page, in: query, description: operation, schema: { type: string } }
|
||||
responses: { '200': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize_document(openapi, &NormalizationConfig::default()).unwrap();
|
||||
assert_eq!(ir.operations[0].parameters.len(), 1);
|
||||
assert_eq!(
|
||||
ir.operations[0].parameters[0].description.as_deref(),
|
||||
Some("operation")
|
||||
);
|
||||
assert_eq!(
|
||||
ir.operations[0].parameters[0].source_location.pointer,
|
||||
"/paths/~1items/get/parameters/0"
|
||||
);
|
||||
|
||||
let swagger = r#"
|
||||
swagger: '2.0'
|
||||
info: { title: Swagger parameter overrides }
|
||||
paths:
|
||||
/items:
|
||||
parameters:
|
||||
- { name: page, in: query, description: path, type: integer }
|
||||
get:
|
||||
operationId: listItems
|
||||
parameters:
|
||||
- { name: page, in: query, description: operation, type: string }
|
||||
responses: { '200': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize_document(swagger, &NormalizationConfig::default()).unwrap();
|
||||
assert_eq!(ir.operations[0].parameters.len(), 1);
|
||||
assert_eq!(
|
||||
ir.operations[0].parameters[0].description.as_deref(),
|
||||
Some("operation")
|
||||
);
|
||||
assert_eq!(
|
||||
ir.operations[0].parameters[0].source_location.pointer,
|
||||
"/paths/~1items/get/parameters/0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openapi_parameter_omissions_are_errors_at_the_dropped_item_pointer() {
|
||||
let document = r#"
|
||||
@@ -220,10 +272,6 @@ components:
|
||||
"/paths/~1items~1{id}/parameters/1",
|
||||
),
|
||||
("invalid_parameter", "/paths/~1items~1{id}/parameters/2",),
|
||||
(
|
||||
"unresolved_parameter_reference",
|
||||
"/paths/~1items~1{id}/parameters/3",
|
||||
),
|
||||
(
|
||||
"missing_parameter_name",
|
||||
"/paths/~1items~1{id}/get/parameters/0",
|
||||
@@ -336,10 +384,6 @@ parameters:
|
||||
"/paths/~1items~1{id}/parameters/1",
|
||||
),
|
||||
("invalid_parameter", "/paths/~1items~1{id}/parameters/2",),
|
||||
(
|
||||
"unresolved_parameter_reference",
|
||||
"/paths/~1items~1{id}/parameters/3",
|
||||
),
|
||||
(
|
||||
"missing_parameter_name",
|
||||
"/paths/~1items~1{id}/get/parameters/0",
|
||||
@@ -688,7 +732,7 @@ paths:
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_every_unresolved_reference_from_the_full_source_tree() {
|
||||
fn resolves_local_reference_graph_and_preserves_external_blocker() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: References }
|
||||
@@ -722,32 +766,24 @@ components:
|
||||
serde_json::to_vec(&first).unwrap(),
|
||||
serde_json::to_vec(&second).unwrap()
|
||||
);
|
||||
assert_eq!(first.unresolved_references.len(), 10);
|
||||
assert_eq!(first.unresolved_references.len(), 1);
|
||||
let references = first
|
||||
.unresolved_references
|
||||
.iter()
|
||||
.map(|reference| (reference.uri.as_str(), reference.location.pointer.as_str()))
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert!(references.contains(&(
|
||||
"#/components/parameters/Id",
|
||||
"/paths/~1ok/parameters/0/$ref"
|
||||
)));
|
||||
assert!(references.contains(&(
|
||||
"#/components/requestBodies/Body",
|
||||
"/paths/~1ok/get/requestBody/$ref"
|
||||
)));
|
||||
assert!(references.contains(&(
|
||||
"#/components/responses/Ok",
|
||||
"/paths/~1ok/get/responses/200/$ref"
|
||||
)));
|
||||
assert!(references.contains(&("#/components/pathItems/Reusable", "/paths/~1reused/$ref")));
|
||||
assert!(
|
||||
references.contains(&("#/components/schemas/Loop", "/components/schemas/Loop/$ref"))
|
||||
);
|
||||
assert!(references.contains(&(
|
||||
"https://example.test/schema.json#/Remote",
|
||||
"/components/schemas/Remote/$ref"
|
||||
)));
|
||||
assert!(first.reference_graph.edges.len() >= 5);
|
||||
assert!(
|
||||
first
|
||||
.reference_graph
|
||||
.edges
|
||||
.iter()
|
||||
.any(|edge| edge.recursive)
|
||||
);
|
||||
assert!(first.unresolved_references.iter().all(|reference| {
|
||||
reference.construct_id
|
||||
== format!(
|
||||
|
||||
@@ -94,7 +94,7 @@ definitions:
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn previews_swagger2_and_preserves_unresolved_definitions() {
|
||||
fn previews_swagger2_and_resolves_local_definitions() {
|
||||
let preview = preview_document(SWAGGER2).unwrap();
|
||||
|
||||
assert_eq!(preview.source.format, "swagger");
|
||||
@@ -105,7 +105,7 @@ definitions:
|
||||
let operation = &preview.groups[0].operations[0];
|
||||
assert_eq!(operation.suggested_name, "get_pet");
|
||||
assert_eq!(operation.input_fields, 1);
|
||||
assert_eq!(operation.output_fields, 0);
|
||||
assert_eq!(operation.output_fields, 2);
|
||||
assert_eq!(operation.draft.target.path_template, "/pets/{id}");
|
||||
assert_eq!(
|
||||
operation.draft.input_mapping.rules[0].target,
|
||||
|
||||
@@ -0,0 +1,793 @@
|
||||
use crank_import::rest::{
|
||||
ExternalDocumentSnapshot, ImportFindingSeverity, NormalizationConfig, NormalizedSchemaKind,
|
||||
SourceDigest, external_reference_uris, normalize_verified_bundle, normalize_verified_document,
|
||||
preview_from_ir, reference_uris,
|
||||
};
|
||||
|
||||
fn digest(byte: char) -> SourceDigest {
|
||||
SourceDigest::parse(byte.to_string().repeat(64)).unwrap()
|
||||
}
|
||||
|
||||
fn normalize(document: &str) -> crank_import::rest::NormalizedIr {
|
||||
normalize_verified_document(document, digest('a'), &NormalizationConfig::default()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_local_schema_and_object_references_with_rfc6901_escaping() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Local refs }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items/{id}:
|
||||
get:
|
||||
operationId: getItem
|
||||
parameters:
|
||||
- { $ref: '#/components/parameters/Id' }
|
||||
responses:
|
||||
'200': { $ref: '#/components/responses/Ok' }
|
||||
components:
|
||||
parameters:
|
||||
Id: { name: id, in: path, required: true, schema: { type: string } }
|
||||
responses:
|
||||
Ok:
|
||||
description: ok
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/a~1b~0c' }
|
||||
schemas:
|
||||
a/b~c:
|
||||
type: object
|
||||
required: [id]
|
||||
properties: { id: { type: string } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
let operation = &ir.operations[0];
|
||||
assert_eq!(operation.parameters.len(), 1);
|
||||
assert_eq!(operation.parameters[0].name, "id");
|
||||
assert!(matches!(
|
||||
operation.response_schema.as_ref().map(|schema| &schema.kind),
|
||||
Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id")
|
||||
));
|
||||
assert!(ir.unresolved_references.is_empty());
|
||||
assert_eq!(ir.reference_graph.edges.len(), 3);
|
||||
assert!(
|
||||
ir.findings
|
||||
.iter()
|
||||
.chain(operation.findings.iter())
|
||||
.all(|finding| finding.code != "unresolved_reference")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn broken_reference_blocks_only_affected_candidate_and_full_preview_remains_visible() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Partial graph }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/broken:
|
||||
get:
|
||||
operationId: broken
|
||||
responses:
|
||||
'200':
|
||||
description: nope
|
||||
content: { application/json: { schema: { $ref: '#/components/schemas/Missing' } } }
|
||||
/healthy:
|
||||
get:
|
||||
operationId: healthy
|
||||
responses: { '204': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
let broken = ir
|
||||
.operations
|
||||
.iter()
|
||||
.find(|operation| operation.path == "/broken")
|
||||
.unwrap();
|
||||
let healthy = ir
|
||||
.operations
|
||||
.iter()
|
||||
.find(|operation| operation.path == "/healthy")
|
||||
.unwrap();
|
||||
assert!(broken.findings.iter().any(|finding| {
|
||||
finding.code == "reference_target_missing"
|
||||
&& finding.severity == ImportFindingSeverity::Error
|
||||
}));
|
||||
assert!(healthy.findings.is_empty());
|
||||
let preview = preview_from_ir(&ir);
|
||||
assert_eq!(
|
||||
preview
|
||||
.groups
|
||||
.iter()
|
||||
.map(|group| group.operations.len())
|
||||
.sum::<usize>(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_references_are_default_deny_without_network_or_snapshot() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: External deny }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
assert!(
|
||||
ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| finding.code == "external_reference_disabled")
|
||||
);
|
||||
assert!(ir.reference_graph.edges.is_empty());
|
||||
assert!(ir.reference_graph.dependency_digests.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_supplied_external_snapshot_and_relative_chain_deterministically() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: External snapshots }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: { application/json: { schema: { $ref: 'https://schemas.example.test/root.yaml#/Item' } } }
|
||||
"#;
|
||||
let snapshots = vec![
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/root.yaml".to_owned(),
|
||||
digest: digest('b'),
|
||||
document: "Item: { $ref: 'child.yaml#/Child' }".to_owned(),
|
||||
},
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/child.yaml".to_owned(),
|
||||
digest: digest('c'),
|
||||
document: "Child: { type: object, properties: { value: { type: integer } } }"
|
||||
.to_owned(),
|
||||
},
|
||||
];
|
||||
let first = normalize_verified_bundle(
|
||||
document,
|
||||
digest('a'),
|
||||
&snapshots,
|
||||
&NormalizationConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let second = normalize_verified_bundle(
|
||||
document,
|
||||
digest('a'),
|
||||
&snapshots,
|
||||
&NormalizationConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_vec(&first).unwrap(),
|
||||
serde_json::to_vec(&second).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
first.reference_graph.dependency_digests,
|
||||
vec![digest('b'), digest('c')]
|
||||
);
|
||||
assert_eq!(first.reference_graph.edges.len(), 2);
|
||||
assert_eq!(
|
||||
first.reference_graph.edges[1].source.snapshot_digest,
|
||||
digest('b')
|
||||
);
|
||||
assert!(first.unresolved_references.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursion_is_a_stable_graph_edge_without_unbounded_expansion() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Recursive }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/nodes:
|
||||
get:
|
||||
operationId: getNode
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: { application/json: { schema: { $ref: '#/components/schemas/Node' } } }
|
||||
components:
|
||||
schemas:
|
||||
Node:
|
||||
type: object
|
||||
properties:
|
||||
child: { $ref: '#/components/schemas/Node' }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
assert!(ir.reference_graph.edges.iter().any(|edge| edge.recursive));
|
||||
assert!(ir.unresolved_references.is_empty());
|
||||
assert!(
|
||||
ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.all(|finding| finding.code != "reference_graph_limit")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merges_compatible_all_of_and_blocks_conflicts_without_first_branch_loss() {
|
||||
let compatible = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: AllOf }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
post:
|
||||
operationId: createItem
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- { type: object, required: [id], properties: { id: { type: string } } }
|
||||
- { type: object, required: [name], properties: { name: { type: string } } }
|
||||
responses: { '204': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize(compatible);
|
||||
assert!(matches!(
|
||||
ir.operations[0].request_body_schema.as_ref().map(|schema| &schema.kind),
|
||||
Some(NormalizedSchemaKind::Object { properties, required })
|
||||
if properties.len() == 2 && required == &vec!["id".to_owned(), "name".to_owned()]
|
||||
));
|
||||
|
||||
let conflict = compatible.replace("{ name: { type: string } }", "{ id: { type: integer } }");
|
||||
let ir = normalize(&conflict);
|
||||
assert!(
|
||||
ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| finding.code == "all_of_conflict")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_one_of_discriminator_and_projects_all_alternatives() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Alternatives }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/events:
|
||||
post:
|
||||
operationId: createEvent
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
discriminator:
|
||||
propertyName: kind
|
||||
mapping: { text: '#/components/schemas/Text' }
|
||||
oneOf:
|
||||
- { type: object, properties: { text: { type: string } } }
|
||||
- { type: object, properties: { count: { type: integer } } }
|
||||
responses: { '204': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
let schema = ir.operations[0].request_body_schema.as_ref().unwrap();
|
||||
assert_eq!(schema.discriminator.as_ref().unwrap().property_name, "kind");
|
||||
assert!(matches!(
|
||||
&schema.kind,
|
||||
NormalizedSchemaKind::Composition { operator, variants }
|
||||
if operator == "oneOf" && variants.len() == 2
|
||||
));
|
||||
let preview = preview_from_ir(&ir);
|
||||
let candidate = &preview.groups[0].operations[0];
|
||||
assert_eq!(
|
||||
candidate
|
||||
.draft
|
||||
.input_schema
|
||||
.fields
|
||||
.get("body")
|
||||
.unwrap()
|
||||
.variants
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
|
||||
let any_of = document
|
||||
.replace("discriminator:\n propertyName: kind\n mapping: { text: '#/components/schemas/Text' }\n oneOf:", "anyOf:");
|
||||
let ir = normalize(&any_of);
|
||||
assert!(matches!(
|
||||
&ir.operations[0].request_body_schema.as_ref().unwrap().kind,
|
||||
NormalizedSchemaKind::Composition { operator, variants }
|
||||
if operator == "anyOf" && variants.len() == 2
|
||||
));
|
||||
let preview = preview_from_ir(&ir);
|
||||
assert_eq!(
|
||||
preview.groups[0].operations[0].draft.input_schema.fields["body"]
|
||||
.variants
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oas_31_applies_ref_siblings_while_oas_30_ignores_them() {
|
||||
let template = |version: &str| {
|
||||
format!(
|
||||
r#"
|
||||
openapi: {version}
|
||||
info: {{ title: Siblings }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Item'
|
||||
description: sibling-description
|
||||
components:
|
||||
schemas:
|
||||
Item: {{ type: string, description: target-description }}
|
||||
"#
|
||||
)
|
||||
};
|
||||
let v30 = normalize(&template("3.0.3"));
|
||||
let v31 = normalize(&template("3.1.0"));
|
||||
assert_eq!(
|
||||
v30.operations[0]
|
||||
.response_schema
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.description
|
||||
.as_deref(),
|
||||
Some("target-description")
|
||||
);
|
||||
assert_eq!(
|
||||
v31.operations[0]
|
||||
.response_schema
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.description
|
||||
.as_deref(),
|
||||
Some("sibling-description")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_pointer_and_type_mismatch_are_exact_blockers_without_panic() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Invalid targets }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/missing:
|
||||
get:
|
||||
operationId: missing
|
||||
responses:
|
||||
'200': { description: ok, content: { application/json: { schema: { $ref: '#not-a-pointer' } } } }
|
||||
/scalar:
|
||||
get:
|
||||
operationId: scalar
|
||||
responses:
|
||||
'200': { description: ok, content: { application/json: { schema: { $ref: '#/info/title' } } } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
let codes = ir
|
||||
.operations
|
||||
.iter()
|
||||
.flat_map(|operation| {
|
||||
operation
|
||||
.findings
|
||||
.iter()
|
||||
.map(move |finding| (operation.path.as_str(), finding.code.as_str()))
|
||||
})
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert!(codes.contains(&("/missing", "reference_target_missing")));
|
||||
assert!(codes.contains(&("/scalar", "reference_type_mismatch")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_depth_and_expanded_node_limits_fail_closed() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Bounded }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200': { description: ok, content: { application/json: { schema: { $ref: '#/components/schemas/A' } } } }
|
||||
components:
|
||||
schemas:
|
||||
A: { $ref: '#/components/schemas/B' }
|
||||
B: { $ref: '#/components/schemas/C' }
|
||||
C: { type: object, properties: { id: { type: string } } }
|
||||
"#;
|
||||
let config = NormalizationConfig {
|
||||
max_reference_depth: 1,
|
||||
..NormalizationConfig::default()
|
||||
};
|
||||
let ir = normalize_verified_document(document, digest('a'), &config).unwrap();
|
||||
assert!(
|
||||
ir.operations
|
||||
.iter()
|
||||
.flat_map(|operation| &operation.findings)
|
||||
.any(|finding| finding.code == "reference_graph_limit")
|
||||
);
|
||||
|
||||
let config = NormalizationConfig {
|
||||
max_expanded_nodes: 8,
|
||||
..NormalizationConfig::default()
|
||||
};
|
||||
let ir = normalize_verified_document(document, digest('a'), &config).unwrap();
|
||||
assert!(
|
||||
ir.findings
|
||||
.iter()
|
||||
.chain(
|
||||
ir.operations
|
||||
.iter()
|
||||
.flat_map(|operation| &operation.findings)
|
||||
)
|
||||
.any(|finding| finding.code == "reference_graph_limit")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_discriminator_and_multiple_composition_operators_are_blockers() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Unsupported composition }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/events:
|
||||
post:
|
||||
operationId: createEvent
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
discriminator: { mapping: { bad: 42 } }
|
||||
oneOf: [{ type: string }, { type: integer }]
|
||||
anyOf: [{ type: boolean }, { type: string }]
|
||||
responses: { '204': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
let codes = ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.map(|finding| finding.code.as_str())
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert!(codes.contains("unsupported_discriminator"));
|
||||
assert!(codes.contains("unsupported_composition"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_uri_scan_applies_external_size_depth_and_alias_limits_before_traversal() {
|
||||
let config = NormalizationConfig {
|
||||
max_bytes: 8,
|
||||
max_external_document_bytes: 8,
|
||||
..NormalizationConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
reference_uris("external-document", &config),
|
||||
Err(crank_import::rest::ImportParseError::LimitExceeded)
|
||||
);
|
||||
|
||||
let config = NormalizationConfig {
|
||||
max_bytes: 8,
|
||||
max_external_document_bytes: 32,
|
||||
..NormalizationConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
external_reference_uris("external-document", &config),
|
||||
Ok(Vec::new())
|
||||
);
|
||||
|
||||
let config = NormalizationConfig {
|
||||
max_depth: 1,
|
||||
..NormalizationConfig::default()
|
||||
};
|
||||
assert_eq!(
|
||||
reference_uris("a: { b: { $ref: '#/x' } }", &config),
|
||||
Err(crank_import::rest::ImportParseError::LimitExceeded)
|
||||
);
|
||||
|
||||
let aliases = format!(
|
||||
"items: [{}]",
|
||||
std::iter::repeat_n("*a", 129)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
);
|
||||
assert_eq!(
|
||||
reference_uris(&aliases, &NormalizationConfig::default()),
|
||||
Err(crank_import::rest::ImportParseError::LimitExceeded)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_relative_references_using_rfc3986_paths_and_decoded_fragments() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Relative refs }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content: { application/json: { schema: { $ref: 'HTTPS://SCHEMAS.EXAMPLE.TEST:443/a/b/root.yaml#/Item' } } }
|
||||
"#;
|
||||
let snapshots = vec![
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/a/b/root.yaml".to_owned(),
|
||||
digest: digest('b'),
|
||||
document: r#"
|
||||
Item:
|
||||
type: object
|
||||
properties:
|
||||
dot: { $ref: './child.yaml#/Value' }
|
||||
parent: { $ref: '../common.yaml#/Value' }
|
||||
absolute: { $ref: '/shared.yaml#/Value' }
|
||||
"#
|
||||
.to_owned(),
|
||||
},
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/a/b/child.yaml".to_owned(),
|
||||
digest: digest('c'),
|
||||
document: "Value: { type: string }".to_owned(),
|
||||
},
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/a/common.yaml".to_owned(),
|
||||
digest: digest('d'),
|
||||
document: "Value: { type: integer }".to_owned(),
|
||||
},
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/shared.yaml".to_owned(),
|
||||
digest: digest('e'),
|
||||
document: "Value: { $ref: '#/a%7E1b' }\na/b: { type: boolean }".to_owned(),
|
||||
},
|
||||
];
|
||||
let ir = normalize_verified_bundle(
|
||||
document,
|
||||
digest('a'),
|
||||
&snapshots,
|
||||
&NormalizationConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let Some(NormalizedSchemaKind::Object { properties, .. }) = ir.operations[0]
|
||||
.response_schema
|
||||
.as_ref()
|
||||
.map(|schema| &schema.kind)
|
||||
else {
|
||||
panic!("response should be an expanded object schema");
|
||||
};
|
||||
assert_eq!(properties.len(), 3);
|
||||
assert_eq!(ir.reference_graph.edges.len(), 5);
|
||||
assert!(ir.findings.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_percent_encoded_fragment_is_a_blocker() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Malformed reference }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200': { description: ok, content: { application/json: { schema: { $ref: '#/components/%ZZ' } } } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
assert!(
|
||||
ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| finding.code == "reference_uri_malformed")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_follow_references_inside_literal_payloads() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Literal payloads }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
example: { $ref: '#/components/schemas/Missing' }
|
||||
examples: { sample: { value: { $ref: '#/components/schemas/Missing' } } }
|
||||
default: { $ref: '#/components/schemas/Missing' }
|
||||
enum: [{ $ref: '#/components/schemas/Missing' }]
|
||||
const: { $ref: '#/components/schemas/Missing' }
|
||||
x-fixture: { $ref: '#/components/schemas/Missing' }
|
||||
properties: { known: { $ref: '#/components/schemas/Known' } }
|
||||
components:
|
||||
schemas:
|
||||
Known: { type: string }
|
||||
"#;
|
||||
assert_eq!(
|
||||
reference_uris(document, &NormalizationConfig::default()),
|
||||
Ok(vec!["#/components/schemas/Known".to_owned()])
|
||||
);
|
||||
let ir = normalize(document);
|
||||
assert_eq!(ir.reference_graph.edges.len(), 1);
|
||||
assert!(
|
||||
ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.all(|finding| finding.code != "reference_target_missing")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn primary_oas31_applies_ref_siblings_in_external_snapshots_without_openapi_field() {
|
||||
let source = |version: &str| {
|
||||
format!(
|
||||
r#"
|
||||
openapi: {version}
|
||||
info: {{ title: External siblings }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200': {{ description: ok, content: {{ application/json: {{ schema: {{ $ref: 'https://schemas.example.test/root.yaml#/Item' }} }} }} }}
|
||||
"#
|
||||
)
|
||||
};
|
||||
let snapshots = vec![
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/root.yaml".to_owned(),
|
||||
digest: digest('b'),
|
||||
document: "Item: { $ref: 'child.yaml#/Base', description: sibling }".to_owned(),
|
||||
},
|
||||
ExternalDocumentSnapshot {
|
||||
canonical_uri: "https://schemas.example.test/child.yaml".to_owned(),
|
||||
digest: digest('c'),
|
||||
document: "Base: { type: string, description: target }".to_owned(),
|
||||
},
|
||||
];
|
||||
let v31 = normalize_verified_bundle(
|
||||
&source("3.1.0"),
|
||||
digest('a'),
|
||||
&snapshots,
|
||||
&NormalizationConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let v30 = normalize_verified_bundle(
|
||||
&source("3.0.3"),
|
||||
digest('a'),
|
||||
&snapshots,
|
||||
&NormalizationConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
v31.operations[0]
|
||||
.response_schema
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.description
|
||||
.as_deref(),
|
||||
Some("sibling")
|
||||
);
|
||||
assert_eq!(
|
||||
v30.operations[0]
|
||||
.response_schema
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.description
|
||||
.as_deref(),
|
||||
Some("target")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_of_intersects_constraints_and_nested_properties_without_losing_conflicts() {
|
||||
let source = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: allOf intersections }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
post:
|
||||
operationId: createItem
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- type: object
|
||||
minimum: 1
|
||||
maximum: 10
|
||||
minLength: 2
|
||||
maxLength: 12
|
||||
properties: { nested: { type: object, properties: { left: { type: string } } } }
|
||||
- type: object
|
||||
minimum: 4
|
||||
maximum: 8
|
||||
minLength: 5
|
||||
maxLength: 9
|
||||
properties: { nested: { type: object, properties: { right: { type: integer } } } }
|
||||
responses: { '204': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize(source);
|
||||
let schema = ir.operations[0].request_body_schema.as_ref().unwrap();
|
||||
assert_eq!(schema.constraints.minimum, Some(4.0));
|
||||
assert_eq!(schema.constraints.maximum, Some(8.0));
|
||||
assert_eq!(schema.constraints.min_length, Some(5));
|
||||
assert_eq!(schema.constraints.max_length, Some(9));
|
||||
let NormalizedSchemaKind::Object { properties, .. } = &schema.kind else {
|
||||
panic!("merged schema should remain an object");
|
||||
};
|
||||
let NormalizedSchemaKind::Object { properties, .. } = &properties["nested"].kind else {
|
||||
panic!("nested property should remain an object");
|
||||
};
|
||||
assert!(properties.contains_key("left") && properties.contains_key("right"));
|
||||
|
||||
let conflicting = source.replace("maximum: 8", "maximum: 3");
|
||||
let ir = normalize(&conflicting);
|
||||
assert!(
|
||||
ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| finding.code == "all_of_conflict")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_array_all_of_is_preserved_and_reported() {
|
||||
let document = r#"
|
||||
openapi: 3.1.0
|
||||
info: { title: Invalid allOf }
|
||||
servers: [{ url: https://api.example.test }]
|
||||
paths:
|
||||
/items:
|
||||
post:
|
||||
operationId: createItem
|
||||
requestBody:
|
||||
content: { application/json: { schema: { allOf: { type: string } } } }
|
||||
responses: { '204': { description: ok } }
|
||||
"#;
|
||||
let ir = normalize(document);
|
||||
assert!(
|
||||
ir.operations[0]
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| finding.code == "unsupported_composition")
|
||||
);
|
||||
assert!(matches!(
|
||||
ir.operations[0]
|
||||
.request_body_schema
|
||||
.as_ref()
|
||||
.map(|schema| &schema.kind),
|
||||
Some(NormalizedSchemaKind::Unknown)
|
||||
));
|
||||
}
|
||||
@@ -122,7 +122,7 @@ paths:
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_references_typed_without_resolving_them() {
|
||||
fn resolves_local_references_into_typed_graph() {
|
||||
let ir = normalize_document(SWAGGER2, &NormalizationConfig::default()).unwrap();
|
||||
let operation = &ir.operations[0];
|
||||
assert!(matches!(
|
||||
@@ -130,13 +130,15 @@ paths:
|
||||
.response_schema
|
||||
.as_ref()
|
||||
.map(|schema| &schema.kind),
|
||||
Some(crank_import::rest::NormalizedSchemaKind::Reference { .. })
|
||||
Some(crank_import::rest::NormalizedSchemaKind::Object { .. })
|
||||
));
|
||||
assert!(ir.unresolved_references.is_empty());
|
||||
assert!(!ir.reference_graph.edges.is_empty());
|
||||
assert!(
|
||||
operation
|
||||
.findings
|
||||
.iter()
|
||||
.any(|finding| finding.code == "unresolved_reference")
|
||||
.all(|finding| finding.code != "unresolved_reference")
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ use crank_artifacts::ArtifactRef;
|
||||
|
||||
const APPLICATION_RESULT_KEY: &str = "_crank_application_result";
|
||||
const IMPORT_JOB_CLEANUP_BATCH: u32 = 128;
|
||||
const MAX_IMPORT_JOB_DOCUMENTS: usize = 32;
|
||||
const DANGLING_OPENAPI_SOURCE_GRACE: time::Duration = time::Duration::minutes(5);
|
||||
|
||||
impl PostgresRegistry {
|
||||
@@ -12,6 +13,7 @@ impl PostgresRegistry {
|
||||
request: CreateImportJobRequest<'_>,
|
||||
) -> Result<(), RegistryError> {
|
||||
validate_source_envelope(request.source, request.preview_payload)?;
|
||||
validate_dependency_envelopes(request.preview_payload)?;
|
||||
let mut transaction = self.pool.begin().await?;
|
||||
sqlx::query(
|
||||
"insert into import_jobs (
|
||||
@@ -81,6 +83,26 @@ impl PostgresRegistry {
|
||||
&self,
|
||||
request: FinishImportJobRequest<'_>,
|
||||
) -> Result<(), RegistryError> {
|
||||
let mut transaction = self.pool.begin().await?;
|
||||
let row = sqlx::query(
|
||||
"select workspace_id, status, preview_payload
|
||||
from import_jobs
|
||||
where id = $1
|
||||
for update",
|
||||
)
|
||||
.bind(request.id.as_str())
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await?
|
||||
.ok_or_else(|| RegistryError::ImportJobNotFound {
|
||||
job_id: request.id.as_str().to_owned(),
|
||||
})?;
|
||||
let current_status =
|
||||
deserialize_enum_text::<ImportJobStatus>(row.try_get("status")?, "status")?;
|
||||
if request.status == ImportJobStatus::Failed && current_status == ImportJobStatus::Completed
|
||||
{
|
||||
transaction.commit().await?;
|
||||
return Ok(());
|
||||
}
|
||||
let result = sqlx::query(
|
||||
"update import_jobs
|
||||
set status = $2,
|
||||
@@ -94,7 +116,7 @@ impl PostgresRegistry {
|
||||
.bind(request.created_operation_ids)
|
||||
.bind(request.error_text)
|
||||
.bind(request.finished_at)
|
||||
.execute(&self.pool)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
|
||||
if result.rows_affected() == 0 {
|
||||
@@ -103,6 +125,25 @@ impl PostgresRegistry {
|
||||
});
|
||||
}
|
||||
|
||||
if request.status == ImportJobStatus::Failed {
|
||||
let workspace_id = WorkspaceId::new(row.try_get::<String, _>("workspace_id")?);
|
||||
let payload = row.try_get::<Value, _>("preview_payload")?;
|
||||
// Failure finalization must itself be fail-safe. The strict
|
||||
// envelope parsers are used before Apply; cleanup only needs the
|
||||
// bounded source identities and must not roll back the terminal
|
||||
// state because some other payload field was corrupted.
|
||||
for source_id in cleanup_source_ids(&payload) {
|
||||
let _ = detach_source_in_transaction(
|
||||
&mut transaction,
|
||||
&workspace_id,
|
||||
&source_id,
|
||||
*request.finished_at,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
transaction.commit().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -120,24 +161,18 @@ impl PostgresRegistry {
|
||||
}
|
||||
Err(error) => {
|
||||
tx.rollback().await?;
|
||||
let error_text = error.to_string();
|
||||
let _ = sqlx::query(
|
||||
"update import_jobs
|
||||
set status = $3,
|
||||
error_text = $4,
|
||||
finished_at = $5::timestamptz
|
||||
where id = $1
|
||||
and workspace_id = $2
|
||||
and status <> $6",
|
||||
)
|
||||
.bind(request.id.as_str())
|
||||
.bind(request.workspace_id.as_str())
|
||||
.bind(serialize_enum_text(&ImportJobStatus::Failed, "status")?)
|
||||
.bind(error_text)
|
||||
.bind(request.finished_at)
|
||||
.bind(serialize_enum_text(&ImportJobStatus::Completed, "status")?)
|
||||
.execute(&self.pool)
|
||||
if !matches!(error, RegistryError::ImportJobAlreadyApplied { .. }) {
|
||||
let empty = serde_json::json!([]);
|
||||
let _ = self
|
||||
.finish_import_job(FinishImportJobRequest {
|
||||
id: request.id,
|
||||
status: ImportJobStatus::Failed,
|
||||
created_operation_ids: &empty,
|
||||
error_text: Some("import_apply_failed"),
|
||||
finished_at: request.finished_at,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
@@ -177,18 +212,14 @@ impl PostgresRegistry {
|
||||
// The job itself is expired regardless of whether a legacy or
|
||||
// corrupt payload can be decoded. Do not let one bad row roll
|
||||
// back cleanup for every tenant.
|
||||
if let Ok(Some(source)) = source_from_payload(&payload)
|
||||
&& detach_source_in_transaction(
|
||||
&mut transaction,
|
||||
&workspace_id,
|
||||
&source.source_id,
|
||||
now,
|
||||
)
|
||||
for source_id in cleanup_source_ids(&payload) {
|
||||
if detach_source_in_transaction(&mut transaction, &workspace_id, &source_id, now)
|
||||
.await?
|
||||
{
|
||||
detached_sources += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let deleted = if expired_ids.is_empty() {
|
||||
0
|
||||
} else {
|
||||
@@ -214,7 +245,14 @@ impl PostgresRegistry {
|
||||
select 1
|
||||
from import_jobs j
|
||||
where j.workspace_id = s.workspace_id
|
||||
and j.preview_payload -> 'source' ->> 'source_id' = s.source_id
|
||||
and (
|
||||
j.preview_payload -> 'source' ->> 'source_id' = s.source_id
|
||||
or exists (
|
||||
select 1
|
||||
from jsonb_array_elements(coalesce(j.preview_payload -> 'dependencies', '[]'::jsonb)) d
|
||||
where d ->> 'source_id' = s.source_id
|
||||
)
|
||||
)
|
||||
)
|
||||
order by s.created_at, s.workspace_id, s.source_id
|
||||
limit $2
|
||||
@@ -283,6 +321,7 @@ async fn apply_import_job_transaction(
|
||||
source_from_payload(&preview_payload)?.ok_or(RegistryError::InvalidArtifactSource {
|
||||
field: "import_job_source",
|
||||
})?;
|
||||
let dependencies = dependencies_from_payload(&preview_payload)?;
|
||||
|
||||
if status == ImportJobStatus::Completed
|
||||
&& let Some(result) = stored_application_result(&preview_payload)?
|
||||
@@ -323,6 +362,31 @@ async fn apply_import_job_transaction(
|
||||
if !source_is_current {
|
||||
return Err(RegistryError::SourceUnavailable);
|
||||
}
|
||||
for dependency in &dependencies {
|
||||
let recorded = sqlx::query(
|
||||
"select s.lifecycle, b.artifact_ref
|
||||
from artifact_sources s
|
||||
join artifact_blobs b on b.digest = s.blob_digest
|
||||
where s.workspace_id = $1 and s.source_id = $2
|
||||
for update",
|
||||
)
|
||||
.bind(request.workspace_id.as_str())
|
||||
.bind(dependency.source_id.as_str())
|
||||
.fetch_optional(&mut **tx)
|
||||
.await?;
|
||||
let current = recorded
|
||||
.as_ref()
|
||||
.map(|row| {
|
||||
let lifecycle = row.try_get::<String, _>("lifecycle").ok();
|
||||
let artifact_ref = row.try_get::<String, _>("artifact_ref").ok();
|
||||
lifecycle.as_deref() == Some("active")
|
||||
&& artifact_ref.as_deref() == Some(dependency.digest.as_str())
|
||||
})
|
||||
.unwrap_or(false);
|
||||
if !current {
|
||||
return Err(RegistryError::SourceUnavailable);
|
||||
}
|
||||
}
|
||||
|
||||
let mut result = ImportJobApplyResult {
|
||||
application_key: request.application_key.to_owned(),
|
||||
@@ -408,6 +472,15 @@ async fn apply_import_job_transaction(
|
||||
*request.finished_at,
|
||||
)
|
||||
.await?;
|
||||
for dependency in dependencies {
|
||||
let _ = detach_source_in_transaction(
|
||||
tx,
|
||||
request.workspace_id,
|
||||
&dependency.source_id,
|
||||
*request.finished_at,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -433,6 +506,11 @@ fn validate_source_envelope(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_dependency_envelopes(preview_payload: &Value) -> Result<(), RegistryError> {
|
||||
dependencies_from_payload(preview_payload)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope>, RegistryError> {
|
||||
let Some(source) = payload.get("source") else {
|
||||
return Ok(None);
|
||||
@@ -466,6 +544,73 @@ fn source_from_payload(payload: &Value) -> Result<Option<ImportJobSourceEnvelope
|
||||
}))
|
||||
}
|
||||
|
||||
fn dependencies_from_payload(
|
||||
payload: &Value,
|
||||
) -> Result<Vec<ImportJobSourceEnvelope>, RegistryError> {
|
||||
let Some(value) = payload.get("dependencies") else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
let items = value
|
||||
.as_array()
|
||||
.filter(|items| items.len() <= MAX_IMPORT_JOB_DOCUMENTS)
|
||||
.ok_or(RegistryError::InvalidArtifactSource {
|
||||
field: "import_job_dependencies",
|
||||
})?;
|
||||
let mut dependencies = Vec::with_capacity(items.len());
|
||||
let mut identities = std::collections::BTreeSet::new();
|
||||
for item in items {
|
||||
let source_id = item
|
||||
.get("source_id")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
.ok_or(RegistryError::InvalidArtifactSource {
|
||||
field: "import_job_dependencies",
|
||||
})?;
|
||||
let digest = item.get("digest").and_then(Value::as_str).ok_or(
|
||||
RegistryError::InvalidArtifactSource {
|
||||
field: "import_job_dependencies",
|
||||
},
|
||||
)?;
|
||||
let digest =
|
||||
ArtifactRef::parse(digest).map_err(|_| RegistryError::InvalidArtifactSource {
|
||||
field: "import_job_dependencies",
|
||||
})?;
|
||||
if !identities.insert((source_id.to_owned(), digest.as_str().to_owned())) {
|
||||
return Err(RegistryError::InvalidArtifactSource {
|
||||
field: "import_job_dependencies",
|
||||
});
|
||||
}
|
||||
dependencies.push(ImportJobSourceEnvelope {
|
||||
source_id: ArtifactSourceId::new(source_id),
|
||||
digest,
|
||||
});
|
||||
}
|
||||
Ok(dependencies)
|
||||
}
|
||||
|
||||
fn cleanup_source_ids(payload: &Value) -> Vec<ArtifactSourceId> {
|
||||
let mut identities = std::collections::BTreeSet::new();
|
||||
if let Some(source_id) = payload
|
||||
.pointer("/source/source_id")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
{
|
||||
identities.insert(source_id.to_owned());
|
||||
}
|
||||
if let Some(dependencies) = payload.get("dependencies").and_then(Value::as_array) {
|
||||
for source_id in dependencies
|
||||
.iter()
|
||||
.take(MAX_IMPORT_JOB_DOCUMENTS)
|
||||
.filter_map(|item| item.get("source_id"))
|
||||
.filter_map(Value::as_str)
|
||||
.filter(|value| value.len() <= 132)
|
||||
{
|
||||
identities.insert(source_id.to_owned());
|
||||
}
|
||||
}
|
||||
identities.into_iter().map(ArtifactSourceId::new).collect()
|
||||
}
|
||||
|
||||
async fn detach_source_in_transaction(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
workspace_id: &WorkspaceId,
|
||||
|
||||
@@ -25,7 +25,9 @@ pub use cache::{
|
||||
pub use cache_factory::{
|
||||
BuiltinCacheBackendFactory, CacheBackendFactory, SharedCacheBackendFactory,
|
||||
};
|
||||
pub use crank_adapter_rest::OutboundHttpPolicy;
|
||||
pub use crank_adapter_rest::{
|
||||
ExternalReferenceFetchError, ExternalReferenceFetcher, OutboundHttpPolicy,
|
||||
};
|
||||
pub use error::RuntimeError;
|
||||
pub use execution_failure::{normalize_runtime_error, normalize_runtime_error_for_operation};
|
||||
pub use execution_request::{
|
||||
|
||||
@@ -25,6 +25,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS=
|
||||
CRANK_OUTBOUND_DENIED_HOSTS=
|
||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES=
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000
|
||||
CRANK_ENVIRONMENT=production
|
||||
CRANK_LOG_LEVEL=
|
||||
CRANK_SENTRY_DSN=
|
||||
|
||||
@@ -27,6 +27,12 @@ CRANK_OUTBOUND_ALLOWED_HOSTS=
|
||||
CRANK_OUTBOUND_DENIED_HOSTS=
|
||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES=4194304
|
||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES=4194304
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES=
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH=8
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS=32
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES=262144
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS=10000
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES=10000
|
||||
CRANK_ENVIRONMENT=production
|
||||
CRANK_LOG_LEVEL=
|
||||
CRANK_SENTRY_DSN=
|
||||
|
||||
@@ -127,6 +127,12 @@ services:
|
||||
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000}
|
||||
volumes:
|
||||
- artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage}
|
||||
ports:
|
||||
|
||||
@@ -130,6 +130,12 @@ services:
|
||||
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000}
|
||||
volumes:
|
||||
- artifact_storage:${CRANK_STORAGE_ROOT:-/var/lib/crank/storage}
|
||||
ports:
|
||||
|
||||
@@ -109,6 +109,12 @@ services:
|
||||
CRANK_OUTBOUND_DENIED_HOSTS: ${CRANK_OUTBOUND_DENIED_HOSTS:-}
|
||||
CRANK_OUTBOUND_MAX_REQUEST_BYTES: ${CRANK_OUTBOUND_MAX_REQUEST_BYTES:-4194304}
|
||||
CRANK_OUTBOUND_MAX_RESPONSE_BYTES: ${CRANK_OUTBOUND_MAX_RESPONSE_BYTES:-4194304}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES:-}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH:-8}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS:-32}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES:-262144}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS: ${CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS:-10000}
|
||||
CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES: ${CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES:-10000}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
|
||||
@@ -27,6 +27,12 @@ Crank настраивается через переменные окружен
|
||||
| `CRANK_OUTBOUND_DENIED_HOSTS` | `outbound.denied_hosts` | `Shared` | `host_list/-` | `` | `-` | `Internal` | `Effective` |
|
||||
| `CRANK_OUTBOUND_MAX_REQUEST_BYTES` | `outbound.max_request_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` |
|
||||
| `CRANK_OUTBOUND_MAX_RESPONSE_BYTES` | `outbound.max_response_bytes` | `Shared` | `u64/bytes` | `4194304` | `1..=67108864` | `Public` | `Effective` |
|
||||
| `CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES` | `import.external_references.allowed_url_prefixes` | `AdminApi` | `url_prefix_list/-` | `` | `-` | `Internal` | `Effective` |
|
||||
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH` | `import.external_references.max_depth` | `AdminApi` | `u32/edges` | `8` | `1..=32` | `Public` | `Effective` |
|
||||
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS` | `import.external_references.max_documents` | `AdminApi` | `u32/documents` | `32` | `1..=32` | `Public` | `Effective` |
|
||||
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES` | `import.external_references.max_fetch_bytes` | `AdminApi` | `u64/bytes` | `262144` | `1..=4194304` | `Public` | `Effective` |
|
||||
| `CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS` | `import.external_references.fetch_timeout_ms` | `AdminApi` | `u64/milliseconds` | `10000` | `1..=300000` | `Public` | `Effective` |
|
||||
| `CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES` | `import.external_references.max_expanded_nodes` | `AdminApi` | `u32/nodes` | `10000` | `1..=100000` | `Public` | `Effective` |
|
||||
| `CRANK_ENVIRONMENT` | `observability.environment` | `Shared` | `label/-` | `development` | `-` | `Public` | `Effective` |
|
||||
| `CRANK_LOG_LEVEL` | `observability.log_filter` | `Shared` | `string/-` | `blank` | `-` | `Public` | `Effective` |
|
||||
| `CRANK_SENTRY_DSN` | `observability.sentry_dsn` | `Shared` | `url/-` | `blank` | `-` | `Secret` | `Effective` |
|
||||
|
||||
@@ -330,6 +330,99 @@
|
||||
"compatibility": null,
|
||||
"rules": []
|
||||
},
|
||||
{
|
||||
"semantic_path": "import.external_references.allowed_url_prefixes",
|
||||
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_ALLOWED_URL_PREFIXES",
|
||||
"process": "admin_api",
|
||||
"value_type": "url_prefix_list",
|
||||
"unit": null,
|
||||
"default": "",
|
||||
"required": false,
|
||||
"minimum": null,
|
||||
"maximum": null,
|
||||
"sensitivity": "internal",
|
||||
"mode": "effective",
|
||||
"compatibility": null,
|
||||
"rules": [
|
||||
"empty list disables external OpenAPI reference fetching",
|
||||
"each prefix must be canonical HTTP(S) without userinfo, query, or fragment"
|
||||
]
|
||||
},
|
||||
{
|
||||
"semantic_path": "import.external_references.max_depth",
|
||||
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DEPTH",
|
||||
"process": "admin_api",
|
||||
"value_type": "u32",
|
||||
"unit": "edges",
|
||||
"default": "8",
|
||||
"required": false,
|
||||
"minimum": 1,
|
||||
"maximum": 32,
|
||||
"sensitivity": "public",
|
||||
"mode": "effective",
|
||||
"compatibility": null,
|
||||
"rules": []
|
||||
},
|
||||
{
|
||||
"semantic_path": "import.external_references.max_documents",
|
||||
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_DOCUMENTS",
|
||||
"process": "admin_api",
|
||||
"value_type": "u32",
|
||||
"unit": "documents",
|
||||
"default": "32",
|
||||
"required": false,
|
||||
"minimum": 1,
|
||||
"maximum": 32,
|
||||
"sensitivity": "public",
|
||||
"mode": "effective",
|
||||
"compatibility": null,
|
||||
"rules": []
|
||||
},
|
||||
{
|
||||
"semantic_path": "import.external_references.max_fetch_bytes",
|
||||
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_FETCH_BYTES",
|
||||
"process": "admin_api",
|
||||
"value_type": "u64",
|
||||
"unit": "bytes",
|
||||
"default": "262144",
|
||||
"required": false,
|
||||
"minimum": 1,
|
||||
"maximum": 4194304,
|
||||
"sensitivity": "public",
|
||||
"mode": "effective",
|
||||
"compatibility": null,
|
||||
"rules": []
|
||||
},
|
||||
{
|
||||
"semantic_path": "import.external_references.fetch_timeout_ms",
|
||||
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_FETCH_TIMEOUT_MS",
|
||||
"process": "admin_api",
|
||||
"value_type": "u64",
|
||||
"unit": "milliseconds",
|
||||
"default": "10000",
|
||||
"required": false,
|
||||
"minimum": 1,
|
||||
"maximum": 300000,
|
||||
"sensitivity": "public",
|
||||
"mode": "effective",
|
||||
"compatibility": null,
|
||||
"rules": []
|
||||
},
|
||||
{
|
||||
"semantic_path": "import.external_references.max_expanded_nodes",
|
||||
"env_name": "CRANK_IMPORT_EXTERNAL_REFERENCE_MAX_EXPANDED_NODES",
|
||||
"process": "admin_api",
|
||||
"value_type": "u32",
|
||||
"unit": "nodes",
|
||||
"default": "10000",
|
||||
"required": false,
|
||||
"minimum": 1,
|
||||
"maximum": 100000,
|
||||
"sensitivity": "public",
|
||||
"mode": "effective",
|
||||
"compatibility": null,
|
||||
"rules": []
|
||||
},
|
||||
{
|
||||
"semantic_path": "observability.environment",
|
||||
"env_name": "CRANK_ENVIRONMENT",
|
||||
|
||||
Reference in New Issue
Block a user