From 1b6db09bd162fbb2b4495a801ee69de53645508d Mon Sep 17 00:00:00 2001 From: bsodfather Date: Sun, 11 Oct 2026 06:19:16 +0000 Subject: [PATCH] =?UTF-8?q?fix(import):=20=D0=B5=D0=B4=D0=B8=D0=BD=D0=B0?= =?UTF-8?q?=D1=8F=20=D0=B8=D0=B4=D0=B5=D0=BD=D1=82=D0=B8=D1=87=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8C=20=D0=B2=D0=BD=D0=B5=D1=88=D0=BD=D0=B8=D1=85?= =?UTF-8?q?=20URI=20(#96)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Единая transport-canonical идентичность внешних документов; normalizer v6 с immutable historical replay; matrix 1.0.4 / corpus 1.0.5. PR CI run840 success, локальная приёмка и independent review выполнены. Co-authored-by: bsodfather --- apps/admin-api/src/service/imports.rs | 24 +- .../src/service/imports/verified_preview.rs | 4 + .../openapi_import/external_references.rs | 7 +- .../external_references/uri_transport.rs | 603 ++++++++++++++++++ .../fixtures/v5-v6-uri-preview.json | 170 +++++ .../import_plan_apply/support_matrix.rs | 173 +++-- .../import_plan_create/creation.rs | 2 +- .../integration/openapi_import/v5_replay.rs | 170 +++++ crates/crank-import/src/rest/mod.rs | 2 + crates/crank-import/src/rest/model.rs | 2 +- crates/crank-import/src/rest/reference.rs | 210 +----- crates/crank-import/src/rest/uri.rs | 96 +++ crates/crank-import/tests/corpus.rs | 23 +- .../crank-import/tests/corpus/manifest.json | 28 +- crates/crank-import/tests/corpus/matrix.rs | 149 +++-- .../tests/corpus/reference_uri.rs | 123 ++++ .../corpus/regression-uri-encoded-dot.yaml | 13 + .../corpus/regression-uri-query-quote.yaml | 13 + .../tests/reference_resolution.rs | 6 + .../reference_resolution/uri_transport.rs | 194 ++++++ crates/crank-import/tests/support-matrix.yaml | 4 +- docs/openapi-import.md | 6 +- 22 files changed, 1669 insertions(+), 353 deletions(-) create mode 100644 apps/admin-api/tests/integration/openapi_import/external_references/uri_transport.rs create mode 100644 apps/admin-api/tests/integration/openapi_import/fixtures/v5-v6-uri-preview.json create mode 100644 crates/crank-import/src/rest/uri.rs create mode 100644 crates/crank-import/tests/corpus/reference_uri.rs create mode 100644 crates/crank-import/tests/corpus/regression-uri-encoded-dot.yaml create mode 100644 crates/crank-import/tests/corpus/regression-uri-query-quote.yaml create mode 100644 crates/crank-import/tests/reference_resolution/uri_transport.rs diff --git a/apps/admin-api/src/service/imports.rs b/apps/admin-api/src/service/imports.rs index a7d74ad..4b10d54 100644 --- a/apps/admin-api/src/service/imports.rs +++ b/apps/admin-api/src/service/imports.rs @@ -7,8 +7,8 @@ use crank_core::{ }; use crank_import::rest::{ ImportFinding, ImportFindingSeverity, ImportNameConflict, ImportOperationCandidate, - NORMALIZER_VERSION, PROJECTION_VERSION, operation_draft_from_candidate, - validate_document_source_safe, + NORMALIZER_VERSION, PROJECTION_VERSION, canonical_external_document_uri, + operation_draft_from_candidate, validate_document_source_safe, }; use crank_registry::{ ApplyImportJobRequest, ArtifactSourceId, ArtifactSourceSensitivity, @@ -55,7 +55,7 @@ const IMPORT_JOB_TTL_HOURS: i64 = 24; const OPENAPI_PARSE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); const OPENAPI_PARSE_CONCURRENCY: usize = 4; const MAX_IMPORT_PLAN_KEYS: usize = 2_000; -const OPENAPI_SUPPORT_MATRIX_VERSION: &str = "1.0.3"; +const OPENAPI_SUPPORT_MATRIX_VERSION: &str = "1.0.4"; const MAX_IMPORT_PLAN_KEY_BYTES: usize = 4_096; const MAX_IMPORT_PLAN_SERVER_URL_BYTES: usize = 8_192; const MAX_IMPORT_PLAN_NAME_BYTES: usize = 256; @@ -664,24 +664,6 @@ impl AdminService { } } -fn canonical_external_document_uri(base: Option<&str>, reference: &str) -> Option { - if reference.starts_with('#') { - return None; - } - let mut url = match base { - Some(base) => url::Url::parse(base).ok()?.join(reference).ok()?, - None => url::Url::parse(reference).ok()?, - }; - if !matches!(url.scheme(), "http" | "https") - || !url.username().is_empty() - || url.password().is_some() - { - return None; - } - url.set_fragment(None); - Some(url.to_string()) -} - fn openapi_import_response(applied: ImportJobApplyResult) -> OpenApiImportCreateResponse { let created = applied .created diff --git a/apps/admin-api/src/service/imports/verified_preview.rs b/apps/admin-api/src/service/imports/verified_preview.rs index d06cbb1..cf2e8c7 100644 --- a/apps/admin-api/src/service/imports/verified_preview.rs +++ b/apps/admin-api/src/service/imports/verified_preview.rs @@ -210,6 +210,10 @@ fn import_replay_contract( | ("normalized-ir-v3", "preview-v3") | ("normalized-ir-v4", "preview-v4") | ("normalized-ir-v5", "preview-v5") + // The v5/preview-v6 pair predates the shared external-URI identity + // (normalized-ir-v6). Its saved projection is authoritative and is + // replayed verbatim, never reparsed under the current normalizer. + | ("normalized-ir-v5", "preview-v6") ) { return Err(ApiError::openapi_upload(locale, "source_integrity")); } diff --git a/apps/admin-api/tests/integration/openapi_import/external_references.rs b/apps/admin-api/tests/integration/openapi_import/external_references.rs index d9c0c96..0f3c23a 100644 --- a/apps/admin-api/tests/integration/openapi_import/external_references.rs +++ b/apps/admin-api/tests/integration/openapi_import/external_references.rs @@ -1,11 +1,16 @@ -use super::super::common::test_service_with_external_references; +use super::super::common::{ + authorized_client, build_test_app_with_external_references, spawn_admin_api, + test_service_with_external_references, +}; mod isolation; +mod uri_transport; use super::*; use admin_api::error::ApiError; use axum::{Router, body::to_bytes, http::StatusCode, response::IntoResponse, routing::get}; use crank_artifacts::ArtifactStore; use crank_core::{Workspace, WorkspaceStatus}; use crank_registry::{ArtifactSourceId, CreateWorkspaceRequest, RegistryError}; +use reqwest::multipart::{Form, Part}; use sha2::{Digest, Sha256}; use std::sync::{ Arc, diff --git a/apps/admin-api/tests/integration/openapi_import/external_references/uri_transport.rs b/apps/admin-api/tests/integration/openapi_import/external_references/uri_transport.rs new file mode 100644 index 0000000..6a88015 --- /dev/null +++ b/apps/admin-api/tests/integration/openapi_import/external_references/uri_transport.rs @@ -0,0 +1,603 @@ +//! F1 transport-parity cases for the Admin external-reference import. +//! +//! The external materializer stores snapshot keys produced by the URL transport +//! (`url::Url`), which collapses a `%2e%2e` path segment and percent-encodes an +//! apostrophe in a query (`'` -> `%27`). Because the materializer and the pure +//! resolver used different URI keys, both references produced a false +//! `external_reference_unavailable` blocker. These cases run the real allowlisted +//! fixture server through preview -> apply -> replay (service) and through the +//! authenticated multipart route (outer). They are registered from +//! `external_references.rs` so the parent file stays within the repository +//! file-size budget; shared helpers and imports come from `super`. + +use super::*; + +// F1 acceptance (both cases): the service preview -> apply -> replay path. The +// assertions verify the transport-canonical keys resolve, the dependency +// bytes/digests verify, apply never refetches, and replay is a stable no-op. +#[tokio::test] +#[serial] +async fn transport_canonical_external_uris_resolve_without_false_blocker_or_refetch() { + // Distinct bytes so each canonical URI owns a distinct content-addressed + // blob; byte-identical documents collapse into a single dependency envelope + // (the recorded alias/digest observation) and could not be told apart. + const ROOT_ITEM: &str = + "Item: { type: object, required: [id], properties: { id: { type: string } } }"; + const QUERY_ITEM: &str = + "Item: { type: object, required: [id], properties: { id: { type: integer } } }"; + let fetches = Arc::new(AtomicUsize::new(0)); + let root_fetches = Arc::clone(&fetches); + let query_fetches = Arc::clone(&fetches); + let app = Router::new() + .route( + "/root.yaml", + get(move || { + let fetches = Arc::clone(&root_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + ROOT_ITEM + } + }), + ) + .route( + "/schema.json", + get(move || { + let fetches = Arc::clone(&query_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + QUERY_ITEM + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let fixture_server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let origin = format!("http://{address}"); + + let registry = test_registry().await; + let storage_root = test_storage_root("openapi_import_external_uri_transport_parity"); + let service = test_service_with_external_references( + registry.clone(), + storage_root.clone(), + vec![format!("{origin}/")], + ); + let workspace_id = WorkspaceId::new("ws_default"); + let upload = OpenApiUpload { + bytes: format!( + r##" +openapi: 3.1.0 +info: {{ title: External URI transport parity }} +servers: [{{ url: https://api.example.test }}] +paths: + /dot: + get: + operationId: encodedDot + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: '{origin}/a/%2e%2e/root.yaml#/Item' }} }} }} + /query: + get: + operationId: quotedQuery + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: "{origin}/schema.json?filter='x'#/Item" }} }} }} +"## + ) + .into_bytes(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }; + let preview = service + .preview_openapi_import(&workspace_id, upload) + .await + .unwrap(); + + // Materialization first: the external materializer must have downloaded and + // stored both snapshots under the transport-canonical keys before resolution + // is judged. This separates the F1 resolver key mismatch from a snapshot that + // was never fetched at all. + let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into(); + let job = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + let dependencies = job.preview_payload["dependencies"].as_array().unwrap(); + assert_eq!(dependencies.len(), 2); + let canonical_uris = dependencies + .iter() + .map(|dependency| dependency["canonical_uri"].as_str().unwrap().to_owned()) + .collect::>(); + assert_eq!( + canonical_uris, + std::collections::BTreeSet::from([ + format!("{origin}/root.yaml"), + format!("{origin}/schema.json?filter=%27x%27"), + ]), + "materialized dependency keys must be the transport-canonical URIs" + ); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "both external documents must be fetched exactly once during preview" + ); + + // Dependency bytes and digests verify against the workspace-scoped store. + let store = Arc::new(ArtifactStore::open(&storage_root).unwrap()); + for dependency in dependencies { + let source_id = ArtifactSourceId::new(dependency["source_id"].as_str().unwrap()); + let digest = dependency["digest"].as_str().unwrap(); + let verified = registry + .read_artifact_source(Arc::clone(&store), &workspace_id, &source_id) + .await + .unwrap(); + assert_eq!(verified.source.blob.artifact_ref.as_str(), digest); + let canonical_uri = dependency["canonical_uri"].as_str().unwrap(); + let expected = if canonical_uri.ends_with("/root.yaml") { + ROOT_ITEM + } else { + QUERY_ITEM + }; + assert_eq!(verified.bytes, expected.as_bytes()); + } + + // Criterion-defining: neither transport-canonical URI may become a false + // blocker. Collected across all operations so one failure reports both. + let false_blockers = preview + .preview + .groups + .iter() + .flat_map(|group| &group.operations) + .flat_map(|operation| { + operation + .findings + .iter() + .filter(|finding| finding.code == "external_reference_unavailable") + .map(|_| operation.key.clone()) + }) + .collect::>(); + assert!( + false_blockers.is_empty(), + "both transport-canonical URIs must resolve; false external blockers: {false_blockers:?}" + ); + let resolved = preview + .preview + .groups + .iter() + .flat_map(|group| &group.operations) + .map(|operation| (operation.key.clone(), operation.output_fields)) + .collect::>(); + assert_eq!(resolved["GET /dot"], 1, "encoded-dot schema must resolve"); + assert_eq!( + resolved["GET /query"], 1, + "quoted-query schema must resolve" + ); + + // Apply consumes the saved snapshots and must not refetch. + let created = service + .create_openapi_import( + &workspace_id, + &job_id, + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /dot".to_owned(), "GET /query".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + assert_eq!(created.created.len(), 2); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "apply must reuse the saved dependency snapshots" + ); + let drafts_after_apply = registry.list_operations(&workspace_id).await.unwrap().len(); + + // AC-2 replay: re-issue the SAME request key/payload. The import is already + // applied, so the service must return the stored application result, create + // no extra Draft, and refetch nothing. The fixture server stays alive until + // after the replay so the counter would observe any real request. + let replay = service + .create_openapi_import( + &workspace_id, + &job_id, + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /dot".to_owned(), "GET /query".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + assert_eq!( + replay.created, created.created, + "replay must return the stored application result for the same request key and payload" + ); + assert_eq!(replay.created.len(), 2); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "replay must not refetch dependency snapshots" + ); + + fixture_server.abort(); + let completed = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.status, ImportJobStatus::Completed); + assert_eq!( + registry.list_operations(&workspace_id).await.unwrap().len(), + drafts_after_apply, + "replay must not create extra Drafts" + ); +} + +// F1 outer path: the same references uploaded as authenticated multipart through +// the outer router must preview without a false blocker, and the materializer +// must have stored both transport-canonical snapshots before resolution is +// judged. The run then continues through the real router apply and a replay of +// the same payload key while the fixture is still alive: the stored application +// result must be returned unchanged, no extra Draft may appear, and the fetch +// counter must not grow. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn outer_authenticated_multipart_preview_matches_transport_canonical_uri() { + // Distinct bytes so the two canonical URIs cannot collapse into one + // dependency envelope; both remain valid resolvable schemas. + const ROOT_ITEM: &str = + "Item: { type: object, required: [id], properties: { id: { type: string } } }"; + const QUERY_ITEM: &str = + "Item: { type: object, required: [id], properties: { id: { type: integer } } }"; + let fetches = Arc::new(AtomicUsize::new(0)); + let root_fetches = Arc::clone(&fetches); + let query_fetches = Arc::clone(&fetches); + let fixture = Router::new() + .route( + "/root.yaml", + get(move || { + let fetches = Arc::clone(&root_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + ROOT_ITEM + } + }), + ) + .route( + "/schema.json", + get(move || { + let fetches = Arc::clone(&query_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + QUERY_ITEM + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let fixture_server = tokio::spawn(async move { axum::serve(listener, fixture).await.unwrap() }); + let origin = format!("http://{address}"); + + let registry = test_registry().await; + let app = build_test_app_with_external_references( + registry.clone(), + test_storage_root("openapi_import_external_uri_multipart"), + vec![format!("{origin}/")], + ); + let server = spawn_admin_api(app).await; + let client = authorized_client(&server).await; + let document = format!( + r##" +openapi: 3.1.0 +info: {{ title: Multipart transport parity }} +servers: [{{ url: https://api.example.test }}] +paths: + /dot: + get: + operationId: encodedDot + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: '{origin}/a/%2e%2e/root.yaml#/Item' }} }} }} + /query: + get: + operationId: quotedQuery + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: "{origin}/schema.json?filter='x'#/Item" }} }} }} +"## + ); + let response = client + .post(format!("{server}/imports/openapi/preview")) + .multipart( + Form::new().part( + "file", + Part::bytes(document.into_bytes()) + .file_name("openapi.yaml".to_owned()) + .mime_str("application/yaml") + .unwrap(), + ), + ) + .send() + .await + .unwrap(); + let status = response.status(); + let body = response.json::().await.unwrap(); + assert_eq!(status, reqwest::StatusCode::OK, "{body}"); + + // The materializer must have downloaded both snapshots under the + // transport-canonical keys before the blocker check is meaningful. + let job_id_str = body["job_id"].as_str().unwrap().to_owned(); + let job_id: crank_registry::ImportJobId = job_id_str.as_str().into(); + let job = registry + .get_import_job(&WorkspaceId::new("ws_default"), &job_id) + .await + .unwrap() + .unwrap(); + let canonical_uris = job.preview_payload["dependencies"] + .as_array() + .unwrap() + .iter() + .map(|dependency| dependency["canonical_uri"].as_str().unwrap().to_owned()) + .collect::>(); + assert_eq!( + canonical_uris, + std::collections::BTreeSet::from([ + format!("{origin}/root.yaml"), + format!("{origin}/schema.json?filter=%27x%27"), + ]), + "both transport-canonical external URIs must be materialized" + ); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "preview must fetch each external document exactly once" + ); + + assert!( + !body.to_string().contains("external_reference_unavailable"), + "outer multipart preview must not report a false external blocker: {body}" + ); + let resolved = body["preview"]["groups"][0]["operations"] + .as_array() + .unwrap() + .iter() + .map(|operation| { + ( + operation["key"].as_str().unwrap().to_owned(), + operation["output_fields"].clone(), + ) + }) + .collect::>(); + assert_eq!(resolved["GET /dot"], serde_json::json!(1)); + assert_eq!(resolved["GET /query"], serde_json::json!(1)); + + // AC-2: apply through the real authenticated router, then replay the SAME + // payload key. The fixture stays alive so a real refetch would move the + // counter, and the same application key must not create another Draft. + let create_payload = serde_json::json!({ + "selected_operation_keys": ["GET /dot", "GET /query"], + "server_url": null, + "conflict_mode": "skip" + }); + let create_response = client + .post(format!("{server}/imports/openapi/{job_id_str}/create")) + .json(&create_payload) + .send() + .await + .unwrap(); + assert_eq!( + create_response.status(), + reqwest::StatusCode::OK, + "apply through the router must succeed" + ); + let created: serde_json::Value = create_response.json().await.unwrap(); + assert_eq!(created["created"].as_array().unwrap().len(), 2); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "apply must reuse the saved dependency snapshots" + ); + let drafts_after_apply = client + .get(format!("{server}/operations")) + .send() + .await + .unwrap() + .json::() + .await + .unwrap()["total"] + .clone(); + assert_eq!(drafts_after_apply, serde_json::json!(2)); + + let replay_response = client + .post(format!("{server}/imports/openapi/{job_id_str}/create")) + .json(&create_payload) + .send() + .await + .unwrap(); + assert_eq!(replay_response.status(), reqwest::StatusCode::OK); + let replay: serde_json::Value = replay_response.json().await.unwrap(); + assert_eq!( + replay, created, + "replaying the same payload key must return the stored application result" + ); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "replay must not refetch dependency snapshots" + ); + let drafts_after_replay = client + .get(format!("{server}/operations")) + .send() + .await + .unwrap() + .json::() + .await + .unwrap()["total"] + .clone(); + assert_eq!( + drafts_after_replay, drafts_after_apply, + "replay must not create extra Drafts" + ); + + fixture_server.abort(); +} + +// AC-3 over real HTTP: the primary document references an encoded-dot root alias +// (`/a/%2e%2e/root.yaml`) whose document contains a relative child `$ref`. The +// materializer must canonicalize the alias to `/root.yaml`, then join the child +// against that real canonical root (not the raw alias), so `/child.yaml` is +// fetched exactly once and the derived schema resolves through the chain. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn encoded_dot_root_alias_chain_resolves_relative_child_against_canonical_root_over_http() { + const ROOT_ALIAS_DOCUMENT: &str = "Item: { $ref: './child.yaml#/Item' }"; + const CHILD_ITEM: &str = + "Item: { type: object, required: [id], properties: { id: { type: string } } }"; + let fetches = Arc::new(AtomicUsize::new(0)); + let root_fetches = Arc::clone(&fetches); + let child_fetches = Arc::clone(&fetches); + let app = Router::new() + .route( + "/root.yaml", + get(move || { + let fetches = Arc::clone(&root_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + ROOT_ALIAS_DOCUMENT + } + }), + ) + .route( + "/child.yaml", + get(move || { + let fetches = Arc::clone(&child_fetches); + async move { + fetches.fetch_add(1, Ordering::SeqCst); + CHILD_ITEM + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let fixture_server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let origin = format!("http://{address}"); + + let registry = test_registry().await; + let storage_root = test_storage_root("openapi_import_external_uri_dot_root_chain"); + let service = test_service_with_external_references( + registry.clone(), + storage_root.clone(), + vec![format!("{origin}/")], + ); + let workspace_id = WorkspaceId::new("ws_default"); + let upload = OpenApiUpload { + bytes: format!( + r##" +openapi: 3.1.0 +info: {{ title: Canonical root alias chain }} +servers: [{{ url: https://api.example.test }}] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: {{ application/json: {{ schema: {{ $ref: '{origin}/a/%2e%2e/root.yaml#/Item' }} }} }} +"## + ) + .into_bytes(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + }; + let preview = service + .preview_openapi_import(&workspace_id, upload) + .await + .unwrap(); + + let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into(); + let job = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + let dependencies = job.preview_payload["dependencies"].as_array().unwrap(); + assert_eq!(dependencies.len(), 2); + let canonical_uris = dependencies + .iter() + .map(|dependency| dependency["canonical_uri"].as_str().unwrap().to_owned()) + .collect::>(); + assert_eq!( + canonical_uris, + std::collections::BTreeSet::from([ + format!("{origin}/root.yaml"), + format!("{origin}/child.yaml"), + ]), + "the child must be materialized relative to the canonical root, not the raw encoded-dot alias" + ); + assert_eq!( + fetches.load(Ordering::SeqCst), + 2, + "the canonical root alias and its relative child must each be fetched exactly once" + ); + + // Dependency bytes and digests verify against the workspace-scoped store. + let store = Arc::new(ArtifactStore::open(&storage_root).unwrap()); + for dependency in dependencies { + let source_id = ArtifactSourceId::new(dependency["source_id"].as_str().unwrap()); + let digest = dependency["digest"].as_str().unwrap(); + let verified = registry + .read_artifact_source(Arc::clone(&store), &workspace_id, &source_id) + .await + .unwrap(); + assert_eq!(verified.source.blob.artifact_ref.as_str(), digest); + let canonical_uri = dependency["canonical_uri"].as_str().unwrap(); + let expected = if canonical_uri.ends_with("/root.yaml") { + ROOT_ALIAS_DOCUMENT + } else { + CHILD_ITEM + }; + assert_eq!(verified.bytes, expected.as_bytes()); + } + + // No false blocker: the chain resolves through the canonical root. + let false_blockers = preview + .preview + .groups + .iter() + .flat_map(|group| &group.operations) + .flat_map(|operation| { + operation + .findings + .iter() + .filter(|finding| finding.code == "external_reference_unavailable") + .map(|_| operation.key.clone()) + }) + .collect::>(); + assert!( + false_blockers.is_empty(), + "the encoded-dot root alias chain must resolve; false blockers: {false_blockers:?}" + ); + assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1); + + // The derived schema must be the child object reached through the canonical + // root, projected into the persisted preview. + let output_schema = + &job.preview_payload["preview"]["groups"][0]["operations"][0]["draft"]["output_schema"]; + assert_eq!( + output_schema["type"], "object", + "the operation output schema must be the child object derived through the chain: {output_schema}" + ); + assert_eq!( + output_schema["fields"]["id"]["type"], "string", + "the child `id` property must survive resolution: {output_schema}" + ); + + fixture_server.abort(); +} diff --git a/apps/admin-api/tests/integration/openapi_import/fixtures/v5-v6-uri-preview.json b/apps/admin-api/tests/integration/openapi_import/fixtures/v5-v6-uri-preview.json new file mode 100644 index 0000000..c6008bb --- /dev/null +++ b/apps/admin-api/tests/integration/openapi_import/fixtures/v5-v6-uri-preview.json @@ -0,0 +1,170 @@ +{ + "dependencies": [], + "dependency_snapshots": [], + "normalization": { + "config": { + "external_references_enabled": false, + "max_aliases": 128, + "max_bytes": 262144, + "max_collection_items": 10000, + "max_depth": 64, + "max_expanded_nodes": 100000, + "max_external_document_bytes": 262144, + "max_nodes": 50000, + "max_reference_depth": 32, + "max_reference_documents": 32, + "max_references": 4096, + "max_scalar_bytes": 262144, + "normalizer_version": "normalized-ir-v5", + "projection_version": "preview-v6" + }, + "ir_fingerprint": "1e7a724e849b657d79d78f494d962ba5ebc164f154902153a8ed0610e0744679", + "normalizer_version": "normalized-ir-v5", + "projection_version": "preview-v6" + }, + "preview": { + "findings": [], + "groups": [ + { + "key": "imported_operation", + "operations": [ + { + "category": "imported", + "description": "Выполняет GET /items", + "draft": { + "category": "imported", + "display_name": "List Items", + "input_mapping": {}, + "input_schema": { + "description": "Входные параметры MCP-инструмента", + "nullable": false, + "required": true, + "type": "object" + }, + "name": "list_items", + "output_mapping": { + "rules": [ + { + "required": true, + "source": "$.response.body.id", + "target": "$.output.id" + } + ] + }, + "output_schema": { + "description": "Ответ API", + "fields": { + "id": { + "nullable": false, + "required": true, + "type": "string" + } + }, + "nullable": false, + "required": true, + "type": "object" + }, + "target": { + "base_url": "https://api.example.test", + "method": "GET", + "path_template": "/items", + "response_contract": [ + { + "media_type": "application/json", + "schema_variant": 0, + "status_code": 200 + } + ] + }, + "tool_description": { + "description": "Выполняет GET /items", + "examples": [ + { + "input": {} + } + ], + "title": "List Items" + }, + "wizard_state": {} + }, + "findings": [ + { + "code": "missing_summary", + "message": "У метода нет summary, отображаемое имя будет сгенерировано автоматически.", + "operation_key": "GET /items", + "severity": "warning", + "source_location": { + "pointer": "/paths/~1items/get" + } + }, + { + "code": "missing_description", + "message": "У метода нет description. Перед публикацией лучше описать, когда агенту стоит вызывать этот инструмент.", + "operation_key": "GET /items", + "severity": "warning", + "source_location": { + "pointer": "/paths/~1items/get" + } + }, + { + "code": "no_input_fields", + "message": "У инструмента нет входных параметров. Это нормально для справочных методов, но проверьте, что агенту не нужно передавать фильтры.", + "operation_key": "GET /items", + "severity": "info", + "source_location": { + "pointer": "/paths/~1items/get" + } + }, + { + "code": "weak_tool_description", + "message": "Описание инструмента слишком короткое или техническое. Перед публикацией добавьте, когда агент должен вызывать инструмент и что будет в успешном ответе.", + "operation_key": "GET /items", + "severity": "warning", + "source_location": { + "pointer": "/paths/~1items/get" + } + }, + { + "code": "weak_tool_name", + "message": "Имя инструмента `list_items` выглядит слишком общим. Лучше использовать имя с конкретным действием и объектом.", + "operation_key": "GET /items", + "severity": "warning", + "source_location": { + "pointer": "/paths/~1items/get" + } + } + ], + "input_fields": 0, + "key": "GET /items", + "method": "GET", + "operation_id": "listItems", + "output_fields": 1, + "path": "/items", + "server_urls": [ + "https://api.example.test" + ], + "source_location": { + "pointer": "/paths/~1items/get" + }, + "suggested_display_name": "List Items", + "suggested_name": "list_items" + } + ], + "title": "Без группы" + } + ], + "source": { + "format": "openapi", + "servers": [ + "https://api.example.test" + ], + "title": "URI preview parity", + "version": "3.1.0" + } + }, + "preview_digest": "ab615bbb932979b3c9d7d78223fc932069e1d7443860a24c682edddc0de11637", + "source": { + "digest": "sha256:edc9f4ec760d76a6830ae5b9932881d898626faffbc0067162ddb44ebdf8b94c", + "source_id": "src_openapi_01a1278095397d018780ab3dafd4b464" + } +} diff --git a/apps/admin-api/tests/integration/openapi_import/import_plan_apply/support_matrix.rs b/apps/admin-api/tests/integration/openapi_import/import_plan_apply/support_matrix.rs index a9ccf69..98d027c 100644 --- a/apps/admin-api/tests/integration/openapi_import/import_plan_apply/support_matrix.rs +++ b/apps/admin-api/tests/integration/openapi_import/import_plan_apply/support_matrix.rs @@ -8,9 +8,17 @@ use admin_api::service::{ use crank_core::WorkspaceId; use crank_registry::{CreateImportPlanRequest, ImportPlanId}; +fn single_operation_upload() -> OpenApiUpload { + OpenApiUpload { + bytes: OPENAPI_SINGLE.as_bytes().to_vec(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + } +} + #[tokio::test] #[serial] -async fn older_v2_support_matrix_rejects_new_apply_but_keeps_plan_auditable() { +async fn older_support_matrix_rejects_new_apply_but_keeps_plan_auditable() { let registry = test_registry().await; let service = test_service( registry.clone(), @@ -20,14 +28,7 @@ async fn older_v2_support_matrix_rejects_new_apply_but_keeps_plan_auditable() { ); let workspace_id = WorkspaceId::new("ws_default"); let preview = service - .preview_openapi_import( - &workspace_id, - OpenApiUpload { - bytes: OPENAPI_SINGLE.as_bytes().to_vec(), - mime_type: "application/yaml".to_owned(), - locale: OpenApiUploadLocale::En, - }, - ) + .preview_openapi_import(&workspace_id, single_operation_upload()) .await .unwrap(); let job_id = preview.job_id.as_str().into(); @@ -51,56 +52,120 @@ async fn older_v2_support_matrix_rejects_new_apply_but_keeps_plan_auditable() { .unwrap() .unwrap(); - // Model an immutable plan saved under an earlier support matrix. The - // registry derives its valid semantic digest from the changed payload. - let old_id: ImportPlanId = "plan_older_support_matrix".into(); - let mut old_payload = original.payload.clone(); - old_payload["plan_reference"] = json!(old_id.as_str()); - old_payload["support_matrix_version"] = json!("1.0.2"); - let older = registry - .create_import_plan(CreateImportPlanRequest { - id: &old_id, - workspace_id: &workspace_id, - job_id: &original.job_id, - payload: &old_payload, - created_at: &original.created_at, - expires_at: &original.expires_at, - }) + // 1.0.2 is the historical matrix contract and 1.0.3 the immediate previous + // one; against the shipped 1.0.4 matrix both must stay stale and must not + // authorize a new Apply, while the original immutable plan stays auditable. + for version in ["1.0.2", "1.0.3"] { + // Model an immutable plan saved under an earlier support matrix. The + // registry derives its valid semantic digest from the changed payload. + let old_id: ImportPlanId = + format!("plan_older_support_matrix_{}", version.replace('.', "_")).into(); + let mut old_payload = original.payload.clone(); + old_payload["plan_reference"] = json!(old_id.as_str()); + old_payload["support_matrix_version"] = json!(version); + let older = registry + .create_import_plan(CreateImportPlanRequest { + id: &old_id, + workspace_id: &workspace_id, + job_id: &original.job_id, + payload: &old_payload, + created_at: &original.created_at, + expires_at: &original.expires_at, + }) + .await + .unwrap(); + assert_ne!(older.digest, original.digest); + + let audit = service + .revalidate_openapi_import_plan(&workspace_id, &old_id) + .await + .unwrap(); + assert_eq!(audit.revalidation_status, "stale", "{version}"); + assert!(audit.revalidation_required, "{version}"); + assert_eq!( + audit.reason.as_deref(), + Some("importer_version_changed"), + "{version}" + ); + + let error = service + .apply_openapi_import_plan( + &workspace_id, + &old_id, + OpenApiImportPlanApplyPayload { + plan_digest: older.digest, + idempotency_key: format!("old-matrix-{version}"), + }, + ) + .await + .expect_err("old support matrix must not authorize a new Apply"); + match error { + ApiError::Conflict { + context: Some(context), + .. + } => assert_eq!(context["error_code"], "importer_version_changed"), + other => panic!("expected typed re-preview conflict, got {other:?}"), + } + assert!( + service + .list_operations(&workspace_id) + .await + .unwrap() + .is_empty(), + "{version} apply must not mutate the workspace" + ); + } +} + +// The runtime matrix version stamped into a fresh plan must equal the version +// recorded in the machine matrix the corpus validates. Reading the machine YAML +// (rather than a second literal) keeps an app release honest: it can never +// advertise a matrix the repository does not actually ship. +#[tokio::test] +#[serial] +async fn fresh_plan_support_matrix_version_matches_machine_matrix() { + let machine_text = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../crates/crank-import/tests/support-matrix.yaml"), + ) + .unwrap(); + let machine: serde_yaml::Value = serde_yaml::from_str(&machine_text).unwrap(); + let machine_version = machine["matrix_version"].as_str().unwrap().to_owned(); + + let registry = test_registry().await; + let service = test_service( + registry.clone(), + test_storage_root("openapi_support_matrix_runtime_version"), + test_auth_settings(), + test_secret_crypto(), + ); + let workspace_id = WorkspaceId::new("ws_default"); + let preview = service + .preview_openapi_import(&workspace_id, single_operation_upload()) .await .unwrap(); - assert_ne!(older.digest, original.digest); - - let audit = service - .revalidate_openapi_import_plan(&workspace_id, &old_id) - .await - .unwrap(); - assert_eq!(audit.revalidation_status, "stale"); - assert!(audit.revalidation_required); - assert_eq!(audit.reason.as_deref(), Some("importer_version_changed")); - - let error = service - .apply_openapi_import_plan( + let job_id = preview.job_id.as_str().into(); + let created = service + .create_openapi_import_plan( &workspace_id, - &old_id, - OpenApiImportPlanApplyPayload { - plan_digest: older.digest, - idempotency_key: "old-matrix-new-attempt".to_owned(), + &job_id, + OpenApiImportPlanCreatePayload { + selection_mode: "full".to_owned(), + selected_operation_keys: Vec::new(), + server_url: None, + decisions: Vec::new(), }, ) .await - .expect_err("old support matrix must not authorize a new Apply"); - match error { - ApiError::Conflict { - context: Some(context), - .. - } => assert_eq!(context["error_code"], "importer_version_changed"), - other => panic!("expected typed re-preview conflict, got {other:?}"), - } - assert!( - service - .list_operations(&workspace_id) - .await - .unwrap() - .is_empty() + .unwrap(); + let stored = registry + .get_import_plan(&workspace_id, &created.plan_reference.as_str().into()) + .await + .unwrap() + .unwrap(); + assert_eq!( + stored.payload["support_matrix_version"], + json!(machine_version), + "the runtime support-matrix version must match the machine matrix" ); } diff --git a/apps/admin-api/tests/integration/openapi_import/import_plan_create/creation.rs b/apps/admin-api/tests/integration/openapi_import/import_plan_create/creation.rs index e40bf72..df192ba 100644 --- a/apps/admin-api/tests/integration/openapi_import/import_plan_create/creation.rs +++ b/apps/admin-api/tests/integration/openapi_import/import_plan_create/creation.rs @@ -208,7 +208,7 @@ async fn full_plan_is_server_authored_with_bounded_confirmation_and_no_mutation( assert_eq!(stored.digest, plan.plan_digest); assert_eq!(stored.payload["schema_version"], "import-plan-v2"); assert_eq!(stored.payload["catalog_revision"], plan.catalog_revision); - assert_eq!(stored.payload["support_matrix_version"], "1.0.3"); + assert_eq!(stored.payload["support_matrix_version"], "1.0.4"); assert!(stored.payload["findings"]["document"].is_array()); assert!(stored.payload["findings"]["candidates"].is_array()); assert_eq!(stored.expires_at, expires_at); diff --git a/apps/admin-api/tests/integration/openapi_import/v5_replay.rs b/apps/admin-api/tests/integration/openapi_import/v5_replay.rs index f7538d4..c219445 100644 --- a/apps/admin-api/tests/integration/openapi_import/v5_replay.rs +++ b/apps/admin-api/tests/integration/openapi_import/v5_replay.rs @@ -91,3 +91,173 @@ async fn exact_v5_contract_replays_saved_preview_without_source_locations() { .unwrap(); assert_eq!(completed.status, ImportJobStatus::Completed); } + +// Deterministic baseline source for the v5/v6 replay characterization. It uses +// a local reference so the preview is self-contained (no external dependency +// or ephemeral fixture port) while still exercising the reference graph. +const URI_PREVIEW_DOCUMENT: &str = r#" +openapi: 3.1.0 +info: { title: URI preview parity, version: '1' } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: '#/components/schemas/Item' } } } +components: + schemas: + Item: { type: object, required: [id], properties: { id: { type: string } } } +"#; + +fn uri_preview_upload() -> OpenApiUpload { + OpenApiUpload { + bytes: URI_PREVIEW_DOCUMENT.as_bytes().to_vec(), + mime_type: "application/yaml".to_owned(), + locale: OpenApiUploadLocale::En, + } +} + +fn v5_v6_fixture_path() -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/integration/openapi_import/fixtures/v5-v6-uri-preview.json") +} + +// AC-4 (history): a preview persisted under the historical (v5, v6) contract +// must replay unchanged, and a new preview of the same source must use the +// current contract. Captured from baseline execution before the URI change. +#[tokio::test] +#[serial] +async fn v5_v6_uri_preview_replays_unchanged_and_new_preview_uses_current_contract() { + let fixture_text = std::fs::read_to_string(v5_v6_fixture_path()) + .expect("baseline v5/v6 URI preview fixture must be captured before the change"); + let fixture: serde_json::Value = serde_json::from_str(&fixture_text).unwrap(); + assert_eq!( + fixture["normalization"]["normalizer_version"], + serde_json::json!("normalized-ir-v5") + ); + assert_eq!( + fixture["normalization"]["projection_version"], + serde_json::json!("preview-v6") + ); + let fixture_preview = fixture["preview"].clone(); + let fixture_digest = fixture["preview_digest"].clone(); + + let registry = test_registry().await; + let service = test_service( + registry.clone(), + test_storage_root("openapi_import_v5_v6_replay"), + test_auth_settings(), + test_secret_crypto(), + ); + let workspace_id = WorkspaceId::new("ws_default"); + + // A current preview of the same bytes must use the current contract. + let fresh = service + .preview_openapi_import(&workspace_id, uri_preview_upload()) + .await + .unwrap(); + let job_id: crank_registry::ImportJobId = fresh.job_id.as_str().into(); + let fresh_job = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + assert_eq!( + fresh_job.preview_payload["source"]["digest"], fixture["source"]["digest"], + "fixture must refer to the exact captured source bytes" + ); + assert_eq!( + fresh_job.preview_payload["normalization"]["normalizer_version"], + serde_json::json!(crank_import::rest::NORMALIZER_VERSION) + ); + assert_eq!( + fresh_job.preview_payload["normalization"]["projection_version"], + serde_json::json!(crank_import::rest::PROJECTION_VERSION) + ); + + // Bind the historical payload to this run's freshly registered source blob. + let mut payload = fixture.clone(); + payload["source"]["source_id"] = fresh_job.preview_payload["source"]["source_id"].clone(); + sqlx::query("update import_jobs set preview_payload = $1::jsonb where id = $2") + .bind(serde_json::to_string(&payload).unwrap()) + .bind(job_id.as_str()) + .execute(registry.pool()) + .await + .unwrap(); + + let result = service + .create_openapi_import( + &workspace_id, + &job_id, + OpenApiImportCreatePayload { + selected_operation_keys: vec!["GET /items".to_owned()], + server_url: None, + conflict_mode: "skip".to_owned(), + }, + ) + .await + .unwrap(); + assert_eq!(result.created.len(), 1); + + // The historical preview is replayed from its saved projection, never + // recomputed under the current normalizer. + let completed = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + assert_eq!(completed.status, ImportJobStatus::Completed); + assert_eq!(completed.preview_payload["preview"], fixture_preview); + assert_eq!( + completed.preview_payload["normalization"]["normalizer_version"], + serde_json::json!("normalized-ir-v5") + ); + assert_eq!( + completed.preview_payload["normalization"]["projection_version"], + serde_json::json!("preview-v6") + ); + assert_eq!(completed.preview_payload["preview_digest"], fixture_digest); +} + +// AC-4 (version RED): a fresh preview of the fixture source must be stamped with +// the approved v6 normalizer contract. This is the version RED, distinct from +// the URI transport RED: it pins the exact approved literals instead of +// following the current `NORMALIZER_VERSION`/`PROJECTION_VERSION` constants, +// which do not by themselves prove the approved v6 value. Deliberately no +// if-current branch and no skip: the assertion is unconditional and is expected +// to fail until the implementation bumps the normalizer to `normalized-ir-v6`. +#[tokio::test] +#[serial] +async fn fresh_preview_matches_approved_v6_normalizer_contract() { + let registry = test_registry().await; + let service = test_service( + registry.clone(), + test_storage_root("openapi_import_v5_v6_version_contract"), + test_auth_settings(), + test_secret_crypto(), + ); + let workspace_id = WorkspaceId::new("ws_default"); + let fresh = service + .preview_openapi_import(&workspace_id, uri_preview_upload()) + .await + .unwrap(); + let job_id: crank_registry::ImportJobId = fresh.job_id.as_str().into(); + let fresh_job = registry + .get_import_job(&workspace_id, &job_id) + .await + .unwrap() + .unwrap(); + assert_eq!( + fresh_job.preview_payload["normalization"]["normalizer_version"], + serde_json::json!("normalized-ir-v6"), + "approved normalizer contract is normalized-ir-v6" + ); + assert_eq!( + fresh_job.preview_payload["normalization"]["projection_version"], + serde_json::json!("preview-v6"), + "approved projection contract is preview-v6" + ); +} diff --git a/crates/crank-import/src/rest/mod.rs b/crates/crank-import/src/rest/mod.rs index 1ddcfb8..47d6633 100644 --- a/crates/crank-import/src/rest/mod.rs +++ b/crates/crank-import/src/rest/mod.rs @@ -27,6 +27,7 @@ mod source_safety; mod strict_decode; mod swagger2; mod swagger2_findings; +mod uri; pub use model::{ ExternalDocumentSnapshot, HISTORICAL_NORMALIZER_VERSION, HISTORICAL_PROJECTION_VERSION, @@ -51,3 +52,4 @@ pub use payload::operation_draft_from_candidate; pub use source_safety::{ validate_document_bundle_source_safe, validate_resolved_bundle_source_safe, }; +pub use uri::canonical_external_document_uri; diff --git a/crates/crank-import/src/rest/model.rs b/crates/crank-import/src/rest/model.rs index 1bfeca5..c177fdb 100644 --- a/crates/crank-import/src/rest/model.rs +++ b/crates/crank-import/src/rest/model.rs @@ -9,7 +9,7 @@ use serde_json::Value; /// The immutable contract used to normalize an OpenAPI source. These names /// deliberately travel with an import job: changing either contract must not /// silently reinterpret a pending preview. -pub const NORMALIZER_VERSION: &str = "normalized-ir-v5"; +pub const NORMALIZER_VERSION: &str = "normalized-ir-v6"; pub const PROJECTION_VERSION: &str = "preview-v6"; pub const HISTORICAL_NORMALIZER_VERSION: &str = "normalized-ir-v4"; pub const HISTORICAL_PROJECTION_VERSION: &str = "preview-v4"; diff --git a/crates/crank-import/src/rest/reference.rs b/crates/crank-import/src/rest/reference.rs index 9edfbc1..184b176 100644 --- a/crates/crank-import/src/rest/reference.rs +++ b/crates/crank-import/src/rest/reference.rs @@ -9,6 +9,7 @@ use crate::rest::model::{ }; use super::reference_budget::{CopyBudget, count_branch_bytes, count_copy_bytes, merge_object}; +use super::uri::canonical_external_document_uri; use super::{ImportParseError, decode_budget, normalize_coverage::escape_pointer}; pub(super) struct ResolutionResult { @@ -467,14 +468,11 @@ impl Resolver<'_> { if document.is_empty() { return Ok(Some((current, pointer))); } - let canonical = if has_uri_scheme(document) { - canonical_absolute_uri(document) - } else { - let Some(base) = self.documents[current].uri.as_deref() else { - return Ok(None); - }; - join_relative(base, document) - }; + // The document part is canonicalized exactly like the Admin URL + // transport, so a supplied snapshot key matches its `$ref`. The shared + // helper also rejects non-HTTP(S) schemes and userinfo targets. + let canonical = + canonical_external_document_uri(self.documents[current].uri.as_deref(), document); Ok(canonical.and_then(|canonical| { self.by_uri .get(&canonical) @@ -565,161 +563,6 @@ fn is_oas31(root: &Value) -> bool { .is_some_and(|version| version.starts_with("3.1.")) } -fn join_relative(base: &str, relative: &str) -> Option { - let base = UriReference::parse(base)?; - let relative = UriReference::parse(relative)?; - let scheme = relative.scheme.or(base.scheme)?; - let authority = if relative.scheme.is_some() || relative.authority.is_some() { - relative.authority - } else { - base.authority - }; - let (path, query) = if relative.scheme.is_some() || relative.authority.is_some() { - (remove_dot_segments(&relative.path), relative.query) - } else if relative.path.is_empty() { - (base.path, relative.query.or(base.query)) - } else if relative.path.starts_with('/') { - (remove_dot_segments(&relative.path), relative.query) - } else { - ( - remove_dot_segments(&merge_paths( - &base.path, - authority.is_some(), - &relative.path, - )), - relative.query, - ) - }; - UriReference { - scheme: Some(scheme), - authority, - path, - query, - } - .render() -} - -fn canonical_absolute_uri(uri: &str) -> Option { - let reference = UriReference::parse(uri)?; - let scheme = reference.scheme?; - UriReference { - scheme: Some(scheme), - authority: reference.authority, - path: remove_dot_segments(&reference.path), - query: reference.query, - } - .render() -} - -#[derive(Clone, Debug)] -struct UriReference<'a> { - scheme: Option<&'a str>, - authority: Option<&'a str>, - path: String, - query: Option<&'a str>, -} - -impl<'a> UriReference<'a> { - fn parse(value: &'a str) -> Option { - if value.contains('\\') || value.contains('#') { - return None; - } - let (without_query, query) = value - .split_once('?') - .map_or((value, None), |(path, query)| (path, Some(query))); - let (scheme, rest) = if let Some(index) = without_query.find(':') { - let candidate = &without_query[..index]; - if is_uri_scheme(candidate) { - (Some(candidate), &without_query[index + 1..]) - } else { - (None, without_query) - } - } else { - (None, without_query) - }; - let (authority, path) = if let Some(rest) = rest.strip_prefix("//") { - match rest.find('/') { - Some(index) => (Some(&rest[..index]), rest[index..].to_owned()), - None => (Some(rest), String::new()), - } - } else { - (None, rest.to_owned()) - }; - Some(Self { - scheme, - authority, - path, - query, - }) - } - - fn render(&self) -> Option { - let scheme = self.scheme?; - let scheme = scheme.to_ascii_lowercase(); - let mut value = format!("{scheme}:"); - if let Some(authority) = self.authority { - value.push_str("//"); - value.push_str(&canonical_authority(authority, &scheme)); - } - if self.authority.is_some() && self.path.is_empty() { - value.push('/'); - } else { - value.push_str(&self.path); - } - if let Some(query) = self.query { - value.push('?'); - value.push_str(query); - } - Some(value) - } -} - -fn canonical_authority(authority: &str, scheme: &str) -> String { - let authority = authority.to_ascii_lowercase(); - let default_port = match scheme { - "http" => Some("80"), - "https" => Some("443"), - _ => None, - }; - let authority = if let Some(default_port) = default_port - && let Some((host, port)) = authority.rsplit_once(':') - && port == default_port - && (!host.contains(':') || host.ends_with(']')) - { - host.to_owned() - } else { - authority - }; - punycode_authority(authority, scheme) -} - -/// Applies the same IDNA/punycode host identity that the external-snapshot -/// materializer gets from `url::Url`, so a Unicode `$ref` and the stored -/// punycode `canonical_uri` resolve to the same snapshot. ASCII-only -/// authorities (including IPv6 literals) and authorities with userinfo keep the -/// previous ASCII-lowercase rendering, preserving RFC 3986 behavior. -fn punycode_authority(authority: String, scheme: &str) -> String { - if authority.is_ascii() || !matches!(scheme, "http" | "https") || authority.contains('@') { - return authority; - } - let Ok(parsed) = url::Url::parse(&format!("{scheme}://{authority}")) else { - return authority; - }; - let Some(host) = parsed.host_str() else { - return authority; - }; - match parsed.port() { - Some(port) => format!("{host}:{port}"), - None => host.to_owned(), - } -} - -fn has_uri_scheme(value: &str) -> bool { - value - .split_once(':') - .is_some_and(|(candidate, _)| is_uri_scheme(candidate)) -} - /// URI schemes are case-insensitive (RFC 3986 §3.1), so the failed-target /// classification must not depend on the literal case of `http`/`https`. fn is_external_http_scheme(reference: &str) -> bool { @@ -728,47 +571,6 @@ fn is_external_http_scheme(reference: &str) -> bool { }) } -fn is_uri_scheme(candidate: &str) -> bool { - let Some(first) = candidate.as_bytes().first() else { - return false; - }; - first.is_ascii_alphabetic() - && candidate - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'-' | b'.')) -} - -fn merge_paths(base_path: &str, has_authority: bool, relative_path: &str) -> String { - match base_path.rfind('/') { - Some(index) => format!("{}{}", &base_path[..=index], relative_path), - None if has_authority => format!("/{relative_path}"), - None => relative_path.to_owned(), - } -} - -fn remove_dot_segments(path: &str) -> String { - let leading_slash = path.starts_with('/'); - let trailing_slash = path.ends_with('/'); - let mut output = Vec::new(); - for segment in path.split('/') { - match segment { - "." => {} - ".." => { - output.pop(); - } - _ => output.push(segment), - } - } - let mut result = output.join("/"); - if leading_slash && !result.starts_with('/') { - result.insert(0, '/'); - } - if trailing_slash && !result.ends_with('/') { - result.push('/'); - } - result -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum TargetError { MalformedFragment, diff --git a/crates/crank-import/src/rest/uri.rs b/crates/crank-import/src/rest/uri.rs new file mode 100644 index 0000000..ad1d879 --- /dev/null +++ b/crates/crank-import/src/rest/uri.rs @@ -0,0 +1,96 @@ +//! Shared, pure canonical identity for external HTTP(S) document references. +//! +//! The Admin external-snapshot materializer keys every fetched dependency by the +//! URL transport canonical form (`url::Url`, i.e. the WHATWG URL standard). +//! The pure reference resolver must derive the exact same key, otherwise a +//! supplied snapshot never matches and the reference becomes a false +//! `external_reference_unavailable`. Both callers delegate here so the identity +//! rule (IDNA punycode host, default-port removal, percent-encoded dot +//! segments, WHATWG special-query encoding) lives in one place and stays +//! I/O-free. +//! +//! Semantics deliberately follow `url::Url` rather than a hand-rolled RFC 3986 +//! renderer: the transport already stores the `url::Url` string, so any other +//! renderer would reintroduce the divergent keys this helper removes. + +use url::Url; + +/// Canonical absolute form of an HTTP(S) URI reference. +/// +/// `base` is the already-canonical URI of the document that contains the +/// reference; pass `None` for an absolute reference or for the primary document. +/// The fragment is always dropped because it addresses a JSON Pointer inside the +/// fetched document, not a network resource. Returns `None` when the result is +/// not an absolute `http`/`https` URL or when it carries userinfo credentials, +/// so a caller can never derive an identity for a disallowed target. +pub fn canonical_external_document_uri(base: Option<&str>, reference: &str) -> Option { + if reference.starts_with('#') { + return None; + } + let mut url = match base { + Some(base) => Url::parse(base).ok()?.join(reference).ok()?, + None => Url::parse(reference).ok()?, + }; + if !matches!(url.scheme(), "http" | "https") + || !url.username().is_empty() + || url.password().is_some() + { + return None; + } + url.set_fragment(None); + Some(url.to_string()) +} + +#[cfg(test)] +mod tests { + use super::canonical_external_document_uri; + + #[test] + fn collapses_percent_encoded_dot_segments_like_the_transport() { + assert_eq!( + canonical_external_document_uri( + None, + "https://schemas.example.test/a/%2e%2e/root.yaml#/Item" + ) + .as_deref(), + Some("https://schemas.example.test/root.yaml") + ); + } + + #[test] + fn applies_the_special_query_percent_encode_set_like_the_transport() { + assert_eq!( + canonical_external_document_uri( + None, + "https://schemas.example.test/schema.json?filter='x'#/Item" + ) + .as_deref(), + Some("https://schemas.example.test/schema.json?filter=%27x%27") + ); + } + + #[test] + fn joins_relative_references_against_a_canonical_base() { + assert_eq!( + canonical_external_document_uri( + Some("https://schemas.example.test/root.yaml"), + "./child.yaml#/Item" + ) + .as_deref(), + Some("https://schemas.example.test/child.yaml") + ); + } + + #[test] + fn rejects_local_non_http_and_userinfo_references() { + assert_eq!(canonical_external_document_uri(None, "#/Item"), None); + assert_eq!( + canonical_external_document_uri(None, "urn:example:item"), + None + ); + assert_eq!( + canonical_external_document_uri(None, "https://user:pass@host/schema.json"), + None + ); + } +} diff --git a/crates/crank-import/tests/corpus.rs b/crates/crank-import/tests/corpus.rs index c9e1df3..8b9b290 100644 --- a/crates/crank-import/tests/corpus.rs +++ b/crates/crank-import/tests/corpus.rs @@ -16,6 +16,9 @@ use std::{ #[path = "corpus/matrix.rs"] mod matrix; +#[path = "corpus/reference_uri.rs"] +mod reference_uri; + fn tests_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("tests") } @@ -117,8 +120,8 @@ fn normalize(source: &str) -> Result= 50); let root = tests_root().join("corpus"); @@ -687,6 +690,21 @@ fn ac_2_manifest_covers_fifty_provenanced_specs_with_exact_expectations() { document.file ); } + "regression-uri-encoded-dot.yaml" | "regression-uri-query-quote.yaml" => { + // Default-deny characterization: without a supplied + // snapshot the external `$ref` stays blocked at its + // source. The positive snapshot oracle lives in + // `corpus/reference_uri.rs`. + assert!( + ir.findings + .iter() + .chain(ir.operations.iter().flat_map(|operation| &operation.findings)) + .any(|finding| finding.code == "external_reference_disabled"), + "{} must default-deny its external $ref: {:?}", + document.file, + ir.findings + ); + } _ => unreachable!("hostile fixture must have a dedicated expectation"), } } @@ -713,6 +731,7 @@ fn ac_2_manifest_covers_fifty_provenanced_specs_with_exact_expectations() { .into_iter() .chain(std::iter::once("manifest.json".to_owned())) .chain(std::iter::once("matrix.rs".to_owned())) + .chain(std::iter::once("reference_uri.rs".to_owned())) .collect::>(); let actual_files = fs::read_dir(&root) .expect("corpus directory exists") diff --git a/crates/crank-import/tests/corpus/manifest.json b/crates/crank-import/tests/corpus/manifest.json index 03c5bc5..59f8b68 100644 --- a/crates/crank-import/tests/corpus/manifest.json +++ b/crates/crank-import/tests/corpus/manifest.json @@ -1,6 +1,6 @@ { - "corpus_version": "1.0.4", - "matrix_version": "1.0.3", + "corpus_version": "1.0.5", + "matrix_version": "1.0.4", "digest_algorithm": "sha256", "canonical_digest_definition": "SHA-256 of compact JSON with lexicographically ordered keys; null for malformed JSON", "documents": [ @@ -14407,7 +14407,29 @@ "warning_findings": [], "blocker_findings": [] } + }, + { + "id": "hostile-13", + "file": "regression-uri-encoded-dot.yaml", + "sha256": "9c3f73debd2f51449904c6691939e3bb3f8958f9562eac8b105cdf23a92cbc2d", + "canonical_digest": null, + "origin": "synthetic: Epic 2 external URI transport regression", + "license": "CC0-1.0 synthetic; no third-party content", + "specification_version": "OpenAPI 3.1", + "classification": "hostile", + "expected_preview": {} + }, + { + "id": "hostile-14", + "file": "regression-uri-query-quote.yaml", + "sha256": "b1c879285d4f5bfa64004fcc0dced1e115a1423d8a4c3a4c9a0699894e776c8f", + "canonical_digest": null, + "origin": "synthetic: Epic 2 external URI transport regression", + "license": "CC0-1.0 synthetic; no third-party content", + "specification_version": "OpenAPI 3.1", + "classification": "hostile", + "expected_preview": {} } ], - "compatibility_impact": "Corpus 1.0.4 is a metadata-only SemVer increment aligning its manifest with support matrix 1.0.3; fixture contents and checksums are unchanged. Corpus 1.0.3 added hostile-security-scope.json: a document-level basic requirement remains selectable on the inherited candidate while an explicit security: [] override stays public; malformed or unsupported requirements retain scoped blockers. Support matrix 1.0.3 adds executable blockers for undeclared path placeholders and object/array schema defaults across OpenAPI 3.0, OpenAPI 3.1, and Swagger 2.0 while preserving scalar defaults and previously supported behavior." + "compatibility_impact": "Corpus 1.0.5 adds two hostile external-URI regression fixtures (regression-uri-encoded-dot.yaml, regression-uri-query-quote.yaml) and the reference_uri.rs positive-snapshot oracle: an encoded-dot path segment and an apostrophe query now canonicalize to the URL transport identity, so a supplied snapshot resolves while the default-deny pass and all statuses/denominators are unchanged. Support matrix 1.0.4 records the shared external HTTP(S) URI canonicalization contract without changing any construct status. Corpus 1.0.4 was a metadata-only SemVer increment aligning its manifest with support matrix 1.0.3; fixture contents and checksums were unchanged." } diff --git a/crates/crank-import/tests/corpus/matrix.rs b/crates/crank-import/tests/corpus/matrix.rs index 0097dfc..682fae3 100644 --- a/crates/crank-import/tests/corpus/matrix.rs +++ b/crates/crank-import/tests/corpus/matrix.rs @@ -31,7 +31,7 @@ fn ac_1_support_matrix_rows_have_exact_construct_contracts() { let matrix_text = fs::read_to_string(tests_root().join("support-matrix.yaml")) .expect("support matrix exists"); let matrix: serde_yaml::Value = serde_yaml::from_str(&matrix_text).expect("valid YAML"); - assert_eq!(matrix["matrix_version"].as_str(), Some("1.0.3")); + assert_eq!(matrix["matrix_version"].as_str(), Some("1.0.4")); assert!( matrix["compatibility_impact"] .as_str() @@ -696,73 +696,100 @@ fn ac_1_support_matrix_rows_have_exact_construct_contracts() { row["specification_version"] == version && row["construct_id"] == construct }) .unwrap(); - let reference = if external { - "https://schemas.example.test/root.yaml#/Item" - } else if version == "Swagger 2.0" { + // The allowlisted-external row must exercise the same transport + // canonicalization the materializer keys snapshots with. The + // expected keys below are static literals, independent of the + // production `canonical_external_document_uri` helper: a + // percent-encoded dot segment collapses to its parent path and an + // apostrophe in the query is percent-encoded (`%27`). + let external_document = "Item: { type: object, properties: { id: { type: string } } }"; + let external_variants: [(&str, &str); 3] = [ + ( + "https://schemas.example.test/root.yaml#/Item", + "https://schemas.example.test/root.yaml", + ), + ( + "https://schemas.example.test/a/%2e%2e/root.yaml#/Item", + "https://schemas.example.test/root.yaml", + ), + ( + "https://schemas.example.test/schema.json?filter='x'#/Item", + "https://schemas.example.test/schema.json?filter=%27x%27", + ), + ]; + let local_reference = if version == "Swagger 2.0" { "#/definitions/Item" } else { "#/components/schemas/Item" }; - let schema = json!({"$ref":reference}); - let response = if version == "Swagger 2.0" { - json!({"description":"ok","schema":schema}) + let variants = if external { + external_variants + .iter() + .map(|(reference, canonical_uri)| (*reference, Some(*canonical_uri))) + .collect::>() } else { - json!({"description":"ok","content":{"application/json":{"schema":schema}}}) + vec![(local_reference, None)] }; - let probe = if version == "Swagger 2.0" { - let mut root = json!({"swagger":"2.0","info":{"title":"Reference probe","version":"1"},"paths":{"/probe":{"get":{"operationId":"referenceProbe","responses":{"200":response}}}}}); - if !external { - root["definitions"] = - json!({"Item":{"type":"object","properties":{"id":{"type":"string"}}}}); - } - root - } else { - let mut root = json!({"openapi":if version == "OpenAPI 3.0" {"3.0.3"} else {"3.1.0"}, - "info":{"title":"Reference probe","version":"1"},"servers":[{"url":"https://matrix.example.test"}],"paths":{"/probe":{"get":{"operationId":"referenceProbe","responses":{"200":response}}}}}); - if !external { - root["components"] = json!({"schemas":{"Item":{"type":"object","properties":{"id":{"type":"string"}}}}}); - } - root - }; - let source = serde_json::to_string(&probe).unwrap(); - let ir = if external { - let external_source = - "Item: { type: object, properties: { id: { type: string } } }"; - let snapshot = crank_import::rest::ExternalDocumentSnapshot { - canonical_uri: "https://schemas.example.test/root.yaml".to_owned(), - digest: SourceDigest::parse(hex(external_source.as_bytes())).unwrap(), - document: external_source.to_owned(), + for (reference, canonical_uri) in variants { + let schema = json!({"$ref":reference}); + let response = if version == "Swagger 2.0" { + json!({"description":"ok","schema":schema}) + } else { + json!({"description":"ok","content":{"application/json":{"schema":schema}}}) }; - let digest = SourceDigest::parse(hex(source.as_bytes())).unwrap(); - normalize_verified_bundle( - &source, - digest, - &[snapshot], - &NormalizationConfig::default(), - ) - .unwrap() - } else { - normalize(&source).unwrap() - }; - validate_normalized_ir(&ir).unwrap(); - assert_eq!(row["status"], "supported"); - assert!( - !ir.reference_graph.edges.is_empty(), - "{version} {construct} did not resolve through a source-backed edge" - ); - assert!( - ir.unresolved_references.is_empty(), - "{version} {construct} remains unresolved" - ); - assert!( - matches!( - ir.operations[0].response_variants[0].schema.as_ref().map(|schema| &schema.kind), - Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id") - ), - "{version} {construct} resolved schema was not projected into the operation" - ); - assert_actionable(&ir, &format!("{version} {construct}")); - assert_eq!(row["expected_finding"], "none"); + let probe = if version == "Swagger 2.0" { + let mut root = json!({"swagger":"2.0","info":{"title":"Reference probe","version":"1"},"paths":{"/probe":{"get":{"operationId":"referenceProbe","responses":{"200":response}}}}}); + if !external { + root["definitions"] = + json!({"Item":{"type":"object","properties":{"id":{"type":"string"}}}}); + } + root + } else { + let mut root = json!({"openapi":if version == "OpenAPI 3.0" {"3.0.3"} else {"3.1.0"}, + "info":{"title":"Reference probe","version":"1"},"servers":[{"url":"https://matrix.example.test"}],"paths":{"/probe":{"get":{"operationId":"referenceProbe","responses":{"200":response}}}}}); + if !external { + root["components"] = json!({"schemas":{"Item":{"type":"object","properties":{"id":{"type":"string"}}}}}); + } + root + }; + let source = serde_json::to_string(&probe).unwrap(); + let ir = if let Some(canonical_uri) = canonical_uri { + let snapshot = crank_import::rest::ExternalDocumentSnapshot { + canonical_uri: canonical_uri.to_owned(), + digest: SourceDigest::parse(hex(external_document.as_bytes())).unwrap(), + document: external_document.to_owned(), + }; + let digest = SourceDigest::parse(hex(source.as_bytes())).unwrap(); + normalize_verified_bundle( + &source, + digest, + &[snapshot], + &NormalizationConfig::default(), + ) + .unwrap() + } else { + normalize(&source).unwrap() + }; + validate_normalized_ir(&ir).unwrap(); + assert_eq!(row["status"], "supported"); + assert!( + !ir.reference_graph.edges.is_empty(), + "{version} {construct} did not resolve through a source-backed edge ({reference})" + ); + assert!( + ir.unresolved_references.is_empty(), + "{version} {construct} remains unresolved ({reference})" + ); + assert!( + matches!( + ir.operations[0].response_variants[0].schema.as_ref().map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id") + ), + "{version} {construct} resolved schema was not projected into the operation ({reference})" + ); + assert_actionable(&ir, &format!("{version} {construct} {reference}")); + assert_eq!(row["expected_finding"], "none"); + } probed(construct); } for code in 200..300 { diff --git a/crates/crank-import/tests/corpus/reference_uri.rs b/crates/crank-import/tests/corpus/reference_uri.rs new file mode 100644 index 0000000..2201aec --- /dev/null +++ b/crates/crank-import/tests/corpus/reference_uri.rs @@ -0,0 +1,123 @@ +//! Positive and default-deny oracle for external HTTP(S) URI transport parity. +//! +//! These two regression fixtures pin the F1 defect: an encoded-dot path segment +//! (`%2e%2e`) and an apostrophe query (`'`). Under the default configuration the +//! external `$ref` must stay default-denied (the document-level pass +//! `corpus.rs` characterizes); once a snapshot keyed by the URL transport +//! canonical form is supplied, the reference must resolve. Keeping the positive +//! snapshot oracle here, separate from the default-deny pass, means neither can +//! mask the other, and both run the real pure resolver. + +use super::*; +use crank_import::rest::{ + ExternalDocumentSnapshot, NormalizedSchemaKind, SourceDigest, normalize_verified_bundle, +}; + +struct Case { + file: &'static str, + /// The raw `$ref` exactly as it appears in the fixture. + reference: &'static str, + /// The `url::Url` transport canonical form the materializer would store. + canonical_uri: &'static str, +} + +const CASES: &[Case] = &[ + Case { + file: "regression-uri-encoded-dot.yaml", + reference: "https://schemas.example.test/a/%2e%2e/root.yaml#/Item", + canonical_uri: "https://schemas.example.test/root.yaml", + }, + Case { + file: "regression-uri-query-quote.yaml", + reference: "https://schemas.example.test/schema.json?filter='x'#/Item", + canonical_uri: "https://schemas.example.test/schema.json?filter=%27x%27", + }, +]; + +fn fixture(file: &str) -> String { + fs::read_to_string(tests_root().join("corpus").join(file)) + .unwrap_or_else(|error| panic!("{file}: {error}")) +} + +fn digest(byte: char) -> SourceDigest { + SourceDigest::parse(byte.to_string().repeat(64)).unwrap() +} + +#[test] +fn default_deny_pass_keeps_each_external_uri_blocked_without_snapshots() { + for case in CASES { + let source = fixture(case.file); + assert!( + source.contains(case.reference), + "{} fixture lost its raw $ref", + case.file + ); + let ir = normalize(&source).unwrap_or_else(|error| panic!("{}: {error}", case.file)); + validate_normalized_ir(&ir).unwrap(); + assert!( + ir.findings + .iter() + .chain( + ir.operations + .iter() + .flat_map(|operation| &operation.findings) + ) + .any(|finding| finding.code == "external_reference_disabled"), + "{} must default-deny its external $ref without a snapshot", + case.file + ); + assert!( + ir.reference_graph.edges.is_empty(), + "{} must not resolve without a supplied snapshot", + case.file + ); + } +} + +#[test] +fn positive_snapshot_oracle_resolves_each_transport_canonical_uri() { + let snapshot_document = + "Item: { type: object, required: [id], properties: { id: { type: string } } }"; + for case in CASES { + let source = fixture(case.file); + let snapshots = vec![ExternalDocumentSnapshot { + canonical_uri: case.canonical_uri.to_owned(), + digest: digest('b'), + document: snapshot_document.to_owned(), + }]; + let config = NormalizationConfig { + external_references_enabled: true, + ..NormalizationConfig::default() + }; + let ir = normalize_verified_bundle(&source, digest('a'), &snapshots, &config) + .unwrap_or_else(|error| panic!("{}: {error}", case.file)); + assert!( + !ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "external_reference_unavailable"), + "{} resolved against the transport key but still reported unavailable: {:?}", + case.file, + ir.operations[0].findings + ); + assert!( + matches!( + ir.operations[0] + .response_schema + .as_ref() + .map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Object { properties, .. }) + if properties.contains_key("id") + ), + "{} resolved schema must keep its `id` property", + case.file + ); + assert_eq!(ir.reference_graph.edges.len(), 1, "{}", case.file); + assert_eq!( + ir.reference_graph.edges[0].target.snapshot_digest, + digest('b') + ); + assert_eq!(ir.reference_graph.dependency_digests, vec![digest('b')]); + assert!(ir.unresolved_references.is_empty()); + } +} diff --git a/crates/crank-import/tests/corpus/regression-uri-encoded-dot.yaml b/crates/crank-import/tests/corpus/regression-uri-encoded-dot.yaml new file mode 100644 index 0000000..456e6a6 --- /dev/null +++ b/crates/crank-import/tests/corpus/regression-uri-encoded-dot.yaml @@ -0,0 +1,13 @@ +openapi: 3.1.0 +info: { title: External URI encoded-dot regression, version: '1' } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: encodedDot + responses: + '200': + description: ok + content: + application/json: + schema: { $ref: 'https://schemas.example.test/a/%2e%2e/root.yaml#/Item' } diff --git a/crates/crank-import/tests/corpus/regression-uri-query-quote.yaml b/crates/crank-import/tests/corpus/regression-uri-query-quote.yaml new file mode 100644 index 0000000..161b86e --- /dev/null +++ b/crates/crank-import/tests/corpus/regression-uri-query-quote.yaml @@ -0,0 +1,13 @@ +openapi: 3.1.0 +info: { title: External URI query-quote regression, version: '1' } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: quotedQuery + responses: + '200': + description: ok + content: + application/json: + schema: { $ref: "https://schemas.example.test/schema.json?filter='x'#/Item" } diff --git a/crates/crank-import/tests/reference_resolution.rs b/crates/crank-import/tests/reference_resolution.rs index ea0e398..130cd89 100644 --- a/crates/crank-import/tests/reference_resolution.rs +++ b/crates/crank-import/tests/reference_resolution.rs @@ -4,6 +4,12 @@ use crank_import::rest::{ normalize_verified_document, preview_from_ir, reference_uris, }; +// `reference_resolution.rs` is an integration-test crate root, so a nested +// module must point at the file explicitly to keep the path +// `tests/reference_resolution/uri_transport.rs` (relative to this file's dir). +#[path = "reference_resolution/uri_transport.rs"] +mod uri_transport; + fn digest(byte: char) -> SourceDigest { SourceDigest::parse(byte.to_string().repeat(64)).unwrap() } diff --git a/crates/crank-import/tests/reference_resolution/uri_transport.rs b/crates/crank-import/tests/reference_resolution/uri_transport.rs new file mode 100644 index 0000000..d173a82 --- /dev/null +++ b/crates/crank-import/tests/reference_resolution/uri_transport.rs @@ -0,0 +1,194 @@ +//! F1 transport-parity cases for the pure resolver. +//! +//! Each case supplies a snapshot keyed exactly as the Admin URL transport +//! (`url::Url`) keys it, then drives `normalize_verified_bundle`. The cases are +//! registered from `reference_resolution.rs` so the parent file stays within the +//! repository file-size budget; the shared helpers (`digest`, `normalize_*`, +//! and the `rest` imports) come from `super` through `use super::*`. + +use super::*; + +// F1 (a): the external materializer derives its snapshot key from the URL +// transport (`url::Url`), which collapses a percent-encoded dot segment +// (`%2e%2e`) into the parent path. The pure resolver must derive the same +// canonical key, otherwise a supplied snapshot becomes a false +// `external_reference_unavailable` blocker. RED before the shared URI helper. +#[test] +fn encoded_dot_path_reference_matches_transport_canonical_snapshot() { + let document = r#" +openapi: 3.1.0 +info: { title: Encoded dot transport parity } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: 'https://schemas.example.test/a/%2e%2e/root.yaml#/Item' } } } +"#; + let snapshots = vec![ExternalDocumentSnapshot { + // Transport (`url::Url`) canonical form of the same reference. + canonical_uri: "https://schemas.example.test/root.yaml".to_owned(), + digest: digest('b'), + document: "Item: { type: object, required: [id], properties: { id: { type: string } } }" + .to_owned(), + }]; + let config = NormalizationConfig { + external_references_enabled: true, + ..NormalizationConfig::default() + }; + let ir = normalize_verified_bundle(document, digest('a'), &snapshots, &config).unwrap(); + + assert!( + !ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "external_reference_unavailable"), + "encoded-dot $ref must match its transport-canonical snapshot, got: {:?}", + ir.operations[0].findings + ); + assert!( + matches!( + ir.operations[0] + .response_schema + .as_ref() + .map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id") + ), + "resolved schema must preserve its `id` property" + ); + assert_eq!(ir.reference_graph.edges.len(), 1); + assert_eq!( + ir.reference_graph.edges[0].target.snapshot_digest, + digest('b') + ); + assert!(ir.unresolved_references.is_empty()); +} + +// F1 (b): the URL transport percent-encodes an apostrophe inside a query string +// (`'` -> `%27`, the WHATWG special-query percent-encode set for http/https). +// The resolver must canonicalize the query the same way or the stored snapshot +// key never matches. RED before the shared URI helper. +// +// The `$ref` sits in a YAML double-quoted scalar: the reference itself contains +// an apostrophe, so a single-quoted scalar would terminate early and turn this +// into a parser syntax failure instead of the intended resolution RED. +#[test] +fn quoted_query_reference_matches_transport_canonical_snapshot() { + let document = r##" +openapi: 3.1.0 +info: { title: Quoted query transport parity } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: "https://schemas.example.test/schema.json?filter='x'#/Item" } } } +"##; + let snapshots = vec![ExternalDocumentSnapshot { + // Transport (`url::Url`) canonical form of the apostrophe query. + canonical_uri: "https://schemas.example.test/schema.json?filter=%27x%27".to_owned(), + digest: digest('b'), + document: "Item: { type: object, required: [id], properties: { id: { type: string } } }" + .to_owned(), + }]; + let config = NormalizationConfig { + external_references_enabled: true, + ..NormalizationConfig::default() + }; + let ir = normalize_verified_bundle(document, digest('a'), &snapshots, &config).unwrap(); + + assert!( + !ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "external_reference_unavailable"), + "quoted-query $ref must match its transport-canonical snapshot, got: {:?}", + ir.operations[0].findings + ); + assert!( + matches!( + ir.operations[0] + .response_schema + .as_ref() + .map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id") + ), + "resolved schema must preserve its `id` property" + ); + assert_eq!(ir.reference_graph.edges.len(), 1); + assert_eq!( + ir.reference_graph.edges[0].target.snapshot_digest, + digest('b') + ); + assert!(ir.unresolved_references.is_empty()); +} + +// AC-3: after the encoded-dot alias is canonicalized, a relative child must be +// joined against the real canonical root, not the raw alias. RED while the +// resolver still keys the alias literally. +#[test] +fn encoded_dot_root_alias_resolves_relative_child_against_canonical_root() { + let document = r#" +openapi: 3.1.0 +info: { title: Canonical root alias } +servers: [{ url: https://api.example.test }] +paths: + /items: + get: + operationId: listItems + responses: + '200': + description: ok + content: { application/json: { schema: { $ref: 'https://schemas.example.test/a/%2e%2e/root.yaml#/Item' } } } +"#; + let snapshots = vec![ + ExternalDocumentSnapshot { + // Transport canonical of the encoded-dot root alias. + canonical_uri: "https://schemas.example.test/root.yaml".to_owned(), + digest: digest('b'), + document: "Item: { $ref: './child.yaml#/Item' }".to_owned(), + }, + ExternalDocumentSnapshot { + canonical_uri: "https://schemas.example.test/child.yaml".to_owned(), + digest: digest('c'), + document: + "Item: { type: object, required: [id], properties: { id: { type: string } } }" + .to_owned(), + }, + ]; + let config = NormalizationConfig { + external_references_enabled: true, + ..NormalizationConfig::default() + }; + let ir = normalize_verified_bundle(document, digest('a'), &snapshots, &config).unwrap(); + + assert!( + !ir.operations[0] + .findings + .iter() + .any(|finding| finding.code == "external_reference_unavailable"), + "encoded-dot root alias must resolve and join its child against the canonical root, got: {:?}", + ir.operations[0].findings + ); + assert!( + matches!( + ir.operations[0] + .response_schema + .as_ref() + .map(|schema| &schema.kind), + Some(NormalizedSchemaKind::Object { properties, .. }) if properties.contains_key("id") + ), + "child schema must resolve through the canonical root" + ); + assert_eq!( + ir.reference_graph.dependency_digests, + vec![digest('b'), digest('c')] + ); + assert!(ir.unresolved_references.is_empty()); +} diff --git a/crates/crank-import/tests/support-matrix.yaml b/crates/crank-import/tests/support-matrix.yaml index 0b8666b..dd820b1 100644 --- a/crates/crank-import/tests/support-matrix.yaml +++ b/crates/crank-import/tests/support-matrix.yaml @@ -1,5 +1,5 @@ -matrix_version: 1.0.3 -compatibility_impact: "1.0.3 adds executable blockers for undeclared path placeholders and object/array schema defaults in OpenAPI 3.0, OpenAPI 3.1, and Swagger 2.0. The 1.0.2 effective-security inheritance/override behavior remains covered; scalar defaults and previously supported importer behavior remain unchanged." +matrix_version: 1.0.4 +compatibility_impact: "1.0.4 records the shared external HTTP(S) URI canonicalization contract (percent-encoded dot segments, WHATWG special-query encoding, IDNA punycode host and default-port removal) now used by both the Admin snapshot materializer and the pure resolver; no construct status or previously supported behavior changes. The 1.0.3 executable blockers for undeclared path placeholders and object/array schema defaults across OpenAPI 3.0, OpenAPI 3.1, and Swagger 2.0 remain covered." constructs: - specification_version: "OpenAPI 3.0" construct_id: "http.method.get" diff --git a/docs/openapi-import.md b/docs/openapi-import.md index fa7fe90..04d5a07 100644 --- a/docs/openapi-import.md +++ b/docs/openapi-import.md @@ -212,7 +212,7 @@ Semantic loss получает blocker с source location: такой candidate Один endpoint с несколькими request formats остаётся одним Tool. Поля `content_type` и `body` выбирают одну schema и соответствующую сериализацию; неизвестный selector или body другой ветви отклоняются до отправки запроса. Для единственного формата selector не нужен. Подробный inline-file contract описан в [REST](protocols/rest.md). -Новые imports используют `normalized-ir-v5` / `preview-v6`: source-derived candidate, finding и naming conflict включают безопасный RFC 6901 `source_location`; typed findings сохраняют стабильный `code`. Схемы и маппинги принадлежат candidate с его source pointer и проверяемой provenance в IR/reference graph. Производные группы, counts и summary связаны с исходником через membership candidates; единственный pointer или finding code для такой агрегации был бы ложной provenance. Исторические v2/v3/v4/v5 jobs воспроизводят сохранённый preview, а legacy-v1 использует отдельную историческую projection. Обновление importer не меняет inputs существующих jobs. +Новые imports используют `normalized-ir-v6` / `preview-v6`: source-derived candidate, finding и naming conflict включают безопасный RFC 6901 `source_location`; typed findings сохраняют стабильный `code`. Схемы и маппинги принадлежат candidate с его source pointer и проверяемой provenance в IR/reference graph. Производные группы, counts и summary связаны с исходником через membership candidates; единственный pointer или finding code для такой агрегации был бы ложной provenance. Общая pure HTTP(S) URI identity (`url::Url`) появилась в `normalized-ir-v6`: materializer и resolver больше не расходятся на percent-encoded dot segments и special-query символах. Исторические v2/v3/v4/v5 jobs, включая прежнюю пару `normalized-ir-v5`/`preview-v6`, воспроизводят сохранённый preview без повторного разбора, а legacy-v1 использует отдельную историческую projection. Обновление importer не меняет inputs существующих jobs. Другие форматы спецификаций и протоколов в Community-импорт не входят. @@ -256,7 +256,7 @@ Harness отправляет реальный `openapi.yaml` с 2 000 operations ## Нормативная матрица и regression corpus -Машинная матрица версии `1.0.3` хранится в [`crates/crank-import/tests/support-matrix.yaml`](../crates/crank-import/tests/support-matrix.yaml); corpus version — `1.0.4`. Матрица фиксирует контракт отдельно для OpenAPI 3.0, OpenAPI 3.1 и Swagger 2.0. Acceptance test запускает нормализатор на отдельном probe для каждой строки, включая все 100 успешных кодов в каждом формате, и сверяет сохранённую semantics или точный blocker; выполненные probes списываются со всех строк версии, поэтому неподвергнутая проверкой строка матрицы падает gate. Corpus version `1.0.4` — metadata-only SemVer increment для согласования manifest с неизменной matrix `1.0.3`; fixture contents и checksums не менялись. Предыдущий corpus release `1.0.3` добавил hostile fixture `hostile-security-scope.json`: document-level требование безопасности применяется только к унаследовавшему его secured candidate (`GET /secure`, location `/security/0`), а операция с явным `security: []` (`GET /public`) остаётся actionable без глобального blocker; oversized hostile JSON остаётся тем же canonical JSON с ожидаемым parser rejection. +Машинная матрица версии `1.0.4` хранится в [`crates/crank-import/tests/support-matrix.yaml`](../crates/crank-import/tests/support-matrix.yaml); corpus version — `1.0.5`. Матрица фиксирует контракт отдельно для OpenAPI 3.0, OpenAPI 3.1 и Swagger 2.0. Acceptance test запускает нормализатор на отдельном probe для каждой строки, включая все 100 успешных кодов в каждом формате, и сверяет сохранённую semantics или точный blocker; выполненные probes списываются со всех строк версии, поэтому неподвергнутая проверкой строка матрицы падает gate. Corpus version `1.0.5` добавляет два hostile external-URI regression fixture (`regression-uri-encoded-dot.yaml`, `regression-uri-query-quote.yaml`) и модуль `reference_uri.rs`: encoded-dot path segment и апостроф в query канонизируются в URL transport identity, поэтому поставленный snapshot разрешается, а default-deny pass, statuses и denominators не ослабляются. Matrix `1.0.4` фиксирует общий external HTTP(S) URI canonicalization contract без изменения statuses. Предыдущий corpus release `1.0.4` был metadata-only SemVer increment; `1.0.3` добавил hostile fixture `hostile-security-scope.json`: document-level требование безопасности применяется только к унаследовавшему его secured candidate (`GET /secure`, location `/security/0`), а операция с явным `security: []` (`GET /public`) остаётся actionable без глобального blocker. | Конструкция | OpenAPI 3.0/3.1 | Swagger 2.0 | Сохраняемый контракт или finding | |---|---|---|---| @@ -276,6 +276,6 @@ Harness отправляет реальный `openapi.yaml` с 2 000 operations | Микс public/secured operations | Поддерживается | Подвергается проверке | Blocker принадлежит только затронутому candidate; global blocker не создаётся для требований одного operation | | XML, `TRACE`, `CONNECT` | Unsupported | Unsupported | Source-located finding; blocker не становится actionable Draft | -Матрица `1.0.3` добавляет исполняемые probes обоих blockers во всех трёх форматах; coverage validation отвергает неподтверждённые source findings. Новые primary документы и external snapshots отвергают повторённые JSON/YAML mapping keys вместо last-key-wins; корректный YAML flow syntax сохраняется. HTTP(S) scheme регистронезависима, а Unicode hostname сопоставляется с punycode snapshot identity. Historical legacy v1 decoder и уже сохранённые completed reports не переписываются; новый preview с изменившейся semantics требует нового подтверждённого плана. +Матрица `1.0.4` фиксирует общий pure HTTP(S) URI canonicalization contract во всех трёх форматах; matrix `1.0.3` добавила исполняемые probes обоих blockers во всех трёх форматах; coverage validation отвергает неподтверждённые source findings. Новые primary документы и external snapshots отвергают повторённые JSON/YAML mapping keys вместо last-key-wins; корректный YAML flow syntax сохраняется. HTTP(S) scheme регистронезависима, percent-encoded dot segments и special-query символы приводятся к `url::Url` identity, а Unicode hostname сопоставляется с punycode snapshot identity. Historical legacy v1 decoder и уже сохранённые completed reports не переписываются; новый preview с изменившейся semantics требует нового подтверждённого плана. Offline corpus в `crates/crank-import/tests/corpus/` содержит 50 synthetic representative документов и hostile subset. Manifest фиксирует SHA-256 исходных файлов, canonical digest JSON, происхождение и лицензию; для representative документов он также фиксирует полный canonical preview и frozen candidates (schemas, mappings, actionable/blocker), а для hostile — ожидаемые отказы и findings. Команда `just openapi-corpus` прогоняет настоящий normalizer, сравнивает независимо перечисленные raw source pointers с canonical source tree и coverage ledger, проверяет порог 95% actionable от всех representative Operations и отдельно нулевой silent loss. Evidence связывает digest manifest и matrix, включает content digest и создаётся без перезаписи; конфликт содержимого для того же identity останавливает gate. CI прикладывает `crates/crank-import/target/openapi-corpus/evidence-.json`. Artifact не содержит загруженный исходник или значения secret-like examples.