fix(import): единая идентичность внешних URI (#96)
CI / Rust Checks (push) Successful in 25m54s
CI / UI Checks (push) Successful in 5s
CI / Community Image Smoke (push) Successful in 1m47s
CI / Frontend E2E (push) Successful in 10m12s
CI / Deploy (push) Successful in 1m22s

Единая transport-canonical идентичность внешних документов; normalizer v6 с immutable historical replay; matrix 1.0.4 / corpus 1.0.5. PR CI run840 success, локальная приёмка и independent review выполнены.

Co-authored-by: bsodfather <bsodfather@gmail.com>
This commit was merged in pull request #96.
This commit is contained in:
bsodfather authored and bsodfather committed 2026-10-11 06:19:16 +00:00
1 parent 178093e81b
commit 1b6db09bd1
22 files changed
+1669 -353

No files matched your search

@@ -0,0 +1,603 @@
//! F1 transport-parity cases for the Admin external-reference import.
//!
//! The external materializer stores snapshot keys produced by the URL transport
//! (`url::Url`), which collapses a `%2e%2e` path segment and percent-encodes an
//! apostrophe in a query (`'` -> `%27`). Because the materializer and the pure
//! resolver used different URI keys, both references produced a false
//! `external_reference_unavailable` blocker. These cases run the real allowlisted
//! fixture server through preview -> apply -> replay (service) and through the
//! authenticated multipart route (outer). They are registered from
//! `external_references.rs` so the parent file stays within the repository
//! file-size budget; shared helpers and imports come from `super`.
use super::*;
// F1 acceptance (both cases): the service preview -> apply -> replay path. The
// assertions verify the transport-canonical keys resolve, the dependency
// bytes/digests verify, apply never refetches, and replay is a stable no-op.
#[tokio::test]
#[serial]
async fn transport_canonical_external_uris_resolve_without_false_blocker_or_refetch() {
// Distinct bytes so each canonical URI owns a distinct content-addressed
// blob; byte-identical documents collapse into a single dependency envelope
// (the recorded alias/digest observation) and could not be told apart.
const ROOT_ITEM: &str =
"Item: { type: object, required: [id], properties: { id: { type: string } } }";
const QUERY_ITEM: &str =
"Item: { type: object, required: [id], properties: { id: { type: integer } } }";
let fetches = Arc::new(AtomicUsize::new(0));
let root_fetches = Arc::clone(&fetches);
let query_fetches = Arc::clone(&fetches);
let app = Router::new()
.route(
"/root.yaml",
get(move || {
let fetches = Arc::clone(&root_fetches);
async move {
fetches.fetch_add(1, Ordering::SeqCst);
ROOT_ITEM
}
}),
)
.route(
"/schema.json",
get(move || {
let fetches = Arc::clone(&query_fetches);
async move {
fetches.fetch_add(1, Ordering::SeqCst);
QUERY_ITEM
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let fixture_server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let origin = format!("http://{address}");
let registry = test_registry().await;
let storage_root = test_storage_root("openapi_import_external_uri_transport_parity");
let service = test_service_with_external_references(
registry.clone(),
storage_root.clone(),
vec![format!("{origin}/")],
);
let workspace_id = WorkspaceId::new("ws_default");
let upload = OpenApiUpload {
bytes: format!(
r##"
openapi: 3.1.0
info: {{ title: External URI transport parity }}
servers: [{{ url: https://api.example.test }}]
paths:
/dot:
get:
operationId: encodedDot
responses:
'200':
description: ok
content: {{ application/json: {{ schema: {{ $ref: '{origin}/a/%2e%2e/root.yaml#/Item' }} }} }}
/query:
get:
operationId: quotedQuery
responses:
'200':
description: ok
content: {{ application/json: {{ schema: {{ $ref: "{origin}/schema.json?filter='x'#/Item" }} }} }}
"##
)
.into_bytes(),
mime_type: "application/yaml".to_owned(),
locale: OpenApiUploadLocale::En,
};
let preview = service
.preview_openapi_import(&workspace_id, upload)
.await
.unwrap();
// Materialization first: the external materializer must have downloaded and
// stored both snapshots under the transport-canonical keys before resolution
// is judged. This separates the F1 resolver key mismatch from a snapshot that
// was never fetched at all.
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
let job = registry
.get_import_job(&workspace_id, &job_id)
.await
.unwrap()
.unwrap();
let dependencies = job.preview_payload["dependencies"].as_array().unwrap();
assert_eq!(dependencies.len(), 2);
let canonical_uris = dependencies
.iter()
.map(|dependency| dependency["canonical_uri"].as_str().unwrap().to_owned())
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(
canonical_uris,
std::collections::BTreeSet::from([
format!("{origin}/root.yaml"),
format!("{origin}/schema.json?filter=%27x%27"),
]),
"materialized dependency keys must be the transport-canonical URIs"
);
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"both external documents must be fetched exactly once during preview"
);
// Dependency bytes and digests verify against the workspace-scoped store.
let store = Arc::new(ArtifactStore::open(&storage_root).unwrap());
for dependency in dependencies {
let source_id = ArtifactSourceId::new(dependency["source_id"].as_str().unwrap());
let digest = dependency["digest"].as_str().unwrap();
let verified = registry
.read_artifact_source(Arc::clone(&store), &workspace_id, &source_id)
.await
.unwrap();
assert_eq!(verified.source.blob.artifact_ref.as_str(), digest);
let canonical_uri = dependency["canonical_uri"].as_str().unwrap();
let expected = if canonical_uri.ends_with("/root.yaml") {
ROOT_ITEM
} else {
QUERY_ITEM
};
assert_eq!(verified.bytes, expected.as_bytes());
}
// Criterion-defining: neither transport-canonical URI may become a false
// blocker. Collected across all operations so one failure reports both.
let false_blockers = preview
.preview
.groups
.iter()
.flat_map(|group| &group.operations)
.flat_map(|operation| {
operation
.findings
.iter()
.filter(|finding| finding.code == "external_reference_unavailable")
.map(|_| operation.key.clone())
})
.collect::<Vec<_>>();
assert!(
false_blockers.is_empty(),
"both transport-canonical URIs must resolve; false external blockers: {false_blockers:?}"
);
let resolved = preview
.preview
.groups
.iter()
.flat_map(|group| &group.operations)
.map(|operation| (operation.key.clone(), operation.output_fields))
.collect::<std::collections::BTreeMap<_, _>>();
assert_eq!(resolved["GET /dot"], 1, "encoded-dot schema must resolve");
assert_eq!(
resolved["GET /query"], 1,
"quoted-query schema must resolve"
);
// Apply consumes the saved snapshots and must not refetch.
let created = service
.create_openapi_import(
&workspace_id,
&job_id,
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /dot".to_owned(), "GET /query".to_owned()],
server_url: None,
conflict_mode: "skip".to_owned(),
},
)
.await
.unwrap();
assert_eq!(created.created.len(), 2);
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"apply must reuse the saved dependency snapshots"
);
let drafts_after_apply = registry.list_operations(&workspace_id).await.unwrap().len();
// AC-2 replay: re-issue the SAME request key/payload. The import is already
// applied, so the service must return the stored application result, create
// no extra Draft, and refetch nothing. The fixture server stays alive until
// after the replay so the counter would observe any real request.
let replay = service
.create_openapi_import(
&workspace_id,
&job_id,
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /dot".to_owned(), "GET /query".to_owned()],
server_url: None,
conflict_mode: "skip".to_owned(),
},
)
.await
.unwrap();
assert_eq!(
replay.created, created.created,
"replay must return the stored application result for the same request key and payload"
);
assert_eq!(replay.created.len(), 2);
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"replay must not refetch dependency snapshots"
);
fixture_server.abort();
let completed = registry
.get_import_job(&workspace_id, &job_id)
.await
.unwrap()
.unwrap();
assert_eq!(completed.status, ImportJobStatus::Completed);
assert_eq!(
registry.list_operations(&workspace_id).await.unwrap().len(),
drafts_after_apply,
"replay must not create extra Drafts"
);
}
// F1 outer path: the same references uploaded as authenticated multipart through
// the outer router must preview without a false blocker, and the materializer
// must have stored both transport-canonical snapshots before resolution is
// judged. The run then continues through the real router apply and a replay of
// the same payload key while the fixture is still alive: the stored application
// result must be returned unchanged, no extra Draft may appear, and the fetch
// counter must not grow.
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn outer_authenticated_multipart_preview_matches_transport_canonical_uri() {
// Distinct bytes so the two canonical URIs cannot collapse into one
// dependency envelope; both remain valid resolvable schemas.
const ROOT_ITEM: &str =
"Item: { type: object, required: [id], properties: { id: { type: string } } }";
const QUERY_ITEM: &str =
"Item: { type: object, required: [id], properties: { id: { type: integer } } }";
let fetches = Arc::new(AtomicUsize::new(0));
let root_fetches = Arc::clone(&fetches);
let query_fetches = Arc::clone(&fetches);
let fixture = Router::new()
.route(
"/root.yaml",
get(move || {
let fetches = Arc::clone(&root_fetches);
async move {
fetches.fetch_add(1, Ordering::SeqCst);
ROOT_ITEM
}
}),
)
.route(
"/schema.json",
get(move || {
let fetches = Arc::clone(&query_fetches);
async move {
fetches.fetch_add(1, Ordering::SeqCst);
QUERY_ITEM
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let fixture_server = tokio::spawn(async move { axum::serve(listener, fixture).await.unwrap() });
let origin = format!("http://{address}");
let registry = test_registry().await;
let app = build_test_app_with_external_references(
registry.clone(),
test_storage_root("openapi_import_external_uri_multipart"),
vec![format!("{origin}/")],
);
let server = spawn_admin_api(app).await;
let client = authorized_client(&server).await;
let document = format!(
r##"
openapi: 3.1.0
info: {{ title: Multipart transport parity }}
servers: [{{ url: https://api.example.test }}]
paths:
/dot:
get:
operationId: encodedDot
responses:
'200':
description: ok
content: {{ application/json: {{ schema: {{ $ref: '{origin}/a/%2e%2e/root.yaml#/Item' }} }} }}
/query:
get:
operationId: quotedQuery
responses:
'200':
description: ok
content: {{ application/json: {{ schema: {{ $ref: "{origin}/schema.json?filter='x'#/Item" }} }} }}
"##
);
let response = client
.post(format!("{server}/imports/openapi/preview"))
.multipart(
Form::new().part(
"file",
Part::bytes(document.into_bytes())
.file_name("openapi.yaml".to_owned())
.mime_str("application/yaml")
.unwrap(),
),
)
.send()
.await
.unwrap();
let status = response.status();
let body = response.json::<serde_json::Value>().await.unwrap();
assert_eq!(status, reqwest::StatusCode::OK, "{body}");
// The materializer must have downloaded both snapshots under the
// transport-canonical keys before the blocker check is meaningful.
let job_id_str = body["job_id"].as_str().unwrap().to_owned();
let job_id: crank_registry::ImportJobId = job_id_str.as_str().into();
let job = registry
.get_import_job(&WorkspaceId::new("ws_default"), &job_id)
.await
.unwrap()
.unwrap();
let canonical_uris = job.preview_payload["dependencies"]
.as_array()
.unwrap()
.iter()
.map(|dependency| dependency["canonical_uri"].as_str().unwrap().to_owned())
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(
canonical_uris,
std::collections::BTreeSet::from([
format!("{origin}/root.yaml"),
format!("{origin}/schema.json?filter=%27x%27"),
]),
"both transport-canonical external URIs must be materialized"
);
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"preview must fetch each external document exactly once"
);
assert!(
!body.to_string().contains("external_reference_unavailable"),
"outer multipart preview must not report a false external blocker: {body}"
);
let resolved = body["preview"]["groups"][0]["operations"]
.as_array()
.unwrap()
.iter()
.map(|operation| {
(
operation["key"].as_str().unwrap().to_owned(),
operation["output_fields"].clone(),
)
})
.collect::<std::collections::BTreeMap<_, _>>();
assert_eq!(resolved["GET /dot"], serde_json::json!(1));
assert_eq!(resolved["GET /query"], serde_json::json!(1));
// AC-2: apply through the real authenticated router, then replay the SAME
// payload key. The fixture stays alive so a real refetch would move the
// counter, and the same application key must not create another Draft.
let create_payload = serde_json::json!({
"selected_operation_keys": ["GET /dot", "GET /query"],
"server_url": null,
"conflict_mode": "skip"
});
let create_response = client
.post(format!("{server}/imports/openapi/{job_id_str}/create"))
.json(&create_payload)
.send()
.await
.unwrap();
assert_eq!(
create_response.status(),
reqwest::StatusCode::OK,
"apply through the router must succeed"
);
let created: serde_json::Value = create_response.json().await.unwrap();
assert_eq!(created["created"].as_array().unwrap().len(), 2);
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"apply must reuse the saved dependency snapshots"
);
let drafts_after_apply = client
.get(format!("{server}/operations"))
.send()
.await
.unwrap()
.json::<serde_json::Value>()
.await
.unwrap()["total"]
.clone();
assert_eq!(drafts_after_apply, serde_json::json!(2));
let replay_response = client
.post(format!("{server}/imports/openapi/{job_id_str}/create"))
.json(&create_payload)
.send()
.await
.unwrap();
assert_eq!(replay_response.status(), reqwest::StatusCode::OK);
let replay: serde_json::Value = replay_response.json().await.unwrap();
assert_eq!(
replay, created,
"replaying the same payload key must return the stored application result"
);
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"replay must not refetch dependency snapshots"
);
let drafts_after_replay = client
.get(format!("{server}/operations"))
.send()
.await
.unwrap()
.json::<serde_json::Value>()
.await
.unwrap()["total"]
.clone();
assert_eq!(
drafts_after_replay, drafts_after_apply,
"replay must not create extra Drafts"
);
fixture_server.abort();
}
// AC-3 over real HTTP: the primary document references an encoded-dot root alias
// (`/a/%2e%2e/root.yaml`) whose document contains a relative child `$ref`. The
// materializer must canonicalize the alias to `/root.yaml`, then join the child
// against that real canonical root (not the raw alias), so `/child.yaml` is
// fetched exactly once and the derived schema resolves through the chain.
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn encoded_dot_root_alias_chain_resolves_relative_child_against_canonical_root_over_http() {
const ROOT_ALIAS_DOCUMENT: &str = "Item: { $ref: './child.yaml#/Item' }";
const CHILD_ITEM: &str =
"Item: { type: object, required: [id], properties: { id: { type: string } } }";
let fetches = Arc::new(AtomicUsize::new(0));
let root_fetches = Arc::clone(&fetches);
let child_fetches = Arc::clone(&fetches);
let app = Router::new()
.route(
"/root.yaml",
get(move || {
let fetches = Arc::clone(&root_fetches);
async move {
fetches.fetch_add(1, Ordering::SeqCst);
ROOT_ALIAS_DOCUMENT
}
}),
)
.route(
"/child.yaml",
get(move || {
let fetches = Arc::clone(&child_fetches);
async move {
fetches.fetch_add(1, Ordering::SeqCst);
CHILD_ITEM
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let fixture_server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let origin = format!("http://{address}");
let registry = test_registry().await;
let storage_root = test_storage_root("openapi_import_external_uri_dot_root_chain");
let service = test_service_with_external_references(
registry.clone(),
storage_root.clone(),
vec![format!("{origin}/")],
);
let workspace_id = WorkspaceId::new("ws_default");
let upload = OpenApiUpload {
bytes: format!(
r##"
openapi: 3.1.0
info: {{ title: Canonical root alias chain }}
servers: [{{ url: https://api.example.test }}]
paths:
/items:
get:
operationId: listItems
responses:
'200':
description: ok
content: {{ application/json: {{ schema: {{ $ref: '{origin}/a/%2e%2e/root.yaml#/Item' }} }} }}
"##
)
.into_bytes(),
mime_type: "application/yaml".to_owned(),
locale: OpenApiUploadLocale::En,
};
let preview = service
.preview_openapi_import(&workspace_id, upload)
.await
.unwrap();
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
let job = registry
.get_import_job(&workspace_id, &job_id)
.await
.unwrap()
.unwrap();
let dependencies = job.preview_payload["dependencies"].as_array().unwrap();
assert_eq!(dependencies.len(), 2);
let canonical_uris = dependencies
.iter()
.map(|dependency| dependency["canonical_uri"].as_str().unwrap().to_owned())
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(
canonical_uris,
std::collections::BTreeSet::from([
format!("{origin}/root.yaml"),
format!("{origin}/child.yaml"),
]),
"the child must be materialized relative to the canonical root, not the raw encoded-dot alias"
);
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"the canonical root alias and its relative child must each be fetched exactly once"
);
// Dependency bytes and digests verify against the workspace-scoped store.
let store = Arc::new(ArtifactStore::open(&storage_root).unwrap());
for dependency in dependencies {
let source_id = ArtifactSourceId::new(dependency["source_id"].as_str().unwrap());
let digest = dependency["digest"].as_str().unwrap();
let verified = registry
.read_artifact_source(Arc::clone(&store), &workspace_id, &source_id)
.await
.unwrap();
assert_eq!(verified.source.blob.artifact_ref.as_str(), digest);
let canonical_uri = dependency["canonical_uri"].as_str().unwrap();
let expected = if canonical_uri.ends_with("/root.yaml") {
ROOT_ALIAS_DOCUMENT
} else {
CHILD_ITEM
};
assert_eq!(verified.bytes, expected.as_bytes());
}
// No false blocker: the chain resolves through the canonical root.
let false_blockers = preview
.preview
.groups
.iter()
.flat_map(|group| &group.operations)
.flat_map(|operation| {
operation
.findings
.iter()
.filter(|finding| finding.code == "external_reference_unavailable")
.map(|_| operation.key.clone())
})
.collect::<Vec<_>>();
assert!(
false_blockers.is_empty(),
"the encoded-dot root alias chain must resolve; false blockers: {false_blockers:?}"
);
assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);
// The derived schema must be the child object reached through the canonical
// root, projected into the persisted preview.
let output_schema =
&job.preview_payload["preview"]["groups"][0]["operations"][0]["draft"]["output_schema"];
assert_eq!(
output_schema["type"], "object",
"the operation output schema must be the child object derived through the chain: {output_schema}"
);
assert_eq!(
output_schema["fields"]["id"]["type"], "string",
"the child `id` property must survive resolution: {output_schema}"
);
fixture_server.abort();
}