Единая transport-canonical идентичность внешних документов; normalizer v6 с immutable historical replay; matrix 1.0.4 / corpus 1.0.5. PR CI run840 success, локальная приёмка и independent review выполнены. Co-authored-by: bsodfather <bsodfather@gmail.com>
934 lines
39 KiB
Rust
934 lines
39 KiB
Rust
use super::super::common::{
|
||
authorized_client, build_test_app_with_external_references, spawn_admin_api,
|
||
test_service_with_external_references,
|
||
};
|
||
mod isolation;
|
||
mod uri_transport;
|
||
use super::*;
|
||
use admin_api::error::ApiError;
|
||
use axum::{Router, body::to_bytes, http::StatusCode, response::IntoResponse, routing::get};
|
||
use crank_artifacts::ArtifactStore;
|
||
use crank_core::{Workspace, WorkspaceStatus};
|
||
use crank_registry::{ArtifactSourceId, CreateWorkspaceRequest, RegistryError};
|
||
use reqwest::multipart::{Form, Part};
|
||
use sha2::{Digest, Sha256};
|
||
use std::sync::{
|
||
Arc,
|
||
atomic::{AtomicUsize, Ordering},
|
||
};
|
||
use time::OffsetDateTime;
|
||
|
||
// Serialized by the integration harness at baseline f5f40cb167a82b1e3c1ea2475db5cbaed9d9abfa; this v4 payload preserves the then-current security blocker and digest.
|
||
const BASELINE_V4_SECURITY_PREVIEW_PAYLOAD: &str = r###"{"dependencies":[],"dependency_snapshots":[],"normalization":{"config":{"external_references_enabled":false,"max_aliases":128,"max_bytes":262144,"max_collection_items":10000,"max_depth":64,"max_expanded_nodes":100000,"max_external_document_bytes":262144,"max_nodes":50000,"max_reference_depth":32,"max_reference_documents":32,"max_references":4096,"max_scalar_bytes":262144,"normalizer_version":"normalized-ir-v4","projection_version":"preview-v4"},"ir_fingerprint":"ae2319de76c13d631b26843623092d966e5deea7cda9e3e40e4acc6fa36a25a8","normalizer_version":"normalized-ir-v4","projection_version":"preview-v4"},"preview":{"findings":[{"code":"unsupported_security_requirement","message":"Требование безопасности должно быть привязано к настроенному Auth Profile перед публикацией этой операции; остальные операции не блокируются.","operation_key":"GET /secure","severity":"error"}],"groups":[{"key":"imported_operation","operations":[{"category":"imported","description":"Выполняет GET /public","draft":{"category":"imported","display_name":"Public","input_mapping":{},"input_schema":{"description":"Входные параметры MCP-инструмента","nullable":false,"required":true,"type":"object"},"name":"public","output_mapping":{"rules":[{"required":true,"source":"$.response.body","target":"$.output"}]},"output_schema":{"description":"Успешный ответ без тела","nullable":false,"required":true,"type":"null"},"target":{"base_url":"https://api.example.test","method":"GET","path_template":"/public","response_contract":[{"schema_variant":0,"status_code":200}]},"tool_description":{"description":"Выполняет GET /public","examples":[{"input":{}}],"title":"Public"},"wizard_state":{}},"findings":[{"code":"missing_summary","message":"У метода нет summary, отображаемое имя будет сгенерировано автоматически.","operation_key":"GET /public","severity":"warning"},{"code":"missing_description","message":"У метода нет description. Перед публикацией лучше описать, когда агенту стоит вызывать этот инструмент.","operation_key":"GET /public","severity":"warning"},{"code":"missing_response_schema","message":"У метода не найдена схема успешного ответа, результат будет описан как общий объект.","operation_key":"GET /public","severity":"warning"},{"code":"no_input_fields","message":"У инструмента нет входных параметров. Это нормально для справочных методов, но проверьте, что агенту не нужно передавать фильтры.","operation_key":"GET /public","severity":"info"},{"code":"empty_output_schema","message":"В ответе не найдено отдельных полей. Перед публикацией проверьте схему ответа и маппинг результата.","operation_key":"GET /public","severity":"warning"},{"code":"weak_tool_description","message":"Описание инструмента слишком короткое или техническое. Перед публикацией добавьте, когда агент должен вызывать инструмент и что будет в успешном ответе.","operation_key":"GET /public","severity":"warning"},{"code":"weak_tool_name","message":"Имя инструмента `public` выглядит слишком общим. Лучше использовать имя с конкретным действием и объектом.","operation_key":"GET /public","severity":"warning"}],"input_fields":0,"key":"GET /public","method":"GET","operation_id":"public","output_fields":0,"path":"/public","server_urls":["https://api.example.test"],"suggested_display_name":"Public","suggested_name":"public"},{"category":"imported","description":"Выполняет GET /secure","draft":{"category":"imported","display_name":"Secure","input_mapping":{},"input_schema":{"description":"Входные параметры MCP-инструмента","nullable":false,"required":true,"type":"object"},"name":"secure","output_mapping":{"rules":[{"required":true,"source":"$.response.body","target":"$.output"}]},"output_schema":{"description":"Успешный ответ без тела","nullable":false,"required":true,"type":"null"},"target":{"base_url":"https://api.example.test","method":"GET","path_template":"/secure","response_contract":[{"schema_variant":0,"status_code":200}]},"tool_description":{"description":"Выполняет GET /secure","examples":[{"input":{}}],"title":"Secure"},"wizard_state":{}},"findings":[{"code":"missing_summary","message":"У метода нет summary, отображаемое имя будет сгенерировано автоматически.","operation_key":"GET /secure","severity":"warning"},{"code":"missing_description","message":"У метода нет description. Перед публикацией лучше описать, когда агенту стоит вызывать этот инструмент.","operation_key":"GET /secure","severity":"warning"},{"code":"missing_response_schema","message":"У метода не найдена схема успешного ответа, результат будет описан как общий объект.","operation_key":"GET /secure","severity":"warning"},{"code":"no_input_fields","message":"У инструмента нет входных параметров. Это нормально для справочных методов, но проверьте, что агенту не нужно передавать фильтры.","operation_key":"GET /secure","severity":"info"},{"code":"empty_output_schema","message":"В ответе не найдено отдельных полей. Перед публикацией проверьте схему ответа и маппинг результата.","operation_key":"GET /secure","severity":"warning"},{"code":"weak_tool_description","message":"Описание инструмента слишком короткое или техническое. Перед публикацией добавьте, когда агент должен вызывать инструмент и что будет в успешном ответе.","operation_key":"GET /secure","severity":"warning"},{"code":"weak_tool_name","message":"Имя инструмента `secure` выглядит слишком общим. Лучше использовать имя с конкретным действием и объектом.","operation_key":"GET /secure","severity":"warning"},{"code":"unsupported_security_requirement","message":"Требование безопасности должно быть привязано к настроенному Auth Profile перед публикацией этой операции; остальные операции не блокируются.","operation_key":"GET /secure","severity":"error"}],"input_fields":0,"key":"GET /secure","method":"GET","operation_id":"secure","output_fields":0,"path":"/secure","server_urls":["https://api.example.test"],"suggested_display_name":"Secure","suggested_name":"secure"}],"title":"Без группы"}],"source":{"format":"openapi","servers":["https://api.example.test"],"title":"Historical security","version":"3.0.3"}},"preview_digest":"85435e01b37d8da35ebc58910404fa364abf55e0089b6593e32fa285c45957c8","source":{"digest":"sha256:a7ef659aff5f5dde6ec38d689746ed4653b1db8038ceb630f6735de16802a999","source_id":"src_openapi_01a0f9b410007852872bb8888caf5690"}}"###;
|
||
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn external_relative_chain_survives_reversed_builder_order_and_apply_never_refetches() {
|
||
let fetches = Arc::new(AtomicUsize::new(0));
|
||
let root_fetches = Arc::clone(&fetches);
|
||
let child_fetches = Arc::clone(&fetches);
|
||
let app = Router::new()
|
||
.route(
|
||
"/root.yaml",
|
||
get(move || {
|
||
let fetches = Arc::clone(&root_fetches);
|
||
async move {
|
||
fetches.fetch_add(1, Ordering::SeqCst);
|
||
"Item: { $ref: './child.yaml#/Item' }"
|
||
}
|
||
}),
|
||
)
|
||
.route(
|
||
"/child.yaml",
|
||
get(move || {
|
||
let fetches = Arc::clone(&child_fetches);
|
||
async move {
|
||
fetches.fetch_add(1, Ordering::SeqCst);
|
||
"Item: { type: object, required: [id], properties: { id: { type: string } } }"
|
||
}
|
||
}),
|
||
);
|
||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||
let address = listener.local_addr().unwrap();
|
||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||
let origin = format!("http://{address}");
|
||
|
||
let registry = test_registry().await;
|
||
let storage_root = test_storage_root("openapi_import_external_snapshot");
|
||
let service = test_service_with_external_references(
|
||
registry.clone(),
|
||
storage_root.clone(),
|
||
vec![format!("{origin}/")],
|
||
);
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let foreign_workspace_id = create_workspace(®istry, "ws_external_snapshot_foreign").await;
|
||
let upload = OpenApiUpload {
|
||
bytes: format!(
|
||
r#"
|
||
openapi: 3.1.0
|
||
info: {{ title: External }}
|
||
servers: [{{ url: https://api.example.test }}]
|
||
paths:
|
||
/items:
|
||
get:
|
||
operationId: listItems
|
||
responses:
|
||
'200':
|
||
description: ok
|
||
content:
|
||
application/json:
|
||
schema: {{ $ref: '{origin}/root.yaml#/Item' }}
|
||
"#
|
||
)
|
||
.into_bytes(),
|
||
mime_type: "application/yaml".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
};
|
||
let preview = service
|
||
.preview_openapi_import(&workspace_id, upload)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||
assert_eq!(preview.preview.groups[0].operations[0].output_fields, 1);
|
||
let job = registry
|
||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||
.await
|
||
.unwrap()
|
||
.unwrap();
|
||
assert_eq!(
|
||
job.preview_payload["dependencies"]
|
||
.as_array()
|
||
.unwrap()
|
||
.len(),
|
||
2
|
||
);
|
||
assert_eq!(
|
||
job.preview_payload["dependency_snapshots"]
|
||
.as_array()
|
||
.unwrap()
|
||
.len(),
|
||
2
|
||
);
|
||
|
||
// The preview response contains only the normalized result. Ownership
|
||
// envelopes, storage digests, and remote origins stay in the persisted
|
||
// job payload and are verified through the workspace-scoped registry API.
|
||
let public_preview = serde_json::to_string(&preview).unwrap();
|
||
assert!(!public_preview.contains("src_openapi_dep_"));
|
||
assert!(!public_preview.contains(&origin));
|
||
let dependencies = job.preview_payload["dependencies"].as_array().unwrap();
|
||
let store = Arc::new(ArtifactStore::open(&storage_root).unwrap());
|
||
for dependency in dependencies {
|
||
let source_id = ArtifactSourceId::new(dependency["source_id"].as_str().unwrap());
|
||
let digest = dependency["digest"].as_str().unwrap();
|
||
let canonical_uri = dependency["canonical_uri"].as_str().unwrap();
|
||
let source = registry
|
||
.get_artifact_source(&workspace_id, &source_id)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(source.workspace_id, workspace_id);
|
||
assert_eq!(source.blob.artifact_ref.as_str(), digest);
|
||
let verified = registry
|
||
.read_artifact_source(Arc::clone(&store), &workspace_id, &source_id)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(verified.source.blob.artifact_ref.as_str(), digest);
|
||
let expected_bytes = if canonical_uri.ends_with("/root.yaml") {
|
||
b"Item: { $ref: './child.yaml#/Item' }".as_slice()
|
||
} else {
|
||
assert!(canonical_uri.ends_with("/child.yaml"));
|
||
b"Item: { type: object, required: [id], properties: { id: { type: string } } }"
|
||
.as_slice()
|
||
};
|
||
assert_eq!(verified.bytes, expected_bytes);
|
||
|
||
let foreign_read = registry
|
||
.read_artifact_source(Arc::clone(&store), &foreign_workspace_id, &source_id)
|
||
.await
|
||
.expect_err("foreign workspace must not read an owned snapshot");
|
||
let unknown_source_id = ArtifactSourceId::new("src_unknown_scope_probe");
|
||
let unknown_read = registry
|
||
.read_artifact_source(
|
||
Arc::clone(&store),
|
||
&foreign_workspace_id,
|
||
&unknown_source_id,
|
||
)
|
||
.await
|
||
.expect_err("unknown source must have the same scoped outcome");
|
||
let foreign_body = source_not_found_body(foreign_read).await;
|
||
let unknown_body = source_not_found_body(unknown_read).await;
|
||
assert_eq!(foreign_body, unknown_body);
|
||
let rendered_body = foreign_body.to_string();
|
||
assert!(!rendered_body.contains(digest));
|
||
assert!(!rendered_body.contains(canonical_uri));
|
||
assert!(!rendered_body.contains(source_id.as_str()));
|
||
}
|
||
|
||
let applied = service
|
||
.create_openapi_import(
|
||
&workspace_id,
|
||
&preview.job_id.as_str().into(),
|
||
OpenApiImportCreatePayload {
|
||
selected_operation_keys: vec!["GET /items".to_owned()],
|
||
server_url: None,
|
||
conflict_mode: "skip".to_owned(),
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
server.abort();
|
||
assert_eq!(applied.created.len(), 1);
|
||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||
let active_sources: i64 = sqlx::query_scalar(
|
||
"select count(*) from artifact_sources
|
||
where source_id like 'src_openapi_%' and lifecycle = 'active'",
|
||
)
|
||
.fetch_one(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(active_sources, 0);
|
||
}
|
||
|
||
fn external_upload(origin: &str) -> OpenApiUpload {
|
||
OpenApiUpload {
|
||
bytes: format!(
|
||
r#"
|
||
openapi: 3.1.0
|
||
info: {{ title: Scoped external snapshot }}
|
||
servers: [{{ url: https://api.example.test }}]
|
||
paths:
|
||
/items:
|
||
get:
|
||
operationId: listItems
|
||
responses:
|
||
'200':
|
||
description: ok
|
||
content:
|
||
application/json:
|
||
schema: {{ $ref: '{origin}/schema.yaml#/Item' }}
|
||
"#
|
||
)
|
||
.into_bytes(),
|
||
mime_type: "application/yaml".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
}
|
||
}
|
||
|
||
async fn create_workspace(registry: &crank_registry::PostgresRegistry, id: &str) -> WorkspaceId {
|
||
let workspace_id = WorkspaceId::new(id);
|
||
let now = OffsetDateTime::now_utc();
|
||
registry
|
||
.create_workspace(CreateWorkspaceRequest {
|
||
workspace: &Workspace {
|
||
id: workspace_id.clone(),
|
||
slug: id.replace('_', "-"),
|
||
display_name: id.to_owned(),
|
||
status: WorkspaceStatus::Active,
|
||
settings: serde_json::json!({}),
|
||
created_at: now,
|
||
updated_at: now,
|
||
},
|
||
})
|
||
.await
|
||
.unwrap();
|
||
workspace_id
|
||
}
|
||
|
||
async fn source_not_found_body(error: RegistryError) -> serde_json::Value {
|
||
assert!(matches!(&error, RegistryError::SourceNotFound { .. }));
|
||
let response = ApiError::from(error).into_response();
|
||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||
serde_json::from_slice(&body).unwrap()
|
||
}
|
||
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn external_secret_document_is_rejected_before_blob_registration() {
|
||
let external_document = "Item: { type: object, x-api-key: external-secret-canary-42, properties: { id: { type: string } } }";
|
||
let fetched = Arc::new(AtomicUsize::new(0));
|
||
let fetch_count = Arc::clone(&fetched);
|
||
let app = Router::new().route(
|
||
"/schemas.yaml",
|
||
get(move || {
|
||
let fetch_count = Arc::clone(&fetch_count);
|
||
async move {
|
||
fetch_count.fetch_add(1, Ordering::SeqCst);
|
||
external_document
|
||
}
|
||
}),
|
||
);
|
||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||
let address = listener.local_addr().unwrap();
|
||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||
let origin = format!("http://{address}");
|
||
let registry = test_registry().await;
|
||
let storage_root = test_storage_root("openapi_import_external_secret_rejected");
|
||
let service = test_service_with_external_references(
|
||
registry.clone(),
|
||
storage_root.clone(),
|
||
vec![format!("{origin}/")],
|
||
);
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let error = service
|
||
.preview_openapi_import(
|
||
&workspace_id,
|
||
OpenApiUpload {
|
||
bytes: format!(
|
||
r#"
|
||
openapi: 3.1.0
|
||
info: {{ title: Safe primary with unsafe external }}
|
||
servers: [{{ url: https://api.example.test }}]
|
||
paths:
|
||
/items:
|
||
get:
|
||
operationId: listItems
|
||
responses:
|
||
'200':
|
||
description: ok
|
||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||
"#
|
||
)
|
||
.into_bytes(),
|
||
mime_type: "application/yaml".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
},
|
||
)
|
||
.await
|
||
.expect_err("unsafe external bundle must be rejected before source persistence");
|
||
server.abort();
|
||
assert_eq!(fetched.load(Ordering::SeqCst), 1);
|
||
let rendered_error = format!("{error:?}");
|
||
assert!(rendered_error.contains("invalid_document"));
|
||
assert!(!rendered_error.contains("external-secret-canary-42"));
|
||
|
||
// The raw dependency's content-addressed path must not exist even as an
|
||
// unregistered blob, and no other stored file may contain its literal.
|
||
let external_digest = format!("{:x}", Sha256::digest(external_document.as_bytes()));
|
||
let external_blob = storage_root
|
||
.join("sha256")
|
||
.join(&external_digest[..2])
|
||
.join(&external_digest);
|
||
assert!(
|
||
!external_blob.exists(),
|
||
"secret-bearing external response was written to the artifact store"
|
||
);
|
||
assert!(!storage_contains_literal(
|
||
&storage_root,
|
||
b"external-secret-canary-42"
|
||
));
|
||
|
||
let dependency_sources: i64 = sqlx::query_scalar(
|
||
"select count(*) from artifact_sources
|
||
where workspace_id = $1 and source_id like 'src_openapi_dep_%'",
|
||
)
|
||
.bind(workspace_id.as_str())
|
||
.fetch_one(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(dependency_sources, 0);
|
||
let active_sources: i64 = sqlx::query_scalar(
|
||
"select count(*) from artifact_sources where workspace_id = $1 and lifecycle = 'active'",
|
||
)
|
||
.bind(workspace_id.as_str())
|
||
.fetch_one(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(active_sources, 0);
|
||
}
|
||
|
||
fn storage_contains_literal(root: &std::path::Path, literal: &[u8]) -> bool {
|
||
std::fs::read_dir(root).unwrap().any(|entry| {
|
||
let path = entry.unwrap().path();
|
||
if path.is_dir() {
|
||
storage_contains_literal(&path, literal)
|
||
} else {
|
||
std::fs::read(path)
|
||
.unwrap()
|
||
.windows(literal.len())
|
||
.any(|window| window == literal)
|
||
}
|
||
})
|
||
}
|
||
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn missing_external_snapshot_fails_closed_before_draft_mutation() {
|
||
let app = Router::new().route(
|
||
"/schemas.yaml",
|
||
get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
|
||
);
|
||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||
let address = listener.local_addr().unwrap();
|
||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||
let origin = format!("http://{address}");
|
||
let registry = test_registry().await;
|
||
let storage_root = test_storage_root("openapi_import_missing_external_snapshot");
|
||
let service = test_service_with_external_references(
|
||
registry.clone(),
|
||
storage_root.clone(),
|
||
vec![format!("{origin}/")],
|
||
);
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let preview = service
|
||
.preview_openapi_import(
|
||
&workspace_id,
|
||
OpenApiUpload {
|
||
bytes: format!(
|
||
r#"
|
||
openapi: 3.1.0
|
||
info: {{ title: External integrity }}
|
||
servers: [{{ url: https://api.example.test }}]
|
||
paths:
|
||
/items:
|
||
get:
|
||
operationId: listItems
|
||
responses:
|
||
'200':
|
||
description: ok
|
||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||
"#
|
||
)
|
||
.into_bytes(),
|
||
mime_type: "application/yaml".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
let job = registry
|
||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||
.await
|
||
.unwrap()
|
||
.unwrap();
|
||
let dependency_source_id = job.preview_payload["dependencies"][0]["source_id"]
|
||
.as_str()
|
||
.unwrap();
|
||
sqlx::query(
|
||
"update artifact_sources
|
||
set lifecycle = 'detached', updated_at = now(), detached_at = now()
|
||
where workspace_id = $1 and source_id = $2",
|
||
)
|
||
.bind(workspace_id.as_str())
|
||
.bind(dependency_source_id)
|
||
.execute(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
|
||
assert!(
|
||
service
|
||
.create_openapi_import(
|
||
&workspace_id,
|
||
&preview.job_id.as_str().into(),
|
||
OpenApiImportCreatePayload {
|
||
selected_operation_keys: vec!["GET /items".to_owned()],
|
||
server_url: None,
|
||
conflict_mode: "skip".to_owned(),
|
||
},
|
||
)
|
||
.await
|
||
.is_err()
|
||
);
|
||
assert!(
|
||
service
|
||
.list_operations(&workspace_id)
|
||
.await
|
||
.unwrap()
|
||
.is_empty()
|
||
);
|
||
assert_failed_job_and_detached_sources(
|
||
®istry,
|
||
&workspace_id,
|
||
&preview.job_id.as_str().into(),
|
||
"import_dependency_verification_failed",
|
||
)
|
||
.await;
|
||
|
||
let corrupt_preview = service
|
||
.preview_openapi_import(
|
||
&workspace_id,
|
||
OpenApiUpload {
|
||
bytes: format!(
|
||
r#"
|
||
openapi: 3.1.0
|
||
info: {{ title: Corrupt external integrity }}
|
||
servers: [{{ url: https://api.example.test }}]
|
||
paths:
|
||
/items:
|
||
get:
|
||
operationId: listItemsCorrupt
|
||
responses:
|
||
'200':
|
||
description: ok
|
||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||
"#
|
||
)
|
||
.into_bytes(),
|
||
mime_type: "application/yaml".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
let corrupt_job = registry
|
||
.get_import_job(&workspace_id, &corrupt_preview.job_id.as_str().into())
|
||
.await
|
||
.unwrap()
|
||
.unwrap();
|
||
let dependency_ref: crank_artifacts::ArtifactRef = corrupt_job.preview_payload["dependencies"]
|
||
[0]["digest"]
|
||
.as_str()
|
||
.unwrap()
|
||
.parse()
|
||
.unwrap();
|
||
let dependency_path = storage_root
|
||
.join("sha256")
|
||
.join(&dependency_ref.digest_hex()[..2])
|
||
.join(dependency_ref.digest_hex());
|
||
std::fs::set_permissions(
|
||
&dependency_path,
|
||
std::os::unix::fs::PermissionsExt::from_mode(0o600),
|
||
)
|
||
.unwrap();
|
||
std::fs::write(&dependency_path, b"corrupt external dependency").unwrap();
|
||
std::fs::set_permissions(
|
||
&dependency_path,
|
||
std::os::unix::fs::PermissionsExt::from_mode(0o400),
|
||
)
|
||
.unwrap();
|
||
|
||
assert!(
|
||
service
|
||
.create_openapi_import(
|
||
&workspace_id,
|
||
&corrupt_preview.job_id.as_str().into(),
|
||
OpenApiImportCreatePayload {
|
||
selected_operation_keys: vec!["GET /items".to_owned()],
|
||
server_url: None,
|
||
conflict_mode: "skip".to_owned(),
|
||
},
|
||
)
|
||
.await
|
||
.is_err()
|
||
);
|
||
assert_failed_job_and_detached_sources(
|
||
®istry,
|
||
&workspace_id,
|
||
&corrupt_preview.job_id.as_str().into(),
|
||
"import_dependency_verification_failed",
|
||
)
|
||
.await;
|
||
}
|
||
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn external_materialization_uses_one_wall_clock_timeout_and_cleans_cancelled_sources() {
|
||
let fetches = Arc::new(AtomicUsize::new(0));
|
||
let root_fetches = Arc::clone(&fetches);
|
||
let child_fetches = Arc::clone(&fetches);
|
||
let app = Router::new()
|
||
.route(
|
||
"/root.yaml",
|
||
get(move || {
|
||
let fetches = Arc::clone(&root_fetches);
|
||
async move {
|
||
fetches.fetch_add(1, Ordering::SeqCst);
|
||
"Item: { $ref: './slow-child.yaml#/Item' }"
|
||
}
|
||
}),
|
||
)
|
||
.route(
|
||
"/slow-child.yaml",
|
||
get(move || {
|
||
let fetches = Arc::clone(&child_fetches);
|
||
async move {
|
||
fetches.fetch_add(1, Ordering::SeqCst);
|
||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||
"Item: { type: object, properties: { id: { type: string } } }"
|
||
}
|
||
}),
|
||
);
|
||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||
let address = listener.local_addr().unwrap();
|
||
let timeout_server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||
let origin = format!("http://{address}");
|
||
|
||
let registry = test_registry().await;
|
||
let outbound_policy = crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]);
|
||
let runtime = crank_runtime::community_with_outbound_policy(outbound_policy.clone()).build();
|
||
let service = AdminServiceBuilder::new(
|
||
registry.clone(),
|
||
test_storage_root("openapi_import_chain_timeout"),
|
||
test_auth_settings(),
|
||
test_secret_crypto(),
|
||
runtime,
|
||
)
|
||
.with_external_reference_import(&crank_config::ExternalReferenceSettings {
|
||
allowed_url_prefixes: vec![format!("{origin}/")],
|
||
max_depth: 8,
|
||
max_documents: 32,
|
||
max_fetch_bytes: 64 * 1024,
|
||
fetch_timeout_ms: 200,
|
||
max_expanded_nodes: 10_000,
|
||
})
|
||
.unwrap()
|
||
.with_outbound_http_policy(outbound_policy)
|
||
.build();
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let started = tokio::time::Instant::now();
|
||
let preview = service
|
||
.preview_openapi_import(
|
||
&workspace_id,
|
||
OpenApiUpload {
|
||
bytes: format!(
|
||
r#"
|
||
openapi: 3.1.0
|
||
info: {{ title: Timed chain }}
|
||
servers: [{{ url: https://api.example.test }}]
|
||
paths:
|
||
/items:
|
||
get:
|
||
operationId: timedItems
|
||
responses:
|
||
'200':
|
||
description: ok
|
||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/root.yaml#/Item' }} }} }}
|
||
"#
|
||
)
|
||
.into_bytes(),
|
||
mime_type: "application/yaml".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
timeout_server.abort();
|
||
assert!(started.elapsed() < std::time::Duration::from_millis(800));
|
||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||
let job = registry
|
||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||
.await
|
||
.unwrap()
|
||
.unwrap();
|
||
assert_eq!(
|
||
job.preview_payload["dependencies"]
|
||
.as_array()
|
||
.unwrap()
|
||
.len(),
|
||
0
|
||
);
|
||
|
||
for _ in 0..100 {
|
||
let active_dependencies: i64 = sqlx::query_scalar(
|
||
"select count(*) from artifact_sources
|
||
where source_id like 'src_openapi_dep_%' and lifecycle = 'active'",
|
||
)
|
||
.fetch_one(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
if active_dependencies == 0 {
|
||
break;
|
||
}
|
||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||
}
|
||
let detached_dependencies: i64 = sqlx::query_scalar(
|
||
"select count(*) from artifact_sources
|
||
where source_id like 'src_openapi_dep_%' and lifecycle = 'detached'",
|
||
)
|
||
.fetch_one(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(
|
||
detached_dependencies, 0,
|
||
"staged dependencies must not be persisted before the fetch chain completes"
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn expired_job_detaches_primary_and_external_dependency_without_orphan() {
|
||
let app = Router::new().route(
|
||
"/schemas.yaml",
|
||
get(|| async { "Item: { type: object, properties: { id: { type: string } } }" }),
|
||
);
|
||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||
let address = listener.local_addr().unwrap();
|
||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||
let origin = format!("http://{address}");
|
||
let registry = test_registry().await;
|
||
let service = test_service_with_external_references(
|
||
registry.clone(),
|
||
test_storage_root("openapi_import_external_expiry"),
|
||
vec![format!("{origin}/")],
|
||
);
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let preview = service
|
||
.preview_openapi_import(
|
||
&workspace_id,
|
||
OpenApiUpload {
|
||
bytes: format!(
|
||
r#"
|
||
openapi: 3.1.0
|
||
info: {{ title: Expiring external }}
|
||
servers: [{{ url: https://api.example.test }}]
|
||
paths:
|
||
/items:
|
||
get:
|
||
operationId: listItems
|
||
responses:
|
||
'200':
|
||
description: ok
|
||
content: {{ application/json: {{ schema: {{ $ref: '{origin}/schemas.yaml#/Item' }} }} }}
|
||
"#
|
||
)
|
||
.into_bytes(),
|
||
mime_type: "application/yaml".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
sqlx::query("update import_jobs set expires_at = now() - interval '1 second' where id = $1")
|
||
.bind(preview.job_id.as_str())
|
||
.execute(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
let report = registry.cleanup_expired_import_jobs(16).await.unwrap();
|
||
assert_eq!(report.deleted_jobs, 1);
|
||
assert_eq!(report.detached_sources, 2);
|
||
let active_sources: i64 = sqlx::query_scalar(
|
||
"select count(*) from artifact_sources
|
||
where source_id like 'src_openapi_%' and lifecycle = 'active'",
|
||
)
|
||
.fetch_one(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(active_sources, 0);
|
||
}
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn exact_v2_contract_replays_persisted_preview_without_v3_or_dependency_reads() {
|
||
let registry = test_registry().await;
|
||
let service = test_service(
|
||
registry.clone(),
|
||
test_storage_root("openapi_import_v2_compatibility"),
|
||
test_auth_settings(),
|
||
test_secret_crypto(),
|
||
);
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let preview = service
|
||
.preview_openapi_import(&workspace_id, openapi_upload())
|
||
.await
|
||
.unwrap();
|
||
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||
let missing_dependency = serde_json::json!([{
|
||
"source_id": "src_openapi_dep_v2_must_not_be_read",
|
||
"digest": format!("sha256:{}", "0".repeat(64)),
|
||
"canonical_uri": "https://schemas.example.test/v2.yaml"
|
||
}]);
|
||
sqlx::query(
|
||
"update import_jobs
|
||
set preview_payload = jsonb_set(
|
||
jsonb_set(
|
||
jsonb_set(
|
||
jsonb_set(
|
||
preview_payload,
|
||
'{normalization,normalizer_version}',
|
||
to_jsonb('normalized-ir-v2'::text)
|
||
),
|
||
'{normalization,projection_version}',
|
||
to_jsonb('preview-v2'::text)
|
||
),
|
||
'{normalization,ir_fingerprint}',
|
||
to_jsonb($1::text)
|
||
),
|
||
'{dependency_snapshots}',
|
||
$2::jsonb
|
||
)
|
||
where id = $3",
|
||
)
|
||
.bind("a".repeat(64))
|
||
.bind(missing_dependency)
|
||
.bind(job_id.as_str())
|
||
.execute(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
|
||
let result = service
|
||
.create_openapi_import(
|
||
&workspace_id,
|
||
&job_id,
|
||
OpenApiImportCreatePayload {
|
||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||
conflict_mode: "skip".to_owned(),
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(result.created.len(), 1);
|
||
let completed = registry
|
||
.get_import_job(&workspace_id, &job_id)
|
||
.await
|
||
.unwrap()
|
||
.unwrap();
|
||
assert_eq!(completed.status, ImportJobStatus::Completed);
|
||
}
|
||
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn exact_v3_contract_replays_persisted_preview_without_v4_or_dependency_reads() {
|
||
let registry = test_registry().await;
|
||
let service = test_service(
|
||
registry.clone(),
|
||
test_storage_root("openapi_import_v3_compatibility"),
|
||
test_auth_settings(),
|
||
test_secret_crypto(),
|
||
);
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let preview = service
|
||
.preview_openapi_import(&workspace_id, openapi_upload())
|
||
.await
|
||
.unwrap();
|
||
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||
let missing_dependency = serde_json::json!([{
|
||
"source_id": "src_openapi_dep_v3_must_not_be_read",
|
||
"digest": format!("sha256:{}", "0".repeat(64)),
|
||
"canonical_uri": "https://schemas.example.test/v3.yaml"
|
||
}]);
|
||
sqlx::query(
|
||
"update import_jobs
|
||
set preview_payload = jsonb_set(
|
||
jsonb_set(
|
||
jsonb_set(
|
||
jsonb_set(
|
||
preview_payload,
|
||
'{normalization,normalizer_version}',
|
||
to_jsonb('normalized-ir-v3'::text)
|
||
),
|
||
'{normalization,projection_version}',
|
||
to_jsonb('preview-v3'::text)
|
||
),
|
||
'{normalization,ir_fingerprint}',
|
||
to_jsonb($1::text)
|
||
),
|
||
'{dependency_snapshots}',
|
||
$2::jsonb
|
||
)
|
||
where id = $3",
|
||
)
|
||
.bind("a".repeat(64))
|
||
.bind(missing_dependency)
|
||
.bind(job_id.as_str())
|
||
.execute(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
|
||
let result = service
|
||
.create_openapi_import(
|
||
&workspace_id,
|
||
&job_id,
|
||
OpenApiImportCreatePayload {
|
||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||
conflict_mode: "skip".to_owned(),
|
||
},
|
||
)
|
||
.await
|
||
.unwrap();
|
||
assert_eq!(result.created.len(), 1);
|
||
let completed = registry
|
||
.get_import_job(&workspace_id, &job_id)
|
||
.await
|
||
.unwrap()
|
||
.unwrap();
|
||
assert_eq!(completed.status, ImportJobStatus::Completed);
|
||
}
|
||
|
||
#[tokio::test]
|
||
#[serial]
|
||
async fn exact_v4_security_contract_replays_its_persisted_security_classification() {
|
||
let registry = test_registry().await;
|
||
let service = test_service(
|
||
registry.clone(),
|
||
test_storage_root("openapi_import_v4_security_compatibility"),
|
||
test_auth_settings(),
|
||
test_secret_crypto(),
|
||
);
|
||
let workspace_id = WorkspaceId::new("ws_default");
|
||
let upload = OpenApiUpload {
|
||
bytes: br#"{
|
||
"openapi":"3.0.3",
|
||
"info":{"title":"Historical security","version":"1"},
|
||
"servers":[{"url":"https://api.example.test"}],
|
||
"components":{"securitySchemes":{"basic":{"type":"http","scheme":"basic"}}},
|
||
"security":[{"basic":[]}],
|
||
"paths":{
|
||
"/secure":{"get":{"operationId":"secure","responses":{"200":{"description":"ok"}}}},
|
||
"/public":{"get":{"operationId":"public","security":[],"responses":{"200":{"description":"ok"}}}}
|
||
}
|
||
}"#
|
||
.to_vec(),
|
||
mime_type: "application/json".to_owned(),
|
||
locale: OpenApiUploadLocale::En,
|
||
};
|
||
let preview = service
|
||
.preview_openapi_import(&workspace_id, upload)
|
||
.await
|
||
.unwrap();
|
||
let job_id: crank_registry::ImportJobId = preview.job_id.as_str().into();
|
||
let job = registry
|
||
.get_import_job(&workspace_id, &job_id)
|
||
.await
|
||
.unwrap()
|
||
.unwrap();
|
||
// Keep the genuine baseline-v4 normalization/projection, preview and digest intact.
|
||
// Only bind its source envelope to this test's newly registered immutable source blob.
|
||
let mut payload: serde_json::Value =
|
||
serde_json::from_str(BASELINE_V4_SECURITY_PREVIEW_PAYLOAD).unwrap();
|
||
assert_eq!(
|
||
payload.pointer("/source/digest"),
|
||
job.preview_payload.pointer("/source/digest"),
|
||
"baseline fixture must refer to the exact uploaded source bytes",
|
||
);
|
||
payload["source"]["source_id"] = job.preview_payload["source"]["source_id"].clone();
|
||
assert_eq!(
|
||
payload["normalization"]["normalizer_version"],
|
||
"normalized-ir-v4"
|
||
);
|
||
assert_eq!(payload["normalization"]["projection_version"], "preview-v4");
|
||
sqlx::query("update import_jobs set preview_payload = $1::jsonb where id = $2")
|
||
.bind(serde_json::to_string(&payload).unwrap())
|
||
.bind(job_id.as_str())
|
||
.execute(registry.pool())
|
||
.await
|
||
.unwrap();
|
||
|
||
let result = service
|
||
.create_openapi_import(
|
||
&workspace_id,
|
||
&job_id,
|
||
OpenApiImportCreatePayload {
|
||
selected_operation_keys: vec!["GET /secure".to_owned()],
|
||
server_url: Some("https://api.example.test".to_owned()),
|
||
conflict_mode: "skip".to_owned(),
|
||
},
|
||
)
|
||
.await;
|
||
let error =
|
||
result.expect_err("historical v4 security blocker must refuse the selected candidate");
|
||
assert!(
|
||
format!("{error:?}").contains("invalid_document"),
|
||
"v4 contract was rejected before candidate validation: {error:?}"
|
||
);
|
||
assert!(
|
||
service
|
||
.list_operations(&workspace_id)
|
||
.await
|
||
.unwrap()
|
||
.is_empty()
|
||
);
|
||
}
|
||
|
||
mod security_bundle;
|