fix(import): подтвердить обязательные гарантии второго эпика
This commit is contained in:
1 parent
31f6f00d3b
commit
ad23cd9a46
33 files changed
+3328
-18
No files matched your search
@@ -1,11 +1,17 @@
|
||||
use super::super::common::test_service_with_external_references;
|
||||
mod isolation;
|
||||
use super::*;
|
||||
use axum::{Router, routing::get};
|
||||
use admin_api::error::ApiError;
|
||||
use axum::{Router, body::to_bytes, http::StatusCode, response::IntoResponse, routing::get};
|
||||
use crank_artifacts::ArtifactStore;
|
||||
use crank_core::{Workspace, WorkspaceStatus};
|
||||
use crank_registry::{ArtifactSourceId, CreateWorkspaceRequest, RegistryError};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
// Serialized by the integration harness at baseline f5f40cb167a82b1e3c1ea2475db5cbaed9d9abfa; this v4 payload preserves the then-current security blocker and digest.
|
||||
const BASELINE_V4_SECURITY_PREVIEW_PAYLOAD: &str = r###"{"dependencies":[],"dependency_snapshots":[],"normalization":{"config":{"external_references_enabled":false,"max_aliases":128,"max_bytes":262144,"max_collection_items":10000,"max_depth":64,"max_expanded_nodes":100000,"max_external_document_bytes":262144,"max_nodes":50000,"max_reference_depth":32,"max_reference_documents":32,"max_references":4096,"max_scalar_bytes":262144,"normalizer_version":"normalized-ir-v4","projection_version":"preview-v4"},"ir_fingerprint":"ae2319de76c13d631b26843623092d966e5deea7cda9e3e40e4acc6fa36a25a8","normalizer_version":"normalized-ir-v4","projection_version":"preview-v4"},"preview":{"findings":[{"code":"unsupported_security_requirement","message":"Требование безопасности должно быть привязано к настроенному Auth Profile перед публикацией этой операции; остальные операции не блокируются.","operation_key":"GET /secure","severity":"error"}],"groups":[{"key":"imported_operation","operations":[{"category":"imported","description":"Выполняет GET /public","draft":{"category":"imported","display_name":"Public","input_mapping":{},"input_schema":{"description":"Входные параметры MCP-инструмента","nullable":false,"required":true,"type":"object"},"name":"public","output_mapping":{"rules":[{"required":true,"source":"$.response.body","target":"$.output"}]},"output_schema":{"description":"Успешный ответ без тела","nullable":false,"required":true,"type":"null"},"target":{"base_url":"https://api.example.test","method":"GET","path_template":"/public","response_contract":[{"schema_variant":0,"status_code":200}]},"tool_description":{"description":"Выполняет GET /public","examples":[{"input":{}}],"title":"Public"},"wizard_state":{}},"findings":[{"code":"missing_summary","message":"У метода нет summary, отображаемое имя будет сгенерировано автоматически.","operation_key":"GET /public","severity":"warning"},{"code":"missing_description","message":"У метода нет description. Перед публикацией лучше описать, когда агенту стоит вызывать этот инструмент.","operation_key":"GET /public","severity":"warning"},{"code":"missing_response_schema","message":"У метода не найдена схема успешного ответа, результат будет описан как общий объект.","operation_key":"GET /public","severity":"warning"},{"code":"no_input_fields","message":"У инструмента нет входных параметров. Это нормально для справочных методов, но проверьте, что агенту не нужно передавать фильтры.","operation_key":"GET /public","severity":"info"},{"code":"empty_output_schema","message":"В ответе не найдено отдельных полей. Перед публикацией проверьте схему ответа и маппинг результата.","operation_key":"GET /public","severity":"warning"},{"code":"weak_tool_description","message":"Описание инструмента слишком короткое или техническое. Перед публикацией добавьте, когда агент должен вызывать инструмент и что будет в успешном ответе.","operation_key":"GET /public","severity":"warning"},{"code":"weak_tool_name","message":"Имя инструмента `public` выглядит слишком общим. Лучше использовать имя с конкретным действием и объектом.","operation_key":"GET /public","severity":"warning"}],"input_fields":0,"key":"GET /public","method":"GET","operation_id":"public","output_fields":0,"path":"/public","server_urls":["https://api.example.test"],"suggested_display_name":"Public","suggested_name":"public"},{"category":"imported","description":"Выполняет GET /secure","draft":{"category":"imported","display_name":"Secure","input_mapping":{},"input_schema":{"description":"Входные параметры MCP-инструмента","nullable":false,"required":true,"type":"object"},"name":"secure","output_mapping":{"rules":[{"required":true,"source":"$.response.body","target":"$.output"}]},"output_schema":{"description":"Успешный ответ без тела","nullable":false,"required":true,"type":"null"},"target":{"base_url":"https://api.example.test","method":"GET","path_template":"/secure","response_contract":[{"schema_variant":0,"status_code":200}]},"tool_description":{"description":"Выполняет GET /secure","examples":[{"input":{}}],"title":"Secure"},"wizard_state":{}},"findings":[{"code":"missing_summary","message":"У мет
Line truncated
|
||||
@@ -43,12 +49,14 @@ async fn external_relative_chain_survives_reversed_builder_order_and_apply_never
|
||||
let origin = format!("http://{address}");
|
||||
|
||||
let registry = test_registry().await;
|
||||
let storage_root = test_storage_root("openapi_import_external_snapshot");
|
||||
let service = test_service_with_external_references(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_external_snapshot"),
|
||||
storage_root.clone(),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let foreign_workspace_id = create_workspace(®istry, "ws_external_snapshot_foreign").await;
|
||||
let upload = OpenApiUpload {
|
||||
bytes: format!(
|
||||
r#"
|
||||
@@ -97,6 +105,60 @@ paths:
|
||||
2
|
||||
);
|
||||
|
||||
// The preview response contains only the normalized result. Ownership
|
||||
// envelopes, storage digests, and remote origins stay in the persisted
|
||||
// job payload and are verified through the workspace-scoped registry API.
|
||||
let public_preview = serde_json::to_string(&preview).unwrap();
|
||||
assert!(!public_preview.contains("src_openapi_dep_"));
|
||||
assert!(!public_preview.contains(&origin));
|
||||
let dependencies = job.preview_payload["dependencies"].as_array().unwrap();
|
||||
let store = Arc::new(ArtifactStore::open(&storage_root).unwrap());
|
||||
for dependency in dependencies {
|
||||
let source_id = ArtifactSourceId::new(dependency["source_id"].as_str().unwrap());
|
||||
let digest = dependency["digest"].as_str().unwrap();
|
||||
let canonical_uri = dependency["canonical_uri"].as_str().unwrap();
|
||||
let source = registry
|
||||
.get_artifact_source(&workspace_id, &source_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(source.workspace_id, workspace_id);
|
||||
assert_eq!(source.blob.artifact_ref.as_str(), digest);
|
||||
let verified = registry
|
||||
.read_artifact_source(Arc::clone(&store), &workspace_id, &source_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(verified.source.blob.artifact_ref.as_str(), digest);
|
||||
let expected_bytes = if canonical_uri.ends_with("/root.yaml") {
|
||||
b"Item: { $ref: './child.yaml#/Item' }".as_slice()
|
||||
} else {
|
||||
assert!(canonical_uri.ends_with("/child.yaml"));
|
||||
b"Item: { type: object, required: [id], properties: { id: { type: string } } }"
|
||||
.as_slice()
|
||||
};
|
||||
assert_eq!(verified.bytes, expected_bytes);
|
||||
|
||||
let foreign_read = registry
|
||||
.read_artifact_source(Arc::clone(&store), &foreign_workspace_id, &source_id)
|
||||
.await
|
||||
.expect_err("foreign workspace must not read an owned snapshot");
|
||||
let unknown_source_id = ArtifactSourceId::new("src_unknown_scope_probe");
|
||||
let unknown_read = registry
|
||||
.read_artifact_source(
|
||||
Arc::clone(&store),
|
||||
&foreign_workspace_id,
|
||||
&unknown_source_id,
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown source must have the same scoped outcome");
|
||||
let foreign_body = source_not_found_body(foreign_read).await;
|
||||
let unknown_body = source_not_found_body(unknown_read).await;
|
||||
assert_eq!(foreign_body, unknown_body);
|
||||
let rendered_body = foreign_body.to_string();
|
||||
assert!(!rendered_body.contains(digest));
|
||||
assert!(!rendered_body.contains(canonical_uri));
|
||||
assert!(!rendered_body.contains(source_id.as_str()));
|
||||
}
|
||||
|
||||
let applied = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
@@ -122,6 +184,59 @@ paths:
|
||||
assert_eq!(active_sources, 0);
|
||||
}
|
||||
|
||||
fn external_upload(origin: &str) -> OpenApiUpload {
|
||||
OpenApiUpload {
|
||||
bytes: format!(
|
||||
r#"
|
||||
openapi: 3.1.0
|
||||
info: {{ title: Scoped external snapshot }}
|
||||
servers: [{{ url: https://api.example.test }}]
|
||||
paths:
|
||||
/items:
|
||||
get:
|
||||
operationId: listItems
|
||||
responses:
|
||||
'200':
|
||||
description: ok
|
||||
content:
|
||||
application/json:
|
||||
schema: {{ $ref: '{origin}/schema.yaml#/Item' }}
|
||||
"#
|
||||
)
|
||||
.into_bytes(),
|
||||
mime_type: "application/yaml".to_owned(),
|
||||
locale: OpenApiUploadLocale::En,
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_workspace(registry: &crank_registry::PostgresRegistry, id: &str) -> WorkspaceId {
|
||||
let workspace_id = WorkspaceId::new(id);
|
||||
let now = OffsetDateTime::now_utc();
|
||||
registry
|
||||
.create_workspace(CreateWorkspaceRequest {
|
||||
workspace: &Workspace {
|
||||
id: workspace_id.clone(),
|
||||
slug: id.replace('_', "-"),
|
||||
display_name: id.to_owned(),
|
||||
status: WorkspaceStatus::Active,
|
||||
settings: serde_json::json!({}),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
},
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
workspace_id
|
||||
}
|
||||
|
||||
async fn source_not_found_body(error: RegistryError) -> serde_json::Value {
|
||||
assert!(matches!(&error, RegistryError::SourceNotFound { .. }));
|
||||
let response = ApiError::from(error).into_response();
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
|
||||
serde_json::from_slice(&body).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn external_secret_document_is_rejected_before_blob_registration() {
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
use super::*;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn concurrent_external_previews_keep_snapshot_ownership_workspace_scoped() {
|
||||
let fetches = Arc::new(AtomicUsize::new(0));
|
||||
let fetch_count = Arc::clone(&fetches);
|
||||
let app = Router::new().route(
|
||||
"/schema.yaml",
|
||||
get(move || {
|
||||
let fetch_count = Arc::clone(&fetch_count);
|
||||
async move {
|
||||
fetch_count.fetch_add(1, Ordering::SeqCst);
|
||||
"Item: { type: object, properties: { id: { type: string } } }"
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
let registry = test_registry().await;
|
||||
let storage_root = test_storage_root("openapi_import_external_workspace_isolation");
|
||||
let service = test_service_with_external_references(
|
||||
registry.clone(),
|
||||
storage_root.clone(),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let workspace_a = create_workspace(®istry, "ws_external_concurrent_a").await;
|
||||
let workspace_b = create_workspace(®istry, "ws_external_concurrent_b").await;
|
||||
|
||||
let (preview_a, preview_b) = tokio::join!(
|
||||
service.preview_openapi_import(&workspace_a, external_upload(&origin)),
|
||||
service.preview_openapi_import(&workspace_b, external_upload(&origin)),
|
||||
);
|
||||
let preview_a = preview_a.unwrap();
|
||||
let preview_b = preview_b.unwrap();
|
||||
server.abort();
|
||||
assert_eq!(
|
||||
fetches.load(Ordering::SeqCst),
|
||||
2,
|
||||
"each workspace fetches its own preview snapshot"
|
||||
);
|
||||
assert_ne!(preview_a.job_id, preview_b.job_id);
|
||||
|
||||
let job_a = registry
|
||||
.get_import_job(&workspace_a, &preview_a.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let job_b = registry
|
||||
.get_import_job(&workspace_b, &preview_b.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let dependency_a = &job_a.preview_payload["dependencies"][0];
|
||||
let dependency_b = &job_b.preview_payload["dependencies"][0];
|
||||
let source_a = ArtifactSourceId::new(dependency_a["source_id"].as_str().unwrap());
|
||||
let source_b = ArtifactSourceId::new(dependency_b["source_id"].as_str().unwrap());
|
||||
let digest_a = dependency_a["digest"].as_str().unwrap();
|
||||
let digest_b = dependency_b["digest"].as_str().unwrap();
|
||||
let canonical_uri = dependency_a["canonical_uri"].as_str().unwrap();
|
||||
assert_eq!(
|
||||
digest_a, digest_b,
|
||||
"identical bytes may share the immutable blob"
|
||||
);
|
||||
assert_eq!(dependency_b["canonical_uri"], dependency_a["canonical_uri"]);
|
||||
assert_ne!(
|
||||
source_a, source_b,
|
||||
"each workspace owns a distinct source relation"
|
||||
);
|
||||
|
||||
let public_a = serde_json::to_string(&preview_a).unwrap();
|
||||
let public_b = serde_json::to_string(&preview_b).unwrap();
|
||||
for response in [&public_a, &public_b] {
|
||||
assert!(!response.contains("src_openapi_dep_"));
|
||||
assert!(!response.contains(digest_a));
|
||||
assert!(!response.contains(canonical_uri));
|
||||
}
|
||||
|
||||
let store = Arc::new(ArtifactStore::open(&storage_root).unwrap());
|
||||
for (workspace_id, source_id) in [(&workspace_a, &source_a), (&workspace_b, &source_b)] {
|
||||
let source = registry
|
||||
.get_artifact_source(workspace_id, source_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(source.workspace_id, *workspace_id);
|
||||
assert_eq!(source.blob.artifact_ref.as_str(), digest_a);
|
||||
let verified = registry
|
||||
.read_artifact_source(Arc::clone(&store), workspace_id, source_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
verified.bytes,
|
||||
b"Item: { type: object, properties: { id: { type: string } } }"
|
||||
);
|
||||
}
|
||||
|
||||
let foreign_body = source_not_found_body(
|
||||
registry
|
||||
.read_artifact_source(Arc::clone(&store), &workspace_b, &source_a)
|
||||
.await
|
||||
.expect_err("workspace B cannot resolve workspace A source"),
|
||||
)
|
||||
.await;
|
||||
let unknown_source = ArtifactSourceId::new("src_unknown_workspace_probe");
|
||||
let unknown_body = source_not_found_body(
|
||||
registry
|
||||
.read_artifact_source(Arc::clone(&store), &workspace_b, &unknown_source)
|
||||
.await
|
||||
.expect_err("unknown source is indistinguishable from a foreign source"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(foreign_body, unknown_body);
|
||||
let response = foreign_body.to_string();
|
||||
assert!(!response.contains(digest_a));
|
||||
assert!(!response.contains(canonical_uri));
|
||||
assert!(!response.contains(source_a.as_str()));
|
||||
assert!(!response.contains("exists"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn existing_foreign_snapshot_does_not_short_circuit_second_workspace_fetch() {
|
||||
let fetches = Arc::new(AtomicUsize::new(0));
|
||||
let fetch_count = Arc::clone(&fetches);
|
||||
let second_fetch_entered = Arc::new(Notify::new());
|
||||
let entered = Arc::clone(&second_fetch_entered);
|
||||
let release_second_fetch = Arc::new(Notify::new());
|
||||
let release = Arc::clone(&release_second_fetch);
|
||||
let app = Router::new().route(
|
||||
"/schema.yaml",
|
||||
get(move || {
|
||||
let fetch_count = Arc::clone(&fetch_count);
|
||||
let entered = Arc::clone(&entered);
|
||||
let release = Arc::clone(&release);
|
||||
async move {
|
||||
let request_number = fetch_count.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if request_number == 2 {
|
||||
entered.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
"Item: { type: object, properties: { id: { type: string } } }"
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
let registry = test_registry().await;
|
||||
let service = test_service_with_external_references(
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import_external_fetch_oracle"),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let workspace_a = create_workspace(®istry, "ws_external_oracle_a").await;
|
||||
let workspace_b = create_workspace(®istry, "ws_external_oracle_b").await;
|
||||
|
||||
let preview_a = service
|
||||
.preview_openapi_import(&workspace_a, external_upload(&origin))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(fetches.load(Ordering::SeqCst), 1);
|
||||
|
||||
let preview_b_origin = origin.clone();
|
||||
let preview_b_workspace = workspace_b.clone();
|
||||
let mut preview_b_task = tokio::spawn(async move {
|
||||
service
|
||||
.preview_openapi_import(&preview_b_workspace, external_upload(&preview_b_origin))
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), second_fetch_entered.notified())
|
||||
.await
|
||||
.expect("the second preview must reach its HTTP fetch");
|
||||
|
||||
// The fixture holds the real HTTP response. Poll the preview once after
|
||||
// observing that fetch; it must remain pending until the fixture releases it.
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut preview_b_task => panic!("second preview bypassed its held fetch: {result:?}"),
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||
release_second_fetch.notify_one();
|
||||
let preview_b = tokio::time::timeout(Duration::from_secs(10), &mut preview_b_task)
|
||||
.await
|
||||
.expect("the released second preview should complete")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
server.abort();
|
||||
|
||||
let job_a = registry
|
||||
.get_import_job(&workspace_a, &preview_a.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let job_b = registry
|
||||
.get_import_job(&workspace_b, &preview_b.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let source_a = ArtifactSourceId::new(
|
||||
job_a.preview_payload["dependencies"][0]["source_id"]
|
||||
.as_str()
|
||||
.unwrap(),
|
||||
);
|
||||
let source_b = ArtifactSourceId::new(
|
||||
job_b.preview_payload["dependencies"][0]["source_id"]
|
||||
.as_str()
|
||||
.unwrap(),
|
||||
);
|
||||
assert_ne!(source_a, source_b);
|
||||
assert_eq!(fetches.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn file_origin_and_relative_chain_are_blocked_without_file_or_network_access() {
|
||||
const PRIVATE_FILE_BYTES: &str = "PrivateCanary: file-origin-secret-must-never-be-read";
|
||||
const HYPOTHETICAL_FILE_ROOT: &str = "Item: { $ref: './private/canary.yaml#/Secret' }";
|
||||
let file_fixture_root = test_storage_root("openapi_import_file_origin_fixture");
|
||||
let private_directory = file_fixture_root.join("private");
|
||||
std::fs::create_dir_all(&private_directory).unwrap();
|
||||
std::fs::write(file_fixture_root.join("root.yaml"), HYPOTHETICAL_FILE_ROOT).unwrap();
|
||||
std::fs::write(
|
||||
private_directory.join("canary.yaml"),
|
||||
format!("Secret: {{ type: string, example: '{PRIVATE_FILE_BYTES}' }}"),
|
||||
)
|
||||
.unwrap();
|
||||
let file_origin = url::Url::from_file_path(file_fixture_root.join("root.yaml")).unwrap();
|
||||
let file_origin_for_response = file_origin.clone();
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let request_count = Arc::clone(&requests);
|
||||
let app = Router::new().route(
|
||||
"/schema.yaml",
|
||||
get(move || {
|
||||
let request_count = Arc::clone(&request_count);
|
||||
async move {
|
||||
request_count.fetch_add(1, Ordering::SeqCst);
|
||||
format!("Item: {{ $ref: '{file_origin_for_response}#/Item' }}")
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let origin = format!("http://{address}");
|
||||
let registry = test_registry().await;
|
||||
let service = test_service_with_external_references(
|
||||
registry,
|
||||
test_storage_root("openapi_import_file_origin_blocked"),
|
||||
vec![format!("{origin}/")],
|
||||
);
|
||||
let preview = service
|
||||
.preview_openapi_import(&WorkspaceId::new("ws_default"), external_upload(&origin))
|
||||
.await
|
||||
.unwrap();
|
||||
server.abort();
|
||||
|
||||
assert_eq!(
|
||||
requests.load(Ordering::SeqCst),
|
||||
1,
|
||||
"only the explicitly allowlisted HTTP root is fetched"
|
||||
);
|
||||
let findings = preview
|
||||
.preview
|
||||
.groups
|
||||
.iter()
|
||||
.flat_map(|group| &group.operations)
|
||||
.flat_map(|operation| &operation.findings)
|
||||
.chain(preview.preview.findings.iter())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
findings.iter().any(|finding| {
|
||||
finding.severity == crank_import::rest::ImportFindingSeverity::Error
|
||||
&& finding.code == "reference_target_missing"
|
||||
}),
|
||||
"file origin must produce a blocking unresolved-reference finding"
|
||||
);
|
||||
let report = serde_json::to_string(&preview).unwrap();
|
||||
assert!(!report.contains(file_origin.as_str()));
|
||||
assert!(!report.contains(PRIVATE_FILE_BYTES));
|
||||
assert!(!report.contains(HYPOTHETICAL_FILE_ROOT));
|
||||
}
|
||||
@@ -32,6 +32,9 @@ impl MachineCredentialVerifier for CorpusMcpCredentialVerifier {
|
||||
|
||||
use super::super::common::*;
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
mod crash_recovery;
|
||||
|
||||
const OPENAPI_SINGLE: &str = r#"
|
||||
openapi: 3.0.3
|
||||
info: { title: Apply test }
|
||||
@@ -98,6 +101,7 @@ paths:
|
||||
responses: { '200': { description: ok } }
|
||||
"#;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CreatedPlan {
|
||||
reference: String,
|
||||
digest: String,
|
||||
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Kill the actual HTTP-serving process at durable Apply boundaries, then
|
||||
//! restart the same production router against the parent's database/storage.
|
||||
|
||||
use super::*;
|
||||
use crank_artifacts::test_support::{FaultAction, set_checkpoint};
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
process::{Child, Command, Stdio},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const CHILD_TEST: &str =
|
||||
"integration::openapi_import::import_plan_apply::crash_recovery::http_apply_crash_child";
|
||||
const CHILD_CONFIG: &str = "CRANK_TEST_APPLY_CRASH_CONFIG";
|
||||
|
||||
struct PrivateConfigGuard(PathBuf);
|
||||
impl Drop for PrivateConfigGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
struct ChildGuard(Child);
|
||||
impl Drop for ChildGuard {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
// Child configuration contains only isolated test fixture credentials. Keep it
|
||||
// in a private file and never include it in assertions, stdout or task logs.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn http_apply_crash_child() {
|
||||
let Ok(path) = std::env::var(CHILD_CONFIG) else {
|
||||
return;
|
||||
};
|
||||
let config: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();
|
||||
let registry = crank_registry::PostgresRegistry::connect(config["database"].as_str().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
let root = PathBuf::from(config["storage"].as_str().unwrap());
|
||||
let server = spawn_admin_api(build_test_app(registry, root)).await;
|
||||
set_checkpoint(config["stage"].as_str().unwrap(), FaultAction::Exit);
|
||||
std::fs::write(config["ready"].as_str().unwrap(), server.as_ref()).unwrap();
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
|
||||
async fn counts(registry: &crank_registry::PostgresRegistry) -> (i64, i64, i64) {
|
||||
sqlx::query_as(
|
||||
"select (select count(*) from operations),
|
||||
(select count(*) from operation_versions),
|
||||
(select count(*) from import_plan_applications)",
|
||||
)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn crash_then_restart(stage: &str, committed: bool) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let (registry, database_url) = test_registry_with_single_connection().await;
|
||||
let root = test_storage_root(stage);
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
let server = spawn_admin_api(build_test_app(registry.clone(), root.clone())).await;
|
||||
let client = authorized_client(&server).await;
|
||||
let plan = create_plan(&client, &server, OPENAPI_SINGLE, full_plan()).await;
|
||||
let store = crank_artifacts::ArtifactStore::open(&root).unwrap();
|
||||
let artifact = store.put_registered(OPENAPI_SINGLE.as_bytes()).unwrap();
|
||||
let key = format!("crash-{stage}");
|
||||
assert_eq!(counts(®istry).await, (0, 0, 0));
|
||||
drop(server);
|
||||
|
||||
let ready = root.join("child.ready");
|
||||
let config_path = root.join("child-config.json");
|
||||
let _private_config = PrivateConfigGuard(config_path.clone());
|
||||
{
|
||||
use std::io::Write;
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(&config_path)
|
||||
.unwrap();
|
||||
let config = json!({"database":database_url,"storage":root,"stage":stage,"ready":ready});
|
||||
file.write_all(&serde_json::to_vec(&config).unwrap())
|
||||
.unwrap();
|
||||
}
|
||||
let mut child = ChildGuard(
|
||||
Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", CHILD_TEST, "--nocapture"])
|
||||
.env(CHILD_CONFIG, &config_path)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap(),
|
||||
);
|
||||
let start = Instant::now();
|
||||
let child_url = loop {
|
||||
if let Ok(url) = std::fs::read_to_string(&ready) {
|
||||
break url;
|
||||
}
|
||||
assert!(
|
||||
child.0.try_wait().unwrap().is_none(),
|
||||
"child exited before HTTP readiness"
|
||||
);
|
||||
assert!(
|
||||
start.elapsed() < Duration::from_secs(30),
|
||||
"child HTTP readiness deadline"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
let response = client
|
||||
.post(format!(
|
||||
"{child_url}/imports/openapi/{}/apply",
|
||||
plan.reference
|
||||
))
|
||||
.json(&json!({"plan_digest":plan.digest,"idempotency_key":key}))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.send()
|
||||
.await;
|
||||
assert!(response.is_err(), "crash must prevent an HTTP response");
|
||||
let status = loop {
|
||||
if let Some(status) = child.0.try_wait().unwrap() {
|
||||
break status;
|
||||
}
|
||||
assert!(
|
||||
start.elapsed() < Duration::from_secs(35),
|
||||
"child crash deadline"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
assert_eq!(
|
||||
status.code(),
|
||||
Some(86),
|
||||
"must hit the exact armed commit boundary"
|
||||
);
|
||||
std::fs::remove_file(config_path).unwrap();
|
||||
std::fs::remove_file(ready).unwrap();
|
||||
let expected = if committed { (1, 1, 1) } else { (0, 0, 0) };
|
||||
// An independent connection can observe only fully committed state.
|
||||
assert_eq!(counts(®istry).await, expected);
|
||||
let before: Option<Value> = sqlx::query_scalar(
|
||||
"select report from import_plan_applications where workspace_id='ws_default' and idempotency_key=$1",
|
||||
).bind(&key).fetch_optional(registry.pool()).await.unwrap();
|
||||
assert_eq!(before.is_some(), committed);
|
||||
|
||||
// The source of this nonexpired job remains referenced after process loss;
|
||||
// cleanup cannot make its plan snapshot eligible for orphan deletion.
|
||||
assert_eq!(sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from artifact_sources where workspace_id='ws_default' and lifecycle='active'",
|
||||
).fetch_one(registry.pool()).await.unwrap(), 1);
|
||||
let cleanup = registry.cleanup_expired_import_jobs(32).await;
|
||||
assert!(cleanup.is_ok());
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let claim = registry
|
||||
.claim_artifact_reconciliation(crank_registry::ClaimArtifactReconciliationRequest {
|
||||
artifact: &artifact,
|
||||
token: crank_registry::ArtifactClaimToken::generate(),
|
||||
claimed_at: now,
|
||||
lease_expires_at: now + time::Duration::minutes(5),
|
||||
detached_grace: time::Duration::hours(24),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
claim,
|
||||
crank_registry::ArtifactClaimOutcome::ActiveReference
|
||||
));
|
||||
assert_eq!(
|
||||
store.read(artifact.artifact_ref()).unwrap(),
|
||||
OPENAPI_SINGLE.as_bytes()
|
||||
);
|
||||
assert_eq!(sqlx::query_scalar::<_, i64>(
|
||||
"select count(*) from artifact_sources where workspace_id='ws_default' and lifecycle='active'",
|
||||
).fetch_one(registry.pool()).await.unwrap(), 1);
|
||||
|
||||
let restarted = spawn_admin_api(build_test_app(registry.clone(), root)).await;
|
||||
let (status, report) = apply_plan(&client, &restarted, &plan, &key).await;
|
||||
assert_eq!(status, reqwest::StatusCode::OK);
|
||||
if let Some(before) = before {
|
||||
assert_eq!(
|
||||
report, before,
|
||||
"restart must return the committed immutable report"
|
||||
);
|
||||
}
|
||||
assert_eq!(report["plan_digest"], plan.digest);
|
||||
assert_eq!(report["counts"]["created"], 1);
|
||||
let (status, replay) = apply_plan(&client, &restarted, &plan, &key).await;
|
||||
assert_eq!(status, reqwest::StatusCode::OK);
|
||||
assert_eq!(
|
||||
replay, report,
|
||||
"exact retry must be byte-equivalent structured report"
|
||||
);
|
||||
assert_eq!(counts(®istry).await, (1, 1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn http_process_crash_before_commit_rolls_back_and_retries_after_restart() {
|
||||
crash_then_restart("apply_before_commit", false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn http_process_crash_after_commit_before_response_replays_after_restart() {
|
||||
crash_then_restart("apply_after_commit", true).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn concurrent_distinct_overlapping_http_plans_have_one_coherent_commit() {
|
||||
let (seed_registry, database_url) = test_registry_with_single_connection().await;
|
||||
// The ordinary test registry has only one connection and would serialize
|
||||
// both requests before their transactions even start.
|
||||
let registry = crank_registry::PostgresRegistry::connect_with_options_and_pool_config(
|
||||
database_url
|
||||
.parse::<sqlx::postgres::PgConnectOptions>()
|
||||
.unwrap(),
|
||||
crank_registry::PostgresPoolConfig::try_new(4, 0, 5_000, 60_000, 1_800_000).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
drop(seed_registry);
|
||||
let server = spawn_admin_api(build_test_app(
|
||||
registry.clone(),
|
||||
test_storage_root("distinct-overlap-plans"),
|
||||
))
|
||||
.await;
|
||||
let client = authorized_client(&server).await;
|
||||
let a = create_plan(&client, &server, OPENAPI_SINGLE, full_plan()).await;
|
||||
let b = create_plan(&client, &server, OPENAPI_TWO_OPERATIONS, full_plan()).await;
|
||||
assert_ne!(a.reference, b.reference);
|
||||
assert_ne!(a.digest, b.digest);
|
||||
let a_key = "overlap-distinct-a";
|
||||
let b_key = "overlap-distinct-b";
|
||||
// Hold the production workspace row lock until BOTH independent Apply
|
||||
// transactions are blocked on it. Merely joining requests could otherwise
|
||||
// exercise two sequential transactions.
|
||||
let mut gate = registry.pool().begin().await.unwrap();
|
||||
let gate_pid: i32 = sqlx::query_scalar("select pg_backend_pid()")
|
||||
.fetch_one(&mut *gate)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("select id from workspaces where id = 'ws_default' for update")
|
||||
.fetch_one(&mut *gate)
|
||||
.await
|
||||
.unwrap();
|
||||
let a_task = tokio::spawn({
|
||||
let (client, server, a) = (client.clone(), server.to_string(), a.clone());
|
||||
async move { apply_plan(&client, &server, &a, a_key).await }
|
||||
});
|
||||
let b_task = tokio::spawn({
|
||||
let (client, server, b) = (client.clone(), server.to_string(), b.clone());
|
||||
async move { apply_plan(&client, &server, &b, b_key).await }
|
||||
});
|
||||
let mut observed_waiting = 0;
|
||||
let barrier_result = tokio::time::timeout(Duration::from_secs(15), async {
|
||||
loop {
|
||||
let waiting: i64 = sqlx::query_scalar(
|
||||
"with recursive blocked(pid) as (
|
||||
select pid from pg_catalog.pg_stat_activity
|
||||
where $1 = any(pg_catalog.pg_blocking_pids(pid))
|
||||
union
|
||||
select activity.pid from pg_catalog.pg_stat_activity activity
|
||||
join blocked on blocked.pid = any(pg_catalog.pg_blocking_pids(activity.pid))
|
||||
)
|
||||
select count(distinct pid) from blocked",
|
||||
)
|
||||
.bind(gate_pid)
|
||||
.fetch_one(registry.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
observed_waiting = waiting;
|
||||
if waiting == 2 {
|
||||
break;
|
||||
}
|
||||
assert!(!a_task.is_finished() && !b_task.is_finished());
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
barrier_result.is_ok(),
|
||||
"both distinct Apply transactions must reach the workspace lock; observed {observed_waiting}"
|
||||
);
|
||||
gate.commit().await.unwrap();
|
||||
let ((a_status, a_report), (b_status, b_report)) =
|
||||
(a_task.await.unwrap(), b_task.await.unwrap());
|
||||
assert!(
|
||||
a_status.is_success() ^ b_status.is_success(),
|
||||
"exactly one overlapping plan commits: {a_status}/{b_status}"
|
||||
);
|
||||
let (winner, loser_status, loser) = if a_status.is_success() {
|
||||
(&a_report, b_status, &b_report)
|
||||
} else {
|
||||
(&b_report, a_status, &a_report)
|
||||
};
|
||||
assert_eq!(loser_status, reqwest::StatusCode::CONFLICT);
|
||||
let error_code = loser
|
||||
.pointer("/error/context/error_code")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(
|
||||
error_code,
|
||||
"import_plan_fingerprint_stale" | "import_plan_name_conflict"
|
||||
),
|
||||
"loser must get the exact typed catalog/name conflict, got {error_code}"
|
||||
);
|
||||
let expected = winner["counts"]["created"].as_i64().unwrap();
|
||||
assert!(expected == 1 || expected == 2);
|
||||
assert_eq!(
|
||||
counts(®istry).await,
|
||||
(expected, expected, 1),
|
||||
"loser cannot leave partial Operations/versions/application"
|
||||
);
|
||||
let (winning_plan, winning_key) = if a_status.is_success() {
|
||||
(&a, a_key)
|
||||
} else {
|
||||
(&b, b_key)
|
||||
};
|
||||
let (status, replay) = apply_plan(&client, &server, winning_plan, winning_key).await;
|
||||
assert_eq!(status, reqwest::StatusCode::OK);
|
||||
assert_eq!(&replay, winner);
|
||||
assert_eq!(counts(®istry).await, (expected, expected, 1));
|
||||
}
|
||||
Reference in new issue
Block a user