fix(import): подтвердить обязательные гарантии второго эпика
CI / Rust Checks (pull_request) Successful in 24m43s
CI / UI Checks (pull_request) Successful in 6s
CI / Community Image Smoke (pull_request) Successful in 9m5s
CI / Frontend E2E (pull_request) Successful in 12m43s
CI / Deploy (pull_request) Skipped

This commit is contained in:
bsodfather committed 2026-10-10 13:25:05 +03:00
1 parent 31f6f00d3b
commit ad23cd9a46
33 files changed
+3328 -18

No files matched your search

+2
View File
@@ -52,7 +52,9 @@ crank-test-support = { path = "../../crates/crank-test-support" }
metrics.workspace = true
metrics-util = "0.20.4"
opentelemetry.workspace = true
opentelemetry-proto.workspace = true
opentelemetry_sdk.workspace = true
prost.workspace = true
reqwest.workspace = true
serial_test = "3"
tower.workspace = true
@@ -0,0 +1,49 @@
//! Read-only database observation for the local Playwright harness.
//! This example is never packaged as an application endpoint or executable.
#[tokio::main]
async fn main() {
let workspace = std::env::args().nth(1).expect("test workspace argument");
assert!(
!workspace.is_empty()
&& workspace.len() <= 128
&& workspace
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b"_-".contains(&byte)),
"invalid test workspace argument"
);
let config = crank_config::parse_migrator(
crank_config::ConfigSource::from_os_for_migrator().expect("test configuration source"),
)
.expect("test database configuration");
let database = config.database;
let options = if let Some(url) = &database.url {
url.expose_secret()
.parse::<sqlx::postgres::PgConnectOptions>()
.expect("test database URL")
} else {
sqlx::postgres::PgConnectOptions::new()
.host(&database.host)
.port(database.port)
.database(&database.database)
.username(&database.username)
.password(database.password.expose_secret())
};
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(1)
.connect_with(options)
.await
.expect("test database connection");
let (sources, operations, versions): (i64, i64, i64) = sqlx::query_as(
"select
(select count(*) from artifact_sources where workspace_id=$1),
(select count(*) from operations where workspace_id=$1),
(select count(*) from operation_versions v
join operations o on o.id=v.operation_id where o.workspace_id=$1)",
)
.bind(workspace)
.fetch_one(&pool)
.await
.expect("test storage observation");
println!("{{\"sources\":{sources},\"operations\":{operations},\"versions\":{versions}}}");
}
@@ -12,6 +12,7 @@ use std::{
use async_trait::async_trait;
use crank_artifacts::{ReconciliationRegistration, RegisteredArtifact};
use crank_registry::MigrationAuthority;
use sqlx::Row;
use super::*;
@@ -688,3 +689,6 @@ async fn expired_claim_cursor_prevents_unsafe_prefix_starvation_across_ticks() {
.unwrap();
assert_eq!(second_lifecycle, "unavailable");
}
#[path = "tests/commit_failure.rs"]
mod commit_failure;
@@ -0,0 +1,149 @@
use super::*;
#[tokio::test]
async fn finalize_commit_failure_stays_retryable_and_recovers_on_a_fresh_cycle() {
let database_url =
crank_test_support::postgres_schema_url("admin_reconciliation_commit_failure").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
MigrationAuthority::apply(&pool).await.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let root = TestRoot::new();
let store = ArtifactStore::open(&root.0).unwrap();
let now = OffsetDateTime::now_utc();
let artifact = store
.put_registered(b"commit-failure physical absence\n")
.unwrap();
let outcome = registry
.claim_artifact_reconciliation(ClaimArtifactReconciliationRequest {
artifact: &artifact,
token: ArtifactClaimToken::generate(),
claimed_at: now - time::Duration::minutes(10),
lease_expires_at: now - time::Duration::minutes(5),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap();
assert!(matches!(outcome, ArtifactClaimOutcome::Claimed(_)));
remove_registered_artifact(&store, &artifact);
sqlx::query(
"update artifact_blobs
set claim_expires_at = clock_timestamp() - interval '1 second'
where digest = $1",
)
.bind(artifact.artifact_ref().digest_hex())
.execute(registry.pool())
.await
.unwrap();
// A deferred constraint trigger fails the COMMIT that records
// `unavailable`, modelling a real failure after the physical unlink that
// removed the last bytes already succeeded. It lives only in this isolated
// test schema.
sqlx::raw_sql(
"create function crank_test_fail_unavailable() returns trigger language plpgsql as $$
begin
if new.storage_lifecycle = 'unavailable' then
raise exception 'injected finalize commit failure';
end if;
return new;
end $$;
create constraint trigger crank_test_fail_unavailable_guard
after update on artifact_blobs
deferrable initially deferred
for each row execute function crank_test_fail_unavailable();",
)
.execute(registry.pool())
.await
.unwrap();
let backend = PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone()));
let mut coordinator = ReconciliationCoordinator::new(backend);
let failed = coordinator.tick().await.unwrap();
// A real commit failure must surface as retryable work, never as idle.
assert!(failed.retryable);
assert_eq!(failed.mutation_attempts, 1);
assert_eq!(
store
.reconciliation_presence(artifact.artifact_ref())
.unwrap(),
ReconciliationPresence::Absent
);
let retained = sqlx::query(
"select claim_token, claim_expires_at, storage_lifecycle
from artifact_blobs where digest = $1",
)
.bind(artifact.artifact_ref().digest_hex())
.fetch_one(registry.pool())
.await
.unwrap();
assert!(
retained
.try_get::<Option<String>, _>("claim_token")
.unwrap()
.is_some(),
"the failed commit must roll back the claim clear"
);
assert!(
retained
.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")
.unwrap()
.is_some()
);
assert_eq!(
retained.get::<String, _>("storage_lifecycle"),
"available",
"the failed commit must roll back the unavailable transition"
);
// Removing the injected trigger and retrying a fresh coordinator clears the
// retained claim and records the confirmed absence as `unavailable`.
sqlx::raw_sql(
"drop trigger crank_test_fail_unavailable_guard on artifact_blobs;
drop function crank_test_fail_unavailable();",
)
.execute(registry.pool())
.await
.unwrap();
sqlx::query(
"update artifact_blobs
set claim_expires_at = clock_timestamp() - interval '1 second'
where digest = $1",
)
.bind(artifact.artifact_ref().digest_hex())
.execute(registry.pool())
.await
.unwrap();
let mut recovered_coordinator = ReconciliationCoordinator::new(
PostgresReconciliationBackend::new(registry.clone(), Arc::new(store.clone())),
);
let recovered = recovered_coordinator.tick().await.unwrap();
assert!(!recovered.retryable);
assert_eq!(
store
.reconciliation_presence(artifact.artifact_ref())
.unwrap(),
ReconciliationPresence::Absent
);
let cleared = sqlx::query(
"select claim_token, claim_expires_at, storage_lifecycle
from artifact_blobs where digest = $1",
)
.bind(artifact.artifact_ref().digest_hex())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
cleared.try_get::<Option<String>, _>("claim_token").unwrap(),
None
);
assert_eq!(
cleared
.try_get::<Option<time::OffsetDateTime>, _>("claim_expires_at")
.unwrap(),
None
);
assert_eq!(cleared.get::<String, _>("storage_lifecycle"), "unavailable");
}
+10
View File
@@ -16,6 +16,13 @@ const REGISTERED_MASTER_KEY: &str = "registered-master-key-CANARY_SECRET_VALUE-0
const WRONG_MASTER_KEY: &str = "wrong-master-key-CANARY_SECRET_VALUE-0000000000000";
static NEXT_STORAGE_ROOT: AtomicU64 = AtomicU64::new(0);
fn isolated_metrics_bind() -> String {
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("ephemeral metrics port");
let addr = listener.local_addr().unwrap().to_string();
drop(listener);
addr
}
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
@@ -27,6 +34,7 @@ fn run_with(entries: &[(&str, &str)]) -> String {
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_METRICS_BIND", isolated_metrics_bind().as_str()),
]);
for (name, value) in entries {
command.env(name, value);
@@ -137,6 +145,7 @@ async fn real_admin_startup_runs_immediate_import_maintenance() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let bind_addr = listener.local_addr().unwrap().to_string();
drop(listener);
let metrics_bind = isolated_metrics_bind();
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
@@ -151,6 +160,7 @@ async fn real_admin_startup_runs_immediate_import_maintenance() {
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
("CRANK_ADMIN_BIND", bind_addr.as_str()),
("CRANK_ADMIN_METRICS_BIND", metrics_bind.as_str()),
("CRANK_STORAGE_ROOT", storage_root.to_str().unwrap()),
])
.stdout(Stdio::null())
@@ -6,6 +6,8 @@ use serde_json::{Value, json};
use serial_test::serial;
mod replay;
mod telemetry;
mod telemetry_startup;
#[tokio::test(flavor = "multi_thread")]
#[serial]
@@ -0,0 +1,301 @@
use super::*;
use std::{collections::BTreeSet, time::Duration};
use axum::{
Router,
body::Bytes,
extract::Request,
http::StatusCode,
middleware::{self, Next},
routing::post,
};
use crank_core::{ProductEventKind, WorkspaceId};
use crank_observability::{
OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider,
};
use opentelemetry::trace::TracerProvider as _;
use opentelemetry_proto::tonic::{
collector::trace::v1::ExportTraceServiceRequest,
common::v1::{KeyValue, any_value},
};
use opentelemetry_sdk::trace::SdkTracerProvider;
use prost::Message;
use tokio::{net::TcpListener, sync::mpsc};
use tracing::instrument::WithSubscriber;
use tracing_subscriber::layer::SubscriberExt;
pub(super) const RAW_DOCUMENT_CANARY: &str = "openapi: 3.1.0 raw-document-canary";
pub(super) const USER_ID_CANARY: &str = "user-id-canary-should-not-export";
const REQUEST_ID: &str = "req_onboarding_optin_0001";
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn onboarding_http_event_keeps_default_trace_export_disabled() {
let registry = test_registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let config = OtlpTraceConfig::default();
let identity = service_identity();
assert!(!config.is_enabled());
assert!(
build_tracer_provider(&identity, &config)
.expect("disabled OTLP config is valid")
.is_none(),
"default config must not construct an exporter or batch queue"
);
let (collector_listener, _) = loopback_listener().await;
let (dispatch, provider) = local_trace_dispatch();
let app = app_with_request_dispatch(
build_test_app(
registry.clone(),
test_storage_root("onboarding_telemetry_default"),
),
dispatch,
);
let base_url = spawn_admin_api(app).await;
let client = authorized_client(&base_url).await;
let revision = onboarding_revision(&client, &base_url).await;
reject_canary_payload(&client, &base_url, revision).await;
let event = post_started_event(&client, &base_url, revision, false).await;
assert_eq!(event.status(), reqwest::StatusCode::OK);
assert_eq!(event.json::<Value>().await.unwrap()["accepted"], true);
assert_event_is_local(&registry, &workspace_id, "ui:telemetry-default-event").await;
assert!(
tokio::time::timeout(Duration::from_millis(300), collector_listener.accept())
.await
.is_err(),
"default telemetry config must make no connection to the external collector"
);
provider
.shutdown()
.expect("local trace provider shuts down");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn onboarding_http_event_exports_only_sanitized_trace_when_operator_opts_in() {
let registry = test_registry().await;
let workspace_id = WorkspaceId::new("ws_default");
let (endpoint, mut exported_requests, collector_task) = spawn_otlp_collector().await;
let config = OtlpTraceConfig::try_new(
Some(endpoint),
Some("http/protobuf".to_owned()),
Duration::from_secs(2),
OtlpBatchConfig::try_new(128, 128, Duration::from_secs(60), Duration::from_secs(2))
.unwrap(),
)
.unwrap();
let identity = service_identity();
assert!(config.is_enabled());
let (provider, tracer) = build_tracer_provider(&identity, &config)
.expect("explicit OTLP config is valid")
.expect("operator endpoint enables the real exporter");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let app = app_with_request_dispatch(
build_test_app(
registry.clone(),
test_storage_root("onboarding_telemetry_optin"),
),
dispatch,
);
let base_url = spawn_admin_api(app).await;
let client = authorized_client(&base_url).await;
let revision = onboarding_revision(&client, &base_url).await;
reject_canary_payload(&client, &base_url, revision).await;
let event = post_started_event(&client, &base_url, revision, true).await;
assert_eq!(event.status(), reqwest::StatusCode::OK);
assert_eq!(event.json::<Value>().await.unwrap()["accepted"], true);
assert_event_is_local(&registry, &workspace_id, "ui:telemetry-optin-event").await;
provider
.force_flush()
.expect("flush trace spans to the operator collector");
provider.shutdown().expect("exporter shuts down cleanly");
let mut request_count = 0;
let mut onboarding_request_id_exported = false;
while let Ok(body) = exported_requests.try_recv() {
request_count += 1;
let export = ExportTraceServiceRequest::decode(body.as_ref())
.expect("collector receives valid OTLP protobuf");
for resource_spans in export.resource_spans {
for scope_spans in resource_spans.scope_spans {
for span in scope_spans.spans {
assert_eq!(span.name, "http.request");
assert!(span.events.is_empty());
assert!(span.links.is_empty());
let attribute_keys: BTreeSet<&str> = span
.attributes
.iter()
.map(|attribute| attribute.key.as_str())
.collect();
assert!(
attribute_keys.iter().all(|key| *key == "request_id"),
"request spans may export only the request_id attribute"
);
if string_attribute(&span.attributes, "request_id") == Some(REQUEST_ID) {
onboarding_request_id_exported = true;
}
}
}
}
assert!(!body_contains(&body, RAW_DOCUMENT_CANARY));
assert!(!body_contains(&body, USER_ID_CANARY));
}
assert!(
request_count > 0,
"explicit opt-in must reach the collector"
);
assert!(
onboarding_request_id_exported,
"the collected trace must include the real onboarding HTTP request"
);
collector_task.abort();
}
fn service_identity() -> ServiceIdentity {
ServiceIdentity::try_new("admin-api", "integration-test", "test")
.expect("static service identity is valid")
}
fn local_trace_dispatch() -> (tracing::Dispatch, SdkTracerProvider) {
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("crank");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
(tracing::Dispatch::new(subscriber), provider)
}
fn app_with_request_dispatch(app: Router, dispatch: tracing::Dispatch) -> Router {
app.layer(middleware::from_fn(move |request: Request, next: Next| {
let dispatch = dispatch.clone();
async move { next.run(request).with_subscriber(dispatch).await }
}))
}
pub(super) async fn loopback_listener() -> (TcpListener, String) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}/v1/traces", listener.local_addr().unwrap());
(listener, endpoint)
}
pub(super) async fn spawn_otlp_collector() -> (
String,
mpsc::UnboundedReceiver<Bytes>,
tokio::task::JoinHandle<()>,
) {
let (listener, endpoint) = loopback_listener().await;
let (sender, receiver) = mpsc::unbounded_channel();
let app = Router::new().route(
"/v1/traces",
post(move |body: Bytes| {
let sender = sender.clone();
async move {
let _ = sender.send(body);
StatusCode::OK
}
}),
);
let handle = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(endpoint, receiver, handle)
}
pub(super) async fn onboarding_revision(
client: &reqwest::Client,
base_url: &(impl AsRef<str> + ?Sized),
) -> i64 {
let base_url = base_url.as_ref();
let response = client
.get(format!("{base_url}/onboarding"))
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
response.json::<Value>().await.unwrap()["revision"]
.as_i64()
.expect("onboarding returns a server revision")
}
pub(super) async fn reject_canary_payload(
client: &reqwest::Client,
base_url: &(impl AsRef<str> + ?Sized),
revision: i64,
) {
let base_url = base_url.as_ref();
let response = client
.post(format!("{base_url}/onboarding/events"))
.json(&json!({
"event": "started",
"idempotency_key": "ui:telemetry-canary-rejection",
"expected_revision": revision,
"user_id": USER_ID_CANARY,
"openapi_document": RAW_DOCUMENT_CANARY,
}))
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNPROCESSABLE_ENTITY);
let body = response.text().await.unwrap();
assert!(!body.contains(USER_ID_CANARY));
assert!(!body.contains(RAW_DOCUMENT_CANARY));
}
async fn post_started_event(
client: &reqwest::Client,
base_url: &(impl AsRef<str> + ?Sized),
revision: i64,
with_request_id: bool,
) -> reqwest::Response {
let base_url = base_url.as_ref();
let mut request = client
.post(format!("{base_url}/onboarding/events"))
.json(&json!({
"event": "started",
"idempotency_key": if with_request_id {
"ui:telemetry-optin-event"
} else {
"ui:telemetry-default-event"
},
"expected_revision": revision,
}));
if with_request_id {
request = request.header("x-request-id", REQUEST_ID);
}
request.send().await.unwrap()
}
async fn assert_event_is_local(
registry: &crank_registry::PostgresRegistry,
workspace_id: &WorkspaceId,
idempotency_key: &str,
) {
let record = registry
.get_product_event_by_idempotency_key(workspace_id, idempotency_key)
.await
.unwrap()
.expect("successful HTTP event is persisted in the local product event store");
assert_eq!(record.event.kind, ProductEventKind::OnboardingStarted);
assert_eq!(record.event.idempotency_key, idempotency_key);
}
pub(super) fn string_attribute<'a>(attributes: &'a [KeyValue], key: &str) -> Option<&'a str> {
attributes.iter().find_map(|attribute| {
(attribute.key == key)
.then_some(attribute.value.as_ref()?.value.as_ref()?)
.and_then(|value| match value {
any_value::Value::StringValue(value) => Some(value.as_str()),
_ => None,
})
})
}
pub(super) fn body_contains(body: &[u8], canary: &str) -> bool {
body.windows(canary.len())
.any(|window| window == canary.as_bytes())
}
@@ -0,0 +1,290 @@
use super::telemetry::{
RAW_DOCUMENT_CANARY, USER_ID_CANARY, body_contains, loopback_listener, onboarding_revision,
reject_canary_payload, string_attribute,
};
use super::*;
use axum::{
Router,
body::{self, Bytes},
extract::Request,
http::StatusCode,
routing::any,
};
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
use prost::Message;
use std::{
collections::BTreeSet,
fs::{self, OpenOptions},
net::TcpListener as StdTcpListener,
os::unix::fs::{OpenOptionsExt, PermissionsExt},
path::PathBuf,
process::{Child, Command, Stdio},
time::{Duration, Instant},
};
const MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
const SESSION_SECRET: &str = "test-session-secret";
const PASSWORD_PEPPER: &str = "test-password-pepper";
const ADMIN_EMAIL: &str = "owner@crank.local";
const ADMIN_PASSWORD: &str = "test-password";
const REQUEST_ID: &str = "req_admin_process_optin_0001";
struct AdminProcess {
child: Child,
log_path: PathBuf,
secrets: Vec<String>,
}
impl AdminProcess {
fn stop_and_assert_logs_are_safe(&mut self) {
let _ = self.child.kill();
self.child.wait().expect("admin-api child must be reaped");
let logs = fs::read_to_string(&self.log_path).expect("child log must be readable");
for secret in &self.secrets {
assert!(
!logs.contains(secret),
"admin-api process log must not contain configured secrets or request canaries"
);
}
}
}
impl Drop for AdminProcess {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn production_admin_process_keeps_default_otlp_export_disabled() {
let (registry, database_url) = test_registry_with_single_connection().await;
let _registry = registry;
let storage_root = test_storage_root("onboarding_telemetry_startup_default");
let (collector_listener, _) = loopback_listener().await;
let (mut process, base_url) = start_admin_process(&database_url, &storage_root, None);
wait_until_ready(&mut process, &base_url).await;
let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default");
let client = authorized_client(&workspace_url).await;
let revision = onboarding_revision(&client, &workspace_url).await;
reject_canary_payload(&client, &workspace_url, revision).await;
let response = post_startup_event(&client, &workspace_url, revision, false).await;
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(response.json::<Value>().await.unwrap()["accepted"], true);
assert!(
tokio::time::timeout(Duration::from_secs(2), collector_listener.accept())
.await
.is_err(),
"default production startup must not connect to an OTLP collector"
);
process.stop_and_assert_logs_are_safe();
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn production_admin_process_exports_only_sanitized_request_trace_when_opted_in() {
let (registry, database_url) = test_registry_with_single_connection().await;
let _registry = registry;
let storage_root = test_storage_root("onboarding_telemetry_startup_optin");
let (endpoint, mut exported_requests, collector_task) = spawn_startup_collector().await;
let (mut process, base_url) =
start_admin_process(&database_url, &storage_root, Some(&endpoint));
wait_until_ready(&mut process, &base_url).await;
let workspace_url = format!("{base_url}/api/admin/workspaces/ws_default");
let client = authorized_client(&workspace_url).await;
let revision = onboarding_revision(&client, &workspace_url).await;
reject_canary_payload(&client, &workspace_url, revision).await;
let response = post_startup_event(&client, &workspace_url, revision, true).await;
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(response.json::<Value>().await.unwrap()["accepted"], true);
let mut request_id_exported = false;
tokio::time::timeout(Duration::from_secs(12), async {
while !request_id_exported {
let (path, body) = exported_requests
.recv()
.await
.expect("collector task must remain active");
assert_eq!(path, "/v1/traces", "trace-specific endpoint path is exact");
let export = ExportTraceServiceRequest::decode(body.as_ref())
.expect("production process must send valid OTLP protobuf");
for resource_spans in export.resource_spans {
for scope_spans in resource_spans.scope_spans {
for span in scope_spans.spans {
assert_eq!(span.name, "http.request");
assert!(span.events.is_empty());
assert!(span.links.is_empty());
let attribute_keys: BTreeSet<&str> = span
.attributes
.iter()
.map(|attribute| attribute.key.as_str())
.collect();
assert!(
attribute_keys.iter().all(|key| *key == "request_id"),
"production request spans may export only the request_id attribute"
);
if string_attribute(&span.attributes, "request_id") == Some(REQUEST_ID) {
request_id_exported = true;
}
let encoded = format!("{} {:?}", span.name, span.attributes);
assert!(!encoded.contains(USER_ID_CANARY));
assert!(!encoded.contains(RAW_DOCUMENT_CANARY));
}
}
}
assert!(!body_contains(&body, USER_ID_CANARY));
assert!(!body_contains(&body, RAW_DOCUMENT_CANARY));
}
})
.await
.expect("production exporter must send the onboarding request in a bounded OTLP batch");
assert!(
request_id_exported,
"collector must receive the trace for the onboarding event request"
);
process.stop_and_assert_logs_are_safe();
collector_task.abort();
}
fn start_admin_process(
database_url: &str,
storage_root: &std::path::Path,
otlp_endpoint: Option<&str>,
) -> (AdminProcess, String) {
fs::create_dir_all(storage_root).expect("isolated storage root must be created");
#[cfg(unix)]
fs::set_permissions(storage_root, fs::Permissions::from_mode(0o700))
.expect("isolated storage root must be private");
let listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve an ephemeral API port");
let address = listener.local_addr().unwrap();
drop(listener);
let base_url = format!("http://{address}");
let log_path = storage_root.join("admin-api.log");
let stdout = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&log_path)
.expect("private child log must be created");
let stderr = stdout.try_clone().expect("child stderr log handle");
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
command
.env_clear()
.env("CRANK_DATABASE_URL", database_url)
.env("CRANK_ADMIN_BIND", address.to_string())
.env("CRANK_STORAGE_ROOT", storage_root)
.env("CRANK_MASTER_KEY", MASTER_KEY)
.env("CRANK_SESSION_SECRET", SESSION_SECRET)
.env("CRANK_PASSWORD_PEPPER", PASSWORD_PEPPER)
.env("CRANK_SESSION_TTL_HOURS", "24")
.env("CRANK_BOOTSTRAP_ADMIN_EMAIL", ADMIN_EMAIL)
.env("CRANK_BOOTSTRAP_ADMIN_PASSWORD", ADMIN_PASSWORD)
.env("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME", "Integration Test")
.env("CRANK_METRICS_ENABLED", "false")
.env("CRANK_BASE_URL", &base_url)
.env("CRANK_LOG_LEVEL", "info")
.stdout(Stdio::from(stdout))
.stderr(Stdio::from(stderr));
if let Some(endpoint) = otlp_endpoint {
command
.env("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", endpoint)
.env("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "http/protobuf")
.env("OTEL_BSP_MAX_QUEUE_SIZE", "16")
.env("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "1")
.env("OTEL_BSP_SCHEDULE_DELAY", "100")
.env("OTEL_BSP_EXPORT_TIMEOUT", "2000");
}
let child = command.spawn().expect("production admin-api binary starts");
(
AdminProcess {
child,
log_path,
secrets: vec![
database_url.to_owned(),
MASTER_KEY.to_owned(),
SESSION_SECRET.to_owned(),
PASSWORD_PEPPER.to_owned(),
ADMIN_PASSWORD.to_owned(),
USER_ID_CANARY.to_owned(),
RAW_DOCUMENT_CANARY.to_owned(),
],
},
base_url,
)
}
async fn spawn_startup_collector() -> (
String,
tokio::sync::mpsc::UnboundedReceiver<(String, Bytes)>,
tokio::task::JoinHandle<()>,
) {
let (listener, endpoint) = loopback_listener().await;
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
let app = Router::new().fallback(any(move |request: Request| {
let sender = sender.clone();
async move {
let path = request.uri().path().to_owned();
let body = body::to_bytes(request.into_body(), usize::MAX)
.await
.expect("collector request body must be readable");
let _ = sender.send((path, body));
StatusCode::OK
}
}));
let handle = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(endpoint, receiver, handle)
}
async fn wait_until_ready(process: &mut AdminProcess, base_url: &str) {
let deadline = Instant::now() + Duration::from_secs(45);
loop {
if let Some(status) = process.child.try_wait().expect("child status is readable") {
panic!("production admin-api exited before readiness: {status}");
}
if reqwest::Client::new()
.get(format!("{base_url}/health"))
.timeout(Duration::from_millis(500))
.send()
.await
.is_ok_and(|response| response.status().is_success())
{
return;
}
assert!(
Instant::now() < deadline,
"production admin-api readiness deadline"
);
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
async fn post_startup_event(
client: &reqwest::Client,
workspace_url: &str,
revision: i64,
with_request_id: bool,
) -> reqwest::Response {
let mut request = client
.post(format!("{workspace_url}/onboarding/events"))
.json(&json!({
"event": "started",
"idempotency_key": if with_request_id {
"ui:telemetry-process-optin"
} else {
"ui:telemetry-process-default"
},
"expected_revision": revision,
}));
if with_request_id {
request = request.header("x-request-id", REQUEST_ID);
}
request.send().await.unwrap()
}
@@ -1,11 +1,17 @@
use super::super::common::test_service_with_external_references;
mod isolation;
use super::*;
use axum::{Router, routing::get};
use admin_api::error::ApiError;
use axum::{Router, body::to_bytes, http::StatusCode, response::IntoResponse, routing::get};
use crank_artifacts::ArtifactStore;
use crank_core::{Workspace, WorkspaceStatus};
use crank_registry::{ArtifactSourceId, CreateWorkspaceRequest, RegistryError};
use sha2::{Digest, Sha256};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use time::OffsetDateTime;
// Serialized by the integration harness at baseline f5f40cb167a82b1e3c1ea2475db5cbaed9d9abfa; this v4 payload preserves the then-current security blocker and digest.
const BASELINE_V4_SECURITY_PREVIEW_PAYLOAD: &str = r###"{"dependencies":[],"dependency_snapshots":[],"normalization":{"config":{"external_references_enabled":false,"max_aliases":128,"max_bytes":262144,"max_collection_items":10000,"max_depth":64,"max_expanded_nodes":100000,"max_external_document_bytes":262144,"max_nodes":50000,"max_reference_depth":32,"max_reference_documents":32,"max_references":4096,"max_scalar_bytes":262144,"normalizer_version":"normalized-ir-v4","projection_version":"preview-v4"},"ir_fingerprint":"ae2319de76c13d631b26843623092d966e5deea7cda9e3e40e4acc6fa36a25a8","normalizer_version":"normalized-ir-v4","projection_version":"preview-v4"},"preview":{"findings":[{"code":"unsupported_security_requirement","message":"Требование безопасности должно быть привязано к настроенному Auth Profile перед публикацией этой операции; остальные операции не блокируются.","operation_key":"GET /secure","severity":"error"}],"groups":[{"key":"imported_operation","operations":[{"category":"imported","description":"Выполняет GET /public","draft":{"category":"imported","display_name":"Public","input_mapping":{},"input_schema":{"description":"Входные параметры MCP-инструмента","nullable":false,"required":true,"type":"object"},"name":"public","output_mapping":{"rules":[{"required":true,"source":"$.response.body","target":"$.output"}]},"output_schema":{"description":"Успешный ответ без тела","nullable":false,"required":true,"type":"null"},"target":{"base_url":"https://api.example.test","method":"GET","path_template":"/public","response_contract":[{"schema_variant":0,"status_code":200}]},"tool_description":{"description":"Выполняет GET /public","examples":[{"input":{}}],"title":"Public"},"wizard_state":{}},"findings":[{"code":"missing_summary","message":"У метода нет summary, отображаемое имя будет сгенерировано автоматически.","operation_key":"GET /public","severity":"warning"},{"code":"missing_description","message":"У метода нет description. Перед публикацией лучше описать, когда агенту стоит вызывать этот инструмент.","operation_key":"GET /public","severity":"warning"},{"code":"missing_response_schema","message":"У метода не найдена схема успешного ответа, результат будет описан как общий объект.","operation_key":"GET /public","severity":"warning"},{"code":"no_input_fields","message":"У инструмента нет входных параметров. Это нормально для справочных методов, но проверьте, что агенту не нужно передавать фильтры.","operation_key":"GET /public","severity":"info"},{"code":"empty_output_schema","message":"В ответе не найдено отдельных полей. Перед публикацией проверьте схему ответа и маппинг результата.","operation_key":"GET /public","severity":"warning"},{"code":"weak_tool_description","message":"Описание инструмента слишком короткое или техническое. Перед публикацией добавьте, когда агент должен вызывать инструмент и что будет в успешном ответе.","operation_key":"GET /public","severity":"warning"},{"code":"weak_tool_name","message":"Имя инструмента `public` выглядит слишком общим. Лучше использовать имя с конкретным действием и объектом.","operation_key":"GET /public","severity":"warning"}],"input_fields":0,"key":"GET /public","method":"GET","operation_id":"public","output_fields":0,"path":"/public","server_urls":["https://api.example.test"],"suggested_display_name":"Public","suggested_name":"public"},{"category":"imported","description":"Выполняет GET /secure","draft":{"category":"imported","display_name":"Secure","input_mapping":{},"input_schema":{"description":"Входные параметры MCP-инструмента","nullable":false,"required":true,"type":"object"},"name":"secure","output_mapping":{"rules":[{"required":true,"source":"$.response.body","target":"$.output"}]},"output_schema":{"description":"Успешный ответ без тела","nullable":false,"required":true,"type":"null"},"target":{"base_url":"https://api.example.test","method":"GET","path_template":"/secure","response_contract":[{"schema_variant":0,"status_code":200}]},"tool_description":{"description":"Выполняет GET /secure","examples":[{"input":{}}],"title":"Secure"},"wizard_state":{}},"findings":[{"code":"missing_summary","message":"У мет Line truncated
@@ -43,12 +49,14 @@ async fn external_relative_chain_survives_reversed_builder_order_and_apply_never
let origin = format!("http://{address}");
let registry = test_registry().await;
let storage_root = test_storage_root("openapi_import_external_snapshot");
let service = test_service_with_external_references(
registry.clone(),
test_storage_root("openapi_import_external_snapshot"),
storage_root.clone(),
vec![format!("{origin}/")],
);
let workspace_id = WorkspaceId::new("ws_default");
let foreign_workspace_id = create_workspace(&registry, "ws_external_snapshot_foreign").await;
let upload = OpenApiUpload {
bytes: format!(
r#"
@@ -97,6 +105,60 @@ paths:
2
);
// The preview response contains only the normalized result. Ownership
// envelopes, storage digests, and remote origins stay in the persisted
// job payload and are verified through the workspace-scoped registry API.
let public_preview = serde_json::to_string(&preview).unwrap();
assert!(!public_preview.contains("src_openapi_dep_"));
assert!(!public_preview.contains(&origin));
let dependencies = job.preview_payload["dependencies"].as_array().unwrap();
let store = Arc::new(ArtifactStore::open(&storage_root).unwrap());
for dependency in dependencies {
let source_id = ArtifactSourceId::new(dependency["source_id"].as_str().unwrap());
let digest = dependency["digest"].as_str().unwrap();
let canonical_uri = dependency["canonical_uri"].as_str().unwrap();
let source = registry
.get_artifact_source(&workspace_id, &source_id)
.await
.unwrap();
assert_eq!(source.workspace_id, workspace_id);
assert_eq!(source.blob.artifact_ref.as_str(), digest);
let verified = registry
.read_artifact_source(Arc::clone(&store), &workspace_id, &source_id)
.await
.unwrap();
assert_eq!(verified.source.blob.artifact_ref.as_str(), digest);
let expected_bytes = if canonical_uri.ends_with("/root.yaml") {
b"Item: { $ref: './child.yaml#/Item' }".as_slice()
} else {
assert!(canonical_uri.ends_with("/child.yaml"));
b"Item: { type: object, required: [id], properties: { id: { type: string } } }"
.as_slice()
};
assert_eq!(verified.bytes, expected_bytes);
let foreign_read = registry
.read_artifact_source(Arc::clone(&store), &foreign_workspace_id, &source_id)
.await
.expect_err("foreign workspace must not read an owned snapshot");
let unknown_source_id = ArtifactSourceId::new("src_unknown_scope_probe");
let unknown_read = registry
.read_artifact_source(
Arc::clone(&store),
&foreign_workspace_id,
&unknown_source_id,
)
.await
.expect_err("unknown source must have the same scoped outcome");
let foreign_body = source_not_found_body(foreign_read).await;
let unknown_body = source_not_found_body(unknown_read).await;
assert_eq!(foreign_body, unknown_body);
let rendered_body = foreign_body.to_string();
assert!(!rendered_body.contains(digest));
assert!(!rendered_body.contains(canonical_uri));
assert!(!rendered_body.contains(source_id.as_str()));
}
let applied = service
.create_openapi_import(
&workspace_id,
@@ -122,6 +184,59 @@ paths:
assert_eq!(active_sources, 0);
}
fn external_upload(origin: &str) -> OpenApiUpload {
OpenApiUpload {
bytes: format!(
r#"
openapi: 3.1.0
info: {{ title: Scoped external snapshot }}
servers: [{{ url: https://api.example.test }}]
paths:
/items:
get:
operationId: listItems
responses:
'200':
description: ok
content:
application/json:
schema: {{ $ref: '{origin}/schema.yaml#/Item' }}
"#
)
.into_bytes(),
mime_type: "application/yaml".to_owned(),
locale: OpenApiUploadLocale::En,
}
}
async fn create_workspace(registry: &crank_registry::PostgresRegistry, id: &str) -> WorkspaceId {
let workspace_id = WorkspaceId::new(id);
let now = OffsetDateTime::now_utc();
registry
.create_workspace(CreateWorkspaceRequest {
workspace: &Workspace {
id: workspace_id.clone(),
slug: id.replace('_', "-"),
display_name: id.to_owned(),
status: WorkspaceStatus::Active,
settings: serde_json::json!({}),
created_at: now,
updated_at: now,
},
})
.await
.unwrap();
workspace_id
}
async fn source_not_found_body(error: RegistryError) -> serde_json::Value {
assert!(matches!(&error, RegistryError::SourceNotFound { .. }));
let response = ApiError::from(error).into_response();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
serde_json::from_slice(&body).unwrap()
}
#[tokio::test]
#[serial]
async fn external_secret_document_is_rejected_before_blob_registration() {
@@ -0,0 +1,287 @@
use super::*;
use std::time::Duration;
use tokio::sync::Notify;
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn concurrent_external_previews_keep_snapshot_ownership_workspace_scoped() {
let fetches = Arc::new(AtomicUsize::new(0));
let fetch_count = Arc::clone(&fetches);
let app = Router::new().route(
"/schema.yaml",
get(move || {
let fetch_count = Arc::clone(&fetch_count);
async move {
fetch_count.fetch_add(1, Ordering::SeqCst);
"Item: { type: object, properties: { id: { type: string } } }"
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let origin = format!("http://{address}");
let registry = test_registry().await;
let storage_root = test_storage_root("openapi_import_external_workspace_isolation");
let service = test_service_with_external_references(
registry.clone(),
storage_root.clone(),
vec![format!("{origin}/")],
);
let workspace_a = create_workspace(&registry, "ws_external_concurrent_a").await;
let workspace_b = create_workspace(&registry, "ws_external_concurrent_b").await;
let (preview_a, preview_b) = tokio::join!(
service.preview_openapi_import(&workspace_a, external_upload(&origin)),
service.preview_openapi_import(&workspace_b, external_upload(&origin)),
);
let preview_a = preview_a.unwrap();
let preview_b = preview_b.unwrap();
server.abort();
assert_eq!(
fetches.load(Ordering::SeqCst),
2,
"each workspace fetches its own preview snapshot"
);
assert_ne!(preview_a.job_id, preview_b.job_id);
let job_a = registry
.get_import_job(&workspace_a, &preview_a.job_id.as_str().into())
.await
.unwrap()
.unwrap();
let job_b = registry
.get_import_job(&workspace_b, &preview_b.job_id.as_str().into())
.await
.unwrap()
.unwrap();
let dependency_a = &job_a.preview_payload["dependencies"][0];
let dependency_b = &job_b.preview_payload["dependencies"][0];
let source_a = ArtifactSourceId::new(dependency_a["source_id"].as_str().unwrap());
let source_b = ArtifactSourceId::new(dependency_b["source_id"].as_str().unwrap());
let digest_a = dependency_a["digest"].as_str().unwrap();
let digest_b = dependency_b["digest"].as_str().unwrap();
let canonical_uri = dependency_a["canonical_uri"].as_str().unwrap();
assert_eq!(
digest_a, digest_b,
"identical bytes may share the immutable blob"
);
assert_eq!(dependency_b["canonical_uri"], dependency_a["canonical_uri"]);
assert_ne!(
source_a, source_b,
"each workspace owns a distinct source relation"
);
let public_a = serde_json::to_string(&preview_a).unwrap();
let public_b = serde_json::to_string(&preview_b).unwrap();
for response in [&public_a, &public_b] {
assert!(!response.contains("src_openapi_dep_"));
assert!(!response.contains(digest_a));
assert!(!response.contains(canonical_uri));
}
let store = Arc::new(ArtifactStore::open(&storage_root).unwrap());
for (workspace_id, source_id) in [(&workspace_a, &source_a), (&workspace_b, &source_b)] {
let source = registry
.get_artifact_source(workspace_id, source_id)
.await
.unwrap();
assert_eq!(source.workspace_id, *workspace_id);
assert_eq!(source.blob.artifact_ref.as_str(), digest_a);
let verified = registry
.read_artifact_source(Arc::clone(&store), workspace_id, source_id)
.await
.unwrap();
assert_eq!(
verified.bytes,
b"Item: { type: object, properties: { id: { type: string } } }"
);
}
let foreign_body = source_not_found_body(
registry
.read_artifact_source(Arc::clone(&store), &workspace_b, &source_a)
.await
.expect_err("workspace B cannot resolve workspace A source"),
)
.await;
let unknown_source = ArtifactSourceId::new("src_unknown_workspace_probe");
let unknown_body = source_not_found_body(
registry
.read_artifact_source(Arc::clone(&store), &workspace_b, &unknown_source)
.await
.expect_err("unknown source is indistinguishable from a foreign source"),
)
.await;
assert_eq!(foreign_body, unknown_body);
let response = foreign_body.to_string();
assert!(!response.contains(digest_a));
assert!(!response.contains(canonical_uri));
assert!(!response.contains(source_a.as_str()));
assert!(!response.contains("exists"));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn existing_foreign_snapshot_does_not_short_circuit_second_workspace_fetch() {
let fetches = Arc::new(AtomicUsize::new(0));
let fetch_count = Arc::clone(&fetches);
let second_fetch_entered = Arc::new(Notify::new());
let entered = Arc::clone(&second_fetch_entered);
let release_second_fetch = Arc::new(Notify::new());
let release = Arc::clone(&release_second_fetch);
let app = Router::new().route(
"/schema.yaml",
get(move || {
let fetch_count = Arc::clone(&fetch_count);
let entered = Arc::clone(&entered);
let release = Arc::clone(&release);
async move {
let request_number = fetch_count.fetch_add(1, Ordering::SeqCst) + 1;
if request_number == 2 {
entered.notify_one();
release.notified().await;
}
"Item: { type: object, properties: { id: { type: string } } }"
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let origin = format!("http://{address}");
let registry = test_registry().await;
let service = test_service_with_external_references(
registry.clone(),
test_storage_root("openapi_import_external_fetch_oracle"),
vec![format!("{origin}/")],
);
let workspace_a = create_workspace(&registry, "ws_external_oracle_a").await;
let workspace_b = create_workspace(&registry, "ws_external_oracle_b").await;
let preview_a = service
.preview_openapi_import(&workspace_a, external_upload(&origin))
.await
.unwrap();
assert_eq!(fetches.load(Ordering::SeqCst), 1);
let preview_b_origin = origin.clone();
let preview_b_workspace = workspace_b.clone();
let mut preview_b_task = tokio::spawn(async move {
service
.preview_openapi_import(&preview_b_workspace, external_upload(&preview_b_origin))
.await
});
tokio::time::timeout(Duration::from_secs(10), second_fetch_entered.notified())
.await
.expect("the second preview must reach its HTTP fetch");
// The fixture holds the real HTTP response. Poll the preview once after
// observing that fetch; it must remain pending until the fixture releases it.
tokio::select! {
biased;
result = &mut preview_b_task => panic!("second preview bypassed its held fetch: {result:?}"),
_ = tokio::task::yield_now() => {}
}
assert_eq!(fetches.load(Ordering::SeqCst), 2);
release_second_fetch.notify_one();
let preview_b = tokio::time::timeout(Duration::from_secs(10), &mut preview_b_task)
.await
.expect("the released second preview should complete")
.unwrap()
.unwrap();
server.abort();
let job_a = registry
.get_import_job(&workspace_a, &preview_a.job_id.as_str().into())
.await
.unwrap()
.unwrap();
let job_b = registry
.get_import_job(&workspace_b, &preview_b.job_id.as_str().into())
.await
.unwrap()
.unwrap();
let source_a = ArtifactSourceId::new(
job_a.preview_payload["dependencies"][0]["source_id"]
.as_str()
.unwrap(),
);
let source_b = ArtifactSourceId::new(
job_b.preview_payload["dependencies"][0]["source_id"]
.as_str()
.unwrap(),
);
assert_ne!(source_a, source_b);
assert_eq!(fetches.load(Ordering::SeqCst), 2);
}
#[tokio::test]
#[serial]
async fn file_origin_and_relative_chain_are_blocked_without_file_or_network_access() {
const PRIVATE_FILE_BYTES: &str = "PrivateCanary: file-origin-secret-must-never-be-read";
const HYPOTHETICAL_FILE_ROOT: &str = "Item: { $ref: './private/canary.yaml#/Secret' }";
let file_fixture_root = test_storage_root("openapi_import_file_origin_fixture");
let private_directory = file_fixture_root.join("private");
std::fs::create_dir_all(&private_directory).unwrap();
std::fs::write(file_fixture_root.join("root.yaml"), HYPOTHETICAL_FILE_ROOT).unwrap();
std::fs::write(
private_directory.join("canary.yaml"),
format!("Secret: {{ type: string, example: '{PRIVATE_FILE_BYTES}' }}"),
)
.unwrap();
let file_origin = url::Url::from_file_path(file_fixture_root.join("root.yaml")).unwrap();
let file_origin_for_response = file_origin.clone();
let requests = Arc::new(AtomicUsize::new(0));
let request_count = Arc::clone(&requests);
let app = Router::new().route(
"/schema.yaml",
get(move || {
let request_count = Arc::clone(&request_count);
async move {
request_count.fetch_add(1, Ordering::SeqCst);
format!("Item: {{ $ref: '{file_origin_for_response}#/Item' }}")
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let origin = format!("http://{address}");
let registry = test_registry().await;
let service = test_service_with_external_references(
registry,
test_storage_root("openapi_import_file_origin_blocked"),
vec![format!("{origin}/")],
);
let preview = service
.preview_openapi_import(&WorkspaceId::new("ws_default"), external_upload(&origin))
.await
.unwrap();
server.abort();
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"only the explicitly allowlisted HTTP root is fetched"
);
let findings = preview
.preview
.groups
.iter()
.flat_map(|group| &group.operations)
.flat_map(|operation| &operation.findings)
.chain(preview.preview.findings.iter())
.collect::<Vec<_>>();
assert!(
findings.iter().any(|finding| {
finding.severity == crank_import::rest::ImportFindingSeverity::Error
&& finding.code == "reference_target_missing"
}),
"file origin must produce a blocking unresolved-reference finding"
);
let report = serde_json::to_string(&preview).unwrap();
assert!(!report.contains(file_origin.as_str()));
assert!(!report.contains(PRIVATE_FILE_BYTES));
assert!(!report.contains(HYPOTHETICAL_FILE_ROOT));
}
@@ -32,6 +32,9 @@ impl MachineCredentialVerifier for CorpusMcpCredentialVerifier {
use super::super::common::*;
#[cfg(debug_assertions)]
mod crash_recovery;
const OPENAPI_SINGLE: &str = r#"
openapi: 3.0.3
info: { title: Apply test }
@@ -98,6 +101,7 @@ paths:
responses: { '200': { description: ok } }
"#;
#[derive(Clone)]
struct CreatedPlan {
reference: String,
digest: String,
@@ -0,0 +1,330 @@
//! Kill the actual HTTP-serving process at durable Apply boundaries, then
//! restart the same production router against the parent's database/storage.
use super::*;
use crank_artifacts::test_support::{FaultAction, set_checkpoint};
use std::{
path::PathBuf,
process::{Child, Command, Stdio},
time::Instant,
};
const CHILD_TEST: &str =
"integration::openapi_import::import_plan_apply::crash_recovery::http_apply_crash_child";
const CHILD_CONFIG: &str = "CRANK_TEST_APPLY_CRASH_CONFIG";
struct PrivateConfigGuard(PathBuf);
impl Drop for PrivateConfigGuard {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
struct ChildGuard(Child);
impl Drop for ChildGuard {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
// Child configuration contains only isolated test fixture credentials. Keep it
// in a private file and never include it in assertions, stdout or task logs.
#[tokio::test(flavor = "multi_thread")]
async fn http_apply_crash_child() {
let Ok(path) = std::env::var(CHILD_CONFIG) else {
return;
};
let config: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();
let registry = crank_registry::PostgresRegistry::connect(config["database"].as_str().unwrap())
.await
.unwrap();
let root = PathBuf::from(config["storage"].as_str().unwrap());
let server = spawn_admin_api(build_test_app(registry, root)).await;
set_checkpoint(config["stage"].as_str().unwrap(), FaultAction::Exit);
std::fs::write(config["ready"].as_str().unwrap(), server.as_ref()).unwrap();
std::future::pending::<()>().await;
}
async fn counts(registry: &crank_registry::PostgresRegistry) -> (i64, i64, i64) {
sqlx::query_as(
"select (select count(*) from operations),
(select count(*) from operation_versions),
(select count(*) from import_plan_applications)",
)
.fetch_one(registry.pool())
.await
.unwrap()
}
async fn crash_then_restart(stage: &str, committed: bool) {
use std::os::unix::fs::PermissionsExt;
let (registry, database_url) = test_registry_with_single_connection().await;
let root = test_storage_root(stage);
std::fs::create_dir_all(&root).unwrap();
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
let server = spawn_admin_api(build_test_app(registry.clone(), root.clone())).await;
let client = authorized_client(&server).await;
let plan = create_plan(&client, &server, OPENAPI_SINGLE, full_plan()).await;
let store = crank_artifacts::ArtifactStore::open(&root).unwrap();
let artifact = store.put_registered(OPENAPI_SINGLE.as_bytes()).unwrap();
let key = format!("crash-{stage}");
assert_eq!(counts(&registry).await, (0, 0, 0));
drop(server);
let ready = root.join("child.ready");
let config_path = root.join("child-config.json");
let _private_config = PrivateConfigGuard(config_path.clone());
{
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&config_path)
.unwrap();
let config = json!({"database":database_url,"storage":root,"stage":stage,"ready":ready});
file.write_all(&serde_json::to_vec(&config).unwrap())
.unwrap();
}
let mut child = ChildGuard(
Command::new(std::env::current_exe().unwrap())
.args(["--exact", CHILD_TEST, "--nocapture"])
.env(CHILD_CONFIG, &config_path)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.unwrap(),
);
let start = Instant::now();
let child_url = loop {
if let Ok(url) = std::fs::read_to_string(&ready) {
break url;
}
assert!(
child.0.try_wait().unwrap().is_none(),
"child exited before HTTP readiness"
);
assert!(
start.elapsed() < Duration::from_secs(30),
"child HTTP readiness deadline"
);
tokio::time::sleep(Duration::from_millis(10)).await;
};
let response = client
.post(format!(
"{child_url}/imports/openapi/{}/apply",
plan.reference
))
.json(&json!({"plan_digest":plan.digest,"idempotency_key":key}))
.timeout(Duration::from_secs(30))
.send()
.await;
assert!(response.is_err(), "crash must prevent an HTTP response");
let status = loop {
if let Some(status) = child.0.try_wait().unwrap() {
break status;
}
assert!(
start.elapsed() < Duration::from_secs(35),
"child crash deadline"
);
tokio::time::sleep(Duration::from_millis(10)).await;
};
assert_eq!(
status.code(),
Some(86),
"must hit the exact armed commit boundary"
);
std::fs::remove_file(config_path).unwrap();
std::fs::remove_file(ready).unwrap();
let expected = if committed { (1, 1, 1) } else { (0, 0, 0) };
// An independent connection can observe only fully committed state.
assert_eq!(counts(&registry).await, expected);
let before: Option<Value> = sqlx::query_scalar(
"select report from import_plan_applications where workspace_id='ws_default' and idempotency_key=$1",
).bind(&key).fetch_optional(registry.pool()).await.unwrap();
assert_eq!(before.is_some(), committed);
// The source of this nonexpired job remains referenced after process loss;
// cleanup cannot make its plan snapshot eligible for orphan deletion.
assert_eq!(sqlx::query_scalar::<_, i64>(
"select count(*) from artifact_sources where workspace_id='ws_default' and lifecycle='active'",
).fetch_one(registry.pool()).await.unwrap(), 1);
let cleanup = registry.cleanup_expired_import_jobs(32).await;
assert!(cleanup.is_ok());
let now = time::OffsetDateTime::now_utc();
let claim = registry
.claim_artifact_reconciliation(crank_registry::ClaimArtifactReconciliationRequest {
artifact: &artifact,
token: crank_registry::ArtifactClaimToken::generate(),
claimed_at: now,
lease_expires_at: now + time::Duration::minutes(5),
detached_grace: time::Duration::hours(24),
})
.await
.unwrap();
assert!(matches!(
claim,
crank_registry::ArtifactClaimOutcome::ActiveReference
));
assert_eq!(
store.read(artifact.artifact_ref()).unwrap(),
OPENAPI_SINGLE.as_bytes()
);
assert_eq!(sqlx::query_scalar::<_, i64>(
"select count(*) from artifact_sources where workspace_id='ws_default' and lifecycle='active'",
).fetch_one(registry.pool()).await.unwrap(), 1);
let restarted = spawn_admin_api(build_test_app(registry.clone(), root)).await;
let (status, report) = apply_plan(&client, &restarted, &plan, &key).await;
assert_eq!(status, reqwest::StatusCode::OK);
if let Some(before) = before {
assert_eq!(
report, before,
"restart must return the committed immutable report"
);
}
assert_eq!(report["plan_digest"], plan.digest);
assert_eq!(report["counts"]["created"], 1);
let (status, replay) = apply_plan(&client, &restarted, &plan, &key).await;
assert_eq!(status, reqwest::StatusCode::OK);
assert_eq!(
replay, report,
"exact retry must be byte-equivalent structured report"
);
assert_eq!(counts(&registry).await, (1, 1, 1));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn http_process_crash_before_commit_rolls_back_and_retries_after_restart() {
crash_then_restart("apply_before_commit", false).await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn http_process_crash_after_commit_before_response_replays_after_restart() {
crash_then_restart("apply_after_commit", true).await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn concurrent_distinct_overlapping_http_plans_have_one_coherent_commit() {
let (seed_registry, database_url) = test_registry_with_single_connection().await;
// The ordinary test registry has only one connection and would serialize
// both requests before their transactions even start.
let registry = crank_registry::PostgresRegistry::connect_with_options_and_pool_config(
database_url
.parse::<sqlx::postgres::PgConnectOptions>()
.unwrap(),
crank_registry::PostgresPoolConfig::try_new(4, 0, 5_000, 60_000, 1_800_000).unwrap(),
)
.await
.unwrap();
drop(seed_registry);
let server = spawn_admin_api(build_test_app(
registry.clone(),
test_storage_root("distinct-overlap-plans"),
))
.await;
let client = authorized_client(&server).await;
let a = create_plan(&client, &server, OPENAPI_SINGLE, full_plan()).await;
let b = create_plan(&client, &server, OPENAPI_TWO_OPERATIONS, full_plan()).await;
assert_ne!(a.reference, b.reference);
assert_ne!(a.digest, b.digest);
let a_key = "overlap-distinct-a";
let b_key = "overlap-distinct-b";
// Hold the production workspace row lock until BOTH independent Apply
// transactions are blocked on it. Merely joining requests could otherwise
// exercise two sequential transactions.
let mut gate = registry.pool().begin().await.unwrap();
let gate_pid: i32 = sqlx::query_scalar("select pg_backend_pid()")
.fetch_one(&mut *gate)
.await
.unwrap();
sqlx::query("select id from workspaces where id = 'ws_default' for update")
.fetch_one(&mut *gate)
.await
.unwrap();
let a_task = tokio::spawn({
let (client, server, a) = (client.clone(), server.to_string(), a.clone());
async move { apply_plan(&client, &server, &a, a_key).await }
});
let b_task = tokio::spawn({
let (client, server, b) = (client.clone(), server.to_string(), b.clone());
async move { apply_plan(&client, &server, &b, b_key).await }
});
let mut observed_waiting = 0;
let barrier_result = tokio::time::timeout(Duration::from_secs(15), async {
loop {
let waiting: i64 = sqlx::query_scalar(
"with recursive blocked(pid) as (
select pid from pg_catalog.pg_stat_activity
where $1 = any(pg_catalog.pg_blocking_pids(pid))
union
select activity.pid from pg_catalog.pg_stat_activity activity
join blocked on blocked.pid = any(pg_catalog.pg_blocking_pids(activity.pid))
)
select count(distinct pid) from blocked",
)
.bind(gate_pid)
.fetch_one(registry.pool())
.await
.unwrap();
observed_waiting = waiting;
if waiting == 2 {
break;
}
assert!(!a_task.is_finished() && !b_task.is_finished());
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await;
assert!(
barrier_result.is_ok(),
"both distinct Apply transactions must reach the workspace lock; observed {observed_waiting}"
);
gate.commit().await.unwrap();
let ((a_status, a_report), (b_status, b_report)) =
(a_task.await.unwrap(), b_task.await.unwrap());
assert!(
a_status.is_success() ^ b_status.is_success(),
"exactly one overlapping plan commits: {a_status}/{b_status}"
);
let (winner, loser_status, loser) = if a_status.is_success() {
(&a_report, b_status, &b_report)
} else {
(&b_report, a_status, &a_report)
};
assert_eq!(loser_status, reqwest::StatusCode::CONFLICT);
let error_code = loser
.pointer("/error/context/error_code")
.and_then(Value::as_str)
.unwrap();
assert!(
matches!(
error_code,
"import_plan_fingerprint_stale" | "import_plan_name_conflict"
),
"loser must get the exact typed catalog/name conflict, got {error_code}"
);
let expected = winner["counts"]["created"].as_i64().unwrap();
assert!(expected == 1 || expected == 2);
assert_eq!(
counts(&registry).await,
(expected, expected, 1),
"loser cannot leave partial Operations/versions/application"
);
let (winning_plan, winning_key) = if a_status.is_success() {
(&a, a_key)
} else {
(&b, b_key)
};
let (status, replay) = apply_plan(&client, &server, winning_plan, winning_key).await;
assert_eq!(status, reqwest::StatusCode::OK);
assert_eq!(&replay, winner);
assert_eq!(counts(&registry).await, (expected, expected, 1));
}