feat: harden community production foundation through story 1.5

apps/mcp-server/tests/config_startup.rs

@@ -0,0 +1,65 @@
+use std::process::Command;
+
+fn run_with(entries: &[(&str, &str)]) -> String {
+    let mut command = Command::new(env!("CARGO_BIN_EXE_mcp-server"));
+    for field in crank_config::field_registry() {
+        command.env_remove(field.env_name);
+    }
+    command.env("CRANK_MASTER_KEY", "master");
+    for (name, value) in entries {
+        command.env(name, value);
+    }
+    let output = command.output().expect("MCP binary executes");
+    assert!(!output.status.success());
+    let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8");
+    assert!(!stderr.contains("CANARY_SECRET_VALUE"));
+    assert!(!stderr.contains("connection refused"));
+    assert!(stderr.len() <= 65_536);
+    stderr
+}
+
+#[test]
+fn invalid_config_fails_before_database_or_listener_side_effects() {
+    for (entries, code) in [
+        (
+            vec![("CRANK_CONFIG_CANARY_UNKNOWN", "CANARY_SECRET_VALUE")],
+            "config.unknown_field",
+        ),
+        (vec![("CRANK_MCP_REFRESH_MS", "bad")], "config.invalid_type"),
+        (
+            vec![
+                ("CRANK_DATABASE_URL", "postgres://db/crank"),
+                ("POSTGRES_HOST", "other"),
+            ],
+            "config.conflict",
+        ),
+        (vec![("CRANK_LOG_LEVEL", "[")], "config.invalid_type"),
+    ] {
+        let stderr = run_with(&entries);
+        assert!(stderr.contains(code), "{stderr}");
+    }
+}
+
+#[test]
+fn database_driver_failures_are_normalized_and_redacted() {
+    let stderr = run_with(&[(
+        "CRANK_DATABASE_URL",
+        "postgres://CANARY_SECRET_VALUE:CANARY_SECRET_VALUE@127.0.0.1:1/crank",
+    )]);
+    assert!(stderr.contains("startup_failed"), "{stderr}");
+}
+
+#[tokio::test]
+async fn fresh_database_startup_is_read_only() {
+    let database_url = crank_test_support::postgres_schema_url("mcp_startup_read_only").await;
+    let stderr = run_with(&[("CRANK_DATABASE_URL", &database_url)]);
+    assert!(stderr.contains("schema_missing"), "{stderr}");
+    let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
+    let present: bool = sqlx::query_scalar(
+        "select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
+    )
+    .fetch_one(&pool)
+    .await
+    .unwrap();
+    assert!(!present);
+}

apps/mcp-server/tests/integration.rs

@@ -1,6 +1,9 @@
 mod integration {
     mod catalog_access;
     mod common;
+    mod execution_stages;
+    mod jsonrpc_correlation;
+    mod request_context;
     mod tool_search;
     mod transport_protocol;
 }

apps/mcp-server/tests/integration/common.rs

@@ -466,6 +466,10 @@ pub(super) async fn stream_logs()
 
 pub(super) async fn test_registry() -> PostgresRegistry {
     let database_url = crank_test_support::postgres_schema_url("test_mcp_server").await;
+    let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
+    crank_registry::MigrationAuthority::apply(&pool)
+        .await
+        .unwrap();
     PostgresRegistry::connect(&database_url).await.unwrap()
 }
 

apps/mcp-server/tests/integration/execution_stages.rs

@@ -154,6 +154,20 @@ async fn exports_real_tool_stages_without_sensitive_data() {
     .await;
 
     assert_eq!(call_result.status(), StatusCode::OK);
+    assert_eq!(
+        call_result
+            .headers()
+            .get("x-request-id")
+            .and_then(|value| value.to_str().ok()),
+        Some(REQUEST_ID),
+    );
+    assert_eq!(
+        call_result
+            .headers()
+            .get("x-trace-id")
+            .and_then(|value| value.to_str().ok()),
+        Some(REMOTE_TRACE_ID),
+    );
     let body = to_bytes(call_result.into_body(), 1024 * 1024)
         .await
         .unwrap();
@@ -235,6 +249,16 @@ async fn exports_real_tool_stages_without_sensitive_data() {
     assert!(!runtime.parent_span_id.is_empty());
     assert_eq!(runtime.parent_span_id, root.span_id);
 
+    let upstream = trace_spans
+        .iter()
+        .find(|span| span.name == "upstream.http")
+        .expect("upstream attempt");
+    assert_eq!(
+        decode_span_id(&traceparent[36..52]).as_slice(),
+        upstream.span_id.as_slice(),
+        "outbound traceparent must identify the actual client attempt span",
+    );
+
     let history = trace_spans
         .iter()
         .find(|span| span.name == "history.write")
@@ -263,6 +287,7 @@ async fn exports_real_tool_stages_without_sensitive_data() {
         .unwrap();
     assert_eq!(logs.len(), 1);
     assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID));
+    assert_eq!(logs[0].log.trace_id.as_deref(), Some(REMOTE_TRACE_ID));
 
     provider.shutdown().unwrap();
 }
@@ -310,6 +335,14 @@ fn decode_trace_id(value: &str) -> [u8; 16] {
     bytes
 }
 
+fn decode_span_id(value: &str) -> [u8; 8] {
+    let mut bytes = [0_u8; 8];
+    for (index, byte) in bytes.iter_mut().enumerate() {
+        *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap();
+    }
+    bytes
+}
+
 fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> {
     span.attributes.iter().find_map(|attribute| {
         let value = attribute.value.as_ref()?.value.as_ref()?;

apps/mcp-server/tests/integration/jsonrpc_correlation.rs

@@ -0,0 +1,73 @@
+use std::time::Duration;
+
+use crank_core::PlatformApiKeyScope;
+use crank_registry::PublishRequest;
+use serde_json::{Value, json};
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+
+use super::common::{
+    agent_mcp_url, build_test_app, create_platform_api_key, initialize_session,
+    post_jsonrpc_response, publish_agent_for_operation, spawn_mcp_server, spawn_upstream_server,
+    test_operation, test_registry, test_workspace_id,
+};
+
+#[tokio::test]
+async fn generic_jsonrpc_errors_carry_the_same_safe_response_ids() {
+    let registry = test_registry().await;
+    let upstream_base_url = spawn_upstream_server().await;
+    let operation = test_operation(&upstream_base_url, "crm_error_identity");
+    registry
+        .create_operation(&test_workspace_id(), &operation, Some("alice"))
+        .await
+        .unwrap();
+    registry
+        .publish_operation(PublishRequest {
+            workspace_id: &test_workspace_id(),
+            operation_id: &operation.id,
+            version: 1,
+            published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
+            published_by: Some("alice"),
+        })
+        .await
+        .unwrap();
+    publish_agent_for_operation(&registry, &operation, "sales-error-identity").await;
+    let api_key = create_platform_api_key(
+        &registry,
+        "sales-error-identity",
+        "mcp-error-identity",
+        &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
+    )
+    .await;
+    let base_url = spawn_mcp_server(build_test_app(registry, Duration::ZERO, None)).await;
+    let client = reqwest::Client::new();
+    let mcp_url = agent_mcp_url(&base_url, "sales-error-identity");
+    let session = initialize_session(&client, &mcp_url, &api_key).await;
+
+    let response = post_jsonrpc_response(
+        &client,
+        &mcp_url,
+        &api_key,
+        Some(&session),
+        Some("jsonrpc-error-request"),
+        json!({
+            "jsonrpc": "2.0",
+            "id": 91,
+            "method": "unsupported/method",
+            "params": {}
+        }),
+    )
+    .await;
+    let request_id = response.headers()["x-request-id"]
+        .to_str()
+        .unwrap()
+        .to_owned();
+    let trace_id = response.headers()["x-trace-id"]
+        .to_str()
+        .unwrap()
+        .to_owned();
+    let payload = response.json::<Value>().await.unwrap();
+
+    assert_eq!(request_id, "jsonrpc-error-request");
+    assert_eq!(payload["error"]["data"]["request_id"], request_id);
+    assert_eq!(payload["error"]["data"]["trace_id"], trace_id);
+}

apps/mcp-server/tests/integration/request_context.rs

@@ -9,7 +9,7 @@ use axum::{
 };
 use opentelemetry::{
     global,
-    trace::{TraceId, TracerProvider as _},
+    trace::{SpanId, TraceId, TracerProvider as _},
 };
 use opentelemetry_sdk::{
     error::OTelSdkResult,
@@ -61,6 +61,7 @@ async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
 
     assert_eq!(valid.status, StatusCode::OK);
     assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate"));
+    assert_eq!(valid.trace_id.as_deref(), Some(REMOTE_TRACE_ID));
     assert_eq!(invalid.status, StatusCode::OK);
     assert_eq!(absent.status, StatusCode::OK);
     assert!(valid.traceparent_response.is_none());
@@ -80,6 +81,41 @@ async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
     assert_ne!(trace_ids[2], trace_ids[0]);
     assert_ne!(trace_ids[1], trace_ids[2]);
     assert!(!trace_ids.contains(&TraceId::INVALID));
+    let request_spans = exported
+        .lock()
+        .unwrap()
+        .iter()
+        .filter(|span| span.name.as_ref() == "mcp.request")
+        .cloned()
+        .collect::<Vec<_>>();
+    assert_eq!(
+        request_spans[0].parent_span_id.to_string(),
+        "b7ad6b7169203331"
+    );
+    assert_eq!(request_spans[1].parent_span_id, SpanId::INVALID);
+    assert_eq!(request_spans[2].parent_span_id, SpanId::INVALID);
+    provider.shutdown().unwrap();
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn sampling_off_still_returns_a_local_trace_identity() {
+    let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
+    global::set_text_map_propagator(TraceContextPropagator::new());
+    let provider = SdkTracerProvider::builder()
+        .with_sampler(opentelemetry_sdk::trace::Sampler::AlwaysOff)
+        .build();
+    let tracer = provider.tracer("mcp-request-context-sampling-off-test");
+    let subscriber =
+        tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+    let app = build_test_app(test_registry().await, Duration::ZERO, None);
+
+    let response = send_health(app, None, None)
+        .with_subscriber(subscriber)
+        .await;
+
+    let trace_id = response.trace_id.expect("local trace id");
+    assert_eq!(trace_id.len(), 32);
+    assert_ne!(trace_id, "00000000000000000000000000000000");
     provider.shutdown().unwrap();
 }
 
@@ -98,6 +134,14 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
         "x-request-id",
         HeaderValue::from_static("second-request-id"),
     );
+    request.headers_mut().append(
+        "traceparent",
+        HeaderValue::from_static("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+    );
+    request.headers_mut().append(
+        "traceparent",
+        HeaderValue::from_static("00-1af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
+    );
 
     let response = app
         .oneshot(request)
@@ -112,6 +156,9 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
         uuid::Uuid::parse_str(generated).unwrap().get_version(),
         Some(uuid::Version::SortRand)
     );
+    let trace_id = response.headers()["x-trace-id"].to_str().unwrap();
+    assert_ne!(trace_id, "0af7651916cd43dd8448eb211c80319c");
+    assert_ne!(trace_id, "1af7651916cd43dd8448eb211c80319c");
 }
 
 async fn send_health(
@@ -143,6 +190,11 @@ async fn send_health(
             .get("traceparent")
             .and_then(|value| value.to_str().ok())
             .map(str::to_owned),
+        trace_id: response
+            .headers()
+            .get("x-trace-id")
+            .and_then(|value| value.to_str().ok())
+            .map(str::to_owned),
     }
 }
 
@@ -150,6 +202,7 @@ struct ProbeResponse {
     status: StatusCode,
     request_id: Option<String>,
     traceparent_response: Option<String>,
+    trace_id: Option<String>,
 }
 
 #[derive(Clone, Debug)]

apps/mcp-server/tests/integration/transport_protocol.rs

@@ -322,6 +322,10 @@ async fn preserves_request_id_for_tool_call_invocations() {
         response.headers()["x-request-id"].to_str().unwrap(),
         "req_test_123"
     );
+    let trace_id = response.headers()["x-trace-id"]
+        .to_str()
+        .unwrap()
+        .to_owned();
     let call_result = response.json::<Value>().await.unwrap();
     assert_eq!(call_result["result"]["isError"], false);
 
@@ -341,6 +345,7 @@ async fn preserves_request_id_for_tool_call_invocations() {
 
     assert_eq!(logs.len(), 1);
     assert_eq!(logs[0].log.request_id.as_deref(), Some("req_test_123"));
+    assert_eq!(logs[0].log.trace_id.as_deref(), Some(trace_id.as_str()));
 }
 
 #[tokio::test]
@@ -408,6 +413,13 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
         .to_str()
         .unwrap()
         .to_owned();
+    let trace_id = response
+        .headers()
+        .get("x-trace-id")
+        .unwrap()
+        .to_str()
+        .unwrap()
+        .to_owned();
     assert_eq!(
         uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
         Some(Version::SortRand)
@@ -432,6 +444,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
 
     assert_eq!(logs.len(), 1);
     assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str()));
+    assert_eq!(logs[0].log.trace_id.as_deref(), Some(trace_id.as_str()));
 }
 
 #[tokio::test(flavor = "current_thread")]