bsodfather/crank
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat: harden community production foundation through story 1.5

Browse Source
This commit is contained in:
bsodfather
2026-08-14 00:21:59 +03:00
parent c30461cc92
commit f6fc2e5c9b
161 changed files with 16758 additions and 2515 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/admin-api/Cargo.toml
+5
View File
@@ -10,12 +10,17 @@ version.workspace = true
name = "admin-api"
path = "src/main.rs"
[[bin]]
name = "crank-migrate"
path = "src/bin/crank-migrate.rs"
[dependencies]
argon2.workspace = true
axum.workspace = true
axum-extra.workspace = true
base64.workspace = true
crank-community-auth = { path = "../../crates/crank-community-auth" }
crank-config = { path = "../../crates/crank-config" }
crank-core = { path = "../../crates/crank-core" }
crank-import = { path = "../../crates/crank-import" }
crank-mapping = { path = "../../crates/crank-mapping" }
apps/admin-api/Dockerfile
+6 -42
View File
@@ -1,41 +1,3 @@
FROM rust:1.96.1-bookworm AS deps
WORKDIR /app
COPY Cargo.toml Cargo.lock ./
COPY .sqlx ./.sqlx
COPY apps/admin-api/Cargo.toml apps/admin-api/Cargo.toml
COPY apps/mcp-server/Cargo.toml apps/mcp-server/Cargo.toml
COPY crates/crank-core/Cargo.toml crates/crank-core/Cargo.toml
COPY crates/crank-schema/Cargo.toml crates/crank-schema/Cargo.toml
COPY crates/crank-mapping/Cargo.toml crates/crank-mapping/Cargo.toml
COPY crates/crank-registry/Cargo.toml crates/crank-registry/Cargo.toml
COPY crates/crank-runtime/Cargo.toml crates/crank-runtime/Cargo.toml
COPY crates/crank-adapter-rest/Cargo.toml crates/crank-adapter-rest/Cargo.toml
RUN mkdir -p \
apps/admin-api/src \
apps/mcp-server/src \
crates/crank-core/src \
crates/crank-schema/src \
crates/crank-mapping/src \
crates/crank-registry/src \
crates/crank-runtime/src \
crates/crank-adapter-rest/src \
&& printf 'fn main() {}\n' > apps/admin-api/src/main.rs \
&& printf 'fn main() {}\n' > apps/mcp-server/src/main.rs \
&& printf 'pub fn placeholder() {}\n' > crates/crank-core/src/lib.rs \
&& printf 'pub fn placeholder() {}\n' > crates/crank-schema/src/lib.rs \
&& printf 'pub fn placeholder() {}\n' > crates/crank-mapping/src/lib.rs \
&& printf 'pub fn placeholder() {}\n' > crates/crank-registry/src/lib.rs \
&& printf 'pub fn placeholder() {}\n' > crates/crank-runtime/src/lib.rs \
&& printf 'pub fn placeholder() {}\n' > crates/crank-adapter-rest/src/lib.rs
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git/db \
--mount=type=cache,target=/app/target \
SQLX_OFFLINE=true cargo build --release -p admin-api
FROM rust:1.96.1-bookworm AS builder
WORKDIR /app
@@ -45,11 +7,12 @@ COPY .sqlx ./.sqlx
COPY apps ./apps
COPY crates ./crates
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git/db \
--mount=type=cache,target=/app/target \
RUN --mount=type=cache,id=crank-admin-cargo-registry,target=/usr/local/cargo/registry \
--mount=type=cache,id=crank-admin-cargo-git,target=/usr/local/cargo/git/db \
--mount=type=cache,id=crank-admin-target,target=/app/target \
SQLX_OFFLINE=true cargo build --release -p admin-api \
&& cp /app/target/release/admin-api /tmp/admin-api
&& cp /app/target/release/admin-api /tmp/admin-api \
&& cp /app/target/release/crank-migrate /tmp/crank-migrate
FROM debian:bookworm-slim
@@ -60,6 +23,7 @@ RUN apt-get update \
WORKDIR /app
COPY --from=builder /tmp/admin-api /usr/local/bin/admin-api
COPY --from=builder /tmp/crank-migrate /usr/local/bin/crank-migrate
ENV CRANK_ADMIN_BIND=0.0.0.0:3001
apps/admin-api/src/bin/crank-migrate.rs
+242
View File
@@ -0,0 +1,242 @@
use std::{process::ExitCode, time::Duration};
use crank_config::{ConfigSource, DatabaseSettings, parse_migrator};
use crank_registry::{
BackfillPolicy, MigrationApplyResult, MigrationAuthority, MigrationPreflight,
};
use serde_json::json;
use sqlx::{
PgPool,
postgres::{PgConnectOptions, PgPoolOptions},
};
#[tokio::main]
async fn main() -> ExitCode {
match run().await {
Ok(code) => code,
Err(error) => {
eprintln!(
"{}",
json!({
"status": "error",
"code": error.code,
"stage": error.stage,
"version": error.version,
"recovery": error.recovery,
})
);
ExitCode::FAILURE
}
}
}
#[derive(Clone, Copy)]
struct CliError {
code: &'static str,
stage: &'static str,
recovery: &'static str,
version: Option<i64>,
}
impl CliError {
const fn new(code: &'static str, stage: &'static str, recovery: &'static str) -> Self {
Self {
code,
stage,
recovery,
version: None,
}
}
fn from_migration(error: crank_registry::MigrationError) -> Self {
Self {
code: error.code(),
stage: error.stage(),
recovery: error.recovery(),
version: error.version(),
}
}
}
async fn run() -> Result<ExitCode, CliError> {
let mut arguments = std::env::args().skip(1);
let requested = arguments.next();
let option = arguments.next();
if arguments.next().is_some() {
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_preflight",
));
}
let command = match requested.as_deref() {
None | Some("preflight") => "preflight",
Some("plan") => "plan",
Some("apply") => "apply",
Some(_) => {
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_preflight",
));
}
};
if command == "plan" {
MigrationAuthority::validate_sequence().map_err(CliError::from_migration)?;
let sequence = MigrationAuthority::sequence()
.into_iter()
.map(|migration| {
let backfill = match migration.backfill {
BackfillPolicy::None => json!({ "kind": "none" }),
BackfillPolicy::Bounded {
max_batch_rows,
max_batch_ms,
resumable,
} => json!({
"kind": "bounded",
"max_batch_rows": max_batch_rows,
"max_batch_ms": max_batch_ms,
"resumable": resumable,
}),
};
json!({
"version": migration.version,
"name": migration.name,
"checksum": migration.checksum,
"source_digest": migration.source_digest,
"phase": migration.phase,
"compatibility": migration.compatibility,
"owner": migration.owner,
"transactional": migration.transactional,
"backfill": backfill,
"readable_schema_min": migration.readable_schema_min,
"readable_schema_max": migration.readable_schema_max,
"contract_evidence": migration.contract_evidence,
})
})
.collect::<Vec<_>>();
let plan = json!({ "schema_version": 1, "sequence": sequence });
if option.as_deref() == Some("--check") {
let bytes = std::fs::read("docs/schemas/migration-sequence.json")
.map_err(|_| CliError::new("contract_drift", "plan.read", "contact_operator"))?;
if bytes.len() > 65_536 {
return Err(CliError::new(
"contract_drift",
"plan.size",
"contact_operator",
));
}
let committed: serde_json::Value = serde_json::from_slice(&bytes)
.map_err(|_| CliError::new("contract_drift", "plan.parse", "contact_operator"))?;
if committed != plan {
return Err(CliError::new(
"contract_drift",
"plan.compare",
"contact_operator",
));
}
println!("{}", json!({ "status": "contract_current" }));
return Ok(ExitCode::SUCCESS);
}
if option.is_some() {
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_preflight",
));
}
println!("{plan}");
return Ok(ExitCode::SUCCESS);
}
if option.is_some() {
return Err(CliError::new(
"invalid_command",
"cli.arguments",
"run_preflight",
));
}
let config = parse_migrator(
ConfigSource::from_os_for_migrator()
.map_err(|_| CliError::new("config_invalid", "config.source", "run_preflight"))?,
)
.map_err(|_| CliError::new("config_invalid", "config.validate", "run_preflight"))?;
let pool = connect(&config.database).await?;
if command == "apply" {
let result = MigrationAuthority::apply(&pool)
.await
.map_err(CliError::from_migration)?;
let (status, from, to) = match result {
MigrationApplyResult::Applied { from, to } => ("applied", from, to),
MigrationApplyResult::AlreadyCurrent { version } => {
("already_current", version, version)
}
};
println!(
"{}",
json!({ "status": status, "from_version": from, "to_version": to })
);
return Ok(ExitCode::SUCCESS);
}
match MigrationAuthority::preflight(&pool)
.await
.map_err(CliError::from_migration)?
{
MigrationPreflight::Current { version } => {
println!("{}", json!({ "status": "current", "version": version }));
Ok(ExitCode::SUCCESS)
}
MigrationPreflight::MigrationRequired { current, target } => {
println!(
"{}",
json!({
"status": "migration_required",
"current_version": current,
"target_version": target,
"recovery": "run_controlled_migration",
})
);
Ok(ExitCode::from(2))
}
}
}
async fn connect(config: &DatabaseSettings) -> Result<PgPool, CliError> {
let options = if let Some(url) = &config.url {
url.expose_secret()
.parse::<PgConnectOptions>()
.map_err(|_| CliError::new("config_invalid", "database.source", "run_preflight"))?
} else {
PgConnectOptions::new()
.host(&config.host)
.port(config.port)
.database(&config.database)
.username(&config.username)
.password(config.password.expose_secret())
};
for attempt in 1..=10 {
let result = PgPoolOptions::new()
.max_connections(config.pool.max_connections)
.min_connections(config.pool.min_connections)
.acquire_timeout(Duration::from_millis(config.pool.acquire_timeout_ms))
.idle_timeout(Duration::from_millis(config.pool.idle_timeout_ms))
.max_lifetime(Duration::from_millis(config.pool.max_lifetime_ms))
.connect_with(options.clone())
.await;
match result {
Ok(pool) => return Ok(pool),
Err(_) if attempt < 10 => {
tokio::time::sleep(Duration::from_secs(1)).await;
}
Err(_) => break,
}
}
Err(CliError::new(
"storage_unavailable",
"database.connect",
"contact_operator",
))
}
apps/admin-api/src/dto.rs
+1
View File
@@ -551,6 +551,7 @@ pub(crate) struct InvocationRecordRequest<'a> {
pub agent_id: Option<&'a AgentId>,
pub operation: &'a RegistryOperation,
pub request_id: Option<&'a str>,
pub trace_id: Option<&'a str>,
pub source: InvocationSource,
pub level: InvocationLevel,
pub status: InvocationStatus,
apps/admin-api/src/error.rs
+44 -14
View File
@@ -74,9 +74,9 @@ impl ApiError {
}
}
pub fn internal(message: impl Into<String>) -> Self {
pub fn internal(_message: impl Into<String>) -> Self {
Self::Internal {
message: message.into(),
message: "internal server error".to_owned(),
context: None,
}
}
@@ -165,6 +165,13 @@ impl IntoResponse for ApiError {
if let Some(context) = self.context() {
error["context"] = context;
}
let (request_id, trace_id) = crank_observability::current_request_correlation();
if let Some(request_id) = request_id {
error["request_id"] = Value::String(request_id);
}
if let Some(trace_id) = trace_id {
error["trace_id"] = Value::String(trace_id);
}
let body = Json(json!({
"error": error
@@ -363,9 +370,10 @@ impl From<RegistryError> for ApiError {
format!("import job {job_id} was already applied with different parameters"),
json!({ "job_id": job_id }),
),
RegistryError::Storage(_) | RegistryError::Serialization(_) => {
Self::internal(value.to_string())
}
RegistryError::Migration(_)
| RegistryError::Storage(_)
| RegistryError::Serialization(_)
| RegistryError::InvalidCorrelationIdentity { .. } => Self::internal(value.to_string()),
}
}
}
@@ -399,7 +407,7 @@ impl From<StorageError> for ApiError {
pub fn runtime_test_failure(error: &RuntimeError) -> Value {
let mut payload = json!({
"code": runtime_test_failure_code(error),
"message": error.to_string()
"message": safe_runtime_test_failure_message(error)
});
if let Some(context) = runtime_error_context(error) {
payload["context"] = context;
@@ -407,6 +415,33 @@ pub fn runtime_test_failure(error: &RuntimeError) -> Value {
payload
}
fn safe_runtime_test_failure_message(error: &RuntimeError) -> &'static str {
match error {
RuntimeError::Schema(_) => "input schema validation failed",
RuntimeError::Mapping(_) => "input mapping failed",
RuntimeError::RestAdapter(_) | RuntimeError::ProtocolAdapter(_) => {
"upstream execution failed"
}
RuntimeError::UnsupportedProtocol { .. } => "operation protocol is unsupported",
RuntimeError::ConcurrencyLimitExceeded { .. } => "runtime concurrency limit exceeded",
RuntimeError::InvalidPreparedRequest { .. } => "prepared request is invalid",
RuntimeError::ConfirmationRequired { .. } => "operation confirmation is required",
RuntimeError::InvalidConfirmationToken { .. } => "confirmation token is invalid",
RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation store is unavailable",
RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency store is unavailable",
RuntimeError::IdempotencyInProgress { .. } => "idempotent execution is in progress",
RuntimeError::IdempotencyConflict { .. } => "idempotency key conflicts with the request",
RuntimeError::IdempotencyOutcomeUnknown { .. } => "previous execution outcome is unknown",
RuntimeError::UnsupportedExecutionMode { .. } => "execution mode is unsupported",
RuntimeError::MissingAuthProfile { .. } => "authorization profile is missing",
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
"authorization secret is missing"
}
RuntimeError::InvalidAuthSecretValue { .. } => "authorization secret is invalid",
RuntimeError::SecretCrypto { .. } => "authorization secret processing failed",
}
}
fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
match error {
RuntimeError::Schema(_) => "runtime_schema_error",
@@ -435,9 +470,8 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
match error {
RuntimeError::InvalidPreparedRequest { field, reason } => Some(json!({
RuntimeError::InvalidPreparedRequest { field, .. } => Some(json!({
"field": field,
"reason": reason,
})),
RuntimeError::ConfirmationRequired {
confirmation_token,
@@ -457,14 +491,10 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
| RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({
"operation_id": operation_id,
})),
RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({
RuntimeError::InvalidAuthSecretValue { secret_id, .. } => Some(json!({
"secret_id": secret_id,
"reason": reason,
})),
RuntimeError::SecretCrypto { operation, details } => Some(json!({
"operation": operation,
"details": details,
})),
RuntimeError::SecretCrypto { .. } => None,
RuntimeError::MissingAuthProfile { auth_profile_id } => Some(json!({
"auth_profile_id": auth_profile_id,
})),
apps/admin-api/src/main.rs
+280 -114
View File
@@ -1,4 +1,4 @@
use std::{env, net::SocketAddr, path::PathBuf, time::Duration};
use std::{io, net::SocketAddr, process::ExitCode, time::Duration};
use admin_api::{
app::build_app,
@@ -8,9 +8,14 @@ use admin_api::{
state::AppState,
};
use crank_community_auth::PasswordIdentityProvider;
use crank_config::{
AdminProcessConfig, CacheBackend as ConfigCacheBackend, ConfigSource, DatabaseSettings,
DiagnosticCode, ObservabilitySettings, ProcessKind, parse_process,
};
use crank_core::CacheBackend;
use crank_observability::{
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
capture_critical_error,
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
};
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
use crank_runtime::{
@@ -21,17 +26,77 @@ use sqlx::postgres::PgConnectOptions;
use tokio::net::TcpListener;
use tracing::{info, warn};
const MAX_INVOCATION_LOG_RETENTION_DAYS: i64 = 36_500;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let observability = crank_observability::init(ObservabilityConfig::from_env(
"admin-api",
env!("CARGO_PKG_VERSION"),
"admin_api=info,tower_http=info",
)?)?;
async fn main() -> ExitCode {
match main_result().await {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("{}", safe_startup_diagnostic(error.as_ref()));
ExitCode::FAILURE
}
}
}
fn safe_startup_diagnostic(error: &(dyn std::error::Error + 'static)) -> String {
let mut current = Some(error);
while let Some(cause) = current {
if let Some(config) = cause.downcast_ref::<crank_config::ConfigError>() {
return config.to_json();
}
if let Some(migration) = cause.downcast_ref::<crank_registry::MigrationError>() {
return serde_json::json!({
"status": "error",
"code": migration.code(),
"stage": migration.stage(),
"version": migration.version(),
"recovery": migration.recovery(),
})
.to_string();
}
if let Some(crank_registry::RegistryError::Migration(migration)) =
cause.downcast_ref::<crank_registry::RegistryError>()
{
return serde_json::json!({
"status": "error",
"code": migration.code(),
"stage": migration.stage(),
"version": migration.version(),
"recovery": migration.recovery(),
})
.to_string();
}
current = cause.source();
}
serde_json::json!({
"status": "error",
"code": "startup_failed",
"stage": "startup",
"version": null,
"recovery": "contact_operator",
})
.to_string()
}
async fn main_result() -> Result<(), Box<dyn std::error::Error>> {
let effective = parse_process(ProcessKind::AdminApi, ConfigSource::from_os()?)?;
let config = effective
.admin()
.cloned()
.ok_or_else(|| io::Error::other("admin configuration projection is unavailable"))?;
preflight_config(&config)?;
let observability = init_observability(&config.observability)?;
for deprecation in effective.deprecations() {
warn!(
name: "config.deprecated",
field = deprecation.field,
source_class = deprecation.source_class,
replacement = deprecation.replacement,
removal_window = deprecation.removal_window,
"deprecated configuration accepted"
);
}
let mut startup_completed = false;
let result = run(&observability, &mut startup_completed).await;
let result = run(config, &observability, &mut startup_completed).await;
if result.is_err() {
capture_critical_error(if startup_completed {
CriticalErrorCategory::Internal
@@ -43,54 +108,67 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
async fn run(
config: AdminProcessConfig,
observability: &ObservabilityLifecycle,
startup_completed: &mut bool,
) -> Result<(), Box<dyn std::error::Error>> {
let metrics_config =
MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?;
let metrics_config = MetricsConfig::new(
config.observability.metrics.enabled,
config.observability.metrics.bind_addr,
config
.observability
.metrics
.bearer_token
.as_ref()
.map(|token| token.expose_secret().to_owned()),
)?;
let metrics_enabled = metrics_config.enabled();
let metrics_server = if metrics_config.enabled() {
let pool_config = postgres_pool_config(&config.database)?;
let registry = PostgresRegistry::connect_with_options_and_pool_config(
database_options(&config.database)?,
pool_config,
)
.await?;
let metrics_server = if metrics_enabled {
Some(observability.metrics_surface(metrics_config).bind().await?)
} else {
None
};
let storage_root = PathBuf::from(
env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()),
);
let bind_addr = env::var("CRANK_ADMIN_BIND").unwrap_or_else(|_| "0.0.0.0:3001".into());
let base_url = env::var("CRANK_BASE_URL").unwrap_or_else(|_| "http://localhost:3000".into());
let socket_addr: SocketAddr = bind_addr.parse()?;
let pool_config = PostgresPoolConfig::from_env()?;
let registry = PostgresRegistry::connect_with_options_and_pool_config(
database_options_from_env()?,
pool_config,
)
.await?;
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
}
let base_url = config
.runtime
.base_url
.clone()
.unwrap_or_else(|| "http://localhost:3000".to_owned());
let auth_settings = AuthSettings {
session_secret: env::var("CRANK_SESSION_SECRET")?,
password_pepper: env::var("CRANK_PASSWORD_PEPPER")?,
session_ttl_hours: env::var("CRANK_SESSION_TTL_HOURS")
.ok()
.and_then(|value| value.parse::<i64>().ok())
.unwrap_or(24),
session_secret: config.session_secret.expose_secret().to_owned(),
password_pepper: config.password_pepper.expose_secret().to_owned(),
session_ttl_hours: config.session_ttl_hours,
cookie_secure: base_url.starts_with("https://"),
bootstrap_admin: BootstrapAdminConfig {
email: env::var("CRANK_BOOTSTRAP_ADMIN_EMAIL")?,
password: env::var("CRANK_BOOTSTRAP_ADMIN_PASSWORD")?,
display_name: env::var("CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME")
.unwrap_or_else(|_| "Crank Owner".into()),
email: config.bootstrap_email.clone(),
password: config.bootstrap_password.expose_secret().to_owned(),
display_name: config.bootstrap_display_name.clone(),
},
};
let runtime_limits = RuntimeLimits::from_env()?;
let cache_config = RuntimeCacheConfig::from_env()?;
let runtime_limits = RuntimeLimits::try_new(
config.runtime.max_concurrent_unary,
config.runtime.max_concurrent_sessions,
)?;
let cache_config = runtime_cache_config(&config)?;
let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
let api_rate_limit = admin_api_rate_limit_config_from_env()?;
let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?;
let outbound_http_policy = crank_runtime::OutboundHttpPolicy::from_env()?;
let api_rate_limit = RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
config.rate_limit.burst,
)?;
let secret_crypto = SecretCrypto::new(config.runtime.master_key.expose_secret())?;
let outbound_http_policy = crank_runtime::OutboundHttpPolicy::try_new(
config.runtime.outbound.allowed_hosts.clone(),
config.runtime.outbound.denied_hosts.clone(),
config.runtime.outbound.max_response_bytes,
)?;
let runtime = crank_runtime::community_with_outbound_policy(outbound_http_policy.clone())
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
@@ -100,7 +178,7 @@ async fn run(
PasswordIdentityProvider::new(registry.clone(), auth_settings.password_pepper.clone());
let service = AdminServiceBuilder::new(
registry,
storage_root,
config.storage_root.clone(),
auth_settings,
secret_crypto,
runtime,
@@ -108,12 +186,11 @@ async fn run(
.with_outbound_http_policy(outbound_http_policy)
.with_identity_provider(std::sync::Arc::new(identity_provider))
.build();
let invocation_log_retention_days = invocation_log_retention_days_from_env()?;
service.bootstrap_admin_user().await?;
if env_flag("CRANK_DEMO_SEED") {
if config.demo_seed {
service.seed_demo_assets().await?;
}
spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days);
spawn_invocation_log_cleanup(service.clone(), config.invocation_log_retention_days);
let state = AppState {
service,
api_rate_limiter: if cache_config.backend.is_external() {
@@ -121,10 +198,10 @@ async fn run(
} else {
RequestRateLimiter::new(api_rate_limit)
},
trust_forwarded_headers: env_flag("CRANK_TRUST_FORWARDED_HEADERS"),
trust_forwarded_headers: config.trust_forwarded_headers,
};
let app = build_app(state);
let listener = TcpListener::bind(socket_addr).await?;
let listener = TcpListener::bind(config.bind_addr).await?;
let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
info!(
@@ -138,14 +215,10 @@ async fn run(
acquire_timeout_ms = pool_config.acquire_timeout_ms,
idle_timeout_ms = pool_config.idle_timeout_ms,
max_lifetime_ms = pool_config.max_lifetime_ms,
invocation_log_retention_days,
invocation_log_retention_days = config.invocation_log_retention_days,
"postgres pool configured"
);
info!(
name: "admin.server.listening",
bind_address = %socket_addr,
"admin-api listening"
);
info!(name: "admin.server.listening", bind_address = %config.bind_addr, "admin-api listening");
*startup_completed = true;
if let Some(metrics_server) = metrics_server {
@@ -156,23 +229,163 @@ async fn run(
} else {
axum::serve(listener, make_service).await?;
}
Ok(())
}
fn invocation_log_retention_days_from_env() -> Result<i64, Box<dyn std::error::Error>> {
const NAME: &str = "CRANK_INVOCATION_LOG_RETENTION_DAYS";
let value = match env::var(NAME) {
Ok(raw) => raw.parse::<i64>()?,
Err(env::VarError::NotPresent) => 30,
Err(error) => return Err(error.into()),
};
if !(1..=MAX_INVOCATION_LOG_RETENTION_DAYS).contains(&value) {
return Err(
format!("{NAME} must be between 1 and {MAX_INVOCATION_LOG_RETENTION_DAYS}").into(),
);
fn init_observability(
config: &ObservabilitySettings,
) -> Result<ObservabilityLifecycle, Box<dyn std::error::Error>> {
let identity = ServiceIdentity::try_new(
"admin-api",
env!("CARGO_PKG_VERSION"),
config.environment.clone(),
)?;
let base = ObservabilityConfig::try_new(
identity,
config.log_filter.clone(),
RedactionLimits::default(),
)?;
let sentry = SentryConfig::parse(
config
.sentry_dsn
.as_ref()
.map(|value| value.expose_secret()),
)?;
let otlp = otlp_config(config)?;
Ok(ObservabilityLifecycle::init_with_exporters(
base, sentry, otlp,
)?)
}
fn preflight_config(config: &AdminProcessConfig) -> Result<(), crank_config::ConfigError> {
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
database_options(&config.database).map_err(|_| invalid("database.source"))?;
postgres_pool_config(&config.database).map_err(|_| invalid("database.pool"))?;
MetricsConfig::new(
config.observability.metrics.enabled,
config.observability.metrics.bind_addr,
config
.observability
.metrics
.bearer_token
.as_ref()
.map(|v| v.expose_secret().to_owned()),
)
.map_err(|_| invalid("observability.metrics"))?;
RuntimeLimits::try_new(
config.runtime.max_concurrent_unary,
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
config.rate_limit.burst,
)
.map_err(|_| invalid("admin.rate_limit"))?;
SecretCrypto::new(config.runtime.master_key.expose_secret())
.map_err(|_| invalid("runtime.master_key"))?;
crank_runtime::OutboundHttpPolicy::try_new(
config.runtime.outbound.allowed_hosts.clone(),
config.runtime.outbound.denied_hosts.clone(),
config.runtime.outbound.max_response_bytes,
)
.map_err(|_| invalid("runtime.outbound"))?;
let identity = ServiceIdentity::try_new(
"admin-api",
env!("CARGO_PKG_VERSION"),
config.observability.environment.clone(),
)
.map_err(|_| invalid("observability.environment"))?;
ObservabilityConfig::try_new(
identity,
config.observability.log_filter.clone(),
RedactionLimits::default(),
)
.map_err(|_| invalid("observability.log_filter"))?;
SentryConfig::parse(
config
.observability
.sentry_dsn
.as_ref()
.map(|v| v.expose_secret()),
)
.map_err(|_| invalid("observability.sentry_dsn"))?;
otlp_config(&config.observability).map_err(|_| invalid("observability.otlp"))?;
Ok(())
}
fn otlp_config(
config: &ObservabilitySettings,
) -> Result<OtlpTraceConfig, crank_observability::OtlpTraceConfigError> {
let values = &config.otlp;
OtlpTraceConfig::from_values(
values.endpoint.clone(),
values.traces_endpoint.clone(),
values.protocol.clone(),
values.traces_protocol.clone(),
values.timeout.clone(),
values.traces_timeout.clone(),
values
.headers
.as_ref()
.map(|value| value.expose_secret().to_owned()),
values
.traces_headers
.as_ref()
.map(|value| value.expose_secret().to_owned()),
values.max_queue_size,
values.max_export_batch_size,
values.schedule_delay.clone(),
values.export_timeout.clone(),
)
}
fn postgres_pool_config(
config: &DatabaseSettings,
) -> Result<PostgresPoolConfig, crank_registry::PostgresPoolConfigError> {
PostgresPoolConfig::try_new(
config.pool.max_connections,
config.pool.min_connections,
config.pool.acquire_timeout_ms,
config.pool.idle_timeout_ms,
config.pool.max_lifetime_ms,
)
}
fn database_options(
config: &DatabaseSettings,
) -> Result<PgConnectOptions, Box<dyn std::error::Error>> {
if let Some(url) = &config.url {
return url
.expose_secret()
.parse::<PgConnectOptions>()
.map_err(|_| io::Error::other("database URL is invalid").into());
}
Ok(value)
Ok(PgConnectOptions::new()
.host(&config.host)
.port(config.port)
.database(&config.database)
.username(&config.username)
.password(config.password.expose_secret()))
}
fn runtime_cache_config(
config: &AdminProcessConfig,
) -> Result<RuntimeCacheConfig, crank_runtime::RuntimeCacheConfigError> {
RuntimeCacheConfig::try_new(
match config.runtime.cache.backend {
ConfigCacheBackend::Memory => CacheBackend::Memory,
ConfigCacheBackend::Valkey => CacheBackend::Valkey,
ConfigCacheBackend::Redis => CacheBackend::Redis,
},
config
.runtime
.cache
.url
.as_ref()
.map(|value| value.expose_secret().to_owned()),
)
}
fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
@@ -197,50 +410,3 @@ fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, reten
}
});
}
fn env_flag(name: &str) -> bool {
matches!(
env::var(name)
.ok()
.as_deref()
.map(str::to_ascii_lowercase)
.as_deref(),
Some("1" | "true" | "yes" | "on")
)
}
fn admin_api_rate_limit_config_from_env()
-> Result<RequestRateLimitConfig, Box<dyn std::error::Error>> {
let requests_per_second = env::var("CRANK_ADMIN_RATE_LIMIT_RPS")
.ok()
.and_then(|value| value.parse::<u32>().ok())
.unwrap_or(30);
let burst = env::var("CRANK_ADMIN_RATE_LIMIT_BURST")
.ok()
.and_then(|value| value.parse::<u32>().ok())
.unwrap_or(60);
Ok(RequestRateLimitConfig::new(requests_per_second, burst)?)
}
fn database_options_from_env() -> Result<PgConnectOptions, Box<dyn std::error::Error>> {
if let Ok(database_url) = env::var("CRANK_DATABASE_URL") {
return Ok(database_url.parse::<PgConnectOptions>()?);
}
let host = env::var("POSTGRES_HOST").unwrap_or_else(|_| "postgres".into());
let port = env::var("POSTGRES_PORT")
.ok()
.and_then(|value| value.parse::<u16>().ok())
.unwrap_or(5432);
let database = env::var("POSTGRES_DB").unwrap_or_else(|_| "crank".into());
let username = env::var("POSTGRES_USER").unwrap_or_else(|_| "crank".into());
let password = env::var("POSTGRES_PASSWORD").unwrap_or_else(|_| "crank".into());
Ok(PgConnectOptions::new()
.host(&host)
.port(port)
.database(&database)
.username(&username)
.password(&password))
}
apps/admin-api/src/request_context.rs
+101 -27
View File
@@ -4,20 +4,30 @@ use axum::{
middleware::Next,
response::Response,
};
use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
use crank_core::{CorrelationContext, RequestId, TraceContext};
use crank_observability::{set_remote_trace_parent, with_request_correlation};
use tracing::{Instrument, info, info_span};
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
pub const TRACE_ID_HEADER: HeaderName = HeaderName::from_static("x-trace-id");
#[derive(Clone, Debug)]
pub struct RequestContext {
pub request_id: String,
pub correlation: CorrelationContext,
}
impl RequestContext {
pub fn request_id(&self) -> &str {
self.correlation.request_id().as_str()
}
pub fn trace_id(&self) -> &str {
self.correlation.trace_id().as_str()
}
}
pub async fn apply_request_context(mut request: Request, next: Next) -> Response {
let context = RequestContext {
request_id: RequestId::resolve_from_headers(request.headers()).into_string(),
};
let (request_id, remote_parent) = resolve_correlation(request.headers());
let method = request.method().clone();
let route = request
.extensions()
@@ -27,41 +37,105 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
let span = info_span!(
target: "crank::trace",
"http.request",
request_id = %context.request_id,
request_id = %request_id,
trace_id = tracing::field::Empty,
);
set_remote_trace_parent(&span, request.headers());
if let Some(remote_parent) = remote_parent.as_ref() {
set_canonical_parent(&span, remote_parent);
}
let trace_context = crank_trace::trace_context_for_span(&span).unwrap_or_else(|| {
remote_parent
.as_ref()
.map_or_else(TraceContext::generate, TraceContext::continue_local)
});
span.record("trace_id", trace_context.trace_id().as_str());
let context = RequestContext {
correlation: CorrelationContext::new(request_id, trace_context),
};
request.extensions_mut().insert(context.clone());
with_request_correlation(context.request_id.clone(), async move {
let mut response = next.run(request).instrument(span).await;
info!(
name: "admin.request.completed",
request_id = %context.request_id,
method = %method,
route,
status = response.status().as_u16(),
"admin request completed"
);
if let Ok(value) = HeaderValue::from_str(&context.request_id) {
response.headers_mut().insert(REQUEST_ID_HEADER, value);
}
response
})
with_request_correlation(
context.correlation.request_id().to_string(),
context.correlation.trace_id().to_string(),
async move {
let mut response = next.run(request).instrument(span).await;
info!(
name: "admin.request.completed",
request_id = %context.correlation.request_id(),
trace_id = %context.correlation.trace_id(),
method = %method,
route,
status = response.status().as_u16(),
"admin request completed"
);
if let Ok(value) = HeaderValue::from_str(context.correlation.request_id().as_str()) {
response.headers_mut().insert(REQUEST_ID_HEADER, value);
}
if let Ok(value) = HeaderValue::from_str(context.correlation.trace_id().as_str()) {
response.headers_mut().insert(TRACE_ID_HEADER, value);
}
response
},
)
.await
}
fn resolve_correlation(headers: &axum::http::HeaderMap) -> (RequestId, Option<TraceContext>) {
let _tracestate_accepted = one_auxiliary_header_within_budget(
headers,
"tracestate",
TraceContext::tracestate_within_budget,
);
let _baggage_accepted =
one_auxiliary_header_within_budget(headers, "baggage", TraceContext::baggage_within_budget);
let mut request_ids = headers.get_all(REQUEST_ID_HEADER).iter();
let request_id = request_ids.next().and_then(|value| value.to_str().ok());
let request_id = if request_ids.next().is_some() {
RequestId::generate()
} else {
RequestId::resolve(request_id)
};
let mut traceparents = headers.get_all("traceparent").iter();
let traceparent = traceparents.next().and_then(|value| value.to_str().ok());
let remote_parent = if traceparents.next().is_some() {
None
} else {
traceparent.and_then(|value| TraceContext::parse(value).ok())
};
(request_id, remote_parent)
}
fn one_auxiliary_header_within_budget(
headers: &axum::http::HeaderMap,
name: &'static str,
validate: fn(&str) -> bool,
) -> bool {
let mut values = headers.get_all(name).iter();
let value = values.next().and_then(|value| value.to_str().ok());
values.next().is_none() && value.is_some_and(validate)
}
fn set_canonical_parent(span: &tracing::Span, context: &TraceContext) {
let mut headers = axum::http::HeaderMap::new();
if let Ok(value) = HeaderValue::from_str(context.traceparent()) {
headers.insert("traceparent", value);
set_remote_trace_parent(span, &headers);
}
}
#[cfg(test)]
mod tests {
#[test]
fn accepts_visible_ascii_request_ids() {
assert!(crank_observability::RequestId::is_valid("req_test_123"));
assert!(crank_observability::RequestId::is_valid("trace-123/abc"));
assert!(crank_core::RequestId::is_valid("req_test_123"));
assert!(crank_core::RequestId::is_valid("trace-123/abc"));
}
#[test]
fn rejects_empty_or_control_request_ids() {
assert!(!crank_observability::RequestId::is_valid(""));
assert!(!crank_observability::RequestId::is_valid("bad value"));
assert!(!crank_observability::RequestId::is_valid("bad\nvalue"));
assert!(!crank_core::RequestId::is_valid(""));
assert!(!crank_core::RequestId::is_valid("bad value"));
assert!(!crank_core::RequestId::is_valid("bad\nvalue"));
}
}
apps/admin-api/src/routes/operations.rs
+1 -1
View File
@@ -193,7 +193,7 @@ pub async fn run_test(
&path.workspace_id.as_str().into(),
&path.operation_id.as_str().into(),
payload,
&request_context.request_id,
&request_context.correlation,
)
.await?;
Ok(Json(json!(result)))
apps/admin-api/src/service.rs
+5
View File
@@ -469,6 +469,7 @@ impl AdminService {
tool_name: request.operation.name.clone(),
message: request.message,
request_id: request.request_id.map(ToOwned::to_owned),
trace_id: request.trace_id.map(ToOwned::to_owned),
status_code: request.status_code,
duration_ms: request.duration_ms,
error_kind: request.error_kind,
@@ -506,6 +507,7 @@ impl AdminService {
observe_invocation_history_outcome(
outcome,
request.request_id,
request.trace_id,
request.status,
request.source,
);
@@ -516,6 +518,7 @@ impl AdminService {
fn observe_invocation_history_outcome(
outcome: InvocationHistoryWriteOutcome,
request_id: Option<&str>,
trace_id: Option<&str>,
status: crank_core::InvocationStatus,
source: InvocationSource,
) {
@@ -528,6 +531,7 @@ fn observe_invocation_history_outcome(
tracing::warn!(
name: "admin.invocation_history.lost",
request_id = request_id.unwrap_or_default(),
trace_id = trace_id.unwrap_or_default(),
source = invocation_source_label(source),
invocation_status = invocation_status_label(status),
error_category = loss.category.as_str(),
@@ -935,6 +939,7 @@ mod tests {
category: InvocationHistoryLossCategory::InvalidRecord,
}),
Some("req_admin_dc08"),
Some("0af7651916cd43dd8448eb211c80319c"),
InvocationStatus::Error,
InvocationSource::AgentToolCall,
);
apps/admin-api/src/service/demo.rs
+3 -1
View File
@@ -316,11 +316,13 @@ impl AdminService {
.current_draft_version,
)
.await?;
let correlation = crank_core::CorrelationContext::generate();
self.record_invocation(InvocationRecordRequest {
workspace_id,
agent_id: Some(currency_agent_id),
operation: &rest_operation.snapshot,
request_id: None,
request_id: Some(correlation.request_id().as_str()),
trace_id: Some(correlation.trace_id().as_str()),
source: InvocationSource::AgentToolCall,
level: InvocationLevel::Info,
status: InvocationStatus::Ok,
apps/admin-api/src/service/operations.rs
+7 -2
View File
@@ -444,9 +444,11 @@ impl AdminService {
workspace_id: &WorkspaceId,
operation_id: &OperationId,
payload: TestRunPayload,
request_id: &str,
correlation: &crank_core::CorrelationContext,
) -> Result<TestRunResult, ApiError> {
let runtime_request_context = RuntimeRequestContext::from_request_id(request_id)
let request_id = correlation.request_id().as_str();
let trace_id = correlation.trace_id().as_str();
let runtime_request_context = RuntimeRequestContext::from_correlation(correlation)
.with_metering_context(workspace_id.clone(), None, InvocationSource::AdminTestRun);
let record = self
.get_operation_version(workspace_id, operation_id, payload.version)
@@ -467,6 +469,7 @@ impl AdminService {
agent_id: None,
operation: &record.snapshot,
request_id: Some(request_id),
trace_id: Some(trace_id),
source: InvocationSource::AdminTestRun,
level: InvocationLevel::Error,
status: InvocationStatus::Error,
@@ -516,6 +519,7 @@ impl AdminService {
agent_id: None,
operation: &record.snapshot,
request_id: Some(request_id),
trace_id: Some(trace_id),
source: InvocationSource::AdminTestRun,
level: InvocationLevel::Info,
status: InvocationStatus::Ok,
@@ -543,6 +547,7 @@ impl AdminService {
agent_id: None,
operation: &record.snapshot,
request_id: Some(request_id),
trace_id: Some(trace_id),
source: InvocationSource::AdminTestRun,
level: InvocationLevel::Error,
status: InvocationStatus::Error,
apps/admin-api/tests/config_startup.rs
+71
View File
@@ -0,0 +1,71 @@
use std::process::Command;
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_admin-api"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.envs([
("CRANK_MASTER_KEY", "master"),
("CRANK_SESSION_SECRET", "session"),
("CRANK_PASSWORD_PEPPER", "pepper"),
("CRANK_BOOTSTRAP_ADMIN_EMAIL", "owner@example.test"),
("CRANK_BOOTSTRAP_ADMIN_PASSWORD", "password"),
]);
for (name, value) in entries {
command.env(name, value);
}
let output = command.output().expect("admin binary executes");
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8");
assert!(!stderr.contains("CANARY_SECRET_VALUE"));
assert!(!stderr.contains("connection refused"));
assert!(stderr.len() <= 65_536);
stderr
}
#[test]
fn invalid_config_fails_before_database_or_listener_side_effects() {
for (entries, code) in [
(
vec![("CRANK_CONFIG_CANARY_UNKNOWN", "CANARY_SECRET_VALUE")],
"config.unknown_field",
),
(vec![("POSTGRES_PORT", "bad")], "config.invalid_type"),
(
vec![
("CRANK_DATABASE_URL", "postgres://db/crank"),
("POSTGRES_HOST", "other"),
],
"config.conflict",
),
(vec![("CRANK_LOG_LEVEL", "[")], "config.invalid_type"),
] {
let stderr = run_with(&entries);
assert!(stderr.contains(code), "{stderr}");
}
}
#[test]
fn database_driver_failures_are_normalized_and_redacted() {
let stderr = run_with(&[(
"CRANK_DATABASE_URL",
"postgres://CANARY_SECRET_VALUE:CANARY_SECRET_VALUE@127.0.0.1:1/crank",
)]);
assert!(stderr.contains("startup_failed"), "{stderr}");
}
#[tokio::test]
async fn fresh_database_startup_is_read_only() {
let database_url = crank_test_support::postgres_schema_url("admin_startup_read_only").await;
let stderr = run_with(&[("CRANK_DATABASE_URL", &database_url)]);
assert!(stderr.contains("schema_missing"), "{stderr}");
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present);
}
apps/admin-api/tests/integration.rs
+1
View File
@@ -4,5 +4,6 @@ mod integration {
mod community_access_usage;
mod openapi_import;
mod operations_agents;
mod request_context;
mod secrets_import_auth;
}
apps/admin-api/tests/integration/common.rs
+4
View File
@@ -211,6 +211,10 @@ pub(super) async fn create_lead(Json(payload): Json<Value>) -> Json<Value> {
pub(super) async fn test_registry() -> PostgresRegistry {
let database_url = crank_test_support::postgres_schema_url("test_admin_api").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
crank_registry::MigrationAuthority::apply(&pool)
.await
.unwrap();
let registry = PostgresRegistry::connect(&database_url).await.unwrap();
let password_hash = hash_password(TEST_AUTH_PASSWORD, TEST_PASSWORD_PEPPER).unwrap();
let user_id = registry
apps/admin-api/tests/integration/operations_agents.rs
+73 -7
View File
@@ -10,7 +10,7 @@ use std::{
};
use async_trait::async_trait;
use axum::{Json, Router, routing::post};
use axum::{Json, Router, extract::State, http::HeaderMap, routing::post};
use crank_core::{
ExecutionConfig, HttpMethod, MembershipRole, OperationSecurityLevel, Protocol,
ResponseCachePolicy, RestTarget, SecretKind, Target, ToolDescription, WorkspaceId,
@@ -97,8 +97,8 @@ impl IdentityProvider for RejectingIdentityProvider {
async fn creates_publishes_and_tests_rest_operation() {
let registry = test_registry().await;
let storage_root = test_storage_root("lifecycle");
let upstream_base_url = spawn_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let (upstream_base_url, observed_upstream_headers) = spawn_correlation_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry.clone(), storage_root)).await;
let client = authorized_client(&base_url).await;
let created = client
@@ -132,18 +132,25 @@ async fn creates_publishes_and_tests_rest_operation() {
.json::<Value>()
.await
.unwrap();
let test_run = client
let test_run_response = client
.post(format!("{base_url}/operations/{operation_id}/test-runs"))
.header("x-request-id", "req_admin_test_run")
.header(
"traceparent",
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
)
.json(&json!({
"version": 1,
"input": { "email": "user@example.com" }
}))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert_eq!(
test_run_response.headers()["x-trace-id"].to_str().unwrap(),
"0af7651916cd43dd8448eb211c80319c"
);
let test_run = test_run_response.json::<Value>().await.unwrap();
assert_eq!(listed["items"][0]["name"], "crm_create_lead");
assert_eq!(
@@ -158,6 +165,65 @@ async fn creates_publishes_and_tests_rest_operation() {
"user@example.com"
);
assert_eq!(test_run["response_preview"]["id"], "lead_123");
let logs = registry
.list_invocation_logs(crank_registry::ListInvocationLogsQuery {
workspace_id: &WorkspaceId::new(DEFAULT_WORKSPACE_ID),
level: None,
search_text: None,
source: Some(crank_core::InvocationSource::AdminTestRun),
operation_id: Some(&crank_core::OperationId::new(&operation_id)),
agent_id: None,
created_after: None,
limit: 10,
})
.await
.unwrap();
assert_eq!(logs.len(), 1);
assert_eq!(
logs[0].log.request_id.as_deref(),
Some("req_admin_test_run")
);
assert_eq!(
logs[0].log.trace_id.as_deref(),
Some("0af7651916cd43dd8448eb211c80319c")
);
let upstream_headers = observed_upstream_headers.lock().await;
let upstream_headers = upstream_headers.as_ref().unwrap();
assert_eq!(
upstream_headers["x-request-id"].to_str().unwrap(),
"req_admin_test_run"
);
assert_eq!(
&upstream_headers["traceparent"].to_str().unwrap()[3..35],
"0af7651916cd43dd8448eb211c80319c"
);
}
async fn spawn_correlation_upstream_server() -> (String, Arc<tokio::sync::Mutex<Option<HeaderMap>>>)
{
let observed = Arc::new(tokio::sync::Mutex::new(None));
let app = Router::new()
.route("/crm/leads", post(capture_correlation_and_create_lead))
.with_state(Arc::clone(&observed));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(format!("http://{address}"), observed)
}
async fn capture_correlation_and_create_lead(
State(observed): State<Arc<tokio::sync::Mutex<Option<HeaderMap>>>>,
headers: HeaderMap,
Json(payload): Json<Value>,
) -> Json<Value> {
*observed.lock().await = Some(headers);
Json(json!({
"id": "lead_123",
"status": "created",
"input": payload,
}))
}
#[tokio::test(flavor = "multi_thread")]
apps/admin-api/tests/integration/request_context.rs
+162 -2
View File
@@ -3,10 +3,10 @@ use std::{
sync::{Arc, Mutex},
};
use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
use admin_api::request_context::{REQUEST_ID_HEADER, TRACE_ID_HEADER, apply_request_context};
use axum::{
Router,
body::Body,
body::{Body, to_bytes},
http::{HeaderMap, HeaderValue, Request, StatusCode},
routing::get,
};
@@ -64,6 +64,11 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
.find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
.unwrap();
assert_eq!(event["request_id"], "req_admin_trace_123");
let trace_id = response.headers()[TRACE_ID_HEADER.as_str()]
.to_str()
.unwrap();
assert_eq!(trace_id.len(), 32);
assert_eq!(event["trace_id"], trace_id);
assert_eq!(event["fields"]["status"], 200);
assert_eq!(event["fields"]["route"], "/probe");
@@ -85,9 +90,144 @@ async fn logs_request_completion_and_rejects_untrusted_values() {
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
let generated_trace = invalid_response.headers()[TRACE_ID_HEADER.as_str()]
.to_str()
.unwrap();
assert_eq!(generated_trace.len(), 32);
assert_ne!(generated_trace, "canary-invalid-traceparent");
assert!(!writer.output().contains("canary-invalid-traceparent"));
}
#[tokio::test(flavor = "current_thread")]
async fn structured_boundary_error_carries_the_same_safe_ids() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let dispatch = tracing::Dispatch::new(tracing_subscriber::registry());
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let response = error_probe_app()
.oneshot(
Request::builder()
.uri("/error")
.header("x-request-id", "request-boundary-error")
.header(
"traceparent",
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
assert_eq!(
response.headers()[REQUEST_ID_HEADER],
"request-boundary-error"
);
assert_eq!(
response.headers()[TRACE_ID_HEADER],
"0af7651916cd43dd8448eb211c80319c"
);
let body = to_bytes(response.into_body(), 16 * 1024).await.unwrap();
let payload: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(payload["error"]["request_id"], "request-boundary-error");
assert_eq!(
payload["error"]["trace_id"],
"0af7651916cd43dd8448eb211c80319c"
);
}
#[tokio::test(flavor = "current_thread")]
async fn boundary_status_matrix_keeps_ids_and_redacts_internal_causes() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let app = Router::new()
.route(
"/bad-request",
get(|| async { Err::<(), _>(admin_api::error::ApiError::validation("invalid")) }),
)
.route(
"/unauthorized",
get(|| async {
Err::<(), _>(admin_api::error::ApiError::unauthorized("unauthorized"))
}),
)
.route(
"/forbidden",
get(|| async { Err::<(), _>(admin_api::error::ApiError::forbidden("forbidden")) }),
)
.route(
"/internal",
get(|| async {
Err::<(), _>(admin_api::error::ApiError::internal(
"postgres://canary-user:canary-password@private-host/database",
))
}),
)
.route(
"/rate-limited",
get(|| async { StatusCode::TOO_MANY_REQUESTS }),
)
.layer(axum::middleware::from_fn(apply_request_context));
let responses = async {
let mut responses = Vec::new();
for (path, status) in [
("/bad-request", StatusCode::BAD_REQUEST),
("/unauthorized", StatusCode::UNAUTHORIZED),
("/forbidden", StatusCode::FORBIDDEN),
("/missing", StatusCode::NOT_FOUND),
("/rate-limited", StatusCode::TOO_MANY_REQUESTS),
("/internal", StatusCode::INTERNAL_SERVER_ERROR),
] {
let response = app
.clone()
.oneshot(
Request::builder()
.uri(path)
.header(REQUEST_ID_HEADER.as_str(), "matrix-request-id")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), status);
assert_eq!(response.headers()[REQUEST_ID_HEADER], "matrix-request-id");
assert_eq!(response.headers()[TRACE_ID_HEADER].as_bytes().len(), 32);
responses.push(to_bytes(response.into_body(), 16 * 1024).await.unwrap());
}
responses
};
let dispatch = tracing::Dispatch::new(subscriber);
let _dispatch_guard = tracing::dispatcher::set_default(&dispatch);
let responses = responses.await;
let combined = responses
.iter()
.flat_map(|body| body.iter().copied())
.collect::<Vec<_>>();
assert!(
!combined
.windows("canary-password".len())
.any(|value| value == b"canary-password")
);
assert!(!writer.output().contains("canary-password"));
let internal_event: serde_json::Value = writer
.output()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.find(|event: &serde_json::Value| event["event"] == "admin.response.internal_error")
.unwrap();
assert_eq!(internal_event["request_id"], "matrix-request-id");
assert_eq!(internal_event["trace_id"].as_str().unwrap().len(), 32);
}
#[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
@@ -163,6 +303,14 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
REQUEST_ID_HEADER,
HeaderValue::from_static("second-request-id"),
);
request.headers_mut().append(
"traceparent",
HeaderValue::from_static("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
);
request.headers_mut().append(
"traceparent",
HeaderValue::from_static("00-1af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
);
let response = probe_app().oneshot(request).await.unwrap();
let generated = response.headers()[REQUEST_ID_HEADER].to_str().unwrap();
@@ -173,6 +321,9 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
let trace_id = response.headers()[TRACE_ID_HEADER].to_str().unwrap();
assert_ne!(trace_id, "0af7651916cd43dd8448eb211c80319c");
assert_ne!(trace_id, "1af7651916cd43dd8448eb211c80319c");
}
fn probe_app() -> Router {
@@ -181,6 +332,15 @@ fn probe_app() -> Router {
.layer(axum::middleware::from_fn(apply_request_context))
}
fn error_probe_app() -> Router {
Router::new()
.route(
"/error",
get(|| async { Err::<(), _>(admin_api::error::ApiError::validation("invalid")) }),
)
.layer(axum::middleware::from_fn(apply_request_context))
}
fn trace_probe_app() -> Router {
Router::new()
.route("/trace", get(observed_traceparent))
apps/admin-api/tests/migration_command.rs
+90
View File
@@ -0,0 +1,90 @@
use std::process::{Command, Output};
fn command(arguments: &[&str], database_url: Option<&str>) -> Output {
let mut command = Command::new(env!("CARGO_BIN_EXE_crank-migrate"));
command.args(arguments);
command.current_dir(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."));
for (name, _) in std::env::vars() {
if name.starts_with("CRANK_") || name.starts_with("POSTGRES_") || name.starts_with("OTEL_")
{
command.env_remove(name);
}
}
if let Some(database_url) = database_url {
command.env("CRANK_DATABASE_URL", database_url);
}
command.output().expect("migration command must run")
}
#[test]
fn plan_is_deterministic_and_committed_contract_is_current() {
let first = command(&["plan"], None);
let second = command(&["plan"], None);
assert!(
first.status.success(),
"{}",
String::from_utf8_lossy(&first.stderr)
);
assert_eq!(first.stdout, second.stdout);
let plan: serde_json::Value = serde_json::from_slice(&first.stdout).unwrap();
assert_eq!(plan["sequence"].as_array().unwrap().len(), 3);
let checked = command(&["plan", "--check"], None);
assert!(
checked.status.success(),
"{}",
String::from_utf8_lossy(&checked.stderr)
);
}
#[test]
fn invalid_command_is_bounded_and_does_not_echo_arguments() {
let canary = "secret-command-canary";
let output = command(&[canary], None);
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(stderr.len() < 1_024);
assert!(!stderr.contains(canary));
let diagnostic: serde_json::Value = serde_json::from_str(stderr.trim()).unwrap();
assert_eq!(diagnostic["code"], "invalid_command");
}
#[tokio::test]
async fn database_only_config_can_apply_and_preflight_a_fresh_schema() {
let database_url = crank_test_support::postgres_schema_url("test_migration_command").await;
let applied = command(&["apply"], Some(&database_url));
assert!(
applied.status.success(),
"{}",
String::from_utf8_lossy(&applied.stderr)
);
let result: serde_json::Value = serde_json::from_slice(&applied.stdout).unwrap();
assert_eq!(result["status"], "applied");
let preflight = command(&["preflight"], Some(&database_url));
assert!(
preflight.status.success(),
"{}",
String::from_utf8_lossy(&preflight.stderr)
);
let result: serde_json::Value = serde_json::from_slice(&preflight.stdout).unwrap();
assert_eq!(result["status"], "current");
assert_eq!(result["version"], 3);
}
#[tokio::test]
async fn migration_error_json_preserves_affected_version() {
let database_url = crank_test_support::postgres_schema_url("test_migration_cli_version").await;
assert!(command(&["apply"], Some(&database_url)).status.success());
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
sqlx::query("update __crank_migrations set checksum = 'tampered' where version = 2")
.execute(&pool)
.await
.unwrap();
let output = command(&["preflight"], Some(&database_url));
assert!(!output.status.success());
let diagnostic: serde_json::Value = serde_json::from_slice(&output.stderr).unwrap();
assert_eq!(diagnostic["code"], "checksum_mismatch");
assert_eq!(diagnostic["version"], 2);
}
apps/admin-api/tests/unit/error.rs
+4 -12
View File
@@ -13,27 +13,19 @@ fn runtime_test_failure_includes_structured_context() {
assert_eq!(
payload["context"],
json!({
"field": "request.headers",
"reason": "must be an object"
"field": "request.headers"
})
);
}
#[test]
fn runtime_error_context_includes_secret_crypto_operation() {
fn runtime_error_context_does_not_expose_secret_crypto_details() {
let context = runtime_error_context(&RuntimeError::SecretCrypto {
operation: "decode secret envelope",
details: "bad base64".to_owned(),
})
.unwrap();
});
assert_eq!(
context,
json!({
"operation": "decode secret envelope",
"details": "bad base64"
})
);
assert_eq!(context, None);
}
#[test]
apps/mcp-server/Cargo.toml
+1
View File
@@ -15,6 +15,7 @@ async-trait = "0.1"
axum.workspace = true
base64.workspace = true
crank-community-mcp = { path = "../../crates/crank-community-mcp" }
crank-config = { path = "../../crates/crank-config" }
crank-core = { path = "../../crates/crank-core" }
crank-observability = { path = "../../crates/crank-observability" }
crank-registry = { path = "../../crates/crank-registry" }
apps/mcp-server/Dockerfile
+6 -6
View File
@@ -31,9 +31,9 @@ RUN mkdir -p \
&& printf 'pub fn placeholder() {}\n' > crates/crank-runtime/src/lib.rs \
&& printf 'pub fn placeholder() {}\n' > crates/crank-adapter-rest/src/lib.rs
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git/db \
--mount=type=cache,target=/app/target \
RUN --mount=type=cache,id=crank-mcp-cargo-registry,target=/usr/local/cargo/registry \
--mount=type=cache,id=crank-mcp-cargo-git,target=/usr/local/cargo/git/db \
--mount=type=cache,id=crank-mcp-target,target=/app/target \
SQLX_OFFLINE=true cargo build --release -p mcp-server
FROM rust:1.96.1-bookworm AS builder
@@ -45,9 +45,9 @@ COPY .sqlx ./.sqlx
COPY apps ./apps
COPY crates ./crates
RUN --mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git/db \
--mount=type=cache,target=/app/target \
RUN --mount=type=cache,id=crank-mcp-cargo-registry,target=/usr/local/cargo/registry \
--mount=type=cache,id=crank-mcp-cargo-git,target=/usr/local/cargo/git/db \
--mount=type=cache,id=crank-mcp-target,target=/app/target \
SQLX_OFFLINE=true cargo build --release -p mcp-server \
&& cp /app/target/release/mcp-server /tmp/mcp-server
apps/mcp-server/src/main.rs
+280 -71
View File
@@ -1,12 +1,17 @@
use std::{env, net::SocketAddr, time::Duration};
use std::{io, process::ExitCode, time::Duration};
use crank_community_mcp::{
auth::CommunityMachineCredentialVerifier, build_app_with_background_workers_and_limits,
session::PostgresTransportSessionStore,
};
use crank_config::{
CacheBackend as ConfigCacheBackend, ConfigSource, DatabaseSettings, DiagnosticCode,
McpProcessConfig, ObservabilitySettings, ProcessKind, parse_process,
};
use crank_core::CacheBackend;
use crank_observability::{
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
capture_critical_error,
OtlpTraceConfig, RedactionLimits, SentryConfig, ServiceIdentity, capture_critical_error,
};
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
use crank_runtime::{
@@ -19,14 +24,76 @@ use tokio::net::TcpListener;
use tracing::info;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let observability = crank_observability::init(ObservabilityConfig::from_env(
"mcp-server",
env!("CARGO_PKG_VERSION"),
"mcp_server=info,tower_http=info",
)?)?;
async fn main() -> ExitCode {
match main_result().await {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("{}", safe_startup_diagnostic(error.as_ref()));
ExitCode::FAILURE
}
}
}
fn safe_startup_diagnostic(error: &(dyn std::error::Error + 'static)) -> String {
let mut current = Some(error);
while let Some(cause) = current {
if let Some(config) = cause.downcast_ref::<crank_config::ConfigError>() {
return config.to_json();
}
if let Some(migration) = cause.downcast_ref::<crank_registry::MigrationError>() {
return serde_json::json!({
"status": "error",
"code": migration.code(),
"stage": migration.stage(),
"version": migration.version(),
"recovery": migration.recovery(),
})
.to_string();
}
if let Some(crank_registry::RegistryError::Migration(migration)) =
cause.downcast_ref::<crank_registry::RegistryError>()
{
return serde_json::json!({
"status": "error",
"code": migration.code(),
"stage": migration.stage(),
"version": migration.version(),
"recovery": migration.recovery(),
})
.to_string();
}
current = cause.source();
}
serde_json::json!({
"status": "error",
"code": "startup_failed",
"stage": "startup",
"version": null,
"recovery": "contact_operator",
})
.to_string()
}
async fn main_result() -> Result<(), Box<dyn std::error::Error>> {
let effective = parse_process(ProcessKind::McpServer, ConfigSource::from_os()?)?;
let config = effective
.mcp()
.cloned()
.ok_or_else(|| io::Error::other("MCP configuration projection is unavailable"))?;
preflight_config(&config)?;
let observability = init_observability(&config.observability)?;
for deprecation in effective.deprecations() {
tracing::warn!(
name: "config.deprecated",
field = deprecation.field,
source_class = deprecation.source_class,
replacement = deprecation.replacement,
removal_window = deprecation.removal_window,
"deprecated configuration accepted"
);
}
let mut startup_completed = false;
let result = run(&observability, &mut startup_completed).await;
let result = run(config, &observability, &mut startup_completed).await;
if result.is_err() {
capture_critical_error(if startup_completed {
CriticalErrorCategory::Internal
@@ -38,49 +105,61 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
async fn run(
config: McpProcessConfig,
observability: &ObservabilityLifecycle,
startup_completed: &mut bool,
) -> Result<(), Box<dyn std::error::Error>> {
let metrics_config =
MetricsConfig::from_env("CRANK_MCP_METRICS_BIND", "127.0.0.1:9465".parse()?)?;
let metrics_config = MetricsConfig::new(
config.observability.metrics.enabled,
config.observability.metrics.bind_addr,
config
.observability
.metrics
.bearer_token
.as_ref()
.map(|token| token.expose_secret().to_owned()),
)?;
let metrics_enabled = metrics_config.enabled();
let metrics_server = if metrics_config.enabled() {
let pool_config = postgres_pool_config(&config.database)?;
let runtime_limits = RuntimeLimits::try_new(
config.runtime.max_concurrent_unary,
config.runtime.max_concurrent_sessions,
)?;
let cache_config = runtime_cache_config(&config)?;
let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
let api_rate_limit = RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
config.rate_limit.burst,
)?;
let registry = PostgresRegistry::connect_with_options_and_pool_config(
database_options(&config.database)?,
pool_config,
)
.await?;
let metrics_server = if metrics_enabled {
Some(observability.metrics_surface(metrics_config).bind().await?)
} else {
None
};
let bind_addr = env::var("CRANK_MCP_BIND").unwrap_or_else(|_| "0.0.0.0:3002".into());
let base_url = env::var("CRANK_BASE_URL").ok();
let refresh_interval = env::var("CRANK_MCP_REFRESH_MS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.map(Duration::from_millis)
.unwrap_or_else(|| Duration::from_secs(5));
let socket_addr: SocketAddr = bind_addr.parse()?;
let pool_config = PostgresPoolConfig::from_env()?;
let runtime_limits = RuntimeLimits::from_env()?;
let cache_config = RuntimeCacheConfig::from_env()?;
let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
let api_rate_limit = mcp_api_rate_limit_config_from_env()?;
let database_options = database_options_from_env()?;
let registry =
PostgresRegistry::connect_with_options_and_pool_config(database_options, pool_config)
.await?;
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
}
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?;
let runtime = crank_runtime::community_from_env()?
let secret_crypto = SecretCrypto::new(config.runtime.master_key.expose_secret())?;
let outbound_http_policy = crank_runtime::OutboundHttpPolicy::try_new(
config.runtime.outbound.allowed_hosts.clone(),
config.runtime.outbound.denied_hosts.clone(),
config.runtime.outbound.max_response_bytes,
)?;
let runtime = crank_runtime::community_with_outbound_policy(outbound_http_policy)
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.build();
let app = build_app_with_background_workers_and_limits(
registry,
refresh_interval,
base_url,
Duration::from_millis(config.refresh_ms),
config.runtime.base_url.clone(),
secret_crypto,
runtime,
if cache_config.backend.is_external() {
@@ -93,7 +172,7 @@ async fn run(
std::sync::Arc::new(CommunityMachineCredentialVerifier),
runtime_limits.max_concurrent_sessions,
);
let listener = TcpListener::bind(socket_addr).await?;
let listener = TcpListener::bind(config.bind_addr).await?;
info!(
name: "mcp.postgres_pool.configured",
@@ -109,11 +188,7 @@ async fn run(
max_lifetime_ms = pool_config.max_lifetime_ms,
"postgres pool configured"
);
info!(
name: "mcp.server.listening",
bind_address = %socket_addr,
"mcp-server listening"
);
info!(name: "mcp.server.listening", bind_address = %config.bind_addr, "mcp-server listening");
*startup_completed = true;
if let Some(metrics_server) = metrics_server {
@@ -124,42 +199,176 @@ async fn run(
} else {
axum::serve(listener, app).await?;
}
Ok(())
}
fn database_options_from_env() -> Result<PgConnectOptions, Box<dyn std::error::Error>> {
if let Ok(database_url) = env::var("CRANK_DATABASE_URL") {
return Ok(database_url.parse::<PgConnectOptions>()?);
fn init_observability(
config: &ObservabilitySettings,
) -> Result<ObservabilityLifecycle, Box<dyn std::error::Error>> {
let identity = ServiceIdentity::try_new(
"mcp-server",
env!("CARGO_PKG_VERSION"),
config.environment.clone(),
)?;
let base = ObservabilityConfig::try_new(
identity,
config.log_filter.clone(),
RedactionLimits::default(),
)?;
let sentry = SentryConfig::parse(
config
.sentry_dsn
.as_ref()
.map(|value| value.expose_secret()),
)?;
let values = &config.otlp;
let otlp = OtlpTraceConfig::from_values(
values.endpoint.clone(),
values.traces_endpoint.clone(),
values.protocol.clone(),
values.traces_protocol.clone(),
values.timeout.clone(),
values.traces_timeout.clone(),
values
.headers
.as_ref()
.map(|value| value.expose_secret().to_owned()),
values
.traces_headers
.as_ref()
.map(|value| value.expose_secret().to_owned()),
values.max_queue_size,
values.max_export_batch_size,
values.schedule_delay.clone(),
values.export_timeout.clone(),
)?;
Ok(ObservabilityLifecycle::init_with_exporters(
base, sentry, otlp,
)?)
}
fn preflight_config(config: &McpProcessConfig) -> Result<(), crank_config::ConfigError> {
let invalid = |field| crank_config::ConfigError::single(DiagnosticCode::InvalidType, field);
database_options(&config.database).map_err(|_| invalid("database.source"))?;
postgres_pool_config(&config.database).map_err(|_| invalid("database.pool"))?;
MetricsConfig::new(
config.observability.metrics.enabled,
config.observability.metrics.bind_addr,
config
.observability
.metrics
.bearer_token
.as_ref()
.map(|v| v.expose_secret().to_owned()),
)
.map_err(|_| invalid("observability.metrics"))?;
RuntimeLimits::try_new(
config.runtime.max_concurrent_unary,
config.runtime.max_concurrent_sessions,
)
.map_err(|_| invalid("runtime.limits"))?;
runtime_cache_config(config).map_err(|_| invalid("cache"))?;
RequestRateLimitConfig::new(
config.rate_limit.requests_per_second,
config.rate_limit.burst,
)
.map_err(|_| invalid("mcp.rate_limit"))?;
SecretCrypto::new(config.runtime.master_key.expose_secret())
.map_err(|_| invalid("runtime.master_key"))?;
crank_runtime::OutboundHttpPolicy::try_new(
config.runtime.outbound.allowed_hosts.clone(),
config.runtime.outbound.denied_hosts.clone(),
config.runtime.outbound.max_response_bytes,
)
.map_err(|_| invalid("runtime.outbound"))?;
let identity = ServiceIdentity::try_new(
"mcp-server",
env!("CARGO_PKG_VERSION"),
config.observability.environment.clone(),
)
.map_err(|_| invalid("observability.environment"))?;
ObservabilityConfig::try_new(
identity,
config.observability.log_filter.clone(),
RedactionLimits::default(),
)
.map_err(|_| invalid("observability.log_filter"))?;
SentryConfig::parse(
config
.observability
.sentry_dsn
.as_ref()
.map(|v| v.expose_secret()),
)
.map_err(|_| invalid("observability.sentry_dsn"))?;
let values = &config.observability.otlp;
OtlpTraceConfig::from_values(
values.endpoint.clone(),
values.traces_endpoint.clone(),
values.protocol.clone(),
values.traces_protocol.clone(),
values.timeout.clone(),
values.traces_timeout.clone(),
values
.headers
.as_ref()
.map(|v| v.expose_secret().to_owned()),
values
.traces_headers
.as_ref()
.map(|v| v.expose_secret().to_owned()),
values.max_queue_size,
values.max_export_batch_size,
values.schedule_delay.clone(),
values.export_timeout.clone(),
)
.map_err(|_| invalid("observability.otlp"))?;
Ok(())
}
fn postgres_pool_config(
config: &DatabaseSettings,
) -> Result<PostgresPoolConfig, crank_registry::PostgresPoolConfigError> {
PostgresPoolConfig::try_new(
config.pool.max_connections,
config.pool.min_connections,
config.pool.acquire_timeout_ms,
config.pool.idle_timeout_ms,
config.pool.max_lifetime_ms,
)
}
fn database_options(
config: &DatabaseSettings,
) -> Result<PgConnectOptions, Box<dyn std::error::Error>> {
if let Some(url) = &config.url {
return url
.expose_secret()
.parse::<PgConnectOptions>()
.map_err(|_| io::Error::other("database URL is invalid").into());
}
let host = env::var("POSTGRES_HOST").unwrap_or_else(|_| "postgres".into());
let port = env::var("POSTGRES_PORT")
.ok()
.and_then(|value| value.parse::<u16>().ok())
.unwrap_or(5432);
let database = env::var("POSTGRES_DB").unwrap_or_else(|_| "crank".into());
let username = env::var("POSTGRES_USER").unwrap_or_else(|_| "crank".into());
let password = env::var("POSTGRES_PASSWORD").unwrap_or_else(|_| "crank".into());
Ok(PgConnectOptions::new()
.host(&host)
.port(port)
.database(&database)
.username(&username)
.password(&password))
.host(&config.host)
.port(config.port)
.database(&config.database)
.username(&config.username)
.password(config.password.expose_secret()))
}
fn mcp_api_rate_limit_config_from_env() -> Result<RequestRateLimitConfig, Box<dyn std::error::Error>>
{
let requests_per_second = env::var("CRANK_MCP_RATE_LIMIT_RPS")
.ok()
.and_then(|value| value.parse::<u32>().ok())
.unwrap_or(60);
let burst = env::var("CRANK_MCP_RATE_LIMIT_BURST")
.ok()
.and_then(|value| value.parse::<u32>().ok())
.unwrap_or(120);
Ok(RequestRateLimitConfig::new(requests_per_second, burst)?)
fn runtime_cache_config(
config: &McpProcessConfig,
) -> Result<RuntimeCacheConfig, crank_runtime::RuntimeCacheConfigError> {
RuntimeCacheConfig::try_new(
match config.runtime.cache.backend {
ConfigCacheBackend::Memory => CacheBackend::Memory,
ConfigCacheBackend::Valkey => CacheBackend::Valkey,
ConfigCacheBackend::Redis => CacheBackend::Redis,
},
config
.runtime
.cache
.url
.as_ref()
.map(|value| value.expose_secret().to_owned()),
)
}
apps/mcp-server/tests/config_startup.rs
+65
View File
@@ -0,0 +1,65 @@
use std::process::Command;
fn run_with(entries: &[(&str, &str)]) -> String {
let mut command = Command::new(env!("CARGO_BIN_EXE_mcp-server"));
for field in crank_config::field_registry() {
command.env_remove(field.env_name);
}
command.env("CRANK_MASTER_KEY", "master");
for (name, value) in entries {
command.env(name, value);
}
let output = command.output().expect("MCP binary executes");
assert!(!output.status.success());
let stderr = String::from_utf8(output.stderr).expect("stderr is UTF-8");
assert!(!stderr.contains("CANARY_SECRET_VALUE"));
assert!(!stderr.contains("connection refused"));
assert!(stderr.len() <= 65_536);
stderr
}
#[test]
fn invalid_config_fails_before_database_or_listener_side_effects() {
for (entries, code) in [
(
vec![("CRANK_CONFIG_CANARY_UNKNOWN", "CANARY_SECRET_VALUE")],
"config.unknown_field",
),
(vec![("CRANK_MCP_REFRESH_MS", "bad")], "config.invalid_type"),
(
vec![
("CRANK_DATABASE_URL", "postgres://db/crank"),
("POSTGRES_HOST", "other"),
],
"config.conflict",
),
(vec![("CRANK_LOG_LEVEL", "[")], "config.invalid_type"),
] {
let stderr = run_with(&entries);
assert!(stderr.contains(code), "{stderr}");
}
}
#[test]
fn database_driver_failures_are_normalized_and_redacted() {
let stderr = run_with(&[(
"CRANK_DATABASE_URL",
"postgres://CANARY_SECRET_VALUE:CANARY_SECRET_VALUE@127.0.0.1:1/crank",
)]);
assert!(stderr.contains("startup_failed"), "{stderr}");
}
#[tokio::test]
async fn fresh_database_startup_is_read_only() {
let database_url = crank_test_support::postgres_schema_url("mcp_startup_read_only").await;
let stderr = run_with(&[("CRANK_DATABASE_URL", &database_url)]);
assert!(stderr.contains("schema_missing"), "{stderr}");
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
let present: bool = sqlx::query_scalar(
"select to_regclass(format('%I.%I', current_schema(), '__crank_core_migrations')) is not null",
)
.fetch_one(&pool)
.await
.unwrap();
assert!(!present);
}
apps/mcp-server/tests/integration.rs
+3
View File
@@ -1,6 +1,9 @@
mod integration {
mod catalog_access;
mod common;
mod execution_stages;
mod jsonrpc_correlation;
mod request_context;
mod tool_search;
mod transport_protocol;
}
apps/mcp-server/tests/integration/common.rs
+4
View File
@@ -466,6 +466,10 @@ pub(super) async fn stream_logs()
pub(super) async fn test_registry() -> PostgresRegistry {
let database_url = crank_test_support::postgres_schema_url("test_mcp_server").await;
let pool = sqlx::PgPool::connect(&database_url).await.unwrap();
crank_registry::MigrationAuthority::apply(&pool)
.await
.unwrap();
PostgresRegistry::connect(&database_url).await.unwrap()
}
apps/mcp-server/tests/integration/execution_stages.rs
+33
View File
@@ -154,6 +154,20 @@ async fn exports_real_tool_stages_without_sensitive_data() {
.await;
assert_eq!(call_result.status(), StatusCode::OK);
assert_eq!(
call_result
.headers()
.get("x-request-id")
.and_then(|value| value.to_str().ok()),
Some(REQUEST_ID),
);
assert_eq!(
call_result
.headers()
.get("x-trace-id")
.and_then(|value| value.to_str().ok()),
Some(REMOTE_TRACE_ID),
);
let body = to_bytes(call_result.into_body(), 1024 * 1024)
.await
.unwrap();
@@ -235,6 +249,16 @@ async fn exports_real_tool_stages_without_sensitive_data() {
assert!(!runtime.parent_span_id.is_empty());
assert_eq!(runtime.parent_span_id, root.span_id);
let upstream = trace_spans
.iter()
.find(|span| span.name == "upstream.http")
.expect("upstream attempt");
assert_eq!(
decode_span_id(&traceparent[36..52]).as_slice(),
upstream.span_id.as_slice(),
"outbound traceparent must identify the actual client attempt span",
);
let history = trace_spans
.iter()
.find(|span| span.name == "history.write")
@@ -263,6 +287,7 @@ async fn exports_real_tool_stages_without_sensitive_data() {
.unwrap();
assert_eq!(logs.len(), 1);
assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID));
assert_eq!(logs[0].log.trace_id.as_deref(), Some(REMOTE_TRACE_ID));
provider.shutdown().unwrap();
}
@@ -310,6 +335,14 @@ fn decode_trace_id(value: &str) -> [u8; 16] {
bytes
}
fn decode_span_id(value: &str) -> [u8; 8] {
let mut bytes = [0_u8; 8];
for (index, byte) in bytes.iter_mut().enumerate() {
*byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap();
}
bytes
}
fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> {
span.attributes.iter().find_map(|attribute| {
let value = attribute.value.as_ref()?.value.as_ref()?;
apps/mcp-server/tests/integration/jsonrpc_correlation.rs
+73
View File
@@ -0,0 +1,73 @@
use std::time::Duration;
use crank_core::PlatformApiKeyScope;
use crank_registry::PublishRequest;
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use super::common::{
agent_mcp_url, build_test_app, create_platform_api_key, initialize_session,
post_jsonrpc_response, publish_agent_for_operation, spawn_mcp_server, spawn_upstream_server,
test_operation, test_registry, test_workspace_id,
};
#[tokio::test]
async fn generic_jsonrpc_errors_carry_the_same_safe_response_ids() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_error_identity");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-error-identity").await;
let api_key = create_platform_api_key(
&registry,
"sales-error-identity",
"mcp-error-identity",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let base_url = spawn_mcp_server(build_test_app(registry, Duration::ZERO, None)).await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-error-identity");
let session = initialize_session(&client, &mcp_url, &api_key).await;
let response = post_jsonrpc_response(
&client,
&mcp_url,
&api_key,
Some(&session),
Some("jsonrpc-error-request"),
json!({
"jsonrpc": "2.0",
"id": 91,
"method": "unsupported/method",
"params": {}
}),
)
.await;
let request_id = response.headers()["x-request-id"]
.to_str()
.unwrap()
.to_owned();
let trace_id = response.headers()["x-trace-id"]
.to_str()
.unwrap()
.to_owned();
let payload = response.json::<Value>().await.unwrap();
assert_eq!(request_id, "jsonrpc-error-request");
assert_eq!(payload["error"]["data"]["request_id"], request_id);
assert_eq!(payload["error"]["data"]["trace_id"], trace_id);
}
apps/mcp-server/tests/integration/request_context.rs
+54 -1
View File
@@ -9,7 +9,7 @@ use axum::{
};
use opentelemetry::{
global,
trace::{TraceId, TracerProvider as _},
trace::{SpanId, TraceId, TracerProvider as _},
};
use opentelemetry_sdk::{
error::OTelSdkResult,
@@ -61,6 +61,7 @@ async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
assert_eq!(valid.status, StatusCode::OK);
assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate"));
assert_eq!(valid.trace_id.as_deref(), Some(REMOTE_TRACE_ID));
assert_eq!(invalid.status, StatusCode::OK);
assert_eq!(absent.status, StatusCode::OK);
assert!(valid.traceparent_response.is_none());
@@ -80,6 +81,41 @@ async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
assert_ne!(trace_ids[2], trace_ids[0]);
assert_ne!(trace_ids[1], trace_ids[2]);
assert!(!trace_ids.contains(&TraceId::INVALID));
let request_spans = exported
.lock()
.unwrap()
.iter()
.filter(|span| span.name.as_ref() == "mcp.request")
.cloned()
.collect::<Vec<_>>();
assert_eq!(
request_spans[0].parent_span_id.to_string(),
"b7ad6b7169203331"
);
assert_eq!(request_spans[1].parent_span_id, SpanId::INVALID);
assert_eq!(request_spans[2].parent_span_id, SpanId::INVALID);
provider.shutdown().unwrap();
}
#[tokio::test(flavor = "current_thread")]
async fn sampling_off_still_returns_a_local_trace_identity() {
let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder()
.with_sampler(opentelemetry_sdk::trace::Sampler::AlwaysOff)
.build();
let tracer = provider.tracer("mcp-request-context-sampling-off-test");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let app = build_test_app(test_registry().await, Duration::ZERO, None);
let response = send_health(app, None, None)
.with_subscriber(subscriber)
.await;
let trace_id = response.trace_id.expect("local trace id");
assert_eq!(trace_id.len(), 32);
assert_ne!(trace_id, "00000000000000000000000000000000");
provider.shutdown().unwrap();
}
@@ -98,6 +134,14 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
"x-request-id",
HeaderValue::from_static("second-request-id"),
);
request.headers_mut().append(
"traceparent",
HeaderValue::from_static("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
);
request.headers_mut().append(
"traceparent",
HeaderValue::from_static("00-1af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
);
let response = app
.oneshot(request)
@@ -112,6 +156,9 @@ async fn replaces_multiple_request_id_headers_with_one_uuid_v7() {
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(uuid::Version::SortRand)
);
let trace_id = response.headers()["x-trace-id"].to_str().unwrap();
assert_ne!(trace_id, "0af7651916cd43dd8448eb211c80319c");
assert_ne!(trace_id, "1af7651916cd43dd8448eb211c80319c");
}
async fn send_health(
@@ -143,6 +190,11 @@ async fn send_health(
.get("traceparent")
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
trace_id: response
.headers()
.get("x-trace-id")
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
}
}
@@ -150,6 +202,7 @@ struct ProbeResponse {
status: StatusCode,
request_id: Option<String>,
traceparent_response: Option<String>,
trace_id: Option<String>,
}
#[derive(Clone, Debug)]
apps/mcp-server/tests/integration/transport_protocol.rs
+13
View File
@@ -322,6 +322,10 @@ async fn preserves_request_id_for_tool_call_invocations() {
response.headers()["x-request-id"].to_str().unwrap(),
"req_test_123"
);
let trace_id = response.headers()["x-trace-id"]
.to_str()
.unwrap()
.to_owned();
let call_result = response.json::<Value>().await.unwrap();
assert_eq!(call_result["result"]["isError"], false);
@@ -341,6 +345,7 @@ async fn preserves_request_id_for_tool_call_invocations() {
assert_eq!(logs.len(), 1);
assert_eq!(logs[0].log.request_id.as_deref(), Some("req_test_123"));
assert_eq!(logs[0].log.trace_id.as_deref(), Some(trace_id.as_str()));
}
#[tokio::test]
@@ -408,6 +413,13 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
.to_str()
.unwrap()
.to_owned();
let trace_id = response
.headers()
.get("x-trace-id")
.unwrap()
.to_str()
.unwrap()
.to_owned();
assert_eq!(
uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
Some(Version::SortRand)
@@ -432,6 +444,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
assert_eq!(logs.len(), 1);
assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str()));
assert_eq!(logs[0].log.trace_id.as_deref(), Some(trace_id.as_str()));
}
#[tokio::test(flavor = "current_thread")]
apps/ui/js/api.js
+15 -3
View File
@@ -8,6 +8,18 @@
}, extra || {});
}
function attachCorrelation(error, response) {
var requestId = response.headers.get('x-request-id');
var traceId = response.headers.get('x-trace-id');
if (requestId && requestId.length <= 128 && /^[!-~]+$/.test(requestId) && requestId.indexOf(',') === -1 && requestId.indexOf(';') === -1) {
error.requestId = requestId;
}
if (traceId && /^[0-9a-f]{32}$/.test(traceId) && traceId !== '00000000000000000000000000000000') {
error.traceId = traceId;
}
return error;
}
async function request(path, options) {
var response = await fetch(path, Object.assign({
credentials: 'same-origin',
@@ -42,11 +54,11 @@
var error = new Error(message);
error.status = response.status;
error.payload = payload;
throw error;
throw attachCorrelation(error, response);
}
if (text && payload === null) {
throw new Error('Backend returned a non-JSON response');
throw attachCorrelation(new Error('Backend returned a non-JSON response'), response);
}
return payload;
@@ -81,7 +93,7 @@
var error = new Error(message);
error.status = response.status;
error.payload = payload;
throw error;
throw attachCorrelation(error, response);
}
return text;
apps/ui/playwright.config.js
+7 -3
View File
@@ -26,10 +26,14 @@ module.exports = defineConfig({
: {
command: 'bash scripts/playwright-stack.sh',
cwd: __dirname,
url: `${baseURL}/login`,
timeout: 600_000,
reuseExistingServer: false,
url: `${baseURL}/login`,
timeout: 600_000,
reuseExistingServer: false,
gracefulShutdown: {
signal: 'SIGTERM',
timeout: 10_000,
},
},
projects: [
{
name: 'chromium',
apps/ui/scripts/playwright-stack.sh
+15 -4
View File
@@ -67,7 +67,8 @@ cleanup() {
kill_port_processes "$MCP_PORT"
}
trap cleanup EXIT INT TERM
trap cleanup EXIT
trap 'exit 130' INT TERM
cleanup
@@ -131,7 +132,13 @@ mkdir -p "$CRANK_STORAGE_ROOT"
(
cd "$ROOT_DIR"
cargo run -p admin-api >"$LOG_DIR/admin-api.log" 2>&1
cargo run -p admin-api --bin crank-migrate -- apply >"$LOG_DIR/migrate.log" 2>&1
)
(
cd "$ROOT_DIR"
exec env -u CRANK_MCP_BIND -u CRANK_MCP_REFRESH_MS \
cargo run -p admin-api --bin admin-api >"$LOG_DIR/admin-api.log" 2>&1
) &
echo $! > "$TMP_DIR/admin-api.pid"
@@ -141,7 +148,11 @@ done
(
cd "$ROOT_DIR"
cargo run -p mcp-server >"$LOG_DIR/mcp-server.log" 2>&1
exec env -u CRANK_ADMIN_BIND -u CRANK_STORAGE_ROOT -u CRANK_SESSION_SECRET \
-u CRANK_PASSWORD_PEPPER -u CRANK_SESSION_TTL_HOURS \
-u CRANK_BOOTSTRAP_ADMIN_EMAIL -u CRANK_BOOTSTRAP_ADMIN_PASSWORD \
-u CRANK_BOOTSTRAP_ADMIN_DISPLAY_NAME -u CRANK_DEMO_SEED \
cargo run -p mcp-server >"$LOG_DIR/mcp-server.log" 2>&1
) &
echo $! > "$TMP_DIR/mcp-server.pid"
@@ -151,7 +162,7 @@ done
(
cd "$ROOT_DIR/apps/ui"
node scripts/playwright-ui-server.js >"$LOG_DIR/ui-server.log" 2>&1
exec node scripts/playwright-ui-server.js >"$LOG_DIR/ui-server.log" 2>&1
) &
echo $! > "$TMP_DIR/ui-server.pid"
apps/ui/tests/e2e/observability.spec.js
+44
View File
@@ -28,3 +28,47 @@ test('secrets page exposes stable secret management hooks', async ({ page }) =>
await expect(page.locator('[data-testid="secret-submit-button"]')).toBeVisible();
await expect(page.locator('html')).toHaveAttribute('data-crank-bootstrap-state', 'ready');
});
test('API errors retain only bounded canonical support identities', async ({ page }) => {
await login(page);
await page.route('**/api/admin/workspaces/correlation-*/operations', async (route) => {
var hostile = route.request().url().includes('correlation-hostile');
await route.fulfill({
status: 503,
contentType: 'application/json',
headers: hostile
? { 'x-request-id': 'reflected;attacker', 'x-trace-id': 'NOT-A-TRACE' }
: {
'x-request-id': '01J5SAFELOCALREQUEST',
'x-trace-id': '0123456789abcdef0123456789abcdef',
},
body: JSON.stringify({ error: { message: 'safe failure' } }),
});
});
var safe = await page.evaluate(async () => {
try {
await window.CrankApi.listOperations('correlation-safe');
return null;
} catch (error) {
return { requestId: error.requestId, traceId: error.traceId };
}
});
expect(safe).toEqual({
requestId: '01J5SAFELOCALREQUEST',
traceId: '0123456789abcdef0123456789abcdef',
});
var hostile = await page.evaluate(async () => {
try {
await window.CrankApi.listOperations('correlation-hostile');
return null;
} catch (error) {
return {
hasRequestId: Object.prototype.hasOwnProperty.call(error, 'requestId'),
hasTraceId: Object.prototype.hasOwnProperty.call(error, 'traceId'),
};
}
});
expect(hostile).toEqual({ hasRequestId: false, hasTraceId: false });
});