наблюдаемость: завершить базовый контур Community
CI / Rust Checks (push) Failing after 4m28s
CI / UI Checks (push) Has been skipped
CI / Frontend E2E (push) Has been skipped
CI / Community Image Smoke (push) Has been skipped
CI / Deploy (push) Has been skipped

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.
This commit is contained in:
2026-07-31 01:01:14 +03:00
parent 99bd05c145
commit 0e8f1ca03a
160 changed files with 13506 additions and 1499 deletions
+8
View File
@@ -3,6 +3,7 @@ name = "admin-api"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[[bin]]
@@ -18,9 +19,12 @@ crank-community-auth = { path = "../../crates/crank-community-auth" }
crank-core = { path = "../../crates/crank-core" }
crank-import = { path = "../../crates/crank-import" }
crank-mapping = { path = "../../crates/crank-mapping" }
crank-observability = { path = "../../crates/crank-observability" }
crank-registry = { path = "../../crates/crank-registry" }
crank-runtime = { path = "../../crates/crank-runtime" }
crank-schema = { path = "../../crates/crank-schema" }
crank-trace = { path = "../../crates/crank-trace" }
metrics.workspace = true
rand.workspace = true
serde.workspace = true
serde_json.workspace = true
@@ -37,5 +41,9 @@ uuid.workspace = true
[dev-dependencies]
async-trait = "0.1"
crank-test-support = { path = "../../crates/crank-test-support" }
opentelemetry.workspace = true
opentelemetry_sdk.workspace = true
reqwest.workspace = true
serial_test = "3"
tower.workspace = true
tracing-opentelemetry.workspace = true
+4
View File
@@ -166,6 +166,7 @@ pub fn build_app(state: AppState) -> Router {
Router::new()
.route("/health", get(crate::routes::health))
.route("/ready", get(crate::routes::readiness))
.nest(
"/api/auth",
Router::new()
@@ -178,6 +179,9 @@ pub fn build_app(state: AppState) -> Router {
apply_api_rate_limit,
))
.layer(middleware::from_fn(apply_request_context))
.layer(middleware::from_fn(
crank_observability::record_http_request,
))
.with_state(state)
}
+6 -1
View File
@@ -269,7 +269,12 @@ pub struct CreatedPlatformApiKeyResponse {
}
#[derive(Clone, Debug, Serialize)]
pub struct WorkspaceExportResponse {
pub struct WorkspaceCatalogSnapshotResponse {
pub kind: String,
pub format_version: String,
pub restorable: bool,
pub included: Vec<String>,
pub excluded: Vec<String>,
pub workspace: WorkspaceRecord,
pub operations: Vec<OperationSummaryView>,
pub agents: Vec<AgentSummaryView>,
+30 -10
View File
@@ -137,16 +137,24 @@ impl ApiError {
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
match &self {
Self::Internal { message, .. } => {
error!(error_code = self.code(), error_message = %message)
Self::Internal { .. } => {
error!(
name: "admin.response.internal_error",
error_code = self.code(),
"internal API error response"
)
}
Self::Unauthorized { message, .. }
| Self::Forbidden { message, .. }
| Self::Validation { message, .. }
| Self::NotFound { message, .. }
| Self::Conflict { message, .. }
| Self::RateLimited { message, .. } => {
warn!(error_code = self.code(), error_message = %message)
Self::Unauthorized { .. }
| Self::Forbidden { .. }
| Self::Validation { .. }
| Self::NotFound { .. }
| Self::Conflict { .. }
| Self::RateLimited { .. } => {
warn!(
name: "admin.response.rejected",
error_code = self.code(),
"API request rejected"
)
}
}
@@ -351,6 +359,10 @@ impl From<RegistryError> for ApiError {
format!("import job {job_id} was not found"),
json!({ "job_id": job_id }),
),
RegistryError::ImportJobAlreadyApplied { job_id } => Self::conflict_with_context(
format!("import job {job_id} was already applied with different parameters"),
json!({ "job_id": job_id }),
),
RegistryError::Storage(_) | RegistryError::Serialization(_) => {
Self::internal(value.to_string())
}
@@ -407,6 +419,10 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
RuntimeError::ConfirmationRequired { .. } => "runtime_confirmation_required",
RuntimeError::InvalidConfirmationToken { .. } => "runtime_confirmation_error",
RuntimeError::ConfirmationStoreUnavailable { .. } => "runtime_confirmation_unavailable",
RuntimeError::IdempotencyStoreUnavailable { .. } => "runtime_idempotency_unavailable",
RuntimeError::IdempotencyInProgress { .. } => "runtime_idempotency_in_progress",
RuntimeError::IdempotencyConflict { .. } => "runtime_idempotency_conflict",
RuntimeError::IdempotencyOutcomeUnknown { .. } => "runtime_idempotency_outcome_unknown",
RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error",
RuntimeError::MissingAuthProfile { .. } => "runtime_auth_profile_error",
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
@@ -434,7 +450,11 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
"safety_class": safety_class,
})),
RuntimeError::InvalidConfirmationToken { operation_id }
| RuntimeError::ConfirmationStoreUnavailable { operation_id } => Some(json!({
| RuntimeError::ConfirmationStoreUnavailable { operation_id }
| RuntimeError::IdempotencyStoreUnavailable { operation_id }
| RuntimeError::IdempotencyInProgress { operation_id }
| RuntimeError::IdempotencyConflict { operation_id }
| RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({
"operation_id": operation_id,
})),
RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({
+106 -10
View File
@@ -1,4 +1,4 @@
use std::{env, net::SocketAddr, path::PathBuf};
use std::{env, net::SocketAddr, path::PathBuf, time::Duration};
use admin_api::{
app::build_app,
@@ -7,22 +7,50 @@ use admin_api::{
state::AppState,
};
use crank_community_auth::PasswordIdentityProvider;
use crank_observability::{
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
capture_critical_error,
};
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use sqlx::postgres::PgConnectOptions;
use sqlx::{PgPool, postgres::PgConnectOptions};
use tokio::net::TcpListener;
use tracing::info;
use tracing::{info, warn};
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt()
.with_env_filter(
env::var("CRANK_LOG_LEVEL").unwrap_or_else(|_| "admin_api=info,tower_http=info".into()),
)
.init();
let observability = crank_observability::init(ObservabilityConfig::from_env(
"admin-api",
env!("CARGO_PKG_VERSION"),
"admin_api=info,tower_http=info",
)?)?;
let mut startup_completed = false;
let result = run(&observability, &mut startup_completed).await;
if result.is_err() {
capture_critical_error(if startup_completed {
CriticalErrorCategory::Internal
} else {
CriticalErrorCategory::Startup
});
}
result
}
async fn run(
observability: &ObservabilityLifecycle,
startup_completed: &mut bool,
) -> Result<(), Box<dyn std::error::Error>> {
let metrics_config =
MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?;
let metrics_enabled = metrics_config.enabled();
let metrics_server = if metrics_config.enabled() {
Some(observability.metrics_surface(metrics_config).bind().await?)
} else {
None
};
let storage_root = PathBuf::from(
env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()),
@@ -36,6 +64,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
pool_config,
)
.await?;
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
}
let auth_settings = AuthSettings {
session_secret: env::var("CRANK_SESSION_SECRET")?,
password_pepper: env::var("CRANK_PASSWORD_PEPPER")?,
@@ -74,10 +105,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.with_outbound_http_policy(outbound_http_policy)
.with_identity_provider(std::sync::Arc::new(identity_provider))
.build();
let invocation_log_retention_days =
positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
service.bootstrap_admin_user().await?;
if env_flag("CRANK_DEMO_SEED") {
service.seed_demo_assets().await?;
}
spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days);
let state = AppState {
service,
api_rate_limiter: if cache_config.backend.is_external() {
@@ -92,6 +126,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
info!(
name: "admin.postgres_pool.configured",
runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
admin_rate_limit_rps = api_rate_limit.requests_per_second,
admin_rate_limit_burst = api_rate_limit.burst,
@@ -101,15 +136,76 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
acquire_timeout_ms = pool_config.acquire_timeout_ms,
idle_timeout_ms = pool_config.idle_timeout_ms,
max_lifetime_ms = pool_config.max_lifetime_ms,
invocation_log_retention_days,
"postgres pool configured"
);
info!("admin-api listening on {}", socket_addr);
info!(
name: "admin.server.listening",
bind_address = %socket_addr,
"admin-api listening"
);
*startup_completed = true;
axum::serve(listener, make_service).await?;
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = axum::serve(listener, make_service) => result?,
result = metrics_server.serve() => result?,
}
} else {
axum::serve(listener, make_service).await?;
}
Ok(())
}
fn positive_i64_from_env(
name: &'static str,
default: i64,
) -> Result<i64, Box<dyn std::error::Error>> {
let value = match env::var(name) {
Ok(raw) => raw.parse::<i64>()?,
Err(env::VarError::NotPresent) => default,
Err(error) => return Err(error.into()),
};
if value <= 0 {
return Err(format!("{name} must be greater than zero").into());
}
Ok(value)
}
fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60 * 60));
loop {
interval.tick().await;
let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days);
match service.cleanup_invocation_logs_before(cutoff).await {
Ok(removed) if removed > 0 => info!(
name: "admin.invocation_log_cleanup.completed",
removed,
"expired invocation logs removed"
),
Ok(_) => {}
Err(_) => warn!(
name: "admin.invocation_log_cleanup.failed",
error_category = "registry_cleanup",
"failed to remove expired invocation logs"
),
}
}
});
}
fn spawn_postgres_pool_metrics(pool: PgPool) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
}
});
}
fn env_flag(name: &str) -> bool {
matches!(
env::var(name)
+11 -6
View File
@@ -6,7 +6,7 @@ use axum::{
middleware::Next,
response::Response,
};
use crank_runtime::RateLimitRejection;
use crank_runtime::{RateLimitCheckError, RateLimitRejection};
use crate::{error::ApiError, state::AppState};
@@ -25,11 +25,16 @@ pub async fn apply_api_rate_limit(
peer_ip,
state.trust_forwarded_headers,
);
if let Err(rejection) = state.api_rate_limiter.check(&key).await {
return Err(ApiError::rate_limited_with_context(
"request rate limit exceeded",
rejection_context(rejection),
));
if let Err(error) = state.api_rate_limiter.check(&key).await {
return match error {
RateLimitCheckError::Rejected(rejection) => Err(ApiError::rate_limited_with_context(
"request rate limit exceeded",
rejection_context(rejection),
)),
RateLimitCheckError::StoreUnavailable => {
Err(ApiError::internal("rate limit service unavailable"))
}
};
}
Ok(next.run(request).await)
+35 -128
View File
@@ -4,11 +4,10 @@ use axum::{
middleware::Next,
response::Response,
};
use tracing::info;
use uuid::Uuid;
use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
use tracing::{Instrument, info, info_span};
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
const MAX_REQUEST_ID_LEN: usize = 128;
#[derive(Clone, Debug)]
pub struct RequestContext {
@@ -21,145 +20,53 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
};
let method = request.method().clone();
let path = request.uri().path().to_owned();
let span = info_span!(
target: "crank::trace",
"http.request",
request_id = %context.request_id,
);
set_remote_trace_parent(&span, request.headers());
request.extensions_mut().insert(context.clone());
let mut response = next.run(request).await;
info!(
request_id = %context.request_id,
method = %method,
path,
status = response.status().as_u16(),
"admin request completed"
);
if let Ok(value) = HeaderValue::from_str(&context.request_id) {
response.headers_mut().insert(REQUEST_ID_HEADER, value);
}
response
with_request_correlation(context.request_id.clone(), async move {
let mut response = next.run(request).instrument(span).await;
info!(
name: "admin.request.completed",
request_id = %context.request_id,
method = %method,
path,
status = response.status().as_u16(),
"admin request completed"
);
if let Ok(value) = HeaderValue::from_str(&context.request_id) {
response.headers_mut().insert(REQUEST_ID_HEADER, value);
}
response
})
.await
}
fn resolve_request_id(headers: &HeaderMap) -> String {
headers
.get(&REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| is_valid_request_id(value))
.map(ToOwned::to_owned)
.unwrap_or_else(|| Uuid::now_v7().to_string())
}
fn is_valid_request_id(value: &str) -> bool {
!value.is_empty()
&& value.len() <= MAX_REQUEST_ID_LEN
&& value
.bytes()
.all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
RequestId::resolve(
headers
.get(&REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok()),
)
.into_string()
}
#[cfg(test)]
mod tests {
use std::io;
use std::sync::{Arc, Mutex};
use axum::{Router, routing::get};
use reqwest::Client;
use tokio::net::TcpListener;
use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
use super::{REQUEST_ID_HEADER, apply_request_context, is_valid_request_id};
#[test]
fn accepts_visible_ascii_request_ids() {
assert!(is_valid_request_id("req_test_123"));
assert!(is_valid_request_id("trace-123/abc"));
assert!(crank_observability::RequestId::is_valid("req_test_123"));
assert!(crank_observability::RequestId::is_valid("trace-123/abc"));
}
#[test]
fn rejects_empty_or_control_request_ids() {
assert!(!is_valid_request_id(""));
assert!(!is_valid_request_id("bad value"));
assert!(!is_valid_request_id("bad\nvalue"));
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[tokio::test]
async fn logs_request_completion_with_request_id() {
let writer = SharedLogWriter::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(writer.clone())
.without_time()
.with_ansi(false)
.with_target(false)
.compact()
.with_filter(LevelFilter::INFO),
);
let dispatch = tracing::Dispatch::new(subscriber);
let app = Router::new()
.route("/probe", get(|| async { "ok" }))
.layer(axum::middleware::from_fn(apply_request_context));
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let _guard = tracing::dispatcher::set_default(&dispatch);
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let response = Client::new()
.get(format!("http://{address}/probe"))
.header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(
response.headers()[REQUEST_ID_HEADER.as_str()]
.to_str()
.unwrap(),
"req_admin_trace_123"
);
let logs = writer.output();
assert!(logs.contains("admin request completed"));
assert!(logs.contains("req_admin_trace_123"));
assert!(logs.contains("GET"));
assert!(logs.contains("/probe"));
assert!(logs.contains("status=200"));
assert!(!crank_observability::RequestId::is_valid(""));
assert!(!crank_observability::RequestId::is_valid("bad value"));
assert!(!crank_observability::RequestId::is_valid("bad\nvalue"));
}
}
+25 -1
View File
@@ -10,12 +10,36 @@ pub mod secrets;
pub mod upstreams;
pub mod workspaces;
use axum::Json;
use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
use serde_json::json;
use crate::state::AppState;
pub async fn health() -> Json<serde_json::Value> {
Json(json!({
"service": "admin-api",
"status": "ok"
}))
}
pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
match state.service.readiness().await {
Ok(()) => (
StatusCode::OK,
Json(json!({
"service": "admin-api",
"status": "ready",
"checks": { "postgres": "ready" }
})),
),
Err(error) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"service": "admin-api",
"status": "not_ready",
"checks": { "postgres": "not_ready" },
"error": error.to_string()
})),
),
}
}
+1 -1
View File
@@ -18,7 +18,7 @@ pub async fn export_workspace(
) -> Result<Json<Value>, ApiError> {
let exported = state
.service
.export_workspace(&path.workspace_id.as_str().into())
.export_workspace_catalog_snapshot(&path.workspace_id.as_str().into())
.await?;
Ok(Json(json!(exported)))
}
+1 -1
View File
@@ -103,7 +103,7 @@ pub async fn change_password(
) -> Result<StatusCode, ApiError> {
state
.service
.change_password(&session.user.id, payload)
.change_password(&session.user.id, &session.session_id, payload)
.await?;
Ok(StatusCode::NO_CONTENT)
}
+248 -47
View File
@@ -12,16 +12,18 @@ use crank_core::{
};
use crank_mapping::{MappingRule, MappingSet};
use crank_registry::{
AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary,
OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef,
OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
};
use crank_runtime::{
OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
};
use crank_schema::{Schema, SchemaKind};
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tracing::Instrument;
use uuid::Uuid;
mod agents;
@@ -74,6 +76,11 @@ pub struct AdminServiceBuilder {
pub use crate::dto::*;
impl AdminService {
pub async fn readiness(&self) -> Result<(), ApiError> {
self.registry.ping().await?;
Ok(())
}
#[cfg(test)]
pub fn new(
registry: PostgresRegistry,
@@ -199,16 +206,33 @@ impl AdminServiceBuilder {
}
impl AdminService {
pub async fn export_workspace(
pub async fn export_workspace_catalog_snapshot(
&self,
workspace_id: &WorkspaceId,
) -> Result<WorkspaceExportResponse, ApiError> {
) -> Result<WorkspaceCatalogSnapshotResponse, ApiError> {
let workspace = self.get_workspace(workspace_id).await?;
let operations = self.list_operations(workspace_id).await?;
let agents = self.list_agents(workspace_id).await?;
let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?;
Ok(WorkspaceExportResponse {
Ok(WorkspaceCatalogSnapshotResponse {
kind: "workspace_catalog_snapshot".to_owned(),
format_version: "1".to_owned(),
restorable: false,
included: vec![
"workspace_settings".to_owned(),
"operation_summaries".to_owned(),
"agent_summaries".to_owned(),
"platform_api_key_metadata".to_owned(),
],
excluded: vec![
"operation_versions_and_samples".to_owned(),
"agent_versions_and_bindings".to_owned(),
"secret_metadata_and_values".to_owned(),
"secret_values".to_owned(),
"invocation_logs_and_usage".to_owned(),
"authentication_sessions".to_owned(),
],
workspace,
operations,
agents,
@@ -239,9 +263,13 @@ impl AdminService {
return Ok(None);
};
let auth_profile = self
.registry
.get_auth_profile(workspace_id, auth_profile_id)
let span = Stage::AuthResolve.span();
let result = async {
let auth_profile = observe_db_query(
DbOperation::AuthProfileRead,
self.registry
.get_auth_profile(workspace_id, auth_profile_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load auth profile",
@@ -251,9 +279,20 @@ impl AdminService {
auth_profile_id: auth_profile_id.as_str().to_owned(),
})?;
self.resolve_auth_profile(workspace_id, &auth_profile)
.await
.map(Some)
self.resolve_auth_profile(workspace_id, &auth_profile)
.await
.map(Some)
}
.instrument(span.clone())
.await;
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Configuration.record(&span);
}
}
result
}
async fn resolve_auth_profile(
@@ -265,40 +304,46 @@ impl AdminService {
let used_at = OffsetDateTime::now_utc();
for secret_id in auth_profile.config.secret_ids() {
let secret = self
.registry
.get_secret(workspace_id, secret_id)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load secret",
details: error.to_string(),
})?
.ok_or_else(|| RuntimeError::MissingSecret {
secret_id: secret_id.as_str().to_owned(),
})?;
let version = self
.registry
.get_current_secret_version(workspace_id, secret_id)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load current secret version",
details: error.to_string(),
})?
.ok_or_else(|| RuntimeError::MissingSecretVersion {
secret_id: secret_id.as_str().to_owned(),
version: secret.secret.current_version,
})?;
let secret = observe_db_query(
DbOperation::SecretRead,
self.registry.get_secret(workspace_id, secret_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load secret",
details: error.to_string(),
})?
.ok_or_else(|| RuntimeError::MissingSecret {
secret_id: secret_id.as_str().to_owned(),
})?;
let version = observe_db_query(
DbOperation::SecretRead,
self.registry
.get_current_secret_version(workspace_id, secret_id),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "load current secret version",
details: error.to_string(),
})?
.ok_or_else(|| RuntimeError::MissingSecretVersion {
secret_id: secret_id.as_str().to_owned(),
version: secret.secret.current_version,
})?;
let plaintext = self.secret_crypto.decrypt(
&version.secret_version.key_version,
&version.secret_version.ciphertext,
)?;
self.registry
.touch_secret(workspace_id, secret_id, &used_at)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "touch secret",
details: error.to_string(),
})?;
observe_db_query(
DbOperation::SecretTouch,
self.registry
.touch_secret(workspace_id, secret_id, &used_at),
)
.await
.map_err(|error| RuntimeError::SecretCrypto {
operation: "touch secret",
details: error.to_string(),
})?;
secrets.insert(secret_id.clone(), plaintext);
}
@@ -412,7 +457,7 @@ impl AdminService {
async fn record_invocation(
&self,
request: InvocationRecordRequest<'_>,
) -> Result<(), ApiError> {
) -> InvocationHistoryWriteOutcome {
let log = InvocationLog {
id: InvocationLogId::new(new_prefixed_id("log")),
workspace_id: request.workspace_id.clone(),
@@ -432,11 +477,70 @@ impl AdminService {
created_at: OffsetDateTime::now_utc(),
};
self.registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.await?;
let history_span = crank_trace::Stage::HistoryWrite.span();
let (outcome, db_span) = async {
let db_span = crank_trace::Stage::DbQuery
.db_span(crank_trace::DbOperation::InvocationHistoryWrite)
.expect("database stage");
let outcome = self
.registry
.create_invocation_log(CreateInvocationLogRequest { log: &log })
.instrument(db_span.clone())
.await;
(outcome, db_span)
}
.instrument(history_span.clone())
.await;
match outcome {
InvocationHistoryWriteOutcome::Recorded => {
crank_trace::StageOutcome::Success.record(&db_span);
crank_trace::StageOutcome::Success.record(&history_span);
}
InvocationHistoryWriteOutcome::Lost(_) => {
crank_trace::StageOutcome::Error.record(&db_span);
crank_trace::ErrorCategory::Database.record(&db_span);
crank_trace::StageOutcome::Error.record(&history_span);
crank_trace::ErrorCategory::History.record(&history_span);
}
}
drop(db_span);
drop(history_span);
observe_invocation_history_outcome(
outcome,
request.request_id,
request.status,
"admin_test_run",
);
outcome
}
}
Ok(())
fn observe_invocation_history_outcome(
outcome: InvocationHistoryWriteOutcome,
request_id: Option<&str>,
status: crank_core::InvocationStatus,
source: &'static str,
) {
let Some(loss) = outcome.loss() else {
return;
};
crank_observability::record_operational_incident(
crank_observability::OperationalIncident::InvocationHistoryLost,
);
tracing::warn!(
name: "admin.invocation_history.lost",
request_id = request_id.unwrap_or_default(),
source,
invocation_status = invocation_status_label(status),
error_category = loss.category.as_str(),
"invocation history was not recorded"
);
}
fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
match status {
crank_core::InvocationStatus::Ok => "ok",
crank_core::InvocationStatus::Error => "error",
}
}
@@ -543,6 +647,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
RuntimeError::RestAdapter(_) => "rest_error",
RuntimeError::ProtocolAdapter(_) => "adapter_error",
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
@@ -761,7 +869,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule {
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::{validate_profile_display_name, validate_profile_email};
use std::{
io,
sync::{Arc, Mutex},
};
use crank_core::InvocationStatus;
use crank_observability::{
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
operational_incident_total,
};
use crank_registry::{
InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
};
use serde_json::Value;
use tracing_subscriber::fmt::MakeWriter;
use super::{
observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email,
};
#[test]
fn validates_profile_identity_fields() {
@@ -782,6 +908,81 @@ mod tests {
assert!(validate_profile_display_name(&"x".repeat(81)).is_err());
assert!(validate_profile_email("owner <html>@crank.local").is_err());
}
#[test]
fn emits_bounded_history_loss_incident() {
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
observe_invocation_history_outcome(
InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
category: InvocationHistoryLossCategory::InvalidRecord,
}),
Some("req_admin_dc08"),
InvocationStatus::Error,
"admin_test_run",
);
let output = writer.output();
assert!(!output.contains("dc08-canary-secret"));
let event: Value = output
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.find(|event: &Value| event["event"] == "admin.invocation_history.lost")
.unwrap();
assert_eq!(event["request_id"], "req_admin_dc08");
assert_eq!(event["fields"]["source"], "admin_test_run");
assert_eq!(event["fields"]["invocation_status"], "error");
assert_eq!(event["fields"]["error_category"], "invalid_record");
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
}
fn enrich_operation_summary(
+23 -4
View File
@@ -296,7 +296,12 @@ impl AdminService {
bindings: &[],
})
.await?;
info!(agent_id = %agent_id.as_str(), version = 1, "agent created");
info!(
name: "admin.agent.created",
agent_id = %agent_id.as_str(),
version = 1,
"agent created"
);
Ok(CreatedAgentResponse {
agent_id: agent_id.as_str().to_owned(),
@@ -417,6 +422,7 @@ impl AdminService {
})
.await?;
info!(
name: "admin.agent.bindings_saved",
agent_id = %agent_id.as_str(),
version = current_version.version,
binding_count = bindings.len(),
@@ -531,7 +537,12 @@ impl AdminService {
published_by: None,
})
.await?;
info!(agent_id = %agent_id.as_str(), version, "agent published");
info!(
name: "admin.agent.published",
agent_id = %agent_id.as_str(),
version,
"agent published"
);
Ok(PublishAgentResponse {
agent_id: agent_id.as_str().to_owned(),
@@ -589,7 +600,11 @@ impl AdminService {
self.registry
.unpublish_agent(workspace_id, agent_id, &updated_at)
.await?;
info!(agent_id = %agent_id.as_str(), "agent moved to draft");
info!(
name: "admin.agent.unpublished",
agent_id = %agent_id.as_str(),
"agent moved to draft"
);
Ok(AgentMutationResult {
agent_id: agent_id.as_str().to_owned(),
@@ -609,7 +624,11 @@ impl AdminService {
self.registry
.archive_agent(workspace_id, agent_id, &updated_at)
.await?;
info!(agent_id = %agent_id.as_str(), "agent archived");
info!(
name: "admin.agent.archived",
agent_id = %agent_id.as_str(),
"agent archived"
);
Ok(AgentMutationResult {
agent_id: agent_id.as_str().to_owned(),
+7 -2
View File
@@ -22,7 +22,7 @@ impl AdminService {
)?;
let user_id = self
.registry
.upsert_bootstrap_user(
.ensure_bootstrap_user(
&self.auth_settings.bootstrap_admin.email,
&self.auth_settings.bootstrap_admin.display_name,
&password_hash,
@@ -227,6 +227,7 @@ impl AdminService {
pub async fn change_password(
&self,
user_id: &crank_core::UserId,
current_session_id: &UserSessionId,
payload: ChangePasswordPayload,
) -> Result<(), ApiError> {
if payload.new_password.len() < 12 {
@@ -257,7 +258,11 @@ impl AdminService {
let password_hash =
hash_password(&payload.new_password, &self.auth_settings.password_pepper)?;
self.registry
.update_user_password(user_id, &password_hash)
.update_user_password_and_revoke_other_sessions(
user_id,
current_session_id,
&password_hash,
)
.await?;
Ok(())
+2 -1
View File
@@ -131,6 +131,7 @@ impl AdminService {
Ok(()) => Ok(()),
Err(RegistryError::OperationHasPublishedAgentBindings { .. }) => {
tracing::warn!(
name: "admin.demo_operation.cleanup_skipped",
operation_id = %operation_id.as_str(),
"legacy demo operation is still bound to a published agent; leaving it in place"
);
@@ -335,7 +336,7 @@ impl AdminService {
}),
response_preview: demo_rest_response_sample(),
})
.await?;
.await;
Ok(())
}
}
@@ -110,6 +110,7 @@ impl AdminService {
warnings,
};
info!(
name: "admin.operation.imported",
operation_id = %response.operation_id,
version = response.version,
"operation imported by upsert"
@@ -125,6 +126,7 @@ impl AdminService {
warnings,
};
info!(
name: "admin.operation.imported",
operation_id = %response.operation_id,
version = response.version,
"operation imported by upsert"
+86 -76
View File
@@ -8,9 +8,11 @@ use crank_import::rest::{
ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
};
use crank_registry::{
CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus,
ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind,
ImportJobStatus, ImportOperationDraft,
};
use serde_json::json;
use sha2::{Digest, Sha256};
use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
use tracing::{info, instrument};
@@ -50,7 +52,7 @@ impl AdminService {
kind: ImportJobKind::OpenApi,
source_format: &preview.source.format,
source_version: preview.source.version.as_deref(),
status: ImportJobStatus::Completed,
status: ImportJobStatus::Pending,
preview_payload: &preview_payload,
created_at: &now,
expires_at: &expires_at,
@@ -99,9 +101,13 @@ impl AdminService {
return Err(ApiError::validation("import job kind is not openapi"));
}
let preview: crank_import::rest::ImportPreview =
serde_json::from_value(job.preview_payload.clone())
.map_err(|error| ApiError::internal(error.to_string()))?;
let stored_preview = job
.preview_payload
.get("preview")
.cloned()
.unwrap_or_else(|| job.preview_payload.clone());
let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview)
.map_err(|error| ApiError::internal(error.to_string()))?;
let selected = payload
.selected_operation_keys
.iter()
@@ -120,10 +126,8 @@ impl AdminService {
}
}
let mut created = Vec::new();
let mut skipped = Vec::new();
let mut findings = Vec::new();
let mut created_ids = Vec::new();
let mut operations = Vec::new();
for operation_key in selected {
let Some(candidate) = candidates.get(&operation_key) else {
@@ -137,44 +141,7 @@ impl AdminService {
let mut draft =
operation_draft_from_candidate(candidate, payload.server_url.as_deref());
attach_import_findings(&mut draft, candidate);
if let Some(existing_name) = self
.find_operation_by_name(workspace_id, &draft.name)
.await?
.map(|operation| operation.name)
{
if payload.conflict_mode == "skip" {
skipped.push(OpenApiImportSkippedOperation {
operation_key: candidate.key.clone(),
name: draft.name.clone(),
reason: "operation with this name already exists".to_owned(),
});
findings.push(ImportFinding {
code: "operation_name_conflict".to_owned(),
severity: ImportFindingSeverity::Warning,
message: format!(
"Операция {} уже существует и была пропущена.",
draft.name
),
operation_key: Some(candidate.key.clone()),
});
continue;
}
let renamed = self
.next_available_operation_name(workspace_id, &draft.name)
.await?;
findings.push(ImportFinding {
code: "operation_name_renamed".to_owned(),
severity: ImportFindingSeverity::Info,
message: format!(
"Операция {existing_name} уже существует, новый черновик создан как {renamed}."
),
operation_key: Some(candidate.key.clone()),
});
draft.name = renamed;
}
let payload = OperationPayload {
let operation = self.new_operation_snapshot(OperationPayload {
name: draft.name.clone(),
display_name: draft.display_name.clone(),
category: draft.category,
@@ -197,27 +164,74 @@ impl AdminService {
},
tool_description: draft.tool_description,
wizard_state: draft.wizard_state,
};
let result = self.create_operation(workspace_id, payload).await?;
created_ids.push(result.operation_id.clone());
created.push(OpenApiImportCreatedOperation {
operation_id: result.operation_id,
name: draft.name,
version: result.version,
})?;
operations.push(ImportOperationDraft {
operation_key: candidate.key.clone(),
operation,
});
}
let finished_at = OffsetDateTime::now_utc();
self.registry
.finish_import_job(FinishImportJobRequest {
let application_key = openapi_application_key(&payload)?;
let conflict_mode = if payload.conflict_mode == "skip" {
ImportConflictMode::Skip
} else {
ImportConflictMode::Rename
};
let applied = self
.registry
.apply_import_job(ApplyImportJobRequest {
id: job_id,
status: ImportJobStatus::Completed,
created_operation_ids: &json!(created_ids),
error_text: None,
workspace_id,
application_key: &application_key,
conflict_mode,
operations: &operations,
finished_at: &finished_at,
})
.await?;
let created = applied
.created
.iter()
.map(|operation| OpenApiImportCreatedOperation {
operation_id: operation.operation_id.as_str().to_owned(),
name: operation.name.clone(),
version: operation.version,
})
.collect::<Vec<_>>();
let mut findings = applied
.created
.iter()
.filter_map(|operation| {
operation.renamed_from.as_ref().map(|previous_name| ImportFinding {
code: "operation_name_renamed".to_owned(),
severity: ImportFindingSeverity::Info,
message: format!(
"Операция {previous_name} уже существует, новый черновик создан как {}.",
operation.name
),
operation_key: Some(operation.operation_key.clone()),
})
})
.collect::<Vec<_>>();
for operation in applied.skipped {
skipped.push(OpenApiImportSkippedOperation {
operation_key: operation.operation_key.clone(),
name: operation.name.clone(),
reason: "operation with this name already exists".to_owned(),
});
findings.push(ImportFinding {
code: operation.reason,
severity: ImportFindingSeverity::Warning,
message: format!(
"Операция {} уже существует и была пропущена.",
operation.name
),
operation_key: Some(operation.operation_key),
});
}
info!(
name: "admin.openapi_import.completed",
created = created.len(),
skipped = skipped.len(),
"openapi import created drafts"
@@ -229,25 +243,21 @@ impl AdminService {
findings,
})
}
}
async fn next_available_operation_name(
&self,
workspace_id: &WorkspaceId,
base_name: &str,
) -> Result<String, ApiError> {
for index in 2.. {
let candidate = format!("{base_name}_{index}");
if self
.find_operation_by_name(workspace_id, &candidate)
.await?
.is_none()
{
return Ok(candidate);
}
}
unreachable!()
}
fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
let selected_operation_keys = payload
.selected_operation_keys
.iter()
.cloned()
.collect::<BTreeSet<_>>();
let canonical = serde_json::to_vec(&json!({
"selected_operation_keys": selected_operation_keys,
"server_url": payload.server_url.as_deref(),
"conflict_mode": payload.conflict_mode.as_str(),
}))
.map_err(|error| ApiError::internal(error.to_string()))?;
Ok(format!("{:x}", Sha256::digest(canonical)))
}
fn attach_import_findings(
@@ -19,6 +19,16 @@ use crate::{
};
impl AdminService {
pub async fn cleanup_invocation_logs_before(
&self,
cutoff: OffsetDateTime,
) -> Result<u64, ApiError> {
self.registry
.delete_invocation_logs_before(cutoff)
.await
.map_err(ApiError::from)
}
#[instrument(skip(self))]
pub async fn list_logs(
&self,
+81 -52
View File
@@ -141,7 +141,6 @@ impl AdminService {
workspace_id: &WorkspaceId,
payload: OperationPayload,
) -> Result<CreatedOperationResponse, ApiError> {
self.validate_operation_payload(&payload)?;
self.ensure_workspace_exists(workspace_id).await?;
if self
@@ -155,10 +154,36 @@ impl AdminService {
));
}
let snapshot = self.new_operation_snapshot(payload)?;
let operation_id = snapshot.id.clone();
self.registry
.create_operation(workspace_id, &snapshot, None)
.await?;
info!(
name: "admin.operation.created",
operation_id = %operation_id.as_str(),
version = 1,
"operation created"
);
Ok(CreatedOperationResponse {
operation_id: operation_id.as_str().to_owned(),
workspace_id: workspace_id.as_str().to_owned(),
version: 1,
status: OperationStatus::Draft,
updated_at: format_timestamp(snapshot.updated_at),
})
}
pub(super) fn new_operation_snapshot(
&self,
payload: OperationPayload,
) -> Result<RegistryOperation, ApiError> {
self.validate_operation_payload(&payload)?;
let now = OffsetDateTime::now_utc();
let operation_id = OperationId::new(new_prefixed_id("op"));
let snapshot = RegistryOperation {
id: operation_id.clone(),
Ok(RegistryOperation {
id: OperationId::new(new_prefixed_id("op")),
name: payload.name,
display_name: payload.display_name,
category: payload.category,
@@ -183,19 +208,6 @@ impl AdminService {
created_at: now,
updated_at: now,
published_at: None,
};
self.registry
.create_operation(workspace_id, &snapshot, None)
.await?;
info!(operation_id = %operation_id.as_str(), version = 1, "operation created");
Ok(CreatedOperationResponse {
operation_id: operation_id.as_str().to_owned(),
workspace_id: workspace_id.as_str().to_owned(),
version: 1,
status: OperationStatus::Draft,
updated_at: format_timestamp(snapshot.updated_at),
})
}
@@ -279,7 +291,12 @@ impl AdminService {
created_by: None,
})
.await?;
info!(operation_id = %operation_id.as_str(), version, "operation version created");
info!(
name: "admin.operation.version_created",
operation_id = %operation_id.as_str(),
version,
"operation version created"
);
Ok(CreatedOperationResponse {
operation_id: operation_id.as_str().to_owned(),
@@ -364,7 +381,12 @@ impl AdminService {
published_by: None,
})
.await?;
info!(operation_id = %operation_id.as_str(), version, "operation published");
info!(
name: "admin.operation.published",
operation_id = %operation_id.as_str(),
version,
"operation published"
);
Ok(PublishResponse {
operation_id: operation_id.as_str().to_owned(),
@@ -431,37 +453,44 @@ impl AdminService {
.await?;
let runtime = RuntimeOperation::from(record.snapshot.clone());
let mode = ExecutionMode::Unary;
let request_preview =
match build_request_preview(&record.snapshot.input_mapping, &payload.input) {
Ok(preview) => preview,
Err(error) => {
self.record_invocation(InvocationRecordRequest {
workspace_id,
agent_id: None,
operation: &record.snapshot,
request_id: Some(request_id),
source: InvocationSource::AdminTestRun,
level: InvocationLevel::Error,
status: InvocationStatus::Error,
message: "mapping preview failed".to_owned(),
status_code: None,
error_kind: Some("mapping".to_owned()),
duration_ms: 0,
request_preview: Value::Null,
response_preview: Value::Null,
})
.await?;
return Ok(TestRunResult {
ok: false,
mode,
request_preview: Value::Null,
response_preview: Value::Null,
errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping(
error,
))],
});
}
};
let preview_span = crank_trace::Stage::RuntimeArgumentsMap.span();
let preview_result = preview_span
.in_scope(|| build_request_preview(&record.snapshot.input_mapping, &payload.input));
let request_preview = match preview_result {
Ok(preview) => preview,
Err(error) => {
crank_trace::StageOutcome::Error.record(&preview_span);
crank_trace::ErrorCategory::Mapping.record(&preview_span);
drop(preview_span);
self.record_invocation(InvocationRecordRequest {
workspace_id,
agent_id: None,
operation: &record.snapshot,
request_id: Some(request_id),
source: InvocationSource::AdminTestRun,
level: InvocationLevel::Error,
status: InvocationStatus::Error,
message: "mapping preview failed".to_owned(),
status_code: None,
error_kind: Some("mapping".to_owned()),
duration_ms: 0,
request_preview: Value::Null,
response_preview: Value::Null,
})
.await;
return Ok(TestRunResult {
ok: false,
mode,
request_preview: Value::Null,
response_preview: Value::Null,
errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping(
error,
))],
});
}
};
crank_trace::StageOutcome::Success.record(&preview_span);
drop(preview_span);
let resolved_auth = self
.resolve_operation_auth(workspace_id, &runtime.execution_config)
@@ -497,7 +526,7 @@ impl AdminService {
request_preview: request_preview.clone(),
response_preview: response_preview.clone(),
})
.await?;
.await;
Ok(TestRunResult {
ok: true,
mode,
@@ -524,7 +553,7 @@ impl AdminService {
request_preview: request_preview.clone(),
response_preview: Value::Null,
})
.await?;
.await;
Ok(TestRunResult {
ok: false,
mode,
+6 -1
View File
@@ -50,6 +50,7 @@ impl AdminService {
.save_sample_metadata(SaveSampleMetadataRequest { sample: &metadata })
.await?;
info!(
name: "admin.sample.saved",
operation_id = %operation_id.as_str(),
sample_id = %metadata.id.as_str(),
version,
@@ -119,7 +120,11 @@ impl AdminService {
input_mapping,
output_mapping,
};
info!(operation_id = %operation_id.as_str(), "draft generated from samples");
info!(
name: "admin.operation_draft.generated",
operation_id = %operation_id.as_str(),
"draft generated from samples"
);
Ok(result)
}
+20 -4
View File
@@ -92,7 +92,11 @@ impl AdminService {
created_by,
})
.await?;
info!(secret_id = %secret.id.as_str(), "secret created");
info!(
name: "admin.secret.created",
secret_id = %secret.id.as_str(),
"secret created"
);
Ok(secret)
}
@@ -126,7 +130,11 @@ impl AdminService {
created_by,
})
.await?;
info!(secret_id = %secret_id.as_str(), "secret rotated");
info!(
name: "admin.secret.rotated",
secret_id = %secret_id.as_str(),
"secret rotated"
);
self.get_secret(workspace_id, secret_id).await
}
@@ -152,7 +160,11 @@ impl AdminService {
.into());
}
self.registry.delete_secret(workspace_id, secret_id).await?;
info!(secret_id = %secret_id.as_str(), "secret deleted");
info!(
name: "admin.secret.deleted",
secret_id = %secret_id.as_str(),
"secret deleted"
);
Ok(())
}
@@ -201,7 +213,11 @@ impl AdminService {
profile: &profile,
})
.await?;
info!(auth_profile_id = %profile.id.as_str(), "auth profile created");
info!(
name: "admin.auth_profile.created",
auth_profile_id = %profile.id.as_str(),
"auth profile created"
);
Ok(profile)
}
+5 -1
View File
@@ -75,7 +75,11 @@ impl AdminService {
upstream: &upstream,
})
.await?;
info!(upstream_id = %upstream.id.as_str(), "workspace upstream saved");
info!(
name: "admin.upstream.saved",
upstream_id = %upstream.id.as_str(),
"workspace upstream saved"
);
Ok(upstream)
}
@@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind};
use serde_json::{Value, json};
use serial_test::serial;
use tokio::net::TcpListener;
use uuid::Version;
use admin_api::{
app::build_app,
@@ -40,6 +41,8 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
const TEST_SESSION_SECRET: &str = "test-session-secret";
const TEST_MASTER_KEY: &str = "test-master-key";
mod history_loss;
struct TestServer {
base_url: String,
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
@@ -385,6 +388,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() {
exported["workspace"]["workspace"]["id"],
DEFAULT_WORKSPACE_ID
);
assert_eq!(exported["kind"], "workspace_catalog_snapshot");
assert_eq!(exported["format_version"], "1");
assert_eq!(exported["restorable"], false);
assert_eq!(
exported["included"],
json!([
"workspace_settings",
"operation_summaries",
"agent_summaries",
"platform_api_key_metadata"
])
);
assert!(
exported["excluded"]
.as_array()
.unwrap()
.contains(&json!("secret_values"))
);
assert!(
exported["excluded"]
.as_array()
.unwrap()
.contains(&json!("invocation_logs_and_usage"))
);
assert!(exported.get("memberships").is_none());
assert!(exported.get("invitations").is_none());
@@ -574,6 +601,25 @@ async fn updates_profile_and_changes_password() {
.unwrap()
.to_owned();
let client = authorized_client(&base_url).await;
let second_client = reqwest::Client::builder()
.cookie_store(true)
.build()
.unwrap();
let second_login = second_client
.post(format!("{root_url}/api/auth/login"))
.json(&json!({
"email": TEST_AUTH_EMAIL,
"password": TEST_AUTH_PASSWORD,
}))
.send()
.await
.unwrap();
let second_login_status = second_login.status();
let second_login_body = second_login.text().await.unwrap();
assert!(
second_login_status.is_success(),
"second login failed with {second_login_status}: {second_login_body}"
);
let profile = assert_success_json(
client
@@ -615,6 +661,21 @@ async fn updates_profile_and_changes_password() {
.status();
assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);
let current_session_status = client
.get(format!("{root_url}/api/auth/profile"))
.send()
.await
.unwrap()
.status();
let other_session_status = second_client
.get(format!("{root_url}/api/auth/profile"))
.send()
.await
.unwrap()
.status();
assert_eq!(current_session_status, reqwest::StatusCode::OK);
assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
let relogin_client = reqwest::Client::builder()
.cookie_store(true)
.build()
@@ -855,7 +916,10 @@ async fn generates_request_id_for_test_run_invocations() {
.unwrap()
.to_owned();
assert!(!request_id.is_empty());
assert_eq!(
uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
Some(Version::SortRand)
);
response.error_for_status().unwrap();
let logs = client
@@ -0,0 +1,141 @@
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use axum::{Json, Router, extract::State, routing::post};
use tokio::{net::TcpListener, sync::Notify};
use super::*;
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn preserves_external_success_when_invocation_history_is_lost() {
let registry = test_registry().await;
let registry_for_failure = registry.clone();
let storage_root = test_storage_root("observability_history_loss");
let upstream = spawn_blocking_upstream_server().await;
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
let client = authorized_client(&base_url).await;
let created = client
.post(format!("{base_url}/operations"))
.json(&test_operation_payload(
&upstream.base_url,
"crm_history_loss",
))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
let operation_id = created["operation_id"].as_str().unwrap().to_owned();
let request_client = client.clone();
let request_url = format!("{base_url}/operations/{operation_id}/test-runs");
let before = crank_observability::operational_incident_total(
crank_observability::OperationalIncident::InvocationHistoryLost,
);
let request = tokio::spawn(async move {
request_client
.post(request_url)
.header("x-request-id", "req_dc08_admin")
.json(&json!({
"version": 1,
"input": { "email": "dc08-canary-secret@example.com" }
}))
.send()
.await
.unwrap()
});
upstream.started.notified().await;
registry_for_failure
.delete_operation(
&WorkspaceId::new(DEFAULT_WORKSPACE_ID),
&OperationId::new(operation_id.clone()),
)
.await
.unwrap();
upstream.release.notify_one();
let response = request.await.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::OK);
assert_eq!(
response.headers()["x-request-id"].to_str().unwrap(),
"req_dc08_admin"
);
let body = response.json::<Value>().await.unwrap();
assert_eq!(body["ok"], true);
assert_eq!(body["response_preview"]["id"], "lead_123");
assert_eq!(upstream.calls.load(Ordering::SeqCst), 1);
assert!(
crank_observability::operational_incident_total(
crank_observability::OperationalIncident::InvocationHistoryLost
) > before
);
let logs = client
.get(format!("{base_url}/logs?period=7d"))
.send()
.await
.unwrap()
.json::<Value>()
.await
.unwrap();
assert!(logs["items"].as_array().unwrap().is_empty());
}
struct BlockingUpstream {
base_url: String,
started: Arc<Notify>,
release: Arc<Notify>,
calls: Arc<AtomicUsize>,
}
#[derive(Clone)]
struct BlockingUpstreamState {
started: Arc<Notify>,
release: Arc<Notify>,
calls: Arc<AtomicUsize>,
}
async fn spawn_blocking_upstream_server() -> BlockingUpstream {
let state = BlockingUpstreamState {
started: Arc::new(Notify::new()),
release: Arc::new(Notify::new()),
calls: Arc::new(AtomicUsize::new(0)),
};
let app = Router::new()
.route("/crm/leads", post(blocking_create_lead))
.with_state(state.clone());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
BlockingUpstream {
base_url: format!("http://{address}"),
started: state.started,
release: state.release,
calls: state.calls,
}
}
async fn blocking_create_lead(
State(state): State<BlockingUpstreamState>,
Json(payload): Json<Value>,
) -> Json<Value> {
state.calls.fetch_add(1, Ordering::SeqCst);
state.started.notify_one();
state.release.notified().await;
Json(json!({
"id": "lead_123",
"status": "created",
"email": payload["email"]
}))
}
@@ -1,5 +1,6 @@
use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
use crank_core::WorkspaceId;
use crank_registry::ImportJobStatus;
use serial_test::serial;
use super::common::{
@@ -42,7 +43,7 @@ paths:
async fn previews_openapi_and_creates_draft_operations() {
let registry = test_registry().await;
let service = test_service(
registry,
registry.clone(),
test_storage_root("openapi_import"),
test_auth_settings(),
test_secret_crypto(),
@@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() {
preview.preview.groups[0].operations[0].suggested_name,
"latest_rates"
);
let preview_job = registry
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
.await
.unwrap()
.unwrap();
assert_eq!(preview_job.status, ImportJobStatus::Pending);
let created = service
.create_openapi_import(
@@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() {
.any(|finding| finding.code == "openapi_import.weak_tool_description")
);
let skip_preview = service
.preview_openapi_import(
&workspace_id,
OpenApiImportPreviewPayload {
document: OPENAPI3.to_owned(),
},
)
.await
.unwrap();
let skipped = service
.create_openapi_import(
&workspace_id,
&preview.job_id.as_str().into(),
&skip_preview.job_id.as_str().into(),
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() {
assert_eq!(skipped.skipped[0].name, "latest_rates");
assert_eq!(skipped.findings[0].code, "operation_name_conflict");
let rename_preview = service
.preview_openapi_import(
&workspace_id,
OpenApiImportPreviewPayload {
document: OPENAPI3.to_owned(),
},
)
.await
.unwrap();
let renamed = service
.create_openapi_import(
&workspace_id,
&preview.job_id.as_str().into(),
&rename_preview.job_id.as_str().into(),
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
@@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() {
assert_eq!(renamed.created[0].name, "latest_rates_2");
assert_eq!(renamed.findings[0].code, "operation_name_renamed");
}
#[tokio::test]
#[serial]
async fn concurrent_openapi_import_replays_the_same_atomic_result() {
let registry = test_registry().await;
let service = test_service(
registry,
test_storage_root("openapi_import_replay"),
test_auth_settings(),
test_secret_crypto(),
);
let workspace_id = WorkspaceId::new("ws_default");
let preview = service
.preview_openapi_import(
&workspace_id,
OpenApiImportPreviewPayload {
document: OPENAPI3.to_owned(),
},
)
.await
.unwrap();
let job_id = preview.job_id.as_str().into();
let payload = OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
conflict_mode: "rename".to_owned(),
};
let (first, second) = tokio::join!(
service.create_openapi_import(&workspace_id, &job_id, payload.clone()),
service.create_openapi_import(&workspace_id, &job_id, payload),
);
let first = first.unwrap();
let second = second.unwrap();
assert_eq!(first.created.len(), 1);
assert_eq!(second.created.len(), 1);
assert_eq!(
first.created[0].operation_id,
second.created[0].operation_id
);
assert_eq!(first.created[0].name, second.created[0].name);
assert_eq!(
service.list_operations(&workspace_id).await.unwrap().len(),
1
);
let conflicting_replay = service
.create_openapi_import(
&workspace_id,
&job_id,
OpenApiImportCreatePayload {
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
server_url: Some("https://api.frankfurter.dev".to_owned()),
conflict_mode: "skip".to_owned(),
},
)
.await;
assert!(conflicting_replay.is_err());
}
@@ -0,0 +1,211 @@
use std::{
io,
sync::{Arc, Mutex},
};
use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
use axum::{
Router,
body::Body,
http::{HeaderMap, Request, StatusCode},
routing::get,
};
use crank_observability::{
ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
};
use opentelemetry::{global, trace::TracerProvider as _};
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
use uuid::Version;
#[tokio::test(flavor = "current_thread")]
async fn logs_request_completion_and_rejects_untrusted_values() {
let writer = SharedLogWriter::default();
let subscriber = crank_observability::build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let dispatch = tracing::Dispatch::new(subscriber);
let app = probe_app();
let response = app
.clone()
.oneshot(
Request::builder()
.uri("/probe")
.header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers()[REQUEST_ID_HEADER.as_str()]
.to_str()
.unwrap(),
"req_admin_trace_123"
);
let event: serde_json::Value = writer
.output()
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
.unwrap();
assert_eq!(event["request_id"], "req_admin_trace_123");
assert_eq!(event["fields"]["status"], 200);
let invalid_response = app
.oneshot(
Request::builder()
.uri("/probe")
.header(REQUEST_ID_HEADER.as_str(), "bad,value")
.header("traceparent", "canary-invalid-traceparent")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch)
.await
.unwrap();
let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
.to_str()
.unwrap();
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
assert!(!writer.output().contains("canary-invalid-traceparent"));
}
#[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent() {
global::set_text_map_propagator(TraceContextPropagator::new());
let provider = SdkTracerProvider::builder().build();
let tracer = provider.tracer("admin-request-context-test");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let app = trace_probe_app();
let valid = observed_trace_id(
app.clone()
.oneshot(
Request::builder()
.uri("/trace")
.header(
"traceparent",
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
)
.header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap(),
);
let invalid = observed_trace_id(
app.clone()
.oneshot(
Request::builder()
.uri("/trace")
.header("traceparent", "canary-invalid-traceparent")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch.clone())
.await
.unwrap(),
);
let absent = observed_trace_id(
app.oneshot(
Request::builder()
.uri("/trace")
.body(Body::empty())
.unwrap(),
)
.with_subscriber(dispatch)
.await
.unwrap(),
);
assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c");
assert_ne!(invalid, valid);
assert_ne!(absent, valid);
assert_ne!(invalid, absent);
provider.shutdown().unwrap();
}
fn probe_app() -> Router {
Router::new()
.route("/probe", get(|| async { "ok" }))
.layer(axum::middleware::from_fn(apply_request_context))
}
fn trace_probe_app() -> Router {
Router::new()
.route("/trace", get(observed_traceparent))
.layer(axum::middleware::from_fn(apply_request_context))
}
async fn observed_traceparent() -> HeaderMap {
let mut trace_headers = HeaderMap::new();
inject_current_trace_context(&mut trace_headers);
let mut response_headers = HeaderMap::new();
if let Some(traceparent) = trace_headers.remove("traceparent") {
response_headers.insert("x-observed-traceparent", traceparent);
}
response_headers
}
fn observed_trace_id(response: axum::response::Response) -> String {
let traceparent = response.headers()["x-observed-traceparent"]
.to_str()
.unwrap();
traceparent[3..35].to_owned()
}
#[derive(Clone, Default)]
struct SharedLogWriter {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl SharedLogWriter {
fn output(&self) -> String {
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
}
}
impl<'a> MakeWriter<'a> for SharedLogWriter {
type Writer = SharedLogGuard;
fn make_writer(&'a self) -> Self::Writer {
SharedLogGuard {
buffer: Arc::clone(&self.buffer),
}
}
}
struct SharedLogGuard {
buffer: Arc<Mutex<Vec<u8>>>,
}
impl io::Write for SharedLogGuard {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.buffer.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
+2
View File
@@ -0,0 +1,2 @@
#[path = "integration/request_context.rs"]
mod request_context;
+9
View File
@@ -3,6 +3,7 @@ name = "mcp-server"
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
version.workspace = true
[[bin]]
@@ -15,10 +16,12 @@ axum.workspace = true
base64.workspace = true
crank-community-mcp = { path = "../../crates/crank-community-mcp" }
crank-core = { path = "../../crates/crank-core" }
crank-observability = { path = "../../crates/crank-observability" }
crank-registry = { path = "../../crates/crank-registry" }
crank-runtime = { path = "../../crates/crank-runtime" }
crank-schema = { path = "../../crates/crank-schema" }
futures-util = "0.3"
metrics.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
@@ -34,4 +37,10 @@ uuid.workspace = true
crank-mapping = { path = "../../crates/crank-mapping" }
crank-schema = { path = "../../crates/crank-schema" }
crank-test-support = { path = "../../crates/crank-test-support" }
opentelemetry.workspace = true
opentelemetry-proto.workspace = true
opentelemetry_sdk.workspace = true
prost.workspace = true
reqwest.workspace = true
tower.workspace = true
tracing-opentelemetry.workspace = true
+70 -21
View File
@@ -1,26 +1,53 @@
use std::{env, net::SocketAddr, time::Duration};
use crank_community_mcp::{
auth::CommunityMachineCredentialVerifier, build_app_with_background_workers,
auth::CommunityMachineCredentialVerifier, build_app_with_background_workers_and_limits,
session::PostgresTransportSessionStore,
};
use crank_observability::{
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
capture_critical_error,
};
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
use crank_runtime::{
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
RuntimeLimits, SecretCrypto,
};
use sqlx::postgres::PgConnectOptions;
use sqlx::{PgPool, postgres::PgConnectOptions};
use tokio::net::TcpListener;
use tracing::info;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing_subscriber::fmt()
.with_env_filter(
env::var("CRANK_LOG_LEVEL")
.unwrap_or_else(|_| "mcp_server=info,tower_http=info".into()),
)
.init();
let observability = crank_observability::init(ObservabilityConfig::from_env(
"mcp-server",
env!("CARGO_PKG_VERSION"),
"mcp_server=info,tower_http=info",
)?)?;
let mut startup_completed = false;
let result = run(&observability, &mut startup_completed).await;
if result.is_err() {
capture_critical_error(if startup_completed {
CriticalErrorCategory::Internal
} else {
CriticalErrorCategory::Startup
});
}
result
}
async fn run(
observability: &ObservabilityLifecycle,
startup_completed: &mut bool,
) -> Result<(), Box<dyn std::error::Error>> {
let metrics_config =
MetricsConfig::from_env("CRANK_MCP_METRICS_BIND", "127.0.0.1:9465".parse()?)?;
let metrics_enabled = metrics_config.enabled();
let metrics_server = if metrics_config.enabled() {
Some(observability.metrics_surface(metrics_config).bind().await?)
} else {
None
};
let bind_addr = env::var("CRANK_MCP_BIND").unwrap_or_else(|_| "0.0.0.0:3002".into());
let base_url = env::var("CRANK_BASE_URL").ok();
@@ -36,23 +63,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
let api_rate_limit = mcp_api_rate_limit_config_from_env()?;
let database_options = database_options_from_env()?;
let registry = PostgresRegistry::connect_with_options_and_pool_config(
database_options.clone(),
pool_config,
)
.await?;
let session_store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
database_options,
pool_config,
)
.await?;
let registry =
PostgresRegistry::connect_with_options_and_pool_config(database_options, pool_config)
.await?;
if metrics_enabled {
spawn_postgres_pool_metrics(registry.pool().clone());
}
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?;
let runtime = crank_runtime::community_from_env()?
.with_limits(runtime_limits)
.with_response_cache(cache_stores.response.clone())
.with_coordination_store(cache_stores.coordination.clone())
.build();
let app = build_app_with_background_workers(
let app = build_app_with_background_workers_and_limits(
registry,
refresh_interval,
base_url,
@@ -66,11 +90,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
cache_stores.coordination.clone(),
std::sync::Arc::new(session_store),
std::sync::Arc::new(CommunityMachineCredentialVerifier),
runtime_limits.max_concurrent_sessions,
);
let listener = TcpListener::bind(socket_addr).await?;
info!(
name: "mcp.postgres_pool.configured",
runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
runtime_max_concurrent_sessions = runtime_limits.max_concurrent_sessions,
mcp_rate_limit_rps = api_rate_limit.requests_per_second,
mcp_rate_limit_burst = api_rate_limit.burst,
cache_backend = %cache_config.backend,
@@ -81,9 +108,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
max_lifetime_ms = pool_config.max_lifetime_ms,
"postgres pool configured"
);
info!("mcp-server listening on {}", socket_addr);
info!(
name: "mcp.server.listening",
bind_address = %socket_addr,
"mcp-server listening"
);
*startup_completed = true;
axum::serve(listener, app).await?;
if let Some(metrics_server) = metrics_server {
tokio::select! {
result = axum::serve(listener, app) => result?,
result = metrics_server.serve() => result?,
}
} else {
axum::serve(listener, app).await?;
}
Ok(())
}
@@ -123,3 +162,13 @@ fn mcp_api_rate_limit_config_from_env() -> Result<RequestRateLimitConfig, Box<dy
Ok(RequestRateLimitConfig::new(requests_per_second, burst)?)
}
fn spawn_postgres_pool_metrics(pool: PgPool) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(5));
loop {
interval.tick().await;
crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
}
});
}
@@ -0,0 +1,4 @@
#[path = "integration/common.rs"]
mod common;
#[path = "integration/execution_stages.rs"]
mod execution_stages;
@@ -89,11 +89,16 @@ async fn approval_key_lists_and_decides_pending_requests() {
let approved = client
.post(&approve_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.header("x-request-id", "req_approval_execute_123")
.json(&json!({ "approve": "yes", "note": "confirmed by test" }))
.send()
.await
.unwrap();
assert_eq!(approved.status(), reqwest::StatusCode::OK);
assert_eq!(
approved.headers()["x-request-id"].to_str().unwrap(),
"req_approval_execute_123"
);
let approved_body = approved.json::<Value>().await.unwrap();
assert_eq!(
approved_body["approval"]["status"],
@@ -169,6 +174,10 @@ async fn approval_key_lists_and_decides_pending_requests() {
.await
.unwrap();
assert_eq!(logs.len(), 1);
assert_eq!(
logs[0].log.request_id.as_deref(),
Some("req_approval_execute_123")
);
}
#[tokio::test]
@@ -401,22 +410,23 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
let mcp_url = agent_mcp_url(&base_url, "sales-gated");
let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
let tool_call = json!({
"jsonrpc": "2.0",
"id": 9,
"method": "tools/call",
"params": {
"name": "crm_requires_human_approval",
"arguments": {
"email": "ada@example.com"
}
}
});
let tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
json!({
"jsonrpc": "2.0",
"id": 9,
"method": "tools/call",
"params": {
"name": "crm_requires_human_approval",
"arguments": {
"email": "ada@example.com"
}
}
}),
tool_call.clone(),
)
.await;
@@ -430,6 +440,19 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
.unwrap();
assert!(approval_id.starts_with("approval_"));
let repeated_tool_result = post_jsonrpc(
&client,
&mcp_url,
&api_key,
Some(&initialized_session),
tool_call,
)
.await;
assert_eq!(
repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
"deduplicated tools/call must return the persisted approval id",
);
let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
let pending = client
.get(&approvals_url)
@@ -448,6 +471,189 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
);
}
#[tokio::test]
async fn approval_http_endpoints_enforce_request_rate_limit() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
let operation = test_operation(&upstream_base_url, "crm_approval_rate_limit");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-approval-rate-limit").await;
let approval_key = create_approval_platform_api_key(
&registry,
"sales-approval-rate-limit",
"approval-rate-limit",
)
.await;
let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
RequestRateLimitConfig::new(1, 1).unwrap(),
))
.await;
let approvals_url = format!(
"{}/approvals",
agent_mcp_url(&base_url, "sales-approval-rate-limit")
);
let client = reqwest::Client::new();
let allowed = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap();
assert_eq!(allowed.status(), reqwest::StatusCode::OK);
let limited = client
.get(&approvals_url)
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
.send()
.await
.unwrap();
assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
assert!(limited.headers().contains_key(header::RETRY_AFTER));
}
#[tokio::test]
async fn recovery_does_not_repeat_interrupted_mutating_approval() {
let registry = test_registry().await;
let (upstream_base_url, upstream_calls) = spawn_counted_approval_upstream().await;
let operation = test_operation(&upstream_base_url, "crm_interrupted_approval");
registry
.create_operation(&test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "sales-interrupted-approval").await;
let approval_key_name = "approval-interrupted";
create_approval_platform_api_key(&registry, "sales-interrupted-approval", approval_key_name)
.await;
let now = OffsetDateTime::now_utc();
let approval = ApprovalRequest {
id: ApprovalRequestId::new("approval_interrupted_mutation"),
workspace_id: test_workspace_id(),
agent_id: test_agent_id("sales-interrupted-approval"),
operation_id: operation.id.clone(),
operation_version: operation.version,
status: ApprovalRequestStatus::Pending,
risk_level: OperationApprovalRiskLevel::Dangerous,
request_payload: json!({"email": "interrupted@example.com"}),
response_payload: None,
created_at: now - time::Duration::minutes(10),
expires_at: now + time::Duration::minutes(5),
decided_at: None,
decided_by_key_id: None,
decision_note: None,
};
registry
.create_approval_request(CreateApprovalRequest {
approval: &approval,
})
.await
.unwrap();
let approval_key_id = PlatformApiKeyId::new(format!("pk_{approval_key_name}"));
registry
.decide_approval_request(crank_registry::DecideApprovalRequest {
workspace_id: &approval.workspace_id,
agent_id: &approval.agent_id,
approval_id: &approval.id,
status: ApprovalRequestStatus::Approved,
decided_at: now - time::Duration::minutes(10),
decided_by_key_id: &approval_key_id,
response_payload: Some(json!({"approve": "yes"})),
decision_note: None,
})
.await
.unwrap()
.unwrap();
registry
.claim_approval_request(
&approval.workspace_id,
&approval.agent_id,
&approval.id,
now - time::Duration::minutes(7),
)
.await
.unwrap()
.unwrap();
let _app = build_test_app_with_approval_recovery(registry.clone());
let failed = tokio::time::timeout(Duration::from_secs(2), async {
loop {
let current = registry
.get_approval_request_for_agent(
&approval.workspace_id,
&approval.agent_id,
&approval.id,
)
.await
.unwrap()
.unwrap();
if current.approval.status == ApprovalRequestStatus::Failed {
break current;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("recovery must quarantine interrupted execution");
assert_eq!(
failed.approval.response_payload.unwrap()["error"]["code"],
"approval_execution_outcome_unknown"
);
assert_eq!(
upstream_calls.load(std::sync::atomic::Ordering::SeqCst),
0,
"recovery must not repeat a mutating upstream request"
);
}
fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router {
crank_community_mcp::build_app_with_background_workers(
registry,
Duration::from_millis(0),
Some("https://crank.example.com".to_owned()),
SecretCrypto::new("test-master-key").unwrap(),
crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build(),
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
Arc::new(InMemoryCoordinationStateStore::default()),
Arc::new(InMemorySessionStore::default()),
Arc::new(CommunityMachineCredentialVerifier),
)
}
async fn spawn_counted_approval_upstream() -> (String, Arc<std::sync::atomic::AtomicUsize>) {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let handler_calls = Arc::clone(&calls);
let app = Router::new().route(
"/crm/leads",
post(move |Json(payload): Json<Value>| {
let calls = Arc::clone(&handler_calls);
async move {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Json(json!({
"id": "lead_123",
"email": payload["email"]
}))
}
}),
);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(format!("http://{address}"), calls)
}
#[tokio::test]
async fn elicitation_approval_requires_client_capability() {
let registry = test_registry().await;
@@ -0,0 +1,376 @@
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use axum::{
Json, Router,
body::{Body, Bytes, to_bytes},
extract::State,
http::{HeaderMap, Request, StatusCode, header},
routing::post,
};
use crank_core::{InvocationSource, PlatformApiKeyScope};
use crank_observability::{
OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider,
};
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
use opentelemetry::global;
use opentelemetry_proto::tonic::{
collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value, trace::v1::Span,
};
use opentelemetry_sdk::propagation::TraceContextPropagator;
use prost::Message;
use serde_json::{Value, json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::net::TcpListener;
use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::{Layer, filter::filter_fn, layer::SubscriberExt};
use super::common::{
build_test_app, create_platform_api_key, publish_agent_for_operation, test_operation,
test_registry, test_workspace_id,
};
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
const REQUEST_ID: &str = "req_stage_end_to_end";
const CANARY: &str = "dc-stage-canary-secret";
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn exports_real_tool_stages_without_sensitive_data() {
global::set_text_map_propagator(TraceContextPropagator::new());
let (otlp_endpoint, request_rx, collector) = spawn_otlp_collector().await;
let trace_config = OtlpTraceConfig::try_new(
Some(otlp_endpoint),
Some("http/protobuf".to_owned()),
Duration::from_secs(2),
OtlpBatchConfig::try_new(128, 128, Duration::from_secs(300), Duration::from_secs(2))
.unwrap(),
)
.unwrap();
let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "integration-test").unwrap();
let (provider, tracer) = build_tracer_provider(&identity, &trace_config)
.unwrap()
.expect("enabled OTLP provider");
let subscriber = tracing_subscriber::registry().with(
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_filter(filter_fn(|metadata| {
metadata.is_span() && metadata.target() == "crank::trace"
})),
);
let dispatch = tracing::Dispatch::new(subscriber);
let registry = test_registry().await;
let observed_traceparent = Arc::new(Mutex::new(None));
let upstream_base_url = spawn_upstream(Arc::clone(&observed_traceparent)).await;
let operation = test_operation(&upstream_base_url, "stage_end_to_end");
registry
.create_operation(&test_workspace_id(), &operation, Some("test"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
published_by: Some("test"),
})
.await
.unwrap();
publish_agent_for_operation(&registry, &operation, "stage-agent").await;
let api_key = create_platform_api_key(
&registry,
"stage-agent",
"stage-key",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let app = build_test_app(registry.clone(), Duration::ZERO, None);
let call_result = async {
let initialized = send_jsonrpc(
app.clone(),
&api_key,
None,
None,
None,
json!({
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-11-25",
"capabilities": {}
}
}),
)
.await;
assert_eq!(initialized.status(), StatusCode::OK);
let session_id = initialized
.headers()
.get("MCP-Session-Id")
.unwrap()
.to_str()
.unwrap()
.to_owned();
let notification = send_jsonrpc(
app.clone(),
&api_key,
Some(&session_id),
None,
None,
json!({
"jsonrpc": "2.0",
"method": "notifications/initialized",
"params": {}
}),
)
.await;
assert_eq!(notification.status(), StatusCode::ACCEPTED);
send_jsonrpc(
app,
&api_key,
Some(&session_id),
Some(REQUEST_ID),
Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
json!({
"jsonrpc": "2.0",
"id": 2,
"method": "tools/call",
"params": {
"name": "stage_end_to_end",
"arguments": { "email": CANARY }
}
}),
)
.await
}
.with_subscriber(dispatch)
.await;
assert_eq!(call_result.status(), StatusCode::OK);
let body = to_bytes(call_result.into_body(), 1024 * 1024)
.await
.unwrap();
let body: Value = serde_json::from_slice(&body).unwrap();
assert_eq!(body["result"]["isError"], false);
provider.force_flush().unwrap();
let request = tokio::time::timeout(Duration::from_secs(2), request_rx)
.await
.unwrap()
.unwrap();
collector.abort();
assert_eq!(
request
.headers
.get(header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok()),
Some("application/x-protobuf")
);
assert!(
!request
.body
.windows(CANARY.len())
.any(|window| window == CANARY.as_bytes())
);
let export = ExportTraceServiceRequest::decode(request.body).unwrap();
let spans = export
.resource_spans
.iter()
.flat_map(|resource| &resource.scope_spans)
.flat_map(|scope| &scope.spans)
.collect::<Vec<_>>();
let traceparent = observed_traceparent
.lock()
.unwrap()
.clone()
.expect("upstream traceparent");
assert_eq!(&traceparent[3..35], REMOTE_TRACE_ID);
let expected_trace_id = decode_trace_id(REMOTE_TRACE_ID);
let trace_spans = spans
.iter()
.copied()
.filter(|span| span.trace_id.as_slice() == expected_trace_id)
.collect::<Vec<_>>();
for expected in [
"mcp.request",
"mcp.rate_limit",
"mcp.access.check",
"mcp.catalog.load",
"mcp.tools.resolve",
"runtime.execute",
"runtime.arguments.map",
"upstream.http",
"runtime.response.transform",
"history.write",
"db.query",
] {
assert!(
trace_spans.iter().any(|span| span.name == expected),
"missing span {expected}"
);
}
assert!(!trace_spans.iter().any(|span| span.name == "approval.check"));
assert!(
!trace_spans
.iter()
.any(|span| span.name == "runtime.idempotency")
);
let root = trace_spans
.iter()
.find(|span| span.name == "mcp.request")
.expect("mcp root");
let runtime = trace_spans
.iter()
.find(|span| span.name == "runtime.execute")
.expect("runtime");
assert!(!runtime.parent_span_id.is_empty());
assert_eq!(runtime.parent_span_id, root.span_id);
let history = trace_spans
.iter()
.find(|span| span.name == "history.write")
.expect("history write");
let history_db = trace_spans
.iter()
.find(|span| {
span.name == "db.query"
&& string_attribute(span, "db.operation") == Some("invocation_history.write")
})
.expect("history PostgreSQL write");
assert_eq!(history_db.parent_span_id, history.span_id);
let logs = registry
.list_invocation_logs(ListInvocationLogsQuery {
workspace_id: &test_workspace_id(),
level: None,
search_text: None,
source: Some(InvocationSource::AgentToolCall),
operation_id: Some(&operation.id),
agent_id: None,
created_after: None,
limit: 10,
})
.await
.unwrap();
assert_eq!(logs.len(), 1);
assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID));
provider.shutdown().unwrap();
}
struct OtlpRequest {
headers: HeaderMap,
body: Bytes,
}
async fn spawn_otlp_collector() -> (
String,
tokio::sync::oneshot::Receiver<OtlpRequest>,
tokio::task::JoinHandle<()>,
) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let (request_tx, request_rx) = tokio::sync::oneshot::channel();
let sender = Arc::new(Mutex::new(Some(request_tx)));
let app = Router::new().route(
"/v1/traces",
post({
let sender = Arc::clone(&sender);
move |headers: HeaderMap, body: Bytes| {
let sender = Arc::clone(&sender);
async move {
if let Some(sender) = sender.lock().unwrap().take() {
let _ = sender.send(OtlpRequest { headers, body });
}
StatusCode::OK
}
}
}),
);
let collector = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(format!("http://{address}/v1/traces"), request_rx, collector)
}
fn decode_trace_id(value: &str) -> [u8; 16] {
let mut bytes = [0_u8; 16];
for (index, byte) in bytes.iter_mut().enumerate() {
*byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap();
}
bytes
}
fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> {
span.attributes.iter().find_map(|attribute| {
let value = attribute.value.as_ref()?.value.as_ref()?;
(attribute.key == key)
.then_some(value)
.and_then(|value| match value {
any_value::Value::StringValue(value) => Some(value.as_str()),
_ => None,
})
})
}
async fn send_jsonrpc(
app: Router,
api_key: &str,
session_id: Option<&str>,
request_id: Option<&str>,
traceparent: Option<&str>,
payload: Value,
) -> axum::response::Response {
let mut request = Request::builder()
.method("POST")
.uri("/v1/default/stage-agent")
.header(header::CONTENT_TYPE, "application/json")
.header(header::ACCEPT, "application/json, text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("MCP-Protocol-Version", "2025-11-25");
if let Some(session_id) = session_id {
request = request.header("MCP-Session-Id", session_id);
}
if let Some(request_id) = request_id {
request = request.header("x-request-id", request_id);
}
if let Some(traceparent) = traceparent {
request = request.header("traceparent", traceparent);
}
app.oneshot(request.body(Body::from(payload.to_string())).unwrap())
.await
.unwrap()
}
async fn spawn_upstream(observed: Arc<Mutex<Option<String>>>) -> String {
async fn create_lead(
State(observed): State<Arc<Mutex<Option<String>>>>,
headers: HeaderMap,
Json(_payload): Json<Value>,
) -> Json<Value> {
*observed.lock().unwrap() = headers
.get("traceparent")
.and_then(|value| value.to_str().ok())
.map(str::to_owned);
Json(json!({ "id": "lead_123" }))
}
let app = Router::new()
.route("/crm/leads", post(create_lead))
.with_state(observed);
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{address}")
}
@@ -0,0 +1,130 @@
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use axum::{
body::Body,
http::{Request, StatusCode},
};
use opentelemetry::{
global,
trace::{TraceId, TracerProvider as _},
};
use opentelemetry_sdk::{
error::OTelSdkResult,
propagation::TraceContextPropagator,
trace::{SdkTracerProvider, SpanData, SpanExporter},
};
use tower::ServiceExt;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::layer::SubscriberExt;
use super::common::{build_test_app, test_registry};
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
#[tokio::test(flavor = "current_thread")]
async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
global::set_text_map_propagator(TraceContextPropagator::new());
let exported = Arc::new(Mutex::new(Vec::new()));
let provider = SdkTracerProvider::builder()
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
.build();
let tracer = provider.tracer("mcp-request-context-test");
let subscriber =
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
let dispatch = tracing::Dispatch::new(subscriber);
let app = build_test_app(test_registry().await, Duration::ZERO, None);
let (valid, invalid, absent) = async {
let valid = send_health(
app.clone(),
Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
Some("request-id-is-separate"),
)
.await;
let invalid = send_health(
app.clone(),
Some("canary-invalid-traceparent"),
Some("bad,value"),
)
.await;
let absent = send_health(app, None, None).await;
(valid, invalid, absent)
}
.with_subscriber(dispatch)
.await;
provider.force_flush().unwrap();
assert_eq!(valid.status, StatusCode::OK);
assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate"));
assert_eq!(invalid.status, StatusCode::OK);
assert_eq!(absent.status, StatusCode::OK);
assert!(valid.traceparent_response.is_none());
assert!(invalid.traceparent_response.is_none());
assert!(absent.traceparent_response.is_none());
let trace_ids: Vec<_> = exported
.lock()
.unwrap()
.iter()
.filter(|span| span.name.as_ref() == "mcp.request")
.map(|span| span.span_context.trace_id())
.collect();
assert_eq!(trace_ids.len(), 3);
assert_eq!(trace_ids[0].to_string(), REMOTE_TRACE_ID);
assert_ne!(trace_ids[1], trace_ids[0]);
assert_ne!(trace_ids[2], trace_ids[0]);
assert_ne!(trace_ids[1], trace_ids[2]);
assert!(!trace_ids.contains(&TraceId::INVALID));
provider.shutdown().unwrap();
}
async fn send_health(
app: axum::Router,
traceparent: Option<&str>,
request_id: Option<&str>,
) -> ProbeResponse {
let mut request = Request::builder().uri("/health");
if let Some(traceparent) = traceparent {
request = request.header("traceparent", traceparent);
}
if let Some(request_id) = request_id {
request = request.header("x-request-id", request_id);
}
let response = app
.oneshot(request.body(Body::empty()).unwrap())
.await
.unwrap();
ProbeResponse {
status: response.status(),
request_id: response
.headers()
.get("x-request-id")
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
traceparent_response: response
.headers()
.get("traceparent")
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
}
}
struct ProbeResponse {
status: StatusCode,
request_id: Option<String>,
traceparent_response: Option<String>,
}
#[derive(Clone, Debug)]
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
impl SpanExporter for CapturingExporter {
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
self.0.lock().unwrap().extend(batch);
Ok(())
}
}
@@ -38,7 +38,8 @@ use sha2::{Digest, Sha256};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use tokio::net::TcpListener;
use tokio::time::sleep;
use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
use tracing_subscriber::fmt::MakeWriter;
use uuid::Version;
use crank_community_mcp::{
auth::{CommunityMachineCredentialVerifier, SharedMachineCredentialVerifier},
@@ -46,6 +47,9 @@ use crank_community_mcp::{
catalog::PublishedToolCatalog,
session::{InMemorySessionStore, SharedSessionStore, TransportSessionStore},
};
use crank_observability::{
ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber,
};
fn test_workspace_id() -> WorkspaceId {
WorkspaceId::new("ws_default")
@@ -404,7 +408,10 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
.to_str()
.unwrap()
.to_owned();
assert!(!request_id.is_empty());
assert_eq!(
uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
Some(Version::SortRand)
);
let call_result = response.json::<Value>().await.unwrap();
assert_eq!(call_result["result"]["isError"], false);
@@ -427,7 +434,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str()));
}
#[tokio::test]
#[tokio::test(flavor = "current_thread")]
async fn emits_request_id_in_mcp_ingress_logs() {
let registry = test_registry().await;
let upstream_base_url = spawn_upstream_server().await;
@@ -456,6 +463,18 @@ async fn emits_request_id_in_mcp_ingress_logs() {
)
.await;
let writer = SharedLogWriter::default();
let subscriber = build_subscriber(
ObservabilityConfig::new(
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
"info",
RedactionLimits::default(),
),
writer.clone(),
)
.unwrap();
let dispatch = tracing::Dispatch::new(subscriber);
let _guard = tracing::dispatcher::set_default(&dispatch);
let base_url = spawn_mcp_server(build_test_app(
registry,
Duration::from_millis(0),
@@ -464,18 +483,7 @@ async fn emits_request_id_in_mcp_ingress_logs() {
.await;
let client = reqwest::Client::new();
let mcp_url = agent_mcp_url(&base_url, "sales-request-trace");
let writer = SharedLogWriter::default();
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(writer.clone())
.without_time()
.with_ansi(false)
.with_target(false)
.compact()
.with_filter(LevelFilter::INFO),
);
let _ = tracing::subscriber::set_global_default(subscriber);
let response = post_jsonrpc_response(
&client,
&mcp_url,
@@ -500,14 +508,16 @@ async fn emits_request_id_in_mcp_ingress_logs() {
assert_eq!(response.status(), reqwest::StatusCode::OK);
let logs = writer.output();
assert!(
logs.contains("mcp request received"),
"captured logs did not include ingress marker: {logs}"
);
assert!(logs.contains("req_mcp_trace_123"));
assert!(logs.contains("sales-request-trace"));
assert!(logs.contains("default"));
assert!(logs.contains("initialize"));
let event = logs
.lines()
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
.find(|event| event["event"] == "mcp.request.received")
.unwrap();
assert_eq!(event["service"], "mcp-server");
assert_eq!(event["request_id"], "req_mcp_trace_123");
assert_eq!(event["fields"]["agent_slug"], "sales-request-trace");
assert_eq!(event["fields"]["workspace_slug"], "default");
assert_eq!(event["fields"]["jsonrpc_method"], "initialize");
}
#[tokio::test]
@@ -803,11 +813,30 @@ async fn get_requires_session_header() {
.get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
.header(header::ACCEPT, "text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("x-request-id", "req_early_mcp_error")
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
assert_eq!(
response.headers()["x-request-id"].to_str().unwrap(),
"req_early_mcp_error"
);
let invalid_response = client
.get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
.header(header::ACCEPT, "text/event-stream")
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
.header("x-request-id", "bad,value")
.send()
.await
.unwrap();
let generated = invalid_response.headers()["x-request-id"].to_str().unwrap();
assert_eq!(
uuid::Uuid::parse_str(generated).unwrap().get_version(),
Some(Version::SortRand)
);
}
#[tokio::test]
+4
View File
@@ -0,0 +1,4 @@
#[path = "integration/common.rs"]
mod common;
#[path = "integration/request_context.rs"]
mod request_context;
+2 -2
View File
@@ -97,8 +97,8 @@
</div>
<div class="danger-zone-action">
<div class="danger-zone-text">
<div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export all data</div>
<div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.</div>
<div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export workspace catalog</div>
<div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.</div>
</div>
<button class="btn-danger" id="export-workspace-btn" type="button" data-i18n="workspace_setup.export">Export</button>
</div>
+4 -4
View File
@@ -442,8 +442,8 @@ var TRANSLATIONS = {
'workspace_setup.create.subtitle': 'This Community installation uses one workspace for MCP operations and agents.',
'workspace_setup.create.footer': 'This Community installation uses one workspace.',
'workspace_setup.danger.title': 'Danger zone',
'workspace_setup.danger.export_title': 'Export all data',
'workspace_setup.danger.export_body': 'Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.',
'workspace_setup.danger.export_title': 'Export workspace catalog',
'workspace_setup.danger.export_body': 'Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.',
'workspace_setup.export': 'Export',
'workspace_setup.role.owner': 'Owner',
'workspace_setup.role.admin': 'Admin',
@@ -1334,8 +1334,8 @@ var TRANSLATIONS = {
'workspace_setup.create.subtitle': 'В Community используется один воркспейс для MCP-операций и агентов.',
'workspace_setup.create.footer': 'В Community используется один воркспейс.',
'workspace_setup.danger.title': 'Опасная зона',
'workspace_setup.danger.export_title': 'Экспортировать все данные',
'workspace_setup.danger.export_body': 'Скачать JSON-снимок настроек воркспейса, операций, агентов, секретов, данных использования и ключей доступа агентов.',
'workspace_setup.danger.export_title': 'Экспорт каталога рабочего пространства',
'workspace_setup.danger.export_body': 'Скачать невосстанавливаемый JSON-каталог настроек рабочего пространства, сводок операций и агентов, а также метаданных ключей API.',
'workspace_setup.export': 'Экспорт',
'workspace_setup.role.owner': 'Владелец',
'workspace_setup.role.admin': 'Администратор',
+40 -4
View File
@@ -8,6 +8,8 @@ document.addEventListener('DOMContentLoaded', function () {
openId: null,
liveMode: true,
timer: null,
searchTimer: null,
refreshPromise: null,
workspaceId: null,
loading: false,
loadError: '',
@@ -457,7 +459,13 @@ document.addEventListener('DOMContentLoaded', function () {
}
async function refreshOperationalData() {
await Promise.all([loadLogs(), loadApprovals()]);
if (state.refreshPromise) {
return state.refreshPromise;
}
state.refreshPromise = Promise.all([loadLogs(), loadApprovals()]).finally(function () {
state.refreshPromise = null;
});
return state.refreshPromise;
}
async function loadLogDetail(logId) {
@@ -494,10 +502,14 @@ document.addEventListener('DOMContentLoaded', function () {
function startPolling() {
stopPolling();
if (!state.liveMode) {
if (!state.liveMode || document.hidden) {
return;
}
state.timer = setInterval(refreshOperationalData, 4000);
state.timer = setTimeout(async function poll() {
state.timer = null;
await refreshOperationalData();
startPolling();
}, 4000);
}
function toggleLive() {
@@ -526,7 +538,13 @@ document.addEventListener('DOMContentLoaded', function () {
if (logSearch) {
logSearch.addEventListener('input', function () {
state.search = this.value.trim();
loadLogs();
if (state.searchTimer) {
clearTimeout(state.searchTimer);
}
state.searchTimer = setTimeout(function () {
state.searchTimer = null;
loadLogs();
}, 250);
});
}
@@ -572,6 +590,24 @@ document.addEventListener('DOMContentLoaded', function () {
refreshOperationalData();
});
document.addEventListener('visibilitychange', function () {
if (document.hidden) {
stopPolling();
return;
}
if (state.liveMode) {
refreshOperationalData().finally(startPolling);
}
});
window.addEventListener('pagehide', function () {
stopPolling();
if (state.searchTimer) {
clearTimeout(state.searchTimer);
state.searchTimer = null;
}
});
setLiveState();
startPolling();
+1 -1
View File
@@ -196,7 +196,7 @@ async function exportWorkspaceSnapshot() {
var slug = workspaceFormState.workspaceRecord && workspaceFormState.workspaceRecord.workspace
? workspaceFormState.workspaceRecord.workspace.slug
: tKey('settings.nav.workspace');
downloadJsonFile(slug + '-snapshot.json', snapshot);
downloadJsonFile(slug + '-catalog-snapshot.json', snapshot);
} catch (error) {
if (window.CrankUi) {
window.CrankUi.error(error.message || tKey('workspace_setup.export_error'), tKey('workspace_setup.export_error_title'));
+4 -4
View File
@@ -9,7 +9,7 @@
"@fontsource/inter": "5.2.8",
"@fontsource/jetbrains-mono": "5.2.8",
"alpinejs": "3.15.12",
"js-yaml": "5.2.1"
"js-yaml": "5.2.2"
},
"devDependencies": {
"@playwright/test": "1.61.1",
@@ -580,9 +580,9 @@
}
},
"node_modules/js-yaml": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz",
"integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==",
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz",
"integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==",
"funding": [
{
"type": "github",
+1 -1
View File
@@ -11,7 +11,7 @@
"@fontsource/inter": "5.2.8",
"@fontsource/jetbrains-mono": "5.2.8",
"alpinejs": "3.15.12",
"js-yaml": "5.2.1"
"js-yaml": "5.2.2"
},
"devDependencies": {
"@playwright/test": "1.61.1",
+3 -3
View File
@@ -161,15 +161,15 @@ test('wizard builds visual request mappings from JSON sample and path params', a
await page.goto('/wizard/');
await page.locator('[data-testid="wizard-protocol-rest"]').click();
await page.evaluate(() => window.CrankWizardShell.doGoToStep(2));
await page.evaluate(() => window.CrankWizardShell.goToStep(2));
await expect(page.locator('#step-panel-2')).toBeVisible();
await page.locator('#endpoint-path').fill('/rates/{date}');
await page.evaluate(() => window.CrankWizardShell.doGoToStep(3));
await page.evaluate(() => window.CrankWizardShell.goToStep(3));
await expect(page.locator('#step-panel-3-rest')).toBeVisible();
await page.locator('.method-card[data-method="GET"]').click();
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
await page.evaluate(() => window.CrankWizardShell.goToStep(5));
await expect(page.locator('#step-panel-5')).toBeVisible();
await page.locator('#wizard-input-sample').fill(JSON.stringify({
@@ -11,6 +11,12 @@ test('workspace and settings pages show live session data', async ({ page }) =>
await expect(page.locator('#section-members')).toHaveCount(0);
await expect(page.locator('#section-invite')).toHaveCount(0);
await expect(page.locator('#delete-workspace-btn')).toHaveCount(0);
await expect(page.locator('[data-i18n="workspace_setup.danger.export_title"]')).toHaveText(
localized('Export workspace catalog', 'Экспорт каталога рабочего пространства'),
);
await expect(page.locator('[data-i18n="workspace_setup.danger.export_body"]')).not.toContainText(
localized('all data', 'все данные'),
);
await page.goto('/settings');
await expect(page.locator('.page-title')).toHaveText(localized('Account settings', 'Настройки аккаунта'));