наблюдаемость: завершить базовый контур Community
Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.
This commit is contained in:
@@ -3,6 +3,7 @@ name = "admin-api"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
publish.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[[bin]]
|
||||
@@ -18,9 +19,12 @@ crank-community-auth = { path = "../../crates/crank-community-auth" }
|
||||
crank-core = { path = "../../crates/crank-core" }
|
||||
crank-import = { path = "../../crates/crank-import" }
|
||||
crank-mapping = { path = "../../crates/crank-mapping" }
|
||||
crank-observability = { path = "../../crates/crank-observability" }
|
||||
crank-registry = { path = "../../crates/crank-registry" }
|
||||
crank-runtime = { path = "../../crates/crank-runtime" }
|
||||
crank-schema = { path = "../../crates/crank-schema" }
|
||||
crank-trace = { path = "../../crates/crank-trace" }
|
||||
metrics.workspace = true
|
||||
rand.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -37,5 +41,9 @@ uuid.workspace = true
|
||||
[dev-dependencies]
|
||||
async-trait = "0.1"
|
||||
crank-test-support = { path = "../../crates/crank-test-support" }
|
||||
opentelemetry.workspace = true
|
||||
opentelemetry_sdk.workspace = true
|
||||
reqwest.workspace = true
|
||||
serial_test = "3"
|
||||
tower.workspace = true
|
||||
tracing-opentelemetry.workspace = true
|
||||
|
||||
@@ -166,6 +166,7 @@ pub fn build_app(state: AppState) -> Router {
|
||||
|
||||
Router::new()
|
||||
.route("/health", get(crate::routes::health))
|
||||
.route("/ready", get(crate::routes::readiness))
|
||||
.nest(
|
||||
"/api/auth",
|
||||
Router::new()
|
||||
@@ -178,6 +179,9 @@ pub fn build_app(state: AppState) -> Router {
|
||||
apply_api_rate_limit,
|
||||
))
|
||||
.layer(middleware::from_fn(apply_request_context))
|
||||
.layer(middleware::from_fn(
|
||||
crank_observability::record_http_request,
|
||||
))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
|
||||
@@ -269,7 +269,12 @@ pub struct CreatedPlatformApiKeyResponse {
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct WorkspaceExportResponse {
|
||||
pub struct WorkspaceCatalogSnapshotResponse {
|
||||
pub kind: String,
|
||||
pub format_version: String,
|
||||
pub restorable: bool,
|
||||
pub included: Vec<String>,
|
||||
pub excluded: Vec<String>,
|
||||
pub workspace: WorkspaceRecord,
|
||||
pub operations: Vec<OperationSummaryView>,
|
||||
pub agents: Vec<AgentSummaryView>,
|
||||
|
||||
+30
-10
@@ -137,16 +137,24 @@ impl ApiError {
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> Response {
|
||||
match &self {
|
||||
Self::Internal { message, .. } => {
|
||||
error!(error_code = self.code(), error_message = %message)
|
||||
Self::Internal { .. } => {
|
||||
error!(
|
||||
name: "admin.response.internal_error",
|
||||
error_code = self.code(),
|
||||
"internal API error response"
|
||||
)
|
||||
}
|
||||
Self::Unauthorized { message, .. }
|
||||
| Self::Forbidden { message, .. }
|
||||
| Self::Validation { message, .. }
|
||||
| Self::NotFound { message, .. }
|
||||
| Self::Conflict { message, .. }
|
||||
| Self::RateLimited { message, .. } => {
|
||||
warn!(error_code = self.code(), error_message = %message)
|
||||
Self::Unauthorized { .. }
|
||||
| Self::Forbidden { .. }
|
||||
| Self::Validation { .. }
|
||||
| Self::NotFound { .. }
|
||||
| Self::Conflict { .. }
|
||||
| Self::RateLimited { .. } => {
|
||||
warn!(
|
||||
name: "admin.response.rejected",
|
||||
error_code = self.code(),
|
||||
"API request rejected"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,6 +359,10 @@ impl From<RegistryError> for ApiError {
|
||||
format!("import job {job_id} was not found"),
|
||||
json!({ "job_id": job_id }),
|
||||
),
|
||||
RegistryError::ImportJobAlreadyApplied { job_id } => Self::conflict_with_context(
|
||||
format!("import job {job_id} was already applied with different parameters"),
|
||||
json!({ "job_id": job_id }),
|
||||
),
|
||||
RegistryError::Storage(_) | RegistryError::Serialization(_) => {
|
||||
Self::internal(value.to_string())
|
||||
}
|
||||
@@ -407,6 +419,10 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
|
||||
RuntimeError::ConfirmationRequired { .. } => "runtime_confirmation_required",
|
||||
RuntimeError::InvalidConfirmationToken { .. } => "runtime_confirmation_error",
|
||||
RuntimeError::ConfirmationStoreUnavailable { .. } => "runtime_confirmation_unavailable",
|
||||
RuntimeError::IdempotencyStoreUnavailable { .. } => "runtime_idempotency_unavailable",
|
||||
RuntimeError::IdempotencyInProgress { .. } => "runtime_idempotency_in_progress",
|
||||
RuntimeError::IdempotencyConflict { .. } => "runtime_idempotency_conflict",
|
||||
RuntimeError::IdempotencyOutcomeUnknown { .. } => "runtime_idempotency_outcome_unknown",
|
||||
RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error",
|
||||
RuntimeError::MissingAuthProfile { .. } => "runtime_auth_profile_error",
|
||||
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
|
||||
@@ -434,7 +450,11 @@ pub fn runtime_error_context(error: &RuntimeError) -> Option<Value> {
|
||||
"safety_class": safety_class,
|
||||
})),
|
||||
RuntimeError::InvalidConfirmationToken { operation_id }
|
||||
| RuntimeError::ConfirmationStoreUnavailable { operation_id } => Some(json!({
|
||||
| RuntimeError::ConfirmationStoreUnavailable { operation_id }
|
||||
| RuntimeError::IdempotencyStoreUnavailable { operation_id }
|
||||
| RuntimeError::IdempotencyInProgress { operation_id }
|
||||
| RuntimeError::IdempotencyConflict { operation_id }
|
||||
| RuntimeError::IdempotencyOutcomeUnknown { operation_id } => Some(json!({
|
||||
"operation_id": operation_id,
|
||||
})),
|
||||
RuntimeError::InvalidAuthSecretValue { secret_id, reason } => Some(json!({
|
||||
|
||||
+106
-10
@@ -1,4 +1,4 @@
|
||||
use std::{env, net::SocketAddr, path::PathBuf};
|
||||
use std::{env, net::SocketAddr, path::PathBuf, time::Duration};
|
||||
|
||||
use admin_api::{
|
||||
app::build_app,
|
||||
@@ -7,22 +7,50 @@ use admin_api::{
|
||||
state::AppState,
|
||||
};
|
||||
use crank_community_auth::PasswordIdentityProvider;
|
||||
use crank_observability::{
|
||||
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
|
||||
capture_critical_error,
|
||||
};
|
||||
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
|
||||
use crank_runtime::{
|
||||
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
|
||||
RuntimeLimits, SecretCrypto,
|
||||
};
|
||||
use sqlx::postgres::PgConnectOptions;
|
||||
use sqlx::{PgPool, postgres::PgConnectOptions};
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::info;
|
||||
use tracing::{info, warn};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
env::var("CRANK_LOG_LEVEL").unwrap_or_else(|_| "admin_api=info,tower_http=info".into()),
|
||||
)
|
||||
.init();
|
||||
let observability = crank_observability::init(ObservabilityConfig::from_env(
|
||||
"admin-api",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
"admin_api=info,tower_http=info",
|
||||
)?)?;
|
||||
let mut startup_completed = false;
|
||||
let result = run(&observability, &mut startup_completed).await;
|
||||
if result.is_err() {
|
||||
capture_critical_error(if startup_completed {
|
||||
CriticalErrorCategory::Internal
|
||||
} else {
|
||||
CriticalErrorCategory::Startup
|
||||
});
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn run(
|
||||
observability: &ObservabilityLifecycle,
|
||||
startup_completed: &mut bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let metrics_config =
|
||||
MetricsConfig::from_env("CRANK_ADMIN_METRICS_BIND", "127.0.0.1:9464".parse()?)?;
|
||||
let metrics_enabled = metrics_config.enabled();
|
||||
let metrics_server = if metrics_config.enabled() {
|
||||
Some(observability.metrics_surface(metrics_config).bind().await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let storage_root = PathBuf::from(
|
||||
env::var("CRANK_STORAGE_ROOT").unwrap_or_else(|_| "/var/lib/crank/storage".into()),
|
||||
@@ -36,6 +64,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
pool_config,
|
||||
)
|
||||
.await?;
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
}
|
||||
let auth_settings = AuthSettings {
|
||||
session_secret: env::var("CRANK_SESSION_SECRET")?,
|
||||
password_pepper: env::var("CRANK_PASSWORD_PEPPER")?,
|
||||
@@ -74,10 +105,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
.with_outbound_http_policy(outbound_http_policy)
|
||||
.with_identity_provider(std::sync::Arc::new(identity_provider))
|
||||
.build();
|
||||
let invocation_log_retention_days =
|
||||
positive_i64_from_env("CRANK_INVOCATION_LOG_RETENTION_DAYS", 30)?;
|
||||
service.bootstrap_admin_user().await?;
|
||||
if env_flag("CRANK_DEMO_SEED") {
|
||||
service.seed_demo_assets().await?;
|
||||
}
|
||||
spawn_invocation_log_cleanup(service.clone(), invocation_log_retention_days);
|
||||
let state = AppState {
|
||||
service,
|
||||
api_rate_limiter: if cache_config.backend.is_external() {
|
||||
@@ -92,6 +126,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let make_service = app.into_make_service_with_connect_info::<SocketAddr>();
|
||||
|
||||
info!(
|
||||
name: "admin.postgres_pool.configured",
|
||||
runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
|
||||
admin_rate_limit_rps = api_rate_limit.requests_per_second,
|
||||
admin_rate_limit_burst = api_rate_limit.burst,
|
||||
@@ -101,15 +136,76 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
acquire_timeout_ms = pool_config.acquire_timeout_ms,
|
||||
idle_timeout_ms = pool_config.idle_timeout_ms,
|
||||
max_lifetime_ms = pool_config.max_lifetime_ms,
|
||||
invocation_log_retention_days,
|
||||
"postgres pool configured"
|
||||
);
|
||||
info!("admin-api listening on {}", socket_addr);
|
||||
info!(
|
||||
name: "admin.server.listening",
|
||||
bind_address = %socket_addr,
|
||||
"admin-api listening"
|
||||
);
|
||||
*startup_completed = true;
|
||||
|
||||
axum::serve(listener, make_service).await?;
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = axum::serve(listener, make_service) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
}
|
||||
} else {
|
||||
axum::serve(listener, make_service).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn positive_i64_from_env(
|
||||
name: &'static str,
|
||||
default: i64,
|
||||
) -> Result<i64, Box<dyn std::error::Error>> {
|
||||
let value = match env::var(name) {
|
||||
Ok(raw) => raw.parse::<i64>()?,
|
||||
Err(env::VarError::NotPresent) => default,
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if value <= 0 {
|
||||
return Err(format!("{name} must be greater than zero").into());
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn spawn_invocation_log_cleanup(service: admin_api::service::AdminService, retention_days: i64) {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(60 * 60));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let cutoff = time::OffsetDateTime::now_utc() - time::Duration::days(retention_days);
|
||||
match service.cleanup_invocation_logs_before(cutoff).await {
|
||||
Ok(removed) if removed > 0 => info!(
|
||||
name: "admin.invocation_log_cleanup.completed",
|
||||
removed,
|
||||
"expired invocation logs removed"
|
||||
),
|
||||
Ok(_) => {}
|
||||
Err(_) => warn!(
|
||||
name: "admin.invocation_log_cleanup.failed",
|
||||
error_category = "registry_cleanup",
|
||||
"failed to remove expired invocation logs"
|
||||
),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn spawn_postgres_pool_metrics(pool: PgPool) {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn env_flag(name: &str) -> bool {
|
||||
matches!(
|
||||
env::var(name)
|
||||
|
||||
@@ -6,7 +6,7 @@ use axum::{
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use crank_runtime::RateLimitRejection;
|
||||
use crank_runtime::{RateLimitCheckError, RateLimitRejection};
|
||||
|
||||
use crate::{error::ApiError, state::AppState};
|
||||
|
||||
@@ -25,11 +25,16 @@ pub async fn apply_api_rate_limit(
|
||||
peer_ip,
|
||||
state.trust_forwarded_headers,
|
||||
);
|
||||
if let Err(rejection) = state.api_rate_limiter.check(&key).await {
|
||||
return Err(ApiError::rate_limited_with_context(
|
||||
"request rate limit exceeded",
|
||||
rejection_context(rejection),
|
||||
));
|
||||
if let Err(error) = state.api_rate_limiter.check(&key).await {
|
||||
return match error {
|
||||
RateLimitCheckError::Rejected(rejection) => Err(ApiError::rate_limited_with_context(
|
||||
"request rate limit exceeded",
|
||||
rejection_context(rejection),
|
||||
)),
|
||||
RateLimitCheckError::StoreUnavailable => {
|
||||
Err(ApiError::internal("rate limit service unavailable"))
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Ok(next.run(request).await)
|
||||
|
||||
@@ -4,11 +4,10 @@ use axum::{
|
||||
middleware::Next,
|
||||
response::Response,
|
||||
};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
|
||||
use tracing::{Instrument, info, info_span};
|
||||
|
||||
pub const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id");
|
||||
const MAX_REQUEST_ID_LEN: usize = 128;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RequestContext {
|
||||
@@ -21,145 +20,53 @@ pub async fn apply_request_context(mut request: Request, next: Next) -> Response
|
||||
};
|
||||
let method = request.method().clone();
|
||||
let path = request.uri().path().to_owned();
|
||||
let span = info_span!(
|
||||
target: "crank::trace",
|
||||
"http.request",
|
||||
request_id = %context.request_id,
|
||||
);
|
||||
set_remote_trace_parent(&span, request.headers());
|
||||
request.extensions_mut().insert(context.clone());
|
||||
|
||||
let mut response = next.run(request).await;
|
||||
info!(
|
||||
request_id = %context.request_id,
|
||||
method = %method,
|
||||
path,
|
||||
status = response.status().as_u16(),
|
||||
"admin request completed"
|
||||
);
|
||||
if let Ok(value) = HeaderValue::from_str(&context.request_id) {
|
||||
response.headers_mut().insert(REQUEST_ID_HEADER, value);
|
||||
}
|
||||
response
|
||||
with_request_correlation(context.request_id.clone(), async move {
|
||||
let mut response = next.run(request).instrument(span).await;
|
||||
info!(
|
||||
name: "admin.request.completed",
|
||||
request_id = %context.request_id,
|
||||
method = %method,
|
||||
path,
|
||||
status = response.status().as_u16(),
|
||||
"admin request completed"
|
||||
);
|
||||
if let Ok(value) = HeaderValue::from_str(&context.request_id) {
|
||||
response.headers_mut().insert(REQUEST_ID_HEADER, value);
|
||||
}
|
||||
response
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn resolve_request_id(headers: &HeaderMap) -> String {
|
||||
headers
|
||||
.get(&REQUEST_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| is_valid_request_id(value))
|
||||
.map(ToOwned::to_owned)
|
||||
.unwrap_or_else(|| Uuid::now_v7().to_string())
|
||||
}
|
||||
|
||||
fn is_valid_request_id(value: &str) -> bool {
|
||||
!value.is_empty()
|
||||
&& value.len() <= MAX_REQUEST_ID_LEN
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| matches!(byte, 0x21..=0x7e) && byte != b',' && byte != b';')
|
||||
RequestId::resolve(
|
||||
headers
|
||||
.get(&REQUEST_ID_HEADER)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
)
|
||||
.into_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::io;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use axum::{Router, routing::get};
|
||||
use reqwest::Client;
|
||||
use tokio::net::TcpListener;
|
||||
use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
|
||||
|
||||
use super::{REQUEST_ID_HEADER, apply_request_context, is_valid_request_id};
|
||||
|
||||
#[test]
|
||||
fn accepts_visible_ascii_request_ids() {
|
||||
assert!(is_valid_request_id("req_test_123"));
|
||||
assert!(is_valid_request_id("trace-123/abc"));
|
||||
assert!(crank_observability::RequestId::is_valid("req_test_123"));
|
||||
assert!(crank_observability::RequestId::is_valid("trace-123/abc"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_or_control_request_ids() {
|
||||
assert!(!is_valid_request_id(""));
|
||||
assert!(!is_valid_request_id("bad value"));
|
||||
assert!(!is_valid_request_id("bad\nvalue"));
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn logs_request_completion_with_request_id() {
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = tracing_subscriber::registry().with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_writer(writer.clone())
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_target(false)
|
||||
.compact()
|
||||
.with_filter(LevelFilter::INFO),
|
||||
);
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let app = Router::new()
|
||||
.route("/probe", get(|| async { "ok" }))
|
||||
.layer(axum::middleware::from_fn(apply_request_context));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
let response = Client::new()
|
||||
.get(format!("http://{address}/probe"))
|
||||
.header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.headers()[REQUEST_ID_HEADER.as_str()]
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
"req_admin_trace_123"
|
||||
);
|
||||
|
||||
let logs = writer.output();
|
||||
assert!(logs.contains("admin request completed"));
|
||||
assert!(logs.contains("req_admin_trace_123"));
|
||||
assert!(logs.contains("GET"));
|
||||
assert!(logs.contains("/probe"));
|
||||
assert!(logs.contains("status=200"));
|
||||
assert!(!crank_observability::RequestId::is_valid(""));
|
||||
assert!(!crank_observability::RequestId::is_valid("bad value"));
|
||||
assert!(!crank_observability::RequestId::is_valid("bad\nvalue"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,12 +10,36 @@ pub mod secrets;
|
||||
pub mod upstreams;
|
||||
pub mod workspaces;
|
||||
|
||||
use axum::Json;
|
||||
use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
pub async fn health() -> Json<serde_json::Value> {
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "ok"
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
|
||||
match state.service.readiness().await {
|
||||
Ok(()) => (
|
||||
StatusCode::OK,
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "ready",
|
||||
"checks": { "postgres": "ready" }
|
||||
})),
|
||||
),
|
||||
Err(error) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "admin-api",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": "not_ready" },
|
||||
"error": error.to_string()
|
||||
})),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ pub async fn export_workspace(
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let exported = state
|
||||
.service
|
||||
.export_workspace(&path.workspace_id.as_str().into())
|
||||
.export_workspace_catalog_snapshot(&path.workspace_id.as_str().into())
|
||||
.await?;
|
||||
Ok(Json(json!(exported)))
|
||||
}
|
||||
|
||||
@@ -103,7 +103,7 @@ pub async fn change_password(
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state
|
||||
.service
|
||||
.change_password(&session.user.id, payload)
|
||||
.change_password(&session.user.id, &session.session_id, payload)
|
||||
.await?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
+248
-47
@@ -12,16 +12,18 @@ use crank_core::{
|
||||
};
|
||||
use crank_mapping::{MappingRule, MappingSet};
|
||||
use crank_registry::{
|
||||
AgentSummary, CreateInvocationLogRequest, OperationAgentRef, OperationSummary,
|
||||
OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
|
||||
AgentSummary, CreateInvocationLogRequest, InvocationHistoryWriteOutcome, OperationAgentRef,
|
||||
OperationSummary, OperationUsageSummary, PostgresRegistry, RegistryOperation, UsageBucket,
|
||||
};
|
||||
use crank_runtime::{
|
||||
OutboundHttpPolicy, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeExecutor, SecretCrypto,
|
||||
};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest, Sha256};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tracing::Instrument;
|
||||
use uuid::Uuid;
|
||||
|
||||
mod agents;
|
||||
@@ -74,6 +76,11 @@ pub struct AdminServiceBuilder {
|
||||
pub use crate::dto::*;
|
||||
|
||||
impl AdminService {
|
||||
pub async fn readiness(&self) -> Result<(), ApiError> {
|
||||
self.registry.ping().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn new(
|
||||
registry: PostgresRegistry,
|
||||
@@ -199,16 +206,33 @@ impl AdminServiceBuilder {
|
||||
}
|
||||
|
||||
impl AdminService {
|
||||
pub async fn export_workspace(
|
||||
pub async fn export_workspace_catalog_snapshot(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
) -> Result<WorkspaceExportResponse, ApiError> {
|
||||
) -> Result<WorkspaceCatalogSnapshotResponse, ApiError> {
|
||||
let workspace = self.get_workspace(workspace_id).await?;
|
||||
let operations = self.list_operations(workspace_id).await?;
|
||||
let agents = self.list_agents(workspace_id).await?;
|
||||
let platform_api_keys = self.registry.list_platform_api_keys(workspace_id).await?;
|
||||
|
||||
Ok(WorkspaceExportResponse {
|
||||
Ok(WorkspaceCatalogSnapshotResponse {
|
||||
kind: "workspace_catalog_snapshot".to_owned(),
|
||||
format_version: "1".to_owned(),
|
||||
restorable: false,
|
||||
included: vec![
|
||||
"workspace_settings".to_owned(),
|
||||
"operation_summaries".to_owned(),
|
||||
"agent_summaries".to_owned(),
|
||||
"platform_api_key_metadata".to_owned(),
|
||||
],
|
||||
excluded: vec![
|
||||
"operation_versions_and_samples".to_owned(),
|
||||
"agent_versions_and_bindings".to_owned(),
|
||||
"secret_metadata_and_values".to_owned(),
|
||||
"secret_values".to_owned(),
|
||||
"invocation_logs_and_usage".to_owned(),
|
||||
"authentication_sessions".to_owned(),
|
||||
],
|
||||
workspace,
|
||||
operations,
|
||||
agents,
|
||||
@@ -239,9 +263,13 @@ impl AdminService {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let auth_profile = self
|
||||
.registry
|
||||
.get_auth_profile(workspace_id, auth_profile_id)
|
||||
let span = Stage::AuthResolve.span();
|
||||
let result = async {
|
||||
let auth_profile = observe_db_query(
|
||||
DbOperation::AuthProfileRead,
|
||||
self.registry
|
||||
.get_auth_profile(workspace_id, auth_profile_id),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load auth profile",
|
||||
@@ -251,9 +279,20 @@ impl AdminService {
|
||||
auth_profile_id: auth_profile_id.as_str().to_owned(),
|
||||
})?;
|
||||
|
||||
self.resolve_auth_profile(workspace_id, &auth_profile)
|
||||
.await
|
||||
.map(Some)
|
||||
self.resolve_auth_profile(workspace_id, &auth_profile)
|
||||
.await
|
||||
.map(Some)
|
||||
}
|
||||
.instrument(span.clone())
|
||||
.await;
|
||||
match &result {
|
||||
Ok(_) => StageOutcome::Success.record(&span),
|
||||
Err(_) => {
|
||||
StageOutcome::Error.record(&span);
|
||||
ErrorCategory::Configuration.record(&span);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn resolve_auth_profile(
|
||||
@@ -265,40 +304,46 @@ impl AdminService {
|
||||
let used_at = OffsetDateTime::now_utc();
|
||||
|
||||
for secret_id in auth_profile.config.secret_ids() {
|
||||
let secret = self
|
||||
.registry
|
||||
.get_secret(workspace_id, secret_id)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load secret",
|
||||
details: error.to_string(),
|
||||
})?
|
||||
.ok_or_else(|| RuntimeError::MissingSecret {
|
||||
secret_id: secret_id.as_str().to_owned(),
|
||||
})?;
|
||||
let version = self
|
||||
.registry
|
||||
.get_current_secret_version(workspace_id, secret_id)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load current secret version",
|
||||
details: error.to_string(),
|
||||
})?
|
||||
.ok_or_else(|| RuntimeError::MissingSecretVersion {
|
||||
secret_id: secret_id.as_str().to_owned(),
|
||||
version: secret.secret.current_version,
|
||||
})?;
|
||||
let secret = observe_db_query(
|
||||
DbOperation::SecretRead,
|
||||
self.registry.get_secret(workspace_id, secret_id),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load secret",
|
||||
details: error.to_string(),
|
||||
})?
|
||||
.ok_or_else(|| RuntimeError::MissingSecret {
|
||||
secret_id: secret_id.as_str().to_owned(),
|
||||
})?;
|
||||
let version = observe_db_query(
|
||||
DbOperation::SecretRead,
|
||||
self.registry
|
||||
.get_current_secret_version(workspace_id, secret_id),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load current secret version",
|
||||
details: error.to_string(),
|
||||
})?
|
||||
.ok_or_else(|| RuntimeError::MissingSecretVersion {
|
||||
secret_id: secret_id.as_str().to_owned(),
|
||||
version: secret.secret.current_version,
|
||||
})?;
|
||||
let plaintext = self.secret_crypto.decrypt(
|
||||
&version.secret_version.key_version,
|
||||
&version.secret_version.ciphertext,
|
||||
)?;
|
||||
self.registry
|
||||
.touch_secret(workspace_id, secret_id, &used_at)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "touch secret",
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
observe_db_query(
|
||||
DbOperation::SecretTouch,
|
||||
self.registry
|
||||
.touch_secret(workspace_id, secret_id, &used_at),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "touch secret",
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
secrets.insert(secret_id.clone(), plaintext);
|
||||
}
|
||||
|
||||
@@ -412,7 +457,7 @@ impl AdminService {
|
||||
async fn record_invocation(
|
||||
&self,
|
||||
request: InvocationRecordRequest<'_>,
|
||||
) -> Result<(), ApiError> {
|
||||
) -> InvocationHistoryWriteOutcome {
|
||||
let log = InvocationLog {
|
||||
id: InvocationLogId::new(new_prefixed_id("log")),
|
||||
workspace_id: request.workspace_id.clone(),
|
||||
@@ -432,11 +477,70 @@ impl AdminService {
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
};
|
||||
|
||||
self.registry
|
||||
.create_invocation_log(CreateInvocationLogRequest { log: &log })
|
||||
.await?;
|
||||
let history_span = crank_trace::Stage::HistoryWrite.span();
|
||||
let (outcome, db_span) = async {
|
||||
let db_span = crank_trace::Stage::DbQuery
|
||||
.db_span(crank_trace::DbOperation::InvocationHistoryWrite)
|
||||
.expect("database stage");
|
||||
let outcome = self
|
||||
.registry
|
||||
.create_invocation_log(CreateInvocationLogRequest { log: &log })
|
||||
.instrument(db_span.clone())
|
||||
.await;
|
||||
(outcome, db_span)
|
||||
}
|
||||
.instrument(history_span.clone())
|
||||
.await;
|
||||
match outcome {
|
||||
InvocationHistoryWriteOutcome::Recorded => {
|
||||
crank_trace::StageOutcome::Success.record(&db_span);
|
||||
crank_trace::StageOutcome::Success.record(&history_span);
|
||||
}
|
||||
InvocationHistoryWriteOutcome::Lost(_) => {
|
||||
crank_trace::StageOutcome::Error.record(&db_span);
|
||||
crank_trace::ErrorCategory::Database.record(&db_span);
|
||||
crank_trace::StageOutcome::Error.record(&history_span);
|
||||
crank_trace::ErrorCategory::History.record(&history_span);
|
||||
}
|
||||
}
|
||||
drop(db_span);
|
||||
drop(history_span);
|
||||
observe_invocation_history_outcome(
|
||||
outcome,
|
||||
request.request_id,
|
||||
request.status,
|
||||
"admin_test_run",
|
||||
);
|
||||
outcome
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
fn observe_invocation_history_outcome(
|
||||
outcome: InvocationHistoryWriteOutcome,
|
||||
request_id: Option<&str>,
|
||||
status: crank_core::InvocationStatus,
|
||||
source: &'static str,
|
||||
) {
|
||||
let Some(loss) = outcome.loss() else {
|
||||
return;
|
||||
};
|
||||
crank_observability::record_operational_incident(
|
||||
crank_observability::OperationalIncident::InvocationHistoryLost,
|
||||
);
|
||||
tracing::warn!(
|
||||
name: "admin.invocation_history.lost",
|
||||
request_id = request_id.unwrap_or_default(),
|
||||
source,
|
||||
invocation_status = invocation_status_label(status),
|
||||
error_category = loss.category.as_str(),
|
||||
"invocation history was not recorded"
|
||||
);
|
||||
}
|
||||
|
||||
fn invocation_status_label(status: crank_core::InvocationStatus) -> &'static str {
|
||||
match status {
|
||||
crank_core::InvocationStatus::Ok => "ok",
|
||||
crank_core::InvocationStatus::Error => "error",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -543,6 +647,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
|
||||
RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
|
||||
RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
|
||||
RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
|
||||
RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
|
||||
RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
|
||||
RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
|
||||
RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
|
||||
RuntimeError::RestAdapter(_) => "rest_error",
|
||||
RuntimeError::ProtocolAdapter(_) => "adapter_error",
|
||||
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
|
||||
@@ -761,7 +869,25 @@ fn tool_quality_mapping_rule(rule: &MappingRule) -> ToolQualityMappingRule {
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::items_after_test_module)]
|
||||
mod tests {
|
||||
use super::{validate_profile_display_name, validate_profile_email};
|
||||
use std::{
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use crank_core::InvocationStatus;
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, OperationalIncident, RedactionLimits, ServiceIdentity,
|
||||
operational_incident_total,
|
||||
};
|
||||
use crank_registry::{
|
||||
InvocationHistoryLoss, InvocationHistoryLossCategory, InvocationHistoryWriteOutcome,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
use super::{
|
||||
observe_invocation_history_outcome, validate_profile_display_name, validate_profile_email,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn validates_profile_identity_fields() {
|
||||
@@ -782,6 +908,81 @@ mod tests {
|
||||
assert!(validate_profile_display_name(&"x".repeat(81)).is_err());
|
||||
assert!(validate_profile_email("owner <html>@crank.local").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_bounded_history_loss_incident() {
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = crank_observability::build_subscriber(
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
|
||||
"info",
|
||||
RedactionLimits::default(),
|
||||
),
|
||||
writer.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let before = operational_incident_total(OperationalIncident::InvocationHistoryLost);
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
|
||||
observe_invocation_history_outcome(
|
||||
InvocationHistoryWriteOutcome::Lost(InvocationHistoryLoss {
|
||||
category: InvocationHistoryLossCategory::InvalidRecord,
|
||||
}),
|
||||
Some("req_admin_dc08"),
|
||||
InvocationStatus::Error,
|
||||
"admin_test_run",
|
||||
);
|
||||
|
||||
let output = writer.output();
|
||||
assert!(!output.contains("dc08-canary-secret"));
|
||||
let event: Value = output
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str(line).unwrap())
|
||||
.find(|event: &Value| event["event"] == "admin.invocation_history.lost")
|
||||
.unwrap();
|
||||
assert_eq!(event["request_id"], "req_admin_dc08");
|
||||
assert_eq!(event["fields"]["source"], "admin_test_run");
|
||||
assert_eq!(event["fields"]["invocation_status"], "error");
|
||||
assert_eq!(event["fields"]["error_category"], "invalid_record");
|
||||
assert!(operational_incident_total(OperationalIncident::InvocationHistoryLost) > before);
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn enrich_operation_summary(
|
||||
|
||||
@@ -296,7 +296,12 @@ impl AdminService {
|
||||
bindings: &[],
|
||||
})
|
||||
.await?;
|
||||
info!(agent_id = %agent_id.as_str(), version = 1, "agent created");
|
||||
info!(
|
||||
name: "admin.agent.created",
|
||||
agent_id = %agent_id.as_str(),
|
||||
version = 1,
|
||||
"agent created"
|
||||
);
|
||||
|
||||
Ok(CreatedAgentResponse {
|
||||
agent_id: agent_id.as_str().to_owned(),
|
||||
@@ -417,6 +422,7 @@ impl AdminService {
|
||||
})
|
||||
.await?;
|
||||
info!(
|
||||
name: "admin.agent.bindings_saved",
|
||||
agent_id = %agent_id.as_str(),
|
||||
version = current_version.version,
|
||||
binding_count = bindings.len(),
|
||||
@@ -531,7 +537,12 @@ impl AdminService {
|
||||
published_by: None,
|
||||
})
|
||||
.await?;
|
||||
info!(agent_id = %agent_id.as_str(), version, "agent published");
|
||||
info!(
|
||||
name: "admin.agent.published",
|
||||
agent_id = %agent_id.as_str(),
|
||||
version,
|
||||
"agent published"
|
||||
);
|
||||
|
||||
Ok(PublishAgentResponse {
|
||||
agent_id: agent_id.as_str().to_owned(),
|
||||
@@ -589,7 +600,11 @@ impl AdminService {
|
||||
self.registry
|
||||
.unpublish_agent(workspace_id, agent_id, &updated_at)
|
||||
.await?;
|
||||
info!(agent_id = %agent_id.as_str(), "agent moved to draft");
|
||||
info!(
|
||||
name: "admin.agent.unpublished",
|
||||
agent_id = %agent_id.as_str(),
|
||||
"agent moved to draft"
|
||||
);
|
||||
|
||||
Ok(AgentMutationResult {
|
||||
agent_id: agent_id.as_str().to_owned(),
|
||||
@@ -609,7 +624,11 @@ impl AdminService {
|
||||
self.registry
|
||||
.archive_agent(workspace_id, agent_id, &updated_at)
|
||||
.await?;
|
||||
info!(agent_id = %agent_id.as_str(), "agent archived");
|
||||
info!(
|
||||
name: "admin.agent.archived",
|
||||
agent_id = %agent_id.as_str(),
|
||||
"agent archived"
|
||||
);
|
||||
|
||||
Ok(AgentMutationResult {
|
||||
agent_id: agent_id.as_str().to_owned(),
|
||||
|
||||
@@ -22,7 +22,7 @@ impl AdminService {
|
||||
)?;
|
||||
let user_id = self
|
||||
.registry
|
||||
.upsert_bootstrap_user(
|
||||
.ensure_bootstrap_user(
|
||||
&self.auth_settings.bootstrap_admin.email,
|
||||
&self.auth_settings.bootstrap_admin.display_name,
|
||||
&password_hash,
|
||||
@@ -227,6 +227,7 @@ impl AdminService {
|
||||
pub async fn change_password(
|
||||
&self,
|
||||
user_id: &crank_core::UserId,
|
||||
current_session_id: &UserSessionId,
|
||||
payload: ChangePasswordPayload,
|
||||
) -> Result<(), ApiError> {
|
||||
if payload.new_password.len() < 12 {
|
||||
@@ -257,7 +258,11 @@ impl AdminService {
|
||||
let password_hash =
|
||||
hash_password(&payload.new_password, &self.auth_settings.password_pepper)?;
|
||||
self.registry
|
||||
.update_user_password(user_id, &password_hash)
|
||||
.update_user_password_and_revoke_other_sessions(
|
||||
user_id,
|
||||
current_session_id,
|
||||
&password_hash,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -131,6 +131,7 @@ impl AdminService {
|
||||
Ok(()) => Ok(()),
|
||||
Err(RegistryError::OperationHasPublishedAgentBindings { .. }) => {
|
||||
tracing::warn!(
|
||||
name: "admin.demo_operation.cleanup_skipped",
|
||||
operation_id = %operation_id.as_str(),
|
||||
"legacy demo operation is still bound to a published agent; leaving it in place"
|
||||
);
|
||||
@@ -335,7 +336,7 @@ impl AdminService {
|
||||
}),
|
||||
response_preview: demo_rest_response_sample(),
|
||||
})
|
||||
.await?;
|
||||
.await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +110,7 @@ impl AdminService {
|
||||
warnings,
|
||||
};
|
||||
info!(
|
||||
name: "admin.operation.imported",
|
||||
operation_id = %response.operation_id,
|
||||
version = response.version,
|
||||
"operation imported by upsert"
|
||||
@@ -125,6 +126,7 @@ impl AdminService {
|
||||
warnings,
|
||||
};
|
||||
info!(
|
||||
name: "admin.operation.imported",
|
||||
operation_id = %response.operation_id,
|
||||
version = response.version,
|
||||
"operation imported by upsert"
|
||||
|
||||
@@ -8,9 +8,11 @@ use crank_import::rest::{
|
||||
ImportFinding, ImportFindingSeverity, ImportOperationCandidate, operation_draft_from_candidate,
|
||||
};
|
||||
use crank_registry::{
|
||||
CreateImportJobRequest, FinishImportJobRequest, ImportJobId, ImportJobKind, ImportJobStatus,
|
||||
ApplyImportJobRequest, CreateImportJobRequest, ImportConflictMode, ImportJobId, ImportJobKind,
|
||||
ImportJobStatus, ImportOperationDraft,
|
||||
};
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tracing::{info, instrument};
|
||||
|
||||
@@ -50,7 +52,7 @@ impl AdminService {
|
||||
kind: ImportJobKind::OpenApi,
|
||||
source_format: &preview.source.format,
|
||||
source_version: preview.source.version.as_deref(),
|
||||
status: ImportJobStatus::Completed,
|
||||
status: ImportJobStatus::Pending,
|
||||
preview_payload: &preview_payload,
|
||||
created_at: &now,
|
||||
expires_at: &expires_at,
|
||||
@@ -99,9 +101,13 @@ impl AdminService {
|
||||
return Err(ApiError::validation("import job kind is not openapi"));
|
||||
}
|
||||
|
||||
let preview: crank_import::rest::ImportPreview =
|
||||
serde_json::from_value(job.preview_payload.clone())
|
||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
let stored_preview = job
|
||||
.preview_payload
|
||||
.get("preview")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| job.preview_payload.clone());
|
||||
let preview: crank_import::rest::ImportPreview = serde_json::from_value(stored_preview)
|
||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
let selected = payload
|
||||
.selected_operation_keys
|
||||
.iter()
|
||||
@@ -120,10 +126,8 @@ impl AdminService {
|
||||
}
|
||||
}
|
||||
|
||||
let mut created = Vec::new();
|
||||
let mut skipped = Vec::new();
|
||||
let mut findings = Vec::new();
|
||||
let mut created_ids = Vec::new();
|
||||
let mut operations = Vec::new();
|
||||
|
||||
for operation_key in selected {
|
||||
let Some(candidate) = candidates.get(&operation_key) else {
|
||||
@@ -137,44 +141,7 @@ impl AdminService {
|
||||
let mut draft =
|
||||
operation_draft_from_candidate(candidate, payload.server_url.as_deref());
|
||||
attach_import_findings(&mut draft, candidate);
|
||||
if let Some(existing_name) = self
|
||||
.find_operation_by_name(workspace_id, &draft.name)
|
||||
.await?
|
||||
.map(|operation| operation.name)
|
||||
{
|
||||
if payload.conflict_mode == "skip" {
|
||||
skipped.push(OpenApiImportSkippedOperation {
|
||||
operation_key: candidate.key.clone(),
|
||||
name: draft.name.clone(),
|
||||
reason: "operation with this name already exists".to_owned(),
|
||||
});
|
||||
findings.push(ImportFinding {
|
||||
code: "operation_name_conflict".to_owned(),
|
||||
severity: ImportFindingSeverity::Warning,
|
||||
message: format!(
|
||||
"Операция {} уже существует и была пропущена.",
|
||||
draft.name
|
||||
),
|
||||
operation_key: Some(candidate.key.clone()),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let renamed = self
|
||||
.next_available_operation_name(workspace_id, &draft.name)
|
||||
.await?;
|
||||
findings.push(ImportFinding {
|
||||
code: "operation_name_renamed".to_owned(),
|
||||
severity: ImportFindingSeverity::Info,
|
||||
message: format!(
|
||||
"Операция {existing_name} уже существует, новый черновик создан как {renamed}."
|
||||
),
|
||||
operation_key: Some(candidate.key.clone()),
|
||||
});
|
||||
draft.name = renamed;
|
||||
}
|
||||
|
||||
let payload = OperationPayload {
|
||||
let operation = self.new_operation_snapshot(OperationPayload {
|
||||
name: draft.name.clone(),
|
||||
display_name: draft.display_name.clone(),
|
||||
category: draft.category,
|
||||
@@ -197,27 +164,74 @@ impl AdminService {
|
||||
},
|
||||
tool_description: draft.tool_description,
|
||||
wizard_state: draft.wizard_state,
|
||||
};
|
||||
let result = self.create_operation(workspace_id, payload).await?;
|
||||
created_ids.push(result.operation_id.clone());
|
||||
created.push(OpenApiImportCreatedOperation {
|
||||
operation_id: result.operation_id,
|
||||
name: draft.name,
|
||||
version: result.version,
|
||||
})?;
|
||||
operations.push(ImportOperationDraft {
|
||||
operation_key: candidate.key.clone(),
|
||||
operation,
|
||||
});
|
||||
}
|
||||
|
||||
let finished_at = OffsetDateTime::now_utc();
|
||||
self.registry
|
||||
.finish_import_job(FinishImportJobRequest {
|
||||
let application_key = openapi_application_key(&payload)?;
|
||||
let conflict_mode = if payload.conflict_mode == "skip" {
|
||||
ImportConflictMode::Skip
|
||||
} else {
|
||||
ImportConflictMode::Rename
|
||||
};
|
||||
let applied = self
|
||||
.registry
|
||||
.apply_import_job(ApplyImportJobRequest {
|
||||
id: job_id,
|
||||
status: ImportJobStatus::Completed,
|
||||
created_operation_ids: &json!(created_ids),
|
||||
error_text: None,
|
||||
workspace_id,
|
||||
application_key: &application_key,
|
||||
conflict_mode,
|
||||
operations: &operations,
|
||||
finished_at: &finished_at,
|
||||
})
|
||||
.await?;
|
||||
|
||||
let created = applied
|
||||
.created
|
||||
.iter()
|
||||
.map(|operation| OpenApiImportCreatedOperation {
|
||||
operation_id: operation.operation_id.as_str().to_owned(),
|
||||
name: operation.name.clone(),
|
||||
version: operation.version,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut findings = applied
|
||||
.created
|
||||
.iter()
|
||||
.filter_map(|operation| {
|
||||
operation.renamed_from.as_ref().map(|previous_name| ImportFinding {
|
||||
code: "operation_name_renamed".to_owned(),
|
||||
severity: ImportFindingSeverity::Info,
|
||||
message: format!(
|
||||
"Операция {previous_name} уже существует, новый черновик создан как {}.",
|
||||
operation.name
|
||||
),
|
||||
operation_key: Some(operation.operation_key.clone()),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for operation in applied.skipped {
|
||||
skipped.push(OpenApiImportSkippedOperation {
|
||||
operation_key: operation.operation_key.clone(),
|
||||
name: operation.name.clone(),
|
||||
reason: "operation with this name already exists".to_owned(),
|
||||
});
|
||||
findings.push(ImportFinding {
|
||||
code: operation.reason,
|
||||
severity: ImportFindingSeverity::Warning,
|
||||
message: format!(
|
||||
"Операция {} уже существует и была пропущена.",
|
||||
operation.name
|
||||
),
|
||||
operation_key: Some(operation.operation_key),
|
||||
});
|
||||
}
|
||||
info!(
|
||||
name: "admin.openapi_import.completed",
|
||||
created = created.len(),
|
||||
skipped = skipped.len(),
|
||||
"openapi import created drafts"
|
||||
@@ -229,25 +243,21 @@ impl AdminService {
|
||||
findings,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_available_operation_name(
|
||||
&self,
|
||||
workspace_id: &WorkspaceId,
|
||||
base_name: &str,
|
||||
) -> Result<String, ApiError> {
|
||||
for index in 2.. {
|
||||
let candidate = format!("{base_name}_{index}");
|
||||
if self
|
||||
.find_operation_by_name(workspace_id, &candidate)
|
||||
.await?
|
||||
.is_none()
|
||||
{
|
||||
return Ok(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
unreachable!()
|
||||
}
|
||||
fn openapi_application_key(payload: &OpenApiImportCreatePayload) -> Result<String, ApiError> {
|
||||
let selected_operation_keys = payload
|
||||
.selected_operation_keys
|
||||
.iter()
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
let canonical = serde_json::to_vec(&json!({
|
||||
"selected_operation_keys": selected_operation_keys,
|
||||
"server_url": payload.server_url.as_deref(),
|
||||
"conflict_mode": payload.conflict_mode.as_str(),
|
||||
}))
|
||||
.map_err(|error| ApiError::internal(error.to_string()))?;
|
||||
Ok(format!("{:x}", Sha256::digest(canonical)))
|
||||
}
|
||||
|
||||
fn attach_import_findings(
|
||||
|
||||
@@ -19,6 +19,16 @@ use crate::{
|
||||
};
|
||||
|
||||
impl AdminService {
|
||||
pub async fn cleanup_invocation_logs_before(
|
||||
&self,
|
||||
cutoff: OffsetDateTime,
|
||||
) -> Result<u64, ApiError> {
|
||||
self.registry
|
||||
.delete_invocation_logs_before(cutoff)
|
||||
.await
|
||||
.map_err(ApiError::from)
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub async fn list_logs(
|
||||
&self,
|
||||
|
||||
@@ -141,7 +141,6 @@ impl AdminService {
|
||||
workspace_id: &WorkspaceId,
|
||||
payload: OperationPayload,
|
||||
) -> Result<CreatedOperationResponse, ApiError> {
|
||||
self.validate_operation_payload(&payload)?;
|
||||
self.ensure_workspace_exists(workspace_id).await?;
|
||||
|
||||
if self
|
||||
@@ -155,10 +154,36 @@ impl AdminService {
|
||||
));
|
||||
}
|
||||
|
||||
let snapshot = self.new_operation_snapshot(payload)?;
|
||||
let operation_id = snapshot.id.clone();
|
||||
|
||||
self.registry
|
||||
.create_operation(workspace_id, &snapshot, None)
|
||||
.await?;
|
||||
info!(
|
||||
name: "admin.operation.created",
|
||||
operation_id = %operation_id.as_str(),
|
||||
version = 1,
|
||||
"operation created"
|
||||
);
|
||||
|
||||
Ok(CreatedOperationResponse {
|
||||
operation_id: operation_id.as_str().to_owned(),
|
||||
workspace_id: workspace_id.as_str().to_owned(),
|
||||
version: 1,
|
||||
status: OperationStatus::Draft,
|
||||
updated_at: format_timestamp(snapshot.updated_at),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn new_operation_snapshot(
|
||||
&self,
|
||||
payload: OperationPayload,
|
||||
) -> Result<RegistryOperation, ApiError> {
|
||||
self.validate_operation_payload(&payload)?;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let operation_id = OperationId::new(new_prefixed_id("op"));
|
||||
let snapshot = RegistryOperation {
|
||||
id: operation_id.clone(),
|
||||
Ok(RegistryOperation {
|
||||
id: OperationId::new(new_prefixed_id("op")),
|
||||
name: payload.name,
|
||||
display_name: payload.display_name,
|
||||
category: payload.category,
|
||||
@@ -183,19 +208,6 @@ impl AdminService {
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
published_at: None,
|
||||
};
|
||||
|
||||
self.registry
|
||||
.create_operation(workspace_id, &snapshot, None)
|
||||
.await?;
|
||||
info!(operation_id = %operation_id.as_str(), version = 1, "operation created");
|
||||
|
||||
Ok(CreatedOperationResponse {
|
||||
operation_id: operation_id.as_str().to_owned(),
|
||||
workspace_id: workspace_id.as_str().to_owned(),
|
||||
version: 1,
|
||||
status: OperationStatus::Draft,
|
||||
updated_at: format_timestamp(snapshot.updated_at),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -279,7 +291,12 @@ impl AdminService {
|
||||
created_by: None,
|
||||
})
|
||||
.await?;
|
||||
info!(operation_id = %operation_id.as_str(), version, "operation version created");
|
||||
info!(
|
||||
name: "admin.operation.version_created",
|
||||
operation_id = %operation_id.as_str(),
|
||||
version,
|
||||
"operation version created"
|
||||
);
|
||||
|
||||
Ok(CreatedOperationResponse {
|
||||
operation_id: operation_id.as_str().to_owned(),
|
||||
@@ -364,7 +381,12 @@ impl AdminService {
|
||||
published_by: None,
|
||||
})
|
||||
.await?;
|
||||
info!(operation_id = %operation_id.as_str(), version, "operation published");
|
||||
info!(
|
||||
name: "admin.operation.published",
|
||||
operation_id = %operation_id.as_str(),
|
||||
version,
|
||||
"operation published"
|
||||
);
|
||||
|
||||
Ok(PublishResponse {
|
||||
operation_id: operation_id.as_str().to_owned(),
|
||||
@@ -431,37 +453,44 @@ impl AdminService {
|
||||
.await?;
|
||||
let runtime = RuntimeOperation::from(record.snapshot.clone());
|
||||
let mode = ExecutionMode::Unary;
|
||||
let request_preview =
|
||||
match build_request_preview(&record.snapshot.input_mapping, &payload.input) {
|
||||
Ok(preview) => preview,
|
||||
Err(error) => {
|
||||
self.record_invocation(InvocationRecordRequest {
|
||||
workspace_id,
|
||||
agent_id: None,
|
||||
operation: &record.snapshot,
|
||||
request_id: Some(request_id),
|
||||
source: InvocationSource::AdminTestRun,
|
||||
level: InvocationLevel::Error,
|
||||
status: InvocationStatus::Error,
|
||||
message: "mapping preview failed".to_owned(),
|
||||
status_code: None,
|
||||
error_kind: Some("mapping".to_owned()),
|
||||
duration_ms: 0,
|
||||
request_preview: Value::Null,
|
||||
response_preview: Value::Null,
|
||||
})
|
||||
.await?;
|
||||
return Ok(TestRunResult {
|
||||
ok: false,
|
||||
mode,
|
||||
request_preview: Value::Null,
|
||||
response_preview: Value::Null,
|
||||
errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping(
|
||||
error,
|
||||
))],
|
||||
});
|
||||
}
|
||||
};
|
||||
let preview_span = crank_trace::Stage::RuntimeArgumentsMap.span();
|
||||
let preview_result = preview_span
|
||||
.in_scope(|| build_request_preview(&record.snapshot.input_mapping, &payload.input));
|
||||
let request_preview = match preview_result {
|
||||
Ok(preview) => preview,
|
||||
Err(error) => {
|
||||
crank_trace::StageOutcome::Error.record(&preview_span);
|
||||
crank_trace::ErrorCategory::Mapping.record(&preview_span);
|
||||
drop(preview_span);
|
||||
self.record_invocation(InvocationRecordRequest {
|
||||
workspace_id,
|
||||
agent_id: None,
|
||||
operation: &record.snapshot,
|
||||
request_id: Some(request_id),
|
||||
source: InvocationSource::AdminTestRun,
|
||||
level: InvocationLevel::Error,
|
||||
status: InvocationStatus::Error,
|
||||
message: "mapping preview failed".to_owned(),
|
||||
status_code: None,
|
||||
error_kind: Some("mapping".to_owned()),
|
||||
duration_ms: 0,
|
||||
request_preview: Value::Null,
|
||||
response_preview: Value::Null,
|
||||
})
|
||||
.await;
|
||||
return Ok(TestRunResult {
|
||||
ok: false,
|
||||
mode,
|
||||
request_preview: Value::Null,
|
||||
response_preview: Value::Null,
|
||||
errors: vec![crate::error::runtime_test_failure(&RuntimeError::Mapping(
|
||||
error,
|
||||
))],
|
||||
});
|
||||
}
|
||||
};
|
||||
crank_trace::StageOutcome::Success.record(&preview_span);
|
||||
drop(preview_span);
|
||||
|
||||
let resolved_auth = self
|
||||
.resolve_operation_auth(workspace_id, &runtime.execution_config)
|
||||
@@ -497,7 +526,7 @@ impl AdminService {
|
||||
request_preview: request_preview.clone(),
|
||||
response_preview: response_preview.clone(),
|
||||
})
|
||||
.await?;
|
||||
.await;
|
||||
Ok(TestRunResult {
|
||||
ok: true,
|
||||
mode,
|
||||
@@ -524,7 +553,7 @@ impl AdminService {
|
||||
request_preview: request_preview.clone(),
|
||||
response_preview: Value::Null,
|
||||
})
|
||||
.await?;
|
||||
.await;
|
||||
Ok(TestRunResult {
|
||||
ok: false,
|
||||
mode,
|
||||
|
||||
@@ -50,6 +50,7 @@ impl AdminService {
|
||||
.save_sample_metadata(SaveSampleMetadataRequest { sample: &metadata })
|
||||
.await?;
|
||||
info!(
|
||||
name: "admin.sample.saved",
|
||||
operation_id = %operation_id.as_str(),
|
||||
sample_id = %metadata.id.as_str(),
|
||||
version,
|
||||
@@ -119,7 +120,11 @@ impl AdminService {
|
||||
input_mapping,
|
||||
output_mapping,
|
||||
};
|
||||
info!(operation_id = %operation_id.as_str(), "draft generated from samples");
|
||||
info!(
|
||||
name: "admin.operation_draft.generated",
|
||||
operation_id = %operation_id.as_str(),
|
||||
"draft generated from samples"
|
||||
);
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -92,7 +92,11 @@ impl AdminService {
|
||||
created_by,
|
||||
})
|
||||
.await?;
|
||||
info!(secret_id = %secret.id.as_str(), "secret created");
|
||||
info!(
|
||||
name: "admin.secret.created",
|
||||
secret_id = %secret.id.as_str(),
|
||||
"secret created"
|
||||
);
|
||||
|
||||
Ok(secret)
|
||||
}
|
||||
@@ -126,7 +130,11 @@ impl AdminService {
|
||||
created_by,
|
||||
})
|
||||
.await?;
|
||||
info!(secret_id = %secret_id.as_str(), "secret rotated");
|
||||
info!(
|
||||
name: "admin.secret.rotated",
|
||||
secret_id = %secret_id.as_str(),
|
||||
"secret rotated"
|
||||
);
|
||||
|
||||
self.get_secret(workspace_id, secret_id).await
|
||||
}
|
||||
@@ -152,7 +160,11 @@ impl AdminService {
|
||||
.into());
|
||||
}
|
||||
self.registry.delete_secret(workspace_id, secret_id).await?;
|
||||
info!(secret_id = %secret_id.as_str(), "secret deleted");
|
||||
info!(
|
||||
name: "admin.secret.deleted",
|
||||
secret_id = %secret_id.as_str(),
|
||||
"secret deleted"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -201,7 +213,11 @@ impl AdminService {
|
||||
profile: &profile,
|
||||
})
|
||||
.await?;
|
||||
info!(auth_profile_id = %profile.id.as_str(), "auth profile created");
|
||||
info!(
|
||||
name: "admin.auth_profile.created",
|
||||
auth_profile_id = %profile.id.as_str(),
|
||||
"auth profile created"
|
||||
);
|
||||
|
||||
Ok(profile)
|
||||
}
|
||||
|
||||
@@ -75,7 +75,11 @@ impl AdminService {
|
||||
upstream: &upstream,
|
||||
})
|
||||
.await?;
|
||||
info!(upstream_id = %upstream.id.as_str(), "workspace upstream saved");
|
||||
info!(
|
||||
name: "admin.upstream.saved",
|
||||
upstream_id = %upstream.id.as_str(),
|
||||
"workspace upstream saved"
|
||||
);
|
||||
|
||||
Ok(upstream)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ use crank_schema::{Schema, SchemaKind};
|
||||
use serde_json::{Value, json};
|
||||
use serial_test::serial;
|
||||
use tokio::net::TcpListener;
|
||||
use uuid::Version;
|
||||
|
||||
use admin_api::{
|
||||
app::build_app,
|
||||
@@ -40,6 +41,8 @@ const TEST_PASSWORD_PEPPER: &str = "test-password-pepper";
|
||||
const TEST_SESSION_SECRET: &str = "test-session-secret";
|
||||
const TEST_MASTER_KEY: &str = "test-master-key";
|
||||
|
||||
mod history_loss;
|
||||
|
||||
struct TestServer {
|
||||
base_url: String,
|
||||
shutdown: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
@@ -385,6 +388,30 @@ async fn exports_single_workspace_but_rejects_access_lifecycle() {
|
||||
exported["workspace"]["workspace"]["id"],
|
||||
DEFAULT_WORKSPACE_ID
|
||||
);
|
||||
assert_eq!(exported["kind"], "workspace_catalog_snapshot");
|
||||
assert_eq!(exported["format_version"], "1");
|
||||
assert_eq!(exported["restorable"], false);
|
||||
assert_eq!(
|
||||
exported["included"],
|
||||
json!([
|
||||
"workspace_settings",
|
||||
"operation_summaries",
|
||||
"agent_summaries",
|
||||
"platform_api_key_metadata"
|
||||
])
|
||||
);
|
||||
assert!(
|
||||
exported["excluded"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("secret_values"))
|
||||
);
|
||||
assert!(
|
||||
exported["excluded"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.contains(&json!("invocation_logs_and_usage"))
|
||||
);
|
||||
assert!(exported.get("memberships").is_none());
|
||||
assert!(exported.get("invitations").is_none());
|
||||
|
||||
@@ -574,6 +601,25 @@ async fn updates_profile_and_changes_password() {
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
let client = authorized_client(&base_url).await;
|
||||
let second_client = reqwest::Client::builder()
|
||||
.cookie_store(true)
|
||||
.build()
|
||||
.unwrap();
|
||||
let second_login = second_client
|
||||
.post(format!("{root_url}/api/auth/login"))
|
||||
.json(&json!({
|
||||
"email": TEST_AUTH_EMAIL,
|
||||
"password": TEST_AUTH_PASSWORD,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let second_login_status = second_login.status();
|
||||
let second_login_body = second_login.text().await.unwrap();
|
||||
assert!(
|
||||
second_login_status.is_success(),
|
||||
"second login failed with {second_login_status}: {second_login_body}"
|
||||
);
|
||||
|
||||
let profile = assert_success_json(
|
||||
client
|
||||
@@ -615,6 +661,21 @@ async fn updates_profile_and_changes_password() {
|
||||
.status();
|
||||
assert_eq!(password_status, reqwest::StatusCode::NO_CONTENT);
|
||||
|
||||
let current_session_status = client
|
||||
.get(format!("{root_url}/api/auth/profile"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status();
|
||||
let other_session_status = second_client
|
||||
.get(format!("{root_url}/api/auth/profile"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status();
|
||||
assert_eq!(current_session_status, reqwest::StatusCode::OK);
|
||||
assert_eq!(other_session_status, reqwest::StatusCode::UNAUTHORIZED);
|
||||
|
||||
let relogin_client = reqwest::Client::builder()
|
||||
.cookie_store(true)
|
||||
.build()
|
||||
@@ -855,7 +916,10 @@ async fn generates_request_id_for_test_run_invocations() {
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
|
||||
assert!(!request_id.is_empty());
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
|
||||
Some(Version::SortRand)
|
||||
);
|
||||
response.error_for_status().unwrap();
|
||||
|
||||
let logs = client
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use axum::{Json, Router, extract::State, routing::post};
|
||||
use tokio::{net::TcpListener, sync::Notify};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn preserves_external_success_when_invocation_history_is_lost() {
|
||||
let registry = test_registry().await;
|
||||
let registry_for_failure = registry.clone();
|
||||
let storage_root = test_storage_root("observability_history_loss");
|
||||
let upstream = spawn_blocking_upstream_server().await;
|
||||
let base_url = spawn_admin_api(build_test_app(registry, storage_root)).await;
|
||||
let client = authorized_client(&base_url).await;
|
||||
|
||||
let created = client
|
||||
.post(format!("{base_url}/operations"))
|
||||
.json(&test_operation_payload(
|
||||
&upstream.base_url,
|
||||
"crm_history_loss",
|
||||
))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json::<Value>()
|
||||
.await
|
||||
.unwrap();
|
||||
let operation_id = created["operation_id"].as_str().unwrap().to_owned();
|
||||
let request_client = client.clone();
|
||||
let request_url = format!("{base_url}/operations/{operation_id}/test-runs");
|
||||
let before = crank_observability::operational_incident_total(
|
||||
crank_observability::OperationalIncident::InvocationHistoryLost,
|
||||
);
|
||||
|
||||
let request = tokio::spawn(async move {
|
||||
request_client
|
||||
.post(request_url)
|
||||
.header("x-request-id", "req_dc08_admin")
|
||||
.json(&json!({
|
||||
"version": 1,
|
||||
"input": { "email": "dc08-canary-secret@example.com" }
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
upstream.started.notified().await;
|
||||
registry_for_failure
|
||||
.delete_operation(
|
||||
&WorkspaceId::new(DEFAULT_WORKSPACE_ID),
|
||||
&OperationId::new(operation_id.clone()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
upstream.release.notify_one();
|
||||
|
||||
let response = request.await.unwrap();
|
||||
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.headers()["x-request-id"].to_str().unwrap(),
|
||||
"req_dc08_admin"
|
||||
);
|
||||
let body = response.json::<Value>().await.unwrap();
|
||||
assert_eq!(body["ok"], true);
|
||||
assert_eq!(body["response_preview"]["id"], "lead_123");
|
||||
assert_eq!(upstream.calls.load(Ordering::SeqCst), 1);
|
||||
assert!(
|
||||
crank_observability::operational_incident_total(
|
||||
crank_observability::OperationalIncident::InvocationHistoryLost
|
||||
) > before
|
||||
);
|
||||
|
||||
let logs = client
|
||||
.get(format!("{base_url}/logs?period=7d"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json::<Value>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(logs["items"].as_array().unwrap().is_empty());
|
||||
}
|
||||
|
||||
struct BlockingUpstream {
|
||||
base_url: String,
|
||||
started: Arc<Notify>,
|
||||
release: Arc<Notify>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BlockingUpstreamState {
|
||||
started: Arc<Notify>,
|
||||
release: Arc<Notify>,
|
||||
calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
async fn spawn_blocking_upstream_server() -> BlockingUpstream {
|
||||
let state = BlockingUpstreamState {
|
||||
started: Arc::new(Notify::new()),
|
||||
release: Arc::new(Notify::new()),
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
};
|
||||
let app = Router::new()
|
||||
.route("/crm/leads", post(blocking_create_lead))
|
||||
.with_state(state.clone());
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
BlockingUpstream {
|
||||
base_url: format!("http://{address}"),
|
||||
started: state.started,
|
||||
release: state.release,
|
||||
calls: state.calls,
|
||||
}
|
||||
}
|
||||
|
||||
async fn blocking_create_lead(
|
||||
State(state): State<BlockingUpstreamState>,
|
||||
Json(payload): Json<Value>,
|
||||
) -> Json<Value> {
|
||||
state.calls.fetch_add(1, Ordering::SeqCst);
|
||||
state.started.notify_one();
|
||||
state.release.notified().await;
|
||||
|
||||
Json(json!({
|
||||
"id": "lead_123",
|
||||
"status": "created",
|
||||
"email": payload["email"]
|
||||
}))
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
use admin_api::service::{OpenApiImportCreatePayload, OpenApiImportPreviewPayload};
|
||||
use crank_core::WorkspaceId;
|
||||
use crank_registry::ImportJobStatus;
|
||||
use serial_test::serial;
|
||||
|
||||
use super::common::{
|
||||
@@ -42,7 +43,7 @@ paths:
|
||||
async fn previews_openapi_and_creates_draft_operations() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry,
|
||||
registry.clone(),
|
||||
test_storage_root("openapi_import"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
@@ -64,6 +65,12 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
preview.preview.groups[0].operations[0].suggested_name,
|
||||
"latest_rates"
|
||||
);
|
||||
let preview_job = registry
|
||||
.get_import_job(&workspace_id, &preview.job_id.as_str().into())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(preview_job.status, ImportJobStatus::Pending);
|
||||
|
||||
let created = service
|
||||
.create_openapi_import(
|
||||
@@ -112,10 +119,19 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
.any(|finding| finding.code == "openapi_import.weak_tool_description")
|
||||
);
|
||||
|
||||
let skip_preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiImportPreviewPayload {
|
||||
document: OPENAPI3.to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let skipped = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
&skip_preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||
@@ -130,10 +146,19 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
assert_eq!(skipped.skipped[0].name, "latest_rates");
|
||||
assert_eq!(skipped.findings[0].code, "operation_name_conflict");
|
||||
|
||||
let rename_preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiImportPreviewPayload {
|
||||
document: OPENAPI3.to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let renamed = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&preview.job_id.as_str().into(),
|
||||
&rename_preview.job_id.as_str().into(),
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||
@@ -147,3 +172,63 @@ async fn previews_openapi_and_creates_draft_operations() {
|
||||
assert_eq!(renamed.created[0].name, "latest_rates_2");
|
||||
assert_eq!(renamed.findings[0].code, "operation_name_renamed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn concurrent_openapi_import_replays_the_same_atomic_result() {
|
||||
let registry = test_registry().await;
|
||||
let service = test_service(
|
||||
registry,
|
||||
test_storage_root("openapi_import_replay"),
|
||||
test_auth_settings(),
|
||||
test_secret_crypto(),
|
||||
);
|
||||
let workspace_id = WorkspaceId::new("ws_default");
|
||||
let preview = service
|
||||
.preview_openapi_import(
|
||||
&workspace_id,
|
||||
OpenApiImportPreviewPayload {
|
||||
document: OPENAPI3.to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let job_id = preview.job_id.as_str().into();
|
||||
let payload = OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||
conflict_mode: "rename".to_owned(),
|
||||
};
|
||||
|
||||
let (first, second) = tokio::join!(
|
||||
service.create_openapi_import(&workspace_id, &job_id, payload.clone()),
|
||||
service.create_openapi_import(&workspace_id, &job_id, payload),
|
||||
);
|
||||
let first = first.unwrap();
|
||||
let second = second.unwrap();
|
||||
|
||||
assert_eq!(first.created.len(), 1);
|
||||
assert_eq!(second.created.len(), 1);
|
||||
assert_eq!(
|
||||
first.created[0].operation_id,
|
||||
second.created[0].operation_id
|
||||
);
|
||||
assert_eq!(first.created[0].name, second.created[0].name);
|
||||
assert_eq!(
|
||||
service.list_operations(&workspace_id).await.unwrap().len(),
|
||||
1
|
||||
);
|
||||
|
||||
let conflicting_replay = service
|
||||
.create_openapi_import(
|
||||
&workspace_id,
|
||||
&job_id,
|
||||
OpenApiImportCreatePayload {
|
||||
selected_operation_keys: vec!["GET /v2/latest".to_owned()],
|
||||
server_url: Some("https://api.frankfurter.dev".to_owned()),
|
||||
conflict_mode: "skip".to_owned(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(conflicting_replay.is_err());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
use std::{
|
||||
io,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use admin_api::request_context::{REQUEST_ID_HEADER, apply_request_context};
|
||||
use axum::{
|
||||
Router,
|
||||
body::Body,
|
||||
http::{HeaderMap, Request, StatusCode},
|
||||
routing::get,
|
||||
};
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, RedactionLimits, ServiceIdentity, inject_current_trace_context,
|
||||
};
|
||||
use opentelemetry::{global, trace::TracerProvider as _};
|
||||
use opentelemetry_sdk::{propagation::TraceContextPropagator, trace::SdkTracerProvider};
|
||||
use tower::ServiceExt;
|
||||
use tracing::instrument::WithSubscriber;
|
||||
use tracing_subscriber::{fmt::MakeWriter, layer::SubscriberExt};
|
||||
use uuid::Version;
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn logs_request_completion_and_rejects_untrusted_values() {
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = crank_observability::build_subscriber(
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("admin-api", "test", "test").unwrap(),
|
||||
"info",
|
||||
RedactionLimits::default(),
|
||||
),
|
||||
writer.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let app = probe_app();
|
||||
|
||||
let response = app
|
||||
.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/probe")
|
||||
.header(REQUEST_ID_HEADER.as_str(), "req_admin_trace_123")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
response.headers()[REQUEST_ID_HEADER.as_str()]
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
"req_admin_trace_123"
|
||||
);
|
||||
let event: serde_json::Value = writer
|
||||
.output()
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str(line).unwrap())
|
||||
.find(|event: &serde_json::Value| event["event"] == "admin.request.completed")
|
||||
.unwrap();
|
||||
assert_eq!(event["request_id"], "req_admin_trace_123");
|
||||
assert_eq!(event["fields"]["status"], 200);
|
||||
|
||||
let invalid_response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/probe")
|
||||
.header(REQUEST_ID_HEADER.as_str(), "bad,value")
|
||||
.header("traceparent", "canary-invalid-traceparent")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch)
|
||||
.await
|
||||
.unwrap();
|
||||
let generated = invalid_response.headers()[REQUEST_ID_HEADER.as_str()]
|
||||
.to_str()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(generated).unwrap().get_version(),
|
||||
Some(Version::SortRand)
|
||||
);
|
||||
assert!(!writer.output().contains("canary-invalid-traceparent"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn covers_valid_invalid_and_absent_traceparent() {
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
let provider = SdkTracerProvider::builder().build();
|
||||
let tracer = provider.tracer("admin-request-context-test");
|
||||
let subscriber =
|
||||
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let app = trace_probe_app();
|
||||
|
||||
let valid = observed_trace_id(
|
||||
app.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/trace")
|
||||
.header(
|
||||
"traceparent",
|
||||
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
|
||||
)
|
||||
.header(REQUEST_ID_HEADER.as_str(), "request-id-is-separate")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch.clone())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let invalid = observed_trace_id(
|
||||
app.clone()
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/trace")
|
||||
.header("traceparent", "canary-invalid-traceparent")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch.clone())
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let absent = observed_trace_id(
|
||||
app.oneshot(
|
||||
Request::builder()
|
||||
.uri("/trace")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.with_subscriber(dispatch)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
assert_eq!(valid, "0af7651916cd43dd8448eb211c80319c");
|
||||
assert_ne!(invalid, valid);
|
||||
assert_ne!(absent, valid);
|
||||
assert_ne!(invalid, absent);
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
fn probe_app() -> Router {
|
||||
Router::new()
|
||||
.route("/probe", get(|| async { "ok" }))
|
||||
.layer(axum::middleware::from_fn(apply_request_context))
|
||||
}
|
||||
|
||||
fn trace_probe_app() -> Router {
|
||||
Router::new()
|
||||
.route("/trace", get(observed_traceparent))
|
||||
.layer(axum::middleware::from_fn(apply_request_context))
|
||||
}
|
||||
|
||||
async fn observed_traceparent() -> HeaderMap {
|
||||
let mut trace_headers = HeaderMap::new();
|
||||
inject_current_trace_context(&mut trace_headers);
|
||||
let mut response_headers = HeaderMap::new();
|
||||
if let Some(traceparent) = trace_headers.remove("traceparent") {
|
||||
response_headers.insert("x-observed-traceparent", traceparent);
|
||||
}
|
||||
response_headers
|
||||
}
|
||||
|
||||
fn observed_trace_id(response: axum::response::Response) -> String {
|
||||
let traceparent = response.headers()["x-observed-traceparent"]
|
||||
.to_str()
|
||||
.unwrap();
|
||||
traceparent[3..35].to_owned()
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct SharedLogWriter {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl SharedLogWriter {
|
||||
fn output(&self) -> String {
|
||||
String::from_utf8(self.buffer.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for SharedLogWriter {
|
||||
type Writer = SharedLogGuard;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
SharedLogGuard {
|
||||
buffer: Arc::clone(&self.buffer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SharedLogGuard {
|
||||
buffer: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
impl io::Write for SharedLogGuard {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.buffer.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
#[path = "integration/request_context.rs"]
|
||||
mod request_context;
|
||||
@@ -3,6 +3,7 @@ name = "mcp-server"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
publish.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[[bin]]
|
||||
@@ -15,10 +16,12 @@ axum.workspace = true
|
||||
base64.workspace = true
|
||||
crank-community-mcp = { path = "../../crates/crank-community-mcp" }
|
||||
crank-core = { path = "../../crates/crank-core" }
|
||||
crank-observability = { path = "../../crates/crank-observability" }
|
||||
crank-registry = { path = "../../crates/crank-registry" }
|
||||
crank-runtime = { path = "../../crates/crank-runtime" }
|
||||
crank-schema = { path = "../../crates/crank-schema" }
|
||||
futures-util = "0.3"
|
||||
metrics.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
@@ -34,4 +37,10 @@ uuid.workspace = true
|
||||
crank-mapping = { path = "../../crates/crank-mapping" }
|
||||
crank-schema = { path = "../../crates/crank-schema" }
|
||||
crank-test-support = { path = "../../crates/crank-test-support" }
|
||||
opentelemetry.workspace = true
|
||||
opentelemetry-proto.workspace = true
|
||||
opentelemetry_sdk.workspace = true
|
||||
prost.workspace = true
|
||||
reqwest.workspace = true
|
||||
tower.workspace = true
|
||||
tracing-opentelemetry.workspace = true
|
||||
|
||||
+70
-21
@@ -1,26 +1,53 @@
|
||||
use std::{env, net::SocketAddr, time::Duration};
|
||||
|
||||
use crank_community_mcp::{
|
||||
auth::CommunityMachineCredentialVerifier, build_app_with_background_workers,
|
||||
auth::CommunityMachineCredentialVerifier, build_app_with_background_workers_and_limits,
|
||||
session::PostgresTransportSessionStore,
|
||||
};
|
||||
use crank_observability::{
|
||||
CriticalErrorCategory, MetricsConfig, ObservabilityConfig, ObservabilityLifecycle,
|
||||
capture_critical_error,
|
||||
};
|
||||
use crank_registry::{PostgresPoolConfig, PostgresRegistry};
|
||||
use crank_runtime::{
|
||||
RequestRateLimitConfig, RequestRateLimiter, RuntimeCacheConfig, RuntimeCacheStores,
|
||||
RuntimeLimits, SecretCrypto,
|
||||
};
|
||||
use sqlx::postgres::PgConnectOptions;
|
||||
use sqlx::{PgPool, postgres::PgConnectOptions};
|
||||
use tokio::net::TcpListener;
|
||||
use tracing::info;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
env::var("CRANK_LOG_LEVEL")
|
||||
.unwrap_or_else(|_| "mcp_server=info,tower_http=info".into()),
|
||||
)
|
||||
.init();
|
||||
let observability = crank_observability::init(ObservabilityConfig::from_env(
|
||||
"mcp-server",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
"mcp_server=info,tower_http=info",
|
||||
)?)?;
|
||||
let mut startup_completed = false;
|
||||
let result = run(&observability, &mut startup_completed).await;
|
||||
if result.is_err() {
|
||||
capture_critical_error(if startup_completed {
|
||||
CriticalErrorCategory::Internal
|
||||
} else {
|
||||
CriticalErrorCategory::Startup
|
||||
});
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn run(
|
||||
observability: &ObservabilityLifecycle,
|
||||
startup_completed: &mut bool,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let metrics_config =
|
||||
MetricsConfig::from_env("CRANK_MCP_METRICS_BIND", "127.0.0.1:9465".parse()?)?;
|
||||
let metrics_enabled = metrics_config.enabled();
|
||||
let metrics_server = if metrics_config.enabled() {
|
||||
Some(observability.metrics_surface(metrics_config).bind().await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let bind_addr = env::var("CRANK_MCP_BIND").unwrap_or_else(|_| "0.0.0.0:3002".into());
|
||||
let base_url = env::var("CRANK_BASE_URL").ok();
|
||||
@@ -36,23 +63,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let cache_stores = RuntimeCacheStores::from_config(&cache_config).await?;
|
||||
let api_rate_limit = mcp_api_rate_limit_config_from_env()?;
|
||||
let database_options = database_options_from_env()?;
|
||||
let registry = PostgresRegistry::connect_with_options_and_pool_config(
|
||||
database_options.clone(),
|
||||
pool_config,
|
||||
)
|
||||
.await?;
|
||||
let session_store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
|
||||
database_options,
|
||||
pool_config,
|
||||
)
|
||||
.await?;
|
||||
let registry =
|
||||
PostgresRegistry::connect_with_options_and_pool_config(database_options, pool_config)
|
||||
.await?;
|
||||
if metrics_enabled {
|
||||
spawn_postgres_pool_metrics(registry.pool().clone());
|
||||
}
|
||||
let session_store = PostgresTransportSessionStore::from_pool(registry.pool().clone()).await?;
|
||||
let secret_crypto = SecretCrypto::new(&env::var("CRANK_MASTER_KEY")?)?;
|
||||
let runtime = crank_runtime::community_from_env()?
|
||||
.with_limits(runtime_limits)
|
||||
.with_response_cache(cache_stores.response.clone())
|
||||
.with_coordination_store(cache_stores.coordination.clone())
|
||||
.build();
|
||||
let app = build_app_with_background_workers(
|
||||
let app = build_app_with_background_workers_and_limits(
|
||||
registry,
|
||||
refresh_interval,
|
||||
base_url,
|
||||
@@ -66,11 +90,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
cache_stores.coordination.clone(),
|
||||
std::sync::Arc::new(session_store),
|
||||
std::sync::Arc::new(CommunityMachineCredentialVerifier),
|
||||
runtime_limits.max_concurrent_sessions,
|
||||
);
|
||||
let listener = TcpListener::bind(socket_addr).await?;
|
||||
|
||||
info!(
|
||||
name: "mcp.postgres_pool.configured",
|
||||
runtime_max_concurrent_unary = runtime_limits.max_concurrent_unary,
|
||||
runtime_max_concurrent_sessions = runtime_limits.max_concurrent_sessions,
|
||||
mcp_rate_limit_rps = api_rate_limit.requests_per_second,
|
||||
mcp_rate_limit_burst = api_rate_limit.burst,
|
||||
cache_backend = %cache_config.backend,
|
||||
@@ -81,9 +108,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
max_lifetime_ms = pool_config.max_lifetime_ms,
|
||||
"postgres pool configured"
|
||||
);
|
||||
info!("mcp-server listening on {}", socket_addr);
|
||||
info!(
|
||||
name: "mcp.server.listening",
|
||||
bind_address = %socket_addr,
|
||||
"mcp-server listening"
|
||||
);
|
||||
*startup_completed = true;
|
||||
|
||||
axum::serve(listener, app).await?;
|
||||
if let Some(metrics_server) = metrics_server {
|
||||
tokio::select! {
|
||||
result = axum::serve(listener, app) => result?,
|
||||
result = metrics_server.serve() => result?,
|
||||
}
|
||||
} else {
|
||||
axum::serve(listener, app).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -123,3 +162,13 @@ fn mcp_api_rate_limit_config_from_env() -> Result<RequestRateLimitConfig, Box<dy
|
||||
|
||||
Ok(RequestRateLimitConfig::new(requests_per_second, burst)?)
|
||||
}
|
||||
|
||||
fn spawn_postgres_pool_metrics(pool: PgPool) {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
crank_observability::record_db_pool_connections(pool.size(), pool.num_idle());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
#[path = "integration/common.rs"]
|
||||
mod common;
|
||||
#[path = "integration/execution_stages.rs"]
|
||||
mod execution_stages;
|
||||
@@ -89,11 +89,16 @@ async fn approval_key_lists_and_decides_pending_requests() {
|
||||
let approved = client
|
||||
.post(&approve_url)
|
||||
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
|
||||
.header("x-request-id", "req_approval_execute_123")
|
||||
.json(&json!({ "approve": "yes", "note": "confirmed by test" }))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(approved.status(), reqwest::StatusCode::OK);
|
||||
assert_eq!(
|
||||
approved.headers()["x-request-id"].to_str().unwrap(),
|
||||
"req_approval_execute_123"
|
||||
);
|
||||
let approved_body = approved.json::<Value>().await.unwrap();
|
||||
assert_eq!(
|
||||
approved_body["approval"]["status"],
|
||||
@@ -169,6 +174,10 @@ async fn approval_key_lists_and_decides_pending_requests() {
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(logs.len(), 1);
|
||||
assert_eq!(
|
||||
logs[0].log.request_id.as_deref(),
|
||||
Some("req_approval_execute_123")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -401,22 +410,23 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
|
||||
let mcp_url = agent_mcp_url(&base_url, "sales-gated");
|
||||
let initialized_session = initialize_session(&client, &mcp_url, &api_key).await;
|
||||
|
||||
let tool_call = json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 9,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "crm_requires_human_approval",
|
||||
"arguments": {
|
||||
"email": "ada@example.com"
|
||||
}
|
||||
}
|
||||
});
|
||||
let tool_result = post_jsonrpc(
|
||||
&client,
|
||||
&mcp_url,
|
||||
&api_key,
|
||||
Some(&initialized_session),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 9,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "crm_requires_human_approval",
|
||||
"arguments": {
|
||||
"email": "ada@example.com"
|
||||
}
|
||||
}
|
||||
}),
|
||||
tool_call.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -430,6 +440,19 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
|
||||
.unwrap();
|
||||
assert!(approval_id.starts_with("approval_"));
|
||||
|
||||
let repeated_tool_result = post_jsonrpc(
|
||||
&client,
|
||||
&mcp_url,
|
||||
&api_key,
|
||||
Some(&initialized_session),
|
||||
tool_call,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
repeated_tool_result["result"]["structuredContent"]["approval_id"], approval_id,
|
||||
"deduplicated tools/call must return the persisted approval id",
|
||||
);
|
||||
|
||||
let approvals_url = format!("{}/approvals", agent_mcp_url(&base_url, "sales-gated"));
|
||||
let pending = client
|
||||
.get(&approvals_url)
|
||||
@@ -448,6 +471,189 @@ async fn tool_call_with_approval_policy_creates_pending_request() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn approval_http_endpoints_enforce_request_rate_limit() {
|
||||
let registry = test_registry().await;
|
||||
let upstream_base_url = spawn_upstream_server().await;
|
||||
let operation = test_operation(&upstream_base_url, "crm_approval_rate_limit");
|
||||
registry
|
||||
.create_operation(&test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent_for_operation(®istry, &operation, "sales-approval-rate-limit").await;
|
||||
let approval_key = create_approval_platform_api_key(
|
||||
®istry,
|
||||
"sales-approval-rate-limit",
|
||||
"approval-rate-limit",
|
||||
)
|
||||
.await;
|
||||
let base_url = spawn_mcp_server(build_test_app_with_rate_limit(
|
||||
registry,
|
||||
Duration::from_millis(0),
|
||||
Some("https://crank.example.com".to_owned()),
|
||||
RequestRateLimitConfig::new(1, 1).unwrap(),
|
||||
))
|
||||
.await;
|
||||
let approvals_url = format!(
|
||||
"{}/approvals",
|
||||
agent_mcp_url(&base_url, "sales-approval-rate-limit")
|
||||
);
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let allowed = client
|
||||
.get(&approvals_url)
|
||||
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(allowed.status(), reqwest::StatusCode::OK);
|
||||
|
||||
let limited = client
|
||||
.get(&approvals_url)
|
||||
.header(header::AUTHORIZATION, format!("Bearer {approval_key}"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(limited.status(), reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
assert!(limited.headers().contains_key(header::RETRY_AFTER));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_does_not_repeat_interrupted_mutating_approval() {
|
||||
let registry = test_registry().await;
|
||||
let (upstream_base_url, upstream_calls) = spawn_counted_approval_upstream().await;
|
||||
let operation = test_operation(&upstream_base_url, "crm_interrupted_approval");
|
||||
registry
|
||||
.create_operation(&test_workspace_id(), &operation, Some("alice"))
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent_for_operation(®istry, &operation, "sales-interrupted-approval").await;
|
||||
let approval_key_name = "approval-interrupted";
|
||||
create_approval_platform_api_key(®istry, "sales-interrupted-approval", approval_key_name)
|
||||
.await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let approval = ApprovalRequest {
|
||||
id: ApprovalRequestId::new("approval_interrupted_mutation"),
|
||||
workspace_id: test_workspace_id(),
|
||||
agent_id: test_agent_id("sales-interrupted-approval"),
|
||||
operation_id: operation.id.clone(),
|
||||
operation_version: operation.version,
|
||||
status: ApprovalRequestStatus::Pending,
|
||||
risk_level: OperationApprovalRiskLevel::Dangerous,
|
||||
request_payload: json!({"email": "interrupted@example.com"}),
|
||||
response_payload: None,
|
||||
created_at: now - time::Duration::minutes(10),
|
||||
expires_at: now + time::Duration::minutes(5),
|
||||
decided_at: None,
|
||||
decided_by_key_id: None,
|
||||
decision_note: None,
|
||||
};
|
||||
registry
|
||||
.create_approval_request(CreateApprovalRequest {
|
||||
approval: &approval,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let approval_key_id = PlatformApiKeyId::new(format!("pk_{approval_key_name}"));
|
||||
registry
|
||||
.decide_approval_request(crank_registry::DecideApprovalRequest {
|
||||
workspace_id: &approval.workspace_id,
|
||||
agent_id: &approval.agent_id,
|
||||
approval_id: &approval.id,
|
||||
status: ApprovalRequestStatus::Approved,
|
||||
decided_at: now - time::Duration::minutes(10),
|
||||
decided_by_key_id: &approval_key_id,
|
||||
response_payload: Some(json!({"approve": "yes"})),
|
||||
decision_note: None,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
registry
|
||||
.claim_approval_request(
|
||||
&approval.workspace_id,
|
||||
&approval.agent_id,
|
||||
&approval.id,
|
||||
now - time::Duration::minutes(7),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let _app = build_test_app_with_approval_recovery(registry.clone());
|
||||
let failed = tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
let current = registry
|
||||
.get_approval_request_for_agent(
|
||||
&approval.workspace_id,
|
||||
&approval.agent_id,
|
||||
&approval.id,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
if current.approval.status == ApprovalRequestStatus::Failed {
|
||||
break current;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("recovery must quarantine interrupted execution");
|
||||
|
||||
assert_eq!(
|
||||
failed.approval.response_payload.unwrap()["error"]["code"],
|
||||
"approval_execution_outcome_unknown"
|
||||
);
|
||||
assert_eq!(
|
||||
upstream_calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
0,
|
||||
"recovery must not repeat a mutating upstream request"
|
||||
);
|
||||
}
|
||||
|
||||
fn build_test_app_with_approval_recovery(registry: PostgresRegistry) -> Router {
|
||||
crank_community_mcp::build_app_with_background_workers(
|
||||
registry,
|
||||
Duration::from_millis(0),
|
||||
Some("https://crank.example.com".to_owned()),
|
||||
SecretCrypto::new("test-master-key").unwrap(),
|
||||
crank_runtime::community_with_outbound_policy(
|
||||
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
|
||||
)
|
||||
.build(),
|
||||
RequestRateLimiter::new(RequestRateLimitConfig::new(10_000, 10_000).unwrap()),
|
||||
Arc::new(InMemoryCoordinationStateStore::default()),
|
||||
Arc::new(InMemorySessionStore::default()),
|
||||
Arc::new(CommunityMachineCredentialVerifier),
|
||||
)
|
||||
}
|
||||
|
||||
async fn spawn_counted_approval_upstream() -> (String, Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let handler_calls = Arc::clone(&calls);
|
||||
let app = Router::new().route(
|
||||
"/crm/leads",
|
||||
post(move |Json(payload): Json<Value>| {
|
||||
let calls = Arc::clone(&handler_calls);
|
||||
async move {
|
||||
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Json(json!({
|
||||
"id": "lead_123",
|
||||
"email": payload["email"]
|
||||
}))
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
(format!("http://{address}"), calls)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn elicitation_approval_requires_client_capability() {
|
||||
let registry = test_registry().await;
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
use std::{
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
body::{Body, Bytes, to_bytes},
|
||||
extract::State,
|
||||
http::{HeaderMap, Request, StatusCode, header},
|
||||
routing::post,
|
||||
};
|
||||
use crank_core::{InvocationSource, PlatformApiKeyScope};
|
||||
use crank_observability::{
|
||||
OtlpBatchConfig, OtlpTraceConfig, ServiceIdentity, build_tracer_provider,
|
||||
};
|
||||
use crank_registry::{ListInvocationLogsQuery, PublishRequest};
|
||||
use opentelemetry::global;
|
||||
use opentelemetry_proto::tonic::{
|
||||
collector::trace::v1::ExportTraceServiceRequest, common::v1::any_value, trace::v1::Span,
|
||||
};
|
||||
use opentelemetry_sdk::propagation::TraceContextPropagator;
|
||||
use prost::Message;
|
||||
use serde_json::{Value, json};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::net::TcpListener;
|
||||
use tower::ServiceExt;
|
||||
use tracing::instrument::WithSubscriber;
|
||||
use tracing_subscriber::{Layer, filter::filter_fn, layer::SubscriberExt};
|
||||
|
||||
use super::common::{
|
||||
build_test_app, create_platform_api_key, publish_agent_for_operation, test_operation,
|
||||
test_registry, test_workspace_id,
|
||||
};
|
||||
|
||||
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
|
||||
const REQUEST_ID: &str = "req_stage_end_to_end";
|
||||
const CANARY: &str = "dc-stage-canary-secret";
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn exports_real_tool_stages_without_sensitive_data() {
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
let (otlp_endpoint, request_rx, collector) = spawn_otlp_collector().await;
|
||||
let trace_config = OtlpTraceConfig::try_new(
|
||||
Some(otlp_endpoint),
|
||||
Some("http/protobuf".to_owned()),
|
||||
Duration::from_secs(2),
|
||||
OtlpBatchConfig::try_new(128, 128, Duration::from_secs(300), Duration::from_secs(2))
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let identity = ServiceIdentity::try_new("mcp-server", "0.3.1", "integration-test").unwrap();
|
||||
let (provider, tracer) = build_tracer_provider(&identity, &trace_config)
|
||||
.unwrap()
|
||||
.expect("enabled OTLP provider");
|
||||
let subscriber = tracing_subscriber::registry().with(
|
||||
tracing_opentelemetry::layer()
|
||||
.with_tracer(tracer)
|
||||
.with_filter(filter_fn(|metadata| {
|
||||
metadata.is_span() && metadata.target() == "crank::trace"
|
||||
})),
|
||||
);
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
|
||||
let registry = test_registry().await;
|
||||
let observed_traceparent = Arc::new(Mutex::new(None));
|
||||
let upstream_base_url = spawn_upstream(Arc::clone(&observed_traceparent)).await;
|
||||
let operation = test_operation(&upstream_base_url, "stage_end_to_end");
|
||||
registry
|
||||
.create_operation(&test_workspace_id(), &operation, Some("test"))
|
||||
.await
|
||||
.unwrap();
|
||||
registry
|
||||
.publish_operation(PublishRequest {
|
||||
workspace_id: &test_workspace_id(),
|
||||
operation_id: &operation.id,
|
||||
version: 1,
|
||||
published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(),
|
||||
published_by: Some("test"),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
publish_agent_for_operation(®istry, &operation, "stage-agent").await;
|
||||
let api_key = create_platform_api_key(
|
||||
®istry,
|
||||
"stage-agent",
|
||||
"stage-key",
|
||||
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
|
||||
)
|
||||
.await;
|
||||
let app = build_test_app(registry.clone(), Duration::ZERO, None);
|
||||
|
||||
let call_result = async {
|
||||
let initialized = send_jsonrpc(
|
||||
app.clone(),
|
||||
&api_key,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2025-11-25",
|
||||
"capabilities": {}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(initialized.status(), StatusCode::OK);
|
||||
let session_id = initialized
|
||||
.headers()
|
||||
.get("MCP-Session-Id")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
|
||||
let notification = send_jsonrpc(
|
||||
app.clone(),
|
||||
&api_key,
|
||||
Some(&session_id),
|
||||
None,
|
||||
None,
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "notifications/initialized",
|
||||
"params": {}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(notification.status(), StatusCode::ACCEPTED);
|
||||
|
||||
send_jsonrpc(
|
||||
app,
|
||||
&api_key,
|
||||
Some(&session_id),
|
||||
Some(REQUEST_ID),
|
||||
Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "stage_end_to_end",
|
||||
"arguments": { "email": CANARY }
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
.with_subscriber(dispatch)
|
||||
.await;
|
||||
|
||||
assert_eq!(call_result.status(), StatusCode::OK);
|
||||
let body = to_bytes(call_result.into_body(), 1024 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
let body: Value = serde_json::from_slice(&body).unwrap();
|
||||
assert_eq!(body["result"]["isError"], false);
|
||||
provider.force_flush().unwrap();
|
||||
let request = tokio::time::timeout(Duration::from_secs(2), request_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
collector.abort();
|
||||
assert_eq!(
|
||||
request
|
||||
.headers
|
||||
.get(header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("application/x-protobuf")
|
||||
);
|
||||
assert!(
|
||||
!request
|
||||
.body
|
||||
.windows(CANARY.len())
|
||||
.any(|window| window == CANARY.as_bytes())
|
||||
);
|
||||
let export = ExportTraceServiceRequest::decode(request.body).unwrap();
|
||||
let spans = export
|
||||
.resource_spans
|
||||
.iter()
|
||||
.flat_map(|resource| &resource.scope_spans)
|
||||
.flat_map(|scope| &scope.spans)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let traceparent = observed_traceparent
|
||||
.lock()
|
||||
.unwrap()
|
||||
.clone()
|
||||
.expect("upstream traceparent");
|
||||
assert_eq!(&traceparent[3..35], REMOTE_TRACE_ID);
|
||||
|
||||
let expected_trace_id = decode_trace_id(REMOTE_TRACE_ID);
|
||||
let trace_spans = spans
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|span| span.trace_id.as_slice() == expected_trace_id)
|
||||
.collect::<Vec<_>>();
|
||||
for expected in [
|
||||
"mcp.request",
|
||||
"mcp.rate_limit",
|
||||
"mcp.access.check",
|
||||
"mcp.catalog.load",
|
||||
"mcp.tools.resolve",
|
||||
"runtime.execute",
|
||||
"runtime.arguments.map",
|
||||
"upstream.http",
|
||||
"runtime.response.transform",
|
||||
"history.write",
|
||||
"db.query",
|
||||
] {
|
||||
assert!(
|
||||
trace_spans.iter().any(|span| span.name == expected),
|
||||
"missing span {expected}"
|
||||
);
|
||||
}
|
||||
assert!(!trace_spans.iter().any(|span| span.name == "approval.check"));
|
||||
assert!(
|
||||
!trace_spans
|
||||
.iter()
|
||||
.any(|span| span.name == "runtime.idempotency")
|
||||
);
|
||||
|
||||
let root = trace_spans
|
||||
.iter()
|
||||
.find(|span| span.name == "mcp.request")
|
||||
.expect("mcp root");
|
||||
let runtime = trace_spans
|
||||
.iter()
|
||||
.find(|span| span.name == "runtime.execute")
|
||||
.expect("runtime");
|
||||
assert!(!runtime.parent_span_id.is_empty());
|
||||
assert_eq!(runtime.parent_span_id, root.span_id);
|
||||
|
||||
let history = trace_spans
|
||||
.iter()
|
||||
.find(|span| span.name == "history.write")
|
||||
.expect("history write");
|
||||
let history_db = trace_spans
|
||||
.iter()
|
||||
.find(|span| {
|
||||
span.name == "db.query"
|
||||
&& string_attribute(span, "db.operation") == Some("invocation_history.write")
|
||||
})
|
||||
.expect("history PostgreSQL write");
|
||||
assert_eq!(history_db.parent_span_id, history.span_id);
|
||||
|
||||
let logs = registry
|
||||
.list_invocation_logs(ListInvocationLogsQuery {
|
||||
workspace_id: &test_workspace_id(),
|
||||
level: None,
|
||||
search_text: None,
|
||||
source: Some(InvocationSource::AgentToolCall),
|
||||
operation_id: Some(&operation.id),
|
||||
agent_id: None,
|
||||
created_after: None,
|
||||
limit: 10,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(logs.len(), 1);
|
||||
assert_eq!(logs[0].log.request_id.as_deref(), Some(REQUEST_ID));
|
||||
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
struct OtlpRequest {
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
async fn spawn_otlp_collector() -> (
|
||||
String,
|
||||
tokio::sync::oneshot::Receiver<OtlpRequest>,
|
||||
tokio::task::JoinHandle<()>,
|
||||
) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let (request_tx, request_rx) = tokio::sync::oneshot::channel();
|
||||
let sender = Arc::new(Mutex::new(Some(request_tx)));
|
||||
let app = Router::new().route(
|
||||
"/v1/traces",
|
||||
post({
|
||||
let sender = Arc::clone(&sender);
|
||||
move |headers: HeaderMap, body: Bytes| {
|
||||
let sender = Arc::clone(&sender);
|
||||
async move {
|
||||
if let Some(sender) = sender.lock().unwrap().take() {
|
||||
let _ = sender.send(OtlpRequest { headers, body });
|
||||
}
|
||||
StatusCode::OK
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let collector = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
(format!("http://{address}/v1/traces"), request_rx, collector)
|
||||
}
|
||||
|
||||
fn decode_trace_id(value: &str) -> [u8; 16] {
|
||||
let mut bytes = [0_u8; 16];
|
||||
for (index, byte) in bytes.iter_mut().enumerate() {
|
||||
*byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap();
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
fn string_attribute<'a>(span: &'a Span, key: &str) -> Option<&'a str> {
|
||||
span.attributes.iter().find_map(|attribute| {
|
||||
let value = attribute.value.as_ref()?.value.as_ref()?;
|
||||
(attribute.key == key)
|
||||
.then_some(value)
|
||||
.and_then(|value| match value {
|
||||
any_value::Value::StringValue(value) => Some(value.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async fn send_jsonrpc(
|
||||
app: Router,
|
||||
api_key: &str,
|
||||
session_id: Option<&str>,
|
||||
request_id: Option<&str>,
|
||||
traceparent: Option<&str>,
|
||||
payload: Value,
|
||||
) -> axum::response::Response {
|
||||
let mut request = Request::builder()
|
||||
.method("POST")
|
||||
.uri("/v1/default/stage-agent")
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.header(header::ACCEPT, "application/json, text/event-stream")
|
||||
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
|
||||
.header("MCP-Protocol-Version", "2025-11-25");
|
||||
if let Some(session_id) = session_id {
|
||||
request = request.header("MCP-Session-Id", session_id);
|
||||
}
|
||||
if let Some(request_id) = request_id {
|
||||
request = request.header("x-request-id", request_id);
|
||||
}
|
||||
if let Some(traceparent) = traceparent {
|
||||
request = request.header("traceparent", traceparent);
|
||||
}
|
||||
app.oneshot(request.body(Body::from(payload.to_string())).unwrap())
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn spawn_upstream(observed: Arc<Mutex<Option<String>>>) -> String {
|
||||
async fn create_lead(
|
||||
State(observed): State<Arc<Mutex<Option<String>>>>,
|
||||
headers: HeaderMap,
|
||||
Json(_payload): Json<Value>,
|
||||
) -> Json<Value> {
|
||||
*observed.lock().unwrap() = headers
|
||||
.get("traceparent")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned);
|
||||
Json(json!({ "id": "lead_123" }))
|
||||
}
|
||||
|
||||
let app = Router::new()
|
||||
.route("/crm/leads", post(create_lead))
|
||||
.with_state(observed);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
format!("http://{address}")
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
use std::{
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode},
|
||||
};
|
||||
use opentelemetry::{
|
||||
global,
|
||||
trace::{TraceId, TracerProvider as _},
|
||||
};
|
||||
use opentelemetry_sdk::{
|
||||
error::OTelSdkResult,
|
||||
propagation::TraceContextPropagator,
|
||||
trace::{SdkTracerProvider, SpanData, SpanExporter},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
use tracing::instrument::WithSubscriber;
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
|
||||
use super::common::{build_test_app, test_registry};
|
||||
|
||||
const REMOTE_TRACE_ID: &str = "0af7651916cd43dd8448eb211c80319c";
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn covers_valid_invalid_and_absent_traceparent_on_mcp_boundary() {
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
let exported = Arc::new(Mutex::new(Vec::new()));
|
||||
let provider = SdkTracerProvider::builder()
|
||||
.with_simple_exporter(CapturingExporter(Arc::clone(&exported)))
|
||||
.build();
|
||||
let tracer = provider.tracer("mcp-request-context-test");
|
||||
let subscriber =
|
||||
tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let app = build_test_app(test_registry().await, Duration::ZERO, None);
|
||||
|
||||
let (valid, invalid, absent) = async {
|
||||
let valid = send_health(
|
||||
app.clone(),
|
||||
Some("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"),
|
||||
Some("request-id-is-separate"),
|
||||
)
|
||||
.await;
|
||||
let invalid = send_health(
|
||||
app.clone(),
|
||||
Some("canary-invalid-traceparent"),
|
||||
Some("bad,value"),
|
||||
)
|
||||
.await;
|
||||
let absent = send_health(app, None, None).await;
|
||||
(valid, invalid, absent)
|
||||
}
|
||||
.with_subscriber(dispatch)
|
||||
.await;
|
||||
provider.force_flush().unwrap();
|
||||
|
||||
assert_eq!(valid.status, StatusCode::OK);
|
||||
assert_eq!(valid.request_id.as_deref(), Some("request-id-is-separate"));
|
||||
assert_eq!(invalid.status, StatusCode::OK);
|
||||
assert_eq!(absent.status, StatusCode::OK);
|
||||
assert!(valid.traceparent_response.is_none());
|
||||
assert!(invalid.traceparent_response.is_none());
|
||||
assert!(absent.traceparent_response.is_none());
|
||||
|
||||
let trace_ids: Vec<_> = exported
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|span| span.name.as_ref() == "mcp.request")
|
||||
.map(|span| span.span_context.trace_id())
|
||||
.collect();
|
||||
assert_eq!(trace_ids.len(), 3);
|
||||
assert_eq!(trace_ids[0].to_string(), REMOTE_TRACE_ID);
|
||||
assert_ne!(trace_ids[1], trace_ids[0]);
|
||||
assert_ne!(trace_ids[2], trace_ids[0]);
|
||||
assert_ne!(trace_ids[1], trace_ids[2]);
|
||||
assert!(!trace_ids.contains(&TraceId::INVALID));
|
||||
provider.shutdown().unwrap();
|
||||
}
|
||||
|
||||
async fn send_health(
|
||||
app: axum::Router,
|
||||
traceparent: Option<&str>,
|
||||
request_id: Option<&str>,
|
||||
) -> ProbeResponse {
|
||||
let mut request = Request::builder().uri("/health");
|
||||
if let Some(traceparent) = traceparent {
|
||||
request = request.header("traceparent", traceparent);
|
||||
}
|
||||
if let Some(request_id) = request_id {
|
||||
request = request.header("x-request-id", request_id);
|
||||
}
|
||||
let response = app
|
||||
.oneshot(request.body(Body::empty()).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
ProbeResponse {
|
||||
status: response.status(),
|
||||
request_id: response
|
||||
.headers()
|
||||
.get("x-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned),
|
||||
traceparent_response: response
|
||||
.headers()
|
||||
.get("traceparent")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned),
|
||||
}
|
||||
}
|
||||
|
||||
struct ProbeResponse {
|
||||
status: StatusCode,
|
||||
request_id: Option<String>,
|
||||
traceparent_response: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CapturingExporter(Arc<Mutex<Vec<SpanData>>>);
|
||||
|
||||
impl SpanExporter for CapturingExporter {
|
||||
async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
|
||||
self.0.lock().unwrap().extend(batch);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,8 @@ use sha2::{Digest, Sha256};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::sleep;
|
||||
use tracing_subscriber::{filter::LevelFilter, fmt::MakeWriter, prelude::*};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
use uuid::Version;
|
||||
|
||||
use crank_community_mcp::{
|
||||
auth::{CommunityMachineCredentialVerifier, SharedMachineCredentialVerifier},
|
||||
@@ -46,6 +47,9 @@ use crank_community_mcp::{
|
||||
catalog::PublishedToolCatalog,
|
||||
session::{InMemorySessionStore, SharedSessionStore, TransportSessionStore},
|
||||
};
|
||||
use crank_observability::{
|
||||
ObservabilityConfig, RedactionLimits, ServiceIdentity, build_subscriber,
|
||||
};
|
||||
|
||||
fn test_workspace_id() -> WorkspaceId {
|
||||
WorkspaceId::new("ws_default")
|
||||
@@ -404,7 +408,10 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_owned();
|
||||
assert!(!request_id.is_empty());
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(&request_id).unwrap().get_version(),
|
||||
Some(Version::SortRand)
|
||||
);
|
||||
|
||||
let call_result = response.json::<Value>().await.unwrap();
|
||||
assert_eq!(call_result["result"]["isError"], false);
|
||||
@@ -427,7 +434,7 @@ async fn generates_request_id_for_tool_call_responses_and_logs() {
|
||||
assert_eq!(logs[0].log.request_id.as_deref(), Some(request_id.as_str()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn emits_request_id_in_mcp_ingress_logs() {
|
||||
let registry = test_registry().await;
|
||||
let upstream_base_url = spawn_upstream_server().await;
|
||||
@@ -456,6 +463,18 @@ async fn emits_request_id_in_mcp_ingress_logs() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = build_subscriber(
|
||||
ObservabilityConfig::new(
|
||||
ServiceIdentity::try_new("mcp-server", "test", "test").unwrap(),
|
||||
"info",
|
||||
RedactionLimits::default(),
|
||||
),
|
||||
writer.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let base_url = spawn_mcp_server(build_test_app(
|
||||
registry,
|
||||
Duration::from_millis(0),
|
||||
@@ -464,18 +483,7 @@ async fn emits_request_id_in_mcp_ingress_logs() {
|
||||
.await;
|
||||
let client = reqwest::Client::new();
|
||||
let mcp_url = agent_mcp_url(&base_url, "sales-request-trace");
|
||||
let writer = SharedLogWriter::default();
|
||||
let subscriber = tracing_subscriber::registry().with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_writer(writer.clone())
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_target(false)
|
||||
.compact()
|
||||
.with_filter(LevelFilter::INFO),
|
||||
);
|
||||
|
||||
let _ = tracing::subscriber::set_global_default(subscriber);
|
||||
let response = post_jsonrpc_response(
|
||||
&client,
|
||||
&mcp_url,
|
||||
@@ -500,14 +508,16 @@ async fn emits_request_id_in_mcp_ingress_logs() {
|
||||
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||
|
||||
let logs = writer.output();
|
||||
assert!(
|
||||
logs.contains("mcp request received"),
|
||||
"captured logs did not include ingress marker: {logs}"
|
||||
);
|
||||
assert!(logs.contains("req_mcp_trace_123"));
|
||||
assert!(logs.contains("sales-request-trace"));
|
||||
assert!(logs.contains("default"));
|
||||
assert!(logs.contains("initialize"));
|
||||
let event = logs
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
|
||||
.find(|event| event["event"] == "mcp.request.received")
|
||||
.unwrap();
|
||||
assert_eq!(event["service"], "mcp-server");
|
||||
assert_eq!(event["request_id"], "req_mcp_trace_123");
|
||||
assert_eq!(event["fields"]["agent_slug"], "sales-request-trace");
|
||||
assert_eq!(event["fields"]["workspace_slug"], "default");
|
||||
assert_eq!(event["fields"]["jsonrpc_method"], "initialize");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -803,11 +813,30 @@ async fn get_requires_session_header() {
|
||||
.get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
|
||||
.header(header::ACCEPT, "text/event-stream")
|
||||
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
|
||||
.header("x-request-id", "req_early_mcp_error")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
assert_eq!(
|
||||
response.headers()["x-request-id"].to_str().unwrap(),
|
||||
"req_early_mcp_error"
|
||||
);
|
||||
|
||||
let invalid_response = client
|
||||
.get(agent_mcp_url(&base_url, "sales-get-sse-missing"))
|
||||
.header(header::ACCEPT, "text/event-stream")
|
||||
.header(header::AUTHORIZATION, format!("Bearer {api_key}"))
|
||||
.header("x-request-id", "bad,value")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let generated = invalid_response.headers()["x-request-id"].to_str().unwrap();
|
||||
assert_eq!(
|
||||
uuid::Uuid::parse_str(generated).unwrap().get_version(),
|
||||
Some(Version::SortRand)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
#[path = "integration/common.rs"]
|
||||
mod common;
|
||||
#[path = "integration/request_context.rs"]
|
||||
mod request_context;
|
||||
@@ -97,8 +97,8 @@
|
||||
</div>
|
||||
<div class="danger-zone-action">
|
||||
<div class="danger-zone-text">
|
||||
<div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export all data</div>
|
||||
<div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.</div>
|
||||
<div class="danger-zone-title" data-i18n="workspace_setup.danger.export_title">Export workspace catalog</div>
|
||||
<div class="danger-zone-desc" data-i18n="workspace_setup.danger.export_body">Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.</div>
|
||||
</div>
|
||||
<button class="btn-danger" id="export-workspace-btn" type="button" data-i18n="workspace_setup.export">Export</button>
|
||||
</div>
|
||||
|
||||
+4
-4
@@ -442,8 +442,8 @@ var TRANSLATIONS = {
|
||||
'workspace_setup.create.subtitle': 'This Community installation uses one workspace for MCP operations and agents.',
|
||||
'workspace_setup.create.footer': 'This Community installation uses one workspace.',
|
||||
'workspace_setup.danger.title': 'Danger zone',
|
||||
'workspace_setup.danger.export_title': 'Export all data',
|
||||
'workspace_setup.danger.export_body': 'Download a JSON snapshot of workspace settings, operations, agents, secrets, usage data and agent access keys.',
|
||||
'workspace_setup.danger.export_title': 'Export workspace catalog',
|
||||
'workspace_setup.danger.export_body': 'Download a non-restorable JSON catalog of workspace settings, operation summaries, agent summaries and API key metadata.',
|
||||
'workspace_setup.export': 'Export',
|
||||
'workspace_setup.role.owner': 'Owner',
|
||||
'workspace_setup.role.admin': 'Admin',
|
||||
@@ -1334,8 +1334,8 @@ var TRANSLATIONS = {
|
||||
'workspace_setup.create.subtitle': 'В Community используется один воркспейс для MCP-операций и агентов.',
|
||||
'workspace_setup.create.footer': 'В Community используется один воркспейс.',
|
||||
'workspace_setup.danger.title': 'Опасная зона',
|
||||
'workspace_setup.danger.export_title': 'Экспортировать все данные',
|
||||
'workspace_setup.danger.export_body': 'Скачать JSON-снимок настроек воркспейса, операций, агентов, секретов, данных использования и ключей доступа агентов.',
|
||||
'workspace_setup.danger.export_title': 'Экспорт каталога рабочего пространства',
|
||||
'workspace_setup.danger.export_body': 'Скачать невосстанавливаемый JSON-каталог настроек рабочего пространства, сводок операций и агентов, а также метаданных ключей API.',
|
||||
'workspace_setup.export': 'Экспорт',
|
||||
'workspace_setup.role.owner': 'Владелец',
|
||||
'workspace_setup.role.admin': 'Администратор',
|
||||
|
||||
+40
-4
@@ -8,6 +8,8 @@ document.addEventListener('DOMContentLoaded', function () {
|
||||
openId: null,
|
||||
liveMode: true,
|
||||
timer: null,
|
||||
searchTimer: null,
|
||||
refreshPromise: null,
|
||||
workspaceId: null,
|
||||
loading: false,
|
||||
loadError: '',
|
||||
@@ -457,7 +459,13 @@ document.addEventListener('DOMContentLoaded', function () {
|
||||
}
|
||||
|
||||
async function refreshOperationalData() {
|
||||
await Promise.all([loadLogs(), loadApprovals()]);
|
||||
if (state.refreshPromise) {
|
||||
return state.refreshPromise;
|
||||
}
|
||||
state.refreshPromise = Promise.all([loadLogs(), loadApprovals()]).finally(function () {
|
||||
state.refreshPromise = null;
|
||||
});
|
||||
return state.refreshPromise;
|
||||
}
|
||||
|
||||
async function loadLogDetail(logId) {
|
||||
@@ -494,10 +502,14 @@ document.addEventListener('DOMContentLoaded', function () {
|
||||
|
||||
function startPolling() {
|
||||
stopPolling();
|
||||
if (!state.liveMode) {
|
||||
if (!state.liveMode || document.hidden) {
|
||||
return;
|
||||
}
|
||||
state.timer = setInterval(refreshOperationalData, 4000);
|
||||
state.timer = setTimeout(async function poll() {
|
||||
state.timer = null;
|
||||
await refreshOperationalData();
|
||||
startPolling();
|
||||
}, 4000);
|
||||
}
|
||||
|
||||
function toggleLive() {
|
||||
@@ -526,7 +538,13 @@ document.addEventListener('DOMContentLoaded', function () {
|
||||
if (logSearch) {
|
||||
logSearch.addEventListener('input', function () {
|
||||
state.search = this.value.trim();
|
||||
loadLogs();
|
||||
if (state.searchTimer) {
|
||||
clearTimeout(state.searchTimer);
|
||||
}
|
||||
state.searchTimer = setTimeout(function () {
|
||||
state.searchTimer = null;
|
||||
loadLogs();
|
||||
}, 250);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -572,6 +590,24 @@ document.addEventListener('DOMContentLoaded', function () {
|
||||
refreshOperationalData();
|
||||
});
|
||||
|
||||
document.addEventListener('visibilitychange', function () {
|
||||
if (document.hidden) {
|
||||
stopPolling();
|
||||
return;
|
||||
}
|
||||
if (state.liveMode) {
|
||||
refreshOperationalData().finally(startPolling);
|
||||
}
|
||||
});
|
||||
|
||||
window.addEventListener('pagehide', function () {
|
||||
stopPolling();
|
||||
if (state.searchTimer) {
|
||||
clearTimeout(state.searchTimer);
|
||||
state.searchTimer = null;
|
||||
}
|
||||
});
|
||||
|
||||
setLiveState();
|
||||
startPolling();
|
||||
|
||||
|
||||
@@ -196,7 +196,7 @@ async function exportWorkspaceSnapshot() {
|
||||
var slug = workspaceFormState.workspaceRecord && workspaceFormState.workspaceRecord.workspace
|
||||
? workspaceFormState.workspaceRecord.workspace.slug
|
||||
: tKey('settings.nav.workspace');
|
||||
downloadJsonFile(slug + '-snapshot.json', snapshot);
|
||||
downloadJsonFile(slug + '-catalog-snapshot.json', snapshot);
|
||||
} catch (error) {
|
||||
if (window.CrankUi) {
|
||||
window.CrankUi.error(error.message || tKey('workspace_setup.export_error'), tKey('workspace_setup.export_error_title'));
|
||||
|
||||
Generated
+4
-4
@@ -9,7 +9,7 @@
|
||||
"@fontsource/inter": "5.2.8",
|
||||
"@fontsource/jetbrains-mono": "5.2.8",
|
||||
"alpinejs": "3.15.12",
|
||||
"js-yaml": "5.2.1"
|
||||
"js-yaml": "5.2.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "1.61.1",
|
||||
@@ -580,9 +580,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz",
|
||||
"integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==",
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz",
|
||||
"integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
"@fontsource/inter": "5.2.8",
|
||||
"@fontsource/jetbrains-mono": "5.2.8",
|
||||
"alpinejs": "3.15.12",
|
||||
"js-yaml": "5.2.1"
|
||||
"js-yaml": "5.2.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "1.61.1",
|
||||
|
||||
@@ -161,15 +161,15 @@ test('wizard builds visual request mappings from JSON sample and path params', a
|
||||
|
||||
await page.goto('/wizard/');
|
||||
await page.locator('[data-testid="wizard-protocol-rest"]').click();
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(2));
|
||||
await page.evaluate(() => window.CrankWizardShell.goToStep(2));
|
||||
await expect(page.locator('#step-panel-2')).toBeVisible();
|
||||
await page.locator('#endpoint-path').fill('/rates/{date}');
|
||||
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(3));
|
||||
await page.evaluate(() => window.CrankWizardShell.goToStep(3));
|
||||
await expect(page.locator('#step-panel-3-rest')).toBeVisible();
|
||||
await page.locator('.method-card[data-method="GET"]').click();
|
||||
|
||||
await page.evaluate(() => window.CrankWizardShell.doGoToStep(5));
|
||||
await page.evaluate(() => window.CrankWizardShell.goToStep(5));
|
||||
await expect(page.locator('#step-panel-5')).toBeVisible();
|
||||
|
||||
await page.locator('#wizard-input-sample').fill(JSON.stringify({
|
||||
|
||||
@@ -11,6 +11,12 @@ test('workspace and settings pages show live session data', async ({ page }) =>
|
||||
await expect(page.locator('#section-members')).toHaveCount(0);
|
||||
await expect(page.locator('#section-invite')).toHaveCount(0);
|
||||
await expect(page.locator('#delete-workspace-btn')).toHaveCount(0);
|
||||
await expect(page.locator('[data-i18n="workspace_setup.danger.export_title"]')).toHaveText(
|
||||
localized('Export workspace catalog', 'Экспорт каталога рабочего пространства'),
|
||||
);
|
||||
await expect(page.locator('[data-i18n="workspace_setup.danger.export_body"]')).not.toContainText(
|
||||
localized('all data', 'все данные'),
|
||||
);
|
||||
|
||||
await page.goto('/settings');
|
||||
await expect(page.locator('.page-title')).toHaveText(localized('Account settings', 'Настройки аккаунта'));
|
||||
|
||||
Reference in New Issue
Block a user