feat: harden community production foundation through story 1.5
This commit is contained in:
@@ -14,7 +14,7 @@ use axum::{
|
||||
};
|
||||
use crank_core::{
|
||||
ApprovalRequest, ApprovalRequestId, ApprovalRequestStatus, AuthProfile, CoordinationStateStore,
|
||||
InvocationLevel, InvocationSource, InvocationStatus, OperationApprovalMode,
|
||||
CorrelationContext, InvocationLevel, InvocationSource, InvocationStatus, OperationApprovalMode,
|
||||
PlatformApiKeyScope, SecretId,
|
||||
};
|
||||
use crank_registry::{
|
||||
@@ -64,10 +64,12 @@ use crate::{
|
||||
mod invocation_history;
|
||||
mod metrics;
|
||||
mod stages;
|
||||
mod tool_resolution;
|
||||
use self::metrics::{ActiveStreamGuard, McpRequestMetrics};
|
||||
use self::stages::{
|
||||
enforce_traced_rate_limit, require_traced_approval_access, require_traced_machine_access,
|
||||
};
|
||||
pub(super) use self::tool_resolution::{resolve_generated_tool, runtime_operation};
|
||||
#[cfg(test)]
|
||||
use invocation_history::observe_invocation_history_outcome;
|
||||
pub(super) use invocation_history::{InvocationRecord, persist_invocation};
|
||||
@@ -298,13 +300,13 @@ async fn readiness(State(state): State<Arc<AppState>>) -> Response {
|
||||
"checks": { "postgres": "ready" }
|
||||
}))
|
||||
.into_response(),
|
||||
Err(error) => (
|
||||
Err(_) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({
|
||||
"service": "mcp-server",
|
||||
"status": "not_ready",
|
||||
"checks": { "postgres": "not_ready" },
|
||||
"error": error.to_string()
|
||||
"error": "database is unavailable"
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
@@ -363,7 +365,7 @@ async fn approve_request(
|
||||
payload,
|
||||
PlatformApiKeyScope::Approve,
|
||||
ApprovalRequestStatus::Approved,
|
||||
Some(request_context.request_id),
|
||||
Some(request_context.correlation),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -425,7 +427,7 @@ async fn decide_approval_request(
|
||||
payload: ApprovalDecisionPayload,
|
||||
required_scope: PlatformApiKeyScope,
|
||||
status: ApprovalRequestStatus,
|
||||
execution_request_id: Option<String>,
|
||||
execution_correlation: Option<CorrelationContext>,
|
||||
) -> Response {
|
||||
let agent_path = AgentRoutePath {
|
||||
workspace_slug: path.workspace_slug,
|
||||
@@ -496,7 +498,7 @@ async fn decide_approval_request(
|
||||
&state,
|
||||
&agent_path,
|
||||
claimed,
|
||||
execution_request_id.as_deref(),
|
||||
execution_correlation.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -719,11 +721,12 @@ async fn mcp_post(
|
||||
let mut request_metrics = McpRequestMetrics::invalid();
|
||||
let response = rejection.into_response();
|
||||
request_metrics.complete(&response);
|
||||
return with_request_id_header(response, &request_context.request_id);
|
||||
return with_request_id_header(response, request_context.request_id());
|
||||
}
|
||||
};
|
||||
let mut request_metrics = McpRequestMetrics::new(&message);
|
||||
let transport_request_id = request_context.request_id;
|
||||
let transport_correlation = request_context.correlation;
|
||||
let transport_request_id = transport_correlation.request_id().to_string();
|
||||
info!(
|
||||
name: "mcp.request.received",
|
||||
request_id = %transport_request_id,
|
||||
@@ -733,7 +736,8 @@ async fn mcp_post(
|
||||
"mcp request received"
|
||||
);
|
||||
|
||||
let response = mcp_post_response(&path, state, &headers, &message, &transport_request_id).await;
|
||||
let response =
|
||||
mcp_post_response(&path, state, &headers, &message, &transport_correlation).await;
|
||||
request_metrics.complete(&response);
|
||||
with_request_id_header(response, &transport_request_id)
|
||||
}
|
||||
@@ -743,7 +747,7 @@ async fn mcp_post_response(
|
||||
state: Arc<AppState>,
|
||||
headers: &HeaderMap,
|
||||
message: &Value,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Response {
|
||||
if let Err(status) = validate_origin(&state.allowed_origins, headers) {
|
||||
return status.into_response();
|
||||
@@ -851,10 +855,10 @@ async fn mcp_post_response(
|
||||
};
|
||||
let tool_call_params: ToolCallParams = match serde_json::from_value(params(message)) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
Err(_) => {
|
||||
return transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_error(request_id(message), -32602, error.to_string()),
|
||||
jsonrpc_error(request_id(message), -32602, "invalid tool call parameters"),
|
||||
response_mode,
|
||||
None,
|
||||
Some(&session.protocol_version),
|
||||
@@ -883,7 +887,7 @@ async fn mcp_post_response(
|
||||
&catalog,
|
||||
&tool_call_params.name,
|
||||
arguments,
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -925,8 +929,9 @@ pub(super) async fn handle_tool_call(
|
||||
resolved: ResolvedToolCall,
|
||||
arguments: Value,
|
||||
confirmation_token: Option<String>,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Response {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
if !credential_allows_security_level(credential, resolved.tool.operation.security_level) {
|
||||
return tool_error_response(
|
||||
message,
|
||||
@@ -940,6 +945,7 @@ pub(super) async fn handle_tool_call(
|
||||
serialize_security_level(resolved.tool.operation.security_level),
|
||||
),
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
false,
|
||||
Some("Используйте ключ агента с достаточным уровнем доступа."),
|
||||
),
|
||||
@@ -956,7 +962,7 @@ pub(super) async fn handle_tool_call(
|
||||
arguments,
|
||||
confirmation_token,
|
||||
},
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1076,8 +1082,9 @@ async fn handle_base_tool_call(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
execution: ToolCallExecution,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Response {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
let tool = execution.tool;
|
||||
let arguments = execution.arguments;
|
||||
let operation = runtime_operation(&tool);
|
||||
@@ -1096,7 +1103,7 @@ async fn handle_base_tool_call(
|
||||
response_mode,
|
||||
&tool,
|
||||
&arguments,
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)
|
||||
.instrument(approval_span.clone())
|
||||
.await;
|
||||
@@ -1116,16 +1123,17 @@ async fn handle_base_tool_call(
|
||||
StageOutcome::Allowed.record(&approval_span);
|
||||
}
|
||||
|
||||
let mut runtime_request_context = RuntimeRequestContext::from_request_id(transport_request_id)
|
||||
.with_response_cache_scope(
|
||||
tool.workspace_id.as_str().to_owned(),
|
||||
tool.agent_id.as_str().to_owned(),
|
||||
)
|
||||
.with_metering_context(
|
||||
tool.workspace_id.clone(),
|
||||
Some(tool.agent_id.clone()),
|
||||
InvocationSource::AgentToolCall,
|
||||
);
|
||||
let mut runtime_request_context =
|
||||
RuntimeRequestContext::from_correlation(transport_correlation)
|
||||
.with_response_cache_scope(
|
||||
tool.workspace_id.as_str().to_owned(),
|
||||
tool.agent_id.as_str().to_owned(),
|
||||
)
|
||||
.with_metering_context(
|
||||
tool.workspace_id.clone(),
|
||||
Some(tool.agent_id.clone()),
|
||||
InvocationSource::AgentToolCall,
|
||||
);
|
||||
if let Some(token) = execution.confirmation_token {
|
||||
runtime_request_context = runtime_request_context.with_confirmation_token(token);
|
||||
}
|
||||
@@ -1155,6 +1163,7 @@ async fn handle_base_tool_call(
|
||||
&tool,
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
tool_name: &tool.tool_name,
|
||||
status: InvocationStatus::Ok,
|
||||
level: InvocationLevel::Info,
|
||||
@@ -1176,10 +1185,11 @@ async fn handle_base_tool_call(
|
||||
&tool,
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
tool_name: &tool.tool_name,
|
||||
status: InvocationStatus::Error,
|
||||
level: InvocationLevel::Error,
|
||||
message: &error.to_string(),
|
||||
message: runtime_error_code(&error),
|
||||
status_code: None,
|
||||
error_kind: Some(runtime_error_code(&error)),
|
||||
duration: started_at.elapsed(),
|
||||
@@ -1193,7 +1203,11 @@ async fn handle_base_tool_call(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
tool_error_contract_from_runtime(&error, transport_request_id),
|
||||
tool_error_contract_from_runtime(
|
||||
&error,
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1211,7 +1225,7 @@ async fn maybe_handle_approval_policy(
|
||||
response_mode: ResponseMode,
|
||||
tool: &PublishedAgentTool,
|
||||
arguments: &Value,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Option<ApprovalPolicyResult> {
|
||||
let policy = tool.operation.execution_config.approval_policy.as_ref()?;
|
||||
if !policy.required {
|
||||
@@ -1227,7 +1241,7 @@ async fn maybe_handle_approval_policy(
|
||||
response_mode,
|
||||
tool,
|
||||
arguments,
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1238,7 +1252,7 @@ async fn maybe_handle_approval_policy(
|
||||
tool,
|
||||
arguments,
|
||||
policy.elicitation_message.as_deref(),
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)),
|
||||
}
|
||||
}
|
||||
@@ -1250,8 +1264,9 @@ async fn maybe_create_custom_pending_approval(
|
||||
response_mode: ResponseMode,
|
||||
tool: &PublishedAgentTool,
|
||||
arguments: &Value,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Option<ApprovalPolicyResult> {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
let policy = tool.operation.execution_config.approval_policy.as_ref()?;
|
||||
|
||||
let approval_id = ApprovalRequestId::new(format!("approval_{}", uuid::Uuid::now_v7().simple()));
|
||||
@@ -1298,6 +1313,7 @@ async fn maybe_create_custom_pending_approval(
|
||||
tool,
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
tool_name: &tool.tool_name,
|
||||
status: InvocationStatus::Ok,
|
||||
level: InvocationLevel::Info,
|
||||
@@ -1326,8 +1342,9 @@ fn handle_elicitation_approval(
|
||||
tool: &PublishedAgentTool,
|
||||
arguments: &Value,
|
||||
elicitation_message: Option<&str>,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> ApprovalPolicyResult {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
if !session.supports_elicitation {
|
||||
return ApprovalPolicyResult::Error(tool_error_response(
|
||||
message,
|
||||
@@ -1337,6 +1354,7 @@ fn handle_elicitation_approval(
|
||||
"approval_elicitation_not_supported",
|
||||
"operation requires MCP Elicitation, but the MCP client did not advertise elicitation capability",
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
false,
|
||||
Some(
|
||||
"Выберите Custom MCP Approval или подключите MCP-клиент с поддержкой elicitation.",
|
||||
@@ -1375,10 +1393,10 @@ async fn handle_initialize(
|
||||
) -> Response {
|
||||
let initialize_params: InitializeParams = match serde_json::from_value(params(message)) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
Err(_error) => {
|
||||
return transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_error(request_id(message), -32602, error.to_string()),
|
||||
jsonrpc_error(request_id(message), -32602, "invalid initialize parameters"),
|
||||
response_mode,
|
||||
None,
|
||||
Some(DEFAULT_PROTOCOL_VERSION),
|
||||
@@ -1513,10 +1531,10 @@ async fn require_initialized_session(
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
fn internal_jsonrpc_error(message: &Value, error: impl std::fmt::Display) -> Response {
|
||||
fn internal_jsonrpc_error(message: &Value, _error: impl std::fmt::Display) -> Response {
|
||||
transport_response(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
jsonrpc_error(request_id(message), -32603, error.to_string()),
|
||||
jsonrpc_error(request_id(message), -32603, "internal server error"),
|
||||
ResponseMode::Json,
|
||||
None,
|
||||
Some(DEFAULT_PROTOCOL_VERSION),
|
||||
@@ -1614,26 +1632,5 @@ fn add_millis(timestamp: OffsetDateTime, millis: u64) -> OffsetDateTime {
|
||||
timestamp + delta
|
||||
}
|
||||
|
||||
pub(super) fn resolve_generated_tool(
|
||||
tools: &[PublishedAgentTool],
|
||||
tool_name: &str,
|
||||
) -> Option<ResolvedToolCall> {
|
||||
for tool in tools {
|
||||
if tool.tool_name == tool_name {
|
||||
return Some(ResolvedToolCall { tool: tool.clone() });
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub(super) fn runtime_operation(tool: &PublishedAgentTool) -> RuntimeOperation {
|
||||
let mut operation = RuntimeOperation::from(tool.operation.clone());
|
||||
operation.tool_name = tool.tool_name.clone();
|
||||
operation.tool_description.title = tool.tool_title.clone();
|
||||
operation.tool_description.description = tool.tool_description.clone();
|
||||
operation
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
@@ -15,6 +15,7 @@ use super::AppState;
|
||||
|
||||
pub(crate) struct InvocationRecord<'a> {
|
||||
pub(crate) request_id: Option<&'a str>,
|
||||
pub(crate) trace_id: Option<&'a str>,
|
||||
pub(crate) tool_name: &'a str,
|
||||
pub(crate) status: InvocationStatus,
|
||||
pub(crate) level: InvocationLevel,
|
||||
@@ -42,6 +43,7 @@ pub(crate) async fn persist_invocation(
|
||||
tool_name: record.tool_name.to_owned(),
|
||||
message: record.message.to_owned(),
|
||||
request_id: record.request_id.map(ToOwned::to_owned),
|
||||
trace_id: record.trace_id.map(ToOwned::to_owned),
|
||||
status_code: record.status_code,
|
||||
duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX),
|
||||
error_kind: record.error_kind.map(ToOwned::to_owned),
|
||||
@@ -79,6 +81,7 @@ pub(crate) async fn persist_invocation(
|
||||
observe_invocation_history_outcome(
|
||||
outcome,
|
||||
record.request_id,
|
||||
record.trace_id,
|
||||
record.status,
|
||||
InvocationSource::AgentToolCall,
|
||||
);
|
||||
@@ -88,6 +91,7 @@ pub(crate) async fn persist_invocation(
|
||||
pub(super) fn observe_invocation_history_outcome(
|
||||
outcome: InvocationHistoryWriteOutcome,
|
||||
request_id: Option<&str>,
|
||||
trace_id: Option<&str>,
|
||||
status: InvocationStatus,
|
||||
source: InvocationSource,
|
||||
) {
|
||||
@@ -100,6 +104,7 @@ pub(super) fn observe_invocation_history_outcome(
|
||||
warn!(
|
||||
name: "mcp.invocation_history.lost",
|
||||
request_id = request_id.unwrap_or_default(),
|
||||
trace_id = trace_id.unwrap_or_default(),
|
||||
source = invocation_source_label(source),
|
||||
invocation_status = invocation_status_label(status),
|
||||
error_category = loss.category.as_str(),
|
||||
|
||||
@@ -37,6 +37,7 @@ async fn tool_error_response_includes_structured_context() {
|
||||
"streaming_payload_error",
|
||||
"request root must be an object",
|
||||
"req-1",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
false,
|
||||
Some("Проверьте параметры вызова инструмента."),
|
||||
),
|
||||
@@ -59,6 +60,7 @@ async fn tool_error_response_includes_structured_context() {
|
||||
"message": "request root must be an object",
|
||||
"recoverable": false,
|
||||
"request_id": "req-1",
|
||||
"trace_id": "0af7651916cd43dd8448eb211c80319c",
|
||||
"suggested_action": "Проверьте параметры вызова инструмента."
|
||||
})
|
||||
);
|
||||
@@ -146,6 +148,7 @@ fn emits_bounded_history_loss_incident() {
|
||||
category: InvocationHistoryLossCategory::Unavailable,
|
||||
}),
|
||||
Some("req_mcp_dc08"),
|
||||
Some("0af7651916cd43dd8448eb211c80319c"),
|
||||
InvocationStatus::Ok,
|
||||
crank_core::InvocationSource::AgentToolCall,
|
||||
);
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
use crank_registry::PublishedAgentTool;
|
||||
use crank_runtime::RuntimeOperation;
|
||||
|
||||
use super::ResolvedToolCall;
|
||||
|
||||
pub(crate) fn resolve_generated_tool(
|
||||
tools: &[PublishedAgentTool],
|
||||
tool_name: &str,
|
||||
) -> Option<ResolvedToolCall> {
|
||||
tools
|
||||
.iter()
|
||||
.find(|tool| tool.tool_name == tool_name)
|
||||
.cloned()
|
||||
.map(|tool| ResolvedToolCall { tool })
|
||||
}
|
||||
|
||||
pub(crate) fn runtime_operation(tool: &PublishedAgentTool) -> RuntimeOperation {
|
||||
let mut operation = RuntimeOperation::from(tool.operation.clone());
|
||||
operation.tool_name = tool.tool_name.clone();
|
||||
operation.tool_description.title = tool.tool_title.clone();
|
||||
operation.tool_description.description = tool.tool_description.clone();
|
||||
operation
|
||||
}
|
||||
@@ -5,7 +5,7 @@ use axum::{
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use crank_core::{ApprovalRequestStatus, InvocationLevel, InvocationSource, InvocationStatus};
|
||||
use crank_observability::RequestId;
|
||||
use crank_core::{CorrelationContext, RequestId, TraceContext};
|
||||
use crank_registry::{ApprovalRequestRecord, FinishApprovalRequest};
|
||||
use crank_runtime::{RuntimeExecutionRequest, RuntimeRequestContext};
|
||||
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
|
||||
@@ -18,7 +18,7 @@ use crate::{
|
||||
AgentRoutePath, AppState, InvocationRecord, build_request_preview, persist_invocation,
|
||||
resolve_operation_auth, runtime_operation,
|
||||
},
|
||||
tool_error::runtime_error_code,
|
||||
tool_error::{runtime_error_code, safe_runtime_error_message},
|
||||
};
|
||||
|
||||
const RECOVERY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
@@ -64,7 +64,10 @@ async fn recover_approved_requests(state: &Arc<AppState>) {
|
||||
continue;
|
||||
};
|
||||
let recovery_span = Stage::ApprovalRecovery.span();
|
||||
let result = execute_approved_request(state, &path, approval, None)
|
||||
let trace_context = crank_trace::trace_context_for_span(&recovery_span)
|
||||
.unwrap_or_else(TraceContext::generate);
|
||||
let correlation = CorrelationContext::new(RequestId::generate(), trace_context);
|
||||
let result = execute_approved_request(state, &path, approval, Some(&correlation))
|
||||
.instrument(recovery_span.clone())
|
||||
.await;
|
||||
match &result {
|
||||
@@ -162,9 +165,12 @@ pub(super) async fn execute_approved_request(
|
||||
state: &Arc<AppState>,
|
||||
path: &AgentRoutePath,
|
||||
approval: ApprovalRequestRecord,
|
||||
request_id: Option<&str>,
|
||||
correlation: Option<&CorrelationContext>,
|
||||
) -> Result<ApprovalRequestRecord, Response> {
|
||||
let request_id = RequestId::resolve(request_id).into_string();
|
||||
let correlation = correlation
|
||||
.cloned()
|
||||
.unwrap_or_else(CorrelationContext::generate);
|
||||
let request_id = correlation.request_id().as_str();
|
||||
let tools = state
|
||||
.catalog
|
||||
.list_tools(&path.workspace_slug, &path.agent_slug)
|
||||
@@ -184,7 +190,7 @@ pub(super) async fn execute_approved_request(
|
||||
&approval.approval.request_payload,
|
||||
);
|
||||
let started_at = Instant::now();
|
||||
let runtime_request_context = RuntimeRequestContext::from_request_id(request_id.clone())
|
||||
let runtime_request_context = RuntimeRequestContext::from_correlation(&correlation)
|
||||
.with_response_cache_scope(
|
||||
tool.workspace_id.as_str().to_owned(),
|
||||
tool.agent_id.as_str().to_owned(),
|
||||
@@ -226,7 +232,7 @@ pub(super) async fn execute_approved_request(
|
||||
json!({
|
||||
"error": {
|
||||
"code": runtime_error_code(&error),
|
||||
"message": error.to_string(),
|
||||
"message": safe_runtime_error_message(&error),
|
||||
}
|
||||
}),
|
||||
InvocationStatus::Error,
|
||||
@@ -240,7 +246,8 @@ pub(super) async fn execute_approved_request(
|
||||
state,
|
||||
&tool,
|
||||
InvocationRecord {
|
||||
request_id: Some(&request_id),
|
||||
request_id: Some(request_id),
|
||||
trace_id: Some(correlation.trace_id().as_str()),
|
||||
tool_name: &tool.tool_name,
|
||||
status: invocation_status,
|
||||
level: invocation_level,
|
||||
|
||||
@@ -45,16 +45,35 @@ pub fn jsonrpc_result(id: Value, result: Value) -> Value {
|
||||
}
|
||||
|
||||
pub fn jsonrpc_error(id: Value, code: i64, message: impl Into<String>) -> Value {
|
||||
let mut error = json!({
|
||||
"code": code,
|
||||
"message": message.into()
|
||||
});
|
||||
let (request_id, trace_id) = crank_observability::current_request_correlation();
|
||||
if request_id.is_some() && trace_id.is_some() {
|
||||
error["data"] = correlated_error_data(json!({}));
|
||||
}
|
||||
json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message.into()
|
||||
}
|
||||
"error": error
|
||||
})
|
||||
}
|
||||
|
||||
pub fn correlated_error_data(mut data: Value) -> Value {
|
||||
if !data.is_object() {
|
||||
data = json!({});
|
||||
}
|
||||
let (request_id, trace_id) = crank_observability::current_request_correlation();
|
||||
if let Some(request_id) = request_id {
|
||||
data["request_id"] = Value::String(request_id);
|
||||
}
|
||||
if let Some(trace_id) = trace_id {
|
||||
data["trace_id"] = Value::String(trace_id);
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
pub fn negotiated_protocol_version(requested: &str) -> Option<&'static str> {
|
||||
SUPPORTED_PROTOCOL_VERSIONS
|
||||
.iter()
|
||||
|
||||
@@ -11,7 +11,7 @@ use serde_json::{Value, json};
|
||||
use crate::{
|
||||
access::{bearer_token, hash_access_secret},
|
||||
app::{AgentRoutePath, AppState},
|
||||
jsonrpc::request_id,
|
||||
jsonrpc::{correlated_error_data, request_id},
|
||||
transport::{ResponseMode, session_id_from_headers, transport_response},
|
||||
};
|
||||
|
||||
@@ -39,7 +39,7 @@ pub(super) fn rate_limited_jsonrpc_response(
|
||||
"error": {
|
||||
"code": -32603,
|
||||
"message": "rate limit service unavailable",
|
||||
"data": { "code": "rate_limit_unavailable" }
|
||||
"data": correlated_error_data(json!({ "code": "rate_limit_unavailable" }))
|
||||
}
|
||||
}),
|
||||
response_mode,
|
||||
@@ -53,10 +53,10 @@ pub(super) fn rate_limited_jsonrpc_response(
|
||||
"error": {
|
||||
"code": -32029,
|
||||
"message": "request rate limit exceeded",
|
||||
"data": {
|
||||
"data": correlated_error_data(json!({
|
||||
"code": "request_rate_limited",
|
||||
"retry_after_ms": rejection.retry_after_ms,
|
||||
}
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -1,33 +1,101 @@
|
||||
use axum::{extract::Request, http::HeaderValue, middleware::Next, response::Response};
|
||||
use crank_observability::{RequestId, set_remote_trace_parent, with_request_correlation};
|
||||
use crank_core::{CorrelationContext, RequestId, TraceContext};
|
||||
use crank_observability::{set_remote_trace_parent, with_request_correlation};
|
||||
use tracing::{Instrument, info_span};
|
||||
|
||||
use crate::transport::HEADER_X_REQUEST_ID;
|
||||
|
||||
const HEADER_X_TRACE_ID: axum::http::HeaderName = axum::http::HeaderName::from_static("x-trace-id");
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct RequestContext {
|
||||
pub(super) request_id: String,
|
||||
pub(super) correlation: CorrelationContext,
|
||||
}
|
||||
|
||||
impl RequestContext {
|
||||
pub(super) fn request_id(&self) -> &str {
|
||||
self.correlation.request_id().as_str()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn apply_request_context(mut request: Request, next: Next) -> Response {
|
||||
let request_id = RequestId::resolve_from_headers(request.headers()).into_string();
|
||||
let context = RequestContext {
|
||||
request_id: request_id.clone(),
|
||||
};
|
||||
let (request_id, remote_parent) = resolve_correlation(request.headers());
|
||||
let span = info_span!(
|
||||
target: "crank::trace",
|
||||
"mcp.request",
|
||||
request_id = %request_id,
|
||||
trace_id = tracing::field::Empty,
|
||||
);
|
||||
set_remote_trace_parent(&span, request.headers());
|
||||
request.extensions_mut().insert(context);
|
||||
if let Some(remote_parent) = remote_parent.as_ref() {
|
||||
set_canonical_parent(&span, remote_parent);
|
||||
}
|
||||
let trace_context = crank_trace::trace_context_for_span(&span).unwrap_or_else(|| {
|
||||
remote_parent
|
||||
.as_ref()
|
||||
.map_or_else(TraceContext::generate, TraceContext::continue_local)
|
||||
});
|
||||
span.record("trace_id", trace_context.trace_id().as_str());
|
||||
let context = RequestContext {
|
||||
correlation: CorrelationContext::new(request_id, trace_context),
|
||||
};
|
||||
request.extensions_mut().insert(context.clone());
|
||||
|
||||
with_request_correlation(request_id.clone(), async move {
|
||||
let mut response = next.run(request).instrument(span).await;
|
||||
if let Ok(value) = HeaderValue::from_str(&request_id) {
|
||||
response.headers_mut().insert(HEADER_X_REQUEST_ID, value);
|
||||
}
|
||||
response
|
||||
})
|
||||
with_request_correlation(
|
||||
context.correlation.request_id().to_string(),
|
||||
context.correlation.trace_id().to_string(),
|
||||
async move {
|
||||
let mut response = next.run(request).instrument(span).await;
|
||||
if let Ok(value) = HeaderValue::from_str(context.correlation.request_id().as_str()) {
|
||||
response.headers_mut().insert(HEADER_X_REQUEST_ID, value);
|
||||
}
|
||||
if let Ok(value) = HeaderValue::from_str(context.correlation.trace_id().as_str()) {
|
||||
response.headers_mut().insert(HEADER_X_TRACE_ID, value);
|
||||
}
|
||||
response
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn resolve_correlation(headers: &axum::http::HeaderMap) -> (RequestId, Option<TraceContext>) {
|
||||
let _tracestate_accepted = one_auxiliary_header_within_budget(
|
||||
headers,
|
||||
"tracestate",
|
||||
TraceContext::tracestate_within_budget,
|
||||
);
|
||||
let _baggage_accepted =
|
||||
one_auxiliary_header_within_budget(headers, "baggage", TraceContext::baggage_within_budget);
|
||||
let mut request_ids = headers.get_all(HEADER_X_REQUEST_ID).iter();
|
||||
let request_id = request_ids.next().and_then(|value| value.to_str().ok());
|
||||
let request_id = if request_ids.next().is_some() {
|
||||
RequestId::generate()
|
||||
} else {
|
||||
RequestId::resolve(request_id)
|
||||
};
|
||||
let mut traceparents = headers.get_all("traceparent").iter();
|
||||
let traceparent = traceparents.next().and_then(|value| value.to_str().ok());
|
||||
let remote_parent = if traceparents.next().is_some() {
|
||||
None
|
||||
} else {
|
||||
traceparent.and_then(|value| TraceContext::parse(value).ok())
|
||||
};
|
||||
(request_id, remote_parent)
|
||||
}
|
||||
|
||||
fn one_auxiliary_header_within_budget(
|
||||
headers: &axum::http::HeaderMap,
|
||||
name: &'static str,
|
||||
validate: fn(&str) -> bool,
|
||||
) -> bool {
|
||||
let mut values = headers.get_all(name).iter();
|
||||
let value = values.next().and_then(|value| value.to_str().ok());
|
||||
values.next().is_none() && value.is_some_and(validate)
|
||||
}
|
||||
|
||||
fn set_canonical_parent(span: &tracing::Span, context: &TraceContext) {
|
||||
let mut headers = axum::http::HeaderMap::new();
|
||||
if let Ok(value) = HeaderValue::from_str(context.traceparent()) {
|
||||
headers.insert("traceparent", value);
|
||||
set_remote_trace_parent(span, &headers);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,7 +130,11 @@ pub struct PostgresTransportSessionStore {
|
||||
|
||||
impl PostgresTransportSessionStore {
|
||||
pub async fn from_pool(pool: PgPool) -> Result<Self, SessionStoreError> {
|
||||
apply_postgres_migrations(&pool).await?;
|
||||
crank_registry::MigrationAuthority::require_current(&pool)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
Ok(Self { pool })
|
||||
}
|
||||
|
||||
@@ -413,124 +417,6 @@ impl TransportSessionStore for PostgresTransportSessionStore {
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_postgres_migrations(pool: &PgPool) -> Result<(), SessionStoreError> {
|
||||
let mut transaction = pool.begin().await.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
query("select pg_advisory_xact_lock($1)")
|
||||
.bind(0x4352_414E_4B4D_4350_i64)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
query(
|
||||
"create table if not exists __crank_mcp_migrations (
|
||||
version integer primary key,
|
||||
checksum text not null,
|
||||
applied_at timestamptz not null default now()
|
||||
)",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
let applied = query("select checksum from __crank_mcp_migrations where version = 1")
|
||||
.fetch_optional(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
if let Some(row) = applied {
|
||||
let checksum = row.get::<String, _>("checksum");
|
||||
if checksum != "mcp-transport-sessions-v1" {
|
||||
return Err(SessionStoreError {
|
||||
details: format!("modified MCP migration version 1: {checksum}"),
|
||||
});
|
||||
}
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
query(
|
||||
"create table if not exists mcp_transport_sessions (
|
||||
id text primary key,
|
||||
protocol_version text not null,
|
||||
initialized boolean not null default false,
|
||||
supports_elicitation boolean not null default false,
|
||||
workspace_slug text not null,
|
||||
agent_slug text not null,
|
||||
created_at timestamptz not null,
|
||||
updated_at timestamptz not null,
|
||||
expires_at timestamptz null
|
||||
)",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
|
||||
query("alter table mcp_transport_sessions add column if not exists supports_elicitation boolean not null default false")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
|
||||
query(
|
||||
"alter table mcp_transport_sessions add column if not exists expires_at timestamptz null",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
|
||||
query(
|
||||
"create index if not exists mcp_transport_sessions_workspace_agent_idx
|
||||
on mcp_transport_sessions(workspace_slug, agent_slug, updated_at desc)",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
|
||||
query(
|
||||
"create index if not exists mcp_transport_sessions_expires_at_idx
|
||||
on mcp_transport_sessions(expires_at)
|
||||
where expires_at is not null",
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
|
||||
query("insert into __crank_mcp_migrations (version, checksum) values (1, $1)")
|
||||
.bind("mcp-transport-sessions-v1")
|
||||
.execute(&mut *transaction)
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
transaction
|
||||
.commit()
|
||||
.await
|
||||
.map_err(|error| SessionStoreError {
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_expired(session: &SessionState, now: OffsetDateTime) -> bool {
|
||||
session
|
||||
.expires_at
|
||||
|
||||
@@ -14,11 +14,13 @@ pub struct ToolErrorContract {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub upstream_status: Option<u16>,
|
||||
pub request_id: String,
|
||||
pub trace_id: String,
|
||||
}
|
||||
|
||||
pub fn tool_error_contract_from_runtime(
|
||||
error: &RuntimeError,
|
||||
request_id: &str,
|
||||
trace_id: &str,
|
||||
) -> ToolErrorContract {
|
||||
let error_code = runtime_error_code(error);
|
||||
ToolErrorContract {
|
||||
@@ -29,6 +31,7 @@ pub fn tool_error_contract_from_runtime(
|
||||
suggested_action: suggested_action(error),
|
||||
upstream_status: upstream_status(error),
|
||||
request_id: request_id.to_owned(),
|
||||
trace_id: trace_id.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +39,7 @@ pub fn generic_tool_error_contract(
|
||||
error_code: &'static str,
|
||||
message: impl Into<String>,
|
||||
request_id: &str,
|
||||
trace_id: &str,
|
||||
recoverable: bool,
|
||||
suggested_action: Option<&'static str>,
|
||||
) -> ToolErrorContract {
|
||||
@@ -47,6 +51,7 @@ pub fn generic_tool_error_contract(
|
||||
suggested_action,
|
||||
upstream_status: None,
|
||||
request_id: request_id.to_owned(),
|
||||
trace_id: trace_id.to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,7 +69,8 @@ pub fn tool_error_value(error: &ToolErrorContract) -> Value {
|
||||
"error_code": "runtime_error",
|
||||
"message": "Не удалось выполнить инструмент.",
|
||||
"recoverable": false,
|
||||
"request_id": error.request_id
|
||||
"request_id": error.request_id,
|
||||
"trace_id": error.trace_id
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -109,7 +115,7 @@ fn upstream_status_code(status: u16) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_runtime_error_message(error: &RuntimeError) -> String {
|
||||
pub(crate) fn safe_runtime_error_message(error: &RuntimeError) -> String {
|
||||
match error {
|
||||
RuntimeError::Schema(_) => "Входные параметры не прошли проверку схемы.".to_owned(),
|
||||
RuntimeError::Mapping(_) => {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::{collections::BTreeSet, sync::Arc};
|
||||
|
||||
use axum::{http::StatusCode, response::Response};
|
||||
use crank_core::{ToolAccessMode, search_tool_catalog};
|
||||
use crank_core::{CorrelationContext, ToolAccessMode, search_tool_catalog};
|
||||
use crank_registry::PublishedAgentCatalog;
|
||||
use crank_trace::{ErrorCategory, Stage, StageOutcome};
|
||||
use serde::Deserialize;
|
||||
@@ -46,8 +46,9 @@ pub(super) async fn handle_catalog_tool_call(
|
||||
catalog: &PublishedAgentCatalog,
|
||||
tool_name: &str,
|
||||
arguments: Value,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Response {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
match catalog.tool_selection_policy.mode {
|
||||
ToolAccessMode::Direct => {
|
||||
execute_catalog_tool(
|
||||
@@ -59,7 +60,7 @@ pub(super) async fn handle_catalog_tool_call(
|
||||
catalog,
|
||||
tool_name,
|
||||
arguments,
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -90,6 +91,7 @@ pub(super) async fn handle_catalog_tool_call(
|
||||
proxy.catalog_revision
|
||||
),
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
true,
|
||||
Some(
|
||||
"Повторите search_tools и вызовите инструмент с новой версией каталога.",
|
||||
@@ -106,7 +108,7 @@ pub(super) async fn handle_catalog_tool_call(
|
||||
catalog,
|
||||
&proxy.name,
|
||||
proxy.arguments,
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -126,7 +128,7 @@ async fn execute_catalog_tool(
|
||||
catalog: &PublishedAgentCatalog,
|
||||
tool_name: &str,
|
||||
mut arguments: Value,
|
||||
transport_request_id: &str,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Response {
|
||||
let resolve_span = Stage::McpToolsResolve.span();
|
||||
let resolved = resolve_span.in_scope(|| resolve_generated_tool(&catalog.tools, tool_name));
|
||||
@@ -158,7 +160,7 @@ async fn execute_catalog_tool(
|
||||
resolved,
|
||||
arguments,
|
||||
confirmation_token,
|
||||
transport_request_id,
|
||||
transport_correlation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crank_community_mcp::session::{PostgresTransportSessionStore, TransportSessionStore};
|
||||
use crank_registry::PostgresPoolConfig;
|
||||
use crank_registry::{MigrationAuthority, PostgresPoolConfig};
|
||||
use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
@@ -13,9 +13,15 @@ fn truncate_to_micros(value: OffsetDateTime) -> OffsetDateTime {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn migrate(database_url: &str) {
|
||||
let pool = sqlx::PgPool::connect(database_url).await.unwrap();
|
||||
MigrationAuthority::apply(&pool).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_transport_sessions_survive_store_reconnect() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_mcp_transport").await;
|
||||
migrate(&database_url).await;
|
||||
let connect_options = database_url.parse::<PgConnectOptions>().unwrap();
|
||||
let pool_config = PostgresPoolConfig::default();
|
||||
let store_a = PostgresTransportSessionStore::connect_with_options_and_pool_config(
|
||||
@@ -61,6 +67,7 @@ async fn postgres_transport_sessions_survive_store_reconnect() {
|
||||
#[tokio::test]
|
||||
async fn postgres_transport_sessions_evict_expired_rows_on_read() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_mcp_transport").await;
|
||||
migrate(&database_url).await;
|
||||
let connect_options = database_url.parse::<PgConnectOptions>().unwrap();
|
||||
let store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
|
||||
connect_options.clone(),
|
||||
@@ -101,6 +108,7 @@ async fn postgres_transport_sessions_evict_expired_rows_on_read() {
|
||||
#[tokio::test]
|
||||
async fn postgres_transport_session_cleanup_removes_abandoned_expired_rows() {
|
||||
let database_url = crank_test_support::postgres_schema_url("test_mcp_cleanup").await;
|
||||
migrate(&database_url).await;
|
||||
let store = PostgresTransportSessionStore::connect_with_options_and_pool_config(
|
||||
database_url.parse::<PgConnectOptions>().unwrap(),
|
||||
PostgresPoolConfig::default(),
|
||||
|
||||
@@ -14,12 +14,14 @@ fn maps_upstream_429_to_recoverable_structured_tool_error() {
|
||||
}),
|
||||
}),
|
||||
"req-429",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
);
|
||||
|
||||
assert_eq!(contract.error_code, "upstream_rate_limited");
|
||||
assert!(contract.recoverable);
|
||||
assert_eq!(contract.upstream_status, Some(429));
|
||||
assert_eq!(contract.request_id, "req-429");
|
||||
assert_eq!(contract.trace_id, "0af7651916cd43dd8448eb211c80319c");
|
||||
assert_eq!(contract.suggested_action, Some("Повторите запрос позже."));
|
||||
assert!(!contract.message.contains("internal_trace"));
|
||||
}
|
||||
@@ -32,12 +34,14 @@ fn maps_mapping_error_to_non_recoverable_structured_tool_error() {
|
||||
reason: "expected string".to_owned(),
|
||||
},
|
||||
"req-map",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
);
|
||||
|
||||
assert_eq!(contract.error_code, "runtime_error");
|
||||
assert!(!contract.recoverable);
|
||||
assert_eq!(contract.upstream_status, None);
|
||||
assert_eq!(contract.request_id, "req-map");
|
||||
assert_eq!(contract.trace_id, "0af7651916cd43dd8448eb211c80319c");
|
||||
assert_eq!(
|
||||
contract.suggested_action,
|
||||
Some("Проверьте параметры вызова инструмента.")
|
||||
|
||||
Reference in New Issue
Block a user