feat: complete Epic 1 production foundation
This commit is contained in:
@@ -1,7 +1,10 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
http::{HeaderMap, StatusCode, header::AUTHORIZATION},
|
||||
http::{
|
||||
HeaderMap, StatusCode,
|
||||
header::{AUTHORIZATION, ORIGIN},
|
||||
},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
@@ -43,11 +46,14 @@ pub(super) async fn require_machine_access(
|
||||
headers: &HeaderMap,
|
||||
required_scope: PlatformApiKeyScope,
|
||||
) -> Result<VerifiedMachineCredential, MachineAccessError> {
|
||||
let secret =
|
||||
bearer_token(headers).ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))?;
|
||||
let secret = bearer_token(headers).ok_or_else(|| {
|
||||
record_machine_access_denial("missing_or_ambiguous_bearer", None);
|
||||
MachineAccessError::Denied(StatusCode::UNAUTHORIZED)
|
||||
})?;
|
||||
let credential = resolve_machine_credential(state, path, secret).await?;
|
||||
|
||||
if !allows_scope(&credential.scopes, required_scope) {
|
||||
record_machine_access_denial("scope", credential.platform_api_key_id.as_ref());
|
||||
return Err(MachineAccessError::Denied(StatusCode::FORBIDDEN));
|
||||
}
|
||||
|
||||
@@ -60,7 +66,10 @@ pub(super) async fn require_approval_access(
|
||||
headers: &HeaderMap,
|
||||
required_scope: PlatformApiKeyScope,
|
||||
) -> Result<crank_registry::PlatformApiKeyRecord, StatusCode> {
|
||||
let secret = bearer_token(headers).ok_or(StatusCode::UNAUTHORIZED)?;
|
||||
let secret = bearer_token(headers).ok_or_else(|| {
|
||||
record_unknown_approval_access_denial("missing_or_ambiguous_bearer");
|
||||
StatusCode::UNAUTHORIZED
|
||||
})?;
|
||||
let secret_hash = hash_access_secret(secret);
|
||||
let Some(api_key) = observe_db_query(
|
||||
DbOperation::MachineAccessRead,
|
||||
@@ -75,10 +84,16 @@ pub(super) async fn require_approval_access(
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
|
||||
else {
|
||||
record_unknown_approval_access_denial("unknown_or_inactive");
|
||||
return Err(StatusCode::UNAUTHORIZED);
|
||||
};
|
||||
|
||||
if !approval_allows_scope(&api_key.api_key.scopes, required_scope) {
|
||||
record_approval_access_denial(&api_key, "scope");
|
||||
return Err(StatusCode::FORBIDDEN);
|
||||
}
|
||||
if !approval_allows_origin(&api_key.api_key.allowed_origins, headers) {
|
||||
record_approval_access_denial(&api_key, "origin");
|
||||
return Err(StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
@@ -92,13 +107,20 @@ pub(super) async fn require_approval_access(
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
.map_err(|error| match error {
|
||||
crank_registry::RegistryError::PlatformApiKeyInactive { .. } => StatusCode::UNAUTHORIZED,
|
||||
_ => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
})?;
|
||||
|
||||
Ok(api_key)
|
||||
}
|
||||
|
||||
pub(super) fn bearer_token(headers: &HeaderMap) -> Option<&str> {
|
||||
let value = headers.get(AUTHORIZATION)?.to_str().ok()?;
|
||||
let mut values = headers.get_all(AUTHORIZATION).iter();
|
||||
let value = values.next()?.to_str().ok()?;
|
||||
if values.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let (scheme, token) = value.split_once(' ')?;
|
||||
if !scheme.eq_ignore_ascii_case("Bearer") || token.is_empty() {
|
||||
return None;
|
||||
@@ -114,18 +136,6 @@ pub(super) fn credential_allows_security_level(
|
||||
security_level_rank(credential.max_security_level) >= security_level_rank(required_level)
|
||||
}
|
||||
|
||||
pub(super) fn serialize_security_level(level: OperationSecurityLevel) -> &'static str {
|
||||
match level {
|
||||
OperationSecurityLevel::Standard => "standard",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn serialize_machine_access_mode(mode: crank_core::MachineAccessMode) -> &'static str {
|
||||
match mode {
|
||||
crank_core::MachineAccessMode::StaticAgentKey => "static_agent_key",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn hash_access_secret(secret: &str) -> String {
|
||||
let digest = Sha256::digest(secret.as_bytes());
|
||||
URL_SAFE_NO_PAD.encode(digest)
|
||||
@@ -145,7 +155,10 @@ async fn resolve_machine_credential(
|
||||
.verify_bearer_token(&path.workspace_slug, &path.agent_slug, token)
|
||||
.await
|
||||
.map_err(|_| MachineAccessError::Unavailable)?
|
||||
.ok_or(MachineAccessError::Denied(StatusCode::UNAUTHORIZED))
|
||||
.ok_or_else(|| {
|
||||
record_machine_access_denial("unknown_or_inactive", None);
|
||||
MachineAccessError::Denied(StatusCode::UNAUTHORIZED)
|
||||
})
|
||||
}
|
||||
|
||||
async fn verify_static_agent_key(
|
||||
@@ -164,7 +177,13 @@ async fn verify_static_agent_key(
|
||||
)
|
||||
.instrument(read_span.clone())
|
||||
.await
|
||||
.map_err(|_| MachineAccessError::Unavailable);
|
||||
.map_err(|error| match error {
|
||||
crank_registry::RegistryError::PlatformApiKeyInactive { .. } => {
|
||||
record_machine_access_denial("unknown_or_inactive", None);
|
||||
MachineAccessError::Denied(StatusCode::UNAUTHORIZED)
|
||||
}
|
||||
_ => MachineAccessError::Unavailable,
|
||||
});
|
||||
let api_key = match api_key_result {
|
||||
Ok(api_key) => {
|
||||
StageOutcome::Success.record(&read_span);
|
||||
@@ -189,7 +208,13 @@ async fn verify_static_agent_key(
|
||||
.touch_platform_api_key(&api_key.api_key.workspace_id, &api_key.api_key.id, &used_at)
|
||||
.instrument(touch_span.clone())
|
||||
.await
|
||||
.map_err(|_| MachineAccessError::Unavailable);
|
||||
.map_err(|error| match error {
|
||||
crank_registry::RegistryError::PlatformApiKeyInactive { .. } => {
|
||||
record_machine_access_denial("unknown_or_inactive", None);
|
||||
MachineAccessError::Denied(StatusCode::UNAUTHORIZED)
|
||||
}
|
||||
_ => MachineAccessError::Unavailable,
|
||||
});
|
||||
match touch_result {
|
||||
Ok(()) => {
|
||||
StageOutcome::Success.record(&touch_span);
|
||||
@@ -207,9 +232,46 @@ async fn verify_static_agent_key(
|
||||
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
|
||||
max_security_level: crank_core::OperationSecurityLevel::Standard,
|
||||
scopes: api_key.api_key.scopes,
|
||||
platform_api_key_id: Some(api_key.api_key.id),
|
||||
}))
|
||||
}
|
||||
|
||||
fn record_machine_access_denial(
|
||||
reason: &'static str,
|
||||
platform_api_key_id: Option<&crank_core::PlatformApiKeyId>,
|
||||
) {
|
||||
let (request_id, trace_id) = crank_observability::current_request_correlation();
|
||||
let credential_ref = platform_api_key_id
|
||||
.map(crank_core::PlatformApiKeyId::as_str)
|
||||
.unwrap_or("unknown");
|
||||
tracing::warn!(
|
||||
target: "crank::audit",
|
||||
action = "credential.platform_api_key.access_denied",
|
||||
credential_type = "platform_api_key.mcp_client",
|
||||
credential_ref = credential_ref,
|
||||
outcome = "denied",
|
||||
reason = reason,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
trace_id = trace_id.as_deref().unwrap_or(""),
|
||||
"mcp platform key access denied"
|
||||
);
|
||||
}
|
||||
|
||||
fn record_unknown_approval_access_denial(reason: &'static str) {
|
||||
let (request_id, trace_id) = crank_observability::current_request_correlation();
|
||||
tracing::warn!(
|
||||
target: "crank::audit",
|
||||
action = "credential.platform_api_key.access_denied",
|
||||
credential_type = "platform_api_key.approval",
|
||||
credential_ref = "unknown",
|
||||
outcome = "denied",
|
||||
reason = reason,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
trace_id = trace_id.as_deref().unwrap_or(""),
|
||||
"approval platform key access denied"
|
||||
);
|
||||
}
|
||||
|
||||
fn allows_scope(scopes: &[PlatformApiKeyScope], required_scope: PlatformApiKeyScope) -> bool {
|
||||
match required_scope {
|
||||
PlatformApiKeyScope::Read => scopes.iter().any(|scope| {
|
||||
@@ -249,8 +311,286 @@ fn approval_allows_scope(
|
||||
})
|
||||
}
|
||||
|
||||
fn approval_allows_origin(allowed_origins: &[String], headers: &HeaderMap) -> bool {
|
||||
if allowed_origins.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let mut origins = headers.get_all(ORIGIN).iter();
|
||||
let Some(origin) = origins.next() else {
|
||||
return true;
|
||||
};
|
||||
if origins.next().is_some() {
|
||||
return false;
|
||||
};
|
||||
let Ok(origin) = origin.to_str() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
allowed_origins
|
||||
.iter()
|
||||
.any(|allowed_origin| allowed_origin == origin)
|
||||
}
|
||||
|
||||
fn record_approval_access_denial(
|
||||
api_key: &crank_registry::PlatformApiKeyRecord,
|
||||
reason: &'static str,
|
||||
) {
|
||||
let (request_id, trace_id) = crank_observability::current_request_correlation();
|
||||
let agent_id = api_key
|
||||
.api_key
|
||||
.agent_id
|
||||
.as_ref()
|
||||
.map(|id| id.as_str())
|
||||
.unwrap_or("");
|
||||
|
||||
tracing::warn!(
|
||||
target: "crank::audit",
|
||||
action = "credential.platform_api_key.access_denied",
|
||||
credential_type = "platform_api_key.approval",
|
||||
credential_ref = %api_key.api_key.id,
|
||||
workspace_id = %api_key.api_key.workspace_id,
|
||||
agent_id = agent_id,
|
||||
outcome = "denied",
|
||||
reason = reason,
|
||||
request_id = request_id.as_deref().unwrap_or(""),
|
||||
trace_id = trace_id.as_deref().unwrap_or(""),
|
||||
"approval platform key access denied"
|
||||
);
|
||||
}
|
||||
|
||||
fn security_level_rank(level: OperationSecurityLevel) -> u8 {
|
||||
match level {
|
||||
OperationSecurityLevel::Standard => 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use axum::http::{
|
||||
HeaderMap, HeaderValue,
|
||||
header::{AUTHORIZATION, ORIGIN},
|
||||
};
|
||||
use crank_core::{
|
||||
AgentId, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind, PlatformApiKeyScope,
|
||||
PlatformApiKeyStatus, WorkspaceId,
|
||||
};
|
||||
use crank_registry::PlatformApiKeyRecord;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{
|
||||
Event, Id, Metadata, Subscriber,
|
||||
field::{Field, Visit},
|
||||
span::{Attributes, Record},
|
||||
};
|
||||
|
||||
use super::{
|
||||
approval_allows_origin, bearer_token, record_approval_access_denial,
|
||||
record_machine_access_denial,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn approval_origin_policy_allows_non_browser_clients_without_origin() {
|
||||
let allowed_origins = vec!["https://allowed.example.test".to_owned()];
|
||||
let headers = HeaderMap::new();
|
||||
|
||||
assert!(approval_allows_origin(&allowed_origins, &headers));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn approval_origin_policy_rejects_ambiguous_or_malformed_origin() {
|
||||
let allowed_origins = vec!["https://allowed.example.test".to_owned()];
|
||||
|
||||
let mut duplicate = HeaderMap::new();
|
||||
duplicate.append(
|
||||
ORIGIN,
|
||||
HeaderValue::from_static("https://allowed.example.test"),
|
||||
);
|
||||
duplicate.append(
|
||||
ORIGIN,
|
||||
HeaderValue::from_static("https://evil.example.test"),
|
||||
);
|
||||
assert!(!approval_allows_origin(&allowed_origins, &duplicate));
|
||||
|
||||
let mut malformed = HeaderMap::new();
|
||||
malformed.insert(ORIGIN, HeaderValue::from_bytes(b"\xff").unwrap());
|
||||
assert!(!approval_allows_origin(&allowed_origins, &malformed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bearer_token_rejects_duplicate_authorization_headers() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.append(AUTHORIZATION, HeaderValue::from_static("Bearer first"));
|
||||
headers.append(AUTHORIZATION, HeaderValue::from_static("Bearer second"));
|
||||
|
||||
assert_eq!(bearer_token(&headers), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn machine_access_denial_audit_uses_known_key_id_only_when_resolved() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let subscriber = CapturingSubscriber {
|
||||
events: Arc::clone(&events),
|
||||
};
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let key_id = PlatformApiKeyId::new("pk_mcp_scope_audit");
|
||||
|
||||
crank_observability::with_request_correlation(
|
||||
"req_mcp_scope_audit".to_owned(),
|
||||
"0af7651916cd43dd8448eb211c80319c".to_owned(),
|
||||
async {
|
||||
record_machine_access_denial("scope", Some(&key_id));
|
||||
record_machine_access_denial("unknown_or_inactive", None);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let events = events.lock().unwrap();
|
||||
assert_eq!(events.len(), 2);
|
||||
assert_eq!(
|
||||
events[0].get("credential_ref").map(String::as_str),
|
||||
Some("pk_mcp_scope_audit")
|
||||
);
|
||||
assert_eq!(events[0].get("reason").map(String::as_str), Some("scope"));
|
||||
assert_eq!(
|
||||
events[1].get("credential_ref").map(String::as_str),
|
||||
Some("unknown")
|
||||
);
|
||||
assert_eq!(
|
||||
events[1].get("reason").map(String::as_str),
|
||||
Some("unknown_or_inactive")
|
||||
);
|
||||
for event in events.iter() {
|
||||
assert_eq!(
|
||||
event.get("credential_type").map(String::as_str),
|
||||
Some("platform_api_key.mcp_client")
|
||||
);
|
||||
assert_eq!(event.get("outcome").map(String::as_str), Some("denied"));
|
||||
assert_eq!(
|
||||
event.get("request_id").map(String::as_str),
|
||||
Some("req_mcp_scope_audit")
|
||||
);
|
||||
assert_eq!(
|
||||
event.get("trace_id").map(String::as_str),
|
||||
Some("0af7651916cd43dd8448eb211c80319c")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn known_approval_access_denial_emits_bounded_audit_event() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let subscriber = CapturingSubscriber {
|
||||
events: Arc::clone(&events),
|
||||
};
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let _guard = tracing::dispatcher::set_default(&dispatch);
|
||||
let record = PlatformApiKeyRecord {
|
||||
api_key: PlatformApiKey {
|
||||
id: PlatformApiKeyId::new("pk_approval_audit"),
|
||||
workspace_id: WorkspaceId::new("ws_audit"),
|
||||
agent_id: Some(AgentId::new("agent_audit")),
|
||||
key_kind: PlatformApiKeyKind::Approval,
|
||||
name: "approval-audit".to_owned(),
|
||||
prefix: "crk_appr_audit".to_owned(),
|
||||
scopes: vec![PlatformApiKeyScope::ReadPending],
|
||||
status: PlatformApiKeyStatus::Active,
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
last_used_at: None,
|
||||
expires_at: None,
|
||||
allowed_origins: vec!["https://allowed.example.test".to_owned()],
|
||||
},
|
||||
};
|
||||
|
||||
crank_observability::with_request_correlation(
|
||||
"req_approval_audit".to_owned(),
|
||||
"0af7651916cd43dd8448eb211c80319c".to_owned(),
|
||||
async {
|
||||
record_approval_access_denial(&record, "origin");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let events = events.lock().unwrap();
|
||||
assert_eq!(events.len(), 1);
|
||||
let event = &events[0];
|
||||
assert_eq!(
|
||||
event.get("action").map(String::as_str),
|
||||
Some("credential.platform_api_key.access_denied")
|
||||
);
|
||||
assert_eq!(
|
||||
event.get("credential_type").map(String::as_str),
|
||||
Some("platform_api_key.approval")
|
||||
);
|
||||
assert_eq!(
|
||||
event.get("credential_ref").map(String::as_str),
|
||||
Some("pk_approval_audit")
|
||||
);
|
||||
assert_eq!(event.get("outcome").map(String::as_str), Some("denied"));
|
||||
assert_eq!(event.get("reason").map(String::as_str), Some("origin"));
|
||||
assert_eq!(
|
||||
event.get("request_id").map(String::as_str),
|
||||
Some("req_approval_audit")
|
||||
);
|
||||
assert_eq!(
|
||||
event.get("trace_id").map(String::as_str),
|
||||
Some("0af7651916cd43dd8448eb211c80319c")
|
||||
);
|
||||
let serialized = format!("{event:?}");
|
||||
assert!(!serialized.contains("secret"));
|
||||
assert!(!serialized.contains("allowed.example.test"));
|
||||
}
|
||||
|
||||
struct CapturingSubscriber {
|
||||
events: Arc<Mutex<Vec<BTreeMap<String, String>>>>,
|
||||
}
|
||||
|
||||
impl Subscriber for CapturingSubscriber {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
|
||||
metadata.target() == "crank::audit"
|
||||
}
|
||||
|
||||
fn new_span(&self, _span: &Attributes<'_>) -> Id {
|
||||
Id::from_u64(1)
|
||||
}
|
||||
|
||||
fn record(&self, _span: &Id, _values: &Record<'_>) {}
|
||||
|
||||
fn record_follows_from(&self, _span: &Id, _follows: &Id) {}
|
||||
|
||||
fn event(&self, event: &Event<'_>) {
|
||||
if event.metadata().target() != "crank::audit" {
|
||||
return;
|
||||
}
|
||||
let mut visitor = FieldCaptureVisitor::default();
|
||||
event.record(&mut visitor);
|
||||
self.events.lock().unwrap().push(visitor.fields);
|
||||
}
|
||||
|
||||
fn enter(&self, _span: &Id) {}
|
||||
|
||||
fn exit(&self, _span: &Id) {}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FieldCaptureVisitor {
|
||||
fields: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl Visit for FieldCaptureVisitor {
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), value.to_owned());
|
||||
}
|
||||
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
|
||||
self.fields
|
||||
.insert(field.name().to_owned(), format!("{value:?}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,32 +13,29 @@ use axum::{
|
||||
routing::{get, post},
|
||||
};
|
||||
use crank_core::{
|
||||
ApprovalRequest, ApprovalRequestId, ApprovalRequestStatus, AuthProfile, CoordinationStateStore,
|
||||
CorrelationContext, InvocationLevel, InvocationSource, InvocationStatus, OperationApprovalMode,
|
||||
PlatformApiKeyScope, SecretId,
|
||||
ApprovalRequestId, ApprovalRequestStatus, AuthProfile, CoordinationStateStore,
|
||||
CorrelationContext, ExecutionOrigin, InvocationLevel, InvocationSource, InvocationStatus,
|
||||
PlatformApiKeyScope, SecretId, SecretStatus,
|
||||
};
|
||||
use crank_registry::{
|
||||
CreateApprovalRequest, DecideApprovalRequest, ExpireApprovalRequest, PostgresRegistry,
|
||||
ApprovalRequestRecord, DecideApprovalRequest, ExpireApprovalRequest, PostgresRegistry,
|
||||
PublishedAgentTool,
|
||||
};
|
||||
use crank_runtime::{
|
||||
RequestRateLimiter, ResolvedAuth, RuntimeError, RuntimeExecutionRequest, RuntimeExecutor,
|
||||
RuntimeOperation, RuntimeRequestContext, SecretCrypto,
|
||||
ExecutionAuthorization, RequestRateLimiter, ResolvedAuth, RuntimeError,
|
||||
RuntimeExecutionRequest, RuntimeExecutor, RuntimeRequestContext, SecretCrypto,
|
||||
};
|
||||
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
|
||||
use futures_util::stream;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{Instrument, info};
|
||||
use tracing::{Instrument, info, warn};
|
||||
|
||||
use crate::{
|
||||
access::{
|
||||
credential_allows_security_level, serialize_machine_access_mode, serialize_security_level,
|
||||
},
|
||||
access::credential_allows_security_level,
|
||||
approval_execution::{execute_approved_request, spawn_approval_recovery},
|
||||
approval_response::approval_required_response,
|
||||
auth::{SharedMachineCredentialVerifier, VerifiedMachineCredential},
|
||||
catalog::PublishedToolCatalog,
|
||||
jsonrpc::{
|
||||
@@ -49,10 +46,7 @@ use crate::{
|
||||
rate_limit::{rate_limited_jsonrpc_response, rate_limited_status_response},
|
||||
request_context::{RequestContext, apply_request_context},
|
||||
session::{ActiveSessionMetrics, SessionState, SharedSessionStore, spawn_session_cleanup},
|
||||
tool_error::{
|
||||
ToolErrorContract, generic_tool_error_contract, runtime_error_code,
|
||||
tool_error_contract_from_runtime, tool_error_text, tool_error_value,
|
||||
},
|
||||
tool_error::{execution_locale, tool_error_contract_from_failure},
|
||||
tool_search::handle_catalog_tool_call,
|
||||
transport::{
|
||||
AllowedOrigins, ResponseMode, json_response, negotiate_post_response_mode,
|
||||
@@ -61,17 +55,30 @@ use crate::{
|
||||
with_request_id_header,
|
||||
},
|
||||
};
|
||||
mod approval_policy;
|
||||
mod invocation_history;
|
||||
mod metrics;
|
||||
mod request;
|
||||
mod response;
|
||||
mod stages;
|
||||
mod tool_resolution;
|
||||
use self::approval_policy::{
|
||||
ApprovalPolicyContext, ApprovalPolicyResult, maybe_handle_approval_policy,
|
||||
};
|
||||
use self::metrics::{ActiveStreamGuard, McpRequestMetrics};
|
||||
use self::request::{
|
||||
ApprovalDecisionPayload, InitializeParams, ToolCallParams, ToolsListParams, paginate_tools_list,
|
||||
};
|
||||
pub(super) use self::response::{
|
||||
success_tool_response, take_confirmation_token, tool_error_response,
|
||||
};
|
||||
use self::stages::{
|
||||
enforce_traced_rate_limit, require_traced_approval_access, require_traced_machine_access,
|
||||
};
|
||||
pub(super) use self::tool_resolution::{resolve_generated_tool, runtime_operation};
|
||||
#[cfg(test)]
|
||||
use invocation_history::observe_invocation_history_outcome;
|
||||
use invocation_history::persist_invocation_for_key;
|
||||
pub(super) use invocation_history::{InvocationRecord, persist_invocation};
|
||||
|
||||
const TRANSPORT_SESSION_TTL_MS: u64 = 86_400_000;
|
||||
@@ -92,28 +99,6 @@ pub(super) struct AppState {
|
||||
allowed_origins: AllowedOrigins,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct InitializeParams {
|
||||
#[serde(rename = "protocolVersion")]
|
||||
protocol_version: String,
|
||||
#[serde(default)]
|
||||
capabilities: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct ToolCallParams {
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
arguments: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ApprovalDecisionPayload {
|
||||
approve: String,
|
||||
#[serde(default)]
|
||||
note: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct ResolvedToolCall {
|
||||
tool: PublishedAgentTool,
|
||||
@@ -123,6 +108,7 @@ struct ToolCallExecution {
|
||||
tool: PublishedAgentTool,
|
||||
arguments: Value,
|
||||
confirmation_token: Option<String>,
|
||||
platform_api_key_id: Option<crank_core::PlatformApiKeyId>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
@@ -346,7 +332,10 @@ async fn list_pending_approvals(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(items) => Json(json!({ "items": items })).into_response(),
|
||||
Ok(items) => Json(json!({
|
||||
"items": items.into_iter().map(safe_approval_record).collect::<Vec<_>>()
|
||||
}))
|
||||
.into_response(),
|
||||
Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
|
||||
}
|
||||
}
|
||||
@@ -461,6 +450,21 @@ async fn decide_approval_request(
|
||||
};
|
||||
let approval_id = ApprovalRequestId::new(path.approval_id);
|
||||
|
||||
let expected_record = match observe_db_query(
|
||||
DbOperation::ApprovalRead,
|
||||
state.registry.get_approval_request_for_agent(
|
||||
&key.api_key.workspace_id,
|
||||
agent_id,
|
||||
&approval_id,
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(record)) => record,
|
||||
Ok(None) => return StatusCode::NOT_FOUND.into_response(),
|
||||
Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(),
|
||||
};
|
||||
|
||||
match observe_db_query(
|
||||
DbOperation::ApprovalWrite,
|
||||
state
|
||||
@@ -469,9 +473,12 @@ async fn decide_approval_request(
|
||||
workspace_id: &key.api_key.workspace_id,
|
||||
agent_id,
|
||||
approval_id: &approval_id,
|
||||
operation_id: &expected_record.approval.operation_id,
|
||||
operation_version: expected_record.approval.operation_version,
|
||||
request_payload: &expected_record.approval.request_payload,
|
||||
status,
|
||||
decided_at: OffsetDateTime::now_utc(),
|
||||
decided_by_key_id: &key.api_key.id,
|
||||
decided_by_key_id: Some(&key.api_key.id),
|
||||
response_payload: Some(json!({ "approve": payload.approve })),
|
||||
decision_note: payload.note.as_deref(),
|
||||
}),
|
||||
@@ -502,11 +509,11 @@ async fn decide_approval_request(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(record) => Json(json!(record)).into_response(),
|
||||
Ok(record) => Json(json!(safe_approval_record(record))).into_response(),
|
||||
Err(response) => response,
|
||||
}
|
||||
}
|
||||
Ok(Some(record)) => Json(json!(record)).into_response(),
|
||||
Ok(Some(record)) => Json(json!(safe_approval_record(record))).into_response(),
|
||||
Ok(None) => {
|
||||
terminal_decision_response(&state, &key.api_key.workspace_id, agent_id, &approval_id)
|
||||
.await
|
||||
@@ -535,7 +542,7 @@ async fn approval_record_response(
|
||||
{
|
||||
expire_approval_response(state, workspace_id, agent_id, approval_id).await
|
||||
}
|
||||
Ok(Some(record)) => Json(json!(record)).into_response(),
|
||||
Ok(Some(record)) => Json(json!(safe_approval_record(record))).into_response(),
|
||||
Ok(None) => StatusCode::NOT_FOUND.into_response(),
|
||||
Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
|
||||
}
|
||||
@@ -570,7 +577,7 @@ async fn terminal_decision_response(
|
||||
| ApprovalRequestStatus::Expired
|
||||
) =>
|
||||
{
|
||||
Json(json!(record)).into_response()
|
||||
Json(json!(safe_approval_record(record))).into_response()
|
||||
}
|
||||
Ok(Some(_)) => StatusCode::CONFLICT.into_response(),
|
||||
Ok(None) => StatusCode::NOT_FOUND.into_response(),
|
||||
@@ -578,6 +585,17 @@ async fn terminal_decision_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_approval_record(mut record: ApprovalRequestRecord) -> ApprovalRequestRecord {
|
||||
record.approval.request_payload =
|
||||
crank_core::sanitize_invocation_preview(&record.approval.request_payload);
|
||||
record.approval.response_payload = record
|
||||
.approval
|
||||
.response_payload
|
||||
.as_ref()
|
||||
.map(crank_core::sanitize_invocation_preview);
|
||||
record
|
||||
}
|
||||
|
||||
async fn expire_approval_response(
|
||||
state: &Arc<AppState>,
|
||||
workspace_id: &crank_core::WorkspaceId,
|
||||
@@ -597,7 +615,7 @@ async fn expire_approval_response(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(record)) => Json(json!(record)).into_response(),
|
||||
Ok(Some(record)) => Json(json!(safe_approval_record(record))).into_response(),
|
||||
Ok(None) => StatusCode::CONFLICT.into_response(),
|
||||
Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(),
|
||||
}
|
||||
@@ -843,10 +861,39 @@ async fn mcp_post_response(
|
||||
{
|
||||
Ok(catalog) => {
|
||||
let definitions = catalog_tool_definitions(&catalog);
|
||||
let list_params: ToolsListParams = match serde_json::from_value(params(message))
|
||||
{
|
||||
Ok(value) => value,
|
||||
Err(_) => {
|
||||
return transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_error(
|
||||
request_id(message),
|
||||
-32602,
|
||||
"invalid tools/list parameters",
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(&session.protocol_version),
|
||||
);
|
||||
}
|
||||
};
|
||||
let page = match paginate_tools_list(definitions, list_params) {
|
||||
Ok(page) => page,
|
||||
Err(message_text) => {
|
||||
return transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_error(request_id(message), -32602, message_text),
|
||||
response_mode,
|
||||
None,
|
||||
Some(&session.protocol_version),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(request_id(message), json!({ "tools": definitions })),
|
||||
jsonrpc_result(request_id(message), page),
|
||||
response_mode,
|
||||
None,
|
||||
Some(&session.protocol_version),
|
||||
@@ -940,22 +987,38 @@ pub(super) async fn handle_tool_call(
|
||||
) -> Response {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
if !credential_allows_security_level(credential, resolved.tool.operation.security_level) {
|
||||
let failure = crank_core::ExecutionFailure::new(
|
||||
crank_core::ExecutionErrorCode::AuthorizationDenied,
|
||||
transport_correlation.clone(),
|
||||
);
|
||||
persist_invocation_for_key(
|
||||
&state,
|
||||
&resolved.tool,
|
||||
credential.platform_api_key_id.as_ref(),
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
tool_name: &resolved.tool.tool_name,
|
||||
status: InvocationStatus::Error,
|
||||
level: InvocationLevel::Warn,
|
||||
message: failure.error_code().as_str(),
|
||||
status_code: None,
|
||||
error_kind: Some(failure.error_code().as_str()),
|
||||
execution_stage: Some(failure.stage()),
|
||||
execution_error_code: Some(failure.error_code()),
|
||||
retryability: Some(failure.retryability()),
|
||||
outcome_certainty: Some(failure.outcome_certainty()),
|
||||
duration: Duration::ZERO,
|
||||
request_preview: crank_core::sanitize_invocation_preview(&arguments),
|
||||
response_preview: Value::Null,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return tool_error_response(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
generic_tool_error_contract(
|
||||
"machine_access_insufficient",
|
||||
format!(
|
||||
"machine access mode {} does not satisfy {} operation security",
|
||||
serialize_machine_access_mode(credential.machine_access_mode),
|
||||
serialize_security_level(resolved.tool.operation.security_level),
|
||||
),
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
false,
|
||||
Some("Используйте ключ агента с достаточным уровнем доступа."),
|
||||
),
|
||||
tool_error_contract_from_failure(&failure, execution_locale(message)),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -968,6 +1031,7 @@ pub(super) async fn handle_tool_call(
|
||||
tool: resolved.tool,
|
||||
arguments,
|
||||
confirmation_token,
|
||||
platform_api_key_id: credential.platform_api_key_id.clone(),
|
||||
},
|
||||
transport_correlation,
|
||||
)
|
||||
@@ -1016,10 +1080,7 @@ async fn resolve_runtime_auth_for_task(
|
||||
registry.get_auth_profile(workspace_id, auth_profile_id),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load auth profile",
|
||||
details: error.to_string(),
|
||||
})?
|
||||
.map_err(|_| RuntimeError::AuthorizationStoreUnavailable)?
|
||||
.ok_or_else(|| RuntimeError::MissingAuthProfile {
|
||||
auth_profile_id: auth_profile_id.as_str().to_owned(),
|
||||
})?;
|
||||
@@ -1044,39 +1105,47 @@ async fn resolve_auth_profile(
|
||||
registry.get_secret(workspace_id, secret_id),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load secret",
|
||||
details: error.to_string(),
|
||||
})?
|
||||
.map_err(|_| RuntimeError::AuthorizationStoreUnavailable)?
|
||||
.ok_or_else(|| RuntimeError::MissingSecret {
|
||||
secret_id: secret_id.as_str().to_owned(),
|
||||
})?;
|
||||
if secret.secret.status != SecretStatus::Active {
|
||||
return Err(RuntimeError::InvalidAuthSecretValue {
|
||||
secret_id: secret_id.as_str().to_owned(),
|
||||
reason: "secret is not active".to_owned(),
|
||||
});
|
||||
}
|
||||
let version = observe_db_query(
|
||||
DbOperation::SecretRead,
|
||||
registry.get_current_secret_version(workspace_id, secret_id),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "load current secret version",
|
||||
details: error.to_string(),
|
||||
})?
|
||||
.map_err(|_| RuntimeError::AuthorizationStoreUnavailable)?
|
||||
.ok_or_else(|| RuntimeError::MissingSecretVersion {
|
||||
secret_id: secret_id.as_str().to_owned(),
|
||||
version: secret.secret.current_version,
|
||||
})?;
|
||||
let plaintext = secret_crypto.decrypt(
|
||||
let plaintext = secret_crypto.decrypt_for_epoch(
|
||||
&version.secret_version.key_version,
|
||||
version.master_key_epoch,
|
||||
&version.secret_version.ciphertext,
|
||||
)?;
|
||||
observe_db_query(
|
||||
if observe_db_query(
|
||||
DbOperation::SecretTouch,
|
||||
registry.touch_secret(workspace_id, secret_id, &used_at),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| RuntimeError::SecretCrypto {
|
||||
operation: "touch secret",
|
||||
details: error.to_string(),
|
||||
})?;
|
||||
.is_err()
|
||||
{
|
||||
warn!(
|
||||
target: "crank::audit",
|
||||
action = "credential.secret.touch_failed",
|
||||
credential_type = "secret",
|
||||
credential_ref = %secret_id.as_str(),
|
||||
outcome = "metadata_touch_failed",
|
||||
"secret last-used metadata was not updated"
|
||||
);
|
||||
}
|
||||
secrets.insert(SecretId::new(secret_id.as_str()), plaintext);
|
||||
}
|
||||
|
||||
@@ -1092,6 +1161,7 @@ async fn handle_base_tool_call(
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Response {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
let platform_api_key_id = execution.platform_api_key_id;
|
||||
let tool = execution.tool;
|
||||
let arguments = execution.arguments;
|
||||
let operation = runtime_operation(&tool);
|
||||
@@ -1103,15 +1173,16 @@ async fn handle_base_tool_call(
|
||||
.is_some_and(|policy| policy.required)
|
||||
{
|
||||
let approval_span = Stage::ApprovalCheck.span();
|
||||
let response = maybe_handle_approval_policy(
|
||||
&state,
|
||||
let response = maybe_handle_approval_policy(ApprovalPolicyContext {
|
||||
state: &state,
|
||||
session,
|
||||
message,
|
||||
response_mode,
|
||||
&tool,
|
||||
&arguments,
|
||||
tool: &tool,
|
||||
arguments: &arguments,
|
||||
platform_api_key_id: platform_api_key_id.as_ref(),
|
||||
transport_correlation,
|
||||
)
|
||||
})
|
||||
.instrument(approval_span.clone())
|
||||
.await;
|
||||
if let Some(result) = response {
|
||||
@@ -1144,30 +1215,45 @@ async fn handle_base_tool_call(
|
||||
if let Some(token) = execution.confirmation_token {
|
||||
runtime_request_context = runtime_request_context.with_confirmation_token(token);
|
||||
}
|
||||
let request_preview = build_request_preview(&state.runtime, &operation, &arguments);
|
||||
let started_at = Instant::now();
|
||||
let resolved_auth =
|
||||
resolve_operation_auth(&state, &tool.workspace_id, &operation.execution_config).await;
|
||||
|
||||
let result = match resolved_auth {
|
||||
Ok(resolved_auth) => {
|
||||
state
|
||||
.runtime
|
||||
.execute_request(
|
||||
RuntimeExecutionRequest::new(&operation, &arguments)
|
||||
.with_optional_auth(resolved_auth.as_ref())
|
||||
.with_context(&runtime_request_context),
|
||||
)
|
||||
.await
|
||||
match RuntimeExecutionRequest::try_new(
|
||||
&tool.workspace_id,
|
||||
ExecutionOrigin::AgentSnapshot,
|
||||
Some(&tool.agent_id),
|
||||
&operation,
|
||||
&arguments,
|
||||
ExecutionAuthorization::Authorized,
|
||||
resolved_auth.as_ref(),
|
||||
&runtime_request_context,
|
||||
Instant::now()
|
||||
+ Duration::from_millis(operation.execution_config.timeout_ms.max(1)),
|
||||
) {
|
||||
Ok(request) => state.runtime.execute_outcome(request).await,
|
||||
Err(_) => Err(crank_core::ExecutionFailure::new(
|
||||
crank_core::ExecutionErrorCode::RuntimeInternal,
|
||||
transport_correlation.clone(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
Err(error) => Err(crank_runtime::normalize_runtime_error(
|
||||
&error,
|
||||
transport_correlation,
|
||||
)),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
persist_invocation(
|
||||
Ok(success) => {
|
||||
let request_preview = success.request_preview;
|
||||
let output = success.output;
|
||||
persist_invocation_for_key(
|
||||
&state,
|
||||
&tool,
|
||||
platform_api_key_id.as_ref(),
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
@@ -1177,6 +1263,10 @@ async fn handle_base_tool_call(
|
||||
message: "agent tool call completed",
|
||||
status_code: None,
|
||||
error_kind: None,
|
||||
execution_stage: Some(crank_core::ExecutionStage::Runtime),
|
||||
execution_error_code: None,
|
||||
retryability: Some(crank_core::Retryability::Never),
|
||||
outcome_certainty: Some(crank_core::OutcomeCertainty::Certain),
|
||||
duration: started_at.elapsed(),
|
||||
request_preview,
|
||||
response_preview: output.clone(),
|
||||
@@ -1186,21 +1276,26 @@ async fn handle_base_tool_call(
|
||||
|
||||
success_tool_response(message, response_mode, &session.protocol_version, output)
|
||||
}
|
||||
Err(error) => {
|
||||
persist_invocation(
|
||||
Err(failure) => {
|
||||
persist_invocation_for_key(
|
||||
&state,
|
||||
&tool,
|
||||
platform_api_key_id.as_ref(),
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
tool_name: &tool.tool_name,
|
||||
status: InvocationStatus::Error,
|
||||
level: InvocationLevel::Error,
|
||||
message: runtime_error_code(&error),
|
||||
status_code: None,
|
||||
error_kind: Some(runtime_error_code(&error)),
|
||||
message: failure.error_code().as_str(),
|
||||
status_code: failure.upstream_status(),
|
||||
error_kind: Some(failure.error_code().as_str()),
|
||||
execution_stage: Some(failure.stage()),
|
||||
execution_error_code: Some(failure.error_code()),
|
||||
retryability: Some(failure.retryability()),
|
||||
outcome_certainty: Some(failure.outcome_certainty()),
|
||||
duration: started_at.elapsed(),
|
||||
request_preview,
|
||||
request_preview: Value::Null,
|
||||
response_preview: Value::Null,
|
||||
},
|
||||
)
|
||||
@@ -1210,188 +1305,12 @@ async fn handle_base_tool_call(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
tool_error_contract_from_runtime(
|
||||
&error,
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
),
|
||||
tool_error_contract_from_failure(&failure, execution_locale(message)),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum ApprovalPolicyResult {
|
||||
Required(Response),
|
||||
Error(Response),
|
||||
}
|
||||
|
||||
async fn maybe_handle_approval_policy(
|
||||
state: &Arc<AppState>,
|
||||
session: &SessionState,
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
tool: &PublishedAgentTool,
|
||||
arguments: &Value,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Option<ApprovalPolicyResult> {
|
||||
let policy = tool.operation.execution_config.approval_policy.as_ref()?;
|
||||
if !policy.required {
|
||||
return None;
|
||||
}
|
||||
|
||||
match policy.mode {
|
||||
OperationApprovalMode::Custom => {
|
||||
maybe_create_custom_pending_approval(
|
||||
state,
|
||||
session,
|
||||
message,
|
||||
response_mode,
|
||||
tool,
|
||||
arguments,
|
||||
transport_correlation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
OperationApprovalMode::Elicitation => Some(handle_elicitation_approval(
|
||||
session,
|
||||
message,
|
||||
response_mode,
|
||||
tool,
|
||||
arguments,
|
||||
policy.elicitation_message.as_deref(),
|
||||
transport_correlation,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn maybe_create_custom_pending_approval(
|
||||
state: &Arc<AppState>,
|
||||
session: &SessionState,
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
tool: &PublishedAgentTool,
|
||||
arguments: &Value,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> Option<ApprovalPolicyResult> {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
let policy = tool.operation.execution_config.approval_policy.as_ref()?;
|
||||
|
||||
let approval_id = ApprovalRequestId::new(format!("approval_{}", uuid::Uuid::now_v7().simple()));
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let expires_at = now + time::Duration::seconds(i64::from(policy.ttl_seconds));
|
||||
let approval = ApprovalRequest {
|
||||
id: approval_id,
|
||||
workspace_id: tool.workspace_id.clone(),
|
||||
agent_id: tool.agent_id.clone(),
|
||||
operation_id: tool.operation.id.clone(),
|
||||
operation_version: tool.operation.version,
|
||||
status: ApprovalRequestStatus::Pending,
|
||||
risk_level: policy.risk_level,
|
||||
request_payload: arguments.clone(),
|
||||
response_payload: None,
|
||||
created_at: now,
|
||||
expires_at,
|
||||
decided_at: None,
|
||||
decided_by_key_id: None,
|
||||
decision_note: None,
|
||||
};
|
||||
|
||||
let persisted_approval = match observe_db_query(
|
||||
DbOperation::ApprovalWrite,
|
||||
state
|
||||
.registry
|
||||
.create_approval_request(CreateApprovalRequest {
|
||||
approval: &approval,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(approval) => approval,
|
||||
Err(error) => {
|
||||
return Some(ApprovalPolicyResult::Error(internal_jsonrpc_error(
|
||||
message, error,
|
||||
)));
|
||||
}
|
||||
};
|
||||
let response_payload = approval_required_response(tool, &persisted_approval.approval, policy);
|
||||
|
||||
persist_invocation(
|
||||
state,
|
||||
tool,
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
tool_name: &tool.tool_name,
|
||||
status: InvocationStatus::Ok,
|
||||
level: InvocationLevel::Info,
|
||||
message: "agent tool call is waiting for human approval",
|
||||
status_code: None,
|
||||
error_kind: None,
|
||||
duration: Duration::from_millis(0),
|
||||
request_preview: arguments.clone(),
|
||||
response_preview: response_payload.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
Some(ApprovalPolicyResult::Required(success_tool_response(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
response_payload,
|
||||
)))
|
||||
}
|
||||
|
||||
fn handle_elicitation_approval(
|
||||
session: &SessionState,
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
tool: &PublishedAgentTool,
|
||||
arguments: &Value,
|
||||
elicitation_message: Option<&str>,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> ApprovalPolicyResult {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
if !session.supports_elicitation {
|
||||
return ApprovalPolicyResult::Error(tool_error_response(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
generic_tool_error_contract(
|
||||
"approval_elicitation_not_supported",
|
||||
"operation requires MCP Elicitation, but the MCP client did not advertise elicitation capability",
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
false,
|
||||
Some(
|
||||
"Выберите Custom MCP Approval или подключите MCP-клиент с поддержкой elicitation.",
|
||||
),
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
let payload_preview = tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()
|
||||
.and_then(|policy| policy.show_payload_preview.then(|| arguments.clone()))
|
||||
.unwrap_or(Value::Null);
|
||||
|
||||
ApprovalPolicyResult::Required(success_tool_response(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
json!({
|
||||
"status": "elicitation_required",
|
||||
"message": elicitation_message.unwrap_or("Confirm operation execution."),
|
||||
"tool": tool.tool_name,
|
||||
"payload_preview": payload_preview,
|
||||
"note": "This MCP client advertised elicitation support. Full elicitation/create continuation is handled by compatible client integrations.",
|
||||
}),
|
||||
))
|
||||
}
|
||||
|
||||
async fn handle_initialize(
|
||||
state: Arc<AppState>,
|
||||
path: &AgentRoutePath,
|
||||
@@ -1548,91 +1467,6 @@ fn internal_jsonrpc_error(message: &Value, _error: impl std::fmt::Display) -> Re
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn take_confirmation_token(arguments: &mut Value) -> Option<String> {
|
||||
let Value::Object(object) = arguments else {
|
||||
return None;
|
||||
};
|
||||
object
|
||||
.remove("_crank_confirmation_token")
|
||||
.and_then(|value| value.as_str().map(str::to_owned))
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
}
|
||||
|
||||
pub(super) fn build_request_preview(
|
||||
runtime: &RuntimeExecutor,
|
||||
operation: &RuntimeOperation,
|
||||
arguments: &Value,
|
||||
) -> Value {
|
||||
match runtime.prepare_request(operation, arguments) {
|
||||
Ok(prepared) => json!({
|
||||
"path": prepared.path_params,
|
||||
"query": prepared.query_params,
|
||||
"headers": prepared.headers,
|
||||
"body": prepared.body.unwrap_or(Value::Null)
|
||||
}),
|
||||
Err(_) => Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
fn success_tool_response(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
output: Value,
|
||||
) -> Response {
|
||||
transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
|
||||
}
|
||||
],
|
||||
"structuredContent": output,
|
||||
"isError": false
|
||||
}),
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn tool_error_response(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
error: ToolErrorContract,
|
||||
) -> Response {
|
||||
let error_message = tool_error_text(&error);
|
||||
let error_value = tool_error_value(&error);
|
||||
|
||||
transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": error_message
|
||||
}
|
||||
],
|
||||
"structuredContent": {
|
||||
"error": error_value
|
||||
},
|
||||
"isError": true
|
||||
}),
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
)
|
||||
}
|
||||
|
||||
fn add_millis(timestamp: OffsetDateTime, millis: u64) -> OffsetDateTime {
|
||||
let delta = time::Duration::milliseconds(i64::try_from(millis).unwrap_or(i64::MAX));
|
||||
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use axum::response::Response;
|
||||
use crank_core::{
|
||||
ApprovalRequest, ApprovalRequestId, ApprovalRequestStatus, CorrelationContext, InvocationLevel,
|
||||
InvocationStatus, OperationApprovalMode,
|
||||
};
|
||||
use crank_registry::{CreateApprovalRequest, PublishedAgentTool};
|
||||
use crank_trace::{DbOperation, observe_db_query};
|
||||
use serde_json::{Value, json};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use super::{
|
||||
AppState, InvocationRecord, internal_jsonrpc_error, persist_invocation_for_key,
|
||||
response::{success_tool_response, tool_error_response},
|
||||
};
|
||||
use crate::{
|
||||
approval_response::{
|
||||
approval_history_request_preview, approval_history_response_preview,
|
||||
approval_required_response,
|
||||
},
|
||||
session::SessionState,
|
||||
tool_error::{McpControlError, control_tool_error_contract, execution_locale},
|
||||
transport::ResponseMode,
|
||||
};
|
||||
|
||||
pub(super) enum ApprovalPolicyResult {
|
||||
Required(Response),
|
||||
Error(Response),
|
||||
}
|
||||
|
||||
pub(super) struct ApprovalPolicyContext<'a> {
|
||||
pub state: &'a Arc<AppState>,
|
||||
pub session: &'a SessionState,
|
||||
pub message: &'a Value,
|
||||
pub response_mode: ResponseMode,
|
||||
pub tool: &'a PublishedAgentTool,
|
||||
pub arguments: &'a Value,
|
||||
pub platform_api_key_id: Option<&'a crank_core::PlatformApiKeyId>,
|
||||
pub transport_correlation: &'a CorrelationContext,
|
||||
}
|
||||
|
||||
pub(super) async fn maybe_handle_approval_policy(
|
||||
context: ApprovalPolicyContext<'_>,
|
||||
) -> Option<ApprovalPolicyResult> {
|
||||
let policy = context
|
||||
.tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()?;
|
||||
if !policy.required {
|
||||
return None;
|
||||
}
|
||||
|
||||
match policy.mode {
|
||||
OperationApprovalMode::Custom => maybe_create_custom_pending_approval(&context).await,
|
||||
OperationApprovalMode::Elicitation => Some(handle_elicitation_approval(
|
||||
context.session,
|
||||
context.message,
|
||||
context.response_mode,
|
||||
context.tool,
|
||||
context.arguments,
|
||||
policy.elicitation_message.as_deref(),
|
||||
context.transport_correlation,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn maybe_create_custom_pending_approval(
|
||||
context: &ApprovalPolicyContext<'_>,
|
||||
) -> Option<ApprovalPolicyResult> {
|
||||
let ApprovalPolicyContext {
|
||||
state,
|
||||
session,
|
||||
message,
|
||||
response_mode,
|
||||
tool,
|
||||
arguments,
|
||||
platform_api_key_id,
|
||||
transport_correlation,
|
||||
} = context;
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
let policy = tool.operation.execution_config.approval_policy.as_ref()?;
|
||||
|
||||
let approval_id = ApprovalRequestId::new(format!("approval_{}", uuid::Uuid::now_v7().simple()));
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let expires_at = now + time::Duration::seconds(i64::from(policy.ttl_seconds));
|
||||
let approval = ApprovalRequest {
|
||||
id: approval_id,
|
||||
workspace_id: tool.workspace_id.clone(),
|
||||
agent_id: tool.agent_id.clone(),
|
||||
operation_id: tool.operation.id.clone(),
|
||||
operation_version: tool.operation.version,
|
||||
status: ApprovalRequestStatus::Pending,
|
||||
risk_level: policy.risk_level,
|
||||
request_id: Some(transport_correlation.request_id().as_str().to_owned()),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str().to_owned()),
|
||||
request_payload: (*arguments).clone(),
|
||||
response_payload: None,
|
||||
created_at: now,
|
||||
expires_at,
|
||||
decided_at: None,
|
||||
decided_by_key_id: None,
|
||||
decision_note: None,
|
||||
};
|
||||
|
||||
let persisted_approval = match observe_db_query(
|
||||
DbOperation::ApprovalWrite,
|
||||
state
|
||||
.registry
|
||||
.create_approval_request(CreateApprovalRequest {
|
||||
approval: &approval,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(approval) => approval,
|
||||
Err(error) => {
|
||||
return Some(ApprovalPolicyResult::Error(internal_jsonrpc_error(
|
||||
message, error,
|
||||
)));
|
||||
}
|
||||
};
|
||||
let response_payload = approval_required_response(tool, &persisted_approval.approval, policy);
|
||||
|
||||
persist_invocation_for_key(
|
||||
state,
|
||||
tool,
|
||||
*platform_api_key_id,
|
||||
InvocationRecord {
|
||||
request_id: Some(transport_request_id),
|
||||
trace_id: Some(transport_correlation.trace_id().as_str()),
|
||||
tool_name: &tool.tool_name,
|
||||
status: InvocationStatus::Ok,
|
||||
level: InvocationLevel::Info,
|
||||
message: "agent tool call is waiting for human approval",
|
||||
status_code: None,
|
||||
error_kind: None,
|
||||
execution_stage: Some(crank_core::ExecutionStage::Admission),
|
||||
execution_error_code: None,
|
||||
retryability: Some(crank_core::Retryability::RequiresConfirmation),
|
||||
outcome_certainty: Some(crank_core::OutcomeCertainty::Certain),
|
||||
duration: Duration::from_millis(0),
|
||||
request_preview: approval_history_request_preview(&persisted_approval.approval),
|
||||
response_preview: approval_history_response_preview(&response_payload),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
Some(ApprovalPolicyResult::Required(success_tool_response(
|
||||
message,
|
||||
*response_mode,
|
||||
&session.protocol_version,
|
||||
response_payload,
|
||||
)))
|
||||
}
|
||||
|
||||
fn handle_elicitation_approval(
|
||||
session: &SessionState,
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
tool: &PublishedAgentTool,
|
||||
arguments: &Value,
|
||||
elicitation_message: Option<&str>,
|
||||
transport_correlation: &CorrelationContext,
|
||||
) -> ApprovalPolicyResult {
|
||||
let transport_request_id = transport_correlation.request_id().as_str();
|
||||
if !session.supports_elicitation {
|
||||
return ApprovalPolicyResult::Error(tool_error_response(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
control_tool_error_contract(
|
||||
McpControlError::ApprovalElicitationNotSupported,
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
execution_locale(message),
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
let payload_preview = tool
|
||||
.operation
|
||||
.execution_config
|
||||
.approval_policy
|
||||
.as_ref()
|
||||
.and_then(|policy| policy.show_payload_preview.then(|| arguments.clone()))
|
||||
.unwrap_or(Value::Null);
|
||||
|
||||
ApprovalPolicyResult::Required(success_tool_response(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
json!({
|
||||
"status": "elicitation_required",
|
||||
"message": elicitation_message.unwrap_or("Confirm operation execution."),
|
||||
"tool": tool.tool_name,
|
||||
"payload_preview": payload_preview,
|
||||
"note": "This MCP client advertised elicitation support. Full elicitation/create continuation is handled by compatible client integrations.",
|
||||
}),
|
||||
))
|
||||
}
|
||||
@@ -22,6 +22,10 @@ pub(crate) struct InvocationRecord<'a> {
|
||||
pub(crate) message: &'a str,
|
||||
pub(crate) status_code: Option<u16>,
|
||||
pub(crate) error_kind: Option<&'a str>,
|
||||
pub(crate) execution_stage: Option<crank_core::ExecutionStage>,
|
||||
pub(crate) execution_error_code: Option<crank_core::ExecutionErrorCode>,
|
||||
pub(crate) retryability: Option<crank_core::Retryability>,
|
||||
pub(crate) outcome_certainty: Option<crank_core::OutcomeCertainty>,
|
||||
pub(crate) duration: Duration,
|
||||
pub(crate) request_preview: Value,
|
||||
pub(crate) response_preview: Value,
|
||||
@@ -31,12 +35,23 @@ pub(crate) async fn persist_invocation(
|
||||
state: &Arc<AppState>,
|
||||
tool: &PublishedAgentTool,
|
||||
record: InvocationRecord<'_>,
|
||||
) -> InvocationHistoryWriteOutcome {
|
||||
persist_invocation_for_key(state, tool, None, record).await
|
||||
}
|
||||
|
||||
pub(crate) async fn persist_invocation_for_key(
|
||||
state: &Arc<AppState>,
|
||||
tool: &PublishedAgentTool,
|
||||
platform_api_key_id: Option<&crank_core::PlatformApiKeyId>,
|
||||
record: InvocationRecord<'_>,
|
||||
) -> InvocationHistoryWriteOutcome {
|
||||
let log = InvocationLog {
|
||||
id: InvocationLogId::new(format!("log_{}", uuid::Uuid::now_v7().simple())),
|
||||
workspace_id: tool.workspace_id.clone(),
|
||||
agent_id: Some(tool.agent_id.clone()),
|
||||
platform_api_key_id: platform_api_key_id.cloned(),
|
||||
operation_id: tool.operation.id.clone(),
|
||||
operation_version: Some(tool.operation.version),
|
||||
source: InvocationSource::AgentToolCall,
|
||||
level: record.level,
|
||||
status: record.status,
|
||||
@@ -47,6 +62,10 @@ pub(crate) async fn persist_invocation(
|
||||
status_code: record.status_code,
|
||||
duration_ms: u64::try_from(record.duration.as_millis()).unwrap_or(u64::MAX),
|
||||
error_kind: record.error_kind.map(ToOwned::to_owned),
|
||||
execution_stage: record.execution_stage,
|
||||
execution_error_code: record.execution_error_code,
|
||||
retryability: record.retryability,
|
||||
outcome_certainty: record.outcome_certainty,
|
||||
request_preview: record.request_preview,
|
||||
response_preview: record.response_preview,
|
||||
created_at: OffsetDateTime::now_utc(),
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub(super) struct InitializeParams {
|
||||
#[serde(rename = "protocolVersion")]
|
||||
pub(super) protocol_version: String,
|
||||
#[serde(default)]
|
||||
pub(super) capabilities: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub(super) struct ToolCallParams {
|
||||
pub(super) name: String,
|
||||
#[serde(default)]
|
||||
pub(super) arguments: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct ToolsListParams {
|
||||
#[serde(default)]
|
||||
pub(super) cursor: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(super) limit: Option<usize>,
|
||||
}
|
||||
|
||||
pub(super) fn paginate_tools_list(
|
||||
definitions: Vec<Value>,
|
||||
params: ToolsListParams,
|
||||
) -> Result<Value, &'static str> {
|
||||
const MAX_TOOLS_LIST_LIMIT: usize = 100;
|
||||
let start = match params.cursor {
|
||||
Some(cursor) if cursor.is_empty() => 0,
|
||||
Some(cursor) => cursor
|
||||
.parse::<usize>()
|
||||
.map_err(|_| "invalid tools/list cursor")?,
|
||||
None => 0,
|
||||
};
|
||||
if start > definitions.len() {
|
||||
return Err("tools/list cursor is out of range");
|
||||
}
|
||||
let limit = params
|
||||
.limit
|
||||
.unwrap_or(definitions.len())
|
||||
.clamp(1, MAX_TOOLS_LIST_LIMIT);
|
||||
let end = start.saturating_add(limit).min(definitions.len());
|
||||
let page = definitions[start..end].to_vec();
|
||||
let mut result = json!({ "tools": page });
|
||||
if end < definitions.len() {
|
||||
result["nextCursor"] = json!(end.to_string());
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(super) struct ApprovalDecisionPayload {
|
||||
pub(super) approve: String,
|
||||
#[serde(default)]
|
||||
pub(super) note: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
use axum::{http::StatusCode, response::Response};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{
|
||||
jsonrpc::{jsonrpc_result, request_id},
|
||||
tool_error::{ToolErrorContract, tool_error_text, tool_error_value},
|
||||
transport::{ResponseMode, transport_response},
|
||||
};
|
||||
|
||||
pub(crate) fn take_confirmation_token(arguments: &mut Value) -> Option<String> {
|
||||
let Value::Object(object) = arguments else {
|
||||
return None;
|
||||
};
|
||||
object
|
||||
.remove("_crank_confirmation_token")
|
||||
.and_then(|value| value.as_str().map(str::to_owned))
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn success_tool_response(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
output: Value,
|
||||
) -> Response {
|
||||
transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": serde_json::to_string_pretty(&output).unwrap_or_else(|_| "{}".to_owned())
|
||||
}
|
||||
],
|
||||
"structuredContent": output,
|
||||
"isError": false
|
||||
}),
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn tool_error_response(
|
||||
message: &Value,
|
||||
response_mode: ResponseMode,
|
||||
protocol_version: &str,
|
||||
error: ToolErrorContract,
|
||||
) -> Response {
|
||||
let error_message = tool_error_text(&error);
|
||||
let error_value = tool_error_value(&error);
|
||||
|
||||
transport_response(
|
||||
StatusCode::OK,
|
||||
jsonrpc_result(
|
||||
request_id(message),
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": error_message
|
||||
}
|
||||
],
|
||||
"structuredContent": {
|
||||
"error": error_value
|
||||
},
|
||||
"isError": true
|
||||
}),
|
||||
),
|
||||
response_mode,
|
||||
None,
|
||||
Some(protocol_version),
|
||||
)
|
||||
}
|
||||
@@ -22,7 +22,7 @@ use super::{
|
||||
observe_invocation_history_outcome, tool_error_response,
|
||||
};
|
||||
use crate::jsonrpc::{CURRENT_PROTOCOL_VERSION, jsonrpc_error};
|
||||
use crate::tool_error::generic_tool_error_contract;
|
||||
use crate::tool_error::{McpControlError, control_tool_error_contract};
|
||||
use crate::transport::transport_response;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -33,13 +33,11 @@ async fn tool_error_response_includes_structured_context() {
|
||||
&message,
|
||||
ResponseMode::Json,
|
||||
CURRENT_PROTOCOL_VERSION,
|
||||
generic_tool_error_contract(
|
||||
"streaming_payload_error",
|
||||
"request root must be an object",
|
||||
control_tool_error_contract(
|
||||
McpControlError::CatalogRevisionChanged,
|
||||
"req-1",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
false,
|
||||
Some("Проверьте параметры вызова инструмента."),
|
||||
crank_core::ExecutionLocale::Ru,
|
||||
),
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -55,13 +53,16 @@ async fn tool_error_response_includes_structured_context() {
|
||||
assert_eq!(
|
||||
payload["result"]["structuredContent"]["error"],
|
||||
json!({
|
||||
"code": "streaming_payload_error",
|
||||
"error_code": "streaming_payload_error",
|
||||
"message": "request root must be an object",
|
||||
"recoverable": false,
|
||||
"code": "agent_catalog_result_stale",
|
||||
"error_code": "agent_catalog_result_stale",
|
||||
"message": "Версия каталога изменилась.",
|
||||
"stage": "authorization",
|
||||
"retryability": "after_delay",
|
||||
"outcome_certainty": "certain",
|
||||
"recoverable": true,
|
||||
"request_id": "req-1",
|
||||
"trace_id": "0af7651916cd43dd8448eb211c80319c",
|
||||
"suggested_action": "Проверьте параметры вызова инструмента."
|
||||
"suggested_action": "Повторите search_tools и вызовите инструмент с новой версией каталога."
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@@ -9,16 +9,13 @@ use crank_core::{CorrelationContext, RequestId, TraceContext};
|
||||
use crank_registry::{ApprovalRequestRecord, FinishApprovalRequest};
|
||||
use crank_runtime::{RuntimeExecutionRequest, RuntimeRequestContext};
|
||||
use crank_trace::{DbOperation, ErrorCategory, Stage, StageOutcome, observe_db_query};
|
||||
use serde_json::json;
|
||||
use serde_json::{Value, json};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::{Instrument, warn};
|
||||
|
||||
use crate::{
|
||||
app::{
|
||||
AgentRoutePath, AppState, InvocationRecord, build_request_preview, persist_invocation,
|
||||
resolve_operation_auth, runtime_operation,
|
||||
},
|
||||
tool_error::{runtime_error_code, safe_runtime_error_message},
|
||||
use crate::app::{
|
||||
AgentRoutePath, AppState, InvocationRecord, persist_invocation, resolve_operation_auth,
|
||||
runtime_operation,
|
||||
};
|
||||
|
||||
const RECOVERY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
@@ -184,11 +181,6 @@ pub(super) async fn execute_approved_request(
|
||||
};
|
||||
|
||||
let operation = runtime_operation(&tool);
|
||||
let request_preview = build_request_preview(
|
||||
&state.runtime,
|
||||
&operation,
|
||||
&approval.approval.request_payload,
|
||||
);
|
||||
let started_at = Instant::now();
|
||||
let runtime_request_context = RuntimeRequestContext::from_correlation(&correlation)
|
||||
.with_response_cache_scope(
|
||||
@@ -205,42 +197,86 @@ pub(super) async fn execute_approved_request(
|
||||
resolve_operation_auth(state, &tool.workspace_id, &operation.execution_config).await;
|
||||
let result = match resolved_auth {
|
||||
Ok(resolved_auth) => {
|
||||
state
|
||||
.runtime
|
||||
.execute_request(
|
||||
RuntimeExecutionRequest::new(&operation, &approval.approval.request_payload)
|
||||
.with_optional_auth(resolved_auth.as_ref())
|
||||
.with_context(&runtime_request_context),
|
||||
)
|
||||
.await
|
||||
match RuntimeExecutionRequest::try_new(
|
||||
&tool.workspace_id,
|
||||
crank_core::ExecutionOrigin::AgentSnapshot,
|
||||
Some(&tool.agent_id),
|
||||
&operation,
|
||||
&approval.approval.request_payload,
|
||||
crank_runtime::ExecutionAuthorization::Authorized,
|
||||
resolved_auth.as_ref(),
|
||||
&runtime_request_context,
|
||||
Instant::now()
|
||||
+ std::time::Duration::from_millis(
|
||||
operation.execution_config.timeout_ms.max(1),
|
||||
),
|
||||
) {
|
||||
Ok(request) => state.runtime.execute_outcome(request).await,
|
||||
Err(_) => Err(crank_core::ExecutionFailure::new(
|
||||
crank_core::ExecutionErrorCode::RuntimeInternal,
|
||||
correlation.clone(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
Err(error) => Err(crank_runtime::normalize_runtime_error(&error, &correlation)),
|
||||
};
|
||||
|
||||
let (status, response_payload, invocation_status, invocation_level, message, error_kind) =
|
||||
match result {
|
||||
Ok(output) => (
|
||||
let (
|
||||
status,
|
||||
response_payload,
|
||||
invocation_status,
|
||||
invocation_level,
|
||||
message,
|
||||
error_kind,
|
||||
execution_stage,
|
||||
execution_error_code,
|
||||
retryability,
|
||||
outcome_certainty,
|
||||
upstream_status,
|
||||
request_preview,
|
||||
) = match result {
|
||||
Ok(success) => {
|
||||
let request_preview = success.request_preview;
|
||||
(
|
||||
ApprovalRequestStatus::Completed,
|
||||
output,
|
||||
success.output,
|
||||
InvocationStatus::Ok,
|
||||
InvocationLevel::Info,
|
||||
"approved tool call completed",
|
||||
None,
|
||||
),
|
||||
Err(error) => (
|
||||
ApprovalRequestStatus::Failed,
|
||||
json!({
|
||||
"error": {
|
||||
"code": runtime_error_code(&error),
|
||||
"message": safe_runtime_error_message(&error),
|
||||
}
|
||||
}),
|
||||
InvocationStatus::Error,
|
||||
InvocationLevel::Error,
|
||||
"approved tool call failed",
|
||||
Some(runtime_error_code(&error)),
|
||||
),
|
||||
};
|
||||
Some(crank_core::ExecutionStage::Runtime),
|
||||
None,
|
||||
Some(crank_core::Retryability::Never),
|
||||
Some(crank_core::OutcomeCertainty::Certain),
|
||||
None,
|
||||
request_preview,
|
||||
)
|
||||
}
|
||||
Err(failure) => (
|
||||
ApprovalRequestStatus::Failed,
|
||||
json!({
|
||||
"error": {
|
||||
"code": failure.error_code().as_str(),
|
||||
"message": failure.error_code().message(crank_core::ExecutionLocale::Ru),
|
||||
"stage": failure.stage().as_str(),
|
||||
"retryability": failure.retryability().as_str(),
|
||||
"outcome_certainty": failure.outcome_certainty().as_str(),
|
||||
"request_id": request_id,
|
||||
"trace_id": correlation.trace_id().as_str(),
|
||||
}
|
||||
}),
|
||||
InvocationStatus::Error,
|
||||
InvocationLevel::Error,
|
||||
"approved tool call failed",
|
||||
Some(failure.error_code().as_str()),
|
||||
Some(failure.stage()),
|
||||
Some(failure.error_code()),
|
||||
Some(failure.retryability()),
|
||||
Some(failure.outcome_certainty()),
|
||||
failure.upstream_status(),
|
||||
Value::Null,
|
||||
),
|
||||
};
|
||||
|
||||
persist_invocation(
|
||||
state,
|
||||
@@ -252,8 +288,12 @@ pub(super) async fn execute_approved_request(
|
||||
status: invocation_status,
|
||||
level: invocation_level,
|
||||
message,
|
||||
status_code: None,
|
||||
status_code: upstream_status,
|
||||
error_kind,
|
||||
execution_stage,
|
||||
execution_error_code,
|
||||
retryability,
|
||||
outcome_certainty,
|
||||
duration: started_at.elapsed(),
|
||||
request_preview,
|
||||
response_preview: response_payload.clone(),
|
||||
@@ -275,8 +315,47 @@ pub(super) async fn execute_approved_request(
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR.into_response())?
|
||||
.ok_or_else(|| StatusCode::CONFLICT.into_response())
|
||||
.map_err(|_| approved_completion_persistence_error(&correlation))?
|
||||
.ok_or_else(|| approved_completion_conflict_error(&correlation))
|
||||
}
|
||||
|
||||
fn approved_completion_conflict_error(correlation: &CorrelationContext) -> Response {
|
||||
(
|
||||
StatusCode::CONFLICT,
|
||||
axum::Json(json!({
|
||||
"error": {
|
||||
"code": "approval_state_conflict",
|
||||
"stage": "mandatory_persistence",
|
||||
"retryability": "manual_reconcile",
|
||||
"outcome_certainty": "outcome_unknown",
|
||||
"request_id": correlation.request_id().as_str(),
|
||||
"trace_id": correlation.trace_id().as_str(),
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
fn approved_completion_persistence_error(correlation: &CorrelationContext) -> Response {
|
||||
let failure = crank_core::ExecutionFailure::new(
|
||||
crank_core::ExecutionErrorCode::PersistenceUnavailable,
|
||||
correlation.clone(),
|
||||
)
|
||||
.with_dispatch_uncertainty();
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
axum::Json(json!({
|
||||
"error": {
|
||||
"code": failure.error_code().as_str(),
|
||||
"stage": failure.stage().as_str(),
|
||||
"retryability": failure.retryability().as_str(),
|
||||
"outcome_certainty": failure.outcome_certainty().as_str(),
|
||||
"request_id": failure.correlation().request_id().as_str(),
|
||||
"trace_id": failure.correlation().trace_id().as_str(),
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn finish_unavailable_approval(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crank_core::{ApprovalRequest, OperationApprovalPolicy};
|
||||
use crank_core::{ApprovalRequest, OperationApprovalPolicy, sanitize_invocation_preview};
|
||||
use crank_registry::PublishedAgentTool;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -30,9 +30,25 @@ pub(super) fn approval_required_response(
|
||||
"expires_at": approval.expires_at,
|
||||
"risk_level": approval.risk_level,
|
||||
"payload_preview": if policy.show_payload_preview {
|
||||
approval.request_payload.clone()
|
||||
sanitize_invocation_preview(&approval.request_payload)
|
||||
} else {
|
||||
Value::Null
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn approval_history_request_preview(approval: &ApprovalRequest) -> Value {
|
||||
json!({
|
||||
"status": "approval_required",
|
||||
"approval_id": approval.id.as_str(),
|
||||
"risk_level": approval.risk_level,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn approval_history_response_preview(response: &Value) -> Value {
|
||||
json!({
|
||||
"status": response.get("status").cloned().unwrap_or(Value::Null),
|
||||
"approval_id": response.get("approval_id").cloned().unwrap_or(Value::Null),
|
||||
"risk_level": response.get("risk_level").cloned().unwrap_or(Value::Null),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -108,14 +108,7 @@ impl PublishedToolCatalog {
|
||||
agent_slug: &str,
|
||||
) -> Result<(), RegistryError> {
|
||||
let key = CatalogKey::new(workspace_slug, agent_slug);
|
||||
let should_refresh = {
|
||||
let guard = self.cached.read().await;
|
||||
|
||||
match guard.get(&key).and_then(|entry| entry.loaded_at) {
|
||||
Some(loaded_at) => loaded_at.elapsed() >= self.refresh_interval,
|
||||
None => true,
|
||||
}
|
||||
};
|
||||
let should_refresh = true;
|
||||
|
||||
if !should_refresh {
|
||||
return Ok(());
|
||||
@@ -133,32 +126,11 @@ impl PublishedToolCatalog {
|
||||
}
|
||||
};
|
||||
let _refresh_guard = refresh_lock.lock().await;
|
||||
let still_stale = {
|
||||
let guard = self.cached.read().await;
|
||||
match guard.get(&key).and_then(|entry| entry.loaded_at) {
|
||||
Some(loaded_at) => loaded_at.elapsed() >= self.refresh_interval,
|
||||
None => true,
|
||||
}
|
||||
};
|
||||
let still_stale = true;
|
||||
if !still_stale {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some((catalog, age)) = self.load_shared_snapshot(workspace_slug, agent_slug).await {
|
||||
let metrics =
|
||||
log_catalog_analysis(workspace_slug, agent_slug, "shared_cache", &catalog.tools);
|
||||
self.store_local_catalog(
|
||||
key,
|
||||
CachedCatalog {
|
||||
loaded_at: Instant::now().checked_sub(age),
|
||||
catalog,
|
||||
metrics,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let db_span = DbOperation::CatalogLoad.span();
|
||||
let catalog_result = self
|
||||
.registry
|
||||
@@ -224,29 +196,6 @@ impl PublishedToolCatalog {
|
||||
previous_count
|
||||
}
|
||||
|
||||
async fn load_shared_snapshot(
|
||||
&self,
|
||||
workspace_slug: &str,
|
||||
agent_slug: &str,
|
||||
) -> Option<(PublishedAgentCatalog, Duration)> {
|
||||
if self.refresh_interval.is_zero() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let key = catalog_snapshot_key(workspace_slug, agent_slug);
|
||||
let value = match self
|
||||
.coordination_store
|
||||
.get_value(CacheScope::Coordination, &key)
|
||||
.await
|
||||
{
|
||||
Ok(value) => value?,
|
||||
Err(_) => return None,
|
||||
};
|
||||
let snapshot = serde_json::from_value::<CatalogSnapshot>(value.payload).ok()?;
|
||||
let age = Duration::from_millis(now_unix_ms().saturating_sub(snapshot.generated_at_ms));
|
||||
(age < self.refresh_interval).then_some((snapshot.catalog, age))
|
||||
}
|
||||
|
||||
async fn store_shared_snapshot(
|
||||
&self,
|
||||
workspace_slug: &str,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use crank_adapter_rest::RestAdapterError;
|
||||
use crank_runtime::RuntimeError;
|
||||
use crank_core::{
|
||||
CorrelationContext, ExecutionFailure, ExecutionLocale, RequestId, Retryability, TraceContext,
|
||||
};
|
||||
use crank_runtime::{RuntimeError, normalize_runtime_error};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
@@ -7,6 +9,9 @@ use serde_json::{Value, json};
|
||||
pub struct ToolErrorContract {
|
||||
pub code: &'static str,
|
||||
pub error_code: &'static str,
|
||||
pub stage: &'static str,
|
||||
pub retryability: &'static str,
|
||||
pub outcome_certainty: &'static str,
|
||||
pub message: String,
|
||||
pub recoverable: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -21,32 +26,95 @@ pub fn tool_error_contract_from_runtime(
|
||||
error: &RuntimeError,
|
||||
request_id: &str,
|
||||
trace_id: &str,
|
||||
locale: ExecutionLocale,
|
||||
) -> ToolErrorContract {
|
||||
let error_code = runtime_error_code(error);
|
||||
let correlation = correlation_from_safe_ids(request_id, trace_id);
|
||||
project_failure(&normalize_runtime_error(error, &correlation), locale)
|
||||
}
|
||||
|
||||
pub fn tool_error_contract_from_failure(
|
||||
failure: &ExecutionFailure,
|
||||
locale: ExecutionLocale,
|
||||
) -> ToolErrorContract {
|
||||
project_failure(failure, locale)
|
||||
}
|
||||
|
||||
fn project_failure(failure: &ExecutionFailure, locale: ExecutionLocale) -> ToolErrorContract {
|
||||
let code = failure.error_code();
|
||||
let retryability = failure.retryability();
|
||||
let message = if let Some(challenge) = failure.confirmation() {
|
||||
confirmation_message(
|
||||
code.message(locale),
|
||||
challenge.token(),
|
||||
challenge.expires_in_ms(),
|
||||
locale,
|
||||
)
|
||||
} else {
|
||||
code.message(locale).to_owned()
|
||||
};
|
||||
ToolErrorContract {
|
||||
code: error_code,
|
||||
error_code,
|
||||
message: safe_runtime_error_message(error),
|
||||
recoverable: is_recoverable_runtime_error(error),
|
||||
suggested_action: suggested_action(error),
|
||||
upstream_status: upstream_status(error),
|
||||
request_id: request_id.to_owned(),
|
||||
trace_id: trace_id.to_owned(),
|
||||
code: code.as_str(),
|
||||
error_code: code.as_str(),
|
||||
stage: failure.stage().as_str(),
|
||||
retryability: retryability.as_str(),
|
||||
outcome_certainty: failure.outcome_certainty().as_str(),
|
||||
message,
|
||||
recoverable: matches!(
|
||||
retryability,
|
||||
Retryability::Safe | Retryability::AfterDelay | Retryability::RequiresConfirmation
|
||||
),
|
||||
suggested_action: suggested_action(retryability, locale),
|
||||
upstream_status: failure.upstream_status(),
|
||||
request_id: failure.correlation().request_id().to_string(),
|
||||
trace_id: failure.correlation().trace_id().to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn generic_tool_error_contract(
|
||||
error_code: &'static str,
|
||||
message: impl Into<String>,
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum McpControlError {
|
||||
CatalogRevisionChanged,
|
||||
ApprovalElicitationNotSupported,
|
||||
}
|
||||
|
||||
pub fn control_tool_error_contract(
|
||||
error: McpControlError,
|
||||
request_id: &str,
|
||||
trace_id: &str,
|
||||
recoverable: bool,
|
||||
suggested_action: Option<&'static str>,
|
||||
locale: ExecutionLocale,
|
||||
) -> ToolErrorContract {
|
||||
let (error_code, message, recoverable, suggested_action) = match (error, locale) {
|
||||
(McpControlError::CatalogRevisionChanged, ExecutionLocale::Ru) => (
|
||||
"agent_catalog_result_stale",
|
||||
"Версия каталога изменилась.",
|
||||
true,
|
||||
Some("Повторите search_tools и вызовите инструмент с новой версией каталога."),
|
||||
),
|
||||
(McpControlError::CatalogRevisionChanged, ExecutionLocale::En) => (
|
||||
"agent_catalog_result_stale",
|
||||
"The catalog revision changed.",
|
||||
true,
|
||||
Some("Run search_tools again and call the tool with the new catalog revision."),
|
||||
),
|
||||
(McpControlError::ApprovalElicitationNotSupported, ExecutionLocale::Ru) => (
|
||||
"approval_elicitation_not_supported",
|
||||
"Клиент не поддерживает MCP Elicitation, обязательный для этой операции.",
|
||||
false,
|
||||
Some("Используйте Custom MCP Approval или клиент с поддержкой elicitation."),
|
||||
),
|
||||
(McpControlError::ApprovalElicitationNotSupported, ExecutionLocale::En) => (
|
||||
"approval_elicitation_not_supported",
|
||||
"The client does not support MCP Elicitation required by this operation.",
|
||||
false,
|
||||
Some("Use Custom MCP Approval or an elicitation-capable client."),
|
||||
),
|
||||
};
|
||||
ToolErrorContract {
|
||||
code: error_code,
|
||||
error_code,
|
||||
message: message.into(),
|
||||
stage: "authorization",
|
||||
retryability: if recoverable { "after_delay" } else { "never" },
|
||||
outcome_certainty: "certain",
|
||||
message: message.to_owned(),
|
||||
recoverable,
|
||||
suggested_action,
|
||||
upstream_status: None,
|
||||
@@ -55,6 +123,19 @@ pub fn generic_tool_error_contract(
|
||||
}
|
||||
}
|
||||
|
||||
pub fn execution_locale(message: &Value) -> ExecutionLocale {
|
||||
let locale = message
|
||||
.pointer("/params/_meta/locale")
|
||||
.or_else(|| message.pointer("/params/locale"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("en");
|
||||
if locale.eq_ignore_ascii_case("ru") || locale.to_ascii_lowercase().starts_with("ru-") {
|
||||
ExecutionLocale::Ru
|
||||
} else {
|
||||
ExecutionLocale::En
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tool_error_text(error: &ToolErrorContract) -> String {
|
||||
match error.suggested_action {
|
||||
Some(action) => format!("{} {}", error.message, action),
|
||||
@@ -65,9 +146,12 @@ pub fn tool_error_text(error: &ToolErrorContract) -> String {
|
||||
pub fn tool_error_value(error: &ToolErrorContract) -> Value {
|
||||
serde_json::to_value(error).unwrap_or_else(|_| {
|
||||
json!({
|
||||
"code": "runtime_error",
|
||||
"error_code": "runtime_error",
|
||||
"message": "Не удалось выполнить инструмент.",
|
||||
"code": "runtime_internal",
|
||||
"error_code": "runtime_internal",
|
||||
"stage": "runtime",
|
||||
"retryability": "never",
|
||||
"outcome_certainty": "certain",
|
||||
"message": "Внутренняя ошибка выполнения.",
|
||||
"recoverable": false,
|
||||
"request_id": error.request_id,
|
||||
"trace_id": error.trace_id
|
||||
@@ -75,178 +159,54 @@ pub fn tool_error_value(error: &ToolErrorContract) -> Value {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn runtime_error_code(error: &RuntimeError) -> &'static str {
|
||||
match error {
|
||||
RuntimeError::Schema(_) => "schema_validation_error",
|
||||
RuntimeError::Mapping(_) => "mapping_error",
|
||||
RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus { status, .. }) => {
|
||||
upstream_status_code(*status)
|
||||
}
|
||||
RuntimeError::RestAdapter(RestAdapterError::Transport(_)) => "upstream_transport_error",
|
||||
RuntimeError::RestAdapter(_) => "adapter_execution_error",
|
||||
RuntimeError::ProtocolAdapter(_) => "adapter_execution_error",
|
||||
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => "runtime_overloaded",
|
||||
RuntimeError::UnsupportedExecutionMode { .. } => "streaming_mode_error",
|
||||
RuntimeError::InvalidPreparedRequest { .. } => "runtime_error",
|
||||
RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
|
||||
RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
|
||||
RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_unavailable",
|
||||
RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_unavailable",
|
||||
RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
|
||||
RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
|
||||
RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
|
||||
RuntimeError::MissingAuthProfile { .. } => "auth_profile_not_found",
|
||||
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
|
||||
"secret_not_found"
|
||||
}
|
||||
RuntimeError::InvalidAuthSecretValue { .. } => "secret_value_error",
|
||||
RuntimeError::SecretCrypto { .. } => "secret_crypto_error",
|
||||
}
|
||||
fn correlation_from_safe_ids(request_id: &str, trace_id: &str) -> CorrelationContext {
|
||||
let request_id = RequestId::parse(request_id).unwrap_or_else(|_| RequestId::generate());
|
||||
let trace_context = TraceContext::from_span_parts(trace_id, "0000000000000001", false)
|
||||
.unwrap_or_else(|_| TraceContext::generate());
|
||||
CorrelationContext::new(request_id, trace_context)
|
||||
}
|
||||
|
||||
fn upstream_status_code(status: u16) -> &'static str {
|
||||
match status {
|
||||
401 | 403 => "upstream_auth_error",
|
||||
404 => "upstream_not_found",
|
||||
408 | 429 => "upstream_rate_limited",
|
||||
500..=599 => "upstream_server_error",
|
||||
_ => "upstream_status_error",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn safe_runtime_error_message(error: &RuntimeError) -> String {
|
||||
match error {
|
||||
RuntimeError::Schema(_) => "Входные параметры не прошли проверку схемы.".to_owned(),
|
||||
RuntimeError::Mapping(_) => {
|
||||
"Не удалось сопоставить параметры инструмента с API-запросом.".to_owned()
|
||||
}
|
||||
RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus { status, .. }) => {
|
||||
format!("Внешний API вернул HTTP {}.", status)
|
||||
}
|
||||
RuntimeError::RestAdapter(RestAdapterError::Transport(_)) => {
|
||||
"Не удалось подключиться к внешнему API.".to_owned()
|
||||
}
|
||||
RuntimeError::RestAdapter(_) => "Не удалось выполнить запрос к внешнему API.".to_owned(),
|
||||
RuntimeError::ProtocolAdapter(_) => "Не удалось выполнить протокольный адаптер.".to_owned(),
|
||||
RuntimeError::UnsupportedProtocol { .. } => {
|
||||
"Протокол операции не поддерживается.".to_owned()
|
||||
}
|
||||
RuntimeError::ConcurrencyLimitExceeded { .. } => {
|
||||
"Сервис временно перегружен и не может выполнить инструмент.".to_owned()
|
||||
}
|
||||
RuntimeError::UnsupportedExecutionMode { .. } => {
|
||||
"Режим выполнения операции не поддерживается.".to_owned()
|
||||
}
|
||||
RuntimeError::InvalidPreparedRequest { .. } => {
|
||||
"Не удалось подготовить корректный API-запрос.".to_owned()
|
||||
}
|
||||
RuntimeError::ConfirmationRequired {
|
||||
confirmation_token,
|
||||
expires_in_ms,
|
||||
..
|
||||
} => format!(
|
||||
"Операция требует подтверждения. Повторите вызов с _crank_confirmation_token=\"{}\" в течение {} секунд.",
|
||||
confirmation_token,
|
||||
fn confirmation_message(
|
||||
base_message: &str,
|
||||
token: &str,
|
||||
expires_in_ms: u64,
|
||||
locale: ExecutionLocale,
|
||||
) -> String {
|
||||
match locale {
|
||||
ExecutionLocale::Ru => format!(
|
||||
"{base_message} Повторите вызов с _crank_confirmation_token=\"{token}\" в течение {} секунд.",
|
||||
expires_in_ms / 1000
|
||||
),
|
||||
ExecutionLocale::En => format!(
|
||||
"{base_message} Repeat the call with _crank_confirmation_token=\"{token}\" within {} seconds.",
|
||||
expires_in_ms / 1000
|
||||
),
|
||||
RuntimeError::InvalidConfirmationToken { .. } => {
|
||||
"Токен подтверждения недействителен, истек или уже был использован.".to_owned()
|
||||
}
|
||||
RuntimeError::ConfirmationStoreUnavailable { .. } => {
|
||||
"Хранилище подтверждений временно недоступно.".to_owned()
|
||||
}
|
||||
RuntimeError::IdempotencyStoreUnavailable { .. } => {
|
||||
"Хранилище идемпотентности временно недоступно.".to_owned()
|
||||
}
|
||||
RuntimeError::IdempotencyInProgress { .. } => {
|
||||
"Операция с этим ключом идемпотентности уже выполняется.".to_owned()
|
||||
}
|
||||
RuntimeError::IdempotencyConflict { .. } => {
|
||||
"Ключ идемпотентности уже использован с другими параметрами.".to_owned()
|
||||
}
|
||||
RuntimeError::IdempotencyOutcomeUnknown { .. } => {
|
||||
"Результат предыдущего выполнения неизвестен; автоматический повтор заблокирован."
|
||||
.to_owned()
|
||||
}
|
||||
RuntimeError::MissingAuthProfile { .. } => "Профиль авторизации не найден.".to_owned(),
|
||||
RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } => {
|
||||
"Секрет авторизации не найден.".to_owned()
|
||||
}
|
||||
RuntimeError::InvalidAuthSecretValue { .. } => {
|
||||
"Секрет авторизации имеет неподходящий формат.".to_owned()
|
||||
}
|
||||
RuntimeError::SecretCrypto { .. } => {
|
||||
"Не удалось расшифровать секрет авторизации.".to_owned()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_recoverable_runtime_error(error: &RuntimeError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus {
|
||||
status: 408 | 429 | 500..=599,
|
||||
..
|
||||
}) | RuntimeError::RestAdapter(RestAdapterError::Transport(_))
|
||||
| RuntimeError::ConcurrencyLimitExceeded { .. }
|
||||
| RuntimeError::SecretCrypto { .. }
|
||||
| RuntimeError::ConfirmationRequired { .. }
|
||||
| RuntimeError::IdempotencyStoreUnavailable { .. }
|
||||
| RuntimeError::IdempotencyInProgress { .. }
|
||||
)
|
||||
}
|
||||
|
||||
fn suggested_action(error: &RuntimeError) -> Option<&'static str> {
|
||||
match error {
|
||||
RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus {
|
||||
status: 401 | 403, ..
|
||||
}) => Some("Проверьте настройки авторизации внешнего API."),
|
||||
RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus { status: 404, .. }) => {
|
||||
Some("Проверьте путь endpoint-а и параметры запроса.")
|
||||
const fn suggested_action(
|
||||
retryability: Retryability,
|
||||
locale: ExecutionLocale,
|
||||
) -> Option<&'static str> {
|
||||
match (retryability, locale) {
|
||||
(Retryability::AfterDelay | Retryability::Safe, ExecutionLocale::Ru) => {
|
||||
Some("Повторите запрос позже.")
|
||||
}
|
||||
RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus {
|
||||
status: 408 | 429 | 500..=599,
|
||||
..
|
||||
})
|
||||
| RuntimeError::RestAdapter(RestAdapterError::Transport(_))
|
||||
| RuntimeError::ConcurrencyLimitExceeded { .. } => Some("Повторите запрос позже."),
|
||||
RuntimeError::Schema(_)
|
||||
| RuntimeError::Mapping(_)
|
||||
| RuntimeError::InvalidPreparedRequest { .. } => {
|
||||
Some("Проверьте параметры вызова инструмента.")
|
||||
(Retryability::AfterDelay | Retryability::Safe, ExecutionLocale::En) => {
|
||||
Some("Retry the request later.")
|
||||
}
|
||||
RuntimeError::ConfirmationRequired { .. } => {
|
||||
Some("Повторите вызов с указанным токеном подтверждения.")
|
||||
}
|
||||
RuntimeError::InvalidConfirmationToken { .. } => {
|
||||
Some("Запросите новый токен подтверждения.")
|
||||
}
|
||||
RuntimeError::ConfirmationStoreUnavailable { .. } => Some("Повторите запрос позже."),
|
||||
RuntimeError::IdempotencyStoreUnavailable { .. }
|
||||
| RuntimeError::IdempotencyInProgress { .. } => Some("Повторите запрос позже."),
|
||||
RuntimeError::IdempotencyConflict { .. } => {
|
||||
Some("Используйте новый ключ идемпотентности для изменённого запроса.")
|
||||
}
|
||||
RuntimeError::IdempotencyOutcomeUnknown { .. } => {
|
||||
(Retryability::ManualReconcile, ExecutionLocale::Ru) => {
|
||||
Some("Проверьте результат во внешней системе перед ручным повтором.")
|
||||
}
|
||||
RuntimeError::MissingAuthProfile { .. }
|
||||
| RuntimeError::MissingSecret { .. }
|
||||
| RuntimeError::MissingSecretVersion { .. }
|
||||
| RuntimeError::InvalidAuthSecretValue { .. }
|
||||
| RuntimeError::SecretCrypto { .. } => {
|
||||
Some("Проверьте настройки авторизации и секретов в Crank.")
|
||||
(Retryability::ManualReconcile, ExecutionLocale::En) => {
|
||||
Some("Check the result in the external system before retrying manually.")
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn upstream_status(error: &RuntimeError) -> Option<u16> {
|
||||
match error {
|
||||
RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus { status, .. }) => {
|
||||
Some(*status)
|
||||
}
|
||||
_ => None,
|
||||
(Retryability::RequiresConfirmation, ExecutionLocale::Ru) => {
|
||||
Some("Повторите вызов с указанным токеном подтверждения.")
|
||||
}
|
||||
(Retryability::RequiresConfirmation, ExecutionLocale::En) => {
|
||||
Some("Repeat the call with the provided confirmation token.")
|
||||
}
|
||||
(Retryability::Never, _) => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::{
|
||||
jsonrpc::{jsonrpc_error, jsonrpc_result, request_id},
|
||||
manifest::{CALL_TOOL_NAME, SEARCH_TOOLS_NAME, searchable_tools},
|
||||
session::SessionState,
|
||||
tool_error::generic_tool_error_contract,
|
||||
tool_error::{McpControlError, control_tool_error_contract, execution_locale},
|
||||
transport::{ResponseMode, transport_response},
|
||||
};
|
||||
|
||||
@@ -84,18 +84,11 @@ pub(super) async fn handle_catalog_tool_call(
|
||||
message,
|
||||
response_mode,
|
||||
&session.protocol_version,
|
||||
generic_tool_error_contract(
|
||||
"catalog_revision_changed",
|
||||
format!(
|
||||
"catalog revision {} is no longer current",
|
||||
proxy.catalog_revision
|
||||
),
|
||||
control_tool_error_contract(
|
||||
McpControlError::CatalogRevisionChanged,
|
||||
transport_request_id,
|
||||
transport_correlation.trace_id().as_str(),
|
||||
true,
|
||||
Some(
|
||||
"Повторите search_tools и вызовите инструмент с новой версией каталога.",
|
||||
),
|
||||
execution_locale(message),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -253,7 +246,7 @@ fn handle_search_tools(
|
||||
}
|
||||
|
||||
fn catalog_revision(catalog: &PublishedAgentCatalog) -> String {
|
||||
format!("agent-version-{}", catalog.agent_version)
|
||||
catalog.catalog_revision.clone()
|
||||
}
|
||||
|
||||
fn invalid_arguments_response(
|
||||
|
||||
@@ -1,12 +1,16 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Instant;
|
||||
|
||||
use crank_community_mcp::manifest::{analyze_published_tool_catalog, tool_definitions};
|
||||
use crank_community_mcp::manifest::{
|
||||
analyze_published_tool_catalog, catalog_tool_definitions, tool_definitions,
|
||||
};
|
||||
use crank_core::{
|
||||
ExecutionConfig, HttpMethod, Operation, OperationId, OperationSecurityLevel, OperationStatus,
|
||||
Protocol, RestTarget, Target, ToolDescription, WorkspaceId,
|
||||
Protocol, RestTarget, Target, ToolAccessMode, ToolDescription, ToolSelectionPolicy,
|
||||
WorkspaceId,
|
||||
};
|
||||
use crank_mapping::MappingSet;
|
||||
use crank_registry::{PublishedAgentTool, RegistryOperation};
|
||||
use crank_registry::{PublishedAgentCatalog, PublishedAgentTool, RegistryOperation};
|
||||
use crank_schema::{Schema, SchemaKind};
|
||||
use serde_json::json;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
@@ -71,6 +75,57 @@ fn catalog_budget_uses_the_same_definitions_as_tools_list() {
|
||||
assert!(!analysis.budget.exceeds_recommended_budget);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_catalog_discovery_profile_handles_200_agents_2000_operations_under_budget() {
|
||||
const AGENT_COUNT: usize = 200;
|
||||
const TOOLS_PER_AGENT: usize = 10;
|
||||
const P95_BUDGET_MS: u128 = 150;
|
||||
|
||||
let mut durations = Vec::with_capacity(AGENT_COUNT);
|
||||
let mut total_definitions = 0usize;
|
||||
let started = Instant::now();
|
||||
|
||||
for agent_index in 0..AGENT_COUNT {
|
||||
let tools = (0..TOOLS_PER_AGENT)
|
||||
.map(|tool_index| {
|
||||
let mut tool = published_tool();
|
||||
tool.agent_id = format!("agent_{agent_index:03}").into();
|
||||
tool.agent_slug = format!("agent-{agent_index:03}");
|
||||
tool.operation.id =
|
||||
OperationId::new(format!("op_{agent_index:03}_{tool_index:03}"));
|
||||
tool.tool_name = format!("tool_{agent_index:03}_{tool_index:03}");
|
||||
tool.tool_title = format!("Tool {agent_index:03}-{tool_index:03}");
|
||||
tool
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let catalog = PublishedAgentCatalog {
|
||||
agent_version: 1,
|
||||
catalog_revision: format!("agent-{agent_index:03}-revision-1"),
|
||||
tool_selection_policy: ToolSelectionPolicy {
|
||||
mode: ToolAccessMode::Direct,
|
||||
..Default::default()
|
||||
},
|
||||
tools,
|
||||
};
|
||||
|
||||
let catalog_started = Instant::now();
|
||||
let definitions = catalog_tool_definitions(&catalog);
|
||||
durations.push(catalog_started.elapsed());
|
||||
total_definitions += definitions.len();
|
||||
}
|
||||
|
||||
durations.sort_unstable();
|
||||
let p95 = durations[((durations.len() * 95).div_ceil(100)).saturating_sub(1)];
|
||||
|
||||
assert_eq!(total_definitions, AGENT_COUNT * TOOLS_PER_AGENT);
|
||||
assert!(
|
||||
p95.as_millis() <= P95_BUDGET_MS,
|
||||
"agent catalog discovery p95={}ms total={}ms",
|
||||
p95.as_millis(),
|
||||
started.elapsed().as_millis()
|
||||
);
|
||||
}
|
||||
|
||||
fn published_tool() -> PublishedAgentTool {
|
||||
PublishedAgentTool {
|
||||
workspace_id: WorkspaceId::new("ws_01"),
|
||||
|
||||
@@ -1,23 +1,28 @@
|
||||
use crank_adapter_rest::RestAdapterError;
|
||||
use crank_community_mcp::tool_error::tool_error_contract_from_runtime;
|
||||
use crank_community_mcp::tool_error::{
|
||||
tool_error_contract_from_failure, tool_error_contract_from_runtime,
|
||||
};
|
||||
use crank_core::{
|
||||
CorrelationContext, ExecutionErrorCode, ExecutionFailure, ExecutionLocale,
|
||||
ProtocolAdapterError, Retryability,
|
||||
};
|
||||
use crank_runtime::RuntimeError;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn maps_upstream_429_to_recoverable_structured_tool_error() {
|
||||
let contract = tool_error_contract_from_runtime(
|
||||
&RuntimeError::RestAdapter(RestAdapterError::UnexpectedStatus {
|
||||
&RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
||||
status: 429,
|
||||
body: json!({
|
||||
"error": "rate limit exceeded",
|
||||
"internal_trace": "do not leak"
|
||||
}),
|
||||
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
||||
}),
|
||||
"req-429",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
ExecutionLocale::Ru,
|
||||
);
|
||||
|
||||
assert_eq!(contract.error_code, "upstream_rate_limited");
|
||||
assert_eq!(contract.stage, "upstream");
|
||||
assert_eq!(contract.retryability, "after_delay");
|
||||
assert_eq!(contract.outcome_certainty, "certain");
|
||||
assert!(contract.recoverable);
|
||||
assert_eq!(contract.upstream_status, Some(429));
|
||||
assert_eq!(contract.request_id, "req-429");
|
||||
@@ -26,6 +31,22 @@ fn maps_upstream_429_to_recoverable_structured_tool_error() {
|
||||
assert!(!contract.message.contains("internal_trace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_upstream_429_to_english_suggested_action() {
|
||||
let contract = tool_error_contract_from_runtime(
|
||||
&RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
||||
status: 429,
|
||||
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
||||
}),
|
||||
"req-429",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
ExecutionLocale::En,
|
||||
);
|
||||
|
||||
assert_eq!(contract.error_code, "upstream_rate_limited");
|
||||
assert_eq!(contract.suggested_action, Some("Retry the request later."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_mapping_error_to_non_recoverable_structured_tool_error() {
|
||||
let contract = tool_error_contract_from_runtime(
|
||||
@@ -35,15 +56,91 @@ fn maps_mapping_error_to_non_recoverable_structured_tool_error() {
|
||||
},
|
||||
"req-map",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
ExecutionLocale::Ru,
|
||||
);
|
||||
|
||||
assert_eq!(contract.error_code, "runtime_error");
|
||||
assert_eq!(contract.error_code, "prepared_request_invalid");
|
||||
assert_eq!(contract.stage, "request_preparation");
|
||||
assert_eq!(contract.retryability, "never");
|
||||
assert!(!contract.recoverable);
|
||||
assert_eq!(contract.upstream_status, None);
|
||||
assert_eq!(contract.request_id, "req-map");
|
||||
assert_eq!(contract.trace_id, "0af7651916cd43dd8448eb211c80319c");
|
||||
assert_eq!(contract.suggested_action, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguous_timeout_requires_manual_reconciliation() {
|
||||
let contract = tool_error_contract_from_runtime(
|
||||
&RuntimeError::ExecutionDeadlineElapsed {
|
||||
may_have_dispatched: true,
|
||||
},
|
||||
"req-timeout",
|
||||
"0af7651916cd43dd8448eb211c80319c",
|
||||
ExecutionLocale::Ru,
|
||||
);
|
||||
assert_eq!(contract.error_code, "upstream_timeout");
|
||||
assert_eq!(contract.retryability, "manual_reconcile");
|
||||
assert_eq!(contract.outcome_certainty, "outcome_unknown");
|
||||
assert!(!contract.recoverable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_execution_code_projects_the_same_canonical_semantics_as_admin() {
|
||||
for code in ExecutionErrorCode::ALL {
|
||||
let contract = tool_error_contract_from_failure(
|
||||
&ExecutionFailure::new(code, CorrelationContext::generate()),
|
||||
ExecutionLocale::En,
|
||||
);
|
||||
assert_eq!(contract.error_code, code.as_str());
|
||||
assert_eq!(contract.stage, code.stage().as_str());
|
||||
assert_eq!(contract.retryability, code.retryability().as_str());
|
||||
assert_eq!(
|
||||
contract.outcome_certainty,
|
||||
code.outcome_certainty().as_str()
|
||||
);
|
||||
assert_eq!(
|
||||
contract.recoverable,
|
||||
matches!(
|
||||
code.retryability(),
|
||||
Retryability::Safe | Retryability::AfterDelay | Retryability::RequiresConfirmation
|
||||
)
|
||||
);
|
||||
assert!(!contract.message.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confirmation_challenge_is_localized_for_mcp_clients() {
|
||||
let failure = ExecutionFailure::new(
|
||||
ExecutionErrorCode::ConfirmationRequired,
|
||||
CorrelationContext::generate(),
|
||||
)
|
||||
.try_with_confirmation("ct_safe", 30_000)
|
||||
.expect("valid confirmation challenge");
|
||||
|
||||
let en = tool_error_contract_from_failure(&failure, ExecutionLocale::En);
|
||||
assert!(
|
||||
en.message.contains(
|
||||
"Repeat the call with _crank_confirmation_token=\"ct_safe\" within 30 seconds."
|
||||
),
|
||||
"{:?}",
|
||||
en.message
|
||||
);
|
||||
assert_eq!(
|
||||
contract.suggested_action,
|
||||
Some("Проверьте параметры вызова инструмента.")
|
||||
en.suggested_action,
|
||||
Some("Repeat the call with the provided confirmation token.")
|
||||
);
|
||||
|
||||
let ru = tool_error_contract_from_failure(&failure, ExecutionLocale::Ru);
|
||||
assert!(
|
||||
ru.message
|
||||
.contains("Повторите вызов с _crank_confirmation_token=\"ct_safe\""),
|
||||
"{:?}",
|
||||
ru.message
|
||||
);
|
||||
assert_eq!(
|
||||
ru.suggested_action,
|
||||
Some("Повторите вызов с указанным токеном подтверждения.")
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user