use std::sync::Arc; use std::time::Instant; use crank_core::{ AdapterRegistry, CoordinationStateStore, ExecutionMode, InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter, }; use crank_metrics::{ CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard, InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics, ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome, record_limit_rejection, }; use crank_trace::{ ErrorCategory, Stage, StageOutcome, set_parent_from_trace_context, trace_context_for_span, }; use serde_json::{Map, Value, json}; use time::OffsetDateTime; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tracing::{Instrument, Span, debug, warn}; use uuid::Uuid; use crate::{ AdapterResponse, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeLimits, RuntimeOperation, RuntimeRequestContext, auth::apply_resolved_auth, request_preparation::adapter_prepared_request, response_cache::{ adapter_response_from_cached, cached_response_from_adapter, response_cache_key, response_cache_ttl, }, }; #[derive(Clone)] pub struct RuntimeExecutor { adapters: AdapterRegistry, limits: RuntimeLimits, unary_limiter: Arc, response_cache: Option>, coordination_store: Option>, metering_sink: SharedMeteringSink, } #[derive(Clone, Copy)] pub struct RuntimeExecutionRequest<'a> { pub operation: &'a RuntimeOperation, pub input: &'a Value, pub resolved_auth: Option<&'a ResolvedAuth>, pub request_context: Option<&'a RuntimeRequestContext>, } struct IdempotencyCancellationGuard { cleanup: Option, runtime: tokio::runtime::Handle, } struct IdempotencyCancellationCleanup { store: Arc, operation: RuntimeOperation, reservation: crate::idempotency::IdempotencyReservation, } impl IdempotencyCancellationGuard { fn new( store: Arc, operation: RuntimeOperation, reservation: crate::idempotency::IdempotencyReservation, ) -> Self { Self { cleanup: Some(IdempotencyCancellationCleanup { store, operation, reservation, }), runtime: tokio::runtime::Handle::current(), } } fn disarm(&mut self) { self.cleanup = None; } } impl Drop for IdempotencyCancellationGuard { fn drop(&mut self) { let Some(cleanup) = self.cleanup.take() else { return; }; self.runtime.spawn(async move { let result = crate::idempotency::mark_outcome_unknown( cleanup.store.as_ref(), &cleanup.operation, &cleanup.reservation, ) .await; record_idempotency_outcome(match &result { Ok(()) => IdempotencyOutcome::OutcomeUnknown, Err(error) => idempotency_error_outcome(error), }); if result.is_err() { warn!( name: "runtime.idempotency.cancellation_cleanup_failed", error_category = "idempotency_store", "failed to finalize cancelled idempotent execution" ); } }); } } impl<'a> RuntimeExecutionRequest<'a> { pub fn new(operation: &'a RuntimeOperation, input: &'a Value) -> Self { Self { operation, input, resolved_auth: None, request_context: None, } } pub fn with_auth(mut self, resolved_auth: &'a ResolvedAuth) -> Self { self.resolved_auth = Some(resolved_auth); self } pub fn with_optional_auth(mut self, resolved_auth: Option<&'a ResolvedAuth>) -> Self { self.resolved_auth = resolved_auth; self } pub fn with_context(mut self, request_context: &'a RuntimeRequestContext) -> Self { self.request_context = Some(request_context); self } pub fn with_optional_context( mut self, request_context: Option<&'a RuntimeRequestContext>, ) -> Self { self.request_context = request_context; self } } impl Default for RuntimeExecutor { fn default() -> Self { Self::new() } } impl RuntimeExecutor { pub fn new() -> Self { crate::executor_builder::community_default().build() } pub fn with_limits(limits: RuntimeLimits) -> Self { crate::executor_builder::community_default() .with_limits(limits) .build() } pub(crate) fn from_builder_parts( limits: RuntimeLimits, adapters: AdapterRegistry, response_cache: Option>, coordination_store: Option>, metering_sink: SharedMeteringSink, ) -> Self { Self { adapters, unary_limiter: Arc::new(Semaphore::new(limits.max_concurrent_unary)), limits, response_cache, coordination_store, metering_sink, } } pub fn with_response_cache_store( mut self, response_cache: Arc, ) -> Self { self.response_cache = Some(response_cache); self } pub fn with_coordination_store( mut self, coordination_store: Arc, ) -> Self { self.coordination_store = Some(coordination_store); self } pub async fn execute( &self, operation: &RuntimeOperation, input: &Value, ) -> Result { self.execute_request(RuntimeExecutionRequest::new(operation, input)) .await } pub async fn execute_with_auth( &self, operation: &RuntimeOperation, input: &Value, resolved_auth: Option<&ResolvedAuth>, ) -> Result { self.execute_request( RuntimeExecutionRequest::new(operation, input).with_optional_auth(resolved_auth), ) .await } pub async fn execute_with_context( &self, operation: &RuntimeOperation, input: &Value, request_context: Option<&RuntimeRequestContext>, ) -> Result { self.execute_request( RuntimeExecutionRequest::new(operation, input).with_optional_context(request_context), ) .await } pub async fn execute_with_auth_and_context( &self, operation: &RuntimeOperation, input: &Value, resolved_auth: Option<&ResolvedAuth>, request_context: Option<&RuntimeRequestContext>, ) -> Result { self.execute_request( RuntimeExecutionRequest::new(operation, input) .with_optional_auth(resolved_auth) .with_optional_context(request_context), ) .await } pub async fn execute_request( &self, request: RuntimeExecutionRequest<'_>, ) -> Result { let runtime_span = Stage::RuntimeExecute.span(); if let Some(context) = request.request_context { set_parent_from_trace_context(&runtime_span, &context.trace_context); } let generated_context = request.request_context.is_none().then(|| { RuntimeRequestContext::new( crank_core::RequestId::generate(), trace_context_for_span(&runtime_span) .unwrap_or_else(crank_core::TraceContext::generate), ) }); let request_context = request.request_context.or(generated_context.as_ref()); log_runtime_event("unary.execute", request.operation, request_context); let started_at = Instant::now(); let invocation_metrics = ToolInvocationMetrics::start_with_exemplar( metric_invocation_source(request_context), request_context.and_then(|context| { context .trace_context .is_sampled() .then(|| { crank_metrics::ExemplarTraceId::parse( &context.trace_context.trace_id().to_string(), ) }) .flatten() }), ); let result = async { let _permit = self.acquire_unary_permit(request.operation)?; let _inflight = InFlightGuard::runtime(); let mapping_span = Stage::RuntimeArgumentsMap.span(); let prepared_request = mapping_span.in_scope(|| self.prepare_request(request.operation, request.input)); record_runtime_result(&mapping_span, &prepared_request); drop(mapping_span); let prepared_request = prepared_request?; let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth); self.execute_prepared( request.operation, request.input, prepared_request, request_context, ) .await } .instrument(runtime_span.clone()) .await; record_runtime_result(&runtime_span, &result); drop(runtime_span); let (outcome, error_kind) = match &result { Ok(_) => (ToolOutcome::Success, ToolErrorKind::None), Err(error) => (ToolOutcome::Error, runtime_error_kind(error)), }; invocation_metrics.complete(outcome, error_kind); self.record_metering(request.operation, request_context, &result, started_at) .await; result } pub fn prepare_request( &self, operation: &RuntimeOperation, input: &Value, ) -> Result { operation.input_schema.validate_shape(input)?; if operation.input_mapping.is_empty() { return Ok(PreparedRequest::default()); } let mapped_input = operation.input_mapping.apply(&json!({ "mcp": input }))?; PreparedRequest::from_mapping_output(&mapped_input) } async fn execute_prepared( &self, operation: &RuntimeOperation, input: &Value, prepared_request: PreparedRequest, request_context: Option<&RuntimeRequestContext>, ) -> Result { let mut prepared_request = prepared_request; let idempotency_applicable = crate::idempotency::policy(operation).is_some(); let idempotency_key = match crate::idempotency::prepare_idempotency( operation, input, &mut prepared_request, ) { Ok(key) => key, Err(error) if idempotency_applicable => { record_idempotency_outcome(idempotency_error_outcome(&error)); let span = Stage::RuntimeIdempotency.span(); StageOutcome::Error.record(&span); ErrorCategory::Idempotency.record(&span); return Err(error); } Err(error) => return Err(error), }; if crate::confirmation::is_applicable(operation) { let approval_span = Stage::ApprovalCheck.span(); let approval_result = crate::confirmation::confirm_operation( self.coordination_store.as_deref(), operation, input, request_context, ) .instrument(approval_span.clone()) .await; match &approval_result { Ok(()) => { StageOutcome::Success.record(&approval_span); record_confirmation_outcome(ConfirmationOutcome::Approved); } Err(RuntimeError::ConfirmationRequired { .. }) => { StageOutcome::Required.record(&approval_span); ErrorCategory::Approval.record(&approval_span); record_confirmation_outcome(ConfirmationOutcome::Required); } Err(error) => { StageOutcome::Error.record(&approval_span); runtime_error_category(error).record(&approval_span); record_confirmation_outcome(confirmation_error_outcome(error)); } } drop(approval_span); approval_result?; } let idempotency = if idempotency_applicable { let idempotency_span = Stage::RuntimeIdempotency.span(); let result = crate::idempotency::begin( self.coordination_store.as_deref(), operation, input, idempotency_key.as_deref(), request_context, ) .instrument(idempotency_span.clone()) .await; match &result { Ok(crate::idempotency::IdempotencyAction::Execute(_)) => { StageOutcome::Execute.record(&idempotency_span); record_idempotency_outcome(IdempotencyOutcome::Execute); } Ok(crate::idempotency::IdempotencyAction::Replay(_)) => { StageOutcome::Replay.record(&idempotency_span); record_idempotency_outcome(IdempotencyOutcome::Replay); } Ok(crate::idempotency::IdempotencyAction::Disabled) => { StageOutcome::Skipped.record(&idempotency_span); } Err(error) => { StageOutcome::Error.record(&idempotency_span); runtime_error_category(error).record(&idempotency_span); record_idempotency_outcome(idempotency_error_outcome(error)); } } drop(idempotency_span); result? } else { crate::idempotency::IdempotencyAction::Disabled }; if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency { return transform_response(operation, response); } let mut cancellation_guard = if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency { self.coordination_store.as_ref().map(|store| { IdempotencyCancellationGuard::new( Arc::clone(store), operation.clone(), reservation.clone(), ) }) } else { None }; let adapter_result = match self .load_cached_adapter_response(operation, &prepared_request, request_context) .await { Some(response) => Ok(response), None => { let adapter_response = self .execute_adapter(operation, prepared_request.clone(), request_context) .await; if let Ok(response) = &adapter_response { self.store_cached_adapter_response( operation, &prepared_request, request_context, response, ) .await; } adapter_response } }; let adapter_response = match adapter_result { Ok(response) => response, Err(error) => { if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency && let Some(store) = self.coordination_store.as_deref() { let idempotency_span = Stage::RuntimeIdempotency.span(); let cleanup_result = crate::idempotency::mark_outcome_unknown(store, operation, reservation) .instrument(idempotency_span.clone()) .await; record_runtime_result(&idempotency_span, &cleanup_result); record_idempotency_outcome(match &cleanup_result { Ok(()) => IdempotencyOutcome::OutcomeUnknown, Err(error) => idempotency_error_outcome(error), }); if let Some(guard) = &mut cancellation_guard { guard.disarm(); } } return Err(error); } }; if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency && let Some(store) = self.coordination_store.as_deref() { let idempotency_span = Stage::RuntimeIdempotency.span(); let completion_result = crate::idempotency::complete(store, operation, reservation, &adapter_response) .instrument(idempotency_span.clone()) .await; record_runtime_result(&idempotency_span, &completion_result); record_idempotency_outcome(match &completion_result { Ok(()) => IdempotencyOutcome::Completed, Err(error) => idempotency_error_outcome(error), }); if let Some(guard) = &mut cancellation_guard { guard.disarm(); } drop(idempotency_span); completion_result?; } transform_response(operation, &adapter_response) } async fn record_metering( &self, operation: &RuntimeOperation, request_context: Option<&RuntimeRequestContext>, result: &Result, started_at: Instant, ) { let Some(context) = request_context.and_then(RuntimeRequestContext::metering_context) else { return; }; self.metering_sink .record(MeteringEvent { workspace_id: context.workspace_id.clone(), agent_id: context.agent_id.clone(), operation_id: operation.operation_id.clone(), source: context.source, status: if result.is_ok() { InvocationStatus::Ok } else { InvocationStatus::Error }, duration_ms: started_at.elapsed().as_millis() as u64, created_at: OffsetDateTime::now_utc(), }) .await; } async fn execute_adapter( &self, operation: &RuntimeOperation, prepared_request: PreparedRequest, request_context: Option<&RuntimeRequestContext>, ) -> Result { log_runtime_event("adapter.dispatch", operation, request_context); let adapter = self.adapter_for(operation)?; if !adapter.supports_mode(ExecutionMode::Unary) { return Err(RuntimeError::UnsupportedExecutionMode { operation_id: operation.operation_id.as_str().to_owned(), mode: ExecutionMode::Unary, }); } let prepared_request = adapter_prepared_request( operation, &prepared_request, operation.execution_config.timeout_ms, ); let adapter_context = adapter_request_context(request_context); adapter .invoke_unary( &operation.target, &prepared_request.into(), &adapter_context, ) .await .map(Into::into) .map_err(|error| map_protocol_adapter_error(operation, error)) } fn acquire_unary_permit( &self, _operation: &RuntimeOperation, ) -> Result { try_acquire_limit( Arc::clone(&self.unary_limiter), "unary", self.limits.max_concurrent_unary, ) } async fn load_cached_adapter_response( &self, operation: &RuntimeOperation, prepared_request: &PreparedRequest, request_context: Option<&RuntimeRequestContext>, ) -> Option { let response_cache = self.response_cache.as_ref()?; let cache_key = response_cache_key(operation, prepared_request, request_context)?; let cached = match response_cache.get(&cache_key).await { Ok(Some(cached)) => cached, Ok(None) => { record_cache_outcome(CacheOutcome::Miss); return None; } Err(_) => { record_cache_outcome(CacheOutcome::ReadError); debug!( name: "runtime.response_cache.read_failed", operation_id = operation.operation_id.as_str(), error_category = "response_cache", "response cache lookup skipped" ); return None; } }; match adapter_response_from_cached(cached) { Ok(response) => { record_cache_outcome(CacheOutcome::Hit); Some(response) } Err(_) => { record_cache_outcome(CacheOutcome::DecodeError); debug!( name: "runtime.response_cache.decode_failed", operation_id = operation.operation_id.as_str(), error_category = "cached_response", "cached response payload was invalid" ); if response_cache.delete(&cache_key).await.is_err() { record_cache_outcome(CacheOutcome::EvictError); } None } } } async fn store_cached_adapter_response( &self, operation: &RuntimeOperation, prepared_request: &PreparedRequest, request_context: Option<&RuntimeRequestContext>, adapter_response: &AdapterResponse, ) { let Some(response_cache) = self.response_cache.as_ref() else { return; }; if !(200..=299).contains(&adapter_response.status_code) { return; } let Some(cache_ttl) = response_cache_ttl(operation) else { return; }; let Some(cache_key) = response_cache_key(operation, prepared_request, request_context) else { return; }; let Some(cached_response) = cached_response_from_adapter(adapter_response) else { return; }; if response_cache .put(&cache_key, cached_response, cache_ttl) .await .is_err() { record_cache_outcome(CacheOutcome::WriteError); debug!( name: "runtime.response_cache.write_failed", operation_id = operation.operation_id.as_str(), error_category = "response_cache", "response cache write skipped" ); } else { record_cache_outcome(CacheOutcome::Stored); } } } fn adapter_request_context( request_context: Option<&RuntimeRequestContext>, ) -> crank_core::RuntimeRequestContext { request_context.map(Into::into).unwrap_or_else(|| { crank_core::RuntimeRequestContext::from_request_id(Uuid::now_v7().to_string()) }) } fn map_protocol_adapter_error( operation: &RuntimeOperation, error: crank_core::ProtocolAdapterError, ) -> RuntimeError { match error { crank_core::ProtocolAdapterError::UnsupportedMode { mode, .. } => { RuntimeError::UnsupportedExecutionMode { operation_id: operation.operation_id.as_str().to_owned(), mode, } } crank_core::ProtocolAdapterError::Message(message) => RuntimeError::ProtocolAdapter( format!("operation {}: {message}", operation.operation_id), ), } } impl RuntimeExecutor { fn adapter_for( &self, operation: &RuntimeOperation, ) -> Result { self.adapters .get(operation.protocol) .ok_or(RuntimeError::UnsupportedProtocol { protocol: operation.protocol, }) } } fn finalize_output( operation: &RuntimeOperation, response: &AdapterResponse, ) -> Result { let mapped = operation.output_mapping.apply(&json!({ "response": { "body": response.body, "data": response.data, "headers": response.headers, "status": response.status_code } }))?; Ok(mapped .get("output") .cloned() .unwrap_or_else(|| Value::Object(Map::new()))) } fn transform_response( operation: &RuntimeOperation, response: &AdapterResponse, ) -> Result { let span = Stage::RuntimeResponseTransform.span(); let result = span.in_scope(|| { let finalized_output = finalize_output(operation, response)?; operation.output_schema.validate_shape(&finalized_output)?; Ok(finalized_output) }); match &result { Ok(_) => StageOutcome::Success.record(&span), Err(_) => { StageOutcome::Error.record(&span); ErrorCategory::Transformation.record(&span); } } result } fn record_runtime_result(span: &Span, result: &Result) { match result { Ok(_) => StageOutcome::Success.record(span), Err(error) => { StageOutcome::Error.record(span); runtime_error_category(error).record(span); } } } fn runtime_error_category(error: &RuntimeError) -> ErrorCategory { match error { RuntimeError::Schema(_) => ErrorCategory::Schema, RuntimeError::Mapping(_) | RuntimeError::InvalidPreparedRequest { .. } => { ErrorCategory::Mapping } RuntimeError::RestAdapter(_) | RuntimeError::ProtocolAdapter(_) | RuntimeError::UnsupportedProtocol { .. } | RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream, RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency, RuntimeError::ConfirmationRequired { .. } | RuntimeError::InvalidConfirmationToken { .. } | RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval, RuntimeError::IdempotencyStoreUnavailable { .. } | RuntimeError::IdempotencyInProgress { .. } | RuntimeError::IdempotencyConflict { .. } | RuntimeError::IdempotencyOutcomeUnknown { .. } => ErrorCategory::Idempotency, RuntimeError::MissingAuthProfile { .. } | RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } | RuntimeError::InvalidAuthSecretValue { .. } | RuntimeError::SecretCrypto { .. } => ErrorCategory::Configuration, } } fn try_acquire_limit( limiter: Arc, kind: &'static str, limit: usize, ) -> Result { limiter.try_acquire_owned().map_err(|_| { record_limit_rejection(LimitStage::Concurrency); RuntimeError::ConcurrencyLimitExceeded { kind, limit } }) } fn metric_invocation_source( request_context: Option<&RuntimeRequestContext>, ) -> MetricInvocationSource { request_context .and_then(RuntimeRequestContext::metering_context) .map_or(MetricInvocationSource::Internal, |context| { match context.source { InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun, InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall, } }) } fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind { match error { RuntimeError::Schema(_) => ToolErrorKind::Schema, RuntimeError::Mapping(_) => ToolErrorKind::Mapping, RuntimeError::RestAdapter(_) => ToolErrorKind::RestAdapter, RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter, RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol, RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode, RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit, RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest, RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired, RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken, RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore, RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore, RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress, RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict, RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown, RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile, RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret, RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion, RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret, RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto, } } fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome { match error { RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict, RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress, RuntimeError::IdempotencyOutcomeUnknown { .. } => IdempotencyOutcome::OutcomeUnknown, RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable, _ => IdempotencyOutcome::Error, } } fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome { match error { RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken, RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable, _ => ConfirmationOutcome::Error, } } fn log_runtime_event( stage: &'static str, operation: &RuntimeOperation, request_context: Option<&RuntimeRequestContext>, ) { let protocol = match operation.protocol { crank_core::Protocol::Rest => "rest", }; if let Some(context) = request_context { debug!( name: "runtime.execution.stage_reached", stage, operation_id = operation.operation_id.as_str(), protocol, request_id = context.request_id.as_str(), trace_id = context.trace_context.trace_id().as_str(), "runtime execution" ); } else { debug!( name: "runtime.execution.stage_reached", stage, operation_id = operation.operation_id.as_str(), protocol, "runtime execution" ); } }