use std::sync::{ Arc, atomic::{AtomicBool, Ordering}, }; use std::time::{Duration, Instant}; const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100); use crank_core::{ AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod, InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter, Target, }; use crank_metrics::{ CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard, InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics, ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome, record_limit_rejection, }; use crank_trace::{ErrorCategory, Stage, StageOutcome, set_parent_from_trace_context}; use serde_json::{Map, Value, json}; use time::OffsetDateTime; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tracing::{Instrument, Span, debug, warn}; use uuid::Uuid; use crate::task_worker::TaskDispatchPermit; use crate::{ AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError, RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext, auth::apply_resolved_auth, request_preparation::{adapter_prepared_request, validate_selected_body_variant}, response_cache::{ adapter_response_from_cached, cached_response_from_adapter, response_cache_key, response_cache_ttl, }, }; #[derive(Clone)] pub struct RuntimeExecutor { adapters: AdapterRegistry, limits: RuntimeLimits, unary_limiter: Arc, response_cache: Option>, coordination_store: Option>, admission: Option>, metering_sink: SharedMeteringSink, } struct IdempotencyCancellationGuard { cleanup: Option, runtime: tokio::runtime::Handle, } struct IdempotencyCancellationCleanup { store: Arc, operation: RuntimeOperation, reservation: crate::idempotency::IdempotencyReservation, dispatch_started: Arc, } impl IdempotencyCancellationGuard { fn new( store: Arc, operation: RuntimeOperation, reservation: crate::idempotency::IdempotencyReservation, dispatch_started: Arc, ) -> Self { Self { cleanup: Some(IdempotencyCancellationCleanup { store, operation, reservation, dispatch_started, }), runtime: tokio::runtime::Handle::current(), } } fn disarm(&mut self) { self.cleanup = None; } } impl Drop for IdempotencyCancellationGuard { fn drop(&mut self) { let Some(cleanup) = self.cleanup.take() else { return; }; self.runtime.spawn(async move { let may_have_dispatched = cleanup.dispatch_started.load(Ordering::Acquire); let result = if may_have_dispatched { crate::idempotency::mark_outcome_unknown( cleanup.store.as_ref(), &cleanup.operation, &cleanup.reservation, ) .await } else { crate::idempotency::release( cleanup.store.as_ref(), &cleanup.operation, &cleanup.reservation, ) .await }; record_idempotency_outcome(match &result { Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown, Ok(()) => IdempotencyOutcome::Completed, Err(error) => idempotency_error_outcome(error), }); if result.is_err() { warn!( name: "runtime.idempotency.cancellation_cleanup_failed", error_category = "idempotency_store", "failed to finalize cancelled idempotent execution" ); } }); } } impl Default for RuntimeExecutor { fn default() -> Self { Self::new() } } impl RuntimeExecutor { pub fn new() -> Self { crate::executor_builder::community_default().build() } pub fn with_limits(limits: RuntimeLimits) -> Self { crate::executor_builder::community_default() .with_limits(limits) .build() } pub(crate) fn from_builder_parts( limits: RuntimeLimits, adapters: AdapterRegistry, response_cache: Option>, coordination_store: Option>, admission: Option>, metering_sink: SharedMeteringSink, ) -> Self { Self { adapters, unary_limiter: Arc::new(Semaphore::new(limits.max_concurrent_unary)), limits, response_cache, coordination_store, admission, metering_sink, } } pub fn with_response_cache_store( mut self, response_cache: Arc, ) -> Self { self.response_cache = Some(response_cache); self } pub fn with_coordination_store( mut self, coordination_store: Arc, ) -> Self { self.coordination_store = Some(coordination_store); self } pub fn with_admission(mut self, admission: Arc) -> Self { self.admission = Some(admission); self } pub(crate) async fn execute_request( &self, request: RuntimeExecutionRequest<'_>, ) -> Result<(Value, Value), RuntimeError> { self.execute_request_impl(request, None).await } pub async fn execute_task_outcome( &self, request: RuntimeExecutionRequest<'_>, permit: TaskDispatchPermit, ) -> Result { let operation = request.operation().clone(); let operation_id = operation.operation_id.clone(); let operation_version = operation.operation_version; let origin = request.origin(); let correlation = crank_core::CorrelationContext::new( request.request_context().request_id.clone(), request.request_context().trace_context.clone(), ); // Validate permit binding before any adapter dispatch. if permit.operation_id() != operation_id.as_str() || permit.operation_version() != operation_version || permit.workspace_id() != request.workspace_id().as_str() || permit.agent_id() != request.agent_id().map(|value| value.as_str()) { return Err(crate::normalize_runtime_error_for_operation( &RuntimeError::InvalidExecutionRequest, &correlation, &operation, )); } if !permit.consume() { return Err(crate::normalize_runtime_error_for_operation( &RuntimeError::InvalidExecutionRequest, &correlation, &operation, )); } match self.execute_request_impl(request, Some(permit)).await { Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess { operation_id: operation_id.clone(), operation_version, origin, correlation, request_preview, output, }), Err(error) => Err(crate::normalize_runtime_error_for_operation( &error, &correlation, &operation, )), } } pub(crate) async fn execute_request_impl( &self, request: RuntimeExecutionRequest<'_>, permit: Option, ) -> Result<(Value, Value), RuntimeError> { let operation = request.operation(); // Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission. let trusted_task_dispatch = permit.is_some(); // The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the // trace names the real persisted attempt instead of inventing one (Story 4.3 AC). let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned()); let input = request.input(); let request_context = request.request_context(); debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized); debug_assert_eq!( request_context .metering_context() .map(|value| &value.workspace_id), Some(request.workspace_id()) ); debug_assert_eq!( request_context .metering_context() .and_then(|value| value.agent_id.as_ref()), request.agent_id() ); let operation_may_mutate = operation_may_mutate(operation); let execution_deadline = crate::admission::effective_deadline(operation, request.deadline()); let runtime_span = Stage::RuntimeExecute.span(); set_parent_from_trace_context(&runtime_span, &request_context.trace_context); log_runtime_event("unary.execute", operation, Some(request_context)); let started_at = Instant::now(); let dispatch_started = Arc::new(AtomicBool::new(false)); // True until the admission gate answered. A deadline that fires while the request is still // waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout. let admission_pending = Arc::new(AtomicBool::new( !trusted_task_dispatch && self.admission.is_some(), )); let invocation_metrics = ToolInvocationMetrics::start_with_exemplar( metric_invocation_source(Some(request_context)), request_context .trace_context .is_sampled() .then(|| { crank_metrics::ExemplarTraceId::parse( &request_context.trace_context.trace_id().to_string(), ) }) .flatten(), ); let execution = async { // One admission gate for every executable path (Story 4.1): the published policy decides // the effective mode, the absolute deadline includes queue waiting and the scoped permit // is held for the whole local execution future, including retries. let mut lease = if trusted_task_dispatch { None } else { let result = crate::admission::acquire( self.admission.as_ref(), self.coordination_store.as_deref(), &request, execution_deadline, ) .await?; admission_pending.store(false, Ordering::Release); result }; let _permit = self.acquire_unary_permit(operation)?; let _inflight = InFlightGuard::runtime(); let mapping_span = Stage::RuntimeArgumentsMap.span(); let prepared_request = mapping_span.in_scope(|| self.prepare_request(operation, input)); record_runtime_result(&mapping_span, &prepared_request); drop(mapping_span); let prepared_request = prepared_request?; let request_preview = sanitized_request_preview(&prepared_request); let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth()); let result = self .execute_prepared( operation, input, prepared_request, Some(request_context), Arc::clone(&dispatch_started), durable_attempt_id.as_deref(), ) .await; if let Some(lease) = lease.as_mut() { lease.release().await; } let output = result?; Ok((output, request_preview)) }; let result = tokio::time::timeout_at( execution_deadline.into(), execution.instrument(runtime_span.clone()), ) .await .unwrap_or_else(|_| { if admission_pending.load(Ordering::Acquire) { Err(crate::admission::deadline_refusal(operation)) } else { Err(RuntimeError::ExecutionDeadlineElapsed { may_have_dispatched: operation_may_mutate && dispatch_started.load(Ordering::Acquire), }) } }); record_runtime_result(&runtime_span, &result); drop(runtime_span); let (outcome, error_kind) = match &result { Ok(_) => (ToolOutcome::Success, ToolErrorKind::None), Err(error) => (ToolOutcome::Error, runtime_error_kind(error)), }; invocation_metrics.complete(outcome, error_kind); self.record_metering(operation, Some(request_context), &result, started_at) .await; result } pub async fn execute_outcome( &self, request: RuntimeExecutionRequest<'_>, ) -> Result { let operation_id = request.operation().operation_id.clone(); let operation_version = request.operation().operation_version; let operation = request.operation().clone(); let origin = request.origin(); let correlation = crank_core::CorrelationContext::new( request.request_context().request_id.clone(), request.request_context().trace_context.clone(), ); match self.execute_request(request).await { Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess { operation_id, operation_version, origin, correlation, request_preview, output, }), Err(error) => Err(crate::normalize_runtime_error_for_operation( &error, &correlation, &operation, )), } } fn prepare_request( &self, operation: &RuntimeOperation, input: &Value, ) -> Result { operation .input_schema .validate_shape(input) .map_err(RuntimeError::InputSchema)?; validate_selected_body_variant(operation, input)?; if operation.input_mapping.is_empty() { return Ok(PreparedRequest::default()); } let mapped_input = operation .input_mapping .apply(&json!({ "mcp": input })) .map_err(RuntimeError::InputMapping)?; let mut prepared = PreparedRequest::from_mapping_output(&mapped_input, &operation.target)?; let Target::Rest(target) = &operation.target; if let Some(contract) = &target.request_contract && input.get(&contract.body_field).is_none() { // Mapping initializes an empty body object for legacy field mappings. // A typed optional body is absent unless its public argument was supplied. prepared.body = None; } Ok(prepared) } async fn execute_prepared( &self, operation: &RuntimeOperation, input: &Value, prepared_request: PreparedRequest, request_context: Option<&RuntimeRequestContext>, dispatch_started: Arc, durable_attempt_id: Option<&str>, ) -> Result { let mut prepared_request = prepared_request; let idempotency_applicable = crate::idempotency::policy(operation).is_some(); let idempotency_key = match crate::idempotency::prepare_idempotency( operation, input, &mut prepared_request, ) { Ok(key) => key, Err(error) if idempotency_applicable => { record_idempotency_outcome(idempotency_error_outcome(&error)); let span = Stage::RuntimeIdempotency.span(); StageOutcome::Error.record(&span); ErrorCategory::Idempotency.record(&span); return Err(error); } Err(error) => return Err(error), }; if crate::confirmation::is_applicable(operation) { let approval_span = Stage::ApprovalCheck.span(); let approval_result = crate::confirmation::confirm_operation( self.coordination_store.as_deref(), operation, input, request_context, ) .instrument(approval_span.clone()) .await; match &approval_result { Ok(()) => { StageOutcome::Success.record(&approval_span); record_confirmation_outcome(ConfirmationOutcome::Approved); } Err(RuntimeError::ConfirmationRequired { .. }) => { StageOutcome::Required.record(&approval_span); ErrorCategory::Approval.record(&approval_span); record_confirmation_outcome(ConfirmationOutcome::Required); } Err(error) => { StageOutcome::Error.record(&approval_span); runtime_error_category(error).record(&approval_span); record_confirmation_outcome(confirmation_error_outcome(error)); } } drop(approval_span); approval_result?; } let idempotency = if idempotency_applicable { let idempotency_span = Stage::RuntimeIdempotency.span(); let result = crate::idempotency::begin( self.coordination_store.as_deref(), operation, input, idempotency_key.as_deref(), request_context, ) .instrument(idempotency_span.clone()) .await; match &result { Ok(crate::idempotency::IdempotencyAction::Execute(_)) => { StageOutcome::Execute.record(&idempotency_span); record_idempotency_outcome(IdempotencyOutcome::Execute); } Ok(crate::idempotency::IdempotencyAction::Replay(_)) => { StageOutcome::Replay.record(&idempotency_span); record_idempotency_outcome(IdempotencyOutcome::Replay); } Ok(crate::idempotency::IdempotencyAction::Disabled) => { StageOutcome::Skipped.record(&idempotency_span); } Err(error) => { StageOutcome::Error.record(&idempotency_span); runtime_error_category(error).record(&idempotency_span); record_idempotency_outcome(idempotency_error_outcome(error)); } } drop(idempotency_span); result? } else { crate::idempotency::IdempotencyAction::Disabled }; if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency { return transform_response(operation, response); } let mut cancellation_guard = if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency { self.coordination_store.as_ref().map(|store| { IdempotencyCancellationGuard::new( Arc::clone(store), operation.clone(), reservation.clone(), Arc::clone(&dispatch_started), ) }) } else { None }; let adapter_result = match self .load_cached_adapter_response(operation, &prepared_request, request_context) .await { Some(response) => Ok(response), None => { let adapter_response = self .execute_adapter_with_retry( operation, &prepared_request, request_context, Arc::clone(&dispatch_started), crate::idempotency::verified_retry_key( operation, idempotency_key.as_deref(), ), durable_attempt_id, ) .await; if let Ok(response) = &adapter_response { self.store_cached_adapter_response( operation, &prepared_request, request_context, response, ) .await; } adapter_response } }; let adapter_response = match adapter_result { Ok(response) => response, Err(error) => { if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency && let Some(store) = self.coordination_store.as_deref() { let idempotency_span = Stage::RuntimeIdempotency.span(); let may_have_dispatched = runtime_error_may_have_dispatched(&error); let cleanup_result = if may_have_dispatched { crate::idempotency::mark_outcome_unknown(store, operation, reservation) .instrument(idempotency_span.clone()) .await } else { crate::idempotency::release(store, operation, reservation) .instrument(idempotency_span.clone()) .await }; record_runtime_result(&idempotency_span, &cleanup_result); record_idempotency_outcome(match &cleanup_result { Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown, Ok(()) => IdempotencyOutcome::Completed, Err(error) => idempotency_error_outcome(error), }); if let Some(guard) = &mut cancellation_guard { guard.disarm(); } } return Err(error); } }; if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency && let Some(store) = self.coordination_store.as_deref() { let idempotency_span = Stage::RuntimeIdempotency.span(); let completion_result = crate::idempotency::complete(store, operation, reservation, &adapter_response) .instrument(idempotency_span.clone()) .await; record_runtime_result(&idempotency_span, &completion_result); record_idempotency_outcome(match &completion_result { Ok(()) => IdempotencyOutcome::Completed, Err(error) => idempotency_error_outcome(error), }); if let Some(guard) = &mut cancellation_guard { guard.disarm(); } drop(idempotency_span); if completion_result.is_err() { return Err(RuntimeError::IdempotencyCompletionUnknown { operation_id: operation.operation_id.as_str().to_owned(), }); } } transform_response(operation, &adapter_response) } async fn record_metering( &self, operation: &RuntimeOperation, request_context: Option<&RuntimeRequestContext>, result: &Result, started_at: Instant, ) { let Some(context) = request_context.and_then(RuntimeRequestContext::metering_context) else { return; }; self.metering_sink .record(MeteringEvent { workspace_id: context.workspace_id.clone(), agent_id: context.agent_id.clone(), operation_id: operation.operation_id.clone(), source: context.source, status: if result.is_ok() { InvocationStatus::Ok } else { InvocationStatus::Error }, duration_ms: started_at.elapsed().as_millis() as u64, created_at: OffsetDateTime::now_utc(), }) .await; } async fn execute_adapter( &self, operation: &RuntimeOperation, prepared_request: PreparedRequest, request_context: Option<&RuntimeRequestContext>, dispatch_started: Arc, ) -> Result { log_runtime_event("adapter.dispatch", operation, request_context); let adapter = self.adapter_for(operation)?; if !adapter.supports_mode(ExecutionMode::Unary) { return Err(RuntimeError::UnsupportedExecutionMode { operation_id: operation.operation_id.as_str().to_owned(), mode: ExecutionMode::Unary, }); } let prepared_request = adapter_prepared_request( operation, &prepared_request, operation.execution_config.timeout_ms, ); let adapter_context = adapter_request_context(request_context, Arc::clone(&dispatch_started))?; adapter .invoke_unary( &operation.target, &prepared_request.into(), &adapter_context, ) .await .map(Into::into) .map_err(|error| map_protocol_adapter_error(operation, error)) } async fn execute_adapter_with_retry( &self, operation: &RuntimeOperation, prepared_request: &PreparedRequest, request_context: Option<&RuntimeRequestContext>, dispatch_started: Arc, verified_idempotency: bool, durable_attempt_id: Option<&str>, ) -> Result { let max_attempts = retry_attempts(operation); let mut attempt = 1; loop { // Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint; the // retry index discriminates the attempts inside it. Only paths without a durable attempt // (synchronous tools/call) mint a synthetic per-attempt id. let attempt_id = durable_attempt_id .map(str::to_owned) .unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7())); let attempt_span = tracing::info_span!( "adapter_attempt", attempt, max_attempts, attempt_id = %attempt_id ); let result = self .execute_adapter( operation, prepared_request.clone(), request_context, Arc::clone(&dispatch_started), ) .instrument(attempt_span) .await; match result { Ok(response) => return Ok(response), Err(error) if attempt < max_attempts && retry_is_allowed(operation, &error, verified_idempotency) => { attempt += 1; tokio::time::sleep(retry_backoff(attempt)).await; } Err(error) => return Err(error), } } } fn acquire_unary_permit( &self, _operation: &RuntimeOperation, ) -> Result { try_acquire_limit( Arc::clone(&self.unary_limiter), "unary", self.limits.max_concurrent_unary, ) } async fn load_cached_adapter_response( &self, operation: &RuntimeOperation, prepared_request: &PreparedRequest, request_context: Option<&RuntimeRequestContext>, ) -> Option { let response_cache = self.response_cache.as_ref()?; let cache_key = response_cache_key(operation, prepared_request, request_context)?; let cached = match tokio::time::timeout(RESPONSE_CACHE_IO_TIMEOUT, response_cache.get(&cache_key)) .await { Ok(Ok(Some(cached))) => cached, Ok(Ok(None)) => { record_cache_outcome(CacheOutcome::Miss); return None; } Ok(Err(_)) | Err(_) => { record_cache_outcome(CacheOutcome::ReadError); debug!( name: "runtime.response_cache.read_failed", operation_id = operation.operation_id.as_str(), error_category = "response_cache", "response cache lookup skipped" ); return None; } }; match adapter_response_from_cached(cached) { Ok(response) => { record_cache_outcome(CacheOutcome::Hit); Some(response) } Err(_) => { record_cache_outcome(CacheOutcome::DecodeError); debug!( name: "runtime.response_cache.decode_failed", operation_id = operation.operation_id.as_str(), error_category = "cached_response", "cached response payload was invalid" ); if response_cache.delete(&cache_key).await.is_err() { record_cache_outcome(CacheOutcome::EvictError); } None } } } async fn store_cached_adapter_response( &self, operation: &RuntimeOperation, prepared_request: &PreparedRequest, request_context: Option<&RuntimeRequestContext>, adapter_response: &AdapterResponse, ) { let Some(response_cache) = self.response_cache.as_ref() else { return; }; if !(200..=299).contains(&adapter_response.status_code) { return; } if request_context .and_then(RuntimeRequestContext::resource_response_cache) .is_some_and(|policy| { serde_json::to_vec(&adapter_response.data) .map_or(true, |bytes| bytes.len() as u64 > policy.max_bytes) }) { return; } let Some(cache_ttl) = response_cache_ttl(operation, request_context) else { return; }; let Some(cache_key) = response_cache_key(operation, prepared_request, request_context) else { return; }; let Some(cached_response) = cached_response_from_adapter(adapter_response) else { return; }; if !matches!( tokio::time::timeout( RESPONSE_CACHE_IO_TIMEOUT, response_cache.put(&cache_key, cached_response, cache_ttl), ) .await, Ok(Ok(())) ) { record_cache_outcome(CacheOutcome::WriteError); debug!( name: "runtime.response_cache.write_failed", operation_id = operation.operation_id.as_str(), error_category = "response_cache", "response cache write skipped" ); } else { record_cache_outcome(CacheOutcome::Stored); } } } fn sanitized_request_preview(prepared: &PreparedRequest) -> Value { json!({ "path_parameter_count": prepared.path_params.len(), "query_parameter_count": prepared.query_params.len(), "header_count": prepared.headers.len(), "body_configured": prepared.body.is_some(), }) } fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool { matches!( error, RuntimeError::ProtocolAdapter( crank_core::ProtocolAdapterError::Transport { dispatch: crank_core::DispatchEvidence::MayHaveDispatched, } | crank_core::ProtocolAdapterError::Timeout { dispatch: crank_core::DispatchEvidence::MayHaveDispatched, } | crank_core::ProtocolAdapterError::ResponseTooLarge { dispatch: crank_core::DispatchEvidence::MayHaveDispatched, } | crank_core::ProtocolAdapterError::UnexpectedStatus { dispatch: crank_core::DispatchEvidence::MayHaveDispatched, .. } | crank_core::ProtocolAdapterError::InvalidResponse { dispatch: crank_core::DispatchEvidence::MayHaveDispatched, }, ) | RuntimeError::ExecutionDeadlineElapsed { may_have_dispatched: true, } | RuntimeError::IdempotencyCompletionUnknown { .. } ) } fn retry_attempts(operation: &RuntimeOperation) -> u32 { operation .execution_config .retry_policy .as_ref() .map_or(1, |policy| policy.max_attempts.clamp(1, 5)) } fn retry_backoff(attempt: u32) -> Duration { Duration::from_millis(u64::from(attempt.saturating_sub(1)).min(4)) } fn retry_is_allowed( operation: &RuntimeOperation, error: &RuntimeError, verified_idempotency: bool, ) -> bool { let is_read = matches!( &operation.target, crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get ); let safe_write = operation_may_mutate(operation) && verified_idempotency; if !(is_read || safe_write) { return false; } match error { RuntimeError::ProtocolAdapter( crank_core::ProtocolAdapterError::Transport { .. } | crank_core::ProtocolAdapterError::Timeout { .. }, ) => true, RuntimeError::ProtocolAdapter(crank_core::ProtocolAdapterError::UnexpectedStatus { status, .. }) => matches!(status, 408 | 429 | 500..=599), _ => false, } } fn operation_may_mutate(operation: &RuntimeOperation) -> bool { matches!( &operation.target, crank_core::Target::Rest(target) if matches!( target.method, crank_core::HttpMethod::Post | crank_core::HttpMethod::Put | crank_core::HttpMethod::Patch | crank_core::HttpMethod::Delete ) ) } fn adapter_request_context( request_context: Option<&RuntimeRequestContext>, dispatch_started: Arc, ) -> Result { request_context .map(|context| { crank_core::RuntimeRequestContext::from(context).with_dispatch_started(dispatch_started) }) .ok_or(RuntimeError::InvalidExecutionRequest) } fn map_protocol_adapter_error( operation: &RuntimeOperation, error: crank_core::ProtocolAdapterError, ) -> RuntimeError { match error { crank_core::ProtocolAdapterError::UnsupportedMode { mode, .. } => { RuntimeError::UnsupportedExecutionMode { operation_id: operation.operation_id.as_str().to_owned(), mode, } } error => RuntimeError::ProtocolAdapter(error), } } impl RuntimeExecutor { fn adapter_for( &self, operation: &RuntimeOperation, ) -> Result { self.adapters .get(operation.protocol) .ok_or(RuntimeError::UnsupportedProtocol { protocol: operation.protocol, }) } } fn finalize_output( operation: &RuntimeOperation, response: &AdapterResponse, ) -> Result { let response_data = if matches!( &operation.target, Target::Rest(target) if target.method == HttpMethod::Head && response.body.is_null() ) { // HEAD has no body; expose only safe status metadata for an explicit output mapping. json!({"status": response.status_code}) } else { response.data.clone() }; let mapped = operation .output_mapping .apply(&json!({ "response": { "body": response.body, "data": response_data, "headers": response.headers, "status": response.status_code } })) .map_err(RuntimeError::OutputMapping)?; Ok(mapped .get("output") .cloned() .unwrap_or_else(|| Value::Object(Map::new()))) } fn transform_response( operation: &RuntimeOperation, response: &AdapterResponse, ) -> Result { let span = Stage::RuntimeResponseTransform.span(); let result = span.in_scope(|| { let finalized_output = finalize_output(operation, response)?; output_schema_for_response(operation, response)? .validate_shape(&finalized_output) .map_err(RuntimeError::OutputSchema)?; Ok(finalized_output) }); match &result { Ok(_) => StageOutcome::Success.record(&span), Err(_) => { StageOutcome::Error.record(&span); ErrorCategory::Transformation.record(&span); } } result } fn output_schema_for_response<'a>( operation: &'a RuntimeOperation, response: &AdapterResponse, ) -> Result<&'a crank_schema::Schema, RuntimeError> { let Target::Rest(target) = &operation.target; if target.response_contract.is_empty() { return Ok(&operation.output_schema); } let media_type = response .headers .get("content-type") .and_then(|value| value.split(';').next()) .map(str::trim); let descriptor = target .response_contract .iter() .filter(|descriptor| descriptor.status_code == response.status_code) .find(|descriptor| match (&descriptor.media_type, media_type) { (Some(expected), Some(actual)) => expected.eq_ignore_ascii_case(actual), (None, _) => response.body.is_null(), _ => false, }) .ok_or(RuntimeError::InvalidExecutionRequest)?; if target.response_contract.len() > 1 && operation.output_schema.kind == crank_schema::SchemaKind::Oneof { return operation .output_schema .variants .get(descriptor.schema_variant) .ok_or(RuntimeError::InvalidExecutionRequest); } (descriptor.schema_variant == 0) .then_some(&operation.output_schema) .ok_or(RuntimeError::InvalidExecutionRequest) } fn record_runtime_result(span: &Span, result: &Result) { match result { Ok(_) => StageOutcome::Success.record(span), Err(error) => { StageOutcome::Error.record(span); runtime_error_category(error).record(span); } } } fn runtime_error_category(error: &RuntimeError) -> ErrorCategory { match error { RuntimeError::InvalidExecutionRequest => ErrorCategory::Configuration, RuntimeError::ExecutionDeadlineElapsed { .. } => ErrorCategory::Upstream, RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => { ErrorCategory::Schema } RuntimeError::Mapping(_) | RuntimeError::InputMapping(_) | RuntimeError::OutputMapping(_) | RuntimeError::InvalidPreparedRequest { .. } => ErrorCategory::Mapping, RuntimeError::ProtocolAdapter(_) | RuntimeError::UnsupportedProtocol { .. } | RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream, RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency, RuntimeError::AdmissionOverloaded { .. } | RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency, RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration, RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval, RuntimeError::ConfirmationRequired { .. } | RuntimeError::InvalidConfirmationToken { .. } | RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval, RuntimeError::IdempotencyStoreUnavailable { .. } | RuntimeError::IdempotencyInProgress { .. } | RuntimeError::IdempotencyConflict { .. } | RuntimeError::IdempotencyOutcomeUnknown { .. } | RuntimeError::IdempotencyCompletionUnknown { .. } => ErrorCategory::Idempotency, RuntimeError::MissingAuthProfile { .. } | RuntimeError::InvalidAuthProfileConfig { .. } | RuntimeError::AuthorizationStoreUnavailable | RuntimeError::MissingSecret { .. } | RuntimeError::MissingSecretVersion { .. } | RuntimeError::InvalidAuthSecretValue { .. } | RuntimeError::SecretCrypto { .. } => ErrorCategory::Configuration, } } fn try_acquire_limit( limiter: Arc, kind: &'static str, limit: usize, ) -> Result { limiter.try_acquire_owned().map_err(|_| { record_limit_rejection(LimitStage::Concurrency); RuntimeError::ConcurrencyLimitExceeded { kind, limit } }) } fn metric_invocation_source( request_context: Option<&RuntimeRequestContext>, ) -> MetricInvocationSource { request_context .and_then(RuntimeRequestContext::metering_context) .map_or(MetricInvocationSource::Internal, |context| { match context.source { InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun, InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall, } }) } fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind { match error { RuntimeError::InvalidExecutionRequest => ToolErrorKind::InvalidPreparedRequest, RuntimeError::ExecutionDeadlineElapsed { .. } => ToolErrorKind::RestAdapter, RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => { ToolErrorKind::Schema } RuntimeError::Mapping(_) | RuntimeError::InputMapping(_) | RuntimeError::OutputMapping(_) => ToolErrorKind::Mapping, RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter, RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol, RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode, RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit, // Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation // label domain stays fixed; the precise code lives in the execution failure/invocation log // (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`). RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit, RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore, RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired, RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode, RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest, RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired, RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken, RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore, RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore, RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress, RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict, RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown, RuntimeError::IdempotencyCompletionUnknown { .. } => { ToolErrorKind::IdempotencyOutcomeUnknown } RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile, RuntimeError::InvalidAuthProfileConfig { .. } => ToolErrorKind::InvalidAuthSecret, RuntimeError::AuthorizationStoreUnavailable => ToolErrorKind::IdempotencyStore, RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret, RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion, RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret, RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto, } } fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome { match error { RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict, RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress, RuntimeError::IdempotencyOutcomeUnknown { .. } | RuntimeError::IdempotencyCompletionUnknown { .. } => IdempotencyOutcome::OutcomeUnknown, RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable, _ => IdempotencyOutcome::Error, } } fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome { match error { RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken, RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable, _ => ConfirmationOutcome::Error, } } fn log_runtime_event( stage: &'static str, operation: &RuntimeOperation, request_context: Option<&RuntimeRequestContext>, ) { let protocol = match operation.protocol { crank_core::Protocol::Rest => "rest", }; if let Some(context) = request_context { debug!( name: "runtime.execution.stage_reached", stage, operation_id = operation.operation_id.as_str(), protocol, request_id = context.request_id.as_str(), trace_id = context.trace_context.trace_id().as_str(), "runtime execution" ); } else { debug!( name: "runtime.execution.stage_reached", stage, operation_id = operation.operation_id.as_str(), protocol, "runtime execution" ); } }