865 lines
31 KiB
Rust
865 lines
31 KiB
Rust
use std::sync::Arc;
|
|
use std::time::Instant;
|
|
|
|
use crank_core::{
|
|
AdapterRegistry, CoordinationStateStore, ExecutionMode, InvocationSource, InvocationStatus,
|
|
MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
|
|
};
|
|
use crank_metrics::{
|
|
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
|
|
InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics,
|
|
ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome,
|
|
record_limit_rejection,
|
|
};
|
|
use crank_trace::{
|
|
ErrorCategory, Stage, StageOutcome, set_parent_from_trace_context, trace_context_for_span,
|
|
};
|
|
use serde_json::{Map, Value, json};
|
|
use time::OffsetDateTime;
|
|
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
|
use tracing::{Instrument, Span, debug, warn};
|
|
use uuid::Uuid;
|
|
|
|
use crate::{
|
|
AdapterResponse, PreparedRequest, ResolvedAuth, RuntimeError, RuntimeLimits, RuntimeOperation,
|
|
RuntimeRequestContext,
|
|
auth::apply_resolved_auth,
|
|
request_preparation::adapter_prepared_request,
|
|
response_cache::{
|
|
adapter_response_from_cached, cached_response_from_adapter, response_cache_key,
|
|
response_cache_ttl,
|
|
},
|
|
};
|
|
|
|
#[derive(Clone)]
|
|
pub struct RuntimeExecutor {
|
|
adapters: AdapterRegistry,
|
|
limits: RuntimeLimits,
|
|
unary_limiter: Arc<Semaphore>,
|
|
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
|
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
|
metering_sink: SharedMeteringSink,
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
pub struct RuntimeExecutionRequest<'a> {
|
|
pub operation: &'a RuntimeOperation,
|
|
pub input: &'a Value,
|
|
pub resolved_auth: Option<&'a ResolvedAuth>,
|
|
pub request_context: Option<&'a RuntimeRequestContext>,
|
|
}
|
|
|
|
struct IdempotencyCancellationGuard {
|
|
cleanup: Option<IdempotencyCancellationCleanup>,
|
|
runtime: tokio::runtime::Handle,
|
|
}
|
|
|
|
struct IdempotencyCancellationCleanup {
|
|
store: Arc<dyn CoordinationStateStore>,
|
|
operation: RuntimeOperation,
|
|
reservation: crate::idempotency::IdempotencyReservation,
|
|
}
|
|
|
|
impl IdempotencyCancellationGuard {
|
|
fn new(
|
|
store: Arc<dyn CoordinationStateStore>,
|
|
operation: RuntimeOperation,
|
|
reservation: crate::idempotency::IdempotencyReservation,
|
|
) -> Self {
|
|
Self {
|
|
cleanup: Some(IdempotencyCancellationCleanup {
|
|
store,
|
|
operation,
|
|
reservation,
|
|
}),
|
|
runtime: tokio::runtime::Handle::current(),
|
|
}
|
|
}
|
|
|
|
fn disarm(&mut self) {
|
|
self.cleanup = None;
|
|
}
|
|
}
|
|
|
|
impl Drop for IdempotencyCancellationGuard {
|
|
fn drop(&mut self) {
|
|
let Some(cleanup) = self.cleanup.take() else {
|
|
return;
|
|
};
|
|
self.runtime.spawn(async move {
|
|
let result = crate::idempotency::mark_outcome_unknown(
|
|
cleanup.store.as_ref(),
|
|
&cleanup.operation,
|
|
&cleanup.reservation,
|
|
)
|
|
.await;
|
|
record_idempotency_outcome(match &result {
|
|
Ok(()) => IdempotencyOutcome::OutcomeUnknown,
|
|
Err(error) => idempotency_error_outcome(error),
|
|
});
|
|
if result.is_err() {
|
|
warn!(
|
|
name: "runtime.idempotency.cancellation_cleanup_failed",
|
|
error_category = "idempotency_store",
|
|
"failed to finalize cancelled idempotent execution"
|
|
);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
impl<'a> RuntimeExecutionRequest<'a> {
|
|
pub fn new(operation: &'a RuntimeOperation, input: &'a Value) -> Self {
|
|
Self {
|
|
operation,
|
|
input,
|
|
resolved_auth: None,
|
|
request_context: None,
|
|
}
|
|
}
|
|
|
|
pub fn with_auth(mut self, resolved_auth: &'a ResolvedAuth) -> Self {
|
|
self.resolved_auth = Some(resolved_auth);
|
|
self
|
|
}
|
|
|
|
pub fn with_optional_auth(mut self, resolved_auth: Option<&'a ResolvedAuth>) -> Self {
|
|
self.resolved_auth = resolved_auth;
|
|
self
|
|
}
|
|
|
|
pub fn with_context(mut self, request_context: &'a RuntimeRequestContext) -> Self {
|
|
self.request_context = Some(request_context);
|
|
self
|
|
}
|
|
|
|
pub fn with_optional_context(
|
|
mut self,
|
|
request_context: Option<&'a RuntimeRequestContext>,
|
|
) -> Self {
|
|
self.request_context = request_context;
|
|
self
|
|
}
|
|
}
|
|
|
|
impl Default for RuntimeExecutor {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl RuntimeExecutor {
|
|
pub fn new() -> Self {
|
|
crate::executor_builder::community_default().build()
|
|
}
|
|
|
|
pub fn with_limits(limits: RuntimeLimits) -> Self {
|
|
crate::executor_builder::community_default()
|
|
.with_limits(limits)
|
|
.build()
|
|
}
|
|
|
|
pub(crate) fn from_builder_parts(
|
|
limits: RuntimeLimits,
|
|
adapters: AdapterRegistry,
|
|
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
|
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
|
metering_sink: SharedMeteringSink,
|
|
) -> Self {
|
|
Self {
|
|
adapters,
|
|
unary_limiter: Arc::new(Semaphore::new(limits.max_concurrent_unary)),
|
|
limits,
|
|
response_cache,
|
|
coordination_store,
|
|
metering_sink,
|
|
}
|
|
}
|
|
|
|
pub fn with_response_cache_store(
|
|
mut self,
|
|
response_cache: Arc<dyn ResponseCacheStore>,
|
|
) -> Self {
|
|
self.response_cache = Some(response_cache);
|
|
self
|
|
}
|
|
|
|
pub fn with_coordination_store(
|
|
mut self,
|
|
coordination_store: Arc<dyn CoordinationStateStore>,
|
|
) -> Self {
|
|
self.coordination_store = Some(coordination_store);
|
|
self
|
|
}
|
|
|
|
pub async fn execute(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
) -> Result<Value, RuntimeError> {
|
|
self.execute_request(RuntimeExecutionRequest::new(operation, input))
|
|
.await
|
|
}
|
|
|
|
pub async fn execute_with_auth(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
resolved_auth: Option<&ResolvedAuth>,
|
|
) -> Result<Value, RuntimeError> {
|
|
self.execute_request(
|
|
RuntimeExecutionRequest::new(operation, input).with_optional_auth(resolved_auth),
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn execute_with_context(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Result<Value, RuntimeError> {
|
|
self.execute_request(
|
|
RuntimeExecutionRequest::new(operation, input).with_optional_context(request_context),
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn execute_with_auth_and_context(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
resolved_auth: Option<&ResolvedAuth>,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Result<Value, RuntimeError> {
|
|
self.execute_request(
|
|
RuntimeExecutionRequest::new(operation, input)
|
|
.with_optional_auth(resolved_auth)
|
|
.with_optional_context(request_context),
|
|
)
|
|
.await
|
|
}
|
|
|
|
pub async fn execute_request(
|
|
&self,
|
|
request: RuntimeExecutionRequest<'_>,
|
|
) -> Result<Value, RuntimeError> {
|
|
let runtime_span = Stage::RuntimeExecute.span();
|
|
if let Some(context) = request.request_context {
|
|
set_parent_from_trace_context(&runtime_span, &context.trace_context);
|
|
}
|
|
let generated_context = request.request_context.is_none().then(|| {
|
|
RuntimeRequestContext::new(
|
|
crank_core::RequestId::generate(),
|
|
trace_context_for_span(&runtime_span)
|
|
.unwrap_or_else(crank_core::TraceContext::generate),
|
|
)
|
|
});
|
|
let request_context = request.request_context.or(generated_context.as_ref());
|
|
log_runtime_event("unary.execute", request.operation, request_context);
|
|
let started_at = Instant::now();
|
|
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
|
|
metric_invocation_source(request_context),
|
|
request_context.and_then(|context| {
|
|
context
|
|
.trace_context
|
|
.is_sampled()
|
|
.then(|| {
|
|
crank_metrics::ExemplarTraceId::parse(
|
|
&context.trace_context.trace_id().to_string(),
|
|
)
|
|
})
|
|
.flatten()
|
|
}),
|
|
);
|
|
let result = async {
|
|
let _permit = self.acquire_unary_permit(request.operation)?;
|
|
let _inflight = InFlightGuard::runtime();
|
|
let mapping_span = Stage::RuntimeArgumentsMap.span();
|
|
let prepared_request =
|
|
mapping_span.in_scope(|| self.prepare_request(request.operation, request.input));
|
|
record_runtime_result(&mapping_span, &prepared_request);
|
|
drop(mapping_span);
|
|
let prepared_request = prepared_request?;
|
|
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth);
|
|
self.execute_prepared(
|
|
request.operation,
|
|
request.input,
|
|
prepared_request,
|
|
request_context,
|
|
)
|
|
.await
|
|
}
|
|
.instrument(runtime_span.clone())
|
|
.await;
|
|
record_runtime_result(&runtime_span, &result);
|
|
drop(runtime_span);
|
|
let (outcome, error_kind) = match &result {
|
|
Ok(_) => (ToolOutcome::Success, ToolErrorKind::None),
|
|
Err(error) => (ToolOutcome::Error, runtime_error_kind(error)),
|
|
};
|
|
invocation_metrics.complete(outcome, error_kind);
|
|
self.record_metering(request.operation, request_context, &result, started_at)
|
|
.await;
|
|
result
|
|
}
|
|
|
|
pub fn prepare_request(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
) -> Result<PreparedRequest, RuntimeError> {
|
|
operation.input_schema.validate_shape(input)?;
|
|
|
|
if operation.input_mapping.is_empty() {
|
|
return Ok(PreparedRequest::default());
|
|
}
|
|
|
|
let mapped_input = operation.input_mapping.apply(&json!({ "mcp": input }))?;
|
|
PreparedRequest::from_mapping_output(&mapped_input)
|
|
}
|
|
|
|
async fn execute_prepared(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
prepared_request: PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Result<Value, RuntimeError> {
|
|
let mut prepared_request = prepared_request;
|
|
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
|
|
let idempotency_key = match crate::idempotency::prepare_idempotency(
|
|
operation,
|
|
input,
|
|
&mut prepared_request,
|
|
) {
|
|
Ok(key) => key,
|
|
Err(error) if idempotency_applicable => {
|
|
record_idempotency_outcome(idempotency_error_outcome(&error));
|
|
let span = Stage::RuntimeIdempotency.span();
|
|
StageOutcome::Error.record(&span);
|
|
ErrorCategory::Idempotency.record(&span);
|
|
return Err(error);
|
|
}
|
|
Err(error) => return Err(error),
|
|
};
|
|
|
|
if crate::confirmation::is_applicable(operation) {
|
|
let approval_span = Stage::ApprovalCheck.span();
|
|
let approval_result = crate::confirmation::confirm_operation(
|
|
self.coordination_store.as_deref(),
|
|
operation,
|
|
input,
|
|
request_context,
|
|
)
|
|
.instrument(approval_span.clone())
|
|
.await;
|
|
match &approval_result {
|
|
Ok(()) => {
|
|
StageOutcome::Success.record(&approval_span);
|
|
record_confirmation_outcome(ConfirmationOutcome::Approved);
|
|
}
|
|
Err(RuntimeError::ConfirmationRequired { .. }) => {
|
|
StageOutcome::Required.record(&approval_span);
|
|
ErrorCategory::Approval.record(&approval_span);
|
|
record_confirmation_outcome(ConfirmationOutcome::Required);
|
|
}
|
|
Err(error) => {
|
|
StageOutcome::Error.record(&approval_span);
|
|
runtime_error_category(error).record(&approval_span);
|
|
record_confirmation_outcome(confirmation_error_outcome(error));
|
|
}
|
|
}
|
|
drop(approval_span);
|
|
approval_result?;
|
|
}
|
|
|
|
let idempotency = if idempotency_applicable {
|
|
let idempotency_span = Stage::RuntimeIdempotency.span();
|
|
let result = crate::idempotency::begin(
|
|
self.coordination_store.as_deref(),
|
|
operation,
|
|
input,
|
|
idempotency_key.as_deref(),
|
|
request_context,
|
|
)
|
|
.instrument(idempotency_span.clone())
|
|
.await;
|
|
match &result {
|
|
Ok(crate::idempotency::IdempotencyAction::Execute(_)) => {
|
|
StageOutcome::Execute.record(&idempotency_span);
|
|
record_idempotency_outcome(IdempotencyOutcome::Execute);
|
|
}
|
|
Ok(crate::idempotency::IdempotencyAction::Replay(_)) => {
|
|
StageOutcome::Replay.record(&idempotency_span);
|
|
record_idempotency_outcome(IdempotencyOutcome::Replay);
|
|
}
|
|
Ok(crate::idempotency::IdempotencyAction::Disabled) => {
|
|
StageOutcome::Skipped.record(&idempotency_span);
|
|
}
|
|
Err(error) => {
|
|
StageOutcome::Error.record(&idempotency_span);
|
|
runtime_error_category(error).record(&idempotency_span);
|
|
record_idempotency_outcome(idempotency_error_outcome(error));
|
|
}
|
|
}
|
|
drop(idempotency_span);
|
|
result?
|
|
} else {
|
|
crate::idempotency::IdempotencyAction::Disabled
|
|
};
|
|
if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency {
|
|
return transform_response(operation, response);
|
|
}
|
|
let mut cancellation_guard =
|
|
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency {
|
|
self.coordination_store.as_ref().map(|store| {
|
|
IdempotencyCancellationGuard::new(
|
|
Arc::clone(store),
|
|
operation.clone(),
|
|
reservation.clone(),
|
|
)
|
|
})
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let adapter_result = match self
|
|
.load_cached_adapter_response(operation, &prepared_request, request_context)
|
|
.await
|
|
{
|
|
Some(response) => Ok(response),
|
|
None => {
|
|
let adapter_response = self
|
|
.execute_adapter(operation, prepared_request.clone(), request_context)
|
|
.await;
|
|
if let Ok(response) = &adapter_response {
|
|
self.store_cached_adapter_response(
|
|
operation,
|
|
&prepared_request,
|
|
request_context,
|
|
response,
|
|
)
|
|
.await;
|
|
}
|
|
adapter_response
|
|
}
|
|
};
|
|
let adapter_response = match adapter_result {
|
|
Ok(response) => response,
|
|
Err(error) => {
|
|
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
|
|
&& let Some(store) = self.coordination_store.as_deref()
|
|
{
|
|
let idempotency_span = Stage::RuntimeIdempotency.span();
|
|
let cleanup_result =
|
|
crate::idempotency::mark_outcome_unknown(store, operation, reservation)
|
|
.instrument(idempotency_span.clone())
|
|
.await;
|
|
record_runtime_result(&idempotency_span, &cleanup_result);
|
|
record_idempotency_outcome(match &cleanup_result {
|
|
Ok(()) => IdempotencyOutcome::OutcomeUnknown,
|
|
Err(error) => idempotency_error_outcome(error),
|
|
});
|
|
if let Some(guard) = &mut cancellation_guard {
|
|
guard.disarm();
|
|
}
|
|
}
|
|
return Err(error);
|
|
}
|
|
};
|
|
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
|
|
&& let Some(store) = self.coordination_store.as_deref()
|
|
{
|
|
let idempotency_span = Stage::RuntimeIdempotency.span();
|
|
let completion_result =
|
|
crate::idempotency::complete(store, operation, reservation, &adapter_response)
|
|
.instrument(idempotency_span.clone())
|
|
.await;
|
|
record_runtime_result(&idempotency_span, &completion_result);
|
|
record_idempotency_outcome(match &completion_result {
|
|
Ok(()) => IdempotencyOutcome::Completed,
|
|
Err(error) => idempotency_error_outcome(error),
|
|
});
|
|
if let Some(guard) = &mut cancellation_guard {
|
|
guard.disarm();
|
|
}
|
|
drop(idempotency_span);
|
|
completion_result?;
|
|
}
|
|
transform_response(operation, &adapter_response)
|
|
}
|
|
|
|
async fn record_metering<T>(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
result: &Result<T, RuntimeError>,
|
|
started_at: Instant,
|
|
) {
|
|
let Some(context) = request_context.and_then(RuntimeRequestContext::metering_context)
|
|
else {
|
|
return;
|
|
};
|
|
|
|
self.metering_sink
|
|
.record(MeteringEvent {
|
|
workspace_id: context.workspace_id.clone(),
|
|
agent_id: context.agent_id.clone(),
|
|
operation_id: operation.operation_id.clone(),
|
|
source: context.source,
|
|
status: if result.is_ok() {
|
|
InvocationStatus::Ok
|
|
} else {
|
|
InvocationStatus::Error
|
|
},
|
|
duration_ms: started_at.elapsed().as_millis() as u64,
|
|
created_at: OffsetDateTime::now_utc(),
|
|
})
|
|
.await;
|
|
}
|
|
|
|
async fn execute_adapter(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
prepared_request: PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Result<AdapterResponse, RuntimeError> {
|
|
log_runtime_event("adapter.dispatch", operation, request_context);
|
|
let adapter = self.adapter_for(operation)?;
|
|
if !adapter.supports_mode(ExecutionMode::Unary) {
|
|
return Err(RuntimeError::UnsupportedExecutionMode {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
mode: ExecutionMode::Unary,
|
|
});
|
|
}
|
|
|
|
let prepared_request = adapter_prepared_request(
|
|
operation,
|
|
&prepared_request,
|
|
operation.execution_config.timeout_ms,
|
|
);
|
|
let adapter_context = adapter_request_context(request_context);
|
|
|
|
adapter
|
|
.invoke_unary(
|
|
&operation.target,
|
|
&prepared_request.into(),
|
|
&adapter_context,
|
|
)
|
|
.await
|
|
.map(Into::into)
|
|
.map_err(|error| map_protocol_adapter_error(operation, error))
|
|
}
|
|
|
|
fn acquire_unary_permit(
|
|
&self,
|
|
_operation: &RuntimeOperation,
|
|
) -> Result<OwnedSemaphorePermit, RuntimeError> {
|
|
try_acquire_limit(
|
|
Arc::clone(&self.unary_limiter),
|
|
"unary",
|
|
self.limits.max_concurrent_unary,
|
|
)
|
|
}
|
|
|
|
async fn load_cached_adapter_response(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
prepared_request: &PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Option<AdapterResponse> {
|
|
let response_cache = self.response_cache.as_ref()?;
|
|
let cache_key = response_cache_key(operation, prepared_request, request_context)?;
|
|
let cached = match response_cache.get(&cache_key).await {
|
|
Ok(Some(cached)) => cached,
|
|
Ok(None) => {
|
|
record_cache_outcome(CacheOutcome::Miss);
|
|
return None;
|
|
}
|
|
Err(_) => {
|
|
record_cache_outcome(CacheOutcome::ReadError);
|
|
debug!(
|
|
name: "runtime.response_cache.read_failed",
|
|
operation_id = operation.operation_id.as_str(),
|
|
error_category = "response_cache",
|
|
"response cache lookup skipped"
|
|
);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
match adapter_response_from_cached(cached) {
|
|
Ok(response) => {
|
|
record_cache_outcome(CacheOutcome::Hit);
|
|
Some(response)
|
|
}
|
|
Err(_) => {
|
|
record_cache_outcome(CacheOutcome::DecodeError);
|
|
debug!(
|
|
name: "runtime.response_cache.decode_failed",
|
|
operation_id = operation.operation_id.as_str(),
|
|
error_category = "cached_response",
|
|
"cached response payload was invalid"
|
|
);
|
|
if response_cache.delete(&cache_key).await.is_err() {
|
|
record_cache_outcome(CacheOutcome::EvictError);
|
|
}
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn store_cached_adapter_response(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
prepared_request: &PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
adapter_response: &AdapterResponse,
|
|
) {
|
|
let Some(response_cache) = self.response_cache.as_ref() else {
|
|
return;
|
|
};
|
|
|
|
if !(200..=299).contains(&adapter_response.status_code) {
|
|
return;
|
|
}
|
|
|
|
let Some(cache_ttl) = response_cache_ttl(operation) else {
|
|
return;
|
|
};
|
|
let Some(cache_key) = response_cache_key(operation, prepared_request, request_context)
|
|
else {
|
|
return;
|
|
};
|
|
let Some(cached_response) = cached_response_from_adapter(adapter_response) else {
|
|
return;
|
|
};
|
|
|
|
if response_cache
|
|
.put(&cache_key, cached_response, cache_ttl)
|
|
.await
|
|
.is_err()
|
|
{
|
|
record_cache_outcome(CacheOutcome::WriteError);
|
|
debug!(
|
|
name: "runtime.response_cache.write_failed",
|
|
operation_id = operation.operation_id.as_str(),
|
|
error_category = "response_cache",
|
|
"response cache write skipped"
|
|
);
|
|
} else {
|
|
record_cache_outcome(CacheOutcome::Stored);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn adapter_request_context(
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> crank_core::RuntimeRequestContext {
|
|
request_context.map(Into::into).unwrap_or_else(|| {
|
|
crank_core::RuntimeRequestContext::from_request_id(Uuid::now_v7().to_string())
|
|
})
|
|
}
|
|
|
|
fn map_protocol_adapter_error(
|
|
operation: &RuntimeOperation,
|
|
error: crank_core::ProtocolAdapterError,
|
|
) -> RuntimeError {
|
|
match error {
|
|
crank_core::ProtocolAdapterError::UnsupportedMode { mode, .. } => {
|
|
RuntimeError::UnsupportedExecutionMode {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
mode,
|
|
}
|
|
}
|
|
crank_core::ProtocolAdapterError::Message(message) => RuntimeError::ProtocolAdapter(
|
|
format!("operation {}: {message}", operation.operation_id),
|
|
),
|
|
}
|
|
}
|
|
|
|
impl RuntimeExecutor {
|
|
fn adapter_for(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
) -> Result<SharedProtocolAdapter, RuntimeError> {
|
|
self.adapters
|
|
.get(operation.protocol)
|
|
.ok_or(RuntimeError::UnsupportedProtocol {
|
|
protocol: operation.protocol,
|
|
})
|
|
}
|
|
}
|
|
|
|
fn finalize_output(
|
|
operation: &RuntimeOperation,
|
|
response: &AdapterResponse,
|
|
) -> Result<Value, RuntimeError> {
|
|
let mapped = operation.output_mapping.apply(&json!({
|
|
"response": {
|
|
"body": response.body,
|
|
"data": response.data,
|
|
"headers": response.headers,
|
|
"status": response.status_code
|
|
}
|
|
}))?;
|
|
|
|
Ok(mapped
|
|
.get("output")
|
|
.cloned()
|
|
.unwrap_or_else(|| Value::Object(Map::new())))
|
|
}
|
|
|
|
fn transform_response(
|
|
operation: &RuntimeOperation,
|
|
response: &AdapterResponse,
|
|
) -> Result<Value, RuntimeError> {
|
|
let span = Stage::RuntimeResponseTransform.span();
|
|
let result = span.in_scope(|| {
|
|
let finalized_output = finalize_output(operation, response)?;
|
|
operation.output_schema.validate_shape(&finalized_output)?;
|
|
Ok(finalized_output)
|
|
});
|
|
match &result {
|
|
Ok(_) => StageOutcome::Success.record(&span),
|
|
Err(_) => {
|
|
StageOutcome::Error.record(&span);
|
|
ErrorCategory::Transformation.record(&span);
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
fn record_runtime_result<T>(span: &Span, result: &Result<T, RuntimeError>) {
|
|
match result {
|
|
Ok(_) => StageOutcome::Success.record(span),
|
|
Err(error) => {
|
|
StageOutcome::Error.record(span);
|
|
runtime_error_category(error).record(span);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
|
|
match error {
|
|
RuntimeError::Schema(_) => ErrorCategory::Schema,
|
|
RuntimeError::Mapping(_) | RuntimeError::InvalidPreparedRequest { .. } => {
|
|
ErrorCategory::Mapping
|
|
}
|
|
RuntimeError::RestAdapter(_)
|
|
| RuntimeError::ProtocolAdapter(_)
|
|
| RuntimeError::UnsupportedProtocol { .. }
|
|
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
|
|
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
|
|
RuntimeError::ConfirmationRequired { .. }
|
|
| RuntimeError::InvalidConfirmationToken { .. }
|
|
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
|
|
RuntimeError::IdempotencyStoreUnavailable { .. }
|
|
| RuntimeError::IdempotencyInProgress { .. }
|
|
| RuntimeError::IdempotencyConflict { .. }
|
|
| RuntimeError::IdempotencyOutcomeUnknown { .. } => ErrorCategory::Idempotency,
|
|
RuntimeError::MissingAuthProfile { .. }
|
|
| RuntimeError::MissingSecret { .. }
|
|
| RuntimeError::MissingSecretVersion { .. }
|
|
| RuntimeError::InvalidAuthSecretValue { .. }
|
|
| RuntimeError::SecretCrypto { .. } => ErrorCategory::Configuration,
|
|
}
|
|
}
|
|
|
|
fn try_acquire_limit(
|
|
limiter: Arc<Semaphore>,
|
|
kind: &'static str,
|
|
limit: usize,
|
|
) -> Result<OwnedSemaphorePermit, RuntimeError> {
|
|
limiter.try_acquire_owned().map_err(|_| {
|
|
record_limit_rejection(LimitStage::Concurrency);
|
|
RuntimeError::ConcurrencyLimitExceeded { kind, limit }
|
|
})
|
|
}
|
|
|
|
fn metric_invocation_source(
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> MetricInvocationSource {
|
|
request_context
|
|
.and_then(RuntimeRequestContext::metering_context)
|
|
.map_or(MetricInvocationSource::Internal, |context| {
|
|
match context.source {
|
|
InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun,
|
|
InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall,
|
|
}
|
|
})
|
|
}
|
|
|
|
fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
|
|
match error {
|
|
RuntimeError::Schema(_) => ToolErrorKind::Schema,
|
|
RuntimeError::Mapping(_) => ToolErrorKind::Mapping,
|
|
RuntimeError::RestAdapter(_) => ToolErrorKind::RestAdapter,
|
|
RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter,
|
|
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
|
|
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
|
|
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
|
|
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
|
|
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
|
|
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
|
|
RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
|
|
RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore,
|
|
RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress,
|
|
RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict,
|
|
RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown,
|
|
RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile,
|
|
RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret,
|
|
RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion,
|
|
RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret,
|
|
RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto,
|
|
}
|
|
}
|
|
|
|
fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome {
|
|
match error {
|
|
RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict,
|
|
RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress,
|
|
RuntimeError::IdempotencyOutcomeUnknown { .. } => IdempotencyOutcome::OutcomeUnknown,
|
|
RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable,
|
|
_ => IdempotencyOutcome::Error,
|
|
}
|
|
}
|
|
|
|
fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome {
|
|
match error {
|
|
RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken,
|
|
RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable,
|
|
_ => ConfirmationOutcome::Error,
|
|
}
|
|
}
|
|
|
|
fn log_runtime_event(
|
|
stage: &'static str,
|
|
operation: &RuntimeOperation,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) {
|
|
let protocol = match operation.protocol {
|
|
crank_core::Protocol::Rest => "rest",
|
|
};
|
|
if let Some(context) = request_context {
|
|
debug!(
|
|
name: "runtime.execution.stage_reached",
|
|
stage,
|
|
operation_id = operation.operation_id.as_str(),
|
|
protocol,
|
|
request_id = context.request_id.as_str(),
|
|
trace_id = context.trace_context.trace_id().as_str(),
|
|
"runtime execution"
|
|
);
|
|
} else {
|
|
debug!(
|
|
name: "runtime.execution.stage_reached",
|
|
stage,
|
|
operation_id = operation.operation_id.as_str(),
|
|
protocol,
|
|
"runtime execution"
|
|
);
|
|
}
|
|
}
|