CI / UI Checks (pull_request) Successful in 11s
CI / Rust Checks (pull_request) Successful in 12m45s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Community Release / release (push) Successful in 40m35s
Tool Draft фиксирует synchronous/background/adaptive, deadline и fallback; runtime применяет единый снимок политики (concurrency, очередь, approval, deadline) до внешнего действия. Насыщение отклоняется typed overload с retry guidance, mandatory safety store fail closed, background/adaptive используют опубликованный synchronous/unsupported fallback. Refs: #29
1122 lines
43 KiB
Rust
1122 lines
43 KiB
Rust
use std::sync::{
|
|
Arc,
|
|
atomic::{AtomicBool, Ordering},
|
|
};
|
|
use std::time::{Duration, Instant};
|
|
|
|
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
|
|
|
|
use crank_core::{
|
|
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
|
|
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
|
|
SharedProtocolAdapter, Target,
|
|
};
|
|
use crank_metrics::{
|
|
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
|
|
InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics,
|
|
ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome,
|
|
record_limit_rejection,
|
|
};
|
|
use crank_trace::{ErrorCategory, Stage, StageOutcome, set_parent_from_trace_context};
|
|
use serde_json::{Map, Value, json};
|
|
use time::OffsetDateTime;
|
|
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
|
use tracing::{Instrument, Span, debug, warn};
|
|
|
|
use crate::{
|
|
AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError,
|
|
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
|
|
auth::apply_resolved_auth,
|
|
request_preparation::{adapter_prepared_request, validate_selected_body_variant},
|
|
response_cache::{
|
|
adapter_response_from_cached, cached_response_from_adapter, response_cache_key,
|
|
response_cache_ttl,
|
|
},
|
|
};
|
|
|
|
#[derive(Clone)]
|
|
pub struct RuntimeExecutor {
|
|
adapters: AdapterRegistry,
|
|
limits: RuntimeLimits,
|
|
unary_limiter: Arc<Semaphore>,
|
|
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
|
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
|
admission: Option<Arc<dyn ExecutionAdmission>>,
|
|
metering_sink: SharedMeteringSink,
|
|
}
|
|
|
|
struct IdempotencyCancellationGuard {
|
|
cleanup: Option<IdempotencyCancellationCleanup>,
|
|
runtime: tokio::runtime::Handle,
|
|
}
|
|
|
|
struct IdempotencyCancellationCleanup {
|
|
store: Arc<dyn CoordinationStateStore>,
|
|
operation: RuntimeOperation,
|
|
reservation: crate::idempotency::IdempotencyReservation,
|
|
dispatch_started: Arc<AtomicBool>,
|
|
}
|
|
|
|
impl IdempotencyCancellationGuard {
|
|
fn new(
|
|
store: Arc<dyn CoordinationStateStore>,
|
|
operation: RuntimeOperation,
|
|
reservation: crate::idempotency::IdempotencyReservation,
|
|
dispatch_started: Arc<AtomicBool>,
|
|
) -> Self {
|
|
Self {
|
|
cleanup: Some(IdempotencyCancellationCleanup {
|
|
store,
|
|
operation,
|
|
reservation,
|
|
dispatch_started,
|
|
}),
|
|
runtime: tokio::runtime::Handle::current(),
|
|
}
|
|
}
|
|
|
|
fn disarm(&mut self) {
|
|
self.cleanup = None;
|
|
}
|
|
}
|
|
|
|
impl Drop for IdempotencyCancellationGuard {
|
|
fn drop(&mut self) {
|
|
let Some(cleanup) = self.cleanup.take() else {
|
|
return;
|
|
};
|
|
self.runtime.spawn(async move {
|
|
let may_have_dispatched = cleanup.dispatch_started.load(Ordering::Acquire);
|
|
let result = if may_have_dispatched {
|
|
crate::idempotency::mark_outcome_unknown(
|
|
cleanup.store.as_ref(),
|
|
&cleanup.operation,
|
|
&cleanup.reservation,
|
|
)
|
|
.await
|
|
} else {
|
|
crate::idempotency::release(
|
|
cleanup.store.as_ref(),
|
|
&cleanup.operation,
|
|
&cleanup.reservation,
|
|
)
|
|
.await
|
|
};
|
|
record_idempotency_outcome(match &result {
|
|
Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown,
|
|
Ok(()) => IdempotencyOutcome::Completed,
|
|
Err(error) => idempotency_error_outcome(error),
|
|
});
|
|
if result.is_err() {
|
|
warn!(
|
|
name: "runtime.idempotency.cancellation_cleanup_failed",
|
|
error_category = "idempotency_store",
|
|
"failed to finalize cancelled idempotent execution"
|
|
);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
impl Default for RuntimeExecutor {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
impl RuntimeExecutor {
|
|
pub fn new() -> Self {
|
|
crate::executor_builder::community_default().build()
|
|
}
|
|
|
|
pub fn with_limits(limits: RuntimeLimits) -> Self {
|
|
crate::executor_builder::community_default()
|
|
.with_limits(limits)
|
|
.build()
|
|
}
|
|
|
|
pub(crate) fn from_builder_parts(
|
|
limits: RuntimeLimits,
|
|
adapters: AdapterRegistry,
|
|
response_cache: Option<Arc<dyn ResponseCacheStore>>,
|
|
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
|
|
admission: Option<Arc<dyn ExecutionAdmission>>,
|
|
metering_sink: SharedMeteringSink,
|
|
) -> Self {
|
|
Self {
|
|
adapters,
|
|
unary_limiter: Arc::new(Semaphore::new(limits.max_concurrent_unary)),
|
|
limits,
|
|
response_cache,
|
|
coordination_store,
|
|
admission,
|
|
metering_sink,
|
|
}
|
|
}
|
|
|
|
pub fn with_response_cache_store(
|
|
mut self,
|
|
response_cache: Arc<dyn ResponseCacheStore>,
|
|
) -> Self {
|
|
self.response_cache = Some(response_cache);
|
|
self
|
|
}
|
|
|
|
pub fn with_coordination_store(
|
|
mut self,
|
|
coordination_store: Arc<dyn CoordinationStateStore>,
|
|
) -> Self {
|
|
self.coordination_store = Some(coordination_store);
|
|
self
|
|
}
|
|
|
|
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
|
|
self.admission = Some(admission);
|
|
self
|
|
}
|
|
|
|
pub(crate) async fn execute_request(
|
|
&self,
|
|
request: RuntimeExecutionRequest<'_>,
|
|
) -> Result<(Value, Value), RuntimeError> {
|
|
let operation = request.operation();
|
|
let input = request.input();
|
|
let request_context = request.request_context();
|
|
debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized);
|
|
debug_assert_eq!(
|
|
request_context
|
|
.metering_context()
|
|
.map(|value| &value.workspace_id),
|
|
Some(request.workspace_id())
|
|
);
|
|
debug_assert_eq!(
|
|
request_context
|
|
.metering_context()
|
|
.and_then(|value| value.agent_id.as_ref()),
|
|
request.agent_id()
|
|
);
|
|
let operation_may_mutate = operation_may_mutate(operation);
|
|
let execution_deadline =
|
|
crate::admission::effective_deadline(operation, request.deadline());
|
|
let runtime_span = Stage::RuntimeExecute.span();
|
|
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
|
|
log_runtime_event("unary.execute", operation, Some(request_context));
|
|
let started_at = Instant::now();
|
|
let dispatch_started = Arc::new(AtomicBool::new(false));
|
|
// True until the admission gate answered. A deadline that fires while the request is still
|
|
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
|
|
let admission_pending = Arc::new(AtomicBool::new(self.admission.is_some()));
|
|
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
|
|
metric_invocation_source(Some(request_context)),
|
|
request_context
|
|
.trace_context
|
|
.is_sampled()
|
|
.then(|| {
|
|
crank_metrics::ExemplarTraceId::parse(
|
|
&request_context.trace_context.trace_id().to_string(),
|
|
)
|
|
})
|
|
.flatten(),
|
|
);
|
|
let execution = async {
|
|
// One admission gate for every executable path (Story 4.1): the published policy decides
|
|
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
|
|
// is held for the whole local execution future, including retries.
|
|
let mut lease = crate::admission::acquire(
|
|
self.admission.as_ref(),
|
|
self.coordination_store.as_deref(),
|
|
&request,
|
|
execution_deadline,
|
|
)
|
|
.await?;
|
|
admission_pending.store(false, Ordering::Release);
|
|
let _permit = self.acquire_unary_permit(operation)?;
|
|
let _inflight = InFlightGuard::runtime();
|
|
let mapping_span = Stage::RuntimeArgumentsMap.span();
|
|
let prepared_request = mapping_span.in_scope(|| self.prepare_request(operation, input));
|
|
record_runtime_result(&mapping_span, &prepared_request);
|
|
drop(mapping_span);
|
|
let prepared_request = prepared_request?;
|
|
let request_preview = sanitized_request_preview(&prepared_request);
|
|
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
|
|
let result = self
|
|
.execute_prepared(
|
|
operation,
|
|
input,
|
|
prepared_request,
|
|
Some(request_context),
|
|
Arc::clone(&dispatch_started),
|
|
)
|
|
.await;
|
|
if let Some(lease) = lease.as_mut() {
|
|
lease.release().await;
|
|
}
|
|
let output = result?;
|
|
Ok((output, request_preview))
|
|
};
|
|
let result = tokio::time::timeout_at(
|
|
execution_deadline.into(),
|
|
execution.instrument(runtime_span.clone()),
|
|
)
|
|
.await
|
|
.unwrap_or_else(|_| {
|
|
if admission_pending.load(Ordering::Acquire) {
|
|
Err(crate::admission::deadline_refusal(operation))
|
|
} else {
|
|
Err(RuntimeError::ExecutionDeadlineElapsed {
|
|
may_have_dispatched: operation_may_mutate
|
|
&& dispatch_started.load(Ordering::Acquire),
|
|
})
|
|
}
|
|
});
|
|
record_runtime_result(&runtime_span, &result);
|
|
drop(runtime_span);
|
|
let (outcome, error_kind) = match &result {
|
|
Ok(_) => (ToolOutcome::Success, ToolErrorKind::None),
|
|
Err(error) => (ToolOutcome::Error, runtime_error_kind(error)),
|
|
};
|
|
invocation_metrics.complete(outcome, error_kind);
|
|
self.record_metering(operation, Some(request_context), &result, started_at)
|
|
.await;
|
|
result
|
|
}
|
|
|
|
pub async fn execute_outcome(
|
|
&self,
|
|
request: RuntimeExecutionRequest<'_>,
|
|
) -> Result<crank_core::ExecutionSuccess, crank_core::ExecutionFailure> {
|
|
let operation_id = request.operation().operation_id.clone();
|
|
let operation_version = request.operation().operation_version;
|
|
let operation = request.operation().clone();
|
|
let origin = request.origin();
|
|
let correlation = crank_core::CorrelationContext::new(
|
|
request.request_context().request_id.clone(),
|
|
request.request_context().trace_context.clone(),
|
|
);
|
|
match self.execute_request(request).await {
|
|
Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess {
|
|
operation_id,
|
|
operation_version,
|
|
origin,
|
|
correlation,
|
|
request_preview,
|
|
output,
|
|
}),
|
|
Err(error) => Err(crate::normalize_runtime_error_for_operation(
|
|
&error,
|
|
&correlation,
|
|
&operation,
|
|
)),
|
|
}
|
|
}
|
|
|
|
fn prepare_request(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
) -> Result<PreparedRequest, RuntimeError> {
|
|
operation
|
|
.input_schema
|
|
.validate_shape(input)
|
|
.map_err(RuntimeError::InputSchema)?;
|
|
validate_selected_body_variant(operation, input)?;
|
|
|
|
if operation.input_mapping.is_empty() {
|
|
return Ok(PreparedRequest::default());
|
|
}
|
|
|
|
let mapped_input = operation
|
|
.input_mapping
|
|
.apply(&json!({ "mcp": input }))
|
|
.map_err(RuntimeError::InputMapping)?;
|
|
let mut prepared = PreparedRequest::from_mapping_output(&mapped_input, &operation.target)?;
|
|
let Target::Rest(target) = &operation.target;
|
|
if let Some(contract) = &target.request_contract
|
|
&& input.get(&contract.body_field).is_none()
|
|
{
|
|
// Mapping initializes an empty body object for legacy field mappings.
|
|
// A typed optional body is absent unless its public argument was supplied.
|
|
prepared.body = None;
|
|
}
|
|
Ok(prepared)
|
|
}
|
|
|
|
async fn execute_prepared(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
prepared_request: PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
dispatch_started: Arc<AtomicBool>,
|
|
) -> Result<Value, RuntimeError> {
|
|
let mut prepared_request = prepared_request;
|
|
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
|
|
let idempotency_key = match crate::idempotency::prepare_idempotency(
|
|
operation,
|
|
input,
|
|
&mut prepared_request,
|
|
) {
|
|
Ok(key) => key,
|
|
Err(error) if idempotency_applicable => {
|
|
record_idempotency_outcome(idempotency_error_outcome(&error));
|
|
let span = Stage::RuntimeIdempotency.span();
|
|
StageOutcome::Error.record(&span);
|
|
ErrorCategory::Idempotency.record(&span);
|
|
return Err(error);
|
|
}
|
|
Err(error) => return Err(error),
|
|
};
|
|
|
|
if crate::confirmation::is_applicable(operation) {
|
|
let approval_span = Stage::ApprovalCheck.span();
|
|
let approval_result = crate::confirmation::confirm_operation(
|
|
self.coordination_store.as_deref(),
|
|
operation,
|
|
input,
|
|
request_context,
|
|
)
|
|
.instrument(approval_span.clone())
|
|
.await;
|
|
match &approval_result {
|
|
Ok(()) => {
|
|
StageOutcome::Success.record(&approval_span);
|
|
record_confirmation_outcome(ConfirmationOutcome::Approved);
|
|
}
|
|
Err(RuntimeError::ConfirmationRequired { .. }) => {
|
|
StageOutcome::Required.record(&approval_span);
|
|
ErrorCategory::Approval.record(&approval_span);
|
|
record_confirmation_outcome(ConfirmationOutcome::Required);
|
|
}
|
|
Err(error) => {
|
|
StageOutcome::Error.record(&approval_span);
|
|
runtime_error_category(error).record(&approval_span);
|
|
record_confirmation_outcome(confirmation_error_outcome(error));
|
|
}
|
|
}
|
|
drop(approval_span);
|
|
approval_result?;
|
|
}
|
|
|
|
let idempotency = if idempotency_applicable {
|
|
let idempotency_span = Stage::RuntimeIdempotency.span();
|
|
let result = crate::idempotency::begin(
|
|
self.coordination_store.as_deref(),
|
|
operation,
|
|
input,
|
|
idempotency_key.as_deref(),
|
|
request_context,
|
|
)
|
|
.instrument(idempotency_span.clone())
|
|
.await;
|
|
match &result {
|
|
Ok(crate::idempotency::IdempotencyAction::Execute(_)) => {
|
|
StageOutcome::Execute.record(&idempotency_span);
|
|
record_idempotency_outcome(IdempotencyOutcome::Execute);
|
|
}
|
|
Ok(crate::idempotency::IdempotencyAction::Replay(_)) => {
|
|
StageOutcome::Replay.record(&idempotency_span);
|
|
record_idempotency_outcome(IdempotencyOutcome::Replay);
|
|
}
|
|
Ok(crate::idempotency::IdempotencyAction::Disabled) => {
|
|
StageOutcome::Skipped.record(&idempotency_span);
|
|
}
|
|
Err(error) => {
|
|
StageOutcome::Error.record(&idempotency_span);
|
|
runtime_error_category(error).record(&idempotency_span);
|
|
record_idempotency_outcome(idempotency_error_outcome(error));
|
|
}
|
|
}
|
|
drop(idempotency_span);
|
|
result?
|
|
} else {
|
|
crate::idempotency::IdempotencyAction::Disabled
|
|
};
|
|
if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency {
|
|
return transform_response(operation, response);
|
|
}
|
|
let mut cancellation_guard =
|
|
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency {
|
|
self.coordination_store.as_ref().map(|store| {
|
|
IdempotencyCancellationGuard::new(
|
|
Arc::clone(store),
|
|
operation.clone(),
|
|
reservation.clone(),
|
|
Arc::clone(&dispatch_started),
|
|
)
|
|
})
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let adapter_result = match self
|
|
.load_cached_adapter_response(operation, &prepared_request, request_context)
|
|
.await
|
|
{
|
|
Some(response) => Ok(response),
|
|
None => {
|
|
let adapter_response = self
|
|
.execute_adapter_with_retry(
|
|
operation,
|
|
&prepared_request,
|
|
request_context,
|
|
Arc::clone(&dispatch_started),
|
|
crate::idempotency::verified_retry_key(
|
|
operation,
|
|
idempotency_key.as_deref(),
|
|
),
|
|
)
|
|
.await;
|
|
if let Ok(response) = &adapter_response {
|
|
self.store_cached_adapter_response(
|
|
operation,
|
|
&prepared_request,
|
|
request_context,
|
|
response,
|
|
)
|
|
.await;
|
|
}
|
|
adapter_response
|
|
}
|
|
};
|
|
let adapter_response = match adapter_result {
|
|
Ok(response) => response,
|
|
Err(error) => {
|
|
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
|
|
&& let Some(store) = self.coordination_store.as_deref()
|
|
{
|
|
let idempotency_span = Stage::RuntimeIdempotency.span();
|
|
let may_have_dispatched = runtime_error_may_have_dispatched(&error);
|
|
let cleanup_result = if may_have_dispatched {
|
|
crate::idempotency::mark_outcome_unknown(store, operation, reservation)
|
|
.instrument(idempotency_span.clone())
|
|
.await
|
|
} else {
|
|
crate::idempotency::release(store, operation, reservation)
|
|
.instrument(idempotency_span.clone())
|
|
.await
|
|
};
|
|
record_runtime_result(&idempotency_span, &cleanup_result);
|
|
record_idempotency_outcome(match &cleanup_result {
|
|
Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown,
|
|
Ok(()) => IdempotencyOutcome::Completed,
|
|
Err(error) => idempotency_error_outcome(error),
|
|
});
|
|
if let Some(guard) = &mut cancellation_guard {
|
|
guard.disarm();
|
|
}
|
|
}
|
|
return Err(error);
|
|
}
|
|
};
|
|
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
|
|
&& let Some(store) = self.coordination_store.as_deref()
|
|
{
|
|
let idempotency_span = Stage::RuntimeIdempotency.span();
|
|
let completion_result =
|
|
crate::idempotency::complete(store, operation, reservation, &adapter_response)
|
|
.instrument(idempotency_span.clone())
|
|
.await;
|
|
record_runtime_result(&idempotency_span, &completion_result);
|
|
record_idempotency_outcome(match &completion_result {
|
|
Ok(()) => IdempotencyOutcome::Completed,
|
|
Err(error) => idempotency_error_outcome(error),
|
|
});
|
|
if let Some(guard) = &mut cancellation_guard {
|
|
guard.disarm();
|
|
}
|
|
drop(idempotency_span);
|
|
if completion_result.is_err() {
|
|
return Err(RuntimeError::IdempotencyCompletionUnknown {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
});
|
|
}
|
|
}
|
|
transform_response(operation, &adapter_response)
|
|
}
|
|
|
|
async fn record_metering<T>(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
result: &Result<T, RuntimeError>,
|
|
started_at: Instant,
|
|
) {
|
|
let Some(context) = request_context.and_then(RuntimeRequestContext::metering_context)
|
|
else {
|
|
return;
|
|
};
|
|
|
|
self.metering_sink
|
|
.record(MeteringEvent {
|
|
workspace_id: context.workspace_id.clone(),
|
|
agent_id: context.agent_id.clone(),
|
|
operation_id: operation.operation_id.clone(),
|
|
source: context.source,
|
|
status: if result.is_ok() {
|
|
InvocationStatus::Ok
|
|
} else {
|
|
InvocationStatus::Error
|
|
},
|
|
duration_ms: started_at.elapsed().as_millis() as u64,
|
|
created_at: OffsetDateTime::now_utc(),
|
|
})
|
|
.await;
|
|
}
|
|
|
|
async fn execute_adapter(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
prepared_request: PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
dispatch_started: Arc<AtomicBool>,
|
|
) -> Result<AdapterResponse, RuntimeError> {
|
|
log_runtime_event("adapter.dispatch", operation, request_context);
|
|
let adapter = self.adapter_for(operation)?;
|
|
if !adapter.supports_mode(ExecutionMode::Unary) {
|
|
return Err(RuntimeError::UnsupportedExecutionMode {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
mode: ExecutionMode::Unary,
|
|
});
|
|
}
|
|
|
|
let prepared_request = adapter_prepared_request(
|
|
operation,
|
|
&prepared_request,
|
|
operation.execution_config.timeout_ms,
|
|
);
|
|
let adapter_context =
|
|
adapter_request_context(request_context, Arc::clone(&dispatch_started))?;
|
|
adapter
|
|
.invoke_unary(
|
|
&operation.target,
|
|
&prepared_request.into(),
|
|
&adapter_context,
|
|
)
|
|
.await
|
|
.map(Into::into)
|
|
.map_err(|error| map_protocol_adapter_error(operation, error))
|
|
}
|
|
|
|
async fn execute_adapter_with_retry(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
prepared_request: &PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
dispatch_started: Arc<AtomicBool>,
|
|
verified_idempotency: bool,
|
|
) -> Result<AdapterResponse, RuntimeError> {
|
|
let max_attempts = retry_attempts(operation);
|
|
let mut attempt = 1;
|
|
loop {
|
|
let result = self
|
|
.execute_adapter(
|
|
operation,
|
|
prepared_request.clone(),
|
|
request_context,
|
|
Arc::clone(&dispatch_started),
|
|
)
|
|
.await;
|
|
match result {
|
|
Ok(response) => return Ok(response),
|
|
Err(error)
|
|
if attempt < max_attempts
|
|
&& retry_is_allowed(operation, &error, verified_idempotency) =>
|
|
{
|
|
attempt += 1;
|
|
tokio::time::sleep(retry_backoff(attempt)).await;
|
|
}
|
|
Err(error) => return Err(error),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn acquire_unary_permit(
|
|
&self,
|
|
_operation: &RuntimeOperation,
|
|
) -> Result<OwnedSemaphorePermit, RuntimeError> {
|
|
try_acquire_limit(
|
|
Arc::clone(&self.unary_limiter),
|
|
"unary",
|
|
self.limits.max_concurrent_unary,
|
|
)
|
|
}
|
|
|
|
async fn load_cached_adapter_response(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
prepared_request: &PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Option<AdapterResponse> {
|
|
let response_cache = self.response_cache.as_ref()?;
|
|
let cache_key = response_cache_key(operation, prepared_request, request_context)?;
|
|
let cached =
|
|
match tokio::time::timeout(RESPONSE_CACHE_IO_TIMEOUT, response_cache.get(&cache_key))
|
|
.await
|
|
{
|
|
Ok(Ok(Some(cached))) => cached,
|
|
Ok(Ok(None)) => {
|
|
record_cache_outcome(CacheOutcome::Miss);
|
|
return None;
|
|
}
|
|
Ok(Err(_)) | Err(_) => {
|
|
record_cache_outcome(CacheOutcome::ReadError);
|
|
debug!(
|
|
name: "runtime.response_cache.read_failed",
|
|
operation_id = operation.operation_id.as_str(),
|
|
error_category = "response_cache",
|
|
"response cache lookup skipped"
|
|
);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
match adapter_response_from_cached(cached) {
|
|
Ok(response) => {
|
|
record_cache_outcome(CacheOutcome::Hit);
|
|
Some(response)
|
|
}
|
|
Err(_) => {
|
|
record_cache_outcome(CacheOutcome::DecodeError);
|
|
debug!(
|
|
name: "runtime.response_cache.decode_failed",
|
|
operation_id = operation.operation_id.as_str(),
|
|
error_category = "cached_response",
|
|
"cached response payload was invalid"
|
|
);
|
|
if response_cache.delete(&cache_key).await.is_err() {
|
|
record_cache_outcome(CacheOutcome::EvictError);
|
|
}
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn store_cached_adapter_response(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
prepared_request: &PreparedRequest,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
adapter_response: &AdapterResponse,
|
|
) {
|
|
let Some(response_cache) = self.response_cache.as_ref() else {
|
|
return;
|
|
};
|
|
|
|
if !(200..=299).contains(&adapter_response.status_code) {
|
|
return;
|
|
}
|
|
if request_context
|
|
.and_then(RuntimeRequestContext::resource_response_cache)
|
|
.is_some_and(|policy| {
|
|
serde_json::to_vec(&adapter_response.data)
|
|
.map_or(true, |bytes| bytes.len() as u64 > policy.max_bytes)
|
|
})
|
|
{
|
|
return;
|
|
}
|
|
|
|
let Some(cache_ttl) = response_cache_ttl(operation, request_context) else {
|
|
return;
|
|
};
|
|
let Some(cache_key) = response_cache_key(operation, prepared_request, request_context)
|
|
else {
|
|
return;
|
|
};
|
|
let Some(cached_response) = cached_response_from_adapter(adapter_response) else {
|
|
return;
|
|
};
|
|
|
|
if !matches!(
|
|
tokio::time::timeout(
|
|
RESPONSE_CACHE_IO_TIMEOUT,
|
|
response_cache.put(&cache_key, cached_response, cache_ttl),
|
|
)
|
|
.await,
|
|
Ok(Ok(()))
|
|
) {
|
|
record_cache_outcome(CacheOutcome::WriteError);
|
|
debug!(
|
|
name: "runtime.response_cache.write_failed",
|
|
operation_id = operation.operation_id.as_str(),
|
|
error_category = "response_cache",
|
|
"response cache write skipped"
|
|
);
|
|
} else {
|
|
record_cache_outcome(CacheOutcome::Stored);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn sanitized_request_preview(prepared: &PreparedRequest) -> Value {
|
|
json!({
|
|
"path_parameter_count": prepared.path_params.len(),
|
|
"query_parameter_count": prepared.query_params.len(),
|
|
"header_count": prepared.headers.len(),
|
|
"body_configured": prepared.body.is_some(),
|
|
})
|
|
}
|
|
|
|
fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool {
|
|
matches!(
|
|
error,
|
|
RuntimeError::ProtocolAdapter(
|
|
crank_core::ProtocolAdapterError::Transport {
|
|
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
|
} | crank_core::ProtocolAdapterError::Timeout {
|
|
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
|
} | crank_core::ProtocolAdapterError::ResponseTooLarge {
|
|
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
|
} | crank_core::ProtocolAdapterError::UnexpectedStatus {
|
|
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
|
..
|
|
} | crank_core::ProtocolAdapterError::InvalidResponse {
|
|
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
|
|
},
|
|
) | RuntimeError::ExecutionDeadlineElapsed {
|
|
may_have_dispatched: true,
|
|
} | RuntimeError::IdempotencyCompletionUnknown { .. }
|
|
)
|
|
}
|
|
|
|
fn retry_attempts(operation: &RuntimeOperation) -> u32 {
|
|
operation
|
|
.execution_config
|
|
.retry_policy
|
|
.as_ref()
|
|
.map_or(1, |policy| policy.max_attempts.clamp(1, 5))
|
|
}
|
|
|
|
fn retry_backoff(attempt: u32) -> Duration {
|
|
Duration::from_millis(u64::from(attempt.saturating_sub(1)).min(4))
|
|
}
|
|
|
|
fn retry_is_allowed(
|
|
operation: &RuntimeOperation,
|
|
error: &RuntimeError,
|
|
verified_idempotency: bool,
|
|
) -> bool {
|
|
let is_read = matches!(
|
|
&operation.target,
|
|
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
|
|
);
|
|
let safe_write = operation_may_mutate(operation) && verified_idempotency;
|
|
if !(is_read || safe_write) {
|
|
return false;
|
|
}
|
|
match error {
|
|
RuntimeError::ProtocolAdapter(
|
|
crank_core::ProtocolAdapterError::Transport { .. }
|
|
| crank_core::ProtocolAdapterError::Timeout { .. },
|
|
) => true,
|
|
RuntimeError::ProtocolAdapter(crank_core::ProtocolAdapterError::UnexpectedStatus {
|
|
status,
|
|
..
|
|
}) => matches!(status, 408 | 429 | 500..=599),
|
|
_ => false,
|
|
}
|
|
}
|
|
|
|
fn operation_may_mutate(operation: &RuntimeOperation) -> bool {
|
|
matches!(
|
|
&operation.target,
|
|
crank_core::Target::Rest(target)
|
|
if matches!(
|
|
target.method,
|
|
crank_core::HttpMethod::Post
|
|
| crank_core::HttpMethod::Put
|
|
| crank_core::HttpMethod::Patch
|
|
| crank_core::HttpMethod::Delete
|
|
)
|
|
)
|
|
}
|
|
|
|
fn adapter_request_context(
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
dispatch_started: Arc<AtomicBool>,
|
|
) -> Result<crank_core::RuntimeRequestContext, RuntimeError> {
|
|
request_context
|
|
.map(|context| {
|
|
crank_core::RuntimeRequestContext::from(context).with_dispatch_started(dispatch_started)
|
|
})
|
|
.ok_or(RuntimeError::InvalidExecutionRequest)
|
|
}
|
|
|
|
fn map_protocol_adapter_error(
|
|
operation: &RuntimeOperation,
|
|
error: crank_core::ProtocolAdapterError,
|
|
) -> RuntimeError {
|
|
match error {
|
|
crank_core::ProtocolAdapterError::UnsupportedMode { mode, .. } => {
|
|
RuntimeError::UnsupportedExecutionMode {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
mode,
|
|
}
|
|
}
|
|
error => RuntimeError::ProtocolAdapter(error),
|
|
}
|
|
}
|
|
|
|
impl RuntimeExecutor {
|
|
fn adapter_for(
|
|
&self,
|
|
operation: &RuntimeOperation,
|
|
) -> Result<SharedProtocolAdapter, RuntimeError> {
|
|
self.adapters
|
|
.get(operation.protocol)
|
|
.ok_or(RuntimeError::UnsupportedProtocol {
|
|
protocol: operation.protocol,
|
|
})
|
|
}
|
|
}
|
|
|
|
fn finalize_output(
|
|
operation: &RuntimeOperation,
|
|
response: &AdapterResponse,
|
|
) -> Result<Value, RuntimeError> {
|
|
let response_data = if matches!(
|
|
&operation.target,
|
|
Target::Rest(target) if target.method == HttpMethod::Head && response.body.is_null()
|
|
) {
|
|
// HEAD has no body; expose only safe status metadata for an explicit output mapping.
|
|
json!({"status": response.status_code})
|
|
} else {
|
|
response.data.clone()
|
|
};
|
|
let mapped = operation
|
|
.output_mapping
|
|
.apply(&json!({
|
|
"response": {
|
|
"body": response.body,
|
|
"data": response_data,
|
|
"headers": response.headers,
|
|
"status": response.status_code
|
|
}
|
|
}))
|
|
.map_err(RuntimeError::OutputMapping)?;
|
|
|
|
Ok(mapped
|
|
.get("output")
|
|
.cloned()
|
|
.unwrap_or_else(|| Value::Object(Map::new())))
|
|
}
|
|
|
|
fn transform_response(
|
|
operation: &RuntimeOperation,
|
|
response: &AdapterResponse,
|
|
) -> Result<Value, RuntimeError> {
|
|
let span = Stage::RuntimeResponseTransform.span();
|
|
let result = span.in_scope(|| {
|
|
let finalized_output = finalize_output(operation, response)?;
|
|
output_schema_for_response(operation, response)?
|
|
.validate_shape(&finalized_output)
|
|
.map_err(RuntimeError::OutputSchema)?;
|
|
Ok(finalized_output)
|
|
});
|
|
match &result {
|
|
Ok(_) => StageOutcome::Success.record(&span),
|
|
Err(_) => {
|
|
StageOutcome::Error.record(&span);
|
|
ErrorCategory::Transformation.record(&span);
|
|
}
|
|
}
|
|
result
|
|
}
|
|
|
|
fn output_schema_for_response<'a>(
|
|
operation: &'a RuntimeOperation,
|
|
response: &AdapterResponse,
|
|
) -> Result<&'a crank_schema::Schema, RuntimeError> {
|
|
let Target::Rest(target) = &operation.target;
|
|
if target.response_contract.is_empty() {
|
|
return Ok(&operation.output_schema);
|
|
}
|
|
let media_type = response
|
|
.headers
|
|
.get("content-type")
|
|
.and_then(|value| value.split(';').next())
|
|
.map(str::trim);
|
|
let descriptor = target
|
|
.response_contract
|
|
.iter()
|
|
.filter(|descriptor| descriptor.status_code == response.status_code)
|
|
.find(|descriptor| match (&descriptor.media_type, media_type) {
|
|
(Some(expected), Some(actual)) => expected.eq_ignore_ascii_case(actual),
|
|
(None, _) => response.body.is_null(),
|
|
_ => false,
|
|
})
|
|
.ok_or(RuntimeError::InvalidExecutionRequest)?;
|
|
if target.response_contract.len() > 1
|
|
&& operation.output_schema.kind == crank_schema::SchemaKind::Oneof
|
|
{
|
|
return operation
|
|
.output_schema
|
|
.variants
|
|
.get(descriptor.schema_variant)
|
|
.ok_or(RuntimeError::InvalidExecutionRequest);
|
|
}
|
|
(descriptor.schema_variant == 0)
|
|
.then_some(&operation.output_schema)
|
|
.ok_or(RuntimeError::InvalidExecutionRequest)
|
|
}
|
|
|
|
fn record_runtime_result<T>(span: &Span, result: &Result<T, RuntimeError>) {
|
|
match result {
|
|
Ok(_) => StageOutcome::Success.record(span),
|
|
Err(error) => {
|
|
StageOutcome::Error.record(span);
|
|
runtime_error_category(error).record(span);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
|
|
match error {
|
|
RuntimeError::InvalidExecutionRequest => ErrorCategory::Configuration,
|
|
RuntimeError::ExecutionDeadlineElapsed { .. } => ErrorCategory::Upstream,
|
|
RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => {
|
|
ErrorCategory::Schema
|
|
}
|
|
RuntimeError::Mapping(_)
|
|
| RuntimeError::InputMapping(_)
|
|
| RuntimeError::OutputMapping(_)
|
|
| RuntimeError::InvalidPreparedRequest { .. } => ErrorCategory::Mapping,
|
|
RuntimeError::ProtocolAdapter(_)
|
|
| RuntimeError::UnsupportedProtocol { .. }
|
|
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
|
|
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
|
|
RuntimeError::AdmissionOverloaded { .. }
|
|
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
|
|
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
|
|
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
|
|
RuntimeError::ConfirmationRequired { .. }
|
|
| RuntimeError::InvalidConfirmationToken { .. }
|
|
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
|
|
RuntimeError::IdempotencyStoreUnavailable { .. }
|
|
| RuntimeError::IdempotencyInProgress { .. }
|
|
| RuntimeError::IdempotencyConflict { .. }
|
|
| RuntimeError::IdempotencyOutcomeUnknown { .. }
|
|
| RuntimeError::IdempotencyCompletionUnknown { .. } => ErrorCategory::Idempotency,
|
|
RuntimeError::MissingAuthProfile { .. }
|
|
| RuntimeError::InvalidAuthProfileConfig { .. }
|
|
| RuntimeError::AuthorizationStoreUnavailable
|
|
| RuntimeError::MissingSecret { .. }
|
|
| RuntimeError::MissingSecretVersion { .. }
|
|
| RuntimeError::InvalidAuthSecretValue { .. }
|
|
| RuntimeError::SecretCrypto { .. } => ErrorCategory::Configuration,
|
|
}
|
|
}
|
|
|
|
fn try_acquire_limit(
|
|
limiter: Arc<Semaphore>,
|
|
kind: &'static str,
|
|
limit: usize,
|
|
) -> Result<OwnedSemaphorePermit, RuntimeError> {
|
|
limiter.try_acquire_owned().map_err(|_| {
|
|
record_limit_rejection(LimitStage::Concurrency);
|
|
RuntimeError::ConcurrencyLimitExceeded { kind, limit }
|
|
})
|
|
}
|
|
|
|
fn metric_invocation_source(
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> MetricInvocationSource {
|
|
request_context
|
|
.and_then(RuntimeRequestContext::metering_context)
|
|
.map_or(MetricInvocationSource::Internal, |context| {
|
|
match context.source {
|
|
InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun,
|
|
InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall,
|
|
}
|
|
})
|
|
}
|
|
|
|
fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
|
|
match error {
|
|
RuntimeError::InvalidExecutionRequest => ToolErrorKind::InvalidPreparedRequest,
|
|
RuntimeError::ExecutionDeadlineElapsed { .. } => ToolErrorKind::RestAdapter,
|
|
RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => {
|
|
ToolErrorKind::Schema
|
|
}
|
|
RuntimeError::Mapping(_)
|
|
| RuntimeError::InputMapping(_)
|
|
| RuntimeError::OutputMapping(_) => ToolErrorKind::Mapping,
|
|
RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter,
|
|
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
|
|
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
|
|
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
|
|
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
|
|
// label domain stays fixed; the precise code lives in the execution failure/invocation log
|
|
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
|
|
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
|
|
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
|
|
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
|
|
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
|
|
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
|
|
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
|
|
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
|
|
RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
|
|
RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore,
|
|
RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress,
|
|
RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict,
|
|
RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown,
|
|
RuntimeError::IdempotencyCompletionUnknown { .. } => {
|
|
ToolErrorKind::IdempotencyOutcomeUnknown
|
|
}
|
|
RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile,
|
|
RuntimeError::InvalidAuthProfileConfig { .. } => ToolErrorKind::InvalidAuthSecret,
|
|
RuntimeError::AuthorizationStoreUnavailable => ToolErrorKind::IdempotencyStore,
|
|
RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret,
|
|
RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion,
|
|
RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret,
|
|
RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto,
|
|
}
|
|
}
|
|
|
|
fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome {
|
|
match error {
|
|
RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict,
|
|
RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress,
|
|
RuntimeError::IdempotencyOutcomeUnknown { .. }
|
|
| RuntimeError::IdempotencyCompletionUnknown { .. } => IdempotencyOutcome::OutcomeUnknown,
|
|
RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable,
|
|
_ => IdempotencyOutcome::Error,
|
|
}
|
|
}
|
|
|
|
fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome {
|
|
match error {
|
|
RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken,
|
|
RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable,
|
|
_ => ConfirmationOutcome::Error,
|
|
}
|
|
}
|
|
|
|
fn log_runtime_event(
|
|
stage: &'static str,
|
|
operation: &RuntimeOperation,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) {
|
|
let protocol = match operation.protocol {
|
|
crank_core::Protocol::Rest => "rest",
|
|
};
|
|
if let Some(context) = request_context {
|
|
debug!(
|
|
name: "runtime.execution.stage_reached",
|
|
stage,
|
|
operation_id = operation.operation_id.as_str(),
|
|
protocol,
|
|
request_id = context.request_id.as_str(),
|
|
trace_id = context.trace_context.trace_id().as_str(),
|
|
"runtime execution"
|
|
);
|
|
} else {
|
|
debug!(
|
|
name: "runtime.execution.stage_reached",
|
|
stage,
|
|
operation_id = operation.operation_id.as_str(),
|
|
protocol,
|
|
"runtime execution"
|
|
);
|
|
}
|
|
}
|