Files
crank/crates/crank-runtime/src/executor.rs
T
bsodfather f9f8c625d1
CI / UI Checks (pull_request) Successful in 15s
CI / Rust Checks (pull_request) Failing after 4m58s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Failing after 1s
CI / Deploy (pull_request) Skipped
feat(epic4): исполнять durable Task через PG lease/fence и восстанавливать после сбоев (Story 4.3)
mcp-server запускает сохранённые фоновые задачи: захват по PostgreSQL lease с
fence-токеном, зашифрованный input и повторная проверка доступа до сетевого
вызова, checkpoint с AttemptID до dispatch и восстановление только по PG.
Устаревший worker не может зафиксировать результат.

Добавлены миграции v26–v28 (V25 не меняется), guard их отсутствия в старой
схеме, метрики/трассы попыток (attempt_trace) и остановка до отправки запроса:
стоп в окне после checkpoint и до permit завершает задачу как
failed(shutdown_before_dispatch) без сетевых вызовов, соседние queued-задачи
остаются доступными. Retry-попытки получают отдельные дочерние spans с durable
current_attempt_id. Shared admission/fairness из Story 4.1 сохранены.

Проверки: durable_tasks 13/13, stages 11/11 (retry 5), runtime lib 47,
bin shutdown 1, registry tasks 16, admission 11, migrations 58 (независимая QA
и исходный runtime-прогон), точный exact-head CI остаётся обязательным.
2026-10-29 22:26:52 +00:00

1212 lines
46 KiB
Rust

use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
use std::time::{Duration, Instant};
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
use crank_core::{
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
SharedProtocolAdapter, Target,
};
use crank_metrics::{
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics,
ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome,
record_limit_rejection,
};
use crank_trace::{ErrorCategory, Stage, StageOutcome, set_parent_from_trace_context};
use serde_json::{Map, Value, json};
use time::OffsetDateTime;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tracing::{Instrument, Span, debug, warn};
use uuid::Uuid;
use crate::task_worker::TaskDispatchPermit;
use crate::{
AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError,
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
auth::apply_resolved_auth,
request_preparation::{adapter_prepared_request, validate_selected_body_variant},
response_cache::{
adapter_response_from_cached, cached_response_from_adapter, response_cache_key,
response_cache_ttl,
},
};
#[derive(Clone)]
pub struct RuntimeExecutor {
adapters: AdapterRegistry,
limits: RuntimeLimits,
unary_limiter: Arc<Semaphore>,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
struct IdempotencyCancellationGuard {
cleanup: Option<IdempotencyCancellationCleanup>,
runtime: tokio::runtime::Handle,
}
struct IdempotencyCancellationCleanup {
store: Arc<dyn CoordinationStateStore>,
operation: RuntimeOperation,
reservation: crate::idempotency::IdempotencyReservation,
dispatch_started: Arc<AtomicBool>,
}
impl IdempotencyCancellationGuard {
fn new(
store: Arc<dyn CoordinationStateStore>,
operation: RuntimeOperation,
reservation: crate::idempotency::IdempotencyReservation,
dispatch_started: Arc<AtomicBool>,
) -> Self {
Self {
cleanup: Some(IdempotencyCancellationCleanup {
store,
operation,
reservation,
dispatch_started,
}),
runtime: tokio::runtime::Handle::current(),
}
}
fn disarm(&mut self) {
self.cleanup = None;
}
}
impl Drop for IdempotencyCancellationGuard {
fn drop(&mut self) {
let Some(cleanup) = self.cleanup.take() else {
return;
};
self.runtime.spawn(async move {
let may_have_dispatched = cleanup.dispatch_started.load(Ordering::Acquire);
let result = if may_have_dispatched {
crate::idempotency::mark_outcome_unknown(
cleanup.store.as_ref(),
&cleanup.operation,
&cleanup.reservation,
)
.await
} else {
crate::idempotency::release(
cleanup.store.as_ref(),
&cleanup.operation,
&cleanup.reservation,
)
.await
};
record_idempotency_outcome(match &result {
Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown,
Ok(()) => IdempotencyOutcome::Completed,
Err(error) => idempotency_error_outcome(error),
});
if result.is_err() {
warn!(
name: "runtime.idempotency.cancellation_cleanup_failed",
error_category = "idempotency_store",
"failed to finalize cancelled idempotent execution"
);
}
});
}
}
impl Default for RuntimeExecutor {
fn default() -> Self {
Self::new()
}
}
impl RuntimeExecutor {
pub fn new() -> Self {
crate::executor_builder::community_default().build()
}
pub fn with_limits(limits: RuntimeLimits) -> Self {
crate::executor_builder::community_default()
.with_limits(limits)
.build()
}
pub(crate) fn from_builder_parts(
limits: RuntimeLimits,
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
) -> Self {
Self {
adapters,
unary_limiter: Arc::new(Semaphore::new(limits.max_concurrent_unary)),
limits,
response_cache,
coordination_store,
admission,
metering_sink,
}
}
pub fn with_response_cache_store(
mut self,
response_cache: Arc<dyn ResponseCacheStore>,
) -> Self {
self.response_cache = Some(response_cache);
self
}
pub fn with_coordination_store(
mut self,
coordination_store: Arc<dyn CoordinationStateStore>,
) -> Self {
self.coordination_store = Some(coordination_store);
self
}
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub(crate) async fn execute_request(
&self,
request: RuntimeExecutionRequest<'_>,
) -> Result<(Value, Value), RuntimeError> {
self.execute_request_impl(request, None).await
}
pub async fn execute_task_outcome(
&self,
request: RuntimeExecutionRequest<'_>,
permit: TaskDispatchPermit,
) -> Result<crank_core::ExecutionSuccess, crank_core::ExecutionFailure> {
let operation = request.operation().clone();
let operation_id = operation.operation_id.clone();
let operation_version = operation.operation_version;
let origin = request.origin();
let correlation = crank_core::CorrelationContext::new(
request.request_context().request_id.clone(),
request.request_context().trace_context.clone(),
);
// Validate permit binding before any adapter dispatch.
if permit.operation_id() != operation_id.as_str()
|| permit.operation_version() != operation_version
|| permit.workspace_id() != request.workspace_id().as_str()
|| permit.agent_id() != request.agent_id().map(|value| value.as_str())
{
return Err(crate::normalize_runtime_error_for_operation(
&RuntimeError::InvalidExecutionRequest,
&correlation,
&operation,
));
}
if !permit.consume() {
return Err(crate::normalize_runtime_error_for_operation(
&RuntimeError::InvalidExecutionRequest,
&correlation,
&operation,
));
}
match self.execute_request_impl(request, Some(permit)).await {
Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess {
operation_id: operation_id.clone(),
operation_version,
origin,
correlation,
request_preview,
output,
}),
Err(error) => Err(crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&operation,
)),
}
}
pub(crate) async fn execute_request_impl(
&self,
request: RuntimeExecutionRequest<'_>,
permit: Option<TaskDispatchPermit>,
) -> Result<(Value, Value), RuntimeError> {
let operation = request.operation();
// Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission.
let trusted_task_dispatch = permit.is_some();
// The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the
// trace names the real persisted attempt instead of inventing one (Story 4.3 AC).
let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned());
let input = request.input();
let request_context = request.request_context();
debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized);
debug_assert_eq!(
request_context
.metering_context()
.map(|value| &value.workspace_id),
Some(request.workspace_id())
);
debug_assert_eq!(
request_context
.metering_context()
.and_then(|value| value.agent_id.as_ref()),
request.agent_id()
);
let operation_may_mutate = operation_may_mutate(operation);
let execution_deadline =
crate::admission::effective_deadline(operation, request.deadline());
let runtime_span = Stage::RuntimeExecute.span();
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
log_runtime_event("unary.execute", operation, Some(request_context));
let started_at = Instant::now();
let dispatch_started = Arc::new(AtomicBool::new(false));
// True until the admission gate answered. A deadline that fires while the request is still
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
let admission_pending = Arc::new(AtomicBool::new(
!trusted_task_dispatch && self.admission.is_some(),
));
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
metric_invocation_source(Some(request_context)),
request_context
.trace_context
.is_sampled()
.then(|| {
crank_metrics::ExemplarTraceId::parse(
&request_context.trace_context.trace_id().to_string(),
)
})
.flatten(),
);
let execution = async {
// One admission gate for every executable path (Story 4.1): the published policy decides
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
// is held for the whole local execution future, including retries.
let mut lease = if trusted_task_dispatch {
None
} else {
let result = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
result
};
let _permit = self.acquire_unary_permit(operation)?;
let _inflight = InFlightGuard::runtime();
let mapping_span = Stage::RuntimeArgumentsMap.span();
let prepared_request = mapping_span.in_scope(|| self.prepare_request(operation, input));
record_runtime_result(&mapping_span, &prepared_request);
drop(mapping_span);
let prepared_request = prepared_request?;
let request_preview = sanitized_request_preview(&prepared_request);
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
let result = self
.execute_prepared(
operation,
input,
prepared_request,
Some(request_context),
Arc::clone(&dispatch_started),
durable_attempt_id.as_deref(),
)
.await;
if let Some(lease) = lease.as_mut() {
lease.release().await;
}
let output = result?;
Ok((output, request_preview))
};
let result = tokio::time::timeout_at(
execution_deadline.into(),
execution.instrument(runtime_span.clone()),
)
.await
.unwrap_or_else(|_| {
if admission_pending.load(Ordering::Acquire) {
Err(crate::admission::deadline_refusal(operation))
} else {
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
}
});
record_runtime_result(&runtime_span, &result);
drop(runtime_span);
let (outcome, error_kind) = match &result {
Ok(_) => (ToolOutcome::Success, ToolErrorKind::None),
Err(error) => (ToolOutcome::Error, runtime_error_kind(error)),
};
invocation_metrics.complete(outcome, error_kind);
self.record_metering(operation, Some(request_context), &result, started_at)
.await;
result
}
pub async fn execute_outcome(
&self,
request: RuntimeExecutionRequest<'_>,
) -> Result<crank_core::ExecutionSuccess, crank_core::ExecutionFailure> {
let operation_id = request.operation().operation_id.clone();
let operation_version = request.operation().operation_version;
let operation = request.operation().clone();
let origin = request.origin();
let correlation = crank_core::CorrelationContext::new(
request.request_context().request_id.clone(),
request.request_context().trace_context.clone(),
);
match self.execute_request(request).await {
Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess {
operation_id,
operation_version,
origin,
correlation,
request_preview,
output,
}),
Err(error) => Err(crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&operation,
)),
}
}
fn prepare_request(
&self,
operation: &RuntimeOperation,
input: &Value,
) -> Result<PreparedRequest, RuntimeError> {
operation
.input_schema
.validate_shape(input)
.map_err(RuntimeError::InputSchema)?;
validate_selected_body_variant(operation, input)?;
if operation.input_mapping.is_empty() {
return Ok(PreparedRequest::default());
}
let mapped_input = operation
.input_mapping
.apply(&json!({ "mcp": input }))
.map_err(RuntimeError::InputMapping)?;
let mut prepared = PreparedRequest::from_mapping_output(&mapped_input, &operation.target)?;
let Target::Rest(target) = &operation.target;
if let Some(contract) = &target.request_contract
&& input.get(&contract.body_field).is_none()
{
// Mapping initializes an empty body object for legacy field mappings.
// A typed optional body is absent unless its public argument was supplied.
prepared.body = None;
}
Ok(prepared)
}
async fn execute_prepared(
&self,
operation: &RuntimeOperation,
input: &Value,
prepared_request: PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
durable_attempt_id: Option<&str>,
) -> Result<Value, RuntimeError> {
let mut prepared_request = prepared_request;
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
let idempotency_key = match crate::idempotency::prepare_idempotency(
operation,
input,
&mut prepared_request,
) {
Ok(key) => key,
Err(error) if idempotency_applicable => {
record_idempotency_outcome(idempotency_error_outcome(&error));
let span = Stage::RuntimeIdempotency.span();
StageOutcome::Error.record(&span);
ErrorCategory::Idempotency.record(&span);
return Err(error);
}
Err(error) => return Err(error),
};
if crate::confirmation::is_applicable(operation) {
let approval_span = Stage::ApprovalCheck.span();
let approval_result = crate::confirmation::confirm_operation(
self.coordination_store.as_deref(),
operation,
input,
request_context,
)
.instrument(approval_span.clone())
.await;
match &approval_result {
Ok(()) => {
StageOutcome::Success.record(&approval_span);
record_confirmation_outcome(ConfirmationOutcome::Approved);
}
Err(RuntimeError::ConfirmationRequired { .. }) => {
StageOutcome::Required.record(&approval_span);
ErrorCategory::Approval.record(&approval_span);
record_confirmation_outcome(ConfirmationOutcome::Required);
}
Err(error) => {
StageOutcome::Error.record(&approval_span);
runtime_error_category(error).record(&approval_span);
record_confirmation_outcome(confirmation_error_outcome(error));
}
}
drop(approval_span);
approval_result?;
}
let idempotency = if idempotency_applicable {
let idempotency_span = Stage::RuntimeIdempotency.span();
let result = crate::idempotency::begin(
self.coordination_store.as_deref(),
operation,
input,
idempotency_key.as_deref(),
request_context,
)
.instrument(idempotency_span.clone())
.await;
match &result {
Ok(crate::idempotency::IdempotencyAction::Execute(_)) => {
StageOutcome::Execute.record(&idempotency_span);
record_idempotency_outcome(IdempotencyOutcome::Execute);
}
Ok(crate::idempotency::IdempotencyAction::Replay(_)) => {
StageOutcome::Replay.record(&idempotency_span);
record_idempotency_outcome(IdempotencyOutcome::Replay);
}
Ok(crate::idempotency::IdempotencyAction::Disabled) => {
StageOutcome::Skipped.record(&idempotency_span);
}
Err(error) => {
StageOutcome::Error.record(&idempotency_span);
runtime_error_category(error).record(&idempotency_span);
record_idempotency_outcome(idempotency_error_outcome(error));
}
}
drop(idempotency_span);
result?
} else {
crate::idempotency::IdempotencyAction::Disabled
};
if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency {
return transform_response(operation, response);
}
let mut cancellation_guard =
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency {
self.coordination_store.as_ref().map(|store| {
IdempotencyCancellationGuard::new(
Arc::clone(store),
operation.clone(),
reservation.clone(),
Arc::clone(&dispatch_started),
)
})
} else {
None
};
let adapter_result = match self
.load_cached_adapter_response(operation, &prepared_request, request_context)
.await
{
Some(response) => Ok(response),
None => {
let adapter_response = self
.execute_adapter_with_retry(
operation,
&prepared_request,
request_context,
Arc::clone(&dispatch_started),
crate::idempotency::verified_retry_key(
operation,
idempotency_key.as_deref(),
),
durable_attempt_id,
)
.await;
if let Ok(response) = &adapter_response {
self.store_cached_adapter_response(
operation,
&prepared_request,
request_context,
response,
)
.await;
}
adapter_response
}
};
let adapter_response = match adapter_result {
Ok(response) => response,
Err(error) => {
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
&& let Some(store) = self.coordination_store.as_deref()
{
let idempotency_span = Stage::RuntimeIdempotency.span();
let may_have_dispatched = runtime_error_may_have_dispatched(&error);
let cleanup_result = if may_have_dispatched {
crate::idempotency::mark_outcome_unknown(store, operation, reservation)
.instrument(idempotency_span.clone())
.await
} else {
crate::idempotency::release(store, operation, reservation)
.instrument(idempotency_span.clone())
.await
};
record_runtime_result(&idempotency_span, &cleanup_result);
record_idempotency_outcome(match &cleanup_result {
Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown,
Ok(()) => IdempotencyOutcome::Completed,
Err(error) => idempotency_error_outcome(error),
});
if let Some(guard) = &mut cancellation_guard {
guard.disarm();
}
}
return Err(error);
}
};
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
&& let Some(store) = self.coordination_store.as_deref()
{
let idempotency_span = Stage::RuntimeIdempotency.span();
let completion_result =
crate::idempotency::complete(store, operation, reservation, &adapter_response)
.instrument(idempotency_span.clone())
.await;
record_runtime_result(&idempotency_span, &completion_result);
record_idempotency_outcome(match &completion_result {
Ok(()) => IdempotencyOutcome::Completed,
Err(error) => idempotency_error_outcome(error),
});
if let Some(guard) = &mut cancellation_guard {
guard.disarm();
}
drop(idempotency_span);
if completion_result.is_err() {
return Err(RuntimeError::IdempotencyCompletionUnknown {
operation_id: operation.operation_id.as_str().to_owned(),
});
}
}
transform_response(operation, &adapter_response)
}
async fn record_metering<T>(
&self,
operation: &RuntimeOperation,
request_context: Option<&RuntimeRequestContext>,
result: &Result<T, RuntimeError>,
started_at: Instant,
) {
let Some(context) = request_context.and_then(RuntimeRequestContext::metering_context)
else {
return;
};
self.metering_sink
.record(MeteringEvent {
workspace_id: context.workspace_id.clone(),
agent_id: context.agent_id.clone(),
operation_id: operation.operation_id.clone(),
source: context.source,
status: if result.is_ok() {
InvocationStatus::Ok
} else {
InvocationStatus::Error
},
duration_ms: started_at.elapsed().as_millis() as u64,
created_at: OffsetDateTime::now_utc(),
})
.await;
}
async fn execute_adapter(
&self,
operation: &RuntimeOperation,
prepared_request: PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
) -> Result<AdapterResponse, RuntimeError> {
log_runtime_event("adapter.dispatch", operation, request_context);
let adapter = self.adapter_for(operation)?;
if !adapter.supports_mode(ExecutionMode::Unary) {
return Err(RuntimeError::UnsupportedExecutionMode {
operation_id: operation.operation_id.as_str().to_owned(),
mode: ExecutionMode::Unary,
});
}
let prepared_request = adapter_prepared_request(
operation,
&prepared_request,
operation.execution_config.timeout_ms,
);
let adapter_context =
adapter_request_context(request_context, Arc::clone(&dispatch_started))?;
adapter
.invoke_unary(
&operation.target,
&prepared_request.into(),
&adapter_context,
)
.await
.map(Into::into)
.map_err(|error| map_protocol_adapter_error(operation, error))
}
async fn execute_adapter_with_retry(
&self,
operation: &RuntimeOperation,
prepared_request: &PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
verified_idempotency: bool,
durable_attempt_id: Option<&str>,
) -> Result<AdapterResponse, RuntimeError> {
let max_attempts = retry_attempts(operation);
let mut attempt = 1;
loop {
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint; the
// retry index discriminates the attempts inside it. Only paths without a durable attempt
// (synchronous tools/call) mint a synthetic per-attempt id.
let attempt_id = durable_attempt_id
.map(str::to_owned)
.unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7()));
let attempt_span = tracing::info_span!(
"adapter_attempt",
attempt,
max_attempts,
attempt_id = %attempt_id
);
let result = self
.execute_adapter(
operation,
prepared_request.clone(),
request_context,
Arc::clone(&dispatch_started),
)
.instrument(attempt_span)
.await;
match result {
Ok(response) => return Ok(response),
Err(error)
if attempt < max_attempts
&& retry_is_allowed(operation, &error, verified_idempotency) =>
{
attempt += 1;
tokio::time::sleep(retry_backoff(attempt)).await;
}
Err(error) => return Err(error),
}
}
}
fn acquire_unary_permit(
&self,
_operation: &RuntimeOperation,
) -> Result<OwnedSemaphorePermit, RuntimeError> {
try_acquire_limit(
Arc::clone(&self.unary_limiter),
"unary",
self.limits.max_concurrent_unary,
)
}
async fn load_cached_adapter_response(
&self,
operation: &RuntimeOperation,
prepared_request: &PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
) -> Option<AdapterResponse> {
let response_cache = self.response_cache.as_ref()?;
let cache_key = response_cache_key(operation, prepared_request, request_context)?;
let cached =
match tokio::time::timeout(RESPONSE_CACHE_IO_TIMEOUT, response_cache.get(&cache_key))
.await
{
Ok(Ok(Some(cached))) => cached,
Ok(Ok(None)) => {
record_cache_outcome(CacheOutcome::Miss);
return None;
}
Ok(Err(_)) | Err(_) => {
record_cache_outcome(CacheOutcome::ReadError);
debug!(
name: "runtime.response_cache.read_failed",
operation_id = operation.operation_id.as_str(),
error_category = "response_cache",
"response cache lookup skipped"
);
return None;
}
};
match adapter_response_from_cached(cached) {
Ok(response) => {
record_cache_outcome(CacheOutcome::Hit);
Some(response)
}
Err(_) => {
record_cache_outcome(CacheOutcome::DecodeError);
debug!(
name: "runtime.response_cache.decode_failed",
operation_id = operation.operation_id.as_str(),
error_category = "cached_response",
"cached response payload was invalid"
);
if response_cache.delete(&cache_key).await.is_err() {
record_cache_outcome(CacheOutcome::EvictError);
}
None
}
}
}
async fn store_cached_adapter_response(
&self,
operation: &RuntimeOperation,
prepared_request: &PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
adapter_response: &AdapterResponse,
) {
let Some(response_cache) = self.response_cache.as_ref() else {
return;
};
if !(200..=299).contains(&adapter_response.status_code) {
return;
}
if request_context
.and_then(RuntimeRequestContext::resource_response_cache)
.is_some_and(|policy| {
serde_json::to_vec(&adapter_response.data)
.map_or(true, |bytes| bytes.len() as u64 > policy.max_bytes)
})
{
return;
}
let Some(cache_ttl) = response_cache_ttl(operation, request_context) else {
return;
};
let Some(cache_key) = response_cache_key(operation, prepared_request, request_context)
else {
return;
};
let Some(cached_response) = cached_response_from_adapter(adapter_response) else {
return;
};
if !matches!(
tokio::time::timeout(
RESPONSE_CACHE_IO_TIMEOUT,
response_cache.put(&cache_key, cached_response, cache_ttl),
)
.await,
Ok(Ok(()))
) {
record_cache_outcome(CacheOutcome::WriteError);
debug!(
name: "runtime.response_cache.write_failed",
operation_id = operation.operation_id.as_str(),
error_category = "response_cache",
"response cache write skipped"
);
} else {
record_cache_outcome(CacheOutcome::Stored);
}
}
}
fn sanitized_request_preview(prepared: &PreparedRequest) -> Value {
json!({
"path_parameter_count": prepared.path_params.len(),
"query_parameter_count": prepared.query_params.len(),
"header_count": prepared.headers.len(),
"body_configured": prepared.body.is_some(),
})
}
fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool {
matches!(
error,
RuntimeError::ProtocolAdapter(
crank_core::ProtocolAdapterError::Transport {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
} | crank_core::ProtocolAdapterError::Timeout {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
} | crank_core::ProtocolAdapterError::ResponseTooLarge {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
} | crank_core::ProtocolAdapterError::UnexpectedStatus {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
..
} | crank_core::ProtocolAdapterError::InvalidResponse {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
},
) | RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: true,
} | RuntimeError::IdempotencyCompletionUnknown { .. }
)
}
fn retry_attempts(operation: &RuntimeOperation) -> u32 {
operation
.execution_config
.retry_policy
.as_ref()
.map_or(1, |policy| policy.max_attempts.clamp(1, 5))
}
fn retry_backoff(attempt: u32) -> Duration {
Duration::from_millis(u64::from(attempt.saturating_sub(1)).min(4))
}
fn retry_is_allowed(
operation: &RuntimeOperation,
error: &RuntimeError,
verified_idempotency: bool,
) -> bool {
let is_read = matches!(
&operation.target,
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
);
let safe_write = operation_may_mutate(operation) && verified_idempotency;
if !(is_read || safe_write) {
return false;
}
match error {
RuntimeError::ProtocolAdapter(
crank_core::ProtocolAdapterError::Transport { .. }
| crank_core::ProtocolAdapterError::Timeout { .. },
) => true,
RuntimeError::ProtocolAdapter(crank_core::ProtocolAdapterError::UnexpectedStatus {
status,
..
}) => matches!(status, 408 | 429 | 500..=599),
_ => false,
}
}
fn operation_may_mutate(operation: &RuntimeOperation) -> bool {
matches!(
&operation.target,
crank_core::Target::Rest(target)
if matches!(
target.method,
crank_core::HttpMethod::Post
| crank_core::HttpMethod::Put
| crank_core::HttpMethod::Patch
| crank_core::HttpMethod::Delete
)
)
}
fn adapter_request_context(
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
) -> Result<crank_core::RuntimeRequestContext, RuntimeError> {
request_context
.map(|context| {
crank_core::RuntimeRequestContext::from(context).with_dispatch_started(dispatch_started)
})
.ok_or(RuntimeError::InvalidExecutionRequest)
}
fn map_protocol_adapter_error(
operation: &RuntimeOperation,
error: crank_core::ProtocolAdapterError,
) -> RuntimeError {
match error {
crank_core::ProtocolAdapterError::UnsupportedMode { mode, .. } => {
RuntimeError::UnsupportedExecutionMode {
operation_id: operation.operation_id.as_str().to_owned(),
mode,
}
}
error => RuntimeError::ProtocolAdapter(error),
}
}
impl RuntimeExecutor {
fn adapter_for(
&self,
operation: &RuntimeOperation,
) -> Result<SharedProtocolAdapter, RuntimeError> {
self.adapters
.get(operation.protocol)
.ok_or(RuntimeError::UnsupportedProtocol {
protocol: operation.protocol,
})
}
}
fn finalize_output(
operation: &RuntimeOperation,
response: &AdapterResponse,
) -> Result<Value, RuntimeError> {
let response_data = if matches!(
&operation.target,
Target::Rest(target) if target.method == HttpMethod::Head && response.body.is_null()
) {
// HEAD has no body; expose only safe status metadata for an explicit output mapping.
json!({"status": response.status_code})
} else {
response.data.clone()
};
let mapped = operation
.output_mapping
.apply(&json!({
"response": {
"body": response.body,
"data": response_data,
"headers": response.headers,
"status": response.status_code
}
}))
.map_err(RuntimeError::OutputMapping)?;
Ok(mapped
.get("output")
.cloned()
.unwrap_or_else(|| Value::Object(Map::new())))
}
fn transform_response(
operation: &RuntimeOperation,
response: &AdapterResponse,
) -> Result<Value, RuntimeError> {
let span = Stage::RuntimeResponseTransform.span();
let result = span.in_scope(|| {
let finalized_output = finalize_output(operation, response)?;
output_schema_for_response(operation, response)?
.validate_shape(&finalized_output)
.map_err(RuntimeError::OutputSchema)?;
Ok(finalized_output)
});
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Transformation.record(&span);
}
}
result
}
fn output_schema_for_response<'a>(
operation: &'a RuntimeOperation,
response: &AdapterResponse,
) -> Result<&'a crank_schema::Schema, RuntimeError> {
let Target::Rest(target) = &operation.target;
if target.response_contract.is_empty() {
return Ok(&operation.output_schema);
}
let media_type = response
.headers
.get("content-type")
.and_then(|value| value.split(';').next())
.map(str::trim);
let descriptor = target
.response_contract
.iter()
.filter(|descriptor| descriptor.status_code == response.status_code)
.find(|descriptor| match (&descriptor.media_type, media_type) {
(Some(expected), Some(actual)) => expected.eq_ignore_ascii_case(actual),
(None, _) => response.body.is_null(),
_ => false,
})
.ok_or(RuntimeError::InvalidExecutionRequest)?;
if target.response_contract.len() > 1
&& operation.output_schema.kind == crank_schema::SchemaKind::Oneof
{
return operation
.output_schema
.variants
.get(descriptor.schema_variant)
.ok_or(RuntimeError::InvalidExecutionRequest);
}
(descriptor.schema_variant == 0)
.then_some(&operation.output_schema)
.ok_or(RuntimeError::InvalidExecutionRequest)
}
fn record_runtime_result<T>(span: &Span, result: &Result<T, RuntimeError>) {
match result {
Ok(_) => StageOutcome::Success.record(span),
Err(error) => {
StageOutcome::Error.record(span);
runtime_error_category(error).record(span);
}
}
}
fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
match error {
RuntimeError::InvalidExecutionRequest => ErrorCategory::Configuration,
RuntimeError::ExecutionDeadlineElapsed { .. } => ErrorCategory::Upstream,
RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => {
ErrorCategory::Schema
}
RuntimeError::Mapping(_)
| RuntimeError::InputMapping(_)
| RuntimeError::OutputMapping(_)
| RuntimeError::InvalidPreparedRequest { .. } => ErrorCategory::Mapping,
RuntimeError::ProtocolAdapter(_)
| RuntimeError::UnsupportedProtocol { .. }
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
RuntimeError::AdmissionOverloaded { .. }
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
RuntimeError::ConfirmationRequired { .. }
| RuntimeError::InvalidConfirmationToken { .. }
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
RuntimeError::IdempotencyStoreUnavailable { .. }
| RuntimeError::IdempotencyInProgress { .. }
| RuntimeError::IdempotencyConflict { .. }
| RuntimeError::IdempotencyOutcomeUnknown { .. }
| RuntimeError::IdempotencyCompletionUnknown { .. } => ErrorCategory::Idempotency,
RuntimeError::MissingAuthProfile { .. }
| RuntimeError::InvalidAuthProfileConfig { .. }
| RuntimeError::AuthorizationStoreUnavailable
| RuntimeError::MissingSecret { .. }
| RuntimeError::MissingSecretVersion { .. }
| RuntimeError::InvalidAuthSecretValue { .. }
| RuntimeError::SecretCrypto { .. } => ErrorCategory::Configuration,
}
}
fn try_acquire_limit(
limiter: Arc<Semaphore>,
kind: &'static str,
limit: usize,
) -> Result<OwnedSemaphorePermit, RuntimeError> {
limiter.try_acquire_owned().map_err(|_| {
record_limit_rejection(LimitStage::Concurrency);
RuntimeError::ConcurrencyLimitExceeded { kind, limit }
})
}
fn metric_invocation_source(
request_context: Option<&RuntimeRequestContext>,
) -> MetricInvocationSource {
request_context
.and_then(RuntimeRequestContext::metering_context)
.map_or(MetricInvocationSource::Internal, |context| {
match context.source {
InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun,
InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall,
}
})
}
fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
match error {
RuntimeError::InvalidExecutionRequest => ToolErrorKind::InvalidPreparedRequest,
RuntimeError::ExecutionDeadlineElapsed { .. } => ToolErrorKind::RestAdapter,
RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => {
ToolErrorKind::Schema
}
RuntimeError::Mapping(_)
| RuntimeError::InputMapping(_)
| RuntimeError::OutputMapping(_) => ToolErrorKind::Mapping,
RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter,
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
// label domain stays fixed; the precise code lives in the execution failure/invocation log
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore,
RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress,
RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict,
RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown,
RuntimeError::IdempotencyCompletionUnknown { .. } => {
ToolErrorKind::IdempotencyOutcomeUnknown
}
RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile,
RuntimeError::InvalidAuthProfileConfig { .. } => ToolErrorKind::InvalidAuthSecret,
RuntimeError::AuthorizationStoreUnavailable => ToolErrorKind::IdempotencyStore,
RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret,
RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion,
RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret,
RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto,
}
}
fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome {
match error {
RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict,
RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress,
RuntimeError::IdempotencyOutcomeUnknown { .. }
| RuntimeError::IdempotencyCompletionUnknown { .. } => IdempotencyOutcome::OutcomeUnknown,
RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable,
_ => IdempotencyOutcome::Error,
}
}
fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome {
match error {
RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken,
RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable,
_ => ConfirmationOutcome::Error,
}
}
fn log_runtime_event(
stage: &'static str,
operation: &RuntimeOperation,
request_context: Option<&RuntimeRequestContext>,
) {
let protocol = match operation.protocol {
crank_core::Protocol::Rest => "rest",
};
if let Some(context) = request_context {
debug!(
name: "runtime.execution.stage_reached",
stage,
operation_id = operation.operation_id.as_str(),
protocol,
request_id = context.request_id.as_str(),
trace_id = context.trace_context.trace_id().as_str(),
"runtime execution"
);
} else {
debug!(
name: "runtime.execution.stage_reached",
stage,
operation_id = operation.operation_id.as_str(),
protocol,
"runtime execution"
);
}
}