Files
crank/crates/crank-runtime/src/executor.rs
T
bsodfather 90e5357a08
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): продолжение ожидающей Task после ввода (Story 4.5)
Durable worker приостанавливает Task как input_required до dispatch, если
закреплённая схема требует отсутствующее поле; authorized schema-validated
update атомарно и с аудитом возобновляет тот же Task ровно один раз.
Replay/conflict/TTL безопасны, ожидание удерживает admission permit.
Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
2026-10-30 16:24:12 +00:00

1498 lines
60 KiB
Rust

use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
use std::time::{Duration, Instant};
const RESPONSE_CACHE_IO_TIMEOUT: Duration = Duration::from_millis(100);
use crank_core::{
AdapterRegistry, CoordinationStateStore, ExecutionAdmission, ExecutionMode, HttpMethod,
InvocationSource, InvocationStatus, MeteringEvent, ResponseCacheStore, SharedMeteringSink,
SharedProtocolAdapter, Target,
};
use crank_metrics::{
CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics,
ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome,
record_limit_rejection,
};
use crank_trace::{ErrorCategory, Stage, StageOutcome, set_parent_from_trace_context};
use serde_json::{Map, Value, json};
use time::OffsetDateTime;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tracing::{Instrument, Span, debug, warn};
use uuid::Uuid;
use crate::task_attempt::{
PrepareAttempt, TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal,
};
use crate::task_worker::TaskDispatchPermit;
use crate::{
AdapterResponse, ExecutionAuthorization, ExecutionRequestError, PreparedRequest, RuntimeError,
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
auth::apply_resolved_auth,
request_preparation::{adapter_prepared_request, validate_selected_body_variant},
response_cache::{
adapter_response_from_cached, cached_response_from_adapter, response_cache_key,
response_cache_ttl,
},
};
/// How many outbound sends one prepared request may perform.
///
/// `Legacy` keeps the existing synchronous `tools/call` retry behaviour inside the transport loop.
/// `Single` performs exactly one send, because a durable Task attempt already has its own committed
/// checkpoint and resolution around every send; it carries the pinned attempt ordinal and budget so
/// the one `adapter_attempt` span still names the real durable attempt.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum SendMode {
Legacy,
Single { attempt: u32, max_attempts: u32 },
}
#[derive(Clone)]
pub struct RuntimeExecutor {
adapters: AdapterRegistry,
limits: RuntimeLimits,
unary_limiter: Arc<Semaphore>,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
}
struct IdempotencyCancellationGuard {
cleanup: Option<IdempotencyCancellationCleanup>,
runtime: tokio::runtime::Handle,
}
struct IdempotencyCancellationCleanup {
store: Arc<dyn CoordinationStateStore>,
operation: RuntimeOperation,
reservation: crate::idempotency::IdempotencyReservation,
dispatch_started: Arc<AtomicBool>,
}
impl IdempotencyCancellationGuard {
fn new(
store: Arc<dyn CoordinationStateStore>,
operation: RuntimeOperation,
reservation: crate::idempotency::IdempotencyReservation,
dispatch_started: Arc<AtomicBool>,
) -> Self {
Self {
cleanup: Some(IdempotencyCancellationCleanup {
store,
operation,
reservation,
dispatch_started,
}),
runtime: tokio::runtime::Handle::current(),
}
}
fn disarm(&mut self) {
self.cleanup = None;
}
}
impl Drop for IdempotencyCancellationGuard {
fn drop(&mut self) {
let Some(cleanup) = self.cleanup.take() else {
return;
};
self.runtime.spawn(async move {
let may_have_dispatched = cleanup.dispatch_started.load(Ordering::Acquire);
let result = if may_have_dispatched {
crate::idempotency::mark_outcome_unknown(
cleanup.store.as_ref(),
&cleanup.operation,
&cleanup.reservation,
)
.await
} else {
crate::idempotency::release(
cleanup.store.as_ref(),
&cleanup.operation,
&cleanup.reservation,
)
.await
};
record_idempotency_outcome(match &result {
Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown,
Ok(()) => IdempotencyOutcome::Completed,
Err(error) => idempotency_error_outcome(error),
});
if result.is_err() {
warn!(
name: "runtime.idempotency.cancellation_cleanup_failed",
error_category = "idempotency_store",
"failed to finalize cancelled idempotent execution"
);
}
});
}
}
impl Default for RuntimeExecutor {
fn default() -> Self {
Self::new()
}
}
impl RuntimeExecutor {
pub fn new() -> Self {
crate::executor_builder::community_default().build()
}
pub fn with_limits(limits: RuntimeLimits) -> Self {
crate::executor_builder::community_default()
.with_limits(limits)
.build()
}
pub(crate) fn from_builder_parts(
limits: RuntimeLimits,
adapters: AdapterRegistry,
response_cache: Option<Arc<dyn ResponseCacheStore>>,
coordination_store: Option<Arc<dyn CoordinationStateStore>>,
admission: Option<Arc<dyn ExecutionAdmission>>,
metering_sink: SharedMeteringSink,
) -> Self {
Self {
adapters,
unary_limiter: Arc::new(Semaphore::new(limits.max_concurrent_unary)),
limits,
response_cache,
coordination_store,
admission,
metering_sink,
}
}
pub fn with_response_cache_store(
mut self,
response_cache: Arc<dyn ResponseCacheStore>,
) -> Self {
self.response_cache = Some(response_cache);
self
}
pub fn with_coordination_store(
mut self,
coordination_store: Arc<dyn CoordinationStateStore>,
) -> Self {
self.coordination_store = Some(coordination_store);
self
}
pub fn with_admission(mut self, admission: Arc<dyn ExecutionAdmission>) -> Self {
self.admission = Some(admission);
self
}
pub(crate) async fn execute_request(
&self,
request: RuntimeExecutionRequest<'_>,
) -> Result<(Value, Value), RuntimeError> {
self.execute_request_impl(request, None, SendMode::Legacy)
.await
}
/// Drives the bounded, durable retry budget of one leased background Task (Story 4.4).
///
/// Each outbound send is preceded by its own committed attempt checkpoint and followed by one
/// atomic resolution; the pinned budget, the safe-retry decision and the accepted absolute
/// deadline are enforced by PostgreSQL, so a restart or a second worker cannot extend them.
pub async fn execute_task_outcome(
&self,
context: &TaskAttemptContext,
authority: &dyn TaskAttemptAuthority,
) -> TaskAttemptOutcome {
let store = context.store().clone();
let lease = context.lease().clone();
let mut record = lease.record.clone();
loop {
if context.is_cancelled() || context.is_stopping() {
return TaskAttemptOutcome::Stopped;
}
let current_lease = crank_core::TaskLease {
record: record.clone(),
owner: lease.owner.clone(),
fence: lease.fence,
lease_until: lease.lease_until,
};
let plan = match authority.prepare_attempt(&current_lease).await {
Ok(PrepareAttempt::Ready(plan)) => *plan,
Ok(PrepareAttempt::InputRequired(request)) => {
// A missing required input is suspended before the dispatch checkpoint, so this
// generation provably produced no outbound side effect. A CAS/store failure stops
// without writing anything new; the lease expiry recovery makes it retryable.
return match store.suspend_for_input(&current_lease, request).await {
Ok(_) => TaskAttemptOutcome::WaitingForInput,
Err(crank_core::TaskError::StateConflict)
| Err(crank_core::TaskError::NotFound)
| Err(crank_core::TaskError::FingerprintConflict)
| Err(crank_core::TaskError::InputExpired) => TaskAttemptOutcome::Stopped,
Err(error) => {
warn!(error = %error, "input suspension failed; no outbound call");
TaskAttemptOutcome::Stopped
}
};
}
Err(TaskAttemptRefusal::Stopped) => return TaskAttemptOutcome::Stopped,
Err(TaskAttemptRefusal::Failed { code }) => {
// A refused attempt never erases a proven ambiguity: if an earlier attempt of this
// Task may already have reached the upstream, the store keeps the Task in the
// non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
return match store.refuse_attempt(&current_lease, &code).await {
Ok(record) if record.state == crank_core::TaskState::OutcomeUnknown => {
TaskAttemptOutcome::OutcomeUnknown
}
_ => TaskAttemptOutcome::Failed { code },
};
}
};
let attempt_id = format!("attempt-{}", Uuid::now_v7());
let begin = crank_core::BeginAttempt {
task_id: record.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: record.version,
attempt_id: attempt_id.clone(),
input_fingerprint: record.effective_input_fingerprint.clone(),
upstream_key_digest: plan.upstream_key_digest.clone(),
retry_allowed: plan.retry_allowed,
};
let attempt = match store.checkpoint_dispatch(begin).await {
Ok(attempt) => attempt,
// Cancellation, lease loss or a concurrent lifecycle change: nothing was dispatched.
Err(crank_core::TaskError::StateConflict)
| Err(crank_core::TaskError::NotFound)
| Err(crank_core::TaskError::FingerprintConflict) => {
return TaskAttemptOutcome::Stopped;
}
// An unproven commit is never retried by guess: no outbound call without a proven
// durable checkpoint.
Err(error) => {
warn!(error = %error, "durable dispatch checkpoint failed; no outbound call");
return TaskAttemptOutcome::Stopped;
}
};
if context.is_stopping() {
let code = "shutdown_before_dispatch".to_owned();
// A stop that lands before the send is a proven failure of *this* attempt only: when
// an earlier attempt of the same Task already lost its confirmation, the store keeps
// the non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
return match store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Failed {
error: bounded_attempt_error(
&code,
crank_core::AttemptPhase::BeforeDispatch,
),
retryable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await
{
Ok(crank_core::AttemptResolution::Unknown(_)) => {
TaskAttemptOutcome::OutcomeUnknown
}
_ => TaskAttemptOutcome::Failed { code },
};
}
if context.is_cancelled() {
let _ = store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Uncertain {
error: bounded_attempt_error(
"outcome_unknown",
crank_core::AttemptPhase::BeforeDispatch,
),
repeatable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await;
return TaskAttemptOutcome::OutcomeUnknown;
}
// Each attempt owns exactly one `adapter_attempt` span, created around the single send
// inside the transport loop so the accepted trace shape (durable attempt → runtime
// execution → adapter attempt) is preserved.
let result = self.execute_task_send(&plan, &attempt).await;
let (outcome, code) = self.classify_task_attempt(&plan, result);
match store.resolve_attempt(&attempt, outcome).await {
Ok(crank_core::AttemptResolution::Finished(task)) => {
return if task.state == crank_core::TaskState::Completed {
TaskAttemptOutcome::Completed
} else {
TaskAttemptOutcome::Failed {
code: code.unwrap_or_else(|| "task_failed".to_owned()),
}
};
}
Ok(crank_core::AttemptResolution::Unknown(_)) => {
return TaskAttemptOutcome::OutcomeUnknown;
}
Ok(crank_core::AttemptResolution::RetryAt {
task,
next_attempt_at_unix_ms,
}) => {
record = task;
let remaining =
next_attempt_at_unix_ms.saturating_sub(crank_core::current_unix_ms());
let delay = Duration::from_millis(u64::try_from(remaining).unwrap_or(0));
if !self.wait_before_attempt(context, delay).await {
return TaskAttemptOutcome::Stopped;
}
}
// A stale fence or version lost the resolution CAS: another writer owns the Task now.
Err(_) => return TaskAttemptOutcome::Stopped,
}
}
}
/// One outbound send of one committed durable attempt.
async fn execute_task_send(
&self,
plan: &TaskAttemptPlan,
attempt: &crank_core::AttemptLease,
) -> Result<Value, RuntimeError> {
let permit = TaskDispatchPermit::new(
attempt.attempt.task_id,
plan.operation.operation_id.as_str().to_owned(),
plan.operation.operation_version,
plan.workspace_id.as_str().to_owned(),
plan.agent_id.as_ref().map(|id| id.as_str().to_owned()),
attempt.attempt.owner.clone(),
attempt.attempt.fence,
attempt.attempt.attempt_id.clone(),
);
let request = match RuntimeExecutionRequest::try_new(
&plan.workspace_id,
plan.origin,
plan.agent_id.as_ref(),
&plan.operation,
&plan.input,
plan.authorization,
plan.resolved_auth.as_ref(),
&plan.request_context,
plan.deadline,
) {
Ok(request) => request,
Err(ExecutionRequestError::DeadlineElapsed) => {
return Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: false,
});
}
Err(_) => return Err(RuntimeError::InvalidExecutionRequest),
};
let (output, _preview) = self
.execute_request_impl(
request,
Some(permit),
SendMode::Single {
attempt: attempt.attempt.ordinal,
max_attempts: attempt.task.pins.retry.max_attempts,
},
)
.await?;
Ok(output)
}
/// Maps one outbound result of a durable attempt onto its persisted attempt outcome.
fn classify_task_attempt(
&self,
plan: &TaskAttemptPlan,
result: Result<Value, RuntimeError>,
) -> (crank_core::AttemptOutcome, Option<String>) {
match result {
Ok(output) => match crank_core::TaskEnvelope::try_redacted(&output) {
Ok(envelope) => (crank_core::AttemptOutcome::Completed(envelope), None),
Err(_) => {
let code = "result_not_redactable".to_owned();
(
crank_core::AttemptOutcome::Failed {
error: bounded_attempt_error(
&code,
crank_core::AttemptPhase::Dispatched,
),
retryable: false,
phase: crank_core::AttemptPhase::Dispatched,
},
Some(code),
)
}
},
Err(error) => {
let phase = if runtime_error_may_have_dispatched(&error) {
crank_core::AttemptPhase::Dispatched
} else {
crank_core::AttemptPhase::BeforeDispatch
};
let correlation = crank_core::CorrelationContext::new(
plan.request_context.request_id.clone(),
plan.request_context.trace_context.clone(),
);
let failure = crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&plan.operation,
);
let code = format!("{:?}", failure.error_code());
let safe_to_repeat = retry_is_allowed(&plan.operation, &error, plan.retry_allowed);
let envelope = bounded_attempt_error(&code, phase);
let outcome = match failure.outcome_certainty() {
crank_core::OutcomeCertainty::OutcomeUnknown => {
crank_core::AttemptOutcome::Uncertain {
error: envelope,
repeatable: safe_to_repeat,
phase,
}
}
_ => crank_core::AttemptOutcome::Failed {
error: envelope,
retryable: safe_to_repeat,
phase,
},
};
(outcome, Some(code))
}
}
}
/// Bounded cooperative wait inside the pinned backoff window. The lease keeps being renewed by
/// the worker, so a durable cancellation or lease loss still stops the local work here.
async fn wait_before_attempt(&self, context: &TaskAttemptContext, delay: Duration) -> bool {
let deadline = Instant::now() + delay;
loop {
if context.is_cancelled() || context.is_stopping() {
return false;
}
let now = Instant::now();
if now >= deadline {
return true;
}
tokio::time::sleep((deadline - now).min(Duration::from_millis(50))).await;
}
}
pub(crate) async fn execute_request_impl(
&self,
request: RuntimeExecutionRequest<'_>,
permit: Option<TaskDispatchPermit>,
send_mode: SendMode,
) -> Result<(Value, Value), RuntimeError> {
let operation = request.operation();
// Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission.
let trusted_task_dispatch = permit.is_some();
// The durable Attempt ID written by the dispatch checkpoint travels with the permit, so the
// trace names the real persisted attempt instead of inventing one (Story 4.3 AC).
let durable_attempt_id = permit.as_ref().map(|permit| permit.attempt_id().to_owned());
let input = request.input();
let request_context = request.request_context();
debug_assert_eq!(request.authorization(), ExecutionAuthorization::Authorized);
debug_assert_eq!(
request_context
.metering_context()
.map(|value| &value.workspace_id),
Some(request.workspace_id())
);
debug_assert_eq!(
request_context
.metering_context()
.and_then(|value| value.agent_id.as_ref()),
request.agent_id()
);
let operation_may_mutate = operation_may_mutate(operation);
let execution_deadline =
crate::admission::effective_deadline(operation, request.deadline());
let runtime_span = Stage::RuntimeExecute.span();
set_parent_from_trace_context(&runtime_span, &request_context.trace_context);
log_runtime_event("unary.execute", operation, Some(request_context));
let started_at = Instant::now();
let dispatch_started = Arc::new(AtomicBool::new(false));
// True until the admission gate answered. A deadline that fires while the request is still
// waiting for a permit is an admission refusal with zero dispatch, never an upstream timeout.
let admission_pending = Arc::new(AtomicBool::new(
!trusted_task_dispatch && self.admission.is_some(),
));
let invocation_metrics = ToolInvocationMetrics::start_with_exemplar(
metric_invocation_source(Some(request_context)),
request_context
.trace_context
.is_sampled()
.then(|| {
crank_metrics::ExemplarTraceId::parse(
&request_context.trace_context.trace_id().to_string(),
)
})
.flatten(),
);
let execution = async {
// One admission gate for every executable path (Story 4.1): the published policy decides
// the effective mode, the absolute deadline includes queue waiting and the scoped permit
// is held for the whole local execution future, including retries.
let mut lease = if trusted_task_dispatch {
None
} else {
let result = crate::admission::acquire(
self.admission.as_ref(),
self.coordination_store.as_deref(),
&request,
execution_deadline,
)
.await?;
admission_pending.store(false, Ordering::Release);
result
};
let _permit = self.acquire_unary_permit(operation)?;
let _inflight = InFlightGuard::runtime();
let mapping_span = Stage::RuntimeArgumentsMap.span();
let prepared_request = mapping_span.in_scope(|| self.prepare_request(operation, input));
record_runtime_result(&mapping_span, &prepared_request);
drop(mapping_span);
let prepared_request = prepared_request?;
let request_preview = sanitized_request_preview(&prepared_request);
let prepared_request = apply_resolved_auth(prepared_request, request.resolved_auth());
let result = self
.execute_prepared(
operation,
input,
prepared_request,
Some(request_context),
Arc::clone(&dispatch_started),
durable_attempt_id.as_deref(),
send_mode,
)
.await;
if let Some(lease) = lease.as_mut() {
lease.release().await;
}
let output = result?;
Ok((output, request_preview))
};
let result = tokio::time::timeout_at(
execution_deadline.into(),
execution.instrument(runtime_span.clone()),
)
.await
.unwrap_or_else(|_| {
if admission_pending.load(Ordering::Acquire) {
Err(crate::admission::deadline_refusal(operation))
} else {
Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: operation_may_mutate
&& dispatch_started.load(Ordering::Acquire),
})
}
});
record_runtime_result(&runtime_span, &result);
drop(runtime_span);
let (outcome, error_kind) = match &result {
Ok(_) => (ToolOutcome::Success, ToolErrorKind::None),
Err(error) => (ToolOutcome::Error, runtime_error_kind(error)),
};
invocation_metrics.complete(outcome, error_kind);
self.record_metering(operation, Some(request_context), &result, started_at)
.await;
result
}
pub async fn execute_outcome(
&self,
request: RuntimeExecutionRequest<'_>,
) -> Result<crank_core::ExecutionSuccess, crank_core::ExecutionFailure> {
let operation_id = request.operation().operation_id.clone();
let operation_version = request.operation().operation_version;
let operation = request.operation().clone();
let origin = request.origin();
let correlation = crank_core::CorrelationContext::new(
request.request_context().request_id.clone(),
request.request_context().trace_context.clone(),
);
match self.execute_request(request).await {
Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess {
operation_id,
operation_version,
origin,
correlation,
request_preview,
output,
}),
Err(error) => Err(crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&operation,
)),
}
}
fn prepare_request(
&self,
operation: &RuntimeOperation,
input: &Value,
) -> Result<PreparedRequest, RuntimeError> {
operation
.input_schema
.validate_shape(input)
.map_err(RuntimeError::InputSchema)?;
validate_selected_body_variant(operation, input)?;
if operation.input_mapping.is_empty() {
return Ok(PreparedRequest::default());
}
let mapped_input = operation
.input_mapping
.apply(&json!({ "mcp": input }))
.map_err(RuntimeError::InputMapping)?;
let mut prepared = PreparedRequest::from_mapping_output(&mapped_input, &operation.target)?;
let Target::Rest(target) = &operation.target;
if let Some(contract) = &target.request_contract
&& input.get(&contract.body_field).is_none()
{
// Mapping initializes an empty body object for legacy field mappings.
// A typed optional body is absent unless its public argument was supplied.
prepared.body = None;
}
Ok(prepared)
}
#[allow(clippy::too_many_arguments)]
async fn execute_prepared(
&self,
operation: &RuntimeOperation,
input: &Value,
prepared_request: PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
durable_attempt_id: Option<&str>,
send_mode: SendMode,
) -> Result<Value, RuntimeError> {
let mut prepared_request = prepared_request;
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
let idempotency_key = match crate::idempotency::prepare_idempotency(
operation,
input,
&mut prepared_request,
) {
Ok(key) => key,
Err(error) if idempotency_applicable => {
record_idempotency_outcome(idempotency_error_outcome(&error));
let span = Stage::RuntimeIdempotency.span();
StageOutcome::Error.record(&span);
ErrorCategory::Idempotency.record(&span);
return Err(error);
}
Err(error) => return Err(error),
};
if crate::confirmation::is_applicable(operation) {
let approval_span = Stage::ApprovalCheck.span();
let approval_result = crate::confirmation::confirm_operation(
self.coordination_store.as_deref(),
operation,
input,
request_context,
)
.instrument(approval_span.clone())
.await;
match &approval_result {
Ok(()) => {
StageOutcome::Success.record(&approval_span);
record_confirmation_outcome(ConfirmationOutcome::Approved);
}
Err(RuntimeError::ConfirmationRequired { .. }) => {
StageOutcome::Required.record(&approval_span);
ErrorCategory::Approval.record(&approval_span);
record_confirmation_outcome(ConfirmationOutcome::Required);
}
Err(error) => {
StageOutcome::Error.record(&approval_span);
runtime_error_category(error).record(&approval_span);
record_confirmation_outcome(confirmation_error_outcome(error));
}
}
drop(approval_span);
approval_result?;
}
// The synchronous coordination guard is the replay authority of one `tools/call`. A durable
// Task attempt already has a PostgreSQL attempt authority around every send, so the guard is
// skipped there: an ambiguous first response must not block the bounded durable retry. The
// stable idempotency key is still delivered to the upstream by `prepare_idempotency` above.
let idempotency = if idempotency_applicable && matches!(send_mode, SendMode::Legacy) {
let idempotency_span = Stage::RuntimeIdempotency.span();
let result = crate::idempotency::begin(
self.coordination_store.as_deref(),
operation,
input,
idempotency_key.as_deref(),
request_context,
)
.instrument(idempotency_span.clone())
.await;
match &result {
Ok(crate::idempotency::IdempotencyAction::Execute(_)) => {
StageOutcome::Execute.record(&idempotency_span);
record_idempotency_outcome(IdempotencyOutcome::Execute);
}
Ok(crate::idempotency::IdempotencyAction::Replay(_)) => {
StageOutcome::Replay.record(&idempotency_span);
record_idempotency_outcome(IdempotencyOutcome::Replay);
}
Ok(crate::idempotency::IdempotencyAction::Disabled) => {
StageOutcome::Skipped.record(&idempotency_span);
}
Err(error) => {
StageOutcome::Error.record(&idempotency_span);
runtime_error_category(error).record(&idempotency_span);
record_idempotency_outcome(idempotency_error_outcome(error));
}
}
drop(idempotency_span);
result?
} else {
crate::idempotency::IdempotencyAction::Disabled
};
if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency {
return transform_response(operation, response);
}
let mut cancellation_guard =
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency {
self.coordination_store.as_ref().map(|store| {
IdempotencyCancellationGuard::new(
Arc::clone(store),
operation.clone(),
reservation.clone(),
Arc::clone(&dispatch_started),
)
})
} else {
None
};
let adapter_result = match self
.load_cached_adapter_response(operation, &prepared_request, request_context)
.await
{
Some(response) => Ok(response),
None => {
let adapter_response = self
.execute_adapter_with_retry(
operation,
&prepared_request,
request_context,
Arc::clone(&dispatch_started),
crate::idempotency::verified_retry_key(
operation,
idempotency_key.as_deref(),
),
durable_attempt_id,
send_mode,
)
.await;
if let Ok(response) = &adapter_response {
self.store_cached_adapter_response(
operation,
&prepared_request,
request_context,
response,
)
.await;
}
adapter_response
}
};
let adapter_response = match adapter_result {
Ok(response) => response,
Err(error) => {
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
&& let Some(store) = self.coordination_store.as_deref()
{
let idempotency_span = Stage::RuntimeIdempotency.span();
let may_have_dispatched = runtime_error_may_have_dispatched(&error);
let cleanup_result = if may_have_dispatched {
crate::idempotency::mark_outcome_unknown(store, operation, reservation)
.instrument(idempotency_span.clone())
.await
} else {
crate::idempotency::release(store, operation, reservation)
.instrument(idempotency_span.clone())
.await
};
record_runtime_result(&idempotency_span, &cleanup_result);
record_idempotency_outcome(match &cleanup_result {
Ok(()) if may_have_dispatched => IdempotencyOutcome::OutcomeUnknown,
Ok(()) => IdempotencyOutcome::Completed,
Err(error) => idempotency_error_outcome(error),
});
if let Some(guard) = &mut cancellation_guard {
guard.disarm();
}
}
return Err(error);
}
};
if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency
&& let Some(store) = self.coordination_store.as_deref()
{
let idempotency_span = Stage::RuntimeIdempotency.span();
let completion_result =
crate::idempotency::complete(store, operation, reservation, &adapter_response)
.instrument(idempotency_span.clone())
.await;
record_runtime_result(&idempotency_span, &completion_result);
record_idempotency_outcome(match &completion_result {
Ok(()) => IdempotencyOutcome::Completed,
Err(error) => idempotency_error_outcome(error),
});
if let Some(guard) = &mut cancellation_guard {
guard.disarm();
}
drop(idempotency_span);
if completion_result.is_err() {
return Err(RuntimeError::IdempotencyCompletionUnknown {
operation_id: operation.operation_id.as_str().to_owned(),
});
}
}
transform_response(operation, &adapter_response)
}
async fn record_metering<T>(
&self,
operation: &RuntimeOperation,
request_context: Option<&RuntimeRequestContext>,
result: &Result<T, RuntimeError>,
started_at: Instant,
) {
let Some(context) = request_context.and_then(RuntimeRequestContext::metering_context)
else {
return;
};
self.metering_sink
.record(MeteringEvent {
workspace_id: context.workspace_id.clone(),
agent_id: context.agent_id.clone(),
operation_id: operation.operation_id.clone(),
source: context.source,
status: if result.is_ok() {
InvocationStatus::Ok
} else {
InvocationStatus::Error
},
duration_ms: started_at.elapsed().as_millis() as u64,
created_at: OffsetDateTime::now_utc(),
})
.await;
}
async fn execute_adapter(
&self,
operation: &RuntimeOperation,
prepared_request: PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
) -> Result<AdapterResponse, RuntimeError> {
log_runtime_event("adapter.dispatch", operation, request_context);
let adapter = self.adapter_for(operation)?;
if !adapter.supports_mode(ExecutionMode::Unary) {
return Err(RuntimeError::UnsupportedExecutionMode {
operation_id: operation.operation_id.as_str().to_owned(),
mode: ExecutionMode::Unary,
});
}
let prepared_request = adapter_prepared_request(
operation,
&prepared_request,
operation.execution_config.timeout_ms,
);
let adapter_context =
adapter_request_context(request_context, Arc::clone(&dispatch_started))?;
adapter
.invoke_unary(
&operation.target,
&prepared_request.into(),
&adapter_context,
)
.await
.map(Into::into)
.map_err(|error| map_protocol_adapter_error(operation, error))
}
#[allow(clippy::too_many_arguments)]
async fn execute_adapter_with_retry(
&self,
operation: &RuntimeOperation,
prepared_request: &PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
verified_idempotency: bool,
durable_attempt_id: Option<&str>,
send_mode: SendMode,
) -> Result<AdapterResponse, RuntimeError> {
// `max_attempts` bounds the transport loop; `reported_max_attempts` is the pinned Task budget
// the attempt span names. A durable attempt performs exactly one send, so its transport loop
// always has a single iteration even when the Task may retry later.
let (max_attempts, first_attempt, reported_max_attempts) = match send_mode {
SendMode::Single {
attempt,
max_attempts,
} => (1, attempt, max_attempts),
SendMode::Legacy => {
let attempts = retry_attempts(operation);
(attempts, 1, attempts)
}
};
let mut attempt = first_attempt;
loop {
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint. Only
// paths without a durable attempt (synchronous tools/call) mint a synthetic identity.
let attempt_id = durable_attempt_id
.map(str::to_owned)
.unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7()));
let attempt_span = tracing::info_span!(
"adapter_attempt",
attempt,
max_attempts = reported_max_attempts,
attempt_id = %attempt_id
);
let result = self
.execute_adapter(
operation,
prepared_request.clone(),
request_context,
Arc::clone(&dispatch_started),
)
.instrument(attempt_span)
.await;
match result {
Ok(response) => return Ok(response),
Err(error)
if attempt < max_attempts
&& retry_is_allowed(operation, &error, verified_idempotency) =>
{
attempt += 1;
tokio::time::sleep(retry_backoff(attempt)).await;
}
Err(error) => return Err(error),
}
}
}
fn acquire_unary_permit(
&self,
_operation: &RuntimeOperation,
) -> Result<OwnedSemaphorePermit, RuntimeError> {
try_acquire_limit(
Arc::clone(&self.unary_limiter),
"unary",
self.limits.max_concurrent_unary,
)
}
async fn load_cached_adapter_response(
&self,
operation: &RuntimeOperation,
prepared_request: &PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
) -> Option<AdapterResponse> {
let response_cache = self.response_cache.as_ref()?;
let cache_key = response_cache_key(operation, prepared_request, request_context)?;
let cached =
match tokio::time::timeout(RESPONSE_CACHE_IO_TIMEOUT, response_cache.get(&cache_key))
.await
{
Ok(Ok(Some(cached))) => cached,
Ok(Ok(None)) => {
record_cache_outcome(CacheOutcome::Miss);
return None;
}
Ok(Err(_)) | Err(_) => {
record_cache_outcome(CacheOutcome::ReadError);
debug!(
name: "runtime.response_cache.read_failed",
operation_id = operation.operation_id.as_str(),
error_category = "response_cache",
"response cache lookup skipped"
);
return None;
}
};
match adapter_response_from_cached(cached) {
Ok(response) => {
record_cache_outcome(CacheOutcome::Hit);
Some(response)
}
Err(_) => {
record_cache_outcome(CacheOutcome::DecodeError);
debug!(
name: "runtime.response_cache.decode_failed",
operation_id = operation.operation_id.as_str(),
error_category = "cached_response",
"cached response payload was invalid"
);
if response_cache.delete(&cache_key).await.is_err() {
record_cache_outcome(CacheOutcome::EvictError);
}
None
}
}
}
async fn store_cached_adapter_response(
&self,
operation: &RuntimeOperation,
prepared_request: &PreparedRequest,
request_context: Option<&RuntimeRequestContext>,
adapter_response: &AdapterResponse,
) {
let Some(response_cache) = self.response_cache.as_ref() else {
return;
};
if !(200..=299).contains(&adapter_response.status_code) {
return;
}
if request_context
.and_then(RuntimeRequestContext::resource_response_cache)
.is_some_and(|policy| {
serde_json::to_vec(&adapter_response.data)
.map_or(true, |bytes| bytes.len() as u64 > policy.max_bytes)
})
{
return;
}
let Some(cache_ttl) = response_cache_ttl(operation, request_context) else {
return;
};
let Some(cache_key) = response_cache_key(operation, prepared_request, request_context)
else {
return;
};
let Some(cached_response) = cached_response_from_adapter(adapter_response) else {
return;
};
if !matches!(
tokio::time::timeout(
RESPONSE_CACHE_IO_TIMEOUT,
response_cache.put(&cache_key, cached_response, cache_ttl),
)
.await,
Ok(Ok(()))
) {
record_cache_outcome(CacheOutcome::WriteError);
debug!(
name: "runtime.response_cache.write_failed",
operation_id = operation.operation_id.as_str(),
error_category = "response_cache",
"response cache write skipped"
);
} else {
record_cache_outcome(CacheOutcome::Stored);
}
}
}
fn sanitized_request_preview(prepared: &PreparedRequest) -> Value {
json!({
"path_parameter_count": prepared.path_params.len(),
"query_parameter_count": prepared.query_params.len(),
"header_count": prepared.headers.len(),
"body_configured": prepared.body.is_some(),
})
}
/// Bounded persisted error envelope of one durable attempt: a closed code and the attempt phase.
/// It never carries an upstream body, status text, URL or credential.
fn bounded_attempt_error(code: &str, phase: crank_core::AttemptPhase) -> crank_core::TaskEnvelope {
crank_core::TaskEnvelope::try_redacted(&json!({
"code": code,
"phase": phase.as_str(),
}))
.unwrap_or_else(|_| {
crank_core::TaskEnvelope::try_redacted(&json!({
"code": "task_error_not_recordable",
"phase": phase.as_str(),
}))
.expect("bounded attempt error envelope is always valid")
})
}
fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool {
matches!(
error,
RuntimeError::ProtocolAdapter(
crank_core::ProtocolAdapterError::Transport {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
} | crank_core::ProtocolAdapterError::Timeout {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
} | crank_core::ProtocolAdapterError::ResponseTooLarge {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
} | crank_core::ProtocolAdapterError::UnexpectedStatus {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
..
} | crank_core::ProtocolAdapterError::InvalidResponse {
dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
},
) | RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: true,
} | RuntimeError::IdempotencyCompletionUnknown { .. }
)
}
fn retry_attempts(operation: &RuntimeOperation) -> u32 {
operation
.execution_config
.retry_policy
.as_ref()
.map_or(1, |policy| policy.max_attempts.clamp(1, 5))
}
fn retry_backoff(attempt: u32) -> Duration {
Duration::from_millis(u64::from(attempt.saturating_sub(1)).min(4))
}
fn retry_is_allowed(
operation: &RuntimeOperation,
error: &RuntimeError,
verified_idempotency: bool,
) -> bool {
let is_read = matches!(
&operation.target,
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
);
let safe_write = operation_may_mutate(operation) && verified_idempotency;
if !(is_read || safe_write) {
return false;
}
match error {
RuntimeError::ProtocolAdapter(
crank_core::ProtocolAdapterError::Transport { .. }
| crank_core::ProtocolAdapterError::Timeout { .. },
) => true,
RuntimeError::ProtocolAdapter(crank_core::ProtocolAdapterError::UnexpectedStatus {
status,
..
}) => matches!(status, 408 | 429 | 500..=599),
_ => false,
}
}
fn operation_may_mutate(operation: &RuntimeOperation) -> bool {
matches!(
&operation.target,
crank_core::Target::Rest(target)
if matches!(
target.method,
crank_core::HttpMethod::Post
| crank_core::HttpMethod::Put
| crank_core::HttpMethod::Patch
| crank_core::HttpMethod::Delete
)
)
}
fn adapter_request_context(
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
) -> Result<crank_core::RuntimeRequestContext, RuntimeError> {
request_context
.map(|context| {
crank_core::RuntimeRequestContext::from(context).with_dispatch_started(dispatch_started)
})
.ok_or(RuntimeError::InvalidExecutionRequest)
}
fn map_protocol_adapter_error(
operation: &RuntimeOperation,
error: crank_core::ProtocolAdapterError,
) -> RuntimeError {
match error {
crank_core::ProtocolAdapterError::UnsupportedMode { mode, .. } => {
RuntimeError::UnsupportedExecutionMode {
operation_id: operation.operation_id.as_str().to_owned(),
mode,
}
}
error => RuntimeError::ProtocolAdapter(error),
}
}
impl RuntimeExecutor {
fn adapter_for(
&self,
operation: &RuntimeOperation,
) -> Result<SharedProtocolAdapter, RuntimeError> {
self.adapters
.get(operation.protocol)
.ok_or(RuntimeError::UnsupportedProtocol {
protocol: operation.protocol,
})
}
}
fn finalize_output(
operation: &RuntimeOperation,
response: &AdapterResponse,
) -> Result<Value, RuntimeError> {
let response_data = if matches!(
&operation.target,
Target::Rest(target) if target.method == HttpMethod::Head && response.body.is_null()
) {
// HEAD has no body; expose only safe status metadata for an explicit output mapping.
json!({"status": response.status_code})
} else {
response.data.clone()
};
let mapped = operation
.output_mapping
.apply(&json!({
"response": {
"body": response.body,
"data": response_data,
"headers": response.headers,
"status": response.status_code
}
}))
.map_err(RuntimeError::OutputMapping)?;
Ok(mapped
.get("output")
.cloned()
.unwrap_or_else(|| Value::Object(Map::new())))
}
fn transform_response(
operation: &RuntimeOperation,
response: &AdapterResponse,
) -> Result<Value, RuntimeError> {
let span = Stage::RuntimeResponseTransform.span();
let result = span.in_scope(|| {
let finalized_output = finalize_output(operation, response)?;
output_schema_for_response(operation, response)?
.validate_shape(&finalized_output)
.map_err(RuntimeError::OutputSchema)?;
Ok(finalized_output)
});
match &result {
Ok(_) => StageOutcome::Success.record(&span),
Err(_) => {
StageOutcome::Error.record(&span);
ErrorCategory::Transformation.record(&span);
}
}
result
}
fn output_schema_for_response<'a>(
operation: &'a RuntimeOperation,
response: &AdapterResponse,
) -> Result<&'a crank_schema::Schema, RuntimeError> {
let Target::Rest(target) = &operation.target;
if target.response_contract.is_empty() {
return Ok(&operation.output_schema);
}
let media_type = response
.headers
.get("content-type")
.and_then(|value| value.split(';').next())
.map(str::trim);
let descriptor = target
.response_contract
.iter()
.filter(|descriptor| descriptor.status_code == response.status_code)
.find(|descriptor| match (&descriptor.media_type, media_type) {
(Some(expected), Some(actual)) => expected.eq_ignore_ascii_case(actual),
(None, _) => response.body.is_null(),
_ => false,
})
.ok_or(RuntimeError::InvalidExecutionRequest)?;
if target.response_contract.len() > 1
&& operation.output_schema.kind == crank_schema::SchemaKind::Oneof
{
return operation
.output_schema
.variants
.get(descriptor.schema_variant)
.ok_or(RuntimeError::InvalidExecutionRequest);
}
(descriptor.schema_variant == 0)
.then_some(&operation.output_schema)
.ok_or(RuntimeError::InvalidExecutionRequest)
}
fn record_runtime_result<T>(span: &Span, result: &Result<T, RuntimeError>) {
match result {
Ok(_) => StageOutcome::Success.record(span),
Err(error) => {
StageOutcome::Error.record(span);
runtime_error_category(error).record(span);
}
}
}
fn runtime_error_category(error: &RuntimeError) -> ErrorCategory {
match error {
RuntimeError::InvalidExecutionRequest => ErrorCategory::Configuration,
RuntimeError::ExecutionDeadlineElapsed { .. } => ErrorCategory::Upstream,
RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => {
ErrorCategory::Schema
}
RuntimeError::Mapping(_)
| RuntimeError::InputMapping(_)
| RuntimeError::OutputMapping(_)
| RuntimeError::InvalidPreparedRequest { .. } => ErrorCategory::Mapping,
RuntimeError::ProtocolAdapter(_)
| RuntimeError::UnsupportedProtocol { .. }
| RuntimeError::UnsupportedExecutionMode { .. } => ErrorCategory::Upstream,
RuntimeError::ConcurrencyLimitExceeded { .. } => ErrorCategory::Concurrency,
RuntimeError::AdmissionOverloaded { .. }
| RuntimeError::AdmissionStoreUnavailable { .. } => ErrorCategory::Concurrency,
RuntimeError::ExecutionPolicyUnsupported { .. } => ErrorCategory::Configuration,
RuntimeError::AdmissionApprovalRequired { .. } => ErrorCategory::Approval,
RuntimeError::ConfirmationRequired { .. }
| RuntimeError::InvalidConfirmationToken { .. }
| RuntimeError::ConfirmationStoreUnavailable { .. } => ErrorCategory::Approval,
RuntimeError::IdempotencyStoreUnavailable { .. }
| RuntimeError::IdempotencyInProgress { .. }
| RuntimeError::IdempotencyConflict { .. }
| RuntimeError::IdempotencyOutcomeUnknown { .. }
| RuntimeError::IdempotencyCompletionUnknown { .. } => ErrorCategory::Idempotency,
RuntimeError::MissingAuthProfile { .. }
| RuntimeError::InvalidAuthProfileConfig { .. }
| RuntimeError::AuthorizationStoreUnavailable
| RuntimeError::MissingSecret { .. }
| RuntimeError::MissingSecretVersion { .. }
| RuntimeError::InvalidAuthSecretValue { .. }
| RuntimeError::SecretCrypto { .. } => ErrorCategory::Configuration,
}
}
fn try_acquire_limit(
limiter: Arc<Semaphore>,
kind: &'static str,
limit: usize,
) -> Result<OwnedSemaphorePermit, RuntimeError> {
limiter.try_acquire_owned().map_err(|_| {
record_limit_rejection(LimitStage::Concurrency);
RuntimeError::ConcurrencyLimitExceeded { kind, limit }
})
}
fn metric_invocation_source(
request_context: Option<&RuntimeRequestContext>,
) -> MetricInvocationSource {
request_context
.and_then(RuntimeRequestContext::metering_context)
.map_or(MetricInvocationSource::Internal, |context| {
match context.source {
InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun,
InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall,
}
})
}
fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
match error {
RuntimeError::InvalidExecutionRequest => ToolErrorKind::InvalidPreparedRequest,
RuntimeError::ExecutionDeadlineElapsed { .. } => ToolErrorKind::RestAdapter,
RuntimeError::Schema(_) | RuntimeError::InputSchema(_) | RuntimeError::OutputSchema(_) => {
ToolErrorKind::Schema
}
RuntimeError::Mapping(_)
| RuntimeError::InputMapping(_)
| RuntimeError::OutputMapping(_) => ToolErrorKind::Mapping,
RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter,
RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
// Story 4.1: admission failures reuse the closed error-kind vocabulary so the tool-invocation
// label domain stays fixed; the precise code lives in the execution failure/invocation log
// (`ExecutionOverloaded`, `SafetyStoreUnavailable`, `ExecutionModeUnsupported`).
RuntimeError::AdmissionOverloaded { .. } => ToolErrorKind::ConcurrencyLimit,
RuntimeError::AdmissionStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
RuntimeError::AdmissionApprovalRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::ExecutionPolicyUnsupported { .. } => ToolErrorKind::UnsupportedExecutionMode,
RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore,
RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress,
RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict,
RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown,
RuntimeError::IdempotencyCompletionUnknown { .. } => {
ToolErrorKind::IdempotencyOutcomeUnknown
}
RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile,
RuntimeError::InvalidAuthProfileConfig { .. } => ToolErrorKind::InvalidAuthSecret,
RuntimeError::AuthorizationStoreUnavailable => ToolErrorKind::IdempotencyStore,
RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret,
RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion,
RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret,
RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto,
}
}
fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome {
match error {
RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict,
RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress,
RuntimeError::IdempotencyOutcomeUnknown { .. }
| RuntimeError::IdempotencyCompletionUnknown { .. } => IdempotencyOutcome::OutcomeUnknown,
RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable,
_ => IdempotencyOutcome::Error,
}
}
fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome {
match error {
RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken,
RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable,
_ => ConfirmationOutcome::Error,
}
}
fn log_runtime_event(
stage: &'static str,
operation: &RuntimeOperation,
request_context: Option<&RuntimeRequestContext>,
) {
let protocol = match operation.protocol {
crank_core::Protocol::Rest => "rest",
};
if let Some(context) = request_context {
debug!(
name: "runtime.execution.stage_reached",
stage,
operation_id = operation.operation_id.as_str(),
protocol,
request_id = context.request_id.as_str(),
trace_id = context.trace_context.trace_id().as_str(),
"runtime execution"
);
} else {
debug!(
name: "runtime.execution.stage_reached",
stage,
operation_id = operation.operation_id.as_str(),
protocol,
"runtime execution"
);
}
}