feat: complete Epic 1 production foundation

crates/crank-runtime/tests/integration.rs

@@ -1,3 +1,5 @@
+mod support;
+
 mod integration {
     mod confirmation;
     mod idempotency;

crates/crank-runtime/tests/integration/confirmation.rs

@@ -7,14 +7,14 @@ use std::sync::{
 use async_trait::async_trait;
 use crank_core::{
     AdapterResponse, CacheScope, CacheStoreError, ConfirmationPolicy, CoordinationStateReservation,
-    CoordinationStateStore, CoordinationStateValue, ExecutionConfig, ExecutionMode, HttpMethod,
-    Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
-    OperationStatus, Protocol, ProtocolAdapter, ProtocolAdapterError, RestTarget, Target,
-    ToolDescription,
+    CoordinationStateStore, CoordinationStateValue, ExecutionConfig, ExecutionErrorCode,
+    ExecutionMode, HttpMethod, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy,
+    OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter, ProtocolAdapterError,
+    RestTarget, Target, ToolDescription,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_runtime::{
-    InMemoryCoordinationStateStore, RuntimeError, RuntimeExecutorBuilder, RuntimeRequestContext,
+    InMemoryCoordinationStateStore, RuntimeExecutorBuilder, RuntimeRequestContext,
 };
 use crank_schema::{Schema, SchemaKind};
 use futures_util::future::join_all;
@@ -42,12 +42,12 @@ async fn destructive_operation_requires_single_use_confirmation() {
         )
         .await
         .expect_err("first destructive call must only stage confirmation");
-    let confirmation_token = match first {
-        RuntimeError::ConfirmationRequired {
-            confirmation_token, ..
-        } => confirmation_token,
-        error => panic!("expected confirmation required, got {error:?}"),
-    };
+    assert_eq!(first.error_code(), ExecutionErrorCode::ConfirmationRequired);
+    let confirmation_token = first
+        .confirmation()
+        .expect("expected confirmation challenge")
+        .token()
+        .to_owned();
     assert_eq!(call_count.load(Ordering::SeqCst), 0);
 
     let confirmed_context = context
@@ -72,10 +72,7 @@ async fn destructive_operation_requires_single_use_confirmation() {
         )
         .await
         .expect_err("confirmation token must be single-use");
-    assert!(matches!(
-        replay,
-        RuntimeError::InvalidConfirmationToken { .. }
-    ));
+    assert_eq!(replay.error_code(), ExecutionErrorCode::ConfirmationInvalid);
     assert_eq!(call_count.load(Ordering::SeqCst), 1);
 
     let approved_context = context.with_approval_granted();
@@ -111,12 +108,12 @@ async fn confirmation_token_allows_only_one_concurrent_execution() {
         )
         .await
         .unwrap_err();
-    let RuntimeError::ConfirmationRequired {
-        confirmation_token, ..
-    } = first
-    else {
-        panic!("expected confirmation token")
-    };
+    assert_eq!(first.error_code(), ExecutionErrorCode::ConfirmationRequired);
+    let confirmation_token = first
+        .confirmation()
+        .expect("expected confirmation token")
+        .token()
+        .to_owned();
 
     let attempts = (0..16).map(|_| {
         let executor = executor.clone();
@@ -143,7 +140,7 @@ async fn confirmation_token_allows_only_one_concurrent_execution() {
         results
             .iter()
             .filter_map(|result| result.as_ref().err())
-            .all(|error| matches!(error, RuntimeError::InvalidConfirmationToken { .. }))
+            .all(|error| error.error_code() == ExecutionErrorCode::ConfirmationInvalid)
     );
 }
 
@@ -167,11 +164,10 @@ async fn unavailable_confirmation_store_is_preserved_as_a_typed_error() {
         )
         .await
         .expect_err("unavailable store must prevent issuing a token");
-    assert!(matches!(
-        issue_error,
-        RuntimeError::ConfirmationStoreUnavailable { ref operation_id }
-            if operation_id == "op_delete_order"
-    ));
+    assert_eq!(
+        issue_error.error_code(),
+        ExecutionErrorCode::SafetyStoreUnavailable
+    );
 
     let consume_error = executor
         .execute_with_context(
@@ -181,11 +177,10 @@ async fn unavailable_confirmation_store_is_preserved_as_a_typed_error() {
         )
         .await
         .expect_err("unavailable store must not look like an invalid token");
-    assert!(matches!(
-        consume_error,
-        RuntimeError::ConfirmationStoreUnavailable { ref operation_id }
-            if operation_id == "op_delete_order"
-    ));
+    assert_eq!(
+        consume_error.error_code(),
+        ExecutionErrorCode::SafetyStoreUnavailable
+    );
 }
 
 struct UnavailableCoordinationStore;
@@ -396,3 +391,4 @@ fn bool_schema() -> Schema {
         variants: Vec::new(),
     }
 }
+use crate::support::RuntimeExecutorTestExt;

crates/crank-runtime/tests/integration/idempotency.rs

@@ -6,15 +6,14 @@ use std::sync::{
 
 use async_trait::async_trait;
 use crank_core::{
-    AdapterResponse, ExecutionConfig, ExecutionMode, HttpMethod, IdempotencyMode,
-    IdempotencyPolicy, Operation, OperationId, OperationSecurityLevel, OperationStatus, Protocol,
-    ProtocolAdapter, ProtocolAdapterError, RestTarget, RuntimeRequestContext, Target,
-    ToolDescription,
+    AdapterResponse, ExecutionConfig, ExecutionErrorCode, ExecutionMode, HttpMethod,
+    IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSecurityLevel,
+    OperationStatus, Protocol, ProtocolAdapter, ProtocolAdapterError, RestTarget,
+    RuntimeRequestContext, Target, ToolDescription,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_runtime::{
-    InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeError,
-    RuntimeExecutorBuilder,
+    InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeExecutorBuilder,
 };
 use crank_schema::{Schema, SchemaKind};
 use serde_json::json;
@@ -71,10 +70,10 @@ async fn required_idempotency_rejects_missing_key_before_adapter_call() {
         .await
         .expect_err("missing idempotency key must fail");
 
-    assert!(matches!(
-        error,
-        crank_runtime::RuntimeError::InvalidPreparedRequest { .. }
-    ));
+    assert_eq!(
+        error.error_code(),
+        ExecutionErrorCode::PreparedRequestInvalid
+    );
     assert_eq!(call_count.load(Ordering::SeqCst), 0);
 }
 
@@ -100,10 +99,10 @@ async fn required_idempotency_fails_closed_without_coordination_store() {
         .await
         .expect_err("required idempotency must not execute without an atomic store");
 
-    assert!(matches!(
-        error,
-        RuntimeError::IdempotencyStoreUnavailable { .. }
-    ));
+    assert_eq!(
+        error.error_code(),
+        ExecutionErrorCode::SafetyStoreUnavailable
+    );
     assert_eq!(call_count.load(Ordering::SeqCst), 0);
 }
 
@@ -186,7 +185,7 @@ async fn same_key_with_different_input_is_rejected() {
         .await
         .expect_err("same key must not accept a different request fingerprint");
 
-    assert!(matches!(error, RuntimeError::IdempotencyConflict { .. }));
+    assert_eq!(error.error_code(), ExecutionErrorCode::IdempotencyConflict);
     assert_eq!(call_count.load(Ordering::SeqCst), 1);
 }
 
@@ -208,16 +207,19 @@ async fn uncertain_adapter_failure_blocks_automatic_retry() {
         .execute_with_context(&operation, &input, Some(&context))
         .await
         .expect_err("adapter failure must be returned");
-    assert!(matches!(first, RuntimeError::ProtocolAdapter(_)));
+    assert_eq!(
+        first.error_code(),
+        ExecutionErrorCode::UpstreamTransportError
+    );
 
     let retry = executor
         .execute_with_context(&operation, &input, Some(&context))
         .await
         .expect_err("an uncertain external outcome must not be retried automatically");
-    assert!(matches!(
-        retry,
-        RuntimeError::IdempotencyOutcomeUnknown { .. }
-    ));
+    assert_eq!(
+        retry.error_code(),
+        ExecutionErrorCode::IdempotencyOutcomeUnknown
+    );
     assert_eq!(call_count.load(Ordering::SeqCst), 1);
 }
 
@@ -257,9 +259,9 @@ impl ProtocolAdapter for FailingAdapter {
         _context: &RuntimeRequestContext,
     ) -> Result<AdapterResponse, ProtocolAdapterError> {
         self.call_count.fetch_add(1, Ordering::SeqCst);
-        Err(ProtocolAdapterError::Message(
-            "upstream outcome is unknown".to_owned(),
-        ))
+        Err(ProtocolAdapterError::Transport {
+            dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
+        })
     }
 }
 
@@ -381,3 +383,4 @@ fn string_schema() -> Schema {
         variants: Vec::new(),
     }
 }
+use crate::support::RuntimeExecutorTestExt;

crates/crank-runtime/tests/integration/no_input_get.rs

@@ -1,28 +1,60 @@
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, sync::Arc};
 
+use async_trait::async_trait;
 use crank_core::{
-    ExecutionConfig, HttpMethod, Operation, OperationId, OperationSecurityLevel, OperationStatus,
-    Protocol, RestTarget, Target, ToolDescription,
+    AdapterResponse, ExecutionConfig, ExecutionMode, HttpMethod, Operation, OperationId,
+    OperationSecurityLevel, OperationStatus, PreparedRequest, Protocol, ProtocolAdapter,
+    ProtocolAdapterError, RestTarget, RuntimeRequestContext as ProtocolRequestContext, Target,
+    ToolDescription,
 };
 use crank_mapping::MappingSet;
-use crank_runtime::RuntimeExecutor;
+use crank_runtime::RuntimeExecutorBuilder;
 use crank_schema::{Schema, SchemaKind};
 use serde_json::json;
 use time::OffsetDateTime;
 
-#[test]
-fn prepares_empty_request_for_no_input_get_with_empty_mapping() {
-    let executor = RuntimeExecutor::new();
-    let operation = no_input_get_operation();
+use crate::support::RuntimeExecutorTestExt;
 
-    let request = executor
-        .prepare_request(&operation.into(), &json!({}))
-        .unwrap();
+#[tokio::test]
+async fn prepares_empty_request_for_no_input_get_with_empty_mapping() {
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(EmptyRequestAdapter))
+        .build();
+    let operation = no_input_get_operation().into();
 
-    assert!(request.path_params.is_empty());
-    assert!(request.query_params.is_empty());
-    assert!(request.headers.is_empty());
-    assert!(request.body.is_none());
+    let response = executor.execute(&operation, &json!({})).await.unwrap();
+    assert_eq!(response, json!({}));
+}
+
+struct EmptyRequestAdapter;
+
+#[async_trait]
+impl ProtocolAdapter for EmptyRequestAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        request: &PreparedRequest,
+        _context: &ProtocolRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        assert!(request.path_params.is_empty());
+        assert!(request.query_params.is_empty());
+        assert!(request.headers.is_empty());
+        assert!(request.body.is_none());
+        Ok(AdapterResponse {
+            status_code: 200,
+            headers: BTreeMap::new(),
+            body: json!({}),
+            data: json!({}),
+        })
+    }
 }
 
 fn no_input_get_operation() -> Operation<Schema, MappingSet> {

crates/crank-runtime/tests/product_metrics.rs

@@ -8,16 +8,16 @@ use std::{
 
 use async_trait::async_trait;
 use crank_core::{
-    AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod,
-    IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass,
+    AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionErrorCode, ExecutionMode,
+    HttpMethod, IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass,
     OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter,
     ProtocolAdapterError, ResponseCachePolicy, RestTarget,
     RuntimeRequestContext as ProtocolRequestContext, Target, ToolDescription,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_runtime::{
-    InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeError,
-    RuntimeExecutorBuilder, RuntimeRequestContext,
+    InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeExecutorBuilder,
+    RuntimeRequestContext,
 };
 use crank_schema::{Schema, SchemaKind};
 use metrics_util::debugging::DebuggingRecorder;
@@ -110,10 +110,10 @@ async fn exercise_cancelled_idempotency() {
         .execute_with_context(&operation, &input, Some(&context))
         .await
         .expect_err("cancelled external outcome must not be retried automatically");
-    assert!(matches!(
-        retry,
-        RuntimeError::IdempotencyOutcomeUnknown { .. }
-    ));
+    assert_eq!(
+        retry.error_code(),
+        ExecutionErrorCode::IdempotencyOutcomeUnknown
+    );
     assert_eq!(calls.load(Ordering::SeqCst), 1);
 }
 
@@ -180,7 +180,10 @@ async fn exercise_idempotency() {
         .await
         .expect_err("same idempotency key with a different input must conflict");
 
-    assert!(matches!(conflict, RuntimeError::IdempotencyConflict { .. }));
+    assert_eq!(
+        conflict.error_code(),
+        ExecutionErrorCode::IdempotencyConflict
+    );
     assert_eq!(calls.load(Ordering::SeqCst), 1);
 }
 
@@ -206,12 +209,15 @@ async fn exercise_confirmation() {
         .execute_with_context(&operation, &input, Some(&context))
         .await
         .expect_err("destructive operation must require confirmation");
-    let RuntimeError::ConfirmationRequired {
-        confirmation_token, ..
-    } = required
-    else {
-        panic!("expected confirmation token");
-    };
+    assert_eq!(
+        required.error_code(),
+        ExecutionErrorCode::ConfirmationRequired
+    );
+    let confirmation_token = required
+        .confirmation()
+        .expect("expected confirmation token")
+        .token()
+        .to_owned();
     let confirmed = context
         .clone()
         .with_confirmation_token(confirmation_token.clone());
@@ -224,10 +230,10 @@ async fn exercise_confirmation() {
         .await
         .expect_err("confirmation token must be single-use");
 
-    assert!(matches!(
-        invalid,
-        RuntimeError::InvalidConfirmationToken { .. }
-    ));
+    assert_eq!(
+        invalid.error_code(),
+        ExecutionErrorCode::ConfirmationInvalid
+    );
     assert_eq!(calls.load(Ordering::SeqCst), 1);
 }
 
@@ -261,9 +267,10 @@ impl ProtocolAdapter for BlockingAdapter {
         &self,
         _target: &Target,
         _prepared: &crank_core::PreparedRequest,
-        _context: &ProtocolRequestContext,
+        context: &ProtocolRequestContext,
     ) -> Result<AdapterResponse, ProtocolAdapterError> {
         self.calls.fetch_add(1, Ordering::SeqCst);
+        context.mark_dispatch_started();
         std::future::pending().await
     }
 }
@@ -412,3 +419,6 @@ async fn wait_for_calls(calls: &AtomicUsize, expected: usize) {
     }
     panic!("adapter did not receive {expected} call(s)");
 }
+mod support;
+
+use support::RuntimeExecutorTestExt;

crates/crank-runtime/tests/stages.rs

@@ -1,3 +1,7 @@
+mod support;
+
+use support::RuntimeExecutorTestExt;
+
 use std::{
     collections::BTreeMap,
     sync::{Arc, Mutex},
@@ -5,14 +9,17 @@ use std::{
 
 use async_trait::async_trait;
 use crank_core::{
-    AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod,
-    IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass,
-    OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter,
-    ProtocolAdapterError, RestTarget, Target, ToolDescription,
+    AdapterResponse, ConfirmationPolicy, CorrelationContext, ExecutionConfig, ExecutionErrorCode,
+    ExecutionMode, ExecutionOrigin, HttpMethod, IdempotencyMode, IdempotencyPolicy,
+    InvocationSource, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy,
+    OperationSecurityLevel, OperationStatus, OutcomeCertainty, Protocol, ProtocolAdapter,
+    ProtocolAdapterError, RestTarget, RetryPolicy, Retryability, Target, ToolDescription,
+    WorkspaceId,
 };
 use crank_mapping::{MappingRule, MappingSet};
 use crank_runtime::{
-    InMemoryCoordinationStateStore, RuntimeError, RuntimeExecutorBuilder, RuntimeRequestContext,
+    ExecutionAuthorization, InMemoryCoordinationStateStore, RuntimeExecutionRequest,
+    RuntimeExecutorBuilder, RuntimeRequestContext,
 };
 use crank_schema::{Schema, SchemaKind};
 use crank_trace::{Stage, StageOutcome};
@@ -24,6 +31,161 @@ use uuid::Version;
 
 static TRACING_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
 
+#[tokio::test]
+async fn deadline_after_dispatch_is_normalized_as_unknown_outcome() {
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(DelayedAdapter))
+        .build();
+    let operation = operation().into();
+    let input = json!({"name": "deadline"});
+    let workspace = WorkspaceId::new("ws_deadline");
+    let correlation = CorrelationContext::generate();
+    let context = RuntimeRequestContext::from_correlation(&correlation).with_metering_context(
+        workspace.clone(),
+        None,
+        InvocationSource::AdminTestRun,
+    );
+    let request = RuntimeExecutionRequest::try_new(
+        &workspace,
+        ExecutionOrigin::AdminDraft,
+        None,
+        &operation,
+        &input,
+        ExecutionAuthorization::Authorized,
+        None,
+        &context,
+        std::time::Instant::now() + std::time::Duration::from_millis(5),
+    )
+    .unwrap();
+
+    let failure = executor.execute_outcome(request).await.unwrap_err();
+    assert_eq!(failure.retryability(), Retryability::ManualReconcile);
+    assert_eq!(
+        failure.outcome_certainty(),
+        OutcomeCertainty::OutcomeUnknown
+    );
+}
+
+#[tokio::test]
+async fn deadline_after_read_only_dispatch_remains_safe_to_retry() {
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(DelayedAdapter))
+        .build();
+    let mut operation: crank_runtime::RuntimeOperation = operation().into();
+    let Target::Rest(target) = &mut operation.target;
+    target.method = HttpMethod::Get;
+    let input = json!({"name": "deadline"});
+    let workspace = WorkspaceId::new("ws_read_deadline");
+    let correlation = CorrelationContext::generate();
+    let context = RuntimeRequestContext::from_correlation(&correlation).with_metering_context(
+        workspace.clone(),
+        None,
+        InvocationSource::AdminTestRun,
+    );
+    let request = RuntimeExecutionRequest::try_new(
+        &workspace,
+        ExecutionOrigin::AdminDraft,
+        None,
+        &operation,
+        &input,
+        ExecutionAuthorization::Authorized,
+        None,
+        &context,
+        std::time::Instant::now() + std::time::Duration::from_millis(5),
+    )
+    .unwrap();
+
+    let failure = executor.execute_outcome(request).await.unwrap_err();
+    assert_eq!(failure.retryability(), Retryability::AfterDelay);
+    assert_eq!(failure.outcome_certainty(), OutcomeCertainty::Certain);
+}
+
+#[tokio::test]
+async fn read_retry_revalidates_adapter_attempt_and_keeps_one_success_outcome() {
+    let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(FlakyReadAdapter {
+            call_count: Arc::clone(&call_count),
+        }))
+        .build();
+    let mut operation: crank_runtime::RuntimeOperation = operation().into();
+    let Target::Rest(target) = &mut operation.target;
+    target.method = HttpMethod::Get;
+    operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
+
+    let result = executor
+        .execute(&operation, &json!({"name": "retry-read"}))
+        .await
+        .expect("read retry should recover after a known pre-dispatch transport failure");
+
+    assert_eq!(result, json!({"accepted": true}));
+    assert_eq!(call_count.load(std::sync::atomic::Ordering::SeqCst), 2);
+}
+
+#[tokio::test]
+async fn mutating_retry_requires_verified_idempotency_contract() {
+    let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(FlakyMayDispatchAdapter {
+            call_count: Arc::clone(&call_count),
+            captured_keys: Arc::new(Mutex::new(Vec::new())),
+        }))
+        .build();
+    let mut operation: crank_runtime::RuntimeOperation = operation().into();
+    operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
+
+    let failure = executor
+        .execute(&operation, &json!({"name": "write-no-idempotency"}))
+        .await
+        .expect_err("write without idempotency must not retry after dispatch uncertainty");
+
+    assert_eq!(failure.retryability(), Retryability::ManualReconcile);
+    assert_eq!(
+        failure.outcome_certainty(),
+        OutcomeCertainty::OutcomeUnknown
+    );
+    assert_eq!(call_count.load(std::sync::atomic::Ordering::SeqCst), 1);
+}
+
+#[tokio::test]
+async fn mutating_retry_with_verified_idempotency_reuses_same_key() {
+    let call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+    let captured_keys = Arc::new(Mutex::new(Vec::new()));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(FlakyMayDispatchAdapter {
+            call_count: Arc::clone(&call_count),
+            captured_keys: Arc::clone(&captured_keys),
+        }))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let mut operation: crank_runtime::RuntimeOperation = operation().into();
+    operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
+    operation.execution_config.idempotency = Some(IdempotencyPolicy {
+        mode: IdempotencyMode::Required,
+        ttl_ms: 60_000,
+        input_field: Some("name".to_owned()),
+        header_name: Some("Idempotency-Key".to_owned()),
+    });
+    let context = RuntimeRequestContext::from_request_id("req_retry_write")
+        .with_response_cache_scope("workspace", "agent");
+
+    let result = executor
+        .execute_with_context(
+            &operation,
+            &json!({"name": "write-with-idempotency"}),
+            Some(&context),
+        )
+        .await
+        .expect("idempotent write may retry with the same upstream key");
+
+    assert_eq!(result, json!({"accepted": true}));
+    assert_eq!(call_count.load(std::sync::atomic::Ordering::SeqCst), 2);
+    let keys = captured_keys.lock().expect("captured keys lock").clone();
+    assert_eq!(keys.len(), 2);
+    assert_eq!(keys[0], "write-with-idempotency");
+    assert_eq!(keys[1], keys[0]);
+}
+
 #[tokio::test]
 async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
     let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
@@ -116,7 +278,7 @@ async fn failed_mapping_records_closed_category_and_stops_later_stages() {
 }
 
 #[tokio::test]
-async fn execution_without_context_sends_generated_correlation_headers() {
+async fn explicit_test_fixture_sends_generated_correlation_headers() {
     let _tracing_test_guard = TRACING_TEST_LOCK.lock().await;
     let captured_headers = Arc::new(Mutex::new(None));
     let executor = RuntimeExecutorBuilder::new()
@@ -184,7 +346,7 @@ async fn approval_stage_is_present_only_when_confirmation_is_required() {
 
     assert!(matches!(
         result,
-        Err(RuntimeError::ConfirmationRequired { .. })
+        Err(failure) if failure.error_code() == ExecutionErrorCode::ConfirmationRequired
     ));
     let spans = capture.snapshot();
     assert_stage(&spans, "approval.check", "required");
@@ -254,6 +416,113 @@ fn assert_stage(spans: &[CapturedSpan], name: &str, outcome: &str) {
 
 struct SuccessAdapter;
 
+struct DelayedAdapter;
+
+struct FlakyReadAdapter {
+    call_count: Arc<std::sync::atomic::AtomicUsize>,
+}
+
+struct FlakyMayDispatchAdapter {
+    call_count: Arc<std::sync::atomic::AtomicUsize>,
+    captured_keys: Arc<Mutex<Vec<String>>>,
+}
+
+#[async_trait]
+impl ProtocolAdapter for DelayedAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        _prepared: &crank_core::PreparedRequest,
+        context: &crank_core::RuntimeRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        context.mark_dispatch_started();
+        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+        unreachable!("deadline must cancel the adapter future")
+    }
+}
+
+#[async_trait]
+impl ProtocolAdapter for FlakyReadAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        _prepared: &crank_core::PreparedRequest,
+        _context: &crank_core::RuntimeRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        if self
+            .call_count
+            .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
+            == 0
+        {
+            return Err(ProtocolAdapterError::Transport {
+                dispatch: crank_core::DispatchEvidence::NotDispatched,
+            });
+        }
+        Ok(AdapterResponse {
+            status_code: 200,
+            headers: BTreeMap::new(),
+            body: json!({"accepted": true}),
+            data: json!({"accepted": true}),
+        })
+    }
+}
+
+#[async_trait]
+impl ProtocolAdapter for FlakyMayDispatchAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        prepared: &crank_core::PreparedRequest,
+        _context: &crank_core::RuntimeRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        if let Some(key) = prepared.headers.get("Idempotency-Key") {
+            self.captured_keys
+                .lock()
+                .expect("captured keys lock")
+                .push(key.clone());
+        }
+        if self
+            .call_count
+            .fetch_add(1, std::sync::atomic::Ordering::SeqCst)
+            == 0
+        {
+            return Err(ProtocolAdapterError::Transport {
+                dispatch: crank_core::DispatchEvidence::MayHaveDispatched,
+            });
+        }
+        Ok(AdapterResponse {
+            status_code: 200,
+            headers: BTreeMap::new(),
+            body: json!({"accepted": true}),
+            data: json!({"accepted": true}),
+        })
+    }
+}
+
 #[async_trait]
 impl ProtocolAdapter for SuccessAdapter {
     fn protocol(&self) -> Protocol {

crates/crank-runtime/tests/support/mod.rs

@@ -0,0 +1,101 @@
+use std::time::{Duration, Instant};
+
+use crank_core::{ExecutionFailure, ExecutionOrigin, InvocationSource, WorkspaceId};
+use crank_runtime::{
+    ExecutionAuthorization, ResolvedAuth, RuntimeExecutionRequest, RuntimeExecutor,
+    RuntimeOperation, RuntimeRequestContext,
+};
+use serde_json::Value;
+
+#[allow(dead_code)]
+pub trait RuntimeExecutorTestExt {
+    async fn execute(
+        &self,
+        operation: &RuntimeOperation,
+        input: &Value,
+    ) -> Result<Value, ExecutionFailure>;
+
+    async fn execute_with_context(
+        &self,
+        operation: &RuntimeOperation,
+        input: &Value,
+        context: Option<&RuntimeRequestContext>,
+    ) -> Result<Value, ExecutionFailure>;
+
+    async fn execute_with_auth_and_context(
+        &self,
+        operation: &RuntimeOperation,
+        input: &Value,
+        resolved_auth: Option<&ResolvedAuth>,
+        context: Option<&RuntimeRequestContext>,
+    ) -> Result<Value, ExecutionFailure>;
+}
+
+impl RuntimeExecutorTestExt for RuntimeExecutor {
+    async fn execute(
+        &self,
+        operation: &RuntimeOperation,
+        input: &Value,
+    ) -> Result<Value, ExecutionFailure> {
+        self.execute_with_auth_and_context(operation, input, None, None)
+            .await
+    }
+
+    async fn execute_with_context(
+        &self,
+        operation: &RuntimeOperation,
+        input: &Value,
+        context: Option<&RuntimeRequestContext>,
+    ) -> Result<Value, ExecutionFailure> {
+        self.execute_with_auth_and_context(operation, input, None, context)
+            .await
+    }
+
+    async fn execute_with_auth_and_context(
+        &self,
+        operation: &RuntimeOperation,
+        input: &Value,
+        resolved_auth: Option<&ResolvedAuth>,
+        context: Option<&RuntimeRequestContext>,
+    ) -> Result<Value, ExecutionFailure> {
+        let mut context = context.cloned().unwrap_or_else(|| {
+            RuntimeRequestContext::from_correlation(&crank_core::CorrelationContext::generate())
+        });
+        if context.metering_context().is_none() {
+            context = context.with_metering_context(
+                WorkspaceId::new("ws_test"),
+                None,
+                InvocationSource::AdminTestRun,
+            );
+        }
+        let metering = context.metering_context().expect("test metering context");
+        let origin = if metering.agent_id.is_some() {
+            ExecutionOrigin::AgentSnapshot
+        } else {
+            ExecutionOrigin::AdminDraft
+        };
+        let request = RuntimeExecutionRequest::try_new(
+            &metering.workspace_id,
+            origin,
+            metering.agent_id.as_ref(),
+            operation,
+            input,
+            ExecutionAuthorization::Authorized,
+            resolved_auth,
+            &context,
+            Instant::now() + Duration::from_secs(30),
+        )
+        .map_err(|_| {
+            crank_core::ExecutionFailure::new(
+                crank_core::ExecutionErrorCode::RuntimeInternal,
+                crank_core::CorrelationContext::new(
+                    context.request_id.clone(),
+                    context.trace_context.clone(),
+                ),
+            )
+        })?;
+        self.execute_outcome(request)
+            .await
+            .map(|success| success.output)
+    }
+}