наблюдаемость: завершить базовый контур Community

Добавить структурированные журналы, метрики, трассировку и безопасный канал критических ошибок. Усилить границы рантайма, тесты, проверку зависимостей и сценарии развёртывания.

crates/crank-runtime/tests/integration.rs

@@ -2,4 +2,6 @@ mod integration {
     mod confirmation;
     mod idempotency;
     mod no_input_get;
+    mod stages;
+    mod valkey;
 }

crates/crank-runtime/tests/integration/confirmation.rs

@@ -16,6 +16,7 @@ use crank_runtime::{
     InMemoryCoordinationStateStore, RuntimeError, RuntimeExecutorBuilder, RuntimeRequestContext,
 };
 use crank_schema::{Schema, SchemaKind};
+use futures_util::future::join_all;
 use serde_json::json;
 use time::OffsetDateTime;
 
@@ -89,6 +90,62 @@ async fn destructive_operation_requires_single_use_confirmation() {
     assert_eq!(call_count.load(Ordering::SeqCst), 2);
 }
 
+#[tokio::test]
+async fn confirmation_token_allows_only_one_concurrent_execution() {
+    let call_count = Arc::new(AtomicUsize::new(0));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(CountingAdapter {
+            call_count: Arc::clone(&call_count),
+        }))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let operation: crank_runtime::RuntimeOperation = destructive_delete_operation().into();
+    let context = RuntimeRequestContext::from_request_id("req_confirm_concurrent")
+        .with_response_cache_scope("workspace_1", "agent_1");
+    let first = executor
+        .execute_with_context(
+            &operation,
+            &json!({ "order_id": "ord_123" }),
+            Some(&context),
+        )
+        .await
+        .unwrap_err();
+    let RuntimeError::ConfirmationRequired {
+        confirmation_token, ..
+    } = first
+    else {
+        panic!("expected confirmation token")
+    };
+
+    let attempts = (0..16).map(|_| {
+        let executor = executor.clone();
+        let operation = operation.clone();
+        let context = context
+            .clone()
+            .with_confirmation_token(confirmation_token.clone());
+        async move {
+            executor
+                .execute_with_context(
+                    &operation,
+                    &json!({ "order_id": "ord_123" }),
+                    Some(&context),
+                )
+                .await
+        }
+    });
+    let results = join_all(attempts).await;
+    let successful = results.iter().filter(|result| result.is_ok()).count();
+
+    assert_eq!(successful, 1);
+    assert_eq!(call_count.load(Ordering::SeqCst), 1);
+    assert!(
+        results
+            .iter()
+            .filter_map(|result| result.as_ref().err())
+            .all(|error| matches!(error, RuntimeError::InvalidConfirmationToken { .. }))
+    );
+}
+
 struct CountingAdapter {
     call_count: Arc<AtomicUsize>,
 }

crates/crank-runtime/tests/integration/idempotency.rs

@@ -12,7 +12,10 @@ use crank_core::{
     ToolDescription,
 };
 use crank_mapping::{MappingRule, MappingSet};
-use crank_runtime::{InMemoryResponseCacheStore, RuntimeExecutorBuilder};
+use crank_runtime::{
+    InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeError,
+    RuntimeExecutorBuilder,
+};
 use crank_schema::{Schema, SchemaKind};
 use serde_json::json;
 use time::OffsetDateTime;
@@ -23,8 +26,10 @@ async fn replays_mutation_result_for_same_idempotency_key() {
     let executor = RuntimeExecutorBuilder::new()
         .register_adapter(Arc::new(CountingAdapter {
             call_count: Arc::clone(&call_count),
+            release: None,
         }))
         .with_response_cache(Arc::new(InMemoryResponseCacheStore::default()))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
         .build();
     let operation = idempotent_post_operation().into();
     let context = crank_runtime::RuntimeRequestContext::from_request_id("req_1")
@@ -51,8 +56,10 @@ async fn required_idempotency_rejects_missing_key_before_adapter_call() {
     let executor = RuntimeExecutorBuilder::new()
         .register_adapter(Arc::new(CountingAdapter {
             call_count: Arc::clone(&call_count),
+            release: None,
         }))
         .with_response_cache(Arc::new(InMemoryResponseCacheStore::default()))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
         .build();
     let mut operation: crank_runtime::RuntimeOperation = idempotent_post_operation().into();
     let policy = operation.execution_config.idempotency.as_mut().unwrap();
@@ -71,8 +78,189 @@ async fn required_idempotency_rejects_missing_key_before_adapter_call() {
     assert_eq!(call_count.load(Ordering::SeqCst), 0);
 }
 
+#[tokio::test]
+async fn required_idempotency_fails_closed_without_coordination_store() {
+    let call_count = Arc::new(AtomicUsize::new(0));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(CountingAdapter {
+            call_count: Arc::clone(&call_count),
+            release: None,
+        }))
+        .build();
+    let operation = idempotent_post_operation().into();
+    let context = crank_runtime::RuntimeRequestContext::from_request_id("req_no_store")
+        .with_response_cache_scope("workspace_1", "agent_1");
+
+    let error = executor
+        .execute_with_context(
+            &operation,
+            &json!({ "request_id": "must-not-run" }),
+            Some(&context),
+        )
+        .await
+        .expect_err("required idempotency must not execute without an atomic store");
+
+    assert!(matches!(
+        error,
+        RuntimeError::IdempotencyStoreUnavailable { .. }
+    ));
+    assert_eq!(call_count.load(Ordering::SeqCst), 0);
+}
+
+#[tokio::test]
+async fn concurrent_calls_with_same_key_execute_adapter_once() {
+    let call_count = Arc::new(AtomicUsize::new(0));
+    let release = Arc::new(tokio::sync::Notify::new());
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(CountingAdapter {
+            call_count: Arc::clone(&call_count),
+            release: Some(Arc::clone(&release)),
+        }))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let operation: crank_runtime::RuntimeOperation = idempotent_post_operation().into();
+    let context = crank_runtime::RuntimeRequestContext::from_request_id("req_concurrent")
+        .with_response_cache_scope("workspace_1", "agent_1");
+    let input = json!({ "request_id": "order-concurrent" });
+
+    let first_executor = executor.clone();
+    let first_operation = operation.clone();
+    let first_context = context.clone();
+    let first_input = input.clone();
+    let first = tokio::spawn(async move {
+        first_executor
+            .execute_with_context(&first_operation, &first_input, Some(&first_context))
+            .await
+    });
+    wait_for_call_count(&call_count, 1).await;
+
+    let second_executor = executor.clone();
+    let second_operation = operation.clone();
+    let second_context = context.clone();
+    let second_input = input.clone();
+    let second = tokio::spawn(async move {
+        second_executor
+            .execute_with_context(&second_operation, &second_input, Some(&second_context))
+            .await
+    });
+    for _ in 0..100 {
+        tokio::task::yield_now().await;
+    }
+    assert_eq!(call_count.load(Ordering::SeqCst), 1);
+
+    release.notify_waiters();
+    let first_result = first.await.unwrap().unwrap();
+    let second_result = second.await.unwrap().unwrap();
+    assert_eq!(first_result, second_result);
+    assert_eq!(call_count.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test]
+async fn same_key_with_different_input_is_rejected() {
+    let call_count = Arc::new(AtomicUsize::new(0));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(CountingAdapter {
+            call_count: Arc::clone(&call_count),
+            release: None,
+        }))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let operation = idempotent_post_operation().into();
+    let context = crank_runtime::RuntimeRequestContext::from_request_id("req_conflict")
+        .with_response_cache_scope("workspace_1", "agent_1");
+
+    executor
+        .execute_with_context(
+            &operation,
+            &json!({ "request_id": "stable-key", "amount": 10 }),
+            Some(&context),
+        )
+        .await
+        .unwrap();
+    let error = executor
+        .execute_with_context(
+            &operation,
+            &json!({ "request_id": "stable-key", "amount": 20 }),
+            Some(&context),
+        )
+        .await
+        .expect_err("same key must not accept a different request fingerprint");
+
+    assert!(matches!(error, RuntimeError::IdempotencyConflict { .. }));
+    assert_eq!(call_count.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test]
+async fn uncertain_adapter_failure_blocks_automatic_retry() {
+    let call_count = Arc::new(AtomicUsize::new(0));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(FailingAdapter {
+            call_count: Arc::clone(&call_count),
+        }))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let operation = idempotent_post_operation().into();
+    let context = crank_runtime::RuntimeRequestContext::from_request_id("req_unknown")
+        .with_response_cache_scope("workspace_1", "agent_1");
+    let input = json!({ "request_id": "uncertain-outcome" });
+
+    let first = executor
+        .execute_with_context(&operation, &input, Some(&context))
+        .await
+        .expect_err("adapter failure must be returned");
+    assert!(matches!(first, RuntimeError::ProtocolAdapter(_)));
+
+    let retry = executor
+        .execute_with_context(&operation, &input, Some(&context))
+        .await
+        .expect_err("an uncertain external outcome must not be retried automatically");
+    assert!(matches!(
+        retry,
+        RuntimeError::IdempotencyOutcomeUnknown { .. }
+    ));
+    assert_eq!(call_count.load(Ordering::SeqCst), 1);
+}
+
+async fn wait_for_call_count(call_count: &AtomicUsize, expected: usize) {
+    for _ in 0..1_000 {
+        if call_count.load(Ordering::SeqCst) >= expected {
+            return;
+        }
+        tokio::task::yield_now().await;
+    }
+    panic!("adapter did not receive {expected} call(s)");
+}
+
 struct CountingAdapter {
     call_count: Arc<AtomicUsize>,
+    release: Option<Arc<tokio::sync::Notify>>,
+}
+
+struct FailingAdapter {
+    call_count: Arc<AtomicUsize>,
+}
+
+#[async_trait]
+impl ProtocolAdapter for FailingAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        _prepared: &crank_core::PreparedRequest,
+        _context: &RuntimeRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        self.call_count.fetch_add(1, Ordering::SeqCst);
+        Err(ProtocolAdapterError::Message(
+            "upstream outcome is unknown".to_owned(),
+        ))
+    }
 }
 
 #[async_trait]
@@ -91,11 +279,11 @@ impl ProtocolAdapter for CountingAdapter {
         prepared: &crank_core::PreparedRequest,
         _context: &RuntimeRequestContext,
     ) -> Result<AdapterResponse, ProtocolAdapterError> {
-        assert_eq!(
-            prepared.headers.get("Idempotency-Key").map(String::as_str),
-            Some("order-123")
-        );
+        assert!(prepared.headers.contains_key("Idempotency-Key"));
         let call_number = self.call_count.fetch_add(1, Ordering::SeqCst) + 1;
+        if let Some(release) = &self.release {
+            release.notified().await;
+        }
         Ok(AdapterResponse {
             status_code: 201,
             headers: BTreeMap::new(),

crates/crank-runtime/tests/integration/stages.rs

@@ -0,0 +1,426 @@
+use std::{
+    collections::BTreeMap,
+    sync::{Arc, Mutex},
+};
+
+use async_trait::async_trait;
+use crank_core::{
+    AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod,
+    IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass,
+    OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter,
+    ProtocolAdapterError, RestTarget, Target, ToolDescription,
+};
+use crank_mapping::{MappingRule, MappingSet};
+use crank_runtime::{
+    InMemoryCoordinationStateStore, RuntimeError, RuntimeExecutorBuilder, RuntimeRequestContext,
+};
+use crank_schema::{Schema, SchemaKind};
+use crank_trace::{Stage, StageOutcome};
+use serde_json::json;
+use time::OffsetDateTime;
+use tracing::{Id, Instrument, Subscriber, field::Visit, instrument::WithSubscriber};
+use tracing_subscriber::{Layer, layer::SubscriberExt, registry::LookupSpan};
+
+#[tokio::test]
+async fn successful_execution_has_real_stages_and_omits_inapplicable_ones() {
+    let capture = TraceCapture::default();
+    let subscriber = tracing_subscriber::registry().with(capture.clone());
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(SuccessAdapter))
+        .build();
+    let operation = operation().into();
+    let context = RuntimeRequestContext::from_request_id("req_stage_test");
+    let result = async {
+        let root = tracing::info_span!(target: "crank::trace", "mcp.request");
+        executor
+            .execute_with_context(
+                &operation,
+                &json!({"name": "canary-secret"}),
+                Some(&context),
+            )
+            .instrument(root)
+            .await
+    }
+    .with_subscriber(subscriber)
+    .await;
+
+    assert_eq!(result.unwrap(), json!({"accepted": true}));
+    let spans = capture.snapshot();
+    assert_stage(&spans, "runtime.execute", "success");
+    assert_stage(&spans, "runtime.arguments.map", "success");
+    assert_stage(&spans, "upstream.http", "success");
+    assert_stage(&spans, "runtime.response.transform", "success");
+    assert!(!spans.iter().any(|span| span.name == "approval.check"));
+    assert!(!spans.iter().any(|span| span.name == "runtime.idempotency"));
+    assert!(
+        spans
+            .iter()
+            .flat_map(|span| span.fields.values())
+            .all(|value| !value.contains("canary-secret"))
+    );
+
+    let runtime = spans
+        .iter()
+        .find(|span| span.name == "runtime.execute")
+        .expect("runtime span");
+    assert_eq!(runtime.parent_name, Some("mcp.request"));
+    for child in [
+        "runtime.arguments.map",
+        "upstream.http",
+        "runtime.response.transform",
+    ] {
+        assert_eq!(
+            spans
+                .iter()
+                .find(|span| span.name == child)
+                .and_then(|span| span.parent_name),
+            Some("runtime.execute"),
+            "{child} must be a runtime child"
+        );
+    }
+}
+
+#[tokio::test]
+async fn failed_mapping_records_closed_category_and_stops_later_stages() {
+    let capture = TraceCapture::default();
+    let subscriber = tracing_subscriber::registry().with(capture.clone());
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(SuccessAdapter))
+        .build();
+    let operation = operation().into();
+
+    let result = async { executor.execute(&operation, &json!({})).await }
+        .with_subscriber(subscriber)
+        .await;
+
+    assert!(result.is_err());
+    let spans = capture.snapshot();
+    let runtime = spans
+        .iter()
+        .find(|span| span.name == "runtime.execute")
+        .expect("runtime span");
+    assert_eq!(runtime.fields["outcome"], "error");
+    assert_eq!(runtime.fields["error.category"], "schema");
+    assert_stage(&spans, "runtime.arguments.map", "error");
+    assert!(!spans.iter().any(|span| span.name == "upstream.http"));
+    assert!(
+        !spans
+            .iter()
+            .any(|span| span.name == "runtime.response.transform")
+    );
+}
+
+#[tokio::test]
+async fn approval_stage_is_present_only_when_confirmation_is_required() {
+    let capture = TraceCapture::default();
+    let subscriber = tracing_subscriber::registry().with(capture.clone());
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(SuccessAdapter))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let mut source = operation();
+    source.execution_config.safety = Some(OperationSafetyPolicy {
+        class: OperationSafetyClass::Destructive,
+        confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
+    });
+    let operation = source.into();
+    let context = RuntimeRequestContext::from_request_id("req_approval_stage")
+        .with_response_cache_scope("workspace", "agent");
+
+    let result = async {
+        executor
+            .execute_with_context(
+                &operation,
+                &json!({"name": "requires-confirmation"}),
+                Some(&context),
+            )
+            .await
+    }
+    .with_subscriber(subscriber)
+    .await;
+
+    assert!(matches!(
+        result,
+        Err(RuntimeError::ConfirmationRequired { .. })
+    ));
+    let spans = capture.snapshot();
+    assert_stage(&spans, "approval.check", "required");
+    assert!(!spans.iter().any(|span| span.name == "upstream.http"));
+    assert!(!spans.iter().any(|span| span.name == "runtime.idempotency"));
+}
+
+#[tokio::test]
+async fn idempotency_stage_distinguishes_execution_from_replay() {
+    let capture = TraceCapture::default();
+    let subscriber = tracing_subscriber::registry().with(capture.clone());
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(SuccessAdapter))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let mut source = operation();
+    source.execution_config.idempotency = Some(IdempotencyPolicy {
+        mode: IdempotencyMode::Required,
+        ttl_ms: 60_000,
+        input_field: Some("name".to_owned()),
+        header_name: Some("Idempotency-Key".to_owned()),
+    });
+    let operation = source.into();
+    let context = RuntimeRequestContext::from_request_id("req_idempotency_stage")
+        .with_response_cache_scope("workspace", "agent");
+
+    let (first, replay) = async {
+        let first = executor
+            .execute_with_context(&operation, &json!({"name": "stable-key"}), Some(&context))
+            .await;
+        let replay = executor
+            .execute_with_context(&operation, &json!({"name": "stable-key"}), Some(&context))
+            .await;
+        (first, replay)
+    }
+    .with_subscriber(subscriber)
+    .await;
+
+    assert!(first.is_ok());
+    assert!(replay.is_ok());
+    let spans = capture.snapshot();
+    let idempotency_outcomes = spans
+        .iter()
+        .filter(|span| span.name == "runtime.idempotency")
+        .map(|span| span.fields["outcome"].as_str())
+        .collect::<Vec<_>>();
+    assert!(idempotency_outcomes.contains(&"execute"));
+    assert!(idempotency_outcomes.contains(&"replay"));
+    assert_eq!(
+        spans
+            .iter()
+            .filter(|span| span.name == "upstream.http")
+            .count(),
+        1,
+        "replay must not pretend to call upstream"
+    );
+}
+
+fn assert_stage(spans: &[CapturedSpan], name: &str, outcome: &str) {
+    let span = spans
+        .iter()
+        .find(|span| span.name == name)
+        .unwrap_or_else(|| panic!("missing stage {name}"));
+    assert_eq!(span.fields["outcome"], outcome);
+}
+
+struct SuccessAdapter;
+
+#[async_trait]
+impl ProtocolAdapter for SuccessAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        _prepared: &crank_core::PreparedRequest,
+        _context: &crank_core::RuntimeRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        let span = Stage::UpstreamHttp.span();
+        let response = Ok(AdapterResponse {
+            status_code: 200,
+            headers: BTreeMap::new(),
+            body: json!({"accepted": true}),
+            data: json!({"accepted": true}),
+        });
+        StageOutcome::Success.record(&span);
+        response
+    }
+}
+
+fn operation() -> Operation<Schema, MappingSet> {
+    Operation {
+        id: OperationId::new("op_stage_test"),
+        name: "stage_test".to_owned(),
+        display_name: "Stage test".to_owned(),
+        category: "test".to_owned(),
+        protocol: Protocol::Rest,
+        security_level: OperationSecurityLevel::Standard,
+        status: OperationStatus::Published,
+        version: 1,
+        target: Target::Rest(RestTarget {
+            base_url: "https://example.invalid".to_owned(),
+            method: HttpMethod::Post,
+            path_template: "/test".to_owned(),
+            static_headers: BTreeMap::new(),
+        }),
+        input_schema: object_schema(BTreeMap::from([("name".to_owned(), string_schema())])),
+        output_schema: object_schema(BTreeMap::from([("accepted".to_owned(), bool_schema())])),
+        input_mapping: MappingSet {
+            rules: vec![MappingRule {
+                source: "$.mcp.name".to_owned(),
+                target: "$.request.body.name".to_owned(),
+                required: true,
+                default_value: None,
+                transform: None,
+                condition: None,
+                notes: None,
+            }],
+        },
+        output_mapping: MappingSet {
+            rules: vec![MappingRule {
+                source: "$.response.body.accepted".to_owned(),
+                target: "$.output.accepted".to_owned(),
+                required: true,
+                default_value: None,
+                transform: None,
+                condition: None,
+                notes: None,
+            }],
+        },
+        execution_config: ExecutionConfig {
+            timeout_ms: 1_000,
+            retry_policy: None,
+            response_cache: None,
+            idempotency: None,
+            safety: None,
+            approval_policy: None,
+            auth_profile_ref: None,
+            headers: BTreeMap::new(),
+        },
+        tool_description: ToolDescription {
+            title: "Stage test".to_owned(),
+            description: "Tests trace stages.".to_owned(),
+            tags: Vec::new(),
+            examples: Vec::new(),
+        },
+        samples: None,
+        generated_draft: None,
+        config_export: None,
+        wizard_state: None,
+        created_at: OffsetDateTime::UNIX_EPOCH,
+        updated_at: OffsetDateTime::UNIX_EPOCH,
+        published_at: None,
+    }
+}
+
+fn object_schema(fields: BTreeMap<String, Schema>) -> Schema {
+    Schema {
+        kind: SchemaKind::Object,
+        description: None,
+        required: true,
+        nullable: false,
+        default_value: None,
+        fields,
+        items: None,
+        enum_values: Vec::new(),
+        variants: Vec::new(),
+    }
+}
+
+fn string_schema() -> Schema {
+    Schema {
+        kind: SchemaKind::String,
+        description: None,
+        required: true,
+        nullable: false,
+        default_value: None,
+        fields: BTreeMap::new(),
+        items: None,
+        enum_values: Vec::new(),
+        variants: Vec::new(),
+    }
+}
+
+fn bool_schema() -> Schema {
+    Schema {
+        kind: SchemaKind::Boolean,
+        ..string_schema()
+    }
+}
+
+#[derive(Clone, Default)]
+struct TraceCapture {
+    spans: Arc<Mutex<Vec<CapturedSpan>>>,
+}
+
+impl TraceCapture {
+    fn snapshot(&self) -> Vec<CapturedSpan> {
+        self.spans.lock().expect("span lock").clone()
+    }
+}
+
+#[derive(Clone, Debug)]
+struct CapturedSpan {
+    name: &'static str,
+    parent_name: Option<&'static str>,
+    fields: BTreeMap<String, String>,
+}
+
+impl<S> Layer<S> for TraceCapture
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
+    fn on_new_span(
+        &self,
+        attributes: &tracing::span::Attributes<'_>,
+        id: &Id,
+        context: tracing_subscriber::layer::Context<'_, S>,
+    ) {
+        let parent = attributes
+            .parent()
+            .and_then(|parent| context.span(parent))
+            .or_else(|| {
+                attributes
+                    .is_contextual()
+                    .then(|| context.lookup_current())
+                    .flatten()
+            });
+        let mut visitor = FieldVisitor::default();
+        attributes.record(&mut visitor);
+        let mut spans = self.spans.lock().expect("span lock");
+        let index = spans.len();
+        spans.push(CapturedSpan {
+            name: attributes.metadata().name(),
+            parent_name: parent.map(|span| span.metadata().name()),
+            fields: visitor.fields,
+        });
+        context
+            .span(id)
+            .expect("span exists")
+            .extensions_mut()
+            .insert(index);
+    }
+
+    fn on_record(
+        &self,
+        id: &Id,
+        values: &tracing::span::Record<'_>,
+        context: tracing_subscriber::layer::Context<'_, S>,
+    ) {
+        let mut visitor = FieldVisitor::default();
+        values.record(&mut visitor);
+        let span = context.span(id).expect("span exists");
+        let index = *span.extensions().get::<usize>().expect("capture index");
+        self.spans.lock().expect("span lock")[index]
+            .fields
+            .extend(visitor.fields);
+    }
+}
+
+#[derive(Default)]
+struct FieldVisitor {
+    fields: BTreeMap<String, String>,
+}
+
+impl Visit for FieldVisitor {
+    fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
+        self.fields
+            .insert(field.name().to_owned(), value.to_owned());
+    }
+
+    fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
+        self.fields
+            .insert(field.name().to_owned(), format!("{value:?}"));
+    }
+}

crates/crank-runtime/tests/integration/valkey.rs

@@ -0,0 +1,167 @@
+use std::{sync::Arc, time::Duration};
+
+use crank_core::{
+    CacheBackend, CacheScope, CoordinationStateReservation, CoordinationStateStore,
+    CoordinationStateValue, RateLimitDecision, RateLimitStateStore,
+};
+use crank_runtime::RedisCacheStore;
+use futures_util::future::join_all;
+use serde_json::json;
+use testcontainers::{
+    GenericImage,
+    core::{IntoContainerPort, WaitFor},
+    runners::AsyncRunner,
+};
+
+#[tokio::test]
+async fn valkey_coordination_and_rate_limit_operations_are_atomic() {
+    let container = GenericImage::new("valkey/valkey", "8-alpine")
+        .with_exposed_port(6379.tcp())
+        .with_wait_for(WaitFor::message_on_stdout("Ready to accept connections"))
+        .start()
+        .await
+        .expect("Valkey test container must start");
+    let port = container
+        .get_host_port_ipv4(6379.tcp())
+        .await
+        .expect("Valkey port must be mapped");
+    let store = Arc::new(
+        RedisCacheStore::connect(CacheBackend::Valkey, &format!("redis://127.0.0.1:{port}/0"))
+            .await
+            .expect("runtime store must connect to Valkey"),
+    );
+
+    verify_atomic_coordination(store.as_ref()).await;
+    verify_atomic_rate_limit(store).await;
+}
+
+async fn verify_atomic_coordination(store: &RedisCacheStore) {
+    let pending = CoordinationStateValue {
+        payload: json!({ "state": "pending" }),
+    };
+    let attempts = (0..32).map(|_| {
+        store.reserve_value(
+            CacheScope::Coordination,
+            "valkey-reservation",
+            pending.clone(),
+            Duration::from_secs(30),
+        )
+    });
+    let results = join_all(attempts).await;
+    assert_eq!(
+        results
+            .iter()
+            .filter(|result| matches!(result, Ok(CoordinationStateReservation::Reserved)))
+            .count(),
+        1
+    );
+
+    let completed = CoordinationStateValue {
+        payload: json!({ "state": "completed" }),
+    };
+    assert!(
+        store
+            .compare_and_set_value(
+                CacheScope::Coordination,
+                "valkey-reservation",
+                &pending,
+                completed.clone(),
+                Duration::from_secs(30),
+            )
+            .await
+            .unwrap()
+    );
+    assert_eq!(
+        store
+            .take_value(CacheScope::Coordination, "valkey-reservation")
+            .await
+            .unwrap(),
+        Some(completed)
+    );
+    assert_eq!(
+        store
+            .take_value(CacheScope::Coordination, "valkey-reservation")
+            .await
+            .unwrap(),
+        None
+    );
+}
+
+async fn verify_atomic_rate_limit(store: Arc<RedisCacheStore>) {
+    let attempts = (0..64).map(|_| {
+        let store = Arc::clone(&store);
+        async move {
+            store
+                .consume_token(
+                    "valkey-burst",
+                    8_000_000,
+                    1_000_000,
+                    0,
+                    Duration::from_secs(30),
+                )
+                .await
+        }
+    });
+    let results = join_all(attempts).await;
+    assert_eq!(
+        results
+            .iter()
+            .filter(|result| matches!(result, Ok(RateLimitDecision::Allowed)))
+            .count(),
+        8
+    );
+    assert!(
+        results
+            .iter()
+            .filter_map(|result| result.as_ref().ok())
+            .all(|decision| matches!(
+                decision,
+                RateLimitDecision::Allowed
+                    | RateLimitDecision::Rejected {
+                        retry_after_ms: 1_000
+                    }
+            ))
+    );
+
+    assert_eq!(
+        store
+            .consume_token(
+                "valkey-retry-after",
+                2_000_000,
+                2_000_000,
+                0,
+                Duration::from_secs(30),
+            )
+            .await
+            .unwrap(),
+        RateLimitDecision::Allowed
+    );
+    assert_eq!(
+        store
+            .consume_token(
+                "valkey-retry-after",
+                2_000_000,
+                2_000_000,
+                0,
+                Duration::from_secs(30),
+            )
+            .await
+            .unwrap(),
+        RateLimitDecision::Allowed
+    );
+    assert_eq!(
+        store
+            .consume_token(
+                "valkey-retry-after",
+                2_000_000,
+                2_000_000,
+                0,
+                Duration::from_secs(30),
+            )
+            .await
+            .unwrap(),
+        RateLimitDecision::Rejected {
+            retry_after_ms: 500
+        }
+    );
+}

crates/crank-runtime/tests/unit/cache.rs

@@ -1,9 +1,14 @@
-use std::time::Duration;
+use std::{
+    ffi::OsString,
+    sync::{Mutex, MutexGuard},
+    time::Duration,
+};
 
 use crank_core::{
     CacheBackend, CacheScope, CacheStoreError, CachedHeader, CachedResponse,
-    CoordinationStateStore, CoordinationStateValue, RateLimitBucketState, RateLimitStateStore,
-    ReplayGuardStatus, ReplayGuardStore, ResponseCacheStore,
+    CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue,
+    RateLimitBucketState, RateLimitStateStore, ReplayGuardStatus, ReplayGuardStore,
+    ResponseCacheStore,
 };
 use crank_runtime::{
     InMemoryCoordinationStateStore, InMemoryRateLimitStateStore, InMemoryReplayGuardStore,
@@ -12,6 +17,13 @@ use crank_runtime::{
 };
 use serde_json::json;
 
+const CACHE_ENV_NAMES: [&str; 3] = [
+    "CRANK_CACHE_BACKEND",
+    "CRANK_CACHE_URL",
+    "CRANK_CACHE_DEFAULT_TTL_MS",
+];
+static CACHE_ENV_LOCK: Mutex<()> = Mutex::new(());
+
 #[test]
 fn defaults_to_in_memory_cache_without_url() {
     let config = RuntimeCacheConfig::default();
@@ -21,8 +33,24 @@ fn defaults_to_in_memory_cache_without_url() {
     assert_eq!(config.default_ttl_ms, None);
 }
 
+#[test]
+fn treats_blank_optional_cache_values_as_unset() {
+    let _env = IsolatedCacheEnv::new();
+    unsafe {
+        std::env::set_var("CRANK_CACHE_URL", "  ");
+        std::env::set_var("CRANK_CACHE_DEFAULT_TTL_MS", "  ");
+    }
+
+    let config = RuntimeCacheConfig::from_env().unwrap();
+
+    assert_eq!(config.backend, CacheBackend::Memory);
+    assert_eq!(config.url, None);
+    assert_eq!(config.default_ttl_ms, None);
+}
+
 #[test]
 fn loads_valkey_config_from_env() {
+    let _env = IsolatedCacheEnv::new();
     unsafe {
         std::env::set_var("CRANK_CACHE_BACKEND", "valkey");
         std::env::set_var("CRANK_CACHE_URL", "redis://cache:6379/0");
@@ -34,16 +62,11 @@ fn loads_valkey_config_from_env() {
     assert_eq!(config.backend, CacheBackend::Valkey);
     assert_eq!(config.url.as_deref(), Some("redis://cache:6379/0"));
     assert_eq!(config.default_ttl_ms, Some(15_000));
-
-    unsafe {
-        std::env::remove_var("CRANK_CACHE_BACKEND");
-        std::env::remove_var("CRANK_CACHE_URL");
-        std::env::remove_var("CRANK_CACHE_DEFAULT_TTL_MS");
-    }
 }
 
 #[test]
 fn rejects_external_backend_without_url() {
+    let _env = IsolatedCacheEnv::new();
     unsafe {
         std::env::set_var("CRANK_CACHE_BACKEND", "redis");
         std::env::remove_var("CRANK_CACHE_URL");
@@ -57,14 +80,11 @@ fn rejects_external_backend_without_url() {
             backend: CacheBackend::Redis
         }
     ));
-
-    unsafe {
-        std::env::remove_var("CRANK_CACHE_BACKEND");
-    }
 }
 
 #[test]
 fn rejects_zero_ttl() {
+    let _env = IsolatedCacheEnv::new();
     unsafe {
         std::env::set_var("CRANK_CACHE_DEFAULT_TTL_MS", "0");
     }
@@ -77,9 +97,44 @@ fn rejects_zero_ttl() {
             name: "CRANK_CACHE_DEFAULT_TTL_MS"
         }
     ));
+}
 
-    unsafe {
-        std::env::remove_var("CRANK_CACHE_DEFAULT_TTL_MS");
+struct IsolatedCacheEnv {
+    _lock: MutexGuard<'static, ()>,
+    previous: Vec<(&'static str, Option<OsString>)>,
+}
+
+impl IsolatedCacheEnv {
+    fn new() -> Self {
+        let lock = CACHE_ENV_LOCK
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        let previous = CACHE_ENV_NAMES
+            .iter()
+            .map(|name| (*name, std::env::var_os(name)))
+            .collect();
+        for name in CACHE_ENV_NAMES {
+            unsafe {
+                std::env::remove_var(name);
+            }
+        }
+        Self {
+            _lock: lock,
+            previous,
+        }
+    }
+}
+
+impl Drop for IsolatedCacheEnv {
+    fn drop(&mut self) {
+        for (name, value) in &self.previous {
+            unsafe {
+                match value {
+                    Some(value) => std::env::set_var(name, value),
+                    None => std::env::remove_var(name),
+                }
+            }
+        }
     }
 }
 
@@ -209,6 +264,81 @@ async fn in_memory_coordination_store_scopes_keys() {
     );
 }
 
+#[tokio::test]
+async fn in_memory_coordination_store_atomically_takes_and_reserves_values() {
+    let store = InMemoryCoordinationStateStore::default();
+    let value = CoordinationStateValue {
+        payload: json!({ "state": "pending" }),
+    };
+    store
+        .put_value(
+            CacheScope::Coordination,
+            "atomic-job",
+            value.clone(),
+            Duration::from_secs(20),
+        )
+        .await
+        .unwrap();
+
+    let (first, second) = tokio::join!(
+        store.take_value(CacheScope::Coordination, "atomic-job"),
+        store.take_value(CacheScope::Coordination, "atomic-job")
+    );
+    assert_eq!(
+        usize::from(first.unwrap().is_some()) + usize::from(second.unwrap().is_some()),
+        1
+    );
+
+    assert_eq!(
+        store
+            .reserve_value(
+                CacheScope::Coordination,
+                "reservation",
+                value.clone(),
+                Duration::from_secs(20),
+            )
+            .await
+            .unwrap(),
+        CoordinationStateReservation::Reserved
+    );
+    assert_eq!(
+        store
+            .reserve_value(
+                CacheScope::Coordination,
+                "reservation",
+                CoordinationStateValue {
+                    payload: json!({ "state": "other" }),
+                },
+                Duration::from_secs(20),
+            )
+            .await
+            .unwrap(),
+        CoordinationStateReservation::Existing(value.clone())
+    );
+    let completed = CoordinationStateValue {
+        payload: json!({ "state": "completed" }),
+    };
+    assert!(
+        store
+            .compare_and_set_value(
+                CacheScope::Coordination,
+                "reservation",
+                &value,
+                completed.clone(),
+                Duration::from_secs(20),
+            )
+            .await
+            .unwrap()
+    );
+    assert_eq!(
+        store
+            .get_value(CacheScope::Coordination, "reservation")
+            .await
+            .unwrap(),
+        Some(completed)
+    );
+}
+
 #[tokio::test]
 async fn in_memory_stores_reject_empty_keys() {
     let response_store = InMemoryResponseCacheStore::default();