наблюдаемость: ввести безопасный контракт метрик

crates/crank-runtime/Cargo.toml

@@ -19,10 +19,10 @@ base64.workspace = true
 crank-adapter-rest = { path = "../crank-adapter-rest" }
 crank-core = { path = "../crank-core" }
 crank-mapping = { path = "../crank-mapping" }
+crank-metrics = { path = "../crank-metrics" }
 crank-schema = { path = "../crank-schema" }
 crank-trace = { path = "../crank-trace" }
 hkdf.workspace = true
-metrics.workspace = true
 redis = { version = "0.29", features = ["tokio-comp", "connection-manager"] }
 serde.workspace = true
 serde_json.workspace = true
@@ -36,6 +36,8 @@ uuid.workspace = true
 [dev-dependencies]
 axum.workspace = true
 futures-util = "0.3"
+metrics.workspace = true
+metrics-util = "0.20.4"
 testcontainers.workspace = true
 time.workspace = true
 tracing-subscriber.workspace = true

crates/crank-runtime/src/confirmation.rs

@@ -38,7 +38,14 @@ pub async fn confirm_operation(
         .or_else(|| confirmation_token_from_input(input));
 
     let Some(provided_token) = provided_token else {
-        let token = issue_confirmation_token(store, &scope, &input_hash, &safety).await?;
+        let token = issue_confirmation_token(
+            store,
+            operation.operation_id.as_str(),
+            &scope,
+            &input_hash,
+            &safety,
+        )
+        .await?;
         return Err(RuntimeError::ConfirmationRequired {
             operation_id: operation.operation_id.as_str().to_owned(),
             safety_class: safety.class,
@@ -47,7 +54,14 @@ pub async fn confirm_operation(
         });
     };
 
-    consume_confirmation_token(store, &scope, provided_token, &input_hash).await
+    consume_confirmation_token(
+        store,
+        operation.operation_id.as_str(),
+        &scope,
+        provided_token,
+        &input_hash,
+    )
+    .await
 }
 
 pub(crate) fn is_applicable(operation: &RuntimeOperation) -> bool {
@@ -108,6 +122,7 @@ fn confirmation_scope(
 
 async fn issue_confirmation_token(
     store: &dyn CoordinationStateStore,
+    operation_id: &str,
     scope: &str,
     input_hash: &str,
     safety: &OperationSafetyPolicy,
@@ -128,15 +143,15 @@ async fn issue_confirmation_token(
             Duration::from_millis(ttl_ms),
         )
         .await
-        .map_err(|error| RuntimeError::InvalidPreparedRequest {
-            field: "confirmation_token".to_owned(),
-            reason: error.to_string(),
+        .map_err(|_| RuntimeError::ConfirmationStoreUnavailable {
+            operation_id: operation_id.to_owned(),
         })?;
     Ok(token)
 }
 
 async fn consume_confirmation_token(
     store: &dyn CoordinationStateStore,
+    operation_id: &str,
     operation_scope: &str,
     token: &str,
     input_hash: &str,
@@ -145,9 +160,8 @@ async fn consume_confirmation_token(
     let stored = store
         .take_value(CacheScope::Coordination, &key)
         .await
-        .map_err(|error| RuntimeError::InvalidPreparedRequest {
-            field: "confirmation_token".to_owned(),
-            reason: error.to_string(),
+        .map_err(|_| RuntimeError::ConfirmationStoreUnavailable {
+            operation_id: operation_id.to_owned(),
         })?;
 
     let Some(stored) = stored else {

crates/crank-runtime/src/executor.rs

@@ -5,12 +5,17 @@ use crank_core::{
     AdapterRegistry, CoordinationStateStore, ExecutionMode, InvocationSource, InvocationStatus,
     MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
 };
+use crank_metrics::{
+    CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
+    InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics,
+    ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome,
+    record_limit_rejection,
+};
 use crank_trace::{ErrorCategory, Stage, StageOutcome};
-use metrics::Gauge;
 use serde_json::{Map, Value, json};
 use time::OffsetDateTime;
 use tokio::sync::{OwnedSemaphorePermit, Semaphore};
-use tracing::{Instrument, Span, debug};
+use tracing::{Instrument, Span, debug, warn};
 use uuid::Uuid;
 
 use crate::{
@@ -42,6 +47,65 @@ pub struct RuntimeExecutionRequest<'a> {
     pub request_context: Option<&'a RuntimeRequestContext>,
 }
 
+struct IdempotencyCancellationGuard {
+    cleanup: Option<IdempotencyCancellationCleanup>,
+    runtime: tokio::runtime::Handle,
+}
+
+struct IdempotencyCancellationCleanup {
+    store: Arc<dyn CoordinationStateStore>,
+    operation: RuntimeOperation,
+    reservation: crate::idempotency::IdempotencyReservation,
+}
+
+impl IdempotencyCancellationGuard {
+    fn new(
+        store: Arc<dyn CoordinationStateStore>,
+        operation: RuntimeOperation,
+        reservation: crate::idempotency::IdempotencyReservation,
+    ) -> Self {
+        Self {
+            cleanup: Some(IdempotencyCancellationCleanup {
+                store,
+                operation,
+                reservation,
+            }),
+            runtime: tokio::runtime::Handle::current(),
+        }
+    }
+
+    fn disarm(&mut self) {
+        self.cleanup = None;
+    }
+}
+
+impl Drop for IdempotencyCancellationGuard {
+    fn drop(&mut self) {
+        let Some(cleanup) = self.cleanup.take() else {
+            return;
+        };
+        self.runtime.spawn(async move {
+            let result = crate::idempotency::mark_outcome_unknown(
+                cleanup.store.as_ref(),
+                &cleanup.operation,
+                &cleanup.reservation,
+            )
+            .await;
+            record_idempotency_outcome(match &result {
+                Ok(()) => IdempotencyOutcome::OutcomeUnknown,
+                Err(error) => idempotency_error_outcome(error),
+            });
+            if result.is_err() {
+                warn!(
+                    name: "runtime.idempotency.cancellation_cleanup_failed",
+                    error_category = "idempotency_store",
+                    "failed to finalize cancelled idempotent execution"
+                );
+            }
+        });
+    }
+}
+
 impl<'a> RuntimeExecutionRequest<'a> {
     pub fn new(operation: &'a RuntimeOperation, input: &'a Value) -> Self {
         Self {
@@ -180,10 +244,12 @@ impl RuntimeExecutor {
     ) -> Result<Value, RuntimeError> {
         log_runtime_event("unary.execute", request.operation, request.request_context);
         let started_at = Instant::now();
+        let invocation_metrics =
+            ToolInvocationMetrics::start(metric_invocation_source(request.request_context));
         let runtime_span = Stage::RuntimeExecute.span();
         let result = async {
             let _permit = self.acquire_unary_permit(request.operation)?;
-            let _inflight = RuntimeInFlightGuard::new();
+            let _inflight = InFlightGuard::runtime();
             let mapping_span = Stage::RuntimeArgumentsMap.span();
             let prepared_request =
                 mapping_span.in_scope(|| self.prepare_request(request.operation, request.input));
@@ -203,7 +269,11 @@ impl RuntimeExecutor {
         .await;
         record_runtime_result(&runtime_span, &result);
         drop(runtime_span);
-        record_execution_metrics(request.request_context, &result, started_at);
+        let (outcome, error_kind) = match &result {
+            Ok(_) => (ToolOutcome::Success, ToolErrorKind::None),
+            Err(error) => (ToolOutcome::Error, runtime_error_kind(error)),
+        };
+        invocation_metrics.complete(outcome, error_kind);
         self.record_metering(
             request.operation,
             request.request_context,
@@ -245,6 +315,7 @@ impl RuntimeExecutor {
         ) {
             Ok(key) => key,
             Err(error) if idempotency_applicable => {
+                record_idempotency_outcome(idempotency_error_outcome(&error));
                 let span = Stage::RuntimeIdempotency.span();
                 StageOutcome::Error.record(&span);
                 ErrorCategory::Idempotency.record(&span);
@@ -264,14 +335,19 @@ impl RuntimeExecutor {
             .instrument(approval_span.clone())
             .await;
             match &approval_result {
-                Ok(()) => StageOutcome::Success.record(&approval_span),
+                Ok(()) => {
+                    StageOutcome::Success.record(&approval_span);
+                    record_confirmation_outcome(ConfirmationOutcome::Approved);
+                }
                 Err(RuntimeError::ConfirmationRequired { .. }) => {
                     StageOutcome::Required.record(&approval_span);
                     ErrorCategory::Approval.record(&approval_span);
+                    record_confirmation_outcome(ConfirmationOutcome::Required);
                 }
                 Err(error) => {
                     StageOutcome::Error.record(&approval_span);
                     runtime_error_category(error).record(&approval_span);
+                    record_confirmation_outcome(confirmation_error_outcome(error));
                 }
             }
             drop(approval_span);
@@ -292,9 +368,11 @@ impl RuntimeExecutor {
             match &result {
                 Ok(crate::idempotency::IdempotencyAction::Execute(_)) => {
                     StageOutcome::Execute.record(&idempotency_span);
+                    record_idempotency_outcome(IdempotencyOutcome::Execute);
                 }
                 Ok(crate::idempotency::IdempotencyAction::Replay(_)) => {
                     StageOutcome::Replay.record(&idempotency_span);
+                    record_idempotency_outcome(IdempotencyOutcome::Replay);
                 }
                 Ok(crate::idempotency::IdempotencyAction::Disabled) => {
                     StageOutcome::Skipped.record(&idempotency_span);
@@ -302,6 +380,7 @@ impl RuntimeExecutor {
                 Err(error) => {
                     StageOutcome::Error.record(&idempotency_span);
                     runtime_error_category(error).record(&idempotency_span);
+                    record_idempotency_outcome(idempotency_error_outcome(error));
                 }
             }
             drop(idempotency_span);
@@ -312,6 +391,18 @@ impl RuntimeExecutor {
         if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency {
             return transform_response(operation, response);
         }
+        let mut cancellation_guard =
+            if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency {
+                self.coordination_store.as_ref().map(|store| {
+                    IdempotencyCancellationGuard::new(
+                        Arc::clone(store),
+                        operation.clone(),
+                        reservation.clone(),
+                    )
+                })
+            } else {
+                None
+            };
 
         let adapter_result = match self
             .load_cached_adapter_response(operation, &prepared_request, request_context)
@@ -346,6 +437,13 @@ impl RuntimeExecutor {
                             .instrument(idempotency_span.clone())
                             .await;
                     record_runtime_result(&idempotency_span, &cleanup_result);
+                    record_idempotency_outcome(match &cleanup_result {
+                        Ok(()) => IdempotencyOutcome::OutcomeUnknown,
+                        Err(error) => idempotency_error_outcome(error),
+                    });
+                    if let Some(guard) = &mut cancellation_guard {
+                        guard.disarm();
+                    }
                 }
                 return Err(error);
             }
@@ -359,6 +457,13 @@ impl RuntimeExecutor {
                     .instrument(idempotency_span.clone())
                     .await;
             record_runtime_result(&idempotency_span, &completion_result);
+            record_idempotency_outcome(match &completion_result {
+                Ok(()) => IdempotencyOutcome::Completed,
+                Err(error) => idempotency_error_outcome(error),
+            });
+            if let Some(guard) = &mut cancellation_guard {
+                guard.disarm();
+            }
             drop(idempotency_span);
             completion_result?;
         }
@@ -447,8 +552,13 @@ impl RuntimeExecutor {
         let response_cache = self.response_cache.as_ref()?;
         let cache_key = response_cache_key(operation, prepared_request, request_context)?;
         let cached = match response_cache.get(&cache_key).await {
-            Ok(cached) => cached?,
+            Ok(Some(cached)) => cached,
+            Ok(None) => {
+                record_cache_outcome(CacheOutcome::Miss);
+                return None;
+            }
             Err(_) => {
+                record_cache_outcome(CacheOutcome::ReadError);
                 debug!(
                     name: "runtime.response_cache.read_failed",
                     operation_id = operation.operation_id.as_str(),
@@ -460,15 +570,21 @@ impl RuntimeExecutor {
         };
 
         match adapter_response_from_cached(cached) {
-            Ok(response) => Some(response),
+            Ok(response) => {
+                record_cache_outcome(CacheOutcome::Hit);
+                Some(response)
+            }
             Err(_) => {
+                record_cache_outcome(CacheOutcome::DecodeError);
                 debug!(
                     name: "runtime.response_cache.decode_failed",
                     operation_id = operation.operation_id.as_str(),
                     error_category = "cached_response",
                     "cached response payload was invalid"
                 );
-                let _ = response_cache.delete(&cache_key).await;
+                if response_cache.delete(&cache_key).await.is_err() {
+                    record_cache_outcome(CacheOutcome::EvictError);
+                }
                 None
             }
         }
@@ -505,12 +621,15 @@ impl RuntimeExecutor {
             .await
             .is_err()
         {
+            record_cache_outcome(CacheOutcome::WriteError);
             debug!(
                 name: "runtime.response_cache.write_failed",
                 operation_id = operation.operation_id.as_str(),
                 error_category = "response_cache",
                 "response cache write skipped"
             );
+        } else {
+            record_cache_outcome(CacheOutcome::Stored);
         }
     }
 }
@@ -634,86 +753,64 @@ fn try_acquire_limit(
     limit: usize,
 ) -> Result<OwnedSemaphorePermit, RuntimeError> {
     limiter.try_acquire_owned().map_err(|_| {
-        metrics::counter!(
-            "crank_runtime_limit_rejections_total",
-            "stage" => "concurrency"
-        )
-        .increment(1);
+        record_limit_rejection(LimitStage::Concurrency);
         RuntimeError::ConcurrencyLimitExceeded { kind, limit }
     })
 }
 
-fn record_execution_metrics<T>(
+fn metric_invocation_source(
     request_context: Option<&RuntimeRequestContext>,
-    result: &Result<T, RuntimeError>,
-    started_at: Instant,
-) {
-    let source = request_context
+) -> MetricInvocationSource {
+    request_context
         .and_then(RuntimeRequestContext::metering_context)
-        .map_or("internal", |context| match context.source {
-            InvocationSource::AdminTestRun => "admin_test_run",
-            InvocationSource::AgentToolCall => "agent_tool_call",
-        });
-    let (outcome, error_kind) = match result {
-        Ok(_) => ("success", "none"),
-        Err(error) => ("error", runtime_error_kind(error)),
-    };
-
-    metrics::counter!(
-        "crank_tool_invocations_total",
-        "source" => source,
-        "outcome" => outcome,
-        "error_kind" => error_kind
-    )
-    .increment(1);
-    metrics::histogram!(
-        "crank_tool_invocation_duration_seconds",
-        "source" => source,
-        "outcome" => outcome
-    )
-    .record(started_at.elapsed().as_secs_f64());
+        .map_or(MetricInvocationSource::Internal, |context| {
+            match context.source {
+                InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun,
+                InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall,
+            }
+        })
 }
 
-fn runtime_error_kind(error: &RuntimeError) -> &'static str {
+fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
     match error {
-        RuntimeError::Schema(_) => "schema",
-        RuntimeError::Mapping(_) => "mapping",
-        RuntimeError::RestAdapter(_) => "rest_adapter",
-        RuntimeError::ProtocolAdapter(_) => "protocol_adapter",
-        RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
-        RuntimeError::UnsupportedExecutionMode { .. } => "unsupported_execution_mode",
-        RuntimeError::ConcurrencyLimitExceeded { .. } => "concurrency_limit",
-        RuntimeError::InvalidPreparedRequest { .. } => "invalid_prepared_request",
-        RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
-        RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
-        RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_store",
-        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_store",
-        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
-        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
-        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
-        RuntimeError::MissingAuthProfile { .. } => "missing_auth_profile",
-        RuntimeError::MissingSecret { .. } => "missing_secret",
-        RuntimeError::MissingSecretVersion { .. } => "missing_secret_version",
-        RuntimeError::InvalidAuthSecretValue { .. } => "invalid_auth_secret",
-        RuntimeError::SecretCrypto { .. } => "secret_crypto",
+        RuntimeError::Schema(_) => ToolErrorKind::Schema,
+        RuntimeError::Mapping(_) => ToolErrorKind::Mapping,
+        RuntimeError::RestAdapter(_) => ToolErrorKind::RestAdapter,
+        RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter,
+        RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
+        RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
+        RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
+        RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
+        RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
+        RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
+        RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
+        RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore,
+        RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress,
+        RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict,
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown,
+        RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile,
+        RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret,
+        RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion,
+        RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret,
+        RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto,
     }
 }
 
-struct RuntimeInFlightGuard {
-    gauge: Gauge,
-}
-
-impl RuntimeInFlightGuard {
-    fn new() -> Self {
-        let gauge = metrics::gauge!("crank_runtime_inflight");
-        gauge.increment(1.0);
-        Self { gauge }
+fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome {
+    match error {
+        RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict,
+        RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress,
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => IdempotencyOutcome::OutcomeUnknown,
+        RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable,
+        _ => IdempotencyOutcome::Error,
     }
 }
 
-impl Drop for RuntimeInFlightGuard {
-    fn drop(&mut self) {
-        self.gauge.decrement(1.0);
+fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome {
+    match error {
+        RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken,
+        RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable,
+        _ => ConfirmationOutcome::Error,
     }
 }
 

crates/crank-runtime/src/idempotency.rs

@@ -21,6 +21,7 @@ pub(crate) enum IdempotencyAction {
     Replay(AdapterResponse),
 }
 
+#[derive(Clone)]
 pub(crate) struct IdempotencyReservation {
     key: String,
     initial: CoordinationStateValue,

crates/crank-runtime/src/rate_limit.rs

@@ -5,6 +5,7 @@ use std::{
 };
 
 use crank_core::{RateLimitDecision, RateLimitStateStore};
+use crank_metrics::{LimitStage, record_limit_rejection};
 use thiserror::Error;
 use tracing::warn;
 
@@ -105,11 +106,7 @@ impl RequestRateLimiter {
             }
         };
         if matches!(result, Err(RateLimitCheckError::Rejected(_))) {
-            metrics::counter!(
-                "crank_runtime_limit_rejections_total",
-                "stage" => "rate_limit"
-            )
-            .increment(1);
+            record_limit_rejection(LimitStage::RateLimit);
         }
         result
     }

crates/crank-runtime/tests/integration/confirmation.rs

@@ -6,8 +6,9 @@ use std::sync::{
 
 use async_trait::async_trait;
 use crank_core::{
-    AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod, Operation,
-    OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
+    AdapterResponse, CacheScope, CacheStoreError, ConfirmationPolicy, CoordinationStateReservation,
+    CoordinationStateStore, CoordinationStateValue, ExecutionConfig, ExecutionMode, HttpMethod,
+    Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel,
     OperationStatus, Protocol, ProtocolAdapter, ProtocolAdapterError, RestTarget, Target,
     ToolDescription,
 };
@@ -146,6 +147,111 @@ async fn confirmation_token_allows_only_one_concurrent_execution() {
     );
 }
 
+#[tokio::test]
+async fn unavailable_confirmation_store_is_preserved_as_a_typed_error() {
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(CountingAdapter {
+            call_count: Arc::new(AtomicUsize::new(0)),
+        }))
+        .with_coordination_store(Arc::new(UnavailableCoordinationStore))
+        .build();
+    let operation: crank_runtime::RuntimeOperation = destructive_delete_operation().into();
+    let context = RuntimeRequestContext::from_request_id("req_confirm_unavailable")
+        .with_response_cache_scope("workspace_1", "agent_1");
+
+    let issue_error = executor
+        .execute_with_context(
+            &operation,
+            &json!({ "order_id": "ord_123" }),
+            Some(&context),
+        )
+        .await
+        .expect_err("unavailable store must prevent issuing a token");
+    assert!(matches!(
+        issue_error,
+        RuntimeError::ConfirmationStoreUnavailable { ref operation_id }
+            if operation_id == "op_delete_order"
+    ));
+
+    let consume_error = executor
+        .execute_with_context(
+            &operation,
+            &json!({ "order_id": "ord_123" }),
+            Some(&context.with_confirmation_token("ct_unavailable")),
+        )
+        .await
+        .expect_err("unavailable store must not look like an invalid token");
+    assert!(matches!(
+        consume_error,
+        RuntimeError::ConfirmationStoreUnavailable { ref operation_id }
+            if operation_id == "op_delete_order"
+    ));
+}
+
+struct UnavailableCoordinationStore;
+
+impl UnavailableCoordinationStore {
+    fn error() -> CacheStoreError {
+        CacheStoreError::Unavailable {
+            message: "test backend unavailable".to_owned(),
+        }
+    }
+}
+
+#[async_trait]
+impl CoordinationStateStore for UnavailableCoordinationStore {
+    async fn get_value(
+        &self,
+        _scope: CacheScope,
+        _key: &str,
+    ) -> Result<Option<CoordinationStateValue>, CacheStoreError> {
+        Err(Self::error())
+    }
+
+    async fn put_value(
+        &self,
+        _scope: CacheScope,
+        _key: &str,
+        _value: CoordinationStateValue,
+        _ttl: std::time::Duration,
+    ) -> Result<(), CacheStoreError> {
+        Err(Self::error())
+    }
+
+    async fn delete_value(&self, _scope: CacheScope, _key: &str) -> Result<(), CacheStoreError> {
+        Err(Self::error())
+    }
+
+    async fn take_value(
+        &self,
+        _scope: CacheScope,
+        _key: &str,
+    ) -> Result<Option<CoordinationStateValue>, CacheStoreError> {
+        Err(Self::error())
+    }
+
+    async fn reserve_value(
+        &self,
+        _scope: CacheScope,
+        _key: &str,
+        _value: CoordinationStateValue,
+        _ttl: std::time::Duration,
+    ) -> Result<CoordinationStateReservation, CacheStoreError> {
+        Err(Self::error())
+    }
+
+    async fn compare_and_set_value(
+        &self,
+        _scope: CacheScope,
+        _key: &str,
+        _expected: &CoordinationStateValue,
+        _value: CoordinationStateValue,
+        _ttl: std::time::Duration,
+    ) -> Result<bool, CacheStoreError> {
+        Err(Self::error())
+    }
+}
+
 struct CountingAdapter {
     call_count: Arc<AtomicUsize>,
 }

crates/crank-runtime/tests/product_metrics.rs

@@ -0,0 +1,414 @@
+use std::{
+    collections::BTreeMap,
+    sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    },
+};
+
+use async_trait::async_trait;
+use crank_core::{
+    AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod,
+    IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass,
+    OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter,
+    ProtocolAdapterError, ResponseCachePolicy, RestTarget,
+    RuntimeRequestContext as ProtocolRequestContext, Target, ToolDescription,
+};
+use crank_mapping::{MappingRule, MappingSet};
+use crank_runtime::{
+    InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeError,
+    RuntimeExecutorBuilder, RuntimeRequestContext,
+};
+use crank_schema::{Schema, SchemaKind};
+use metrics_util::debugging::DebuggingRecorder;
+use serde_json::json;
+use time::OffsetDateTime;
+
+#[tokio::test]
+async fn real_runtime_paths_emit_cache_idempotency_and_confirmation_outcomes() {
+    let recorder = DebuggingRecorder::new();
+    let snapshotter = recorder.snapshotter();
+    recorder
+        .install()
+        .expect("isolated integration test recorder");
+
+    exercise_response_cache().await;
+    exercise_idempotency().await;
+    exercise_cancelled_idempotency().await;
+    exercise_confirmation().await;
+
+    let snapshot = snapshotter.snapshot().into_vec();
+    for outcome in ["miss", "stored", "hit"] {
+        assert!(has_series(
+            &snapshot,
+            "crank_runtime_cache_total",
+            "outcome",
+            outcome
+        ));
+    }
+    for outcome in [
+        "execute",
+        "completed",
+        "replay",
+        "conflict",
+        "outcome_unknown",
+    ] {
+        assert!(has_series(
+            &snapshot,
+            "crank_idempotency_total",
+            "outcome",
+            outcome
+        ));
+    }
+    for outcome in ["required", "approved", "invalid_token"] {
+        assert!(
+            has_series(&snapshot, "crank_confirmation_total", "outcome", outcome),
+            "missing confirmation outcome {outcome}: {snapshot:?}"
+        );
+    }
+}
+
+async fn exercise_cancelled_idempotency() {
+    let calls = Arc::new(AtomicUsize::new(0));
+    let executor = RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(BlockingAdapter {
+            calls: Arc::clone(&calls),
+        }))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build();
+    let operation: crank_runtime::RuntimeOperation = operation(
+        "cancelled_idempotent_write",
+        HttpMethod::Post,
+        None,
+        Some(IdempotencyPolicy {
+            mode: IdempotencyMode::Required,
+            ttl_ms: 60_000,
+            input_field: Some("key".to_owned()),
+            header_name: Some("Idempotency-Key".to_owned()),
+        }),
+        None,
+    )
+    .into();
+    let context = RuntimeRequestContext::from_request_id("req_cancelled_idempotency")
+        .with_response_cache_scope("workspace_cancelled", "agent_cancelled");
+    let input = json!({"key": "cancelled-key"});
+
+    let running_executor = executor.clone();
+    let running_operation = operation.clone();
+    let running_context = context.clone();
+    let running_input = input.clone();
+    let task = tokio::spawn(async move {
+        running_executor
+            .execute_with_context(&running_operation, &running_input, Some(&running_context))
+            .await
+    });
+    wait_for_calls(&calls, 1).await;
+    task.abort();
+    let _ = task.await;
+
+    let retry = executor
+        .execute_with_context(&operation, &input, Some(&context))
+        .await
+        .expect_err("cancelled external outcome must not be retried automatically");
+    assert!(matches!(
+        retry,
+        RuntimeError::IdempotencyOutcomeUnknown { .. }
+    ));
+    assert_eq!(calls.load(Ordering::SeqCst), 1);
+}
+
+async fn exercise_response_cache() {
+    let calls = Arc::new(AtomicUsize::new(0));
+    let executor = executor(Arc::clone(&calls));
+    let operation: crank_runtime::RuntimeOperation = operation(
+        "cached_lookup",
+        HttpMethod::Get,
+        Some(ResponseCachePolicy { ttl_ms: 60_000 }),
+        None,
+        None,
+    )
+    .into();
+    let context = RuntimeRequestContext::from_request_id("req_cache")
+        .with_response_cache_scope("workspace_cache", "agent_cache");
+
+    executor
+        .execute_with_context(&operation, &json!({"key": "cache-key"}), Some(&context))
+        .await
+        .unwrap();
+    executor
+        .execute_with_context(&operation, &json!({"key": "cache-key"}), Some(&context))
+        .await
+        .unwrap();
+
+    assert_eq!(calls.load(Ordering::SeqCst), 1);
+}
+
+async fn exercise_idempotency() {
+    let calls = Arc::new(AtomicUsize::new(0));
+    let executor = executor(Arc::clone(&calls));
+    let operation: crank_runtime::RuntimeOperation = operation(
+        "idempotent_write",
+        HttpMethod::Post,
+        None,
+        Some(IdempotencyPolicy {
+            mode: IdempotencyMode::Required,
+            ttl_ms: 60_000,
+            input_field: Some("key".to_owned()),
+            header_name: Some("Idempotency-Key".to_owned()),
+        }),
+        None,
+    )
+    .into();
+    let context = RuntimeRequestContext::from_request_id("req_idempotency")
+        .with_response_cache_scope("workspace_idempotency", "agent_idempotency");
+
+    let first_input = json!({"key": "stable-key", "variant": "first"});
+    executor
+        .execute_with_context(&operation, &first_input, Some(&context))
+        .await
+        .unwrap();
+    executor
+        .execute_with_context(&operation, &first_input, Some(&context))
+        .await
+        .unwrap();
+    let conflict = executor
+        .execute_with_context(
+            &operation,
+            &json!({"key": "stable-key", "variant": "different"}),
+            Some(&context),
+        )
+        .await
+        .expect_err("same idempotency key with a different input must conflict");
+
+    assert!(matches!(conflict, RuntimeError::IdempotencyConflict { .. }));
+    assert_eq!(calls.load(Ordering::SeqCst), 1);
+}
+
+async fn exercise_confirmation() {
+    let calls = Arc::new(AtomicUsize::new(0));
+    let executor = executor(Arc::clone(&calls));
+    let operation: crank_runtime::RuntimeOperation = operation(
+        "destructive_write",
+        HttpMethod::Delete,
+        None,
+        None,
+        Some(OperationSafetyPolicy {
+            class: OperationSafetyClass::Destructive,
+            confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }),
+        }),
+    )
+    .into();
+    let context = RuntimeRequestContext::from_request_id("req_confirmation")
+        .with_response_cache_scope("workspace_confirmation", "agent_confirmation");
+    let input = json!({"key": "delete-key"});
+
+    let required = executor
+        .execute_with_context(&operation, &input, Some(&context))
+        .await
+        .expect_err("destructive operation must require confirmation");
+    let RuntimeError::ConfirmationRequired {
+        confirmation_token, ..
+    } = required
+    else {
+        panic!("expected confirmation token");
+    };
+    let confirmed = context
+        .clone()
+        .with_confirmation_token(confirmation_token.clone());
+    executor
+        .execute_with_context(&operation, &input, Some(&confirmed))
+        .await
+        .unwrap();
+    let invalid = executor
+        .execute_with_context(&operation, &input, Some(&confirmed))
+        .await
+        .expect_err("confirmation token must be single-use");
+
+    assert!(matches!(
+        invalid,
+        RuntimeError::InvalidConfirmationToken { .. }
+    ));
+    assert_eq!(calls.load(Ordering::SeqCst), 1);
+}
+
+fn executor(calls: Arc<AtomicUsize>) -> crank_runtime::RuntimeExecutor {
+    RuntimeExecutorBuilder::new()
+        .register_adapter(Arc::new(CountingAdapter { calls }))
+        .with_response_cache(Arc::new(InMemoryResponseCacheStore::default()))
+        .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
+        .build()
+}
+
+struct CountingAdapter {
+    calls: Arc<AtomicUsize>,
+}
+
+struct BlockingAdapter {
+    calls: Arc<AtomicUsize>,
+}
+
+#[async_trait]
+impl ProtocolAdapter for BlockingAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        _prepared: &crank_core::PreparedRequest,
+        _context: &ProtocolRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        self.calls.fetch_add(1, Ordering::SeqCst);
+        std::future::pending().await
+    }
+}
+
+#[async_trait]
+impl ProtocolAdapter for CountingAdapter {
+    fn protocol(&self) -> Protocol {
+        Protocol::Rest
+    }
+
+    fn supports_mode(&self, mode: ExecutionMode) -> bool {
+        mode == ExecutionMode::Unary
+    }
+
+    async fn invoke_unary(
+        &self,
+        _target: &Target,
+        _prepared: &crank_core::PreparedRequest,
+        _context: &ProtocolRequestContext,
+    ) -> Result<AdapterResponse, ProtocolAdapterError> {
+        self.calls.fetch_add(1, Ordering::SeqCst);
+        Ok(AdapterResponse {
+            status_code: 200,
+            headers: BTreeMap::new(),
+            body: json!({"result": "ok"}),
+            data: json!({"result": "ok"}),
+        })
+    }
+}
+
+fn operation(
+    name: &str,
+    method: HttpMethod,
+    response_cache: Option<ResponseCachePolicy>,
+    idempotency: Option<IdempotencyPolicy>,
+    safety: Option<OperationSafetyPolicy>,
+) -> Operation<Schema, MappingSet> {
+    Operation {
+        id: OperationId::new(format!("op_{name}")),
+        name: name.to_owned(),
+        display_name: name.to_owned(),
+        category: "metrics".to_owned(),
+        protocol: Protocol::Rest,
+        security_level: OperationSecurityLevel::Standard,
+        status: OperationStatus::Published,
+        version: 1,
+        target: Target::Rest(RestTarget {
+            base_url: "https://metrics.example.invalid".to_owned(),
+            method,
+            path_template: "/resource".to_owned(),
+            static_headers: BTreeMap::new(),
+        }),
+        input_schema: object_schema("key"),
+        output_schema: object_schema("result"),
+        input_mapping: MappingSet { rules: Vec::new() },
+        output_mapping: MappingSet {
+            rules: vec![MappingRule {
+                source: "$.response.body.result".to_owned(),
+                target: "$.output.result".to_owned(),
+                required: true,
+                default_value: None,
+                transform: None,
+                condition: None,
+                notes: None,
+            }],
+        },
+        execution_config: ExecutionConfig {
+            timeout_ms: 1_000,
+            retry_policy: None,
+            response_cache,
+            idempotency,
+            safety,
+            approval_policy: None,
+            auth_profile_ref: None,
+            headers: BTreeMap::new(),
+        },
+        tool_description: ToolDescription {
+            title: name.to_owned(),
+            description: "Exercises a real runtime metrics path.".to_owned(),
+            tags: Vec::new(),
+            examples: Vec::new(),
+        },
+        samples: None,
+        generated_draft: None,
+        config_export: None,
+        wizard_state: None,
+        created_at: OffsetDateTime::UNIX_EPOCH,
+        updated_at: OffsetDateTime::UNIX_EPOCH,
+        published_at: None,
+    }
+}
+
+fn object_schema(field: &str) -> Schema {
+    Schema {
+        kind: SchemaKind::Object,
+        description: None,
+        required: true,
+        nullable: false,
+        default_value: None,
+        fields: BTreeMap::from([(
+            field.to_owned(),
+            Schema {
+                kind: SchemaKind::String,
+                description: None,
+                required: true,
+                nullable: false,
+                default_value: None,
+                fields: BTreeMap::new(),
+                items: None,
+                enum_values: Vec::new(),
+                variants: Vec::new(),
+            },
+        )]),
+        items: None,
+        enum_values: Vec::new(),
+        variants: Vec::new(),
+    }
+}
+
+fn has_series(
+    snapshot: &[(
+        metrics_util::CompositeKey,
+        Option<metrics::Unit>,
+        Option<metrics::SharedString>,
+        metrics_util::debugging::DebugValue,
+    )],
+    metric: &str,
+    label_name: &str,
+    label_value: &str,
+) -> bool {
+    snapshot.iter().any(|(key, _, _, _)| {
+        key.key().name() == metric
+            && key
+                .key()
+                .labels()
+                .any(|label| label.key() == label_name && label.value() == label_value)
+    })
+}
+
+async fn wait_for_calls(calls: &AtomicUsize, expected: usize) {
+    for _ in 0..1_000 {
+        if calls.load(Ordering::SeqCst) >= expected {
+            return;
+        }
+        tokio::task::yield_now().await;
+    }
+    panic!("adapter did not receive {expected} call(s)");
+}