наблюдаемость: ввести безопасный контракт метрик

crates/crank-runtime/src/executor.rs

@@ -5,12 +5,17 @@ use crank_core::{
     AdapterRegistry, CoordinationStateStore, ExecutionMode, InvocationSource, InvocationStatus,
     MeteringEvent, ResponseCacheStore, SharedMeteringSink, SharedProtocolAdapter,
 };
+use crank_metrics::{
+    CacheOutcome, ConfirmationOutcome, IdempotencyOutcome, InFlightGuard,
+    InvocationSource as MetricInvocationSource, LimitStage, ToolErrorKind, ToolInvocationMetrics,
+    ToolOutcome, record_cache_outcome, record_confirmation_outcome, record_idempotency_outcome,
+    record_limit_rejection,
+};
 use crank_trace::{ErrorCategory, Stage, StageOutcome};
-use metrics::Gauge;
 use serde_json::{Map, Value, json};
 use time::OffsetDateTime;
 use tokio::sync::{OwnedSemaphorePermit, Semaphore};
-use tracing::{Instrument, Span, debug};
+use tracing::{Instrument, Span, debug, warn};
 use uuid::Uuid;
 
 use crate::{
@@ -42,6 +47,65 @@ pub struct RuntimeExecutionRequest<'a> {
     pub request_context: Option<&'a RuntimeRequestContext>,
 }
 
+struct IdempotencyCancellationGuard {
+    cleanup: Option<IdempotencyCancellationCleanup>,
+    runtime: tokio::runtime::Handle,
+}
+
+struct IdempotencyCancellationCleanup {
+    store: Arc<dyn CoordinationStateStore>,
+    operation: RuntimeOperation,
+    reservation: crate::idempotency::IdempotencyReservation,
+}
+
+impl IdempotencyCancellationGuard {
+    fn new(
+        store: Arc<dyn CoordinationStateStore>,
+        operation: RuntimeOperation,
+        reservation: crate::idempotency::IdempotencyReservation,
+    ) -> Self {
+        Self {
+            cleanup: Some(IdempotencyCancellationCleanup {
+                store,
+                operation,
+                reservation,
+            }),
+            runtime: tokio::runtime::Handle::current(),
+        }
+    }
+
+    fn disarm(&mut self) {
+        self.cleanup = None;
+    }
+}
+
+impl Drop for IdempotencyCancellationGuard {
+    fn drop(&mut self) {
+        let Some(cleanup) = self.cleanup.take() else {
+            return;
+        };
+        self.runtime.spawn(async move {
+            let result = crate::idempotency::mark_outcome_unknown(
+                cleanup.store.as_ref(),
+                &cleanup.operation,
+                &cleanup.reservation,
+            )
+            .await;
+            record_idempotency_outcome(match &result {
+                Ok(()) => IdempotencyOutcome::OutcomeUnknown,
+                Err(error) => idempotency_error_outcome(error),
+            });
+            if result.is_err() {
+                warn!(
+                    name: "runtime.idempotency.cancellation_cleanup_failed",
+                    error_category = "idempotency_store",
+                    "failed to finalize cancelled idempotent execution"
+                );
+            }
+        });
+    }
+}
+
 impl<'a> RuntimeExecutionRequest<'a> {
     pub fn new(operation: &'a RuntimeOperation, input: &'a Value) -> Self {
         Self {
@@ -180,10 +244,12 @@ impl RuntimeExecutor {
     ) -> Result<Value, RuntimeError> {
         log_runtime_event("unary.execute", request.operation, request.request_context);
         let started_at = Instant::now();
+        let invocation_metrics =
+            ToolInvocationMetrics::start(metric_invocation_source(request.request_context));
         let runtime_span = Stage::RuntimeExecute.span();
         let result = async {
             let _permit = self.acquire_unary_permit(request.operation)?;
-            let _inflight = RuntimeInFlightGuard::new();
+            let _inflight = InFlightGuard::runtime();
             let mapping_span = Stage::RuntimeArgumentsMap.span();
             let prepared_request =
                 mapping_span.in_scope(|| self.prepare_request(request.operation, request.input));
@@ -203,7 +269,11 @@ impl RuntimeExecutor {
         .await;
         record_runtime_result(&runtime_span, &result);
         drop(runtime_span);
-        record_execution_metrics(request.request_context, &result, started_at);
+        let (outcome, error_kind) = match &result {
+            Ok(_) => (ToolOutcome::Success, ToolErrorKind::None),
+            Err(error) => (ToolOutcome::Error, runtime_error_kind(error)),
+        };
+        invocation_metrics.complete(outcome, error_kind);
         self.record_metering(
             request.operation,
             request.request_context,
@@ -245,6 +315,7 @@ impl RuntimeExecutor {
         ) {
             Ok(key) => key,
             Err(error) if idempotency_applicable => {
+                record_idempotency_outcome(idempotency_error_outcome(&error));
                 let span = Stage::RuntimeIdempotency.span();
                 StageOutcome::Error.record(&span);
                 ErrorCategory::Idempotency.record(&span);
@@ -264,14 +335,19 @@ impl RuntimeExecutor {
             .instrument(approval_span.clone())
             .await;
             match &approval_result {
-                Ok(()) => StageOutcome::Success.record(&approval_span),
+                Ok(()) => {
+                    StageOutcome::Success.record(&approval_span);
+                    record_confirmation_outcome(ConfirmationOutcome::Approved);
+                }
                 Err(RuntimeError::ConfirmationRequired { .. }) => {
                     StageOutcome::Required.record(&approval_span);
                     ErrorCategory::Approval.record(&approval_span);
+                    record_confirmation_outcome(ConfirmationOutcome::Required);
                 }
                 Err(error) => {
                     StageOutcome::Error.record(&approval_span);
                     runtime_error_category(error).record(&approval_span);
+                    record_confirmation_outcome(confirmation_error_outcome(error));
                 }
             }
             drop(approval_span);
@@ -292,9 +368,11 @@ impl RuntimeExecutor {
             match &result {
                 Ok(crate::idempotency::IdempotencyAction::Execute(_)) => {
                     StageOutcome::Execute.record(&idempotency_span);
+                    record_idempotency_outcome(IdempotencyOutcome::Execute);
                 }
                 Ok(crate::idempotency::IdempotencyAction::Replay(_)) => {
                     StageOutcome::Replay.record(&idempotency_span);
+                    record_idempotency_outcome(IdempotencyOutcome::Replay);
                 }
                 Ok(crate::idempotency::IdempotencyAction::Disabled) => {
                     StageOutcome::Skipped.record(&idempotency_span);
@@ -302,6 +380,7 @@ impl RuntimeExecutor {
                 Err(error) => {
                     StageOutcome::Error.record(&idempotency_span);
                     runtime_error_category(error).record(&idempotency_span);
+                    record_idempotency_outcome(idempotency_error_outcome(error));
                 }
             }
             drop(idempotency_span);
@@ -312,6 +391,18 @@ impl RuntimeExecutor {
         if let crate::idempotency::IdempotencyAction::Replay(response) = &idempotency {
             return transform_response(operation, response);
         }
+        let mut cancellation_guard =
+            if let crate::idempotency::IdempotencyAction::Execute(reservation) = &idempotency {
+                self.coordination_store.as_ref().map(|store| {
+                    IdempotencyCancellationGuard::new(
+                        Arc::clone(store),
+                        operation.clone(),
+                        reservation.clone(),
+                    )
+                })
+            } else {
+                None
+            };
 
         let adapter_result = match self
             .load_cached_adapter_response(operation, &prepared_request, request_context)
@@ -346,6 +437,13 @@ impl RuntimeExecutor {
                             .instrument(idempotency_span.clone())
                             .await;
                     record_runtime_result(&idempotency_span, &cleanup_result);
+                    record_idempotency_outcome(match &cleanup_result {
+                        Ok(()) => IdempotencyOutcome::OutcomeUnknown,
+                        Err(error) => idempotency_error_outcome(error),
+                    });
+                    if let Some(guard) = &mut cancellation_guard {
+                        guard.disarm();
+                    }
                 }
                 return Err(error);
             }
@@ -359,6 +457,13 @@ impl RuntimeExecutor {
                     .instrument(idempotency_span.clone())
                     .await;
             record_runtime_result(&idempotency_span, &completion_result);
+            record_idempotency_outcome(match &completion_result {
+                Ok(()) => IdempotencyOutcome::Completed,
+                Err(error) => idempotency_error_outcome(error),
+            });
+            if let Some(guard) = &mut cancellation_guard {
+                guard.disarm();
+            }
             drop(idempotency_span);
             completion_result?;
         }
@@ -447,8 +552,13 @@ impl RuntimeExecutor {
         let response_cache = self.response_cache.as_ref()?;
         let cache_key = response_cache_key(operation, prepared_request, request_context)?;
         let cached = match response_cache.get(&cache_key).await {
-            Ok(cached) => cached?,
+            Ok(Some(cached)) => cached,
+            Ok(None) => {
+                record_cache_outcome(CacheOutcome::Miss);
+                return None;
+            }
             Err(_) => {
+                record_cache_outcome(CacheOutcome::ReadError);
                 debug!(
                     name: "runtime.response_cache.read_failed",
                     operation_id = operation.operation_id.as_str(),
@@ -460,15 +570,21 @@ impl RuntimeExecutor {
         };
 
         match adapter_response_from_cached(cached) {
-            Ok(response) => Some(response),
+            Ok(response) => {
+                record_cache_outcome(CacheOutcome::Hit);
+                Some(response)
+            }
             Err(_) => {
+                record_cache_outcome(CacheOutcome::DecodeError);
                 debug!(
                     name: "runtime.response_cache.decode_failed",
                     operation_id = operation.operation_id.as_str(),
                     error_category = "cached_response",
                     "cached response payload was invalid"
                 );
-                let _ = response_cache.delete(&cache_key).await;
+                if response_cache.delete(&cache_key).await.is_err() {
+                    record_cache_outcome(CacheOutcome::EvictError);
+                }
                 None
             }
         }
@@ -505,12 +621,15 @@ impl RuntimeExecutor {
             .await
             .is_err()
         {
+            record_cache_outcome(CacheOutcome::WriteError);
             debug!(
                 name: "runtime.response_cache.write_failed",
                 operation_id = operation.operation_id.as_str(),
                 error_category = "response_cache",
                 "response cache write skipped"
             );
+        } else {
+            record_cache_outcome(CacheOutcome::Stored);
         }
     }
 }
@@ -634,86 +753,64 @@ fn try_acquire_limit(
     limit: usize,
 ) -> Result<OwnedSemaphorePermit, RuntimeError> {
     limiter.try_acquire_owned().map_err(|_| {
-        metrics::counter!(
-            "crank_runtime_limit_rejections_total",
-            "stage" => "concurrency"
-        )
-        .increment(1);
+        record_limit_rejection(LimitStage::Concurrency);
         RuntimeError::ConcurrencyLimitExceeded { kind, limit }
     })
 }
 
-fn record_execution_metrics<T>(
+fn metric_invocation_source(
     request_context: Option<&RuntimeRequestContext>,
-    result: &Result<T, RuntimeError>,
-    started_at: Instant,
-) {
-    let source = request_context
+) -> MetricInvocationSource {
+    request_context
         .and_then(RuntimeRequestContext::metering_context)
-        .map_or("internal", |context| match context.source {
-            InvocationSource::AdminTestRun => "admin_test_run",
-            InvocationSource::AgentToolCall => "agent_tool_call",
-        });
-    let (outcome, error_kind) = match result {
-        Ok(_) => ("success", "none"),
-        Err(error) => ("error", runtime_error_kind(error)),
-    };
-
-    metrics::counter!(
-        "crank_tool_invocations_total",
-        "source" => source,
-        "outcome" => outcome,
-        "error_kind" => error_kind
-    )
-    .increment(1);
-    metrics::histogram!(
-        "crank_tool_invocation_duration_seconds",
-        "source" => source,
-        "outcome" => outcome
-    )
-    .record(started_at.elapsed().as_secs_f64());
+        .map_or(MetricInvocationSource::Internal, |context| {
+            match context.source {
+                InvocationSource::AdminTestRun => MetricInvocationSource::AdminTestRun,
+                InvocationSource::AgentToolCall => MetricInvocationSource::AgentToolCall,
+            }
+        })
 }
 
-fn runtime_error_kind(error: &RuntimeError) -> &'static str {
+fn runtime_error_kind(error: &RuntimeError) -> ToolErrorKind {
     match error {
-        RuntimeError::Schema(_) => "schema",
-        RuntimeError::Mapping(_) => "mapping",
-        RuntimeError::RestAdapter(_) => "rest_adapter",
-        RuntimeError::ProtocolAdapter(_) => "protocol_adapter",
-        RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
-        RuntimeError::UnsupportedExecutionMode { .. } => "unsupported_execution_mode",
-        RuntimeError::ConcurrencyLimitExceeded { .. } => "concurrency_limit",
-        RuntimeError::InvalidPreparedRequest { .. } => "invalid_prepared_request",
-        RuntimeError::ConfirmationRequired { .. } => "confirmation_required",
-        RuntimeError::InvalidConfirmationToken { .. } => "invalid_confirmation_token",
-        RuntimeError::ConfirmationStoreUnavailable { .. } => "confirmation_store",
-        RuntimeError::IdempotencyStoreUnavailable { .. } => "idempotency_store",
-        RuntimeError::IdempotencyInProgress { .. } => "idempotency_in_progress",
-        RuntimeError::IdempotencyConflict { .. } => "idempotency_conflict",
-        RuntimeError::IdempotencyOutcomeUnknown { .. } => "idempotency_outcome_unknown",
-        RuntimeError::MissingAuthProfile { .. } => "missing_auth_profile",
-        RuntimeError::MissingSecret { .. } => "missing_secret",
-        RuntimeError::MissingSecretVersion { .. } => "missing_secret_version",
-        RuntimeError::InvalidAuthSecretValue { .. } => "invalid_auth_secret",
-        RuntimeError::SecretCrypto { .. } => "secret_crypto",
+        RuntimeError::Schema(_) => ToolErrorKind::Schema,
+        RuntimeError::Mapping(_) => ToolErrorKind::Mapping,
+        RuntimeError::RestAdapter(_) => ToolErrorKind::RestAdapter,
+        RuntimeError::ProtocolAdapter(_) => ToolErrorKind::ProtocolAdapter,
+        RuntimeError::UnsupportedProtocol { .. } => ToolErrorKind::UnsupportedProtocol,
+        RuntimeError::UnsupportedExecutionMode { .. } => ToolErrorKind::UnsupportedExecutionMode,
+        RuntimeError::ConcurrencyLimitExceeded { .. } => ToolErrorKind::ConcurrencyLimit,
+        RuntimeError::InvalidPreparedRequest { .. } => ToolErrorKind::InvalidPreparedRequest,
+        RuntimeError::ConfirmationRequired { .. } => ToolErrorKind::ConfirmationRequired,
+        RuntimeError::InvalidConfirmationToken { .. } => ToolErrorKind::InvalidConfirmationToken,
+        RuntimeError::ConfirmationStoreUnavailable { .. } => ToolErrorKind::ConfirmationStore,
+        RuntimeError::IdempotencyStoreUnavailable { .. } => ToolErrorKind::IdempotencyStore,
+        RuntimeError::IdempotencyInProgress { .. } => ToolErrorKind::IdempotencyInProgress,
+        RuntimeError::IdempotencyConflict { .. } => ToolErrorKind::IdempotencyConflict,
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => ToolErrorKind::IdempotencyOutcomeUnknown,
+        RuntimeError::MissingAuthProfile { .. } => ToolErrorKind::MissingAuthProfile,
+        RuntimeError::MissingSecret { .. } => ToolErrorKind::MissingSecret,
+        RuntimeError::MissingSecretVersion { .. } => ToolErrorKind::MissingSecretVersion,
+        RuntimeError::InvalidAuthSecretValue { .. } => ToolErrorKind::InvalidAuthSecret,
+        RuntimeError::SecretCrypto { .. } => ToolErrorKind::SecretCrypto,
     }
 }
 
-struct RuntimeInFlightGuard {
-    gauge: Gauge,
-}
-
-impl RuntimeInFlightGuard {
-    fn new() -> Self {
-        let gauge = metrics::gauge!("crank_runtime_inflight");
-        gauge.increment(1.0);
-        Self { gauge }
+fn idempotency_error_outcome(error: &RuntimeError) -> IdempotencyOutcome {
+    match error {
+        RuntimeError::IdempotencyConflict { .. } => IdempotencyOutcome::Conflict,
+        RuntimeError::IdempotencyInProgress { .. } => IdempotencyOutcome::InProgress,
+        RuntimeError::IdempotencyOutcomeUnknown { .. } => IdempotencyOutcome::OutcomeUnknown,
+        RuntimeError::IdempotencyStoreUnavailable { .. } => IdempotencyOutcome::StoreUnavailable,
+        _ => IdempotencyOutcome::Error,
     }
 }
 
-impl Drop for RuntimeInFlightGuard {
-    fn drop(&mut self) {
-        self.gauge.decrement(1.0);
+fn confirmation_error_outcome(error: &RuntimeError) -> ConfirmationOutcome {
+    match error {
+        RuntimeError::InvalidConfirmationToken { .. } => ConfirmationOutcome::InvalidToken,
+        RuntimeError::ConfirmationStoreUnavailable { .. } => ConfirmationOutcome::StoreUnavailable,
+        _ => ConfirmationOutcome::Error,
     }
 }