use std::{ collections::BTreeMap, sync::{ Arc, atomic::{AtomicUsize, Ordering}, }, }; use async_trait::async_trait; use crank_core::{ AdapterResponse, ConfirmationPolicy, ExecutionConfig, ExecutionMode, HttpMethod, IdempotencyMode, IdempotencyPolicy, Operation, OperationId, OperationSafetyClass, OperationSafetyPolicy, OperationSecurityLevel, OperationStatus, Protocol, ProtocolAdapter, ProtocolAdapterError, ResponseCachePolicy, RestTarget, RuntimeRequestContext as ProtocolRequestContext, Target, ToolDescription, }; use crank_mapping::{MappingRule, MappingSet}; use crank_runtime::{ InMemoryCoordinationStateStore, InMemoryResponseCacheStore, RuntimeError, RuntimeExecutorBuilder, RuntimeRequestContext, }; use crank_schema::{Schema, SchemaKind}; use metrics_util::debugging::DebuggingRecorder; use serde_json::json; use time::OffsetDateTime; #[tokio::test] async fn real_runtime_paths_emit_cache_idempotency_and_confirmation_outcomes() { let recorder = DebuggingRecorder::new(); let snapshotter = recorder.snapshotter(); recorder .install() .expect("isolated integration test recorder"); exercise_response_cache().await; exercise_idempotency().await; exercise_cancelled_idempotency().await; exercise_confirmation().await; let snapshot = snapshotter.snapshot().into_vec(); for outcome in ["miss", "stored", "hit"] { assert!(has_series( &snapshot, "crank_runtime_cache_total", "outcome", outcome )); } for outcome in [ "execute", "completed", "replay", "conflict", "outcome_unknown", ] { assert!(has_series( &snapshot, "crank_idempotency_total", "outcome", outcome )); } for outcome in ["required", "approved", "invalid_token"] { assert!( has_series(&snapshot, "crank_confirmation_total", "outcome", outcome), "missing confirmation outcome {outcome}: {snapshot:?}" ); } } async fn exercise_cancelled_idempotency() { let calls = Arc::new(AtomicUsize::new(0)); let executor = RuntimeExecutorBuilder::new() .register_adapter(Arc::new(BlockingAdapter { calls: Arc::clone(&calls), })) .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) .build(); let operation: crank_runtime::RuntimeOperation = operation( "cancelled_idempotent_write", HttpMethod::Post, None, Some(IdempotencyPolicy { mode: IdempotencyMode::Required, ttl_ms: 60_000, input_field: Some("key".to_owned()), header_name: Some("Idempotency-Key".to_owned()), }), None, ) .into(); let context = RuntimeRequestContext::from_request_id("req_cancelled_idempotency") .with_response_cache_scope("workspace_cancelled", "agent_cancelled"); let input = json!({"key": "cancelled-key"}); let running_executor = executor.clone(); let running_operation = operation.clone(); let running_context = context.clone(); let running_input = input.clone(); let task = tokio::spawn(async move { running_executor .execute_with_context(&running_operation, &running_input, Some(&running_context)) .await }); wait_for_calls(&calls, 1).await; task.abort(); let _ = task.await; let retry = executor .execute_with_context(&operation, &input, Some(&context)) .await .expect_err("cancelled external outcome must not be retried automatically"); assert!(matches!( retry, RuntimeError::IdempotencyOutcomeUnknown { .. } )); assert_eq!(calls.load(Ordering::SeqCst), 1); } async fn exercise_response_cache() { let calls = Arc::new(AtomicUsize::new(0)); let executor = executor(Arc::clone(&calls)); let operation: crank_runtime::RuntimeOperation = operation( "cached_lookup", HttpMethod::Get, Some(ResponseCachePolicy { ttl_ms: 60_000 }), None, None, ) .into(); let context = RuntimeRequestContext::from_request_id("req_cache") .with_response_cache_scope("workspace_cache", "agent_cache"); executor .execute_with_context(&operation, &json!({"key": "cache-key"}), Some(&context)) .await .unwrap(); executor .execute_with_context(&operation, &json!({"key": "cache-key"}), Some(&context)) .await .unwrap(); assert_eq!(calls.load(Ordering::SeqCst), 1); } async fn exercise_idempotency() { let calls = Arc::new(AtomicUsize::new(0)); let executor = executor(Arc::clone(&calls)); let operation: crank_runtime::RuntimeOperation = operation( "idempotent_write", HttpMethod::Post, None, Some(IdempotencyPolicy { mode: IdempotencyMode::Required, ttl_ms: 60_000, input_field: Some("key".to_owned()), header_name: Some("Idempotency-Key".to_owned()), }), None, ) .into(); let context = RuntimeRequestContext::from_request_id("req_idempotency") .with_response_cache_scope("workspace_idempotency", "agent_idempotency"); let first_input = json!({"key": "stable-key", "variant": "first"}); executor .execute_with_context(&operation, &first_input, Some(&context)) .await .unwrap(); executor .execute_with_context(&operation, &first_input, Some(&context)) .await .unwrap(); let conflict = executor .execute_with_context( &operation, &json!({"key": "stable-key", "variant": "different"}), Some(&context), ) .await .expect_err("same idempotency key with a different input must conflict"); assert!(matches!(conflict, RuntimeError::IdempotencyConflict { .. })); assert_eq!(calls.load(Ordering::SeqCst), 1); } async fn exercise_confirmation() { let calls = Arc::new(AtomicUsize::new(0)); let executor = executor(Arc::clone(&calls)); let operation: crank_runtime::RuntimeOperation = operation( "destructive_write", HttpMethod::Delete, None, None, Some(OperationSafetyPolicy { class: OperationSafetyClass::Destructive, confirmation: Some(ConfirmationPolicy { ttl_ms: 60_000 }), }), ) .into(); let context = RuntimeRequestContext::from_request_id("req_confirmation") .with_response_cache_scope("workspace_confirmation", "agent_confirmation"); let input = json!({"key": "delete-key"}); let required = executor .execute_with_context(&operation, &input, Some(&context)) .await .expect_err("destructive operation must require confirmation"); let RuntimeError::ConfirmationRequired { confirmation_token, .. } = required else { panic!("expected confirmation token"); }; let confirmed = context .clone() .with_confirmation_token(confirmation_token.clone()); executor .execute_with_context(&operation, &input, Some(&confirmed)) .await .unwrap(); let invalid = executor .execute_with_context(&operation, &input, Some(&confirmed)) .await .expect_err("confirmation token must be single-use"); assert!(matches!( invalid, RuntimeError::InvalidConfirmationToken { .. } )); assert_eq!(calls.load(Ordering::SeqCst), 1); } fn executor(calls: Arc) -> crank_runtime::RuntimeExecutor { RuntimeExecutorBuilder::new() .register_adapter(Arc::new(CountingAdapter { calls })) .with_response_cache(Arc::new(InMemoryResponseCacheStore::default())) .with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default())) .build() } struct CountingAdapter { calls: Arc, } struct BlockingAdapter { calls: Arc, } #[async_trait] impl ProtocolAdapter for BlockingAdapter { fn protocol(&self) -> Protocol { Protocol::Rest } fn supports_mode(&self, mode: ExecutionMode) -> bool { mode == ExecutionMode::Unary } async fn invoke_unary( &self, _target: &Target, _prepared: &crank_core::PreparedRequest, _context: &ProtocolRequestContext, ) -> Result { self.calls.fetch_add(1, Ordering::SeqCst); std::future::pending().await } } #[async_trait] impl ProtocolAdapter for CountingAdapter { fn protocol(&self) -> Protocol { Protocol::Rest } fn supports_mode(&self, mode: ExecutionMode) -> bool { mode == ExecutionMode::Unary } async fn invoke_unary( &self, _target: &Target, _prepared: &crank_core::PreparedRequest, _context: &ProtocolRequestContext, ) -> Result { self.calls.fetch_add(1, Ordering::SeqCst); Ok(AdapterResponse { status_code: 200, headers: BTreeMap::new(), body: json!({"result": "ok"}), data: json!({"result": "ok"}), }) } } fn operation( name: &str, method: HttpMethod, response_cache: Option, idempotency: Option, safety: Option, ) -> Operation { Operation { id: OperationId::new(format!("op_{name}")), name: name.to_owned(), display_name: name.to_owned(), category: "metrics".to_owned(), protocol: Protocol::Rest, security_level: OperationSecurityLevel::Standard, status: OperationStatus::Published, version: 1, target: Target::Rest(RestTarget { base_url: "https://metrics.example.invalid".to_owned(), method, path_template: "/resource".to_owned(), static_headers: BTreeMap::new(), }), input_schema: object_schema("key"), output_schema: object_schema("result"), input_mapping: MappingSet { rules: Vec::new() }, output_mapping: MappingSet { rules: vec![MappingRule { source: "$.response.body.result".to_owned(), target: "$.output.result".to_owned(), required: true, default_value: None, transform: None, condition: None, notes: None, }], }, execution_config: ExecutionConfig { timeout_ms: 1_000, retry_policy: None, response_cache, idempotency, safety, approval_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, tool_description: ToolDescription { title: name.to_owned(), description: "Exercises a real runtime metrics path.".to_owned(), tags: Vec::new(), examples: Vec::new(), }, samples: None, generated_draft: None, config_export: None, wizard_state: None, created_at: OffsetDateTime::UNIX_EPOCH, updated_at: OffsetDateTime::UNIX_EPOCH, published_at: None, } } fn object_schema(field: &str) -> Schema { Schema { kind: SchemaKind::Object, description: None, required: true, nullable: false, default_value: None, fields: BTreeMap::from([( field.to_owned(), Schema { kind: SchemaKind::String, description: None, required: true, nullable: false, default_value: None, fields: BTreeMap::new(), items: None, enum_values: Vec::new(), variants: Vec::new(), }, )]), items: None, enum_values: Vec::new(), variants: Vec::new(), } } fn has_series( snapshot: &[( metrics_util::CompositeKey, Option, Option, metrics_util::debugging::DebugValue, )], metric: &str, label_name: &str, label_value: &str, ) -> bool { snapshot.iter().any(|(key, _, _, _)| { key.key().name() == metric && key .key() .labels() .any(|label| label.key() == label_name && label.value() == label_value) }) } async fn wait_for_calls(calls: &AtomicUsize, expected: usize) { for _ in 0..1_000 { if calls.load(Ordering::SeqCst) >= expected { return; } tokio::task::yield_now().await; } panic!("adapter did not receive {expected} call(s)"); }