use std::{ collections::BTreeMap, time::{Duration, Instant}, }; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use crank_core::{ CacheScope, CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue, HttpMethod, IdempotencyMode, IdempotencyPolicy, Target, }; use serde_json::{Map, Value, json}; use sha2::{Digest, Sha256}; use uuid::Uuid; use crate::{ AdapterResponse, PreparedRequest, RuntimeError, RuntimeOperation, RuntimeRequestContext, }; const POLL_INTERVAL: Duration = Duration::from_millis(10); pub(crate) enum IdempotencyAction { Disabled, Execute(IdempotencyReservation), Replay(AdapterResponse), } #[derive(Clone)] pub(crate) struct IdempotencyReservation { key: String, initial: CoordinationStateValue, fingerprint: String, result_ttl: Duration, } pub fn prepare_idempotency( operation: &RuntimeOperation, input: &Value, prepared_request: &mut PreparedRequest, ) -> Result, RuntimeError> { let Some(policy) = policy(operation) else { return Ok(None); }; let key = key_from_policy(policy, input); let Some(key) = key else { if policy.mode == IdempotencyMode::Required { return Err(RuntimeError::InvalidPreparedRequest { field: "execution_config.idempotency".to_owned(), reason: "required idempotency key was not provided".to_owned(), }); } return Ok(None); }; if let Some(header_name) = policy .header_name .as_deref() .filter(|value| !value.is_empty()) { remove_header_case_insensitive(&mut prepared_request.headers, header_name); prepared_request .headers .insert(header_name.to_owned(), key.clone()); prepared_request .trusted_header_names .insert(header_name.to_ascii_lowercase()); } Ok(Some(key)) } pub(crate) async fn begin( store: Option<&dyn CoordinationStateStore>, operation: &RuntimeOperation, input: &Value, idempotency_key: Option<&str>, request_context: Option<&RuntimeRequestContext>, ) -> Result { let Some(policy) = policy(operation) else { return Ok(IdempotencyAction::Disabled); }; let Some(idempotency_key) = idempotency_key else { return Ok(IdempotencyAction::Disabled); }; let Some(key) = cache_key(operation, idempotency_key, request_context) else { return Err(RuntimeError::IdempotencyStoreUnavailable { operation_id: operation.operation_id.as_str().to_owned(), }); }; let Some(store) = store else { return Err(RuntimeError::IdempotencyStoreUnavailable { operation_id: operation.operation_id.as_str().to_owned(), }); }; let fingerprint = request_fingerprint(input)?; let result_ttl = Duration::from_millis(policy.ttl_ms); let reservation_ttl = result_ttl.max(Duration::from_millis( operation.execution_config.timeout_ms.saturating_add(1_000), )); let initial = in_progress_value(&fingerprint); let reservation = store .reserve_value( CacheScope::Coordination, &key, initial.clone(), reservation_ttl, ) .await .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { operation_id: operation.operation_id.as_str().to_owned(), })?; match reservation { CoordinationStateReservation::Reserved => { Ok(IdempotencyAction::Execute(IdempotencyReservation { key, initial, fingerprint, result_ttl, })) } CoordinationStateReservation::Existing(existing) => { resolve_existing(store, operation, &key, &fingerprint, existing).await } } } pub(crate) async fn complete( store: &dyn CoordinationStateStore, operation: &RuntimeOperation, reservation: &IdempotencyReservation, response: &AdapterResponse, ) -> Result<(), RuntimeError> { let completed = CoordinationStateValue { payload: json!({ "state": "completed", "fingerprint": reservation.fingerprint, "response": response, }), }; let replaced = store .compare_and_set_value( CacheScope::Coordination, &reservation.key, &reservation.initial, completed, reservation.result_ttl, ) .await .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { operation_id: operation.operation_id.as_str().to_owned(), })?; if replaced { Ok(()) } else { Err(RuntimeError::IdempotencyOutcomeUnknown { operation_id: operation.operation_id.as_str().to_owned(), }) } } pub(crate) async fn mark_outcome_unknown( store: &dyn CoordinationStateStore, operation: &RuntimeOperation, reservation: &IdempotencyReservation, ) -> Result<(), RuntimeError> { let unknown = CoordinationStateValue { payload: json!({ "state": "outcome_unknown", "fingerprint": reservation.fingerprint, }), }; let replaced = store .compare_and_set_value( CacheScope::Coordination, &reservation.key, &reservation.initial, unknown, reservation.result_ttl, ) .await .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { operation_id: operation.operation_id.as_str().to_owned(), })?; if replaced { Ok(()) } else { Err(RuntimeError::IdempotencyOutcomeUnknown { operation_id: operation.operation_id.as_str().to_owned(), }) } } pub(crate) async fn release( store: &dyn CoordinationStateStore, operation: &RuntimeOperation, reservation: &IdempotencyReservation, ) -> Result<(), RuntimeError> { store .delete_value(CacheScope::Coordination, &reservation.key) .await .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { operation_id: operation.operation_id.as_str().to_owned(), }) } async fn resolve_existing( store: &dyn CoordinationStateStore, operation: &RuntimeOperation, key: &str, fingerprint: &str, mut existing: CoordinationStateValue, ) -> Result { let operation_id = operation.operation_id.as_str().to_owned(); let deadline = Instant::now() + Duration::from_millis(operation.execution_config.timeout_ms.max(1)); loop { let existing_fingerprint = existing.payload.get("fingerprint").and_then(Value::as_str); if existing_fingerprint != Some(fingerprint) { return Err(RuntimeError::IdempotencyConflict { operation_id }); } match existing.payload.get("state").and_then(Value::as_str) { Some("completed") => { let response = existing .payload .get("response") .cloned() .and_then(|value| serde_json::from_value(value).ok()) .ok_or_else(|| RuntimeError::InvalidPreparedRequest { field: "idempotency_state".to_owned(), reason: "completed idempotency state has no valid response".to_owned(), })?; return Ok(IdempotencyAction::Replay(response)); } Some("outcome_unknown") => { return Err(RuntimeError::IdempotencyOutcomeUnknown { operation_id }); } Some("in_progress") if Instant::now() < deadline => { tokio::time::sleep(POLL_INTERVAL).await; existing = store .get_value(CacheScope::Coordination, key) .await .map_err(|_| RuntimeError::IdempotencyStoreUnavailable { operation_id: operation_id.clone(), })? .ok_or_else(|| RuntimeError::IdempotencyOutcomeUnknown { operation_id: operation_id.clone(), })?; } Some("in_progress") => { return Err(RuntimeError::IdempotencyInProgress { operation_id }); } _ => { return Err(RuntimeError::InvalidPreparedRequest { field: "idempotency_state".to_owned(), reason: "unknown idempotency state".to_owned(), }); } } } } pub fn policy(operation: &RuntimeOperation) -> Option<&IdempotencyPolicy> { let is_mutating_rest = matches!( &operation.target, Target::Rest(target) if matches!( target.method, HttpMethod::Post | HttpMethod::Put | HttpMethod::Patch | HttpMethod::Delete ) ); if !is_mutating_rest { return None; } let policy = operation.execution_config.idempotency.as_ref()?; if policy.mode == IdempotencyMode::Disabled || policy.ttl_ms == 0 { return None; } Some(policy) } fn cache_key( operation: &RuntimeOperation, idempotency_key: &str, request_context: Option<&RuntimeRequestContext>, ) -> Option { if idempotency_key.is_empty() { return None; } let scope = request_context?.response_cache_scope()?; let key_hash = URL_SAFE_NO_PAD.encode(Sha256::digest(idempotency_key.as_bytes())); Some(format!( "crank:idempotency:workspace:{}:agent:{}:operation:{}:version:{}:key:{}", scope.workspace_key, scope.agent_key, operation.operation_id.as_str(), operation.operation_version, key_hash )) } fn request_fingerprint(input: &Value) -> Result { let canonical = canonical_json(input); let encoded = serde_json::to_vec(&canonical).map_err(|error| RuntimeError::InvalidPreparedRequest { field: "idempotency_fingerprint".to_owned(), reason: error.to_string(), })?; Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded))) } pub(crate) fn canonical_json(value: &Value) -> Value { match value { Value::Object(object) => { let mut entries = object.iter().collect::>(); entries.sort_unstable_by_key(|(key, _)| *key); let mut canonical = Map::new(); for (key, value) in entries { canonical.insert(key.clone(), canonical_json(value)); } Value::Object(canonical) } Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()), _ => value.clone(), } } fn in_progress_value(fingerprint: &str) -> CoordinationStateValue { CoordinationStateValue { payload: json!({ "state": "in_progress", "fingerprint": fingerprint, "owner": Uuid::now_v7().simple().to_string(), }), } } fn key_from_policy(policy: &IdempotencyPolicy, input: &Value) -> Option { let field_name = policy.input_field.as_deref()?.trim(); if field_name.is_empty() { return None; } input.get(field_name).and_then(|value| match value { Value::String(value) if !value.trim().is_empty() => Some(value.clone()), Value::Number(value) => Some(value.to_string()), _ => None, }) } pub(crate) fn verified_retry_key( operation: &RuntimeOperation, idempotency_key: Option<&str>, ) -> bool { let Some(policy) = policy(operation) else { return false; }; idempotency_key.is_some() && policy .input_field .as_deref() .is_some_and(|field| !field.trim().is_empty()) } fn remove_header_case_insensitive(headers: &mut BTreeMap, header_name: &str) { let Some(existing) = headers .keys() .find(|candidate| candidate.eq_ignore_ascii_case(header_name)) .cloned() else { return; }; headers.remove(&existing); } #[cfg(test)] mod tests { use std::collections::BTreeMap; use crank_core::{ ExecutionConfig, HttpMethod, IdempotencyMode, IdempotencyPolicy, OperationId, Protocol, RestTarget, Target, ToolDescription, }; use crank_mapping::MappingSet; use crank_schema::{Schema, SchemaKind}; use serde_json::json; use crate::{PreparedRequest, RuntimeOperation}; #[test] fn idempotency_header_is_derived_from_input_not_user_mapped_header() { let operation = RuntimeOperation { operation_id: OperationId::new("op_idempotent"), operation_version: 1, tool_name: "idempotent_write".to_owned(), protocol: Protocol::Rest, target: Target::Rest(RestTarget { base_url: "https://api.example.invalid".to_owned(), method: HttpMethod::Post, path_template: "/orders".to_owned(), static_headers: BTreeMap::new(), request_contract: None, response_contract: Vec::new(), multipart_field_mime_types: BTreeMap::new(), }), input_schema: Schema { kind: SchemaKind::Object, format: None, description: None, required: true, nullable: false, default_value: None, fields: BTreeMap::new(), items: None, enum_values: Vec::new(), variants: Vec::new(), }, output_schema: Schema { kind: SchemaKind::Object, format: None, description: None, required: true, nullable: false, default_value: None, fields: BTreeMap::new(), items: None, enum_values: Vec::new(), variants: Vec::new(), }, input_mapping: MappingSet { rules: Vec::new() }, output_mapping: MappingSet { rules: Vec::new() }, execution_config: ExecutionConfig { timeout_ms: 1_000, retry_policy: None, response_cache: None, idempotency: Some(IdempotencyPolicy { mode: IdempotencyMode::Required, ttl_ms: 60_000, input_field: Some("request_id".to_owned()), header_name: Some("Idempotency-Key".to_owned()), }), safety: None, approval_policy: None, execution_policy: None, auth_profile_ref: None, headers: BTreeMap::new(), }, tool_description: ToolDescription { title: "Idempotent write".to_owned(), description: "Test operation".to_owned(), tags: Vec::new(), examples: Vec::new(), }, }; let mut prepared = PreparedRequest { headers: BTreeMap::from([("idempotency-key".to_owned(), "attacker".to_owned())]), ..PreparedRequest::default() }; let key = super::prepare_idempotency( &operation, &json!({"request_id": "trusted-key"}), &mut prepared, ) .unwrap(); assert_eq!(key.as_deref(), Some("trusted-key")); assert_eq!( prepared.headers.get("Idempotency-Key").map(String::as_str), Some("trusted-key") ); assert!(!prepared.headers.contains_key("idempotency-key")); assert!(prepared.trusted_header_names.contains("idempotency-key")); } }