Files
crank/crates/crank-runtime/src/idempotency.rs
T
bsodfather 2219a7217d
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 19m34s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): подтверждать durable Task после атомарного PG-commit (Story 4.2)
Application entry выдаёт Task ID только после атомарного PostgreSQL commit
Task+intent+capacity; replay/conflict, owner scope, bounded envelope,
terminal CAS и fail-closed readiness проверены на реальном PG.

Также включён принятый план перехода на rmcp (только документ, миграция не
запускается).
2026-10-27 00:09:36 +00:00

481 lines
16 KiB
Rust

use std::{
collections::BTreeMap,
time::{Duration, Instant},
};
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
use crank_core::{
CacheScope, CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue,
HttpMethod, IdempotencyMode, IdempotencyPolicy, Target,
};
use serde_json::{Map, Value, json};
use sha2::{Digest, Sha256};
use uuid::Uuid;
use crate::{
AdapterResponse, PreparedRequest, RuntimeError, RuntimeOperation, RuntimeRequestContext,
};
const POLL_INTERVAL: Duration = Duration::from_millis(10);
pub(crate) enum IdempotencyAction {
Disabled,
Execute(IdempotencyReservation),
Replay(AdapterResponse),
}
#[derive(Clone)]
pub(crate) struct IdempotencyReservation {
key: String,
initial: CoordinationStateValue,
fingerprint: String,
result_ttl: Duration,
}
pub fn prepare_idempotency(
operation: &RuntimeOperation,
input: &Value,
prepared_request: &mut PreparedRequest,
) -> Result<Option<String>, RuntimeError> {
let Some(policy) = policy(operation) else {
return Ok(None);
};
let key = key_from_policy(policy, input);
let Some(key) = key else {
if policy.mode == IdempotencyMode::Required {
return Err(RuntimeError::InvalidPreparedRequest {
field: "execution_config.idempotency".to_owned(),
reason: "required idempotency key was not provided".to_owned(),
});
}
return Ok(None);
};
if let Some(header_name) = policy
.header_name
.as_deref()
.filter(|value| !value.is_empty())
{
remove_header_case_insensitive(&mut prepared_request.headers, header_name);
prepared_request
.headers
.insert(header_name.to_owned(), key.clone());
prepared_request
.trusted_header_names
.insert(header_name.to_ascii_lowercase());
}
Ok(Some(key))
}
pub(crate) async fn begin(
store: Option<&dyn CoordinationStateStore>,
operation: &RuntimeOperation,
input: &Value,
idempotency_key: Option<&str>,
request_context: Option<&RuntimeRequestContext>,
) -> Result<IdempotencyAction, RuntimeError> {
let Some(policy) = policy(operation) else {
return Ok(IdempotencyAction::Disabled);
};
let Some(idempotency_key) = idempotency_key else {
return Ok(IdempotencyAction::Disabled);
};
let Some(key) = cache_key(operation, idempotency_key, request_context) else {
return Err(RuntimeError::IdempotencyStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
});
};
let Some(store) = store else {
return Err(RuntimeError::IdempotencyStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
});
};
let fingerprint = request_fingerprint(input)?;
let result_ttl = Duration::from_millis(policy.ttl_ms);
let reservation_ttl = result_ttl.max(Duration::from_millis(
operation.execution_config.timeout_ms.saturating_add(1_000),
));
let initial = in_progress_value(&fingerprint);
let reservation = store
.reserve_value(
CacheScope::Coordination,
&key,
initial.clone(),
reservation_ttl,
)
.await
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
})?;
match reservation {
CoordinationStateReservation::Reserved => {
Ok(IdempotencyAction::Execute(IdempotencyReservation {
key,
initial,
fingerprint,
result_ttl,
}))
}
CoordinationStateReservation::Existing(existing) => {
resolve_existing(store, operation, &key, &fingerprint, existing).await
}
}
}
pub(crate) async fn complete(
store: &dyn CoordinationStateStore,
operation: &RuntimeOperation,
reservation: &IdempotencyReservation,
response: &AdapterResponse,
) -> Result<(), RuntimeError> {
let completed = CoordinationStateValue {
payload: json!({
"state": "completed",
"fingerprint": reservation.fingerprint,
"response": response,
}),
};
let replaced = store
.compare_and_set_value(
CacheScope::Coordination,
&reservation.key,
&reservation.initial,
completed,
reservation.result_ttl,
)
.await
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
})?;
if replaced {
Ok(())
} else {
Err(RuntimeError::IdempotencyOutcomeUnknown {
operation_id: operation.operation_id.as_str().to_owned(),
})
}
}
pub(crate) async fn mark_outcome_unknown(
store: &dyn CoordinationStateStore,
operation: &RuntimeOperation,
reservation: &IdempotencyReservation,
) -> Result<(), RuntimeError> {
let unknown = CoordinationStateValue {
payload: json!({
"state": "outcome_unknown",
"fingerprint": reservation.fingerprint,
}),
};
let replaced = store
.compare_and_set_value(
CacheScope::Coordination,
&reservation.key,
&reservation.initial,
unknown,
reservation.result_ttl,
)
.await
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
})?;
if replaced {
Ok(())
} else {
Err(RuntimeError::IdempotencyOutcomeUnknown {
operation_id: operation.operation_id.as_str().to_owned(),
})
}
}
pub(crate) async fn release(
store: &dyn CoordinationStateStore,
operation: &RuntimeOperation,
reservation: &IdempotencyReservation,
) -> Result<(), RuntimeError> {
store
.delete_value(CacheScope::Coordination, &reservation.key)
.await
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
operation_id: operation.operation_id.as_str().to_owned(),
})
}
async fn resolve_existing(
store: &dyn CoordinationStateStore,
operation: &RuntimeOperation,
key: &str,
fingerprint: &str,
mut existing: CoordinationStateValue,
) -> Result<IdempotencyAction, RuntimeError> {
let operation_id = operation.operation_id.as_str().to_owned();
let deadline =
Instant::now() + Duration::from_millis(operation.execution_config.timeout_ms.max(1));
loop {
let existing_fingerprint = existing.payload.get("fingerprint").and_then(Value::as_str);
if existing_fingerprint != Some(fingerprint) {
return Err(RuntimeError::IdempotencyConflict { operation_id });
}
match existing.payload.get("state").and_then(Value::as_str) {
Some("completed") => {
let response = existing
.payload
.get("response")
.cloned()
.and_then(|value| serde_json::from_value(value).ok())
.ok_or_else(|| RuntimeError::InvalidPreparedRequest {
field: "idempotency_state".to_owned(),
reason: "completed idempotency state has no valid response".to_owned(),
})?;
return Ok(IdempotencyAction::Replay(response));
}
Some("outcome_unknown") => {
return Err(RuntimeError::IdempotencyOutcomeUnknown { operation_id });
}
Some("in_progress") if Instant::now() < deadline => {
tokio::time::sleep(POLL_INTERVAL).await;
existing = store
.get_value(CacheScope::Coordination, key)
.await
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
operation_id: operation_id.clone(),
})?
.ok_or_else(|| RuntimeError::IdempotencyOutcomeUnknown {
operation_id: operation_id.clone(),
})?;
}
Some("in_progress") => {
return Err(RuntimeError::IdempotencyInProgress { operation_id });
}
_ => {
return Err(RuntimeError::InvalidPreparedRequest {
field: "idempotency_state".to_owned(),
reason: "unknown idempotency state".to_owned(),
});
}
}
}
}
pub fn policy(operation: &RuntimeOperation) -> Option<&IdempotencyPolicy> {
let is_mutating_rest = matches!(
&operation.target,
Target::Rest(target)
if matches!(
target.method,
HttpMethod::Post | HttpMethod::Put | HttpMethod::Patch | HttpMethod::Delete
)
);
if !is_mutating_rest {
return None;
}
let policy = operation.execution_config.idempotency.as_ref()?;
if policy.mode == IdempotencyMode::Disabled || policy.ttl_ms == 0 {
return None;
}
Some(policy)
}
fn cache_key(
operation: &RuntimeOperation,
idempotency_key: &str,
request_context: Option<&RuntimeRequestContext>,
) -> Option<String> {
if idempotency_key.is_empty() {
return None;
}
let scope = request_context?.response_cache_scope()?;
let key_hash = URL_SAFE_NO_PAD.encode(Sha256::digest(idempotency_key.as_bytes()));
Some(format!(
"crank:idempotency:workspace:{}:agent:{}:operation:{}:version:{}:key:{}",
scope.workspace_key,
scope.agent_key,
operation.operation_id.as_str(),
operation.operation_version,
key_hash
))
}
fn request_fingerprint(input: &Value) -> Result<String, RuntimeError> {
let canonical = canonical_json(input);
let encoded =
serde_json::to_vec(&canonical).map_err(|error| RuntimeError::InvalidPreparedRequest {
field: "idempotency_fingerprint".to_owned(),
reason: error.to_string(),
})?;
Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded)))
}
pub(crate) fn canonical_json(value: &Value) -> Value {
match value {
Value::Object(object) => {
let mut entries = object.iter().collect::<Vec<_>>();
entries.sort_unstable_by_key(|(key, _)| *key);
let mut canonical = Map::new();
for (key, value) in entries {
canonical.insert(key.clone(), canonical_json(value));
}
Value::Object(canonical)
}
Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()),
_ => value.clone(),
}
}
fn in_progress_value(fingerprint: &str) -> CoordinationStateValue {
CoordinationStateValue {
payload: json!({
"state": "in_progress",
"fingerprint": fingerprint,
"owner": Uuid::now_v7().simple().to_string(),
}),
}
}
fn key_from_policy(policy: &IdempotencyPolicy, input: &Value) -> Option<String> {
let field_name = policy.input_field.as_deref()?.trim();
if field_name.is_empty() {
return None;
}
input.get(field_name).and_then(|value| match value {
Value::String(value) if !value.trim().is_empty() => Some(value.clone()),
Value::Number(value) => Some(value.to_string()),
_ => None,
})
}
pub(crate) fn verified_retry_key(
operation: &RuntimeOperation,
idempotency_key: Option<&str>,
) -> bool {
let Some(policy) = policy(operation) else {
return false;
};
idempotency_key.is_some()
&& policy
.input_field
.as_deref()
.is_some_and(|field| !field.trim().is_empty())
}
fn remove_header_case_insensitive(headers: &mut BTreeMap<String, String>, header_name: &str) {
let Some(existing) = headers
.keys()
.find(|candidate| candidate.eq_ignore_ascii_case(header_name))
.cloned()
else {
return;
};
headers.remove(&existing);
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use crank_core::{
ExecutionConfig, HttpMethod, IdempotencyMode, IdempotencyPolicy, OperationId, Protocol,
RestTarget, Target, ToolDescription,
};
use crank_mapping::MappingSet;
use crank_schema::{Schema, SchemaKind};
use serde_json::json;
use crate::{PreparedRequest, RuntimeOperation};
#[test]
fn idempotency_header_is_derived_from_input_not_user_mapped_header() {
let operation = RuntimeOperation {
operation_id: OperationId::new("op_idempotent"),
operation_version: 1,
tool_name: "idempotent_write".to_owned(),
protocol: Protocol::Rest,
target: Target::Rest(RestTarget {
base_url: "https://api.example.invalid".to_owned(),
method: HttpMethod::Post,
path_template: "/orders".to_owned(),
static_headers: BTreeMap::new(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: BTreeMap::new(),
}),
input_schema: Schema {
kind: SchemaKind::Object,
format: None,
description: None,
required: true,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
},
output_schema: Schema {
kind: SchemaKind::Object,
format: None,
description: None,
required: true,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
},
input_mapping: MappingSet { rules: Vec::new() },
output_mapping: MappingSet { rules: Vec::new() },
execution_config: ExecutionConfig {
timeout_ms: 1_000,
retry_policy: None,
response_cache: None,
idempotency: Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("request_id".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
}),
safety: None,
approval_policy: None,
execution_policy: None,
auth_profile_ref: None,
headers: BTreeMap::new(),
},
tool_description: ToolDescription {
title: "Idempotent write".to_owned(),
description: "Test operation".to_owned(),
tags: Vec::new(),
examples: Vec::new(),
},
};
let mut prepared = PreparedRequest {
headers: BTreeMap::from([("idempotency-key".to_owned(), "attacker".to_owned())]),
..PreparedRequest::default()
};
let key = super::prepare_idempotency(
&operation,
&json!({"request_id": "trusted-key"}),
&mut prepared,
)
.unwrap();
assert_eq!(key.as_deref(), Some("trusted-key"));
assert_eq!(
prepared.headers.get("Idempotency-Key").map(String::as_str),
Some("trusted-key")
);
assert!(!prepared.headers.contains_key("idempotency-key"));
assert!(prepared.trusted_header_names.contains("idempotency-key"));
}
}