CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 19m34s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Application entry выдаёт Task ID только после атомарного PostgreSQL commit Task+intent+capacity; replay/conflict, owner scope, bounded envelope, terminal CAS и fail-closed readiness проверены на реальном PG. Также включён принятый план перехода на rmcp (только документ, миграция не запускается).
481 lines
16 KiB
Rust
481 lines
16 KiB
Rust
use std::{
|
|
collections::BTreeMap,
|
|
time::{Duration, Instant},
|
|
};
|
|
|
|
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
|
use crank_core::{
|
|
CacheScope, CoordinationStateReservation, CoordinationStateStore, CoordinationStateValue,
|
|
HttpMethod, IdempotencyMode, IdempotencyPolicy, Target,
|
|
};
|
|
use serde_json::{Map, Value, json};
|
|
use sha2::{Digest, Sha256};
|
|
use uuid::Uuid;
|
|
|
|
use crate::{
|
|
AdapterResponse, PreparedRequest, RuntimeError, RuntimeOperation, RuntimeRequestContext,
|
|
};
|
|
|
|
const POLL_INTERVAL: Duration = Duration::from_millis(10);
|
|
|
|
pub(crate) enum IdempotencyAction {
|
|
Disabled,
|
|
Execute(IdempotencyReservation),
|
|
Replay(AdapterResponse),
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub(crate) struct IdempotencyReservation {
|
|
key: String,
|
|
initial: CoordinationStateValue,
|
|
fingerprint: String,
|
|
result_ttl: Duration,
|
|
}
|
|
|
|
pub fn prepare_idempotency(
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
prepared_request: &mut PreparedRequest,
|
|
) -> Result<Option<String>, RuntimeError> {
|
|
let Some(policy) = policy(operation) else {
|
|
return Ok(None);
|
|
};
|
|
|
|
let key = key_from_policy(policy, input);
|
|
let Some(key) = key else {
|
|
if policy.mode == IdempotencyMode::Required {
|
|
return Err(RuntimeError::InvalidPreparedRequest {
|
|
field: "execution_config.idempotency".to_owned(),
|
|
reason: "required idempotency key was not provided".to_owned(),
|
|
});
|
|
}
|
|
return Ok(None);
|
|
};
|
|
|
|
if let Some(header_name) = policy
|
|
.header_name
|
|
.as_deref()
|
|
.filter(|value| !value.is_empty())
|
|
{
|
|
remove_header_case_insensitive(&mut prepared_request.headers, header_name);
|
|
prepared_request
|
|
.headers
|
|
.insert(header_name.to_owned(), key.clone());
|
|
prepared_request
|
|
.trusted_header_names
|
|
.insert(header_name.to_ascii_lowercase());
|
|
}
|
|
|
|
Ok(Some(key))
|
|
}
|
|
|
|
pub(crate) async fn begin(
|
|
store: Option<&dyn CoordinationStateStore>,
|
|
operation: &RuntimeOperation,
|
|
input: &Value,
|
|
idempotency_key: Option<&str>,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Result<IdempotencyAction, RuntimeError> {
|
|
let Some(policy) = policy(operation) else {
|
|
return Ok(IdempotencyAction::Disabled);
|
|
};
|
|
let Some(idempotency_key) = idempotency_key else {
|
|
return Ok(IdempotencyAction::Disabled);
|
|
};
|
|
let Some(key) = cache_key(operation, idempotency_key, request_context) else {
|
|
return Err(RuntimeError::IdempotencyStoreUnavailable {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
});
|
|
};
|
|
let Some(store) = store else {
|
|
return Err(RuntimeError::IdempotencyStoreUnavailable {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
});
|
|
};
|
|
|
|
let fingerprint = request_fingerprint(input)?;
|
|
let result_ttl = Duration::from_millis(policy.ttl_ms);
|
|
let reservation_ttl = result_ttl.max(Duration::from_millis(
|
|
operation.execution_config.timeout_ms.saturating_add(1_000),
|
|
));
|
|
let initial = in_progress_value(&fingerprint);
|
|
let reservation = store
|
|
.reserve_value(
|
|
CacheScope::Coordination,
|
|
&key,
|
|
initial.clone(),
|
|
reservation_ttl,
|
|
)
|
|
.await
|
|
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
})?;
|
|
|
|
match reservation {
|
|
CoordinationStateReservation::Reserved => {
|
|
Ok(IdempotencyAction::Execute(IdempotencyReservation {
|
|
key,
|
|
initial,
|
|
fingerprint,
|
|
result_ttl,
|
|
}))
|
|
}
|
|
CoordinationStateReservation::Existing(existing) => {
|
|
resolve_existing(store, operation, &key, &fingerprint, existing).await
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn complete(
|
|
store: &dyn CoordinationStateStore,
|
|
operation: &RuntimeOperation,
|
|
reservation: &IdempotencyReservation,
|
|
response: &AdapterResponse,
|
|
) -> Result<(), RuntimeError> {
|
|
let completed = CoordinationStateValue {
|
|
payload: json!({
|
|
"state": "completed",
|
|
"fingerprint": reservation.fingerprint,
|
|
"response": response,
|
|
}),
|
|
};
|
|
let replaced = store
|
|
.compare_and_set_value(
|
|
CacheScope::Coordination,
|
|
&reservation.key,
|
|
&reservation.initial,
|
|
completed,
|
|
reservation.result_ttl,
|
|
)
|
|
.await
|
|
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
})?;
|
|
if replaced {
|
|
Ok(())
|
|
} else {
|
|
Err(RuntimeError::IdempotencyOutcomeUnknown {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
})
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn mark_outcome_unknown(
|
|
store: &dyn CoordinationStateStore,
|
|
operation: &RuntimeOperation,
|
|
reservation: &IdempotencyReservation,
|
|
) -> Result<(), RuntimeError> {
|
|
let unknown = CoordinationStateValue {
|
|
payload: json!({
|
|
"state": "outcome_unknown",
|
|
"fingerprint": reservation.fingerprint,
|
|
}),
|
|
};
|
|
let replaced = store
|
|
.compare_and_set_value(
|
|
CacheScope::Coordination,
|
|
&reservation.key,
|
|
&reservation.initial,
|
|
unknown,
|
|
reservation.result_ttl,
|
|
)
|
|
.await
|
|
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
})?;
|
|
if replaced {
|
|
Ok(())
|
|
} else {
|
|
Err(RuntimeError::IdempotencyOutcomeUnknown {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
})
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn release(
|
|
store: &dyn CoordinationStateStore,
|
|
operation: &RuntimeOperation,
|
|
reservation: &IdempotencyReservation,
|
|
) -> Result<(), RuntimeError> {
|
|
store
|
|
.delete_value(CacheScope::Coordination, &reservation.key)
|
|
.await
|
|
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
|
|
operation_id: operation.operation_id.as_str().to_owned(),
|
|
})
|
|
}
|
|
|
|
async fn resolve_existing(
|
|
store: &dyn CoordinationStateStore,
|
|
operation: &RuntimeOperation,
|
|
key: &str,
|
|
fingerprint: &str,
|
|
mut existing: CoordinationStateValue,
|
|
) -> Result<IdempotencyAction, RuntimeError> {
|
|
let operation_id = operation.operation_id.as_str().to_owned();
|
|
let deadline =
|
|
Instant::now() + Duration::from_millis(operation.execution_config.timeout_ms.max(1));
|
|
loop {
|
|
let existing_fingerprint = existing.payload.get("fingerprint").and_then(Value::as_str);
|
|
if existing_fingerprint != Some(fingerprint) {
|
|
return Err(RuntimeError::IdempotencyConflict { operation_id });
|
|
}
|
|
match existing.payload.get("state").and_then(Value::as_str) {
|
|
Some("completed") => {
|
|
let response = existing
|
|
.payload
|
|
.get("response")
|
|
.cloned()
|
|
.and_then(|value| serde_json::from_value(value).ok())
|
|
.ok_or_else(|| RuntimeError::InvalidPreparedRequest {
|
|
field: "idempotency_state".to_owned(),
|
|
reason: "completed idempotency state has no valid response".to_owned(),
|
|
})?;
|
|
return Ok(IdempotencyAction::Replay(response));
|
|
}
|
|
Some("outcome_unknown") => {
|
|
return Err(RuntimeError::IdempotencyOutcomeUnknown { operation_id });
|
|
}
|
|
Some("in_progress") if Instant::now() < deadline => {
|
|
tokio::time::sleep(POLL_INTERVAL).await;
|
|
existing = store
|
|
.get_value(CacheScope::Coordination, key)
|
|
.await
|
|
.map_err(|_| RuntimeError::IdempotencyStoreUnavailable {
|
|
operation_id: operation_id.clone(),
|
|
})?
|
|
.ok_or_else(|| RuntimeError::IdempotencyOutcomeUnknown {
|
|
operation_id: operation_id.clone(),
|
|
})?;
|
|
}
|
|
Some("in_progress") => {
|
|
return Err(RuntimeError::IdempotencyInProgress { operation_id });
|
|
}
|
|
_ => {
|
|
return Err(RuntimeError::InvalidPreparedRequest {
|
|
field: "idempotency_state".to_owned(),
|
|
reason: "unknown idempotency state".to_owned(),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn policy(operation: &RuntimeOperation) -> Option<&IdempotencyPolicy> {
|
|
let is_mutating_rest = matches!(
|
|
&operation.target,
|
|
Target::Rest(target)
|
|
if matches!(
|
|
target.method,
|
|
HttpMethod::Post | HttpMethod::Put | HttpMethod::Patch | HttpMethod::Delete
|
|
)
|
|
);
|
|
if !is_mutating_rest {
|
|
return None;
|
|
}
|
|
|
|
let policy = operation.execution_config.idempotency.as_ref()?;
|
|
if policy.mode == IdempotencyMode::Disabled || policy.ttl_ms == 0 {
|
|
return None;
|
|
}
|
|
|
|
Some(policy)
|
|
}
|
|
|
|
fn cache_key(
|
|
operation: &RuntimeOperation,
|
|
idempotency_key: &str,
|
|
request_context: Option<&RuntimeRequestContext>,
|
|
) -> Option<String> {
|
|
if idempotency_key.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
let scope = request_context?.response_cache_scope()?;
|
|
let key_hash = URL_SAFE_NO_PAD.encode(Sha256::digest(idempotency_key.as_bytes()));
|
|
|
|
Some(format!(
|
|
"crank:idempotency:workspace:{}:agent:{}:operation:{}:version:{}:key:{}",
|
|
scope.workspace_key,
|
|
scope.agent_key,
|
|
operation.operation_id.as_str(),
|
|
operation.operation_version,
|
|
key_hash
|
|
))
|
|
}
|
|
|
|
fn request_fingerprint(input: &Value) -> Result<String, RuntimeError> {
|
|
let canonical = canonical_json(input);
|
|
let encoded =
|
|
serde_json::to_vec(&canonical).map_err(|error| RuntimeError::InvalidPreparedRequest {
|
|
field: "idempotency_fingerprint".to_owned(),
|
|
reason: error.to_string(),
|
|
})?;
|
|
Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(encoded)))
|
|
}
|
|
|
|
pub(crate) fn canonical_json(value: &Value) -> Value {
|
|
match value {
|
|
Value::Object(object) => {
|
|
let mut entries = object.iter().collect::<Vec<_>>();
|
|
entries.sort_unstable_by_key(|(key, _)| *key);
|
|
let mut canonical = Map::new();
|
|
for (key, value) in entries {
|
|
canonical.insert(key.clone(), canonical_json(value));
|
|
}
|
|
Value::Object(canonical)
|
|
}
|
|
Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()),
|
|
_ => value.clone(),
|
|
}
|
|
}
|
|
|
|
fn in_progress_value(fingerprint: &str) -> CoordinationStateValue {
|
|
CoordinationStateValue {
|
|
payload: json!({
|
|
"state": "in_progress",
|
|
"fingerprint": fingerprint,
|
|
"owner": Uuid::now_v7().simple().to_string(),
|
|
}),
|
|
}
|
|
}
|
|
|
|
fn key_from_policy(policy: &IdempotencyPolicy, input: &Value) -> Option<String> {
|
|
let field_name = policy.input_field.as_deref()?.trim();
|
|
if field_name.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
input.get(field_name).and_then(|value| match value {
|
|
Value::String(value) if !value.trim().is_empty() => Some(value.clone()),
|
|
Value::Number(value) => Some(value.to_string()),
|
|
_ => None,
|
|
})
|
|
}
|
|
|
|
pub(crate) fn verified_retry_key(
|
|
operation: &RuntimeOperation,
|
|
idempotency_key: Option<&str>,
|
|
) -> bool {
|
|
let Some(policy) = policy(operation) else {
|
|
return false;
|
|
};
|
|
idempotency_key.is_some()
|
|
&& policy
|
|
.input_field
|
|
.as_deref()
|
|
.is_some_and(|field| !field.trim().is_empty())
|
|
}
|
|
|
|
fn remove_header_case_insensitive(headers: &mut BTreeMap<String, String>, header_name: &str) {
|
|
let Some(existing) = headers
|
|
.keys()
|
|
.find(|candidate| candidate.eq_ignore_ascii_case(header_name))
|
|
.cloned()
|
|
else {
|
|
return;
|
|
};
|
|
headers.remove(&existing);
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use std::collections::BTreeMap;
|
|
|
|
use crank_core::{
|
|
ExecutionConfig, HttpMethod, IdempotencyMode, IdempotencyPolicy, OperationId, Protocol,
|
|
RestTarget, Target, ToolDescription,
|
|
};
|
|
use crank_mapping::MappingSet;
|
|
use crank_schema::{Schema, SchemaKind};
|
|
use serde_json::json;
|
|
|
|
use crate::{PreparedRequest, RuntimeOperation};
|
|
|
|
#[test]
|
|
fn idempotency_header_is_derived_from_input_not_user_mapped_header() {
|
|
let operation = RuntimeOperation {
|
|
operation_id: OperationId::new("op_idempotent"),
|
|
operation_version: 1,
|
|
tool_name: "idempotent_write".to_owned(),
|
|
protocol: Protocol::Rest,
|
|
target: Target::Rest(RestTarget {
|
|
base_url: "https://api.example.invalid".to_owned(),
|
|
method: HttpMethod::Post,
|
|
path_template: "/orders".to_owned(),
|
|
static_headers: BTreeMap::new(),
|
|
request_contract: None,
|
|
response_contract: Vec::new(),
|
|
multipart_field_mime_types: BTreeMap::new(),
|
|
}),
|
|
input_schema: Schema {
|
|
kind: SchemaKind::Object,
|
|
format: None,
|
|
description: None,
|
|
required: true,
|
|
nullable: false,
|
|
default_value: None,
|
|
fields: BTreeMap::new(),
|
|
items: None,
|
|
enum_values: Vec::new(),
|
|
variants: Vec::new(),
|
|
},
|
|
output_schema: Schema {
|
|
kind: SchemaKind::Object,
|
|
format: None,
|
|
description: None,
|
|
required: true,
|
|
nullable: false,
|
|
default_value: None,
|
|
fields: BTreeMap::new(),
|
|
items: None,
|
|
enum_values: Vec::new(),
|
|
variants: Vec::new(),
|
|
},
|
|
input_mapping: MappingSet { rules: Vec::new() },
|
|
output_mapping: MappingSet { rules: Vec::new() },
|
|
execution_config: ExecutionConfig {
|
|
timeout_ms: 1_000,
|
|
retry_policy: None,
|
|
response_cache: None,
|
|
idempotency: Some(IdempotencyPolicy {
|
|
mode: IdempotencyMode::Required,
|
|
ttl_ms: 60_000,
|
|
input_field: Some("request_id".to_owned()),
|
|
header_name: Some("Idempotency-Key".to_owned()),
|
|
}),
|
|
safety: None,
|
|
approval_policy: None,
|
|
execution_policy: None,
|
|
auth_profile_ref: None,
|
|
headers: BTreeMap::new(),
|
|
},
|
|
tool_description: ToolDescription {
|
|
title: "Idempotent write".to_owned(),
|
|
description: "Test operation".to_owned(),
|
|
tags: Vec::new(),
|
|
examples: Vec::new(),
|
|
},
|
|
};
|
|
let mut prepared = PreparedRequest {
|
|
headers: BTreeMap::from([("idempotency-key".to_owned(), "attacker".to_owned())]),
|
|
..PreparedRequest::default()
|
|
};
|
|
|
|
let key = super::prepare_idempotency(
|
|
&operation,
|
|
&json!({"request_id": "trusted-key"}),
|
|
&mut prepared,
|
|
)
|
|
.unwrap();
|
|
|
|
assert_eq!(key.as_deref(), Some("trusted-key"));
|
|
assert_eq!(
|
|
prepared.headers.get("Idempotency-Key").map(String::as_str),
|
|
Some("trusted-key")
|
|
);
|
|
assert!(!prepared.headers.contains_key("idempotency-key"));
|
|
assert!(prepared.trusted_header_names.contains("idempotency-key"));
|
|
}
|
|
}
|