feat: add window execution to runtime
This commit is contained in:
@@ -2,19 +2,20 @@
|
|||||||
|
|
||||||
## Current
|
## Current
|
||||||
|
|
||||||
### `feat/stream-session-store`
|
### `feat/runtime-window-mode`
|
||||||
|
|
||||||
Status: completed
|
Status: completed
|
||||||
|
|
||||||
DoD:
|
DoD:
|
||||||
- persistent tables exist for `stream_sessions` and `async_jobs`
|
- runtime exposes bounded `window` execution
|
||||||
- registry supports create/get/update/close/list/cleanup for both stores
|
- item and byte limits are enforced
|
||||||
- invalid state transitions are rejected explicitly
|
- `window_complete`, `truncated`, `has_more` are returned
|
||||||
- migration and storage tests cover lifecycle and cleanup
|
- aggregation and redaction are covered by tests
|
||||||
|
- unary execution remains intact
|
||||||
|
|
||||||
## Next
|
## Next
|
||||||
|
|
||||||
- `feat/runtime-window-mode`
|
- `feat/rest-sse-adapter`
|
||||||
|
|
||||||
## Backlog
|
## Backlog
|
||||||
|
|
||||||
|
|||||||
@@ -240,5 +240,8 @@ fn runtime_test_failure_code(error: &RuntimeError) -> &'static str {
|
|||||||
RuntimeError::RestAdapter(_) => "runtime_rest_error",
|
RuntimeError::RestAdapter(_) => "runtime_rest_error",
|
||||||
RuntimeError::UnsupportedProtocol { .. } => "runtime_protocol_error",
|
RuntimeError::UnsupportedProtocol { .. } => "runtime_protocol_error",
|
||||||
RuntimeError::InvalidPreparedRequest { .. } => "runtime_request_error",
|
RuntimeError::InvalidPreparedRequest { .. } => "runtime_request_error",
|
||||||
|
RuntimeError::MissingStreamingConfig { .. } => "runtime_streaming_config_error",
|
||||||
|
RuntimeError::UnsupportedExecutionMode { .. } => "runtime_streaming_mode_error",
|
||||||
|
RuntimeError::InvalidStreamingPayload { .. } => "runtime_streaming_payload_error",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3756,6 +3756,9 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
|
|||||||
RuntimeError::GrpcAdapter(_) => "grpc_error",
|
RuntimeError::GrpcAdapter(_) => "grpc_error",
|
||||||
RuntimeError::RestAdapter(_) => "rest_error",
|
RuntimeError::RestAdapter(_) => "rest_error",
|
||||||
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
|
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
|
||||||
|
RuntimeError::MissingStreamingConfig { .. } => "streaming_config_error",
|
||||||
|
RuntimeError::UnsupportedExecutionMode { .. } => "streaming_mode_error",
|
||||||
|
RuntimeError::InvalidStreamingPayload { .. } => "streaming_payload_error",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -846,7 +846,10 @@ fn runtime_error_code(error: &RuntimeError) -> &'static str {
|
|||||||
RuntimeError::GrpcAdapter(_) => "adapter_execution_error",
|
RuntimeError::GrpcAdapter(_) => "adapter_execution_error",
|
||||||
RuntimeError::RestAdapter(_) => "adapter_execution_error",
|
RuntimeError::RestAdapter(_) => "adapter_execution_error",
|
||||||
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
|
RuntimeError::UnsupportedProtocol { .. } => "unsupported_protocol",
|
||||||
|
RuntimeError::MissingStreamingConfig { .. } => "streaming_config_error",
|
||||||
|
RuntimeError::UnsupportedExecutionMode { .. } => "streaming_mode_error",
|
||||||
RuntimeError::InvalidPreparedRequest { .. } => "runtime_error",
|
RuntimeError::InvalidPreparedRequest { .. } => "runtime_error",
|
||||||
|
RuntimeError::InvalidStreamingPayload { .. } => "streaming_payload_error",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,292 @@
|
|||||||
|
use serde_json::{Map, Value, json};
|
||||||
|
|
||||||
|
use crate::{RuntimeError, WindowExecutionResult};
|
||||||
|
|
||||||
|
pub fn collect_window_result(
|
||||||
|
response: &Value,
|
||||||
|
config: &crank_core::StreamingConfig,
|
||||||
|
) -> Result<WindowExecutionResult, RuntimeError> {
|
||||||
|
let mut items = extract_path_value(response, config.items_path.as_deref())
|
||||||
|
.and_then(|value| value.as_array().cloned())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let cursor = extract_path_value(response, config.cursor_path.as_deref()).cloned();
|
||||||
|
let done = extract_path_value(response, config.done_path.as_deref())
|
||||||
|
.and_then(Value::as_bool)
|
||||||
|
.unwrap_or(true);
|
||||||
|
|
||||||
|
let mut truncated = false;
|
||||||
|
let mut has_more = !done;
|
||||||
|
|
||||||
|
if let Some(max_items) = config.max_items.map(|value| value as usize) {
|
||||||
|
if items.len() > max_items {
|
||||||
|
items.truncate(max_items);
|
||||||
|
truncated = true;
|
||||||
|
has_more = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut summary = build_summary(response, &items, config);
|
||||||
|
redact_and_truncate(&mut summary, &mut items, config);
|
||||||
|
|
||||||
|
if let Some(max_bytes) = config.max_bytes.map(|value| value as usize) {
|
||||||
|
while payload_size_bytes(&summary, &items, cursor.as_ref()) > max_bytes && !items.is_empty()
|
||||||
|
{
|
||||||
|
items.pop();
|
||||||
|
truncated = true;
|
||||||
|
has_more = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if payload_size_bytes(&summary, &items, cursor.as_ref()) > max_bytes {
|
||||||
|
summary = json!({
|
||||||
|
"notice": "window payload exceeded byte limit",
|
||||||
|
"items_returned": items.len()
|
||||||
|
});
|
||||||
|
truncated = true;
|
||||||
|
has_more = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if matches!(
|
||||||
|
config.aggregation_mode,
|
||||||
|
crank_core::AggregationMode::SummaryOnly
|
||||||
|
) {
|
||||||
|
items.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(WindowExecutionResult {
|
||||||
|
summary,
|
||||||
|
items,
|
||||||
|
cursor,
|
||||||
|
window_complete: !has_more,
|
||||||
|
truncated,
|
||||||
|
has_more,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_summary(response: &Value, items: &[Value], config: &crank_core::StreamingConfig) -> Value {
|
||||||
|
match config.aggregation_mode {
|
||||||
|
crank_core::AggregationMode::RawItems => Value::Null,
|
||||||
|
crank_core::AggregationMode::SummaryOnly
|
||||||
|
| crank_core::AggregationMode::SummaryPlusSamples => {
|
||||||
|
extract_path_value(response, config.summary_path.as_deref())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| stats_summary(items))
|
||||||
|
}
|
||||||
|
crank_core::AggregationMode::Stats => stats_summary(items),
|
||||||
|
crank_core::AggregationMode::LatestState => items.last().cloned().unwrap_or(Value::Null),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stats_summary(items: &[Value]) -> Value {
|
||||||
|
let mut summary = Map::new();
|
||||||
|
summary.insert("items_total".to_owned(), Value::from(items.len() as u64));
|
||||||
|
summary.insert("items_sampled".to_owned(), Value::from(items.len() as u64));
|
||||||
|
Value::Object(summary)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn redact_and_truncate(
|
||||||
|
summary: &mut Value,
|
||||||
|
items: &mut [Value],
|
||||||
|
config: &crank_core::StreamingConfig,
|
||||||
|
) {
|
||||||
|
crate::redaction::redact_paths(summary, &config.redacted_paths);
|
||||||
|
|
||||||
|
for item in items.iter_mut() {
|
||||||
|
crate::redaction::redact_paths(item, &config.redacted_paths);
|
||||||
|
}
|
||||||
|
|
||||||
|
if config.truncate_item_fields {
|
||||||
|
if let Some(max_len) = config.max_field_length.map(|value| value as usize) {
|
||||||
|
for item in items.iter_mut() {
|
||||||
|
crate::redaction::truncate_item_fields(item, max_len);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn payload_size_bytes(summary: &Value, items: &[Value], cursor: Option<&Value>) -> usize {
|
||||||
|
serde_json::to_vec(&json!({
|
||||||
|
"summary": summary,
|
||||||
|
"items": items,
|
||||||
|
"cursor": cursor
|
||||||
|
}))
|
||||||
|
.map(|value| value.len())
|
||||||
|
.unwrap_or(usize::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract_path_value<'a>(value: &'a Value, path: Option<&str>) -> Option<&'a Value> {
|
||||||
|
let path = path?;
|
||||||
|
let path = path.strip_prefix("$.").or_else(|| path.strip_prefix('$'))?;
|
||||||
|
|
||||||
|
if path.is_empty() {
|
||||||
|
return Some(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut current = value;
|
||||||
|
for segment in path.split('.') {
|
||||||
|
current = current.get(segment)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(current)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use serde_json::{Value, json};
|
||||||
|
|
||||||
|
use crank_core::{
|
||||||
|
AggregationMode, ExecutionMode, StreamingConfig, ToolFamilyConfig, TransportBehavior,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::collect_window_result;
|
||||||
|
|
||||||
|
fn window_config() -> StreamingConfig {
|
||||||
|
StreamingConfig {
|
||||||
|
mode: ExecutionMode::Window,
|
||||||
|
transport_behavior: TransportBehavior::ServerStream,
|
||||||
|
window_duration_ms: Some(3_000),
|
||||||
|
poll_interval_ms: None,
|
||||||
|
upstream_timeout_ms: Some(5_000),
|
||||||
|
idle_timeout_ms: None,
|
||||||
|
max_session_lifetime_ms: None,
|
||||||
|
max_items: Some(10),
|
||||||
|
max_bytes: Some(10_000),
|
||||||
|
aggregation_mode: AggregationMode::RawItems,
|
||||||
|
summary_path: Some("$.summary".to_owned()),
|
||||||
|
items_path: Some("$.items".to_owned()),
|
||||||
|
cursor_path: Some("$.cursor".to_owned()),
|
||||||
|
status_path: None,
|
||||||
|
done_path: Some("$.done".to_owned()),
|
||||||
|
redacted_paths: Vec::new(),
|
||||||
|
truncate_item_fields: false,
|
||||||
|
max_field_length: None,
|
||||||
|
drop_duplicates: false,
|
||||||
|
sampling_rate: None,
|
||||||
|
tool_family: ToolFamilyConfig::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn collects_raw_items_mode() {
|
||||||
|
let config = window_config();
|
||||||
|
let result = collect_window_result(
|
||||||
|
&json!({
|
||||||
|
"items": [{ "message": "one" }, { "message": "two" }],
|
||||||
|
"summary": { "count": 2 },
|
||||||
|
"done": true
|
||||||
|
}),
|
||||||
|
&config,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.summary, Value::Null);
|
||||||
|
assert_eq!(result.items.len(), 2);
|
||||||
|
assert!(result.window_complete);
|
||||||
|
assert!(!result.truncated);
|
||||||
|
assert!(!result.has_more);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn builds_summary_only_mode() {
|
||||||
|
let mut config = window_config();
|
||||||
|
config.aggregation_mode = AggregationMode::SummaryOnly;
|
||||||
|
|
||||||
|
let result = collect_window_result(
|
||||||
|
&json!({
|
||||||
|
"items": [{ "message": "one" }],
|
||||||
|
"summary": { "count": 1 },
|
||||||
|
"done": true
|
||||||
|
}),
|
||||||
|
&config,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.summary, json!({ "count": 1 }));
|
||||||
|
assert!(result.items.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn keeps_summary_plus_samples() {
|
||||||
|
let mut config = window_config();
|
||||||
|
config.aggregation_mode = AggregationMode::SummaryPlusSamples;
|
||||||
|
|
||||||
|
let result = collect_window_result(
|
||||||
|
&json!({
|
||||||
|
"items": [{ "message": "one" }, { "message": "two" }],
|
||||||
|
"summary": { "count": 2 },
|
||||||
|
"done": true
|
||||||
|
}),
|
||||||
|
&config,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.summary, json!({ "count": 2 }));
|
||||||
|
assert_eq!(result.items.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncates_by_item_count() {
|
||||||
|
let mut config = window_config();
|
||||||
|
config.max_items = Some(1);
|
||||||
|
|
||||||
|
let result = collect_window_result(
|
||||||
|
&json!({
|
||||||
|
"items": [{ "message": "one" }, { "message": "two" }],
|
||||||
|
"cursor": "next",
|
||||||
|
"done": false
|
||||||
|
}),
|
||||||
|
&config,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.items.len(), 1);
|
||||||
|
assert!(result.truncated);
|
||||||
|
assert!(result.has_more);
|
||||||
|
assert!(!result.window_complete);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncates_by_byte_limit() {
|
||||||
|
let mut config = window_config();
|
||||||
|
config.max_bytes = Some(120);
|
||||||
|
config.aggregation_mode = AggregationMode::SummaryPlusSamples;
|
||||||
|
|
||||||
|
let result = collect_window_result(
|
||||||
|
&json!({
|
||||||
|
"items": [
|
||||||
|
{ "message": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
|
||||||
|
{ "message": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }
|
||||||
|
],
|
||||||
|
"summary": { "count": 2 },
|
||||||
|
"done": true
|
||||||
|
}),
|
||||||
|
&config,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(result.truncated);
|
||||||
|
assert!(result.items.len() < 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redacts_and_truncates_values() {
|
||||||
|
let mut config = window_config();
|
||||||
|
config.redacted_paths = vec!["$.secret".to_owned()];
|
||||||
|
config.truncate_item_fields = true;
|
||||||
|
config.max_field_length = Some(4);
|
||||||
|
|
||||||
|
let result = collect_window_result(
|
||||||
|
&json!({
|
||||||
|
"items": [
|
||||||
|
{ "message": "abcdefghijklmnop", "secret": "token" }
|
||||||
|
],
|
||||||
|
"done": true
|
||||||
|
}),
|
||||||
|
&config,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.items[0]["secret"], json!("[REDACTED]"));
|
||||||
|
assert_eq!(result.items[0]["message"], json!("abcd..."));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
use crank_adapter_graphql::GraphqlAdapterError;
|
use crank_adapter_graphql::GraphqlAdapterError;
|
||||||
use crank_adapter_grpc::GrpcAdapterError;
|
use crank_adapter_grpc::GrpcAdapterError;
|
||||||
use crank_adapter_rest::RestAdapterError;
|
use crank_adapter_rest::RestAdapterError;
|
||||||
use crank_core::Protocol;
|
use crank_core::{ExecutionMode, Protocol};
|
||||||
use crank_mapping::MappingError;
|
use crank_mapping::MappingError;
|
||||||
use crank_schema::SchemaError;
|
use crank_schema::SchemaError;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
@@ -20,6 +20,15 @@ pub enum RuntimeError {
|
|||||||
RestAdapter(#[from] RestAdapterError),
|
RestAdapter(#[from] RestAdapterError),
|
||||||
#[error("protocol {protocol:?} is not supported by runtime")]
|
#[error("protocol {protocol:?} is not supported by runtime")]
|
||||||
UnsupportedProtocol { protocol: Protocol },
|
UnsupportedProtocol { protocol: Protocol },
|
||||||
|
#[error("operation {operation_id} does not define streaming config")]
|
||||||
|
MissingStreamingConfig { operation_id: String },
|
||||||
|
#[error("operation {operation_id} does not support requested execution mode {mode:?}")]
|
||||||
|
UnsupportedExecutionMode {
|
||||||
|
operation_id: String,
|
||||||
|
mode: ExecutionMode,
|
||||||
|
},
|
||||||
#[error("invalid prepared request: {details}")]
|
#[error("invalid prepared request: {details}")]
|
||||||
InvalidPreparedRequest { details: String },
|
InvalidPreparedRequest { details: String },
|
||||||
|
#[error("invalid streaming payload: {details}")]
|
||||||
|
InvalidStreamingPayload { details: String },
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,12 @@ use std::collections::BTreeMap;
|
|||||||
use crank_adapter_graphql::{GraphqlAdapter, GraphqlRequest};
|
use crank_adapter_graphql::{GraphqlAdapter, GraphqlRequest};
|
||||||
use crank_adapter_grpc::{GrpcAdapter, GrpcRequest};
|
use crank_adapter_grpc::{GrpcAdapter, GrpcRequest};
|
||||||
use crank_adapter_rest::{RestAdapter, RestRequest};
|
use crank_adapter_rest::{RestAdapter, RestRequest};
|
||||||
use crank_core::Target;
|
use crank_core::{ExecutionMode, Target};
|
||||||
use serde_json::{Map, Value, json};
|
use serde_json::{Map, Value, json};
|
||||||
|
|
||||||
use crate::{AdapterResponse, PreparedRequest, RuntimeError, RuntimeOperation};
|
use crate::{
|
||||||
|
AdapterResponse, PreparedRequest, RuntimeError, RuntimeOperation, WindowExecutionResult,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct RuntimeExecutor {
|
pub struct RuntimeExecutor {
|
||||||
@@ -39,6 +41,30 @@ impl RuntimeExecutor {
|
|||||||
self.execute_prepared(operation, prepared_request).await
|
self.execute_prepared(operation, prepared_request).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn execute_window(
|
||||||
|
&self,
|
||||||
|
operation: &RuntimeOperation,
|
||||||
|
input: &Value,
|
||||||
|
) -> Result<WindowExecutionResult, RuntimeError> {
|
||||||
|
let Some(streaming) = operation.execution_config.streaming.as_ref() else {
|
||||||
|
return Err(RuntimeError::MissingStreamingConfig {
|
||||||
|
operation_id: operation.operation_id.as_str().to_owned(),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
if streaming.mode != ExecutionMode::Window {
|
||||||
|
return Err(RuntimeError::UnsupportedExecutionMode {
|
||||||
|
operation_id: operation.operation_id.as_str().to_owned(),
|
||||||
|
mode: streaming.mode,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let prepared_request = self.prepare_request(operation, input)?;
|
||||||
|
let adapter_response = self.execute_adapter(operation, prepared_request).await?;
|
||||||
|
|
||||||
|
crate::aggregation::collect_window_result(&adapter_response.body, streaming)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn prepare_request(
|
pub fn prepare_request(
|
||||||
&self,
|
&self,
|
||||||
operation: &RuntimeOperation,
|
operation: &RuntimeOperation,
|
||||||
@@ -55,7 +81,20 @@ impl RuntimeExecutor {
|
|||||||
operation: &RuntimeOperation,
|
operation: &RuntimeOperation,
|
||||||
prepared_request: PreparedRequest,
|
prepared_request: PreparedRequest,
|
||||||
) -> Result<Value, RuntimeError> {
|
) -> Result<Value, RuntimeError> {
|
||||||
let adapter_response = match &operation.target {
|
let adapter_response = self.execute_adapter(operation, prepared_request).await?;
|
||||||
|
let finalized_output = finalize_output(operation, &adapter_response)?;
|
||||||
|
|
||||||
|
operation.output_schema.validate_shape(&finalized_output)?;
|
||||||
|
|
||||||
|
Ok(finalized_output)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn execute_adapter(
|
||||||
|
&self,
|
||||||
|
operation: &RuntimeOperation,
|
||||||
|
prepared_request: PreparedRequest,
|
||||||
|
) -> Result<AdapterResponse, RuntimeError> {
|
||||||
|
match &operation.target {
|
||||||
Target::Grpc(target) => {
|
Target::Grpc(target) => {
|
||||||
let request = GrpcRequest {
|
let request = GrpcRequest {
|
||||||
headers: merge_headers(
|
headers: merge_headers(
|
||||||
@@ -71,12 +110,12 @@ impl RuntimeExecutor {
|
|||||||
};
|
};
|
||||||
let response = self.grpc_adapter.execute(target, &request).await?;
|
let response = self.grpc_adapter.execute(target, &request).await?;
|
||||||
|
|
||||||
AdapterResponse {
|
Ok(AdapterResponse {
|
||||||
status_code: response.status_code,
|
status_code: response.status_code,
|
||||||
headers: response.headers,
|
headers: response.headers,
|
||||||
body: response.body.clone(),
|
body: response.body.clone(),
|
||||||
data: response.body,
|
data: response.body,
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
Target::Graphql(target) => {
|
Target::Graphql(target) => {
|
||||||
let request = GraphqlRequest {
|
let request = GraphqlRequest {
|
||||||
@@ -90,12 +129,12 @@ impl RuntimeExecutor {
|
|||||||
};
|
};
|
||||||
let response = self.graphql_adapter.execute(target, &request).await?;
|
let response = self.graphql_adapter.execute(target, &request).await?;
|
||||||
|
|
||||||
AdapterResponse {
|
Ok(AdapterResponse {
|
||||||
status_code: response.status_code,
|
status_code: response.status_code,
|
||||||
headers: response.headers,
|
headers: response.headers,
|
||||||
body: response.body,
|
body: response.body,
|
||||||
data: response.data,
|
data: response.data,
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
Target::Rest(target) => {
|
Target::Rest(target) => {
|
||||||
let request = RestRequest {
|
let request = RestRequest {
|
||||||
@@ -111,20 +150,14 @@ impl RuntimeExecutor {
|
|||||||
};
|
};
|
||||||
let response = self.rest_adapter.execute(target, &request).await?;
|
let response = self.rest_adapter.execute(target, &request).await?;
|
||||||
|
|
||||||
AdapterResponse {
|
Ok(AdapterResponse {
|
||||||
status_code: response.status_code,
|
status_code: response.status_code,
|
||||||
headers: response.headers,
|
headers: response.headers,
|
||||||
body: response.body,
|
body: response.body,
|
||||||
data: Value::Null,
|
data: Value::Null,
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
let finalized_output = finalize_output(operation, &adapter_response)?;
|
|
||||||
|
|
||||||
operation.output_schema.validate_shape(&finalized_output)?;
|
|
||||||
|
|
||||||
Ok(finalized_output)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -234,9 +267,10 @@ mod tests {
|
|||||||
use axum::{Json, Router, routing::post};
|
use axum::{Json, Router, routing::post};
|
||||||
use crank_adapter_grpc::test_support as grpc_test_support;
|
use crank_adapter_grpc::test_support as grpc_test_support;
|
||||||
use crank_core::{
|
use crank_core::{
|
||||||
DescriptorId, ExecutionConfig, GeneratedDraft, GeneratedDraftStatus, GraphqlOperationType,
|
AggregationMode, DescriptorId, ExecutionConfig, ExecutionMode, GeneratedDraft,
|
||||||
GraphqlTarget, GrpcTarget, HttpMethod, Operation, OperationId, OperationStatus, Protocol,
|
GeneratedDraftStatus, GraphqlOperationType, GraphqlTarget, GrpcTarget, HttpMethod,
|
||||||
RestTarget, Samples, Target, ToolDescription, ToolExample,
|
Operation, OperationId, OperationStatus, Protocol, RestTarget, Samples, StreamingConfig,
|
||||||
|
Target, ToolDescription, ToolExample, ToolFamilyConfig, TransportBehavior,
|
||||||
};
|
};
|
||||||
use crank_mapping::{MappingRule, MappingSet};
|
use crank_mapping::{MappingRule, MappingSet};
|
||||||
use crank_schema::{Schema, SchemaKind};
|
use crank_schema::{Schema, SchemaKind};
|
||||||
@@ -340,8 +374,81 @@ mod tests {
|
|||||||
assert!(matches!(error, RuntimeError::GraphqlAdapter(_)));
|
assert!(matches!(error, RuntimeError::GraphqlAdapter(_)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn executes_window_mode_with_raw_items() {
|
||||||
|
let base_url = spawn_runtime_server().await;
|
||||||
|
let executor = RuntimeExecutor::new();
|
||||||
|
let operation = test_window_operation(&base_url, AggregationMode::RawItems, None, None);
|
||||||
|
|
||||||
|
let result = executor
|
||||||
|
.execute_window(&operation, &json!({}))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.items.len(), 3);
|
||||||
|
assert_eq!(result.summary, Value::Null);
|
||||||
|
assert!(result.window_complete);
|
||||||
|
assert!(!result.truncated);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn executes_window_mode_with_summary_only() {
|
||||||
|
let base_url = spawn_runtime_server().await;
|
||||||
|
let executor = RuntimeExecutor::new();
|
||||||
|
let operation =
|
||||||
|
test_window_operation(&base_url, AggregationMode::SummaryOnly, Some(10), None);
|
||||||
|
|
||||||
|
let result = executor
|
||||||
|
.execute_window(&operation, &json!({}))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result.summary, json!({ "service": "billing", "errors": 1 }));
|
||||||
|
assert!(result.items.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn executes_window_mode_with_truncation_and_redaction() {
|
||||||
|
let base_url = spawn_runtime_server().await;
|
||||||
|
let executor = RuntimeExecutor::new();
|
||||||
|
let operation = test_window_operation(
|
||||||
|
&base_url,
|
||||||
|
AggregationMode::SummaryPlusSamples,
|
||||||
|
Some(2),
|
||||||
|
Some(120),
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = executor
|
||||||
|
.execute_window(&operation, &json!({}))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(result.truncated);
|
||||||
|
assert!(result.has_more);
|
||||||
|
assert_eq!(result.items.len(), 1);
|
||||||
|
assert_eq!(result.items[0]["secret"], json!("[REDACTED]"));
|
||||||
|
assert_eq!(result.items[0]["message"], json!("disk..."));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn propagates_timeout_in_window_mode() {
|
||||||
|
let base_url = spawn_runtime_server().await;
|
||||||
|
let executor = RuntimeExecutor::new();
|
||||||
|
let operation = test_slow_window_operation(&base_url);
|
||||||
|
|
||||||
|
let error = executor
|
||||||
|
.execute_window(&operation, &json!({}))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
assert!(matches!(error, RuntimeError::RestAdapter(_)));
|
||||||
|
}
|
||||||
|
|
||||||
async fn spawn_runtime_server() -> String {
|
async fn spawn_runtime_server() -> String {
|
||||||
let app = Router::new().route("/leads", post(create_lead));
|
let app = Router::new()
|
||||||
|
.route("/leads", post(create_lead))
|
||||||
|
.route("/events", post(events_window))
|
||||||
|
.route("/slow-events", post(slow_events_window));
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let address = listener.local_addr().unwrap();
|
let address = listener.local_addr().unwrap();
|
||||||
|
|
||||||
@@ -383,6 +490,27 @@ mod tests {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn events_window() -> (axum::http::StatusCode, Json<Value>) {
|
||||||
|
(
|
||||||
|
axum::http::StatusCode::OK,
|
||||||
|
Json(json!({
|
||||||
|
"summary": { "service": "billing", "errors": 1 },
|
||||||
|
"items": [
|
||||||
|
{ "message": "disk pressure detected", "secret": "token-a" },
|
||||||
|
{ "message": "error budget exhausted", "secret": "token-b" },
|
||||||
|
{ "message": "node restarted", "secret": "token-c" }
|
||||||
|
],
|
||||||
|
"cursor": "cursor_02",
|
||||||
|
"done": true
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn slow_events_window() -> (axum::http::StatusCode, Json<Value>) {
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||||
|
events_window().await
|
||||||
|
}
|
||||||
|
|
||||||
async fn graphql_handler(Json(payload): Json<Value>) -> Json<Value> {
|
async fn graphql_handler(Json(payload): Json<Value>) -> Json<Value> {
|
||||||
let email = payload
|
let email = payload
|
||||||
.get("variables")
|
.get("variables")
|
||||||
@@ -653,6 +781,107 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn test_window_operation(
|
||||||
|
base_url: &str,
|
||||||
|
aggregation_mode: AggregationMode,
|
||||||
|
max_items: Option<u32>,
|
||||||
|
max_bytes: Option<u32>,
|
||||||
|
) -> RuntimeOperation {
|
||||||
|
RuntimeOperation::from(Operation {
|
||||||
|
id: OperationId::new("op_window_runtime"),
|
||||||
|
name: "billing_log_window".to_owned(),
|
||||||
|
display_name: "Billing Log Window".to_owned(),
|
||||||
|
category: "ops".to_owned(),
|
||||||
|
protocol: Protocol::Rest,
|
||||||
|
status: OperationStatus::Published,
|
||||||
|
version: 1,
|
||||||
|
target: Target::Rest(RestTarget {
|
||||||
|
base_url: base_url.to_owned(),
|
||||||
|
method: HttpMethod::Post,
|
||||||
|
path_template: "/events".to_owned(),
|
||||||
|
static_headers: BTreeMap::new(),
|
||||||
|
}),
|
||||||
|
input_schema: Schema {
|
||||||
|
kind: SchemaKind::Object,
|
||||||
|
description: None,
|
||||||
|
required: true,
|
||||||
|
nullable: false,
|
||||||
|
default_value: None,
|
||||||
|
fields: BTreeMap::new(),
|
||||||
|
items: None,
|
||||||
|
enum_values: Vec::new(),
|
||||||
|
variants: Vec::new(),
|
||||||
|
},
|
||||||
|
output_schema: object_schema("ignored", SchemaKind::String),
|
||||||
|
input_mapping: MappingSet {
|
||||||
|
rules: vec![MappingRule {
|
||||||
|
source: "$.mcp".to_owned(),
|
||||||
|
target: "$.request.body".to_owned(),
|
||||||
|
required: false,
|
||||||
|
default_value: None,
|
||||||
|
transform: None,
|
||||||
|
condition: None,
|
||||||
|
notes: None,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
output_mapping: MappingSet { rules: Vec::new() },
|
||||||
|
execution_config: ExecutionConfig {
|
||||||
|
timeout_ms: 1_000,
|
||||||
|
retry_policy: None,
|
||||||
|
auth_profile_ref: None,
|
||||||
|
headers: BTreeMap::new(),
|
||||||
|
protocol_options: None,
|
||||||
|
streaming: Some(StreamingConfig {
|
||||||
|
mode: ExecutionMode::Window,
|
||||||
|
transport_behavior: TransportBehavior::ServerStream,
|
||||||
|
window_duration_ms: Some(3_000),
|
||||||
|
poll_interval_ms: None,
|
||||||
|
upstream_timeout_ms: Some(1_000),
|
||||||
|
idle_timeout_ms: None,
|
||||||
|
max_session_lifetime_ms: None,
|
||||||
|
max_items,
|
||||||
|
max_bytes,
|
||||||
|
aggregation_mode,
|
||||||
|
summary_path: Some("$.summary".to_owned()),
|
||||||
|
items_path: Some("$.items".to_owned()),
|
||||||
|
cursor_path: Some("$.cursor".to_owned()),
|
||||||
|
status_path: None,
|
||||||
|
done_path: Some("$.done".to_owned()),
|
||||||
|
redacted_paths: vec!["$.secret".to_owned()],
|
||||||
|
truncate_item_fields: true,
|
||||||
|
max_field_length: Some(4),
|
||||||
|
drop_duplicates: false,
|
||||||
|
sampling_rate: None,
|
||||||
|
tool_family: ToolFamilyConfig::default(),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
tool_description: ToolDescription {
|
||||||
|
title: "Billing Log Window".to_owned(),
|
||||||
|
description: "Collects bounded billing events.".to_owned(),
|
||||||
|
tags: vec!["logs".to_owned()],
|
||||||
|
examples: Vec::new(),
|
||||||
|
},
|
||||||
|
samples: None,
|
||||||
|
generated_draft: None,
|
||||||
|
config_export: None,
|
||||||
|
created_at: "2026-04-06T12:00:00Z".to_owned(),
|
||||||
|
updated_at: "2026-04-06T12:00:00Z".to_owned(),
|
||||||
|
published_at: Some("2026-04-06T12:00:00Z".to_owned()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn test_slow_window_operation(base_url: &str) -> RuntimeOperation {
|
||||||
|
let mut operation =
|
||||||
|
test_window_operation(base_url, AggregationMode::RawItems, Some(10), None);
|
||||||
|
|
||||||
|
if let Target::Rest(target) = &mut operation.target {
|
||||||
|
target.path_template = "/slow-events".to_owned();
|
||||||
|
}
|
||||||
|
|
||||||
|
operation.execution_config.timeout_ms = 10;
|
||||||
|
operation
|
||||||
|
}
|
||||||
|
|
||||||
fn object_schema(field_name: &str, kind: SchemaKind) -> Schema {
|
fn object_schema(field_name: &str, kind: SchemaKind) -> Schema {
|
||||||
Schema {
|
Schema {
|
||||||
kind: SchemaKind::Object,
|
kind: SchemaKind::Object,
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
|
mod aggregation;
|
||||||
mod error;
|
mod error;
|
||||||
mod executor;
|
mod executor;
|
||||||
mod model;
|
mod model;
|
||||||
|
mod redaction;
|
||||||
|
mod streaming;
|
||||||
|
|
||||||
pub use error::RuntimeError;
|
pub use error::RuntimeError;
|
||||||
pub use executor::RuntimeExecutor;
|
pub use executor::RuntimeExecutor;
|
||||||
pub use model::{AdapterResponse, PreparedRequest, RuntimeOperation};
|
pub use model::{AdapterResponse, PreparedRequest, RuntimeOperation};
|
||||||
|
pub use streaming::WindowExecutionResult;
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
pub fn redact_paths(value: &mut Value, paths: &[String]) {
|
||||||
|
for path in paths {
|
||||||
|
let segments = parse_path(path);
|
||||||
|
if segments.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
redact_path(value, &segments);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn truncate_item_fields(value: &mut Value, max_len: usize) {
|
||||||
|
match value {
|
||||||
|
Value::String(text) => {
|
||||||
|
if text != "[REDACTED]" && text.chars().count() > max_len {
|
||||||
|
let truncated = text.chars().take(max_len).collect::<String>();
|
||||||
|
*text = format!("{truncated}...");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Array(items) => {
|
||||||
|
for item in items {
|
||||||
|
truncate_item_fields(item, max_len);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Object(map) => {
|
||||||
|
for value in map.values_mut() {
|
||||||
|
truncate_item_fields(value, max_len);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Null | Value::Bool(_) | Value::Number(_) => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
enum PathSegment {
|
||||||
|
Field(String),
|
||||||
|
Wildcard,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_path(path: &str) -> Vec<PathSegment> {
|
||||||
|
let path = path.strip_prefix("$.").or_else(|| path.strip_prefix('$'));
|
||||||
|
let Some(path) = path else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
|
||||||
|
path.split('.')
|
||||||
|
.flat_map(|segment| {
|
||||||
|
if let Some(field) = segment.strip_suffix("[*]") {
|
||||||
|
vec![PathSegment::Field(field.to_owned()), PathSegment::Wildcard]
|
||||||
|
} else if segment == "*" {
|
||||||
|
vec![PathSegment::Wildcard]
|
||||||
|
} else {
|
||||||
|
vec![PathSegment::Field(segment.to_owned())]
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn redact_path(value: &mut Value, segments: &[PathSegment]) {
|
||||||
|
let Some((current, rest)) = segments.split_first() else {
|
||||||
|
*value = Value::String("[REDACTED]".to_owned());
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
match current {
|
||||||
|
PathSegment::Field(field) => {
|
||||||
|
if let Some(next) = value
|
||||||
|
.as_object_mut()
|
||||||
|
.and_then(|object| object.get_mut(field))
|
||||||
|
{
|
||||||
|
redact_path(next, rest);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
PathSegment::Wildcard => {
|
||||||
|
if let Some(items) = value.as_array_mut() {
|
||||||
|
for item in items {
|
||||||
|
redact_path(item, rest);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use super::{redact_paths, truncate_item_fields};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redacts_simple_and_wildcard_paths() {
|
||||||
|
let mut value = json!({
|
||||||
|
"summary": { "token": "secret" },
|
||||||
|
"items": [
|
||||||
|
{ "token": "a", "message": "keep" },
|
||||||
|
{ "token": "b", "message": "keep" }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
redact_paths(
|
||||||
|
&mut value,
|
||||||
|
&["$.summary.token".to_owned(), "$.items[*].token".to_owned()],
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(value["summary"]["token"], json!("[REDACTED]"));
|
||||||
|
assert_eq!(value["items"][0]["token"], json!("[REDACTED]"));
|
||||||
|
assert_eq!(value["items"][1]["token"], json!("[REDACTED]"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truncates_long_strings_recursively() {
|
||||||
|
let mut value = json!({
|
||||||
|
"message": "abcdefghijklmnop",
|
||||||
|
"nested": [{ "message": "qrstuvwxyz" }]
|
||||||
|
});
|
||||||
|
|
||||||
|
truncate_item_fields(&mut value, 4);
|
||||||
|
|
||||||
|
assert_eq!(value["message"], json!("abcd..."));
|
||||||
|
assert_eq!(value["nested"][0]["message"], json!("qrst..."));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub struct WindowExecutionResult {
|
||||||
|
pub summary: Value,
|
||||||
|
pub items: Vec<Value>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub cursor: Option<Value>,
|
||||||
|
pub window_complete: bool,
|
||||||
|
pub truncated: bool,
|
||||||
|
pub has_more: bool,
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user