Files
crank/crates/crank-runtime/tests/execution_certainty.rs
T

245 lines
8.5 KiB
Rust

use std::collections::BTreeMap;
use crank_core::{
CorrelationContext, DispatchEvidence, ExecutionErrorCode, ExecutionFailure, HttpMethod,
OutcomeCertainty, Protocol, ProtocolAdapterError, RestTarget, Retryability, Target,
ToolDescription,
};
use crank_mapping::MappingSet;
use crank_runtime::{RuntimeError, RuntimeOperation, normalize_runtime_error_for_operation};
use crank_schema::{Schema, SchemaKind};
const MUTATING_METHODS: [HttpMethod; 4] = [
HttpMethod::Post,
HttpMethod::Put,
HttpMethod::Patch,
HttpMethod::Delete,
];
const READ_METHODS: [HttpMethod; 2] = [HttpMethod::Get, HttpMethod::Head];
const UNCERTAIN_STATUSES: [u16; 4] = [408, 429, 500, 503];
#[test]
fn ambiguous_upstream_failures_are_unknown_only_for_dispatched_mutations() {
let mut failures = Vec::new();
for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
for dispatch in [
DispatchEvidence::MayHaveDispatched,
DispatchEvidence::NotDispatched,
] {
for status in UNCERTAIN_STATUSES {
let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
status,
dispatch,
});
if let Err(failure) = assert_normalization(
method,
&error,
is_ambiguous_mutation(method, dispatch),
Some(status),
format!("{method:?} status {status} {dispatch:?}"),
) {
failures.push(failure);
}
}
for adapter in [
ProtocolAdapterError::ResponseTooLarge { dispatch },
ProtocolAdapterError::InvalidResponse { dispatch },
] {
let error = RuntimeError::ProtocolAdapter(adapter);
if let Err(failure) = assert_normalization(
method,
&error,
is_ambiguous_mutation(method, dispatch),
None,
format!("{method:?} {error:?}"),
) {
failures.push(failure);
}
}
}
}
assert!(
failures.is_empty(),
"unexpected normalization rows:\n{}",
failures.join("\n")
);
}
#[test]
fn nonambiguous_statuses_keep_their_existing_descriptor() {
let mut failures = Vec::new();
for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
for status in [401, 404] {
let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
status,
dispatch: DispatchEvidence::MayHaveDispatched,
});
if let Err(failure) = assert_normalization(
method,
&error,
false,
Some(status),
format!("{method:?} nonambiguous status {status}"),
) {
failures.push(failure);
}
}
}
assert!(
failures.is_empty(),
"unexpected normalization rows:\n{}",
failures.join("\n")
);
}
#[test]
fn dispatch_uncertainty_allowlist_covers_uncertain_upstream_codes_and_drops_retry_delay() {
let uncertain_codes = [
ExecutionErrorCode::UpstreamRateLimited,
ExecutionErrorCode::UpstreamServerError,
ExecutionErrorCode::UpstreamResponseTooLarge,
ExecutionErrorCode::UpstreamStatusError,
];
let mut failures = Vec::new();
for code in uncertain_codes {
let failure = ExecutionFailure::new(code, CorrelationContext::generate());
let failure = if matches!(code.retryability(), Retryability::AfterDelay) {
failure
.try_with_retry_after_ms(1_000)
.expect("retry delay is valid for the original descriptor")
} else {
failure
}
.with_dispatch_uncertainty();
if failure.error_code() != code
|| failure.retryability() != Retryability::ManualReconcile
|| failure.outcome_certainty() != OutcomeCertainty::OutcomeUnknown
|| failure.retry_after_ms().is_some()
{
failures.push(format!(
"{code:?}: actual code={:?}, retryability={:?}, certainty={:?}, retry_after_ms={:?}",
failure.error_code(),
failure.retryability(),
failure.outcome_certainty(),
failure.retry_after_ms(),
));
}
}
assert!(
failures.is_empty(),
"uncertain code allowlist failures:\n{}",
failures.join("\n")
);
}
fn assert_normalization(
method: HttpMethod,
error: &RuntimeError,
unknown: bool,
status: Option<u16>,
context: String,
) -> Result<(), String> {
let correlation = CorrelationContext::generate();
let failure = normalize_runtime_error_for_operation(error, &correlation, &operation(method));
let code = expected_code(error);
let expected = ExecutionFailure::new(code, correlation.clone());
let expected_retryability = if unknown {
Retryability::ManualReconcile
} else {
expected.retryability()
};
let expected_certainty = if unknown {
OutcomeCertainty::OutcomeUnknown
} else {
expected.outcome_certainty()
};
if failure.error_code() != code
|| failure.retryability() != expected_retryability
|| failure.outcome_certainty() != expected_certainty
|| failure.upstream_status() != status
|| failure.retry_after_ms().is_some()
{
return Err(format!(
"{context}: code={:?} (expected {code:?}), retryability={:?} (expected {expected_retryability:?}), certainty={:?} (expected {expected_certainty:?}), upstream_status={:?} (expected {status:?}), retry_after_ms={:?}",
failure.error_code(),
failure.retryability(),
failure.outcome_certainty(),
failure.upstream_status(),
failure.retry_after_ms(),
));
}
Ok(())
}
fn expected_code(error: &RuntimeError) -> ExecutionErrorCode {
match error {
RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
status, ..
}) => match status {
401 | 403 => ExecutionErrorCode::UpstreamAuthError,
404 => ExecutionErrorCode::UpstreamNotFound,
408 | 429 => ExecutionErrorCode::UpstreamRateLimited,
500..=599 => ExecutionErrorCode::UpstreamServerError,
_ => ExecutionErrorCode::UpstreamStatusError,
},
RuntimeError::ProtocolAdapter(ProtocolAdapterError::ResponseTooLarge { .. }) => {
ExecutionErrorCode::UpstreamResponseTooLarge
}
RuntimeError::ProtocolAdapter(ProtocolAdapterError::InvalidResponse { .. }) => {
ExecutionErrorCode::UpstreamStatusError
}
other => panic!("unexpected test fixture: {other:?}"),
}
}
fn is_ambiguous_mutation(method: HttpMethod, dispatch: DispatchEvidence) -> bool {
MUTATING_METHODS.contains(&method) && dispatch == DispatchEvidence::MayHaveDispatched
}
fn operation(method: HttpMethod) -> RuntimeOperation {
RuntimeOperation {
operation_id: crank_core::OperationId::new("certainty_test"),
operation_version: 1,
tool_name: "certainty_test".to_owned(),
protocol: Protocol::Rest,
target: Target::Rest(RestTarget {
base_url: "https://example.invalid".to_owned(),
method,
path_template: "/items".to_owned(),
static_headers: BTreeMap::new(),
request_contract: None,
response_contract: Vec::new(),
multipart_field_mime_types: BTreeMap::new(),
}),
input_schema: empty_schema(),
output_schema: empty_schema(),
input_mapping: MappingSet::default(),
output_mapping: MappingSet::default(),
execution_config: crank_core::ExecutionConfig::default(),
tool_description: ToolDescription {
title: "Certainty test".to_owned(),
description: "Execution certainty fixture".to_owned(),
tags: Vec::new(),
examples: Vec::new(),
},
}
}
fn empty_schema() -> Schema {
Schema {
kind: SchemaKind::Object,
format: None,
description: None,
required: true,
nullable: false,
default_value: None,
fields: BTreeMap::new(),
items: None,
enum_values: Vec::new(),
variants: Vec::new(),
}
}