diff --git a/crates/crank-runtime/tests/execution_certainty.rs b/crates/crank-runtime/tests/execution_certainty.rs new file mode 100644 index 0000000..9c8a4a6 --- /dev/null +++ b/crates/crank-runtime/tests/execution_certainty.rs @@ -0,0 +1,244 @@ +use std::collections::BTreeMap; + +use crank_core::{ + CorrelationContext, DispatchEvidence, ExecutionErrorCode, ExecutionFailure, HttpMethod, + OutcomeCertainty, Protocol, ProtocolAdapterError, RestTarget, Retryability, Target, + ToolDescription, +}; +use crank_mapping::MappingSet; +use crank_runtime::{RuntimeError, RuntimeOperation, normalize_runtime_error_for_operation}; +use crank_schema::{Schema, SchemaKind}; + +const MUTATING_METHODS: [HttpMethod; 4] = [ + HttpMethod::Post, + HttpMethod::Put, + HttpMethod::Patch, + HttpMethod::Delete, +]; +const READ_METHODS: [HttpMethod; 2] = [HttpMethod::Get, HttpMethod::Head]; +const UNCERTAIN_STATUSES: [u16; 4] = [408, 429, 500, 503]; + +#[test] +fn ambiguous_upstream_failures_are_unknown_only_for_dispatched_mutations() { + let mut failures = Vec::new(); + for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) { + for dispatch in [ + DispatchEvidence::MayHaveDispatched, + DispatchEvidence::NotDispatched, + ] { + for status in UNCERTAIN_STATUSES { + let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus { + status, + dispatch, + }); + if let Err(failure) = assert_normalization( + method, + &error, + is_ambiguous_mutation(method, dispatch), + Some(status), + format!("{method:?} status {status} {dispatch:?}"), + ) { + failures.push(failure); + } + } + + for adapter in [ + ProtocolAdapterError::ResponseTooLarge { dispatch }, + ProtocolAdapterError::InvalidResponse { dispatch }, + ] { + let error = RuntimeError::ProtocolAdapter(adapter); + if let Err(failure) = assert_normalization( + method, + &error, + is_ambiguous_mutation(method, dispatch), + None, + format!("{method:?} {error:?}"), + ) { + failures.push(failure); + } + } + } + } + assert!( + failures.is_empty(), + "unexpected normalization rows:\n{}", + failures.join("\n") + ); +} + +#[test] +fn nonambiguous_statuses_keep_their_existing_descriptor() { + let mut failures = Vec::new(); + for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) { + for status in [401, 404] { + let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus { + status, + dispatch: DispatchEvidence::MayHaveDispatched, + }); + if let Err(failure) = assert_normalization( + method, + &error, + false, + Some(status), + format!("{method:?} nonambiguous status {status}"), + ) { + failures.push(failure); + } + } + } + assert!( + failures.is_empty(), + "unexpected normalization rows:\n{}", + failures.join("\n") + ); +} + +#[test] +fn dispatch_uncertainty_allowlist_covers_uncertain_upstream_codes_and_drops_retry_delay() { + let uncertain_codes = [ + ExecutionErrorCode::UpstreamRateLimited, + ExecutionErrorCode::UpstreamServerError, + ExecutionErrorCode::UpstreamResponseTooLarge, + ExecutionErrorCode::UpstreamStatusError, + ]; + let mut failures = Vec::new(); + for code in uncertain_codes { + let failure = ExecutionFailure::new(code, CorrelationContext::generate()); + let failure = if matches!(code.retryability(), Retryability::AfterDelay) { + failure + .try_with_retry_after_ms(1_000) + .expect("retry delay is valid for the original descriptor") + } else { + failure + } + .with_dispatch_uncertainty(); + + if failure.error_code() != code + || failure.retryability() != Retryability::ManualReconcile + || failure.outcome_certainty() != OutcomeCertainty::OutcomeUnknown + || failure.retry_after_ms().is_some() + { + failures.push(format!( + "{code:?}: actual code={:?}, retryability={:?}, certainty={:?}, retry_after_ms={:?}", + failure.error_code(), + failure.retryability(), + failure.outcome_certainty(), + failure.retry_after_ms(), + )); + } + } + assert!( + failures.is_empty(), + "uncertain code allowlist failures:\n{}", + failures.join("\n") + ); +} + +fn assert_normalization( + method: HttpMethod, + error: &RuntimeError, + unknown: bool, + status: Option, + context: String, +) -> Result<(), String> { + let correlation = CorrelationContext::generate(); + let failure = normalize_runtime_error_for_operation(error, &correlation, &operation(method)); + let code = expected_code(error); + let expected = ExecutionFailure::new(code, correlation.clone()); + let expected_retryability = if unknown { + Retryability::ManualReconcile + } else { + expected.retryability() + }; + let expected_certainty = if unknown { + OutcomeCertainty::OutcomeUnknown + } else { + expected.outcome_certainty() + }; + + if failure.error_code() != code + || failure.retryability() != expected_retryability + || failure.outcome_certainty() != expected_certainty + || failure.upstream_status() != status + || failure.retry_after_ms().is_some() + { + return Err(format!( + "{context}: code={:?} (expected {code:?}), retryability={:?} (expected {expected_retryability:?}), certainty={:?} (expected {expected_certainty:?}), upstream_status={:?} (expected {status:?}), retry_after_ms={:?}", + failure.error_code(), + failure.retryability(), + failure.outcome_certainty(), + failure.upstream_status(), + failure.retry_after_ms(), + )); + } + Ok(()) +} + +fn expected_code(error: &RuntimeError) -> ExecutionErrorCode { + match error { + RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus { + status, .. + }) => match status { + 401 | 403 => ExecutionErrorCode::UpstreamAuthError, + 404 => ExecutionErrorCode::UpstreamNotFound, + 408 | 429 => ExecutionErrorCode::UpstreamRateLimited, + 500..=599 => ExecutionErrorCode::UpstreamServerError, + _ => ExecutionErrorCode::UpstreamStatusError, + }, + RuntimeError::ProtocolAdapter(ProtocolAdapterError::ResponseTooLarge { .. }) => { + ExecutionErrorCode::UpstreamResponseTooLarge + } + RuntimeError::ProtocolAdapter(ProtocolAdapterError::InvalidResponse { .. }) => { + ExecutionErrorCode::UpstreamStatusError + } + other => panic!("unexpected test fixture: {other:?}"), + } +} + +fn is_ambiguous_mutation(method: HttpMethod, dispatch: DispatchEvidence) -> bool { + MUTATING_METHODS.contains(&method) && dispatch == DispatchEvidence::MayHaveDispatched +} + +fn operation(method: HttpMethod) -> RuntimeOperation { + RuntimeOperation { + operation_id: crank_core::OperationId::new("certainty_test"), + operation_version: 1, + tool_name: "certainty_test".to_owned(), + protocol: Protocol::Rest, + target: Target::Rest(RestTarget { + base_url: "https://example.invalid".to_owned(), + method, + path_template: "/items".to_owned(), + static_headers: BTreeMap::new(), + request_contract: None, + response_contract: Vec::new(), + multipart_field_mime_types: BTreeMap::new(), + }), + input_schema: empty_schema(), + output_schema: empty_schema(), + input_mapping: MappingSet::default(), + output_mapping: MappingSet::default(), + execution_config: crank_core::ExecutionConfig::default(), + tool_description: ToolDescription { + title: "Certainty test".to_owned(), + description: "Execution certainty fixture".to_owned(), + tags: Vec::new(), + examples: Vec::new(), + }, + } +} + +fn empty_schema() -> Schema { + Schema { + kind: SchemaKind::Object, + format: None, + description: None, + required: true, + nullable: false, + default_value: None, + fields: BTreeMap::new(), + items: None, + enum_values: Vec::new(), + variants: Vec::new(), + } +}