251 lines
8.9 KiB
Diff
251 lines
8.9 KiB
Diff
diff --git a/crates/crank-runtime/tests/execution_certainty.rs b/crates/crank-runtime/tests/execution_certainty.rs
|
|
new file mode 100644
|
|
index 0000000..9c8a4a6
|
|
--- /dev/null
|
|
+++ b/crates/crank-runtime/tests/execution_certainty.rs
|
|
@@ -0,0 +1,244 @@
|
|
+use std::collections::BTreeMap;
|
|
+
|
|
+use crank_core::{
|
|
+ CorrelationContext, DispatchEvidence, ExecutionErrorCode, ExecutionFailure, HttpMethod,
|
|
+ OutcomeCertainty, Protocol, ProtocolAdapterError, RestTarget, Retryability, Target,
|
|
+ ToolDescription,
|
|
+};
|
|
+use crank_mapping::MappingSet;
|
|
+use crank_runtime::{RuntimeError, RuntimeOperation, normalize_runtime_error_for_operation};
|
|
+use crank_schema::{Schema, SchemaKind};
|
|
+
|
|
+const MUTATING_METHODS: [HttpMethod; 4] = [
|
|
+ HttpMethod::Post,
|
|
+ HttpMethod::Put,
|
|
+ HttpMethod::Patch,
|
|
+ HttpMethod::Delete,
|
|
+];
|
|
+const READ_METHODS: [HttpMethod; 2] = [HttpMethod::Get, HttpMethod::Head];
|
|
+const UNCERTAIN_STATUSES: [u16; 4] = [408, 429, 500, 503];
|
|
+
|
|
+#[test]
|
|
+fn ambiguous_upstream_failures_are_unknown_only_for_dispatched_mutations() {
|
|
+ let mut failures = Vec::new();
|
|
+ for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
|
|
+ for dispatch in [
|
|
+ DispatchEvidence::MayHaveDispatched,
|
|
+ DispatchEvidence::NotDispatched,
|
|
+ ] {
|
|
+ for status in UNCERTAIN_STATUSES {
|
|
+ let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
|
+ status,
|
|
+ dispatch,
|
|
+ });
|
|
+ if let Err(failure) = assert_normalization(
|
|
+ method,
|
|
+ &error,
|
|
+ is_ambiguous_mutation(method, dispatch),
|
|
+ Some(status),
|
|
+ format!("{method:?} status {status} {dispatch:?}"),
|
|
+ ) {
|
|
+ failures.push(failure);
|
|
+ }
|
|
+ }
|
|
+
|
|
+ for adapter in [
|
|
+ ProtocolAdapterError::ResponseTooLarge { dispatch },
|
|
+ ProtocolAdapterError::InvalidResponse { dispatch },
|
|
+ ] {
|
|
+ let error = RuntimeError::ProtocolAdapter(adapter);
|
|
+ if let Err(failure) = assert_normalization(
|
|
+ method,
|
|
+ &error,
|
|
+ is_ambiguous_mutation(method, dispatch),
|
|
+ None,
|
|
+ format!("{method:?} {error:?}"),
|
|
+ ) {
|
|
+ failures.push(failure);
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ assert!(
|
|
+ failures.is_empty(),
|
|
+ "unexpected normalization rows:\n{}",
|
|
+ failures.join("\n")
|
|
+ );
|
|
+}
|
|
+
|
|
+#[test]
|
|
+fn nonambiguous_statuses_keep_their_existing_descriptor() {
|
|
+ let mut failures = Vec::new();
|
|
+ for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
|
|
+ for status in [401, 404] {
|
|
+ let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
|
+ status,
|
|
+ dispatch: DispatchEvidence::MayHaveDispatched,
|
|
+ });
|
|
+ if let Err(failure) = assert_normalization(
|
|
+ method,
|
|
+ &error,
|
|
+ false,
|
|
+ Some(status),
|
|
+ format!("{method:?} nonambiguous status {status}"),
|
|
+ ) {
|
|
+ failures.push(failure);
|
|
+ }
|
|
+ }
|
|
+ }
|
|
+ assert!(
|
|
+ failures.is_empty(),
|
|
+ "unexpected normalization rows:\n{}",
|
|
+ failures.join("\n")
|
|
+ );
|
|
+}
|
|
+
|
|
+#[test]
|
|
+fn dispatch_uncertainty_allowlist_covers_uncertain_upstream_codes_and_drops_retry_delay() {
|
|
+ let uncertain_codes = [
|
|
+ ExecutionErrorCode::UpstreamRateLimited,
|
|
+ ExecutionErrorCode::UpstreamServerError,
|
|
+ ExecutionErrorCode::UpstreamResponseTooLarge,
|
|
+ ExecutionErrorCode::UpstreamStatusError,
|
|
+ ];
|
|
+ let mut failures = Vec::new();
|
|
+ for code in uncertain_codes {
|
|
+ let failure = ExecutionFailure::new(code, CorrelationContext::generate());
|
|
+ let failure = if matches!(code.retryability(), Retryability::AfterDelay) {
|
|
+ failure
|
|
+ .try_with_retry_after_ms(1_000)
|
|
+ .expect("retry delay is valid for the original descriptor")
|
|
+ } else {
|
|
+ failure
|
|
+ }
|
|
+ .with_dispatch_uncertainty();
|
|
+
|
|
+ if failure.error_code() != code
|
|
+ || failure.retryability() != Retryability::ManualReconcile
|
|
+ || failure.outcome_certainty() != OutcomeCertainty::OutcomeUnknown
|
|
+ || failure.retry_after_ms().is_some()
|
|
+ {
|
|
+ failures.push(format!(
|
|
+ "{code:?}: actual code={:?}, retryability={:?}, certainty={:?}, retry_after_ms={:?}",
|
|
+ failure.error_code(),
|
|
+ failure.retryability(),
|
|
+ failure.outcome_certainty(),
|
|
+ failure.retry_after_ms(),
|
|
+ ));
|
|
+ }
|
|
+ }
|
|
+ assert!(
|
|
+ failures.is_empty(),
|
|
+ "uncertain code allowlist failures:\n{}",
|
|
+ failures.join("\n")
|
|
+ );
|
|
+}
|
|
+
|
|
+fn assert_normalization(
|
|
+ method: HttpMethod,
|
|
+ error: &RuntimeError,
|
|
+ unknown: bool,
|
|
+ status: Option<u16>,
|
|
+ context: String,
|
|
+) -> Result<(), String> {
|
|
+ let correlation = CorrelationContext::generate();
|
|
+ let failure = normalize_runtime_error_for_operation(error, &correlation, &operation(method));
|
|
+ let code = expected_code(error);
|
|
+ let expected = ExecutionFailure::new(code, correlation.clone());
|
|
+ let expected_retryability = if unknown {
|
|
+ Retryability::ManualReconcile
|
|
+ } else {
|
|
+ expected.retryability()
|
|
+ };
|
|
+ let expected_certainty = if unknown {
|
|
+ OutcomeCertainty::OutcomeUnknown
|
|
+ } else {
|
|
+ expected.outcome_certainty()
|
|
+ };
|
|
+
|
|
+ if failure.error_code() != code
|
|
+ || failure.retryability() != expected_retryability
|
|
+ || failure.outcome_certainty() != expected_certainty
|
|
+ || failure.upstream_status() != status
|
|
+ || failure.retry_after_ms().is_some()
|
|
+ {
|
|
+ return Err(format!(
|
|
+ "{context}: code={:?} (expected {code:?}), retryability={:?} (expected {expected_retryability:?}), certainty={:?} (expected {expected_certainty:?}), upstream_status={:?} (expected {status:?}), retry_after_ms={:?}",
|
|
+ failure.error_code(),
|
|
+ failure.retryability(),
|
|
+ failure.outcome_certainty(),
|
|
+ failure.upstream_status(),
|
|
+ failure.retry_after_ms(),
|
|
+ ));
|
|
+ }
|
|
+ Ok(())
|
|
+}
|
|
+
|
|
+fn expected_code(error: &RuntimeError) -> ExecutionErrorCode {
|
|
+ match error {
|
|
+ RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
|
|
+ status, ..
|
|
+ }) => match status {
|
|
+ 401 | 403 => ExecutionErrorCode::UpstreamAuthError,
|
|
+ 404 => ExecutionErrorCode::UpstreamNotFound,
|
|
+ 408 | 429 => ExecutionErrorCode::UpstreamRateLimited,
|
|
+ 500..=599 => ExecutionErrorCode::UpstreamServerError,
|
|
+ _ => ExecutionErrorCode::UpstreamStatusError,
|
|
+ },
|
|
+ RuntimeError::ProtocolAdapter(ProtocolAdapterError::ResponseTooLarge { .. }) => {
|
|
+ ExecutionErrorCode::UpstreamResponseTooLarge
|
|
+ }
|
|
+ RuntimeError::ProtocolAdapter(ProtocolAdapterError::InvalidResponse { .. }) => {
|
|
+ ExecutionErrorCode::UpstreamStatusError
|
|
+ }
|
|
+ other => panic!("unexpected test fixture: {other:?}"),
|
|
+ }
|
|
+}
|
|
+
|
|
+fn is_ambiguous_mutation(method: HttpMethod, dispatch: DispatchEvidence) -> bool {
|
|
+ MUTATING_METHODS.contains(&method) && dispatch == DispatchEvidence::MayHaveDispatched
|
|
+}
|
|
+
|
|
+fn operation(method: HttpMethod) -> RuntimeOperation {
|
|
+ RuntimeOperation {
|
|
+ operation_id: crank_core::OperationId::new("certainty_test"),
|
|
+ operation_version: 1,
|
|
+ tool_name: "certainty_test".to_owned(),
|
|
+ protocol: Protocol::Rest,
|
|
+ target: Target::Rest(RestTarget {
|
|
+ base_url: "https://example.invalid".to_owned(),
|
|
+ method,
|
|
+ path_template: "/items".to_owned(),
|
|
+ static_headers: BTreeMap::new(),
|
|
+ request_contract: None,
|
|
+ response_contract: Vec::new(),
|
|
+ multipart_field_mime_types: BTreeMap::new(),
|
|
+ }),
|
|
+ input_schema: empty_schema(),
|
|
+ output_schema: empty_schema(),
|
|
+ input_mapping: MappingSet::default(),
|
|
+ output_mapping: MappingSet::default(),
|
|
+ execution_config: crank_core::ExecutionConfig::default(),
|
|
+ tool_description: ToolDescription {
|
|
+ title: "Certainty test".to_owned(),
|
|
+ description: "Execution certainty fixture".to_owned(),
|
|
+ tags: Vec::new(),
|
|
+ examples: Vec::new(),
|
|
+ },
|
|
+ }
|
|
+}
|
|
+
|
|
+fn empty_schema() -> Schema {
|
|
+ Schema {
|
|
+ kind: SchemaKind::Object,
|
|
+ format: None,
|
|
+ description: None,
|
|
+ required: true,
|
|
+ nullable: false,
|
|
+ default_value: None,
|
|
+ fields: BTreeMap::new(),
|
|
+ items: None,
|
|
+ enum_values: Vec::new(),
|
|
+ variants: Vec::new(),
|
|
+ }
|
|
+}
|