Files
crank/docs/epic-1-completion/evidence/runtime-red.patch
T

251 lines
8.9 KiB
Diff

diff --git a/crates/crank-runtime/tests/execution_certainty.rs b/crates/crank-runtime/tests/execution_certainty.rs
new file mode 100644
index 0000000..9c8a4a6
--- /dev/null
+++ b/crates/crank-runtime/tests/execution_certainty.rs
@@ -0,0 +1,244 @@
+use std::collections::BTreeMap;
+
+use crank_core::{
+ CorrelationContext, DispatchEvidence, ExecutionErrorCode, ExecutionFailure, HttpMethod,
+ OutcomeCertainty, Protocol, ProtocolAdapterError, RestTarget, Retryability, Target,
+ ToolDescription,
+};
+use crank_mapping::MappingSet;
+use crank_runtime::{RuntimeError, RuntimeOperation, normalize_runtime_error_for_operation};
+use crank_schema::{Schema, SchemaKind};
+
+const MUTATING_METHODS: [HttpMethod; 4] = [
+ HttpMethod::Post,
+ HttpMethod::Put,
+ HttpMethod::Patch,
+ HttpMethod::Delete,
+];
+const READ_METHODS: [HttpMethod; 2] = [HttpMethod::Get, HttpMethod::Head];
+const UNCERTAIN_STATUSES: [u16; 4] = [408, 429, 500, 503];
+
+#[test]
+fn ambiguous_upstream_failures_are_unknown_only_for_dispatched_mutations() {
+ let mut failures = Vec::new();
+ for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
+ for dispatch in [
+ DispatchEvidence::MayHaveDispatched,
+ DispatchEvidence::NotDispatched,
+ ] {
+ for status in UNCERTAIN_STATUSES {
+ let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
+ status,
+ dispatch,
+ });
+ if let Err(failure) = assert_normalization(
+ method,
+ &error,
+ is_ambiguous_mutation(method, dispatch),
+ Some(status),
+ format!("{method:?} status {status} {dispatch:?}"),
+ ) {
+ failures.push(failure);
+ }
+ }
+
+ for adapter in [
+ ProtocolAdapterError::ResponseTooLarge { dispatch },
+ ProtocolAdapterError::InvalidResponse { dispatch },
+ ] {
+ let error = RuntimeError::ProtocolAdapter(adapter);
+ if let Err(failure) = assert_normalization(
+ method,
+ &error,
+ is_ambiguous_mutation(method, dispatch),
+ None,
+ format!("{method:?} {error:?}"),
+ ) {
+ failures.push(failure);
+ }
+ }
+ }
+ }
+ assert!(
+ failures.is_empty(),
+ "unexpected normalization rows:\n{}",
+ failures.join("\n")
+ );
+}
+
+#[test]
+fn nonambiguous_statuses_keep_their_existing_descriptor() {
+ let mut failures = Vec::new();
+ for method in MUTATING_METHODS.into_iter().chain(READ_METHODS) {
+ for status in [401, 404] {
+ let error = RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
+ status,
+ dispatch: DispatchEvidence::MayHaveDispatched,
+ });
+ if let Err(failure) = assert_normalization(
+ method,
+ &error,
+ false,
+ Some(status),
+ format!("{method:?} nonambiguous status {status}"),
+ ) {
+ failures.push(failure);
+ }
+ }
+ }
+ assert!(
+ failures.is_empty(),
+ "unexpected normalization rows:\n{}",
+ failures.join("\n")
+ );
+}
+
+#[test]
+fn dispatch_uncertainty_allowlist_covers_uncertain_upstream_codes_and_drops_retry_delay() {
+ let uncertain_codes = [
+ ExecutionErrorCode::UpstreamRateLimited,
+ ExecutionErrorCode::UpstreamServerError,
+ ExecutionErrorCode::UpstreamResponseTooLarge,
+ ExecutionErrorCode::UpstreamStatusError,
+ ];
+ let mut failures = Vec::new();
+ for code in uncertain_codes {
+ let failure = ExecutionFailure::new(code, CorrelationContext::generate());
+ let failure = if matches!(code.retryability(), Retryability::AfterDelay) {
+ failure
+ .try_with_retry_after_ms(1_000)
+ .expect("retry delay is valid for the original descriptor")
+ } else {
+ failure
+ }
+ .with_dispatch_uncertainty();
+
+ if failure.error_code() != code
+ || failure.retryability() != Retryability::ManualReconcile
+ || failure.outcome_certainty() != OutcomeCertainty::OutcomeUnknown
+ || failure.retry_after_ms().is_some()
+ {
+ failures.push(format!(
+ "{code:?}: actual code={:?}, retryability={:?}, certainty={:?}, retry_after_ms={:?}",
+ failure.error_code(),
+ failure.retryability(),
+ failure.outcome_certainty(),
+ failure.retry_after_ms(),
+ ));
+ }
+ }
+ assert!(
+ failures.is_empty(),
+ "uncertain code allowlist failures:\n{}",
+ failures.join("\n")
+ );
+}
+
+fn assert_normalization(
+ method: HttpMethod,
+ error: &RuntimeError,
+ unknown: bool,
+ status: Option<u16>,
+ context: String,
+) -> Result<(), String> {
+ let correlation = CorrelationContext::generate();
+ let failure = normalize_runtime_error_for_operation(error, &correlation, &operation(method));
+ let code = expected_code(error);
+ let expected = ExecutionFailure::new(code, correlation.clone());
+ let expected_retryability = if unknown {
+ Retryability::ManualReconcile
+ } else {
+ expected.retryability()
+ };
+ let expected_certainty = if unknown {
+ OutcomeCertainty::OutcomeUnknown
+ } else {
+ expected.outcome_certainty()
+ };
+
+ if failure.error_code() != code
+ || failure.retryability() != expected_retryability
+ || failure.outcome_certainty() != expected_certainty
+ || failure.upstream_status() != status
+ || failure.retry_after_ms().is_some()
+ {
+ return Err(format!(
+ "{context}: code={:?} (expected {code:?}), retryability={:?} (expected {expected_retryability:?}), certainty={:?} (expected {expected_certainty:?}), upstream_status={:?} (expected {status:?}), retry_after_ms={:?}",
+ failure.error_code(),
+ failure.retryability(),
+ failure.outcome_certainty(),
+ failure.upstream_status(),
+ failure.retry_after_ms(),
+ ));
+ }
+ Ok(())
+}
+
+fn expected_code(error: &RuntimeError) -> ExecutionErrorCode {
+ match error {
+ RuntimeError::ProtocolAdapter(ProtocolAdapterError::UnexpectedStatus {
+ status, ..
+ }) => match status {
+ 401 | 403 => ExecutionErrorCode::UpstreamAuthError,
+ 404 => ExecutionErrorCode::UpstreamNotFound,
+ 408 | 429 => ExecutionErrorCode::UpstreamRateLimited,
+ 500..=599 => ExecutionErrorCode::UpstreamServerError,
+ _ => ExecutionErrorCode::UpstreamStatusError,
+ },
+ RuntimeError::ProtocolAdapter(ProtocolAdapterError::ResponseTooLarge { .. }) => {
+ ExecutionErrorCode::UpstreamResponseTooLarge
+ }
+ RuntimeError::ProtocolAdapter(ProtocolAdapterError::InvalidResponse { .. }) => {
+ ExecutionErrorCode::UpstreamStatusError
+ }
+ other => panic!("unexpected test fixture: {other:?}"),
+ }
+}
+
+fn is_ambiguous_mutation(method: HttpMethod, dispatch: DispatchEvidence) -> bool {
+ MUTATING_METHODS.contains(&method) && dispatch == DispatchEvidence::MayHaveDispatched
+}
+
+fn operation(method: HttpMethod) -> RuntimeOperation {
+ RuntimeOperation {
+ operation_id: crank_core::OperationId::new("certainty_test"),
+ operation_version: 1,
+ tool_name: "certainty_test".to_owned(),
+ protocol: Protocol::Rest,
+ target: Target::Rest(RestTarget {
+ base_url: "https://example.invalid".to_owned(),
+ method,
+ path_template: "/items".to_owned(),
+ static_headers: BTreeMap::new(),
+ request_contract: None,
+ response_contract: Vec::new(),
+ multipart_field_mime_types: BTreeMap::new(),
+ }),
+ input_schema: empty_schema(),
+ output_schema: empty_schema(),
+ input_mapping: MappingSet::default(),
+ output_mapping: MappingSet::default(),
+ execution_config: crank_core::ExecutionConfig::default(),
+ tool_description: ToolDescription {
+ title: "Certainty test".to_owned(),
+ description: "Execution certainty fixture".to_owned(),
+ tags: Vec::new(),
+ examples: Vec::new(),
+ },
+ }
+}
+
+fn empty_schema() -> Schema {
+ Schema {
+ kind: SchemaKind::Object,
+ format: None,
+ description: None,
+ required: true,
+ nullable: false,
+ default_value: None,
+ fields: BTreeMap::new(),
+ items: None,
+ enum_values: Vec::new(),
+ variants: Vec::new(),
+ }
+}