use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, }; use std::time::Duration; use axum::{Router, http::StatusCode, routing::post}; use crank_core::{ ExecutionErrorCode, InvocationSource, OutcomeCertainty, PlatformApiKeyScope, Retryability, WorkspaceId, }; use crank_registry::{ListInvocationLogsQuery, PublishRequest}; use serde_json::{Value, json}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::net::TcpListener; use super::common::{ agent_mcp_url, build_test_app, create_platform_api_key, initialize_session, post_jsonrpc_response, publish_agent_for_operation, spawn_mcp_server, test_operation, test_registry, test_workspace_id, }; const REQUEST_ID: &str = "req_mcp_execution_certainty"; #[tokio::test] async fn ambiguous_post_failure_is_reported_and_persisted_as_unknown() { let registry = test_registry().await; let upstream_calls = Arc::new(AtomicUsize::new(0)); let upstream_base_url = spawn_503_upstream(Arc::clone(&upstream_calls)).await; let operation = test_operation(&upstream_base_url, "mcp_execution_certainty"); registry .create_operation(&test_workspace_id(), &operation, Some("test")) .await .unwrap(); registry .publish_operation(PublishRequest { workspace_id: &test_workspace_id(), operation_id: &operation.id, version: 1, published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(), published_by: Some("test"), }) .await .unwrap(); publish_agent_for_operation(®istry, &operation, "mcp-certainty-agent").await; let api_key = create_platform_api_key( ®istry, "mcp-certainty-agent", "mcp-certainty-key", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let app = build_test_app(registry.clone(), Duration::ZERO, None); let base_url = spawn_mcp_server(app).await; let client = reqwest::Client::new(); let mcp_url = agent_mcp_url(&base_url, "mcp-certainty-agent"); let session_id = initialize_session(&client, &mcp_url, &api_key).await; let response = post_jsonrpc_response( &client, &mcp_url, &api_key, Some(&session_id), Some(REQUEST_ID), json!({ "jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": { "name": operation.name, "arguments": { "email": "user@example.com" } } }), ) .await; assert_eq!(response.status(), StatusCode::OK); let response: Value = response.json().await.unwrap(); let tool_result = &response["result"]; let error = &tool_result["structuredContent"]["error"]; assert_eq!(tool_result["isError"], true); assert_eq!(error["code"], "upstream_server_error"); assert_eq!(error["upstream_status"], 503); assert_eq!(error["retryability"], "manual_reconcile"); assert_eq!(error["outcome_certainty"], "outcome_unknown"); assert_eq!(error["recoverable"], false); assert_eq!(upstream_calls.load(Ordering::SeqCst), 1); let logs = registry .list_invocation_logs(ListInvocationLogsQuery { workspace_id: &WorkspaceId::new("ws_default"), level: None, status: None, outcome_group: None, search_text: None, source: Some(InvocationSource::AgentToolCall), operation_id: Some(&operation.id), agent_id: None, created_after: None, created_before: None, cursor_created_at: None, cursor_id: None, limit: 10, }) .await .unwrap(); let invocation = logs .iter() .find(|record| record.log.request_id.as_deref() == Some(REQUEST_ID)) .expect("Invocation History entry for this MCP request"); assert_eq!(invocation.log.tool_name, operation.name); assert_eq!(invocation.log.operation_version, Some(1)); assert_eq!(invocation.log.status_code, Some(503)); assert_eq!( invocation.log.execution_error_code, Some(ExecutionErrorCode::UpstreamServerError) ); assert_eq!( invocation.log.retryability, Some(Retryability::ManualReconcile) ); assert_eq!( invocation.log.outcome_certainty, Some(OutcomeCertainty::OutcomeUnknown) ); } async fn spawn_503_upstream(calls: Arc) -> String { let app = Router::new().route( "/crm/leads", post(move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") } }), ); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); format!("http://{address}") }