//! Story 3.4 (FR-14) ATDD: progress and cancellation of one synchronous `tools/call`. //! //! Every case drives the real HTTP endpoint of the real application against a real (counted, slow) //! upstream and real published catalog/registry state; nothing here is a fake client of the server //! internals. The oracles are the pinned MCP shapes: `notifications/progress` (monotonic `progress`, //! the request's own `progressToken`, no invented `total`) and the terminal `tools/call` result. //! //! Cancellation is revision-correct: `2025-11-25` cancels with `notifications/cancelled`, and //! `2026-07-28` cancels by closing the request's own response stream. Both are proven to stop the //! work, to release the call, and to never produce a `completed` record. use std::{ sync::{ Arc, atomic::{AtomicUsize, Ordering}, }, time::Duration, }; use axum::{Json, Router, routing::post}; use crank_core::{InvocationStatus, OperationId, PlatformApiKeyScope}; use crank_registry::{ListInvocationLogsQuery, PostgresRegistry, PublishRequest}; use serde_json::{Value, json}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use tokio::net::TcpListener; use super::common::{ agent_mcp_url, build_test_app, capturing_subscriber, create_platform_api_key, initialize_session, post_jsonrpc, post_jsonrpc_with_headers, publish_agent_for_operation, spawn_mcp_server, test_operation, test_registry, test_workspace_id, }; const LEGACY: &str = "2025-11-25"; const MODERN: &str = "2026-07-28"; const META_PROTOCOL_VERSION: &str = "io.modelcontextprotocol/protocolVersion"; const META_CLIENT_CAPABILITIES: &str = "io.modelcontextprotocol/clientCapabilities"; const META_CLIENT_INFO: &str = "io.modelcontextprotocol/clientInfo"; const TOOL: &str = "crm_lifecycle_lead"; /// A call that is long enough for the client to observe real progress, short enough for a suite. const HOLD_MS: u64 = 250; /// A call that is still running when the client cancels it. const LONG_HOLD_MS: u64 = 1_500; /// The grace period after a cancellation. It must exceed [`LONG_HOLD_MS`]: a call that was *not* /// stopped completes its upstream request and records the outcome inside this window, so the checks /// after it prove that the cancellation stopped the call rather than only that it had not finished /// yet. const CANCEL_GRACE_MS: u64 = 2_500; /// The suite depends on that order: the constant must not silently drop back below the hold. const _: () = assert!(CANCEL_GRACE_MS > LONG_HOLD_MS); #[derive(Clone, Copy, Debug)] enum Revision { Legacy, Modern, } impl Revision { const fn label(self) -> &'static str { match self { Self::Legacy => LEGACY, Self::Modern => MODERN, } } } struct Upstream { base_url: String, started: Arc, completed: Arc, } /// A real upstream that records when a request starts and when it finishes. A cancelled call must /// increment `started` exactly once and never `completed`: the request really was dispatched (so the /// outcome is not `certain`-not-sent) and its response really was abandoned (so it is not retried). async fn spawn_counted_slow_upstream(hold_ms: u64) -> Upstream { let started = Arc::new(AtomicUsize::new(0)); let completed = Arc::new(AtomicUsize::new(0)); let handler_started = Arc::clone(&started); let handler_completed = Arc::clone(&completed); let app = Router::new().route( "/crm/lifecycle-leads", post(move |Json(payload): Json| { let started = Arc::clone(&handler_started); let completed = Arc::clone(&handler_completed); async move { started.fetch_add(1, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(hold_ms)).await; completed.fetch_add(1, Ordering::SeqCst); Json(json!({ "id": "lead_123", "email": payload["email"] })) } }), ); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); Upstream { base_url: format!("http://{address}"), started, completed, } } struct Fixture { client: reqwest::Client, mcp_url: String, api_key: String, other_api_key: String, registry: PostgresRegistry, operation_id: OperationId, upstream: Upstream, } async fn fixture(agent_slug: &str, hold_ms: u64, timeout_ms: u64) -> Fixture { let upstream = spawn_counted_slow_upstream(hold_ms).await; let registry = test_registry().await; let mut operation = test_operation(&upstream.base_url, TOOL); let crank_core::Target::Rest(target) = &mut operation.target; target.path_template = "/crm/lifecycle-leads".to_owned(); operation.execution_config.timeout_ms = timeout_ms; registry .create_operation(&test_workspace_id(), &operation, Some("alice")) .await .unwrap(); registry .publish_operation(PublishRequest { workspace_id: &test_workspace_id(), operation_id: &operation.id, version: 1, published_at: &OffsetDateTime::parse("2026-03-26T10:00:00Z", &Rfc3339).unwrap(), published_by: Some("alice"), }) .await .unwrap(); publish_agent_for_operation(®istry, &operation, agent_slug).await; let scopes = [PlatformApiKeyScope::Read, PlatformApiKeyScope::Write]; let api_key = create_platform_api_key(®istry, agent_slug, "mcp-lifecycle", &scopes).await; let other_api_key = create_platform_api_key(®istry, agent_slug, "mcp-lifecycle-other", &scopes).await; let base_url = spawn_mcp_server(build_test_app( registry.clone(), Duration::ZERO, Some("https://crank.example.com".to_owned()), )) .await; Fixture { client: reqwest::Client::new(), mcp_url: agent_mcp_url(&base_url, agent_slug), api_key, other_api_key, registry, operation_id: operation.id, upstream, } } /// One parsed answer: the HTTP status, the negotiated media type and every JSON-RPC frame in order. struct Reply { status: reqwest::StatusCode, content_type: String, frames: Vec, raw: String, } impl Reply { fn parse(status: reqwest::StatusCode, content_type: String, body: String) -> Self { let mut frames = Vec::new(); for block in body.split("\n\n") { for line in block.lines() { let Some(data) = line.strip_prefix("data:") else { continue; }; let data = data.trim_start(); if data.is_empty() { continue; } frames.push( serde_json::from_str(data) .unwrap_or_else(|error| panic!("every frame is JSON: {data:?} {error}")), ); } } if frames.is_empty() && !body.trim().is_empty() { frames.push( serde_json::from_str(body.trim()).unwrap_or_else(|error| { panic!("expected a JSON-RPC body, got {body:?}: {error}") }), ); } Self { status, content_type, frames, raw: body, } } fn assert_stream(&self) { assert_eq!(self.status, reqwest::StatusCode::OK); assert!( self.content_type.starts_with("text/event-stream"), "a progress answer must be the request's SSE stream, got {:?}: {}", self.content_type, self.raw ); } fn progress_frames(&self) -> Vec<&Value> { self.frames .iter() .filter(|frame| frame["method"] == json!("notifications/progress")) .collect() } fn terminal(&self) -> &Value { let terminal = self .frames .last() .unwrap_or_else(|| panic!("the stream must carry a terminal frame: {}", self.raw)); assert!( terminal.get("method").is_none(), "the terminal frame must be the JSON-RPC response, got {terminal}" ); terminal } } impl Fixture { fn call_params(&self, token: Option, revision: Revision) -> Value { let mut params = json!({ "name": TOOL, "arguments": { "email": "lifecycle@example.com" }, }); let mut meta = match revision { Revision::Modern => json!({ META_PROTOCOL_VERSION: MODERN, META_CLIENT_CAPABILITIES: {}, META_CLIENT_INFO: { "name": "crank-3.4-atdd", "version": "0.1.0" }, }), Revision::Legacy => json!({}), }; if let Some(token) = token { meta["progressToken"] = token; } if meta.as_object().is_some_and(|object| !object.is_empty()) { params["_meta"] = meta; } params } fn headers(&self, revision: Revision, session: Option<&str>) -> Vec<(&'static str, String)> { let mut headers = vec![("MCP-Protocol-Version", revision.label().to_owned())]; match revision { Revision::Legacy => { if let Some(session) = session { headers.push(("MCP-Session-Id", session.to_owned())); } } Revision::Modern => { headers.push(("Mcp-Method", "tools/call".to_owned())); headers.push(("Mcp-Name", TOOL.to_owned())); } } headers } fn request( &self, accept: &'static str, key: &str, headers: &[(&'static str, String)], payload: Value, ) -> reqwest::RequestBuilder { let mut request = self .client .post(&self.mcp_url) .header(reqwest::header::ACCEPT, accept) .header(reqwest::header::AUTHORIZATION, format!("Bearer {key}")); for (name, value) in headers { request = request.header(*name, value); } request.json(&payload) } /// One `tools/call` whose answer is negotiated as the request's own SSE stream. async fn call_stream( &self, revision: Revision, id: u64, params: Value, session: Option<&str>, request_id: &str, ) -> Reply { self.call_stream_with_key(revision, &self.api_key, id, params, session, request_id) .await } async fn call_stream_with_key( &self, revision: Revision, key: &str, id: u64, params: Value, session: Option<&str>, request_id: &str, ) -> Reply { self.send( "text/event-stream", key, revision, id, params, session, Some(request_id), ) .await } /// One `tools/call` whose answer is negotiated as a single JSON body. async fn call_json( &self, revision: Revision, id: u64, params: Value, session: Option<&str>, ) -> Reply { self.send( "application/json", &self.api_key, revision, id, params, session, None, ) .await } #[allow(clippy::too_many_arguments)] async fn send( &self, accept: &'static str, key: &str, revision: Revision, id: u64, params: Value, session: Option<&str>, request_id: Option<&str>, ) -> Reply { let headers = self.headers(revision, session); let payload = json!({ "jsonrpc": "2.0", "id": id, "method": "tools/call", "params": params }); let mut request = self.request(accept, key, &headers, payload); if let Some(request_id) = request_id { request = request.header("x-request-id", request_id); } let response = request.send().await.expect("the MCP endpoint answers"); let status = response.status(); let content_type = response .headers() .get(reqwest::header::CONTENT_TYPE) .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_owned(); let body = response.text().await.expect("the body is readable"); Reply::parse(status, content_type, body) } async fn cancel(&self, key: &str, session: &str, request_id: Value) -> reqwest::StatusCode { let response = self .request( "application/json", key, &[ ("MCP-Protocol-Version", LEGACY.to_owned()), ("MCP-Session-Id", session.to_owned()), ], json!({ "jsonrpc": "2.0", "method": "notifications/cancelled", "params": { "requestId": request_id, "reason": "user requested cancellation" } }), ) .send() .await .expect("the cancellation is acknowledged"); response.status() } /// Every Invocation History row of this operation and request id, oldest first. async fn history(&self, request_id: &str) -> Vec { self.registry .list_invocation_logs(ListInvocationLogsQuery { workspace_id: &test_workspace_id(), level: None, status: None, outcome_group: None, search_text: None, source: Some(crank_core::InvocationSource::AgentToolCall), operation_id: Some(&self.operation_id), agent_id: None, created_after: None, created_before: None, cursor_created_at: None, cursor_id: None, limit: 50, }) .await .unwrap() .into_iter() .map(|record| record.log) .filter(|log| log.request_id.as_deref() == Some(request_id)) .collect() } /// Wait for the counted upstream to be entered, so a cancellation happens while it really runs. async fn wait_until_dispatched(&self) { for _ in 0..200 { if self.upstream.started.load(Ordering::SeqCst) > 0 { return; } tokio::time::sleep(Duration::from_millis(5)).await; } panic!("the upstream was never dispatched"); } } fn progress_values(frames: &[&Value]) -> Vec { frames .iter() .map(|frame| { frame["params"]["progress"] .as_u64() .unwrap_or_else(|| panic!("progress must be a number, got {frame}")) }) .collect() } // --------------------------------------------------------------------------------------------- // AT-3.4-01: a negotiated stream carries bounded monotonic progress of the real execution, and a // missing token (or a missing carrier) never changes the execution result. // --------------------------------------------------------------------------------------------- #[tokio::test] async fn at_3_4_01_legacy_progress_stream_is_monotonic_and_ends_with_the_real_result() { let fixture = fixture("lifecycle-legacy-progress", HOLD_MS, 5_000).await; let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; let reply = fixture .call_stream( Revision::Legacy, 501, fixture.call_params(Some(json!("tok-legacy-progress")), Revision::Legacy), Some(&session), "req-legacy-progress", ) .await; reply.assert_stream(); let progress = reply.progress_frames(); assert!( !progress.is_empty(), "a token with a stream carrier must produce progress, got {}", reply.raw ); for frame in &progress { assert_eq!(frame["jsonrpc"], json!("2.0")); assert_eq!( frame["params"]["progressToken"], json!("tok-legacy-progress") ); assert!( frame["params"].get("total").is_none(), "this server knows no total and must not invent one: {frame}" ); assert!( frame["params"]["message"].is_string(), "every real step names itself: {frame}" ); } let values = progress_values(&progress); assert!( values.windows(2).all(|pair| pair[0] < pair[1]), "progress must be strictly monotonic, got {values:?}" ); // One event per real execution step this server actually crosses: resolved, authorized, // dispatched. Never a timer-driven percentage and never an unbounded stream. assert_eq!(values, vec![1, 2, 3], "got {values:?}"); let terminal = reply.terminal(); assert_eq!(terminal["id"], json!(501)); assert_eq!(terminal["error"], Value::Null); assert_eq!(terminal["result"]["isError"], json!(false)); assert_eq!( terminal["result"]["structuredContent"], json!({ "id": "lead_123" }) ); assert_eq!(fixture.upstream.started.load(Ordering::SeqCst), 1); assert_eq!(fixture.upstream.completed.load(Ordering::SeqCst), 1); } #[tokio::test] async fn at_3_4_01_modern_progress_stream_is_monotonic_and_ends_with_the_real_result() { let fixture = fixture("lifecycle-modern-progress", HOLD_MS, 5_000).await; let reply = fixture .call_stream( Revision::Modern, 502, fixture.call_params(Some(json!(77)), Revision::Modern), None, "req-modern-progress", ) .await; reply.assert_stream(); let progress = reply.progress_frames(); assert!(!progress.is_empty(), "got {}", reply.raw); for frame in &progress { assert_eq!(frame["params"]["progressToken"], json!(77)); } let values = progress_values(&progress); assert!( values.windows(2).all(|pair| pair[0] < pair[1]), "{values:?}" ); let terminal = reply.terminal(); assert_eq!(terminal["result"]["isError"], json!(false)); assert_eq!( terminal["result"]["structuredContent"], json!({ "id": "lead_123" }) ); } // --------------------------------------------------------------------------------------------- // AT-3.4-01 (TL correction): the official SDK sends the dual `Accept` JSON-first. A `tools/call` // that carries a validated token and offers `text/event-stream` in that same header must still be // answered on its own stream, wherever the media types are ordered. // --------------------------------------------------------------------------------------------- #[tokio::test] async fn at_3_4_01_json_first_dual_accept_keeps_progress_for_a_valid_token() { /// Exactly what the official MCP SDK sends when it can read a stream. const SDK_ACCEPT: &str = "application/json, text/event-stream"; let fixture = fixture("lifecycle-json-first", HOLD_MS, 5_000).await; let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; for revision in [Revision::Legacy, Revision::Modern] { let carrier = match revision { Revision::Legacy => Some(session.as_str()), Revision::Modern => None, }; let token = json!("tok-json-first"); let reply = fixture .send( SDK_ACCEPT, &fixture.api_key, revision, 550, fixture.call_params(Some(token.clone()), revision), carrier, Some("req-json-first"), ) .await; reply.assert_stream(); let progress = reply.progress_frames(); assert!( !progress.is_empty(), "{revision:?}: a valid token with an acceptable stream must produce progress, got {}", reply.raw ); for frame in &progress { assert_eq!(frame["params"]["progressToken"], token, "{frame}"); assert!( frame["params"].get("total").is_none(), "this server knows no total: {frame}" ); } let values = progress_values(&progress); assert!( values.windows(2).all(|pair| pair[0] < pair[1]), "{revision:?}: progress must be strictly monotonic, got {values:?}" ); assert_eq!(values, vec![1, 2, 3], "{revision:?}: got {values:?}"); let terminal = reply.terminal(); assert_eq!(terminal["id"], json!(550), "{revision:?}"); assert_eq!(terminal["result"]["isError"], json!(false), "{revision:?}"); assert_eq!( terminal["result"]["structuredContent"], json!({ "id": "lead_123" }), "{revision:?}" ); } // The correction is scoped to a call that can actually carry progress: the same JSON-first // header without a token, and with an unusable token, keeps the negotiated single JSON body. for token in [None, Some(json!("x".repeat(4096)))] { let reply = fixture .send( SDK_ACCEPT, &fixture.api_key, Revision::Legacy, 551, fixture.call_params(token.clone(), Revision::Legacy), Some(&session), None, ) .await; assert_eq!(reply.status, reqwest::StatusCode::OK); assert!( reply.content_type.starts_with("application/json"), "token {token:?} must keep the negotiated JSON body, got {:?}: {}", reply.content_type, reply.raw ); assert!(reply.progress_frames().is_empty(), "{token:?}"); assert_eq!( reply.terminal()["result"]["structuredContent"], json!({ "id": "lead_123" }), "token {token:?}" ); } } #[tokio::test] async fn at_3_4_01_no_token_and_no_carrier_keep_the_result_identical() { let fixture = fixture("lifecycle-no-token", HOLD_MS, 5_000).await; let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; // Without a token the very same call is one plain JSON body: no notification exists. let without_token = fixture .call_json( Revision::Legacy, 510, fixture.call_params(None, Revision::Legacy), Some(&session), ) .await; assert_eq!(without_token.status, reqwest::StatusCode::OK); assert!(without_token.progress_frames().is_empty()); let plain = without_token.terminal()["result"].clone(); assert_eq!(plain["isError"], json!(false)); // With a token but only the `application/json` carrier the result is the same and progress is // simply not sent (the receiver is never obligated to send it). let token_without_carrier = fixture .call_json( Revision::Legacy, 511, fixture.call_params(Some(json!("tok-no-carrier")), Revision::Legacy), Some(&session), ) .await; assert_eq!(token_without_carrier.status, reqwest::StatusCode::OK); assert!(token_without_carrier.progress_frames().is_empty()); assert_eq!(token_without_carrier.terminal()["result"], plain); // In the stream mode without a token the only frame is the terminal response itself. let stream_without_token = fixture .call_stream( Revision::Legacy, 512, fixture.call_params(None, Revision::Legacy), Some(&session), "req-no-token-stream", ) .await; assert!(stream_without_token.progress_frames().is_empty()); assert_eq!(stream_without_token.frames.len(), 1); assert_eq!(stream_without_token.terminal()["result"], plain); } /// The streamed answer keeps the Story 3.18 projection: a client that opted in to logging gets the /// same single closed `notifications/message` of this call before the terminal frame, next to the /// progress events of Story 3.4. #[tokio::test] async fn at_3_4_01_a_progress_stream_keeps_the_logging_projection() { let fixture = fixture("lifecycle-progress-logging", HOLD_MS, 5_000).await; let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; let set_level = fixture .request( "application/json", &fixture.api_key, &[ ("MCP-Protocol-Version", LEGACY.to_owned()), ("MCP-Session-Id", session.clone()), ], json!({ "jsonrpc": "2.0", "id": 900, "method": "logging/setLevel", "params": { "level": "debug" } }), ) .send() .await .expect("the level is accepted"); assert_eq!(set_level.status(), reqwest::StatusCode::OK); let reply = fixture .call_stream( Revision::Legacy, 901, fixture.call_params(Some(json!("tok-logging")), Revision::Legacy), Some(&session), "req-progress-logging", ) .await; reply.assert_stream(); assert!(!reply.progress_frames().is_empty(), "got {}", reply.raw); let logged: Vec<&Value> = reply .frames .iter() .filter(|frame| frame["method"] == json!("notifications/message")) .collect(); assert_eq!( logged.len(), 1, "exactly one closed log event per response, got {}", reply.raw ); assert_eq!( logged[0]["params"]["data"]["event"], json!("mcp.request.served") ); assert_eq!(logged[0]["params"]["data"]["outcome"], json!("ok")); assert_eq!(reply.terminal()["result"]["isError"], json!(false)); } // --------------------------------------------------------------------------------------------- // AT-3.4-02: a malformed, reused-in-another-scope or oversized token disables progress, and the // token never becomes a metric label, a log field or an authorization scope. // --------------------------------------------------------------------------------------------- #[tokio::test] async fn at_3_4_02_malformed_and_oversized_tokens_disable_progress_without_changing_the_result() { let fixture = fixture("lifecycle-bad-token", HOLD_MS, 5_000).await; let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; // Both supported revisions read the token from the same pinned `RequestMetaObject` field, so an // unusable token must behave identically in each of them: no progress, unchanged result. for revision in [Revision::Legacy, Revision::Modern] { let carrier = match revision { Revision::Legacy => Some(session.as_str()), Revision::Modern => None, }; let tokens = [ json!(""), json!({ "not": "a token" }), json!([1, 2]), json!(true), json!("x".repeat(4096)), ]; let mut observed = Vec::new(); for token in tokens { let reply = fixture .call_stream( revision, 520, fixture.call_params(Some(token.clone()), revision), carrier, "req-bad-token", ) .await; observed.push((token, reply)); } for (token, reply) in observed { assert!( reply.progress_frames().is_empty(), "{revision:?}: token {token} must not produce progress, got {}", reply.raw ); let terminal = reply.terminal(); assert_eq!( terminal["result"]["structuredContent"], json!({ "id": "lead_123" }), "{revision:?}: token {token} must not change the execution result, got {terminal}" ); assert_eq!(terminal["result"]["isError"], json!(false)); assert!(terminal["error"].is_null() || terminal.get("error").is_none()); } } } #[tokio::test] async fn at_3_4_02_a_reused_token_is_scoped_to_its_own_request() { let fixture = fixture("lifecycle-reused-token", HOLD_MS, 5_000).await; let first = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; let second = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; let token = json!("tok-shared"); let (first_reply, second_reply) = tokio::join!( fixture.call_stream( Revision::Legacy, 530, fixture.call_params(Some(token.clone()), Revision::Legacy), Some(&first), "req-first", ), fixture.call_stream( Revision::Legacy, 531, fixture.call_params(Some(token.clone()), Revision::Legacy), Some(&second), "req-second", ) ); for (reply, id) in [(&first_reply, 530), (&second_reply, 531)] { reply.assert_stream(); let values = progress_values(&reply.progress_frames()); // Each request counts its own real steps: a reused token never inherits another call's count. assert_eq!(values, vec![1, 2, 3], "request {id} got {values:?}"); assert_eq!(reply.terminal()["id"], json!(id)); } } #[tokio::test] async fn at_3_4_02_a_progress_token_never_reaches_logs_or_invocation_history() { const CANARY: &str = "crank-progress-canary-4f1c2d"; let fixture = fixture("lifecycle-token-leak", HOLD_MS, 5_000).await; let session = initialize_session(&fixture.client, &fixture.mcp_url, &fixture.api_key).await; let (writer, subscriber) = capturing_subscriber(); let reply = { let _guard = tracing::subscriber::set_default(subscriber); fixture .call_stream( Revision::Legacy, 540, fixture.call_params(Some(json!(CANARY)), Revision::Legacy), Some(&session), "req-token-leak", ) .await }; reply.assert_stream(); let logs = writer.output(); assert!( logs.contains("mcp request received"), "the capture must really see the endpoint's own log projection, got {logs}" ); assert!( !logs.contains(CANARY), "the progress token is client data and must never be logged, got {logs}" ); let history = fixture.history("req-token-leak").await; assert_eq!(history.len(), 1, "got {history:?}"); let row = &history[0]; let serialized = serde_json::to_string(row).unwrap(); assert!( !serialized.contains(CANARY), "the progress token must never reach Invocation History, got {serialized}" ); assert_eq!(row.status, InvocationStatus::Ok); } // AT-3.4-03: a cancellation stops a call that can still be stopped, releases it and never yields a // `completed` result or record. mod cancellation; // AT-3.4-05: the client-visible smoke of both supported revisions. mod smoke;