//! ATDD Story 4.5 (Beam): durable input continuation through TaskApplication -> PG -> worker -> upstream. use super::common; use super::unsampled_correlation; use axum::Router; use axum::http::StatusCode; use axum::routing::post; use crank_community_mcp::{TaskApplication, TaskWorkerHandler}; use crank_core::{ NoTasksFallback, PlatformApiKeyScope, TaskId, TaskState, ToolExecutionMode, ToolExecutionPolicy, }; use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest}; use crank_runtime::{ InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig, }; use serde_json::json; use sqlx::Row; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use time::OffsetDateTime; const AGENT_SLUG: &str = "reports"; pub(crate) const TOOL_NAME: &str = "report_gen"; const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000"; pub(crate) async fn state_of(registry: &PostgresRegistry, task_id: TaskId) -> Option { sqlx::query_scalar("select state from tasks where task_id = $1") .bind(task_id.as_uuid()) .fetch_optional(registry.pool()) .await .unwrap() } pub(crate) async fn wait_for_state( registry: &PostgresRegistry, task_id: TaskId, desired: &str, limit: Duration, ) -> bool { let deadline = tokio::time::Instant::now() + limit; while tokio::time::Instant::now() < deadline { if state_of(registry, task_id).await.as_deref() == Some(desired) { return true; } tokio::time::sleep(Duration::from_millis(50)).await; } state_of(registry, task_id).await.as_deref() == Some(desired) } pub(crate) async fn seed_tool( registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy, ) { let mut operation = common::test_operation(upstream, TOOL_NAME); operation.execution_config.execution_policy = Some(policy); registry .create_operation(&common::test_workspace_id(), &operation, Some("alice")) .await .unwrap(); registry .publish_operation(PublishRequest { workspace_id: &common::test_workspace_id(), operation_id: &operation.id, version: 1, published_at: &OffsetDateTime::parse( "2026-03-26T10:00:00Z", &time::format_description::well_known::Rfc3339, ) .unwrap(), published_by: Some("alice"), }) .await .unwrap(); common::publish_agent_with_bindings( registry, AGENT_SLUG, vec![crank_core::AgentOperationBinding { operation_id: operation.id.clone(), operation_version: 1, tool_name: operation.name.clone(), tool_title: operation.tool_description.title.clone(), tool_description_override: None, enabled: true, }], ) .await; } pub(crate) async fn principal( registry: &PostgresRegistry, name: &str, ) -> crank_community_mcp::TaskPrincipal { let token = common::create_platform_api_key( registry, AGENT_SLUG, name, &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let secret_hash = common::hash_access_secret(&token); let api_key = registry .get_platform_api_key_by_secret_for_agent_slug( common::test_workspace_slug(), AGENT_SLUG, &secret_hash, ) .await .unwrap() .expect("active key"); registry .touch_platform_api_key( &api_key.api_key.workspace_id, &api_key.api_key.id, &OffsetDateTime::now_utc(), ) .await .unwrap(); crank_community_mcp::TaskPrincipal::Agent { workspace_slug: common::test_workspace_slug().to_owned(), agent_slug: AGENT_SLUG.to_owned(), credential: crank_core::VerifiedMachineCredential { machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey, max_security_level: crank_core::OperationSecurityLevel::Standard, scopes: api_key.api_key.scopes.clone(), platform_api_key_id: Some(api_key.api_key.id.clone()), }, } } pub(crate) async fn application(registry: &PostgresRegistry) -> TaskApplication { let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); let observed_at = OffsetDateTime::now_utc(); registry .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { epoch: crypto.master_key_epoch(), fingerprint: crypto.master_key_fingerprint(), cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, observed_at: &observed_at, }) .await .expect("master key"); let store = Arc::new( PostgresTaskStore::new( registry.pool().clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("store"), ); TaskApplication::new( registry.clone(), Duration::ZERO, Arc::new(InMemoryCoordinationStateStore::default()), TaskService::with_secret_crypto(store, Arc::new(crypto)), ) } pub(crate) fn spawn_worker( registry: &PostgresRegistry, app: &TaskApplication, ) -> ( Arc, tokio::task::JoinHandle<()>, ) { let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap(); let pool = registry.pool().clone(); let store = Arc::new( PostgresTaskStore::new( pool, AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("store"), ); let runtime = crank_runtime::community_with_outbound_policy( crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), ) .build(); let handler = Arc::new(TaskWorkerHandler::from_application( app, registry.clone(), runtime, crypto, )); let worker = TaskWorker::new( store, handler, TaskWorkerConfig { owner: "worker-input".to_owned(), batch_limit: 1, lease_ms: 10_000, poll_interval: Duration::from_millis(50), max_inflight: 1, renewal_interval: Duration::from_millis(100), shutdown_timeout: Duration::from_secs(5), }, ) .expect("worker"); let stop_flag = worker.stop_flag(); let jh = tokio::spawn(async move { let _ = worker.run().await; }); (stop_flag, jh) } #[tokio::test] async fn input_continuation_happy_path_accept_update_resume_replay() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::OK, axum::Json(json!({"id":"lead"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let upstream = format!("http://{address}"); let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, &upstream, policy).await; let caller = principal(®istry, "input-happy").await; let app = application(®istry).await; let acceptance = app .accept_background( &caller, TOOL_NAME, None, "input-key-1", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("acceptance"); assert_eq!(acceptance.state, TaskState::Queued); let task_id = acceptance.task_id; let (stop, worker_task) = spawn_worker(®istry, &app); assert!( wait_for_state( ®istry, task_id, "input_required", Duration::from_secs(10) ) .await, "worker must suspend for missing email" ); // positive control: before update no outbound request assert_eq!(calls.load(Ordering::SeqCst), 0); // persisted request fields let row = sqlx::query( "select wait_id, expected_version, schema_snapshot, projected_schema, prompt_code, expires_at from task_input_requests where task_id = $1", ) .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); let wait_id: String = row.get("wait_id"); let expected_version: i32 = row.get("expected_version"); let schema_snapshot: serde_json::Value = row.get("schema_snapshot"); let projected_schema: serde_json::Value = row.get("projected_schema"); let prompt_code: String = row.get("prompt_code"); let _expires_at: OffsetDateTime = row.get("expires_at"); assert!(!wait_id.is_empty()); assert!(expected_version > 0); assert!(schema_snapshot.is_object()); assert!(projected_schema.is_object()); assert_eq!(prompt_code, "required_input_missing"); let audit_count_before: i64 = sqlx::query_scalar( "select count(*)::bigint from task_input_audit where task_id = $1 and event = 'input_required'", ) .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert!(audit_count_before >= 1); // stop worker before update to avoid races stop.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; let outcome = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version as u32, "update-key-1", &json!({"email": "user@example.com"}), ) .await .expect("valid update"); assert_eq!(outcome.state, TaskState::Working); assert!(!outcome.replayed); assert_eq!(outcome.task_id, task_id); assert_eq!(outcome.wait_id, wait_id); let (stop2, worker_task2) = spawn_worker(®istry, &app); assert!( wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await, "worker must complete after update" ); stop2.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await; assert_eq!(calls.load(Ordering::SeqCst), 1); let attempts = app .attempts(&caller, TOOL_NAME, task_id) .await .expect("attempts"); assert_eq!(attempts.len(), 1); let effective_fp: String = sqlx::query_scalar("select effective_input_fingerprint from tasks where task_id = $1") .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!(attempts[0].input_fingerprint, effective_fp); let input_fp: String = sqlx::query_scalar("select input_fingerprint from tasks where task_id = $1") .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert_ne!(effective_fp, input_fp); // An identical answer replay still returns the committed outcome after the Task completed: // no second transition, audit row or outbound call. let completed_replay = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version as u32, "update-key-1", &json!({"email": "user@example.com"}), ) .await .expect("identical replay after completion"); assert!(completed_replay.replayed); assert_eq!(completed_replay.version, outcome.version); assert_eq!(calls.load(Ordering::SeqCst), 1); // replay with original scoped key/partial input: same Task, no fingerprint conflict let replay = app .accept_background( &caller, TOOL_NAME, None, "input-key-1", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("replay accepted"); assert_eq!(replay.task_id, task_id); server.abort(); } pub(crate) async fn start_and_suspend( registry: &PostgresRegistry, app: &TaskApplication, caller: &crank_community_mcp::TaskPrincipal, scoped_key: &str, ) -> (TaskId, String, u32) { let acceptance = app .accept_background( caller, TOOL_NAME, None, scoped_key, &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("acceptance"); let (stop, worker_task) = spawn_worker(registry, app); assert!( wait_for_state( registry, acceptance.task_id, "input_required", Duration::from_secs(10) ) .await, "worker must suspend for missing email" ); stop.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; let (wait_id, expected_version) = wait_row(registry, acceptance.task_id).await; (acceptance.task_id, wait_id, expected_version) } async fn wait_row(registry: &PostgresRegistry, task_id: TaskId) -> (String, u32) { let row = sqlx::query("select wait_id, expected_version from task_input_requests where task_id = $1") .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); ( row.get::("wait_id"), u32::try_from(row.get::("expected_version")).unwrap_or_default(), ) } pub(crate) async fn audit_events( registry: &PostgresRegistry, task_id: TaskId, ) -> Vec<(String, Option)> { sqlx::query("select event, reason_code from task_input_audit where task_id = $1 order by id") .bind(task_id.as_uuid()) .fetch_all(registry.pool()) .await .unwrap() .into_iter() .map(|row| (row.get("event"), row.get("reason_code"))) .collect() } pub(crate) async fn admission_entries( registry: &PostgresRegistry, task_id: TaskId, ) -> Vec<(String, i64)> { sqlx::query( "select state, count(*)::bigint as count from execution_admission_entries where task_id = $1 group by state order by state", ) .bind(task_id.as_uuid()) .fetch_all(registry.pool()) .await .unwrap() .into_iter() .map(|row| (row.get("state"), row.get("count"))) .collect() } fn key_id(principal: &crank_community_mcp::TaskPrincipal) -> crank_core::PlatformApiKeyId { match principal { crank_community_mcp::TaskPrincipal::Agent { credential, .. } => credential .platform_api_key_id .clone() .expect("agent principal carries a key id"), crank_community_mcp::TaskPrincipal::Admin { .. } => panic!("agent principal expected"), } } #[tokio::test] async fn invalid_schema_foreign_scope_and_revoked_key_never_transition_or_dispatch() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::OK, axum::Json(json!({"id": "lead"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, &format!("http://{address}"), policy).await; let caller = principal(®istry, "input-scope").await; let app = application(®istry).await; let (task_id, wait_id, expected_version) = start_and_suspend(®istry, &app, &caller, "input-key-scope").await; // Invalid answer: server-side schema validation refuses it, nothing transitions and nothing leaks. let invalid = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-invalid", &json!({}), ) .await .expect_err("the pinned schema requires email"); assert!( matches!( invalid, crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::Invalid { .. }) ), "unexpected error: {invalid:?}" ); assert_eq!( state_of(®istry, task_id).await.as_deref(), Some("input_required") ); assert_eq!(calls.load(Ordering::SeqCst), 0); assert!( audit_events(®istry, task_id) .await .iter() .all(|(event, _)| event == "input_required"), "a rejected answer must not be audited as accepted" ); let stored_snapshot: serde_json::Value = sqlx::query_scalar("select schema_snapshot from task_input_requests where task_id = $1") .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert!(!stored_snapshot.to_string().contains("user@example.com")); // A different key of the same agent carries a different authorization scope: opaque not-found. let foreign = principal(®istry, "input-foreign").await; let foreign_result = app .update_input( &foreign, TOOL_NAME, task_id, &wait_id, expected_version, "update-foreign", &json!({"email": "user@example.com"}), ) .await .expect_err("a foreign scope cannot continue the Task"); assert_eq!( foreign_result, crank_community_mcp::TaskApplicationError::NotFound ); assert_eq!( state_of(®istry, task_id).await.as_deref(), Some("input_required") ); // Positive control: the accepting principal (same key/scope) completes the transition. let accepted = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-valid", &json!({"email": "user@example.com"}), ) .await .expect("valid answer for the accepting scope"); assert_eq!(accepted.state, TaskState::Working); assert!(!accepted.replayed); // Finish the positive-control Task through a real worker so the revocation case starts from a // known dispatch count instead of leaving resumable work behind. let (finish_stop, finish_worker) = spawn_worker(®istry, &app); assert!( wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await, "the positive-control Task completes" ); finish_stop.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), finish_worker).await; assert_eq!(calls.load(Ordering::SeqCst), 1); // Revocation race: the accepting key is revoked after suspension. The same waiting scope must // fail closed inside the update transaction and never revive or dispatch the Task. let revocable = principal(®istry, "input-revoked").await; let (revoked_task, revoked_wait, revoked_version) = start_and_suspend(®istry, &app, &revocable, "input-key-revoked").await; registry .revoke_platform_api_key( &common::test_workspace_id(), &key_id(&revocable), &OffsetDateTime::now_utc(), ) .await .unwrap(); let revoked_result = app .update_input( &revocable, TOOL_NAME, revoked_task, &revoked_wait, revoked_version, "update-revoked", &json!({"email": "revoked@example.com"}), ) .await .expect_err("a revoked key cannot continue the Task"); assert_eq!( revoked_result, crank_community_mcp::TaskApplicationError::NotFound ); assert_eq!( state_of(®istry, revoked_task).await.as_deref(), Some("input_required") ); assert_eq!( calls.load(Ordering::SeqCst), 1, "the revoked answer must not add an outbound call" ); server.abort(); } #[tokio::test] async fn replay_conflict_and_stale_answers_have_one_typed_outcome() { let registry = common::test_registry().await; let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, "http://127.0.0.1:9", policy).await; let caller = principal(®istry, "input-replay").await; let app = application(®istry).await; let (task_id, wait_id, expected_version) = start_and_suspend(®istry, &app, &caller, "input-key-replay").await; let first = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-key", &json!({"email": "user@example.com"}), ) .await .expect("first valid answer"); assert!(!first.replayed); assert_eq!(first.state, TaskState::Working); // Identical replay returns the stored outcome without a second CAS/audit. let replay = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-key", &json!({"email": "user@example.com"}), ) .await .expect("identical replay"); assert!(replay.replayed); assert_eq!(replay.version, first.version); assert_eq!( state_of(®istry, task_id).await.as_deref(), Some("working") ); let accepted_audit = audit_events(®istry, task_id) .await .into_iter() .filter(|(event, _)| event == "input_accepted") .count(); assert_eq!(accepted_audit, 1); // Same key with another payload is a fingerprint conflict. let conflict = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-key", &json!({"email": "other@example.com"}), ) .await .expect_err("same key with another payload conflicts"); assert_eq!( conflict, crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::FingerprintConflict) ); // Another key after a committed answer, and a foreign wait id, are state conflicts. let other_key = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, first.version, "update-key-2", &json!({"email": "user@example.com"}), ) .await .expect_err("another key after commit conflicts"); assert_eq!( other_key, crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict) ); let foreign_wait = app .update_input( &caller, TOOL_NAME, task_id, "wait-does-not-exist", first.version, "update-key-3", &json!({"email": "user@example.com"}), ) .await .expect_err("a foreign wait id conflicts"); assert_eq!( foreign_wait, crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict) ); // A stale generation on a fresh wait is refused; the valid generation still works. let (stale_task, stale_wait, stale_version) = start_and_suspend(®istry, &app, &caller, "input-key-stale").await; let stale = app .update_input( &caller, TOOL_NAME, stale_task, &stale_wait, stale_version.saturating_sub(1).max(1), "update-stale", &json!({"email": "user@example.com"}), ) .await .expect_err("a stale generation cannot be continued"); assert_eq!( stale, crank_community_mcp::TaskApplicationError::Task(crank_core::TaskError::StateConflict) ); let repaired = app .update_input( &caller, TOOL_NAME, stale_task, &stale_wait, stale_version, "update-stale", &json!({"email": "user@example.com"}), ) .await .expect("the current generation still accepts the answer"); assert!(!repaired.replayed); } #[tokio::test] async fn ttl_expiry_fails_with_audit_and_releases_capacity_without_dispatch() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::OK, axum::Json(json!({"id": "lead"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); // The wait TTL is the remaining pinned absolute execution deadline. let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 1_200, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, &format!("http://{address}"), policy).await; let caller = principal(®istry, "input-ttl").await; let app = application(®istry).await; let acceptance = app .accept_background( &caller, TOOL_NAME, None, "input-key-ttl", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("acceptance"); let task_id = acceptance.task_id; let (stop, worker_task) = spawn_worker(®istry, &app); assert!( wait_for_state(®istry, task_id, "input_required", Duration::from_secs(5)).await, "worker must suspend" ); assert!( wait_for_state(®istry, task_id, "failed", Duration::from_secs(6)).await, "the per-poll sweeper must fail the expired wait" ); stop.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; let (wait_id, expected_version) = wait_row(®istry, task_id).await; let events = audit_events(®istry, task_id).await; assert!( events.iter().any(|(event, reason)| event == "input_expired" && reason.as_deref() == Some("input_ttl_expired")), "expiry reason must be audited: {events:?}" ); assert_eq!(admission_entries(®istry, task_id).await, Vec::new()); assert_eq!(calls.load(Ordering::SeqCst), 0); // A late answer cannot revive the failed Task. let late = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-late", &json!({"email": "user@example.com"}), ) .await .expect_err("an expired wait never revives"); assert!( matches!( late, crank_community_mcp::TaskApplicationError::Task( crank_core::TaskError::InputExpired | crank_core::TaskError::StateConflict ) ), "unexpected late answer error: {late:?}" ); assert_eq!( state_of(®istry, task_id).await.as_deref(), Some("failed") ); assert_eq!(calls.load(Ordering::SeqCst), 0); server.abort(); } #[tokio::test] async fn competing_identical_updates_commit_once_and_keep_one_permit() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::OK, axum::Json(json!({"id": "lead"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, &format!("http://{address}"), policy).await; let caller = principal(®istry, "input-race").await; let app = application(®istry).await; let (task_id, wait_id, expected_version) = start_and_suspend(®istry, &app, &caller, "input-key-race").await; assert_eq!( admission_entries(®istry, task_id).await, vec![("executing".to_owned(), 1)], "the waiting Task keeps exactly one executing permit" ); let payload = json!({"email": "user@example.com"}); let first = app.update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-race", &payload, ); let second = app.update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-race", &payload, ); let (first, second) = tokio::join!(first, second); let mut outcomes = [ first.expect("competing answer"), second.expect("competing answer"), ]; outcomes.sort_by_key(|outcome| outcome.replayed); assert!(!outcomes[0].replayed); assert!(outcomes[1].replayed); assert_eq!(outcomes[0].version, outcomes[1].version); assert_eq!( audit_events(®istry, task_id) .await .into_iter() .filter(|(event, _)| event == "input_accepted") .count(), 1 ); assert_eq!( admission_entries(®istry, task_id).await, vec![("executing".to_owned(), 1)], "the update must not grant a second permit" ); assert_eq!(calls.load(Ordering::SeqCst), 0); let (stop, worker_task) = spawn_worker(®istry, &app); assert!( wait_for_state(®istry, task_id, "completed", Duration::from_secs(10)).await, "the resumed Task completes" ); stop.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; assert_eq!(calls.load(Ordering::SeqCst), 1); assert_eq!(admission_entries(®istry, task_id).await, Vec::new()); server.abort(); }