//! ATDD Story 4.5 QA1 guards (F1/F3): one pinned deadline for several waiting Tasks, cancellation //! while the Task waits for input and refusal of a stale lease/fence around the continuation seam. use super::common; use super::input_continuation::{ TOOL_NAME, admission_entries, application, audit_events, principal, seed_tool, spawn_worker, start_and_suspend, state_of, wait_for_state, }; use super::unsampled_correlation; use axum::Router; use axum::http::StatusCode; use axum::routing::post; use crank_core::{ BeginAttempt, NoTasksFallback, TASK_INPUT_SCHEMA_FORMAT_VERSION, TaskInputRequest, TaskInputStore, TaskState, TaskStore, ToolExecutionMode, ToolExecutionPolicy, }; use crank_registry::{AdmissionLimits, PostgresTaskStore}; use serde_json::json; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use time::OffsetDateTime; #[tokio::test] async fn two_waits_with_one_pinned_deadline_and_cancelled_leftover_do_not_block_suspension() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::OK, axum::Json(json!({"id": "lead"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, &format!("http://{address}"), policy).await; let caller = principal(®istry, "deadline-cancel").await; let app = application(®istry).await; let acc_a = app .accept_background( &caller, TOOL_NAME, None, "deadline-key-a", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("accept a"); let acc_b = app .accept_background( &caller, TOOL_NAME, None, "deadline-key-b", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("accept b"); let task_a = acc_a.task_id; let task_b = acc_b.task_id; let pinned_ms = (OffsetDateTime::now_utc().unix_timestamp() as f64 * 1000.0) + 60_000.0; for tid in [task_a, task_b] { sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2") .bind(pinned_ms) .bind(tid.as_uuid()) .execute(registry.pool()) .await .unwrap(); } let (stop, worker_task) = spawn_worker(®istry, &app); assert!( wait_for_state(®istry, task_a, "input_required", Duration::from_secs(10)).await, "a must suspend" ); assert!( wait_for_state(®istry, task_b, "input_required", Duration::from_secs(10)).await, "b must suspend" ); stop.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; assert_eq!(calls.load(Ordering::SeqCst), 0); // Both active waits carry the same absolute TTL value: the per-Task uniqueness above is the only // uniqueness, so this must not be a UNIQUE index on expires_at. let same_deadline_wait: i64 = sqlx::query_scalar( "select count(*)::bigint from task_input_requests a join task_input_requests b on b.task_id = $2 where a.task_id = $1 and a.accepted_at is null and b.accepted_at is null and a.expires_at = b.expires_at", ) .bind(task_a.as_uuid()) .bind(task_b.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!( same_deadline_wait, 1, "both waits share the pinned deadline" ); let outcome = app .cancel(&caller, TOOL_NAME, task_a, None) .await .expect("cancel a"); assert!(matches!( outcome, crank_core::TaskCancelOutcome::Cancelled(_) )); assert_eq!( state_of(®istry, task_a).await.as_deref(), Some("cancelled") ); assert_eq!(admission_entries(®istry, task_a).await, Vec::new()); // accept C with same pinned deadline let acc_c = app .accept_background( &caller, TOOL_NAME, None, "deadline-key-c", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("accept c"); let task_c = acc_c.task_id; sqlx::query("update tasks set execution_deadline = to_timestamp($1::double precision/1000.0) where task_id = $2") .bind(pinned_ms) .bind(task_c.as_uuid()) .execute(registry.pool()) .await .unwrap(); let (stop2, worker_task2) = spawn_worker(®istry, &app); assert!( wait_for_state(®istry, task_c, "input_required", Duration::from_secs(10)).await, "c must suspend despite leftover a" ); stop2.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task2).await; assert_eq!( state_of(®istry, task_b).await.as_deref(), Some("input_required") ); let c_deadline_ms: i64 = sqlx::query_scalar( "select (extract(epoch from expires_at) * 1000)::bigint from task_input_requests where task_id = $1", ) .bind(task_c.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!( c_deadline_ms, pinned_ms as i64, "the third wait reuses the same pinned deadline value" ); let a_accepted: Option = sqlx::query_scalar("select accepted_at from task_input_requests where task_id = $1") .bind(task_a.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert!( a_accepted.is_none(), "cancellation leaves the wait row without an accepted receipt" ); assert_eq!(calls.load(Ordering::SeqCst), 0); server.abort(); } #[tokio::test] async fn cancel_while_input_required_never_revives_or_dispatches() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::OK, axum::Json(json!({"id": "lead"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, &format!("http://{address}"), policy).await; let caller = principal(®istry, "cancel-wait").await; let app = application(®istry).await; let (task_id, wait_id, expected_version) = start_and_suspend(®istry, &app, &caller, "input-key-cancel").await; assert_eq!( admission_entries(®istry, task_id).await, vec![("executing".to_owned(), 1)] ); let outcome = app .cancel(&caller, TOOL_NAME, task_id, None) .await .expect("cancel"); assert!(matches!( outcome, crank_core::TaskCancelOutcome::Cancelled(_) )); assert_eq!( state_of(®istry, task_id).await.as_deref(), Some("cancelled") ); assert_eq!(admission_entries(®istry, task_id).await, Vec::new()); let late = app .update_input( &caller, TOOL_NAME, task_id, &wait_id, expected_version, "update-late", &json!({"email": "user@example.com"}), ) .await .expect_err("late answer must not revive"); assert!( matches!( late, crank_community_mcp::TaskApplicationError::Task( crank_core::TaskError::StateConflict | crank_core::TaskError::InputExpired ) ), "unexpected: {late:?}" ); assert_eq!( state_of(®istry, task_id).await.as_deref(), Some("cancelled") ); assert_eq!(calls.load(Ordering::SeqCst), 0); let events = audit_events(®istry, task_id).await; assert!(events.iter().any(|(e, _)| e == "input_required")); assert!(!events.iter().any(|(e, _)| e == "input_accepted")); let attempts = app .attempts(&caller, TOOL_NAME, task_id) .await .expect("attempts"); assert!(attempts.is_empty()); server.abort(); } #[tokio::test] async fn stale_lease_fence_cannot_suspend_or_dispatch() { let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let upstream_app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); move || { let calls = Arc::clone(&calls); async move { calls.fetch_add(1, Ordering::SeqCst); (StatusCode::OK, axum::Json(json!({"id": "lead"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let policy = ToolExecutionPolicy { mode: ToolExecutionMode::Background, deadline_ms: 60_000, no_tasks_fallback: NoTasksFallback::Unsupported, }; seed_tool(®istry, &format!("http://{address}"), policy).await; let caller = principal(®istry, "stale-fence").await; let app = application(®istry).await; let acceptance = app .accept_background( &caller, TOOL_NAME, None, "stale-key", &json!({"report": "q3"}), &unsampled_correlation(), ) .await .expect("accept"); let task_id = acceptance.task_id; let store = Arc::new( PostgresTaskStore::new( registry.pool().clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("store"), ); let mut old_claims = store .claim_batch("worker-old", 1, 50) .await .expect("old claim"); assert!(!old_claims.is_empty()); let old = old_claims.remove(0); tokio::time::sleep(Duration::from_millis(120)).await; let mut new_claims = store .claim_batch("worker-new", 1, 10_000) .await .expect("new claim"); assert!(!new_claims.is_empty()); let new = new_claims.remove(0); assert!(new.fence > old.fence); assert_eq!(new.record.task_id, task_id); let request = TaskInputRequest { wait_id: "stale-wait".to_string(), expected_version: old.record.version + 1, schema_revision: 1, schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION, schema_snapshot: json!({}), projected_schema: json!({}), prompt_code: "required_input_missing".to_string(), missing_fields: vec!["$.email".to_string()], expires_at_unix_ms: OffsetDateTime::now_utc().unix_timestamp() * 1000 + 30_000, fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION, }; let suspended = ::suspend_for_input(&*store, &old, request).await; assert!( suspended.is_err(), "stale fence must not suspend: {suspended:?}" ); let attempt_id = "attempt-stale".to_string(); let checkpoint = store .checkpoint_dispatch(BeginAttempt { task_id, owner: old.owner.clone(), fence: old.fence, expected_version: old.record.version, attempt_id, input_fingerprint: old.record.input_fingerprint.clone(), upstream_key_digest: None, retry_allowed: false, }) .await; assert!( checkpoint.is_err(), "stale fence must not checkpoint: {checkpoint:?}" ); assert_ne!( state_of(®istry, task_id).await.as_deref(), Some("input_required") ); let attempt_count: i64 = sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1") .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!(attempt_count, 0); let req_count: i64 = sqlx::query_scalar("select count(*)::bigint from task_input_requests where task_id = $1") .bind(task_id.as_uuid()) .fetch_one(registry.pool()) .await .unwrap(); assert_eq!(req_count, 0); assert_eq!(calls.load(Ordering::SeqCst), 0); // Positive control: the same request under the current lease suspends the Task exactly once, so // the refusal above is the stale fence and not an invalid request shape. let request = TaskInputRequest { wait_id: "current-wait".to_string(), expected_version: new.record.version + 1, schema_revision: 1, schema_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION, fingerprint_format_version: TASK_INPUT_SCHEMA_FORMAT_VERSION, schema_snapshot: json!({}), projected_schema: json!({}), prompt_code: "required_input_missing".to_string(), missing_fields: vec!["$.email".to_string()], expires_at_unix_ms: new.record.pins.execution_deadline_unix_ms, }; let suspended = ::suspend_for_input(&*store, &new, request) .await .expect("the current lease suspends"); assert_eq!(suspended.state, TaskState::InputRequired); assert_eq!( state_of(®istry, task_id).await.as_deref(), Some("input_required") ); assert_eq!( admission_entries(®istry, task_id).await, vec![("executing".to_owned(), 1)] ); // The stale fence still cannot checkpoint a dispatch of this generation. let checkpoint = store .checkpoint_dispatch(BeginAttempt { task_id, owner: old.owner.clone(), fence: old.fence, expected_version: old.record.version, attempt_id: "attempt-stale-after-suspend".to_string(), input_fingerprint: old.record.input_fingerprint.clone(), upstream_key_digest: None, retry_allowed: false, }) .await; assert!( checkpoint.is_err(), "stale fence must not checkpoint: {checkpoint:?}" ); assert_eq!(calls.load(Ordering::SeqCst), 0); server.abort(); }