//! Story 4.3 ATDD: fenced durable Task worker leases, recovery and real upstream dispatch. //! //! Split from `durable_tasks.rs` to keep both integration files inside the code-health limit. use super::*; /// Store-level positive control only: PostgreSQL leases/fencing/recovery work without Valkey or a /// task-worker process. This does not exercise TaskWorker startup, a bounded authoritative scan, /// graceful shutdown, outbound dispatch, or runtime recovery timing, which remain unimplemented. #[tokio::test] async fn task_worker_store_lease_recovery_contract() { let registry = common::test_registry().await; let store = PostgresTaskStore::new( registry.pool().clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"); // Positive control: no Valkey or worker process is involved; PostgreSQL alone accepts and // leases durable work, preserving the same admission-backed intent. let request = TaskAcceptRequest { task_id: TaskId::generate(), workspace_id: common::test_workspace_id().to_string(), owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_1".to_owned(), platform_api_key_id: None, operation_id: "op_report".to_owned(), pins: TaskPins { operation_version: 1, policy_mode: ToolExecutionMode::Background, execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "a".repeat(64), input_fingerprint: "b".repeat(64), input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(), support: TaskSupportIds::new( RequestId::generate(), &crank_core::TraceContext::parse( "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", ) .unwrap(), ), // This store-level control exercises lease/fence behavior, not the encrypted acceptance path. executable_input_ciphertext: None, executable_input_key_version: None, executable_input_master_key_epoch: None, }; let accepted = store.accept(request).await.expect("PG accepts"); let leases = store .claim_batch("worker_1", 4, 10_000) .await .expect("claim"); assert_eq!(leases.len(), 1); assert_eq!(leases[0].record.task_id, accepted.task_id); let lease = leases[0].clone(); // A stale or competing owner cannot win CAS with the issued fencing token. assert!( store .renew_lease("worker_2", lease.record.task_id, lease.fence, 10_000) .await .is_err() ); assert!( store .write_progress( "worker_1", lease.record.task_id, lease.fence + 1, TaskEnvelope::try_redacted(&json!({"progress": 1})).unwrap(), ) .await .is_err() ); // A stale fencing token can never checkpoint dispatch, even under the issuing owner. assert!( store .checkpoint_dispatch(crank_core::BeginAttempt { task_id: lease.record.task_id, owner: "worker_1".to_owned(), fence: lease.fence + 1, expected_version: lease.record.version, attempt_id: "attempt-stale".to_owned(), input_fingerprint: lease.record.input_fingerprint.clone(), upstream_key_digest: None, retry_allowed: false, }) .await .is_err() ); // The valid owner still commits the dispatch checkpoint before any outbound work. let committed = store .checkpoint_dispatch(crank_core::BeginAttempt { task_id: lease.record.task_id, owner: "worker_1".to_owned(), fence: lease.fence, expected_version: lease.record.version, attempt_id: "attempt-1".to_owned(), input_fingerprint: lease.record.input_fingerprint.clone(), upstream_key_digest: None, retry_allowed: false, }) .await .unwrap(); assert_eq!(committed.attempt.attempt_id, "attempt-1"); assert_eq!( committed.task.current_attempt_id.as_deref(), Some("attempt-1") ); } #[tokio::test] async fn task_worker_runtime_dispatch_and_recovery_contract() { use async_trait::async_trait; use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; use crank_runtime::{ TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; let registry = common::test_registry().await; let pool = registry.pool().clone(); let store = Arc::new( PostgresTaskStore::new( pool.clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"), ); let observed = Arc::new(AtomicUsize::new(0)); // Positive control: PostgreSQL alone accepts the durable Task. let request = TaskAcceptRequest { task_id: TaskId::generate(), workspace_id: common::test_workspace_id().to_string(), owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_runtime".to_owned(), platform_api_key_id: None, operation_id: "op_report".to_owned(), pins: TaskPins { operation_version: 1, policy_mode: ToolExecutionMode::Background, execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "r".repeat(64), input_fingerprint: "b".repeat(64), input: TaskEnvelope::try_redacted(&json!({"email": "q3"})).unwrap(), support: TaskSupportIds::new( RequestId::generate(), &crank_core::TraceContext::parse( "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", ) .unwrap(), ), executable_input_ciphertext: None, executable_input_key_version: None, executable_input_master_key_epoch: None, }; let accepted = store.accept(request).await.expect("PG accepts"); let task_id = accepted.task_id; struct RuntimeHandler { observed: Arc, } #[async_trait] impl TaskHandler for RuntimeHandler { async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { Ok(()) } async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome { self.observed.fetch_add(1, Ordering::SeqCst); commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await } } let handler: Arc = Arc::new(RuntimeHandler { observed: observed.clone(), }); // PG-only authoritative dispatch: no Valkey or upstream process is involved. let config = TaskWorkerConfig { owner: "runtime-worker-1".to_owned(), batch_limit: 1, lease_ms: 10_000, poll_interval: Duration::from_millis(100), renewal_interval: Duration::from_millis(1_000), max_inflight: 1, shutdown_timeout: Duration::from_secs(5), }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); let access = TaskAccess { workspace_id: common::test_workspace_id().to_string(), owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_runtime".to_owned(), }; let mut state = None; for _ in 0..100 { tokio::time::sleep(Duration::from_millis(100)).await; if let Ok(record) = store.read(&access, task_id).await { state = Some(record.state); if record.state == TaskState::Completed { break; } } } stop_flag.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; assert_eq!(state, Some(TaskState::Completed)); assert_eq!( observed.load(Ordering::SeqCst), 1, "exactly one outbound attempt" ); // Terminal state is not duplicated. let record = store.read(&access, task_id).await.expect("terminal read"); assert_eq!(record.state, TaskState::Completed); // Bounded shutdown with no fresh work returns promptly. let idle_config = TaskWorkerConfig { owner: "runtime-worker-idle".to_owned(), poll_interval: Duration::from_millis(50), shutdown_timeout: Duration::from_secs(2), ..TaskWorkerConfig::default() }; let idle_worker = TaskWorker::new( store.clone(), handler_for_no_work(store.clone()), idle_config, ) .expect("valid idle worker"); idle_worker.stop(); let started = std::time::Instant::now(); idle_worker.run().await.expect("idle worker stops"); assert!(started.elapsed() <= Duration::from_secs(30)); } /// Story 4.3 ATDD: failed pre-dispatch revalidation finishes the attempt without any outbound call. #[tokio::test] async fn task_worker_revalidation_denial_never_dispatches() { use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind}; use crank_runtime::{ TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; let registry = common::test_registry().await; let store = Arc::new( PostgresTaskStore::new( registry.pool().clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"), ); let request = TaskAcceptRequest { task_id: TaskId::generate(), workspace_id: common::test_workspace_id().to_string(), owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_denied".to_owned(), platform_api_key_id: None, operation_id: "op_report".to_owned(), pins: TaskPins { operation_version: 1, policy_mode: ToolExecutionMode::Background, execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "d".repeat(64), input_fingerprint: "b".repeat(64), input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(), support: TaskSupportIds::new( RequestId::generate(), &crank_core::TraceContext::parse( "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", ) .unwrap(), ), executable_input_ciphertext: None, executable_input_key_version: None, executable_input_master_key_epoch: None, }; let accepted = store.accept(request).await.expect("PG accepts"); struct DeniedHandler { observed: Arc, } #[async_trait::async_trait] impl TaskHandler for DeniedHandler { async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { Err(TaskWorkerError::Store(TaskError::invalid( "publication or credential revoked", ))) } async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome { self.observed.fetch_add(1, Ordering::SeqCst); commit_attempt(&context, completed_outcome(json!({"ok": true}))).await } } let observed = Arc::new(AtomicUsize::new(0)); let handler: Arc = Arc::new(DeniedHandler { observed: observed.clone(), }); let config = TaskWorkerConfig { owner: "runtime-worker-denied".to_owned(), batch_limit: 1, lease_ms: 10_000, poll_interval: Duration::from_millis(50), max_inflight: 1, shutdown_timeout: Duration::from_secs(5), ..TaskWorkerConfig::default() }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); let access = TaskAccess { workspace_id: common::test_workspace_id().to_string(), owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_denied".to_owned(), }; let mut state = None; for _ in 0..100 { tokio::time::sleep(Duration::from_millis(50)).await; if let Ok(record) = store.read(&access, accepted.task_id).await { state = Some(record.state); if record.state == TaskState::Failed { break; } } } stop_flag.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; assert_eq!(state, Some(TaskState::Failed)); assert_eq!( observed.load(Ordering::SeqCst), 0, "invalidated work never reaches the upstream" ); } fn handler_for_no_work( _store: std::sync::Arc, ) -> std::sync::Arc { struct NoWorkHandler; #[async_trait::async_trait] impl crank_runtime::TaskHandler for NoWorkHandler { async fn revalidate( &self, _lease: &crank_core::TaskLease, ) -> Result<(), crank_runtime::TaskWorkerError> { Ok(()) } async fn execute( &self, context: crank_runtime::TaskAttemptContext, ) -> crank_runtime::TaskAttemptOutcome { commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await } } std::sync::Arc::new(NoWorkHandler) } /// Story 4.3 end-to-end ATDD: the production durable-Task handler, the real runtime executor and a /// real local HTTP upstream observe exactly one dispatch after the fenced checkpoint, while the /// accepted parent trace is continued on the wire. #[tokio::test] async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() { use axum::Router; use axum::http::{HeaderMap, StatusCode}; use axum::routing::post; use crank_community_mcp::TaskWorkerHandler; use crank_core::{AgentOperationBinding, PlatformApiKeyScope}; use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; let registry = common::test_registry().await; let calls = Arc::new(AtomicUsize::new(0)); let seen_traceparent = Arc::new(Mutex::new(None::)); let app = Router::new().route( "/crm/leads", post({ let calls = Arc::clone(&calls); let seen = Arc::clone(&seen_traceparent); move |headers: HeaderMap| { let calls = Arc::clone(&calls); let seen = Arc::clone(&seen); async move { calls.fetch_add(1, Ordering::SeqCst); if let Some(value) = headers .get("traceparent") .and_then(|value| value.to_str().ok()) { *seen.lock().unwrap() = Some(value.to_owned()); } (StatusCode::OK, axum::Json(json!({"id": "lead-1"}))) } } }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); let upstream = format!("http://{address}"); let mut operation = common::test_operation(&upstream, TOOL_NAME); operation.execution_config.execution_policy = Some(background_policy()); registry .create_operation(&common::test_workspace_id(), &operation, Some("alice")) .await .unwrap(); publish(®istry, &operation.id, 1).await; common::publish_agent_with_bindings( ®istry, AGENT_SLUG, vec![AgentOperationBinding { operation_id: operation.id.clone(), operation_version: 1, tool_name: operation.name.clone(), tool_title: operation.tool_description.title.clone(), tool_description_override: None, enabled: true, }], ) .await; let caller = principal( ®istry, "task-worker-e2e", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let correlation = unsampled_correlation(); let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(); let observed_at = OffsetDateTime::now_utc(); registry .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { epoch: crypto.master_key_epoch(), fingerprint: crypto.master_key_fingerprint(), cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, observed_at: &observed_at, }) .await .expect("test master-key identity registered"); let store = Arc::new( PostgresTaskStore::new( registry.pool().clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"), ); let tasks = TaskApplication::new( registry.clone(), Duration::ZERO, Arc::new(InMemoryCoordinationStateStore::default()), TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())), ); let acceptance = tasks .accept_background( &caller, TOOL_NAME, None, "worker-e2e-key", &json!({"email": "user@example.com"}), &correlation, ) .await .expect("durable acceptance with executable input"); let runtime = crank_runtime::community_with_outbound_policy( crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), ) .build(); let handler = Arc::new(TaskWorkerHandler::from_application( &tasks, registry.clone(), runtime, crypto, )); let config = TaskWorkerConfig { owner: "runtime-worker-e2e".to_owned(), batch_limit: 1, lease_ms: 10_000, poll_interval: Duration::from_millis(50), max_inflight: 1, shutdown_timeout: Duration::from_secs(5), ..TaskWorkerConfig::default() }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); let mut state = None; for _ in 0..200 { tokio::time::sleep(Duration::from_millis(50)).await; let current: Option = sqlx::query_scalar("select state from tasks where task_id = $1") .bind(acceptance.task_id.as_uuid()) .fetch_optional(registry.pool()) .await .unwrap(); state = current; if state.as_deref() == Some("completed") { break; } } stop_flag.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; server.abort(); assert_eq!(state.as_deref(), Some("completed")); assert_eq!(calls.load(Ordering::SeqCst), 1, "exactly one upstream call"); assert_eq!( seen_traceparent.lock().unwrap().as_deref(), Some(correlation.trace_context().traceparent()), "the durable attempt continues the accepted parent trace" ); } /// Story 4.3 end-to-end ATDD: an ambiguous post-checkpoint upstream failure is persisted as /// `outcome_unknown` (not a retryable failure) and is not redispatched automatically. #[tokio::test] async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() { use axum::Router; use axum::http::StatusCode; use axum::routing::post; use crank_community_mcp::TaskWorkerHandler; use crank_core::{AgentOperationBinding, PlatformApiKeyScope}; use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig}; use std::sync::Arc; use std::sync::atomic::Ordering; let registry = common::test_registry().await; let upstream_app = Router::new().route( "/crm/leads", post(|| async { (StatusCode::SERVICE_UNAVAILABLE, "upstream unavailable") }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() }); let upstream = format!("http://{address}"); let mut operation = common::test_operation(&upstream, TOOL_NAME); operation.execution_config.execution_policy = Some(background_policy()); registry .create_operation(&common::test_workspace_id(), &operation, Some("alice")) .await .unwrap(); publish(®istry, &operation.id, 1).await; common::publish_agent_with_bindings( ®istry, AGENT_SLUG, vec![AgentOperationBinding { operation_id: operation.id.clone(), operation_version: 1, tool_name: operation.name.clone(), tool_title: operation.tool_description.title.clone(), tool_description_override: None, enabled: true, }], ) .await; let caller = principal( ®istry, "task-worker-uncertain", &[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write], ) .await; let correlation = unsampled_correlation(); let crypto = SecretCrypto::new("test-master-key-00000000000000000000000000000000").unwrap(); let observed_at = OffsetDateTime::now_utc(); registry .verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate { epoch: crypto.master_key_epoch(), fingerprint: crypto.master_key_fingerprint(), cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT, observed_at: &observed_at, }) .await .expect("test master-key identity registered"); let store = Arc::new( PostgresTaskStore::new( registry.pool().clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"), ); let tasks = TaskApplication::new( registry.clone(), Duration::ZERO, Arc::new(InMemoryCoordinationStateStore::default()), TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())), ); let acceptance = tasks .accept_background( &caller, TOOL_NAME, None, "worker-uncertain-key", &json!({"email": "user@example.com"}), &correlation, ) .await .expect("durable acceptance with executable input"); let runtime = crank_runtime::community_with_outbound_policy( crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]), ) .build(); let handler = Arc::new(TaskWorkerHandler::from_application( &tasks, registry.clone(), runtime, crypto, )); let config = TaskWorkerConfig { owner: "runtime-worker-uncertain".to_owned(), batch_limit: 1, lease_ms: 10_000, poll_interval: Duration::from_millis(50), max_inflight: 1, shutdown_timeout: Duration::from_secs(5), ..TaskWorkerConfig::default() }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); let mut state = None; for _ in 0..200 { tokio::time::sleep(Duration::from_millis(50)).await; let current: Option = sqlx::query_scalar("select state from tasks where task_id = $1") .bind(acceptance.task_id.as_uuid()) .fetch_optional(registry.pool()) .await .unwrap(); state = current; if state.as_deref() == Some("outcome_unknown") { break; } } stop_flag.store(true, Ordering::SeqCst); let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await; server.abort(); assert_eq!(state.as_deref(), Some("outcome_unknown")); // No automatic retry: another authoritative scan never redispaches the uncertain attempt. assert!( store .claim_batch("worker_after", 4, 10_000) .await .unwrap() .is_empty() ); } /// Story 4.3 ATDD: graceful stop stops new claims but lets an active attempt reach its fenced /// terminal commit within the 30s budget. #[tokio::test] async fn task_worker_graceful_stop_drains_active_attempt() { use crank_core::{TaskAccess, TaskLease, TaskOwnerKind}; use crank_runtime::{ TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; let registry = common::test_registry().await; let store = Arc::new( PostgresTaskStore::new( registry.pool().clone(), AdmissionLimits { global_limit: 4, agent_limit: 4, tool_limit: 4, queue_limit: 8, }, Default::default(), ) .expect("valid task store configuration"), ); let request = TaskAcceptRequest { task_id: TaskId::generate(), workspace_id: common::test_workspace_id().to_string(), owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_drain".to_owned(), platform_api_key_id: None, operation_id: "op_report".to_owned(), pins: TaskPins { operation_version: 1, policy_mode: ToolExecutionMode::Background, execution_deadline_unix_ms: 1_900_000_000_000, retain_until_unix_ms: 1_900_000_000_000 + 60_000, approval_reference: None, retry: crank_core::TaskRetryPolicy::single_attempt(), }, scoped_key: "s".repeat(64), input_fingerprint: "d".repeat(64), input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(), support: TaskSupportIds::new( RequestId::generate(), &crank_core::TraceContext::parse( "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", ) .unwrap(), ), executable_input_ciphertext: None, executable_input_key_version: None, executable_input_master_key_epoch: None, }; let accepted = store.accept(request).await.expect("PG accepts"); struct SlowHandler { started: Arc, } #[async_trait::async_trait] impl TaskHandler for SlowHandler { async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> { Ok(()) } async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome { self.started.fetch_add(1, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(500)).await; commit_attempt(&context, completed_outcome(json!({"ok": true}))).await } } let started = Arc::new(AtomicUsize::new(0)); let handler: Arc = Arc::new(SlowHandler { started: started.clone(), }); let config = TaskWorkerConfig { owner: "runtime-worker-drain".to_owned(), batch_limit: 1, lease_ms: 10_000, poll_interval: Duration::from_millis(50), max_inflight: 1, shutdown_timeout: Duration::from_secs(20), ..TaskWorkerConfig::default() }; let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker"); let stop_flag = worker.stop_flag(); let worker_task = tokio::spawn(async move { worker.run().await }); for _ in 0..100 { if started.load(Ordering::SeqCst) > 0 { break; } tokio::time::sleep(Duration::from_millis(20)).await; } assert_eq!(started.load(Ordering::SeqCst), 1, "attempt started"); let stop_requested = std::time::Instant::now(); stop_flag.store(true, Ordering::SeqCst); tokio::time::timeout(Duration::from_secs(30), worker_task) .await .expect("worker stops inside the 30s budget") .expect("worker exits cleanly") .expect("worker reports no error"); assert!(stop_requested.elapsed() <= Duration::from_secs(30)); let access = TaskAccess { workspace_id: common::test_workspace_id().to_string(), owner_kind: TaskOwnerKind::Agent, owner_id: "agent_a".to_owned(), authorization_scope: "credential:pk_drain".to_owned(), }; let record = store.read(&access, accepted.task_id).await.unwrap(); assert_eq!(record.state, TaskState::Completed); }