//! Fenced durable Task worker loop for Story 4.3. //! //! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded, //! Valkey may only wake it up (never represented here as correctness), and every lease-protected //! write goes through CAS using owner + fencing token. The actual executable input and revalidation //! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a //! redacted projection and is never dispatched. use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use async_trait::async_trait; use crank_core::{TaskError, TaskLease, TaskStore}; use tokio::sync::watch; use tokio::task::JoinSet; use tracing::{Instrument, debug, warn}; use uuid::Uuid; use crate::task_attempt::{TaskAttemptContext, TaskAttemptOutcome}; /// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets. #[derive(Clone, Debug)] pub struct TaskWorkerConfig { /// Stable random-ish owner identity for this process instance. Required lease owner. pub owner: String, /// Maximum number of Tasks claimed in one bounded PG scan. pub batch_limit: u32, /// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used. pub lease_ms: i64, /// Authoritative PG polling interval when no wake-up hint is available. pub poll_interval: Duration, /// Renewal cadence while processing a lease. pub renewal_interval: Duration, /// Concurrent in-flight Tasks for this worker. pub max_inflight: usize, /// Maximum time to wait for active work during graceful shutdown. pub shutdown_timeout: Duration, } impl Default for TaskWorkerConfig { fn default() -> Self { Self { owner: format!("task-worker-{}", Uuid::now_v7()), batch_limit: 8, lease_ms: 30_000, poll_interval: Duration::from_millis(250), renewal_interval: Duration::from_millis(5_000), max_inflight: 4, shutdown_timeout: Duration::from_secs(30), } } } /// Application-owned seam for pre-dispatch revalidation and execution. /// /// The worker owns lease renewal/CAS/fencing and passes a durable [`TaskAttemptContext`]. The /// application implements [`crate::TaskAttemptAuthority`] and drives the pinned retry budget through /// [`crate::RuntimeExecutor::execute_task_outcome`]; it must not trust `TaskRecord.input` as /// executable payload. #[async_trait] pub trait TaskHandler: Send + Sync { async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>; async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome; } /// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and /// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and /// it binds one durable Task attempt to its execution request. pub struct TaskDispatchPermit { task_id: crank_core::TaskId, operation_id: String, operation_version: u32, workspace_id: String, agent_id: Option, owner: String, fence: u64, attempt_id: String, consumed: std::sync::atomic::AtomicBool, } impl TaskDispatchPermit { #[allow(clippy::too_many_arguments)] pub(crate) fn new( task_id: crank_core::TaskId, operation_id: String, operation_version: u32, workspace_id: String, agent_id: Option, owner: String, fence: u64, attempt_id: String, ) -> Self { Self { task_id, operation_id, operation_version, workspace_id, agent_id, owner, fence, attempt_id, consumed: std::sync::atomic::AtomicBool::new(false), } } pub fn task_id(&self) -> crank_core::TaskId { self.task_id } pub fn operation_id(&self) -> &str { &self.operation_id } pub fn operation_version(&self) -> u32 { self.operation_version } pub fn workspace_id(&self) -> &str { &self.workspace_id } pub fn agent_id(&self) -> Option<&str> { self.agent_id.as_deref() } pub fn owner(&self) -> &str { &self.owner } pub fn fence(&self) -> u64 { self.fence } pub fn attempt_id(&self) -> &str { &self.attempt_id } pub fn is_consumed(&self) -> bool { self.consumed.load(std::sync::atomic::Ordering::Acquire) } pub fn consume(&self) -> bool { !self .consumed .swap(true, std::sync::atomic::Ordering::AcqRel) } } #[derive(thiserror::Error, Debug)] pub enum TaskWorkerError { #[error("task store error: {0}")] Store(#[from] TaskError), #[error("invalid worker configuration: {0}")] Config(String), #[error("task terminal transition CAS failed")] CasRejected, } pub struct TaskWorker { store: Arc, handler: Arc, config: TaskWorkerConfig, stopping: Arc, } impl TaskWorker { pub fn new( store: Arc, handler: Arc, config: TaskWorkerConfig, ) -> Result { if config.owner.is_empty() { return Err(TaskWorkerError::Config( "owner must not be empty".to_owned(), )); } if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 { return Err(TaskWorkerError::Config( "batch_limit must be 1..=64 and lease_ms must be positive".to_owned(), )); } if config.lease_ms > 30_000 || config.poll_interval.is_zero() || config.poll_interval > Duration::from_secs(30) || config.renewal_interval.is_zero() || config.renewal_interval >= Duration::from_millis(config.lease_ms as u64) || config.max_inflight == 0 || config.shutdown_timeout.is_zero() || config.shutdown_timeout > Duration::from_secs(30) { return Err(TaskWorkerError::Config( "lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds" .to_owned(), )); } Ok(Self { store, handler, config, stopping: Arc::new(AtomicBool::new(false)), }) } /// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled. pub fn stop(&self) { self.stopping.store(true, Ordering::Release); } /// Returns a handle whose drop does not stop the worker; call `stop()` on the worker. pub fn stop_flag(&self) -> Arc { self.stopping.clone() } /// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan. pub async fn run(&self) -> Result<(), TaskWorkerError> { self.run_inner(None).await } /// Run until either `stop()` or the provided shutdown receiver becomes true. pub async fn run_until_shutdown( &self, mut shutdown: watch::Receiver, ) -> Result<(), TaskWorkerError> { self.run_inner(Some(&mut shutdown)).await } async fn run_inner( &self, mut shutdown: Option<&mut watch::Receiver>, ) -> Result<(), TaskWorkerError> { let mut active: JoinSet<()> = JoinSet::new(); loop { if self.stopping.load(Ordering::Acquire) { break; } if let Some(rx) = shutdown.as_mut() && *rx.borrow() { break; } // Reap finished attempts before claiming more. while active.try_join_next().is_some() {} // The bounded TTL sweep runs on every poll, including when all local slots are busy: an // expired input wait must end `failed` with an audit reason even without free capacity. match self.store.expire_input_waits(self.config.batch_limit).await { Ok(expired) if expired > 0 => { debug!(expired, "expired input waits were failed") } Ok(_) => {} Err(error) => { warn!(error = %error, "input TTL sweep failed; retrying on next scan") } } let available = self.config.max_inflight.saturating_sub(active.len()); if available > 0 { let batch_limit = self .config .batch_limit .min(u32::try_from(available).unwrap_or(u32::MAX)); match self .store .claim_batch(&self.config.owner, batch_limit, self.config.lease_ms) .await { Ok(leases) => { for lease in leases { let store = self.store.clone(); let handler = self.handler.clone(); let owner = self.config.owner.clone(); let lease_ms = self.config.lease_ms; let renewal_interval = self.config.renewal_interval; let stopping = self.stopping.clone(); active.spawn(async move { process_lease( store, handler, owner, lease, lease_ms, renewal_interval, stopping, ) .await; }); } } Err(error) => { warn!(error = %error, "bounded task claim failed; retrying on next scan"); } } } tokio::select! { _ = tokio::time::sleep(self.config.poll_interval) => {} _ = async { if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await } } => {} } } // Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded. // The same stop flag reaches cooperative cancellation and lease-renewal loops. self.stopping.store(true, Ordering::Release); let deadline = Instant::now() + self.config.shutdown_timeout; while !active.is_empty() { if Instant::now() >= deadline { warn!("graceful shutdown timeout reached; aborting remaining task attempts"); active.abort_all(); break; } tokio::select! { _ = active.join_next() => {} _ = tokio::time::sleep(Duration::from_millis(50)) => {} } } Ok(()) } } async fn process_lease( store: Arc, handler: Arc, owner: String, lease: TaskLease, lease_ms: i64, renewal_interval: Duration, stopping: Arc, ) { let task_id = lease.record.task_id; let fence = lease.fence; let cancelled = Arc::new(AtomicBool::new(false)); // Pre-dispatch revalidation under the fresh lease. if let Err(error) = handler.revalidate(&lease).await { // A transient authority-store outage is not evidence that the work is invalid: leave the // lease to expire so the bounded recovery scan can retry it without dispatching now. if matches!( &error, TaskWorkerError::Store(TaskError::Unavailable { .. }) ) { debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery"); return; } debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch"); finish_failed(&store, &lease, "revalidation_failed").await; return; } if stopping.load(Ordering::Acquire) { // Graceful shutdown before the durable dispatch checkpoint: no outbound call can have // happened, so the Task is left leased. The bounded recovery scan requeues it after the // lease expires instead of turning a never-started attempt into a user-visible failure. return; } let attempt_id = format!("attempt-{}", Uuid::now_v7()); // Background lease renewal covers the whole bounded retry budget, including the pinned backoff. // A durable cancellation clears the lease, so the next renewal fails and the local work stops. let renewal_store = store.clone(); let renewal_owner = owner.clone(); let renewal_cancelled = cancelled.clone(); let renewal_stopping = stopping.clone(); let renewal_task_id = task_id; let renewal_handle = tokio::spawn(async move { let mut interval = tokio::time::interval(renewal_interval); interval.tick().await; loop { interval.tick().await; if renewal_cancelled.load(Ordering::Acquire) { break; } if renewal_stopping.load(Ordering::Acquire) { renewal_cancelled.store(true, Ordering::Release); break; } if renewal_store .renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms) .await .is_err() { renewal_cancelled.store(true, Ordering::Release); break; } } }); // The parent span names the leased Task generation; every individual outbound attempt adds its own // `adapter_attempt` child span with the durable attempt ID committed by its checkpoint. let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, lease_ref = %attempt_id, fence); if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) { crank_trace::set_parent_from_trace_context(&span, &parent); } // Every durable write of this Task (checkpoint, attempt resolution, terminal state) happens inside // the executor's single retry loop under this lease/fence; the worker only renews and stops. let context = TaskAttemptContext::new( store.clone(), lease.clone(), cancelled.clone(), stopping.clone(), ); let outcome = handler.execute(context).instrument(span).await; cancelled.store(true, Ordering::Release); renewal_handle.abort(); match outcome { TaskAttemptOutcome::Completed => { debug!(task_id = %task_id, "durable task completed"); } TaskAttemptOutcome::Failed { code } => { debug!(task_id = %task_id, code = %code, "durable task failed"); } TaskAttemptOutcome::OutcomeUnknown => { warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry"); } TaskAttemptOutcome::WaitingForInput => { debug!(task_id = %task_id, "durable task suspended until validated input arrives"); } TaskAttemptOutcome::Stopped => { debug!(task_id = %task_id, "durable task attempt stopped without a new outcome"); } } } async fn finish_failed(store: &Arc, lease: &TaskLease, code: &str) { // The store owns the fate of a refused pre-dispatch attempt: a Task that already carries a proven // ambiguity still ends `outcome_unknown`, never a plain failure (Story 4.4). let _ = store.refuse_attempt(lease, code).await; }