CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если закреплённая схема требует отсутствующее поле; authorized schema-validated update атомарно и с аудитом возобновляет тот же Task ровно один раз. Replay/conflict/TTL безопасны, ожидание удерживает admission permit. Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
434 lines
16 KiB
Rust
434 lines
16 KiB
Rust
//! Fenced durable Task worker loop for Story 4.3.
|
|
//!
|
|
//! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded,
|
|
//! Valkey may only wake it up (never represented here as correctness), and every lease-protected
|
|
//! write goes through CAS using owner + fencing token. The actual executable input and revalidation
|
|
//! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a
|
|
//! redacted projection and is never dispatched.
|
|
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
use std::time::{Duration, Instant};
|
|
|
|
use async_trait::async_trait;
|
|
use crank_core::{TaskError, TaskLease, TaskStore};
|
|
use tokio::sync::watch;
|
|
use tokio::task::JoinSet;
|
|
use tracing::{Instrument, debug, warn};
|
|
use uuid::Uuid;
|
|
|
|
use crate::task_attempt::{TaskAttemptContext, TaskAttemptOutcome};
|
|
|
|
/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets.
|
|
#[derive(Clone, Debug)]
|
|
pub struct TaskWorkerConfig {
|
|
/// Stable random-ish owner identity for this process instance. Required lease owner.
|
|
pub owner: String,
|
|
/// Maximum number of Tasks claimed in one bounded PG scan.
|
|
pub batch_limit: u32,
|
|
/// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used.
|
|
pub lease_ms: i64,
|
|
/// Authoritative PG polling interval when no wake-up hint is available.
|
|
pub poll_interval: Duration,
|
|
/// Renewal cadence while processing a lease.
|
|
pub renewal_interval: Duration,
|
|
/// Concurrent in-flight Tasks for this worker.
|
|
pub max_inflight: usize,
|
|
/// Maximum time to wait for active work during graceful shutdown.
|
|
pub shutdown_timeout: Duration,
|
|
}
|
|
|
|
impl Default for TaskWorkerConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
owner: format!("task-worker-{}", Uuid::now_v7()),
|
|
batch_limit: 8,
|
|
lease_ms: 30_000,
|
|
poll_interval: Duration::from_millis(250),
|
|
renewal_interval: Duration::from_millis(5_000),
|
|
max_inflight: 4,
|
|
shutdown_timeout: Duration::from_secs(30),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Application-owned seam for pre-dispatch revalidation and execution.
|
|
///
|
|
/// The worker owns lease renewal/CAS/fencing and passes a durable [`TaskAttemptContext`]. The
|
|
/// application implements [`crate::TaskAttemptAuthority`] and drives the pinned retry budget through
|
|
/// [`crate::RuntimeExecutor::execute_task_outcome`]; it must not trust `TaskRecord.input` as
|
|
/// executable payload.
|
|
#[async_trait]
|
|
pub trait TaskHandler: Send + Sync {
|
|
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>;
|
|
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome;
|
|
}
|
|
|
|
/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and
|
|
/// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and
|
|
/// it binds one durable Task attempt to its execution request.
|
|
pub struct TaskDispatchPermit {
|
|
task_id: crank_core::TaskId,
|
|
operation_id: String,
|
|
operation_version: u32,
|
|
workspace_id: String,
|
|
agent_id: Option<String>,
|
|
owner: String,
|
|
fence: u64,
|
|
attempt_id: String,
|
|
consumed: std::sync::atomic::AtomicBool,
|
|
}
|
|
|
|
impl TaskDispatchPermit {
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn new(
|
|
task_id: crank_core::TaskId,
|
|
operation_id: String,
|
|
operation_version: u32,
|
|
workspace_id: String,
|
|
agent_id: Option<String>,
|
|
owner: String,
|
|
fence: u64,
|
|
attempt_id: String,
|
|
) -> Self {
|
|
Self {
|
|
task_id,
|
|
operation_id,
|
|
operation_version,
|
|
workspace_id,
|
|
agent_id,
|
|
owner,
|
|
fence,
|
|
attempt_id,
|
|
consumed: std::sync::atomic::AtomicBool::new(false),
|
|
}
|
|
}
|
|
|
|
pub fn task_id(&self) -> crank_core::TaskId {
|
|
self.task_id
|
|
}
|
|
|
|
pub fn operation_id(&self) -> &str {
|
|
&self.operation_id
|
|
}
|
|
|
|
pub fn operation_version(&self) -> u32 {
|
|
self.operation_version
|
|
}
|
|
|
|
pub fn workspace_id(&self) -> &str {
|
|
&self.workspace_id
|
|
}
|
|
|
|
pub fn agent_id(&self) -> Option<&str> {
|
|
self.agent_id.as_deref()
|
|
}
|
|
|
|
pub fn owner(&self) -> &str {
|
|
&self.owner
|
|
}
|
|
|
|
pub fn fence(&self) -> u64 {
|
|
self.fence
|
|
}
|
|
|
|
pub fn attempt_id(&self) -> &str {
|
|
&self.attempt_id
|
|
}
|
|
|
|
pub fn is_consumed(&self) -> bool {
|
|
self.consumed.load(std::sync::atomic::Ordering::Acquire)
|
|
}
|
|
|
|
pub fn consume(&self) -> bool {
|
|
!self
|
|
.consumed
|
|
.swap(true, std::sync::atomic::Ordering::AcqRel)
|
|
}
|
|
}
|
|
|
|
#[derive(thiserror::Error, Debug)]
|
|
pub enum TaskWorkerError {
|
|
#[error("task store error: {0}")]
|
|
Store(#[from] TaskError),
|
|
#[error("invalid worker configuration: {0}")]
|
|
Config(String),
|
|
#[error("task terminal transition CAS failed")]
|
|
CasRejected,
|
|
}
|
|
|
|
pub struct TaskWorker {
|
|
store: Arc<dyn TaskStore>,
|
|
handler: Arc<dyn TaskHandler>,
|
|
config: TaskWorkerConfig,
|
|
stopping: Arc<AtomicBool>,
|
|
}
|
|
|
|
impl TaskWorker {
|
|
pub fn new(
|
|
store: Arc<dyn TaskStore>,
|
|
handler: Arc<dyn TaskHandler>,
|
|
config: TaskWorkerConfig,
|
|
) -> Result<Self, TaskWorkerError> {
|
|
if config.owner.is_empty() {
|
|
return Err(TaskWorkerError::Config(
|
|
"owner must not be empty".to_owned(),
|
|
));
|
|
}
|
|
if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 {
|
|
return Err(TaskWorkerError::Config(
|
|
"batch_limit must be 1..=64 and lease_ms must be positive".to_owned(),
|
|
));
|
|
}
|
|
if config.lease_ms > 30_000
|
|
|| config.poll_interval.is_zero()
|
|
|| config.poll_interval > Duration::from_secs(30)
|
|
|| config.renewal_interval.is_zero()
|
|
|| config.renewal_interval >= Duration::from_millis(config.lease_ms as u64)
|
|
|| config.max_inflight == 0
|
|
|| config.shutdown_timeout.is_zero()
|
|
|| config.shutdown_timeout > Duration::from_secs(30)
|
|
{
|
|
return Err(TaskWorkerError::Config(
|
|
"lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds"
|
|
.to_owned(),
|
|
));
|
|
}
|
|
Ok(Self {
|
|
store,
|
|
handler,
|
|
config,
|
|
stopping: Arc::new(AtomicBool::new(false)),
|
|
})
|
|
}
|
|
|
|
/// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled.
|
|
pub fn stop(&self) {
|
|
self.stopping.store(true, Ordering::Release);
|
|
}
|
|
|
|
/// Returns a handle whose drop does not stop the worker; call `stop()` on the worker.
|
|
pub fn stop_flag(&self) -> Arc<AtomicBool> {
|
|
self.stopping.clone()
|
|
}
|
|
|
|
/// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan.
|
|
pub async fn run(&self) -> Result<(), TaskWorkerError> {
|
|
self.run_inner(None).await
|
|
}
|
|
|
|
/// Run until either `stop()` or the provided shutdown receiver becomes true.
|
|
pub async fn run_until_shutdown(
|
|
&self,
|
|
mut shutdown: watch::Receiver<bool>,
|
|
) -> Result<(), TaskWorkerError> {
|
|
self.run_inner(Some(&mut shutdown)).await
|
|
}
|
|
|
|
async fn run_inner(
|
|
&self,
|
|
mut shutdown: Option<&mut watch::Receiver<bool>>,
|
|
) -> Result<(), TaskWorkerError> {
|
|
let mut active: JoinSet<()> = JoinSet::new();
|
|
loop {
|
|
if self.stopping.load(Ordering::Acquire) {
|
|
break;
|
|
}
|
|
if let Some(rx) = shutdown.as_mut()
|
|
&& *rx.borrow()
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Reap finished attempts before claiming more.
|
|
while active.try_join_next().is_some() {}
|
|
|
|
// The bounded TTL sweep runs on every poll, including when all local slots are busy: an
|
|
// expired input wait must end `failed` with an audit reason even without free capacity.
|
|
match self.store.expire_input_waits(self.config.batch_limit).await {
|
|
Ok(expired) if expired > 0 => {
|
|
debug!(expired, "expired input waits were failed")
|
|
}
|
|
Ok(_) => {}
|
|
Err(error) => {
|
|
warn!(error = %error, "input TTL sweep failed; retrying on next scan")
|
|
}
|
|
}
|
|
|
|
let available = self.config.max_inflight.saturating_sub(active.len());
|
|
if available > 0 {
|
|
let batch_limit = self
|
|
.config
|
|
.batch_limit
|
|
.min(u32::try_from(available).unwrap_or(u32::MAX));
|
|
match self
|
|
.store
|
|
.claim_batch(&self.config.owner, batch_limit, self.config.lease_ms)
|
|
.await
|
|
{
|
|
Ok(leases) => {
|
|
for lease in leases {
|
|
let store = self.store.clone();
|
|
let handler = self.handler.clone();
|
|
let owner = self.config.owner.clone();
|
|
let lease_ms = self.config.lease_ms;
|
|
let renewal_interval = self.config.renewal_interval;
|
|
let stopping = self.stopping.clone();
|
|
active.spawn(async move {
|
|
process_lease(
|
|
store,
|
|
handler,
|
|
owner,
|
|
lease,
|
|
lease_ms,
|
|
renewal_interval,
|
|
stopping,
|
|
)
|
|
.await;
|
|
});
|
|
}
|
|
}
|
|
Err(error) => {
|
|
warn!(error = %error, "bounded task claim failed; retrying on next scan");
|
|
}
|
|
}
|
|
}
|
|
|
|
tokio::select! {
|
|
_ = tokio::time::sleep(self.config.poll_interval) => {}
|
|
_ = async {
|
|
if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await }
|
|
} => {}
|
|
}
|
|
}
|
|
|
|
// Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded.
|
|
// The same stop flag reaches cooperative cancellation and lease-renewal loops.
|
|
self.stopping.store(true, Ordering::Release);
|
|
let deadline = Instant::now() + self.config.shutdown_timeout;
|
|
while !active.is_empty() {
|
|
if Instant::now() >= deadline {
|
|
warn!("graceful shutdown timeout reached; aborting remaining task attempts");
|
|
active.abort_all();
|
|
break;
|
|
}
|
|
tokio::select! {
|
|
_ = active.join_next() => {}
|
|
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
async fn process_lease(
|
|
store: Arc<dyn TaskStore>,
|
|
handler: Arc<dyn TaskHandler>,
|
|
owner: String,
|
|
lease: TaskLease,
|
|
lease_ms: i64,
|
|
renewal_interval: Duration,
|
|
stopping: Arc<AtomicBool>,
|
|
) {
|
|
let task_id = lease.record.task_id;
|
|
let fence = lease.fence;
|
|
let cancelled = Arc::new(AtomicBool::new(false));
|
|
|
|
// Pre-dispatch revalidation under the fresh lease.
|
|
if let Err(error) = handler.revalidate(&lease).await {
|
|
// A transient authority-store outage is not evidence that the work is invalid: leave the
|
|
// lease to expire so the bounded recovery scan can retry it without dispatching now.
|
|
if matches!(
|
|
&error,
|
|
TaskWorkerError::Store(TaskError::Unavailable { .. })
|
|
) {
|
|
debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery");
|
|
return;
|
|
}
|
|
debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch");
|
|
finish_failed(&store, &lease, "revalidation_failed").await;
|
|
return;
|
|
}
|
|
if stopping.load(Ordering::Acquire) {
|
|
// Graceful shutdown before the durable dispatch checkpoint: no outbound call can have
|
|
// happened, so the Task is left leased. The bounded recovery scan requeues it after the
|
|
// lease expires instead of turning a never-started attempt into a user-visible failure.
|
|
return;
|
|
}
|
|
|
|
let attempt_id = format!("attempt-{}", Uuid::now_v7());
|
|
|
|
// Background lease renewal covers the whole bounded retry budget, including the pinned backoff.
|
|
// A durable cancellation clears the lease, so the next renewal fails and the local work stops.
|
|
let renewal_store = store.clone();
|
|
let renewal_owner = owner.clone();
|
|
let renewal_cancelled = cancelled.clone();
|
|
let renewal_stopping = stopping.clone();
|
|
let renewal_task_id = task_id;
|
|
let renewal_handle = tokio::spawn(async move {
|
|
let mut interval = tokio::time::interval(renewal_interval);
|
|
interval.tick().await;
|
|
loop {
|
|
interval.tick().await;
|
|
if renewal_cancelled.load(Ordering::Acquire) {
|
|
break;
|
|
}
|
|
if renewal_stopping.load(Ordering::Acquire) {
|
|
renewal_cancelled.store(true, Ordering::Release);
|
|
break;
|
|
}
|
|
if renewal_store
|
|
.renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms)
|
|
.await
|
|
.is_err()
|
|
{
|
|
renewal_cancelled.store(true, Ordering::Release);
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
|
|
// The parent span names the leased Task generation; every individual outbound attempt adds its own
|
|
// `adapter_attempt` child span with the durable attempt ID committed by its checkpoint.
|
|
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, lease_ref = %attempt_id, fence);
|
|
if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) {
|
|
crank_trace::set_parent_from_trace_context(&span, &parent);
|
|
}
|
|
// Every durable write of this Task (checkpoint, attempt resolution, terminal state) happens inside
|
|
// the executor's single retry loop under this lease/fence; the worker only renews and stops.
|
|
let context = TaskAttemptContext::new(
|
|
store.clone(),
|
|
lease.clone(),
|
|
cancelled.clone(),
|
|
stopping.clone(),
|
|
);
|
|
let outcome = handler.execute(context).instrument(span).await;
|
|
|
|
cancelled.store(true, Ordering::Release);
|
|
renewal_handle.abort();
|
|
|
|
match outcome {
|
|
TaskAttemptOutcome::Completed => {
|
|
debug!(task_id = %task_id, "durable task completed");
|
|
}
|
|
TaskAttemptOutcome::Failed { code } => {
|
|
debug!(task_id = %task_id, code = %code, "durable task failed");
|
|
}
|
|
TaskAttemptOutcome::OutcomeUnknown => {
|
|
warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry");
|
|
}
|
|
TaskAttemptOutcome::WaitingForInput => {
|
|
debug!(task_id = %task_id, "durable task suspended until validated input arrives");
|
|
}
|
|
TaskAttemptOutcome::Stopped => {
|
|
debug!(task_id = %task_id, "durable task attempt stopped without a new outcome");
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn finish_failed(store: &Arc<dyn TaskStore>, lease: &TaskLease, code: &str) {
|
|
// The store owns the fate of a refused pre-dispatch attempt: a Task that already carries a proven
|
|
// ambiguity still ends `outcome_unknown`, never a plain failure (Story 4.4).
|
|
let _ = store.refuse_attempt(lease, code).await;
|
|
}
|