Files
crank/crates/crank-runtime/src/task_worker.rs
T
bsodfather 90e5357a08
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): продолжение ожидающей Task после ввода (Story 4.5)
Durable worker приостанавливает Task как input_required до dispatch, если
закреплённая схема требует отсутствующее поле; authorized schema-validated
update атомарно и с аудитом возобновляет тот же Task ровно один раз.
Replay/conflict/TTL безопасны, ожидание удерживает admission permit.
Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
2026-10-30 16:24:12 +00:00

434 lines
16 KiB
Rust

//! Fenced durable Task worker loop for Story 4.3.
//!
//! The worker is intentionally scheduler-ish: PostgreSQL is authoritative, polling is bounded,
//! Valkey may only wake it up (never represented here as correctness), and every lease-protected
//! write goes through CAS using owner + fencing token. The actual executable input and revalidation
//! policy arrive via [`TaskHandler`] implemented by the application layer; `TaskRecord.input` is a
//! redacted projection and is never dispatched.
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use crank_core::{TaskError, TaskLease, TaskStore};
use tokio::sync::watch;
use tokio::task::JoinSet;
use tracing::{Instrument, debug, warn};
use uuid::Uuid;
use crate::task_attempt::{TaskAttemptContext, TaskAttemptOutcome};
/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets.
#[derive(Clone, Debug)]
pub struct TaskWorkerConfig {
/// Stable random-ish owner identity for this process instance. Required lease owner.
pub owner: String,
/// Maximum number of Tasks claimed in one bounded PG scan.
pub batch_limit: u32,
/// Lease length in milliseconds. Recovery stays within 60s when <= 30s is used.
pub lease_ms: i64,
/// Authoritative PG polling interval when no wake-up hint is available.
pub poll_interval: Duration,
/// Renewal cadence while processing a lease.
pub renewal_interval: Duration,
/// Concurrent in-flight Tasks for this worker.
pub max_inflight: usize,
/// Maximum time to wait for active work during graceful shutdown.
pub shutdown_timeout: Duration,
}
impl Default for TaskWorkerConfig {
fn default() -> Self {
Self {
owner: format!("task-worker-{}", Uuid::now_v7()),
batch_limit: 8,
lease_ms: 30_000,
poll_interval: Duration::from_millis(250),
renewal_interval: Duration::from_millis(5_000),
max_inflight: 4,
shutdown_timeout: Duration::from_secs(30),
}
}
}
/// Application-owned seam for pre-dispatch revalidation and execution.
///
/// The worker owns lease renewal/CAS/fencing and passes a durable [`TaskAttemptContext`]. The
/// application implements [`crate::TaskAttemptAuthority`] and drives the pinned retry budget through
/// [`crate::RuntimeExecutor::execute_task_outcome`]; it must not trust `TaskRecord.input` as
/// executable payload.
#[async_trait]
pub trait TaskHandler: Send + Sync {
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>;
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome;
}
/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and
/// successful `checkpoint_dispatch`. It is neither `Clone` nor constructible outside this crate, and
/// it binds one durable Task attempt to its execution request.
pub struct TaskDispatchPermit {
task_id: crank_core::TaskId,
operation_id: String,
operation_version: u32,
workspace_id: String,
agent_id: Option<String>,
owner: String,
fence: u64,
attempt_id: String,
consumed: std::sync::atomic::AtomicBool,
}
impl TaskDispatchPermit {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
task_id: crank_core::TaskId,
operation_id: String,
operation_version: u32,
workspace_id: String,
agent_id: Option<String>,
owner: String,
fence: u64,
attempt_id: String,
) -> Self {
Self {
task_id,
operation_id,
operation_version,
workspace_id,
agent_id,
owner,
fence,
attempt_id,
consumed: std::sync::atomic::AtomicBool::new(false),
}
}
pub fn task_id(&self) -> crank_core::TaskId {
self.task_id
}
pub fn operation_id(&self) -> &str {
&self.operation_id
}
pub fn operation_version(&self) -> u32 {
self.operation_version
}
pub fn workspace_id(&self) -> &str {
&self.workspace_id
}
pub fn agent_id(&self) -> Option<&str> {
self.agent_id.as_deref()
}
pub fn owner(&self) -> &str {
&self.owner
}
pub fn fence(&self) -> u64 {
self.fence
}
pub fn attempt_id(&self) -> &str {
&self.attempt_id
}
pub fn is_consumed(&self) -> bool {
self.consumed.load(std::sync::atomic::Ordering::Acquire)
}
pub fn consume(&self) -> bool {
!self
.consumed
.swap(true, std::sync::atomic::Ordering::AcqRel)
}
}
#[derive(thiserror::Error, Debug)]
pub enum TaskWorkerError {
#[error("task store error: {0}")]
Store(#[from] TaskError),
#[error("invalid worker configuration: {0}")]
Config(String),
#[error("task terminal transition CAS failed")]
CasRejected,
}
pub struct TaskWorker {
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
config: TaskWorkerConfig,
stopping: Arc<AtomicBool>,
}
impl TaskWorker {
pub fn new(
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
config: TaskWorkerConfig,
) -> Result<Self, TaskWorkerError> {
if config.owner.is_empty() {
return Err(TaskWorkerError::Config(
"owner must not be empty".to_owned(),
));
}
if config.batch_limit == 0 || config.batch_limit > 64 || config.lease_ms <= 0 {
return Err(TaskWorkerError::Config(
"batch_limit must be 1..=64 and lease_ms must be positive".to_owned(),
));
}
if config.lease_ms > 30_000
|| config.poll_interval.is_zero()
|| config.poll_interval > Duration::from_secs(30)
|| config.renewal_interval.is_zero()
|| config.renewal_interval >= Duration::from_millis(config.lease_ms as u64)
|| config.max_inflight == 0
|| config.shutdown_timeout.is_zero()
|| config.shutdown_timeout > Duration::from_secs(30)
{
return Err(TaskWorkerError::Config(
"lease, polling, renewal, concurrency, or shutdown exceeds story timing bounds"
.to_owned(),
));
}
Ok(Self {
store,
handler,
config,
stopping: Arc::new(AtomicBool::new(false)),
})
}
/// Signal graceful shutdown. New claims stop; active handlers are cooperatively cancelled.
pub fn stop(&self) {
self.stopping.store(true, Ordering::Release);
}
/// Returns a handle whose drop does not stop the worker; call `stop()` on the worker.
pub fn stop_flag(&self) -> Arc<AtomicBool> {
self.stopping.clone()
}
/// Run until `stop()` is called. Uses PostgreSQL as the authoritative bounded scan.
pub async fn run(&self) -> Result<(), TaskWorkerError> {
self.run_inner(None).await
}
/// Run until either `stop()` or the provided shutdown receiver becomes true.
pub async fn run_until_shutdown(
&self,
mut shutdown: watch::Receiver<bool>,
) -> Result<(), TaskWorkerError> {
self.run_inner(Some(&mut shutdown)).await
}
async fn run_inner(
&self,
mut shutdown: Option<&mut watch::Receiver<bool>>,
) -> Result<(), TaskWorkerError> {
let mut active: JoinSet<()> = JoinSet::new();
loop {
if self.stopping.load(Ordering::Acquire) {
break;
}
if let Some(rx) = shutdown.as_mut()
&& *rx.borrow()
{
break;
}
// Reap finished attempts before claiming more.
while active.try_join_next().is_some() {}
// The bounded TTL sweep runs on every poll, including when all local slots are busy: an
// expired input wait must end `failed` with an audit reason even without free capacity.
match self.store.expire_input_waits(self.config.batch_limit).await {
Ok(expired) if expired > 0 => {
debug!(expired, "expired input waits were failed")
}
Ok(_) => {}
Err(error) => {
warn!(error = %error, "input TTL sweep failed; retrying on next scan")
}
}
let available = self.config.max_inflight.saturating_sub(active.len());
if available > 0 {
let batch_limit = self
.config
.batch_limit
.min(u32::try_from(available).unwrap_or(u32::MAX));
match self
.store
.claim_batch(&self.config.owner, batch_limit, self.config.lease_ms)
.await
{
Ok(leases) => {
for lease in leases {
let store = self.store.clone();
let handler = self.handler.clone();
let owner = self.config.owner.clone();
let lease_ms = self.config.lease_ms;
let renewal_interval = self.config.renewal_interval;
let stopping = self.stopping.clone();
active.spawn(async move {
process_lease(
store,
handler,
owner,
lease,
lease_ms,
renewal_interval,
stopping,
)
.await;
});
}
}
Err(error) => {
warn!(error = %error, "bounded task claim failed; retrying on next scan");
}
}
}
tokio::select! {
_ = tokio::time::sleep(self.config.poll_interval) => {}
_ = async {
if let Some(rx) = shutdown.as_mut() { let _ = rx.changed().await; } else { std::future::pending::<()>().await }
} => {}
}
}
// Graceful shutdown: stop claims (already) and let active work finish/checkpoint, bounded.
// The same stop flag reaches cooperative cancellation and lease-renewal loops.
self.stopping.store(true, Ordering::Release);
let deadline = Instant::now() + self.config.shutdown_timeout;
while !active.is_empty() {
if Instant::now() >= deadline {
warn!("graceful shutdown timeout reached; aborting remaining task attempts");
active.abort_all();
break;
}
tokio::select! {
_ = active.join_next() => {}
_ = tokio::time::sleep(Duration::from_millis(50)) => {}
}
}
Ok(())
}
}
async fn process_lease(
store: Arc<dyn TaskStore>,
handler: Arc<dyn TaskHandler>,
owner: String,
lease: TaskLease,
lease_ms: i64,
renewal_interval: Duration,
stopping: Arc<AtomicBool>,
) {
let task_id = lease.record.task_id;
let fence = lease.fence;
let cancelled = Arc::new(AtomicBool::new(false));
// Pre-dispatch revalidation under the fresh lease.
if let Err(error) = handler.revalidate(&lease).await {
// A transient authority-store outage is not evidence that the work is invalid: leave the
// lease to expire so the bounded recovery scan can retry it without dispatching now.
if matches!(
&error,
TaskWorkerError::Store(TaskError::Unavailable { .. })
) {
debug!(task_id = %task_id, error = %error, "transient revalidation failure; leaving work for recovery");
return;
}
debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch");
finish_failed(&store, &lease, "revalidation_failed").await;
return;
}
if stopping.load(Ordering::Acquire) {
// Graceful shutdown before the durable dispatch checkpoint: no outbound call can have
// happened, so the Task is left leased. The bounded recovery scan requeues it after the
// lease expires instead of turning a never-started attempt into a user-visible failure.
return;
}
let attempt_id = format!("attempt-{}", Uuid::now_v7());
// Background lease renewal covers the whole bounded retry budget, including the pinned backoff.
// A durable cancellation clears the lease, so the next renewal fails and the local work stops.
let renewal_store = store.clone();
let renewal_owner = owner.clone();
let renewal_cancelled = cancelled.clone();
let renewal_stopping = stopping.clone();
let renewal_task_id = task_id;
let renewal_handle = tokio::spawn(async move {
let mut interval = tokio::time::interval(renewal_interval);
interval.tick().await;
loop {
interval.tick().await;
if renewal_cancelled.load(Ordering::Acquire) {
break;
}
if renewal_stopping.load(Ordering::Acquire) {
renewal_cancelled.store(true, Ordering::Release);
break;
}
if renewal_store
.renew_lease(&renewal_owner, renewal_task_id, fence, lease_ms)
.await
.is_err()
{
renewal_cancelled.store(true, Ordering::Release);
break;
}
}
});
// The parent span names the leased Task generation; every individual outbound attempt adds its own
// `adapter_attempt` child span with the durable attempt ID committed by its checkpoint.
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, lease_ref = %attempt_id, fence);
if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) {
crank_trace::set_parent_from_trace_context(&span, &parent);
}
// Every durable write of this Task (checkpoint, attempt resolution, terminal state) happens inside
// the executor's single retry loop under this lease/fence; the worker only renews and stops.
let context = TaskAttemptContext::new(
store.clone(),
lease.clone(),
cancelled.clone(),
stopping.clone(),
);
let outcome = handler.execute(context).instrument(span).await;
cancelled.store(true, Ordering::Release);
renewal_handle.abort();
match outcome {
TaskAttemptOutcome::Completed => {
debug!(task_id = %task_id, "durable task completed");
}
TaskAttemptOutcome::Failed { code } => {
debug!(task_id = %task_id, code = %code, "durable task failed");
}
TaskAttemptOutcome::OutcomeUnknown => {
warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry");
}
TaskAttemptOutcome::WaitingForInput => {
debug!(task_id = %task_id, "durable task suspended until validated input arrives");
}
TaskAttemptOutcome::Stopped => {
debug!(task_id = %task_id, "durable task attempt stopped without a new outcome");
}
}
}
async fn finish_failed(store: &Arc<dyn TaskStore>, lease: &TaskLease, code: &str) {
// The store owns the fate of a refused pre-dispatch attempt: a Task that already carries a proven
// ambiguity still ends `outcome_unknown`, never a plain failure (Story 4.4).
let _ = store.refuse_attempt(lease, code).await;
}