Files
crank/crates/crank-registry/src/postgres/tasks/attempts.rs
T
bsodfather 90e5357a08
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): продолжение ожидающей Task после ввода (Story 4.5)
Durable worker приостанавливает Task как input_required до dispatch, если
закреплённая схема требует отсутствующее поле; authorized schema-validated
update атомарно и с аудитом возобновляет тот же Task ровно один раз.
Replay/conflict/TTL безопасны, ожидание удерживает admission permit.
Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
2026-10-30 16:24:12 +00:00

586 lines
25 KiB
Rust

//! Durable per-outbound attempt persistence of a Task (Story 4.4): the dispatch checkpoint, the
//! single-winner attempt resolution and the owner-scoped cancellation live here so the acceptance
//! and retention half of the store stays readable.
use crank_core::{
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest,
TaskEnvelope, TaskError, TaskId, TaskRecord, TaskState,
};
use sqlx::Row;
use uuid::Uuid;
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row, unavailable_row};
const ATTEMPT_PROJECTION: &str = "attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
upstream_key_digest, retry_allowed,
(extract(epoch from checkpoint_at) * 1000)::bigint as checkpoint_at_ms,
(extract(epoch from resolved_at) * 1000)::bigint as resolved_at_ms,
outcome, certainty, phase";
/// Bounded attempt-resolution helper: `retry` may only be armed when the attempt itself is marked
/// safe-to-retry, the pinned budget still has room and the accepted absolute deadline still does.
impl PostgresTaskStore {
fn retry_delay(
&self,
record: &TaskRecord,
lease: &AttemptLease,
safe: bool,
now: i64,
) -> Option<i64> {
let delay = record.pins.retry.backoff_before(lease.attempt.ordinal + 1);
(safe
&& lease.attempt.retry_allowed
&& lease.attempt.ordinal < record.pins.retry.max_attempts
&& now.saturating_add(delay) < record.pins.execution_deadline_unix_ms)
.then_some(delay)
}
}
impl PostgresTaskStore {
/// Authoritative word on whether an earlier resolved attempt really reached upstream despite
/// the current one not having dispatched. Returns Some(phase) only when the latest resolved
/// attempt that actually dispatched (phase = 'dispatched') is still of unknown certainty.
pub(super) async fn unproven_dispatch_phase(
&self,
task_id: TaskId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<Option<String>, TaskError> {
let row = sqlx::query(
"select certainty, phase from task_attempts where task_id = $1 and resolved_at is not null and phase = 'dispatched' order by ordinal desc limit 1",
)
.bind(task_id.as_uuid())
.fetch_optional(&mut **transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Ok(None);
};
let certainty: Option<String> = row.try_get("certainty").map_err(Self::unavailable)?;
if certainty.as_deref() != Some(AttemptCertainty::as_str(AttemptCertainty::Unknown)) {
return Ok(None);
}
let phase: Option<String> = row.try_get("phase").map_err(Self::unavailable)?;
Ok(Some(phase.unwrap_or_else(|| "dispatched".to_owned())))
}
}
fn attempt_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskAttempt, TaskError> {
let attempt_id: String = row.try_get("attempt_id").map_err(unavailable_row)?;
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let outcome: Option<String> = row.try_get("outcome").map_err(unavailable_row)?;
let certainty: Option<String> = row.try_get("certainty").map_err(unavailable_row)?;
let phase: Option<String> = row.try_get("phase").map_err(unavailable_row)?;
Ok(TaskAttempt {
attempt_id,
task_id: TaskId::parse(&task_id.to_string())?,
ordinal: u32::try_from(row.try_get::<i32, _>("ordinal").map_err(unavailable_row)?)
.unwrap_or_default(),
owner: row.try_get("owner").map_err(unavailable_row)?,
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
.unwrap_or_default(),
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
upstream_key_digest: row
.try_get("upstream_key_digest")
.map_err(unavailable_row)?,
retry_allowed: row.try_get("retry_allowed").map_err(unavailable_row)?,
checkpoint_at_unix_ms: row.try_get("checkpoint_at_ms").map_err(unavailable_row)?,
resolved_at_unix_ms: row.try_get("resolved_at_ms").map_err(unavailable_row)?,
outcome: outcome
.as_deref()
.map(AttemptOutcomeClass::parse)
.transpose()?,
certainty: certainty
.as_deref()
.map(AttemptCertainty::parse)
.transpose()?,
phase: phase.as_deref().map(AttemptPhase::parse).transpose()?,
})
}
impl PostgresTaskStore {
pub(super) async fn checkpoint_dispatch_attempt(
&self,
request: BeginAttempt,
) -> Result<AttemptLease, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
// One row lock serializes this checkpoint with claim, cancellation and the resolution of the
// same Task, so a cancel that wins first can never be followed by an outbound call.
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
// The checkpoint must bind the payload this attempt actually executes: after a committed
// input continuation that is the effective fingerprint, not the unchanged acceptance one.
if record.effective_input_fingerprint != request.input_fingerprint {
transaction.rollback().await.ok();
return Err(TaskError::FingerprintConflict);
}
let now = crank_core::current_unix_ms();
let eligible = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(request.owner.as_str())
&& record.fence == request.fence
&& record.version == request.expected_version
&& record.current_attempt_id.is_none()
&& record.cancel_requested_at_unix_ms.is_none()
&& record.lease_until_unix_ms.is_some_and(|until| until > now)
&& record.attempts < record.pins.retry.max_attempts
&& record.pins.execution_deadline_unix_ms > now
&& record.next_attempt_at_unix_ms.is_none_or(|at| at <= now);
if !eligible {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let ordinal = record.attempts.saturating_add(1);
sqlx::query(
"insert into task_attempts (
attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
upstream_key_digest, retry_allowed
) values ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(&request.attempt_id)
.bind(request.task_id.as_uuid())
.bind(i32::try_from(ordinal).unwrap_or(i32::MAX))
.bind(&request.owner)
.bind(i64::try_from(request.fence).unwrap_or_default())
.bind(&request.input_fingerprint)
.bind(request.upstream_key_digest.as_deref())
.bind(request.retry_allowed)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set current_attempt_id = $1,
dispatch_started_at = now(),
attempts = attempts + 1,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id = $2 and current_attempt_id is null and version = $3
returning {TASK_PROJECTION}"
)))
.bind(&request.attempt_id)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
transaction.commit().await.map_err(Self::unavailable)?;
let attempt = TaskAttempt {
attempt_id: request.attempt_id,
task_id: request.task_id,
ordinal,
owner: request.owner,
fence: request.fence,
input_fingerprint: request.input_fingerprint,
upstream_key_digest: request.upstream_key_digest,
retry_allowed: request.retry_allowed,
checkpoint_at_unix_ms: task.dispatch_started_at_unix_ms.unwrap_or(now),
resolved_at_unix_ms: None,
outcome: None,
certainty: None,
phase: None,
};
Ok(AttemptLease { attempt, task })
}
pub(super) async fn resolve_attempt_outcome(
&self,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(lease.attempt.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
let now = crank_core::current_unix_ms();
let prior_uncertain_phase = self
.unproven_dispatch_phase(lease.attempt.task_id, &mut transaction)
.await?;
// Only the current attempt of the current owner, under the version it was admitted with, may
// be resolved. A stale worker can never append a second outcome to the history or the Task.
let current = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(lease.attempt.owner.as_str())
&& record.fence == lease.attempt.fence
&& record.version == lease.task.version
&& record.current_attempt_id.as_deref() == Some(lease.attempt.attempt_id.as_str())
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
if !current {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let (class, certainty, phase) = match &outcome {
AttemptOutcome::Completed(_) => (
AttemptOutcomeClass::Completed,
AttemptCertainty::Known,
AttemptPhase::Dispatched,
),
AttemptOutcome::Failed { phase, .. } => {
(AttemptOutcomeClass::Failed, AttemptCertainty::Known, *phase)
}
AttemptOutcome::Uncertain { phase, .. } => (
AttemptOutcomeClass::OutcomeUnknown,
AttemptCertainty::Unknown,
*phase,
),
};
// A failed or uncertain attempt that may be safely repeated keeps the lease and arms the
// pinned backoff. Exhausted budget or deadline never turns uncertainty into a plain failure.
let retry = match &outcome {
AttemptOutcome::Failed {
retryable,
phase: _,
error: _,
} => self.retry_delay(&record, lease, *retryable, now),
AttemptOutcome::Uncertain {
repeatable,
phase: _,
error: _,
} => self.retry_delay(&record, lease, *repeatable, now),
AttemptOutcome::Completed(_) => None,
};
let attempt_class = if retry.is_some() {
AttemptOutcomeClass::RetryScheduled
} else {
class
};
let closed = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = $1, certainty = $2, phase = $3
where attempt_id = $4 and resolved_at is null",
)
.bind(attempt_class.as_str())
.bind(certainty.as_str())
.bind(phase.as_str())
.bind(&lease.attempt.attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if closed.rows_affected() != 1 {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let (state, result, error, keep_lease, next_attempt_at) = match outcome {
AttemptOutcome::Completed(result) => {
(TaskState::Completed, Some(result), None, false, None)
}
AttemptOutcome::Failed {
error,
retryable: _,
phase,
} => match retry {
Some(delay) => (
TaskState::Working,
None,
None,
true,
Some(now.saturating_add(delay)),
),
None => {
if phase == AttemptPhase::BeforeDispatch {
if let Some(prior) = prior_uncertain_phase.as_ref() {
let mut value = error.value().clone();
if let serde_json::Value::Object(map) = &mut value {
map.insert(
"phase".to_string(),
serde_json::Value::String(prior.clone()),
);
}
let replaced = TaskEnvelope::try_redacted(&value)?;
(TaskState::OutcomeUnknown, None, Some(replaced), false, None)
} else {
(TaskState::Failed, None, Some(error), false, None)
}
} else {
(TaskState::Failed, None, Some(error), false, None)
}
}
},
AttemptOutcome::Uncertain {
error,
repeatable: _,
phase: _,
} => match retry {
Some(delay) => (
TaskState::Working,
None,
None,
true,
Some(now.saturating_add(delay)),
),
None => (TaskState::OutcomeUnknown, None, Some(error), false, None),
},
};
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
lease_owner = case when $4 then lease_owner else null end,
lease_until = case when $4 then lease_until else null end,
-- A retry-scheduled attempt is already resolved, so it no longer holds the Task:
-- the next durable attempt needs its own checkpoint under the same lease.
current_attempt_id = case when $8 then null else current_attempt_id end,
next_attempt_at = case when $5::bigint is null then null
else to_timestamp($5::double precision / 1000.0) end,
version = version + 1,
updated_at = now()
where task_id = $6 and version = $7
returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(result.as_ref().map(|value| value.value().clone()))
.bind(error.as_ref().map(|value| value.value().clone()))
.bind(keep_lease)
.bind(next_attempt_at)
.bind(lease.attempt.task_id.as_uuid())
.bind(i64::from(record.version))
.bind(retry.is_some())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
if !keep_lease {
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(lease.attempt.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
}
transaction.commit().await.map_err(Self::unavailable)?;
Ok(match retry {
Some(_) => AttemptResolution::RetryAt {
next_attempt_at_unix_ms: next_attempt_at.unwrap_or(now),
task,
},
None if state == TaskState::OutcomeUnknown => AttemptResolution::Unknown(task),
None => AttemptResolution::Finished(task),
})
}
pub(super) async fn cancel_task(
&self,
request: TaskCancelRequest,
) -> Result<TaskCancelOutcome, TaskError> {
request.validate()?;
let access = &request.access;
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5
for update"
)))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
if let Some(expected) = request.expected_version
&& record.version != expected
{
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::Conflict(record));
}
if record.state.is_terminal() {
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::AlreadyTerminal(record));
}
if record.state == TaskState::OutcomeUnknown {
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::OutcomeUnknown(record));
}
// A checkpointed attempt may already have reached the upstream: the honest answer is the
// non-terminal uncertain state, never a plain cancellation and never a retry.
let checkpointed = record.current_attempt_id.clone();
// Preserve an uncertainty recorded by a previously resolved attempt even after RetryAt
// cleared current_attempt_id: the request already reached the upstream in that case.
let prior_uncertain = self
.unproven_dispatch_phase(request.task_id, &mut transaction)
.await?
.is_some();
let (state, code) = match (&checkpointed, prior_uncertain) {
(Some(_), _) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
(None, true) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
(None, false) => (TaskState::Cancelled, "cancelled_before_dispatch"),
};
if let Some(attempt_id) = checkpointed {
let closed = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
phase = 'dispatched'
where task_id = $1 and attempt_id = $2 and resolved_at is null",
)
.bind(request.task_id.as_uuid())
.bind(&attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let _ = closed;
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
error_envelope = jsonb_build_object('code', $2),
cancel_requested_at = now(),
cancel_reason = $3,
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $4 and version = $5
returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(code)
.bind(&request.reason)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(match state {
TaskState::Cancelled => TaskCancelOutcome::Cancelled(task),
_ => TaskCancelOutcome::OutcomeUnknown(task),
})
}
pub(super) async fn refuse_attempt_resolution(
&self,
lease: &crank_core::TaskLease,
code: &str,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks\n where task_id = $1\n for update"
)))
.bind(lease.record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
let now = crank_core::current_unix_ms();
let eligible = matches!(record.state, TaskState::Queued | TaskState::Working)
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
&& record.fence == lease.fence
&& record.version == lease.record.version
&& record.current_attempt_id.is_none()
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
if !eligible {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let prior_phase = self
.unproven_dispatch_phase(lease.record.task_id, &mut transaction)
.await?;
let uncertain = prior_phase.is_some();
let state = if uncertain {
TaskState::OutcomeUnknown
} else {
TaskState::Failed
};
let phase = prior_phase.unwrap_or_else(|| "before_dispatch".to_owned());
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks\n set state = $1,\n error_envelope = jsonb_build_object('code', $2, 'phase', $3),\n lease_owner = null,\n lease_until = null,\n next_attempt_at = null,\n version = version + 1,\n updated_at = now()\n where task_id = $4 and version = $5\n returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(code)
.bind(&phase)
.bind(lease.record.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row).map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(lease.record.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(task)
}
pub(super) async fn attempt_history(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<TaskAttempt>, TaskError> {
access.validate()?;
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {ATTEMPT_PROJECTION} from task_attempts
where task_id = $1 and exists (
select 1 from tasks
where tasks.task_id = task_attempts.task_id and tasks.workspace_id = $2
and tasks.owner_kind = $3 and tasks.owner_id = $4
and tasks.authorization_scope = $5
)
order by ordinal"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_all(&self.pool)
.await
.map_err(Self::unavailable)?;
rows.iter().map(attempt_from_row).collect()
}
}