CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если закреплённая схема требует отсутствующее поле; authorized schema-validated update атомарно и с аудитом возобновляет тот же Task ровно один раз. Replay/conflict/TTL безопасны, ожидание удерживает admission permit. Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
586 lines
25 KiB
Rust
586 lines
25 KiB
Rust
//! Durable per-outbound attempt persistence of a Task (Story 4.4): the dispatch checkpoint, the
|
|
//! single-winner attempt resolution and the owner-scoped cancellation live here so the acceptance
|
|
//! and retention half of the store stays readable.
|
|
|
|
use crank_core::{
|
|
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
|
|
AttemptResolution, BeginAttempt, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest,
|
|
TaskEnvelope, TaskError, TaskId, TaskRecord, TaskState,
|
|
};
|
|
use sqlx::Row;
|
|
use uuid::Uuid;
|
|
|
|
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row, unavailable_row};
|
|
|
|
const ATTEMPT_PROJECTION: &str = "attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
|
|
upstream_key_digest, retry_allowed,
|
|
(extract(epoch from checkpoint_at) * 1000)::bigint as checkpoint_at_ms,
|
|
(extract(epoch from resolved_at) * 1000)::bigint as resolved_at_ms,
|
|
outcome, certainty, phase";
|
|
|
|
/// Bounded attempt-resolution helper: `retry` may only be armed when the attempt itself is marked
|
|
/// safe-to-retry, the pinned budget still has room and the accepted absolute deadline still does.
|
|
impl PostgresTaskStore {
|
|
fn retry_delay(
|
|
&self,
|
|
record: &TaskRecord,
|
|
lease: &AttemptLease,
|
|
safe: bool,
|
|
now: i64,
|
|
) -> Option<i64> {
|
|
let delay = record.pins.retry.backoff_before(lease.attempt.ordinal + 1);
|
|
(safe
|
|
&& lease.attempt.retry_allowed
|
|
&& lease.attempt.ordinal < record.pins.retry.max_attempts
|
|
&& now.saturating_add(delay) < record.pins.execution_deadline_unix_ms)
|
|
.then_some(delay)
|
|
}
|
|
}
|
|
|
|
impl PostgresTaskStore {
|
|
/// Authoritative word on whether an earlier resolved attempt really reached upstream despite
|
|
/// the current one not having dispatched. Returns Some(phase) only when the latest resolved
|
|
/// attempt that actually dispatched (phase = 'dispatched') is still of unknown certainty.
|
|
pub(super) async fn unproven_dispatch_phase(
|
|
&self,
|
|
task_id: TaskId,
|
|
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
|
) -> Result<Option<String>, TaskError> {
|
|
let row = sqlx::query(
|
|
"select certainty, phase from task_attempts where task_id = $1 and resolved_at is not null and phase = 'dispatched' order by ordinal desc limit 1",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.fetch_optional(&mut **transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
return Ok(None);
|
|
};
|
|
let certainty: Option<String> = row.try_get("certainty").map_err(Self::unavailable)?;
|
|
if certainty.as_deref() != Some(AttemptCertainty::as_str(AttemptCertainty::Unknown)) {
|
|
return Ok(None);
|
|
}
|
|
let phase: Option<String> = row.try_get("phase").map_err(Self::unavailable)?;
|
|
Ok(Some(phase.unwrap_or_else(|| "dispatched".to_owned())))
|
|
}
|
|
}
|
|
|
|
fn attempt_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskAttempt, TaskError> {
|
|
let attempt_id: String = row.try_get("attempt_id").map_err(unavailable_row)?;
|
|
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
|
|
let outcome: Option<String> = row.try_get("outcome").map_err(unavailable_row)?;
|
|
let certainty: Option<String> = row.try_get("certainty").map_err(unavailable_row)?;
|
|
let phase: Option<String> = row.try_get("phase").map_err(unavailable_row)?;
|
|
Ok(TaskAttempt {
|
|
attempt_id,
|
|
task_id: TaskId::parse(&task_id.to_string())?,
|
|
ordinal: u32::try_from(row.try_get::<i32, _>("ordinal").map_err(unavailable_row)?)
|
|
.unwrap_or_default(),
|
|
owner: row.try_get("owner").map_err(unavailable_row)?,
|
|
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
|
|
.unwrap_or_default(),
|
|
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
|
|
upstream_key_digest: row
|
|
.try_get("upstream_key_digest")
|
|
.map_err(unavailable_row)?,
|
|
retry_allowed: row.try_get("retry_allowed").map_err(unavailable_row)?,
|
|
checkpoint_at_unix_ms: row.try_get("checkpoint_at_ms").map_err(unavailable_row)?,
|
|
resolved_at_unix_ms: row.try_get("resolved_at_ms").map_err(unavailable_row)?,
|
|
outcome: outcome
|
|
.as_deref()
|
|
.map(AttemptOutcomeClass::parse)
|
|
.transpose()?,
|
|
certainty: certainty
|
|
.as_deref()
|
|
.map(AttemptCertainty::parse)
|
|
.transpose()?,
|
|
phase: phase.as_deref().map(AttemptPhase::parse).transpose()?,
|
|
})
|
|
}
|
|
|
|
impl PostgresTaskStore {
|
|
pub(super) async fn checkpoint_dispatch_attempt(
|
|
&self,
|
|
request: BeginAttempt,
|
|
) -> Result<AttemptLease, TaskError> {
|
|
request.validate()?;
|
|
let mut transaction = self.start().await?;
|
|
// One row lock serializes this checkpoint with claim, cancellation and the resolution of the
|
|
// same Task, so a cancel that wins first can never be followed by an outbound call.
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
|
|
)))
|
|
.bind(request.task_id.as_uuid())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let record = record_from_row(&row)?;
|
|
// The checkpoint must bind the payload this attempt actually executes: after a committed
|
|
// input continuation that is the effective fingerprint, not the unchanged acceptance one.
|
|
if record.effective_input_fingerprint != request.input_fingerprint {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::FingerprintConflict);
|
|
}
|
|
let now = crank_core::current_unix_ms();
|
|
let eligible = record.state == TaskState::Working
|
|
&& record.lease_owner.as_deref() == Some(request.owner.as_str())
|
|
&& record.fence == request.fence
|
|
&& record.version == request.expected_version
|
|
&& record.current_attempt_id.is_none()
|
|
&& record.cancel_requested_at_unix_ms.is_none()
|
|
&& record.lease_until_unix_ms.is_some_and(|until| until > now)
|
|
&& record.attempts < record.pins.retry.max_attempts
|
|
&& record.pins.execution_deadline_unix_ms > now
|
|
&& record.next_attempt_at_unix_ms.is_none_or(|at| at <= now);
|
|
if !eligible {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
let ordinal = record.attempts.saturating_add(1);
|
|
sqlx::query(
|
|
"insert into task_attempts (
|
|
attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
|
|
upstream_key_digest, retry_allowed
|
|
) values ($1, $2, $3, $4, $5, $6, $7, $8)",
|
|
)
|
|
.bind(&request.attempt_id)
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(i32::try_from(ordinal).unwrap_or(i32::MAX))
|
|
.bind(&request.owner)
|
|
.bind(i64::try_from(request.fence).unwrap_or_default())
|
|
.bind(&request.input_fingerprint)
|
|
.bind(request.upstream_key_digest.as_deref())
|
|
.bind(request.retry_allowed)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set current_attempt_id = $1,
|
|
dispatch_started_at = now(),
|
|
attempts = attempts + 1,
|
|
next_attempt_at = null,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $2 and current_attempt_id is null and version = $3
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(&request.attempt_id)
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(i64::from(record.version))
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let task = record_from_row(&row)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
let attempt = TaskAttempt {
|
|
attempt_id: request.attempt_id,
|
|
task_id: request.task_id,
|
|
ordinal,
|
|
owner: request.owner,
|
|
fence: request.fence,
|
|
input_fingerprint: request.input_fingerprint,
|
|
upstream_key_digest: request.upstream_key_digest,
|
|
retry_allowed: request.retry_allowed,
|
|
checkpoint_at_unix_ms: task.dispatch_started_at_unix_ms.unwrap_or(now),
|
|
resolved_at_unix_ms: None,
|
|
outcome: None,
|
|
certainty: None,
|
|
phase: None,
|
|
};
|
|
Ok(AttemptLease { attempt, task })
|
|
}
|
|
|
|
pub(super) async fn resolve_attempt_outcome(
|
|
&self,
|
|
lease: &AttemptLease,
|
|
outcome: AttemptOutcome,
|
|
) -> Result<AttemptResolution, TaskError> {
|
|
let mut transaction = self.start().await?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
|
|
)))
|
|
.bind(lease.attempt.task_id.as_uuid())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let record = record_from_row(&row)?;
|
|
let now = crank_core::current_unix_ms();
|
|
let prior_uncertain_phase = self
|
|
.unproven_dispatch_phase(lease.attempt.task_id, &mut transaction)
|
|
.await?;
|
|
// Only the current attempt of the current owner, under the version it was admitted with, may
|
|
// be resolved. A stale worker can never append a second outcome to the history or the Task.
|
|
let current = record.state == TaskState::Working
|
|
&& record.lease_owner.as_deref() == Some(lease.attempt.owner.as_str())
|
|
&& record.fence == lease.attempt.fence
|
|
&& record.version == lease.task.version
|
|
&& record.current_attempt_id.as_deref() == Some(lease.attempt.attempt_id.as_str())
|
|
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
|
|
if !current {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
let (class, certainty, phase) = match &outcome {
|
|
AttemptOutcome::Completed(_) => (
|
|
AttemptOutcomeClass::Completed,
|
|
AttemptCertainty::Known,
|
|
AttemptPhase::Dispatched,
|
|
),
|
|
AttemptOutcome::Failed { phase, .. } => {
|
|
(AttemptOutcomeClass::Failed, AttemptCertainty::Known, *phase)
|
|
}
|
|
AttemptOutcome::Uncertain { phase, .. } => (
|
|
AttemptOutcomeClass::OutcomeUnknown,
|
|
AttemptCertainty::Unknown,
|
|
*phase,
|
|
),
|
|
};
|
|
// A failed or uncertain attempt that may be safely repeated keeps the lease and arms the
|
|
// pinned backoff. Exhausted budget or deadline never turns uncertainty into a plain failure.
|
|
let retry = match &outcome {
|
|
AttemptOutcome::Failed {
|
|
retryable,
|
|
phase: _,
|
|
error: _,
|
|
} => self.retry_delay(&record, lease, *retryable, now),
|
|
AttemptOutcome::Uncertain {
|
|
repeatable,
|
|
phase: _,
|
|
error: _,
|
|
} => self.retry_delay(&record, lease, *repeatable, now),
|
|
AttemptOutcome::Completed(_) => None,
|
|
};
|
|
let attempt_class = if retry.is_some() {
|
|
AttemptOutcomeClass::RetryScheduled
|
|
} else {
|
|
class
|
|
};
|
|
let closed = sqlx::query(
|
|
"update task_attempts
|
|
set resolved_at = now(), outcome = $1, certainty = $2, phase = $3
|
|
where attempt_id = $4 and resolved_at is null",
|
|
)
|
|
.bind(attempt_class.as_str())
|
|
.bind(certainty.as_str())
|
|
.bind(phase.as_str())
|
|
.bind(&lease.attempt.attempt_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
if closed.rows_affected() != 1 {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
let (state, result, error, keep_lease, next_attempt_at) = match outcome {
|
|
AttemptOutcome::Completed(result) => {
|
|
(TaskState::Completed, Some(result), None, false, None)
|
|
}
|
|
AttemptOutcome::Failed {
|
|
error,
|
|
retryable: _,
|
|
phase,
|
|
} => match retry {
|
|
Some(delay) => (
|
|
TaskState::Working,
|
|
None,
|
|
None,
|
|
true,
|
|
Some(now.saturating_add(delay)),
|
|
),
|
|
None => {
|
|
if phase == AttemptPhase::BeforeDispatch {
|
|
if let Some(prior) = prior_uncertain_phase.as_ref() {
|
|
let mut value = error.value().clone();
|
|
if let serde_json::Value::Object(map) = &mut value {
|
|
map.insert(
|
|
"phase".to_string(),
|
|
serde_json::Value::String(prior.clone()),
|
|
);
|
|
}
|
|
let replaced = TaskEnvelope::try_redacted(&value)?;
|
|
(TaskState::OutcomeUnknown, None, Some(replaced), false, None)
|
|
} else {
|
|
(TaskState::Failed, None, Some(error), false, None)
|
|
}
|
|
} else {
|
|
(TaskState::Failed, None, Some(error), false, None)
|
|
}
|
|
}
|
|
},
|
|
AttemptOutcome::Uncertain {
|
|
error,
|
|
repeatable: _,
|
|
phase: _,
|
|
} => match retry {
|
|
Some(delay) => (
|
|
TaskState::Working,
|
|
None,
|
|
None,
|
|
true,
|
|
Some(now.saturating_add(delay)),
|
|
),
|
|
None => (TaskState::OutcomeUnknown, None, Some(error), false, None),
|
|
},
|
|
};
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = $1,
|
|
result_envelope = $2,
|
|
error_envelope = $3,
|
|
lease_owner = case when $4 then lease_owner else null end,
|
|
lease_until = case when $4 then lease_until else null end,
|
|
-- A retry-scheduled attempt is already resolved, so it no longer holds the Task:
|
|
-- the next durable attempt needs its own checkpoint under the same lease.
|
|
current_attempt_id = case when $8 then null else current_attempt_id end,
|
|
next_attempt_at = case when $5::bigint is null then null
|
|
else to_timestamp($5::double precision / 1000.0) end,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $6 and version = $7
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(state.as_str())
|
|
.bind(result.as_ref().map(|value| value.value().clone()))
|
|
.bind(error.as_ref().map(|value| value.value().clone()))
|
|
.bind(keep_lease)
|
|
.bind(next_attempt_at)
|
|
.bind(lease.attempt.task_id.as_uuid())
|
|
.bind(i64::from(record.version))
|
|
.bind(retry.is_some())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let task = record_from_row(&row)?;
|
|
if !keep_lease {
|
|
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
|
.bind(lease.attempt.task_id.as_uuid())
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
}
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(match retry {
|
|
Some(_) => AttemptResolution::RetryAt {
|
|
next_attempt_at_unix_ms: next_attempt_at.unwrap_or(now),
|
|
task,
|
|
},
|
|
None if state == TaskState::OutcomeUnknown => AttemptResolution::Unknown(task),
|
|
None => AttemptResolution::Finished(task),
|
|
})
|
|
}
|
|
|
|
pub(super) async fn cancel_task(
|
|
&self,
|
|
request: TaskCancelRequest,
|
|
) -> Result<TaskCancelOutcome, TaskError> {
|
|
request.validate()?;
|
|
let access = &request.access;
|
|
let mut transaction = self.start().await?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks
|
|
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
|
and owner_id = $4 and authorization_scope = $5
|
|
for update"
|
|
)))
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let record = record_from_row(&row)?;
|
|
if let Some(expected) = request.expected_version
|
|
&& record.version != expected
|
|
{
|
|
transaction.rollback().await.ok();
|
|
return Ok(TaskCancelOutcome::Conflict(record));
|
|
}
|
|
if record.state.is_terminal() {
|
|
transaction.rollback().await.ok();
|
|
return Ok(TaskCancelOutcome::AlreadyTerminal(record));
|
|
}
|
|
if record.state == TaskState::OutcomeUnknown {
|
|
transaction.rollback().await.ok();
|
|
return Ok(TaskCancelOutcome::OutcomeUnknown(record));
|
|
}
|
|
// A checkpointed attempt may already have reached the upstream: the honest answer is the
|
|
// non-terminal uncertain state, never a plain cancellation and never a retry.
|
|
let checkpointed = record.current_attempt_id.clone();
|
|
// Preserve an uncertainty recorded by a previously resolved attempt even after RetryAt
|
|
// cleared current_attempt_id: the request already reached the upstream in that case.
|
|
let prior_uncertain = self
|
|
.unproven_dispatch_phase(request.task_id, &mut transaction)
|
|
.await?
|
|
.is_some();
|
|
let (state, code) = match (&checkpointed, prior_uncertain) {
|
|
(Some(_), _) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
|
|
(None, true) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
|
|
(None, false) => (TaskState::Cancelled, "cancelled_before_dispatch"),
|
|
};
|
|
if let Some(attempt_id) = checkpointed {
|
|
let closed = sqlx::query(
|
|
"update task_attempts
|
|
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
|
|
phase = 'dispatched'
|
|
where task_id = $1 and attempt_id = $2 and resolved_at is null",
|
|
)
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(&attempt_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let _ = closed;
|
|
}
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = $1,
|
|
error_envelope = jsonb_build_object('code', $2),
|
|
cancel_requested_at = now(),
|
|
cancel_reason = $3,
|
|
lease_owner = null,
|
|
lease_until = null,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $4 and version = $5
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(state.as_str())
|
|
.bind(code)
|
|
.bind(&request.reason)
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(i64::from(record.version))
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let task = record_from_row(&row)?;
|
|
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
|
.bind(request.task_id.as_uuid())
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(match state {
|
|
TaskState::Cancelled => TaskCancelOutcome::Cancelled(task),
|
|
_ => TaskCancelOutcome::OutcomeUnknown(task),
|
|
})
|
|
}
|
|
|
|
pub(super) async fn refuse_attempt_resolution(
|
|
&self,
|
|
lease: &crank_core::TaskLease,
|
|
code: &str,
|
|
) -> Result<TaskRecord, TaskError> {
|
|
let mut transaction = self.start().await?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks\n where task_id = $1\n for update"
|
|
)))
|
|
.bind(lease.record.task_id.as_uuid())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
|
let now = crank_core::current_unix_ms();
|
|
let eligible = matches!(record.state, TaskState::Queued | TaskState::Working)
|
|
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
|
|
&& record.fence == lease.fence
|
|
&& record.version == lease.record.version
|
|
&& record.current_attempt_id.is_none()
|
|
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
|
|
if !eligible {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
let prior_phase = self
|
|
.unproven_dispatch_phase(lease.record.task_id, &mut transaction)
|
|
.await?;
|
|
let uncertain = prior_phase.is_some();
|
|
let state = if uncertain {
|
|
TaskState::OutcomeUnknown
|
|
} else {
|
|
TaskState::Failed
|
|
};
|
|
let phase = prior_phase.unwrap_or_else(|| "before_dispatch".to_owned());
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks\n set state = $1,\n error_envelope = jsonb_build_object('code', $2, 'phase', $3),\n lease_owner = null,\n lease_until = null,\n next_attempt_at = null,\n version = version + 1,\n updated_at = now()\n where task_id = $4 and version = $5\n returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(state.as_str())
|
|
.bind(code)
|
|
.bind(&phase)
|
|
.bind(lease.record.task_id.as_uuid())
|
|
.bind(i64::from(record.version))
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let task = record_from_row(&row).map_err(Self::unavailable)?;
|
|
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
|
.bind(lease.record.task_id.as_uuid())
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(task)
|
|
}
|
|
|
|
pub(super) async fn attempt_history(
|
|
&self,
|
|
access: &TaskAccess,
|
|
task_id: TaskId,
|
|
) -> Result<Vec<TaskAttempt>, TaskError> {
|
|
access.validate()?;
|
|
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {ATTEMPT_PROJECTION} from task_attempts
|
|
where task_id = $1 and exists (
|
|
select 1 from tasks
|
|
where tasks.task_id = task_attempts.task_id and tasks.workspace_id = $2
|
|
and tasks.owner_kind = $3 and tasks.owner_id = $4
|
|
and tasks.authorization_scope = $5
|
|
)
|
|
order by ordinal"
|
|
)))
|
|
.bind(task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.fetch_all(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
rows.iter().map(attempt_from_row).collect()
|
|
}
|
|
}
|