Files
crank/crates/crank-registry/src/postgres/tasks/mod.rs
T
bsodfather a3057de51e
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): безопасные повторы и отмена durable Task (Story 4.4)
Durable per-outbound attempt до отправки запроса, bounded retry/read,
server-owned verified writes, честный cancel/unknown, deadline/fencing,
additive schema v29. Staged renames task.rs/tasks.rs разложены на модули.

Includes disclosed cosmetic parity exception: единственная правка вне
принятой дельты — удалена лишняя пустая строка в конце
scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
2026-10-30 12:21:57 +00:00

907 lines
37 KiB
Rust

//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
//!
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
//!
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
use async_trait::async_trait;
use crank_core::{
AttemptLease, AttemptOutcome, AttemptResolution, BeginAttempt, PlatformApiKeyId,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskLease,
TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode,
};
use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
use super::admission::{AdmissionLimits, AttemptError};
mod acceptance;
mod attempts;
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version,
policy_mode,
approval_reference,
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
result_envelope, error_envelope, attempts, fence, lease_owner, lease_until,
(extract(epoch from lease_until) * 1000)::bigint as lease_until_ms,
current_attempt_id, dispatch_started_at,
(extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version,
max_attempts, retry_backoff_step_ms,
(extract(epoch from next_attempt_at) * 1000)::bigint as next_attempt_at_ms,
(extract(epoch from cancel_requested_at) * 1000)::bigint as cancel_requested_at_ms,
cancel_reason,
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
(extract(epoch from created_at) * 1000)::bigint as created_ms,
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
#[derive(Clone)]
pub struct PostgresTaskStore {
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
}
impl PostgresTaskStore {
pub fn new(
pool: PgPool,
limits: AdmissionLimits,
scheduler: crank_core::AdmissionSchedulerConfig,
) -> Result<Self, TaskError> {
limits
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
scheduler
.validate()
.map_err(|error| TaskError::invalid(error.to_string()))?;
Ok(Self {
pool,
limits,
scheduler,
})
}
fn unavailable(error: impl std::fmt::Display) -> TaskError {
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
}
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
sqlx::query("set local lock_timeout = '3000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("set local statement_timeout = '5000ms'")
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
Ok(transaction)
}
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
/// query, so a foreign row is never fetched.
async fn stored_acceptance(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&mut **transaction)
.await
.map_err(AttemptError::from_sql)?;
let Some(row) = row else {
return Ok(None);
};
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
Ok(Some((
acceptance_from_record(&record),
record.input_fingerprint,
)))
}
}
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
TaskAcceptance {
task_id: record.task_id,
state: record.state,
operation_version: record.pins.operation_version,
support: record.support.clone(),
created_at_unix_ms: record.created_at_unix_ms,
retain_until_unix_ms: record.pins.retain_until_unix_ms,
replayed: true,
}
}
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(pool)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}
/// One transaction's decision of the acceptance path.
enum Accepted {
Created(TaskAcceptance),
Replayed(TaskAcceptance),
Conflict,
}
#[async_trait]
impl TaskStore for PostgresTaskStore {
async fn health(&self) -> Result<(), TaskError> {
sqlx::query("select count(*)::bigint from tasks where false")
.fetch_one(&self.pool)
.await
.map_err(Self::unavailable)?;
super::admission::admission_health(&self.pool)
.await
.map_err(|error| TaskError::unavailable(error.to_string()))?;
Ok(())
}
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
if let Some(epoch) = request.executable_input_master_key_epoch {
super::master_key::ensure_active_master_key_epoch(&mut transaction, epoch)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
}
let outcome = self.accept_locked(&mut transaction, &request).await;
match outcome {
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
// The acknowledgement of a created Task is only possible after a real COMMIT.
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskAcceptance {
replayed: acceptance.replayed,
..acceptance
})
}
Ok(Accepted::Conflict) => {
transaction.rollback().await.ok();
Err(TaskError::FingerprintConflict)
}
Err(AttemptError::Overloaded) => {
transaction.rollback().await.ok();
Err(TaskError::Overloaded {
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
})
}
Err(AttemptError::Contended) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(
"durable task acceptance could not take the admission locks",
))
}
Err(AttemptError::UniqueViolation) => {
// Roll back the aborted transaction, then reread the scoped key once outside it.
transaction.rollback().await.ok();
self.reread_after_unique(&request).await
}
Err(AttemptError::Failure(error)) => {
transaction.rollback().await.ok();
Err(error)
}
Err(AttemptError::AdmissionFailure(error)) => {
transaction.rollback().await.ok();
Err(TaskError::unavailable(error.to_string()))
}
}
}
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
record_from_row(&row).map_err(Self::unavailable)
}
async fn find(
&self,
access: &TaskAccess,
operation_id: &str,
scoped_key: &str,
) -> Result<Option<TaskRecord>, TaskError> {
access.validate()?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(operation_id)
.bind(scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
row.map(|row| record_from_row(&row)).transpose()
}
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now()
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
and (lease_owner is null or fence = 0)
and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = update else {
transaction.rollback().await.ok();
return Err(self.classify_missed_finish(&request).await);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
let expired = sqlx::query(
"with expired as (
update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
version = version + 1,
updated_at = now()
where task_id in (
select task_id from tasks
where state = 'queued' and execution_deadline <= now()
and lease_owner is null
order by task_id
limit $1
)
returning task_id
), released as (
delete from execution_admission_entries
where task_id in (select task_id from expired)
returning task_id
)
select (select count(*) from expired)::bigint as expired_queued",
)
.bind(batch)
.fetch_one(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let expired_queued: i64 = expired
.try_get("expired_queued")
.map_err(Self::unavailable)?;
let purged = sqlx::query(
"delete from tasks
where task_id in (
select task_id from tasks
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
order by task_id
limit $1
)",
)
.bind(batch)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskPurgeReport {
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
expired_terminal: purged.rows_affected(),
})
}
async fn claim_batch(
&self,
owner: &str,
batch_limit: u32,
lease_ms: i64,
) -> Result<Vec<TaskLease>, TaskError> {
let batch = (batch_limit.clamp(1, 64)) as usize;
let mut transaction = self.start().await?;
// A checkpointed attempt is never redispatched blindly: once its lease expires another worker
// must decide per Task whether the pinned safe-retry evidence, the budget and the deadline
// still allow exactly one bounded new attempt (Story 4.4). Everything else resolves to the
// non-terminal uncertain state and releases its admission slot.
let candidates = sqlx::query(
"select t.task_id, t.current_attempt_id, t.attempts, t.max_attempts,
t.execution_deadline > clock_timestamp() as deadline_left,
coalesce(a.retry_allowed, false) as retry_allowed,
a.resolved_at is null as attempt_open
from tasks t
left join task_attempts a
on a.task_id = t.task_id and a.attempt_id = t.current_attempt_id
where t.state = 'working' and t.lease_owner is not null
and t.current_attempt_id is not null
and t.lease_until < clock_timestamp()
order by t.task_id
limit 64
for update of t skip locked",
)
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
for candidate in candidates {
let task_id: Uuid = candidate.try_get("task_id").map_err(Self::unavailable)?;
let attempt_id: String = candidate
.try_get("current_attempt_id")
.map_err(Self::unavailable)?;
let attempts: i32 = candidate.try_get("attempts").map_err(Self::unavailable)?;
let max_attempts: i32 = candidate
.try_get("max_attempts")
.map_err(Self::unavailable)?;
let deadline_left: bool = candidate
.try_get("deadline_left")
.map_err(Self::unavailable)?;
let retry_allowed: bool = candidate
.try_get("retry_allowed")
.map_err(Self::unavailable)?;
let attempt_open: bool = candidate
.try_get("attempt_open")
.map_err(Self::unavailable)?;
let safe_recovery =
retry_allowed && attempt_open && attempts < max_attempts && deadline_left;
let close = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
phase = 'dispatched'
where task_id = $1 and attempt_id = $2 and resolved_at is null",
)
.bind(task_id)
.bind(&attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let _ = close;
if safe_recovery {
// Interrupted, not failed: re-arm the same admission intent and let the bounded claim
// hand the Task to a new owner with a new fence and exactly one new attempt.
sqlx::query(
"update tasks
set state = 'queued',
lease_owner = null,
lease_until = null,
current_attempt_id = null,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working'
and lease_until < clock_timestamp()",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query(
"update execution_admission_entries
set state = 'reserved', granted_at = null
where task_id = $1 and state = 'executing'",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
} else {
sqlx::query(
"update tasks
set state = 'outcome_unknown',
lease_owner = null,
lease_until = null,
error_envelope = jsonb_build_object('code', 'outcome_unknown', 'phase', 'dispatched'),
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working'
and lease_until < clock_timestamp()",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
}
}
// A crash before the durable dispatch checkpoint left the Task queued for execution: its
// admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is
// bounded by the <=30s lease expiry, inside the 60s story budget. The expiry condition is
// re-checked in the UPDATE so a concurrent renewal can never be clobbered by recovery.
sqlx::query(
"with candidates as (
select task_id from tasks
where state = 'working' and lease_owner is not null
and current_attempt_id is null
and lease_until < clock_timestamp()
order by task_id
limit 64
for update skip locked
), requeued as (
update tasks
set state = 'queued',
lease_owner = null,
lease_until = null,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id in (select task_id from candidates)
and state = 'working'
and current_attempt_id is null
and lease_until < clock_timestamp()
returning task_id
)
update execution_admission_entries
set state = 'reserved', granted_at = null
where task_id in (select task_id from requeued) and state = 'executing'",
)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
let mut leases = Vec::new();
for _ in 0..batch {
let mut transaction = self.start().await?;
let workspace_cursor =
match super::admission::lock_admission_in_tx(&mut transaction, "*").await {
Ok(cursor) => cursor,
Err(error) => {
transaction.rollback().await.ok();
return Err(TaskError::unavailable(format!("{error:?}")));
}
};
let (task_id, _updated_cursor) = match super::admission::grant_next_task(
&mut transaction,
"*",
&workspace_cursor,
&self.limits,
self.scheduler,
)
.await
{
Ok(value) => value,
Err(error) => {
transaction.rollback().await.ok();
return Err(TaskError::unavailable(format!("{error:?}")));
}
};
// Persist the updated controller cursor even when no task is granted.
let Some(task_id) = task_id else {
transaction.commit().await.map_err(Self::unavailable)?;
break;
};
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'working',
lease_owner = $1,
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
fence = fence + 1,
updated_at = now()
where task_id = $3
and (
(state = 'queued' and lease_owner is null)
or (state = 'working' and lease_owner is not null
and lease_until < clock_timestamp() and current_attempt_id is null)
)
and (next_attempt_at is null or next_attempt_at <= clock_timestamp())
returning {TASK_PROJECTION}"
)))
.bind(owner)
.bind(lease_ms)
.bind(task_id)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
// Persist cursor update (grant_next_task already did it) regardless.
let Some(row) = row else {
transaction.rollback().await.ok();
break;
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
leases.push(TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
});
transaction.commit().await.map_err(Self::unavailable)?;
}
Ok(leases)
}
async fn renew_lease(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
lease_ms: i64,
) -> Result<TaskLease, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set lease_until = clock_timestamp() + ($1::bigint || ' milliseconds')::interval,
updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(lease_ms)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
let lease = TaskLease {
owner: owner.to_owned(),
fence: record.fence,
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
record,
};
transaction.commit().await.map_err(Self::unavailable)?;
Ok(lease)
}
async fn write_progress(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
progress: TaskEnvelope,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set progress_envelope = $1, updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(progress.value().clone())
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn read_executable_input(
&self,
owner: &str,
task_id: TaskId,
fence: u64,
) -> Result<(String, String, i64), TaskError> {
let row = sqlx::query(
"select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch
from tasks
where task_id = $1 and state = 'working' and lease_owner = $2 and fence = $3
and lease_until > clock_timestamp()",
)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::StateConflict);
};
let ciphertext: Option<String> = row
.try_get("executable_input_ciphertext")
.map_err(Self::unavailable)?;
let key_version: Option<String> = row
.try_get("executable_input_key_version")
.map_err(Self::unavailable)?;
let epoch: Option<i64> = row
.try_get("executable_input_master_key_epoch")
.map_err(Self::unavailable)?;
if let Some(epoch) = epoch {
let active = sqlx::query_scalar::<_, i64>(
"select epoch from master_key_identities where status = 'active' order by epoch desc limit 1",
)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
if let Some(active) = active
&& active != epoch
{
return Err(TaskError::StateConflict);
}
}
match (ciphertext, key_version, epoch) {
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
Ok((ciphertext, key_version, epoch))
}
_ => Err(TaskError::NotFound),
}
}
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError> {
self.checkpoint_dispatch_attempt(request).await
}
async fn resolve_attempt(
&self,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError> {
self.resolve_attempt_outcome(lease, outcome).await
}
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError> {
self.cancel_task(request).await
}
async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<TaskAttempt>, TaskError> {
self.attempt_history(access, task_id).await
}
async fn finish_leased(
&self,
request: TaskFinishRequest,
owner: &str,
fence: u64,
) -> Result<TaskRecord, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
let access = &request.access;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
version = version + 1,
updated_at = now(),
lease_owner = null,
lease_until = null
where task_id = $4 and workspace_id = $5 and owner_kind = $6
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
and lease_owner = $11 and fence = $12
and lease_until > clock_timestamp()
and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
.bind(request.result.as_ref().map(|value| value.value().clone()))
.bind(request.error.as_ref().map(|value| value.value().clone()))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.bind(request.expected_state.as_str())
.bind(i64::from(request.expected_version))
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError> {
self.refuse_attempt_resolution(lease, code).await
}
}
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
match mode {
ToolExecutionMode::Synchronous => "synchronous",
ToolExecutionMode::Background => "background",
ToolExecutionMode::Adaptive => "adaptive",
}
}
fn task_sql_error(error: sqlx::Error) -> AttemptError {
if error
.as_database_error()
.and_then(|database| database.code())
.as_deref()
== Some("23505")
{
return AttemptError::UniqueViolation;
}
AttemptError::from_sql(error)
}
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let state: String = row.try_get("state").map_err(unavailable_row)?;
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
value.map(TaskEnvelope::from_persisted).transpose()
};
Ok(TaskRecord {
task_id: TaskId::parse(&task_id.to_string())?,
state: TaskState::parse(&state)?,
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
authorization_scope: row
.try_get("authorization_scope")
.map_err(unavailable_row)?,
platform_api_key_id: row
.try_get::<Option<String>, _>("platform_api_key_id")
.map_err(unavailable_row)?
.map(PlatformApiKeyId::new),
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
pins: TaskPins {
operation_version: u32::try_from(
row.try_get::<i32, _>("operation_version")
.map_err(unavailable_row)?,
)
.unwrap_or_default(),
policy_mode: match policy_mode.as_str() {
"background" => ToolExecutionMode::Background,
"adaptive" => ToolExecutionMode::Adaptive,
_ => ToolExecutionMode::Synchronous,
},
execution_deadline_unix_ms: row
.try_get("execution_deadline_ms")
.map_err(unavailable_row)?,
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
retry: TaskRetryPolicy::new(
u32::try_from(
row.try_get::<i32, _>("max_attempts")
.map_err(unavailable_row)?,
)
.unwrap_or(1),
row.try_get::<i64, _>("retry_backoff_step_ms")
.map_err(unavailable_row)?,
),
},
support: TaskSupportIds {
request_id: crank_core::RequestId::parse(&request_id)
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
trace_id: crank_core::TraceId::parse(&trace_id)
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
},
input: envelope("input_envelope")?
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
progress: envelope("progress_envelope")?,
result: envelope("result_envelope")?,
error: envelope("error_envelope")?,
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
.unwrap_or_default(),
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
.unwrap_or_default(),
lease_owner: row.try_get("lease_owner").map_err(unavailable_row)?,
lease_until_unix_ms: row.try_get("lease_until_ms").map_err(unavailable_row)?,
current_attempt_id: row.try_get("current_attempt_id").map_err(unavailable_row)?,
dispatch_started_at_unix_ms: row
.try_get("dispatch_started_at_ms")
.map_err(unavailable_row)?,
next_attempt_at_unix_ms: row.try_get("next_attempt_at_ms").map_err(unavailable_row)?,
cancel_requested_at_unix_ms: row
.try_get("cancel_requested_at_ms")
.map_err(unavailable_row)?,
cancel_reason: row.try_get("cancel_reason").map_err(unavailable_row)?,
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
.unwrap_or_default(),
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
})
}
fn unavailable_row(error: sqlx::Error) -> TaskError {
PostgresTaskStore::unavailable(error)
}