CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable per-outbound attempt до отправки запроса, bounded retry/read, server-owned verified writes, честный cancel/unknown, deadline/fencing, additive schema v29. Staged renames task.rs/tasks.rs разложены на модули. Includes disclosed cosmetic parity exception: единственная правка вне принятой дельты — удалена лишняя пустая строка в конце scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
907 lines
37 KiB
Rust
907 lines
37 KiB
Rust
//! PostgreSQL implementation of the durable Task store (Story 4.2, FR-22/FR-23).
|
|
//!
|
|
//! One PostgreSQL transaction owns the whole acceptance: Task record, dispatch intent
|
|
//! (`execution_admission_entries.task_id`) and its queue slot commit together, so no Task ID and no
|
|
//! acknowledgement can exist without the durable record. The transaction reuses the transaction-aware
|
|
//! primitives of the execution-admission store (`lock_admission_in_tx`, `bounded_queue_len`,
|
|
//! `insert_admission_entry`) instead of copying capacity SQL or starting a nested transaction.
|
|
//!
|
|
//! Replay is resolved before any capacity mutation and never touches the accepted row. An ambiguous
|
|
//! COMMIT is reported as an unavailable store: it is never treated as a proven rollback, so no
|
|
//! compensating delete and no synchronous fallback can happen after an uncertain acceptance.
|
|
|
|
use async_trait::async_trait;
|
|
use crank_core::{
|
|
AttemptLease, AttemptOutcome, AttemptResolution, BeginAttempt, PlatformApiKeyId,
|
|
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
|
|
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskLease,
|
|
TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskStore,
|
|
TaskSupportIds, ToolExecutionMode,
|
|
};
|
|
use serde_json::Value;
|
|
use sqlx::{PgPool, Postgres, Row, Transaction};
|
|
use uuid::Uuid;
|
|
|
|
use super::admission::{AdmissionLimits, AttemptError};
|
|
|
|
mod acceptance;
|
|
mod attempts;
|
|
|
|
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
|
|
authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version,
|
|
policy_mode,
|
|
approval_reference,
|
|
request_id, trace_id, traceparent, trace_sampled, input_envelope, progress_envelope,
|
|
result_envelope, error_envelope, attempts, fence, lease_owner, lease_until,
|
|
(extract(epoch from lease_until) * 1000)::bigint as lease_until_ms,
|
|
current_attempt_id, dispatch_started_at,
|
|
(extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version,
|
|
max_attempts, retry_backoff_step_ms,
|
|
(extract(epoch from next_attempt_at) * 1000)::bigint as next_attempt_at_ms,
|
|
(extract(epoch from cancel_requested_at) * 1000)::bigint as cancel_requested_at_ms,
|
|
cancel_reason,
|
|
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
|
|
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
|
|
(extract(epoch from created_at) * 1000)::bigint as created_ms,
|
|
(extract(epoch from updated_at) * 1000)::bigint as updated_ms";
|
|
|
|
#[derive(Clone)]
|
|
pub struct PostgresTaskStore {
|
|
pool: PgPool,
|
|
limits: AdmissionLimits,
|
|
scheduler: crank_core::AdmissionSchedulerConfig,
|
|
}
|
|
|
|
impl PostgresTaskStore {
|
|
pub fn new(
|
|
pool: PgPool,
|
|
limits: AdmissionLimits,
|
|
scheduler: crank_core::AdmissionSchedulerConfig,
|
|
) -> Result<Self, TaskError> {
|
|
limits
|
|
.validate()
|
|
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
|
scheduler
|
|
.validate()
|
|
.map_err(|error| TaskError::invalid(error.to_string()))?;
|
|
Ok(Self {
|
|
pool,
|
|
limits,
|
|
scheduler,
|
|
})
|
|
}
|
|
|
|
fn unavailable(error: impl std::fmt::Display) -> TaskError {
|
|
TaskError::unavailable(format!("durable task store is unavailable: {error}"))
|
|
}
|
|
|
|
async fn start(&self) -> Result<Transaction<'_, Postgres>, TaskError> {
|
|
let mut transaction = self.pool.begin().await.map_err(Self::unavailable)?;
|
|
sqlx::query("set local lock_timeout = '3000ms'")
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
sqlx::query("set local statement_timeout = '5000ms'")
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
Ok(transaction)
|
|
}
|
|
|
|
/// Scoped lookup of an accepted Task. The full owner/authorization/Tool predicate is part of the
|
|
/// query, so a foreign row is never fetched.
|
|
async fn stored_acceptance(
|
|
&self,
|
|
transaction: &mut Transaction<'_, Postgres>,
|
|
request: &TaskAcceptRequest,
|
|
) -> Result<Option<(TaskAcceptance, String)>, AttemptError> {
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks
|
|
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
|
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
|
)))
|
|
.bind(&request.workspace_id)
|
|
.bind(request.owner_kind.as_str())
|
|
.bind(&request.owner_id)
|
|
.bind(&request.authorization_scope)
|
|
.bind(&request.operation_id)
|
|
.bind(&request.scoped_key)
|
|
.fetch_optional(&mut **transaction)
|
|
.await
|
|
.map_err(AttemptError::from_sql)?;
|
|
let Some(row) = row else {
|
|
return Ok(None);
|
|
};
|
|
let record = record_from_row(&row).map_err(AttemptError::Failure)?;
|
|
Ok(Some((
|
|
acceptance_from_record(&record),
|
|
record.input_fingerprint,
|
|
)))
|
|
}
|
|
}
|
|
|
|
fn acceptance_from_record(record: &TaskRecord) -> TaskAcceptance {
|
|
TaskAcceptance {
|
|
task_id: record.task_id,
|
|
state: record.state,
|
|
operation_version: record.pins.operation_version,
|
|
support: record.support.clone(),
|
|
created_at_unix_ms: record.created_at_unix_ms,
|
|
retain_until_unix_ms: record.pins.retain_until_unix_ms,
|
|
replayed: true,
|
|
}
|
|
}
|
|
|
|
/// Readiness probe of the durable Task state: the table must be reachable and readable. A missing or
|
|
/// unreadable schema is a dependency failure, never a degraded-but-ready state.
|
|
pub async fn tasks_health(pool: &PgPool) -> Result<(), TaskError> {
|
|
sqlx::query("select count(*)::bigint from tasks where false")
|
|
.fetch_one(pool)
|
|
.await
|
|
.map_err(PostgresTaskStore::unavailable)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// One transaction's decision of the acceptance path.
|
|
enum Accepted {
|
|
Created(TaskAcceptance),
|
|
Replayed(TaskAcceptance),
|
|
Conflict,
|
|
}
|
|
|
|
#[async_trait]
|
|
impl TaskStore for PostgresTaskStore {
|
|
async fn health(&self) -> Result<(), TaskError> {
|
|
sqlx::query("select count(*)::bigint from tasks where false")
|
|
.fetch_one(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
super::admission::admission_health(&self.pool)
|
|
.await
|
|
.map_err(|error| TaskError::unavailable(error.to_string()))?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError> {
|
|
request.validate()?;
|
|
let mut transaction = self.start().await?;
|
|
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
|
.await
|
|
.map_err(|error| match error {
|
|
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
|
_ => TaskError::StateConflict,
|
|
})?;
|
|
if let Some(epoch) = request.executable_input_master_key_epoch {
|
|
super::master_key::ensure_active_master_key_epoch(&mut transaction, epoch)
|
|
.await
|
|
.map_err(|error| match error {
|
|
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
|
_ => TaskError::StateConflict,
|
|
})?;
|
|
}
|
|
let outcome = self.accept_locked(&mut transaction, &request).await;
|
|
match outcome {
|
|
Ok(Accepted::Created(acceptance) | Accepted::Replayed(acceptance)) => {
|
|
// The acknowledgement of a created Task is only possible after a real COMMIT.
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(TaskAcceptance {
|
|
replayed: acceptance.replayed,
|
|
..acceptance
|
|
})
|
|
}
|
|
Ok(Accepted::Conflict) => {
|
|
transaction.rollback().await.ok();
|
|
Err(TaskError::FingerprintConflict)
|
|
}
|
|
Err(AttemptError::Overloaded) => {
|
|
transaction.rollback().await.ok();
|
|
Err(TaskError::Overloaded {
|
|
retry_after_ms: crank_core::ADMISSION_OVERLOAD_RETRY_AFTER_MS,
|
|
})
|
|
}
|
|
Err(AttemptError::Contended) => {
|
|
transaction.rollback().await.ok();
|
|
Err(TaskError::unavailable(
|
|
"durable task acceptance could not take the admission locks",
|
|
))
|
|
}
|
|
Err(AttemptError::UniqueViolation) => {
|
|
// Roll back the aborted transaction, then reread the scoped key once outside it.
|
|
transaction.rollback().await.ok();
|
|
self.reread_after_unique(&request).await
|
|
}
|
|
Err(AttemptError::Failure(error)) => {
|
|
transaction.rollback().await.ok();
|
|
Err(error)
|
|
}
|
|
Err(AttemptError::AdmissionFailure(error)) => {
|
|
transaction.rollback().await.ok();
|
|
Err(TaskError::unavailable(error.to_string()))
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError> {
|
|
access.validate()?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks
|
|
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
|
and owner_id = $4 and authorization_scope = $5"
|
|
)))
|
|
.bind(task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.fetch_optional(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
record_from_row(&row).map_err(Self::unavailable)
|
|
}
|
|
|
|
async fn find(
|
|
&self,
|
|
access: &TaskAccess,
|
|
operation_id: &str,
|
|
scoped_key: &str,
|
|
) -> Result<Option<TaskRecord>, TaskError> {
|
|
access.validate()?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks
|
|
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
|
|
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
|
|
)))
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.bind(operation_id)
|
|
.bind(scoped_key)
|
|
.fetch_optional(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
row.map(|row| record_from_row(&row)).transpose()
|
|
}
|
|
|
|
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError> {
|
|
request.validate()?;
|
|
let mut transaction = self.start().await?;
|
|
let access = &request.access;
|
|
let update = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = $1,
|
|
result_envelope = $2,
|
|
error_envelope = $3,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
|
and owner_id = $7 and authorization_scope = $8
|
|
and state = $9 and version = $10
|
|
and (lease_owner is null or fence = 0)
|
|
and current_attempt_id is null
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(request.target_state.as_str())
|
|
.bind(request.result.as_ref().map(|value| value.value().clone()))
|
|
.bind(request.error.as_ref().map(|value| value.value().clone()))
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.bind(request.expected_state.as_str())
|
|
.bind(i64::from(request.expected_version))
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = update else {
|
|
transaction.rollback().await.ok();
|
|
return Err(self.classify_missed_finish(&request).await);
|
|
};
|
|
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
|
// Terminal Task and its dispatch intent leave together: a half-done release is rolled back.
|
|
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
|
.bind(request.task_id.as_uuid())
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(record)
|
|
}
|
|
|
|
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError> {
|
|
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
|
|
let mut transaction = self.start().await?;
|
|
super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
|
.await
|
|
.map_err(|error| match error {
|
|
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
|
_ => TaskError::StateConflict,
|
|
})?;
|
|
let expired = sqlx::query(
|
|
"with expired as (
|
|
update tasks
|
|
set state = 'failed',
|
|
error_envelope = jsonb_build_object('reason', 'execution_deadline_elapsed'),
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id in (
|
|
select task_id from tasks
|
|
where state = 'queued' and execution_deadline <= now()
|
|
and lease_owner is null
|
|
order by task_id
|
|
limit $1
|
|
)
|
|
returning task_id
|
|
), released as (
|
|
delete from execution_admission_entries
|
|
where task_id in (select task_id from expired)
|
|
returning task_id
|
|
)
|
|
select (select count(*) from expired)::bigint as expired_queued",
|
|
)
|
|
.bind(batch)
|
|
.fetch_one(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let expired_queued: i64 = expired
|
|
.try_get("expired_queued")
|
|
.map_err(Self::unavailable)?;
|
|
let purged = sqlx::query(
|
|
"delete from tasks
|
|
where task_id in (
|
|
select task_id from tasks
|
|
where state in ('completed', 'failed', 'cancelled') and retain_until <= now()
|
|
order by task_id
|
|
limit $1
|
|
)",
|
|
)
|
|
.bind(batch)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(TaskPurgeReport {
|
|
expired_queued: u64::try_from(expired_queued).unwrap_or_default(),
|
|
expired_terminal: purged.rows_affected(),
|
|
})
|
|
}
|
|
|
|
async fn claim_batch(
|
|
&self,
|
|
owner: &str,
|
|
batch_limit: u32,
|
|
lease_ms: i64,
|
|
) -> Result<Vec<TaskLease>, TaskError> {
|
|
let batch = (batch_limit.clamp(1, 64)) as usize;
|
|
let mut transaction = self.start().await?;
|
|
// A checkpointed attempt is never redispatched blindly: once its lease expires another worker
|
|
// must decide per Task whether the pinned safe-retry evidence, the budget and the deadline
|
|
// still allow exactly one bounded new attempt (Story 4.4). Everything else resolves to the
|
|
// non-terminal uncertain state and releases its admission slot.
|
|
let candidates = sqlx::query(
|
|
"select t.task_id, t.current_attempt_id, t.attempts, t.max_attempts,
|
|
t.execution_deadline > clock_timestamp() as deadline_left,
|
|
coalesce(a.retry_allowed, false) as retry_allowed,
|
|
a.resolved_at is null as attempt_open
|
|
from tasks t
|
|
left join task_attempts a
|
|
on a.task_id = t.task_id and a.attempt_id = t.current_attempt_id
|
|
where t.state = 'working' and t.lease_owner is not null
|
|
and t.current_attempt_id is not null
|
|
and t.lease_until < clock_timestamp()
|
|
order by t.task_id
|
|
limit 64
|
|
for update of t skip locked",
|
|
)
|
|
.fetch_all(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
for candidate in candidates {
|
|
let task_id: Uuid = candidate.try_get("task_id").map_err(Self::unavailable)?;
|
|
let attempt_id: String = candidate
|
|
.try_get("current_attempt_id")
|
|
.map_err(Self::unavailable)?;
|
|
let attempts: i32 = candidate.try_get("attempts").map_err(Self::unavailable)?;
|
|
let max_attempts: i32 = candidate
|
|
.try_get("max_attempts")
|
|
.map_err(Self::unavailable)?;
|
|
let deadline_left: bool = candidate
|
|
.try_get("deadline_left")
|
|
.map_err(Self::unavailable)?;
|
|
let retry_allowed: bool = candidate
|
|
.try_get("retry_allowed")
|
|
.map_err(Self::unavailable)?;
|
|
let attempt_open: bool = candidate
|
|
.try_get("attempt_open")
|
|
.map_err(Self::unavailable)?;
|
|
let safe_recovery =
|
|
retry_allowed && attempt_open && attempts < max_attempts && deadline_left;
|
|
let close = sqlx::query(
|
|
"update task_attempts
|
|
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
|
|
phase = 'dispatched'
|
|
where task_id = $1 and attempt_id = $2 and resolved_at is null",
|
|
)
|
|
.bind(task_id)
|
|
.bind(&attempt_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let _ = close;
|
|
if safe_recovery {
|
|
// Interrupted, not failed: re-arm the same admission intent and let the bounded claim
|
|
// hand the Task to a new owner with a new fence and exactly one new attempt.
|
|
sqlx::query(
|
|
"update tasks
|
|
set state = 'queued',
|
|
lease_owner = null,
|
|
lease_until = null,
|
|
current_attempt_id = null,
|
|
next_attempt_at = null,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $1 and state = 'working'
|
|
and lease_until < clock_timestamp()",
|
|
)
|
|
.bind(task_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
sqlx::query(
|
|
"update execution_admission_entries
|
|
set state = 'reserved', granted_at = null
|
|
where task_id = $1 and state = 'executing'",
|
|
)
|
|
.bind(task_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
} else {
|
|
sqlx::query(
|
|
"update tasks
|
|
set state = 'outcome_unknown',
|
|
lease_owner = null,
|
|
lease_until = null,
|
|
error_envelope = jsonb_build_object('code', 'outcome_unknown', 'phase', 'dispatched'),
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $1 and state = 'working'
|
|
and lease_until < clock_timestamp()",
|
|
)
|
|
.bind(task_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
|
.bind(task_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
}
|
|
}
|
|
|
|
// A crash before the durable dispatch checkpoint left the Task queued for execution: its
|
|
// admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is
|
|
// bounded by the <=30s lease expiry, inside the 60s story budget. The expiry condition is
|
|
// re-checked in the UPDATE so a concurrent renewal can never be clobbered by recovery.
|
|
sqlx::query(
|
|
"with candidates as (
|
|
select task_id from tasks
|
|
where state = 'working' and lease_owner is not null
|
|
and current_attempt_id is null
|
|
and lease_until < clock_timestamp()
|
|
order by task_id
|
|
limit 64
|
|
for update skip locked
|
|
), requeued as (
|
|
update tasks
|
|
set state = 'queued',
|
|
lease_owner = null,
|
|
lease_until = null,
|
|
next_attempt_at = null,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id in (select task_id from candidates)
|
|
and state = 'working'
|
|
and current_attempt_id is null
|
|
and lease_until < clock_timestamp()
|
|
returning task_id
|
|
)
|
|
update execution_admission_entries
|
|
set state = 'reserved', granted_at = null
|
|
where task_id in (select task_id from requeued) and state = 'executing'",
|
|
)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
|
|
let mut leases = Vec::new();
|
|
for _ in 0..batch {
|
|
let mut transaction = self.start().await?;
|
|
let workspace_cursor =
|
|
match super::admission::lock_admission_in_tx(&mut transaction, "*").await {
|
|
Ok(cursor) => cursor,
|
|
Err(error) => {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::unavailable(format!("{error:?}")));
|
|
}
|
|
};
|
|
let (task_id, _updated_cursor) = match super::admission::grant_next_task(
|
|
&mut transaction,
|
|
"*",
|
|
&workspace_cursor,
|
|
&self.limits,
|
|
self.scheduler,
|
|
)
|
|
.await
|
|
{
|
|
Ok(value) => value,
|
|
Err(error) => {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::unavailable(format!("{error:?}")));
|
|
}
|
|
};
|
|
// Persist the updated controller cursor even when no task is granted.
|
|
let Some(task_id) = task_id else {
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
break;
|
|
};
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = 'working',
|
|
lease_owner = $1,
|
|
lease_until = clock_timestamp() + ($2::bigint || ' milliseconds')::interval,
|
|
fence = fence + 1,
|
|
updated_at = now()
|
|
where task_id = $3
|
|
and (
|
|
(state = 'queued' and lease_owner is null)
|
|
or (state = 'working' and lease_owner is not null
|
|
and lease_until < clock_timestamp() and current_attempt_id is null)
|
|
)
|
|
and (next_attempt_at is null or next_attempt_at <= clock_timestamp())
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(owner)
|
|
.bind(lease_ms)
|
|
.bind(task_id)
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
// Persist cursor update (grant_next_task already did it) regardless.
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
break;
|
|
};
|
|
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
|
leases.push(TaskLease {
|
|
owner: owner.to_owned(),
|
|
fence: record.fence,
|
|
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
|
|
record,
|
|
});
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
}
|
|
Ok(leases)
|
|
}
|
|
|
|
async fn renew_lease(
|
|
&self,
|
|
owner: &str,
|
|
task_id: TaskId,
|
|
fence: u64,
|
|
lease_ms: i64,
|
|
) -> Result<TaskLease, TaskError> {
|
|
let mut transaction = self.start().await?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set lease_until = clock_timestamp() + ($1::bigint || ' milliseconds')::interval,
|
|
updated_at = now()
|
|
where task_id = $2 and lease_owner = $3 and fence = $4
|
|
and lease_until > clock_timestamp()
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(lease_ms)
|
|
.bind(task_id.as_uuid())
|
|
.bind(owner)
|
|
.bind(i64::try_from(fence).unwrap_or_default())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
|
let lease = TaskLease {
|
|
owner: owner.to_owned(),
|
|
fence: record.fence,
|
|
lease_until: record.lease_until_unix_ms.unwrap_or_default(),
|
|
record,
|
|
};
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(lease)
|
|
}
|
|
|
|
async fn write_progress(
|
|
&self,
|
|
owner: &str,
|
|
task_id: TaskId,
|
|
fence: u64,
|
|
progress: TaskEnvelope,
|
|
) -> Result<TaskRecord, TaskError> {
|
|
let mut transaction = self.start().await?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set progress_envelope = $1, updated_at = now()
|
|
where task_id = $2 and lease_owner = $3 and fence = $4
|
|
and lease_until > clock_timestamp()
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(progress.value().clone())
|
|
.bind(task_id.as_uuid())
|
|
.bind(owner)
|
|
.bind(i64::try_from(fence).unwrap_or_default())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(record)
|
|
}
|
|
|
|
async fn read_executable_input(
|
|
&self,
|
|
owner: &str,
|
|
task_id: TaskId,
|
|
fence: u64,
|
|
) -> Result<(String, String, i64), TaskError> {
|
|
let row = sqlx::query(
|
|
"select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch
|
|
from tasks
|
|
where task_id = $1 and state = 'working' and lease_owner = $2 and fence = $3
|
|
and lease_until > clock_timestamp()",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.bind(owner)
|
|
.bind(i64::try_from(fence).unwrap_or_default())
|
|
.fetch_optional(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let ciphertext: Option<String> = row
|
|
.try_get("executable_input_ciphertext")
|
|
.map_err(Self::unavailable)?;
|
|
let key_version: Option<String> = row
|
|
.try_get("executable_input_key_version")
|
|
.map_err(Self::unavailable)?;
|
|
let epoch: Option<i64> = row
|
|
.try_get("executable_input_master_key_epoch")
|
|
.map_err(Self::unavailable)?;
|
|
if let Some(epoch) = epoch {
|
|
let active = sqlx::query_scalar::<_, i64>(
|
|
"select epoch from master_key_identities where status = 'active' order by epoch desc limit 1",
|
|
)
|
|
.fetch_optional(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
if let Some(active) = active
|
|
&& active != epoch
|
|
{
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
}
|
|
match (ciphertext, key_version, epoch) {
|
|
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
|
|
Ok((ciphertext, key_version, epoch))
|
|
}
|
|
_ => Err(TaskError::NotFound),
|
|
}
|
|
}
|
|
|
|
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError> {
|
|
self.checkpoint_dispatch_attempt(request).await
|
|
}
|
|
|
|
async fn resolve_attempt(
|
|
&self,
|
|
lease: &AttemptLease,
|
|
outcome: AttemptOutcome,
|
|
) -> Result<AttemptResolution, TaskError> {
|
|
self.resolve_attempt_outcome(lease, outcome).await
|
|
}
|
|
|
|
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError> {
|
|
self.cancel_task(request).await
|
|
}
|
|
|
|
async fn attempts(
|
|
&self,
|
|
access: &TaskAccess,
|
|
task_id: TaskId,
|
|
) -> Result<Vec<TaskAttempt>, TaskError> {
|
|
self.attempt_history(access, task_id).await
|
|
}
|
|
|
|
async fn finish_leased(
|
|
&self,
|
|
request: TaskFinishRequest,
|
|
owner: &str,
|
|
fence: u64,
|
|
) -> Result<TaskRecord, TaskError> {
|
|
request.validate()?;
|
|
let mut transaction = self.start().await?;
|
|
let access = &request.access;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = $1,
|
|
result_envelope = $2,
|
|
error_envelope = $3,
|
|
version = version + 1,
|
|
updated_at = now(),
|
|
lease_owner = null,
|
|
lease_until = null
|
|
where task_id = $4 and workspace_id = $5 and owner_kind = $6
|
|
and owner_id = $7 and authorization_scope = $8
|
|
and state = $9 and version = $10
|
|
and lease_owner = $11 and fence = $12
|
|
and lease_until > clock_timestamp()
|
|
and current_attempt_id is null
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(request.target_state.as_str())
|
|
.bind(request.result.as_ref().map(|value| value.value().clone()))
|
|
.bind(request.error.as_ref().map(|value| value.value().clone()))
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.bind(request.expected_state.as_str())
|
|
.bind(i64::from(request.expected_version))
|
|
.bind(owner)
|
|
.bind(i64::try_from(fence).unwrap_or_default())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let record = record_from_row(&row).map_err(Self::unavailable)?;
|
|
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
|
.bind(request.task_id.as_uuid())
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(record)
|
|
}
|
|
|
|
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError> {
|
|
self.refuse_attempt_resolution(lease, code).await
|
|
}
|
|
}
|
|
|
|
fn policy_mode_str(mode: ToolExecutionMode) -> &'static str {
|
|
match mode {
|
|
ToolExecutionMode::Synchronous => "synchronous",
|
|
ToolExecutionMode::Background => "background",
|
|
ToolExecutionMode::Adaptive => "adaptive",
|
|
}
|
|
}
|
|
|
|
fn task_sql_error(error: sqlx::Error) -> AttemptError {
|
|
if error
|
|
.as_database_error()
|
|
.and_then(|database| database.code())
|
|
.as_deref()
|
|
== Some("23505")
|
|
{
|
|
return AttemptError::UniqueViolation;
|
|
}
|
|
AttemptError::from_sql(error)
|
|
}
|
|
|
|
fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError> {
|
|
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
|
|
let state: String = row.try_get("state").map_err(unavailable_row)?;
|
|
let owner_kind: String = row.try_get("owner_kind").map_err(unavailable_row)?;
|
|
let policy_mode: String = row.try_get("policy_mode").map_err(unavailable_row)?;
|
|
let trace_id: String = row.try_get("trace_id").map_err(unavailable_row)?;
|
|
let request_id: String = row.try_get("request_id").map_err(unavailable_row)?;
|
|
let envelope = |column: &str| -> Result<Option<TaskEnvelope>, TaskError> {
|
|
let value: Option<Value> = row.try_get(column).map_err(unavailable_row)?;
|
|
value.map(TaskEnvelope::from_persisted).transpose()
|
|
};
|
|
Ok(TaskRecord {
|
|
task_id: TaskId::parse(&task_id.to_string())?,
|
|
state: TaskState::parse(&state)?,
|
|
workspace_id: row.try_get("workspace_id").map_err(unavailable_row)?,
|
|
owner_kind: TaskOwnerKind::parse(&owner_kind)?,
|
|
owner_id: row.try_get("owner_id").map_err(unavailable_row)?,
|
|
authorization_scope: row
|
|
.try_get("authorization_scope")
|
|
.map_err(unavailable_row)?,
|
|
platform_api_key_id: row
|
|
.try_get::<Option<String>, _>("platform_api_key_id")
|
|
.map_err(unavailable_row)?
|
|
.map(PlatformApiKeyId::new),
|
|
operation_id: row.try_get("operation_id").map_err(unavailable_row)?,
|
|
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
|
|
pins: TaskPins {
|
|
operation_version: u32::try_from(
|
|
row.try_get::<i32, _>("operation_version")
|
|
.map_err(unavailable_row)?,
|
|
)
|
|
.unwrap_or_default(),
|
|
policy_mode: match policy_mode.as_str() {
|
|
"background" => ToolExecutionMode::Background,
|
|
"adaptive" => ToolExecutionMode::Adaptive,
|
|
_ => ToolExecutionMode::Synchronous,
|
|
},
|
|
execution_deadline_unix_ms: row
|
|
.try_get("execution_deadline_ms")
|
|
.map_err(unavailable_row)?,
|
|
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
|
|
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
|
|
retry: TaskRetryPolicy::new(
|
|
u32::try_from(
|
|
row.try_get::<i32, _>("max_attempts")
|
|
.map_err(unavailable_row)?,
|
|
)
|
|
.unwrap_or(1),
|
|
row.try_get::<i64, _>("retry_backoff_step_ms")
|
|
.map_err(unavailable_row)?,
|
|
),
|
|
},
|
|
support: TaskSupportIds {
|
|
request_id: crank_core::RequestId::parse(&request_id)
|
|
.map_err(|_| TaskError::unavailable("stored request id is invalid"))?,
|
|
trace_id: crank_core::TraceId::parse(&trace_id)
|
|
.map_err(|_| TaskError::unavailable("stored trace id is invalid"))?,
|
|
traceparent: row.try_get("traceparent").map_err(unavailable_row)?,
|
|
trace_sampled: row.try_get("trace_sampled").map_err(unavailable_row)?,
|
|
},
|
|
input: envelope("input_envelope")?
|
|
.ok_or_else(|| TaskError::unavailable("stored task input envelope is missing"))?,
|
|
progress: envelope("progress_envelope")?,
|
|
result: envelope("result_envelope")?,
|
|
error: envelope("error_envelope")?,
|
|
attempts: u32::try_from(row.try_get::<i32, _>("attempts").map_err(unavailable_row)?)
|
|
.unwrap_or_default(),
|
|
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
|
|
.unwrap_or_default(),
|
|
lease_owner: row.try_get("lease_owner").map_err(unavailable_row)?,
|
|
lease_until_unix_ms: row.try_get("lease_until_ms").map_err(unavailable_row)?,
|
|
current_attempt_id: row.try_get("current_attempt_id").map_err(unavailable_row)?,
|
|
dispatch_started_at_unix_ms: row
|
|
.try_get("dispatch_started_at_ms")
|
|
.map_err(unavailable_row)?,
|
|
next_attempt_at_unix_ms: row.try_get("next_attempt_at_ms").map_err(unavailable_row)?,
|
|
cancel_requested_at_unix_ms: row
|
|
.try_get("cancel_requested_at_ms")
|
|
.map_err(unavailable_row)?,
|
|
cancel_reason: row.try_get("cancel_reason").map_err(unavailable_row)?,
|
|
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
|
|
.unwrap_or_default(),
|
|
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
|
|
updated_at_unix_ms: row.try_get("updated_ms").map_err(unavailable_row)?,
|
|
})
|
|
}
|
|
|
|
fn unavailable_row(error: sqlx::Error) -> TaskError {
|
|
PostgresTaskStore::unavailable(error)
|
|
}
|