CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если закреплённая схема требует отсутствующее поле; authorized schema-validated update атомарно и с аудитом возобновляет тот же Task ровно один раз. Replay/conflict/TTL безопасны, ожидание удерживает admission permit. Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
660 lines
26 KiB
Rust
660 lines
26 KiB
Rust
//! Input-continuation half of the PostgreSQL Task store (Story 4.5, FR-25).
|
|
//!
|
|
//! One transaction owns each transition: suspend (`working` -> `input_required`), the committed
|
|
//! answer (`input_required` -> `working`) and the TTL expiry (`input_required` -> `failed`). Authority,
|
|
//! generation, TTL and receipt checks run under the same Task row lock, so a revocation, a competing
|
|
//! answer, a sweeper or a cancellation has exactly one winner and never a duplicated side effect.
|
|
//!
|
|
//! Raw input never enters a request, receipt, audit row or error text: the validated payload is
|
|
//! encrypted by the runtime service before this module sees it.
|
|
|
|
use crank_core::{
|
|
TaskAccess, TaskError, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
|
|
TaskInputUpdateRequest, TaskRecord, TaskState,
|
|
};
|
|
use sqlx::{Postgres, Row, Transaction};
|
|
|
|
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row};
|
|
|
|
#[async_trait::async_trait]
|
|
impl TaskInputStore for PostgresTaskStore {
|
|
async fn suspend_for_input(
|
|
&self,
|
|
lease: &crank_core::TaskLease,
|
|
request: TaskInputRequest,
|
|
) -> Result<TaskRecord, TaskError> {
|
|
self.suspend_input_locked(lease, request).await
|
|
}
|
|
|
|
async fn input_request(
|
|
&self,
|
|
access: &TaskAccess,
|
|
task_id: crank_core::TaskId,
|
|
) -> Result<TaskInputRequest, TaskError> {
|
|
self.read_input_request(access, task_id).await
|
|
}
|
|
|
|
async fn read_executable_input_for_update(
|
|
&self,
|
|
access: &TaskAccess,
|
|
task_id: crank_core::TaskId,
|
|
) -> Result<(String, String, i64), TaskError> {
|
|
self.read_binding_for_update(access, task_id).await
|
|
}
|
|
|
|
async fn apply_input(
|
|
&self,
|
|
request: TaskInputUpdateRequest,
|
|
) -> Result<TaskInputUpdateOutcome, TaskError> {
|
|
self.apply_input_locked(request).await
|
|
}
|
|
|
|
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError> {
|
|
self.expire_input_waits_locked(batch).await
|
|
}
|
|
}
|
|
|
|
impl PostgresTaskStore {
|
|
/// One transactional suspend of a leased Task before any dispatch checkpoint.
|
|
pub(super) async fn suspend_input_locked(
|
|
&self,
|
|
lease: &crank_core::TaskLease,
|
|
request: TaskInputRequest,
|
|
) -> Result<TaskRecord, TaskError> {
|
|
request.validate()?;
|
|
if request.expected_version != lease.record.version.saturating_add(1) {
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
let mut transaction = self.start().await?;
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
|
|
)))
|
|
.bind(lease.record.task_id.as_uuid())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let record = record_from_row(&row)?;
|
|
let current = record.state == TaskState::Working
|
|
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
|
|
&& record.fence == lease.fence
|
|
&& record.version == lease.record.version
|
|
&& record.current_attempt_id.is_none()
|
|
&& record.attempts == 0
|
|
&& record.cancel_requested_at_unix_ms.is_none()
|
|
&& record
|
|
.lease_until_unix_ms
|
|
.is_some_and(|until| until > crank_core::current_unix_ms())
|
|
&& record.pins.execution_deadline_unix_ms > crank_core::current_unix_ms();
|
|
// A generation that already dispatched may not be suspended: that would hide a possible side
|
|
// effect behind an apparent "waiting for input".
|
|
if !current {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
if request.expires_at_unix_ms > record.pins.execution_deadline_unix_ms {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::invalid(
|
|
"task input TTL exceeds the execution deadline",
|
|
));
|
|
}
|
|
|
|
sqlx::query(
|
|
"insert into task_input_requests (
|
|
task_id, wait_id, expected_version, schema_revision, schema_format_version,
|
|
fingerprint_format_version, schema_snapshot, projected_schema, prompt_code,
|
|
missing_fields, expires_at
|
|
) values (
|
|
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
|
to_timestamp($11::double precision / 1000.0)
|
|
)",
|
|
)
|
|
.bind(lease.record.task_id.as_uuid())
|
|
.bind(&request.wait_id)
|
|
.bind(i32::try_from(request.expected_version).unwrap_or(i32::MAX))
|
|
.bind(i32::try_from(request.schema_revision).unwrap_or(i32::MAX))
|
|
.bind(i32::try_from(request.schema_format_version).unwrap_or(i32::MAX))
|
|
.bind(i32::try_from(request.fingerprint_format_version).unwrap_or(i32::MAX))
|
|
.bind(request.schema_snapshot.clone())
|
|
.bind(request.projected_schema.clone())
|
|
.bind(&request.prompt_code)
|
|
.bind(serde_json::Value::Array(
|
|
request
|
|
.missing_fields
|
|
.iter()
|
|
.cloned()
|
|
.map(serde_json::Value::String)
|
|
.collect(),
|
|
))
|
|
.bind(request.expires_at_unix_ms)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(|error| {
|
|
match error
|
|
.as_database_error()
|
|
.and_then(|db| db.code())
|
|
.as_deref()
|
|
{
|
|
Some("23505") => TaskError::StateConflict,
|
|
_ => Self::unavailable(error),
|
|
}
|
|
})?;
|
|
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = 'input_required',
|
|
lease_owner = null,
|
|
lease_until = null,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $1 and state = 'working' and version = $2
|
|
and lease_owner = $3 and fence = $4 and current_attempt_id is null
|
|
and lease_until > clock_timestamp()
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(lease.record.task_id.as_uuid())
|
|
.bind(i64::from(record.version))
|
|
.bind(&lease.owner)
|
|
.bind(i64::try_from(lease.fence).unwrap_or_default())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let suspended = record_from_row(&row)?;
|
|
insert_audit(
|
|
&mut transaction,
|
|
lease.record.task_id,
|
|
&request.wait_id,
|
|
"input_required",
|
|
TaskState::Working,
|
|
TaskState::InputRequired,
|
|
record.version,
|
|
suspended.version,
|
|
None,
|
|
None,
|
|
)
|
|
.await?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(suspended)
|
|
}
|
|
|
|
/// Owner-scoped read of the durable request; a foreign Task is one opaque not-found.
|
|
pub(super) async fn read_input_request(
|
|
&self,
|
|
access: &TaskAccess,
|
|
task_id: crank_core::TaskId,
|
|
) -> Result<TaskInputRequest, TaskError> {
|
|
access.validate()?;
|
|
let row = sqlx::query(
|
|
"select r.wait_id, r.expected_version, r.schema_revision, r.schema_format_version,
|
|
r.fingerprint_format_version, r.schema_snapshot, r.projected_schema,
|
|
r.prompt_code, r.missing_fields,
|
|
(extract(epoch from r.expires_at) * 1000)::bigint as expires_at_ms
|
|
from task_input_requests r
|
|
join tasks t on t.task_id = r.task_id
|
|
where r.task_id = $1 and t.workspace_id = $2 and t.owner_kind = $3
|
|
and t.owner_id = $4 and t.authorization_scope = $5",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.fetch_optional(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let missing_fields: Vec<String> = serde_json::from_value(
|
|
row.try_get::<serde_json::Value, _>("missing_fields")
|
|
.map_err(Self::unavailable)?,
|
|
)
|
|
.map_err(|_| TaskError::unavailable("stored input request metadata is invalid"))?;
|
|
Ok(TaskInputRequest {
|
|
wait_id: row.try_get("wait_id").map_err(Self::unavailable)?,
|
|
expected_version: u32::try_from(
|
|
row.try_get::<i32, _>("expected_version")
|
|
.map_err(Self::unavailable)?,
|
|
)
|
|
.unwrap_or_default(),
|
|
schema_revision: u32::try_from(
|
|
row.try_get::<i32, _>("schema_revision")
|
|
.map_err(Self::unavailable)?,
|
|
)
|
|
.unwrap_or_default(),
|
|
schema_format_version: u32::try_from(
|
|
row.try_get::<i32, _>("schema_format_version")
|
|
.map_err(Self::unavailable)?,
|
|
)
|
|
.unwrap_or_default(),
|
|
schema_snapshot: row.try_get("schema_snapshot").map_err(Self::unavailable)?,
|
|
projected_schema: row.try_get("projected_schema").map_err(Self::unavailable)?,
|
|
prompt_code: row.try_get("prompt_code").map_err(Self::unavailable)?,
|
|
missing_fields,
|
|
expires_at_unix_ms: row.try_get("expires_at_ms").map_err(Self::unavailable)?,
|
|
fingerprint_format_version: u32::try_from(
|
|
row.try_get::<i32, _>("fingerprint_format_version")
|
|
.map_err(Self::unavailable)?,
|
|
)
|
|
.unwrap_or_default(),
|
|
})
|
|
}
|
|
|
|
/// Owner-scoped read of the binding a continuation may extend. Never lease-authorized.
|
|
///
|
|
/// Deliberately not filtered by lifecycle state: an identical replay of an already committed
|
|
/// answer (including after completion or TTL expiry) must reach the receipt lookup, which is the
|
|
/// single authority for that outcome.
|
|
pub(super) async fn read_binding_for_update(
|
|
&self,
|
|
access: &TaskAccess,
|
|
task_id: crank_core::TaskId,
|
|
) -> Result<(String, String, i64), TaskError> {
|
|
access.validate()?;
|
|
let row = sqlx::query(
|
|
"select executable_input_ciphertext, executable_input_key_version,
|
|
executable_input_master_key_epoch
|
|
from tasks
|
|
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
|
and owner_id = $4 and authorization_scope = $5",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.fetch_optional(&self.pool)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let ciphertext: Option<String> = row
|
|
.try_get("executable_input_ciphertext")
|
|
.map_err(Self::unavailable)?;
|
|
let key_version: Option<String> = row
|
|
.try_get("executable_input_key_version")
|
|
.map_err(Self::unavailable)?;
|
|
let epoch: Option<i64> = row
|
|
.try_get("executable_input_master_key_epoch")
|
|
.map_err(Self::unavailable)?;
|
|
match (ciphertext, key_version, epoch) {
|
|
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
|
|
Ok((ciphertext, key_version, epoch))
|
|
}
|
|
_ => Err(TaskError::NotFound),
|
|
}
|
|
}
|
|
|
|
/// One transactional answer application: authority, receipt, generation, TTL and CAS.
|
|
pub(super) async fn apply_input_locked(
|
|
&self,
|
|
request: TaskInputUpdateRequest,
|
|
) -> Result<TaskInputUpdateOutcome, TaskError> {
|
|
request.validate()?;
|
|
let access = &request.access;
|
|
let mut transaction = self.start().await?;
|
|
super::super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
|
|
.await
|
|
.map_err(|error| match error {
|
|
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
|
_ => TaskError::StateConflict,
|
|
})?;
|
|
super::super::master_key::ensure_active_master_key_epoch(
|
|
&mut transaction,
|
|
request.executable_input_master_key_epoch,
|
|
)
|
|
.await
|
|
.map_err(|error| match error {
|
|
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
|
|
_ => TaskError::StateConflict,
|
|
})?;
|
|
|
|
// Authority is revalidated inside this transaction, before any receipt lookup and under the
|
|
// same lock order: master key check, platform key share lock, Task row lock.
|
|
let key_row = sqlx::query(
|
|
"select k.id from platform_api_keys k
|
|
join tasks t on t.platform_api_key_id = k.id
|
|
where t.task_id = $1 and t.owner_kind = 'agent'
|
|
and k.workspace_id = $2 and k.agent_id = $3
|
|
and k.key_kind = 'mcp_client' and k.status = 'active'
|
|
and (k.expires_at is null or k.expires_at > clock_timestamp())
|
|
and (k.scopes_json ? 'write' or k.scopes_json ? 'deploy')
|
|
for share of k",
|
|
)
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(&access.owner_id)
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
if key_row.is_none() {
|
|
transaction.rollback().await.ok();
|
|
// A revoked key, a foreign scope and an absent Task share one opaque answer.
|
|
return Err(TaskError::NotFound);
|
|
}
|
|
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks
|
|
where task_id = $1 and workspace_id = $2 and owner_kind = $3
|
|
and owner_id = $4 and authorization_scope = $5
|
|
for update"
|
|
)))
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(&access.workspace_id)
|
|
.bind(access.owner_kind.as_str())
|
|
.bind(&access.owner_id)
|
|
.bind(&access.authorization_scope)
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::NotFound);
|
|
};
|
|
let record = record_from_row(&row)?;
|
|
|
|
// Receipt lookup precedes the generation and TTL checks: an authorized identical replay keeps
|
|
// returning the committed outcome even after completion, failure or TTL expiry.
|
|
let receipt = sqlx::query(
|
|
"select wait_id, update_key_digest, canonical_fingerprint, committed_state,
|
|
committed_version
|
|
from task_input_requests where task_id = $1 and accepted_at is not null",
|
|
)
|
|
.bind(request.task_id.as_uuid())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
if let Some(receipt) = receipt {
|
|
let wait_id: String = receipt.try_get("wait_id").map_err(Self::unavailable)?;
|
|
if wait_id != request.wait_id {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
let stored_key: String = receipt
|
|
.try_get("update_key_digest")
|
|
.map_err(Self::unavailable)?;
|
|
if stored_key != request.update_key_digest {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
let stored_fingerprint: String = receipt
|
|
.try_get("canonical_fingerprint")
|
|
.map_err(Self::unavailable)?;
|
|
if stored_fingerprint != request.canonical_fingerprint {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::FingerprintConflict);
|
|
}
|
|
let state: String = receipt
|
|
.try_get("committed_state")
|
|
.map_err(Self::unavailable)?;
|
|
let version: i32 = receipt
|
|
.try_get("committed_version")
|
|
.map_err(Self::unavailable)?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
return Ok(TaskInputUpdateOutcome {
|
|
task_id: request.task_id,
|
|
wait_id,
|
|
state: TaskState::parse(&state)?,
|
|
version: u32::try_from(version).unwrap_or_default(),
|
|
replayed: true,
|
|
});
|
|
}
|
|
|
|
let pending = sqlx::query(
|
|
"select wait_id, expected_version,
|
|
(extract(epoch from expires_at) * 1000)::bigint as expires_at_ms,
|
|
(extract(epoch from clock_timestamp()) * 1000)::bigint as now_ms
|
|
from task_input_requests where task_id = $1 and accepted_at is null",
|
|
)
|
|
.bind(request.task_id.as_uuid())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(pending) = pending else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let wait_id: String = pending.try_get("wait_id").map_err(Self::unavailable)?;
|
|
let expected_version: i32 = pending
|
|
.try_get("expected_version")
|
|
.map_err(Self::unavailable)?;
|
|
let expires_at_ms: i64 = pending
|
|
.try_get("expires_at_ms")
|
|
.map_err(Self::unavailable)?;
|
|
// PostgreSQL clock, read after the row lock was taken: never a client timestamp.
|
|
let now_ms: i64 = pending.try_get("now_ms").map_err(Self::unavailable)?;
|
|
if wait_id != request.wait_id
|
|
|| u32::try_from(expected_version).unwrap_or_default() != request.expected_version
|
|
|| record.version != request.expected_version
|
|
|| record.state != TaskState::InputRequired
|
|
{
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
if record.cancel_requested_at_unix_ms.is_some() {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
if now_ms >= expires_at_ms {
|
|
// The same transactional helper the sweeper uses: the Task ends failed with an audit
|
|
// reason and no dispatch. The update never revives it.
|
|
expire_locked(&mut transaction, &record, &wait_id, now_ms).await?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
return Err(TaskError::InputExpired);
|
|
}
|
|
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = 'working',
|
|
effective_input_fingerprint = $1,
|
|
executable_input_ciphertext = $2,
|
|
executable_input_key_version = $3,
|
|
executable_input_master_key_epoch = $4,
|
|
lease_owner = null,
|
|
lease_until = null,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $5 and state = 'input_required' and version = $6
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(&request.effective_input_fingerprint)
|
|
.bind(&request.executable_input_ciphertext)
|
|
.bind(&request.executable_input_key_version)
|
|
.bind(request.executable_input_master_key_epoch)
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(i64::from(record.version))
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let Some(row) = row else {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
};
|
|
let accepted = record_from_row(&row)?;
|
|
let updated = sqlx::query(
|
|
"update task_input_requests
|
|
set accepted_at = now(),
|
|
update_key_digest = $1,
|
|
canonical_fingerprint = $2,
|
|
effective_input_fingerprint = $3,
|
|
committed_state = 'working',
|
|
committed_version = $4
|
|
where task_id = $5 and wait_id = $6 and accepted_at is null",
|
|
)
|
|
.bind(&request.update_key_digest)
|
|
.bind(&request.canonical_fingerprint)
|
|
.bind(&request.effective_input_fingerprint)
|
|
.bind(i64::from(accepted.version))
|
|
.bind(request.task_id.as_uuid())
|
|
.bind(&request.wait_id)
|
|
.execute(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
if updated.rows_affected() != 1 {
|
|
transaction.rollback().await.ok();
|
|
return Err(TaskError::StateConflict);
|
|
}
|
|
insert_audit(
|
|
&mut transaction,
|
|
request.task_id,
|
|
&request.wait_id,
|
|
"input_accepted",
|
|
TaskState::InputRequired,
|
|
TaskState::Working,
|
|
record.version,
|
|
accepted.version,
|
|
Some(access.owner_id.as_str()),
|
|
None,
|
|
)
|
|
.await?;
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(TaskInputUpdateOutcome {
|
|
task_id: request.task_id,
|
|
wait_id: request.wait_id,
|
|
state: TaskState::Working,
|
|
version: accepted.version,
|
|
replayed: false,
|
|
})
|
|
}
|
|
|
|
/// Bounded TTL sweep, safe to call on every poll while all worker slots are busy.
|
|
pub(super) async fn expire_input_waits_locked(&self, batch: u32) -> Result<u64, TaskError> {
|
|
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
|
|
let mut transaction = self.start().await?;
|
|
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"select {TASK_PROJECTION} from tasks t
|
|
where t.state = 'input_required'
|
|
and exists (
|
|
select 1 from task_input_requests r
|
|
where r.task_id = t.task_id and r.accepted_at is null
|
|
and r.expires_at <= clock_timestamp()
|
|
)
|
|
order by t.task_id
|
|
limit $1
|
|
for update of t skip locked"
|
|
)))
|
|
.bind(batch)
|
|
.fetch_all(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?;
|
|
let mut expired = 0u64;
|
|
for row in rows {
|
|
let record = record_from_row(&row)?;
|
|
let Some(wait_id) = sqlx::query_scalar::<_, String>(
|
|
"select wait_id from task_input_requests
|
|
where task_id = $1 and accepted_at is null and expires_at <= clock_timestamp()",
|
|
)
|
|
.bind(record.task_id.as_uuid())
|
|
.fetch_optional(&mut *transaction)
|
|
.await
|
|
.map_err(Self::unavailable)?
|
|
else {
|
|
continue;
|
|
};
|
|
if expire_locked(
|
|
&mut transaction,
|
|
&record,
|
|
&wait_id,
|
|
crank_core::current_unix_ms(),
|
|
)
|
|
.await?
|
|
{
|
|
expired += 1;
|
|
}
|
|
}
|
|
transaction.commit().await.map_err(Self::unavailable)?;
|
|
Ok(expired)
|
|
}
|
|
}
|
|
|
|
/// Marks one locked `input_required` Task failed, releases its admission permit and writes the audit
|
|
/// reason. Returns whether this call performed the transition.
|
|
async fn expire_locked(
|
|
transaction: &mut Transaction<'_, Postgres>,
|
|
record: &TaskRecord,
|
|
wait_id: &str,
|
|
_now_ms: i64,
|
|
) -> Result<bool, TaskError> {
|
|
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
|
|
"update tasks
|
|
set state = 'failed',
|
|
error_envelope = jsonb_build_object('reason', 'input_ttl_expired'),
|
|
lease_owner = null,
|
|
lease_until = null,
|
|
version = version + 1,
|
|
updated_at = now()
|
|
where task_id = $1 and state = 'input_required' and version = $2
|
|
returning {TASK_PROJECTION}"
|
|
)))
|
|
.bind(record.task_id.as_uuid())
|
|
.bind(i64::from(record.version))
|
|
.fetch_optional(&mut **transaction)
|
|
.await
|
|
.map_err(PostgresTaskStore::unavailable)?;
|
|
let Some(row) = row else {
|
|
return Ok(false);
|
|
};
|
|
let failed = record_from_row(&row)?;
|
|
sqlx::query("delete from execution_admission_entries where task_id = $1")
|
|
.bind(record.task_id.as_uuid())
|
|
.execute(&mut **transaction)
|
|
.await
|
|
.map_err(PostgresTaskStore::unavailable)?;
|
|
insert_audit(
|
|
transaction,
|
|
record.task_id,
|
|
wait_id,
|
|
"input_expired",
|
|
TaskState::InputRequired,
|
|
TaskState::Failed,
|
|
record.version,
|
|
failed.version,
|
|
None,
|
|
Some("input_ttl_expired"),
|
|
)
|
|
.await?;
|
|
Ok(true)
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
async fn insert_audit(
|
|
transaction: &mut Transaction<'_, Postgres>,
|
|
task_id: crank_core::TaskId,
|
|
wait_id: &str,
|
|
event: &str,
|
|
from_state: TaskState,
|
|
to_state: TaskState,
|
|
from_version: u32,
|
|
to_version: u32,
|
|
actor: Option<&str>,
|
|
reason_code: Option<&str>,
|
|
) -> Result<(), TaskError> {
|
|
sqlx::query(
|
|
"insert into task_input_audit (
|
|
task_id, wait_id, event, from_state, to_state, from_version, to_version, actor,
|
|
reason_code
|
|
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
|
|
)
|
|
.bind(task_id.as_uuid())
|
|
.bind(wait_id)
|
|
.bind(event)
|
|
.bind(from_state.as_str())
|
|
.bind(to_state.as_str())
|
|
.bind(i32::try_from(from_version).unwrap_or(i32::MAX))
|
|
.bind(i32::try_from(to_version).unwrap_or(i32::MAX))
|
|
.bind(actor)
|
|
.bind(reason_code)
|
|
.execute(&mut **transaction)
|
|
.await
|
|
.map_err(PostgresTaskStore::unavailable)?;
|
|
Ok(())
|
|
}
|