Files
crank/crates/crank-registry/src/postgres/tasks/input.rs
T
bsodfather 90e5357a08
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
feat(epic4): продолжение ожидающей Task после ввода (Story 4.5)
Durable worker приостанавливает Task как input_required до dispatch, если
закреплённая схема требует отсутствующее поле; authorized schema-validated
update атомарно и с аудитом возобновляет тот же Task ровно один раз.
Replay/conflict/TTL безопасны, ожидание удерживает admission permit.
Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
2026-10-30 16:24:12 +00:00

660 lines
26 KiB
Rust

//! Input-continuation half of the PostgreSQL Task store (Story 4.5, FR-25).
//!
//! One transaction owns each transition: suspend (`working` -> `input_required`), the committed
//! answer (`input_required` -> `working`) and the TTL expiry (`input_required` -> `failed`). Authority,
//! generation, TTL and receipt checks run under the same Task row lock, so a revocation, a competing
//! answer, a sweeper or a cancellation has exactly one winner and never a duplicated side effect.
//!
//! Raw input never enters a request, receipt, audit row or error text: the validated payload is
//! encrypted by the runtime service before this module sees it.
use crank_core::{
TaskAccess, TaskError, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome,
TaskInputUpdateRequest, TaskRecord, TaskState,
};
use sqlx::{Postgres, Row, Transaction};
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row};
#[async_trait::async_trait]
impl TaskInputStore for PostgresTaskStore {
async fn suspend_for_input(
&self,
lease: &crank_core::TaskLease,
request: TaskInputRequest,
) -> Result<TaskRecord, TaskError> {
self.suspend_input_locked(lease, request).await
}
async fn input_request(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<TaskInputRequest, TaskError> {
self.read_input_request(access, task_id).await
}
async fn read_executable_input_for_update(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<(String, String, i64), TaskError> {
self.read_binding_for_update(access, task_id).await
}
async fn apply_input(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError> {
self.apply_input_locked(request).await
}
async fn expire_input_waits(&self, batch: u32) -> Result<u64, TaskError> {
self.expire_input_waits_locked(batch).await
}
}
impl PostgresTaskStore {
/// One transactional suspend of a leased Task before any dispatch checkpoint.
pub(super) async fn suspend_input_locked(
&self,
lease: &crank_core::TaskLease,
request: TaskInputRequest,
) -> Result<TaskRecord, TaskError> {
request.validate()?;
if request.expected_version != lease.record.version.saturating_add(1) {
return Err(TaskError::StateConflict);
}
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(lease.record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
let current = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
&& record.fence == lease.fence
&& record.version == lease.record.version
&& record.current_attempt_id.is_none()
&& record.attempts == 0
&& record.cancel_requested_at_unix_ms.is_none()
&& record
.lease_until_unix_ms
.is_some_and(|until| until > crank_core::current_unix_ms())
&& record.pins.execution_deadline_unix_ms > crank_core::current_unix_ms();
// A generation that already dispatched may not be suspended: that would hide a possible side
// effect behind an apparent "waiting for input".
if !current {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if request.expires_at_unix_ms > record.pins.execution_deadline_unix_ms {
transaction.rollback().await.ok();
return Err(TaskError::invalid(
"task input TTL exceeds the execution deadline",
));
}
sqlx::query(
"insert into task_input_requests (
task_id, wait_id, expected_version, schema_revision, schema_format_version,
fingerprint_format_version, schema_snapshot, projected_schema, prompt_code,
missing_fields, expires_at
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
to_timestamp($11::double precision / 1000.0)
)",
)
.bind(lease.record.task_id.as_uuid())
.bind(&request.wait_id)
.bind(i32::try_from(request.expected_version).unwrap_or(i32::MAX))
.bind(i32::try_from(request.schema_revision).unwrap_or(i32::MAX))
.bind(i32::try_from(request.schema_format_version).unwrap_or(i32::MAX))
.bind(i32::try_from(request.fingerprint_format_version).unwrap_or(i32::MAX))
.bind(request.schema_snapshot.clone())
.bind(request.projected_schema.clone())
.bind(&request.prompt_code)
.bind(serde_json::Value::Array(
request
.missing_fields
.iter()
.cloned()
.map(serde_json::Value::String)
.collect(),
))
.bind(request.expires_at_unix_ms)
.execute(&mut *transaction)
.await
.map_err(|error| {
match error
.as_database_error()
.and_then(|db| db.code())
.as_deref()
{
Some("23505") => TaskError::StateConflict,
_ => Self::unavailable(error),
}
})?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'input_required',
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working' and version = $2
and lease_owner = $3 and fence = $4 and current_attempt_id is null
and lease_until > clock_timestamp()
returning {TASK_PROJECTION}"
)))
.bind(lease.record.task_id.as_uuid())
.bind(i64::from(record.version))
.bind(&lease.owner)
.bind(i64::try_from(lease.fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let suspended = record_from_row(&row)?;
insert_audit(
&mut transaction,
lease.record.task_id,
&request.wait_id,
"input_required",
TaskState::Working,
TaskState::InputRequired,
record.version,
suspended.version,
None,
None,
)
.await?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(suspended)
}
/// Owner-scoped read of the durable request; a foreign Task is one opaque not-found.
pub(super) async fn read_input_request(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<TaskInputRequest, TaskError> {
access.validate()?;
let row = sqlx::query(
"select r.wait_id, r.expected_version, r.schema_revision, r.schema_format_version,
r.fingerprint_format_version, r.schema_snapshot, r.projected_schema,
r.prompt_code, r.missing_fields,
(extract(epoch from r.expires_at) * 1000)::bigint as expires_at_ms
from task_input_requests r
join tasks t on t.task_id = r.task_id
where r.task_id = $1 and t.workspace_id = $2 and t.owner_kind = $3
and t.owner_id = $4 and t.authorization_scope = $5",
)
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
let missing_fields: Vec<String> = serde_json::from_value(
row.try_get::<serde_json::Value, _>("missing_fields")
.map_err(Self::unavailable)?,
)
.map_err(|_| TaskError::unavailable("stored input request metadata is invalid"))?;
Ok(TaskInputRequest {
wait_id: row.try_get("wait_id").map_err(Self::unavailable)?,
expected_version: u32::try_from(
row.try_get::<i32, _>("expected_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_revision: u32::try_from(
row.try_get::<i32, _>("schema_revision")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_format_version: u32::try_from(
row.try_get::<i32, _>("schema_format_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
schema_snapshot: row.try_get("schema_snapshot").map_err(Self::unavailable)?,
projected_schema: row.try_get("projected_schema").map_err(Self::unavailable)?,
prompt_code: row.try_get("prompt_code").map_err(Self::unavailable)?,
missing_fields,
expires_at_unix_ms: row.try_get("expires_at_ms").map_err(Self::unavailable)?,
fingerprint_format_version: u32::try_from(
row.try_get::<i32, _>("fingerprint_format_version")
.map_err(Self::unavailable)?,
)
.unwrap_or_default(),
})
}
/// Owner-scoped read of the binding a continuation may extend. Never lease-authorized.
///
/// Deliberately not filtered by lifecycle state: an identical replay of an already committed
/// answer (including after completion or TTL expiry) must reach the receipt lookup, which is the
/// single authority for that outcome.
pub(super) async fn read_binding_for_update(
&self,
access: &TaskAccess,
task_id: crank_core::TaskId,
) -> Result<(String, String, i64), TaskError> {
access.validate()?;
let row = sqlx::query(
"select executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch
from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::NotFound);
};
let ciphertext: Option<String> = row
.try_get("executable_input_ciphertext")
.map_err(Self::unavailable)?;
let key_version: Option<String> = row
.try_get("executable_input_key_version")
.map_err(Self::unavailable)?;
let epoch: Option<i64> = row
.try_get("executable_input_master_key_epoch")
.map_err(Self::unavailable)?;
match (ciphertext, key_version, epoch) {
(Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => {
Ok((ciphertext, key_version, epoch))
}
_ => Err(TaskError::NotFound),
}
}
/// One transactional answer application: authority, receipt, generation, TTL and CAS.
pub(super) async fn apply_input_locked(
&self,
request: TaskInputUpdateRequest,
) -> Result<TaskInputUpdateOutcome, TaskError> {
request.validate()?;
let access = &request.access;
let mut transaction = self.start().await?;
super::super::master_key::ensure_no_active_master_key_rotation(&mut transaction)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
super::super::master_key::ensure_active_master_key_epoch(
&mut transaction,
request.executable_input_master_key_epoch,
)
.await
.map_err(|error| match error {
crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()),
_ => TaskError::StateConflict,
})?;
// Authority is revalidated inside this transaction, before any receipt lookup and under the
// same lock order: master key check, platform key share lock, Task row lock.
let key_row = sqlx::query(
"select k.id from platform_api_keys k
join tasks t on t.platform_api_key_id = k.id
where t.task_id = $1 and t.owner_kind = 'agent'
and k.workspace_id = $2 and k.agent_id = $3
and k.key_kind = 'mcp_client' and k.status = 'active'
and (k.expires_at is null or k.expires_at > clock_timestamp())
and (k.scopes_json ? 'write' or k.scopes_json ? 'deploy')
for share of k",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(&access.owner_id)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if key_row.is_none() {
transaction.rollback().await.ok();
// A revoked key, a foreign scope and an absent Task share one opaque answer.
return Err(TaskError::NotFound);
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5
for update"
)))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
// Receipt lookup precedes the generation and TTL checks: an authorized identical replay keeps
// returning the committed outcome even after completion, failure or TTL expiry.
let receipt = sqlx::query(
"select wait_id, update_key_digest, canonical_fingerprint, committed_state,
committed_version
from task_input_requests where task_id = $1 and accepted_at is not null",
)
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if let Some(receipt) = receipt {
let wait_id: String = receipt.try_get("wait_id").map_err(Self::unavailable)?;
if wait_id != request.wait_id {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let stored_key: String = receipt
.try_get("update_key_digest")
.map_err(Self::unavailable)?;
if stored_key != request.update_key_digest {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let stored_fingerprint: String = receipt
.try_get("canonical_fingerprint")
.map_err(Self::unavailable)?;
if stored_fingerprint != request.canonical_fingerprint {
transaction.rollback().await.ok();
return Err(TaskError::FingerprintConflict);
}
let state: String = receipt
.try_get("committed_state")
.map_err(Self::unavailable)?;
let version: i32 = receipt
.try_get("committed_version")
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
return Ok(TaskInputUpdateOutcome {
task_id: request.task_id,
wait_id,
state: TaskState::parse(&state)?,
version: u32::try_from(version).unwrap_or_default(),
replayed: true,
});
}
let pending = sqlx::query(
"select wait_id, expected_version,
(extract(epoch from expires_at) * 1000)::bigint as expires_at_ms,
(extract(epoch from clock_timestamp()) * 1000)::bigint as now_ms
from task_input_requests where task_id = $1 and accepted_at is null",
)
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(pending) = pending else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let wait_id: String = pending.try_get("wait_id").map_err(Self::unavailable)?;
let expected_version: i32 = pending
.try_get("expected_version")
.map_err(Self::unavailable)?;
let expires_at_ms: i64 = pending
.try_get("expires_at_ms")
.map_err(Self::unavailable)?;
// PostgreSQL clock, read after the row lock was taken: never a client timestamp.
let now_ms: i64 = pending.try_get("now_ms").map_err(Self::unavailable)?;
if wait_id != request.wait_id
|| u32::try_from(expected_version).unwrap_or_default() != request.expected_version
|| record.version != request.expected_version
|| record.state != TaskState::InputRequired
{
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if record.cancel_requested_at_unix_ms.is_some() {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
if now_ms >= expires_at_ms {
// The same transactional helper the sweeper uses: the Task ends failed with an audit
// reason and no dispatch. The update never revives it.
expire_locked(&mut transaction, &record, &wait_id, now_ms).await?;
transaction.commit().await.map_err(Self::unavailable)?;
return Err(TaskError::InputExpired);
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'working',
effective_input_fingerprint = $1,
executable_input_ciphertext = $2,
executable_input_key_version = $3,
executable_input_master_key_epoch = $4,
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $5 and state = 'input_required' and version = $6
returning {TASK_PROJECTION}"
)))
.bind(&request.effective_input_fingerprint)
.bind(&request.executable_input_ciphertext)
.bind(&request.executable_input_key_version)
.bind(request.executable_input_master_key_epoch)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let accepted = record_from_row(&row)?;
let updated = sqlx::query(
"update task_input_requests
set accepted_at = now(),
update_key_digest = $1,
canonical_fingerprint = $2,
effective_input_fingerprint = $3,
committed_state = 'working',
committed_version = $4
where task_id = $5 and wait_id = $6 and accepted_at is null",
)
.bind(&request.update_key_digest)
.bind(&request.canonical_fingerprint)
.bind(&request.effective_input_fingerprint)
.bind(i64::from(accepted.version))
.bind(request.task_id.as_uuid())
.bind(&request.wait_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if updated.rows_affected() != 1 {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
insert_audit(
&mut transaction,
request.task_id,
&request.wait_id,
"input_accepted",
TaskState::InputRequired,
TaskState::Working,
record.version,
accepted.version,
Some(access.owner_id.as_str()),
None,
)
.await?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(TaskInputUpdateOutcome {
task_id: request.task_id,
wait_id: request.wait_id,
state: TaskState::Working,
version: accepted.version,
replayed: false,
})
}
/// Bounded TTL sweep, safe to call on every poll while all worker slots are busy.
pub(super) async fn expire_input_waits_locked(&self, batch: u32) -> Result<u64, TaskError> {
let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT));
let mut transaction = self.start().await?;
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks t
where t.state = 'input_required'
and exists (
select 1 from task_input_requests r
where r.task_id = t.task_id and r.accepted_at is null
and r.expires_at <= clock_timestamp()
)
order by t.task_id
limit $1
for update of t skip locked"
)))
.bind(batch)
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let mut expired = 0u64;
for row in rows {
let record = record_from_row(&row)?;
let Some(wait_id) = sqlx::query_scalar::<_, String>(
"select wait_id from task_input_requests
where task_id = $1 and accepted_at is null and expires_at <= clock_timestamp()",
)
.bind(record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?
else {
continue;
};
if expire_locked(
&mut transaction,
&record,
&wait_id,
crank_core::current_unix_ms(),
)
.await?
{
expired += 1;
}
}
transaction.commit().await.map_err(Self::unavailable)?;
Ok(expired)
}
}
/// Marks one locked `input_required` Task failed, releases its admission permit and writes the audit
/// reason. Returns whether this call performed the transition.
async fn expire_locked(
transaction: &mut Transaction<'_, Postgres>,
record: &TaskRecord,
wait_id: &str,
_now_ms: i64,
) -> Result<bool, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = 'failed',
error_envelope = jsonb_build_object('reason', 'input_ttl_expired'),
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'input_required' and version = $2
returning {TASK_PROJECTION}"
)))
.bind(record.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
let Some(row) = row else {
return Ok(false);
};
let failed = record_from_row(&row)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(record.task_id.as_uuid())
.execute(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
insert_audit(
transaction,
record.task_id,
wait_id,
"input_expired",
TaskState::InputRequired,
TaskState::Failed,
record.version,
failed.version,
None,
Some("input_ttl_expired"),
)
.await?;
Ok(true)
}
#[allow(clippy::too_many_arguments)]
async fn insert_audit(
transaction: &mut Transaction<'_, Postgres>,
task_id: crank_core::TaskId,
wait_id: &str,
event: &str,
from_state: TaskState,
to_state: TaskState,
from_version: u32,
to_version: u32,
actor: Option<&str>,
reason_code: Option<&str>,
) -> Result<(), TaskError> {
sqlx::query(
"insert into task_input_audit (
task_id, wait_id, event, from_state, to_state, from_version, to_version, actor,
reason_code
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
)
.bind(task_id.as_uuid())
.bind(wait_id)
.bind(event)
.bind(from_state.as_str())
.bind(to_state.as_str())
.bind(i32::try_from(from_version).unwrap_or(i32::MAX))
.bind(i32::try_from(to_version).unwrap_or(i32::MAX))
.bind(actor)
.bind(reason_code)
.execute(&mut **transaction)
.await
.map_err(PostgresTaskStore::unavailable)?;
Ok(())
}