//! Input-continuation half of the PostgreSQL Task store (Story 4.5, FR-25). //! //! One transaction owns each transition: suspend (`working` -> `input_required`), the committed //! answer (`input_required` -> `working`) and the TTL expiry (`input_required` -> `failed`). Authority, //! generation, TTL and receipt checks run under the same Task row lock, so a revocation, a competing //! answer, a sweeper or a cancellation has exactly one winner and never a duplicated side effect. //! //! Raw input never enters a request, receipt, audit row or error text: the validated payload is //! encrypted by the runtime service before this module sees it. use crank_core::{ TaskAccess, TaskError, TaskInputRequest, TaskInputStore, TaskInputUpdateOutcome, TaskInputUpdateRequest, TaskRecord, TaskState, }; use sqlx::{Postgres, Row, Transaction}; use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row}; #[async_trait::async_trait] impl TaskInputStore for PostgresTaskStore { async fn suspend_for_input( &self, lease: &crank_core::TaskLease, request: TaskInputRequest, ) -> Result { self.suspend_input_locked(lease, request).await } async fn input_request( &self, access: &TaskAccess, task_id: crank_core::TaskId, ) -> Result { self.read_input_request(access, task_id).await } async fn read_executable_input_for_update( &self, access: &TaskAccess, task_id: crank_core::TaskId, ) -> Result<(String, String, i64), TaskError> { self.read_binding_for_update(access, task_id).await } async fn apply_input( &self, request: TaskInputUpdateRequest, ) -> Result { self.apply_input_locked(request).await } async fn expire_input_waits(&self, batch: u32) -> Result { self.expire_input_waits_locked(batch).await } } impl PostgresTaskStore { /// One transactional suspend of a leased Task before any dispatch checkpoint. pub(super) async fn suspend_input_locked( &self, lease: &crank_core::TaskLease, request: TaskInputRequest, ) -> Result { request.validate()?; if request.expected_version != lease.record.version.saturating_add(1) { return Err(TaskError::StateConflict); } let mut transaction = self.start().await?; let row = sqlx::query(sqlx::AssertSqlSafe(format!( "select {TASK_PROJECTION} from tasks where task_id = $1 for update" ))) .bind(lease.record.task_id.as_uuid()) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)?; let Some(row) = row else { transaction.rollback().await.ok(); return Err(TaskError::NotFound); }; let record = record_from_row(&row)?; let current = record.state == TaskState::Working && record.lease_owner.as_deref() == Some(lease.owner.as_str()) && record.fence == lease.fence && record.version == lease.record.version && record.current_attempt_id.is_none() && record.attempts == 0 && record.cancel_requested_at_unix_ms.is_none() && record .lease_until_unix_ms .is_some_and(|until| until > crank_core::current_unix_ms()) && record.pins.execution_deadline_unix_ms > crank_core::current_unix_ms(); // A generation that already dispatched may not be suspended: that would hide a possible side // effect behind an apparent "waiting for input". if !current { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); } if request.expires_at_unix_ms > record.pins.execution_deadline_unix_ms { transaction.rollback().await.ok(); return Err(TaskError::invalid( "task input TTL exceeds the execution deadline", )); } sqlx::query( "insert into task_input_requests ( task_id, wait_id, expected_version, schema_revision, schema_format_version, fingerprint_format_version, schema_snapshot, projected_schema, prompt_code, missing_fields, expires_at ) values ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, to_timestamp($11::double precision / 1000.0) )", ) .bind(lease.record.task_id.as_uuid()) .bind(&request.wait_id) .bind(i32::try_from(request.expected_version).unwrap_or(i32::MAX)) .bind(i32::try_from(request.schema_revision).unwrap_or(i32::MAX)) .bind(i32::try_from(request.schema_format_version).unwrap_or(i32::MAX)) .bind(i32::try_from(request.fingerprint_format_version).unwrap_or(i32::MAX)) .bind(request.schema_snapshot.clone()) .bind(request.projected_schema.clone()) .bind(&request.prompt_code) .bind(serde_json::Value::Array( request .missing_fields .iter() .cloned() .map(serde_json::Value::String) .collect(), )) .bind(request.expires_at_unix_ms) .execute(&mut *transaction) .await .map_err(|error| { match error .as_database_error() .and_then(|db| db.code()) .as_deref() { Some("23505") => TaskError::StateConflict, _ => Self::unavailable(error), } })?; let row = sqlx::query(sqlx::AssertSqlSafe(format!( "update tasks set state = 'input_required', lease_owner = null, lease_until = null, version = version + 1, updated_at = now() where task_id = $1 and state = 'working' and version = $2 and lease_owner = $3 and fence = $4 and current_attempt_id is null and lease_until > clock_timestamp() returning {TASK_PROJECTION}" ))) .bind(lease.record.task_id.as_uuid()) .bind(i64::from(record.version)) .bind(&lease.owner) .bind(i64::try_from(lease.fence).unwrap_or_default()) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)?; let Some(row) = row else { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); }; let suspended = record_from_row(&row)?; insert_audit( &mut transaction, lease.record.task_id, &request.wait_id, "input_required", TaskState::Working, TaskState::InputRequired, record.version, suspended.version, None, None, ) .await?; transaction.commit().await.map_err(Self::unavailable)?; Ok(suspended) } /// Owner-scoped read of the durable request; a foreign Task is one opaque not-found. pub(super) async fn read_input_request( &self, access: &TaskAccess, task_id: crank_core::TaskId, ) -> Result { access.validate()?; let row = sqlx::query( "select r.wait_id, r.expected_version, r.schema_revision, r.schema_format_version, r.fingerprint_format_version, r.schema_snapshot, r.projected_schema, r.prompt_code, r.missing_fields, (extract(epoch from r.expires_at) * 1000)::bigint as expires_at_ms from task_input_requests r join tasks t on t.task_id = r.task_id where r.task_id = $1 and t.workspace_id = $2 and t.owner_kind = $3 and t.owner_id = $4 and t.authorization_scope = $5", ) .bind(task_id.as_uuid()) .bind(&access.workspace_id) .bind(access.owner_kind.as_str()) .bind(&access.owner_id) .bind(&access.authorization_scope) .fetch_optional(&self.pool) .await .map_err(Self::unavailable)?; let Some(row) = row else { return Err(TaskError::NotFound); }; let missing_fields: Vec = serde_json::from_value( row.try_get::("missing_fields") .map_err(Self::unavailable)?, ) .map_err(|_| TaskError::unavailable("stored input request metadata is invalid"))?; Ok(TaskInputRequest { wait_id: row.try_get("wait_id").map_err(Self::unavailable)?, expected_version: u32::try_from( row.try_get::("expected_version") .map_err(Self::unavailable)?, ) .unwrap_or_default(), schema_revision: u32::try_from( row.try_get::("schema_revision") .map_err(Self::unavailable)?, ) .unwrap_or_default(), schema_format_version: u32::try_from( row.try_get::("schema_format_version") .map_err(Self::unavailable)?, ) .unwrap_or_default(), schema_snapshot: row.try_get("schema_snapshot").map_err(Self::unavailable)?, projected_schema: row.try_get("projected_schema").map_err(Self::unavailable)?, prompt_code: row.try_get("prompt_code").map_err(Self::unavailable)?, missing_fields, expires_at_unix_ms: row.try_get("expires_at_ms").map_err(Self::unavailable)?, fingerprint_format_version: u32::try_from( row.try_get::("fingerprint_format_version") .map_err(Self::unavailable)?, ) .unwrap_or_default(), }) } /// Owner-scoped read of the binding a continuation may extend. Never lease-authorized. /// /// Deliberately not filtered by lifecycle state: an identical replay of an already committed /// answer (including after completion or TTL expiry) must reach the receipt lookup, which is the /// single authority for that outcome. pub(super) async fn read_binding_for_update( &self, access: &TaskAccess, task_id: crank_core::TaskId, ) -> Result<(String, String, i64), TaskError> { access.validate()?; let row = sqlx::query( "select executable_input_ciphertext, executable_input_key_version, executable_input_master_key_epoch from tasks where task_id = $1 and workspace_id = $2 and owner_kind = $3 and owner_id = $4 and authorization_scope = $5", ) .bind(task_id.as_uuid()) .bind(&access.workspace_id) .bind(access.owner_kind.as_str()) .bind(&access.owner_id) .bind(&access.authorization_scope) .fetch_optional(&self.pool) .await .map_err(Self::unavailable)?; let Some(row) = row else { return Err(TaskError::NotFound); }; let ciphertext: Option = row .try_get("executable_input_ciphertext") .map_err(Self::unavailable)?; let key_version: Option = row .try_get("executable_input_key_version") .map_err(Self::unavailable)?; let epoch: Option = row .try_get("executable_input_master_key_epoch") .map_err(Self::unavailable)?; match (ciphertext, key_version, epoch) { (Some(ciphertext), Some(key_version), Some(epoch)) if !ciphertext.is_empty() => { Ok((ciphertext, key_version, epoch)) } _ => Err(TaskError::NotFound), } } /// One transactional answer application: authority, receipt, generation, TTL and CAS. pub(super) async fn apply_input_locked( &self, request: TaskInputUpdateRequest, ) -> Result { request.validate()?; let access = &request.access; let mut transaction = self.start().await?; super::super::master_key::ensure_no_active_master_key_rotation(&mut transaction) .await .map_err(|error| match error { crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()), _ => TaskError::StateConflict, })?; super::super::master_key::ensure_active_master_key_epoch( &mut transaction, request.executable_input_master_key_epoch, ) .await .map_err(|error| match error { crate::RegistryError::Storage(_) => TaskError::unavailable(error.to_string()), _ => TaskError::StateConflict, })?; // Authority is revalidated inside this transaction, before any receipt lookup and under the // same lock order: master key check, platform key share lock, Task row lock. let key_row = sqlx::query( "select k.id from platform_api_keys k join tasks t on t.platform_api_key_id = k.id where t.task_id = $1 and t.owner_kind = 'agent' and k.workspace_id = $2 and k.agent_id = $3 and k.key_kind = 'mcp_client' and k.status = 'active' and (k.expires_at is null or k.expires_at > clock_timestamp()) and (k.scopes_json ? 'write' or k.scopes_json ? 'deploy') for share of k", ) .bind(request.task_id.as_uuid()) .bind(&access.workspace_id) .bind(&access.owner_id) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)?; if key_row.is_none() { transaction.rollback().await.ok(); // A revoked key, a foreign scope and an absent Task share one opaque answer. return Err(TaskError::NotFound); } let row = sqlx::query(sqlx::AssertSqlSafe(format!( "select {TASK_PROJECTION} from tasks where task_id = $1 and workspace_id = $2 and owner_kind = $3 and owner_id = $4 and authorization_scope = $5 for update" ))) .bind(request.task_id.as_uuid()) .bind(&access.workspace_id) .bind(access.owner_kind.as_str()) .bind(&access.owner_id) .bind(&access.authorization_scope) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)?; let Some(row) = row else { transaction.rollback().await.ok(); return Err(TaskError::NotFound); }; let record = record_from_row(&row)?; // Receipt lookup precedes the generation and TTL checks: an authorized identical replay keeps // returning the committed outcome even after completion, failure or TTL expiry. let receipt = sqlx::query( "select wait_id, update_key_digest, canonical_fingerprint, committed_state, committed_version from task_input_requests where task_id = $1 and accepted_at is not null", ) .bind(request.task_id.as_uuid()) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)?; if let Some(receipt) = receipt { let wait_id: String = receipt.try_get("wait_id").map_err(Self::unavailable)?; if wait_id != request.wait_id { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); } let stored_key: String = receipt .try_get("update_key_digest") .map_err(Self::unavailable)?; if stored_key != request.update_key_digest { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); } let stored_fingerprint: String = receipt .try_get("canonical_fingerprint") .map_err(Self::unavailable)?; if stored_fingerprint != request.canonical_fingerprint { transaction.rollback().await.ok(); return Err(TaskError::FingerprintConflict); } let state: String = receipt .try_get("committed_state") .map_err(Self::unavailable)?; let version: i32 = receipt .try_get("committed_version") .map_err(Self::unavailable)?; transaction.commit().await.map_err(Self::unavailable)?; return Ok(TaskInputUpdateOutcome { task_id: request.task_id, wait_id, state: TaskState::parse(&state)?, version: u32::try_from(version).unwrap_or_default(), replayed: true, }); } let pending = sqlx::query( "select wait_id, expected_version, (extract(epoch from expires_at) * 1000)::bigint as expires_at_ms, (extract(epoch from clock_timestamp()) * 1000)::bigint as now_ms from task_input_requests where task_id = $1 and accepted_at is null", ) .bind(request.task_id.as_uuid()) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)?; let Some(pending) = pending else { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); }; let wait_id: String = pending.try_get("wait_id").map_err(Self::unavailable)?; let expected_version: i32 = pending .try_get("expected_version") .map_err(Self::unavailable)?; let expires_at_ms: i64 = pending .try_get("expires_at_ms") .map_err(Self::unavailable)?; // PostgreSQL clock, read after the row lock was taken: never a client timestamp. let now_ms: i64 = pending.try_get("now_ms").map_err(Self::unavailable)?; if wait_id != request.wait_id || u32::try_from(expected_version).unwrap_or_default() != request.expected_version || record.version != request.expected_version || record.state != TaskState::InputRequired { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); } if record.cancel_requested_at_unix_ms.is_some() { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); } if now_ms >= expires_at_ms { // The same transactional helper the sweeper uses: the Task ends failed with an audit // reason and no dispatch. The update never revives it. expire_locked(&mut transaction, &record, &wait_id, now_ms).await?; transaction.commit().await.map_err(Self::unavailable)?; return Err(TaskError::InputExpired); } let row = sqlx::query(sqlx::AssertSqlSafe(format!( "update tasks set state = 'working', effective_input_fingerprint = $1, executable_input_ciphertext = $2, executable_input_key_version = $3, executable_input_master_key_epoch = $4, lease_owner = null, lease_until = null, version = version + 1, updated_at = now() where task_id = $5 and state = 'input_required' and version = $6 returning {TASK_PROJECTION}" ))) .bind(&request.effective_input_fingerprint) .bind(&request.executable_input_ciphertext) .bind(&request.executable_input_key_version) .bind(request.executable_input_master_key_epoch) .bind(request.task_id.as_uuid()) .bind(i64::from(record.version)) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)?; let Some(row) = row else { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); }; let accepted = record_from_row(&row)?; let updated = sqlx::query( "update task_input_requests set accepted_at = now(), update_key_digest = $1, canonical_fingerprint = $2, effective_input_fingerprint = $3, committed_state = 'working', committed_version = $4 where task_id = $5 and wait_id = $6 and accepted_at is null", ) .bind(&request.update_key_digest) .bind(&request.canonical_fingerprint) .bind(&request.effective_input_fingerprint) .bind(i64::from(accepted.version)) .bind(request.task_id.as_uuid()) .bind(&request.wait_id) .execute(&mut *transaction) .await .map_err(Self::unavailable)?; if updated.rows_affected() != 1 { transaction.rollback().await.ok(); return Err(TaskError::StateConflict); } insert_audit( &mut transaction, request.task_id, &request.wait_id, "input_accepted", TaskState::InputRequired, TaskState::Working, record.version, accepted.version, Some(access.owner_id.as_str()), None, ) .await?; transaction.commit().await.map_err(Self::unavailable)?; Ok(TaskInputUpdateOutcome { task_id: request.task_id, wait_id: request.wait_id, state: TaskState::Working, version: accepted.version, replayed: false, }) } /// Bounded TTL sweep, safe to call on every poll while all worker slots are busy. pub(super) async fn expire_input_waits_locked(&self, batch: u32) -> Result { let batch = i64::from(batch.clamp(1, crank_core::TASK_PURGE_BATCH_LIMIT)); let mut transaction = self.start().await?; let rows = sqlx::query(sqlx::AssertSqlSafe(format!( "select {TASK_PROJECTION} from tasks t where t.state = 'input_required' and exists ( select 1 from task_input_requests r where r.task_id = t.task_id and r.accepted_at is null and r.expires_at <= clock_timestamp() ) order by t.task_id limit $1 for update of t skip locked" ))) .bind(batch) .fetch_all(&mut *transaction) .await .map_err(Self::unavailable)?; let mut expired = 0u64; for row in rows { let record = record_from_row(&row)?; let Some(wait_id) = sqlx::query_scalar::<_, String>( "select wait_id from task_input_requests where task_id = $1 and accepted_at is null and expires_at <= clock_timestamp()", ) .bind(record.task_id.as_uuid()) .fetch_optional(&mut *transaction) .await .map_err(Self::unavailable)? else { continue; }; if expire_locked( &mut transaction, &record, &wait_id, crank_core::current_unix_ms(), ) .await? { expired += 1; } } transaction.commit().await.map_err(Self::unavailable)?; Ok(expired) } } /// Marks one locked `input_required` Task failed, releases its admission permit and writes the audit /// reason. Returns whether this call performed the transition. async fn expire_locked( transaction: &mut Transaction<'_, Postgres>, record: &TaskRecord, wait_id: &str, _now_ms: i64, ) -> Result { let row = sqlx::query(sqlx::AssertSqlSafe(format!( "update tasks set state = 'failed', error_envelope = jsonb_build_object('reason', 'input_ttl_expired'), lease_owner = null, lease_until = null, version = version + 1, updated_at = now() where task_id = $1 and state = 'input_required' and version = $2 returning {TASK_PROJECTION}" ))) .bind(record.task_id.as_uuid()) .bind(i64::from(record.version)) .fetch_optional(&mut **transaction) .await .map_err(PostgresTaskStore::unavailable)?; let Some(row) = row else { return Ok(false); }; let failed = record_from_row(&row)?; sqlx::query("delete from execution_admission_entries where task_id = $1") .bind(record.task_id.as_uuid()) .execute(&mut **transaction) .await .map_err(PostgresTaskStore::unavailable)?; insert_audit( transaction, record.task_id, wait_id, "input_expired", TaskState::InputRequired, TaskState::Failed, record.version, failed.version, None, Some("input_ttl_expired"), ) .await?; Ok(true) } #[allow(clippy::too_many_arguments)] async fn insert_audit( transaction: &mut Transaction<'_, Postgres>, task_id: crank_core::TaskId, wait_id: &str, event: &str, from_state: TaskState, to_state: TaskState, from_version: u32, to_version: u32, actor: Option<&str>, reason_code: Option<&str>, ) -> Result<(), TaskError> { sqlx::query( "insert into task_input_audit ( task_id, wait_id, event, from_state, to_state, from_version, to_version, actor, reason_code ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9)", ) .bind(task_id.as_uuid()) .bind(wait_id) .bind(event) .bind(from_state.as_str()) .bind(to_state.as_str()) .bind(i32::try_from(from_version).unwrap_or(i32::MAX)) .bind(i32::try_from(to_version).unwrap_or(i32::MAX)) .bind(actor) .bind(reason_code) .execute(&mut **transaction) .await .map_err(PostgresTaskStore::unavailable)?; Ok(()) }