CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 21m42s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped
Durable worker приостанавливает Task как input_required до dispatch, если закреплённая схема требует отсутствующее поле; authorized schema-validated update атомарно и с аудитом возобновляет тот же Task ровно один раз. Replay/conflict/TTL безопасны, ожидание удерживает admission permit. Additive schema v30: task_input_requests, fingerprint/version, TTL-индекс.
965 lines
37 KiB
Rust
965 lines
37 KiB
Rust
//! Durable background Task seam (Story 4.2, FR-22/FR-23).
|
|
//!
|
|
//! A Task is the internal, protocol-independent record of one accepted background request. This
|
|
//! module owns the domain vocabulary only: the lifecycle states, the accepted identity/pins, the
|
|
//! bounded persisted envelopes and the `TaskStore` port. The protocol adapter (Story 4.7) and the
|
|
//! worker (Story 4.3) do not define these semantics; the authoritative store lives in PostgreSQL
|
|
//! (`crank-registry`), so an accepted Task survives a crash or restart.
|
|
//!
|
|
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
|
|
//! Task was executed. A terminal state is written by a conditional transition only.
|
|
|
|
mod attempt;
|
|
mod input;
|
|
use async_trait::async_trait;
|
|
pub use attempt::*;
|
|
pub use input::*;
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_json::Value;
|
|
use uuid::Uuid;
|
|
|
|
use crate::correlation::{RequestId, TraceContext, TraceId};
|
|
use crate::ids::PlatformApiKeyId;
|
|
use crate::operation::ToolExecutionMode;
|
|
|
|
/// Byte budget of every persisted Task envelope (input, progress, result, error, schema metadata).
|
|
pub const TASK_ENVELOPE_MAX_BYTES: usize = 32_768;
|
|
/// Maximum byte budget of the decrypted executable Task input.
|
|
pub const TASK_EXECUTABLE_INPUT_MAX_BYTES: usize = TASK_ENVELOPE_MAX_BYTES;
|
|
/// Maximum byte budget of the stored encrypted executable Task input ciphertext.
|
|
pub const TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES: usize = 98_304;
|
|
/// Maximum nesting depth of a persisted Task envelope.
|
|
pub const TASK_ENVELOPE_MAX_DEPTH: usize = 8;
|
|
/// Maximum number of JSON members/nodes of a persisted Task envelope.
|
|
pub const TASK_ENVELOPE_MAX_NODES: usize = 256;
|
|
/// Bound of every scope/owner/operation identity column.
|
|
pub const TASK_IDENTITY_MAX_BYTES: usize = 160;
|
|
/// Bound of the caller-supplied idempotency key before it becomes a scoped digest.
|
|
pub const TASK_SCOPED_KEY_MAX_BYTES: usize = 128;
|
|
/// Bound of a saved approval reference (an identifier, never a bearer token).
|
|
pub const TASK_APPROVAL_REFERENCE_MAX_BYTES: usize = 160;
|
|
/// Default retention of a terminal Task after its execution deadline (internal bounded default).
|
|
pub const TASK_DEFAULT_RETENTION_MS: i64 = 86_400_000;
|
|
/// Upper bound on the retention an accepted Task may pin.
|
|
pub const TASK_MAX_RETENTION_MS: i64 = 30 * 86_400_000;
|
|
/// Upper bound of one retention/housekeeping batch.
|
|
pub const TASK_PURGE_BATCH_LIMIT: u32 = 128;
|
|
|
|
/// Stable identity of one accepted Task. Generated as UUIDv7 and returned only after the accepting
|
|
/// transaction committed, so the ID never exists without its durable record.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
|
#[serde(transparent)]
|
|
pub struct TaskId(Uuid);
|
|
|
|
impl TaskId {
|
|
pub fn generate() -> Self {
|
|
Self(Uuid::now_v7())
|
|
}
|
|
|
|
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
|
Uuid::parse_str(value)
|
|
.map(Self)
|
|
.map_err(|_| TaskError::invalid("task identity is not a UUID"))
|
|
}
|
|
|
|
pub fn as_uuid(self) -> Uuid {
|
|
self.0
|
|
}
|
|
}
|
|
|
|
impl std::fmt::Display for TaskId {
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
self.0.fmt(formatter)
|
|
}
|
|
}
|
|
|
|
/// Closed Task lifecycle vocabulary. `outcome_unknown` is a non-terminal state: a value commit that
|
|
/// failed to confirm its dispatch is never collapsed into a plain failure.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum TaskState {
|
|
Queued,
|
|
Working,
|
|
InputRequired,
|
|
OutcomeUnknown,
|
|
Completed,
|
|
Failed,
|
|
Cancelled,
|
|
}
|
|
|
|
impl TaskState {
|
|
pub const fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::Queued => "queued",
|
|
Self::Working => "working",
|
|
Self::InputRequired => "input_required",
|
|
Self::OutcomeUnknown => "outcome_unknown",
|
|
Self::Completed => "completed",
|
|
Self::Failed => "failed",
|
|
Self::Cancelled => "cancelled",
|
|
}
|
|
}
|
|
|
|
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
|
match value {
|
|
"queued" => Ok(Self::Queued),
|
|
"working" => Ok(Self::Working),
|
|
"input_required" => Ok(Self::InputRequired),
|
|
"outcome_unknown" => Ok(Self::OutcomeUnknown),
|
|
"completed" => Ok(Self::Completed),
|
|
"failed" => Ok(Self::Failed),
|
|
"cancelled" => Ok(Self::Cancelled),
|
|
_ => Err(TaskError::invalid("unknown task state")),
|
|
}
|
|
}
|
|
|
|
pub const fn is_terminal(self) -> bool {
|
|
matches!(self, Self::Completed | Self::Failed | Self::Cancelled)
|
|
}
|
|
|
|
/// One conditional transition may only move along this table. A terminal state has no successor,
|
|
/// so a repeated terminal write can never rewrite an accepted result.
|
|
pub const fn allows_transition_to(self, next: Self) -> bool {
|
|
match self {
|
|
Self::Queued => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
|
|
Self::Working => matches!(
|
|
next,
|
|
Self::InputRequired
|
|
| Self::OutcomeUnknown
|
|
| Self::Completed
|
|
| Self::Failed
|
|
| Self::Cancelled
|
|
),
|
|
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
|
|
// Story 4.4: an uncertain outbound outcome is never resolved by a generic transition.
|
|
// Reconciliation is Story 4.6, so no ordinary finish/cancel/retry may leave this state.
|
|
Self::OutcomeUnknown => false,
|
|
Self::Completed | Self::Failed | Self::Cancelled => false,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Who owns the accepted Task. An administrator is never a wildcard owner of every Task: the owner
|
|
/// identity and the authorization scope are separate, both-pinned columns.
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum TaskOwnerKind {
|
|
Agent,
|
|
Admin,
|
|
}
|
|
|
|
impl TaskOwnerKind {
|
|
pub const fn as_str(self) -> &'static str {
|
|
match self {
|
|
Self::Agent => "agent",
|
|
Self::Admin => "admin",
|
|
}
|
|
}
|
|
|
|
pub fn parse(value: &str) -> Result<Self, TaskError> {
|
|
match value {
|
|
"agent" => Ok(Self::Agent),
|
|
"admin" => Ok(Self::Admin),
|
|
_ => Err(TaskError::invalid("unknown task owner kind")),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Support identities captured at acceptance. They are stored and returned independently of
|
|
/// sampling, and a replay never rewrites them.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskSupportIds {
|
|
pub request_id: RequestId,
|
|
pub trace_id: TraceId,
|
|
pub traceparent: String,
|
|
pub trace_sampled: bool,
|
|
}
|
|
|
|
impl TaskSupportIds {
|
|
pub fn new(request_id: RequestId, trace_context: &TraceContext) -> Self {
|
|
Self {
|
|
request_id,
|
|
trace_id: trace_context.trace_id().clone(),
|
|
traceparent: trace_context.traceparent().to_owned(),
|
|
trace_sampled: trace_context.is_sampled(),
|
|
}
|
|
}
|
|
|
|
/// Re-validates both the trace identity and the parent context: a Task must never persist an
|
|
/// inconsistent or malformed support pair, and an unsampled context is still a valid parent.
|
|
pub fn validate(&self) -> Result<(), TaskError> {
|
|
if self.trace_id.as_str().is_empty() {
|
|
return Err(TaskError::invalid("task trace identity is missing"));
|
|
}
|
|
let context = TraceContext::parse(&self.traceparent)
|
|
.map_err(|_| TaskError::invalid("task trace parent is invalid"))?;
|
|
if context.trace_id() != &self.trace_id || context.is_sampled() != self.trace_sampled {
|
|
return Err(TaskError::invalid("task trace parent is inconsistent"));
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Execution semantics pinned at acceptance: a later publication never rewrites them.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskPins {
|
|
pub operation_version: u32,
|
|
pub policy_mode: ToolExecutionMode,
|
|
pub execution_deadline_unix_ms: i64,
|
|
/// Retention deadline of the record; never the same thing as the execution deadline.
|
|
pub retain_until_unix_ms: i64,
|
|
/// Verified approval evidence reference (identifier only, never a bearer token).
|
|
pub approval_reference: Option<String>,
|
|
/// Bounded retry budget pinned at acceptance (Story 4.4). A later publication, a restart or a
|
|
/// new binary never changes the numbers an accepted Task was admitted with.
|
|
pub retry: TaskRetryPolicy,
|
|
}
|
|
|
|
impl TaskPins {
|
|
pub fn validate(&self) -> Result<(), TaskError> {
|
|
if self.operation_version == 0 {
|
|
return Err(TaskError::invalid(
|
|
"task operation version must be positive",
|
|
));
|
|
}
|
|
if self.execution_deadline_unix_ms <= 0 {
|
|
return Err(TaskError::invalid("task execution deadline is invalid"));
|
|
}
|
|
if self.retain_until_unix_ms < self.execution_deadline_unix_ms {
|
|
return Err(TaskError::invalid(
|
|
"task retention deadline precedes execution",
|
|
));
|
|
}
|
|
if self.retain_until_unix_ms - self.execution_deadline_unix_ms > TASK_MAX_RETENTION_MS {
|
|
return Err(TaskError::invalid("task retention exceeds its bound"));
|
|
}
|
|
if let Some(reference) = self.approval_reference.as_deref()
|
|
&& (reference.is_empty()
|
|
|| reference.len() > TASK_APPROVAL_REFERENCE_MAX_BYTES
|
|
|| reference.bytes().any(|byte| byte.is_ascii_control()))
|
|
{
|
|
return Err(TaskError::invalid("task approval reference is invalid"));
|
|
}
|
|
self.retry.validate()?;
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Bounded, classified payload of a Task. Construction validates the byte/depth/node budget and
|
|
/// redacts secret-bearing fields before anything is hashed or persisted.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskEnvelope {
|
|
value: Value,
|
|
}
|
|
|
|
impl TaskEnvelope {
|
|
/// Validates the caller-provided value, then stores a redacted copy of it. The caller hashes the
|
|
/// original value separately, so redaction never collapses two different requests into one.
|
|
pub fn try_redacted(input: &Value) -> Result<Self, TaskError> {
|
|
validate_task_envelope(input)?;
|
|
Ok(Self {
|
|
value: redact_task_envelope(input),
|
|
})
|
|
}
|
|
|
|
/// Accepts a value that was already validated and redacted at a trust boundary.
|
|
pub fn from_persisted(value: Value) -> Result<Self, TaskError> {
|
|
validate_task_envelope(&value)?;
|
|
Ok(Self { value })
|
|
}
|
|
|
|
pub fn value(&self) -> &Value {
|
|
&self.value
|
|
}
|
|
|
|
pub fn into_value(self) -> Value {
|
|
self.value
|
|
}
|
|
}
|
|
|
|
/// Validates the size, nesting depth and node count of one Task envelope. Every byte, collection and
|
|
/// node is counted before the value is cloned, hashed or persisted.
|
|
pub fn validate_task_envelope(value: &Value) -> Result<(), TaskError> {
|
|
let encoded = serde_json::to_vec(value)
|
|
.map_err(|_| TaskError::invalid("task envelope is not serializable"))?;
|
|
if encoded.len() > TASK_ENVELOPE_MAX_BYTES {
|
|
return Err(TaskError::invalid("task envelope exceeds its byte budget"));
|
|
}
|
|
let mut nodes = 0usize;
|
|
check_budget(value, 1, &mut nodes)?;
|
|
Ok(())
|
|
}
|
|
|
|
fn check_budget(value: &Value, depth: usize, nodes: &mut usize) -> Result<(), TaskError> {
|
|
if depth > TASK_ENVELOPE_MAX_DEPTH {
|
|
return Err(TaskError::invalid("task envelope exceeds its depth budget"));
|
|
}
|
|
*nodes += 1;
|
|
if *nodes > TASK_ENVELOPE_MAX_NODES {
|
|
return Err(TaskError::invalid("task envelope exceeds its node budget"));
|
|
}
|
|
match value {
|
|
Value::Object(object) => {
|
|
for (key, child) in object {
|
|
check_budget_name(key)?;
|
|
check_budget(child, depth + 1, nodes)?;
|
|
}
|
|
}
|
|
Value::Array(values) => {
|
|
for child in values {
|
|
check_budget(child, depth + 1, nodes)?;
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn check_budget_name(key: &str) -> Result<(), TaskError> {
|
|
if key.len() > TASK_IDENTITY_MAX_BYTES {
|
|
return Err(TaskError::invalid("task envelope key exceeds its budget"));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Field-name classification of secret-bearing values. A matching field keeps its position but never
|
|
/// its plaintext in the persisted envelope.
|
|
pub fn sensitive_task_key(name: &str) -> bool {
|
|
let lowered = name.to_ascii_lowercase();
|
|
[
|
|
"password",
|
|
"passwd",
|
|
"secret",
|
|
"token",
|
|
"api_key",
|
|
"apikey",
|
|
"authorization",
|
|
"credential",
|
|
"private_key",
|
|
"cookie",
|
|
"signature",
|
|
]
|
|
.iter()
|
|
.any(|marker| lowered.contains(marker))
|
|
}
|
|
|
|
/// Redacted projection of an already validated envelope.
|
|
pub fn redact_task_envelope(value: &Value) -> Value {
|
|
match value {
|
|
Value::Object(object) => Value::Object(
|
|
object
|
|
.iter()
|
|
.map(|(key, child)| {
|
|
let value = if sensitive_task_key(key) {
|
|
Value::String("[redacted]".to_owned())
|
|
} else {
|
|
redact_task_envelope(child)
|
|
};
|
|
(key.clone(), value)
|
|
})
|
|
.collect(),
|
|
),
|
|
Value::Array(values) => Value::Array(values.iter().map(redact_task_envelope).collect()),
|
|
other => other.clone(),
|
|
}
|
|
}
|
|
|
|
/// Recursive check used to fail closed on sensitive fields before encryption.
|
|
pub fn contains_sensitive_task_keys(value: &Value) -> bool {
|
|
match value {
|
|
Value::Object(object) => object
|
|
.iter()
|
|
.any(|(key, child)| sensitive_task_key(key) || contains_sensitive_task_keys(child)),
|
|
Value::Array(values) => values.iter().any(contains_sensitive_task_keys),
|
|
_ => false,
|
|
}
|
|
}
|
|
/// Write request of a new durable Task. The store owns the transaction: Task, dispatch intent and
|
|
/// queue capacity commit together, and no acknowledgement exists before that commit.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskAcceptRequest {
|
|
pub task_id: TaskId,
|
|
pub workspace_id: String,
|
|
pub owner_kind: TaskOwnerKind,
|
|
pub owner_id: String,
|
|
pub authorization_scope: String,
|
|
/// Typed identity of the Platform API key that authorized the acceptance (Story 4.3). Only the
|
|
/// immutable key ID is stored, never the bearer secret; a worker revalidates it authoritatively.
|
|
/// `None` means the accepting principal has no machine key (administrator) and fail-closes.
|
|
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
|
pub operation_id: String,
|
|
pub pins: TaskPins,
|
|
/// Digest of the caller key over the full owner/authorization/Tool scope.
|
|
pub scoped_key: String,
|
|
/// Digest of the canonical validated executable input.
|
|
pub input_fingerprint: String,
|
|
pub input: TaskEnvelope,
|
|
pub support: TaskSupportIds,
|
|
/// Bounded encrypted executable input, persisted alongside the redacted input projection.
|
|
pub executable_input_ciphertext: Option<String>,
|
|
pub executable_input_key_version: Option<String>,
|
|
pub executable_input_master_key_epoch: Option<i64>,
|
|
}
|
|
|
|
impl TaskAcceptRequest {
|
|
pub fn validate(&self) -> Result<(), TaskError> {
|
|
for (field, value) in [
|
|
("workspace", self.workspace_id.as_str()),
|
|
("owner", self.owner_id.as_str()),
|
|
("authorization scope", self.authorization_scope.as_str()),
|
|
("operation", self.operation_id.as_str()),
|
|
] {
|
|
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
|
|
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
|
}
|
|
}
|
|
if let Some(key_id) = self.platform_api_key_id.as_ref()
|
|
&& (key_id.as_str().is_empty() || key_id.as_str().len() > TASK_IDENTITY_MAX_BYTES)
|
|
{
|
|
return Err(TaskError::invalid(
|
|
"task platform api key reference is invalid",
|
|
));
|
|
}
|
|
if self.scoped_key.len() < 16 || self.scoped_key.len() > TASK_SCOPED_KEY_MAX_BYTES {
|
|
return Err(TaskError::invalid("task scoped key digest is invalid"));
|
|
}
|
|
if self.input_fingerprint.len() != 64
|
|
|| !self
|
|
.input_fingerprint
|
|
.bytes()
|
|
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
|
{
|
|
return Err(TaskError::invalid("task input fingerprint is invalid"));
|
|
}
|
|
self.pins.validate()?;
|
|
self.support.validate()?;
|
|
let has_ciphertext = self.executable_input_ciphertext.is_some();
|
|
let has_key = self.executable_input_key_version.is_some();
|
|
let has_epoch = self.executable_input_master_key_epoch.is_some();
|
|
match (has_ciphertext, has_key, has_epoch) {
|
|
(false, false, false) => {}
|
|
(true, true, true) => {
|
|
let ciphertext = self.executable_input_ciphertext.as_deref().unwrap();
|
|
if ciphertext.is_empty()
|
|
|| ciphertext.len() > TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES
|
|
{
|
|
return Err(TaskError::invalid(
|
|
"task executable input ciphertext is invalid",
|
|
));
|
|
}
|
|
let key_version = self.executable_input_key_version.as_deref().unwrap();
|
|
if key_version.is_empty() || key_version.len() > 64 {
|
|
return Err(TaskError::invalid(
|
|
"task executable input ciphertext is invalid",
|
|
));
|
|
}
|
|
if self.executable_input_master_key_epoch.unwrap_or_default() < 1 {
|
|
return Err(TaskError::invalid(
|
|
"task executable input ciphertext is invalid",
|
|
));
|
|
}
|
|
}
|
|
_ => {
|
|
return Err(TaskError::invalid(
|
|
"task executable input ciphertext is invalid",
|
|
));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Successful acceptance, returned only after the accepting transaction committed. A replay returns
|
|
/// the stored identity and support IDs of the original acceptance.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskAcceptance {
|
|
pub task_id: TaskId,
|
|
pub state: TaskState,
|
|
/// Exact Published Version pinned by this acceptance; a replay returns the stored pin.
|
|
pub operation_version: u32,
|
|
pub support: TaskSupportIds,
|
|
pub created_at_unix_ms: i64,
|
|
pub retain_until_unix_ms: i64,
|
|
pub replayed: bool,
|
|
}
|
|
|
|
/// Owner-side access proof of a read or a terminal write. It is built by the application layer from
|
|
/// the verified principal, never from a caller-supplied marker, and every store query carries the
|
|
/// whole predicate so a foreign row is never fetched.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskAccess {
|
|
pub workspace_id: String,
|
|
pub owner_kind: TaskOwnerKind,
|
|
pub owner_id: String,
|
|
pub authorization_scope: String,
|
|
}
|
|
|
|
impl TaskAccess {
|
|
pub fn validate(&self) -> Result<(), TaskError> {
|
|
for (field, value) in [
|
|
("workspace", self.workspace_id.as_str()),
|
|
("owner", self.owner_id.as_str()),
|
|
("authorization scope", self.authorization_scope.as_str()),
|
|
] {
|
|
if value.is_empty() || value.len() > TASK_IDENTITY_MAX_BYTES {
|
|
return Err(TaskError::invalid(format!("task {field} is invalid")));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Read model of one Task. Lifecycle fields, pins and support IDs stay readable together; the
|
|
/// protocol adapter only maps them.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskRecord {
|
|
pub task_id: TaskId,
|
|
pub state: TaskState,
|
|
pub workspace_id: String,
|
|
pub owner_kind: TaskOwnerKind,
|
|
pub owner_id: String,
|
|
pub authorization_scope: String,
|
|
/// Typed immutable identity of the Platform API key that authorized the acceptance; the worker
|
|
/// revalidates this exact key against the registry before any outbound dispatch.
|
|
pub platform_api_key_id: Option<PlatformApiKeyId>,
|
|
pub operation_id: String,
|
|
/// Canonical digest of the accepted executable input (compare-and-conflict identity).
|
|
pub input_fingerprint: String,
|
|
/// Effective fingerprint of the payload the next attempt will execute. It equals the acceptance
|
|
/// fingerprint until a committed input continuation replaces the effective payload; the
|
|
/// acceptance fingerprint above is never overwritten, so an original accept replay stays replay.
|
|
pub effective_input_fingerprint: String,
|
|
pub pins: TaskPins,
|
|
pub support: TaskSupportIds,
|
|
pub input: TaskEnvelope,
|
|
pub progress: Option<TaskEnvelope>,
|
|
pub result: Option<TaskEnvelope>,
|
|
pub error: Option<TaskEnvelope>,
|
|
pub attempts: u32,
|
|
/// Monotonic lease/fencing token. It never resets and protects Task state mutations.
|
|
pub fence: u64,
|
|
/// Random process-instance lease owner, if a worker currently owns the Task.
|
|
pub lease_owner: Option<String>,
|
|
/// Lease expiry epoch milliseconds, if the Task is claimed.
|
|
pub lease_until_unix_ms: Option<i64>,
|
|
/// Current dispatch attempt identity, committed before any outbound adapter call.
|
|
pub current_attempt_id: Option<String>,
|
|
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
|
|
pub dispatch_started_at_unix_ms: Option<i64>,
|
|
/// Earliest moment the next durable attempt may be dispatched (pinned backoff).
|
|
pub next_attempt_at_unix_ms: Option<i64>,
|
|
/// Durable cancellation request that arrived while an attempt was already checkpointed.
|
|
pub cancel_requested_at_unix_ms: Option<i64>,
|
|
/// Bounded reason of the cancellation request.
|
|
pub cancel_reason: Option<String>,
|
|
/// Monotonic lifecycle version used by the terminal CAS.
|
|
pub version: u32,
|
|
pub created_at_unix_ms: i64,
|
|
pub updated_at_unix_ms: i64,
|
|
}
|
|
|
|
/// One conditional terminal transition. Exactly one concurrent writer wins; the losers observe the
|
|
/// accepted state and never rewrite the result.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct TaskFinishRequest {
|
|
pub access: TaskAccess,
|
|
pub task_id: TaskId,
|
|
pub expected_state: TaskState,
|
|
pub expected_version: u32,
|
|
pub target_state: TaskState,
|
|
pub result: Option<TaskEnvelope>,
|
|
pub error: Option<TaskEnvelope>,
|
|
}
|
|
|
|
impl TaskFinishRequest {
|
|
pub fn validate(&self) -> Result<(), TaskError> {
|
|
self.access.validate()?;
|
|
if !self.target_state.is_terminal() {
|
|
return Err(TaskError::invalid("task finish target is not terminal"));
|
|
}
|
|
if self.expected_state.is_terminal() {
|
|
return Err(TaskError::invalid("task finish source is already terminal"));
|
|
}
|
|
if !self.expected_state.allows_transition_to(self.target_state) {
|
|
return Err(TaskError::invalid("task state transition is not allowed"));
|
|
}
|
|
if self.target_state == TaskState::Completed && self.result.is_none() {
|
|
return Err(TaskError::invalid("a completed task requires a result"));
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Bounded retention outcome.
|
|
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
|
pub struct TaskPurgeReport {
|
|
/// Queued Tasks that passed their execution deadline and were failed with their intent removed.
|
|
pub expired_queued: u64,
|
|
/// Terminal Tasks that passed their retention deadline.
|
|
pub expired_terminal: u64,
|
|
}
|
|
|
|
/// Typed Task failure. A denial and a missing Task share one outward shape, so a foreign caller can
|
|
/// never distinguish "not yours" from "does not exist".
|
|
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
|
pub enum TaskError {
|
|
#[error("task request is invalid: {message}")]
|
|
Invalid { message: String },
|
|
#[error("task store is unavailable: {message}")]
|
|
Unavailable { message: String },
|
|
#[error("task was not found")]
|
|
NotFound,
|
|
#[error("task request conflicts with the accepted request")]
|
|
FingerprintConflict,
|
|
#[error("task lifecycle transition was rejected")]
|
|
StateConflict,
|
|
#[error("task input request has expired")]
|
|
InputExpired,
|
|
#[error("background task capacity is exhausted")]
|
|
Overloaded { retry_after_ms: u64 },
|
|
}
|
|
|
|
impl TaskError {
|
|
pub fn invalid(message: impl Into<String>) -> Self {
|
|
Self::Invalid {
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
pub fn unavailable(message: impl Into<String>) -> Self {
|
|
Self::Unavailable {
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
/// Whether the caller may tell this failure apart from an absent Task.
|
|
pub fn is_not_found(&self) -> bool {
|
|
matches!(self, Self::NotFound)
|
|
}
|
|
}
|
|
|
|
/// Leased view of a claimed Task: its durable record plus lease fencing identity.
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub struct TaskLease {
|
|
pub record: TaskRecord,
|
|
pub owner: String,
|
|
pub fence: u64,
|
|
pub lease_until: i64,
|
|
}
|
|
|
|
/// Authoritative durable Task port.
|
|
#[async_trait]
|
|
pub trait TaskStore: TaskInputStore + Send + Sync {
|
|
/// Proves the durable Task state is usable (schema and operations reachable).
|
|
async fn health(&self) -> Result<(), TaskError>;
|
|
|
|
/// Atomically accepts a Task: Task record, dispatch intent and one queue slot commit together.
|
|
async fn accept(&self, request: TaskAcceptRequest) -> Result<TaskAcceptance, TaskError>;
|
|
|
|
/// Reads one Task inside the caller's owner scope.
|
|
async fn read(&self, access: &TaskAccess, task_id: TaskId) -> Result<TaskRecord, TaskError>;
|
|
|
|
/// Scoped lookup of an already accepted Task by its full-scope idempotency digest. Used to keep a
|
|
/// retry of an explicitly pinned version from becoming new work; `None` means "not accepted yet".
|
|
async fn find(
|
|
&self,
|
|
access: &TaskAccess,
|
|
operation_id: &str,
|
|
scoped_key: &str,
|
|
) -> Result<Option<TaskRecord>, TaskError>;
|
|
|
|
/// Applies exactly one conditional terminal transition and releases its intent atomically.
|
|
async fn finish(&self, request: TaskFinishRequest) -> Result<TaskRecord, TaskError>;
|
|
|
|
/// Reads encrypted executable input only under a valid, unexpired current lease.
|
|
async fn read_executable_input(
|
|
&self,
|
|
owner: &str,
|
|
task_id: TaskId,
|
|
fence: u64,
|
|
) -> Result<(String, String, i64), TaskError>;
|
|
|
|
/// Bounded retention...
|
|
async fn purge_expired(&self, batch: u32) -> Result<TaskPurgeReport, TaskError>;
|
|
|
|
/// Claims a bounded batch of queued or safely recoverable Tasks under leases.
|
|
async fn claim_batch(
|
|
&self,
|
|
owner: &str,
|
|
batch_limit: u32,
|
|
lease_ms: i64,
|
|
) -> Result<Vec<TaskLease>, TaskError>;
|
|
|
|
/// Renews a lease owned by `owner` when fence and expiry still match.
|
|
async fn renew_lease(
|
|
&self,
|
|
owner: &str,
|
|
task_id: TaskId,
|
|
fence: u64,
|
|
lease_ms: i64,
|
|
) -> Result<TaskLease, TaskError>;
|
|
|
|
/// Persists redacted progress under a valid lease.
|
|
async fn write_progress(
|
|
&self,
|
|
owner: &str,
|
|
task_id: TaskId,
|
|
fence: u64,
|
|
progress: TaskEnvelope,
|
|
) -> Result<TaskRecord, TaskError>;
|
|
|
|
/// Commits one durable outbound attempt before network use: attempt row, incremented budget and
|
|
/// the new Task version commit in one transaction. A returned lease is the only authority to
|
|
/// dispatch that attempt; an unknown commit means zero outbound calls.
|
|
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError>;
|
|
|
|
/// Resolves exactly one committed attempt under its lease/fence/version. Only the current attempt
|
|
/// of the current owner may be resolved, and the attempt becomes immutable afterwards.
|
|
async fn resolve_attempt(
|
|
&self,
|
|
lease: &AttemptLease,
|
|
outcome: AttemptOutcome,
|
|
) -> Result<AttemptResolution, TaskError>;
|
|
|
|
/// Owner-scoped atomic cancellation. It serializes with claim and checkpoint on the same Task.
|
|
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError>;
|
|
|
|
/// Owner-scoped bounded history of the durable attempts of one Task.
|
|
async fn attempts(
|
|
&self,
|
|
access: &TaskAccess,
|
|
task_id: TaskId,
|
|
) -> Result<Vec<TaskAttempt>, TaskError>;
|
|
|
|
/// Applies a terminal transition only under a valid lease and releases intent atomically.
|
|
async fn finish_leased(
|
|
&self,
|
|
request: TaskFinishRequest,
|
|
owner: &str,
|
|
fence: u64,
|
|
) -> Result<TaskRecord, TaskError>;
|
|
|
|
/// Resolves a leased Task whose next attempt was refused before its dispatch checkpoint (elapsed
|
|
/// deadline, rejected authority). A Task that still carries a proven ambiguity is never downgraded
|
|
/// to a plain failure: it ends non-terminal `outcome_unknown` with `code`; otherwise `failed`.
|
|
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError>;
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use serde_json::json;
|
|
|
|
fn request() -> TaskAcceptRequest {
|
|
TaskAcceptRequest {
|
|
task_id: TaskId::generate(),
|
|
workspace_id: "ws_default".to_owned(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_1".to_owned(),
|
|
platform_api_key_id: Some(PlatformApiKeyId::new("pk_1")),
|
|
operation_id: "op_report".to_owned(),
|
|
pins: TaskPins {
|
|
operation_version: 3,
|
|
policy_mode: ToolExecutionMode::Background,
|
|
execution_deadline_unix_ms: 1_790_000_000_000,
|
|
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
|
|
approval_reference: Some("appr_1".to_owned()),
|
|
retry: TaskRetryPolicy::new(3, 1),
|
|
},
|
|
scoped_key: "a".repeat(64),
|
|
input_fingerprint: "b".repeat(64),
|
|
input: TaskEnvelope::try_redacted(&json!({"report": "q3"})).unwrap(),
|
|
support: TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
|
.unwrap(),
|
|
),
|
|
executable_input_ciphertext: None,
|
|
executable_input_key_version: None,
|
|
executable_input_master_key_epoch: None,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn uuid_v7_task_ids_are_unique_and_round_trip() {
|
|
let first = TaskId::generate();
|
|
let second = TaskId::generate();
|
|
assert_ne!(first, second);
|
|
assert_eq!(TaskId::parse(&first.to_string()).unwrap(), first);
|
|
}
|
|
|
|
#[test]
|
|
fn terminal_states_have_no_successor() {
|
|
for terminal in [
|
|
TaskState::Completed,
|
|
TaskState::Failed,
|
|
TaskState::Cancelled,
|
|
] {
|
|
assert!(terminal.is_terminal());
|
|
for next in [
|
|
TaskState::Queued,
|
|
TaskState::Working,
|
|
TaskState::Completed,
|
|
TaskState::Failed,
|
|
] {
|
|
assert!(!terminal.allows_transition_to(next));
|
|
}
|
|
}
|
|
assert!(TaskState::Queued.allows_transition_to(TaskState::Working));
|
|
assert!(TaskState::Working.allows_transition_to(TaskState::OutcomeUnknown));
|
|
assert!(!TaskState::Queued.allows_transition_to(TaskState::Completed));
|
|
assert!(!TaskState::InputRequired.allows_transition_to(TaskState::Completed));
|
|
}
|
|
|
|
#[test]
|
|
fn envelope_bounds_reject_bytes_depth_and_nodes() {
|
|
assert!(TaskEnvelope::try_redacted(&json!({"a": 1})).is_ok());
|
|
assert!(
|
|
TaskEnvelope::try_redacted(&json!({"a": "x".repeat(TASK_ENVELOPE_MAX_BYTES)})).is_err()
|
|
);
|
|
|
|
let mut deep = json!(1);
|
|
for _ in 0..=TASK_ENVELOPE_MAX_DEPTH {
|
|
deep = json!({"n": deep});
|
|
}
|
|
assert!(TaskEnvelope::try_redacted(&deep).is_err());
|
|
|
|
let wide = Value::Array((0..=TASK_ENVELOPE_MAX_NODES).map(|_| json!(1)).collect());
|
|
assert!(TaskEnvelope::try_redacted(&wide).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn secret_bearing_fields_are_redacted_and_keep_their_shape() {
|
|
let envelope = TaskEnvelope::try_redacted(&json!({
|
|
"query": "leads",
|
|
"auth": {"api_key": "sk_live_1", "nested": [{"password": "p"}]}
|
|
}))
|
|
.unwrap();
|
|
let value = envelope.value();
|
|
assert_eq!(value["query"], json!("leads"));
|
|
assert_eq!(value["auth"]["api_key"], json!("[redacted]"));
|
|
assert_eq!(value["auth"]["nested"][0]["password"], json!("[redacted]"));
|
|
}
|
|
|
|
#[test]
|
|
fn unsampled_support_ids_are_valid_and_consistent() {
|
|
let support = TaskSupportIds::new(
|
|
RequestId::generate(),
|
|
&TraceContext::parse("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00")
|
|
.unwrap(),
|
|
);
|
|
assert!(!support.trace_sampled);
|
|
assert!(support.traceparent.ends_with("-00"));
|
|
support.validate().unwrap();
|
|
|
|
let mut inconsistent = support.clone();
|
|
inconsistent.traceparent = inconsistent.traceparent.replace("-00", "-01");
|
|
assert!(inconsistent.validate().is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn accept_request_validation_holds_the_identity_and_pin_bounds() {
|
|
assert!(request().validate().is_ok());
|
|
|
|
let mut oversized = request();
|
|
oversized.workspace_id = "w".repeat(TASK_IDENTITY_MAX_BYTES + 1);
|
|
assert!(oversized.validate().is_err());
|
|
|
|
let mut bad_fingerprint = request();
|
|
bad_fingerprint.input_fingerprint = "XYZ".to_owned();
|
|
assert!(bad_fingerprint.validate().is_err());
|
|
|
|
let mut bad_retention = request();
|
|
bad_retention.pins.retain_until_unix_ms = bad_retention.pins.execution_deadline_unix_ms - 1;
|
|
assert!(bad_retention.validate().is_err());
|
|
|
|
// The persisted authority reference is bounded and never empty.
|
|
let mut empty_key_reference = request();
|
|
empty_key_reference.platform_api_key_id = Some(PlatformApiKeyId::new(""));
|
|
assert!(empty_key_reference.validate().is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn outcome_unknown_has_no_generic_successor() {
|
|
// Story 4.4: reconciliation of an uncertain outcome is Story 4.6. Until then no ordinary
|
|
// finish/cancel/retry may leave the non-terminal uncertain state.
|
|
for next in [
|
|
TaskState::Queued,
|
|
TaskState::Working,
|
|
TaskState::InputRequired,
|
|
TaskState::OutcomeUnknown,
|
|
TaskState::Completed,
|
|
TaskState::Failed,
|
|
TaskState::Cancelled,
|
|
] {
|
|
assert!(
|
|
!TaskState::OutcomeUnknown.allows_transition_to(next),
|
|
"outcome_unknown must not transition to {next:?}"
|
|
);
|
|
}
|
|
assert!(!TaskState::OutcomeUnknown.is_terminal());
|
|
}
|
|
|
|
#[test]
|
|
fn retry_policy_bounds_are_pinned_and_bounded() {
|
|
assert!(TaskRetryPolicy::single_attempt().validate().is_ok());
|
|
assert!(
|
|
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS)
|
|
.validate()
|
|
.is_ok()
|
|
);
|
|
assert!(TaskRetryPolicy::new(0, 0).validate().is_err());
|
|
assert!(
|
|
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS + 1, 0)
|
|
.validate()
|
|
.is_err()
|
|
);
|
|
assert!(
|
|
TaskRetryPolicy::new(1, TASK_MAX_BACKOFF_STEP_MS + 1)
|
|
.validate()
|
|
.is_err()
|
|
);
|
|
// The pinned step grows the delay but never beyond four steps.
|
|
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(2), 2);
|
|
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(5), 8);
|
|
assert_eq!(TaskRetryPolicy::single_attempt().backoff_before(2), 0);
|
|
// A pin whose retry snapshot is invalid is rejected at acceptance.
|
|
let mut invalid = request();
|
|
invalid.pins.retry = TaskRetryPolicy::new(0, 0);
|
|
assert!(invalid.validate().is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn finish_requires_a_terminal_reachable_transition() {
|
|
let access = TaskAccess {
|
|
workspace_id: "ws_default".to_owned(),
|
|
owner_kind: TaskOwnerKind::Agent,
|
|
owner_id: "agent_a".to_owned(),
|
|
authorization_scope: "credential:pk_1".to_owned(),
|
|
};
|
|
let base = TaskFinishRequest {
|
|
access,
|
|
task_id: TaskId::generate(),
|
|
expected_state: TaskState::Working,
|
|
expected_version: 1,
|
|
target_state: TaskState::Completed,
|
|
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
|
|
error: None,
|
|
};
|
|
assert!(base.validate().is_ok());
|
|
|
|
let mut non_terminal = base.clone();
|
|
non_terminal.target_state = TaskState::Working;
|
|
assert!(non_terminal.validate().is_err());
|
|
|
|
let mut terminal_source = base.clone();
|
|
terminal_source.expected_state = TaskState::Completed;
|
|
assert!(terminal_source.validate().is_err());
|
|
|
|
let mut completed_without_result = base;
|
|
completed_without_result.result = None;
|
|
assert!(completed_without_result.validate().is_err());
|
|
}
|
|
}
|