feat(epic4): безопасные повторы и отмена durable Task (Story 4.4)
CI / UI Checks (pull_request) Successful in 8s
CI / Rust Checks (pull_request) Successful in 20m46s
CI / Rust Feature Validation (pull_request) Skipped
CI / Frontend E2E (pull_request) Skipped
CI / Community Image Smoke (pull_request) Skipped
CI / Required Checks (pull_request) Successful in 1s
CI / Deploy (pull_request) Skipped

Durable per-outbound attempt до отправки запроса, bounded retry/read,
server-owned verified writes, честный cancel/unknown, deadline/fencing,
additive schema v29. Staged renames task.rs/tasks.rs разложены на модули.

Includes disclosed cosmetic parity exception: единственная правка вне
принятой дельты — удалена лишняя пустая строка в конце
scripts/ci-story-checks.sh (EOF blank line); story-код не менялся.
This commit is contained in:
bsodfather committed 2026-10-30 12:21:57 +00:00
1 parent 8c02380762
commit a3057de51e
44 files changed
+4547 -815

No files matched your search

@@ -0,0 +1,555 @@
//! Story 4.4 ATDD: no outbound send exists without a committed durable attempt, and an elapsed
//! accepted deadline fails before dispatch.
//!
//! Both oracles drive the production composition (application entry → PG store → runtime executor →
//! real local HTTP upstream); only the upstream and the checkpoint failure are test-controlled.
use super::*;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::TaskWorkerHandler;
use crank_core::{
AgentOperationBinding, PlatformApiKeyScope, TaskState, ToolExecutionMode, ToolExecutionPolicy,
};
use crank_runtime::{SecretCrypto, TaskWorker, TaskWorkerConfig};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use time::OffsetDateTime;
async fn composition(
registry: &PostgresRegistry,
upstream: &str,
policy: ToolExecutionPolicy,
caller_name: &str,
scoped_key: &str,
) -> (
Arc<PostgresTaskStore>,
crank_community_mcp::TaskApplication,
SecretCrypto,
crank_community_mcp::TaskPrincipal,
) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
registry,
caller_name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let _ = scoped_key;
(store, tasks, crypto, caller)
}
fn worker_for(
registry: &PostgresRegistry,
store: &Arc<PostgresTaskStore>,
tasks: &TaskApplication,
crypto: &SecretCrypto,
owner: &str,
) -> TaskWorker {
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
tasks,
registry.clone(),
runtime,
crypto.clone(),
));
let config = TaskWorkerConfig {
owner: owner.to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
TaskWorker::new(store.clone(), handler, config).expect("valid worker")
}
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
/// AC1: a dispatch checkpoint that cannot commit produces zero outbound calls and no attempt row.
#[tokio::test]
async fn checkpoint_commit_failure_never_reaches_upstream() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let (store, tasks, crypto, caller) = composition(
&registry,
&upstream,
background_policy(),
"attempt-guards-checkpoint",
"guards-checkpoint-key",
)
.await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-checkpoint-key",
&json!({"email": "user@example.com"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
// The checkpoint is the only statement that commits an attempt; it is refused here.
sqlx::raw_sql(sqlx::AssertSqlSafe(
"create function refuse_task_checkpoint() returns trigger language plpgsql as $$
begin
if new.current_attempt_id is not null and old.current_attempt_id is null then
raise exception 'durable checkpoint refused for test';
end if;
return new;
end $$;
create trigger refuse_task_checkpoint before update on tasks
for each row execute function refuse_task_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint refusal trigger");
let worker = worker_for(
&registry,
&store,
&tasks,
&crypto,
"worker-guards-checkpoint",
);
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::sleep(Duration::from_secs(2)).await;
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an uncommitted durable checkpoint must never produce an outbound call"
);
let attempts: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(
attempts, 0,
"no attempt history without a committed checkpoint"
);
assert_ne!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("completed"),
"the refused attempt cannot complete the Task"
);
sqlx::raw_sql(sqlx::AssertSqlSafe(
"drop trigger refuse_task_checkpoint on tasks; drop function refuse_task_checkpoint();",
))
.execute(registry.pool())
.await
.expect("checkpoint refusal trigger removed");
server.abort();
}
/// AC6: an accepted absolute deadline that already elapsed fails the Task before any dispatch.
#[tokio::test]
async fn elapsed_deadline_fails_before_dispatch_without_upstream_call() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id": "lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let short_deadline = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 300,
no_tasks_fallback: crank_core::NoTasksFallback::Unsupported,
};
let (store, tasks, crypto, caller) = composition(
&registry,
&upstream,
short_deadline,
"attempt-guards-deadline",
"guards-deadline-key",
)
.await;
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-deadline-key",
&json!({"email": "user@example.com"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
// The accepted deadline elapses before any worker claims the Task.
tokio::time::sleep(Duration::from_millis(500)).await;
let worker = worker_for(&registry, &store, &tasks, &crypto, "worker-guards-deadline");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("failed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
assert_eq!(state.as_deref(), Some("failed"));
assert_eq!(
calls.load(Ordering::SeqCst),
0,
"an elapsed deadline must fail before dispatch"
);
let attempts: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts, 0, "no attempt is checkpointed after the deadline");
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("task_deadline_elapsed"));
assert_eq!(
sqlx::query_scalar::<_, String>("select state from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap(),
TaskState::Failed.as_str()
);
server.abort();
}
/// AC2: a write whose upstream idempotency is backed by server-owned evidence is retried with the
/// same stable key, in a bounded number of distinct durable attempts, and applies exactly one effect.
#[tokio::test]
async fn verified_idempotent_write_retries_with_stable_key_and_one_effect() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
use std::collections::HashSet;
use std::sync::Mutex;
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let effects = Arc::new(AtomicUsize::new(0));
let keys = Arc::new(Mutex::new(Vec::<String>::new()));
let seen = Arc::new(Mutex::new(HashSet::<String>::new()));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
let effects = Arc::clone(&effects);
let keys = Arc::clone(&keys);
let seen = Arc::clone(&seen);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
let effects = Arc::clone(&effects);
let keys = Arc::clone(&keys);
let seen = Arc::clone(&seen);
async move {
let key = headers
.get("idempotency-key")
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
keys.lock().expect("keys").push(key.clone());
let first_application = seen.lock().expect("seen").insert(key);
if first_application {
effects.fetch_add(1, Ordering::SeqCst);
}
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// The first application is stored but its acknowledgement is lost.
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error": "acknowledgement lost"})),
);
}
(
StatusCode::OK,
axum::Json(json!({"id": "lead-1", "deduped": !first_application})),
)
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(background_policy());
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
publish(&registry, &operation.id, 1).await;
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(
&registry,
"attempt-guards-verified",
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("test master-key identity registered");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("valid task store configuration"),
);
let tasks = TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store.clone(), Arc::new(crypto.clone())),
);
let acceptance = tasks
.accept_background(
&caller,
TOOL_NAME,
None,
"guards-verified-key",
&json!({"email": "user@example.com", "idempotency_key": "k-1"}),
&unsampled_correlation(),
)
.await
.expect("durable acceptance");
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&tasks, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
let config = TaskWorkerConfig {
owner: "worker-guards-verified".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("completed") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(state.as_deref(), Some("completed"));
assert_eq!(
requests.load(Ordering::SeqCst),
2,
"exactly one retry of the write"
);
assert_eq!(
effects.load(Ordering::SeqCst),
1,
"the upstream applied the write exactly once"
);
assert_eq!(
keys.lock().expect("keys").as_slice(),
["k-1".to_owned(), "k-1".to_owned()],
"the same stable key is delivered on both attempts"
);
let attempts = store
.attempts(
&crank_core::TaskAccess {
workspace_id: common::test_workspace_id().to_string(),
owner_kind: crank_core::TaskOwnerKind::Agent,
owner_id: "agent_attempt_guards".to_owned(),
authorization_scope: String::new(),
},
acceptance.task_id,
)
.await;
// Owner scope of this principal is the agent of the published binding; read the history through
// the application entry instead of guessing the authorization scope string.
assert!(attempts.is_err(), "a foreign scope never reads the history");
let attempts = tasks
.attempts(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("owner-scoped attempt history");
assert_eq!(attempts.len(), 2, "two distinct durable attempts");
assert_eq!(attempts[0].ordinal, 1);
assert_eq!(attempts[1].ordinal, 2);
assert!(attempts.iter().all(|attempt| attempt.retry_allowed));
assert_eq!(
attempts[0].outcome,
Some(crank_core::AttemptOutcomeClass::RetryScheduled)
);
assert_eq!(
attempts[1].outcome,
Some(crank_core::AttemptOutcomeClass::Completed)
);
assert_eq!(
attempts[0].upstream_key_digest, attempts[1].upstream_key_digest,
"the same stable key digest is recorded on both attempts"
);
assert!(attempts[0].upstream_key_digest.is_some());
let record = tasks
.status(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("status");
assert_eq!(record.pins.retry.max_attempts, 2);
}
@@ -215,13 +215,15 @@ async fn adapter_attempt_span_carries_durable_attempt_id() {
runtime,
crypto,
));
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-attempt-trace".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = std::time::Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = std::time::Duration::from_secs(5);
let config = TaskWorkerConfig {
owner: "runtime-worker-attempt-trace".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: std::time::Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: std::time::Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
@@ -263,6 +265,24 @@ async fn adapter_attempt_span_carries_durable_attempt_id() {
assert_eq!(state.as_deref(), Some("completed"));
// Story 4.4: every outbound send is its own durable attempt, so the two retries of one lease carry
// two distinct attempt identities, both recorded in the immutable attempt history.
let attempt_rows: Vec<(String, i32)> = sqlx::query_as(
"select attempt_id, ordinal from task_attempts where task_id = $1 order by ordinal",
)
.bind(acceptance.task_id.as_uuid())
.fetch_all(registry.pool())
.await
.unwrap();
assert_eq!(attempt_rows.len(), 2, "two durable attempts were committed");
assert_eq!(attempt_rows[0].1, 1);
assert_eq!(attempt_rows[1].1, 2);
assert_eq!(
Some(attempt_rows[1].0.as_str()),
durable_attempt_id.as_deref(),
"the Task points at its latest durable attempt"
);
let spans = capture.snapshot();
let attempt_spans: Vec<_> = spans
.iter()
@@ -276,17 +296,16 @@ async fn adapter_attempt_span_carries_durable_attempt_id() {
!aid.contains("traceparent"),
"attempt_id must not contain traceparent"
);
assert_eq!(
Some(aid.as_str()),
durable_attempt_id.as_deref(),
"span attempt_id must equal durable attempt id"
assert!(
attempt_rows.iter().any(|(id, _)| id == &aid),
"every attempt span names an attempt that was durably committed"
);
}
let ids: Vec<String> = attempt_spans
.iter()
.map(|s| s.fields.get("attempt_id").cloned().unwrap_or_default())
.collect();
assert_eq!(ids[0], ids[1], "same durable id on both retries");
assert_ne!(ids[0], ids[1], "each retry has its own durable attempt id");
assert_eq!(
spans
.iter()
@@ -0,0 +1,846 @@
//! ATDD Story 4.4 (Beam): cancel through TaskApplication -> PG -> worker -> upstream.
use super::common;
use super::unsampled_correlation;
use axum::Router;
use axum::http::StatusCode;
use axum::routing::post;
use crank_community_mcp::{TaskApplication, TaskWorkerHandler};
use crank_core::{
AgentOperationBinding, NoTasksFallback, PlatformApiKeyId, PlatformApiKeyScope, TaskState,
TaskStore, ToolExecutionMode, ToolExecutionPolicy, VerifiedMachineCredential,
};
use crank_registry::{AdmissionLimits, PostgresRegistry, PostgresTaskStore, PublishRequest};
use crank_runtime::{
InMemoryCoordinationStateStore, SecretCrypto, TaskService, TaskWorker, TaskWorkerConfig,
};
use serde_json::json;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use time::OffsetDateTime;
const AGENT_SLUG: &str = "reports";
const TOOL_NAME: &str = "report_gen";
const TEST_MASTER_KEY: &str = "test-master-key-00000000000000000000000000000000";
async fn state_of(registry: &PostgresRegistry, task_id: crank_core::TaskId) -> Option<String> {
sqlx::query_scalar("select state from tasks where task_id = $1")
.bind(task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap()
}
async fn seed_tool(registry: &PostgresRegistry, upstream: &str, policy: ToolExecutionPolicy) {
let mut operation = common::test_operation(upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(policy);
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
}
async fn principal(registry: &PostgresRegistry, name: &str) -> crank_community_mcp::TaskPrincipal {
let token = common::create_platform_api_key(
registry,
AGENT_SLUG,
name,
&[PlatformApiKeyScope::Read, PlatformApiKeyScope::Write],
)
.await;
let secret_hash = common::hash_access_secret(&token);
let api_key = registry
.get_platform_api_key_by_secret_for_agent_slug(
common::test_workspace_slug(),
AGENT_SLUG,
&secret_hash,
)
.await
.unwrap()
.expect("active key");
registry
.touch_platform_api_key(
&api_key.api_key.workspace_id,
&api_key.api_key.id,
&OffsetDateTime::now_utc(),
)
.await
.unwrap();
crank_community_mcp::TaskPrincipal::Agent {
workspace_slug: common::test_workspace_slug().to_owned(),
agent_slug: AGENT_SLUG.to_owned(),
credential: VerifiedMachineCredential {
machine_access_mode: crank_core::MachineAccessMode::StaticAgentKey,
max_security_level: crank_core::OperationSecurityLevel::Standard,
scopes: api_key.api_key.scopes.clone(),
platform_api_key_id: Some(api_key.api_key.id.clone()),
},
}
}
async fn application(registry: &PostgresRegistry) -> TaskApplication {
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
let observed_at = OffsetDateTime::now_utc();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &observed_at,
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
TaskApplication::new(
registry.clone(),
Duration::ZERO,
Arc::new(InMemoryCoordinationStateStore::default()),
TaskService::with_secret_crypto(store, Arc::new(crypto)),
)
}
#[tokio::test]
async fn cancel_during_active_request_is_observed_and_leaves_one_unknown_outcome() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
let signal = Arc::new(std::sync::Mutex::new(Some(tx)));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
move || {
let calls = Arc::clone(&calls);
let signal = Arc::clone(&signal);
async move {
calls.fetch_add(1, Ordering::SeqCst);
if let Some(tx) = signal.lock().unwrap().take() {
let _ = tx.send(());
}
tokio::time::sleep(Duration::from_millis(1500)).await;
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s1").await;
let foreign = principal(&registry, "cancel-s1-foreign").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s1-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.build();
let handler = Arc::new(TaskWorkerHandler::from_application(
&app,
registry.clone(),
runtime,
crypto.clone(),
));
let worker = TaskWorker::new(
store,
handler,
TaskWorkerConfig {
owner: "worker-s1".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
renewal_interval: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(5),
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
let _ = tokio::time::timeout(Duration::from_secs(5), rx).await;
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, TaskState::OutcomeUnknown);
}
other => panic!("expected OutcomeUnknown got {other:?}"),
}
let foreign_cancel = app
.cancel(&foreign, TOOL_NAME, acceptance.task_id, None)
.await;
assert!(matches!(
foreign_cancel,
Err(crank_community_mcp::TaskApplicationError::NotFound)
));
tokio::time::sleep(Duration::from_millis(1600)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1);
let cancel2 = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel2");
assert!(matches!(
cancel2,
crank_core::TaskCancelOutcome::OutcomeUnknown(_)
));
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
}
#[tokio::test]
async fn cancel_before_any_dispatch_is_cancelled_without_upstream_call() {
let registry = common::test_registry().await;
let calls = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let calls = Arc::clone(&calls);
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
(StatusCode::OK, axum::Json(json!({"id":"lead"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let policy = ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
};
seed_tool(&registry, &upstream, policy).await;
let caller = principal(&registry, "cancel-s3").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s3-key",
&json!({"email":"u"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::Cancelled(record) => {
assert_eq!(record.state, TaskState::Cancelled)
}
other => panic!("expected Cancelled got {other:?}"),
}
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("cancelled")
);
let attempts = app
.attempts(&caller, TOOL_NAME, acceptance.task_id)
.await
.expect("attempts");
assert!(attempts.is_empty());
assert_eq!(calls.load(Ordering::SeqCst), 0);
server.abort();
}
#[tokio::test]
async fn cancel_during_pinned_backoff_never_redispatches_and_is_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s2").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s2-key",
&json!({"email":"u","idempotency_key":"k-2"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
// Reproduce deterministically the durable state the executor commits after an ambiguous attempt
// whose bounded retry is allowed: one attempt resolved with certainty=unknown, the retry armed and
// `current_attempt_id` cleared. The real retry window is a pinned millisecond backoff, so racing a
// running executor against the cancel would not be a repeatable oracle.
let lease = store
.claim_batch("worker-s2-prep", 1, 10_000)
.await
.expect("claim")
.pop()
.expect("one claimed lease");
let attempt = store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: acceptance.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("durable checkpoint");
let resolution = store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Uncertain {
error: crank_core::TaskEnvelope::try_redacted(
&json!({"code": "upstream_ack_lost", "phase": "dispatched"}),
)
.expect("bounded attempt error"),
repeatable: true,
phase: crank_core::AttemptPhase::Dispatched,
},
)
.await
.expect("attempt resolution");
assert!(
matches!(resolution, crank_core::AttemptResolution::RetryAt { .. }),
"an ambiguous but safely repeatable attempt arms the pinned retry"
);
let cancel = app
.cancel(&caller, TOOL_NAME, acceptance.task_id, None)
.await
.expect("cancel");
match cancel {
crank_core::TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, crank_core::TaskState::OutcomeUnknown)
}
other => panic!("cancel after an ambiguous attempt must stay uncertain, got {other:?}"),
}
// A real worker must never redispatch or terminate the Task afterwards: the uncertain state is not
// claimable and no second terminal write is possible.
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s2".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
tokio::time::sleep(Duration::from_secs(1)).await;
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
requests.load(Ordering::SeqCst),
0,
"no outbound send after the cancel"
);
assert_eq!(
state_of(&registry, acceptance.task_id).await.as_deref(),
Some("outcome_unknown")
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let code: Option<String> =
sqlx::query_scalar("select error_envelope ->> 'code' from tasks where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(code.as_deref(), Some("cancelled_after_dispatch"));
}
#[tokio::test]
async fn refused_attempt_after_lost_ack_keeps_outcome_unknown() {
use crank_community_mcp::VerifiedIdempotentWrite;
use crank_core::{IdempotencyMode, IdempotencyPolicy, RetryPolicy};
let registry = common::test_registry().await;
let requests = Arc::new(AtomicUsize::new(0));
let upstream_app = Router::new().route(
"/crm/leads",
post({
let requests = Arc::clone(&requests);
move |headers: axum::http::HeaderMap| {
let requests = Arc::clone(&requests);
async move {
let _ = headers.get("idempotency-key");
let attempt = requests.fetch_add(1, Ordering::SeqCst);
if attempt == 0 {
// Applied upstream, acknowledgement lost after the caller had time to observe that
// the request really left the worker.
tokio::time::sleep(Duration::from_millis(600)).await;
return (
StatusCode::INTERNAL_SERVER_ERROR,
axum::Json(json!({"error":"acknowledgement lost"})),
);
}
(StatusCode::OK, axum::Json(json!({"id":"lead-1"})))
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move { axum::serve(listener, upstream_app).await.unwrap() });
let upstream = format!("http://{address}");
let mut operation = common::test_operation(&upstream, TOOL_NAME);
operation.execution_config.execution_policy = Some(ToolExecutionPolicy {
mode: ToolExecutionMode::Background,
deadline_ms: 30_000,
no_tasks_fallback: NoTasksFallback::Unsupported,
});
operation.execution_config.retry_policy = Some(RetryPolicy { max_attempts: 2 });
operation.execution_config.idempotency = Some(IdempotencyPolicy {
mode: IdempotencyMode::Required,
ttl_ms: 60_000,
input_field: Some("idempotency_key".to_owned()),
header_name: Some("Idempotency-Key".to_owned()),
});
registry
.create_operation(&common::test_workspace_id(), &operation, Some("alice"))
.await
.unwrap();
registry
.publish_operation(PublishRequest {
workspace_id: &common::test_workspace_id(),
operation_id: &operation.id,
version: 1,
published_at: &OffsetDateTime::parse(
"2026-03-26T10:00:00Z",
&time::format_description::well_known::Rfc3339,
)
.unwrap(),
published_by: Some("alice"),
})
.await
.unwrap();
common::publish_agent_with_bindings(
&registry,
AGENT_SLUG,
vec![AgentOperationBinding {
operation_id: operation.id.clone(),
operation_version: 1,
tool_name: operation.name.clone(),
tool_title: operation.tool_description.title.clone(),
tool_description_override: None,
enabled: true,
}],
)
.await;
let caller = principal(&registry, "cancel-s4").await;
let app = application(&registry).await;
let acceptance = app
.accept_background(
&caller,
TOOL_NAME,
None,
"s4-key",
&json!({"email":"u","idempotency_key":"k-4"}),
&unsampled_correlation(),
)
.await
.expect("accept");
let crypto = SecretCrypto::new(TEST_MASTER_KEY).unwrap();
registry
.verify_or_register_master_key_identity(crank_registry::MasterKeyIdentityCandidate {
epoch: crypto.master_key_epoch(),
fingerprint: crypto.master_key_fingerprint(),
cipher_contract: crank_registry::MASTER_KEY_CIPHER_CONTRACT,
observed_at: &OffsetDateTime::now_utc(),
})
.await
.expect("master key");
let store = Arc::new(
PostgresTaskStore::new(
registry.pool().clone(),
AdmissionLimits {
global_limit: 4,
agent_limit: 4,
tool_limit: 4,
queue_limit: 8,
},
Default::default(),
)
.expect("store"),
);
let runtime = crank_runtime::community_with_outbound_policy(
crank_runtime::OutboundHttpPolicy::allowing_hosts(["127.0.0.1"]),
)
.with_coordination_store(Arc::new(InMemoryCoordinationStateStore::default()))
.build();
let handler = Arc::new(
TaskWorkerHandler::from_application(&app, registry.clone(), runtime, crypto.clone())
.with_verified_writes(vec![VerifiedIdempotentWrite {
operation_id: operation.id.as_str().to_owned(),
operation_version: 1,
key_header: "Idempotency-Key".to_owned(),
validity_ms: 60_000,
}]),
);
let worker = TaskWorker::new(
store.clone(),
handler,
TaskWorkerConfig {
owner: "worker-s4".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
},
)
.expect("worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
// Revoke the key the Task was accepted with while the first outbound write is provably in flight:
// that attempt still resolves as an uncertain write, and the next attempt is refused by the
// authority before its checkpoint.
for _ in 0..200 {
if requests.load(Ordering::SeqCst) >= 1 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the first attempt is dispatched before the refusal"
);
registry
.revoke_platform_api_key_for_agent(
&common::test_workspace_id(),
&common::test_agent_id(AGENT_SLUG),
&PlatformApiKeyId::new("pk_cancel-s4".to_owned()),
&OffsetDateTime::now_utc(),
)
.await
.expect("revoke");
let mut state = None;
for _ in 0..200 {
tokio::time::sleep(Duration::from_millis(50)).await;
state = state_of(&registry, acceptance.task_id).await;
if state.as_deref() == Some("outcome_unknown") {
break;
}
}
stop_flag.store(true, Ordering::SeqCst);
let _ = tokio::time::timeout(Duration::from_secs(10), worker_task).await;
server.abort();
assert_eq!(
state.as_deref(),
Some("outcome_unknown"),
"a refused attempt after a lost acknowledgement must stay uncertain"
);
assert_eq!(
requests.load(Ordering::SeqCst),
1,
"the refused attempt is never dispatched"
);
let attempts_count: i64 =
sqlx::query_scalar("select count(*)::bigint from task_attempts where task_id = $1")
.bind(acceptance.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(attempts_count, 1, "no second durable attempt");
let certainty: Option<String> = sqlx::query_scalar(
"select certainty from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(certainty.as_deref(), Some("unknown"));
let phase: Option<String> = sqlx::query_scalar(
"select phase from task_attempts where task_id = $1 order by ordinal desc limit 1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
assert_eq!(phase.as_deref(), Some("dispatched"));
let error: Option<(Option<String>, Option<String>)> = sqlx::query_as(
"select error_envelope ->> 'code', error_envelope ->> 'phase' from tasks where task_id = $1",
)
.bind(acceptance.task_id.as_uuid())
.fetch_optional(registry.pool())
.await
.unwrap();
let (code, error_phase) = error.expect("the refused Task carries its bounded error");
assert_eq!(code.as_deref(), Some("task_authority_rejected"));
assert_eq!(
error_phase.as_deref(),
Some("dispatched"),
"the error keeps the phase of the attempt that may have reached the upstream"
);
}
@@ -605,6 +605,53 @@ async fn a_missing_task_store_fails_closed_and_readiness_reports_not_ready() {
server.abort();
}
/// Commits and resolves exactly one durable attempt of the current lease through the real store.
///
/// Story 4.4 moved the dispatch checkpoint and the attempt resolution into the runtime seam, so a
/// worker-level test double persists exactly one attempt instead of inventing a second retry loop.
pub(crate) async fn commit_attempt(
context: &crank_runtime::TaskAttemptContext,
outcome: crank_core::AttemptOutcome,
) -> crank_runtime::TaskAttemptOutcome {
use crank_core::{AttemptResolution, BeginAttempt, TaskState};
let lease = context.lease();
let begin = BeginAttempt {
task_id: lease.record.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: format!("attempt-{}", uuid::Uuid::now_v7()),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: lease.record.pins.retry.max_attempts > 1,
};
match context.store().checkpoint_dispatch(begin).await {
Ok(attempt) => match context.store().resolve_attempt(&attempt, outcome).await {
Ok(AttemptResolution::Finished(task)) if task.state == TaskState::Completed => {
crank_runtime::TaskAttemptOutcome::Completed
}
Ok(AttemptResolution::Finished(_)) => crank_runtime::TaskAttemptOutcome::Failed {
code: "test_failure".to_owned(),
},
Ok(AttemptResolution::Unknown(_)) => crank_runtime::TaskAttemptOutcome::OutcomeUnknown,
Ok(AttemptResolution::RetryAt { .. }) | Err(_) => {
crank_runtime::TaskAttemptOutcome::Stopped
}
},
Err(_) => crank_runtime::TaskAttemptOutcome::Stopped,
}
}
/// Bounded success outcome used by the worker-level test doubles.
pub(crate) fn completed_outcome(value: serde_json::Value) -> crank_core::AttemptOutcome {
crank_core::AttemptOutcome::Completed(
crank_core::TaskEnvelope::try_redacted(&value).expect("test result is bounded"),
)
}
mod attempt_guards;
mod attempt_trace;
mod cancel_chain;
mod shutdown_window;
mod task_worker;
@@ -15,8 +15,7 @@ use super::*;
async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -67,6 +66,7 @@ async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: scoped_key.repeat(64),
input_fingerprint: scoped_key.repeat(64),
@@ -101,15 +101,44 @@ async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream
let _ = self.release.acquire().await;
Ok(())
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
// The trigger delays the durable checkpoint; a stop that lands while it is in flight must
// finalize failed(shutdown_before_dispatch) without an outbound send.
let begin = crank_core::BeginAttempt {
task_id: context.lease().record.task_id,
owner: context.lease().owner.clone(),
fence: context.lease().fence,
expected_version: context.lease().record.version,
attempt_id: "attempt-stop-window".to_owned(),
input_fingerprint: context.lease().record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
};
let Ok(attempt) = context.store().checkpoint_dispatch(begin).await else {
return TaskAttemptOutcome::Stopped;
};
if context.is_stopping() {
let _ = context
.store()
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Failed {
error: crank_core::TaskEnvelope::try_redacted(&json!({
"code": "shutdown_before_dispatch",
"phase": "before_dispatch",
}))
.expect("bounded error envelope"),
retryable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await;
return TaskAttemptOutcome::Failed {
code: "shutdown_before_dispatch".to_owned(),
};
}
self.executed.fetch_add(1, Ordering::SeqCst);
TaskAttemptOutcome::Completed(json!({ "ok": true }))
commit_attempt(&context, completed_outcome(json!({ "ok": true }))).await
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(GatedHandler {
@@ -117,14 +146,15 @@ async fn task_worker_stop_between_checkpoint_and_dispatch_never_reaches_upstream
release: revalidate_release.clone(),
executed: executed.clone(),
});
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-stop-window".to_owned();
config.batch_limit = 1;
config.lease_ms = 30_000;
config.poll_interval = Duration::from_millis(50);
config.renewal_interval = Duration::from_secs(20);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(20);
let config = TaskWorkerConfig {
owner: "runtime-worker-stop-window".to_owned(),
batch_limit: 1,
lease_ms: 30_000,
poll_interval: Duration::from_millis(50),
renewal_interval: Duration::from_secs(20),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(20),
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
@@ -38,6 +38,7 @@ async fn task_worker_store_lease_recovery_contract() {
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
@@ -85,29 +86,37 @@ async fn task_worker_store_lease_recovery_contract() {
// A stale fencing token can never checkpoint dispatch, even under the issuing owner.
assert!(
store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence + 1,
"attempt-stale".to_owned()
)
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence + 1,
expected_version: lease.record.version,
attempt_id: "attempt-stale".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.is_err()
);
// The valid owner still commits the dispatch checkpoint before any outbound work.
assert!(
store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence,
"attempt-1".to_owned(),
)
.await
.unwrap()
.current_attempt_id
.as_deref()
== Some("attempt-1")
let committed = store
.checkpoint_dispatch(crank_core::BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-1".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.unwrap();
assert_eq!(committed.attempt.attempt_id, "attempt-1");
assert_eq!(
committed.task.current_attempt_id.as_deref(),
Some("attempt-1")
);
}
@@ -116,8 +125,7 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() {
use async_trait::async_trait;
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -154,6 +162,7 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() {
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "r".repeat(64),
input_fingerprint: "b".repeat(64),
@@ -180,15 +189,9 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
}
}
let handler: Arc<dyn TaskHandler> = Arc::new(RuntimeHandler {
@@ -196,14 +199,15 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() {
});
// PG-only authoritative dispatch: no Valkey or upstream process is involved.
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-1".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(100);
config.renewal_interval = Duration::from_millis(1_000);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let config = TaskWorkerConfig {
owner: "runtime-worker-1".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(100),
renewal_interval: Duration::from_millis(1_000),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
@@ -238,10 +242,12 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() {
assert_eq!(record.state, TaskState::Completed);
// Bounded shutdown with no fresh work returns promptly.
let mut idle_config = TaskWorkerConfig::default();
idle_config.owner = "runtime-worker-idle".to_owned();
idle_config.poll_interval = Duration::from_millis(50);
idle_config.shutdown_timeout = Duration::from_secs(2);
let idle_config = TaskWorkerConfig {
owner: "runtime-worker-idle".to_owned(),
poll_interval: Duration::from_millis(50),
shutdown_timeout: Duration::from_secs(2),
..TaskWorkerConfig::default()
};
let idle_worker = TaskWorker::new(
store.clone(),
handler_for_no_work(store.clone()),
@@ -259,8 +265,7 @@ async fn task_worker_runtime_dispatch_and_recovery_contract() {
async fn task_worker_revalidation_denial_never_dispatches() {
use crank_core::{TaskAccess, TaskError, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -293,6 +298,7 @@ async fn task_worker_revalidation_denial_never_dispatches() {
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "d".repeat(64),
input_fingerprint: "b".repeat(64),
@@ -320,28 +326,24 @@ async fn task_worker_revalidation_denial_never_dispatches() {
"publication or credential revoked",
)))
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.observed.fetch_add(1, Ordering::SeqCst);
TaskAttemptOutcome::Completed(json!({"ok": true}))
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
}
}
let observed = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(DeniedHandler {
observed: observed.clone(),
});
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-denied".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let config = TaskWorkerConfig {
owner: "runtime-worker-denied".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
@@ -387,12 +389,9 @@ fn handler_for_no_work(
}
async fn execute(
&self,
_lease: &crank_core::TaskLease,
_attempt_id: &str,
_cancelled: std::sync::Arc<std::sync::atomic::AtomicBool>,
_permit: crank_runtime::TaskDispatchPermit,
context: crank_runtime::TaskAttemptContext,
) -> crank_runtime::TaskAttemptOutcome {
crank_runtime::TaskAttemptOutcome::Completed(serde_json::json!({"ok": true}))
commit_attempt(&context, completed_outcome(serde_json::json!({"ok": true}))).await
}
}
std::sync::Arc::new(NoWorkHandler)
@@ -522,13 +521,15 @@ async fn task_worker_dispatches_one_real_upstream_attempt_with_parent_trace() {
runtime,
crypto,
));
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-e2e".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let config = TaskWorkerConfig {
owner: "runtime-worker-e2e".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
@@ -663,13 +664,15 @@ async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
runtime,
crypto,
));
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-uncertain".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(5);
let config = TaskWorkerConfig {
owner: "runtime-worker-uncertain".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(5),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
@@ -709,8 +712,7 @@ async fn task_worker_ambiguous_upstream_error_becomes_outcome_unknown() {
async fn task_worker_graceful_stop_drains_active_attempt() {
use crank_core::{TaskAccess, TaskLease, TaskOwnerKind};
use crank_runtime::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
TaskAttemptOutcome, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
@@ -743,6 +745,7 @@ async fn task_worker_graceful_stop_drains_active_attempt() {
execution_deadline_unix_ms: 1_900_000_000_000,
retain_until_unix_ms: 1_900_000_000_000 + 60_000,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: "s".repeat(64),
input_fingerprint: "d".repeat(64),
@@ -768,29 +771,25 @@ async fn task_worker_graceful_stop_drains_active_attempt() {
async fn revalidate(&self, _lease: &TaskLease) -> Result<(), TaskWorkerError> {
Ok(())
}
async fn execute(
&self,
_lease: &TaskLease,
_attempt_id: &str,
_cancelled: Arc<std::sync::atomic::AtomicBool>,
_permit: TaskDispatchPermit,
) -> TaskAttemptOutcome {
async fn execute(&self, context: crank_runtime::TaskAttemptContext) -> TaskAttemptOutcome {
self.started.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(Duration::from_millis(500)).await;
TaskAttemptOutcome::Completed(json!({"ok": true}))
commit_attempt(&context, completed_outcome(json!({"ok": true}))).await
}
}
let started = Arc::new(AtomicUsize::new(0));
let handler: Arc<dyn TaskHandler> = Arc::new(SlowHandler {
started: started.clone(),
});
let mut config = TaskWorkerConfig::default();
config.owner = "runtime-worker-drain".to_owned();
config.batch_limit = 1;
config.lease_ms = 10_000;
config.poll_interval = Duration::from_millis(50);
config.max_inflight = 1;
config.shutdown_timeout = Duration::from_secs(20);
let config = TaskWorkerConfig {
owner: "runtime-worker-drain".to_owned(),
batch_limit: 1,
lease_ms: 10_000,
poll_interval: Duration::from_millis(50),
max_inflight: 1,
shutdown_timeout: Duration::from_secs(20),
..TaskWorkerConfig::default()
};
let worker = TaskWorker::new(store.clone(), handler, config).expect("valid worker");
let stop_flag = worker.stop_flag();
let worker_task = tokio::spawn(async move { worker.run().await });
+1 -1
View File
@@ -58,7 +58,7 @@ use self::approvals::{
};
pub(super) use self::auth_resolution::resolve_operation_auth;
use self::endpoint::{mcp_delete, mcp_get, mcp_post};
pub use self::task_worker::TaskWorkerHandler;
pub use self::task_worker::{TaskWorkerHandler, VerifiedIdempotentWrite};
pub use self::tasks::{TaskApplication, TaskApplicationError, TaskPrincipal};
// Test-only import surface: the BUILD-internal `app/tests.rs` module reaches these two
// through `super`, and the plain lib build does not use them (same pattern as
+151 -103
View File
@@ -1,6 +1,10 @@
//! Application-level durable Task handler for Story 4.3.
//! Application-level durable Task handler and attempt authority (Story 4.3/4.4).
//!
//! The handler owns everything that touches product authority: the exact published pin, the typed
//! Platform API key, the approval policy, the encrypted executable input and the resolved upstream
//! auth. The runtime owns the single bounded retry loop and the durable attempt checkpoint/resolution;
//! this module only revalidates and prepares one attempt at a time.
use std::sync::Arc;
use std::time::{Duration, Instant};
use async_trait::async_trait;
@@ -12,9 +16,11 @@ use crank_core::{
use crank_registry::PostgresRegistry;
use crank_registry::PublishedAgentTool;
use crank_runtime::{
ExecutionAuthorization, ExecutionRequestError, RuntimeExecutionRequest, RuntimeExecutor,
SecretCrypto, TaskDispatchPermit, TaskHandler, TaskService, TaskWorkerError,
ExecutionAuthorization, RuntimeExecutor, RuntimeOperation, SecretCrypto, TaskAttemptAuthority,
TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan, TaskAttemptRefusal, TaskHandler,
TaskService, TaskWorkerError,
};
use sha2::{Digest, Sha256};
use super::TaskApplication;
use super::auth_resolution::resolve_operation_auth_with;
@@ -22,12 +28,31 @@ use super::tool_resolution::runtime_operation;
use crate::access::{allows_scope, credential_allows_security_level};
use crate::catalog::PublishedToolCatalog;
/// Server-owned evidence that one published operation deduplicates repeated sends of the same stable
/// idempotency key.
///
/// The descriptor is constructed by the application for an integration whose upstream contract was
/// actually qualified (key delivery, dedup scope, canonical input binding and a dedup window that
/// covers the pinned retry budget). It is never deserialized from Task input, so a caller key or
/// header alone can never make a write retryable. Production wiring passes an empty set until such a
/// contract exists.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct VerifiedIdempotentWrite {
pub operation_id: String,
pub operation_version: u32,
/// Header through which the stable key is delivered to this exact upstream.
pub key_header: String,
/// Dedup window of the upstream contract, which must cover the pinned retry budget.
pub validity_ms: i64,
}
pub struct TaskWorkerHandler {
registry: PostgresRegistry,
catalog: PublishedToolCatalog,
runtime: RuntimeExecutor,
secret_crypto: SecretCrypto,
service: TaskService,
verified_writes: Vec<VerifiedIdempotentWrite>,
}
impl TaskWorkerHandler {
@@ -46,9 +71,16 @@ impl TaskWorkerHandler {
runtime,
secret_crypto,
service: tasks.task_service(),
verified_writes: Vec::new(),
}
}
/// Registers server-owned verified-idempotency evidence of qualified integrations.
pub fn with_verified_writes(mut self, verified_writes: Vec<VerifiedIdempotentWrite>) -> Self {
self.verified_writes = verified_writes;
self
}
/// Authoritative pre-dispatch authorization of one leased Task (Story 4.3).
///
/// Resolves the exact pinned Published Version in the current publication, repeats the approval
@@ -143,6 +175,74 @@ impl TaskWorkerHandler {
}
Ok(tool)
}
/// Safe-retry classification of one attempt and the stable upstream key digest.
///
/// A read is always safe to repeat. A write is only safe to repeat when this process holds
/// server-owned evidence for the exact published operation and that evidence's dedup window still
/// covers the accepted retry budget; the presence of a client key or an idempotency header alone
/// never qualifies.
fn retry_classification(
&self,
operation: &RuntimeOperation,
input: &serde_json::Value,
retry_window_ms: i64,
) -> (bool, Option<String>) {
if matches!(
&operation.target,
crank_core::Target::Rest(target) if target.method == crank_core::HttpMethod::Get
) {
return (true, None);
}
let Some(policy) = operation.execution_config.idempotency.as_ref() else {
return (false, None);
};
let Some(field) = policy
.input_field
.as_deref()
.filter(|field| !field.trim().is_empty())
else {
return (false, None);
};
let Some(header) = policy
.header_name
.as_deref()
.filter(|value| !value.is_empty())
else {
return (false, None);
};
let key = match input.get(field) {
Some(serde_json::Value::String(value)) if !value.trim().is_empty() => value.clone(),
Some(serde_json::Value::Number(value)) => value.to_string(),
_ => return (false, None),
};
let evidence = self.verified_writes.iter().find(|evidence| {
evidence.operation_id == operation.operation_id.as_str()
&& evidence.operation_version == operation.operation_version
&& evidence.key_header.eq_ignore_ascii_case(header)
&& evidence.validity_ms >= retry_window_ms
});
let Some(evidence) = evidence else {
return (false, None);
};
// Only a digest of the stable key is persisted; the key itself stays in the encrypted input.
let mut digest = Sha256::new();
digest.update(evidence.key_header.as_bytes());
digest.update([0u8]);
digest.update(key.as_bytes());
(true, Some(format!("{:x}", digest.finalize())))
}
/// Maps a preparation failure onto the attempt refusal vocabulary: a transient authority-store
/// outage leaves the Task leased for recovery, everything else is a proven pre-dispatch refusal.
fn refusal(error: TaskError) -> TaskAttemptRefusal {
match error {
TaskError::Unavailable { .. } => TaskAttemptRefusal::Stopped,
_ => TaskAttemptRefusal::Failed {
code: "task_authority_rejected".to_owned(),
},
}
}
}
#[async_trait]
@@ -154,74 +254,54 @@ impl TaskHandler for TaskWorkerHandler {
Ok(())
}
async fn execute(
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome {
self.runtime.execute_task_outcome(&context, self).await
}
}
#[async_trait]
impl TaskAttemptAuthority for TaskWorkerHandler {
async fn prepare_attempt(
&self,
lease: &TaskLease,
_attempt_id: &str,
cancelled: Arc<std::sync::atomic::AtomicBool>,
permit: TaskDispatchPermit,
) -> crank_runtime::TaskAttemptOutcome {
use crank_runtime::TaskAttemptOutcome;
use std::sync::atomic::Ordering;
// The lease may have been lost between the checkpoint and the outbound call; never dispatch
// on behalf of a worker that no longer owns the Task.
if cancelled.load(Ordering::Acquire) {
return TaskAttemptOutcome::OutcomeUnknown;
}
let tool = match self.authorize(lease).await {
Ok(tool) => tool,
Err(error) => {
return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()}));
}
};
let input = match self.service.executable_input(lease).await {
Ok(input) => input,
Err(error) => {
return TaskAttemptOutcome::Failed(serde_json::json!({"code": error.to_string()}));
}
};
) -> Result<TaskAttemptPlan, TaskAttemptRefusal> {
let tool = self.authorize(lease).await.map_err(Self::refusal)?;
let input = self
.service
.executable_input(lease)
.await
.map_err(Self::refusal)?;
let operation = runtime_operation(&tool);
let now_ms = current_unix_ms();
let deadline_ms = lease.record.pins.execution_deadline_unix_ms;
if deadline_ms <= now_ms {
return TaskAttemptOutcome::Failed(
serde_json::json!({"code": "task deadline elapsed"}),
);
// The accepted absolute deadline already elapsed: no further attempt may be dispatched,
// and any previous uncertainty was already recorded as such.
return Err(TaskAttemptRefusal::Failed {
code: "task_deadline_elapsed".to_owned(),
});
}
let deadline = Instant::now()
+ Duration::from_millis((deadline_ms - now_ms).min(i64::MAX as i64) as u64);
let deadline = Instant::now() + Duration::from_millis((deadline_ms - now_ms) as u64);
let workspace_id = WorkspaceId::new(lease.record.workspace_id.as_str());
let agent_id = AgentId::new(lease.record.owner_id.as_str());
let resolved_auth = match resolve_operation_auth_with(
let resolved_auth = resolve_operation_auth_with(
&self.registry,
&self.secret_crypto,
&workspace_id,
&operation.execution_config,
)
.await
{
Ok(auth) => auth,
Err(error) => {
return TaskAttemptOutcome::Failed(
serde_json::json!({ "code": error.to_string() }),
);
}
};
// Continue the parent trace persisted with the accepted Task; the worker's attempt span is a
// child of it, and the executor records the attempt identity. Trace data stays out of labels.
let trace_context = match TraceContext::parse(&lease.record.support.traceparent) {
Ok(context) => context,
Err(_) => {
return TaskAttemptOutcome::Failed(
serde_json::json!({ "code": "task trace parent is invalid" }),
);
}
};
.map_err(|_| TaskAttemptRefusal::Failed {
code: "task_auth_resolution_failed".to_owned(),
})?;
// Continue the parent trace persisted with the accepted Task; every attempt adds its own
// child `adapter_attempt` span carrying the committed durable attempt ID.
let trace_context =
TraceContext::parse(&lease.record.support.traceparent).map_err(|_| {
TaskAttemptRefusal::Failed {
code: "task_trace_parent_invalid".to_owned(),
}
})?;
let request_context = crank_runtime::RuntimeRequestContext::new(
lease.record.support.request_id.clone(),
trace_context,
@@ -231,52 +311,20 @@ impl TaskHandler for TaskWorkerHandler {
Some(agent_id.clone()),
crank_core::InvocationSource::AgentToolCall,
);
let request = match RuntimeExecutionRequest::try_new(
&workspace_id,
ExecutionOrigin::AgentSnapshot,
Some(&agent_id),
&operation,
&input,
ExecutionAuthorization::Authorized,
resolved_auth.as_ref(),
&request_context,
let (retry_allowed, upstream_key_digest) =
self.retry_classification(&operation, &input, deadline_ms - now_ms);
Ok(TaskAttemptPlan {
workspace_id,
origin: ExecutionOrigin::AgentSnapshot,
agent_id: Some(agent_id),
operation,
input,
authorization: ExecutionAuthorization::Authorized,
resolved_auth,
request_context,
deadline,
) {
Ok(request) => request,
Err(error) => {
return TaskAttemptOutcome::Failed(match error {
ExecutionRequestError::InvalidOrigin => {
serde_json::json!({"code": "invalid execution origin"})
}
ExecutionRequestError::InvalidOperationVersion => {
serde_json::json!({"code": "invalid operation version"})
}
ExecutionRequestError::DeadlineElapsed => {
serde_json::json!({"code": "task deadline elapsed"})
}
ExecutionRequestError::InvalidMeteringScope => {
serde_json::json!({"code": "invalid metering scope"})
}
});
}
};
if cancelled.load(Ordering::Acquire) {
return TaskAttemptOutcome::OutcomeUnknown;
}
match self.runtime.execute_task_outcome(request, permit).await {
Ok(success) => TaskAttemptOutcome::Completed(success.output),
Err(failure) => {
// An ambiguous outbound failure is recorded as uncertainty, never as a retryable
// failure: the durable checkpoint already exists, so Story 4.4 may reconcile it.
if failure.outcome_certainty() == crank_core::OutcomeCertainty::OutcomeUnknown {
return TaskAttemptOutcome::OutcomeUnknown;
}
TaskAttemptOutcome::Failed(
serde_json::json!({ "code": format!("{:?}", failure.error_code()) }),
)
}
}
retry_allowed,
upstream_key_digest,
})
}
}
@@ -213,6 +213,7 @@ impl TaskApplication {
deadline: Instant::now() + Duration::from_millis(policy.deadline_ms.max(1)),
retention_ms: TaskService::default_retention_ms(),
approval_reference: None,
retry: task_retry_policy(&tool),
}
};
@@ -274,6 +275,45 @@ impl TaskApplication {
}
}
/// Owner-scoped cancellation of one Task of the addressed published Tool (Story 4.4).
///
/// Cancellation is decided by the durable store under a row lock: before the dispatch checkpoint
/// it is an honest cancellation with zero outbound calls, after it the Task becomes the
/// non-terminal uncertain outcome instead of a plain cancel or a retry.
pub async fn cancel(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
expected_version: Option<u32>,
) -> Result<crank_core::TaskCancelOutcome, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self
.service
.cancel(&access, task_id, expected_version, "user_cancelled")
.await
{
Ok(outcome) => Ok(outcome),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
/// Owner-scoped attempt history of one Task of the addressed published Tool.
pub async fn attempts(
&self,
principal: &TaskPrincipal,
tool_name: &str,
task_id: TaskId,
) -> Result<Vec<crank_core::TaskAttempt>, TaskApplicationError> {
let access = self.task_access(principal, tool_name).await?;
match self.service.attempts(&access, task_id).await {
Ok(attempts) => Ok(attempts),
Err(TaskError::NotFound) => Err(TaskApplicationError::NotFound),
Err(error) => Err(TaskApplicationError::Task(error)),
}
}
async fn task_access(
&self,
principal: &TaskPrincipal,
@@ -320,3 +360,18 @@ impl TaskApplication {
Ok(tool)
}
}
/// Bounded retry snapshot pinned from the exact published operation at acceptance. A later
/// publication, a restart or a new binary never changes the numbers an accepted Task was admitted
/// with; a missing or oversized policy is clamped to the closed story bound.
fn task_retry_policy(tool: &PublishedAgentTool) -> crank_core::TaskRetryPolicy {
let max_attempts = tool
.operation
.execution_config
.retry_policy
.as_ref()
.map_or(1, |policy| {
policy.max_attempts.clamp(1, crank_core::TASK_MAX_ATTEMPTS)
});
crank_core::TaskRetryPolicy::new(max_attempts, 1)
}
+4 -3
View File
@@ -16,7 +16,8 @@ mod tool_search;
mod transport;
pub use app::{
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler, build_app,
build_app_with_artifacts, build_app_with_artifacts_and_tasks,
build_app_with_background_workers, build_app_with_background_workers_and_limits,
TaskApplication, TaskApplicationError, TaskPrincipal, TaskWorkerHandler,
VerifiedIdempotentWrite, build_app, build_app_with_artifacts,
build_app_with_artifacts_and_tasks, build_app_with_background_workers,
build_app_with_background_workers_and_limits,
};
+17 -11
View File
@@ -75,13 +75,16 @@ pub mod domain {
pub use crate::protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use crate::secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use crate::task::{
TASK_APPROVAL_REFERENCE_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TASK_APPROVAL_REFERENCE_MAX_BYTES,
TASK_CANCEL_REASON_MAX_BYTES, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_ENVELOPE_MAX_DEPTH, TASK_ENVELOPE_MAX_NODES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES,
TASK_IDENTITY_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskSupportIds,
TASK_IDENTITY_MAX_BYTES, TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS,
TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT, TASK_SCOPED_KEY_MAX_BYTES,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind,
TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskSupportIds,
};
pub use crate::tool_catalog::{ToolCatalogAnalysis, ToolCatalogBudget};
pub use crate::tool_quality::{
@@ -204,12 +207,15 @@ pub use product_event::{
pub use protocol::{AuthKind, ExportMode, HttpMethod, Protocol};
pub use secret::{Secret, SecretKind, SecretStatus, SecretVersion};
pub use task::{
TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES, TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES,
TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope,
TaskError, TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport,
TaskRecord, TaskState, TaskStore, TaskSupportIds, contains_sensitive_task_keys,
redact_task_envelope, sensitive_task_key, validate_task_envelope,
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TASK_DEFAULT_RETENTION_MS, TASK_ENVELOPE_MAX_BYTES,
TASK_EXECUTABLE_INPUT_CIPHERTEXT_MAX_BYTES, TASK_EXECUTABLE_INPUT_MAX_BYTES, TASK_MAX_ATTEMPTS,
TASK_MAX_BACKOFF_STEP_MS, TASK_MAX_RETENTION_MS, TASK_PURGE_BATCH_LIMIT,
TASK_SCOPED_KEY_MAX_BYTES, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt,
TaskCancelOutcome, TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId,
TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState,
TaskStore, TaskSupportIds, contains_sensitive_task_keys, redact_task_envelope,
sensitive_task_key, validate_task_envelope,
};
pub use tool_catalog::{
RECOMMENDED_TOOL_CATALOG_CONTEXT_TOKENS, ToolCatalogAnalysis, ToolCatalogAnalysisError,
+285
View File
@@ -0,0 +1,285 @@
//! Durable attempt, retry-snapshot and cancellation vocabulary of a Task (Story 4.4).
use serde::{Deserialize, Serialize};
use super::{TASK_IDENTITY_MAX_BYTES, TaskAccess, TaskEnvelope, TaskError, TaskId, TaskRecord};
/// Upper bound of the retry budget a Task may pin at acceptance (Story 4.4).
pub const TASK_MAX_ATTEMPTS: u32 = 5;
/// Upper bound of the pinned backoff step between two durable attempts (Story 4.4).
pub const TASK_MAX_BACKOFF_STEP_MS: i64 = 1_000;
/// Bound of a stored cancellation reason.
pub const TASK_CANCEL_REASON_MAX_BYTES: usize = 160;
/// Immutable retry snapshot of one accepted Task.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TaskRetryPolicy {
/// Total durable outbound attempts (1 = a single attempt, never retried).
pub max_attempts: u32,
/// Pinned base backoff between two attempts; the delay grows by this step per attempt.
pub backoff_step_ms: i64,
}
impl TaskRetryPolicy {
/// A Task that may be dispatched exactly once (legacy Tasks and unverified writes).
pub const fn single_attempt() -> Self {
Self {
max_attempts: 1,
backoff_step_ms: 0,
}
}
pub fn new(max_attempts: u32, backoff_step_ms: i64) -> Self {
Self {
max_attempts,
backoff_step_ms,
}
}
pub fn validate(&self) -> Result<(), TaskError> {
if !(1..=TASK_MAX_ATTEMPTS).contains(&self.max_attempts) {
return Err(TaskError::invalid("task retry budget is invalid"));
}
if !(0..=TASK_MAX_BACKOFF_STEP_MS).contains(&self.backoff_step_ms) {
return Err(TaskError::invalid("task retry backoff is invalid"));
}
Ok(())
}
/// Delay before attempt number `next_ordinal` (2..=max_attempts), capped so a pinned snapshot can
/// never stretch the accepted execution deadline on its own.
pub fn backoff_before(&self, next_ordinal: u32) -> i64 {
let steps = i64::from(next_ordinal.saturating_sub(1)).max(1);
self.backoff_step_ms
.saturating_mul(steps)
.min(self.backoff_step_ms.saturating_mul(4))
}
}
/// Certainty of one durable attempt outcome.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptCertainty {
/// The outcome of the outbound operation is proven (success or a definite failure).
Known,
/// The operation may have been applied upstream; the effect cannot be proven locally.
Unknown,
}
impl AttemptCertainty {
pub const fn as_str(self) -> &'static str {
match self {
Self::Known => "known",
Self::Unknown => "unknown",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"known" => Ok(Self::Known),
"unknown" => Ok(Self::Unknown),
_ => Err(TaskError::invalid("unknown attempt certainty")),
}
}
}
/// Phase of one durable attempt at the moment it was resolved.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptPhase {
/// The durable checkpoint exists but no outbound send was observed.
BeforeDispatch,
/// The request was handed to the transport; its effect may exist upstream.
Dispatched,
}
impl AttemptPhase {
pub const fn as_str(self) -> &'static str {
match self {
Self::BeforeDispatch => "before_dispatch",
Self::Dispatched => "dispatched",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"before_dispatch" => Ok(Self::BeforeDispatch),
"dispatched" => Ok(Self::Dispatched),
_ => Err(TaskError::invalid("unknown attempt phase")),
}
}
}
/// Closed outcome vocabulary of one resolved attempt.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AttemptOutcomeClass {
Completed,
Failed,
/// A bounded further attempt was scheduled instead of a terminal outcome.
RetryScheduled,
OutcomeUnknown,
}
impl AttemptOutcomeClass {
pub const fn as_str(self) -> &'static str {
match self {
Self::Completed => "completed",
Self::Failed => "failed",
Self::RetryScheduled => "retry_scheduled",
Self::OutcomeUnknown => "outcome_unknown",
}
}
pub fn parse(value: &str) -> Result<Self, TaskError> {
match value {
"completed" => Ok(Self::Completed),
"failed" => Ok(Self::Failed),
"retry_scheduled" => Ok(Self::RetryScheduled),
"outcome_unknown" => Ok(Self::OutcomeUnknown),
_ => Err(TaskError::invalid("unknown attempt outcome")),
}
}
}
/// One durable outbound attempt. The row is created by the dispatch checkpoint before any network
/// use and becomes immutable once resolved, so a stale worker can never append to the history.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskAttempt {
pub attempt_id: String,
pub task_id: TaskId,
pub ordinal: u32,
pub owner: String,
pub fence: u64,
/// Canonical fingerprint of the sealed executable input this attempt was admitted for.
pub input_fingerprint: String,
/// Digest of the stable upstream idempotency key (never the raw key or headers).
pub upstream_key_digest: Option<String>,
/// Whether a retry of this attempt is safe (read or server-verified idempotent write).
pub retry_allowed: bool,
pub checkpoint_at_unix_ms: i64,
pub resolved_at_unix_ms: Option<i64>,
pub outcome: Option<AttemptOutcomeClass>,
pub certainty: Option<AttemptCertainty>,
pub phase: Option<AttemptPhase>,
}
/// Committed dispatch intent of one durable attempt (Story 4.4). The store owns the transaction: the
/// attempt row, the incremented budget and the new Task version commit together before the network is
/// touched, so an uncommitted checkpoint can never produce an outbound call.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BeginAttempt {
pub task_id: TaskId,
pub owner: String,
pub fence: u64,
/// Version the worker read from the lease; a concurrent change rejects the checkpoint.
pub expected_version: u32,
pub attempt_id: String,
pub input_fingerprint: String,
pub upstream_key_digest: Option<String>,
/// The runtime's decision that a retry of this attempt is safe; the store still enforces the
/// pinned budget and the absolute deadline.
pub retry_allowed: bool,
}
impl BeginAttempt {
pub fn validate(&self) -> Result<(), TaskError> {
if self.owner.is_empty() || self.owner.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task attempt owner is invalid"));
}
if self.attempt_id.is_empty() || self.attempt_id.len() > TASK_IDENTITY_MAX_BYTES {
return Err(TaskError::invalid("task attempt identity is invalid"));
}
if self.input_fingerprint.len() != 64 {
return Err(TaskError::invalid("task attempt fingerprint is invalid"));
}
if let Some(digest) = self.upstream_key_digest.as_deref()
&& (digest.is_empty() || digest.len() > TASK_IDENTITY_MAX_BYTES)
{
return Err(TaskError::invalid(
"task attempt upstream key digest is invalid",
));
}
Ok(())
}
}
/// Leased, committed view of one in-flight attempt and the Task version it was admitted under.
#[derive(Clone, Debug, PartialEq)]
pub struct AttemptLease {
pub attempt: TaskAttempt,
pub task: TaskRecord,
}
/// Typed outcome of one attempt, handed to the store for a single atomic resolution.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AttemptOutcome {
/// Proven success; the Task becomes `completed` with this result.
Completed(TaskEnvelope),
/// Proven failure. `retryable` only schedules another attempt when the store's pinned budget,
/// the attempt's safe-retry decision and the absolute deadline all still allow it; otherwise the
/// Task fails with this error.
Failed {
error: TaskEnvelope,
retryable: bool,
phase: AttemptPhase,
},
/// The outbound effect cannot be proven. `repeatable` only schedules another bounded attempt when
/// repeating it is safe and the pinned budget/deadline still allow it; otherwise the Task becomes
/// the non-terminal `outcome_unknown`, never a plain failure.
Uncertain {
error: TaskEnvelope,
repeatable: bool,
phase: AttemptPhase,
},
}
/// Single-winner result of resolving one attempt.
#[derive(Clone, Debug, PartialEq)]
pub enum AttemptResolution {
/// A terminal transition committed (completed or failed).
Finished(TaskRecord),
/// The attempt failed in a known way and another bounded attempt was armed.
RetryAt {
task: TaskRecord,
next_attempt_at_unix_ms: i64,
},
/// The attempt resolved to the non-terminal uncertain state.
Unknown(TaskRecord),
}
/// Owner-side cancellation request. `expected_version` is optional so a repeated cancel of the same
/// accepted Task stays idempotent while a stale version is reported as a conflict.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskCancelRequest {
pub access: TaskAccess,
pub task_id: TaskId,
pub expected_version: Option<u32>,
pub reason: String,
}
impl TaskCancelRequest {
pub fn validate(&self) -> Result<(), TaskError> {
self.access.validate()?;
if self.reason.is_empty()
|| self.reason.len() > TASK_CANCEL_REASON_MAX_BYTES
|| self.reason.bytes().any(|byte| byte.is_ascii_control())
{
return Err(TaskError::invalid("task cancellation reason is invalid"));
}
Ok(())
}
}
/// Typed result of an owner-scoped cancellation.
#[derive(Clone, Debug, PartialEq)]
pub enum TaskCancelOutcome {
/// Cancelled before the dispatch checkpoint: no outbound call can have happened.
Cancelled(TaskRecord),
/// The Task already had a checkpointed attempt: non-terminal uncertain, never a plain cancel.
OutcomeUnknown(TaskRecord),
/// The Task was already in a terminal state; its accepted result is unchanged.
AlreadyTerminal(TaskRecord),
/// The caller's expected version is stale; `current` is the authorized current state.
Conflict(TaskRecord),
}
@@ -9,7 +9,9 @@
//! Deliberately absent: credentials, prepared headers, approval bearer tokens and any claim that a
//! Task was executed. A terminal state is written by a conditional transition only.
mod attempt;
use async_trait::async_trait;
pub use attempt::*;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
@@ -127,9 +129,9 @@ impl TaskState {
| Self::Cancelled
),
Self::InputRequired => matches!(next, Self::Working | Self::Failed | Self::Cancelled),
Self::OutcomeUnknown => {
matches!(next, Self::Completed | Self::Failed | Self::Cancelled)
}
// Story 4.4: an uncertain outbound outcome is never resolved by a generic transition.
// Reconciliation is Story 4.6, so no ordinary finish/cancel/retry may leave this state.
Self::OutcomeUnknown => false,
Self::Completed | Self::Failed | Self::Cancelled => false,
}
}
@@ -206,6 +208,9 @@ pub struct TaskPins {
pub retain_until_unix_ms: i64,
/// Verified approval evidence reference (identifier only, never a bearer token).
pub approval_reference: Option<String>,
/// Bounded retry budget pinned at acceptance (Story 4.4). A later publication, a restart or a
/// new binary never changes the numbers an accepted Task was admitted with.
pub retry: TaskRetryPolicy,
}
impl TaskPins {
@@ -233,6 +238,7 @@ impl TaskPins {
{
return Err(TaskError::invalid("task approval reference is invalid"));
}
self.retry.validate()?;
Ok(())
}
}
@@ -534,6 +540,12 @@ pub struct TaskRecord {
pub current_attempt_id: Option<String>,
/// Dispatch checkpoint epoch milliseconds, if an outbound attempt was committed.
pub dispatch_started_at_unix_ms: Option<i64>,
/// Earliest moment the next durable attempt may be dispatched (pinned backoff).
pub next_attempt_at_unix_ms: Option<i64>,
/// Durable cancellation request that arrived while an attempt was already checkpointed.
pub cancel_requested_at_unix_ms: Option<i64>,
/// Bounded reason of the cancellation request.
pub cancel_reason: Option<String>,
/// Monotonic lifecycle version used by the terminal CAS.
pub version: u32,
pub created_at_unix_ms: i64,
@@ -688,14 +700,28 @@ pub trait TaskStore: Send + Sync {
progress: TaskEnvelope,
) -> Result<TaskRecord, TaskError>;
/// Commits an outbound attempt identity and dispatch timestamp before network use.
async fn checkpoint_dispatch(
/// Commits one durable outbound attempt before network use: attempt row, incremented budget and
/// the new Task version commit in one transaction. A returned lease is the only authority to
/// dispatch that attempt; an unknown commit means zero outbound calls.
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError>;
/// Resolves exactly one committed attempt under its lease/fence/version. Only the current attempt
/// of the current owner may be resolved, and the attempt becomes immutable afterwards.
async fn resolve_attempt(
&self,
owner: &str,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError>;
/// Owner-scoped atomic cancellation. It serializes with claim and checkpoint on the same Task.
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError>;
/// Owner-scoped bounded history of the durable attempts of one Task.
async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
fence: u64,
attempt_id: String,
) -> Result<TaskRecord, TaskError>;
) -> Result<Vec<TaskAttempt>, TaskError>;
/// Applies a terminal transition only under a valid lease and releases intent atomically.
async fn finish_leased(
@@ -704,6 +730,11 @@ pub trait TaskStore: Send + Sync {
owner: &str,
fence: u64,
) -> Result<TaskRecord, TaskError>;
/// Resolves a leased Task whose next attempt was refused before its dispatch checkpoint (elapsed
/// deadline, rejected authority). A Task that still carries a proven ambiguity is never downgraded
/// to a plain failure: it ends non-terminal `outcome_unknown` with `code`; otherwise `failed`.
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError>;
}
#[cfg(test)]
@@ -726,6 +757,7 @@ mod tests {
execution_deadline_unix_ms: 1_790_000_000_000,
retain_until_unix_ms: 1_790_000_000_000 + 1_000,
approval_reference: Some("appr_1".to_owned()),
retry: TaskRetryPolicy::new(3, 1),
},
scoped_key: "a".repeat(64),
input_fingerprint: "b".repeat(64),
@@ -840,6 +872,56 @@ mod tests {
assert!(empty_key_reference.validate().is_err());
}
#[test]
fn outcome_unknown_has_no_generic_successor() {
// Story 4.4: reconciliation of an uncertain outcome is Story 4.6. Until then no ordinary
// finish/cancel/retry may leave the non-terminal uncertain state.
for next in [
TaskState::Queued,
TaskState::Working,
TaskState::InputRequired,
TaskState::OutcomeUnknown,
TaskState::Completed,
TaskState::Failed,
TaskState::Cancelled,
] {
assert!(
!TaskState::OutcomeUnknown.allows_transition_to(next),
"outcome_unknown must not transition to {next:?}"
);
}
assert!(!TaskState::OutcomeUnknown.is_terminal());
}
#[test]
fn retry_policy_bounds_are_pinned_and_bounded() {
assert!(TaskRetryPolicy::single_attempt().validate().is_ok());
assert!(
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS, TASK_MAX_BACKOFF_STEP_MS)
.validate()
.is_ok()
);
assert!(TaskRetryPolicy::new(0, 0).validate().is_err());
assert!(
TaskRetryPolicy::new(TASK_MAX_ATTEMPTS + 1, 0)
.validate()
.is_err()
);
assert!(
TaskRetryPolicy::new(1, TASK_MAX_BACKOFF_STEP_MS + 1)
.validate()
.is_err()
);
// The pinned step grows the delay but never beyond four steps.
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(2), 2);
assert_eq!(TaskRetryPolicy::new(5, 2).backoff_before(5), 8);
assert_eq!(TaskRetryPolicy::single_attempt().backoff_before(2), 0);
// A pin whose retry snapshot is invalid is rejected at acceptance.
let mut invalid = request();
invalid.pins.retry = TaskRetryPolicy::new(0, 0);
assert!(invalid.validate().is_err());
}
#[test]
fn finish_requires_a_terminal_reachable_transition() {
let access = TaskAccess {
+2
View File
@@ -10,6 +10,7 @@ mod durable_tasks_v25;
mod durable_tasks_v26;
mod durable_tasks_v27;
mod durable_tasks_v28;
mod durable_tasks_v29;
mod execution_admission_v24;
mod execution_outcome_v5;
mod import_plan_v14;
@@ -45,6 +46,7 @@ mod schema_guard_v25;
mod schema_guard_v26;
mod schema_guard_v27;
mod schema_guard_v28;
mod schema_guard_v29;
mod schema_guard_v7;
mod schema_guard_v8;
mod schema_guard_v9;
@@ -8,6 +8,7 @@ use super::durable_tasks_v25;
use super::durable_tasks_v26;
use super::durable_tasks_v27;
use super::durable_tasks_v28;
use super::durable_tasks_v29;
use super::execution_admission_v24;
use super::execution_outcome_v5;
use super::import_plan_v14;
@@ -44,10 +45,10 @@ use preflight::*;
use contract::baseline_source_digest;
use contract::validate_descriptors;
const MIGRATION_LOCK_ID: i64 = 0x4352_414E_4B4D_4947;
const CURRENT_VERSION: i64 = 28;
const CURRENT_VERSION: i64 = 29;
const IMPLEMENTED_VERSIONS: &[i64] = &[
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28,
27, 28, 29,
];
const BASELINE_SOURCE_SHA256: &str =
"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675";
@@ -366,6 +367,12 @@ impl MigrationAuthority {
durable_tasks_v28::SOURCE_SHA256,
27,
),
expand_descriptor(
29,
durable_tasks_v29::NAME,
durable_tasks_v29::SOURCE_SHA256,
28,
),
]
}
pub fn validate_sequence() -> Result<(), MigrationError> {
@@ -540,6 +547,9 @@ impl MigrationAuthority {
if from < 28 {
durable_tasks_v28::apply(&mut transaction, &Self::sequence()[27]).await?;
}
if from < 29 {
durable_tasks_v29::apply(&mut transaction, &Self::sequence()[28]).await?;
}
match inspect(&mut transaction).await? {
MigrationPreflight::Current {
version: CURRENT_VERSION,
@@ -0,0 +1,45 @@
use sqlx::{Postgres, Transaction, query};
use super::authority::{MigrationDescriptor, MigrationError};
pub(super) const NAME: &str = "durable-tasks-v29";
pub(super) const SOURCE: &str = include_str!("durable_tasks_v29.sql");
pub(super) const SOURCE_SHA256: &str =
"0666c7d2aa440e6aae2170ceaf2fce2549118612e1505ef78817be60436a75dd";
pub(super) async fn apply(
transaction: &mut Transaction<'_, Postgres>,
descriptor: &MigrationDescriptor,
) -> Result<(), MigrationError> {
sqlx::raw_sql(SOURCE)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v29",
Some(29),
"restore_known_good_backup",
)
})?;
query(
"insert into __crank_migrations (version, name, checksum, phase, compatibility)
values ($1, $2, $3, $4, $5)",
)
.bind(descriptor.version)
.bind(descriptor.name)
.bind(&descriptor.checksum)
.bind(descriptor.phase)
.bind(descriptor.compatibility)
.execute(&mut **transaction)
.await
.map_err(|_| {
MigrationError::new(
"apply_failed",
"apply.durable_tasks_v29",
Some(29),
"restore_known_good_backup",
)
})?;
Ok(())
}
@@ -0,0 +1,52 @@
-- V29: durable per-outbound attempt history, pinned retry policy and cancellation (Story 4.4).
-- An accepted Task keeps the retry snapshot it was admitted with; legacy Tasks get a single attempt.
alter table tasks add column if not exists max_attempts integer not null default 1;
alter table tasks add column if not exists retry_backoff_step_ms bigint not null default 0;
alter table tasks add column if not exists next_attempt_at timestamptz;
alter table tasks add column if not exists cancel_requested_at timestamptz;
alter table tasks add column if not exists cancel_reason text;
alter table tasks add constraint tasks_max_attempts_check check (max_attempts between 1 and 5);
alter table tasks add constraint tasks_retry_backoff_step_check
check (retry_backoff_step_ms between 0 and 1000);
alter table tasks add constraint tasks_cancel_reason_check check (
cancel_reason is null or octet_length(cancel_reason) between 1 and 160
);
-- One row per committed outbound attempt. The authoritative outcome of a Task stays on `tasks`;
-- this history only records what was attempted and how each attempt resolved.
create table if not exists task_attempts (
attempt_id text primary key,
task_id uuid not null references tasks (task_id) on delete cascade,
ordinal integer not null,
owner text not null,
fence bigint not null default 0,
input_fingerprint text not null,
upstream_key_digest text,
retry_allowed boolean not null default false,
checkpoint_at timestamptz not null default now(),
resolved_at timestamptz,
outcome text,
certainty text,
phase text,
constraint task_attempts_task_ordinal_unique unique (task_id, ordinal),
constraint task_attempts_ordinal_check check (ordinal between 1 and 5),
constraint task_attempts_outcome_check check (
outcome is null
or outcome in ('completed', 'failed', 'retry_scheduled', 'outcome_unknown')
),
constraint task_attempts_certainty_check check (
certainty is null or certainty in ('known', 'unknown')
),
constraint task_attempts_phase_check check (
phase is null or phase in ('before_dispatch', 'dispatched')
),
-- A resolved attempt is complete: an unresolved row carries no resolution at all, so a stale
-- writer can never append a second outcome to the same attempt.
constraint task_attempts_resolution_check check (
(resolved_at is null and outcome is null and certainty is null and phase is null)
or (resolved_at is not null and outcome is not null and certainty is not null)
)
);
create index if not exists task_attempts_task_id_idx on task_attempts (task_id, ordinal);
@@ -489,6 +489,9 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 26 {
super::schema_guard_v25::validate_v25_durable_tasks(connection).await?;
if !super::schema_guard_v26::validate_v26_absent(connection).await? {
@@ -497,6 +500,9 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 27 {
super::schema_guard_v26::validate_v26_durable_tasks(connection).await?;
if !super::schema_guard_v27::validate_v27_absent(connection).await? {
@@ -505,15 +511,27 @@ pub(super) async fn validate_schema_fingerprint(
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 28 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v27::validate_v27_durable_tasks(connection).await?;
if !super::schema_guard_v28::validate_v28_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
} else if current_version < 29 {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v28::validate_v28_durable_tasks(connection).await?;
if !super::schema_guard_v29::validate_v29_absent(connection).await? {
return Err(schema_error(current_version));
}
} else {
super::schema_guard_v26::validate_v26_base_columns(connection).await?;
super::schema_guard_v29::validate_v29_durable_attempts(connection).await?;
}
Ok(())
}
@@ -73,7 +73,7 @@ pub(super) async fn validate_v27_durable_tasks(
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(37).unwrap_or_default() {
if count != 37_i64 {
return Err(MigrationError::new(
"partial_sequence",
"v27.columns",
@@ -10,10 +10,10 @@ const TASKS: &str = "tasks";
pub(super) async fn validate_v28_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
if super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await? {
return Ok(false);
}
if relation_exists(connection, TASKS).await?
&& super::schema_guard::column_exists(connection, TASKS, "platform_api_key_id").await?
{
return Ok(false);
}
Ok(true)
}
@@ -55,7 +55,7 @@ pub(super) async fn validate_v28_durable_tasks(
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != i64::try_from(38).unwrap_or_default() {
if count != 38_i64 {
return Err(MigrationError::new(
"partial_sequence",
"v28.columns",
@@ -0,0 +1,87 @@
//! V29 schema guard: durable per-outbound attempt history, pinned retry policy and cancellation
//! (Story 4.4).
use sqlx::{PgConnection, Row, query};
use super::authority::MigrationError;
use super::schema_guard::{column_exists, relation_exists};
const TASKS: &str = "tasks";
const ATTEMPTS: &str = "task_attempts";
pub(super) async fn validate_v29_absent(
connection: &mut PgConnection,
) -> Result<bool, MigrationError> {
if relation_exists(connection, TASKS).await? {
if column_exists(connection, TASKS, "max_attempts").await? {
return Ok(false);
}
if relation_exists(connection, ATTEMPTS).await? {
return Ok(false);
}
}
Ok(true)
}
pub(super) async fn validate_v29_durable_attempts(
connection: &mut PgConnection,
) -> Result<(), MigrationError> {
// The V28 column set is still required and unchanged; V29 only appends columns and one table.
for column in [
"platform_api_key_id",
"max_attempts",
"retry_backoff_step_ms",
"next_attempt_at",
"cancel_requested_at",
"cancel_reason",
] {
if !column_exists(connection, TASKS, column).await? {
return Err(invalid(29, "v29.columns"));
}
}
// Exact total column count for version 29: the 38 V28 columns + 5 V29 columns.
let count: i64 = query(
"select count(*)::bigint as count from information_schema.columns
where table_schema = current_schema() and table_name = $1",
)
.bind(TASKS)
.fetch_one(&mut *connection)
.await
.map_err(|_| MigrationError::storage("preflight.schema"))?
.try_get("count")
.map_err(|_| MigrationError::storage("preflight.schema"))?;
if count != 43_i64 {
return Err(invalid(29, "v29.columns"));
}
if !relation_exists(connection, ATTEMPTS).await? {
return Err(invalid(29, "v29.attempts"));
}
for column in [
"attempt_id",
"task_id",
"ordinal",
"owner",
"fence",
"input_fingerprint",
"upstream_key_digest",
"retry_allowed",
"checkpoint_at",
"resolved_at",
"outcome",
"certainty",
"phase",
] {
if !column_exists(connection, ATTEMPTS, column).await? {
return Err(invalid(29, "v29.attempts"));
}
}
Ok(())
}
fn invalid(version: i64, stage: &'static str) -> MigrationError {
MigrationError::new(
"partial_sequence",
stage,
Some(version),
"restore_known_good_backup",
)
}
@@ -69,6 +69,7 @@ impl PostgresRegistry {
rows.into_iter().map(map_master_key_task_version).collect()
}
#[allow(clippy::too_many_arguments)]
pub async fn stage_master_key_rotation_task_ciphertext(
&self,
rotation_id: &str,
@@ -0,0 +1,195 @@
//! Acceptance half of the PostgreSQL Task store (Story 4.2/4.3): the one-transaction acceptance
//! path, its bounded unique-conflict reread and the typed miss classification.
use crank_core::{
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskError, TaskFinishRequest, TaskState,
};
use sqlx::{Postgres, Row, Transaction};
use super::super::admission::{
AttemptError, bounded_queue_len, cleanup_expired_unlinked, insert_admission_entry,
lock_admission_in_tx,
};
use super::{
Accepted, PostgresTaskStore, TASK_PROJECTION, acceptance_from_record, policy_mode_str,
record_from_row, task_sql_error,
};
impl PostgresTaskStore {
pub(super) async fn accept_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Accepted, AttemptError> {
// Replay first: an already accepted Task is returned without capacity mutation, so it works
// with a full queue and after the original execution deadline while the record is retained.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
lock_admission_in_tx(transaction, &request.workspace_id).await?;
// The competitor may have committed while this transaction waited for the controller lock.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
cleanup_expired_unlinked(transaction).await?;
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let created_ms: i64 = sqlx::query(
"insert into tasks (
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch, platform_api_key_id, max_attempts, retry_backoff_step_ms
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22, $23, $24
)
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
)
.bind(request.task_id.as_uuid())
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
.bind(&request.scoped_key)
.bind(&request.input_fingerprint)
.bind(request.support.request_id.as_str())
.bind(request.support.trace_id.as_str())
.bind(&request.support.traceparent)
.bind(request.support.trace_sampled)
.bind(policy_mode_str(request.pins.policy_mode))
.bind(request.pins.approval_reference.as_deref())
.bind(request.input.value().clone())
.bind(request.pins.execution_deadline_unix_ms)
.bind(request.pins.retain_until_unix_ms)
.bind(request.executable_input_ciphertext.as_deref())
.bind(request.executable_input_key_version.as_deref())
.bind(request.executable_input_master_key_epoch)
.bind(
request
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str),
)
.bind(i32::try_from(request.pins.retry.max_attempts).unwrap_or(1))
.bind(request.pins.retry.backoff_step_ms)
.fetch_one(&mut **transaction)
.await
.map_err(task_sql_error)?
.try_get("created_ms")
.map_err(AttemptError::from_sql)?;
let admission_request = AdmissionRequest {
scope: AdmissionScope {
workspace_id: request.workspace_id.clone(),
agent_key: request.owner_id.clone(),
operation_id: request.operation_id.clone(),
operation_version: request.pins.operation_version,
},
mode: AdmissionMode::Background,
policy_mode: request.pins.policy_mode,
no_tasks_fallback: NoTasksFallback::Unsupported,
// Never reached: the intent row is inserted by this transaction, not by `admit`.
approval: ApprovalEvidence::NotRequired,
fingerprint: request.input_fingerprint.clone(),
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
};
let owner_token = format!("task_{}", request.task_id);
insert_admission_entry(
transaction,
&admission_request,
&owner_token,
"reserved",
Some(request.task_id.as_uuid()),
)
.await?;
Ok(Accepted::Created(TaskAcceptance {
task_id: request.task_id,
state: TaskState::Queued,
operation_version: request.pins.operation_version,
support: request.support.clone(),
created_at_unix_ms: created_ms,
retain_until_unix_ms: request.pins.retain_until_unix_ms,
replayed: false,
}))
}
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
pub(super) async fn reread_after_unique(
&self,
request: &TaskAcceptRequest,
) -> Result<TaskAcceptance, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::unavailable(
"durable task acceptance ended with an unresolved unique conflict",
));
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
return Err(TaskError::FingerprintConflict);
}
Ok(acceptance_from_record(&record))
}
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
pub(super) async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
let access = &request.access;
match sqlx::query(
"select state, version from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
{
Ok(Some(_)) => TaskError::StateConflict,
Ok(None) => TaskError::NotFound,
Err(error) => Self::unavailable(error),
}
}
}
@@ -0,0 +1,583 @@
//! Durable per-outbound attempt persistence of a Task (Story 4.4): the dispatch checkpoint, the
//! single-winner attempt resolution and the owner-scoped cancellation live here so the acceptance
//! and retention half of the store stays readable.
use crank_core::{
AttemptCertainty, AttemptLease, AttemptOutcome, AttemptOutcomeClass, AttemptPhase,
AttemptResolution, BeginAttempt, TaskAccess, TaskAttempt, TaskCancelOutcome, TaskCancelRequest,
TaskEnvelope, TaskError, TaskId, TaskRecord, TaskState,
};
use sqlx::Row;
use uuid::Uuid;
use super::{PostgresTaskStore, TASK_PROJECTION, record_from_row, unavailable_row};
const ATTEMPT_PROJECTION: &str = "attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
upstream_key_digest, retry_allowed,
(extract(epoch from checkpoint_at) * 1000)::bigint as checkpoint_at_ms,
(extract(epoch from resolved_at) * 1000)::bigint as resolved_at_ms,
outcome, certainty, phase";
/// Bounded attempt-resolution helper: `retry` may only be armed when the attempt itself is marked
/// safe-to-retry, the pinned budget still has room and the accepted absolute deadline still does.
impl PostgresTaskStore {
fn retry_delay(
&self,
record: &TaskRecord,
lease: &AttemptLease,
safe: bool,
now: i64,
) -> Option<i64> {
let delay = record.pins.retry.backoff_before(lease.attempt.ordinal + 1);
(safe
&& lease.attempt.retry_allowed
&& lease.attempt.ordinal < record.pins.retry.max_attempts
&& now.saturating_add(delay) < record.pins.execution_deadline_unix_ms)
.then_some(delay)
}
}
impl PostgresTaskStore {
/// Authoritative word on whether an earlier resolved attempt really reached upstream despite
/// the current one not having dispatched. Returns Some(phase) only when the latest resolved
/// attempt that actually dispatched (phase = 'dispatched') is still of unknown certainty.
pub(super) async fn unproven_dispatch_phase(
&self,
task_id: TaskId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<Option<String>, TaskError> {
let row = sqlx::query(
"select certainty, phase from task_attempts where task_id = $1 and resolved_at is not null and phase = 'dispatched' order by ordinal desc limit 1",
)
.bind(task_id.as_uuid())
.fetch_optional(&mut **transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Ok(None);
};
let certainty: Option<String> = row.try_get("certainty").map_err(Self::unavailable)?;
if certainty.as_deref() != Some(AttemptCertainty::as_str(AttemptCertainty::Unknown)) {
return Ok(None);
}
let phase: Option<String> = row.try_get("phase").map_err(Self::unavailable)?;
Ok(Some(phase.unwrap_or_else(|| "dispatched".to_owned())))
}
}
fn attempt_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskAttempt, TaskError> {
let attempt_id: String = row.try_get("attempt_id").map_err(unavailable_row)?;
let task_id: Uuid = row.try_get("task_id").map_err(unavailable_row)?;
let outcome: Option<String> = row.try_get("outcome").map_err(unavailable_row)?;
let certainty: Option<String> = row.try_get("certainty").map_err(unavailable_row)?;
let phase: Option<String> = row.try_get("phase").map_err(unavailable_row)?;
Ok(TaskAttempt {
attempt_id,
task_id: TaskId::parse(&task_id.to_string())?,
ordinal: u32::try_from(row.try_get::<i32, _>("ordinal").map_err(unavailable_row)?)
.unwrap_or_default(),
owner: row.try_get("owner").map_err(unavailable_row)?,
fence: u64::try_from(row.try_get::<i64, _>("fence").map_err(unavailable_row)?)
.unwrap_or_default(),
input_fingerprint: row.try_get("input_fingerprint").map_err(unavailable_row)?,
upstream_key_digest: row
.try_get("upstream_key_digest")
.map_err(unavailable_row)?,
retry_allowed: row.try_get("retry_allowed").map_err(unavailable_row)?,
checkpoint_at_unix_ms: row.try_get("checkpoint_at_ms").map_err(unavailable_row)?,
resolved_at_unix_ms: row.try_get("resolved_at_ms").map_err(unavailable_row)?,
outcome: outcome
.as_deref()
.map(AttemptOutcomeClass::parse)
.transpose()?,
certainty: certainty
.as_deref()
.map(AttemptCertainty::parse)
.transpose()?,
phase: phase.as_deref().map(AttemptPhase::parse).transpose()?,
})
}
impl PostgresTaskStore {
pub(super) async fn checkpoint_dispatch_attempt(
&self,
request: BeginAttempt,
) -> Result<AttemptLease, TaskError> {
request.validate()?;
let mut transaction = self.start().await?;
// One row lock serializes this checkpoint with claim, cancellation and the resolution of the
// same Task, so a cancel that wins first can never be followed by an outbound call.
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(request.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
transaction.rollback().await.ok();
return Err(TaskError::FingerprintConflict);
}
let now = crank_core::current_unix_ms();
let eligible = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(request.owner.as_str())
&& record.fence == request.fence
&& record.version == request.expected_version
&& record.current_attempt_id.is_none()
&& record.cancel_requested_at_unix_ms.is_none()
&& record.lease_until_unix_ms.is_some_and(|until| until > now)
&& record.attempts < record.pins.retry.max_attempts
&& record.pins.execution_deadline_unix_ms > now
&& record.next_attempt_at_unix_ms.is_none_or(|at| at <= now);
if !eligible {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let ordinal = record.attempts.saturating_add(1);
sqlx::query(
"insert into task_attempts (
attempt_id, task_id, ordinal, owner, fence, input_fingerprint,
upstream_key_digest, retry_allowed
) values ($1, $2, $3, $4, $5, $6, $7, $8)",
)
.bind(&request.attempt_id)
.bind(request.task_id.as_uuid())
.bind(i32::try_from(ordinal).unwrap_or(i32::MAX))
.bind(&request.owner)
.bind(i64::try_from(request.fence).unwrap_or_default())
.bind(&request.input_fingerprint)
.bind(request.upstream_key_digest.as_deref())
.bind(request.retry_allowed)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set current_attempt_id = $1,
dispatch_started_at = now(),
attempts = attempts + 1,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id = $2 and current_attempt_id is null and version = $3
returning {TASK_PROJECTION}"
)))
.bind(&request.attempt_id)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
transaction.commit().await.map_err(Self::unavailable)?;
let attempt = TaskAttempt {
attempt_id: request.attempt_id,
task_id: request.task_id,
ordinal,
owner: request.owner,
fence: request.fence,
input_fingerprint: request.input_fingerprint,
upstream_key_digest: request.upstream_key_digest,
retry_allowed: request.retry_allowed,
checkpoint_at_unix_ms: task.dispatch_started_at_unix_ms.unwrap_or(now),
resolved_at_unix_ms: None,
outcome: None,
certainty: None,
phase: None,
};
Ok(AttemptLease { attempt, task })
}
pub(super) async fn resolve_attempt_outcome(
&self,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks where task_id = $1 for update"
)))
.bind(lease.attempt.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
let now = crank_core::current_unix_ms();
let prior_uncertain_phase = self
.unproven_dispatch_phase(lease.attempt.task_id, &mut transaction)
.await?;
// Only the current attempt of the current owner, under the version it was admitted with, may
// be resolved. A stale worker can never append a second outcome to the history or the Task.
let current = record.state == TaskState::Working
&& record.lease_owner.as_deref() == Some(lease.attempt.owner.as_str())
&& record.fence == lease.attempt.fence
&& record.version == lease.task.version
&& record.current_attempt_id.as_deref() == Some(lease.attempt.attempt_id.as_str())
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
if !current {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let (class, certainty, phase) = match &outcome {
AttemptOutcome::Completed(_) => (
AttemptOutcomeClass::Completed,
AttemptCertainty::Known,
AttemptPhase::Dispatched,
),
AttemptOutcome::Failed { phase, .. } => {
(AttemptOutcomeClass::Failed, AttemptCertainty::Known, *phase)
}
AttemptOutcome::Uncertain { phase, .. } => (
AttemptOutcomeClass::OutcomeUnknown,
AttemptCertainty::Unknown,
*phase,
),
};
// A failed or uncertain attempt that may be safely repeated keeps the lease and arms the
// pinned backoff. Exhausted budget or deadline never turns uncertainty into a plain failure.
let retry = match &outcome {
AttemptOutcome::Failed {
retryable,
phase: _,
error: _,
} => self.retry_delay(&record, lease, *retryable, now),
AttemptOutcome::Uncertain {
repeatable,
phase: _,
error: _,
} => self.retry_delay(&record, lease, *repeatable, now),
AttemptOutcome::Completed(_) => None,
};
let attempt_class = if retry.is_some() {
AttemptOutcomeClass::RetryScheduled
} else {
class
};
let closed = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = $1, certainty = $2, phase = $3
where attempt_id = $4 and resolved_at is null",
)
.bind(attempt_class.as_str())
.bind(certainty.as_str())
.bind(phase.as_str())
.bind(&lease.attempt.attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
if closed.rows_affected() != 1 {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let (state, result, error, keep_lease, next_attempt_at) = match outcome {
AttemptOutcome::Completed(result) => {
(TaskState::Completed, Some(result), None, false, None)
}
AttemptOutcome::Failed {
error,
retryable: _,
phase,
} => match retry {
Some(delay) => (
TaskState::Working,
None,
None,
true,
Some(now.saturating_add(delay)),
),
None => {
if phase == AttemptPhase::BeforeDispatch {
if let Some(prior) = prior_uncertain_phase.as_ref() {
let mut value = error.value().clone();
if let serde_json::Value::Object(map) = &mut value {
map.insert(
"phase".to_string(),
serde_json::Value::String(prior.clone()),
);
}
let replaced = TaskEnvelope::try_redacted(&value)?;
(TaskState::OutcomeUnknown, None, Some(replaced), false, None)
} else {
(TaskState::Failed, None, Some(error), false, None)
}
} else {
(TaskState::Failed, None, Some(error), false, None)
}
}
},
AttemptOutcome::Uncertain {
error,
repeatable: _,
phase: _,
} => match retry {
Some(delay) => (
TaskState::Working,
None,
None,
true,
Some(now.saturating_add(delay)),
),
None => (TaskState::OutcomeUnknown, None, Some(error), false, None),
},
};
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
result_envelope = $2,
error_envelope = $3,
lease_owner = case when $4 then lease_owner else null end,
lease_until = case when $4 then lease_until else null end,
-- A retry-scheduled attempt is already resolved, so it no longer holds the Task:
-- the next durable attempt needs its own checkpoint under the same lease.
current_attempt_id = case when $8 then null else current_attempt_id end,
next_attempt_at = case when $5::bigint is null then null
else to_timestamp($5::double precision / 1000.0) end,
version = version + 1,
updated_at = now()
where task_id = $6 and version = $7
returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(result.as_ref().map(|value| value.value().clone()))
.bind(error.as_ref().map(|value| value.value().clone()))
.bind(keep_lease)
.bind(next_attempt_at)
.bind(lease.attempt.task_id.as_uuid())
.bind(i64::from(record.version))
.bind(retry.is_some())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
if !keep_lease {
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(lease.attempt.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
}
transaction.commit().await.map_err(Self::unavailable)?;
Ok(match retry {
Some(_) => AttemptResolution::RetryAt {
next_attempt_at_unix_ms: next_attempt_at.unwrap_or(now),
task,
},
None if state == TaskState::OutcomeUnknown => AttemptResolution::Unknown(task),
None => AttemptResolution::Finished(task),
})
}
pub(super) async fn cancel_task(
&self,
request: TaskCancelRequest,
) -> Result<TaskCancelOutcome, TaskError> {
request.validate()?;
let access = &request.access;
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5
for update"
)))
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row)?;
if let Some(expected) = request.expected_version
&& record.version != expected
{
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::Conflict(record));
}
if record.state.is_terminal() {
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::AlreadyTerminal(record));
}
if record.state == TaskState::OutcomeUnknown {
transaction.rollback().await.ok();
return Ok(TaskCancelOutcome::OutcomeUnknown(record));
}
// A checkpointed attempt may already have reached the upstream: the honest answer is the
// non-terminal uncertain state, never a plain cancellation and never a retry.
let checkpointed = record.current_attempt_id.clone();
// Preserve an uncertainty recorded by a previously resolved attempt even after RetryAt
// cleared current_attempt_id: the request already reached the upstream in that case.
let prior_uncertain = self
.unproven_dispatch_phase(request.task_id, &mut transaction)
.await?
.is_some();
let (state, code) = match (&checkpointed, prior_uncertain) {
(Some(_), _) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
(None, true) => (TaskState::OutcomeUnknown, "cancelled_after_dispatch"),
(None, false) => (TaskState::Cancelled, "cancelled_before_dispatch"),
};
if let Some(attempt_id) = checkpointed {
let closed = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
phase = 'dispatched'
where task_id = $1 and attempt_id = $2 and resolved_at is null",
)
.bind(request.task_id.as_uuid())
.bind(&attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let _ = closed;
}
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set state = $1,
error_envelope = jsonb_build_object('code', $2),
cancel_requested_at = now(),
cancel_reason = $3,
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id = $4 and version = $5
returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(code)
.bind(&request.reason)
.bind(request.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(request.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(match state {
TaskState::Cancelled => TaskCancelOutcome::Cancelled(task),
_ => TaskCancelOutcome::OutcomeUnknown(task),
})
}
pub(super) async fn refuse_attempt_resolution(
&self,
lease: &crank_core::TaskLease,
code: &str,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks\n where task_id = $1\n for update"
)))
.bind(lease.record.task_id.as_uuid())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::NotFound);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
let now = crank_core::current_unix_ms();
let eligible = matches!(record.state, TaskState::Queued | TaskState::Working)
&& record.lease_owner.as_deref() == Some(lease.owner.as_str())
&& record.fence == lease.fence
&& record.version == lease.record.version
&& record.current_attempt_id.is_none()
&& record.lease_until_unix_ms.is_some_and(|until| until > now);
if !eligible {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
}
let prior_phase = self
.unproven_dispatch_phase(lease.record.task_id, &mut transaction)
.await?;
let uncertain = prior_phase.is_some();
let state = if uncertain {
TaskState::OutcomeUnknown
} else {
TaskState::Failed
};
let phase = prior_phase.unwrap_or_else(|| "before_dispatch".to_owned());
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks\n set state = $1,\n error_envelope = jsonb_build_object('code', $2, 'phase', $3),\n lease_owner = null,\n lease_until = null,\n next_attempt_at = null,\n version = version + 1,\n updated_at = now()\n where task_id = $4 and version = $5\n returning {TASK_PROJECTION}"
)))
.bind(state.as_str())
.bind(code)
.bind(&phase)
.bind(lease.record.task_id.as_uuid())
.bind(i64::from(record.version))
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let task = record_from_row(&row).map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(lease.record.task_id.as_uuid())
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(task)
}
pub(super) async fn attempt_history(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<TaskAttempt>, TaskError> {
access.validate()?;
let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {ATTEMPT_PROJECTION} from task_attempts
where task_id = $1 and exists (
select 1 from tasks
where tasks.task_id = task_attempts.task_id and tasks.workspace_id = $2
and tasks.owner_kind = $3 and tasks.owner_id = $4
and tasks.authorization_scope = $5
)
order by ordinal"
)))
.bind(task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_all(&self.pool)
.await
.map_err(Self::unavailable)?;
rows.iter().map(attempt_from_row).collect()
}
}
@@ -12,19 +12,20 @@
use async_trait::async_trait;
use crank_core::{
AdmissionMode, AdmissionRequest, AdmissionScope, ApprovalEvidence, NoTasksFallback,
PlatformApiKeyId, TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskEnvelope, TaskError,
TaskFinishRequest, TaskId, TaskLease, TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord,
TaskState, TaskStore, TaskSupportIds, ToolExecutionMode,
AttemptLease, AttemptOutcome, AttemptResolution, BeginAttempt, PlatformApiKeyId,
TaskAcceptRequest, TaskAcceptance, TaskAccess, TaskAttempt, TaskCancelOutcome,
TaskCancelRequest, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskLease,
TaskOwnerKind, TaskPins, TaskPurgeReport, TaskRecord, TaskRetryPolicy, TaskState, TaskStore,
TaskSupportIds, ToolExecutionMode,
};
use serde_json::Value;
use sqlx::{PgPool, Postgres, Row, Transaction};
use uuid::Uuid;
use super::admission::{
AdmissionLimits, AttemptError, bounded_queue_len, cleanup_expired_unlinked,
insert_admission_entry, lock_admission_in_tx,
};
use super::admission::{AdmissionLimits, AttemptError};
mod acceptance;
mod attempts;
const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_id,
authorization_scope, platform_api_key_id, operation_id, input_fingerprint, operation_version,
@@ -35,6 +36,10 @@ const TASK_PROJECTION: &str = "task_id, state, workspace_id, owner_kind, owner_i
(extract(epoch from lease_until) * 1000)::bigint as lease_until_ms,
current_attempt_id, dispatch_started_at,
(extract(epoch from dispatch_started_at) * 1000)::bigint as dispatch_started_at_ms, version,
max_attempts, retry_backoff_step_ms,
(extract(epoch from next_attempt_at) * 1000)::bigint as next_attempt_at_ms,
(extract(epoch from cancel_requested_at) * 1000)::bigint as cancel_requested_at_ms,
cancel_reason,
(extract(epoch from execution_deadline) * 1000)::bigint as execution_deadline_ms,
(extract(epoch from retain_until) * 1000)::bigint as retain_until_ms,
(extract(epoch from created_at) * 1000)::bigint as created_ms,
@@ -276,6 +281,7 @@ impl TaskStore for PostgresTaskStore {
and owner_id = $7 and authorization_scope = $8
and state = $9 and version = $10
and (lease_owner is null or fence = 0)
and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
@@ -371,39 +377,117 @@ and lease_owner is null
lease_ms: i64,
) -> Result<Vec<TaskLease>, TaskError> {
let batch = (batch_limit.clamp(1, 64)) as usize;
// A checkpointed attempt is never redispatched: once its lease expires the outbound outcome
// is uncertain, so it resolves to outcome_unknown and releases its admission slot. Story 4.4
// owns any later retry/cancellation of that uncertain outcome.
let mut transaction = self.start().await?;
sqlx::query(
"with candidates as (
select task_id from tasks
where state = 'working' and lease_owner is not null
and current_attempt_id is not null
and lease_until < clock_timestamp()
order by task_id
limit 64
), resolved as (
update tasks
set state = 'outcome_unknown',
lease_owner = null,
lease_until = null,
version = version + 1,
updated_at = now()
where task_id in (select task_id from candidates)
and state = 'working'
returning task_id
)
delete from execution_admission_entries
where task_id in (select task_id from resolved)",
// A checkpointed attempt is never redispatched blindly: once its lease expires another worker
// must decide per Task whether the pinned safe-retry evidence, the budget and the deadline
// still allow exactly one bounded new attempt (Story 4.4). Everything else resolves to the
// non-terminal uncertain state and releases its admission slot.
let candidates = sqlx::query(
"select t.task_id, t.current_attempt_id, t.attempts, t.max_attempts,
t.execution_deadline > clock_timestamp() as deadline_left,
coalesce(a.retry_allowed, false) as retry_allowed,
a.resolved_at is null as attempt_open
from tasks t
left join task_attempts a
on a.task_id = t.task_id and a.attempt_id = t.current_attempt_id
where t.state = 'working' and t.lease_owner is not null
and t.current_attempt_id is not null
and t.lease_until < clock_timestamp()
order by t.task_id
limit 64
for update of t skip locked",
)
.execute(&mut *transaction)
.fetch_all(&mut *transaction)
.await
.map_err(Self::unavailable)?;
for candidate in candidates {
let task_id: Uuid = candidate.try_get("task_id").map_err(Self::unavailable)?;
let attempt_id: String = candidate
.try_get("current_attempt_id")
.map_err(Self::unavailable)?;
let attempts: i32 = candidate.try_get("attempts").map_err(Self::unavailable)?;
let max_attempts: i32 = candidate
.try_get("max_attempts")
.map_err(Self::unavailable)?;
let deadline_left: bool = candidate
.try_get("deadline_left")
.map_err(Self::unavailable)?;
let retry_allowed: bool = candidate
.try_get("retry_allowed")
.map_err(Self::unavailable)?;
let attempt_open: bool = candidate
.try_get("attempt_open")
.map_err(Self::unavailable)?;
let safe_recovery =
retry_allowed && attempt_open && attempts < max_attempts && deadline_left;
let close = sqlx::query(
"update task_attempts
set resolved_at = now(), outcome = 'outcome_unknown', certainty = 'unknown',
phase = 'dispatched'
where task_id = $1 and attempt_id = $2 and resolved_at is null",
)
.bind(task_id)
.bind(&attempt_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let _ = close;
if safe_recovery {
// Interrupted, not failed: re-arm the same admission intent and let the bounded claim
// hand the Task to a new owner with a new fence and exactly one new attempt.
sqlx::query(
"update tasks
set state = 'queued',
lease_owner = null,
lease_until = null,
current_attempt_id = null,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working'
and lease_until < clock_timestamp()",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query(
"update execution_admission_entries
set state = 'reserved', granted_at = null
where task_id = $1 and state = 'executing'",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
} else {
sqlx::query(
"update tasks
set state = 'outcome_unknown',
lease_owner = null,
lease_until = null,
error_envelope = jsonb_build_object('code', 'outcome_unknown', 'phase', 'dispatched'),
version = version + 1,
updated_at = now()
where task_id = $1 and state = 'working'
and lease_until < clock_timestamp()",
)
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
sqlx::query("delete from execution_admission_entries where task_id = $1")
.bind(task_id)
.execute(&mut *transaction)
.await
.map_err(Self::unavailable)?;
}
}
// A crash before the durable dispatch checkpoint left the Task queued for execution: its
// admission intent is re-armed as reserved and the Task becomes claimable again. Recovery is
// bounded by the <=30s lease expiry, inside the 60s story budget.
// bounded by the <=30s lease expiry, inside the 60s story budget. The expiry condition is
// re-checked in the UPDATE so a concurrent renewal can never be clobbered by recovery.
sqlx::query(
"with candidates as (
select task_id from tasks
@@ -412,15 +496,19 @@ and lease_owner is null
and lease_until < clock_timestamp()
order by task_id
limit 64
for update skip locked
), requeued as (
update tasks
set state = 'queued',
lease_owner = null,
lease_until = null,
next_attempt_at = null,
version = version + 1,
updated_at = now()
where task_id in (select task_id from candidates)
and state = 'working'
and current_attempt_id is null
and lease_until < clock_timestamp()
returning task_id
)
update execution_admission_entries
@@ -476,6 +564,7 @@ and lease_owner is null
or (state = 'working' and lease_owner is not null
and lease_until < clock_timestamp() and current_attempt_id is null)
)
and (next_attempt_at is null or next_attempt_at <= clock_timestamp())
returning {TASK_PROJECTION}"
)))
.bind(owner)
@@ -607,10 +696,10 @@ and lease_owner is null
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
if let Some(active) = active {
if active != epoch {
return Err(TaskError::StateConflict);
}
if let Some(active) = active
&& active != epoch
{
return Err(TaskError::StateConflict);
}
}
match (ciphertext, key_version, epoch) {
@@ -621,38 +710,28 @@ and lease_owner is null
}
}
async fn checkpoint_dispatch(
async fn checkpoint_dispatch(&self, request: BeginAttempt) -> Result<AttemptLease, TaskError> {
self.checkpoint_dispatch_attempt(request).await
}
async fn resolve_attempt(
&self,
owner: &str,
lease: &AttemptLease,
outcome: AttemptOutcome,
) -> Result<AttemptResolution, TaskError> {
self.resolve_attempt_outcome(lease, outcome).await
}
async fn cancel(&self, request: TaskCancelRequest) -> Result<TaskCancelOutcome, TaskError> {
self.cancel_task(request).await
}
async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
fence: u64,
attempt_id: String,
) -> Result<TaskRecord, TaskError> {
let mut transaction = self.start().await?;
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"update tasks
set current_attempt_id = $1,
dispatch_started_at = now(),
attempts = attempts + 1,
updated_at = now()
where task_id = $2 and lease_owner = $3 and fence = $4
and lease_until > clock_timestamp() and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(&attempt_id)
.bind(task_id.as_uuid())
.bind(owner)
.bind(i64::try_from(fence).unwrap_or_default())
.fetch_optional(&mut *transaction)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
transaction.rollback().await.ok();
return Err(TaskError::StateConflict);
};
let record = record_from_row(&row).map_err(Self::unavailable)?;
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
) -> Result<Vec<TaskAttempt>, TaskError> {
self.attempt_history(access, task_id).await
}
async fn finish_leased(
@@ -678,6 +757,7 @@ and lease_owner is null
and state = $9 and version = $10
and lease_owner = $11 and fence = $12
and lease_until > clock_timestamp()
and current_attempt_id is null
returning {TASK_PROJECTION}"
)))
.bind(request.target_state.as_str())
@@ -708,181 +788,9 @@ and lease_owner is null
transaction.commit().await.map_err(Self::unavailable)?;
Ok(record)
}
}
impl PostgresTaskStore {
async fn accept_locked(
&self,
transaction: &mut Transaction<'_, Postgres>,
request: &TaskAcceptRequest,
) -> Result<Accepted, AttemptError> {
// Replay first: an already accepted Task is returned without capacity mutation, so it works
// with a full queue and after the original execution deadline while the record is retained.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
// Serialize with synchronous admission and any concurrent acceptance of the same scope.
lock_admission_in_tx(transaction, &request.workspace_id).await?;
// The competitor may have committed while this transaction waited for the controller lock.
if let Some((acceptance, fingerprint)) =
self.stored_acceptance(transaction, request).await?
{
return Ok(if fingerprint == request.input_fingerprint {
Accepted::Replayed(acceptance)
} else {
Accepted::Conflict
});
}
cleanup_expired_unlinked(transaction).await?;
let queued = bounded_queue_len(transaction).await?;
if queued >= u64::try_from(self.limits.queue_limit).unwrap_or(u64::MAX) {
return Err(AttemptError::Overloaded);
}
let created_ms: i64 = sqlx::query(
"insert into tasks (
task_id, workspace_id, owner_kind, owner_id, authorization_scope, operation_id,
operation_version, scoped_key, input_fingerprint, request_id, trace_id, traceparent,
trace_sampled, policy_mode, approval_reference, state, input_envelope,
execution_deadline, retain_until, executable_input_ciphertext, executable_input_key_version,
executable_input_master_key_epoch, platform_api_key_id
) values (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, 'queued', $16,
to_timestamp($17::double precision / 1000.0),
to_timestamp($18::double precision / 1000.0), $19, $20, $21, $22
)
returning (extract(epoch from created_at) * 1000)::bigint as created_ms",
)
.bind(request.task_id.as_uuid())
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(i32::try_from(request.pins.operation_version).unwrap_or(i32::MAX))
.bind(&request.scoped_key)
.bind(&request.input_fingerprint)
.bind(request.support.request_id.as_str())
.bind(request.support.trace_id.as_str())
.bind(&request.support.traceparent)
.bind(request.support.trace_sampled)
.bind(policy_mode_str(request.pins.policy_mode))
.bind(request.pins.approval_reference.as_deref())
.bind(request.input.value().clone())
.bind(request.pins.execution_deadline_unix_ms)
.bind(request.pins.retain_until_unix_ms)
.bind(request.executable_input_ciphertext.as_deref())
.bind(request.executable_input_key_version.as_deref())
.bind(request.executable_input_master_key_epoch)
.bind(
request
.platform_api_key_id
.as_ref()
.map(PlatformApiKeyId::as_str),
)
.fetch_one(&mut **transaction)
.await
.map_err(task_sql_error)?
.try_get("created_ms")
.map_err(AttemptError::from_sql)?;
let admission_request = AdmissionRequest {
scope: AdmissionScope {
workspace_id: request.workspace_id.clone(),
agent_key: request.owner_id.clone(),
operation_id: request.operation_id.clone(),
operation_version: request.pins.operation_version,
},
mode: AdmissionMode::Background,
policy_mode: request.pins.policy_mode,
no_tasks_fallback: NoTasksFallback::Unsupported,
// Never reached: the intent row is inserted by this transaction, not by `admit`.
approval: ApprovalEvidence::NotRequired,
fingerprint: request.input_fingerprint.clone(),
expires_at_unix_ms: request.pins.execution_deadline_unix_ms,
};
let owner_token = format!("task_{}", request.task_id);
insert_admission_entry(
transaction,
&admission_request,
&owner_token,
"reserved",
Some(request.task_id.as_uuid()),
)
.await?;
Ok(Accepted::Created(TaskAcceptance {
task_id: request.task_id,
state: TaskState::Queued,
operation_version: request.pins.operation_version,
support: request.support.clone(),
created_at_unix_ms: created_ms,
retain_until_unix_ms: request.pins.retain_until_unix_ms,
replayed: false,
}))
}
/// Bounded scoped reread after a foreign unique conflict aborted the transaction.
async fn reread_after_unique(
&self,
request: &TaskAcceptRequest,
) -> Result<TaskAcceptance, TaskError> {
let row = sqlx::query(sqlx::AssertSqlSafe(format!(
"select {TASK_PROJECTION} from tasks
where workspace_id = $1 and owner_kind = $2 and owner_id = $3
and authorization_scope = $4 and operation_id = $5 and scoped_key = $6"
)))
.bind(&request.workspace_id)
.bind(request.owner_kind.as_str())
.bind(&request.owner_id)
.bind(&request.authorization_scope)
.bind(&request.operation_id)
.bind(&request.scoped_key)
.fetch_optional(&self.pool)
.await
.map_err(Self::unavailable)?;
let Some(row) = row else {
return Err(TaskError::unavailable(
"durable task acceptance ended with an unresolved unique conflict",
));
};
let record = record_from_row(&row)?;
if record.input_fingerprint != request.input_fingerprint {
return Err(TaskError::FingerprintConflict);
}
Ok(acceptance_from_record(&record))
}
/// A rejected conditional update is either a lost CAS race or a foreign/absent Task. Both are
/// reported without payload, and a foreign Task is indistinguishable from an absent one.
async fn classify_missed_finish(&self, request: &TaskFinishRequest) -> TaskError {
let access = &request.access;
match sqlx::query(
"select state, version from tasks
where task_id = $1 and workspace_id = $2 and owner_kind = $3
and owner_id = $4 and authorization_scope = $5",
)
.bind(request.task_id.as_uuid())
.bind(&access.workspace_id)
.bind(access.owner_kind.as_str())
.bind(&access.owner_id)
.bind(&access.authorization_scope)
.fetch_optional(&self.pool)
.await
{
Ok(Some(_)) => TaskError::StateConflict,
Ok(None) => TaskError::NotFound,
Err(error) => Self::unavailable(error),
}
async fn refuse_attempt(&self, lease: &TaskLease, code: &str) -> Result<TaskRecord, TaskError> {
self.refuse_attempt_resolution(lease, code).await
}
}
@@ -948,6 +856,15 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError>
.map_err(unavailable_row)?,
retain_until_unix_ms: row.try_get("retain_until_ms").map_err(unavailable_row)?,
approval_reference: row.try_get("approval_reference").map_err(unavailable_row)?,
retry: TaskRetryPolicy::new(
u32::try_from(
row.try_get::<i32, _>("max_attempts")
.map_err(unavailable_row)?,
)
.unwrap_or(1),
row.try_get::<i64, _>("retry_backoff_step_ms")
.map_err(unavailable_row)?,
),
},
support: TaskSupportIds {
request_id: crank_core::RequestId::parse(&request_id)
@@ -972,6 +889,11 @@ fn record_from_row(row: &sqlx::postgres::PgRow) -> Result<TaskRecord, TaskError>
dispatch_started_at_unix_ms: row
.try_get("dispatch_started_at_ms")
.map_err(unavailable_row)?,
next_attempt_at_unix_ms: row.try_get("next_attempt_at_ms").map_err(unavailable_row)?,
cancel_requested_at_unix_ms: row
.try_get("cancel_requested_at_ms")
.map_err(unavailable_row)?,
cancel_reason: row.try_get("cancel_reason").map_err(unavailable_row)?,
version: u32::try_from(row.try_get::<i32, _>("version").map_err(unavailable_row)?)
.unwrap_or_default(),
created_at_unix_ms: row.try_get("created_ms").map_err(unavailable_row)?,
@@ -76,6 +76,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
(26, "durable-tasks-v26"),
(27, "durable-tasks-v27"),
(28, "durable-tasks-v28"),
(29, "durable-tasks-v29"),
];
assert_eq!(rows.len(), expected.len());
for (row, (version, name)) in rows.iter().zip(expected) {
@@ -186,7 +187,7 @@ async fn controlled_authority_is_versioned_and_safe_under_concurrent_apply() {
);
assert_eq!(
MigrationAuthority::preflight(first.pool()).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
#[tokio::test]
@@ -234,7 +235,7 @@ async fn v18_guard_rejects_missing_agent_resource_binding_trigger() {
.expect("the V18-local guard must be restorable");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"restoring the exact V18 contract must return the current verdict"
);
}
@@ -360,7 +361,7 @@ async fn ledgerless_legacy_baseline_is_consolidated_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -473,13 +474,13 @@ async fn healthy_v2_is_reported_as_migration_required_and_upgrades_to_v3() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 2,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
let trace_column: bool = sqlx::query_scalar(
"select exists (
@@ -522,7 +523,7 @@ async fn healthy_v3_upgrades_to_v4_with_honest_legacy_snapshot_provenance() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 3,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -582,7 +583,7 @@ async fn healthy_v4_upgrades_to_v5_without_fabricating_legacy_outcomes() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 4,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -836,6 +837,6 @@ async fn v3_upgrade_ignores_oversized_legacy_request_ids_in_partial_index() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -32,7 +32,7 @@ async fn healthy_v11_upgrades_to_v13_without_rewriting_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 28,
target: 29,
}
);
@@ -177,7 +177,7 @@ async fn v12_guard_still_rejects_column_constraint_index_and_relation_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"restore: {restore}"
);
}
@@ -230,7 +230,7 @@ async fn v13_guard_rejects_wrong_cleanup_index_order_and_predicate() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"restore: {restore}"
);
}
@@ -507,7 +507,7 @@ async fn assert_v12_drift(pool: &sqlx::PgPool, drift: &str, restore: &str) {
.unwrap();
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"restore: {restore}"
);
}
@@ -845,7 +845,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
for (drift, restore) in [
@@ -879,7 +879,7 @@ async fn v17_guard_rejects_publication_schema_drift() {
sqlx::raw_sql(restore).execute(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"restore: {restore}"
);
}
@@ -289,7 +289,7 @@ async fn assert_primary_key_nullability(pool: &sqlx::PgPool, table: &str, column
);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"a rejected primary-key nullability change must leave the schema current"
);
@@ -473,7 +473,7 @@ async fn assert_definition_only_drift(
assert_eq!(super::catalog_constraint_definitions(pool).await, before);
assert_eq!(
MigrationAuthority::preflight(pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -554,7 +554,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"the baseline before the drift matrix must be the current V16 schema"
);
let mut observations: Vec<String> = Vec::new();
@@ -773,7 +773,7 @@ async fn story36_v16_guard_rejects_constraint_and_created_index_definition_drift
.await
.expect("restoring the exact definition must apply");
let restored = MigrationAuthority::preflight(&pool).await;
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 28 }));
let restored_current = matches!(restored, Ok(MigrationPreflight::Current { version: 29 }));
// Точное восстановление доказывается сравнением снятого снимка с baseline, а не
// мнением того же guard-а, который принимает неправильные определения (F4).
let restored_exact = catalog_definition_snapshot(&pool).await == before;
@@ -804,7 +804,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"baseline before inserting the shadow schema"
);
@@ -875,7 +875,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
// Тень не меняет вердикт guard-а по текущей схеме.
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"a same-named object in another schema must not change the verdict for the current schema"
);
@@ -947,7 +947,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 },
MigrationPreflight::Current { version: 29 },
"the exact restoration must return the current V16 schema"
);
@@ -958,7 +958,7 @@ async fn story36_v16_guard_resolves_dependent_source_fk_in_the_current_schema()
.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
assert!(
@@ -22,16 +22,16 @@ async fn clean_legacy_ledger_upgrades_to_v28_after_v25_removal() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 24,
target: 28,
target: 29,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 24, to: 28 }
MigrationApplyResult::Applied { from: 24, to: 29 }
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -71,7 +71,7 @@ async fn fresh_install_exposes_import_plan_objects_with_cascading_job_fk() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
let tables = sqlx::query(
"select table_name
@@ -224,7 +224,7 @@ async fn healthy_v13_upgrades_to_v14_without_changing_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 13,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -88,7 +88,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -101,7 +101,7 @@ async fn published_ledgerless_baseline_upgrades_without_data_loss() {
assert_eq!(operation_count, 1);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -194,7 +194,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -230,7 +230,7 @@ async fn published_ledgerless_historical_column_layout_upgrades_without_data_los
);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -262,7 +262,7 @@ async fn published_initial_mcp_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
@@ -280,7 +280,7 @@ async fn published_initial_mcp_layout_is_accepted() {
assert!(supports_elicitation);
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -314,13 +314,13 @@ async fn published_in_place_mcp_upgrade_layout_is_accepted() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 0,
target: 28,
target: 29,
}
);
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -14,7 +14,7 @@ async fn fresh_install_creates_the_empty_revocation_authority() {
MigrationAuthority::apply(&pool).await.unwrap();
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
let rows: i64 = sqlx::query_scalar("select count(*) from resource_security_revocations")
.fetch_one(&pool)
@@ -42,12 +42,12 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 28,
target: 29,
}
);
assert_eq!(
MigrationAuthority::apply(&pool).await.unwrap(),
MigrationApplyResult::Applied { from: 18, to: 28 }
MigrationApplyResult::Applied { from: 18, to: 29 }
);
let after: Vec<(i64, String, String)> =
sqlx::query_as("select version, name, checksum from __crank_migrations where version < 19 order by version")
@@ -57,7 +57,7 @@ async fn v18_upgrades_to_v19_without_rewriting_the_prior_ledger() {
assert_eq!(before, after, "V16–V18 ledger rows must not be rewritten");
assert_eq!(
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::Current { version: 28 }
MigrationPreflight::Current { version: 29 }
);
}
@@ -161,7 +161,7 @@ async fn failed_v19_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 18,
target: 28,
target: 29,
}
);
}
@@ -63,7 +63,7 @@ async fn failed_artifact_metadata_migration_rolls_back_schema_and_ledger() {
MigrationAuthority::preflight(&pool).await.unwrap(),
MigrationPreflight::MigrationRequired {
current: 11,
target: 28,
target: 29,
}
);
}
@@ -9,10 +9,11 @@ use super::common;
use crank_core::{
AdmissionMode, AdmissionOutcome, AdmissionRequest, AdmissionScope, AgentId, ApprovalEvidence,
ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId, PlatformApiKeyKind,
PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest, TaskAccess,
TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins, TaskState,
TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId, current_unix_ms,
BeginAttempt, ExecutionAdmission, NoTasksFallback, PlatformApiKey, PlatformApiKeyId,
PlatformApiKeyKind, PlatformApiKeyScope, PlatformApiKeyStatus, RequestId, TaskAcceptRequest,
TaskAccess, TaskEnvelope, TaskError, TaskFinishRequest, TaskId, TaskOwnerKind, TaskPins,
TaskState, TaskStore, TaskSupportIds, ToolExecutionMode, TraceContext, WorkspaceId,
current_unix_ms,
};
use crank_registry::{
AdmissionLimits, CreatePlatformApiKeyRequest, PostgresExecutionAdmission, PostgresTaskStore,
@@ -60,6 +61,7 @@ fn request(key: &str, fingerprint: &str, agent: &str, deadline_in_ms: i64) -> Ta
execution_deadline_unix_ms: deadline,
retain_until_unix_ms: deadline + 3_600_000,
approval_reference: Some("appr_1".to_owned()),
retry: crank_core::TaskRetryPolicy::single_attempt(),
},
scoped_key: key.to_owned(),
input_fingerprint: fingerprint.to_owned(),
@@ -708,28 +710,38 @@ async fn task_worker_lease_and_fencing_contract() {
// checkpoint dispatch once
let checkpointed = store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence,
"attempt-1".to_owned(),
)
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-1".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.unwrap();
assert_eq!(checkpointed.attempt.attempt_id, "attempt-1");
assert_eq!(checkpointed.attempt.ordinal, 1);
assert!(checkpointed.task.dispatch_started_at_unix_ms.is_some());
assert_eq!(
checkpointed.current_attempt_id.as_deref(),
checkpointed.task.current_attempt_id.as_deref(),
Some("attempt-1")
);
assert!(checkpointed.dispatch_started_at_unix_ms.is_some());
// second checkpoint rejected
let second = store
.checkpoint_dispatch(
"worker_1",
lease.record.task_id,
lease.fence,
"attempt-2".to_owned(),
)
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_1".to_owned(),
fence: lease.fence,
expected_version: checkpointed.task.version,
attempt_id: "attempt-2".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await;
assert!(second.is_err());
@@ -902,4 +914,5 @@ fn authority_key(
}
}
mod task_attempts;
mod task_worker;
@@ -0,0 +1,534 @@
use super::*;
use crank_core::{
AttemptOutcome, AttemptOutcomeClass, AttemptPhase, AttemptResolution, BeginAttempt,
TaskCancelOutcome, TaskCancelRequest, TaskRetryPolicy,
};
use serde_json::json;
#[tokio::test]
async fn cancel_before_checkpoint_wins_race_with_claim_and_never_dispatches() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(
&key("cancel_before_1"),
&digest("fb1"),
AGENT,
30_000,
))
.await
.unwrap();
let outcome = store
.cancel(TaskCancelRequest {
access: access(AGENT),
task_id: created.task_id,
expected_version: None,
reason: "cancel before dispatch".to_owned(),
})
.await
.unwrap();
match outcome {
TaskCancelOutcome::Cancelled(record) => assert_eq!(record.state, TaskState::Cancelled),
other => panic!("expected Cancelled, got {other:?}"),
}
let claims = store.claim_batch("worker", 4, 10_000).await.unwrap();
assert!(claims.is_empty(), "cancelled task must not be claimed");
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert!(attempts.is_empty(), "no attempts for cancelled task");
let intents: i64 = sqlx::query_scalar(
"select count(*)::bigint from execution_admission_entries where task_id = $1",
)
.bind(created.task_id.as_uuid())
.fetch_one(registry.pool())
.await
.unwrap();
assert_eq!(intents, 0, "dispatch intent must be removed");
database.cleanup().await;
}
#[tokio::test]
async fn cancel_after_checkpoint_is_uncertain_and_never_cancelled_or_retried() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(
&key("cancel_after_1"),
&digest("fa1"),
AGENT,
30_000,
))
.await
.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
let leased = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-cancel-after".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("checkpoint");
let outcome = store
.cancel(TaskCancelRequest {
access: access(AGENT),
task_id: created.task_id,
expected_version: None,
reason: "cancel after checkpoint".to_owned(),
})
.await
.unwrap();
match outcome {
TaskCancelOutcome::OutcomeUnknown(record) => {
assert_eq!(record.state, TaskState::OutcomeUnknown)
}
other => panic!("expected OutcomeUnknown, got {other:?}"),
}
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::OutcomeUnknown);
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert_eq!(attempts.len(), 1);
assert_eq!(
attempts[0].outcome,
Some(AttemptOutcomeClass::OutcomeUnknown)
);
// Repeated cancel stays uncertain.
let outcome2 = store
.cancel(TaskCancelRequest {
access: access(AGENT),
task_id: created.task_id,
expected_version: None,
reason: "cancel after checkpoint".to_owned(),
})
.await
.unwrap();
assert!(matches!(outcome2, TaskCancelOutcome::OutcomeUnknown(_)));
// finish_leased by the original lease must be rejected.
let finish = store
.finish_leased(
TaskFinishRequest {
access: access(AGENT),
task_id: created.task_id,
expected_state: leased.task.state,
expected_version: leased.task.version,
target_state: TaskState::Completed,
result: Some(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
error: None,
},
"worker",
leased.attempt.fence,
)
.await;
assert!(matches!(finish, Err(TaskError::StateConflict)));
let claims2 = store.claim_batch("worker2", 4, 10_000).await.unwrap();
assert!(
!claims2.iter().any(|l| l.record.task_id == created.task_id),
"outcome unknown task must never be claimed"
);
database.cleanup().await;
}
#[tokio::test]
async fn checkpoint_and_resolve_are_cas_guarded_and_history_is_immutable() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("cas_1"), &digest("fc1"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
let first_version = lease.record.version;
let committed = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-cas".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("first checkpoint");
let committed_lease = committed;
// Repeated checkpoint with the same old version -> StateConflict.
let second = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: first_version,
attempt_id: "attempt-cas-2".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
assert!(matches!(second, Err(TaskError::StateConflict)));
// resolve with a modified lease task version -> StateConflict.
let mut bad_lease = committed_lease.clone();
bad_lease.task.version = first_version;
let bad_resolve = store
.resolve_attempt(
&bad_lease,
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
)
.await;
assert!(matches!(bad_resolve, Err(TaskError::StateConflict)));
// Correct resolve -> Finished(Completed).
let ok = store
.resolve_attempt(
&committed_lease,
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
)
.await
.expect("resolve");
match ok {
AttemptResolution::Finished(rec) => assert_eq!(rec.state, TaskState::Completed),
other => panic!("expected Finished, got {other:?}"),
}
// Repeated resolve -> StateConflict.
let repeat = store
.resolve_attempt(
&committed_lease,
AttemptOutcome::Completed(TaskEnvelope::try_redacted(&json!({"ok": true})).unwrap()),
)
.await;
assert!(matches!(repeat, Err(TaskError::StateConflict)));
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert_eq!(attempts.len(), 1, "exactly one immutable attempt record");
assert_eq!(attempts[0].outcome, Some(AttemptOutcomeClass::Completed));
let _ = created;
database.cleanup().await;
}
#[tokio::test]
async fn expired_checkpointed_attempt_recovers_only_with_safe_retry_evidence() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
// Task A: retry policy allows 2 attempts, retry_allowed=true.
let mut req_a = request(&key("rec_a"), &digest("fa"), AGENT, 30_000);
req_a.pins.retry = TaskRetryPolicy::new(2, 0);
let created_a = store.accept(req_a).await.unwrap();
// Task B: single attempt, retry_allowed=false.
let created_b = store
.accept(request(&key("rec_b"), &digest("fb"), AGENT, 30_000))
.await
.unwrap();
let claims = store.claim_batch("worker_dead", 2, 150).await.unwrap();
assert_eq!(claims.len(), 2);
let lease_a = claims
.iter()
.find(|l| l.record.task_id == created_a.task_id)
.cloned()
.expect("claim A");
let lease_b = claims
.iter()
.find(|l| l.record.task_id == created_b.task_id)
.cloned()
.expect("claim B");
store
.checkpoint_dispatch(BeginAttempt {
task_id: lease_b.record.task_id,
owner: "worker_dead".to_owned(),
fence: lease_b.fence,
expected_version: lease_b.record.version,
attempt_id: "attempt-b".to_owned(),
input_fingerprint: lease_b.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("checkpoint B");
store
.checkpoint_dispatch(BeginAttempt {
task_id: lease_a.record.task_id,
owner: "worker_dead".to_owned(),
fence: lease_a.fence,
expected_version: lease_a.record.version,
attempt_id: "attempt-a".to_owned(),
input_fingerprint: lease_a.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("checkpoint A");
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let recovered = store.claim_batch("worker_live", 4, 10_000).await.unwrap();
let lease_a2 = recovered
.iter()
.find(|l| l.record.task_id == created_a.task_id)
.cloned()
.expect("A recovered");
assert!(lease_a2.fence > lease_a.fence, "recovery bumps fence");
assert!(
matches!(
lease_a2.record.state,
TaskState::Working | TaskState::Queued
),
"A state working/queued"
);
assert!(
!recovered
.iter()
.any(|l| l.record.task_id == created_b.task_id),
"B must not be redispatched"
);
let record_b = store.read(&access(AGENT), created_b.task_id).await.unwrap();
assert_eq!(record_b.state, TaskState::OutcomeUnknown);
// A prior attempt resolved as uncertain.
let attempts_a = store
.attempts(&access(AGENT), created_a.task_id)
.await
.unwrap();
assert!(!attempts_a.is_empty());
assert_eq!(
attempts_a[0].outcome,
Some(AttemptOutcomeClass::OutcomeUnknown)
);
// Further scans do not hand B back.
assert!(
store
.claim_batch("worker_live", 4, 10_000)
.await
.unwrap()
.iter()
.all(|l| l.record.task_id != created_b.task_id)
);
database.cleanup().await;
}
#[tokio::test]
async fn before_dispatch_failed_attempt_does_not_erase_recorded_uncertainty() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let mut req = request(&key("bd_1"), &digest("fbd"), AGENT, 30_000);
req.pins.retry = TaskRetryPolicy::new(2, 0);
let created = store.accept(req).await.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease_a = leases.into_iter().next().expect("claim A");
let committed_a = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease_a.record.task_id,
owner: "worker".to_owned(),
fence: lease_a.fence,
expected_version: lease_a.record.version,
attempt_id: "attempt-a".to_owned(),
input_fingerprint: lease_a.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: true,
})
.await
.expect("checkpoint A");
// Attempt A resolves to a bounded retry (uncertain outcome).
let retry = store
.resolve_attempt(
&committed_a,
AttemptOutcome::Uncertain {
error: TaskEnvelope::try_redacted(
&json!({"code":"outcome_unknown","phase":"dispatched"}),
)
.unwrap(),
repeatable: true,
phase: AttemptPhase::Dispatched,
},
)
.await
.expect("resolve A");
let retry_task = match retry {
AttemptResolution::RetryAt { task, .. } => task,
other => panic!("expected RetryAt, got {other:?}"),
};
assert_eq!(retry_task.state, TaskState::Working);
assert!(retry_task.current_attempt_id.is_none());
let retry_version = retry_task.version;
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
// The Task keeps its lease across the pinned backoff: the next attempt checkpoints under the
// same owner/fence and the version the RetryAt record reported.
let committed_b_result = store
.checkpoint_dispatch(BeginAttempt {
task_id: retry_task.task_id,
owner: "worker".to_owned(),
fence: lease_a.fence,
expected_version: retry_version,
attempt_id: "attempt-b".to_owned(),
input_fingerprint: retry_task.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await;
let committed_b = committed_b_result.expect("checkpoint B");
// Attempt B never reached upstream: a before_dispatch failure must NOT overwrite
// the recorded uncertainty of the previously dispatched attempt.
let resolution = store
.resolve_attempt(
&committed_b,
AttemptOutcome::Failed {
error: TaskEnvelope::try_redacted(
&json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}),
)
.unwrap(),
retryable: false,
phase: AttemptPhase::BeforeDispatch,
},
)
.await
.expect("resolve B");
match resolution {
AttemptResolution::Unknown(record) => assert_eq!(record.state, TaskState::OutcomeUnknown),
other => panic!("expected Unknown, got {other:?}"),
}
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::OutcomeUnknown);
assert!(record.result.is_none());
let error = record.error.expect("error envelope present");
assert_eq!(error.value()["code"], json!("shutdown_before_dispatch"));
assert_eq!(error.value()["phase"], json!("dispatched"));
assert_eq!(record.attempts, 2);
let attempts = store
.attempts(&access(AGENT), created.task_id)
.await
.unwrap();
assert_eq!(attempts.len(), 2);
assert_eq!(
attempts[0].outcome,
Some(AttemptOutcomeClass::RetryScheduled)
);
assert_eq!(
attempts[0].certainty,
Some(crank_core::AttemptCertainty::Unknown)
);
assert_eq!(attempts[0].phase, Some(AttemptPhase::Dispatched));
assert_eq!(attempts[1].outcome, Some(AttemptOutcomeClass::Failed));
assert_eq!(
attempts[1].certainty,
Some(crank_core::AttemptCertainty::Known)
);
assert_eq!(attempts[1].phase, Some(AttemptPhase::BeforeDispatch));
// Repeated read/cancel must not turn the state terminal.
let again = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(again.state, TaskState::OutcomeUnknown);
database.cleanup().await;
}
#[tokio::test]
async fn before_dispatch_failed_single_attempt_stays_failed() {
let database = common::TestDatabase::new().await;
let registry = database.registry().await;
let store = store_with(registry.pool().clone(), 8);
let created = store
.accept(request(&key("bd_single"), &digest("fbf"), AGENT, 30_000))
.await
.unwrap();
let leases = store.claim_batch("worker", 1, 10_000).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
let committed = store
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-single".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("checkpoint");
let resolution = store
.resolve_attempt(
&committed,
AttemptOutcome::Failed {
error: TaskEnvelope::try_redacted(
&json!({"code":"shutdown_before_dispatch","phase":"before_dispatch"}),
)
.unwrap(),
retryable: false,
phase: AttemptPhase::BeforeDispatch,
},
)
.await
.expect("resolve");
match resolution {
AttemptResolution::Finished(record) => assert_eq!(record.state, TaskState::Failed),
other => panic!("expected Finished, got {other:?}"),
}
let record = store.read(&access(AGENT), created.task_id).await.unwrap();
assert_eq!(record.state, TaskState::Failed);
let error = record.error.expect("error envelope present");
assert_eq!(error.value()["phase"], json!("before_dispatch"));
database.cleanup().await;
}
@@ -87,12 +87,16 @@ async fn task_worker_expired_checkpointed_attempt_becomes_outcome_unknown_withou
let leases = store.claim_batch("worker_dead", 1, 150).await.unwrap();
let lease = leases.into_iter().next().expect("claim");
store
.checkpoint_dispatch(
"worker_dead",
lease.record.task_id,
lease.fence,
"attempt-uncertain".to_owned(),
)
.checkpoint_dispatch(BeginAttempt {
task_id: lease.record.task_id,
owner: "worker_dead".to_owned(),
fence: lease.fence,
expected_version: lease.record.version,
attempt_id: "attempt-uncertain".to_owned(),
input_fingerprint: lease.record.input_fingerprint.clone(),
upstream_key_digest: None,
retry_allowed: false,
})
.await
.expect("durable dispatch checkpoint");
+321 -51
View File
@@ -24,9 +24,13 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tracing::{Instrument, Span, debug, warn};
use uuid::Uuid;
use crate::task_attempt::{
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal,
};
use crate::task_worker::TaskDispatchPermit;
use crate::{
AdapterResponse, ExecutionAuthorization, PreparedRequest, RuntimeError,
AdapterResponse, ExecutionAuthorization, ExecutionRequestError, PreparedRequest, RuntimeError,
RuntimeExecutionRequest, RuntimeLimits, RuntimeOperation, RuntimeRequestContext,
auth::apply_resolved_auth,
request_preparation::{adapter_prepared_request, validate_selected_body_variant},
@@ -36,6 +40,18 @@ use crate::{
},
};
/// How many outbound sends one prepared request may perform.
///
/// `Legacy` keeps the existing synchronous `tools/call` retry behaviour inside the transport loop.
/// `Single` performs exactly one send, because a durable Task attempt already has its own committed
/// checkpoint and resolution around every send; it carries the pinned attempt ordinal and budget so
/// the one `adapter_attempt` span still names the real durable attempt.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum SendMode {
Legacy,
Single { attempt: u32, max_attempts: u32 },
}
#[derive(Clone)]
pub struct RuntimeExecutor {
adapters: AdapterRegistry,
@@ -181,57 +197,273 @@ impl RuntimeExecutor {
&self,
request: RuntimeExecutionRequest<'_>,
) -> Result<(Value, Value), RuntimeError> {
self.execute_request_impl(request, None).await
self.execute_request_impl(request, None, SendMode::Legacy)
.await
}
/// Drives the bounded, durable retry budget of one leased background Task (Story 4.4).
///
/// Each outbound send is preceded by its own committed attempt checkpoint and followed by one
/// atomic resolution; the pinned budget, the safe-retry decision and the accepted absolute
/// deadline are enforced by PostgreSQL, so a restart or a second worker cannot extend them.
pub async fn execute_task_outcome(
&self,
request: RuntimeExecutionRequest<'_>,
permit: TaskDispatchPermit,
) -> Result<crank_core::ExecutionSuccess, crank_core::ExecutionFailure> {
let operation = request.operation().clone();
let operation_id = operation.operation_id.clone();
let operation_version = operation.operation_version;
let origin = request.origin();
let correlation = crank_core::CorrelationContext::new(
request.request_context().request_id.clone(),
request.request_context().trace_context.clone(),
context: &TaskAttemptContext,
authority: &dyn TaskAttemptAuthority,
) -> TaskAttemptOutcome {
let store = context.store().clone();
let lease = context.lease().clone();
let mut record = lease.record.clone();
loop {
if context.is_cancelled() || context.is_stopping() {
return TaskAttemptOutcome::Stopped;
}
let current_lease = crank_core::TaskLease {
record: record.clone(),
owner: lease.owner.clone(),
fence: lease.fence,
lease_until: lease.lease_until,
};
let plan = match authority.prepare_attempt(&current_lease).await {
Ok(plan) => plan,
Err(TaskAttemptRefusal::Stopped) => return TaskAttemptOutcome::Stopped,
Err(TaskAttemptRefusal::Failed { code }) => {
// A refused attempt never erases a proven ambiguity: if an earlier attempt of this
// Task may already have reached the upstream, the store keeps the Task in the
// non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
return match store.refuse_attempt(&current_lease, &code).await {
Ok(record) if record.state == crank_core::TaskState::OutcomeUnknown => {
TaskAttemptOutcome::OutcomeUnknown
}
_ => TaskAttemptOutcome::Failed { code },
};
}
};
let attempt_id = format!("attempt-{}", Uuid::now_v7());
let begin = crank_core::BeginAttempt {
task_id: record.task_id,
owner: lease.owner.clone(),
fence: lease.fence,
expected_version: record.version,
attempt_id: attempt_id.clone(),
input_fingerprint: record.input_fingerprint.clone(),
upstream_key_digest: plan.upstream_key_digest.clone(),
retry_allowed: plan.retry_allowed,
};
let attempt = match store.checkpoint_dispatch(begin).await {
Ok(attempt) => attempt,
// Cancellation, lease loss or a concurrent lifecycle change: nothing was dispatched.
Err(crank_core::TaskError::StateConflict)
| Err(crank_core::TaskError::NotFound)
| Err(crank_core::TaskError::FingerprintConflict) => {
return TaskAttemptOutcome::Stopped;
}
// An unproven commit is never retried by guess: no outbound call without a proven
// durable checkpoint.
Err(error) => {
warn!(error = %error, "durable dispatch checkpoint failed; no outbound call");
return TaskAttemptOutcome::Stopped;
}
};
if context.is_stopping() {
let code = "shutdown_before_dispatch".to_owned();
// A stop that lands before the send is a proven failure of *this* attempt only: when
// an earlier attempt of the same Task already lost its confirmation, the store keeps
// the non-terminal `outcome_unknown` instead of downgrading it to a plain failure.
return match store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Failed {
error: bounded_attempt_error(
&code,
crank_core::AttemptPhase::BeforeDispatch,
),
retryable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await
{
Ok(crank_core::AttemptResolution::Unknown(_)) => {
TaskAttemptOutcome::OutcomeUnknown
}
_ => TaskAttemptOutcome::Failed { code },
};
}
if context.is_cancelled() {
let _ = store
.resolve_attempt(
&attempt,
crank_core::AttemptOutcome::Uncertain {
error: bounded_attempt_error(
"outcome_unknown",
crank_core::AttemptPhase::BeforeDispatch,
),
repeatable: false,
phase: crank_core::AttemptPhase::BeforeDispatch,
},
)
.await;
return TaskAttemptOutcome::OutcomeUnknown;
}
// Each attempt owns exactly one `adapter_attempt` span, created around the single send
// inside the transport loop so the accepted trace shape (durable attempt → runtime
// execution → adapter attempt) is preserved.
let result = self.execute_task_send(&plan, &attempt).await;
let (outcome, code) = self.classify_task_attempt(&plan, result);
match store.resolve_attempt(&attempt, outcome).await {
Ok(crank_core::AttemptResolution::Finished(task)) => {
return if task.state == crank_core::TaskState::Completed {
TaskAttemptOutcome::Completed
} else {
TaskAttemptOutcome::Failed {
code: code.unwrap_or_else(|| "task_failed".to_owned()),
}
};
}
Ok(crank_core::AttemptResolution::Unknown(_)) => {
return TaskAttemptOutcome::OutcomeUnknown;
}
Ok(crank_core::AttemptResolution::RetryAt {
task,
next_attempt_at_unix_ms,
}) => {
record = task;
let remaining =
next_attempt_at_unix_ms.saturating_sub(crank_core::current_unix_ms());
let delay = Duration::from_millis(u64::try_from(remaining).unwrap_or(0));
if !self.wait_before_attempt(context, delay).await {
return TaskAttemptOutcome::Stopped;
}
}
// A stale fence or version lost the resolution CAS: another writer owns the Task now.
Err(_) => return TaskAttemptOutcome::Stopped,
}
}
}
/// One outbound send of one committed durable attempt.
async fn execute_task_send(
&self,
plan: &TaskAttemptPlan,
attempt: &crank_core::AttemptLease,
) -> Result<Value, RuntimeError> {
let permit = TaskDispatchPermit::new(
attempt.attempt.task_id,
plan.operation.operation_id.as_str().to_owned(),
plan.operation.operation_version,
plan.workspace_id.as_str().to_owned(),
plan.agent_id.as_ref().map(|id| id.as_str().to_owned()),
attempt.attempt.owner.clone(),
attempt.attempt.fence,
attempt.attempt.attempt_id.clone(),
);
let request = match RuntimeExecutionRequest::try_new(
&plan.workspace_id,
plan.origin,
plan.agent_id.as_ref(),
&plan.operation,
&plan.input,
plan.authorization,
plan.resolved_auth.as_ref(),
&plan.request_context,
plan.deadline,
) {
Ok(request) => request,
Err(ExecutionRequestError::DeadlineElapsed) => {
return Err(RuntimeError::ExecutionDeadlineElapsed {
may_have_dispatched: false,
});
}
Err(_) => return Err(RuntimeError::InvalidExecutionRequest),
};
let (output, _preview) = self
.execute_request_impl(
request,
Some(permit),
SendMode::Single {
attempt: attempt.attempt.ordinal,
max_attempts: attempt.task.pins.retry.max_attempts,
},
)
.await?;
Ok(output)
}
// Validate permit binding before any adapter dispatch.
if permit.operation_id() != operation_id.as_str()
|| permit.operation_version() != operation_version
|| permit.workspace_id() != request.workspace_id().as_str()
|| permit.agent_id() != request.agent_id().map(|value| value.as_str())
{
return Err(crate::normalize_runtime_error_for_operation(
&RuntimeError::InvalidExecutionRequest,
&correlation,
&operation,
));
}
if !permit.consume() {
return Err(crate::normalize_runtime_error_for_operation(
&RuntimeError::InvalidExecutionRequest,
&correlation,
&operation,
));
/// Maps one outbound result of a durable attempt onto its persisted attempt outcome.
fn classify_task_attempt(
&self,
plan: &TaskAttemptPlan,
result: Result<Value, RuntimeError>,
) -> (crank_core::AttemptOutcome, Option<String>) {
match result {
Ok(output) => match crank_core::TaskEnvelope::try_redacted(&output) {
Ok(envelope) => (crank_core::AttemptOutcome::Completed(envelope), None),
Err(_) => {
let code = "result_not_redactable".to_owned();
(
crank_core::AttemptOutcome::Failed {
error: bounded_attempt_error(
&code,
crank_core::AttemptPhase::Dispatched,
),
retryable: false,
phase: crank_core::AttemptPhase::Dispatched,
},
Some(code),
)
}
},
Err(error) => {
let phase = if runtime_error_may_have_dispatched(&error) {
crank_core::AttemptPhase::Dispatched
} else {
crank_core::AttemptPhase::BeforeDispatch
};
let correlation = crank_core::CorrelationContext::new(
plan.request_context.request_id.clone(),
plan.request_context.trace_context.clone(),
);
let failure = crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&plan.operation,
);
let code = format!("{:?}", failure.error_code());
let safe_to_repeat = retry_is_allowed(&plan.operation, &error, plan.retry_allowed);
let envelope = bounded_attempt_error(&code, phase);
let outcome = match failure.outcome_certainty() {
crank_core::OutcomeCertainty::OutcomeUnknown => {
crank_core::AttemptOutcome::Uncertain {
error: envelope,
repeatable: safe_to_repeat,
phase,
}
}
_ => crank_core::AttemptOutcome::Failed {
error: envelope,
retryable: safe_to_repeat,
phase,
},
};
(outcome, Some(code))
}
}
}
match self.execute_request_impl(request, Some(permit)).await {
Ok((output, request_preview)) => Ok(crank_core::ExecutionSuccess {
operation_id: operation_id.clone(),
operation_version,
origin,
correlation,
request_preview,
output,
}),
Err(error) => Err(crate::normalize_runtime_error_for_operation(
&error,
&correlation,
&operation,
)),
/// Bounded cooperative wait inside the pinned backoff window. The lease keeps being renewed by
/// the worker, so a durable cancellation or lease loss still stops the local work here.
async fn wait_before_attempt(&self, context: &TaskAttemptContext, delay: Duration) -> bool {
let deadline = Instant::now() + delay;
loop {
if context.is_cancelled() || context.is_stopping() {
return false;
}
let now = Instant::now();
if now >= deadline {
return true;
}
tokio::time::sleep((deadline - now).min(Duration::from_millis(50))).await;
}
}
@@ -239,6 +471,7 @@ impl RuntimeExecutor {
&self,
request: RuntimeExecutionRequest<'_>,
permit: Option<TaskDispatchPermit>,
send_mode: SendMode,
) -> Result<(Value, Value), RuntimeError> {
let operation = request.operation();
// Derive trusted dispatch from a present permit; a missing permit always uses ordinary admission.
@@ -320,6 +553,7 @@ impl RuntimeExecutor {
Some(request_context),
Arc::clone(&dispatch_started),
durable_attempt_id.as_deref(),
send_mode,
)
.await;
if let Some(lease) = lease.as_mut() {
@@ -415,6 +649,7 @@ impl RuntimeExecutor {
Ok(prepared)
}
#[allow(clippy::too_many_arguments)]
async fn execute_prepared(
&self,
operation: &RuntimeOperation,
@@ -423,6 +658,7 @@ impl RuntimeExecutor {
request_context: Option<&RuntimeRequestContext>,
dispatch_started: Arc<AtomicBool>,
durable_attempt_id: Option<&str>,
send_mode: SendMode,
) -> Result<Value, RuntimeError> {
let mut prepared_request = prepared_request;
let idempotency_applicable = crate::idempotency::policy(operation).is_some();
@@ -472,7 +708,11 @@ impl RuntimeExecutor {
approval_result?;
}
let idempotency = if idempotency_applicable {
// The synchronous coordination guard is the replay authority of one `tools/call`. A durable
// Task attempt already has a PostgreSQL attempt authority around every send, so the guard is
// skipped there: an ambiguous first response must not block the bounded durable retry. The
// stable idempotency key is still delivered to the upstream by `prepare_idempotency` above.
let idempotency = if idempotency_applicable && matches!(send_mode, SendMode::Legacy) {
let idempotency_span = Stage::RuntimeIdempotency.span();
let result = crate::idempotency::begin(
self.coordination_store.as_deref(),
@@ -540,6 +780,7 @@ impl RuntimeExecutor {
idempotency_key.as_deref(),
),
durable_attempt_id,
send_mode,
)
.await;
if let Ok(response) = &adapter_response {
@@ -673,6 +914,7 @@ impl RuntimeExecutor {
.map_err(|error| map_protocol_adapter_error(operation, error))
}
#[allow(clippy::too_many_arguments)]
async fn execute_adapter_with_retry(
&self,
operation: &RuntimeOperation,
@@ -681,20 +923,32 @@ impl RuntimeExecutor {
dispatch_started: Arc<AtomicBool>,
verified_idempotency: bool,
durable_attempt_id: Option<&str>,
send_mode: SendMode,
) -> Result<AdapterResponse, RuntimeError> {
let max_attempts = retry_attempts(operation);
let mut attempt = 1;
// `max_attempts` bounds the transport loop; `reported_max_attempts` is the pinned Task budget
// the attempt span names. A durable attempt performs exactly one send, so its transport loop
// always has a single iteration even when the Task may retry later.
let (max_attempts, first_attempt, reported_max_attempts) = match send_mode {
SendMode::Single {
attempt,
max_attempts,
} => (1, attempt, max_attempts),
SendMode::Legacy => {
let attempts = retry_attempts(operation);
(attempts, 1, attempts)
}
};
let mut attempt = first_attempt;
loop {
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint; the
// retry index discriminates the attempts inside it. Only paths without a durable attempt
// (synchronous tools/call) mint a synthetic per-attempt id.
// Task dispatches reuse the durable Attempt ID committed by the dispatch checkpoint. Only
// paths without a durable attempt (synchronous tools/call) mint a synthetic identity.
let attempt_id = durable_attempt_id
.map(str::to_owned)
.unwrap_or_else(|| format!("attempt-{}", Uuid::now_v7()));
let attempt_span = tracing::info_span!(
"adapter_attempt",
attempt,
max_attempts,
max_attempts = reported_max_attempts,
attempt_id = %attempt_id
);
let result = self
@@ -846,6 +1100,22 @@ fn sanitized_request_preview(prepared: &PreparedRequest) -> Value {
})
}
/// Bounded persisted error envelope of one durable attempt: a closed code and the attempt phase.
/// It never carries an upstream body, status text, URL or credential.
fn bounded_attempt_error(code: &str, phase: crank_core::AttemptPhase) -> crank_core::TaskEnvelope {
crank_core::TaskEnvelope::try_redacted(&json!({
"code": code,
"phase": phase.as_str(),
}))
.unwrap_or_else(|_| {
crank_core::TaskEnvelope::try_redacted(&json!({
"code": "task_error_not_recordable",
"phase": phase.as_str(),
}))
.expect("bounded attempt error envelope is always valid")
})
}
fn runtime_error_may_have_dispatched(error: &RuntimeError) -> bool {
matches!(
error,
+6 -2
View File
@@ -15,6 +15,7 @@ mod rate_limit;
mod request_context;
mod request_preparation;
mod response_cache;
mod task_attempt;
mod task_worker;
mod tasks;
pub fn validate_resource_template_parameter(
@@ -116,8 +117,11 @@ pub use rate_limit::{
};
pub use request_context::{MeteringContext, ResponseCacheScope, RuntimeRequestContext};
pub use secret_crypto::SecretCrypto;
pub use task_attempt::{
TaskAttemptAuthority, TaskAttemptContext, TaskAttemptOutcome, TaskAttemptPlan,
TaskAttemptRefusal,
};
pub use task_worker::{
TaskAttemptOutcome, TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig,
TaskWorkerError,
TaskDispatchPermit, TaskHandler, TaskWorker, TaskWorkerConfig, TaskWorkerError,
};
pub use tasks::{TaskAcceptDraft, TaskOutcome, TaskService};
+113
View File
@@ -0,0 +1,113 @@
//! Durable per-outbound attempt execution of one background Task (Story 4.4).
//!
//! The authoritative state of a Task stays in PostgreSQL. This module only carries the owned
//! execution plan of one attempt, the durable control (store + lease + cooperative stop) and the
//! bounded seam the application layer implements to revalidate authority before each attempt.
//! The single retry loop lives in [`crate::RuntimeExecutor::execute_task_outcome`]; there is no second
//! loop in the worker and no retry inside the transport adapter.
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::Instant;
use async_trait::async_trait;
use crank_core::{AgentId, ExecutionOrigin, TaskLease, TaskStore, WorkspaceId};
use crate::{ExecutionAuthorization, ResolvedAuth, RuntimeOperation, RuntimeRequestContext};
/// Owned execution plan of one durable attempt.
///
/// It is rebuilt by the application through [`TaskAttemptAuthority::prepare_attempt`] before every
/// attempt, so a republished catalog, a revoked key or an elapsed deadline is revalidated between two
/// attempts instead of being trusted from the first one.
pub struct TaskAttemptPlan {
pub workspace_id: WorkspaceId,
pub origin: ExecutionOrigin,
pub agent_id: Option<AgentId>,
pub operation: RuntimeOperation,
pub input: serde_json::Value,
pub authorization: ExecutionAuthorization,
pub resolved_auth: Option<ResolvedAuth>,
pub request_context: RuntimeRequestContext,
/// Absolute deadline pinned at acceptance; never re-derived from a later publication.
pub deadline: Instant,
/// Safe-to-retry decision of this attempt: a read, or a write whose upstream idempotency is
/// backed by server-owned evidence. A client-supplied key or header never makes this true.
pub retry_allowed: bool,
/// Digest of the stable upstream idempotency key (never the raw key or a header value).
pub upstream_key_digest: Option<String>,
}
/// Application-owned seam: revalidate authority and prepare exactly one durable attempt.
#[async_trait]
pub trait TaskAttemptAuthority: Send + Sync {
async fn prepare_attempt(
&self,
lease: &TaskLease,
) -> Result<TaskAttemptPlan, TaskAttemptRefusal>;
}
/// Typed refusal of one attempt preparation.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum TaskAttemptRefusal {
/// Proven refusal before any dispatch of this attempt; the bounded code is persisted.
Failed { code: String },
/// Durable control changed (cancellation, lease loss, shutdown): write nothing new from here.
Stopped,
}
/// Durable control of one leased Task: the store, the leased record and cooperative stop signals.
#[derive(Clone)]
pub struct TaskAttemptContext {
pub(crate) store: Arc<dyn TaskStore>,
pub(crate) lease: TaskLease,
pub(crate) cancelled: Arc<AtomicBool>,
pub(crate) stopping: Arc<AtomicBool>,
}
impl TaskAttemptContext {
pub(crate) fn new(
store: Arc<dyn TaskStore>,
lease: TaskLease,
cancelled: Arc<AtomicBool>,
stopping: Arc<AtomicBool>,
) -> Self {
Self {
store,
lease,
cancelled,
stopping,
}
}
pub fn store(&self) -> &Arc<dyn TaskStore> {
&self.store
}
pub fn lease(&self) -> &TaskLease {
&self.lease
}
/// Cooperative stop: the lease renewal or an observed durable cancellation set this flag.
pub fn is_cancelled(&self) -> bool {
self.cancelled.load(std::sync::atomic::Ordering::Acquire)
}
/// Process shutdown was requested: no new outbound send may start after the current checkpoint.
pub fn is_stopping(&self) -> bool {
self.stopping.load(std::sync::atomic::Ordering::Acquire)
}
}
/// Result of driving the whole pinned retry budget of one leased Task.
#[derive(Clone, Debug, PartialEq)]
pub enum TaskAttemptOutcome {
/// The Task committed `completed` (exactly one terminal write).
Completed,
/// The Task committed `failed` without dispatch or with a proven failure.
Failed { code: String },
/// The Task is non-terminal `outcome_unknown`; it is never retried or cancelled generically.
OutcomeUnknown,
/// Nothing new was written by this worker: cancellation, lease loss or an unavailable store.
Stopped,
}
+39 -194
View File
@@ -11,12 +11,14 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use crank_core::{TaskEnvelope, TaskError, TaskFinishRequest, TaskLease, TaskState, TaskStore};
use crank_core::{TaskError, TaskLease, TaskStore};
use tokio::sync::watch;
use tokio::task::JoinSet;
use tracing::{Instrument, debug, warn};
use uuid::Uuid;
use crate::task_attempt::{TaskAttemptContext, TaskAttemptOutcome};
/// Configuration for [`TaskWorker`]. Durations are capped by the caller to story timing budgets.
#[derive(Clone, Debug)]
pub struct TaskWorkerConfig {
@@ -50,29 +52,16 @@ impl Default for TaskWorkerConfig {
}
}
/// Result of one durable Task attempt produced by the application layer.
#[derive(Clone, Debug)]
pub enum TaskAttemptOutcome {
Completed(serde_json::Value),
Failed(serde_json::Value),
OutcomeUnknown,
}
/// Application-owned seam for pre-dispatch revalidation and execution.
///
/// The worker owns lease renewal/CAS/fencing and the dispatch checkpoint. The handler only
/// receives the leased record and attempt identity; it must not trust `TaskRecord.input` as
/// The worker owns lease renewal/CAS/fencing and passes a durable [`TaskAttemptContext`]. The
/// application implements [`crate::TaskAttemptAuthority`] and drives the pinned retry budget through
/// [`crate::RuntimeExecutor::execute_task_outcome`]; it must not trust `TaskRecord.input` as
/// executable payload.
#[async_trait]
pub trait TaskHandler: Send + Sync {
async fn revalidate(&self, lease: &TaskLease) -> Result<(), TaskWorkerError>;
async fn execute(
&self,
lease: &TaskLease,
attempt_id: &str,
cancelled: Arc<AtomicBool>,
permit: TaskDispatchPermit,
) -> TaskAttemptOutcome;
async fn execute(&self, context: TaskAttemptContext) -> TaskAttemptOutcome;
}
/// Sealed, single-use trusted dispatch permit created by [`TaskWorker`] after a durable lease and
@@ -91,6 +80,7 @@ pub struct TaskDispatchPermit {
}
impl TaskDispatchPermit {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
task_id: crank_core::TaskId,
operation_id: String,
@@ -244,10 +234,10 @@ impl TaskWorker {
if self.stopping.load(Ordering::Acquire) {
break;
}
if let Some(rx) = shutdown.as_mut() {
if *rx.borrow() {
break;
}
if let Some(rx) = shutdown.as_mut()
&& *rx.borrow()
{
break;
}
// Reap finished attempts before claiming more.
@@ -344,15 +334,7 @@ async fn process_lease(
return;
}
debug!(task_id = %task_id, error = %error, "task revalidation failed; finishing without dispatch");
finish_failed(
&store,
&owner,
task_id,
fence,
&lease,
"revalidation_failed",
)
.await;
finish_failed(&store, &lease, "revalidation_failed").await;
return;
}
if stopping.load(Ordering::Acquire) {
@@ -364,49 +346,8 @@ async fn process_lease(
let attempt_id = format!("attempt-{}", Uuid::now_v7());
// Durable checkpoint before any network use.
if store
.checkpoint_dispatch(&owner, task_id, fence, attempt_id.clone())
.await
.is_err()
{
// Lease lost or checkpoint rejected: do not dispatch.
cancelled.store(true, Ordering::Release);
return;
}
if stopping.load(Ordering::Acquire) {
// The checkpoint is durable, but we know the outbound callback has not started.
finish_failed(
&store,
&owner,
task_id,
fence,
&lease,
"shutdown_before_dispatch",
)
.await;
return;
}
// Sealed single-use permit for the trusted runtime dispatch bridge. It is created only after
// the current lease and durable checkpoint above.
let agent_id = if lease.record.owner_kind == crank_core::TaskOwnerKind::Agent {
Some(lease.record.owner_id.clone())
} else {
None
};
let permit = TaskDispatchPermit::new(
lease.record.task_id,
lease.record.operation_id.clone(),
lease.record.pins.operation_version,
lease.record.workspace_id.clone(),
agent_id,
owner.clone(),
fence,
attempt_id.clone(),
);
// Background lease renewal while the attempt runs.
// Background lease renewal covers the whole bounded retry budget, including the pinned backoff.
// A durable cancellation clears the lease, so the next renewal fails and the local work stops.
let renewal_store = store.clone();
let renewal_owner = owner.clone();
let renewal_cancelled = cancelled.clone();
@@ -435,139 +376,43 @@ async fn process_lease(
}
});
// Each dispatch gets a child span carrying the durable attempt identity, rooted in the parent
// trace persisted with the accepted Task (never a mutable queue/cache projection).
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, attempt_id = %attempt_id, fence);
// The parent span names the leased Task generation; every individual outbound attempt adds its own
// `adapter_attempt` child span with the durable attempt ID committed by its checkpoint.
let span = tracing::info_span!("durable_task_attempt", task_id = %task_id, lease_ref = %attempt_id, fence);
if let Ok(parent) = crank_core::TraceContext::parse(&lease.record.support.traceparent) {
crank_trace::set_parent_from_trace_context(&span, &parent);
}
let outcome = handler
.execute(&lease, &attempt_id, cancelled.clone(), permit)
.instrument(span)
.await;
// Every durable write of this Task (checkpoint, attempt resolution, terminal state) happens inside
// the executor's single retry loop under this lease/fence; the worker only renews and stops.
let context = TaskAttemptContext::new(
store.clone(),
lease.clone(),
cancelled.clone(),
stopping.clone(),
);
let outcome = handler.execute(context).instrument(span).await;
cancelled.store(true, Ordering::Release);
renewal_handle.abort();
// Terminal CAS under valid lease. Only one writer wins; stale token is rejected by store.
let access = crank_core::TaskAccess {
workspace_id: lease.record.workspace_id.clone(),
owner_kind: lease.record.owner_kind,
owner_id: lease.record.owner_id.clone(),
authorization_scope: lease.record.authorization_scope.clone(),
};
match outcome {
TaskAttemptOutcome::Completed(value) => {
let Ok(result) = TaskEnvelope::try_redacted(&value) else {
finish_failed(
&store,
&owner,
task_id,
fence,
&lease,
"result_not_redactable",
)
.await;
return;
};
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(&lease),
expected_version: lease.record.version,
target_state: TaskState::Completed,
result: Some(result),
error: None,
},
&owner,
fence,
)
.await;
TaskAttemptOutcome::Completed => {
debug!(task_id = %task_id, "durable task completed");
}
TaskAttemptOutcome::Failed(value) => {
let error = TaskEnvelope::try_redacted(&value).or_else(|_| {
TaskEnvelope::try_redacted(&serde_json::json!({
"code": "task_error_not_redactable"
}))
});
let Ok(error) = error else {
return;
};
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(&lease),
expected_version: lease.record.version,
target_state: TaskState::Failed,
result: None,
error: Some(error),
},
&owner,
fence,
)
.await;
TaskAttemptOutcome::Failed { code } => {
debug!(task_id = %task_id, code = %code, "durable task failed");
}
TaskAttemptOutcome::OutcomeUnknown => {
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(&lease),
expected_version: lease.record.version,
target_state: TaskState::OutcomeUnknown,
result: None,
error: None,
},
&owner,
fence,
)
.await;
warn!(task_id = %task_id, "durable task outcome is uncertain; no automatic retry");
}
TaskAttemptOutcome::Stopped => {
debug!(task_id = %task_id, "durable task attempt stopped without a new outcome");
}
}
}
fn leased_expected_state(lease: &TaskLease) -> TaskState {
// Claimed tasks are expected to be in a non-terminal processing state.
if lease.record.state == TaskState::Queued {
TaskState::Queued
} else {
TaskState::Working
}
}
async fn finish_failed(
store: &Arc<dyn TaskStore>,
owner: &str,
task_id: crank_core::TaskId,
fence: u64,
lease: &TaskLease,
code: &str,
) {
let access = crank_core::TaskAccess {
workspace_id: lease.record.workspace_id.clone(),
owner_kind: lease.record.owner_kind,
owner_id: lease.record.owner_id.clone(),
authorization_scope: lease.record.authorization_scope.clone(),
};
let _ = store
.finish_leased(
TaskFinishRequest {
access,
task_id,
expected_state: leased_expected_state(lease),
expected_version: lease.record.version,
target_state: TaskState::Failed,
result: None,
error: TaskEnvelope::try_redacted(&serde_json::json!({ "code": code })).ok(),
},
owner,
fence,
)
.await;
async fn finish_failed(store: &Arc<dyn TaskStore>, lease: &TaskLease, code: &str) {
// The store owns the fate of a refused pre-dispatch attempt: a Task that already carries a proven
// ambiguity still ends `outcome_unknown`, never a plain failure (Story 4.4).
let _ = store.refuse_attempt(lease, code).await;
}
+33
View File
@@ -47,6 +47,8 @@ pub struct TaskAcceptDraft {
/// Retention after the execution deadline.
pub retention_ms: i64,
pub approval_reference: Option<String>,
/// Bounded retry snapshot pinned from the exact published operation at acceptance.
pub retry: crank_core::TaskRetryPolicy,
}
/// Projection of the durable status/result read. `result`/`error` are only present once a terminal
@@ -161,6 +163,7 @@ impl TaskService {
execution_deadline_unix_ms,
retain_until_unix_ms: execution_deadline_unix_ms.saturating_add(retention_ms),
approval_reference: draft.approval_reference,
retry: draft.retry,
},
scoped_key,
input_fingerprint,
@@ -196,6 +199,35 @@ impl TaskService {
))
}
/// Owner-scoped cancellation of one accepted Task. The store decides under a row lock whether the
/// Task can still be cancelled or whether an already checkpointed attempt makes the outcome
/// uncertain; the service only carries the request and a bounded reason.
pub async fn cancel(
&self,
access: &TaskAccess,
task_id: TaskId,
expected_version: Option<u32>,
reason: &str,
) -> Result<crank_core::TaskCancelOutcome, TaskError> {
let request = crank_core::TaskCancelRequest {
access: access.clone(),
task_id,
expected_version,
reason: reason.to_owned(),
};
request.validate()?;
self.store.cancel(request).await
}
/// Bounded owner-scoped attempt history of one Task.
pub async fn attempts(
&self,
access: &TaskAccess,
task_id: TaskId,
) -> Result<Vec<crank_core::TaskAttempt>, TaskError> {
self.store.attempts(access, task_id).await
}
/// Decrypts and validates the executable input bound to a valid current lease.
pub async fn executable_input(
&self,
@@ -383,6 +415,7 @@ mod tests {
deadline: Instant::now() + std::time::Duration::from_secs(30),
retention_ms: TASK_DEFAULT_RETENTION_MS,
approval_reference: None,
retry: crank_core::TaskRetryPolicy::single_attempt(),
}
}
+20
View File
@@ -88,3 +88,23 @@ checkpoint-ятся/дренируются в пределах 30s, а ошиб
не отправляя запрос; это доказано регрессией, которая детерминированно
задерживает checkpoint в его собственной транзакции, проверяет отсутствие вызова
adapter и сохранность ещё queued работы.
## Безопасные повторы, отмена и outcome_unknown (Story 4.4)
До каждого outbound — отдельная durable-попытка в таблице `task_attempts` (attempt_id, ordinal, owner/fence, input fingerprint, digest upstream-ключа, pinned retry snapshot) в одной транзакции с инкрементом бюджета и версии Task; несостоявшийся commit = ноль вызовов.
После каждой попытки — одна атомарная resolution; только текущая попытка/владелец/версия, иначе StateConflict; история попыток после закрытия неизменна.
Bounded retry только для read и для write с server-owned verified-idempotency evidence; клиентский key/header сам по себе не даёт write-retry; бюджет/backoff/абсолютный deadline пинятся при приёме и не меняются новой публикацией/рестартом.
Стабильный upstream-ключ доставляется тем же `prepare_idempotency`, но синхронный coordination guard одного `tools/call` на durable-пути не применяется: авторитет повтора — PG-попытка Task, иначе неоднозначный первый ответ заблокировал бы разрешённый bounded retry. В историю попытки пишется только digest ключа, не значение и не заголовки.
Cancel до checkpoint -> cancelled и ноль outbound (сериализуется с claim и checkpoint на одной строке Task); после checkpoint (в т.ч. verified write) -> non-terminal outcome_unknown, без retry и без обещания rollback; подтверждённый результат, выигравший CAS раньше, неизменяем.
Отмена в окне backoff между попытками и отказ перед следующей попыткой (истёкший deadline, отклонённая authority, shutdown/детерминированная ошибка до отправки) не стирают уже записанную неопределённость: если последняя закрытая **отправленная** (`phase=dispatched`) попытка Task имеет certainty=unknown, Task закрывается non-terminal `outcome_unknown` (cancel -> код `cancelled_after_dispatch`, отказ/резолюция -> код отказа и phase прошлой попытки), а `cancelled`/`failed` выдаются только когда ни одна попытка не отправлялась. Закрытая known-попытка с `phase=before_dispatch` (request вообще не уходил) такую неопределённость не перекрывает — аудит-строка остаётся known/before_dispatch, но Task не становится терминальным `failed`.
Deadline/lease loss: read/safe-retry -> одна bounded new attempt (fence растёт, stale worker fenced), иначе outcome_unknown; сохранённый outcome_unknown обычными finish/cancel/retry не выводится (reconciliation — Story 4.6, вне scope).
Миграция v29 additive; legacy Task получает max_attempts=1.
Отмена после dispatch для read тоже даёт non-terminal `outcome_unknown`: в pinned snapshot Task'а нет read/write-класса, а выводить его из текущей публикации означало бы менять уже принятый snapshot. ARC-различие «read -> failed(`cancellation_after_dispatch`)» поэтому осознанно отложено до reconciliation Story 4.6 — это консервативное отклонение в безопасную сторону, а не забытый сценарий.
+1 -1
View File
@@ -1 +1 @@
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
{"schema_version":1,"sequence":[{"backfill":{"kind":"none"},"checksum":"crank-community-baseline-v1","compatibility":"legacy-baseline","contract_evidence":null,"name":"community-baseline-v1","owner":"crank-registry","phase":"expand","readable_schema_max":1,"readable_schema_min":1,"source_digest":"eb1656fc5b4b5be9ee390d237d1d58e4b2274ae5ba9b7ba06a2f3f860dfda675","transactional":true,"version":1},{"backfill":{"kind":"none"},"checksum":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"legacy-consolidation-v2","owner":"crank-registry","phase":"expand","readable_schema_max":2,"readable_schema_min":1,"source_digest":"1908b97ca7fe8a85d146b0eebf007d5018a9ff406cb099614270f3428ec1db48","transactional":true,"version":2},{"backfill":{"kind":"none"},"checksum":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"request-trace-identity-v3","owner":"crank-registry","phase":"expand","readable_schema_max":3,"readable_schema_min":2,"source_digest":"36487625503a8d4d8f18d5771c3c9b6705f845e267acbfcb7244c330f640cd94","transactional":true,"version":3},{"backfill":{"kind":"none"},"checksum":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"operation-lifecycle-v4","owner":"crank-registry","phase":"expand","readable_schema_max":4,"readable_schema_min":3,"source_digest":"45723712a1ea49cd8bbf59d77148c225f3ec376df7433983d982cc6f9d7fb39c","transactional":true,"version":4},{"backfill":{"kind":"none"},"checksum":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"execution-outcome-v5","owner":"crank-registry","phase":"expand","readable_schema_max":5,"readable_schema_min":4,"source_digest":"bd6703249cc407789586327eb7fbbd5776ba27923c5dc7eb85b05d332585bf42","transactional":true,"version":5},{"backfill":{"kind":"none"},"checksum":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"platform-key-name-reuse-v6","owner":"crank-registry","phase":"expand","readable_schema_max":6,"readable_schema_min":5,"source_digest":"94dba9b9dd3364607bc37e7698478ba6644607a7f268a1621cd3acea6bc96c2d","transactional":true,"version":6},{"backfill":{"kind":"none"},"checksum":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"master-key-identity-v7","owner":"crank-registry","phase":"expand","readable_schema_max":7,"readable_schema_min":6,"source_digest":"06e5d4667d9a7346474e9dcb176b8b2b9680c4b89089bea5a18eb5a88d5ac60e","transactional":true,"version":7},{"backfill":{"kind":"none"},"checksum":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"admin-auth-lifecycle-v8","owner":"crank-registry","phase":"expand","readable_schema_max":8,"readable_schema_min":7,"source_digest":"6361f3321a1442a77c695cad9b30c4702de1aa3e565bfbf1a7415d653302611f","transactional":true,"version":8},{"backfill":{"kind":"none"},"checksum":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"agent-catalog-lifecycle-v9","owner":"crank-registry","phase":"expand","readable_schema_max":9,"readable_schema_min":8,"source_digest":"b49bf4b53691407ec27e3810b0531c0d440da7dddc62ea71eb219933c0a30211","transactional":true,"version":9},{"backfill":{"kind":"none"},"checksum":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"approval-side-effects-v10","owner":"crank-registry","phase":"expand","readable_schema_max":10,"readable_schema_min":9,"source_digest":"6c87f680efdbc275fa2b920826b3e3e390aa34d6f2f1db48fe074a5d7691ba5b","transactional":true,"version":10},{"backfill":{"kind":"none"},"checksum":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"onboarding-product-events-v11","owner":"crank-registry","phase":"expand","readable_schema_max":11,"readable_schema_min":10,"source_digest":"a439bbcdc9cc909d717ed5a868c7a51bd1ad3c49c4e85fd20933743ee3f31166","transactional":true,"version":11},{"backfill":{"kind":"none"},"checksum":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","compatibility":"n-minus-one-readable","contract_evidence":null,"name":"artifact-metadata-v12","owner":"crank-registry","phase":"expand","readable_schema_max":12,"readable_schema_min":11,"source_digest":"052058da53cd861b2a1af81243b34cc35204d665a9276f8ece563e061f8ebbfe","transactional":true,"version":12},{"backfill":{"kind":"none"},"checksum":"abe822c967e1b2cc3966ede988e05eb2d9Line truncated
+7
View File
@@ -85,3 +85,10 @@ run_atdd "durable task migration" -p crank-registry --test integration migration
# claims before a slow HTTP drain, not after it) and one child attempt span per adapter retry.
run_atdd "task worker process shutdown" -p mcp-server --bin mcp-server -- --test-threads=1
run_atdd "durable task retry attempt spans" -p crank-runtime --test stages retry -- --test-threads=1
# Story 4.4 (FR-22, FR-23): safe retries, cancellation and outcome_unknown. No new selector is added
# here on purpose: the Story 4.2 lines above already run `crank-core --lib task::` (attempt/retry
# bounds), `crank-registry --test integration tasks::` (checkpoint, resolution, cancel and recovery
# CAS) and `mcp-server --test integration durable_tasks::` (application → PG → worker attempts,
# cancellation through the application entry and the composition oracles). Duplicating one of them
# here would only run the same bounded set twice.